dwc-archive 0.9.10 → 1.1.2

data/lib/dwc-archive/core.rb

@@ -1,17 +0,0 @@
-class DarwinCore
-  class Core
-    include DarwinCore::Ingester
-    attr_reader :id 
-    def initialize(dwc)
-      @dwc = dwc
-      @archive = @dwc.archive
-      @path = @archive.files_path
-      root_key = @archive.meta.keys[0]
-      @data = @archive.meta[root_key][:core]
-      raise DarwinCore::CoreFileError.
-        new("Cannot find core in meta.xml, is meta.xml valid?") unless @data
-      @id = @data[:id][:attributes] 
-      get_attributes(DarwinCore::CoreFileError)
-    end
-  end
-end

data/lib/dwc-archive/expander.rb

@@ -1,84 +0,0 @@
-class DarwinCore
-  class Expander
-    def initialize(archive_path, tmp_dir)
-      @archive_path = archive_path
-      @tmp_dir = tmp_dir
-      @path = File.join(tmp_dir, 'dwc_' + rand(10_000_000_000).to_s)
-      @unpacker = get_unpacker
-    end
-
-    def unpack
-      clean
-      raise DarwinCore::FileNotFoundError unless File.exists?(@archive_path)
-      success = @unpacker.call(@path, @archive_path) if @unpacker
-      (@unpacker && success && $?.exitstatus == 0) ? 
-        success : 
-        (clean; raise DarwinCore::UnpackingError)
-    end
-
-    def path
-      @files_path ||= files_path
-    end
-
-    def clean
-      FileUtils.rm_rf(@path) if FileTest.exists?(@path)
-    end
-
-    def files
-      return nil unless path && FileTest.exists?(path)
-      Dir.entries(path).select {|e| e !~ /[\.]{1,2}$/}.sort
-    end
-
-    private
-
-    def esc(a_str)
-      "'" + a_str.gsub(92.chr, '\\\\\\').gsub("'", "\\\\'") + "'"
-    end
-
-    def get_unpacker
-      file_command = IO.popen("file -z " + esc(@archive_path))
-      file_type    = file_command.read
-      file_command.close
-
-      if file_type.match(/tar.*gzip/i)
-        return proc do |tmp_path, archive_path|
-          FileUtils.mkdir tmp_path
-          path = esc(archive_path)
-          system("tar -zxf #{path} -C #{tmp_path} > /dev/null 2>&1")
-        end
-      end
-
-      if file_type.match(/Zip/)
-        return proc do |tmp_path, archive_path| 
-          path = esc(archive_path)
-          system("unzip -qq -d #{tmp_path} #{path} > /dev/null 2>&1")
-        end
-      end
-
-      return nil
-    end
-
-    def path_entries(dir)
-      Dir.entries(dir).select {|e| e !~ /[\.]{1,2}$/}.sort
-    end
-
-    def files_path
-      res = nil
-      entries = path_entries(@path)
-      if entries.include?('meta.xml')
-        res = @path
-      else
-        entries.each do |e|
-          check_path = File.join(@path, e)
-          if FileTest.directory?(check_path)
-            if path_entries(check_path).include?('meta.xml')
-              res = check_path
-              break
-            end
-          end
-        end
-      end
-      res
-    end
-  end
-end

data/lib/dwc-archive/generator.rb

@@ -1,85 +0,0 @@
-class DarwinCore
-  class Generator
-    attr_reader :eml_xml_data
-
-    #TODO refactor -- for now copying expander methods
-    def initialize(dwc_path, tmp_dir = DEFAULT_TMP_DIR)
-      @dwc_path = dwc_path
-      @path = File.join(tmp_dir, 'dwc_' + rand(10000000000).to_s)
-      FileUtils.mkdir(@path)
-      @meta_xml_data = {:extensions => []}
-      @eml_xml_data = {:id => nil, :title => nil, 
-        :authors => [], :abstract => nil, :citation => nil, :url => nil}
-      @write = 'w:utf-8'
-    end
-    
-    #TODO refactor!
-    def clean
-      FileUtils.rm_rf(@path) if FileTest.exists?(@path)
-    end
-
-    def add_core(data, file_name, keep_headers = true)
-      c = CSV.open(File.join(@path,file_name), @write)
-      header = data.shift
-      fields = header.map do |f|
-        f.strip!
-        err = 'No header in core data, or header fields are not urls'
-        raise DarwinCore::GeneratorError.new(err) unless f.match(/^http:\/\//)
-        f.split('/')[-1]
-      end
-      data.unshift(fields) if keep_headers
-      ignore_header_lines = keep_headers ? 1 : 0
-      @meta_xml_data[:core] = { fields: header, 
-                                ignoreHeaderLines: ignore_header_lines, 
-                                location:file_name }
-      data.each {|d| c << d}
-      c.close
-    end
-
-    def add_extension(data, file_name, 
-                      keep_headers = true, 
-                      row_type = 'http://rs.tdwg.org/dwc/terms/Taxon')
-      c = CSV.open(File.join(@path,file_name), @write)
-      header = data.shift
-      fields = header.map do |f|
-        f.strip!
-        err = 'No header in core data, or header fields are not urls'
-        raise DarwinCore::GeneratorError.new(err) unless f.match(/^http:\/\//)
-        f.split('/')[-1]
-      end
-      data.unshift(fields) if keep_headers
-      ignore_header_lines = keep_headers ? 1 : 0
-      @meta_xml_data[:extensions] << { fields: header, 
-                                       ignoreHeaderLines: ignore_header_lines, 
-                                       location: file_name, 
-                                       rowType: row_type }
-      data.each { |d| c << d }
-      c.close
-    end
-
-    def add_meta_xml
-      meta = DarwinCore::Generator::MetaXml.new(@meta_xml_data, @path)
-      meta.create
-    end
-
-    def add_eml_xml(data)
-      @eml_xml_data = data
-      eml = DarwinCore::Generator::EmlXml.new(@eml_xml_data, @path)
-      eml.create
-    end
-
-    def path
-      @path
-    end
-    
-    def files
-      return nil unless @path && FileTest.exists?(@path)
-      Dir.entries(@path).select {|e| e !~ /[\.]{1,2}$/}.sort
-    end
-
-    def pack
-      a = "cd #{@path}; tar -zcf #{@dwc_path} *"
-      system(a)
-    end
-  end
-end

data/lib/dwc-archive/generator_eml_xml.rb

@@ -1,86 +0,0 @@
-class DarwinCore
-  class Generator
-    class EmlXml
-
-      def initialize(data, path)
-        @data = data
-        @path = path
-        @write = 'w:utf-8'
-      end
-
-      def create
-        eml_uri =  'eml://ecoinformatics.org/eml-2.1.1' + 
-          ' http://rs.gbif.org/schema/eml-gbif-profile/1.0.1/eml.xsd'
-        builder = Nokogiri::XML::Builder.new do |xml|
-          xml.eml(packageId:      "%s/%s" % [@data[:id], timestamp],
-            system:               @data[:system] || 'http://globalnames.org',
-            :'xml:lang' =>         'en',
-            :'xmlns:eml' =>        'eml://ecoinformatics.org/eml-2.1.1',
-            :'xmlns:md' =>         'eml://ecoinformatics.org/methods-2.1.1',
-            :'xmlns:proj' =>       'eml://ecoinformatics.org/project-2.1.1',
-            :'xmlns:d' =>          'eml://ecoinformatics.org/dataset-2.1.1',
-            :'xmlns:res' =>        'eml://ecoinformatics.org/resource-2.1.1',
-            :'xmlns:dc' =>         'http://purl.org/dc/terms/',
-            :'xmlns:xsi' =>        'http://www.w3.org/2001/XMLSchema-instance',
-            :'xsi:schemaLocation' => 'eml_uri')  do
-            xml.dataset(id: @data[:id]) do
-              xml.title(@data[:title])
-              xml.license(@data[:license])
-              contacts = []
-              @data[:authors].each_with_index do |a, i|
-                creator_id = i + 1
-                contacts << creator_id
-                xml.creator(id: creator_id, scope: 'document') do
-                  xml.individualName do
-                    xml.givenName(a[:first_name])
-                    xml.surName(a[:last_name])
-                  end
-                  xml.organizationName(a[:organization]) if a[:organization]
-                  xml.positionName(a[:position]) if a[:position]
-                  xml.onlineUrl(a[:url]) if a[:url]
-                  xml.electronicMailAddress(a[:email])
-                end
-              end
-              @data[:metadata_providers].each_with_index do |a, i|
-                xml.metadataProvider do
-                  xml.individualName do
-                    xml.givenName(a[:first_name])
-                    xml.surName(a[:last_name])
-                  end
-                  xml.organizationName(a[:organization]) if a[:organization]
-                  xml.positionName(a[:position]) if a[:position]
-                  xml.onlineUrl(a[:url]) if a[:url]
-                  xml.electronicMailAddress(a[:email])
-                end
-              end if @data[:metadata_providers]
-              xml.pubDate(Time.now.to_s)
-              xml.abstract() do
-                xml.para(@data[:abstract])
-              end
-              contacts.each do |contact|
-                xml.contact { xml.references(contact) }
-              end
-            end
-            xml.additionalMetadata do
-              xml.metadata do
-                xml.citation(@data[:citation])
-                xml.resourceLogoUrl(@data[:logo_url]) if @data[:logo_url]
-              end
-            end
-            xml.parent.namespace = xml.parent.namespace_definitions.first
-          end
-        end
-        data = builder.to_xml
-        f = open(File.join(@path, 'eml.xml'), @write)
-        f.write(data)
-        f.close
-      end
-
-      private
-      def timestamp
-        t = Time.now.getutc.to_a[0..5].reverse
-        t[0..2].join('-') + '::' + t[-3..-1].join(':')
-      end
-    end
-  end
-end

data/lib/dwc-archive/generator_meta_xml.rb

@@ -1,58 +0,0 @@
-class DarwinCore
-  class Generator
-    class MetaXml
-      def initialize(data, path)
-        @data = data
-        @path = path
-        @write = 'w:utf-8'
-      end
-
-      def create
-        schema_uri =  'http://rs.tdwg.org/dwc/terms/xsd/archive/' + 
-          ' http://darwincore.googlecode.com/svn/trunk/text/tdwg_dwc_text.xsd'
-        builder = Nokogiri::XML::Builder.new do |xml|
-          opts = { encoding: 'UTF-8', 
-                   fieldsTerminatedBy: ',', 
-                   fieldsEnclosedBy: '"', 
-                   linesTerminatedBy: "\n", 
-                   rowType: 'http://rs.tdwg.org/dwc/terms/Taxon' }
-          xml.archive(xmlns: 'http://rs.tdwg.org/dwc/text/',
-            :'xmlns:xsi' => 'http://www.w3.org/2001/XMLSchema-instance',
-            :'xsi:schemaLocation' => schema_uri) do
-            xml.core(opts.merge(ignoreHeaderLines: 
-                                  @data[:core][:ignoreHeaderLines])) do
-              xml.files { xml.location(@data[:core][:location]) }
-              taxon_id, fields = find_taxon_id(@data[:core][:fields])
-              xml.id_(index: taxon_id[1])
-              fields.each { |f| xml.field(term: f[0], index: f[1]) }
-            end
-            @data[:extensions].each do |e|
-              xml.extension(opts.merge(ignoreHeaderLines: e[:ignoreHeaderLines], 
-                                       rowType: e[:rowType])) do
-                xml.files { xml.location(e[:location]) }
-                taxon_id, fields = find_taxon_id(e[:fields])
-                xml.coreid(index: taxon_id[1])
-                fields.each { |f| xml.field(term: f[0], index: f[1]) }
-              end
-            end
-          end
-        end
-        meta_xml_data = builder.to_xml
-        meta_file = open(File.join(@path, 'meta.xml'), @write)
-        meta_file.write(meta_xml_data)
-        meta_file.close
-      end
-
-      private
-      def find_taxon_id(data)
-        fields = []
-        data.each_with_index { |f, i| fields << [f.strip, i] }
-        taxon_id, fields = fields.partition { |f| f[0].match(%r|/taxonid$|i) }
-        raise DarwinCore::GeneratorError if taxon_id.size != 1
-        [taxon_id[0], fields]
-      end
-
-    end
-  end
-end
-

data/lib/dwc-archive/ingester.rb

@@ -1,101 +0,0 @@
-# encoding: utf-8
-class DarwinCore
-  module Ingester
-    attr_reader :data, :properties, :encoding, :fields_separator, :size
-    attr_reader :file_path, :fields, :line_separator, 
-                :quote_character, :ignore_headers
-
-    def size
-      @size ||= get_size
-    end
-
-    def read(batch_size = 10000)
-      DarwinCore.logger_write(@dwc.object_id, "Reading %s data" % name)
-      res = []
-      errors = []
-      index_fix = 1
-      args = {:col_sep => @field_separator}
-      @quote_character = "\b" if @quote_character.empty?
-      args.merge!({:quote_char => @quote_character})
-      min_size = @fields.map {|f| f[:index].to_i || 0}.sort[-1] + 1
-      csv = CSV.new(open(@file_path), args)
-      csv.each_with_index do |r, i|
-        index_fix = 0; next if @ignore_headers && i == 0
-        min_size > r.size ? errors << r : process_csv_row(res, errors, r)
-        if (i + index_fix) % batch_size == 0
-          DarwinCore.logger_write(@dwc.object_id, 
-                                  "Ingested %s records from %s" % 
-                                  [(i + index_fix), name])
-          if block_given?
-            yield [res, errors]
-            res = []
-            errors = []
-          end
-        end
-      end
-      yield [res, errors] if block_given?
-      [res, errors]
-    end
-
-    private
-    def name
-      self.class.to_s.split('::')[-1].downcase
-    end
-
-    def process_csv_row(result, errors, row)
-      str = row.join('')
-      str = str.force_encoding('utf-8')
-      if str.encoding.name == 'UTF-8' && str.valid_encoding?
-        result << row.map { |f| f.nil? ? nil : f.force_encoding('utf-8') }
-      else
-        errors << row
-      end
-    end
-
-    def get_attributes(exception)
-      @properties = @data[:attributes]
-      @encoding = @properties[:encoding] || 'UTF-8'
-      err_msg = 'No support for encodings other ' + 
-        'than utf-8 or utf-16 at the moment'
-      encodings = ['utf-8', 'utf8', 'utf-16', 'utf16']
-      unless encodings.include? @encoding.downcase
-        raise DarwinCore::EncodingError.new(err_msg) 
-      end
-      @field_separator = get_field_separator
-      @quote_character = @properties[:fieldsEnclosedBy] || ''
-      @line_separator = @properties[:linesTerminatedBy] || '\n'
-      @ignore_headers = @properties[:ignoreHeaderLines] ? 
-                        [1, true].include?(@properties[:ignoreHeaderLines]) : 
-                        false
-      @file_path = get_file_path
-      raise DarwinCore::FileNotFoundError.new("No file data") unless @file_path
-      @fields = get_fields
-      if @fields.empty?
-        raise DarwinCore::InvalidArchiveError.new("No data fields are found")
-      end
-    end
-
-    def get_file_path
-      file = @data[:location] || 
-             @data[:attributes][:location] || 
-             @data[:files][:location]
-      File.join(@path, file)
-    end
-
-    def get_fields
-      @data[:field] = [data[:field]] if data[:field].class != Array
-      @data[:field].map {|f| f[:attributes]}
-    end
-
-    def get_field_separator
-      res = @properties[:fieldsTerminatedBy] || ','
-      res = "\t" if res == "\\t"
-      res
-    end
-
-    def get_size
-      `wc -l #{@file_path}`.match(/^\s*([\d]+)\s/)[1].to_i
-    end
-  end
-end
-

data/lib/dwc-archive/metadata.rb

@@ -1,48 +0,0 @@
-class DarwinCore
-  class Metadata
-    def initialize(archive = nil)
-      @archive = archive
-      @metadata = @archive.eml
-    end
-
-    def data
-      @metadata
-    end
-
-    def id
-      @metadata[:eml][:dataset][:attributes][:id] rescue nil
-    end
-
-    def package_id
-      @metadata.data[:eml][:attributes][:packageId] rescue nil
-    end
-
-    def title
-      @metadata[:eml][:dataset][:title] rescue nil
-    end
-
-    def authors
-      return nil unless defined?(@metadata[:eml][:dataset][:creator])
-      @metadata[:eml][:dataset][:creator] = 
-        [@metadata[:eml][:dataset][:creator]] unless 
-          @metadata[:eml][:dataset][:creator].class == Array 
-      @metadata[:eml][:dataset][:creator].map do |c| 
-        { first_name: c[:individualName][:givenName], 
-          last_name: c[:individualName][:surName], 
-          email: c[:electronicMailAddress] }
-      end
-    end
-
-    def abstract
-      @metadata[:eml][:dataset][:abstract] rescue nil
-    end
-
-    def citation
-      @metadata[:eml][:additionalMetadata][:metadata][:citation] rescue nil
-    end
-
-    def url
-      @metadata[:eml][:dataset][:distribution][:online][:url] rescue nil
-    end
-  end
-end