dwc-archive 0.9.10 → 1.1.2

data/lib/dwc_archive/core.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class DarwinCore
+  # Represents core of the DarwinCore Archive
+  class Core
+    include DarwinCore::Ingester
+    attr_reader :id
+
+    # rubocop:disable Metrics/MethodLength
+    def initialize(dwc)
+      @dwc = dwc
+      @archive = @dwc.archive
+      @path = @archive.files_path
+      root_key = @archive.meta.keys[0]
+      @data = @archive.meta[root_key][:core]
+      unless @data
+        raise DarwinCore::CoreFileError,
+              "Cannot find core in meta.xml, is meta.xml valid?"
+      end
+      @id = @data[:id][:attributes]
+      init_attributes
+    end
+  end
+  # rubocop:enable Metrics/MethodLength
+end

data/lib/{dwc-archive → dwc_archive}/errors.rb

@@ -1,11 +1,21 @@
+# frozen_string_literal: true
+
 class DarwinCore
   class Error < RuntimeError; end
+
   class FileNotFoundError < Error; end
+
   class UnpackingError < Error; end
+
   class InvalidArchiveError < Error; end
+
   class CoreFileError < Error; end
+
   class ExtensionFileError < Error; end
+
   class GeneratorError < Error; end
+
   class ParentNotCurrentError < Error; end
+
   class EncodingError < Error; end
 end

data/lib/dwc_archive/expander.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+class DarwinCore
+  # Unpacks compressed archives into a temp directory
+  class Expander
+    def initialize(archive_path, tmp_dir)
+      @archive_path = archive_path
+      @tmp_dir = tmp_dir
+      @dir_path = DarwinCore.random_path(tmp_dir)
+      @unpacker = init_unpacker
+    end
+
+    def unpack
+      clean
+      raise DarwinCore::FileNotFoundError unless File.exist?(@archive_path)
+
+      success = @unpacker.call(@dir_path, @archive_path) if @unpacker
+      if @unpacker && success && $CHILD_STATUS.exitstatus.zero?
+        success
+      else
+        clean
+        raise DarwinCore::UnpackingError
+      end
+    end
+
+    def path
+      @path ||= files_path
+    end
+
+    def clean
+      DarwinCore.clean(@dir_path)
+    end
+
+    def files
+      DarwinCore.files(path)
+    end
+
+    private
+
+    def init_unpacker
+      return tar_unpacker if @archive_path =~ /tar.gz$/i
+      return zip_unpacker if @archive_path =~ /zip$/i
+
+      nil
+    end
+
+    def tar_unpacker
+      proc do |tmp_path, archive_path|
+        FileUtils.mkdir tmp_path
+        path = esc(archive_path)
+        system("tar -zxf #{path} -C #{tmp_path} > /dev/null 2>&1")
+      end
+    end
+
+    def zip_unpacker
+      proc do |tmp_path, archive_path|
+        path = esc(archive_path)
+        system("unzip -qq -d #{tmp_path} #{path} > /dev/null 2>&1")
+      end
+    end
+
+    def esc(a_str)
+      "'#{a_str.gsub(92.chr, '\\\\\\').gsub("'", "\\\\'")}'"
+    end
+
+    def path_entries(dir)
+      Dir.entries(dir).reject { |e| e.match(/\.{1,2}$/) }.sort
+    end
+
+    def files_path
+      entries = path_entries(@dir_path)
+      entries.include?("meta.xml") ? @dir_path : search_for_file_path(entries)
+    end
+
+    def search_for_file_path(entries)
+      res = nil
+      entries.each do |e|
+        check_path = File.join(@dir_path, e)
+        next unless FileTest.directory?(check_path) &&
+                    path_entries(check_path).include?("meta.xml")
+
+        res = check_path
+        break
+      end
+      res
+    end
+  end
+end

data/lib/{dwc-archive → dwc_archive}/extension.rb

@@ -1,8 +1,11 @@
+# frozen_string_literal: true
+
 class DarwinCore
+  # Represents extensions of DarwinCore Archive
   class Extension
     include DarwinCore::Ingester
     attr_reader :coreid
-    alias :id :coreid
+    alias id coreid
 
     def initialize(dwc, data)
       @dwc = dwc
@@ -10,8 +13,7 @@ class DarwinCore
       @path = @archive.files_path
       @data = data
       @coreid = @data[:coreid][:attributes]
-      get_attributes(DarwinCore::ExtensionFileError)
+      init_attributes
     end
-
   end
 end

data/lib/dwc_archive/generator.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+class DarwinCore
+  # Creates csv files for core and extensions
+  class Generator
+    attr_reader :eml_xml_data, :path
+
+    def initialize(dwc_path, tmp_dir = DEFAULT_TMP_DIR)
+      @dwc_path = dwc_path
+      @path = DarwinCore.random_path(tmp_dir)
+      FileUtils.mkdir(@path)
+      @meta_xml_data = { extensions: [] }
+      @eml_xml_data = { id: nil, title: nil, authors: [], abstrac: nil,
+                        citation: nil, url: nil }
+      @write = 'w:utf-8'
+    end
+
+    def clean
+      DarwinCore.clean(@path)
+    end
+
+    def add_core(data, file_name, keep_headers = true)
+      opts = { type: 'core', data: data, file_name: file_name,
+               keep_headers: keep_headers }
+      prepare_csv_file(opts)
+    end
+
+    def add_extension(data, file_name, keep_headers = true,
+                      row_type = 'http://rs.tdwg.org/dwc/terms/Taxon')
+      opts = { type: 'extension', data: data, file_name: file_name,
+               keep_headers: keep_headers, row_type: row_type }
+      prepare_csv_file(opts)
+    end
+
+    def add_meta_xml
+      meta = DarwinCore::Generator::MetaXml.new(@meta_xml_data, @path)
+      meta.create
+    end
+
+    def add_eml_xml(data)
+      @eml_xml_data = data
+      eml = DarwinCore::Generator::EmlXml.new(@eml_xml_data, @path)
+      eml.create
+    end
+
+    def files
+      DarwinCore.files(@path)
+    end
+
+    def pack
+      a = "cd #{@path}; tar -zcf #{@dwc_path} *"
+      system(a)
+    end
+
+    private
+
+    def prepare_csv_file(opts)
+      c = CSV.open(File.join(@path, opts[:file_name]), @write)
+      attributes = prepare_attributes(opts)
+      if opts[:type] == 'core'
+        @meta_xml_data[:core] = attributes
+      else
+        @meta_xml_data[:extensions] << attributes
+      end
+      opts[:data].each { |d| c << d }
+      c.close
+    end
+
+    def prepare_attributes(opts)
+      header = opts[:data].shift
+      fields = init_fields(header, opts[:type])
+      opts[:data].unshift(fields) if opts[:keep_headers]
+      ignore_header_lines = opts[:keep_headers] ? 1 : 0
+
+      res = { fields: header, ignoreHeaderLines: ignore_header_lines,
+              location: opts[:file_name] }
+      res[:rowType] = opts[:row_type] if opts[:row_type]
+      res
+    end
+
+    def init_fields(header, file_type)
+      header.map do |f|
+        f = f.strip
+        err = "No header in #{file_type} data, or header fields are not urls"
+        raise DarwinCore::GeneratorError, err unless f =~ %r{^http://}
+
+        f.split('/')[-1]
+      end
+    end
+  end
+end

data/lib/dwc_archive/generator_eml_xml.rb

@@ -0,0 +1,116 @@
+class DarwinCore
+  class Generator
+    # Creates EML file with meta information about archive
+    class EmlXml
+      SCHEMA_DATA = {
+        :"xml:lang" =>           "en",
+        :"xmlns:eml" =>          "eml://ecoinformatics.org/eml-2.1.1",
+        :"xmlns:md" =>           "eml://ecoinformatics.org/methods-2.1.1",
+        :"xmlns:proj" =>         "eml://ecoinformatics.org/project-2.1.1",
+        :"xmlns:d" =>            "eml://ecoinformatics.org/dataset-2.1.1",
+        :"xmlns:res" =>          "eml://ecoinformatics.org/resource-2.1.1",
+        :"xmlns:dc" =>           "http://purl.org/dc/terms/",
+        :"xmlns:xsi" =>          "http://www.w3.org/2001/XMLSchema-instance",
+        :"xsi:schemaLocation" => "eml://ecoinformatics.org/eml-2.1.1 "\
+          "http://rs.gbif.org/schema/eml-gbif-profile/1.0.1/eml.xsd"
+      }
+
+      def initialize(data, path)
+        @data = data
+        @path = path
+        @write = "w:utf-8"
+      end
+
+      def create
+        schema_data = {
+          packageId: "#{@data[:id]}/#{timestamp}",
+          system: @data[:system] || "http://globalnames.org"
+        }.merge(SCHEMA_DATA)
+        builder = Nokogiri::XML::Builder.new do |xml|
+          xml.eml(schema_data) do
+            build_body(xml)
+          end
+        end
+        save_eml(builder)
+      end
+
+      private
+
+      def build_body(xml)
+        build_dataset(xml)
+        build_additional_metadata(xml)
+        xml.parent.namespace = xml.parent.namespace_definitions.first
+      end
+
+      def save_eml(builder)
+        data = builder.to_xml
+        f = open(File.join(@path, "eml.xml"), @write)
+        f.write(data)
+        f.close
+      end
+
+      def build_dataset(xml)
+        xml.dataset(id: @data[:id]) do
+          xml.title(@data[:title])
+          xml.license(@data[:license])
+          contacts = []
+          build_authors(xml, contacts)
+          build_metadata_providers(xml)
+          xml.pubDate(Time.now.to_s)
+          build_abstract(xml)
+          build_contacts(xml, contacts)
+        end
+      end
+
+      def build_abstract(xml)
+        xml.abstract { xml.para(@data[:abstract]) }
+      end
+
+      def build_contacts(xml, contacts)
+        contacts.each { |contact| xml.contact { xml.references(contact) } }
+      end
+
+      def build_metadata_providers(xml)
+        @data[:metadata_providers].each do |a|
+          xml.metadataProvider { build_person(xml, a) }
+        end if @data[:metadata_providers]
+      end
+
+      def build_authors(xml, contacts)
+        @data[:authors].each_with_index do |a, i|
+          creator_id = i + 1
+          contacts << creator_id
+          xml.creator(id: creator_id, scope: "document") do
+            build_person(xml, a)
+          end
+        end
+      end
+
+      def build_additional_metadata(xml)
+        xml.additionalMetadata do
+          xml.metadata do
+            xml.citation(@data[:citation])
+            xml.resourceLogoUrl(@data[:logo_url]) if @data[:logo_url]
+          end
+        end
+      end
+
+      def build_person(xml, data)
+        a = data
+        xml.individualName do
+          xml.givenName(a[:first_name])
+          xml.surName(a[:last_name])
+        end
+        xml.organizationName(a[:organization]) if a[:organization]
+        xml.positionName(a[:position]) if a[:position]
+        xml.onlineUrl(a[:url]) if a[:url]
+        xml.electronicMailAddress(a[:email])
+      end
+
+      def timestamp
+        t = Time.now.getutc.to_a[0..5].reverse
+        t[0..2] * ("-") + "::" + t[-3..-1] * (":")
+      end
+    end
+  end
+end

data/lib/dwc_archive/generator_meta_xml.rb

@@ -0,0 +1,72 @@
+class DarwinCore
+  class Generator
+    # Creates DarwinCore meta file
+    class MetaXml
+      def initialize(data, path)
+        @data = data
+        @path = path
+        @write = "w:utf-8"
+      end
+
+      def create
+        schema_uri = "http://rs.tdwg.org/dwc/terms/xsd/archive/ "\
+          "http://darwincore.googlecode.com/svn/trunk/text/tdwg_dwc_text.xsd"
+        builder = Nokogiri::XML::Builder.new do |xml|
+          opts = { encoding: "UTF-8", fieldsTerminatedBy: ",",
+                   fieldsEnclosedBy: '"', linesTerminatedBy: "\n",
+                   rowType: "http://rs.tdwg.org/dwc/terms/Taxon" }
+          build_archive(xml, opts, schema_uri)
+        end
+        save_meta(builder)
+      end
+
+      private
+
+      def save_meta(builder)
+        meta_xml_data = builder.to_xml
+        meta_file = open(File.join(@path, "meta.xml"), @write)
+        meta_file.write(meta_xml_data)
+        meta_file.close
+      end
+
+      def build_archive(xml, opts, schema_uri)
+        xml.archive(xmlns: "http://rs.tdwg.org/dwc/text/",
+                    :"xmlns:xsi" => "http://www.w3.org/2001/XMLSchema-instance",
+                    :"xsi:schemaLocation" => schema_uri) do
+          build_core(xml, opts)
+          build_extensions(xml, opts)
+        end
+      end
+
+      def build_core(xml, opts)
+        xml.core(opts.merge(ignoreHeaderLines:
+                              @data[:core][:ignoreHeaderLines])) do
+          xml.files { xml.location(@data[:core][:location]) }
+          taxon_id, fields = find_taxon_id(@data[:core][:fields])
+          xml.id_(index: taxon_id[1])
+          fields.each { |f| xml.field(term: f[0], index: f[1]) }
+        end
+      end
+
+      def build_extensions(xml, opts)
+        @data[:extensions].each do |e|
+          xml.extension(opts.merge(ignoreHeaderLines: e[:ignoreHeaderLines],
+                                   rowType: e[:rowType])) do
+            xml.files { xml.location(e[:location]) }
+            taxon_id, fields = find_taxon_id(e[:fields])
+            xml.coreid(index: taxon_id[1])
+            fields.each { |f| xml.field(term: f[0], index: f[1]) }
+          end
+        end
+      end
+
+      def find_taxon_id(data)
+        fields = []
+        data.each_with_index { |f, i| fields << [f.strip, i] }
+        taxon_id, fields = fields.partition { |f| f[0].match(/\/taxonid$/i) }
+        fail DarwinCore::GeneratorError if taxon_id.size != 1
+        [taxon_id[0], fields]
+      end
+    end
+  end
+end

data/lib/dwc_archive/gnub_taxon.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class DarwinCore
+  # Covers special case of Global Names Usage Bank data
+  class GnubTaxon < TaxonNormalized
+    attr_accessor :uuid, :uuid_path
+
+    def initialize
+      super
+      @uuid = nil
+      @uuid_path = []
+    end
+  end
+end