dwca_hunter 0.5.3 → 0.7.2

data/lib/dwca_hunter/resource.rb

@@ -4,7 +4,12 @@ module DwcaHunter
 
     def self.unzip(file, dir = nil)
       Dir.chdir(dir) if dir
-      `unzip -qq -u #{file} > /dev/null 2>&1`
+      Zip::File.open(file) do |zip_file|
+        zip_file.each do |entry|
+          puts "Extracting #{entry.name}"
+          entry.extract
+        end
+      end
     end
 
     def self.gunzip(file, dir = nil)
@@ -13,8 +18,8 @@ module DwcaHunter
     end
 
     def initialize(opts)
-      @needs_download = !(opts[:download] == false)
-      @needs_unpack = !(opts[:unpack] == false)
+      @needs_download = (opts[:download] != false)
+      @needs_unpack = (opts[:unpack] != false)
       @download_dir, @download_file = File.split(@download_path)
       prepare_path if needs_download?
     end

data/lib/dwca_hunter/resources/aos-birds.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module DwcaHunter
+  class ResourceAOS < DwcaHunter::Resource
+    def initialize(opts = {})
+      @command = "aos-birds"
+      @title = "American Ornithological Society"
+      @url = "http://checklist.americanornithology.org/taxa.csv"
+      @UUID = "91d38806-8435-479f-a18d-705e5cb0767c"
+      @download_path = File.join(Dir.tmpdir,
+                                 "dwca_hunter",
+                                 "aos",
+                                 "data.csv")
+      @synonyms = []
+      @names = []
+      @vernaculars = []
+      @extensions = []
+      @synonyms_hash = {}
+      @vernaculars_hash = {}
+      super(opts)
+    end
+
+    def download
+      puts "Downloading csv from remote"
+      `curl -s -L #{@url} -o #{@download_path}`
+    end
+
+    def unpack; end
+
+    def make_dwca
+      DwcaHunter.logger_write(object_id, "Extracting data")
+      get_names
+      generate_dwca
+    end
+
+    private
+
+    def get_names
+      Dir.chdir(@download_dir)
+      collect_names
+    end
+
+    def collect_names
+      @names_index = {}
+      file = CSV.open(File.join(@download_dir, "data.csv"),
+                      headers: true)
+      file.each_with_index do |row, _i|
+        taxon_id = row["id"]
+        name_string = row["species"]
+        kingdom = "Animalia"
+        phylum = "Chordata"
+        klass = "Aves"
+        order = row["order"]
+        family = row["family"]
+        genus = row["genus"]
+        code = "ICZN"
+
+        @names << {
+          taxon_id: taxon_id,
+          name_string: name_string,
+          kingdom: kingdom,
+          phylum: phylum,
+          klass: klass,
+          order: order,
+          family: family,
+          genus: genus,
+          code: code
+        }
+        if row["common_name"].to_s != ""
+          @vernaculars << {
+            taxon_id: taxon_id,
+            vern: row["common_name"],
+            lang: "en"
+          }
+        end
+        next unless row["french_name"].to_s != ""
+
+        @vernaculars << {
+          taxon_id: taxon_id,
+          vern: row["french_name"],
+          lang: "fr"
+        }
+      end
+    end
+
+    def generate_dwca
+      DwcaHunter.logger_write(object_id,
+                              "Creating DarwinCore Archive file")
+      @core = [["http://rs.tdwg.org/dwc/terms/taxonID",
+                "http://rs.tdwg.org/dwc/terms/scientificName",
+                "http://rs.tdwg.org/dwc/terms/kingdom",
+                "http://rs.tdwg.org/dwc/terms/phylum",
+                "http://rs.tdwg.org/dwc/terms/class",
+                "http://rs.tdwg.org/dwc/terms/order",
+                "http://rs.tdwg.org/dwc/terms/family",
+                "http://rs.tdwg.org/dwc/terms/genus",
+                "http://rs.tdwg.org/dwc/terms/nomenclaturalCode"]]
+      @names.each do |n|
+        @core << [n[:taxon_id], n[:name_string],
+                  n[:kingdom], n[:phylum], n[:klass], n[:order], n[:family],
+                  n[:genus], n[:code]]
+      end
+      @extensions << {
+        data: [[
+          "http://rs.tdwg.org/dwc/terms/taxonID",
+          "http://rs.tdwg.org/dwc/terms/vernacularName",
+          "http://purl.org/dc/terms/language"
+        ]],
+        file_name: "vernacular_names.txt",
+        row_type: "http://rs.gbif.org/terms/1.0/VernacularName"
+      }
+
+      @vernaculars.each do |v|
+        @extensions[-1][:data] << [v[:taxon_id], v[:vern], v[:lang]]
+      end
+      @eml = {
+        id: @uuid,
+        title: @title,
+        authors: [
+          { first_name: "R. T.",
+            last_name: "Chesser" }
+        ],
+        metadata_providers: [
+          { first_name: "Dmitry",
+            last_name: "Mozzherin",
+            email: "dmozzherin@gmail.com" }
+        ],
+        abstract: "The American Ornithological Society's (AOS) Checklist is " \
+        "the official source on the taxonomy of birds found in North and " \
+        "Middle America, including adjacent islands. This list is produced " \
+        "by the North American Classification and Nomenclature Committee " \
+        "(NACC) of the AOS.\n\n" \
+        "Recommended citation: Chesser, R. T., K. J. Burns, C. Cicero, " \
+        "J. L. Dunn, A. W. Kratter, I. J. Lovette, P. C. Rasmussen, " \
+        "J. V. Remsen, Jr., D. F. Stotz, and K. Winker. 2019. Check-list " \
+        "of North American Birds (online). American Ornithological Society. " \
+        "http://checklist.aou.org/taxa",
+        url: @url
+      }
+      super
+    end
+  end
+end

data/lib/dwca_hunter/resources/arctos.rb

@@ -1,34 +1,36 @@
-# encoding: utf-8
+# frozen_string_literal: true
+
 module DwcaHunter
   class ResourceArctos < DwcaHunter::Resource
-
     def initialize(opts = {})
-      @command = 'arctos'
-      @title = 'Arctos'
-      @url = 'https://www.dropbox.com/s/jo44d1vd9bkdwm8/arctos.zip?dl=1'
-      @UUID =  'eea8315d-a244-4625-859a-226675622312'
+      @command = "arctos"
+      @title = "Arctos"
+      @url = "http://arctos.database.museum/cache/gn_merge.tgz"
+      @UUID = "eea8315d-a244-4625-859a-226675622312"
       @download_path = File.join(Dir.tmpdir,
-                                 'dwca_hunter',
-                                 'arctos',
-                                 'data.zip')
+                                 "dwca_hunter",
+                                 "arctos",
+                                 "data.tar.gz")
       @synonyms = []
       @names = []
       @vernaculars = []
       @extensions = []
+      @synonyms_hash = {}
+      @vernaculars_hash = {}
       super(opts)
     end
 
     def download
-      puts "Downloading cached verion of the file. Ask Arctos to generate new."
-        `curl -s -L #{@url} -o #{@download_path}`
+      puts "Downloading Arctos file."
+      `curl -s #{@url} -o #{@download_path}`
     end
 
     def unpack
-      unpack_zip
+      unpack_tar
     end
 
     def make_dwca
-      DwcaHunter::logger_write(self.object_id, 'Extracting data')
+      DwcaHunter.logger_write(object_id, "Extracting data")
       get_names
       generate_dwca
     end
@@ -37,190 +39,154 @@ module DwcaHunter
 
     def get_names
       Dir.chdir(@download_dir)
-      Dir.entries(@download_dir).grep(/zip$/).each do |file|
-        self.class.unzip(file) unless File.exists?(file.gsub(/zip$/,'csv'))
-      end
-      collect_names
       collect_synonyms
       collect_vernaculars
+      collect_names
     end
 
     def collect_vernaculars
-      file = open(File.join(@download_dir, 'flat_common_name.csv'))
-      fields = {}
+      file = CSV.open(File.join(@download_dir, "globalnames_commonname.csv"),
+                      headers: true)
       file.each_with_index do |row, i|
+        canonical = row["scientific_name"]
+        vernacular_name_string = row["common_name"]
 
-        if i == 0
-          fields = get_fields(row)
-          next
+        if @vernaculars_hash.key?(canonical)
+          @vernaculars_hash[canonical] << vernacular_name_string
+        else
+          @vernaculars_hash[canonical] = [vernacular_name_string]
         end
 
-        row = split_row(row)
-
-        taxon_id = row[fields[:taxon_name_id]]
-        vernacular_name_string = row[fields[:common_name]]
-
-        @vernaculars << {
-          taxon_id: taxon_id,
-          vernacular_name_string: vernacular_name_string
-        }
-
-        puts "Processed %s vernaculars" % i if i % 10000 == 0
+        puts "Processed #{i} vernaculars"if (i % 100_000).zero?
       end
     end
 
     def collect_synonyms
-      file = open(File.join(@download_dir, 'flat_relationships.csv'))
-      fields = {}
+      file = CSV.open(File.join(@download_dir, "globalnames_relationships.csv"),
+                      headers: true)
       file.each_with_index do |row, i|
-        if i == 0
-          fields = get_fields(row)
-          next
+        canonical = row["scientific_name"]
+        if @synonyms_hash.key?(canonical)
+          @synonyms_hash[canonical] <<
+            { name_string: row["related_name"], status: row["taxon_relationship"] }
+        else
+          @synonyms_hash[canonical] = [
+            { name_string: row["related_name"], status: row["taxon_relationship"] }
+          ]
         end
-
-        row = split_row(row)
-        taxon_id = row[fields[:taxon_name_id]]
-        @synonyms << {
-          taxon_id: row[fields[:related_taxon_name_id]],
-          local_id: taxon_id,
-          name_string: @names_index[taxon_id],
-          #synonym_authority:      row[fields[:relation_authority]],
-          taxonomic_status:       row[fields[:taxon_relationship]],
-        }
-        puts "Processed %s synonyms" % i if i % 10000 == 0
+        puts "Processed #{i} synonyms" if (i % 100_000).zero?
       end
     end
 
     def collect_names
       @names_index = {}
-      file = open(File.join(@download_dir, 'flat_classification.csv'))
-      fields = {}
+      file = CSV.open(File.join(@download_dir, "globalnames_classification.csv"),
+                      headers: true)
+
+      names = {}
       file.each_with_index do |row, i|
-        if i == 0
-          fields = get_fields(row)
-          next
+        next if row["term_type"].nil?
+        name = row["scientific_name"]
+        if names.key?(name)
+          names[name] = names[name].
+            merge({row["term_type"].to_sym => row["term"]})
+        else
+          names[name] = {row["term_type"].to_sym => row["term"]}
         end
-
-        next unless  row[fields[:display_name]]
-        row = split_row(row)
-        taxon_id = row[fields[:taxon_name_id]]
-        name_string = row[fields[:display_name]].gsub(/<\/?i>/,'')
-        kingdom = row[fields[:kingdom]]
-        phylum = row[fields[:phylum]]
-        klass = row[fields[:phylclass]]
-        subclass = row[fields[:subclass]]
-        order = row[fields[:phylorder]]
-        suborder = row[fields[:suborder]]
-        superfamily = row[fields[:superfamily]]
-        family = row[fields[:family]]
-        subfamily = row[fields[:subfamily]]
-        tribe = row[fields[:tribe]]
-        genus = row[fields[:genus]]
-        subgenus = row[fields[:subgenus]]
-        species = row[fields[:species]]
-        subspecies = row[fields[:subspecies]]
-        code = row[fields[:nomenclatural_code]]
-
-        @names << { taxon_id: taxon_id,
-          local_id: taxon_id,
-          name_string: name_string,
-          kingdom: kingdom,
-          phylum: phylum,
-          klass: klass,
-          order: order,
-          family: family,
-          genus: genus,
-          code: code,
-        }
-
-        @names_index[taxon_id] = name_string
-        puts "Processed %s names" % i if i % 10000 == 0
+        puts "Preprocessed #{i} rows" if (i % 100_000).zero?
+      end
+      names.each_with_index do |m, i|
+        canonical = m[0]
+        v = m[1]
+        taxon_id = "gn_#{i + 1}"
+        res ={ taxon_id: taxon_id,
+               name_string: canonical,
+               kingdom: v[:kingdom],
+               phylum: v[:phylum],
+               klass: v[:class],
+               order: v[:order],
+               family: v[:family],
+               genus: v[:genus],
+               species: v[:species],
+               authors: v[:author_text],
+               code: v[:nomenclatural_code] }
+        @names << res
+        update_vernacular(taxon_id, canonical)
+        update_synonym(taxon_id, canonical)
+        puts "Processed #{i} names" if (i % 100_000).zero?
       end
     end
 
-    def split_row(row)
-      row = row.strip.gsub(/^"/, '').gsub(/"$/, '')
-      row.split('","')
-    end
+    def update_vernacular(taxon_id, canonical)
+      return unless @vernaculars_hash.key?(canonical)
 
-    def get_fields(row)
-      row = row.split(",")
-      encoding_options = {
-        :invalid           => :replace,
-        :undef             => :replace,
-        :replace           => '',
-        :universal_newline => true
-      }
-      num_ary = (0...row.size).to_a
-      row = row.map do |f|
-        f = f.strip.downcase
-        f = f.encode ::Encoding.find('ASCII'), encoding_options
-        f.to_sym
+      @vernaculars_hash[canonical].each do |vern|
+        @vernaculars << { taxon_id: taxon_id, vern: vern }
       end
-      res = Hash[row.zip(num_ary)]
-      require 'byebug'; byebug
-      puts ''
-      res
     end
 
+    def update_synonym(taxon_id, canonical)
+      return unless @synonyms_hash.key?(canonical)
+
+      @synonyms_hash[canonical].each do |syn|
+        @synonyms << { taxon_id: taxon_id, name_string: syn[:name_string],
+                       status: syn[:status] }
+      end
+    end
 
     def generate_dwca
-      DwcaHunter::logger_write(self.object_id,
-                               'Creating DarwinCore Archive file')
-      @core = [['http://rs.tdwg.org/dwc/terms/taxonID',
-        'http://globalnames.org/terms/localID',
-        'http://rs.tdwg.org/dwc/terms/scientificName',
-        'http://rs.tdwg.org/dwc/terms/kingdom',
-        'http://rs.tdwg.org/dwc/terms/phylum',
-        'http://rs.tdwg.org/dwc/terms/class',
-        'http://rs.tdwg.org/dwc/terms/order',
-        'http://rs.tdwg.org/dwc/terms/family',
-        'http://rs.tdwg.org/dwc/terms/genus',
-        'http://rs.tdwg.org/dwc/terms/nomenclaturalCode',
-        ]]
+      DwcaHunter.logger_write(object_id,
+                              "Creating DarwinCore Archive file")
+      @core = [["http://rs.tdwg.org/dwc/terms/taxonID",
+                "http://rs.tdwg.org/dwc/terms/scientificName",
+                "http://rs.tdwg.org/dwc/terms/kingdom",
+                "http://rs.tdwg.org/dwc/terms/phylum",
+                "http://rs.tdwg.org/dwc/terms/class",
+                "http://rs.tdwg.org/dwc/terms/order",
+                "http://rs.tdwg.org/dwc/terms/family",
+                "http://rs.tdwg.org/dwc/terms/genus",
+                "http://rs.tdwg.org/dwc/terms/nomenclaturalCode"]]
       @names.each do |n|
-        @core << [n[:taxon_id], n[:taxon_id], n[:name_string],
-          n[:kingdom], n[:phylum], n[:klass], n[:order], n[:family],
-          n[:genus], n[:code]]
+        @core << [n[:taxon_id], n[:name_string],
+                  n[:kingdom], n[:phylum], n[:klass], n[:order], n[:family],
+                  n[:genus], n[:code]]
       end
       @extensions << {
         data: [[
-          'http://rs.tdwg.org/dwc/terms/taxonID',
-          'http://rs.tdwg.org/dwc/terms/vernacularName']],
-        file_name: 'vernacular_names.txt',
-        row_type: 'http://rs.gbif.org/terms/1.0/VernacularName' }
+          "http://rs.tdwg.org/dwc/terms/taxonID",
+          "http://rs.tdwg.org/dwc/terms/vernacularName"
+        ]],
+        file_name: "vernacular_names.txt",
+        row_type: "http://rs.gbif.org/terms/1.0/VernacularName"
+      }
 
       @vernaculars.each do |v|
-        @extensions[-1][:data] << [v[:taxon_id], v[:vernacular_name_string]]
+        @extensions[-1][:data] << [v[:taxon_id], v[:vern]]
       end
 
       @extensions << {
         data: [[
-          'http://rs.tdwg.org/dwc/terms/taxonID',
-          'http://globalnames.org/terms/localID',
-          'http://rs.tdwg.org/dwc/terms/scientificName',
-          'http://rs.tdwg.org/dwc/terms/taxonomicStatus',
-          ]],
-        file_name: 'synonyms.txt',
-        }
-
+          "http://rs.tdwg.org/dwc/terms/taxonID",
+          "http://rs.tdwg.org/dwc/terms/scientificName",
+          "http://rs.tdwg.org/dwc/terms/taxonomicStatus"
+        ]],
+        file_name: "synonyms.txt"
+      }
       @synonyms.each do |s|
-        @extensions[-1][:data] << [
-          s[:taxon_id], s[:local_id],
-          s[:name_string], s[:taxonomic_status]]
+        @extensions[-1][:data] << [s[:taxon_id], s[:name_string], s[:status]]
       end
       @eml = {
         id: @uuid,
         title: @title,
         authors: [
-          {email: 'dustymc at gmail dot com'}
-      ],
+          { email: "dustymc at gmail dot com" }
+        ],
         metadata_providers: [
-          { first_name: 'Dmitry',
-            last_name: 'Mozzherin',
-            email: 'dmozzherin@gmail.com' }
-      ],
-        abstract: 'Arctos is an ongoing effort to integrate access to specimen data, collection-management tools, and external resources on the internet.',
+          { first_name: "Dmitry",
+            last_name: "Mozzherin",
+            email: "dmozzherin@gmail.com" }
+        ],
+        abstract: "Arctos is an ongoing effort to integrate access to specimen data, collection-management tools, and external resources on the internet.",
         url: @url
       }
       super