dwca_hunter 0.5.1 → 0.7.0

data/lib/dwca_hunter/resources/arctos.rb

@@ -1,30 +1,36 @@
-# encoding: utf-8
+# frozen_string_literal: true
+
 module DwcaHunter
   class ResourceArctos < DwcaHunter::Resource
-
     def initialize(opts = {})
-      @command = 'arctos'
-      @title = 'Arctos'
-      @url = 'http://arctos.database.museum/download/gncombined.zip'
-      @UUID =  'eea8315d-a244-4625-859a-226675622312'
+      @command = "arctos"
+      @title = "Arctos"
+      @url = "https://www.dropbox.com/s/3rmny5d8cfm9mmp/arctos.tar.gz?dl=1"
+      @UUID = "eea8315d-a244-4625-859a-226675622312"
       @download_path = File.join(Dir.tmpdir,
-                                 'dwca_hunter',
-                                 'arctos',
-                                 'data.tar.gz')
+                                 "dwca_hunter",
+                                 "arctos",
+                                 "data.zip")
       @synonyms = []
       @names = []
       @vernaculars = []
       @extensions = []
+      @synonyms_hash = {}
+      @vernaculars_hash = {}
       super(opts)
-      @gnub_dir = File.join(@download_dir, 'gnub')
+    end
+
+    def download
+      puts "Downloading cached verion of the file. Ask Arctos to generate new."
+      `curl -s -L #{@url} -o #{@download_path}`
     end
 
     def unpack
-      unpack_zip
+      unpack_tar
     end
 
     def make_dwca
-      DwcaHunter::logger_write(self.object_id, 'Extracting data')
+      DwcaHunter.logger_write(object_id, "Extracting data")
       get_names
       generate_dwca
     end
@@ -33,190 +39,160 @@ module DwcaHunter
 
     def get_names
       Dir.chdir(@download_dir)
-      Dir.entries(@download_dir).grep(/zip$/).each do |file|
-        self.class.unzip(file) unless File.exists?(file.gsub(/zip$/,'csv'))
-      end
-      collect_names
       collect_synonyms
       collect_vernaculars
+      collect_names
     end
 
     def collect_vernaculars
-      file = open(File.join(@download_dir, 'common_name.csv'))
-      fields = {}
+      file = CSV.open(File.join(@download_dir, "common_name.csv"),
+                      headers: true)
       file.each_with_index do |row, i|
+        canonical = row["SCIENTIFIC_NAME"]
+        vernacular_name_string = row["COMMON_NAME"]
 
-        if i == 0
-          fields = get_fields(row)
-          next
+        if @vernaculars_hash.key?(canonical)
+          @vernaculars_hash[canonical] << vernacular_name_string
+        else
+          @vernaculars_hash[canonical] = [vernacular_name_string]
         end
 
-        row = split_row(row)
-
-        taxon_id = row[fields[:taxon_name_id]]
-        vernacular_name_string = row[fields[:common_name]]
-
-        @vernaculars << {
-          taxon_id: taxon_id,
-          vernacular_name_string: vernacular_name_string
-        }
-
-        puts "Processed %s vernaculars" % i if i % 10000 == 0
+        puts "Processed %s vernaculars" % i if i % 10_000 == 0
       end
     end
 
     def collect_synonyms
-      file = open(File.join(@download_dir, 'taxon_relations.csv'))
-      fields = {}
+      file = CSV.open(File.join(@download_dir, "relationships.csv"),
+                      headers: true)
       file.each_with_index do |row, i|
-        if i == 0
-          fields = get_fields(row)
-          next
+        canonical = row["scientific_name"]
+        if @synonyms_hash.key?(canonical)
+          @synonyms_hash[canonical] <<
+            { name_string: row["related_name"], status: row["TAXON_RELATIONSHIP"] }
+        else
+          @synonyms_hash[canonical] = [
+            { name_string: row["related_name"], status: row["TAXON_RELATIONSHIP"] }
+          ]
         end
-
-        row = split_row(row)
-        taxon_id = row[fields[:taxon_name_id]]
-        @synonyms << {
-          taxon_id: row[fields[:related_taxon_name_id]],
-          local_id: taxon_id,
-          name_string: @names_index[taxon_id],
-          #synonym_authority:      row[fields[:relation_authority]],
-          taxonomic_status:       row[fields[:taxon_relationship]],
-        }
-        puts "Processed %s synonyms" % i if i % 10000 == 0
+        puts "Processed %s synonyms" % i if i % 10_000 == 0
       end
     end
 
     def collect_names
       @names_index = {}
-      file = open(File.join(@download_dir, 'taxonomy.csv'))
-      fields = {}
+      file = CSV.open(File.join(@download_dir, "classification.csv"),
+                      headers: true)
       file.each_with_index do |row, i|
-        if i == 0
-          fields = get_fields(row)
-          next
-        end
-        next unless  row[fields[:display_name]]
-        row = split_row(row)
-        taxon_id = row[fields[:taxon_name_id]]
-        name_string = row[fields[:display_name]].gsub(/<\/?i>/,'')
-        kingdom = row[fields[:kingdom]]
-        phylum = row[fields[:phylum]]
-        klass = row[fields[:phylclass]]
-        subclass = row[fields[:subclass]]
-        order = row[fields[:phylorder]]
-        suborder = row[fields[:suborder]]
-        superfamily = row[fields[:superfamily]]
-        family = row[fields[:family]]
-        subfamily = row[fields[:subfamily]]
-        tribe = row[fields[:tribe]]
-        genus = row[fields[:genus]]
-        subgenus = row[fields[:subgenus]]
-        species = row[fields[:species]]
-        subspecies = row[fields[:subspecies]]
-        code = row[fields[:nomenclatural_code]]
-
+        next unless row["display_name"]
+
+        name_string = row["display_name"].gsub(%r{</?i>}, "")
+        canonical = row["scientific_name"]
+        kingdom = row["kingdom"]
+        phylum = row["phylum"]
+        klass = row["phylclass"]
+        subclass = row["subclass"]
+        order = row["phylorder"]
+        suborder = row["suborder"]
+        superfamily = row["superfamily"]
+        family = row["family"]
+        subfamily = row["subfamily"]
+        tribe = row["tribe"]
+        genus = row["genus"]
+        subgenus = row["subgenus"]
+        species = row["species"]
+        subspecies = row["subspecies"]
+        code = row["nomenclatural_code"]
+
+        taxon_id = "ARCT_#{i + 1}"
         @names << { taxon_id: taxon_id,
-          local_id: taxon_id,
-          name_string: name_string,
-          kingdom: kingdom,
-          phylum: phylum,
-          klass: klass,
-          order: order,
-          family: family,
-          genus: genus,
-          code: code,
-        }
-
-        @names_index[taxon_id] = name_string
-        puts "Processed %s names" % i if i % 10000 == 0
+                    name_string: name_string,
+                    kingdom: kingdom,
+                    phylum: phylum,
+                    klass: klass,
+                    order: order,
+                    family: family,
+                    genus: genus,
+                    code: code }
+
+        update_vernacular(taxon_id, canonical)
+        update_synonym(taxon_id, canonical)
+        puts "Processed %s names" % i if i % 10_000 == 0
       end
     end
 
-    def split_row(row)
-      row = row.strip.gsub(/^"/, '').gsub(/"$/, '')
-      row.split('","')
-    end
+    def update_vernacular(taxon_id, canonical)
+      return unless @vernaculars_hash.key?(canonical)
 
-    def get_fields(row)
-      row = row.split(",")
-      encoding_options = {
-        :invalid           => :replace,
-        :undef             => :replace,
-        :replace           => '',
-        :universal_newline => true
-      }
-      num_ary = (0...row.size).to_a
-      row = row.map do |f|
-        f = f.strip.downcase
-        f = f.encode ::Encoding.find('ASCII'), encoding_options
-        f.to_sym
+      @vernaculars_hash[canonical].each do |vern|
+        @vernaculars << { taxon_id: taxon_id, vern: vern }
       end
-      Hash[row.zip(num_ary)]
     end
 
+    def update_synonym(taxon_id, canonical)
+      return unless @synonyms_hash.key?(canonical)
+
+      @synonyms_hash[canonical].each do |syn|
+        @synonyms << { taxon_id: taxon_id, name_string: syn[:name_string],
+                       status: syn[:status] }
+      end
+    end
 
     def generate_dwca
-      DwcaHunter::logger_write(self.object_id,
-                               'Creating DarwinCore Archive file')
-      @core = [['http://rs.tdwg.org/dwc/terms/taxonID',
-        'http://globalnames.org/terms/localID',
-        'http://rs.tdwg.org/dwc/terms/scientificName',
-        'http://rs.tdwg.org/dwc/terms/kingdom',
-        'http://rs.tdwg.org/dwc/terms/phylum',
-        'http://rs.tdwg.org/dwc/terms/class',
-        'http://rs.tdwg.org/dwc/terms/order',
-        'http://rs.tdwg.org/dwc/terms/family',
-        'http://rs.tdwg.org/dwc/terms/genus',
-        'http://rs.tdwg.org/dwc/terms/nomenclaturalCode',
-        ]]
+      DwcaHunter.logger_write(object_id,
+                              "Creating DarwinCore Archive file")
+      @core = [["http://rs.tdwg.org/dwc/terms/taxonID",
+                "http://rs.tdwg.org/dwc/terms/scientificName",
+                "http://rs.tdwg.org/dwc/terms/kingdom",
+                "http://rs.tdwg.org/dwc/terms/phylum",
+                "http://rs.tdwg.org/dwc/terms/class",
+                "http://rs.tdwg.org/dwc/terms/order",
+                "http://rs.tdwg.org/dwc/terms/family",
+                "http://rs.tdwg.org/dwc/terms/genus",
+                "http://rs.tdwg.org/dwc/terms/nomenclaturalCode"]]
       @names.each do |n|
-        @core << [n[:taxon_id], n[:taxon_id], n[:name_string],
-          n[:kingdom], n[:phylum], n[:klass], n[:order], n[:family],
-          n[:genus], n[:code]]
+        @core << [n[:taxon_id], n[:name_string],
+                  n[:kingdom], n[:phylum], n[:klass], n[:order], n[:family],
+                  n[:genus], n[:code]]
       end
       @extensions << {
         data: [[
-          'http://rs.tdwg.org/dwc/terms/taxonID',
-          'http://rs.tdwg.org/dwc/terms/vernacularName']],
-        file_name: 'vernacular_names.txt',
-        row_type: 'http://rs.gbif.org/terms/1.0/VernacularName' }
+          "http://rs.tdwg.org/dwc/terms/taxonID",
+          "http://rs.tdwg.org/dwc/terms/vernacularName"
+        ]],
+        file_name: "vernacular_names.txt",
+        row_type: "http://rs.gbif.org/terms/1.0/VernacularName"
+      }
 
       @vernaculars.each do |v|
-        @extensions[-1][:data] << [v[:taxon_id], v[:vernacular_name_string]]
+        @extensions[-1][:data] << [v[:taxon_id], v[:vern]]
       end
 
       @extensions << {
         data: [[
-          'http://rs.tdwg.org/dwc/terms/taxonID',
-          'http://globalnames.org/terms/localID',
-          'http://rs.tdwg.org/dwc/terms/scientificName',
-          'http://rs.tdwg.org/dwc/terms/taxonomicStatus',
-          ]],
-        file_name: 'synonyms.txt',
-        }
-
+          "http://rs.tdwg.org/dwc/terms/taxonID",
+          "http://rs.tdwg.org/dwc/terms/scientificName",
+          "http://rs.tdwg.org/dwc/terms/taxonomicStatus"
+        ]],
+        file_name: "synonyms.txt"
+      }
       @synonyms.each do |s|
-        @extensions[-1][:data] << [
-          s[:taxon_id], s[:local_id],
-          s[:name_string], s[:taxonomic_status]]
+        @extensions[-1][:data] << [s[:taxon_id], s[:name_string], s[:status]]
       end
       @eml = {
         id: @uuid,
         title: @title,
         authors: [
-          {email: 'dustymc at gmail dot com'}
-      ],
+          { email: "dustymc at gmail dot com" }
+        ],
         metadata_providers: [
-          { first_name: 'Dmitry',
-            last_name: 'Mozzherin',
-            email: 'dmozzherin@gmail.com' }
-      ],
-        abstract: 'Arctos is an ongoing effort to integrate access to specimen data, collection-management tools, and external resources on the internet.',
+          { first_name: "Dmitry",
+            last_name: "Mozzherin",
+            email: "dmozzherin@gmail.com" }
+        ],
+        abstract: "Arctos is an ongoing effort to integrate access to specimen data, collection-management tools, and external resources on the internet.",
         url: @url
       }
       super
     end
   end
 end
-

data/lib/dwca_hunter/resources/clements.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module DwcaHunter
+  class ResourceClements < DwcaHunter::Resource
+    def initialize(opts = {})
+      @command = "clements-ebird"
+      @title = "The eBird/Clements Checklist of Birds of the World"
+      @url = "https://uofi.box.com/shared/static/b4n8zqa99hq9rdga27skkh3870yhujgo.csv"
+      @UUID = "577c0b56-4a3c-4314-8724-14b304f601de"
+      @download_path = File.join(Dir.tmpdir,
+                                 "dwca_hunter",
+                                 "clements",
+                                 "data.csv")
+      @synonyms = []
+      @names = []
+      @vernaculars = []
+      @extensions = []
+      @synonyms_hash = {}
+      @vernaculars_hash = {}
+      super(opts)
+    end
+
+    def download
+      puts "Downloading cached and modified version of the file."
+      puts "Go to https://www.birds.cornell.edu/clementschecklist/download/ " \
+        "for updates."
+      `curl -s -L #{@url} -o #{@download_path}`
+    end
+
+    def unpack
+    end
+
+    def make_dwca
+      DwcaHunter.logger_write(object_id, "Extracting data")
+      get_names
+      generate_dwca
+    end
+
+    private
+
+    def get_names
+      Dir.chdir(@download_dir)
+      collect_names
+    end
+
+    def collect_names
+      @names_index = {}
+      file = CSV.open(File.join(@download_dir, "data.csv"),
+                      headers: true)
+      file.each_with_index do |row, i|
+        name_string = row["scientific name"]
+        canonical = name_string
+        kingdom = "Animalia"
+        phylum = "Chordata"
+        klass = "Aves"
+        order = row["order"]
+        family = row["family"]
+        code = "ICZN"
+
+        taxon_id = "gn_#{i + 1}"
+        @names << { taxon_id: taxon_id,
+                    name_string: name_string,
+                    kingdom: kingdom,
+                    phylum: phylum,
+                    klass: klass,
+                    order: order,
+                    family: family,
+                    code: code }
+
+        if row["English name"].to_s != ""
+            @vernaculars << {
+              taxon_id: taxon_id,
+              vern: row["English name"],
+              lang: "end"
+            }
+        end
+
+        puts "Processed %s names" % i if i % 10_000 == 0
+      end
+    end
+
+    def generate_dwca
+      DwcaHunter.logger_write(object_id,
+                              "Creating DarwinCore Archive file")
+      @core = [["http://rs.tdwg.org/dwc/terms/taxonID",
+                "http://rs.tdwg.org/dwc/terms/scientificName",
+                "http://rs.tdwg.org/dwc/terms/kingdom",
+                "http://rs.tdwg.org/dwc/terms/phylum",
+                "http://rs.tdwg.org/dwc/terms/class",
+                "http://rs.tdwg.org/dwc/terms/order",
+                "http://rs.tdwg.org/dwc/terms/family",
+                "http://rs.tdwg.org/dwc/terms/nomenclaturalCode"]]
+      @names.each do |n|
+        @core << [n[:taxon_id], n[:name_string],
+                  n[:kingdom], n[:phylum], n[:klass], n[:order], n[:family],
+                  n[:code]]
+      end
+      @extensions << {
+        data: [[
+          "http://rs.tdwg.org/dwc/terms/taxonID",
+          "http://rs.tdwg.org/dwc/terms/vernacularName",
+          "http://purl.org/dc/terms/language"
+        ]],
+        file_name: "vernacular_names.txt",
+        row_type: "http://rs.gbif.org/terms/1.0/VernacularName"
+      }
+
+      @vernaculars.each do |v|
+        @extensions[-1][:data] << [v[:taxon_id], v[:vern], v[:lang]]
+      end
+
+      @eml = {
+        id: @uuid,
+        title: @title,
+        authors: [
+          { first_name: "G. F.",
+            last_name: "Clements"
+          },
+          { first_name: "T. S.",
+            last_name: "Schulenberg"
+          },
+          { first_name: "M. J.",
+            last_name: "Iliff"
+          },
+          { first_name: "S. M.",
+            last_name: "Billerman"
+          },
+          { first_name: "T. A.",
+            last_name: "Fredericks"
+          },
+          { first_name: "B. L.",
+            last_name: "Sullivan"
+          },
+          { first_name: "C. L.",
+            last_name: "Wood"
+          },
+        ],
+        metadata_providers: [
+          { first_name: "Dmitry",
+            last_name: "Mozzherin",
+            email: "dmozzherin@gmail.com" }
+        ],
+        abstract: "The eBird/Clements Checklist of Birds of the World" \
+        ": v2019. Downloaded from " \
+        "https://www.birds.cornell.edu/clementschecklist/download/",
+        url: @url
+      }
+      super
+    end
+  end
+end