dwca_hunter 0.5.5 → 0.7.0

data/lib/dwca_hunter/resources/arctos.rb

@@ -1,16 +1,16 @@
-# encoding: utf-8
+# frozen_string_literal: true
+
 module DwcaHunter
   class ResourceArctos < DwcaHunter::Resource
-
     def initialize(opts = {})
-      @command = 'arctos'
-      @title = 'Arctos'
-      @url = 'https://www.dropbox.com/s/3rmny5d8cfm9mmp/arctos.tar.gz?dl=1'
-      @UUID =  'eea8315d-a244-4625-859a-226675622312'
+      @command = "arctos"
+      @title = "Arctos"
+      @url = "https://www.dropbox.com/s/3rmny5d8cfm9mmp/arctos.tar.gz?dl=1"
+      @UUID = "eea8315d-a244-4625-859a-226675622312"
       @download_path = File.join(Dir.tmpdir,
-                                 'dwca_hunter',
-                                 'arctos',
-                                 'data.zip')
+                                 "dwca_hunter",
+                                 "arctos",
+                                 "data.zip")
       @synonyms = []
       @names = []
       @vernaculars = []
@@ -22,7 +22,7 @@ module DwcaHunter
 
     def download
       puts "Downloading cached verion of the file. Ask Arctos to generate new."
-        `curl -s -L #{@url} -o #{@download_path}`
+      `curl -s -L #{@url} -o #{@download_path}`
     end
 
     def unpack
@@ -30,7 +30,7 @@ module DwcaHunter
     end
 
     def make_dwca
-      DwcaHunter::logger_write(self.object_id, 'Extracting data')
+      DwcaHunter.logger_write(object_id, "Extracting data")
       get_names
       generate_dwca
     end
@@ -45,121 +45,123 @@ module DwcaHunter
     end
 
     def collect_vernaculars
-      file = CSV.open(File.join(@download_dir, 'common_name.csv'),
-        headers: true)
+      file = CSV.open(File.join(@download_dir, "common_name.csv"),
+                      headers: true)
       file.each_with_index do |row, i|
+        canonical = row["SCIENTIFIC_NAME"]
+        vernacular_name_string = row["COMMON_NAME"]
 
-        canonical = row['SCIENTIFIC_NAME']
-        vernacular_name_string = row['COMMON_NAME']
-
-        if @vernaculars_hash.has_key?(canonical)
+        if @vernaculars_hash.key?(canonical)
           @vernaculars_hash[canonical] << vernacular_name_string
         else
           @vernaculars_hash[canonical] = [vernacular_name_string]
         end
 
-        puts "Processed %s vernaculars" % i if i % 10000 == 0
+        puts "Processed %s vernaculars" % i if i % 10_000 == 0
       end
     end
 
     def collect_synonyms
-      file = CSV.open(File.join(@download_dir, 'relationships.csv'),
-       headers: true)
+      file = CSV.open(File.join(@download_dir, "relationships.csv"),
+                      headers: true)
       file.each_with_index do |row, i|
-        canonical = row['scientific_name']
-        if @synonyms_hash.has_key?(canonical)
+        canonical = row["scientific_name"]
+        if @synonyms_hash.key?(canonical)
           @synonyms_hash[canonical] <<
-          { name_string: row['related_name'], status: row['TAXON_RELATIONSHIP']}
+            { name_string: row["related_name"], status: row["TAXON_RELATIONSHIP"] }
         else
           @synonyms_hash[canonical] = [
-          { name_string: row['related_name'], status: row['TAXON_RELATIONSHIP']}
+            { name_string: row["related_name"], status: row["TAXON_RELATIONSHIP"] }
           ]
         end
-        puts "Processed %s synonyms" % i if i % 10000 == 0
+        puts "Processed %s synonyms" % i if i % 10_000 == 0
       end
     end
 
     def collect_names
       @names_index = {}
-      file = CSV.open(File.join(@download_dir, 'classification.csv'),
-       headers: true)
+      file = CSV.open(File.join(@download_dir, "classification.csv"),
+                      headers: true)
       file.each_with_index do |row, i|
-        next unless  row['display_name']
-        name_string = row['display_name'].gsub(/<\/?i>/,'')
-        canonical = row['scientific_name']
-        kingdom = row['kingdom']
-        phylum = row['phylum']
-        klass = row['phylclass']
-        subclass = row['subclass']
-        order = row['phylorder']
-        suborder = row['suborder']
-        superfamily = row['superfamily']
-        family = row['family']
-        subfamily = row['subfamily']
-        tribe = row['tribe']
-        genus = row['genus']
-        subgenus = row['subgenus']
-        species = row['species']
-        subspecies = row['subspecies']
-        code = row['nomenclatural_code']
-
-        taxon_id = "ARCT_#{i+1}"
+        next unless row["display_name"]
+
+        name_string = row["display_name"].gsub(%r{</?i>}, "")
+        canonical = row["scientific_name"]
+        kingdom = row["kingdom"]
+        phylum = row["phylum"]
+        klass = row["phylclass"]
+        subclass = row["subclass"]
+        order = row["phylorder"]
+        suborder = row["suborder"]
+        superfamily = row["superfamily"]
+        family = row["family"]
+        subfamily = row["subfamily"]
+        tribe = row["tribe"]
+        genus = row["genus"]
+        subgenus = row["subgenus"]
+        species = row["species"]
+        subspecies = row["subspecies"]
+        code = row["nomenclatural_code"]
+
+        taxon_id = "ARCT_#{i + 1}"
         @names << { taxon_id: taxon_id,
-          name_string: name_string,
-          kingdom: kingdom,
-          phylum: phylum,
-          klass: klass,
-          order: order,
-          family: family,
-          genus: genus,
-          code: code,
-        }
+                    name_string: name_string,
+                    kingdom: kingdom,
+                    phylum: phylum,
+                    klass: klass,
+                    order: order,
+                    family: family,
+                    genus: genus,
+                    code: code }
 
         update_vernacular(taxon_id, canonical)
         update_synonym(taxon_id, canonical)
-        puts "Processed %s names" % i if i % 10000 == 0
+        puts "Processed %s names" % i if i % 10_000 == 0
       end
     end
 
     def update_vernacular(taxon_id, canonical)
-      return unless @vernaculars_hash.has_key?(canonical)
+      return unless @vernaculars_hash.key?(canonical)
+
       @vernaculars_hash[canonical].each do |vern|
         @vernaculars << { taxon_id: taxon_id, vern: vern }
       end
     end
 
     def update_synonym(taxon_id, canonical)
-      return unless @synonyms_hash.has_key?(canonical)
+      return unless @synonyms_hash.key?(canonical)
+
       @synonyms_hash[canonical].each do |syn|
         @synonyms << { taxon_id: taxon_id, name_string: syn[:name_string],
-          status: syn[:status] }
+                       status: syn[:status] }
       end
     end
 
     def generate_dwca
-      DwcaHunter::logger_write(self.object_id,
-                               'Creating DarwinCore Archive file')
-      @core = [['http://rs.tdwg.org/dwc/terms/taxonID',
-        'http://rs.tdwg.org/dwc/terms/scientificName',
-        'http://rs.tdwg.org/dwc/terms/kingdom',
-        'http://rs.tdwg.org/dwc/terms/phylum',
-        'http://rs.tdwg.org/dwc/terms/class',
-        'http://rs.tdwg.org/dwc/terms/order',
-        'http://rs.tdwg.org/dwc/terms/family',
-        'http://rs.tdwg.org/dwc/terms/genus',
-        'http://rs.tdwg.org/dwc/terms/nomenclaturalCode',
-        ]]
+      DwcaHunter.logger_write(object_id,
+                              "Creating DarwinCore Archive file")
+      @core = [["http://rs.tdwg.org/dwc/terms/taxonID",
+                "http://rs.tdwg.org/dwc/terms/scientificName",
+                "http://rs.tdwg.org/dwc/terms/kingdom",
+                "http://rs.tdwg.org/dwc/terms/phylum",
+                "http://rs.tdwg.org/dwc/terms/class",
+                "http://rs.tdwg.org/dwc/terms/order",
+                "http://rs.tdwg.org/dwc/terms/family",
+                "http://rs.tdwg.org/dwc/terms/genus",
+                "http://rs.tdwg.org/dwc/terms/nomenclaturalCode"]]
       @names.each do |n|
         @core << [n[:taxon_id], n[:name_string],
-          n[:kingdom], n[:phylum], n[:klass], n[:order], n[:family],
-          n[:genus], n[:code]]
+                  n[:kingdom], n[:phylum], n[:klass], n[:order], n[:family],
+                  n[:genus], n[:code]]
       end
       @extensions << {
         data: [[
-          'http://rs.tdwg.org/dwc/terms/taxonID',
-          'http://rs.tdwg.org/dwc/terms/vernacularName']],
-        file_name: 'vernacular_names.txt',
-        row_type: 'http://rs.gbif.org/terms/1.0/VernacularName' }
+          "http://rs.tdwg.org/dwc/terms/taxonID",
+          "http://rs.tdwg.org/dwc/terms/vernacularName"
+        ]],
+        file_name: "vernacular_names.txt",
+        row_type: "http://rs.gbif.org/terms/1.0/VernacularName"
+      }
 
       @vernaculars.each do |v|
         @extensions[-1][:data] << [v[:taxon_id], v[:vern]]
@@ -167,12 +169,12 @@ module DwcaHunter
 
       @extensions << {
         data: [[
-          'http://rs.tdwg.org/dwc/terms/taxonID',
-          'http://rs.tdwg.org/dwc/terms/scientificName',
-          'http://rs.tdwg.org/dwc/terms/taxonomicStatus',
-          ]],
-        file_name: 'synonyms.txt',
-        }
+          "http://rs.tdwg.org/dwc/terms/taxonID",
+          "http://rs.tdwg.org/dwc/terms/scientificName",
+          "http://rs.tdwg.org/dwc/terms/taxonomicStatus"
+        ]],
+        file_name: "synonyms.txt"
+      }
       @synonyms.each do |s|
         @extensions[-1][:data] << [s[:taxon_id], s[:name_string], s[:status]]
       end
@@ -180,14 +182,14 @@ module DwcaHunter
         id: @uuid,
         title: @title,
         authors: [
-          {email: 'dustymc at gmail dot com'}
-      ],
+          { email: "dustymc at gmail dot com" }
+        ],
         metadata_providers: [
-          { first_name: 'Dmitry',
-            last_name: 'Mozzherin',
-            email: 'dmozzherin@gmail.com' }
-      ],
-        abstract: 'Arctos is an ongoing effort to integrate access to specimen data, collection-management tools, and external resources on the internet.',
+          { first_name: "Dmitry",
+            last_name: "Mozzherin",
+            email: "dmozzherin@gmail.com" }
+        ],
+        abstract: "Arctos is an ongoing effort to integrate access to specimen data, collection-management tools, and external resources on the internet.",
         url: @url
       }
       super

data/lib/dwca_hunter/resources/clements.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module DwcaHunter
+  class ResourceClements < DwcaHunter::Resource
+    def initialize(opts = {})
+      @command = "clements-ebird"
+      @title = "The eBird/Clements Checklist of Birds of the World"
+      @url = "https://uofi.box.com/shared/static/b4n8zqa99hq9rdga27skkh3870yhujgo.csv"
+      @UUID = "577c0b56-4a3c-4314-8724-14b304f601de"
+      @download_path = File.join(Dir.tmpdir,
+                                 "dwca_hunter",
+                                 "clements",
+                                 "data.csv")
+      @synonyms = []
+      @names = []
+      @vernaculars = []
+      @extensions = []
+      @synonyms_hash = {}
+      @vernaculars_hash = {}
+      super(opts)
+    end
+
+    def download
+      puts "Downloading cached and modified version of the file."
+      puts "Go to https://www.birds.cornell.edu/clementschecklist/download/ " \
+        "for updates."
+      `curl -s -L #{@url} -o #{@download_path}`
+    end
+
+    def unpack
+    end
+
+    def make_dwca
+      DwcaHunter.logger_write(object_id, "Extracting data")
+      get_names
+      generate_dwca
+    end
+
+    private
+
+    def get_names
+      Dir.chdir(@download_dir)
+      collect_names
+    end
+
+    def collect_names
+      @names_index = {}
+      file = CSV.open(File.join(@download_dir, "data.csv"),
+                      headers: true)
+      file.each_with_index do |row, i|
+        name_string = row["scientific name"]
+        canonical = name_string
+        kingdom = "Animalia"
+        phylum = "Chordata"
+        klass = "Aves"
+        order = row["order"]
+        family = row["family"]
+        code = "ICZN"
+
+        taxon_id = "gn_#{i + 1}"
+        @names << { taxon_id: taxon_id,
+                    name_string: name_string,
+                    kingdom: kingdom,
+                    phylum: phylum,
+                    klass: klass,
+                    order: order,
+                    family: family,
+                    code: code }
+
+        if row["English name"].to_s != ""
+            @vernaculars << {
+              taxon_id: taxon_id,
+              vern: row["English name"],
+              lang: "end"
+            }
+        end
+
+        puts "Processed %s names" % i if i % 10_000 == 0
+      end
+    end
+
+    def generate_dwca
+      DwcaHunter.logger_write(object_id,
+                              "Creating DarwinCore Archive file")
+      @core = [["http://rs.tdwg.org/dwc/terms/taxonID",
+                "http://rs.tdwg.org/dwc/terms/scientificName",
+                "http://rs.tdwg.org/dwc/terms/kingdom",
+                "http://rs.tdwg.org/dwc/terms/phylum",
+                "http://rs.tdwg.org/dwc/terms/class",
+                "http://rs.tdwg.org/dwc/terms/order",
+                "http://rs.tdwg.org/dwc/terms/family",
+                "http://rs.tdwg.org/dwc/terms/nomenclaturalCode"]]
+      @names.each do |n|
+        @core << [n[:taxon_id], n[:name_string],
+                  n[:kingdom], n[:phylum], n[:klass], n[:order], n[:family],
+                  n[:code]]
+      end
+      @extensions << {
+        data: [[
+          "http://rs.tdwg.org/dwc/terms/taxonID",
+          "http://rs.tdwg.org/dwc/terms/vernacularName",
+          "http://purl.org/dc/terms/language"
+        ]],
+        file_name: "vernacular_names.txt",
+        row_type: "http://rs.gbif.org/terms/1.0/VernacularName"
+      }
+
+      @vernaculars.each do |v|
+        @extensions[-1][:data] << [v[:taxon_id], v[:vern], v[:lang]]
+      end
+
+      @eml = {
+        id: @uuid,
+        title: @title,
+        authors: [
+          { first_name: "G. F.",
+            last_name: "Clements"
+          },
+          { first_name: "T. S.",
+            last_name: "Schulenberg"
+          },
+          { first_name: "M. J.",
+            last_name: "Iliff"
+          },
+          { first_name: "S. M.",
+            last_name: "Billerman"
+          },
+          { first_name: "T. A.",
+            last_name: "Fredericks"
+          },
+          { first_name: "B. L.",
+            last_name: "Sullivan"
+          },
+          { first_name: "C. L.",
+            last_name: "Wood"
+          },
+        ],
+        metadata_providers: [
+          { first_name: "Dmitry",
+            last_name: "Mozzherin",
+            email: "dmozzherin@gmail.com" }
+        ],
+        abstract: "The eBird/Clements Checklist of Birds of the World" \
+        ": v2019. Downloaded from " \
+        "https://www.birds.cornell.edu/clementschecklist/download/",
+        url: @url
+      }
+      super
+    end
+  end
+end

data/lib/dwca_hunter/resources/freebase.rb

@@ -1,15 +1,15 @@
-# encoding: utf-8
+# frozen_string_literal: true
 
 module DwcaHunter
   class ResourceFreebase < DwcaHunter::Resource
     def initialize(opts = {})
       @command = "freebase"
-      @title = 'Freebase'
-      @uuid = 'bacd21f0-44e0-43e2-914c-70929916f257'
+      @title = "Freebase"
+      @uuid = "bacd21f0-44e0-43e2-914c-70929916f257"
       @download_path = File.join(Dir.tmpdir,
-                                 'dwca_hunter',
-                                 'freebase',
-                                 'data.json')
+                                 "dwca_hunter",
+                                 "freebase",
+                                 "data.json")
       @data = []
       @all_taxa = {}
       @cleaned_taxa = {}
@@ -27,11 +27,11 @@ module DwcaHunter
     end
 
     def download
-      DwcaHunter::logger_write(self.object_id,
-                               'Querying freebase for species information...')
+      DwcaHunter.logger_write(object_id,
+                              "Querying freebase for species information...")
       q = {
         query: [{
-          type: '/biology/organism_classification',
+          type: "/biology/organism_classification",
           id: nil,
           guid: nil,
           name: nil,
@@ -41,16 +41,16 @@ module DwcaHunter
             id: nil,
             guid: nil,
             scientific_name: nil,
-            optional: true,
-          },
+            optional: true
+          }
         }],
-        cursor: true,
+        cursor: true
       }
 
       run_query(q)
 
       data = JSON.pretty_generate @data
-      f = open(@download_path, 'w:utf-8')
+      f = open(@download_path, "w:utf-8")
       f.write(data)
       f.close
     end
@@ -60,31 +60,32 @@ module DwcaHunter
     def run_query(q)
       count = 0
       requests_num = 0
-      while true
+      loop do
         freebase_url = "http://api.freebase.com/api/service/mqlread?query=%s" %
-          URI.encode(q.to_json)
+                       URI.encode(q.to_json)
         res = JSON.load RestClient.get(freebase_url)
         requests_num += 1
-        break if res['result'] == nil || res['result'].empty?
+        break if res["result"].nil? || res["result"].empty?
+
         if requests_num % 10 == 0
-          DwcaHunter::logger_write(self.object_id,
-                                   "Received %s names" % count)
+          DwcaHunter.logger_write(object_id,
+                                  "Received %s names" % count)
         end
-        count += res['result'].size
-        res['result'].each { |d| @data << d }
-        q[:cursor] = res['cursor']
+        count += res["result"].size
+        res["result"].each { |d| @data << d }
+        q[:cursor] = res["cursor"]
       end
     end
 
     def organize_data
-      @data = JSON.load(open(@download_path, 'r:utf-8').read)
+      @data = JSON.load(open(@download_path, "r:utf-8").read)
       @data.each do |d|
-        scientific_name = d['scientific_name'].to_s
+        scientific_name = d["scientific_name"].to_s
         id = d["id"]
-        parent_id = d['higher_classification'] ?
-                    d['higher_classification']["id"] :
+        parent_id = d["higher_classification"] ?
+                    d["higher_classification"]["id"] :
                     nil
-        synonyms = d['synonym_scientific_name']
+        synonyms = d["synonym_scientific_name"]
         @all_taxa[id] = { id: id,
                           parent_id: parent_id,
                           scientific_name: scientific_name,
@@ -93,6 +94,7 @@ module DwcaHunter
 
       @all_taxa.each do |k, v|
         next unless v[:scientific_name] && v[:scientific_name].strip != ""
+
         parent_id = v[:parent_id]
         until (@all_taxa[parent_id] &&
                 @all_taxa[parent_id][:scientific_name]) || parent_id.nil?
@@ -103,29 +105,28 @@ module DwcaHunter
         v[:parent_id] = parent_id
         @cleaned_taxa[k] = v
       end
-
     end
 
     def generate_dwca
-      DwcaHunter::logger_write(self.object_id,
-                               'Creating DarwinCore Archive file')
-      @core = [['http://rs.tdwg.org/dwc/terms/taxonID',
-                'http://rs.tdwg.org/dwc/terms/scientificName',
-                'http://rs.tdwg.org/dwc/terms/parentNameUsageID']]
+      DwcaHunter.logger_write(object_id,
+                              "Creating DarwinCore Archive file")
+      @core = [["http://rs.tdwg.org/dwc/terms/taxonID",
+                "http://rs.tdwg.org/dwc/terms/scientificName",
+                "http://rs.tdwg.org/dwc/terms/parentNameUsageID"]]
 
       @extensions << { data: [[
-        'http://rs.tdwg.org/dwc/terms/TaxonID',
-        'http://rs.tdwg.org/dwc/terms/scientificName',
-      ]], file_name: 'synonyms.txt' }
-      DwcaHunter::logger_write(self.object_id,
-                    'Creating synonyms extension for DarwinCore Archive file')
+        "http://rs.tdwg.org/dwc/terms/TaxonID",
+        "http://rs.tdwg.org/dwc/terms/scientificName"
+      ]], file_name: "synonyms.txt" }
+      DwcaHunter.logger_write(object_id,
+                              "Creating synonyms extension for DarwinCore Archive file")
       count = 0
-      @cleaned_taxa.each do |key, taxon|
+      @cleaned_taxa.each do |_key, taxon|
         count += 1
         @core << [taxon[:id], taxon[:scientific_name], taxon[:parent_id]]
         if count % BATCH_SIZE == 0
-          DwcaHunter::logger_write(self.object_id,
-                                 "Traversing %s extension data record" % count)
+          DwcaHunter.logger_write(object_id,
+                                  "Traversing %s extension data record" % count)
         end
         taxon[:synonyms].each do |name|
           @extensions[-1][:data] << [taxon[:id], name]
@@ -134,19 +135,20 @@ module DwcaHunter
       @eml = {
         id: @uuid,
         title: @title,
-        license: 'http://creativecommons.org/licenses/by-sa/3.0/',
+        license: "http://creativecommons.org/licenses/by-sa/3.0/",
         authors: [
-          { url: 'http://www.freebase.com/home' }],
-        abstract: 'An entity graph of people, places and things, ' +
-                  'built by a community that loves open data.',
+          { url: "http://www.freebase.com/home" }
+        ],
+        abstract: "An entity graph of people, places and things, " \
+                  "built by a community that loves open data.",
         metadata_providers: [
-          { first_name: 'Dmitry',
-            last_name: 'Mozzherin',
-            email: 'dmozzherin@mbl.edu' }],
-        url: 'http://www.freebase.com/home'
+          { first_name: "Dmitry",
+            last_name: "Mozzherin",
+            email: "dmozzherin@mbl.edu" }
+        ],
+        url: "http://www.freebase.com/home"
       }
       super
     end
-
   end
 end