dwca_hunter 0.7.1 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 60327e105c53c226f322e3a7272bdc5747d73fac0124887b024f99e3c39c985b
-  data.tar.gz: '09660f8b5feccfaf4caeaec277db4dc4729a973196a77bb02860947ef55bd272'
+  metadata.gz: 26106d03a805f473871c3092a84b8d0498f991ff809a7e3f259f1e20d29bca80
+  data.tar.gz: 28ac068cf264870f67ffdc6697801c683ffec0e101c582ddbd50aa77b7831b8f
 SHA512:
-  metadata.gz: 9b0a621f85535f421eef5a8550ce653c4f3483f563c7b19934a76e8f30b0cdf17e7a8c59945ea31455c57a350a14d345993f5fe6b91d656f5eb40317da6b1af9
-  data.tar.gz: 00a54b23a8588e6d304d35bb8756f633fb409777f07908525d257f4b1a23c5956a9683863b491f50e8d772c8b41343f909e6cb252a326d4f2e662f96f37826ed
+  metadata.gz: 3a1e49e6db3aa0bb79616a512feb8c05438f3534d6c7bdd43a0ae32981e32f83ccd74bdcd682130bdf044dd893e0017eeddf533b170109338bb8d95f0379e2b7
+  data.tar.gz: 35bb769a822639d26380ad70db0b7aeed22483ce7184e62b7fad38c8d5c3b7ab5ea3b82e3dce00bb012bcc18e719077c2c3bb4a2b53830f7babd693c26aa2caf

data/.rubocop.yml

@@ -1,6 +1,6 @@
 AllCops:
   NewCops: disable
-  TargetRubyVersion: 2.6.6
+  TargetRubyVersion: 3.0.0
   Exclude:
     - bin/**/*
     - db/**/*
@@ -28,6 +28,14 @@ Metrics/ClassLength:
 Metrics/MethodLength:
   Enabled: false
 
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/PerceivedComplexity:
+  Enabled: false
+
+
+
 Naming/FileName:
   Exclude:
     - Gemfile

data/.ruby-version

@@ -1 +1 @@
-2.6.6
+3.0.0

data/Gemfile.lock

@@ -1,23 +1,26 @@
 PATH
   remote: .
   specs:
-    dwca_hunter (0.7.0)
-      biodiversity (~> 4)
-      dwc-archive (~> 1.1.1)
+    dwca_hunter (0.7.2)
+      biodiversity (~> 5.1.2)
+      dwc-archive (~> 1.1.3)
       gn_uuid (~> 0.5)
       htmlentities (~> 4.3)
       nokogiri (~> 1.11)
-      rest-client (~> 2.0)
+      rest-client (~> 2.1)
       ruby-xz (~> 1.0)
-      thor (~> 0.19)
+      rubyzip (~> 2.3)
+      thor (~> 1.1)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    ast (2.4.1)
-    biodiversity (4.1.0)
-      ffi (~> 1.11)
-    byebug (10.0.2)
+    ast (2.4.2)
+    backport (1.1.2)
+    benchmark (0.1.1)
+    biodiversity (5.1.2)
+      ffi (~> 1.14)
+    byebug (11.1.3)
     coveralls (0.8.23)
       json (>= 1.8, < 3)
       simplecov (~> 0.16.1)
@@ -25,19 +28,25 @@ GEM
       thor (>= 0.19.4, < 2.0)
       tins (~> 1.6)
     diff-lcs (1.4.4)
-    docile (1.3.4)
+    docile (1.3.5)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    dwc-archive (1.1.1)
-      biodiversity (~> 4)
-      nokogiri (~> 1.10)
+    dwc-archive (1.1.3)
+      biodiversity (~> 5.1.2)
+      nokogiri (~> 1.11)
+    e2mmap (0.1.0)
     ffi (1.14.2)
     gn_uuid (0.5.1)
     htmlentities (4.3.4)
     http-accept (1.7.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
+    jaro_winkler (1.5.4)
     json (2.5.1)
+    kramdown (2.3.0)
+      rexml
+    kramdown-parser-gfm (1.1.0)
+      kramdown (~> 2.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2020.1104)
@@ -58,6 +67,8 @@ GEM
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
+    reverse_markdown (2.0.0)
+      nokogiri
     rexml (3.2.4)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
@@ -72,46 +83,64 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
     rspec-support (3.10.1)
-    rubocop (0.93.1)
+    rubocop (1.9.0)
       parallel (~> 1.10)
-      parser (>= 2.7.1.5)
+      parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8)
+      regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 0.6.0)
+      rubocop-ast (>= 1.2.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (1.4.0)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.4.1)
       parser (>= 2.7.1.5)
     ruby-progressbar (1.11.0)
     ruby-xz (1.0.0)
+    rubyzip (2.3.0)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
+    solargraph (0.40.2)
+      backport (~> 1.1)
+      benchmark
+      bundler (>= 1.17.2)
+      e2mmap
+      jaro_winkler (~> 1.5)
+      kramdown (~> 2.3)
+      kramdown-parser-gfm (~> 1.1)
+      parser (~> 3.0)
+      reverse_markdown (>= 1.0.5, < 3)
+      rubocop (>= 0.52)
+      thor (~> 1.0)
+      tilt (~> 2.0)
+      yard (~> 0.9, >= 0.9.24)
     sync (0.5.0)
     term-ansicolor (1.7.1)
       tins (~> 1.0)
-    thor (0.20.3)
-    tins (1.26.0)
+    thor (1.1.0)
+    tilt (2.0.10)
+    tins (1.28.0)
       sync
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
-    unicode-display_width (1.7.0)
+    unicode-display_width (2.0.0)
+    yard (0.9.26)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 2.0)
-  byebug (~> 10.0)
+  bundler (~> 2.2)
+  byebug (~> 11.1)
   coveralls (~> 0.8)
   dwca_hunter!
   rake (~> 13.0)
-  rspec (~> 3.9)
-  rubocop (~> 0.84)
+  rspec (~> 3.10)
+  rubocop (~> 1.9)
+  solargraph (~> 0.40)
 
 BUNDLED WITH
-   2.1.4
+   2.2.7

data/dwca_hunter.gemspec

@@ -6,7 +6,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "dwca_hunter/version"
 
 Gem::Specification.new do |gem|
-  gem.required_ruby_version = ">= 2.6.6"
+  gem.required_ruby_version = ">= 3.0.0"
   gem.name          = "dwca_hunter"
   gem.version       = DwcaHunter.version
   gem.license       = "MIT"
@@ -26,19 +26,21 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^exe/}) { |f| File.basename(f) }
   gem.require_paths = ["lib"]
 
-  gem.add_dependency "biodiversity", "~> 4"
-  gem.add_dependency "dwc-archive", "~> 1.1.1"
+  gem.add_dependency "biodiversity", "~> 5.1.2"
+  gem.add_dependency "dwc-archive", "~> 1.1.3"
   gem.add_dependency "gn_uuid", "~> 0.5"
   gem.add_dependency "htmlentities", "~> 4.3"
   gem.add_dependency "nokogiri", "~> 1.11"
-  gem.add_dependency "rest-client", "~> 2.0"
+  gem.add_dependency "rest-client", "~> 2.1"
   gem.add_dependency "ruby-xz", "~> 1.0"
-  gem.add_dependency "thor", "~> 0.19"
+  gem.add_dependency "rubyzip", "~> 2.3"
+  gem.add_dependency "thor", "~> 1.1"
 
-  gem.add_development_dependency "bundler", "~> 2.0"
-  gem.add_development_dependency "byebug", "~> 10.0"
+  gem.add_development_dependency "bundler", "~> 2.2"
+  gem.add_development_dependency "byebug", "~> 11.1"
   gem.add_development_dependency "coveralls", "~> 0.8"
   gem.add_development_dependency "rake", "~> 13.0"
-  gem.add_development_dependency "rspec", "~> 3.9"
-  gem.add_development_dependency "rubocop", "~> 0.84"
+  gem.add_development_dependency "rspec", "~> 3.10"
+  gem.add_development_dependency "rubocop", "~> 1.9"
+  gem.add_development_dependency "solargraph", "~> 0.40"
 end

data/exe/dwcahunter

@@ -51,7 +51,6 @@ class DwcaHunterCLI < Thor
                 resource.abbr =~ /#{search}/i)
   end
 
-  # rubocop:disable Metrics/AbcSize
   def print_table(data)
     table = [data.first.keys.map { |k| k.to_s.capitalize }] + data.map(&:values)
     widths = table_widths(table)
@@ -59,7 +58,6 @@ class DwcaHunterCLI < Thor
     format = widths.collect { |n| "%-#{n}s" }.join("  ")
     table.each { |line| printf "  #{format}  \n", *line }
   end
-  # rubocop:enable Metrics/AbcSize
 
   def table_widths(table)
     table.each_with_object([]) do |line, widths|

data/lib/dwca_hunter.rb

@@ -7,11 +7,13 @@ require "dwca_hunter/resource"
 require "fileutils"
 require "htmlentities"
 require "json"
+require "zip"
 require "logger"
 require "net/http"
 require "rest_client"
 require "tmpdir"
 require "uri"
+require "cgi"
 
 Dir[File.join(__dir__, "dwca_hunter", "*.rb")].
   each { |f| require f }
@@ -53,13 +55,14 @@ module DwcaHunter
     end
 
     def normalize_authors(auth)
-      reg = Regexp.new(/^([\(]?)(.*?)(([\s,\)][^[:upper:]]*)?$)/)
-      auth = auth.gsub(/duPont/, 'du Pont')
+      reg = Regexp.new(/^(\(?)(.*?)(([\s,)][^[:upper:]]*)?$)/)
+      auth = auth.gsub(/duPont/, "du Pont")
       match = reg.match(auth)
       return auth if match.nil?
+
       a1, a2, a3 = match[1..3]
-      a2mod = a2.gsub('&', ',')
-      ary2 = a2mod.split(',').map(&:strip)
+      a2mod = a2.gsub("&", ",")
+      ary2 = a2mod.split(",").map(&:strip)
       a2 = move_initials(ary2) if ary2.size > 1
       "#{a1}#{a2}#{a3}"
     end
@@ -73,8 +76,8 @@ module DwcaHunter
         end
         match = /^([[:upper:]]{1,4})(\sJr)?$/.match(a)
         if !match.nil?
-          initialls = match[1].split('').join('. ')
-          res[-1] = "#{initialls}. #{res[-1]}#{match[2].to_s}"
+          initialls = match[1].split("").join(". ")
+          res[-1] = "#{initialls}. #{res[-1]}#{match[2]}"
         else
           res << a
         end
@@ -83,4 +86,3 @@ module DwcaHunter
     end
   end
 end
-

data/lib/dwca_hunter/resource.rb

@@ -4,7 +4,12 @@ module DwcaHunter
 
     def self.unzip(file, dir = nil)
       Dir.chdir(dir) if dir
-      `unzip -qq -u #{file} > /dev/null 2>&1`
+      Zip::File.open(file) do |zip_file|
+        zip_file.each do |entry|
+          puts "Extracting #{entry.name}"
+          entry.extract
+        end
+      end
     end
 
     def self.gunzip(file, dir = nil)
@@ -13,8 +18,8 @@ module DwcaHunter
     end
 
     def initialize(opts)
-      @needs_download = !(opts[:download] == false)
-      @needs_unpack = !(opts[:unpack] == false)
+      @needs_download = (opts[:download] != false)
+      @needs_unpack = (opts[:unpack] != false)
       @download_dir, @download_file = File.split(@download_path)
       prepare_path if needs_download?
     end

data/lib/dwca_hunter/resources/arctos.rb

@@ -5,12 +5,12 @@ module DwcaHunter
     def initialize(opts = {})
       @command = "arctos"
       @title = "Arctos"
-      @url = "https://www.dropbox.com/s/3rmny5d8cfm9mmp/arctos.tar.gz?dl=1"
+      @url = "http://arctos.database.museum/cache/gn_merge.tgz"
       @UUID = "eea8315d-a244-4625-859a-226675622312"
       @download_path = File.join(Dir.tmpdir,
                                  "dwca_hunter",
                                  "arctos",
-                                 "data.zip")
+                                 "data.tar.gz")
       @synonyms = []
       @names = []
       @vernaculars = []
@@ -21,8 +21,8 @@ module DwcaHunter
     end
 
     def download
-      puts "Downloading cached verion of the file. Ask Arctos to generate new."
-      `curl -s -L #{@url} -o #{@download_path}`
+      puts "Downloading Arctos file."
+      `curl -s #{@url} -o #{@download_path}`
     end
 
     def unpack
@@ -45,11 +45,11 @@ module DwcaHunter
     end
 
     def collect_vernaculars
-      file = CSV.open(File.join(@download_dir, "common_name.csv"),
+      file = CSV.open(File.join(@download_dir, "globalnames_commonname.csv"),
                       headers: true)
       file.each_with_index do |row, i|
-        canonical = row["SCIENTIFIC_NAME"]
-        vernacular_name_string = row["COMMON_NAME"]
+        canonical = row["scientific_name"]
+        vernacular_name_string = row["common_name"]
 
         if @vernaculars_hash.key?(canonical)
           @vernaculars_hash[canonical] << vernacular_name_string
@@ -57,66 +57,63 @@ module DwcaHunter
           @vernaculars_hash[canonical] = [vernacular_name_string]
         end
 
-        puts "Processed %s vernaculars" % i if i % 10_000 == 0
+        puts "Processed #{i} vernaculars"if (i % 100_000).zero?
       end
     end
 
     def collect_synonyms
-      file = CSV.open(File.join(@download_dir, "relationships.csv"),
+      file = CSV.open(File.join(@download_dir, "globalnames_relationships.csv"),
                       headers: true)
       file.each_with_index do |row, i|
         canonical = row["scientific_name"]
         if @synonyms_hash.key?(canonical)
           @synonyms_hash[canonical] <<
-            { name_string: row["related_name"], status: row["TAXON_RELATIONSHIP"] }
+            { name_string: row["related_name"], status: row["taxon_relationship"] }
         else
           @synonyms_hash[canonical] = [
-            { name_string: row["related_name"], status: row["TAXON_RELATIONSHIP"] }
+            { name_string: row["related_name"], status: row["taxon_relationship"] }
           ]
         end
-        puts "Processed %s synonyms" % i if i % 10_000 == 0
+        puts "Processed #{i} synonyms" if (i % 100_000).zero?
       end
     end
 
     def collect_names
       @names_index = {}
-      file = CSV.open(File.join(@download_dir, "classification.csv"),
+      file = CSV.open(File.join(@download_dir, "globalnames_classification.csv"),
                       headers: true)
-      file.each_with_index do |row, i|
-        next unless row["display_name"]
-
-        name_string = row["display_name"].gsub(%r{</?i>}, "")
-        canonical = row["scientific_name"]
-        kingdom = row["kingdom"]
-        phylum = row["phylum"]
-        klass = row["phylclass"]
-        subclass = row["subclass"]
-        order = row["phylorder"]
-        suborder = row["suborder"]
-        superfamily = row["superfamily"]
-        family = row["family"]
-        subfamily = row["subfamily"]
-        tribe = row["tribe"]
-        genus = row["genus"]
-        subgenus = row["subgenus"]
-        species = row["species"]
-        subspecies = row["subspecies"]
-        code = row["nomenclatural_code"]
-
-        taxon_id = "ARCT_#{i + 1}"
-        @names << { taxon_id: taxon_id,
-                    name_string: name_string,
-                    kingdom: kingdom,
-                    phylum: phylum,
-                    klass: klass,
-                    order: order,
-                    family: family,
-                    genus: genus,
-                    code: code }
 
+      names = {}
+      file.each_with_index do |row, i|
+        next if row["term_type"].nil?
+        name = row["scientific_name"]
+        if names.key?(name)
+          names[name] = names[name].
+            merge({row["term_type"].to_sym => row["term"]})
+        else
+          names[name] = {row["term_type"].to_sym => row["term"]}
+        end
+        puts "Preprocessed #{i} rows" if (i % 100_000).zero?
+      end
+      names.each_with_index do |m, i|
+        canonical = m[0]
+        v = m[1]
+        taxon_id = "gn_#{i + 1}"
+        res ={ taxon_id: taxon_id,
+               name_string: canonical,
+               kingdom: v[:kingdom],
+               phylum: v[:phylum],
+               klass: v[:class],
+               order: v[:order],
+               family: v[:family],
+               genus: v[:genus],
+               species: v[:species],
+               authors: v[:author_text],
+               code: v[:nomenclatural_code] }
+        @names << res
         update_vernacular(taxon_id, canonical)
         update_synonym(taxon_id, canonical)
-        puts "Processed %s names" % i if i % 10_000 == 0
+        puts "Processed #{i} names" if (i % 100_000).zero?
       end
     end