dwca_hunter 0.5.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e30e9b34ea1c46b021bd3c2ec66ccad4996d4a921c7ce78791b84940bd239f05
-  data.tar.gz: 1be0e7119fd38094f94a53d71460a8f97a59f2e46a5e9740b814f5dcc97b42cd
+  metadata.gz: 76fb831d8f9d33d8cfa32221ce4a5a0e82ee8090ed427c6d2eebc96e267b8151
+  data.tar.gz: 33c8040ec605718963ee46eef4a2a4c27770ce3cfb53baf7486047a3d0528410
 SHA512:
-  metadata.gz: df1d9bebe191ebf8ae72d601f05374edeaeffbb627d08d7981da582559439dd9ad173656d78a7c88b3a9765562efa8a7095c4eaa5ed0f1f3ef856be94f990b63
-  data.tar.gz: 703bbf2d197a55a8d4e8510e940f77562540983bb5eddc7a7496ed77af5e610ce0ce5d95a1faa25878a8f8b12a17613808898453a0bd6b7aa19a87dc7c5f000e
+  metadata.gz: 380a22edce284de5f1e836b8b4ae236b8d43fecb7b72032d4e4d3c071d77da05bb87498f6e2f868b071f935983e64861cd8ce549d6b9ed8d672b8ab7d53da32e
+  data.tar.gz: 97a397e0fc24f5705e46287178f4dd1560621995a19b85007609eba4819127b0a98c49a9dd9ea2d946ead54bf5c4f17e2d85cea738f3c9fdf7ba3de3a5c92f93

data/.byebug_history

@@ -1,4 +1,49 @@
 q
+row
+data
+cc
+c
+data
+c
+data
+c
+data
+c
+data
+c
+data
+q
+p n
+n
+q
+@name_id_json[n]
+n = row[:acc_name].to_sym
+row[:acc_name].to_sym
+row[:acc_name]
+acc_id
+vi acc_id
+q
+@name_id
+acc_id
+id
+q
+DwcaHunter.resources
+q
+DwcaHunter.resources
+q
+res.size
+res.len
+res
+trd
+exit
+pp row
+require "pp"
+pp row
+row["id"]
+row.id
+row
+data
+q
 c
 q
 resource.title =>q

data/.gitignore

@@ -56,3 +56,8 @@ tags
 
 # vscode
 .vs
+
+# misc
+*.json
+*.csv
+*.txt

data/.rubocop.yml

@@ -1,5 +1,6 @@
 AllCops:
-  TargetRubyVersion: 2.4
+  NewCops: disable
+  TargetRubyVersion: 2.6.6
   Exclude:
     - bin/**/*
     - db/**/*
@@ -30,4 +31,4 @@ Metrics/MethodLength:
 Naming/FileName:
   Exclude:
     - Gemfile
-    - Rakefile
+    - Rakefile

data/.ruby-version

@@ -1 +1 @@
-2.5.1
+2.6.6

data/Gemfile.lock

@@ -1,135 +1,113 @@
 PATH
   remote: .
   specs:
-    dwca_hunter (0.5.1)
-      biodiversity (~> 3.5)
-      dwc-archive (~> 1.0)
+    dwca_hunter (0.7.0)
+      biodiversity (~> 4)
+      dwc-archive (~> 1.1.1)
       gn_uuid (~> 0.5)
       htmlentities (~> 4.3)
-      nokogiri (~> 1.8)
+      nokogiri (~> 1.10)
       rest-client (~> 2.0)
+      ruby-xz (~> 1.0)
       thor (~> 0.19)
 
 GEM
   remote: http://rubygems.org/
   specs:
     ast (2.4.0)
-    biodiversity (3.5.0)
-      gn_uuid (~> 0.5)
-      parallel (~> 1.12)
-      treetop (~> 1.6)
-      unicode_utils (~> 1.4)
+    biodiversity (4.1.0)
+      ffi (~> 1.11)
     byebug (10.0.2)
-    coderay (1.1.2)
-    coveralls (0.8.22)
+    coveralls (0.8.23)
       json (>= 1.8, < 3)
       simplecov (~> 0.16.1)
       term-ansicolor (~> 1.3)
-      thor (~> 0.19.4)
+      thor (>= 0.19.4, < 2.0)
       tins (~> 1.6)
     diff-lcs (1.3)
-    docile (1.3.1)
-    domain_name (0.5.20180417)
+    docile (1.3.2)
+    domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    dwc-archive (1.0.1)
-      nokogiri (~> 1.8)
-      parsley-store (~> 0.3)
-    eventmachine (1.2.7)
-    gn_uuid (0.5.0)
+    dwc-archive (1.1.1)
+      biodiversity (~> 4)
+      nokogiri (~> 1.10)
+    ffi (1.13.1)
+    gn_uuid (0.5.1)
     htmlentities (4.3.4)
+    http-accept (1.7.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
-    jaro_winkler (1.5.1)
-    json (2.1.0)
-    kramdown (1.17.0)
-    mime-types (3.1)
+    json (2.3.0)
+    mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2016.0521)
-    mini_portile2 (2.3.0)
+    mime-types-data (3.2020.0512)
+    mini_portile2 (2.4.0)
     netrc (0.11.0)
-    nokogiri (1.8.4)
-      mini_portile2 (~> 2.3.0)
-    parallel (1.12.1)
-    parser (2.5.1.2)
+    nokogiri (1.10.9)
+      mini_portile2 (~> 2.4.0)
+    parallel (1.19.1)
+    parser (2.7.1.3)
       ast (~> 2.4.0)
-    parsley-store (0.3.6)
-      biodiversity (~> 3.1)
-      redis (~> 3.0)
-    polyglot (0.3.5)
-    powerpack (0.1.2)
     rainbow (3.0.0)
-    rake (12.3.1)
-    redis (3.3.5)
-    rest-client (2.0.2)
+    rake (13.0.1)
+    rest-client (2.1.0)
+      http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
-    reverse_markdown (1.1.0)
-      nokogiri
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.1)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
+    rexml (3.2.4)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-mocks (3.7.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-support (3.7.1)
-    rubocop (0.58.1)
-      jaro_winkler (~> 1.5.1)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    rubocop (0.84.0)
       parallel (~> 1.10)
-      parser (>= 2.5, != 2.5.1.1)
-      powerpack (~> 0.1)
+      parser (>= 2.7.0.1)
       rainbow (>= 2.2.2, < 4.0)
+      rexml
+      rubocop-ast (>= 0.0.3)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (~> 1.0, >= 1.0.1)
-    ruby-progressbar (1.9.0)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-ast (0.0.3)
+      parser (>= 2.7.0.1)
+    ruby-progressbar (1.10.1)
+    ruby-xz (1.0.0)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    solargraph (0.23.5)
-      coderay (~> 1.1)
-      eventmachine (~> 1.2, >= 1.2.5)
-      htmlentities (~> 4.3, >= 4.3.4)
-      kramdown (~> 1.16)
-      parser (~> 2.4)
-      reverse_markdown (~> 1.0, >= 1.0.5)
-      rubocop (~> 0.52)
-      thor (~> 0.19, >= 0.19.4)
-      tilt (~> 2.0)
-      yard (~> 0.9)
-    term-ansicolor (1.6.0)
+    sync (0.5.0)
+    term-ansicolor (1.7.1)
       tins (~> 1.0)
-    thor (0.19.4)
-    tilt (2.0.8)
-    tins (1.16.3)
-    treetop (1.6.10)
-      polyglot (~> 0.3)
+    thor (0.20.3)
+    tins (1.25.0)
+      sync
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.5)
-    unicode-display_width (1.4.0)
-    unicode_utils (1.4.0)
-    yard (0.9.15)
+    unf_ext (0.0.7.7)
+    unicode-display_width (1.7.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.16)
+  bundler (~> 2.0)
   byebug (~> 10.0)
   coveralls (~> 0.8)
   dwca_hunter!
-  rake (~> 12.3)
-  rspec (~> 3.7)
-  rubocop (~> 0.58)
-  solargraph (~> 0.23)
+  rake (~> 13.0)
+  rspec (~> 3.9)
+  rubocop (~> 0.84)
 
 BUNDLED WITH
-   1.16.3
+   2.1.4

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2011 Marine Biological Laboratory
+Copyright (c) 2011-2020 Dmitry Mozzherin
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -32,7 +32,7 @@ See dwca_hunter/bin/dwca-hunt.rb as a example how to use the code.
 
 ## Copyright
 
-Copyright (c) 2011-2016 Dmitry Mozzherin. See LICENSE.txt for further details.
+Copyright (c) 2011-2020 Dmitry Mozzherin. See LICENSE.txt for further details.
 
 [code-climate-img]: https://codeclimate.com/badge.png
 [code-climate]: https://codeclimate.com/github/GlobalNamesArchitecture/dwca_hunter

data/dwca_hunter.gemspec

@@ -6,7 +6,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "dwca_hunter/version"
 
 Gem::Specification.new do |gem|
-  gem.required_ruby_version = ">= 2.4"
+  gem.required_ruby_version = ">= 2.6.6"
   gem.name          = "dwca_hunter"
   gem.version       = DwcaHunter.version
   gem.license       = "MIT"
@@ -26,19 +26,19 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^exe/}) { |f| File.basename(f) }
   gem.require_paths = ["lib"]
 
-  gem.add_dependency "biodiversity", "~> 3.5"
-  gem.add_dependency "dwc-archive", "~> 1.0"
+  gem.add_dependency "biodiversity", "~> 4"
+  gem.add_dependency "dwc-archive", "~> 1.1.1"
   gem.add_dependency "gn_uuid", "~> 0.5"
   gem.add_dependency "htmlentities", "~> 4.3"
-  gem.add_dependency "nokogiri", "~> 1.8"
+  gem.add_dependency "nokogiri", "~> 1.10"
   gem.add_dependency "rest-client", "~> 2.0"
+  gem.add_dependency "ruby-xz", "~> 1.0"
   gem.add_dependency "thor", "~> 0.19"
 
-  gem.add_development_dependency "bundler", "~> 1.16"
+  gem.add_development_dependency "bundler", "~> 2.0"
   gem.add_development_dependency "byebug", "~> 10.0"
   gem.add_development_dependency "coveralls", "~> 0.8"
-  gem.add_development_dependency "rake", "~> 12.3"
-  gem.add_development_dependency "rspec", "~> 3.7"
-  gem.add_development_dependency "rubocop", "~> 0.58"
-  gem.add_development_dependency "solargraph", "~> 0.23"
+  gem.add_development_dependency "rake", "~> 13.0"
+  gem.add_development_dependency "rspec", "~> 3.9"
+  gem.add_development_dependency "rubocop", "~> 0.84"
 end

data/exe/dwcahunter

@@ -37,9 +37,7 @@ class DwcaHunterCLI < Thor
     res = []
     DwcaHunter.resources.each do |resource|
       r = resource.new
-      if add_resource?(search, r)
-        res << { n: nil, command: r.command, resource: r.title }
-      end
+      res << { n: nil, command: r.command, resource: r.title } if add_resource?(search, r)
     end
     res.sort_by { |r| r[:command] }.each_with_object([]) do |r, a|
       r[:n] = a.size + 1

data/lib/dwca_hunter.rb

@@ -1,17 +1,17 @@
 # frozen_string_literal: true
 
+require "base64"
 require "biodiversity"
-require "logger"
-require "fileutils"
-require "uri"
-require "tmpdir"
-require "net/http"
-require "json"
-require "htmlentities"
 require "dwc_archive"
 require "dwca_hunter/resource"
+require "fileutils"
+require "htmlentities"
+require "json"
+require "logger"
+require "net/http"
 require "rest_client"
-require "base64"
+require "tmpdir"
+require "uri"
 
 Dir[File.join(__dir__, "dwca_hunter", "*.rb")].
   each { |f| require f }
@@ -51,5 +51,36 @@ module DwcaHunter
         c < Resource
       end
     end
+
+    def normalize_authors(auth)
+      reg = Regexp.new(/^([\(]?)(.*?)(([\s,\)][^[:upper:]]*)?$)/)
+      auth = auth.gsub(/duPont/, 'du Pont')
+      match = reg.match(auth)
+      return auth if match.nil?
+      a1, a2, a3 = match[1..3]
+      a2mod = a2.gsub('&', ',')
+      ary2 = a2mod.split(',').map(&:strip)
+      a2 = move_initials(ary2) if ary2.size > 1
+      "#{a1}#{a2}#{a3}"
+    end
+
+    def move_initials(ary)
+      res = []
+      ary.each do |a|
+        if res.empty?
+          res << a
+          next
+        end
+        match = /^([[:upper:]]{1,4})(\sJr)?$/.match(a)
+        if !match.nil?
+          initialls = match[1].split('').join('. ')
+          res[-1] = "#{initialls}. #{res[-1]}#{match[2].to_s}"
+        else
+          res << a
+        end
+      end
+      res.size == 1 ? res[0] : "#{res[0..-2].join(', ')} & #{res[-1]}"
+    end
   end
 end
+

data/lib/dwca_hunter/resource.rb

@@ -7,6 +7,11 @@ module DwcaHunter
       `unzip -qq -u #{file} > /dev/null 2>&1`
     end
 
+    def self.gunzip(file, dir = nil)
+      Dir.chdir(dir) if dir
+      `gunzip #{file}`
+    end
+
     def initialize(opts)
       @needs_download = !(opts[:download] == false)
       @needs_unpack = !(opts[:unpack] == false)

data/lib/dwca_hunter/resources/aos-birds.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module DwcaHunter
+  class ResourceAOS < DwcaHunter::Resource
+    def initialize(opts = {})
+      @command = "aos-birds"
+      @title = "American Ornithological Society"
+      @url = "http://checklist.americanornithology.org/taxa.csv"
+      @UUID = "91d38806-8435-479f-a18d-705e5cb0767c"
+      @download_path = File.join(Dir.tmpdir,
+                                 "dwca_hunter",
+                                 "aos",
+                                 "data.csv")
+      @synonyms = []
+      @names = []
+      @vernaculars = []
+      @extensions = []
+      @synonyms_hash = {}
+      @vernaculars_hash = {}
+      super(opts)
+    end
+
+    def download
+      puts "Downloading csv from remote"
+      `curl -s -L #{@url} -o #{@download_path}`
+    end
+
+    def unpack; end
+
+    def make_dwca
+      DwcaHunter.logger_write(object_id, "Extracting data")
+      get_names
+      generate_dwca
+    end
+
+    private
+
+    def get_names
+      Dir.chdir(@download_dir)
+      collect_names
+    end
+
+    def collect_names
+      @names_index = {}
+      file = CSV.open(File.join(@download_dir, "data.csv"),
+                      headers: true)
+      file.each_with_index do |row, _i|
+        taxon_id = row["id"]
+        name_string = row["species"]
+        kingdom = "Animalia"
+        phylum = "Chordata"
+        klass = "Aves"
+        order = row["order"]
+        family = row["family"]
+        genus = row["genus"]
+        code = "ICZN"
+
+        @names << {
+          taxon_id: taxon_id,
+          name_string: name_string,
+          kingdom: kingdom,
+          phylum: phylum,
+          klass: klass,
+          order: order,
+          family: family,
+          genus: genus,
+          code: code
+        }
+        if row["common_name"].to_s != ""
+          @vernaculars << {
+            taxon_id: taxon_id,
+            vern: row["common_name"],
+            lang: "en"
+          }
+        end
+        next unless row["french_name"].to_s != ""
+
+        @vernaculars << {
+          taxon_id: taxon_id,
+          vern: row["french_name"],
+          lang: "fr"
+        }
+      end
+    end
+
+    def generate_dwca
+      DwcaHunter.logger_write(object_id,
+                              "Creating DarwinCore Archive file")
+      @core = [["http://rs.tdwg.org/dwc/terms/taxonID",
+                "http://rs.tdwg.org/dwc/terms/scientificName",
+                "http://rs.tdwg.org/dwc/terms/kingdom",
+                "http://rs.tdwg.org/dwc/terms/phylum",
+                "http://rs.tdwg.org/dwc/terms/class",
+                "http://rs.tdwg.org/dwc/terms/order",
+                "http://rs.tdwg.org/dwc/terms/family",
+                "http://rs.tdwg.org/dwc/terms/genus",
+                "http://rs.tdwg.org/dwc/terms/nomenclaturalCode"]]
+      @names.each do |n|
+        @core << [n[:taxon_id], n[:name_string],
+                  n[:kingdom], n[:phylum], n[:klass], n[:order], n[:family],
+                  n[:genus], n[:code]]
+      end
+      @extensions << {
+        data: [[
+          "http://rs.tdwg.org/dwc/terms/taxonID",
+          "http://rs.tdwg.org/dwc/terms/vernacularName",
+          "http://purl.org/dc/terms/language"
+        ]],
+        file_name: "vernacular_names.txt",
+        row_type: "http://rs.gbif.org/terms/1.0/VernacularName"
+      }
+
+      @vernaculars.each do |v|
+        @extensions[-1][:data] << [v[:taxon_id], v[:vern], v[:lang]]
+      end
+      @eml = {
+        id: @uuid,
+        title: @title,
+        authors: [
+          { first_name: "R. T.",
+            last_name: "Chesser" }
+        ],
+        metadata_providers: [
+          { first_name: "Dmitry",
+            last_name: "Mozzherin",
+            email: "dmozzherin@gmail.com" }
+        ],
+        abstract: "The American Ornithological Society's (AOS) Checklist is " \
+        "the official source on the taxonomy of birds found in North and " \
+        "Middle America, including adjacent islands. This list is produced " \
+        "by the North American Classification and Nomenclature Committee " \
+        "(NACC) of the AOS.\n\n" \
+        "Recommended citation: Chesser, R. T., K. J. Burns, C. Cicero, " \
+        "J. L. Dunn, A. W. Kratter, I. J. Lovette, P. C. Rasmussen, " \
+        "J. V. Remsen, Jr., D. F. Stotz, and K. Winker. 2019. Check-list " \
+        "of North American Birds (online). American Ornithological Society. " \
+        "http://checklist.aou.org/taxa",
+        url: @url
+      }
+      super
+    end
+  end
+end