dwca_hunter 0.5.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e30e9b34ea1c46b021bd3c2ec66ccad4996d4a921c7ce78791b84940bd239f05
-  data.tar.gz: 1be0e7119fd38094f94a53d71460a8f97a59f2e46a5e9740b814f5dcc97b42cd
+  metadata.gz: fb05e834a8403ae6b6cce3fc9c913b38d7111808ec355f4ad659d74b3960697f
+  data.tar.gz: cb30906b445212fa52d4ab536610d77050bcfa417fdbb0546c93d5f004a266d9
 SHA512:
-  metadata.gz: df1d9bebe191ebf8ae72d601f05374edeaeffbb627d08d7981da582559439dd9ad173656d78a7c88b3a9765562efa8a7095c4eaa5ed0f1f3ef856be94f990b63
-  data.tar.gz: 703bbf2d197a55a8d4e8510e940f77562540983bb5eddc7a7496ed77af5e610ce0ce5d95a1faa25878a8f8b12a17613808898453a0bd6b7aa19a87dc7c5f000e
+  metadata.gz: 8ee016bf36ca9bab6ed6d65475b52e231c13cae276482ac34787b5680d3953ca5672b73428b33e1834823ad90411464567617b9a7bf25f359a09e5f6f7a8122c
+  data.tar.gz: 233dd03050a99fc016e1c78326d76476f0fbf3f81743a28c1ec842e4b5755a2162eb1951035192149b9da537811dedad20c046b31e962eeeabd89550b146cec5

data/.byebug_history

@@ -1,3 +1,11 @@
+exit
+pp row
+require "pp"
+pp row
+row["id"]
+row.id
+row
+data
 q
 c
 q

data/.ruby-version

@@ -1 +1 @@
-2.5.1
+2.5.3

data/Gemfile.lock

@@ -1,128 +1,186 @@
 PATH
   remote: .
   specs:
-    dwca_hunter (0.5.1)
+    dwca_hunter (0.5.2)
       biodiversity (~> 3.5)
       dwc-archive (~> 1.0)
       gn_uuid (~> 0.5)
+      google-cloud-storage (~> 1.23)
       htmlentities (~> 4.3)
       nokogiri (~> 1.8)
       rest-client (~> 2.0)
+      ruby-xz (~> 1.0)
       thor (~> 0.19)
 
 GEM
   remote: http://rubygems.org/
   specs:
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.0)
-    biodiversity (3.5.0)
+    backport (0.3.0)
+    biodiversity (3.5.1)
       gn_uuid (~> 0.5)
       parallel (~> 1.12)
       treetop (~> 1.6)
       unicode_utils (~> 1.4)
     byebug (10.0.2)
-    coderay (1.1.2)
     coveralls (0.8.22)
       json (>= 1.8, < 3)
       simplecov (~> 0.16.1)
       term-ansicolor (~> 1.3)
       thor (~> 0.19.4)
       tins (~> 1.6)
+    declarative (0.0.10)
+    declarative-option (0.1.0)
     diff-lcs (1.3)
+    digest-crc (0.4.1)
     docile (1.3.1)
-    domain_name (0.5.20180417)
+    domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     dwc-archive (1.0.1)
       nokogiri (~> 1.8)
       parsley-store (~> 0.3)
-    eventmachine (1.2.7)
-    gn_uuid (0.5.0)
+    faraday (0.17.0)
+      multipart-post (>= 1.2, < 3)
+    gn_uuid (0.5.1)
+    google-api-client (0.34.1)
+      addressable (~> 2.5, >= 2.5.1)
+      googleauth (~> 0.9)
+      httpclient (>= 2.8.1, < 3.0)
+      mini_mime (~> 1.0)
+      representable (~> 3.0)
+      retriable (>= 2.0, < 4.0)
+      signet (~> 0.12)
+    google-cloud-core (1.4.1)
+      google-cloud-env (~> 1.0)
+    google-cloud-env (1.3.0)
+      faraday (~> 0.11)
+    google-cloud-storage (1.23.0)
+      addressable (~> 2.5)
+      digest-crc (~> 0.4)
+      google-api-client (~> 0.33)
+      google-cloud-core (~> 1.2)
+      googleauth (~> 0.9)
+      mini_mime (~> 1.0)
+    googleauth (0.10.0)
+      faraday (~> 0.12)
+      jwt (>= 1.4, < 3.0)
+      memoist (~> 0.16)
+      multi_json (~> 1.11)
+      os (>= 0.9, < 2.0)
+      signet (~> 0.12)
     htmlentities (4.3.4)
+    http-accept (1.7.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
-    jaro_winkler (1.5.1)
-    json (2.1.0)
+    httpclient (2.8.3)
+    jaro_winkler (1.5.2)
+    json (2.2.0)
+    jwt (2.2.1)
     kramdown (1.17.0)
-    mime-types (3.1)
+    memoist (0.16.1)
+    mime-types (3.3)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2016.0521)
-    mini_portile2 (2.3.0)
+    mime-types-data (3.2019.1009)
+    mini_mime (1.0.2)
+    mini_portile2 (2.4.0)
+    multi_json (1.14.1)
+    multipart-post (2.1.1)
     netrc (0.11.0)
-    nokogiri (1.8.4)
-      mini_portile2 (~> 2.3.0)
-    parallel (1.12.1)
-    parser (2.5.1.2)
+    nokogiri (1.10.1)
+      mini_portile2 (~> 2.4.0)
+    os (1.0.1)
+    parallel (1.14.0)
+    parser (2.6.0.0)
       ast (~> 2.4.0)
     parsley-store (0.3.6)
       biodiversity (~> 3.1)
       redis (~> 3.0)
     polyglot (0.3.5)
     powerpack (0.1.2)
+    psych (3.1.0)
+    public_suffix (4.0.1)
     rainbow (3.0.0)
-    rake (12.3.1)
+    rake (12.3.2)
     redis (3.3.5)
-    rest-client (2.0.2)
+    representable (3.0.4)
+      declarative (< 0.1.0)
+      declarative-option (< 0.2.0)
+      uber (< 0.2.0)
+    rest-client (2.1.0)
+      http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
+    retriable (3.1.2)
     reverse_markdown (1.1.0)
       nokogiri
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.1)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-mocks (3.7.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-support (3.7.1)
-    rubocop (0.58.1)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    rubocop (0.65.0)
       jaro_winkler (~> 1.5.1)
       parallel (~> 1.10)
       parser (>= 2.5, != 2.5.1.1)
       powerpack (~> 0.1)
+      psych (>= 3.1.0)
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (~> 1.0, >= 1.0.1)
-    ruby-progressbar (1.9.0)
+      unicode-display_width (~> 1.4.0)
+    ruby-progressbar (1.10.0)
+    ruby-xz (1.0.0)
+    signet (0.12.0)
+      addressable (~> 2.3)
+      faraday (~> 0.9)
+      jwt (>= 1.5, < 3.0)
+      multi_json (~> 1.10)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    solargraph (0.23.5)
-      coderay (~> 1.1)
-      eventmachine (~> 1.2, >= 1.2.5)
+    solargraph (0.31.3)
+      backport (~> 0.3)
       htmlentities (~> 4.3, >= 4.3.4)
+      jaro_winkler (~> 1.5)
       kramdown (~> 1.16)
-      parser (~> 2.4)
+      parser (~> 2.3)
       reverse_markdown (~> 1.0, >= 1.0.5)
       rubocop (~> 0.52)
       thor (~> 0.19, >= 0.19.4)
       tilt (~> 2.0)
       yard (~> 0.9)
-    term-ansicolor (1.6.0)
+    term-ansicolor (1.7.1)
       tins (~> 1.0)
     thor (0.19.4)
-    tilt (2.0.8)
-    tins (1.16.3)
+    tilt (2.0.9)
+    tins (1.20.2)
     treetop (1.6.10)
       polyglot (~> 0.3)
+    uber (0.1.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.5)
-    unicode-display_width (1.4.0)
+    unf_ext (0.0.7.6)
+    unicode-display_width (1.4.1)
     unicode_utils (1.4.0)
-    yard (0.9.15)
+    yard (0.9.18)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.16)
+  bundler (~> 2.0)
   byebug (~> 10.0)
   coveralls (~> 0.8)
   dwca_hunter!
@@ -132,4 +190,4 @@ DEPENDENCIES
   solargraph (~> 0.23)
 
 BUNDLED WITH
-   1.16.3
+   2.0.1

data/dwca_hunter.gemspec

@@ -32,9 +32,10 @@ Gem::Specification.new do |gem|
   gem.add_dependency "htmlentities", "~> 4.3"
   gem.add_dependency "nokogiri", "~> 1.8"
   gem.add_dependency "rest-client", "~> 2.0"
+  gem.add_dependency "ruby-xz", "~> 1.0"
   gem.add_dependency "thor", "~> 0.19"
 
-  gem.add_development_dependency "bundler", "~> 1.16"
+  gem.add_development_dependency "bundler", "~> 2.0"
   gem.add_development_dependency "byebug", "~> 10.0"
   gem.add_development_dependency "coveralls", "~> 0.8"
   gem.add_development_dependency "rake", "~> 12.3"

data/lib/dwca_hunter/resources/ipni.rb

@@ -0,0 +1,110 @@
+require "xz"
+
+module DwcaHunter
+  # Resource for FishBase
+  class ResourceIPNI < DwcaHunter::Resource
+    attr_reader :title, :abbr
+    def initialize(opts = {}) #download: false, unpack: false})
+      @command = "ipni"
+      @title = "The International Plant Names Index"
+      @abbr = "IPNI"
+      @url = "https://www.dropbox.com/s/1n0sn80vkdir5nu/ipniWebName.csv.xz"
+      @uuid = "6b3905ce-5025-49f3-9697-ddd5bdfb4ff0"
+      @download_path = File.join(Dir.tmpdir, "dwca_hunter", "ipni",
+                                 "ipni.csv.xz")
+      @extensions = []
+      super
+    end
+
+    def unpack
+      puts "Unpacking #{@download_path}"
+      XZ.decompress_file(@download_path, @download_path[0...-3] )
+    end
+
+    def download
+      puts "Downloading cached verion of the file. Get daily updated one from"
+      puts "https://storage.cloud.google.com/ipni-data/ipniWebName.csv.xz"
+        `curl -s -L #{@url} -o #{@download_path}`
+    end
+
+    def make_dwca
+      organize_data
+      generate_dwca
+    end
+
+    private
+
+    def organize_data
+      DwcaHunter::logger_write(self.object_id,
+                               "Organizing data")
+      # snp = ScientificNameParser.new
+      @data = CSV.open(@download_path[0...-3],
+         col_sep: "|", quote_char: "щ", headers: true)
+        .each_with_object([]) do |row, data|
+        name = row['taxon_scientific_name_s_lower'].strip
+        au = row['authors_t'].to_s.strip
+        name = "#{name} #{au}" if au != ''
+        id = row["id"].split(":")[-1]
+        data << { taxon_id: id,
+                  local_id: id,
+                  family: row["family_s_lower"],
+                  genus: row["genus_s_lower"],
+                  scientific_name: name,
+                  rank: row["rank_s_alphanum"]
+                }
+
+      end
+    end
+
+    def generate_dwca
+      DwcaHunter::logger_write(self.object_id,
+                               'Creating DarwinCore Archive file')
+      core_init
+      eml_init
+      DwcaHunter::logger_write(self.object_id, 'Assembling Core Data')
+      count = 0
+      @data.each do |d|
+        count += 1
+        if count % 10000 == 0
+          DwcaHunter::logger_write(self.object_id, "Core row #{count}")
+        end
+        @core << [d[:taxon_id], d[:local_id],
+                  d[:scientific_name], d[:rank],
+                  d[:family], d[:genus]]
+      end
+      super
+    end
+
+    def eml_init
+      @eml = {
+        id: @uuid,
+        title: @title,
+        authors: [],
+        metadata_providers: [
+          { first_name: "Dmitry",
+            last_name: "Mozzherin",
+          }
+      ],
+        abstract: "The International Plant Names Index (IPNI) is a database " \
+                  "of the names and associated basic bibliographical " \
+                  "details of seed plants, ferns and lycophytes. Its goal " \
+                  "is to eliminate the need for repeated reference to " \
+                  "primary sources for basic bibliographic information " \
+                  "about plant names. The data are freely available and are " \
+                  "gradually being standardized and checked. IPNI will be a " \
+                  "dynamic resource, depending on direct contributions by " \
+                  "all members of the botanical community.",
+        url: "http://www.ipni.org"
+      }
+    end
+
+    def core_init
+      @core = [["http://rs.tdwg.org/dwc/terms/taxonID",
+                "http://globalnames.org/terms/localID",
+                "http://rs.tdwg.org/dwc/terms/scientificName",
+                "http://rs.tdwg.org/dwc/terms/taxonRank",
+                "http://rs.tdwg.org/dwc/terms/family",
+                "http://rs.tdwg.org/dwc/terms/genus"]]
+    end
+  end
+end

data/lib/dwca_hunter/version.rb

@@ -1,5 +1,5 @@
 module DwcaHunter
-  VERSION = "0.5.1"
+  VERSION = "0.5.2"
 
   def self.version
     VERSION

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dwca_hunter
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors:
 - Dmitry Mozzherin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-08-04 00:00:00.000000000 Z
+date: 2019-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: biodiversity
@@ -94,6 +94,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: ruby-xz
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -114,14 +128,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -233,6 +247,8 @@ files:
 - files/fishbase_taxon_cache.tsv
 - files/reptile_checklist_2014_12.csv
 - files/species-black.txt
+- ipni.csv.gz
+- ipniWebName.csv.xz?dl=1
 - lib/dwca_hunter.rb
 - lib/dwca_hunter/downloader.rb
 - lib/dwca_hunter/encoding.rb
@@ -242,6 +258,7 @@ files:
 - lib/dwca_hunter/resources/fishbase.rb
 - lib/dwca_hunter/resources/freebase.rb
 - lib/dwca_hunter/resources/gnub.rb
+- lib/dwca_hunter/resources/ipni.rb
 - lib/dwca_hunter/resources/itis.rb
 - lib/dwca_hunter/resources/mammal_species.rb
 - lib/dwca_hunter/resources/ncbi.rb