dwca_hunter 0.5.1 → 0.7.0

data/lib/dwca_hunter/resources/ipni.rb

@@ -0,0 +1,111 @@
+require "xz"
+
+module DwcaHunter
+  # Resource for FishBase
+  class ResourceIPNI < DwcaHunter::Resource
+    attr_reader :title, :abbr
+    def initialize(opts = {}) #download: false, unpack: false})
+      @command = "ipni"
+      @title = "The International Plant Names Index"
+      @abbr = "IPNI"
+      @url = "https://uofi.box.com/shared/static/s0x4xjonxt54pi89n543gdmttrdqd6iv.xz"
+      @uuid = "6b3905ce-5025-49f3-9697-ddd5bdfb4ff0"
+      @download_path = File.join(Dir.tmpdir, "dwca_hunter", "ipni",
+                                 "ipni.csv.xz")
+      @extensions = []
+      super
+    end
+
+    def unpack
+      puts "Unpacking #{@download_path}"
+      XZ.decompress_file(@download_path, @download_path[0...-3] )
+    end
+
+    def download
+      puts "Download by hand from"
+      puts "https://storage.cloud.google.com/ipni-data/ipniWebName.csv.xz"
+      puts "and copy to given url"
+        `curl -s -L #{@url} -o #{@download_path}`
+    end
+
+    def make_dwca
+      organize_data
+      generate_dwca
+    end
+
+    private
+
+    def organize_data
+      DwcaHunter::logger_write(self.object_id,
+                               "Organizing data")
+      # snp = ScientificNameParser.new
+      @data = CSV.open(@download_path[0...-3],
+         col_sep: "|", quote_char: "щ", headers: true)
+        .each_with_object([]) do |row, data|
+        name = row['taxon_scientific_name_s_lower'].strip
+        au = row['authors_t'].to_s.strip
+        name = "#{name} #{au}" if au != ''
+        id = row["id"].split(":")[-1]
+        data << { taxon_id: id,
+                  local_id: id,
+                  family: row["family_s_lower"],
+                  genus: row["genus_s_lower"],
+                  scientific_name: name,
+                  rank: row["rank_s_alphanum"]
+                }
+
+      end
+    end
+
+    def generate_dwca
+      DwcaHunter::logger_write(self.object_id,
+                               'Creating DarwinCore Archive file')
+      core_init
+      eml_init
+      DwcaHunter::logger_write(self.object_id, 'Assembling Core Data')
+      count = 0
+      @data.each do |d|
+        count += 1
+        if count % 10000 == 0
+          DwcaHunter::logger_write(self.object_id, "Core row #{count}")
+        end
+        @core << [d[:taxon_id], d[:local_id],
+                  d[:scientific_name], d[:rank],
+                  d[:family], d[:genus]]
+      end
+      super
+    end
+
+    def eml_init
+      @eml = {
+        id: @uuid,
+        title: @title,
+        authors: [],
+        metadata_providers: [
+          { first_name: "Dmitry",
+            last_name: "Mozzherin",
+          }
+      ],
+        abstract: "The International Plant Names Index (IPNI) is a database " \
+                  "of the names and associated basic bibliographical " \
+                  "details of seed plants, ferns and lycophytes. Its goal " \
+                  "is to eliminate the need for repeated reference to " \
+                  "primary sources for basic bibliographic information " \
+                  "about plant names. The data are freely available and are " \
+                  "gradually being standardized and checked. IPNI will be a " \
+                  "dynamic resource, depending on direct contributions by " \
+                  "all members of the botanical community.",
+        url: "http://www.ipni.org"
+      }
+    end
+
+    def core_init
+      @core = [["http://rs.tdwg.org/dwc/terms/taxonID",
+                "http://globalnames.org/terms/localID",
+                "http://rs.tdwg.org/dwc/terms/scientificName",
+                "http://rs.tdwg.org/dwc/terms/taxonRank",
+                "http://rs.tdwg.org/dwc/terms/family",
+                "http://rs.tdwg.org/dwc/terms/genus"]]
+    end
+  end
+end

data/lib/dwca_hunter/resources/itis.rb

@@ -1,15 +1,16 @@
-# encoding: utf-8
+# frozen_string_literal: true
+
 module DwcaHunter
   class ResourceITIS < DwcaHunter::Resource
     def initialize(opts = {})
-      @command = 'itis'
-      @title = 'ITIS'
-      @url = 'https://www.itis.gov/downloads/itisMySQLTables.tar.gz'
-      @uuid =  '5d066e84-e512-4a2f-875c-0a605d3d9f35'
+      @command = "itis"
+      @title = "ITIS"
+      @url = "https://www.itis.gov/downloads/itisMySQLTables.tar.gz"
+      @uuid = "5d066e84-e512-4a2f-875c-0a605d3d9f35"
       @download_path = File.join(Dir.tmpdir,
-                                 'dwca_hunter',
-                                 'itis',
-                                 'data.tar.gz')
+                                 "dwca_hunter",
+                                 "itis",
+                                 "data.tar.gz")
       @ranks = {}
       @kingdoms = {}
       @authors = {}
@@ -19,20 +20,20 @@ module DwcaHunter
       @names = {}
       @extensions = []
       super(opts)
-      @itis_dir = File.join(@download_dir, 'itis')
+      @itis_dir = File.join(@download_dir, "itis")
     end
 
     def unpack
       unpack_tar
-      dir = Dir.entries(@download_dir).select {|e| e.match(/itisMySQL/)}[0]
+      dir = Dir.entries(@download_dir).select { |e| e.match(/itisMySQL/) }[0]
       FileUtils.mv(File.join(@download_dir, dir), @itis_dir)
 
       # Create a file with the same name as the directory we extracted.
-      FileUtils.touch(File.join(@itis_dir, 'version_' + dir))
+      FileUtils.touch(File.join(@itis_dir, "version_" + dir))
     end
 
     def make_dwca
-      DwcaHunter::logger_write(self.object_id, 'Extracting data')
+      DwcaHunter.logger_write(object_id, "Extracting data")
       get_ranks
       get_kingdoms
       get_authors
@@ -42,7 +43,8 @@ module DwcaHunter
       generate_dwca
     end
 
-  private
+    private
+
     def get_ranks
       # 0    kingdom_id integer not null
       # 1    rank_id smallint not null
@@ -50,15 +52,15 @@ module DwcaHunter
       # 3    dir_parent_rank_id smallint not null
       # 4    req_parent_rank_id smallint not null
       # 5    update_date date not null
-      rank_file = File.join(@itis_dir, 'taxon_unit_types')
-      f = open(rank_file, 'r:utf-8')
+      rank_file = File.join(@itis_dir, "taxon_unit_types")
+      f = open(rank_file, "r:utf-8")
       f.each do |l|
-        l.encode!('UTF-8',
-                  'ISO-8859-1',
+        l.encode!("UTF-8",
+                  "ISO-8859-1",
                   invalid: :replace,
-                  replace: '?')
-        row = l.strip.split('|')
-        @ranks[row[0].strip + '/' + row[1].strip] = row[2].strip
+                  replace: "?")
+        row = l.strip.split("|")
+        @ranks[row[0].strip + "/" + row[1].strip] = row[2].strip
       end
     end
 
@@ -67,9 +69,9 @@ module DwcaHunter
       # 1    kingdom_name char(10) not null
       # 2    update_date date not null
 
-      f = open(File.join(@itis_dir, 'kingdoms'))
+      f = open(File.join(@itis_dir, "kingdoms"))
       f.each do |l|
-        data = l.strip.split('|')
+        data = l.strip.split("|")
         @kingdoms[data[0].strip] = data[1].strip
       end
     end
@@ -80,13 +82,13 @@ module DwcaHunter
       # 2    update_date date not null
       # 3    kingdom_id smallint not null
 
-      f = open(File.join(@itis_dir, 'taxon_authors_lkp'))
+      f = open(File.join(@itis_dir, "taxon_authors_lkp"))
       f.each do |l|
-        l.encode!('UTF-8',
-                  'ISO-8859-1',
+        l.encode!("UTF-8",
+                  "ISO-8859-1",
                   invalid: :replace,
-                  replace: '?')
-        data = l.strip.split('|')
+                  replace: "?")
+        data = l.strip.split("|")
         @authors[data[0].strip] = data[1].strip
       end
     end
@@ -100,22 +102,22 @@ module DwcaHunter
       # 5    primary key (tsn,vernacular_name,language)
       #      constraint "itis".vernaculars_key
 
-      f = open(File.join(@itis_dir, 'vernaculars'))
+      f = open(File.join(@itis_dir, "vernaculars"))
       f.each_with_index do |l, i|
         if i % BATCH_SIZE == 0
-          DwcaHunter::logger_write(self.object_id,
-                                   "Extracted %s vernacular names" % i)
+          DwcaHunter.logger_write(object_id,
+                                  "Extracted %s vernacular names" % i)
         end
-        l.encode!('UTF-8',
-                  'ISO-8859-1',
+        l.encode!("UTF-8",
+                  "ISO-8859-1",
                   invalid: :replace,
-                  replace: '?')
-        data = l.split('|').map { |d| d.strip }
+                  replace: "?")
+        data = l.split("|").map(&:strip)
         name_tsn = data[0]
         string   = data[1]
         language = data[2]
-        language = 'Common name' if language == 'unspecified'
-        @vernaculars[name_tsn] = { name:string, language:language }
+        language = "Common name" if language == "unspecified"
+        @vernaculars[name_tsn] = { name: string, language: language }
       end
     end
 
@@ -124,17 +126,17 @@ module DwcaHunter
       # 1    tsn_accepted integer not null
       # 2    update_date date not null
 
-      f = open(File.join(@itis_dir, 'synonym_links'))
+      f = open(File.join(@itis_dir, "synonym_links"))
       f.each_with_index do |l, i|
         if i % BATCH_SIZE == 0
-          DwcaHunter::logger_write(self.object_id,
-                                   "Extracted %s synonyms" % i)
+          DwcaHunter.logger_write(object_id,
+                                  "Extracted %s synonyms" % i)
         end
-        l.encode!('UTF-8',
-                  'ISO-8859-1',
+        l.encode!("UTF-8",
+                  "ISO-8859-1",
                   invalid: :replace,
-                  replace: '?')
-        data = l.split('|').map { |d| d.strip }
+                  replace: "?")
+        data = l.split("|").map(&:strip)
         synonym_name_tsn = data[0]
         accepted_name_tsn = data[1]
         @synonyms[synonym_name_tsn] = accepted_name_tsn
@@ -167,19 +169,19 @@ module DwcaHunter
       # 22   update_date date not null
       # 23   uncertain_prnt_ind char(3)
 
-      f = open(File.join(@itis_dir, 'taxonomic_units'))
+      f = open(File.join(@itis_dir, "taxonomic_units"))
       f.each_with_index do |l, i|
         if i % BATCH_SIZE == 0
-          DwcaHunter::logger_write(self.object_id,
-                                   "Extracted %s names" % i)
+          DwcaHunter.logger_write(object_id,
+                                  "Extracted %s names" % i)
         end
-        l.encode!('UTF-8',
-                  'ISO-8859-1',
+        l.encode!("UTF-8",
+                  "ISO-8859-1",
                   invalid: :replace,
-                  replace: '?')
-        data = l.split("|").map { |d| d.strip }
-        name_tsn   = data[0]
-        x1         = data[1]
+                  replace: "?")
+        data = l.split("|").map(&:strip)
+        name_tsn = data[0]
+        x1 = data[1]
         name_part1 = data[2]
         x2         = data[3]
         name_part2 = data[4]
@@ -193,16 +195,15 @@ module DwcaHunter
         kingdom_id = data[20]
         rank_id    = data[21]
 
-        parent_tsn = nil if parent_tsn == ''
+        parent_tsn = nil if parent_tsn == ""
         name = [x1, name_part1, x2, name_part2,
                 sp_marker1, name_part3, sp_marker2, name_part4]
         canonical_name = name.clone
         name << @authors[author_id] if @authors[author_id]
-        name = name.join(' ').strip.gsub(/\s+/, ' ')
-        canonical_name = canonical_name.join(' ').strip.gsub(/\s+/, ' ')
-        rank = @ranks[kingdom_id + '/' + rank_id] ?
-               @ranks[kingdom_id + '/' + rank_id] :
-               ''
+        name = name.join(" ").strip.gsub(/\s+/, " ")
+        canonical_name = canonical_name.join(" ").strip.gsub(/\s+/, " ")
+        rank = @ranks[kingdom_id + "/" + rank_id] ||
+               ""
         @names[name_tsn] = { name: name,
                              canonical_name: canonical_name,
                              status: status,
@@ -212,58 +213,57 @@ module DwcaHunter
     end
 
     def generate_dwca
-      DwcaHunter::logger_write(self.object_id,
-                               'Creating DarwinCore Archive file')
-      @core = [['http://rs.tdwg.org/dwc/terms/taxonID',
-                'http://rs.tdwg.org/dwc/terms/parentNameUsageID',
-                'http://rs.tdwg.org/dwc/terms/acceptedNameUsageID',
-                'http://rs.tdwg.org/dwc/terms/scientificName',
-                'http://rs.tdwg.org/ontology/voc/TaxonName#nameComplete',
-                'http://rs.tdwg.org/dwc/terms/taxonomicStatus',
-                'http://rs.tdwg.org/dwc/terms/taxonRank']]
-      @extensions << { data: [['http://rs.tdwg.org/dwc/terms/taxonID',
-                               'http://rs.tdwg.org/dwc/terms/vernacularName',
-                               'http://purl.org/dc/terms/language']],
-                       file_name: 'vernacular_names.txt',
-                       row_type: 'http://rs.gbif.org/terms/1.0/VernacularName'
-                     }
-      @names.keys.each_with_index do |k, i|
+      DwcaHunter.logger_write(object_id,
+                              "Creating DarwinCore Archive file")
+      @core = [["http://rs.tdwg.org/dwc/terms/taxonID",
+                "http://rs.tdwg.org/dwc/terms/parentNameUsageID",
+                "http://rs.tdwg.org/dwc/terms/acceptedNameUsageID",
+                "http://rs.tdwg.org/dwc/terms/scientificName",
+                "http://rs.tdwg.org/ontology/voc/TaxonName#nameComplete",
+                "http://rs.tdwg.org/dwc/terms/taxonomicStatus",
+                "http://rs.tdwg.org/dwc/terms/taxonRank"]]
+      @extensions << { data: [["http://rs.tdwg.org/dwc/terms/taxonID",
+                               "http://rs.tdwg.org/dwc/terms/vernacularName",
+                               "http://purl.org/dc/terms/language"]],
+                       file_name: "vernacular_names.txt",
+                       row_type: "http://rs.gbif.org/terms/1.0/VernacularName" }
+      @names.keys.each_with_index do |k, _i|
         d = @names[k]
-        accepted_id = @synonyms[k] ? @synonyms[k] : nil
+        accepted_id = @synonyms[k] || nil
         parent_id = d[:parent_tsn].to_i == 0 ? nil : d[:parent_tsn]
         row = [k, parent_id, accepted_id, d[:name], d[:canonical_name], d[:status], d[:rank]]
         @core << row
       end
 
-      @vernaculars.keys.each_with_index do |k, i|
+      @vernaculars.keys.each_with_index do |k, _i|
         d = @vernaculars[k]
         @extensions[0][:data] << [k, d[:name], d[:language]]
       end
 
       @eml = {
-          id: @uuid,
-          title: @title,
-          authors: [
-            {email: 'itiswebmaster@itis.gov'}
-          ],
-          metadata_providers: [
-            { first_name: 'Dmitry',
-              last_name: 'Mozzherin',
-              email: 'dmozzherin@gmail.com' }
-            ],
-          abstract: 'The White House Subcommittee on Biodiversity and ' +
-                    'Ecosystem Dynamics has identified systematics as a ' +
-                    'research priority that is fundamental to ecosystem ' +
-                    'management and biodiversity conservation. This primary ' +
-                    'need identified by the Subcommittee requires ' +
-                    'improvements in the organization of, and access to, ' +
-                    'standardized nomenclature. ITIS (originally referred ' +
-                    'to as the Interagency Taxonomic Information System) ' +
-                    'was designed to fulfill these requirements. In the ' +
-                    'future, the ITIS will provide taxonomic data and a ' +
-                    'directory of taxonomic expertise that will support ' +
-                    'the system',
-          url: 'http://www.itis.gov'
+        id: @uuid,
+        title: @title,
+        authors: [
+          { email: "itiswebmaster@itis.gov" }
+        ],
+        metadata_providers: [
+          { first_name: "Dmitry",
+            last_name: "Mozzherin",
+            email: "dmozzherin@gmail.com" }
+        ],
+        abstract: "The White House Subcommittee on Biodiversity and " \
+                    "Ecosystem Dynamics has identified systematics as a " \
+                    "research priority that is fundamental to ecosystem " \
+                    "management and biodiversity conservation. This primary " \
+                    "need identified by the Subcommittee requires " \
+                    "improvements in the organization of, and access to, " \
+                    "standardized nomenclature. ITIS (originally referred " \
+                    "to as the Interagency Taxonomic Information System) " \
+                    "was designed to fulfill these requirements. In the " \
+                    "future, the ITIS will provide taxonomic data and a " \
+                    "directory of taxonomic expertise that will support " \
+                    "the system",
+        url: "http://www.itis.gov"
       }
       super
     end

data/lib/dwca_hunter/resources/mammal_divdb.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+module DwcaHunter
+  class ResourceMammalDiversityDb < DwcaHunter::Resource
+    def initialize(opts = {})
+      @command = "mammal-div-db"
+      @title = "ASM Mammal Diversity Database"
+      @url = "https://mammaldiversity.org/species-account/api.php?q=*"
+      @UUID = "94270cdd-5424-4bb1-8324-46ccc5386dc7"
+      @download_path = File.join(Dir.tmpdir,
+                                 "dwca_hunter",
+                                 "mammal-div-db",
+                                 "data.json")
+      @synonyms = []
+      @names = []
+      @vernaculars = []
+      @extensions = []
+      @synonyms_hash = {}
+      @vernaculars_hash = {}
+      super(opts)
+    end
+
+    def download
+      DwcaHunter.logger_write(object_id, "Downloading")
+      `curl '#{@url}' -H 'User-Agent:' -o #{@download_path}`
+    end
+
+    def unpack; end
+
+    def make_dwca
+      DwcaHunter.logger_write(object_id, "Extracting data")
+      get_names
+      generate_dwca
+    end
+
+    private
+
+    def get_names
+      Dir.chdir(@download_dir)
+      collect_names
+    end
+
+    def collect_names
+      @names_index = {}
+      decoder = HTMLEntities.new
+      data = File.read(File.join(@download_dir, "data.json"))
+      data = JSON.parse(data, symbolize_names: true)
+      data[:result].each_with_index do |e, _i|
+        e = e[1]
+        order = e[:dwc][:order].capitalize
+        order = nil if order.match(/incertae/)
+        family = e[:dwc][:family].capitalize
+        family = nil if family.match(/incertae/)
+        genus = e[:dwc][:genus].capitalize
+        genus = nil if genus.match(/incertae/)
+        name = {
+          taxon_id: e[:internal_id],
+          kingdom: "Animalia",
+          phylum: "Chordata",
+          klass: "Mammalia",
+          order: order,
+          family: family,
+          genus: genus,
+          name_string: "#{e[:dwc][:scientificName]} " \
+          "#{e[:dwc][:scientificNameAuthorship][:species]}".strip,
+          rank: e[:dwc][:taxonRank],
+          status: e[:dwc][:taxonRank],
+          code: "ICZN"
+        }
+        if e[:dwc][:taxonomicStatus] == "accepted"
+          @names << name
+        else
+          @synonyms << name
+        end
+        vern = e[:dwc][:vernacularName]
+        next unless vern.to_s != ""
+        vern = decoder.decode(vern)
+        vernacular = {
+          taxon_id: e[:id],
+          vern: vern,
+          lang: "en"
+        }
+        @vernaculars << vernacular
+      end
+      puts data[:result].size
+    end
+
+    def generate_dwca
+      DwcaHunter.logger_write(object_id,
+                              "Creating DarwinCore Archive file")
+      @core = [["http://rs.tdwg.org/dwc/terms/taxonID",
+                "http://rs.tdwg.org/dwc/terms/scientificName",
+                "http://rs.tdwg.org/dwc/terms/kingdom",
+                "http://rs.tdwg.org/dwc/terms/phylum",
+                "http://rs.tdwg.org/dwc/terms/class",
+                "http://rs.tdwg.org/dwc/terms/order",
+                "http://rs.tdwg.org/dwc/terms/family",
+                "http://rs.tdwg.org/dwc/terms/genus",
+                "http://rs.tdwg.org/dwc/terms/nomenclaturalCode"]]
+      @names.each do |n|
+        @core << [n[:taxon_id], n[:name_string],
+                  n[:kingdom], n[:phylum], n[:klass], n[:order], n[:family],
+                  n[:genus], n[:code]]
+      end
+      @extensions << {
+        data: [[
+          "http://rs.tdwg.org/dwc/terms/taxonID",
+          "http://rs.tdwg.org/dwc/terms/vernacularName",
+          "http://purl.org/dc/terms/language"
+        ]],
+        file_name: "vernacular_names.txt",
+        row_type: "http://rs.gbif.org/terms/1.0/VernacularName"
+      }
+
+      @vernaculars.each do |v|
+        @extensions[-1][:data] << [v[:taxon_id], v[:vern], v[:lang]]
+      end
+
+      @extensions << {
+        data: [[
+          "http://rs.tdwg.org/dwc/terms/taxonID",
+          "http://rs.tdwg.org/dwc/terms/scientificName",
+          "http://rs.tdwg.org/dwc/terms/taxonomicStatus"
+        ]],
+        file_name: "synonyms.txt"
+      }
+      @synonyms.each do |s|
+        @extensions[-1][:data] << [s[:taxon_id], s[:name_string], s[:status]]
+      end
+      @eml = {
+        id: @uuid,
+        title: @title,
+        authors: [
+          { first_name: "C. J.",
+            last_name: "Burgin" },
+          { first_name: "J. P.",
+            last_name: "Colella" },
+          { first_name: "P. L.",
+            last_name: "Kahn" },
+          { first_name: "N. S.",
+            last_name: "Upham" }
+        ],
+        metadata_providers: [
+          { first_name: "Dmitry",
+            last_name: "Mozzherin",
+            email: "dmozzherin@gmail.com" }
+        ],
+        abstract: "Mammal Diversity Database. 2020. www.mammaldiversity.org. " \
+        "American Society of Mammalogists. Accessed 2020-05-24 .",
+        url: @url
+      }
+      super
+    end
+  end
+end