dwca_hunter 0.5.0

data/lib/dwca_hunter/resources/freebase.rb

@@ -0,0 +1,152 @@
+# encoding: utf-8
+
+module DwcaHunter
+  class ResourceFreebase < DwcaHunter::Resource
+    def initialize(opts = {})
+      @command = "freebase"
+      @title = 'Freebase'
+      @uuid = 'bacd21f0-44e0-43e2-914c-70929916f257'
+      @download_path = File.join(Dir.tmpdir,
+                                 'dwca_hunter',
+                                 'freebase',
+                                 'data.json')
+      @data = []
+      @all_taxa = {}
+      @cleaned_taxa = {}
+      @extensions = []
+      super
+    end
+
+    def needs_unpack?
+      false
+    end
+
+    def make_dwca
+      organize_data
+      generate_dwca
+    end
+
+    def download
+      DwcaHunter::logger_write(self.object_id,
+                               'Querying freebase for species information...')
+      q = {
+        query: [{
+          type: '/biology/organism_classification',
+          id: nil,
+          guid: nil,
+          name: nil,
+          scientific_name: nil,
+          synonym_scientific_name: [],
+          higher_classification: {
+            id: nil,
+            guid: nil,
+            scientific_name: nil,
+            optional: true,
+          },
+        }],
+        cursor: true,
+      }
+
+      run_query(q)
+
+      data = JSON.pretty_generate @data
+      f = open(@download_path, 'w:utf-8')
+      f.write(data)
+      f.close
+    end
+
+    private
+
+    def run_query(q)
+      count = 0
+      requests_num = 0
+      while true
+        freebase_url = "http://api.freebase.com/api/service/mqlread?query=%s" %
+          URI.encode(q.to_json)
+        res = JSON.load RestClient.get(freebase_url)
+        requests_num += 1
+        break if res['result'] == nil || res['result'].empty?
+        if requests_num % 10 == 0
+          DwcaHunter::logger_write(self.object_id,
+                                   "Received %s names" % count)
+        end
+        count += res['result'].size
+        res['result'].each { |d| @data << d }
+        q[:cursor] = res['cursor']
+      end
+    end
+
+    def organize_data
+      @data = JSON.load(open(@download_path, 'r:utf-8').read)
+      @data.each do |d|
+        scientific_name = d['scientific_name'].to_s
+        id = d["id"]
+        parent_id = d['higher_classification'] ?
+                    d['higher_classification']["id"] :
+                    nil
+        synonyms = d['synonym_scientific_name']
+        @all_taxa[id] = { id: id,
+                          parent_id: parent_id,
+                          scientific_name: scientific_name,
+                          synonyms: synonyms }
+      end
+
+      @all_taxa.each do |k, v|
+        next unless v[:scientific_name] && v[:scientific_name].strip != ""
+        parent_id = v[:parent_id]
+        until (@all_taxa[parent_id] &&
+                @all_taxa[parent_id][:scientific_name]) || parent_id.nil?
+          puts "did not find parent %s" % parent_id
+          parent_id = @all_taxa[parent_id]
+        end
+        parent_id = nil if v[:id] == parent_id
+        v[:parent_id] = parent_id
+        @cleaned_taxa[k] = v
+      end
+
+    end
+
+    def generate_dwca
+      DwcaHunter::logger_write(self.object_id,
+                               'Creating DarwinCore Archive file')
+      @core = [['http://rs.tdwg.org/dwc/terms/taxonID',
+                'http://rs.tdwg.org/dwc/terms/scientificName',
+                'http://rs.tdwg.org/dwc/terms/parentNameUsageID']]
+
+      @extensions << { data: [[
+        'http://rs.tdwg.org/dwc/terms/TaxonID',
+        'http://rs.tdwg.org/dwc/terms/scientificName',
+      ]], file_name: 'synonyms.txt' }
+      DwcaHunter::logger_write(self.object_id,
+                    'Creating synonyms extension for DarwinCore Archive file')
+      count = 0
+      @cleaned_taxa.each do |key, taxon|
+        count += 1
+        @core << [taxon[:id], taxon[:scientific_name], taxon[:parent_id]]
+        if count % BATCH_SIZE == 0
+          DwcaHunter::logger_write(self.object_id,
+                                 "Traversing %s extension data record" % count)
+        end
+        taxon[:synonyms].each do |name|
+          @extensions[-1][:data] << [taxon[:id], name]
+        end
+      end
+      @eml = {
+        id: @uuid,
+        title: @title,
+        license: 'http://creativecommons.org/licenses/by-sa/3.0/',
+        authors: [
+          { url: 'http://www.freebase.com/home' }],
+        abstract: 'An entity graph of people, places and things, ' +
+                  'built by a community that loves open data.',
+        metadata_providers: [
+          { first_name: 'Dmitry',
+            last_name: 'Mozzherin',
+            email: 'dmozzherin@mbl.edu' }],
+        url: 'http://www.freebase.com/home'
+      }
+      super
+    end
+
+  end
+end

data/lib/dwca_hunter/resources/gnub.rb

@@ -0,0 +1,101 @@
+# encoding: utf-8
+module DwcaHunter
+  class ResourceGNUB < DwcaHunter::Resource
+    def initialize(opts = {})
+      @command = 'gnub'
+      @title = 'GNUB'
+      @url = 'http://gnub.org/datadump/gni_export.zip'
+      @UUID =  'd34ed224-78e7-485d-a478-adc2558a0f68'
+      @download_path = File.join(Dir.tmpdir,
+                                 'dwca_hunter',
+                                 'gnub',
+                                 'data.tar.gz')
+      @ranks = {}
+      @kingdoms = {}
+      @authors = {}
+      @vernaculars = {}
+      @synonyms = {}
+      @synonym_of = {}
+      @names = []
+      @extensions = []
+      super(opts)
+      @gnub_dir = File.join(@download_dir, 'gnub')
+    end
+
+    def unpack
+      unpack_zip
+    end
+
+    def make_dwca
+      DwcaHunter::logger_write(self.object_id, 'Extracting data')
+      get_names
+      generate_dwca
+    end
+
+    private
+
+    def get_names
+      codes = get_codes
+      file = Dir.entries(@download_dir).grep(/txt$/).first
+      open(File.join(@download_dir, file)).each_with_index do |line, i|
+        next if i == 0 || (data = line.strip) == ''
+        data = data.split("\t")
+        protolog = data[0].downcase
+        protolog_path = data[1].downcase
+        name_string = data[2]
+        rank = data[3]
+        code = codes[data[4].to_i]
+        taxon_id = UUID.create_v5(name_string +
+                                  protolog_path +
+                                  rank, GNA_NAMESPACE)
+        @names << { taxon_id: taxon_id,
+                    name_string: name_string,
+                    protolog: protolog,
+                    protolog_path: protolog_path,
+                    code: code,
+                    rank: rank }
+      end
+    end
+
+    def get_codes
+      codes_url = 'http://resolver.globalnames.org/nomenclatural_codes.json'
+      codes = RestClient.get(codes_url)
+      codes = JSON.parse(codes, symbolize_names: true)
+      codes.inject({}) do |res, c|
+        res[c[:id]] = c[:code]
+        res
+      end
+    end
+
+    def generate_dwca
+      DwcaHunter::logger_write(self.object_id,
+                               'Creating DarwinCore Archive file')
+      @core = [['http://rs.tdwg.org/dwc/terms/taxonID',
+                'http://rs.tdwg.org/dwc/terms/originalNameUsageID',
+                'http://globalnames.org/terms/originalNameUsageIDPath',
+                'http://rs.tdwg.org/dwc/terms/scientificName',
+                'http://rs.tdwg.org/dwc/terms/nomenclaturalCode',
+                'http://rs.tdwg.org/dwc/terms/taxonRank']]
+      @names.each do |n|
+        @core << [n[:taxon_id], n[:protolog], n[:name_string],
+                  n[:protolog_path], n[:code], n[:rank]]
+      end
+      @eml = {
+          id: @uuid,
+          title: @title,
+          authors: [
+            {email: 'deepreef@bishopmuseum.org'}
+          ],
+          metadata_providers: [
+            { first_name: 'Dmitry',
+              last_name: 'Mozzherin',
+              email: 'dmozzherin@gmail.com' }
+            ],
+          abstract: 'Global Names Usage Bank',
+          url: 'http://www.zoobank.org'
+      }
+      super
+    end
+  end
+end
+

data/lib/dwca_hunter/resources/itis.rb

@@ -0,0 +1,271 @@
+# encoding: utf-8
+module DwcaHunter
+  class ResourceITIS < DwcaHunter::Resource
+    def initialize(opts = {})
+      @command = 'itis'
+      @title = 'ITIS'
+      @url = 'https://www.itis.gov/downloads/itisMySQLTables.tar.gz'
+      @uuid =  '5d066e84-e512-4a2f-875c-0a605d3d9f35'
+      @download_path = File.join(Dir.tmpdir,
+                                 'dwca_hunter',
+                                 'itis',
+                                 'data.tar.gz')
+      @ranks = {}
+      @kingdoms = {}
+      @authors = {}
+      @vernaculars = {}
+      @synonyms = {}
+      @synonym_of = {}
+      @names = {}
+      @extensions = []
+      super(opts)
+      @itis_dir = File.join(@download_dir, 'itis')
+    end
+
+    def unpack
+      unpack_tar
+      dir = Dir.entries(@download_dir).select {|e| e.match(/itisMySQL/)}[0]
+      FileUtils.mv(File.join(@download_dir, dir), @itis_dir)
+
+      # Create a file with the same name as the directory we extracted.
+      FileUtils.touch(File.join(@itis_dir, 'version_' + dir))
+    end
+
+    def make_dwca
+      DwcaHunter::logger_write(self.object_id, 'Extracting data')
+      get_ranks
+      get_kingdoms
+      get_authors
+      get_vernaculars
+      get_synonyms
+      get_names
+      generate_dwca
+    end
+
+  private
+    def get_ranks
+      # 0    kingdom_id integer not null
+      # 1    rank_id smallint not null
+      # 2    rank_name char(15) not null
+      # 3    dir_parent_rank_id smallint not null
+      # 4    req_parent_rank_id smallint not null
+      # 5    update_date date not null
+      rank_file = File.join(@itis_dir, 'taxon_unit_types')
+      f = open(rank_file, 'r:utf-8')
+      f.each do |l|
+        l.encode!('UTF-8',
+                  'ISO-8859-1',
+                  invalid: :replace,
+                  replace: '?')
+        row = l.strip.split('|')
+        @ranks[row[0].strip + '/' + row[1].strip] = row[2].strip
+      end
+    end
+
+    def get_kingdoms
+      # 0    kingdom_id serial not null
+      # 1    kingdom_name char(10) not null
+      # 2    update_date date not null
+
+      f = open(File.join(@itis_dir, 'kingdoms'))
+      f.each do |l|
+        data = l.strip.split('|')
+        @kingdoms[data[0].strip] = data[1].strip
+      end
+    end
+
+    def get_authors
+      # 0    taxon_author_id serial not null
+      # 1    taxon_author varchar(100,30) not null
+      # 2    update_date date not null
+      # 3    kingdom_id smallint not null
+
+      f = open(File.join(@itis_dir, 'taxon_authors_lkp'))
+      f.each do |l|
+        l.encode!('UTF-8',
+                  'ISO-8859-1',
+                  invalid: :replace,
+                  replace: '?')
+        data = l.strip.split('|')
+        @authors[data[0].strip] = data[1].strip
+      end
+    end
+
+    def get_vernaculars
+      # 0    tsn integer not null
+      # 1    vernacular_name varchar(80,5) not null
+      # 2    language varchar(15) not null
+      # 3    approved_ind char(1)
+      # 4    update_date date not null
+      # 5    primary key (tsn,vernacular_name,language)
+      #      constraint "itis".vernaculars_key
+
+      f = open(File.join(@itis_dir, 'vernaculars'))
+      f.each_with_index do |l, i|
+        if i % BATCH_SIZE == 0
+          DwcaHunter::logger_write(self.object_id,
+                                   "Extracted %s vernacular names" % i)
+        end
+        l.encode!('UTF-8',
+                  'ISO-8859-1',
+                  invalid: :replace,
+                  replace: '?')
+        data = l.split('|').map { |d| d.strip }
+        name_tsn = data[0]
+        string   = data[1]
+        language = data[2]
+        language = 'Common name' if language == 'unspecified'
+        @vernaculars[name_tsn] = { name:string, language:language }
+      end
+    end
+
+    def get_synonyms
+      # 0    tsn integer not null
+      # 1    tsn_accepted integer not null
+      # 2    update_date date not null
+
+      f = open(File.join(@itis_dir, 'synonym_links'))
+      f.each_with_index do |l, i|
+        if i % BATCH_SIZE == 0
+          DwcaHunter::logger_write(self.object_id,
+                                   "Extracted %s synonyms" % i)
+        end
+        l.encode!('UTF-8',
+                  'ISO-8859-1',
+                  invalid: :replace,
+                  replace: '?')
+        data = l.split('|').map { |d| d.strip }
+        synonym_name_tsn = data[0]
+        accepted_name_tsn = data[1]
+        @synonyms[synonym_name_tsn] = accepted_name_tsn
+      end
+    end
+
+    def get_names
+      # 0    tsn serial not null
+      # 1    unit_ind1 char(1)
+      # 2    unit_name1 char(35) not null
+      # 3    unit_ind2 char(1)
+      # 4    unit_name2 varchar(35)
+      # 5    unit_ind3 varchar(7)
+      # 6    unit_name3 varchar(35)
+      # 7    unit_ind4 varchar(7)
+      # 8    unit_name4 varchar(35)
+      # 9    unnamed_taxon_ind char(1)
+      # 10   usage varchar(12,5) not null
+      # 11   unaccept_reason varchar(50,9)
+      # 12   credibility_rtng varchar(40,17) not null
+      # 13   completeness_rtng char(10)
+      # 14   currency_rating char(7)
+      # 15   phylo_sort_seq smallint
+      # 16   initial_time_stamp datetime year to second not null
+      # 17   parent_tsn integer
+      # 18   taxon_author_id integer
+      # 19   hybrid_author_id integer
+      # 20   kingdom_id smallint not null
+      # 21   rank_id smallint not null
+      # 22   update_date date not null
+      # 23   uncertain_prnt_ind char(3)
+
+      f = open(File.join(@itis_dir, 'taxonomic_units'))
+      f.each_with_index do |l, i|
+        if i % BATCH_SIZE == 0
+          DwcaHunter::logger_write(self.object_id,
+                                   "Extracted %s names" % i)
+        end
+        l.encode!('UTF-8',
+                  'ISO-8859-1',
+                  invalid: :replace,
+                  replace: '?')
+        data = l.split("|").map { |d| d.strip }
+        name_tsn   = data[0]
+        x1         = data[1]
+        name_part1 = data[2]
+        x2         = data[3]
+        name_part2 = data[4]
+        sp_marker1 = data[5]
+        name_part3 = data[6]
+        sp_marker2 = data[7]
+        name_part4 = data[8]
+        status     = data[10]
+        parent_tsn = data[17]
+        author_id  = data[18]
+        kingdom_id = data[20]
+        rank_id    = data[21]
+
+        parent_tsn = nil if parent_tsn == ''
+        name = [x1, name_part1, x2, name_part2,
+                sp_marker1, name_part3, sp_marker2, name_part4]
+        canonical_name = name.clone
+        name << @authors[author_id] if @authors[author_id]
+        name = name.join(' ').strip.gsub(/\s+/, ' ')
+        canonical_name = canonical_name.join(' ').strip.gsub(/\s+/, ' ')
+        rank = @ranks[kingdom_id + '/' + rank_id] ?
+               @ranks[kingdom_id + '/' + rank_id] :
+               ''
+        @names[name_tsn] = { name: name,
+                             canonical_name: canonical_name,
+                             status: status,
+                             parent_tsn: parent_tsn,
+                             rank: rank }
+      end
+    end
+
+    def generate_dwca
+      DwcaHunter::logger_write(self.object_id,
+                               'Creating DarwinCore Archive file')
+      @core = [['http://rs.tdwg.org/dwc/terms/taxonID',
+                'http://rs.tdwg.org/dwc/terms/parentNameUsageID',
+                'http://rs.tdwg.org/dwc/terms/acceptedNameUsageID',
+                'http://rs.tdwg.org/dwc/terms/scientificName',
+                'http://rs.tdwg.org/ontology/voc/TaxonName#nameComplete',
+                'http://rs.tdwg.org/dwc/terms/taxonomicStatus',
+                'http://rs.tdwg.org/dwc/terms/taxonRank']]
+      @extensions << { data: [['http://rs.tdwg.org/dwc/terms/taxonID',
+                               'http://rs.tdwg.org/dwc/terms/vernacularName',
+                               'http://purl.org/dc/terms/language']],
+                       file_name: 'vernacular_names.txt',
+                       row_type: 'http://rs.gbif.org/terms/1.0/VernacularName'
+                     }
+      @names.keys.each_with_index do |k, i|
+        d = @names[k]
+        accepted_id = @synonyms[k] ? @synonyms[k] : nil
+        parent_id = d[:parent_tsn].to_i == 0 ? nil : d[:parent_tsn]
+        row = [k, parent_id, accepted_id, d[:name], d[:canonical_name], d[:status], d[:rank]]
+        @core << row
+      end
+
+      @vernaculars.keys.each_with_index do |k, i|
+        d = @vernaculars[k]
+        @extensions[0][:data] << [k, d[:name], d[:language]]
+      end
+
+      @eml = {
+          id: @uuid,
+          title: @title,
+          authors: [
+            {email: 'itiswebmaster@itis.gov'}
+          ],
+          metadata_providers: [
+            { first_name: 'Dmitry',
+              last_name: 'Mozzherin',
+              email: 'dmozzherin@gmail.com' }
+            ],
+          abstract: 'The White House Subcommittee on Biodiversity and ' +
+                    'Ecosystem Dynamics has identified systematics as a ' +
+                    'research priority that is fundamental to ecosystem ' +
+                    'management and biodiversity conservation. This primary ' +
+                    'need identified by the Subcommittee requires ' +
+                    'improvements in the organization of, and access to, ' +
+                    'standardized nomenclature. ITIS (originally referred ' +
+                    'to as the Interagency Taxonomic Information System) ' +
+                    'was designed to fulfill these requirements. In the ' +
+                    'future, the ITIS will provide taxonomic data and a ' +
+                    'directory of taxonomic expertise that will support ' +
+                    'the system',
+          url: 'http://www.itis.gov'
+      }
+      super
+    end
+  end
+end

data/lib/dwca_hunter/resources/mammal_species.rb

@@ -0,0 +1,179 @@
+module DwcaHunter
+  class ResourceMammalSpecies < DwcaHunter::Resource
+    def initialize(opts = {})
+      @command = "mammal-species"
+      @title = "The Mammal Species of The World"
+      @uuid = "464dafec-1037-432d-8449-c0b309e0a030"
+      @data = []
+      @extensions = []
+      @count = 1
+      @clades = {"Mammalia" => { rank: "class", id: @count}}
+      @url = "http://www.departments.bucknell.edu"\
+             "/biology/resources/msw3/export.asp"
+      @download_path = File.join(Dir.tmpdir, "dwca_hunter",
+                                 "mammalsp", "msw3-all.csv")
+      super
+    end
+
+    def needs_unpack?
+      false
+    end
+
+    def make_dwca
+      DwcaHunter::logger_write(self.object_id, "Extracting data")
+      encode
+      collect_data
+      generate_dwca
+    end
+
+    def download
+      DwcaHunter::logger_write(self.object_id, "Downloading file -- "\
+                               "it will take some time...")
+      dlr = DwcaHunter::Downloader.new(url, @download_path)
+      dlr.download
+    end
+
+    private
+
+    def encode
+      DwcaHunter::Encoding.latin1_to_utf8(@download_path)
+    end
+
+    def collect_data
+      opts = { headers: true, header_converters: :symbol }
+      CSV.open(@download_path + ".utf_8", opts).each do |row|
+        @data << row.to_hash
+      end
+    end
+
+    def generate_dwca
+      DwcaHunter::logger_write(self.object_id,
+                               'Creating DarwinCore Archive file')
+      core_init
+      extensions_init
+      eml_init
+      @data.each do |rec|
+        taxon = process_hierarchy(rec)
+        process_vernaculars(rec, taxon)
+        process_synonyms(rec, taxon)
+      end
+      super
+    end
+
+    def process_vernaculars(rec, taxon)
+      return if rec[:commonname].to_s == ""
+      taxon_id = taxon[0]
+      lang = "en"
+      name = rec[:commonname].gsub("\u{0092}", "'")
+      @extensions[0][:data] << [taxon_id, name, lang]
+
+    end
+
+    def process_synonyms(rec, taxon)
+      accepted_id = taxon[0]
+      parent_id = taxon[2]
+      rank = taxon[-1]
+      return unless ['species', 'subspecies'].include? rank
+      synonyms = rec[:synonyms].gsub(/\.$/, "").
+        gsub(/<[\/ib]+>/, "").gsub(/[\s]+/, " ").split(";")
+      synonyms = synonyms.map(&:strip)
+      synonyms = synonyms.map do |s|
+        next if s.match(/<u>/)
+        if s.match(/^[a-z]/)
+          s = rec[:genus] + " " + s
+        end
+        @count += 1
+        id = @count
+        @core << [id, nil, parent_id, accepted_id, s, "synonym", rank]
+      end
+    end
+
+    def process_name(rec, rank)
+      name =[@core.last[4], rec[:author], rec[:date]]
+      @core.last[4] = name.join(" ").gsub(/[\s]+/, " ").strip
+      @core.last[1] = rec[:id]
+    end
+
+    def process_hierarchy(rec)
+      parent_id = @clades["Mammalia"][:id]
+      is_row_rank = false
+      [:order, :suborder, :infraorder, :superfamily, :family,
+       :subfamily, :tribe, :genus, :subgenus,
+       :species, :subspecies].each do |rank|
+       is_row_rank = true if rank == rec[:taxonlevel].downcase.to_sym
+        clade = rec[rank]
+        clade = clade.capitalize if clade.match(/^[A-Z]+$/)
+        next if clade.to_s == ""
+        clade_id = nil
+        clade = adjust_clade(rec, rank, clade)
+        if @clades.key?(clade)
+          clade_id = @clades[clade][:id]
+        else
+          @count += 1
+          clade_id = @count
+          @clades[clade] = { id: clade_id, rank: rank }
+          @core << [clade_id, nil, parent_id, clade_id, clade, nil, rank.to_s]
+          if is_row_rank
+            process_name(rec, rank)
+            return @core.last
+          end
+        end
+        parent_id = clade_id
+      end
+    end
+
+    def adjust_clade(rec, rank, clade)
+      if [:species, :subspecies].include? rank
+        clade = [rec[:genus], rec[:species]]
+        clade << rec[:subspecies] if rank == :subspecies
+        clade.join(" ").gsub(/[\s]+/, " ").strip
+      else
+        clade
+      end
+    end
+
+    def eml_init
+      @eml = {
+        id: @uuid,
+        title: @title,
+        authors: [
+          { first_name: "Don",
+            last_name: "Wilson" },
+          { first_name: "DeeAnn",
+            last_name: "Reader" },
+      ],
+        metadata_providers: [
+          { first_name: "Dmitry",
+            last_name: "Mozzherin",
+            email: "dmozzherin@gmail.com" }
+      ],
+        abstract: "Mammal Species of the World, 3rd edition (MSW3) is "\
+        "a database of mammalian taxonomy, based upon the 2005 book "\
+        "Mammal Species of the World. A Taxonomic and Geographic Reference "\
+        "(3rd ed). Don E. Wilson & DeeAnn M. Reeder (editors).",
+        url: "http://www.vertebrates.si.edu/msw/mswcfapp/msw/index.cfm"
+      }
+    end
+
+    def core_init
+      @core = [['http://rs.tdwg.org/dwc/terms/taxonID',
+                'http://globalnames.org/terms/localID',
+                'http://rs.tdwg.org/dwc/terms/parentNameUsageID',
+                'http://rs.tdwg.org/dwc/terms/acceptedNameUsageID',
+                'http://rs.tdwg.org/dwc/terms/scientificName',
+                'http://rs.tdwg.org/dwc/terms/taxonomicStatus',
+                'http://rs.tdwg.org/dwc/terms/taxonRank']]
+      m = @clades["Mammalia"]
+      @core << [m[:id], nil, nil, m[:id], "Mammalia", nil, "class"]
+    end
+
+    def extensions_init
+      @extensions << { data: [['http://rs.tdwg.org/dwc/terms/taxonID',
+                               'http://rs.tdwg.org/dwc/terms/vernacularName',
+                               'http://purl.org/dc/terms/language']],
+                       file_name: 'vernacular_names.txt',
+                       row_type: 'http://rs.gbif.org/terms/1.0/VernacularName'
+                     }
+    end
+  end
+end