toccatore 0.2.3 → 0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 18794a290d5b37831e228d170849338b67e48cc9
-  data.tar.gz: d7a8d75e76ac3611375895cfa5baf552daf1579c
+  metadata.gz: e67c0fc653ac9fc750cd7a75ecd0056c72dcdfec
+  data.tar.gz: cf181d9073928728a382e6740912a982cb02fd61
 SHA512:
-  metadata.gz: 394448d66f7e3673579d8ac0d0c93403ccd3d5e9549106dd50e4248c51d9a2f120842f4a5e6402e68c5d85283f3041667511e9273a4d1a4e90b53059c6b7c7e4
-  data.tar.gz: 4e3b4fa820d446eb2a7558adcf314a371a2bac1af8f2ed1f08aef1a93f27cffa5f1f8ebf74264124d4db3ad86bab1bde53c2b3d00ff7525fb0303f5ab3ce9775
+  metadata.gz: 4a1c3d00575f9289cc7938028829b49a1cddbb397419e17483360e4a6586566fbf3e20a1f760727e8a90df20b4dcfce06097ac87e2a96b09ffc12c567f555e16
+  data.tar.gz: 43ae2875879cf644783244d554d52a6bde83c615bd5dbea12a441ac9291c61d65b2bc7fb8bc03a597f880676499d76c19c908d43cd490e8fbeb1ac52705d177d

data/Dockerfile

@@ -1,7 +1,10 @@
-FROM phusion/passenger-full:0.9.19
+FROM phusion/passenger-full:0.9.20
 MAINTAINER Martin Fenner "mfenner@datacite.org"
 
-ENV PATH="/usr/local/rvm/gems/ruby-2.3.1/bin:${PATH}"
+# Install Ruby 2.3.3
+RUN bash -lc 'rvm --default use ruby-2.3.3'
+
+ENV PATH="/usr/local/rvm/gems/ruby-2.3.3/bin:${PATH}"
 
 # Update installed APT packages, clean up APT when done.
 RUN apt-get update && apt-get upgrade -y -o Dpkg::Options::="--force-confold" && \
@@ -10,4 +13,4 @@ RUN apt-get update && apt-get upgrade -y -o Dpkg::Options::="--force-confold" &&
 # Install toccatore gem
 RUN /sbin/setuser app gem install toccatore
 
-CMD toccatore orcid_update --push_url $VOLPINO_URL --access_token $VOLPINO_TOKEN --from_date $FROM_DATE --until_date $UNTIL_DATE
+CMD toccatore --version

data/Gemfile.lock

@@ -1,27 +1,26 @@
 PATH
   remote: .
   specs:
-    toccatore (0.2.3)
+    toccatore (0.3)
       activesupport (~> 4.2, >= 4.2.5)
       dotenv (~> 2.1, >= 2.1.1)
       gender_detector (~> 1.0)
-      maremma (~> 3.1)
+      maremma (~> 3.5)
       namae (~> 0.11.0)
       thor (~> 0.19)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (4.2.7.1)
+    activesupport (4.2.8)
       i18n (~> 0.7)
-      json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
       thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
     addressable (2.5.0)
       public_suffix (~> 2.0, >= 2.0.2)
     builder (3.2.3)
-    codeclimate-test-reporter (1.0.5)
+    codeclimate-test-reporter (1.0.7)
       simplecov
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
@@ -37,11 +36,11 @@ GEM
       faraday (>= 0.7.4, < 1.0)
     gender_detector (1.0.0)
     hashdiff (0.3.2)
-    i18n (0.8.0)
-    json (1.8.6)
-    maremma (3.1.2)
+    i18n (0.8.1)
+    json (2.0.3)
+    maremma (3.5.1)
       activesupport (~> 4.2, >= 4.2.5)
-      addressable (~> 2.5)
+      addressable (>= 2.3.6)
       builder (~> 3.2, >= 3.2.2)
       excon (~> 0.45.0)
       faraday (~> 0.9.2)
@@ -49,7 +48,7 @@ GEM
       faraday_middleware (~> 0.10.0)
       multi_json (~> 1.11.2)
       nokogiri (~> 1.6.7)
-      oj (~> 2.13.1)
+      oj (~> 2.18, >= 2.18.1)
     mini_portile2 (2.1.0)
     minitest (5.10.1)
     multi_json (1.11.3)
@@ -57,7 +56,7 @@ GEM
     namae (0.11.3)
     nokogiri (1.6.8.1)
       mini_portile2 (~> 2.1.0)
-    oj (2.13.1)
+    oj (2.18.2)
     public_suffix (2.0.5)
     rack (2.0.1)
     rack-test (0.6.3)
@@ -83,7 +82,7 @@ GEM
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.0)
     thor (0.19.4)
-    thread_safe (0.3.5)
+    thread_safe (0.3.6)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
     vcr (3.0.3)

data/README.md

@@ -4,7 +4,7 @@
 [![Code Climate](https://codeclimate.com/github/datacite/toccatore/badges/gpa.svg)](https://codeclimate.com/github/datacite/toccatore)
 [![Test Coverage](https://codeclimate.com/github/datacite/toccatore/badges/coverage.svg)](https://codeclimate.com/github/datacite/toccatore/coverage)
 
-Command-line client for finding ORCID IDs in DataCite metadata.
+Command-line client for finding ORCID IDs and related identifiers in DataCite metadata.
 
 ## Development
 

data/lib/toccatore.rb

@@ -1,2 +1,4 @@
 require "toccatore/orcid_update"
+require "toccatore/datacite_related"
 require "toccatore/cli"
+require "toccatore/version"

data/lib/toccatore/base.rb

@@ -28,6 +28,8 @@ module Toccatore
         q = "doi:#{options[:doi]}"
       elsif options[:orcid].present?
         q = "nameIdentifier:ORCID\\:#{options[:orcid]}"
+      elsif options[:related_identifier].present?
+        q = "relatedIdentifier:DOI\\:#{options[:related_identifier]}"
       elsif options[:query].present?
         q = options[:query]
       else
@@ -37,7 +39,7 @@ module Toccatore
       params = { q: q,
                  start: options[:offset],
                  rows: options[:rows],
-                 fl: "doi,creator,title,publisher,publicationYear,resourceTypeGeneral,datacentre_symbol,relatedIdentifier,nameIdentifier,xml,minted,updated",
+                 fl: "doi,resourceTypeGeneral,relatedIdentifier,nameIdentifier,minted,updated",
                  fq: fq,
                  wt: "json" }
       url +  URI.encode_www_form(params)
@@ -87,206 +89,46 @@ module Toccatore
       Maremma.get(query_url, options)
     end
 
-    def parse_data(result, options={})
-      return result.body.fetch("errors") if result.body.fetch("errors", nil).present?
-
-      items = result.fetch("data", {}).fetch('response', {}).fetch('docs', nil)
-      get_relations_with_related_works(items)
-    end
-
-    # push to Lagotto deposit API if no error and we have collected works
-    def push_data(items, options={})
-      if items.empty?
-        puts "No works found for date range #{options[:from_date]} - #{options[:until_date]}."
-      else
-        Array(items).map do |item|
-          relation = item.fetch(:relation, {})
-          deposit = { "deposit" => { "subj_id" => relation.fetch("subj_id", nil),
-                                     "obj_id" => relation.fetch("obj_id", nil),
-                                     "relation_type_id" => relation.fetch("relation_type_id", nil),
-                                     "source_id" => relation.fetch("source_id", nil),
-                                     "publisher_id" => relation.fetch("publisher_id", nil),
-                                     "subj" => item.fetch(:subj, {}),
-                                     "obj" => item.fetch(:obj, {}),
-                                     "message_type" => item.fetch(:message_type, "relation"),
-                                     "prefix" => item.fetch(:prefix, nil),
-                                     "source_token" => uuid } }
-
-          Maremma.post push_url, data: deposit.to_json, content_type: 'json', token: access_token
-        end
-      end
+    def url
+      "https://search.datacite.org/api?"
     end
 
-    def get_relations_with_related_works(items)
-      Array(items).reduce([]) do |sum, item|
-        doi = item.fetch("doi", nil)
-        prefix = doi[/^10\.\d{4,5}/]
-        pid = doi_as_url(doi)
-        type = item.fetch("resourceTypeGeneral", nil)
-        publisher_id = item.fetch("datacentre_symbol", nil)
-
-        xml = Base64.decode64(item.fetch('xml', "PGhzaD48L2hzaD4=\n"))
-        xml = Hash.from_xml(xml).fetch("resource", {})
-        authors = xml.fetch("creators", {}).fetch("creator", [])
-        authors = [authors] if authors.is_a?(Hash)
-
-        subj = { "pid" => pid,
-                 "DOI" => doi,
-                 "author" => get_hashed_authors(authors),
-                 "title" => item.fetch("title", []).first,
-                 "container-title" => item.fetch("publisher", nil),
-                 "published" => item.fetch("publicationYear", nil),
-                 "issued" => item.fetch("minted", nil),
-                 "publisher_id" => publisher_id,
-                 "registration_agency_id" => "datacite",
-                 "tracked" => true,
-                 "type" => type }
-
-        related_doi_identifiers = item.fetch('relatedIdentifier', []).select { |id| id =~ /:DOI:.+/ }
-        sum += get_doi_relations(subj, related_doi_identifiers)
-
-        related_github_identifiers = item.fetch('relatedIdentifier', []).select { |id| id =~ /:URL:https:\/\/github.com.+/ }
-        sum += get_github_relations(subj, related_github_identifiers)
-
-        name_identifiers = item.fetch('nameIdentifier', []).select { |id| id =~ /^ORCID:.+/ }
-        sum += get_contributions(subj, name_identifiers)
-
-        if source_id == "datacite_import"
-          sum += [{ prefix: prefix,
-                    relation: { "subj_id" => subj["pid"],
-                                "source_id" => source_id,
-                                "publisher_id" => subj["publisher_id"],
-                                "occurred_at" => subj["issued"] },
-                    subj: subj }]
-        end
-
-        sum
-      end
+    def timeout
+      120
     end
 
-    def get_github_relations(subj, items)
-      prefix = subj["DOI"][/^10\.\d{4,5}/]
-
-      Array(items).reduce([]) do |sum, item|
-        raw_relation_type, _related_identifier_type, related_identifier = item.split(':', 3)
-
-        # get parent repo
-        # code from https://github.com/octokit/octokit.rb/blob/master/lib/octokit/repository.rb
-        related_identifier = PostRank::URI.clean(related_identifier)
-        github_hash = github_from_url(related_identifier)
-        owner_url = github_as_owner_url(github_hash)
-        repo_url = github_as_repo_url(github_hash)
-
-        sum << { prefix: prefix,
-                 relation: { "subj_id" => subj["pid"],
-                             "obj_id" => related_identifier,
-                             "relation_type_id" => raw_relation_type.underscore,
-                             "source_id" => source_id,
-                             "publisher_id" => subj["publisher_id"],
-                             "registration_agency_id" => "github",
-                             "occurred_at" => subj["issued"] },
-                 subj: subj }
-
-        # if relatedIdentifier is release URL rather than repo URL
-        if related_identifier != repo_url
-          sum << { relation: { "subj_id" => related_identifier,
-                               "obj_id" => repo_url,
-                               "relation_type_id" => "is_part_of",
-                               "source_id" => source_id,
-                               "publisher_id" => "github",
-                               "registration_agency_id" => "github" } }
-        end
-
-        sum << {  message_type: "contribution",
-                  relation: { "subj_id" => owner_url,
-                              "obj_id" => repo_url,
-                              "source_id" => "github_contributor",
-                              "registration_agency_id" => "github" }}
-      end
+    def job_batch_size
+      1000
     end
 
-    def get_doi_relations(subj, items)
-      prefix = subj["DOI"][/^10\.\d{4,5}/]
-
-      Array(items).reduce([]) do |sum, item|
-        raw_relation_type, _related_identifier_type, related_identifier = item.split(':', 3)
-        doi = related_identifier.strip.upcase
-        registration_agency = get_doi_ra(doi)
-
-        if source_id == "datacite_crossref" && registration_agency == "datacite"
-          sum
-        else
-          _source_id = registration_agency == "crossref" ? "datacite_crossref" : "datacite_related"
-          pid = doi_as_url(doi)
-
-          sum << { prefix: prefix,
-                   relation: { "subj_id" => subj["pid"],
-                               "obj_id" => pid,
-                               "relation_type_id" => raw_relation_type.underscore,
-                               "source_id" => _source_id,
-                               "publisher_id" => subj["publisher_id"],
-                               "registration_agency_id" => registration_agency,
-                               "occurred_at" => subj["issued"] },
-                   subj: subj }
-        end
-      end
-    end
+    def get_doi_ra(prefix)
+      return nil if prefix.blank?
 
-    # we are flipping subj and obj for contributions
-    def get_contributions(obj, items)
-      prefix = obj["DOI"][/^10\.\d{4,5}/]
-
-      Array(items).reduce([]) do |sum, item|
-        orcid = item.split(':', 2).last
-        orcid = validate_orcid(orcid)
-
-        return sum if orcid.nil?
-
-        sum << { prefix: prefix,
-                 message_type: "contribution",
-                 relation: { "subj_id" => orcid_as_url(orcid),
-                             "obj_id" => obj["pid"],
-                             "relation_type_id" => nil,
-                             "source_id" => source_id,
-                             "publisher_id" => obj["publisher_id"],
-                             "registration_agency_id" => "datacite",
-                             "occurred_at" => obj["issued"] },
-                 obj: obj }
-      end
-    end
+      url = "https://api.datacite.org/prefixes/#{prefix}"
+      result = Maremma.get(url)
 
-    def config_fields
-      [:url, :push_url, :access_token]
-    end
+      return result.body.fetch("errors") if result.body.fetch("errors", nil).present?
 
-    def url
-      "https://search.datacite.org/api?"
+      result.body.fetch("data", {}).fetch('attributes', {}).fetch('registration-agency', nil)
     end
 
-    def timeout
-      120
+    def validate_doi(doi)
+      Array(/\A(?:(http|https):\/\/(dx\.)?doi.org\/)?(doi:)?(10\.\d{4,5}\/.+)\z/.match(doi)).last
     end
 
-    def job_batch_size
-      1000
+    def validate_prefix(doi)
+      Array(/\A(?:(http|https):\/\/(dx\.)?doi.org\/)?(doi:)?(10\.\d{4,5})\/.+\z/.match(doi)).last
     end
 
-    # remove non-printing whitespace
-    def clean_doi(doi)
-      doi.gsub(/\u200B/, '')
-    end
+    def normalize_doi(doi)
+      doi = validate_doi(doi)
+      return nil unless doi.present?
 
-    def doi_from_url(url)
-      if /(http|https):\/\/(dx\.)?doi\.org\/(\w+)/.match(url)
-        uri = Addressable::URI.parse(url)
-        uri.path[1..-1].upcase
-      elsif url.starts_with?("doi:")
-        url[4..-1].upcase
-      end
-    end
+      # remove non-printing whitespace and downcase
+      doi = doi.delete("\u200B").downcase
 
-    def doi_as_url(doi)
-      Addressable::URI.encode("https://doi.org/#{clean_doi(doi)}") if doi.present?
+      # turn DOI into URL, escape unsafe characters
+      "https://doi.org/" + Addressable::URI.encode(doi)
     end
 
     def orcid_from_url(url)

data/lib/toccatore/cli.rb

@@ -1,7 +1,6 @@
 # encoding: UTF-8
 
 require "thor"
-require_relative 'orcid_update'
 
 module Toccatore
   class CLI < Thor
@@ -30,5 +29,18 @@ module Toccatore
       orcid_update = Toccatore::OrcidUpdate.new
       orcid_update.queue_jobs(orcid_update.unfreeze(options))
     end
+
+    desc "datacite_related", "push non-DataCite DOIs from DataCite MDS to Event Data"
+    method_option :access_token, type: :string, required: true
+    method_option :push_url, type: :string
+    method_option :from_date, type: :string, default: (Time.now.to_date - 1.day).iso8601
+    method_option :until_date, type: :string, default: Time.now.to_date.iso8601
+    method_option :q, type: :string
+    method_option :related_identifier, type: :string
+    method_option :doi, type: :string
+    def datacite_related
+      datacite_related = Toccatore::DataciteRelated.new
+      datacite_related.queue_jobs(datacite_related.unfreeze(options))
+    end
   end
 end

data/lib/toccatore/datacite_related.rb

@@ -0,0 +1,77 @@
+require_relative 'base'
+
+module Toccatore
+  class DataciteRelated < Base
+    def source_id
+      "datacite_related"
+    end
+
+    def query
+      "relatedIdentifier:DOI\\:*"
+    end
+
+    def parse_data(result, options={})
+      return result.body.fetch("errors") if result.body.fetch("errors", nil).present?
+
+      items = result.body.fetch("data", {}).fetch('response', {}).fetch('docs', nil)
+      registration_agencies = {}
+
+      Array.wrap(items).reduce([]) do |sum, item|
+        doi = item.fetch("doi")
+        pid = normalize_doi(doi)
+        related_doi_identifiers = item.fetch('relatedIdentifier', []).select { |id| id =~ /:DOI:.+/ }
+
+        sum += Array(related_doi_identifiers).reduce([]) do |ssum, iitem|
+          raw_relation_type, _related_identifier_type, related_identifier = iitem.split(':', 3)
+          related_identifier = related_identifier.strip.downcase
+          prefix = validate_prefix(related_identifier)
+          registration_agencies[prefix] = get_doi_ra(prefix) unless registration_agencies[prefix]
+
+          # check whether this is a DataCite DOI
+          if registration_agencies[prefix] == "DataCite"
+            ssum
+          else
+            ssum << { "id" => SecureRandom.uuid,
+                      "message_action" => "create",
+                      "subj_id" => pid,
+                      "obj_id" => normalize_doi(related_identifier),
+                      "relation_type_id" => raw_relation_type.underscore,
+                      "source_id" => "datacite",
+                      "occurred_at" => item.fetch("minted") }
+          end
+        end
+
+        sum
+      end
+    end
+
+    # push to Event Data API if no error and we have collected works
+    def push_data(items, options={})
+      if items.empty?
+        puts "No works found for date range #{options[:from_date]} - #{options[:until_date]}."
+      elsif options[:access_token].blank?
+        puts "An error occured: Access token missing."
+      else
+        Array(items).each { |item| push_item(item, options) }
+      end
+    end
+
+    def push_item(item, options={})
+      return OpenStruct.new(body: { "errors" => [{ "title" => "Access token missing." }] }) if options[:access_token].blank?
+
+      host = options[:push_url].presence || "https://bus.eventdata.crossref.org"
+      push_url = host + "/events"
+
+      response = Maremma.post(push_url, data: item.to_json,
+                                        bearer: options[:access_token],
+                                        content_type: 'json',
+                                        host: host)
+
+      if response.status == 201
+        puts "#{item['subj_id']} #{item['relation_type_id']} #{item['obj_id']} pushed to Event Data service."
+      elsif response.body["errors"].present?
+        puts "An error occured: #{response.body['errors'].first['title']}"
+      end
+    end
+  end
+end