toccatore 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 13870f2c02d23ff37a9f4de7af84bfa353af388d
+  data.tar.gz: a356318891ee353f9dab772c9f74b1629a21b6dc
+SHA512:
+  metadata.gz: 63cd648459e66ce0bda3434ec311e35b12fd66d617882cb59c78a1c9091a6448c9ff2f43c0bd187d1b27c8e0cfabfd861f11325d73de8245be8419f3f79a6086
+  data.tar.gz: c0a6a5c484535dace3a5a677e73c97858f3f27bcab2383917d9d7ea631242abd869ed85b6d5eca275a87ab55ac072938b2a6c96093e50cdc13d3b259fea2573c

data/.gitignore

@@ -0,0 +1,50 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+.env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/.travis.yml

@@ -0,0 +1,19 @@
+language: ruby
+cache: bundler
+sudo: false
+rvm:
+- 2.3.1
+script:
+- bundle exec rspec
+- bundle exec codeclimate-test-reporter
+notifications:
+  slack: datacite:Wt8En0ALoTA6Kjc5EOKNDWxN
+  email: false
+deploy:
+  provider: rubygems
+  api_key:
+    secure: eLWrKUf5as7RNOJpm4/viPR97yOpYjj7yiwblXf1axtBr/nJCi/ZZ3UyakHbsM88DI+sF52A56cum+0KkWTQuzBVR6TXB3u/UclC0z1pyjv6QClm3qLi5/lx6f//7K9FmdB130CuAUbJgUBiIyDElPJK4bE+teBUWft/Pb49Yy1/5M5F0VV/lZrOQ/O6js9cdmxxmp8DfC+UMmw1I982VGJ1xTW6vhWlZ3pA+PLi7KkdxzA5f3/SQLIC8ij6i9FLFXz37qs5ynumzDKiyshKoVZ7mVeR0SjmGAteAXDqkwmknJPMJTHxc2dvxDpZjB1KguBw6Ohs/Bv+R14bzyXepkBaZ8Mo++Ro0EqRdP9tdLbhhtJJ4+MrVPqYVL+JakAcJY3Y58e/j4ZOvbjrWFJ3oMljDpxzSUNvMvpWCQ8NlEDAhOEG3b4jbERl+vEhjYlcDVeSxBrxA02wXCoWTrZxpkRZY8qdgA3O21W+pcixEGIYT/Ox0jTfWdQUhqjJM16qcN13i6SMzeC1FaihXkA1AltUtEIgXA+uJA9aMrhHGYktMc2XkUqO4blna6ExzBwvafzZgor47oCOh1VLxpas+5Hui3YKEnmHn8sxKa26WuQJomnXgXfhB8n+eB/KWunWOPiBvJZDMU2C6AvP6N1MDTKvQWp2bwU5Jamt9vNjTtM=
+  gem: toccatore
+  on:
+    tags: true
+    repo: datacite/toccatore

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## v.0.1 (January 30, 2017)
+
+[toccatore 0.1](https://github.com/datacite/toccatore/releases/tag/v.0.1) was released on January 30, 2017:
+
+* initial release

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,110 @@
+PATH
+  remote: .
+  specs:
+    toccatore (0.1)
+      activesupport (~> 4.2, >= 4.2.5)
+      dotenv (~> 2.1, >= 2.1.1)
+      gender_detector (~> 1.0)
+      maremma (~> 3.1)
+      namae (~> 0.11.0)
+      thor (~> 0.19)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.2.7.1)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    addressable (2.5.0)
+      public_suffix (~> 2.0, >= 2.0.2)
+    builder (3.2.3)
+    codeclimate-test-reporter (1.0.5)
+      simplecov
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    diff-lcs (1.3)
+    docile (1.1.5)
+    dotenv (2.2.0)
+    excon (0.45.4)
+    faraday (0.9.2)
+      multipart-post (>= 1.2, < 3)
+    faraday-encoding (0.0.4)
+      faraday
+    faraday_middleware (0.10.1)
+      faraday (>= 0.7.4, < 1.0)
+    gender_detector (1.0.0)
+    hashdiff (0.3.2)
+    i18n (0.7.0)
+    json (1.8.6)
+    maremma (3.1.2)
+      activesupport (~> 4.2, >= 4.2.5)
+      addressable (~> 2.5)
+      builder (~> 3.2, >= 3.2.2)
+      excon (~> 0.45.0)
+      faraday (~> 0.9.2)
+      faraday-encoding (~> 0.0.1)
+      faraday_middleware (~> 0.10.0)
+      multi_json (~> 1.11.2)
+      nokogiri (~> 1.6.7)
+      oj (~> 2.13.1)
+    mini_portile2 (2.1.0)
+    minitest (5.10.1)
+    multi_json (1.11.3)
+    multipart-post (2.0.0)
+    namae (0.11.3)
+    nokogiri (1.6.8.1)
+      mini_portile2 (~> 2.1.0)
+    oj (2.13.1)
+    public_suffix (2.0.5)
+    rack (2.0.1)
+    rack-test (0.6.3)
+      rack (>= 1.0)
+    rake (12.0.0)
+    rspec (3.5.0)
+      rspec-core (~> 3.5.0)
+      rspec-expectations (~> 3.5.0)
+      rspec-mocks (~> 3.5.0)
+    rspec-core (3.5.4)
+      rspec-support (~> 3.5.0)
+    rspec-expectations (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-mocks (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-support (3.5.0)
+    safe_yaml (1.0.4)
+    simplecov (0.12.0)
+      docile (~> 1.1.0)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    thor (0.19.4)
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    vcr (3.0.3)
+    webmock (1.24.6)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0)
+  codeclimate-test-reporter (~> 1.0, >= 1.0.0)
+  rack-test (~> 0)
+  rake (~> 12.0)
+  rspec (~> 3.4)
+  simplecov (~> 0.12.0)
+  toccatore!
+  vcr (~> 3.0, >= 3.0.3)
+  webmock (~> 1.22, >= 1.22.3)
+
+BUNDLED WITH
+   1.12.5

data/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 DataCite
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,28 @@
+# Toccatore
+
+[![Build Status](https://travis-ci.org/datacite/toccatore.svg?branch=master)](https://travis-ci.org/datacite/toccatore)
+[![Code Climate](https://codeclimate.com/github/datacite/toccatore/badges/gpa.svg)](https://codeclimate.com/github/datacite/toccatore)
+[![Test Coverage](https://codeclimate.com/github/datacite/toccatore/badges/coverage.svg)](https://codeclimate.com/github/datacite/toccatore/coverage)
+
+Command-line client for finding ORCID IDs in DataCite metadata.
+
+## Development
+
+We use rspec for unit testing:
+
+```
+bundle exec rspec
+```
+
+Follow along via [Github Issues](https://github.com/datacite/toccatore/issues).
+
+### Note on Patches/Pull Requests
+
+* Fork the project
+* Write tests for your new feature or a test that reproduces a bug
+* Implement your feature or make a bug fix
+* Do not mess with Rakefile, version or history
+* Commit, push and make a pull request. Bonus points for topical branches.
+
+## License
+**toccatore** is released under the [MIT License](https://github.com/datacite/toccatore/blob/master/LICENSE.md).

data/bin/toccatore

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require File.expand_path("../../lib/toccatore", __FILE__)
+
+Toccatore::CLI.start

data/lib/toccatore.rb

@@ -0,0 +1 @@
+require "toccatore/orcid_update"

data/lib/toccatore/base.rb

@@ -0,0 +1,361 @@
+require 'namae'
+require 'gender_detector'
+
+module Toccatore
+  class Base
+    # load ENV variables from .env file if it exists
+    env_file = File.expand_path("../../../.env", __FILE__)
+    if File.exist?(env_file)
+      require 'dotenv'
+      Dotenv.load! env_file
+    end
+
+    # load ENV variables from container environment if json file exists
+    # see https://github.com/phusion/baseimage-docker#envvar_dumps
+    env_json_file = "/etc/container_environment.json"
+    if File.exist?(env_json_file)
+      env_vars = JSON.parse(File.read(env_json_file))
+      env_vars.each { |k, v| ENV[k] = v }
+    end
+
+    def get_query_url(options={})
+      offset = options[:offset].to_i || 0
+      rows = options[:rows].presence || job_batch_size
+      from_date = options[:from_date].presence || (Time.now.to_date - 1.day).iso8601
+      until_date = options[:until_date].presence || Time.now.to_date.iso8601
+
+      updated = "updated:[#{from_date}T00:00:00Z TO #{until_date}T23:59:59Z]"
+      fq = "#{updated} AND has_metadata:true AND is_active:true"
+
+      params = { q: q,
+                 start: offset,
+                 rows: rows,
+                 fl: "doi,creator,title,publisher,publicationYear,resourceTypeGeneral,datacentre_symbol,relatedIdentifier,nameIdentifier,xml,minted,updated",
+                 fq: fq,
+                 wt: "json" }
+      url +  URI.encode_www_form(params)
+    end
+
+    def get_total(options={})
+      query_url = get_query_url(options.merge(rows: 0))
+      result = Maremma.get(query_url, options)
+      result.body.fetch("data", {}).fetch("response", {}).fetch("numFound", 0)
+    end
+
+    def queue_jobs(options={})
+      total = get_total(options)
+
+      if total > 0
+        # walk through paginated results
+        total_pages = (total.to_f / job_batch_size).ceil
+
+        (0...total_pages).each do |page|
+          options[:offset] = page * job_batch_size
+          process_data(options)
+        end
+      end
+
+      # return number of works queued
+      total
+    end
+
+    def process_data(options = {})
+      data = get_data(options.merge(timeout: timeout, source_id: source_id))
+      data = parse_data(data, options.merge(source_id: source_id))
+
+      # push to deposit API if no error and we have collected works and/or events
+      # returns hash with number of deposits created, e.g. { total: 10 }
+      push_data(data, options)
+    end
+
+    def get_data(options={})
+      query_url = get_query_url(options)
+      Maremma.get(query_url, options)
+    end
+
+    def parse_data(result, options={})
+      return result.body.fetch("errors") if result.body.fetch("errors", nil).present?
+
+      items = result.fetch("data", {}).fetch('response', {}).fetch('docs', nil)
+      get_relations_with_related_works(items)
+    end
+
+    # push to Lagotto deposit API if no error and we have collected works
+    def push_data(items, options={})
+      return [] if items.empty?
+
+      Array(items).map do |item|
+        relation = item.fetch(:relation, {})
+        deposit = { "deposit" => { "subj_id" => relation.fetch("subj_id", nil),
+                                   "obj_id" => relation.fetch("obj_id", nil),
+                                   "relation_type_id" => relation.fetch("relation_type_id", nil),
+                                   "source_id" => relation.fetch("source_id", nil),
+                                   "publisher_id" => relation.fetch("publisher_id", nil),
+                                   "subj" => item.fetch(:subj, {}),
+                                   "obj" => item.fetch(:obj, {}),
+                                   "message_type" => item.fetch(:message_type, "relation"),
+                                   "prefix" => item.fetch(:prefix, nil),
+                                   "source_token" => uuid } }
+
+        Maremma.post push_url, data: deposit.to_json, content_type: 'json', token: access_token
+      end
+    end
+
+    def get_relations_with_related_works(items)
+      Array(items).reduce([]) do |sum, item|
+        doi = item.fetch("doi", nil)
+        prefix = doi[/^10\.\d{4,5}/]
+        pid = doi_as_url(doi)
+        type = item.fetch("resourceTypeGeneral", nil)
+        publisher_id = item.fetch("datacentre_symbol", nil)
+
+        xml = Base64.decode64(item.fetch('xml', "PGhzaD48L2hzaD4=\n"))
+        xml = Hash.from_xml(xml).fetch("resource", {})
+        authors = xml.fetch("creators", {}).fetch("creator", [])
+        authors = [authors] if authors.is_a?(Hash)
+
+        subj = { "pid" => pid,
+                 "DOI" => doi,
+                 "author" => get_hashed_authors(authors),
+                 "title" => item.fetch("title", []).first,
+                 "container-title" => item.fetch("publisher", nil),
+                 "published" => item.fetch("publicationYear", nil),
+                 "issued" => item.fetch("minted", nil),
+                 "publisher_id" => publisher_id,
+                 "registration_agency_id" => "datacite",
+                 "tracked" => true,
+                 "type" => type }
+
+        related_doi_identifiers = item.fetch('relatedIdentifier', []).select { |id| id =~ /:DOI:.+/ }
+        sum += get_doi_relations(subj, related_doi_identifiers)
+
+        related_github_identifiers = item.fetch('relatedIdentifier', []).select { |id| id =~ /:URL:https:\/\/github.com.+/ }
+        sum += get_github_relations(subj, related_github_identifiers)
+
+        name_identifiers = item.fetch('nameIdentifier', []).select { |id| id =~ /^ORCID:.+/ }
+        sum += get_contributions(subj, name_identifiers)
+
+        if source_id == "datacite_import"
+          sum += [{ prefix: prefix,
+                    relation: { "subj_id" => subj["pid"],
+                                "source_id" => source_id,
+                                "publisher_id" => subj["publisher_id"],
+                                "occurred_at" => subj["issued"] },
+                    subj: subj }]
+        end
+
+        sum
+      end
+    end
+
+    def get_github_relations(subj, items)
+      prefix = subj["DOI"][/^10\.\d{4,5}/]
+
+      Array(items).reduce([]) do |sum, item|
+        raw_relation_type, _related_identifier_type, related_identifier = item.split(':', 3)
+
+        # get parent repo
+        # code from https://github.com/octokit/octokit.rb/blob/master/lib/octokit/repository.rb
+        related_identifier = PostRank::URI.clean(related_identifier)
+        github_hash = github_from_url(related_identifier)
+        owner_url = github_as_owner_url(github_hash)
+        repo_url = github_as_repo_url(github_hash)
+
+        sum << { prefix: prefix,
+                 relation: { "subj_id" => subj["pid"],
+                             "obj_id" => related_identifier,
+                             "relation_type_id" => raw_relation_type.underscore,
+                             "source_id" => source_id,
+                             "publisher_id" => subj["publisher_id"],
+                             "registration_agency_id" => "github",
+                             "occurred_at" => subj["issued"] },
+                 subj: subj }
+
+        # if relatedIdentifier is release URL rather than repo URL
+        if related_identifier != repo_url
+          sum << { relation: { "subj_id" => related_identifier,
+                               "obj_id" => repo_url,
+                               "relation_type_id" => "is_part_of",
+                               "source_id" => source_id,
+                               "publisher_id" => "github",
+                               "registration_agency_id" => "github" } }
+        end
+
+        sum << {  message_type: "contribution",
+                  relation: { "subj_id" => owner_url,
+                              "obj_id" => repo_url,
+                              "source_id" => "github_contributor",
+                              "registration_agency_id" => "github" }}
+      end
+    end
+
+    def get_doi_relations(subj, items)
+      prefix = subj["DOI"][/^10\.\d{4,5}/]
+
+      Array(items).reduce([]) do |sum, item|
+        raw_relation_type, _related_identifier_type, related_identifier = item.split(':', 3)
+        doi = related_identifier.strip.upcase
+        registration_agency = get_doi_ra(doi)
+
+        if source_id == "datacite_crossref" && registration_agency == "datacite"
+          sum
+        else
+          _source_id = registration_agency == "crossref" ? "datacite_crossref" : "datacite_related"
+          pid = doi_as_url(doi)
+
+          sum << { prefix: prefix,
+                   relation: { "subj_id" => subj["pid"],
+                               "obj_id" => pid,
+                               "relation_type_id" => raw_relation_type.underscore,
+                               "source_id" => _source_id,
+                               "publisher_id" => subj["publisher_id"],
+                               "registration_agency_id" => registration_agency,
+                               "occurred_at" => subj["issued"] },
+                   subj: subj }
+        end
+      end
+    end
+
+    # we are flipping subj and obj for contributions
+    def get_contributions(obj, items)
+      prefix = obj["DOI"][/^10\.\d{4,5}/]
+
+      Array(items).reduce([]) do |sum, item|
+        orcid = item.split(':', 2).last
+        orcid = validate_orcid(orcid)
+
+        return sum if orcid.nil?
+
+        sum << { prefix: prefix,
+                 message_type: "contribution",
+                 relation: { "subj_id" => orcid_as_url(orcid),
+                             "obj_id" => obj["pid"],
+                             "relation_type_id" => nil,
+                             "source_id" => source_id,
+                             "publisher_id" => obj["publisher_id"],
+                             "registration_agency_id" => "datacite",
+                             "occurred_at" => obj["issued"] },
+                 obj: obj }
+      end
+    end
+
+    def config_fields
+      [:url, :push_url, :access_token]
+    end
+
+    def url
+      "https://search.datacite.org/api?"
+    end
+
+    def timeout
+      120
+    end
+
+    def job_batch_size
+      1000
+    end
+
+    # remove non-printing whitespace
+    def clean_doi(doi)
+      doi.gsub(/\u200B/, '')
+    end
+
+    def doi_from_url(url)
+      if /(http|https):\/\/(dx\.)?doi\.org\/(\w+)/.match(url)
+        uri = Addressable::URI.parse(url)
+        uri.path[1..-1].upcase
+      elsif url.starts_with?("doi:")
+        url[4..-1].upcase
+      end
+    end
+
+    def doi_as_url(doi)
+      Addressable::URI.encode("https://doi.org/#{clean_doi(doi)}") if doi.present?
+    end
+
+    def orcid_from_url(url)
+      Array(/\Ahttp:\/\/orcid\.org\/(.+)/.match(url)).last
+    end
+
+    def orcid_as_url(orcid)
+      "http://orcid.org/#{orcid}" if orcid.present?
+    end
+
+    def validate_orcid(orcid)
+      Array(/\A(?:http:\/\/orcid\.org\/)?(\d{4}-\d{4}-\d{4}-\d{3}[0-9X]+)\z/.match(orcid)).last
+    end
+
+    # parse author string into CSL format
+    # only assume personal name when using sort-order: "Turing, Alan"
+    def get_one_author(author, options = {})
+      return { "literal" => "" } if author.strip.blank?
+
+      author = cleanup_author(author)
+      names = Namae.parse(author)
+
+      if names.blank? || is_personal_name?(author).blank?
+        { "literal" => author }
+      else
+        name = names.first
+
+        { "family" => name.family,
+          "given" => name.given }.compact
+      end
+    end
+
+    def cleanup_author(author)
+      # detect pattern "Smith J.", but not "Smith, John K."
+      author = author.gsub(/[[:space:]]([A-Z]\.)?(-?[A-Z]\.)$/, ', \1\2') unless author.include?(",")
+
+      # titleize strings
+      # remove non-standard space characters
+      author.my_titleize
+            .gsub(/[[:space:]]/, ' ')
+    end
+
+    def is_personal_name?(author)
+      return true if author.include?(",")
+
+      # lookup given name
+      name_detector.name_exists?(author.split.first)
+    end
+
+    # parse array of author strings into CSL format
+    def get_authors(authors, options={})
+      Array(authors).map { |author| get_one_author(author, options) }
+    end
+
+    # parse array of author hashes into CSL format
+    def get_hashed_authors(authors)
+      Array(authors).map { |author| get_one_hashed_author(author) }
+    end
+
+    def get_one_hashed_author(author)
+      raw_name = author.fetch("creatorName", nil)
+
+      author_hsh = get_one_author(raw_name)
+      author_hsh["ORCID"] = get_name_identifier(author)
+      author_hsh.compact
+    end
+
+    def get_name_identifier(author)
+      name_identifier = author.fetch("nameIdentifier", nil)
+      name_identifier_scheme = author.fetch("nameIdentifierScheme", "orcid").downcase
+      if name_identifier_scheme == "orcid" && name_identifier = validate_orcid(name_identifier)
+        "http://orcid.org/#{name_identifier}"
+      else
+        nil
+      end
+    end
+
+    def name_detector
+      GenderDetector.new
+    end
+  end
+end
+
+class String
+  def my_titleize
+    self.gsub(/(\b|_)(.)/) { "#{$1}#{$2.upcase}" }
+  end
+end