briard 2.0

data/Rakefile

@@ -0,0 +1,13 @@
+#!/usr/bin/env rake
+# frozen_string_literal: true
+
+require 'bundler'
+require 'rake'
+require 'yaml'
+require 'rspec/core/rake_task'
+
+Bundler::GemHelper.install_tasks
+RSpec::Core::RakeTask.new('spec')
+
+# default task is running rspec tests
+task :default => :spec

data/bin/briard

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require File.expand_path("../../lib/briard", __FILE__)
+
+if (ARGV & %w(--version -v help --help)).empty?
+  Briard::CLI.start(ARGV.dup.unshift("convert"))
+else
+  Briard::CLI.start
+end

data/bolognese.gemspec

@@ -0,0 +1,59 @@
+require "date"
+require File.expand_path("../lib/briard/version", __FILE__)
+
+Gem::Specification.new do |s|
+  s.authors       = "Martin Fenner"
+  s.email         = "martin@front-matter.io"
+  s.name          = "briard"
+  s.homepage      = "https://github.com/front-matter/briard"
+  s.summary       = "Ruby client library for conversion of DOI Metadata"
+  s.date          = Date.today
+  s.description   = "Ruby gem and command-line utility for conversion of DOI metadata from and to different metadata formats, including schema.org. Fork of bolognese gem."
+  s.require_paths = ["lib"]
+  s.version       = Briard::VERSION
+  s.extra_rdoc_files = ["README.md"]
+  s.license       = 'MIT'
+  s.required_ruby_version = '~> 2.3'
+
+  # Declare dependencies here, rather than in the Gemfile
+  s.add_dependency 'maremma', '>= 4.9.5', '< 5'
+  s.add_dependency 'faraday', '~> 0.17.3'
+  s.add_dependency 'nokogiri', '~> 1.11.2'
+  s.add_dependency 'loofah', '~> 2.0', '>= 2.0.3'
+  s.add_dependency 'builder', '~> 3.2', '>= 3.2.2'
+  s.add_dependency 'activesupport', '>= 4.2.5'
+  s.add_dependency 'bibtex-ruby', '>= 5.1.0'
+  s.add_dependency 'thor', '>= 1.1.0'
+  s.add_dependency 'colorize', '~> 0.8.1'
+  s.add_dependency 'namae', '~> 1.0'
+  s.add_dependency 'edtf', '~> 3.0', '>= 3.0.4'
+  s.add_dependency 'citeproc-ruby', '~> 1.1', '>= 1.1.12'
+  s.add_dependency 'csl-styles', '~> 1.0', '>= 1.0.1.10'
+  s.add_dependency 'iso8601', '~> 0.9.1'
+  s.add_dependency 'postrank-uri', '~> 1.0', '>= 1.0.18'
+  s.add_dependency 'public_suffix', '2.0.5'
+  s.add_dependency 'json-ld-preloaded', '~> 3.1', '>= 3.1.3'
+  s.add_dependency 'jsonlint', '~> 0.3.0'
+  s.add_dependency 'oj', '~> 3.10'
+  s.add_dependency "oj_mimic_json", "~> 1.0", ">= 1.0.1"
+  s.add_dependency 'rdf-turtle', '~> 3.1'
+  s.add_dependency 'rdf-rdfxml', '~> 3.1'
+  s.add_dependency 'benchmark_methods', '~> 0.7'
+  s.add_dependency 'dotenv', '~> 2.1', '>= 2.1.1'
+  s.add_dependency 'gender_detector', '~> 0.1.2'
+  s.add_dependency 'concurrent-ruby', '~> 1.1', '>= 1.1.5'
+  s.add_development_dependency 'bundler', '>= 1.0'
+  s.add_development_dependency 'rspec', '~> 3.4'
+  s.add_development_dependency 'rspec-xsd', '~> 0.1.0'
+  s.add_development_dependency 'rake', '~> 12.0'
+  s.add_development_dependency 'rack-test', '~> 0'
+  s.add_development_dependency 'vcr', '~> 3.0', '>= 3.0.3'
+  s.add_development_dependency 'webmock', '~> 3.0', '>= 3.0.1'
+  s.add_development_dependency 'simplecov', '0.17.1'
+  s.add_development_dependency 'hashdiff', ['>= 1.0.0.beta1', '< 2.0.0']
+
+  s.require_paths = ["lib"]
+  s.files       = `git ls-files`.split($/)
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables = ["briard"]
+end

data/lib/briard/.env.example

@@ -0,0 +1,3 @@
+CROSSREF_DEPOSITOR_NAME=
+CROSSREF_DEPOSITOR_EMAIL=
+CROSSREF_REGISTRANT=

data/lib/briard/array.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# turn array into hash or nil, depending on array size.
+# Reverses Array.wrap, but uses self to allow chaining with Array.wrap
+class Array
+  def unwrap
+    case self.length
+    when 0 then nil
+    when 1 then self.first
+    else self
+    end
+  end
+end

data/lib/briard/author_utils.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+require 'namae'
+
+module Briard
+  module AuthorUtils
+    # include BenchmarkMethods
+    #
+    # benchmark :get_authors
+
+    IDENTIFIER_SCHEME_URIS = {
+      "ORCID" => "https://orcid.org/"
+    }
+
+    def get_one_author(author)
+      # author is a string
+      author = { "creatorName" => author } if author.is_a?(String)
+
+      # malformed XML
+      return nil if author.fetch("creatorName", nil).is_a?(Array)
+
+      name = parse_attributes(author.fetch("creatorName", nil)) ||
+             parse_attributes(author.fetch("contributorName", nil))
+      given_name = parse_attributes(author.fetch("givenName", nil))
+      family_name = parse_attributes(author.fetch("familyName", nil))
+      name = cleanup_author(name)
+      name = [family_name, given_name].join(", ") if family_name.present? && given_name.present?
+      contributor_type = parse_attributes(author.fetch("contributorType", nil))
+
+      name_type = parse_attributes(author.fetch("creatorName", nil), content: "nameType", first: true) || parse_attributes(author.fetch("contributorName", nil), content: "nameType", first: true)
+
+      name_identifiers = Array.wrap(author.fetch("nameIdentifier", nil)).map do |ni|
+        if ni["nameIdentifierScheme"] == "ORCID"
+          {
+            "nameIdentifier" => normalize_orcid(ni["__content__"]),
+            "schemeUri" => "https://orcid.org",
+            "nameIdentifierScheme" => "ORCID" }.compact
+        elsif ni["schemeURI"].present?
+          {
+            "nameIdentifier" => ni["schemeURI"].to_s + ni["__content__"].to_s,
+            "schemeUri" => ni["schemeURI"].to_s,
+            "nameIdentifierScheme" => ni["nameIdentifierScheme"] }.compact
+        else
+          {
+            "nameIdentifier" => ni["__content__"],
+            "schemeUri" => nil,
+            "nameIdentifierScheme" => ni["nameIdentifierScheme"] }.compact
+        end
+      end.presence
+
+      author = { "nameType" => name_type,
+                 "name" => name,
+                 "givenName" => given_name,
+                 "familyName" => family_name,
+                 "nameIdentifiers" => name_identifiers,
+                 "affiliation" => get_affiliations(author.fetch("affiliation", nil)),
+                 "contributorType" => contributor_type }.compact
+
+      return author if family_name.present?
+
+      if is_personal_name?(author)
+        Namae.options[:include_particle_in_family] = true
+        names = Namae.parse(name)
+        parsed_name = names.first
+
+        if parsed_name.present?
+          given_name = parsed_name.given
+          family_name = parsed_name.family
+          name = [family_name, given_name].join(", ")
+        else
+          given_name = nil
+          family_name = nil
+        end
+
+        { "nameType" => "Personal",
+          "name" => name,
+          "givenName" => given_name,
+          "familyName" => family_name,
+          "nameIdentifiers" => Array.wrap(name_identifiers),
+          "affiliation" => Array.wrap(author.fetch("affiliation", nil)),
+          "contributorType" => contributor_type }.compact
+      else
+        { "nameType" => name_type,
+          "name" => name,
+          "nameIdentifiers" => Array.wrap(name_identifiers),
+          "affiliation" => Array.wrap(author.fetch("affiliation", nil)),
+          "contributorType" => contributor_type }.compact
+      end
+    end
+
+    def cleanup_author(author)
+      return nil unless author.present?
+
+      # detect pattern "Smith J.", but not "Smith, John K."
+      author = author.gsub(/[[:space:]]([A-Z]\.)?(-?[A-Z]\.)$/, ', \1\2') unless author.include?(",")
+
+      # remove spaces around hyphens
+      author = author.gsub(" - ", "-")
+
+      # titleize strings
+      # remove non-standard space characters
+      author.my_titleize.gsub(/[[:space:]]/, ' ')
+    end
+
+    def is_personal_name?(author)
+      return false if author.fetch("nameType", nil) == "Organizational"
+      return true if Array.wrap(author.fetch("nameIdentifiers", nil)).find { |a| a["nameIdentifierScheme"] == "ORCID" }.present? ||
+                     author.fetch("familyName", "").present? ||
+                     (author.fetch("name", "").include?(",") &&
+                     author.fetch("name", "").exclude?(";")) ||
+                     name_exists?(author.fetch("name", "").split(" ").first)
+      false
+    end
+
+    # recognize given name if we have loaded ::NameDetector data, e.g. in a Rails initializer
+    def name_exists?(name)
+      return false unless name_detector.present?
+
+      name_detector.name_exists?(name)
+    end
+
+    # parse array of author strings into CSL format
+    def get_authors(authors)
+      Array.wrap(authors).map { |author| get_one_author(author) }.compact
+    end
+
+    def authors_as_string(authors)
+      Array.wrap(authors).map do |a|
+        if a["familyName"].present?
+          [a["familyName"], a["givenName"]].join(", ")
+        elsif a["type"] == "Person"
+          a["name"]
+        elsif a["name"].present?
+          "{" + a["name"] + "}"
+        end
+      end.join(" and ").presence
+    end
+
+    def get_affiliations(affiliations)
+      Array.wrap(affiliations).map do |a|
+        affiliation_identifier = nil
+        if a.is_a?(String)
+          name = a.squish
+          affiliation_identifier_scheme = nil
+          scheme_uri = nil
+        else
+          if a["affiliationIdentifier"].present?
+            affiliation_identifier = a["affiliationIdentifier"]
+            if a["schemeURI"].present?
+              schemeURI = a["schemeURI"].end_with?("/") ? a["schemeURI"] : a["schemeURI"] + "/"
+            end
+            affiliation_identifier = !affiliation_identifier.to_s.start_with?("https://") && schemeURI.present? ? normalize_id(schemeURI + affiliation_identifier) : normalize_id(affiliation_identifier)
+          end
+          name = a["__content__"].to_s.squish.presence
+          affiliation_identifier_scheme = a["affiliationIdentifierScheme"]
+          scheme_uri = a["SchemeURI"]
+        end
+
+        { "name" => name,
+          "affiliationIdentifier" => affiliation_identifier,
+          "affiliationIdentifierScheme" => affiliation_identifier_scheme,
+          "schemeUri" => scheme_uri }.compact
+      end.presence
+    end
+  end
+end

data/lib/briard/cli.rb

@@ -0,0 +1,58 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+
+require "thor"
+
+require_relative 'doi_utils'
+require_relative 'utils'
+
+module Briard
+  class CLI < Thor
+    include Briard::DoiUtils
+    include Briard::Utils
+
+    def self.exit_on_failure?
+      true
+    end
+
+    # from http://stackoverflow.com/questions/22809972/adding-a-version-option-to-a-ruby-thor-cli
+    map %w[--version -v] => :__print_version
+
+    desc "--version, -v", "print the version"
+    def __print_version
+      puts Briard::VERSION
+    end
+
+    desc "", "convert metadata"
+    method_option :from, aliases: "-f"
+    method_option :to, aliases: "-t", default: "schema_org"
+    method_option :regenerate, :type => :boolean, :force => false
+    method_option :style, aliases: "-s", default: "apa"
+    method_option :locale, aliases: "-l", default: "en-US"
+    method_option :show_errors, :type => :boolean, :force => false
+    method_option :depositor
+    method_option :email
+    method_option :registrant
+
+    def convert(input)
+      metadata = Metadata.new(input: input,
+                              from: options[:from],
+                              regenerate: options[:regenerate],
+                              style: options[:style],
+                              locale: options[:locale],
+                              show_errors: options[:show_errors],
+                              depositor: options[:depositor],
+                              email: options[:email],
+                              registrant: options[:registrant])
+      to = options[:to] || "schema_org"
+
+      if options[:show_errors] && !metadata.valid?
+        $stderr.puts metadata.errors
+      else
+        puts metadata.send(to)
+      end
+    end
+
+    default_task :convert
+  end
+end

data/lib/briard/crossref_utils.rb

@@ -0,0 +1,336 @@
+# frozen_string_literal: true
+
+module Briard
+  module CrossrefUtils
+    # To configure the writing of Crossref metadata, use environmental
+    # variables CROSSREF_DEPOSITOR_NAME, CROSSREF_DEPOSITOR_EMAIL and CROSSREF_REGISTRANT,
+    # e.g. in a .env file
+    def crossref_xml
+      @crossref_xml ||= Nokogiri::XML::Builder.new(:encoding => 'UTF-8') do |xml|
+        xml.doi_batch(crossref_root_attributes) do
+          xml.head do
+            # we use a uuid as batch_id
+            xml.doi_batch_id(SecureRandom.uuid)
+            xml.timestamp(Time.now.utc.strftime('%Y%m%d%H%M%S'))
+            xml.depositor do
+              xml.depositor_name(ENV['CROSSREF_DEPOSITOR_NAME'])
+              xml.email_address(ENV['CROSSREF_DEPOSITOR_EMAIL'])
+            end
+            xml.registrant(ENV['CROSSREF_REGISTRANT'])
+          end
+          xml.body do
+            insert_crossref_work(xml)
+          end
+        end
+      end.to_xml
+    end
+
+    def crossref_errors(xml: nil)
+      filepath = File.expand_path("../../../resources/crossref/crossref5.3.1.xsd", __FILE__)
+      schema = Nokogiri::XML::Schema(open(filepath))
+
+      schema.validate(Nokogiri::XML(xml, nil, 'UTF-8')).map { |error| error.to_s }.unwrap
+    rescue Nokogiri::XML::SyntaxError => e
+      e.message
+    end
+
+    def insert_crossref_work(xml)
+      return xml if doi.blank?
+
+      if types["resourceTypeGeneral"] == "JournalArticle"
+        insert_journal(xml)
+      elsif types["resourceTypeGeneral"] == "Preprint"
+        insert_posted_content(xml)
+      end
+    end
+
+    def insert_journal(xml)
+      xml.journal do
+        if language.present?
+          xml.journal_metadata("language" => language) do
+            xml.full_title(container["title"])
+          end
+        else
+          xml.journal_metadata do
+            xml.full_title(container["title"])
+          end
+        end
+        xml.journal_article("publication_type" => "full_text") do
+          insert_crossref_titles(xml)
+          insert_crossref_creators(xml)
+          insert_crossref_publication_date(xml)
+          insert_crossref_abstract(xml)
+          insert_crossref_issn(xml)
+          insert_crossref_alternate_identifiers(xml)
+          insert_crossref_access_indicators(xml)
+          insert_doi_data(xml)
+          insert_citation_list(xml)
+        end
+      end
+    end
+
+    def insert_posted_content(xml)
+      posted_content = { "type" => "other", "language" => language }.compact
+
+      xml.posted_content(posted_content) do
+        insert_group_title(xml)
+        insert_crossref_creators(xml)
+        insert_crossref_titles(xml)
+        insert_posted_date(xml)
+        insert_institution(xml)
+        insert_crossref_abstract(xml)
+        insert_crossref_alternate_identifiers(xml)
+        insert_crossref_access_indicators(xml)
+        insert_doi_data(xml)
+        insert_citation_list(xml)
+      end
+    end
+
+    def insert_group_title(xml)
+      return xml if subjects.blank?
+
+      xml.group_title(subjects.first["subject"].titleize)
+    end
+
+    def insert_crossref_creators(xml)
+      xml.contributors do
+        Array.wrap(creators).each_with_index do |au, index|
+          xml.person_name("contributor_role" => "author", "sequence" => index == 0 ? "first" : "additional") do
+            insert_crossref_person(xml, au, "author")
+          end
+        end
+      end
+    end
+
+    def insert_crossref_person(xml, person, type)
+      person_name = person["familyName"].present? ? [person["familyName"], person["givenName"]].compact.join(", ") : person["name"]
+      xml.given_name(person["givenName"]) if person["givenName"].present?
+      xml.surname(person["familyName"]) if person["familyName"].present?
+      if person.dig("nameIdentifiers", 0, "nameIdentifierScheme") == "ORCID"
+        xml.ORCID(person.dig("nameIdentifiers", 0, "nameIdentifier")) 
+      end
+      Array.wrap(person["affiliation"]).each do |affiliation|
+        attributes = { "affiliationIdentifier" => affiliation["affiliationIdentifier"], "affiliationIdentifierScheme" => affiliation["affiliationIdentifierScheme"], "schemeURI" => affiliation["schemeUri"] }.compact
+        xml.affiliation(affiliation["name"], attributes)
+      end
+    end
+
+    def insert_crossref_titles(xml)
+      xml.titles do
+        Array.wrap(titles).each do |title|
+          if title.is_a?(Hash)
+            xml.title(title["title"])
+          else
+            xml.title(title)
+          end
+        end
+      end
+    end
+
+    def insert_citation_list(xml)
+      # filter out references
+      references = related_identifiers.find_all { |ri| ri["relationType"] == "References" }
+      return xml if references.blank?
+
+      xml.citation_list do
+        references.each do |ref|
+          xml.citation do
+            xml.doi(ref["relatedIdentifier"])
+          end
+        end
+      end
+    end
+
+    # def insert_publisher(xml)
+    #   xml.publisher(publisher || container && container["title"])
+    # end
+
+    # def insert_publication_year(xml)
+    #   xml.publicationYear(publication_year)
+    # end
+
+    # def insert_resource_type(xml)
+    #   return xml unless types.is_a?(Hash) && (types["schemaOrg"].present? || types["resourceTypeGeneral"])
+
+    #   xml.resourceType(types["resourceType"] || types["schemaOrg"],
+    #     'resourceTypeGeneral' => types["resourceTypeGeneral"] || Metadata::SO_TO_DC_TRANSLATIONS[types["schemaOrg"]] || "Other")
+    # end
+
+    def insert_crossref_alternate_identifiers(xml)
+      alternate_identifier = Array.wrap(identifiers).select { |r| r["identifierType"] != "DOI" }.first
+      return xml if alternate_identifier.blank?
+
+      xml.item_number(alternate_identifier["identifier"], "item_number_type" => alternate_identifier["identifierType"])
+    end
+
+    def insert_crossref_access_indicators(xml)
+      return xml if rights_list.blank?
+
+      rights_uri = Array.wrap(rights_list).map { |l| l["rightsUri"] }.first
+
+      xml.program("xmlns" => "http://www.crossref.org/AccessIndicators.xsd", "name" => "AccessIndicators") do
+        xml.license_ref(rights_uri, "applies_to" => "vor")
+        xml.license_ref(rights_uri, "applies_to" => "tdm")
+      end
+    end
+
+    # def insert_dates(xml)
+    #   return xml unless Array.wrap(dates).present?
+
+    #   xml.dates do
+    #     Array.wrap(dates).each do |date|
+    #       attributes = { 'dateType' => date["dateType"] || "Issued", 'dateInformation' => date["dateInformation"] }.compact
+    #       xml.date(date["date"], attributes)
+    #     end
+    #   end
+    # end
+
+    # def insert_funding_references(xml)
+    #   return xml unless Array.wrap(funding_references).present?
+
+    #   xml.fundingReferences do
+    #     Array.wrap(funding_references).each do |funding_reference|
+    #       xml.fundingReference do
+    #         xml.funderName(funding_reference["funderName"])
+    #         xml.funderIdentifier(funding_reference["funderIdentifier"], { "funderIdentifierType" => funding_reference["funderIdentifierType"] }.compact) if funding_reference["funderIdentifier"].present?
+    #         xml.awardNumber(funding_reference["awardNumber"], { "awardURI" => funding_reference["awardUri"] }.compact) if funding_reference["awardNumber"].present? || funding_reference["awardUri"].present?
+    #         xml.awardTitle(funding_reference["awardTitle"]) if funding_reference["awardTitle"].present?
+    #       end
+    #     end
+    #   end
+    # end
+
+    def insert_crossref_subjects(xml)
+      return xml unless subjects.present?
+
+      xml.subjects do
+        subjects.each do |subject|
+          if subject.is_a?(Hash)
+            xml.subject(subject["subject"])
+          else
+            xml.subject(subject)
+          end
+        end
+      end
+    end
+
+    # def insert_version(xml)
+    #   return xml unless version_info.present?
+
+    #   xml.version(version_info)
+    # end
+
+
+    def insert_crossref_language(xml)
+      return xml unless language.present?
+
+      xml.language(language)
+    end
+
+    def insert_crossref_publication_date(xml)
+      return xml if date_registered.blank?
+
+      date = get_datetime_from_iso8601(date_registered)
+      
+      xml.publication_date("media_type" => "online") do
+        xml.month(date.month) if date.month.present?
+        xml.day(date.day) if date.day.present?
+        xml.year(date.year) if date.year.present?
+      end
+    end
+
+    def insert_posted_date(xml)
+      date_posted = get_date(dates, "Issued")
+      return xml if date_posted.blank?
+
+      date = get_datetime_from_iso8601(date_posted)
+
+      xml.posted_date do
+        xml.month(date.month) if date.month.present?
+        xml.day(date.day) if date.day.present?
+        xml.year(date.year) if date.year.present?
+      end
+    end
+
+    def insert_institution(xml)
+      return xml if publisher.blank?
+
+      xml.institution do
+        xml.institution_name(publisher)
+      end
+    end
+
+    def insert_doi_data(xml)
+      return xml if doi.blank? || url.blank?
+
+      xml.doi_data do
+        xml.doi(doi)
+        xml.resource(url)
+        xml.collection("property" => "text-mining") do
+          xml.item do
+            xml.resource(url, "mime_type" => "text/html")
+          end
+        end
+      end
+    end
+
+    def insert_crossref_rights_list(xml)
+      return xml unless rights_list.present?
+
+      xml.rightsList do
+        Array.wrap(rights_list).each do |rights|
+          if rights.is_a?(Hash)
+            r = rights
+          else
+            r = {}
+            r["rights"] = rights
+            r["rightsUri"] = normalize_id(rights)
+          end
+
+          attributes = {
+            "rightsURI" => r["rightsUri"],
+            "rightsIdentifier" => r["rightsIdentifier"],
+            "rightsIdentifierScheme" => r["rightsIdentifierScheme"],
+            "schemeURI" => r["schemeUri"],
+            "xml:lang" => r["lang"]
+          }.compact
+
+          xml.rights(r["rights"], attributes)
+        end
+      end
+    end
+
+    def insert_crossref_issn(xml)
+      issn = container.to_h.fetch('identifierType', nil) == "ISSN" ? container.to_h.fetch('identifier', nil) : nil
+
+      return xml if issn.blank?
+
+      xml.issn(issn)
+    end
+
+    def insert_crossref_abstract(xml)
+      return xml if descriptions.blank?
+
+      if descriptions.first.is_a?(Hash)
+        d = descriptions.first
+      else
+        d = {}
+        d["description"] = descriptions.first
+      end
+
+      xml.abstract("xmlns" => "http://www.ncbi.nlm.nih.gov/JATS1") do
+        xml.p(d["description"])
+      end
+    end
+
+    def crossref_root_attributes
+      { :'xmlns:xsi' => 'http://www.w3.org/2001/XMLSchema-instance',
+        :'xsi:schemaLocation' => 'http://www.crossref.org/schema/5.3.1 https://www.crossref.org/schemas/crossref5.3.1.xsd',
+        :'xmlns' => 'http://www.crossref.org/schema/5.3.1',
+        :'xmlns:jats' => 'http://www.ncbi.nlm.nih.gov/JATS1',
+        :'xmlns:fr' => 'http://www.crossref.org/fundref.xsd',
+        :'xmlns:mml' => 'http://www.w3.org/1998/Math/MathML',
+        :'version' => '5.3.1' }
+    end
+  end
+end