briard 2.0

data/lib/briard/metadata.rb

@@ -0,0 +1,285 @@
+# frozen_string_literal: true
+
+require_relative 'metadata_utils'
+
+module Briard
+  class Metadata
+    include Briard::MetadataUtils
+
+    attr_accessor :string, :from, :sandbox, :meta, :regenerate, :issue, :show_errors
+    attr_reader :doc, :page_start, :page_end
+    attr_writer :id, :provider_id, :client_id, :doi, :identifiers, :creators, :contributors, :titles, :publisher,
+                :rights_list, :dates, :publication_year, :volume, :url, :version_info,
+                :subjects, :contributor, :descriptions, :language, :sizes,
+                :formats, :schema_version, :meta, :container, :agency,
+                :format, :funding_references, :state, :geo_locations,
+                :types, :content_url, :related_identifiers, :related_items, :style, :locale, :date_registered,
+                :depositor, :email, :registrant
+
+    def initialize(options={})
+      options.symbolize_keys!
+
+      id = normalize_id(options[:input], options)
+      ra = nil
+
+      if id.present?
+        @from = options[:from] || find_from_format(id: id)
+
+        # mEDRA, KISTI, JaLC and OP DOIs are found in the Crossref index
+        if @from == "medra"
+          ra = "mEDRA"
+        elsif @from == "kisti"
+          ra = "KISTI"
+        elsif @from == "jalc"
+          ra = "JaLC"
+        elsif @from == "op"
+          ra = "OP"
+        end
+
+        # generate name for method to call dynamically
+        hsh = @from.present? ? send("get_" + @from, id: id, sandbox: options[:sandbox]) : {}
+        string = hsh.fetch("string", nil)
+
+      elsif options[:input].present? && File.exist?(options[:input])
+        filename = File.basename(options[:input])
+        ext = File.extname(options[:input])
+        if %w(.bib .ris .xml .json).include?(ext)
+          hsh = {
+            "url" => options[:url],
+            "state" => options[:state],
+            "date_registered" => options[:date_registered],
+            "date_updated" => options[:date_updated],
+            "provider_id" => options[:provider_id],
+            "client_id" => options[:client_id],
+            "depositor" => options[:depositor],
+            "email" => options[:email],
+            "registrant" => options[:registrant],
+            "content_url" => options[:content_url] }
+          string = IO.read(options[:input])
+          @from = options[:from] || find_from_format(string: string, filename: filename, ext: ext)
+        else
+          $stderr.puts "File type #{ext} not supported"
+          exit 1
+        end
+      else
+        hsh = {
+          "url" => options[:url],
+          "state" => options[:state],
+          "date_registered" => options[:date_registered],
+          "date_updated" => options[:date_updated],
+          "provider_id" => options[:provider_id],
+          "client_id" => options[:client_id],
+          "depositor" => options[:depositor],
+          "email" => options[:email],
+          "registrant" => options[:registrant],
+          "content_url" => options[:content_url],
+          "creators" => options[:creators],
+          "contributors" => options[:contributors],
+          "titles" => options[:titles],
+          "publisher" => options[:publisher],
+          "publication_year" => options[:publication_year] }
+        string = options[:input]
+        @from = options[:from] || find_from_format(string: string)
+      end
+
+      # make sure input is encoded as utf8
+      string = string.force_encoding("UTF-8") if string.present?
+      @string = string
+
+      # input options for citation formatting
+      @style = options[:style]
+      @locale = options[:locale]
+
+      @sandbox = options[:sandbox]
+
+      # options that come from the datacite database
+      @url = hsh.to_h["url"].presence || options[:url].presence
+      @state = hsh.to_h["state"].presence
+      @date_registered = hsh.to_h["date_registered"].presence
+      @date_updated = hsh.to_h["date_updated"].presence
+      @provider_id = hsh.to_h["provider_id"].presence
+      @client_id = hsh.to_h["client_id"].presence
+      @content_url = hsh.to_h["content_url"].presence
+
+      # options that come from the submission, needed
+      # for crossref doi registration
+      @depositor = hsh.to_h["depositor"].presence
+      @email = hsh.to_h["email"].presence
+      @registrant = hsh.to_h["registrant"].presence
+
+      # set attributes directly
+      read_options = options.slice(
+        :creators,
+        :contributors,
+        :titles,
+        :types,
+        :identifiers,
+        :container,
+        :publisher,
+        :funding_references,
+        :dates,
+        :publication_year,
+        :descriptions,
+        :rights_list,
+        :version_info,
+        :subjects,
+        :language,
+        :geo_locations,
+        :related_identifiers,
+        :related_items,
+        :formats,
+        :sizes
+      ).compact
+
+      @regenerate = options[:regenerate] || read_options.present?
+      # generate name for method to call dynamically
+      opts = { string: string, sandbox: options[:sandbox], doi: options[:doi], id: id, ra: ra }.merge(read_options)
+      @meta = @from.present? ? send("read_" + @from, **opts) : {}
+    end
+
+    def depositor
+      @depositor
+    end
+
+    def email
+      @email
+    end
+
+    def registrant
+      @registrant
+    end
+
+    def id
+      @id ||= meta.fetch("id", nil)
+    end
+
+    def doi
+      @doi ||= meta.fetch("doi", nil)
+    end
+
+    def provider_id
+      @provider_id ||= meta.fetch("provider_id", nil)
+    end
+
+    def client_id
+      @client_id ||= meta.fetch("client_id", nil)
+    end
+
+    def exists?
+      (@state || meta.fetch("state", nil)) != "not_found"
+    end
+
+    def valid?
+      exists? && errors.nil?
+    end
+
+    # validate against DataCite schema, unless already errors in the reader
+    def errors
+      meta.fetch("errors", nil) || datacite_errors(xml: datacite, schema_version: schema_version)
+    end
+
+    def descriptions
+      @descriptions ||= meta.fetch("descriptions", nil)
+    end
+
+    def rights_list
+      @rights_list ||= meta.fetch("rights_list", nil)
+    end
+
+    def subjects
+      @subjects ||= meta.fetch("subjects", nil)
+    end
+
+    def language
+      @language ||= meta.fetch("language", nil)
+    end
+
+    def sizes
+      @sizes ||= meta.fetch("sizes", nil)
+    end
+
+    def formats
+      @formats ||= meta.fetch("formats", nil)
+    end
+
+    def schema_version
+      @schema_version ||= meta.fetch("schema_version", nil)
+    end
+
+    def funding_references
+      @funding_references ||= meta.fetch("funding_references", nil)
+    end
+
+    def related_identifiers
+      @related_identifiers ||= meta.fetch("related_identifiers", nil)
+    end
+
+    def related_items
+      @related_items ||= meta.fetch("related_items", nil)
+    end
+
+    def url
+      @url ||= meta.fetch("url", nil)
+    end
+
+    def version_info
+      @version_info ||= meta.fetch("version_info", nil) || meta.fetch("version", nil)
+    end
+
+    def publication_year
+      @publication_year ||= meta.fetch("publication_year", nil)
+    end
+
+    def container
+      @container ||= meta.fetch("container", nil)
+    end
+
+    def geo_locations
+      @geo_locations ||= meta.fetch("geo_locations", nil)
+    end
+
+    def dates
+      @dates ||= meta.fetch("dates", nil)
+    end
+
+    def publisher
+      @publisher ||= meta.fetch("publisher", nil)
+    end
+
+    def identifiers
+      @identifiers ||= meta.fetch("identifiers", nil)
+    end
+
+    def content_url
+      @content_url ||= meta.fetch("content_url", nil)
+    end
+
+    def agency
+      @agency ||= meta.fetch("agency", nil)
+    end
+
+    def state
+      @state ||= meta.fetch("state", nil)
+    end
+
+    def date_registered
+      @date_registered ||= meta.fetch("date_registered", nil)
+    end
+
+    def types
+      @types ||= meta.fetch("types", nil)
+    end
+
+    def titles
+      @titles ||= meta.fetch("titles", nil)
+    end
+
+    def creators
+      @creators ||= meta.fetch("creators", nil)
+    end
+
+    def contributors
+      @contributors ||= meta.fetch("contributors", nil)
+    end
+  end
+end

data/lib/briard/metadata_utils.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+require_relative 'doi_utils'
+require_relative 'author_utils'
+require_relative 'crossref_utils'
+require_relative 'datacite_utils'
+require_relative 'utils'
+
+require_relative 'readers/bibtex_reader'
+require_relative 'readers/citeproc_reader'
+require_relative 'readers/codemeta_reader'
+require_relative 'readers/crosscite_reader'
+require_relative 'readers/crossref_reader'
+require_relative 'readers/datacite_json_reader'
+require_relative 'readers/datacite_reader'
+require_relative 'readers/npm_reader'
+require_relative 'readers/ris_reader'
+require_relative 'readers/schema_org_reader'
+
+require_relative 'writers/bibtex_writer'
+require_relative 'writers/citation_writer'
+require_relative 'writers/citeproc_writer'
+require_relative 'writers/codemeta_writer'
+require_relative 'writers/crosscite_writer'
+require_relative 'writers/crossref_writer'
+require_relative 'writers/csv_writer'
+require_relative 'writers/datacite_writer'
+require_relative 'writers/datacite_json_writer'
+require_relative 'writers/jats_writer'
+require_relative 'writers/rdf_xml_writer'
+require_relative 'writers/ris_writer'
+require_relative 'writers/schema_org_writer'
+require_relative 'writers/turtle_writer'
+
+module Briard
+  module MetadataUtils
+    # include BenchmarkMethods
+    include Briard::DoiUtils
+    include Briard::AuthorUtils
+    include Briard::CrossrefUtils
+    include Briard::DataciteUtils
+    include Briard::Utils
+
+    include Briard::Readers::BibtexReader
+    include Briard::Readers::CiteprocReader
+    include Briard::Readers::CodemetaReader
+    include Briard::Readers::CrossciteReader
+    include Briard::Readers::CrossrefReader
+    include Briard::Readers::DataciteReader
+    include Briard::Readers::DataciteJsonReader
+    include Briard::Readers::NpmReader
+    include Briard::Readers::RisReader
+    include Briard::Readers::SchemaOrgReader
+
+    include Briard::Writers::BibtexWriter
+    include Briard::Writers::CitationWriter
+    include Briard::Writers::CiteprocWriter
+    include Briard::Writers::CodemetaWriter
+    include Briard::Writers::CrossciteWriter
+    include Briard::Writers::CrossrefWriter
+    include Briard::Writers::CsvWriter
+    include Briard::Writers::DataciteWriter
+    include Briard::Writers::DataciteJsonWriter
+    include Briard::Writers::JatsWriter
+    include Briard::Writers::RdfXmlWriter
+    include Briard::Writers::RisWriter
+    include Briard::Writers::SchemaOrgWriter
+    include Briard::Writers::TurtleWriter
+
+    attr_reader :name_detector, :reverse
+
+    # some dois in the Crossref index are from other registration agencies
+    alias get_medra get_crossref
+    alias read_medra read_crossref
+    alias get_kisti get_crossref
+    alias read_kisti read_crossref
+    alias get_jalc get_crossref
+    alias read_jalc read_crossref
+    alias get_op get_crossref
+    alias read_op read_crossref
+
+    # replace DOI in XML if provided in options
+    def raw
+      r = string.present? ? string.strip : nil
+      return r unless (from == "datacite" && r.present?)
+
+      doc = Nokogiri::XML(string, nil, 'UTF-8', &:noblanks)
+      node = doc.at_css("identifier")
+      node.content = doi.to_s.upcase if node.present? && doi.present?
+      doc.to_xml.strip
+    end
+
+    def should_passthru
+      (from == "datacite") && regenerate.blank? && raw.present?
+    end
+
+    def container_title
+      if container.present?
+        container["title"]
+      elsif types["citeproc"] == "article-journal"
+        publisher
+      else
+        nil
+      end
+    end
+
+    # recognize given name. Can be loaded once as ::NameDetector, e.g. in a Rails initializer
+    def name_detector
+      @name_detector ||= defined?(::NameDetector) ? ::NameDetector : nil
+    end
+
+    def reverse
+      { "citation" => Array.wrap(related_identifiers).select { |ri| ri["relationType"] == "IsReferencedBy" }.map do |r|
+        { "@id" => normalize_doi(r["relatedIdentifier"]),
+          "@type" => r["resourceTypeGeneral"] || "ScholarlyArticle",
+          "identifier" => r["relatedIdentifierType"] == "DOI" ? nil : to_identifier(r) }.compact
+        end.unwrap,
+        "isBasedOn" => Array.wrap(related_identifiers).select { |ri| ri["relationType"] == "IsSupplementTo" }.map do |r|
+          { "@id" => normalize_doi(r["relatedIdentifier"]),
+            "@type" => r["resourceTypeGeneral"] || "ScholarlyArticle",
+            "identifier" => r["relatedIdentifierType"] == "DOI" ? nil : to_identifier(r) }.compact
+        end.unwrap }.compact
+    end
+
+    def graph
+      # preload schema_org context
+      JSON::LD::Context.add_preloaded(
+        'http://schema.org/',
+        JSON::LD::Context.new.parse('resources/schema_org/jsonldcontext.json')
+      )
+
+      RDF::Graph.new << JSON::LD::API.toRdf(schema_hsh)
+    rescue NameError
+      nil
+    end
+
+    def citeproc_hsh
+      page = container.to_h["firstPage"].present? ? [container["firstPage"], container["lastPage"]].compact.join("-") : nil
+      if Array.wrap(creators).size == 1 && Array.wrap(creators).first.fetch("name", nil) == ":(unav)"
+        author = nil
+      else
+        author = to_citeproc(creators)
+      end
+
+      if types["resourceTypeGeneral"] == "Software" && version_info.present?
+        type = "book"
+      else
+        type = types["citeproc"]
+      end
+
+      {
+        "type" => type,
+        "id" => normalize_doi(doi),
+        "categories" => Array.wrap(subjects).map { |k| parse_attributes(k, content: "subject", first: true) }.presence,
+        "language" => language,
+        "author" => author,
+        "contributor" => to_citeproc(contributors),
+        "issued" => get_date(dates, "Issued") ? get_date_parts(get_date(dates, "Issued")) : get_date_parts(publication_year.to_s),
+        "submitted" => Array.wrap(dates).find { |d| d["dateType"] == "Submitted" }.to_h.fetch("__content__", nil),
+        "abstract" => parse_attributes(descriptions, content: "description", first: true),
+        "container-title" => container_title,
+        "DOI" => doi,
+        "volume" => container.to_h["volume"],
+        "issue" => container.to_h["issue"],
+        "page" => page,
+        "publisher" => publisher,
+        "title" => parse_attributes(titles, content: "title", first: true),
+        "URL" => url,
+        "copyright" => Array.wrap(rights_list).map { |l| l["rights"] }.first,
+        "version" => version_info
+      }.compact.symbolize_keys
+    end
+
+    def crosscite_hsh
+      {
+        "id" => normalize_doi(doi),
+        "doi" => doi,
+        "url" => url,
+        "types" => types,
+        "creators" => creators,
+        "titles" => titles,
+        "publisher" => publisher,
+        "container" => container,
+        "subjects" => subjects,
+        "contributors" => contributors,
+        "dates" => dates,
+        "publication_year" => publication_year,
+        "language" => language,
+        "identifiers" => identifiers,
+        "sizes" => sizes,
+        "formats" => formats,
+        "version" => version_info,
+        "rights_list" => rights_list,
+        "descriptions" => descriptions,
+        "geo_locations" => geo_locations,
+        "funding_references" => funding_references,
+        "related_identifiers" => related_identifiers,
+        "related_items" => related_items,
+        "schema_version" => schema_version,
+        "provider_id" => provider_id,
+        "client_id" => client_id,
+        "agency" => agency,
+        "state" => state
+      }.compact
+    end
+
+    def style
+      @style ||= "apa"
+    end
+
+    def locale
+      @locale ||= "en-US"
+    end
+  end
+end

data/lib/briard/pubmed.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Briard
+  module Pubmed
+    # def get_pubmed_metadata(pmid, options = {})
+    #   return {} if pmid.blank?
+
+    #   url = "http://www.ebi.ac.uk/europepmc/webservices/rest/search/query=ext_id:#{pmid}&format=json"
+    #   response = Maremma.get(url, options)
+
+    #   metadata = response.body.fetch("data", {}).fetch("resultList", {}).fetch("result", []).first
+    #   return { error: 'Resource not found.', status: 404 } if metadata.blank?
+
+    #   metadata["issued"] = metadata.fetch("pubYear", nil)
+
+    #   author_string = metadata.fetch("authorString", "").chomp(".")
+    #   metadata["author"] = get_authors(author_string.split(", "))
+
+    #   metadata["title"] = metadata.fetch("title", "").chomp(".")
+    #   metadata["container-title"] = metadata.fetch("journalTitle", nil)
+    #   metadata["volume"] = metadata.fetch("journalVolume", nil)
+    #   metadata["page"] = metadata.fetch("pageInfo", nil)
+    #   metadata["type"] = "article-journal"
+
+    #   metadata
+    # end
+
+    def pmid_as_url(pmid)
+      "http://www.ncbi.nlm.nih.gov/pubmed/#{pmid}" if pmid.present?
+    end
+
+    def pmcid_as_url(pmcid)
+      "http://www.ncbi.nlm.nih.gov/pmc/articles/PMC#{pmcid}" if pmcid.present?
+    end
+  end
+end

data/lib/briard/readers/bibtex_reader.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Briard
+  module Readers
+    module BibtexReader
+      BIB_TO_CP_TRANSLATIONS = {
+        "article" => "article-journal",
+        "phdthesis" => "thesis"
+      }
+
+      BIB_TO_RIS_TRANSLATIONS = {
+        "article" => "JOUR",
+        "book" => "BOOK",
+        "inbook" => "CHAP",
+        "inproceedings" => "CPAPER",
+        "manual" => nil,
+        "misc" => "GEN",
+        "phdthesis" => "THES",
+        "proceedings" => "CONF",
+        "techreport" => "RPRT",
+        "unpublished" => "UNPD"
+      }
+
+      BIB_TO_SO_TRANSLATIONS = {
+        "article" => "ScholarlyArticle",
+        "phdthesis" => "Thesis"
+      }
+
+      def read_bibtex(string: nil, **options)
+        read_options = ActiveSupport::HashWithIndifferentAccess.new(options.except(:doi, :id, :url, :sandbox, :validate, :ra))
+
+        meta = string.present? ? BibTeX.parse(string).first : OpenStruct.new
+
+        bibtex_type = meta.try(:type).to_s
+        schema_org = BIB_TO_SO_TRANSLATIONS[bibtex_type] || "ScholarlyArticle"
+        types = {
+          "resourceTypeGeneral" => Metadata::BIB_TO_DC_TRANSLATIONS[bibtex_type],
+          "resourceType" => Briard::Utils::BIB_TO_CR_TRANSLATIONS[meta.try(:type).to_s] || meta.try(:type).to_s,
+          "schemaOrg" => schema_org,
+          "bibtex" => bibtex_type,
+          "citeproc" => BIB_TO_CP_TRANSLATIONS[meta.try(:type).to_s] || "misc",
+          "ris" => BIB_TO_RIS_TRANSLATIONS[meta.try(:type).to_s] || "GEN"
+        }.compact
+        doi = meta.try(:doi).to_s.presence || options[:doi]
+
+        creators = Array(meta.try(:author)).map do |a|
+          { "nameType" => "Personal",
+            "name" => [a.last, a.first].join(", "),
+            "givenName" => a.first,
+            "familyName" => a.last }.compact
+        end
+
+        related_identifiers = if meta.try(:journal).present? && meta.try(:issn).to_s.presence
+                                [{ "type" => "Periodical",
+                                  "relationType" => "IsPartOf",
+                                  "relatedIdentifierType" => "ISSN",
+                                  "title" => meta.journal.to_s,
+                                  "relatedIdentifier" => meta.try(:issn).to_s.presence }.compact]
+                              end
+
+        container = if meta.try(:journal).present?
+                      first_page = meta.try(:pages).present? ? meta.try(:pages).split("-").map(&:strip)[0] : nil
+                      last_page = meta.try(:pages).present? ? meta.try(:pages).split("-").map(&:strip)[1] : nil
+
+                      { "type" => "Journal",
+                        "title" => meta.journal.to_s,
+                        "identifier" => meta.try(:issn).to_s.presence,
+                        "identifierType" => meta.try(:issn).present? ? "ISSN" : nil,
+                        "volume" => meta.try(:volume).to_s.presence,
+                        "firstPage" => first_page,
+                        "lastPage" => last_page }.compact
+                    end
+
+        state = meta.try(:doi).to_s.present? || read_options.present? ? "findable" : "not_found"
+        dates = if meta.try(:date).present? && Date.edtf(meta.date.to_s).present?
+                  [{ "date" => meta.date.to_s,
+                    "dateType" => "Issued" }]
+                end
+        publication_year =  meta.try(:date).present? ? meta.date.to_s[0..3] : nil
+        rights_list = meta.try(:copyright).present? ? [hsh_to_spdx("rightsURI" => meta[:copyright])] : []
+
+        { "id" => normalize_doi(doi),
+          "types" => types,
+          "doi" => doi,
+          "url" => meta.try(:url).to_s.presence,
+          "titles" => meta.try(:title).present? ? [{ "title" => meta.try(:title).to_s }] : [],
+          "creators" => creators,
+          "container" => container,
+          "publisher" => meta.try(:publisher).to_s.presence,
+          "related_identifiers" => related_identifiers,
+          "dates" => dates,
+          "publication_year" => publication_year,
+          "descriptions" => meta.try(:abstract).present? ? [{ "description" => meta.try(:abstract) && sanitize(meta.abstract.to_s).presence, "descriptionType" => "Abstract" }] : [],
+          "rights_list" => rights_list,
+          "state" => state
+        }.merge(read_options)
+      end
+    end
+  end
+end