bolognese 0.9.56 → 0.9.58

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1022c2ebb6dac109eff056a0ebec7717a590af87
-  data.tar.gz: 92f75fd535284fea56aa0d1b28ed63444c9c9ce7
+  metadata.gz: 325e7c67f90b425a1498add3572540a3f5172e46
+  data.tar.gz: 43d1bc36ba1a0a9135349e5d4189ab2bb2387b79
 SHA512:
-  metadata.gz: 9fe97b6eb3be9a18a3b87877713a9573a422971a9f92b97006cf57f5ca7db7663fabb760f0237f4cb249a6e3277bddfd748f192341a0b370e340c84c0af8663d
-  data.tar.gz: cd915315950204de4bbc5737c1476f361cda398907f5a29943e54c158239f3a6529cda8ac848257577a57dfefe095203c6d92fa79562c7aaddf5868d29c334c7
+  metadata.gz: dd66c9e2b9b6f4966f7daa4c620664faf66310a770d7229714ebc294681f1d906dd776150ecf919cb805926deb9901f0ada550c2fe03ec7cf48707e1b049572e
+  data.tar.gz: a7ace8155d4e47908d867f7dca076f8de4de90f96a20b3479db8ee16d3a5b32d401a1f3359d3f8c7cd0afff9a3d0032c843d16b4bb5282bad034b1f86cf9b157

data/.travis.yml

@@ -15,7 +15,7 @@ install:
 
 script:
 - bundle exec rspec
-- bundle exec codeclimate-test-reporter
+- bundle exec codeclimate-test-reporter || true
 
 notifications:
   email: false

data/Gemfile.lock

@@ -1,13 +1,14 @@
 PATH
   remote: .
   specs:
-    bolognese (0.9.56)
+    bolognese (0.9.58)
       activesupport (>= 4.2.5, < 6)
       benchmark_methods (~> 0.7)
       bibtex-ruby (~> 4.1)
       builder (~> 3.2, >= 3.2.2)
       colorize (~> 0.8.1)
       gender_detector (~> 0.1.2)
+      iso8601 (~> 0.9.1)
       json-ld (~> 2.1, >= 2.1.2)
       jsonlint (~> 0.2.0)
       loofah (~> 2.0, >= 2.0.3)
@@ -64,6 +65,7 @@ GEM
     htmlentities (4.3.4)
     i18n (0.9.0)
       concurrent-ruby (~> 1.0)
+    iso8601 (0.9.1)
     json (2.1.0)
     json-ld (2.1.7)
       multi_json (~> 1.12)
@@ -104,7 +106,7 @@ GEM
     rack (2.0.3)
     rack-test (0.7.0)
       rack (>= 1.0, < 3)
-    rake (12.1.0)
+    rake (12.2.1)
     rdf (2.2.11)
       hamster (~> 3.0)
       link_header (~> 0.0, >= 0.0.8)
@@ -155,7 +157,7 @@ GEM
     thread_safe (0.3.6)
     tilt (2.0.8)
     trollop (2.1.2)
-    tzinfo (1.2.3)
+    tzinfo (1.2.4)
       thread_safe (~> 0.1)
     unicode (0.4.4.4)
     unicode_utils (1.4.0)
@@ -181,4 +183,4 @@ DEPENDENCIES
   webmock (~> 3.0, >= 3.0.1)
 
 BUNDLED WITH
-   1.15.3
+   1.15.4

data/README.md

@@ -79,6 +79,13 @@ Bolognese reads and/or writes these metadata formats:
       <td>Yes</td>
       <td>Yes</td>
     </tr>
+    <tr>
+      <td><a href='https://jats.nlm.nih.gov/'>JATS</a></td>
+      <td>jats</td>
+      <td>application/vnd.jats+xml</td>
+      <td>No</td>
+      <td>Yes</td>
+    </tr>
     <tr>
       <td><a href='http://en.wikipedia.org/wiki/BibTeX'>BibTeX</a></td>
       <td>bibtex</td>

data/bolognese.gemspec

@@ -24,6 +24,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'thor', '~> 0.19'
   s.add_dependency 'colorize', '~> 0.8.1'
   s.add_dependency 'namae', '~> 0.11.3'
+  s.add_dependency 'iso8601', '~> 0.9.1'
   s.add_dependency 'postrank-uri', '~> 1.0', '>= 1.0.18'
   s.add_dependency 'json-ld', '~> 2.1', '>= 2.1.2'
   s.add_dependency 'jsonlint', '~> 0.2.0'

data/lib/bolognese.rb

@@ -9,6 +9,7 @@ require 'json/ld'
 require 'rdf/turtle'
 require 'rdf/rdfxml'
 require 'logger'
+require 'iso8601'
 require 'jsonlint'
 require 'benchmark_methods'
 require 'gender_detector'

data/lib/bolognese/metadata.rb

@@ -22,6 +22,7 @@ require_relative 'writers/crosscite_writer'
 require_relative 'writers/crossref_writer'
 require_relative 'writers/datacite_writer'
 require_relative 'writers/datacite_json_writer'
+require_relative 'writers/jats_writer'
 require_relative 'writers/rdf_xml_writer'
 require_relative 'writers/ris_writer'
 require_relative 'writers/schema_org_writer'
@@ -54,6 +55,7 @@ module Bolognese
     include Bolognese::Writers::CrossrefWriter
     include Bolognese::Writers::DataciteWriter
     include Bolognese::Writers::DataciteJsonWriter
+    include Bolognese::Writers::JatsWriter
     include Bolognese::Writers::RdfXmlWriter
     include Bolognese::Writers::RisWriter
     include Bolognese::Writers::SchemaOrgWriter
@@ -63,7 +65,7 @@ module Bolognese
       :date_accepted, :date_available, :date_copyrighted, :date_collected,
       :date_submitted, :date_valid, :date_created, :date_modified,
       :date_registered, :date_updated, :provider_id, :client_id, :journal,
-      :volume, :issue, :pagination, :url, :version, :keywords, :editor,
+      :volume, :issue, :first_page, :last_page, :url, :version, :keywords, :editor,
       :description, :alternate_name, :language, :content_size, :spatial_coverage,
       :schema_version, :additional_type, :has_part, :same_as,
       :is_previous_version_of, :is_new_version_of,   :is_cited_by, :cites,
@@ -97,7 +99,8 @@ module Bolognese
           exit 1
         end
       else
-        hsh = { "date_registered" => options[:date_registered],
+        hsh = { "url" => options[:url],
+                "date_registered" => options[:date_registered],
                 "date_updated" => options[:date_updated],
                 "provider_id" => options[:provider_id],
                 "client_id" => options[:client_id] }
@@ -111,6 +114,7 @@ module Bolognese
 
       @should_passthru = (@from == "datacite") && !regenerate
 
+      @url = hsh.to_h["url"].presence
       @date_registered = hsh.to_h["date_registered"].presence
       @date_updated = hsh.to_h["date_updated"].presence
       @provider_id = hsh.to_h["provider_id"].presence
@@ -243,8 +247,12 @@ module Bolognese
       @volume ||= metadata.fetch("volume", nil)
     end
 
-    def pagination
-      @pagination ||= metadata.fetch("pagination", nil)
+    def first_page
+      @first_page ||= metadata.fetch("first_page", nil)
+    end
+
+    def last_page
+      @last_page ||= metadata.fetch("last_page", nil)
     end
 
     def description

data/lib/bolognese/readers/bibtex_reader.rb

@@ -43,6 +43,8 @@ module Bolognese
                   nil
                 end
 
+        page_first, page_last = meta.pages.to_s.split("-")
+
         { "id" => normalize_doi(doi),
           "type" => type,
           "citeproc_type" => BIB_TO_CP_TRANSLATIONS[meta.type.to_s] || "misc",
@@ -56,7 +58,8 @@ module Bolognese
           "is_part_of" => is_part_of,
           "date_published" => meta.date.to_s.presence,
           "volume" => meta.volume.to_s.presence,
-          "pagination" => meta.pages.to_s.presence,
+          "page_first" => page_first,
+          "page_last" => page_last,
           "description" => { "text" => meta.field?(:abstract) && sanitize(meta.abstract.to_s).presence },
           "license" => { "id" => meta.field?(:copyright) && meta.copyright.to_s.presence }
         }

data/lib/bolognese/readers/crossref_reader.rb

@@ -84,20 +84,48 @@ module Bolognese
 
         return meta unless meta["crossref"].present?
 
-        journal_metadata = meta.dig("crossref", "journal", "journal_metadata").presence || {}
-        bibliographic_metadata = meta.dig("crossref", "journal", "journal_article").presence ||
-                                 meta.dig("crossref", "conference", "conference_paper").presence ||
-                                 meta.dig("crossref", "sa_component", "component_list", "component").presence ||
-                                 meta.dig("crossref", meta.fetch("crossref", {}).keys.last).presence || {}
-        program_metadata = bibliographic_metadata.dig("program") ||
-                           bibliographic_metadata.dig("crossmark", "custom_metadata", "program") || {}
-        journal_issue = meta.dig("crossref", "journal", "journal_issue").presence || {}
-
-        additional_type = if meta.dig("crossref", "journal").present?
-                            meta.dig("crossref", "journal").keys.last.camelize
-                          else
-                            meta.dig("crossref").keys.last.camelize
-                          end
+        # model should be one of book, conference, database, dissertation, journal, peer_review, posted_content,
+        # report-paper, sa_component, standard
+        model = meta.dig("crossref").keys.first
+
+        additional_type = nil
+        program_metadata = {}
+        journal_metadata = nil
+        journal_issue = {}
+        publisher = nil
+
+        case model
+        when "book"
+          book_metadata = meta.dig("crossref", "book", "book_metadata")
+          book_series_metadata = meta.dig("crossref", "book", "book_series_metadata")
+          book_set_metadata = meta.dig("crossref", "book", "book_set_metadata")
+          bibliographic_metadata = meta.dig("crossref", "book", "content_item")
+          additional_type = bibliographic_metadata.fetch("component_type", nil) ? "book-" + bibliographic_metadata.fetch("component_type") : "book"
+          publisher = book_metadata.dig("publisher", "publisher_name")
+        when "conference"
+          event_metadata = meta.dig("crossref", "conference", "event_metadata")
+          bibliographic_metadata = meta.dig("crossref", "conference", "conference_paper")
+        when "journal"
+          journal_metadata = meta.dig("crossref", "journal", "journal_metadata")
+          bibliographic_metadata = meta.dig("crossref", "journal", "journal_article")
+          program_metadata = bibliographic_metadata.dig("crossmark", "custom_metadata", "program") || bibliographic_metadata.dig("program")
+          journal_issue = meta.dig("crossref", "journal", "journal_issue")
+          journal_article = meta.dig("crossref", "journal", "journal_article")
+
+          additional_type = if journal_article.present?
+                              "journal_article"
+                            elsif journal_issue.present?
+                              "journal_issue"
+                            else
+                              "journal"
+                            end
+        when "posted_content"
+          bibliographic_metadata = meta.dig("crossref", "posted_content")
+        when "sa_component"
+          bibliographic_metadata = meta.dig("crossref", "sa_component", "component_list", "component")
+        end
+
+        additional_type = (additional_type || model).underscore.camelize
         type = CR_TO_SO_TRANSLATIONS[additional_type] || "ScholarlyArticle"
         doi = bibliographic_metadata.dig("doi_data", "doi").to_s.downcase
 
@@ -105,16 +133,6 @@ module Bolognese
         Time.zone = 'Eastern Time (US & Canada)'
         date_modified = Time.zone.parse(meta.fetch("timestamp", "")).utc.iso8601
 
-        prefix = validate_prefix(doi)
-        if prefix.present?
-          prefix_url = "http://api.crossref.org/prefixes/#{prefix}"
-          response = Maremma.get(prefix_url, host: true)
-          publisher = response.body.dig("data", "message", "name")
-        else
-          publisher = nil
-        end
-
-
         { "id" => normalize_doi(doi),
           "type" => type,
           "additional_type" => additional_type,
@@ -137,7 +155,8 @@ module Bolognese
           "date_modified" => date_modified,
           "volume" => journal_issue.dig("journal_volume", "volume"),
           "issue" => journal_issue.dig("issue"),
-          "pagination" => [bibliographic_metadata.dig("pages", "first_page"), bibliographic_metadata.dig("pages", "last_page")].compact.join("-").presence,
+          "first_page" => bibliographic_metadata.dig("pages", "first_page"),
+          "last_page" => bibliographic_metadata.dig("pages", "last_page"),
           "description" => crossref_description(bibliographic_metadata),
           "license" => crossref_license(program_metadata),
           "version" => nil,
@@ -157,12 +176,19 @@ module Bolognese
       end
 
       def crossref_description(bibliographic_metadata)
-        des = bibliographic_metadata.fetch("abstract", {}).values.first || bibliographic_metadata.fetch("description", nil)
-        if des.is_a?(Hash)
-          sanitize(des.fetch("__content__", nil))
-        elsif des.is_a?(String)
-          sanitize(des)
+        abstract = Array.wrap(bibliographic_metadata.dig("abstract")).map do |r|
+          { "type" => "Abstract", "text" => sanitize(parse_attributes(r, content: 'p')) }.compact
         end
+
+        description = Array.wrap(bibliographic_metadata.dig("description")).map do |r|
+          if abstract.present?
+            { "text" => sanitize(parse_attributes(r)) }.compact
+          else
+            sanitize(parse_attributes(r))
+          end
+        end
+
+        (abstract + description).unwrap
       end
 
       def crossref_license(program_metadata)
@@ -232,11 +258,11 @@ module Bolognese
         end
       end
 
-      def crossref_is_part_of(journal_metadata)
-        if journal_metadata.present?
+      def crossref_is_part_of(model_metadata)
+        if model_metadata.present?
           { "type" => "Periodical",
-            "title" => journal_metadata["full_title"],
-            "issn" => parse_attributes(journal_metadata.fetch("issn", nil)) }.compact
+            "title" => model_metadata["full_title"],
+            "issn" => parse_attributes(model_metadata.fetch("issn", nil), first: true) }.compact
         else
           nil
         end

data/lib/bolognese/readers/datacite_reader.rb

@@ -6,9 +6,9 @@ module Bolognese
 
         doi = doi_from_url(id)
         search_url = doi_search(id, options)
-        url = search_url + "?q=doi:#{doi}&fl=doi,xml,allocator_symbol,datacentre_symbol,media,minted,updated&wt=json"
+        search_url += "?q=doi:#{doi}&fl=doi,url,xml,allocator_symbol,datacentre_symbol,media,minted,updated&wt=json"
 
-        response = Maremma.get url
+        response = Maremma.get search_url
         attributes = response.body.dig("data", "response", "docs").first
         return nil unless attributes.present?
 
@@ -30,6 +30,7 @@ module Bolognese
         end
 
         { "string" => string,
+          "url" => attributes.fetch("url", nil),
           "date_registered" => attributes.fetch("minted", nil),
           "date_updated" => attributes.fetch("updated", nil),
           "provider_id" => attributes.fetch("allocator_symbol", nil),
@@ -56,14 +57,6 @@ module Bolognese
         end
 
         doi = doi_from_url(id)
-
-        if options[:url]
-          url = options[:url]
-        else
-          response = Maremma.head(id, limit: 0)
-          url = response.headers.present? ? response.headers["location"] : nil
-        end
-
         resource_type_general = meta.dig("resourceType", "resourceTypeGeneral")
         type = Bolognese::Utils::DC_TO_SO_TRANSLATIONS[resource_type_general.to_s.dasherize] || "CreativeWork"
         title = Array.wrap(meta.dig("titles", "title")).map do |r|
@@ -106,7 +99,7 @@ module Bolognese
           "ris_type" => Bolognese::Utils::DC_TO_RIS_TRANSLATIONS[resource_type_general.to_s.dasherize] || "GEN",
           "resource_type_general" => resource_type_general,
           "doi" => doi,
-          "url" => url,
+          "url" => options.fetch(:url, nil),
           "title" => title,
           "alternate_name" => alternate_name,
           "author" => get_authors(meta.dig("creators", "creator")),

data/lib/bolognese/readers/ris_reader.rb

@@ -65,7 +65,8 @@ module Bolognese
           "description" => meta.fetch("AB", nil).present? ? { "text" => sanitize(meta.fetch("AB")) } : nil,
           "volume" => meta.fetch("VL", nil),
           "issue" => meta.fetch("IS", nil),
-          "pagination" => [meta.fetch("SP", nil), meta.fetch("EP", nil)].compact.join("-").presence,
+          "first_page" => meta.fetch("SP", nil),
+          "last_page" => meta.fetch("EP", nil),
           "keywords" => meta.fetch("KW", nil),
           "language" => meta.fetch("LA", nil)
         }

data/lib/bolognese/utils.rb

@@ -43,31 +43,59 @@ module Bolognese
     }
 
     CR_TO_CP_TRANSLATIONS = {
-      "proceedings" => nil,
-      "reference-book" => nil,
-      "journal-issue" => nil,
-      "proceedings-article" => "paper-conference",
-      "other" => nil,
-      "dissertation" => "thesis",
-      "dataset" => "dataset",
-      "edited-book" => "book",
-      "journal-article" => "article-journal",
-      "journal" => nil,
-      "report" => "report",
-      "book-series" => nil,
-      "report-series" => nil,
-      "book-track" => nil,
-      "standard" => nil,
-      "book-section" => "chapter",
-      "book-part" => nil,
-      "book" => "book",
-      "book-chapter" => "chapter",
-      "standard-series" => nil,
-      "monograph" => "book",
-      "component" => nil,
-      "reference-entry" => "entry-dictionary",
-      "journal-volume" => nil,
-      "book-set" => nil
+      "Proceedings" => nil,
+      "ReferenceBook" => nil,
+      "JournalIssue" => nil,
+      "ProceedingsArticle" => "paper-conference",
+      "Other" => nil,
+      "Dissertation" => "thesis",
+      "Dataset" => "dataset",
+      "EditedBook" => "book",
+      "JournalArticle" => "article-journal",
+      "Journal" => nil,
+      "Report" => "report",
+      "BookSeries" => nil,
+      "ReportSeries" => nil,
+      "BookTrack" => nil,
+      "Standard" => nil,
+      "BookSection" => "chapter",
+      "BookPart" => nil,
+      "Book" => "book",
+      "BookChapter" => "chapter",
+      "StandardSeries" => nil,
+      "Monograph" => "book",
+      "Component" => nil,
+      "ReferenceEntry" => "entry-dictionary",
+      "JournalVolume" => nil,
+      "BookSet" => nil
+    }
+
+    CR_TO_JATS_TRANSLATIONS = {
+      "Proceedings" => "working-paper",
+      "ReferenceBook" => "book",
+      "JournalIssue" => "journal",
+      "ProceedingsArticle" => "working-paper",
+      "Other" => nil,
+      "Dissertation" => nil,
+      "Dataset" => "data",
+      "EditedBook" => "book",
+      "JournalArticle" => "journal",
+      "Journal" => "journal",
+      "Report" => "report",
+      "BookSeries" => "book",
+      "ReportSeries" => "report",
+      "BookTrack" => "book",
+      "Standard" => "standard",
+      "BookSection" => "book",
+      "BookPart" => "book",
+      "Book" => "book",
+      "BookChapter" => "book",
+      "StandardSeries" => "standard",
+      "Monograph" => "book",
+      "Component" => nil,
+      "ReferenceEntry" => nil,
+      "JournalVolume" => "journal",
+      "BookSet" => "book"
     }
 
     SO_TO_DC_TRANSLATIONS = {
@@ -75,6 +103,7 @@ module Bolognese
       "AudioObject" => "Sound",
       "Blog" => "Text",
       "BlogPosting" => "Text",
+      "Chapter" => "Text",
       "Collection" => "Collection",
       "CreativeWork" => "Other",
       "DataCatalog" => "Dataset",
@@ -91,6 +120,27 @@ module Bolognese
       "WebSite" => "Text"
     }
 
+    SO_TO_JATS_TRANSLATIONS = {
+      "Article" => "journal",
+      "AudioObject" => nil,
+      "Blog" => nil,
+      "BlogPosting" => nil,
+      "Collection" => nil,
+      "CreativeWork" => nil,
+      "DataCatalog" => "data",
+      "Dataset" => "data",
+      "Event" => nil,
+      "ImageObject" => nil,
+      "Movie" => nil,
+      "PublicationIssue" => "journal",
+      "ScholarlyArticle" => "journal",
+      "Service" => nil,
+      "SoftwareSourceCode" => "software",
+      "VideoObject" => nil,
+      "WebPage" => nil,
+      "WebSite" => "website"
+    }
+
     SO_TO_CP_TRANSLATIONS = {
       "Article" => "",
       "AudioObject" => "song",
@@ -133,31 +183,31 @@ module Bolognese
     }
 
     CR_TO_RIS_TRANSLATIONS = {
-      "proceedings" => "CONF",
-      "reference-book" => "BOOK",
-      "journal-issue" => nil,
-      "proceedings-article" => "CPAPER",
-      "other" => "GEN",
-      "dissertation" => "THES",
-      "dataset" => "DATA",
-      "edited-book" => "BOOK",
-      "journal-article" => "JOUR",
-      "journal" => nil,
-      "report" => nil,
-      "book-series" => nil,
-      "report-series" => nil,
-      "book-track" => nil,
-      "standard" => nil,
-      "book-section" => "CHAP",
-      "book-part" => "CHAP",
-      "book" => "BOOK",
-      "book-chapter" => "CHAP",
-      "standard-series" => nil,
-      "monograph" => "BOOK",
-      "component" => nil,
-      "reference-entry" => "DICT",
-      "journal-volume" => nil,
-      "book-set" => nil
+      "Proceedings" => "CONF",
+      "ReferenceBook" => "BOOK",
+      "JournalIssue" => nil,
+      "ProceedingsArticle" => "CPAPER",
+      "Other" => "GEN",
+      "Dissertation" => "THES",
+      "Dataset" => "DATA",
+      "EditedBook" => "BOOK",
+      "JournalArticle" => "JOUR",
+      "Journal" => nil,
+      "Report" => nil,
+      "BookSeries" => nil,
+      "ReportSeries" => nil,
+      "BookTrack" => nil,
+      "Standard" => nil,
+      "BookSection" => "CHAP",
+      "BookPart" => "CHAP",
+      "Book" => "BOOK",
+      "BookChapter" => "CHAP",
+      "StandardSeries" => nil,
+      "Monograph" => "BOOK",
+      "Component" => nil,
+      "ReferenceEntry" => "DICT",
+      "JournalVolume" => nil,
+      "BookSet" => nil
     }
 
     DC_TO_RIS_TRANSLATIONS = {
@@ -450,9 +500,19 @@ module Bolognese
 
     def sanitize(text, options={})
       options[:tags] ||= Set.new(%w(strong em b i code pre sub sup br))
+      content = options[:content] || "__content__"
       custom_scrubber = Bolognese::WhitelistScrubber.new(options)
 
-      Loofah.scrub_fragment(text, custom_scrubber).to_s.gsub(/\u00a0/, ' ').strip
+      if text.is_a?(String)
+        Loofah.scrub_fragment(text, custom_scrubber).to_s.gsub(/\u00a0/, ' ').strip
+      elsif text.is_a?(Hash)
+        sanitize(text.fetch(content, nil))
+      elsif text.is_a?(Array)
+        a = text.map { |e| e.is_a?(Hash) ? sanitize(e.fetch(content, nil)) : sanitize(e) }.uniq
+        a = options[:first] ? a.first : a.unwrap
+      else
+        nil
+      end
     end
 
     def github_from_url(url)