bolognese 1.6.9 → 1.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 25ab3b7ce79c4ab19b986230a63bd1bb5e583f257fa2281432d10e08cdec529f
-  data.tar.gz: db1f7a4704f00cefe310edba0eca9dd69181f87861d8b9693b05056347cce759
+  metadata.gz: 7446775419fc2bf90ef92cae706c719008e19c5916eca5d9fa7fae985814103d
+  data.tar.gz: 54c8eba0583ce6238d7adb52f19323de12f93b7ff8df787d19f5167ac636df72
 SHA512:
-  metadata.gz: b795e4deddb046f5cd6acfdb1b75f8455428facc31aed10a2cb4215dbda6d7ad22344df8a470ebe18c7672f30a76476570aca151f2006dd2c3166f9984468bd5
-  data.tar.gz: a3b5aa9640cf3fdd4b3cd2c6d7eac32be36adf242acd585277a01ce2355a8c61243aae170622b113aeb0ca16c92da7cf7c80f38b2bc76acbf9bdf7e066525389
+  metadata.gz: 0452a44753fb0ed02b0a9f03a477b52a8edd569822ac972a13b0b950d0819023ada9bcac8801fd2b178d986b5617c895971deee44948960ead9dfef765c93f01
+  data.tar.gz: e7fbfe6eb47e5111cb12b53bcf1125d97b2727221b2ffbf6b804496cb5585fdd17bb53573357aa2c8046045e6ed474c0da355c58672e4353cba5b714b6158798

data/Gemfile.lock

@@ -1,23 +1,23 @@
 PATH
   remote: .
   specs:
-    bolognese (1.6.9)
+    bolognese (1.7.2)
       activesupport (>= 4.2.5)
       benchmark_methods (~> 0.7)
       bibtex-ruby (>= 5.1.0)
       builder (~> 3.2, >= 3.2.2)
-      citeproc-ruby (~> 1.1, >= 1.1.10)
+      citeproc-ruby (~> 1.1, >= 1.1.12)
       colorize (~> 0.8.1)
       concurrent-ruby (~> 1.1, >= 1.1.5)
-      csl-styles (~> 1.0, >= 1.0.1.8)
+      csl-styles (~> 1.0, >= 1.0.1.10)
       edtf (~> 3.0, >= 3.0.4)
-      faraday (= 0.17.0)
+      faraday (~> 0.17.3)
       gender_detector (~> 0.1.2)
       iso8601 (~> 0.9.1)
-      json-ld (~> 3.1, >= 3.1.4)
+      json-ld-preloaded (~> 3.1, >= 3.1.3)
       jsonlint (~> 0.3.0)
       loofah (~> 2.0, >= 2.0.3)
-      maremma (>= 4.3, < 5)
+      maremma (>= 4.7, < 5)
       namae (~> 1.0)
       nokogiri (~> 1.10.4)
       oj (~> 3.10)
@@ -56,7 +56,7 @@ GEM
       namae (~> 1.0)
     csl-styles (1.0.1.10)
       csl (~> 1.0)
-    diff-lcs (1.4.3)
+    diff-lcs (1.4.4)
     docile (1.3.2)
     ebnf (1.2.0)
       rdf (~> 3.1)
@@ -64,11 +64,11 @@ GEM
     edtf (3.0.5)
       activesupport (>= 3.0, < 7.0)
     excon (0.71.1)
-    faraday (0.17.0)
+    faraday (0.17.3)
       multipart-post (>= 1.2, < 3)
     faraday-encoding (0.0.5)
       faraday
-    faraday_middleware (0.13.1)
+    faraday_middleware (0.14.0)
       faraday (>= 0.7.4, < 1.0)
     gender_detector (0.1.2)
       unicode_utils (>= 1.3.0)
@@ -90,6 +90,9 @@ GEM
       multi_json (~> 1.14)
       rack (~> 2.0)
       rdf (~> 3.1)
+    json-ld-preloaded (3.1.3)
+      json-ld (~> 3.1)
+      rdf (~> 3.1)
     jsonlint (0.3.0)
       oj (~> 3)
       optimist (~> 3)
@@ -98,23 +101,23 @@ GEM
     loofah (2.6.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
-    maremma (4.7)
+    maremma (4.7.1)
       activesupport (>= 4.2.5)
       addressable (>= 2.3.6)
       builder (~> 3.2, >= 3.2.2)
       excon (~> 0.71.0)
-      faraday (= 0.17.0)
+      faraday (~> 0.17.3)
       faraday-encoding (~> 0.0.4)
-      faraday_middleware (~> 0.13.1)
+      faraday_middleware (~> 0.14.0)
       nokogiri (~> 1.10.4)
       oj (>= 2.8.3)
       oj_mimic_json (~> 1.0, >= 1.0.1)
     mini_portile2 (2.4.0)
     minitest (5.14.1)
-    multi_json (1.14.1)
+    multi_json (1.15.0)
     multipart-post (2.1.1)
     namae (1.0.1)
-    nokogiri (1.10.9)
+    nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     oj (3.10.6)
     oj_mimic_json (1.0.1)
@@ -128,12 +131,12 @@ GEM
     rack-test (0.8.3)
       rack (>= 1.0, < 3)
     rake (12.3.3)
-    rdf (3.1.3)
+    rdf (3.1.4)
       hamster (~> 3.0)
       link_header (~> 0.0, >= 0.0.8)
     rdf-aggregate-repo (3.1.0)
       rdf (~> 3.1)
-    rdf-rdfa (3.1.1)
+    rdf-rdfa (3.1.2)
       haml (~> 5.1)
       htmlentities (~> 4.3)
       rdf (~> 3.1, >= 3.1.2)
@@ -148,7 +151,7 @@ GEM
     rdf-turtle (3.1.1)
       ebnf (~> 1.2)
       rdf (~> 3.1, >= 3.1.2)
-    rdf-vocab (3.1.5)
+    rdf-vocab (3.1.7)
       rdf (~> 3.1, >= 3.1.2)
     rdf-xsd (3.1.0)
       rdf (~> 3.1)

data/bolognese.gemspec

@@ -16,8 +16,8 @@ Gem::Specification.new do |s|
   s.required_ruby_version = '~> 2.3'
 
   # Declare dependencies here, rather than in the Gemfile
-  s.add_dependency 'maremma', '>= 4.3', '< 5'
-  s.add_dependency 'faraday', "0.17.0"
+  s.add_dependency 'maremma', '>= 4.7', '< 5'
+  s.add_dependency 'faraday', '~> 0.17.3'
   s.add_dependency 'nokogiri', '~> 1.10.4'
   s.add_dependency 'loofah', '~> 2.0', '>= 2.0.3'
   s.add_dependency 'builder', '~> 3.2', '>= 3.2.2'
@@ -27,11 +27,11 @@ Gem::Specification.new do |s|
   s.add_dependency 'colorize', '~> 0.8.1'
   s.add_dependency 'namae', '~> 1.0'
   s.add_dependency 'edtf', '~> 3.0', '>= 3.0.4'
-  s.add_dependency 'citeproc-ruby', '~> 1.1', '>= 1.1.10'
-  s.add_dependency 'csl-styles', '~> 1.0', '>= 1.0.1.8'
+  s.add_dependency 'citeproc-ruby', '~> 1.1', '>= 1.1.12'
+  s.add_dependency 'csl-styles', '~> 1.0', '>= 1.0.1.10'
   s.add_dependency 'iso8601', '~> 0.9.1'
   s.add_dependency 'postrank-uri', '~> 1.0', '>= 1.0.18'
-  s.add_dependency 'json-ld', '~> 3.1', '>= 3.1.4'
+  s.add_dependency 'json-ld-preloaded', '~> 3.1', '>= 3.1.3'
   s.add_dependency 'jsonlint', '~> 0.3.0'
   s.add_dependency 'oj', '~> 3.10'
   s.add_dependency "oj_mimic_json", "~> 1.0", ">= 1.0.1"

data/lib/bolognese/datacite_utils.rb

@@ -116,11 +116,10 @@ module Bolognese
     end
 
     def insert_alternate_identifiers(xml)
-      alternate_identifiers = Array.wrap(identifiers).select { |r| r["identifierType"] != "DOI" }
-      return xml unless alternate_identifiers.present?
+      return xml unless identifiers.present?
 
       xml.alternateIdentifiers do
-        Array.wrap(alternate_identifiers).each do |alternate_identifier|
+        Array.wrap(identifiers).each do |alternate_identifier|
           xml.alternateIdentifier(alternate_identifier["identifier"], 'alternateIdentifierType' => alternate_identifier["identifierType"])
         end
       end

data/lib/bolognese/doi_utils.rb

@@ -28,7 +28,7 @@ module Bolognese
 
     def doi_api_url(doi, options = {})
       sandbox = Array(/handle.test.datacite.org/.match(doi)).last
-      sandbox.present? || options[:sandbox] ? "https://api.test.datacite.org/dois/" + doi_from_url(doi) : "https://api.datacite.org/dois/" + doi_from_url(doi)
+      sandbox.present? || options[:sandbox] ? "https://api.test.datacite.org/dois/#{doi_from_url(doi)}?include=media,client"  : "https://api.datacite.org/dois/#{doi_from_url(doi)}?include=media,client"
     end
 
     def normalize_doi(doi, options = {})

data/lib/bolognese/metadata_utils.rb

@@ -153,7 +153,7 @@ module Bolognese
         "language" => language,
         "author" => author,
         "contributor" => to_citeproc(contributors),
-        "issued" => get_date(dates, "Issued") ? get_date_parts(get_date(dates, "Issued")) : nil,
+        "issued" => get_date(dates, "Issued") ? get_date_parts(get_date(dates, "Issued")) : get_date_parts(publication_year),
         "submitted" => Array.wrap(dates).find { |d| d["dateType"] == "Submitted" }.to_h.fetch("__content__", nil),
         "abstract" => parse_attributes(descriptions, content: "description", first: true),
         "container-title" => container_title,

data/lib/bolognese/readers/bibtex_reader.rb

@@ -80,7 +80,6 @@ module Bolognese
 
         { "id" => normalize_doi(doi),
           "types" => types,
-          "identifiers" => [{ "identifier" => normalize_doi(doi), "identifierType" => "DOI" }],
           "doi" => doi,
           "url" => meta.try(:url).to_s.presence,
           "titles" => meta.try(:title).present? ? [{ "title" => meta.try(:title).to_s }] : [],

data/lib/bolognese/readers/citeproc_reader.rb

@@ -86,19 +86,8 @@ module Bolognese
           nil
         end
 
-        identifiers = [normalize_id(meta.fetch("id", nil)), normalize_doi(meta.fetch("DOI", nil))].compact.map do |r|
-          r = normalize_id(r)
+        id = normalize_id(meta.fetch("id", nil) || meta.fetch("DOI", nil))
 
-          if r.start_with?("https://doi.org")
-            { "identifierType" => "DOI", "identifier" => r }
-          else
-              { "identifierType" => "URL", "identifier" => r }
-          end
-        end.uniq
-
-        id = Array.wrap(identifiers).first.to_h.fetch("identifier", nil)
-        doi = Array.wrap(identifiers).find { |r| r["identifierType"] == "DOI" }.to_h.fetch("identifier", nil)
-          
         state = id.present? || read_options.present? ? "findable" : "not_found"
         subjects = Array.wrap(meta.fetch("categories", nil)).reduce([]) do |sum, subject|
           sum += name_to_fos(subject)
@@ -107,9 +96,8 @@ module Bolognese
         end
 
         { "id" => id,
-          "identifiers" => identifiers,
           "types" => types,
-          "doi" => doi_from_url(doi),
+          "doi" => doi_from_url(id),
           "url" => normalize_id(meta.fetch("URL", nil)),
           "titles" => [{ "title" => meta.fetch("title", nil) }],
           "creators" => creators,

data/lib/bolognese/readers/codemeta_reader.rb

@@ -22,19 +22,16 @@ module Bolognese
 
         meta = string.present? ? Maremma.from_json(string) : {}
 
-        identifiers = ([meta.fetch("@id", nil)] + Array.wrap(meta.fetch("identifier", nil))).map do |r|
+        identifiers = Array.wrap(meta.fetch("identifier", nil)).map do |r|
           r = normalize_id(r) if r.is_a?(String)
-          if r.is_a?(String) && r.start_with?("https://doi.org")
-            { "identifierType" => "DOI", "identifier" => r }
-          elsif r.is_a?(String)
+          if r.is_a?(String) && !r.start_with?("https://doi.org")
               { "identifierType" => "URL", "identifier" => r }
           elsif r.is_a?(Hash)
             { "identifierType" => get_identifier_type(r["propertyID"]), "identifier" => r["value"] }
           end
         end.compact.uniq
 
-        id = Array.wrap(identifiers).first.to_h.fetch("identifier", nil)
-        doi = Array.wrap(identifiers).find { |r| r["identifierType"] == "DOI" }.to_h.fetch("identifier", nil)
+        id = normalize_id(options[:doi] || meta.fetch("@id", nil) || meta.fetch("identifier", nil))
 
         has_agents = meta.fetch("agents", nil)
         authors =  has_agents.nil? ? meta.fetch("authors", nil) : has_agents
@@ -70,7 +67,7 @@ module Bolognese
         { "id" => id,
           "types" => types,
           "identifiers" => identifiers,
-          "doi" => doi_from_url(doi),
+          "doi" => doi_from_url(id),
           "url" => normalize_id(meta.fetch("codeRepository", nil)),
           "titles" => titles,
           "creators" => creators,

data/lib/bolognese/readers/crossref_reader.rb

@@ -150,14 +150,12 @@ module Bolognese
                         "volume" => bibliographic_metadata.fetch("volume", nil) }.compact
                     end
 
-        identifiers = [{ "identifierType" => "DOI", "identifier" => normalize_doi(options[:doi] || options[:id] || bibliographic_metadata.dig("doi_data", "doi")) }, crossref_alternate_identifiers(bibliographic_metadata)].compact
-
-        id = Array.wrap(identifiers).first.to_h.fetch("identifier", nil)
-        doi = Array.wrap(identifiers).find { |r| r["identifierType"] == "DOI" }.to_h.fetch("identifier", nil)
+        id = normalize_doi(options[:doi] || options[:id] || bibliographic_metadata.dig("doi_data", "doi"))
+        identifiers = [crossref_alternate_identifiers(bibliographic_metadata)].compact
 
         { "id" => id,
           "types" => types,
-          "doi" => doi_from_url(doi),
+          "doi" => doi_from_url(id),
           "url" => parse_attributes(bibliographic_metadata.dig("doi_data", "resource"), first: true),
           "titles" => titles,
           "identifiers" => identifiers,

data/lib/bolognese/readers/datacite_reader.rb

@@ -74,14 +74,12 @@ module Bolognese
           id = normalize_doi(meta.dig("identifier", "__content__") || options[:id], sandbox: options[:sandbox])
         end
 
-        identifiers = [{ "identifierType" => "DOI", "identifier" => id }] + Array.wrap(meta.dig("alternateIdentifiers", "alternateIdentifier")).map do |r|
+        identifiers = Array.wrap(meta.dig("alternateIdentifiers", "alternateIdentifier")).map do |r|
           if r["__content__"].present?
             { "identifierType" => get_identifier_type(r["alternateIdentifierType"]), "identifier" => r["__content__"] }
           end
         end.compact
 
-        doi = Array.wrap(identifiers).find { |r| r["identifierType"] == "DOI" }.to_h.fetch("identifier", nil)
-
         resource_type_general = meta.dig("resourceType", "resourceTypeGeneral")
         resource_type = meta.dig("resourceType", "__content__")
         schema_org = Bolognese::Utils::CR_TO_SO_TRANSLATIONS[resource_type.to_s.underscore.camelcase] || Bolognese::Utils::DC_TO_SO_TRANSLATIONS[resource_type_general.to_s.dasherize] || "CreativeWork"
@@ -217,11 +215,11 @@ module Bolognese
           end
         end.compact
 
-        state = doi.present? || read_options.present? ? "findable" : "not_found"
+        state = id.present? || read_options.present? ? "findable" : "not_found"
 
         { "id" => id,
           "types" => types,
-          "doi" => doi_from_url(doi),
+          "doi" => doi_from_url(id),
           "identifiers" => identifiers,
           "url" => options.fetch(:url, nil).to_s.strip.presence,
           "titles" => titles,

data/lib/bolognese/readers/ris_reader.rb

@@ -47,13 +47,8 @@ module Bolognese
           "ris" => ris_type
         }.compact
 
-        identifiers = [normalize_doi(options[:doi]) || normalize_doi(meta.fetch("DO", nil))].map do |r|
-          { "identifierType" => "DOI", "identifier" => normalize_id(r) }
-        end.compact
-
-        id = Array.wrap(identifiers).first.to_h.fetch("identifier", nil)
-        doi = Array.wrap(identifiers).find { |r| r["identifierType"] == "DOI" }.to_h.fetch("identifier", nil)
-
+        id = normalize_doi(options[:doi] || meta.fetch("DO", nil))
+        
         author = Array.wrap(meta.fetch("AU", nil)).map { |a| { "creatorName" => a } }
         date_parts = meta.fetch("PY", nil).to_s.split("/")
         created_date_parts = meta.fetch("Y1", nil).to_s.split("/")
@@ -90,8 +85,7 @@ module Bolognese
 
         { "id" => id,
           "types" => types,
-          "identifiers" => identifiers,
-          "doi" => doi_from_url(doi),
+          "doi" => doi_from_url(id),
           "url" => meta.fetch("UR", nil),
           "titles" => meta.fetch("T1", nil).present? ? [{ "title" => meta.fetch("T1", nil) }] : nil,
           "creators" => get_authors(author),

data/lib/bolognese/readers/schema_org_reader.rb

@@ -48,18 +48,16 @@ module Bolognese
 
         meta = string.present? ? Maremma.from_json(string) : {}
 
-        identifiers = ([options[:doi] || meta.fetch("@id", nil)] + Array.wrap(meta.fetch("identifier", nil))).map do |r|
+        identifiers = Array.wrap(meta.fetch("identifier", nil)).map do |r|
           r = normalize_id(r) if r.is_a?(String)
-          if r.is_a?(String) && r.start_with?("https://doi.org")
-            { "identifierType" => "DOI", "identifier" => r }
-          elsif r.is_a?(String)
+          if r.is_a?(String) && !r.start_with?("https://doi.org")
               { "identifierType" => "URL", "identifier" => r }
           elsif r.is_a?(Hash)
             { "identifierType" => get_identifier_type(r["propertyID"]), "identifier" => r["value"] }
           end
         end.compact.uniq
 
-        id = Array.wrap(identifiers).first.to_h.fetch("identifier", nil)
+        id = normalize_id(options[:doi] || meta.fetch("@id", nil) || meta.fetch("identifier", nil))
 
         schema_org = meta.fetch("@type", nil) && meta.fetch("@type").camelcase
         resource_type_general = Bolognese::Utils::SO_TO_DC_TRANSLATIONS[schema_org]
@@ -128,6 +126,14 @@ module Bolognese
         dates << { "date" => meta.fetch("dateModified"), "dateType" => "Updated" } if Date.edtf(meta.fetch("dateModified", nil)).present?
         publication_year = meta.fetch("datePublished")[0..3] if meta.fetch("datePublished", nil).present?
 
+        if meta.fetch("inLanguage", nil).is_a?(String)
+          language = meta.fetch("inLanguage")
+        elsif meta.fetch("inLanguage", nil).is_a?(Object)
+          language = meta.dig("inLanguage", 'alternateName') || meta.dig("inLanguage", 'name')
+        else
+          language = nil
+        end
+
         state = meta.present? || read_options.present? ? "findable" : "not_found"
         geo_locations = Array.wrap(meta.fetch("spatialCoverage", nil)).map do |gl|
           if gl.dig("geo", "box")
@@ -136,7 +142,7 @@ module Bolognese
               "westBoundLongitude" => w,
               "eastBoundLongitude" => e,
               "southBoundLatitude" => s,
-              "northBoundLatitude" => n
+              "northBoundLatitude" => n,
             }.compact.presence
           else
             geo_location_box = nil
@@ -179,6 +185,7 @@ module Bolognese
           "rights_list" => rights_list,
           "version_info" => meta.fetch("version", nil).to_s.presence,
           "subjects" => subjects,
+          "language" => language,
           "state" => state,
           "schema_version" => meta.fetch("schemaVersion", nil).to_s.presence,
           "funding_references" => funding_references,

data/lib/bolognese/utils.rb

@@ -1177,11 +1177,11 @@ module Bolognese
     def hsh_to_fos(hsh)
       # first find subject in Fields of Science (OECD)
       fos = JSON.load(File.read(File.expand_path('../../../resources/oecd/fos-mappings.json', __FILE__))).fetch("fosFields")
-      subject = fos.find { |l| l["fosLabel"] == hsh["__content__"] || "FOS: " + l["fosLabel"] == hsh["__content__"] }
+      subject = fos.find { |l| l["fosLabel"] == hsh["__content__"] || "FOS: " + l["fosLabel"] == hsh["__content__"] || l["fosLabel"] == hsh["subject"]}
 
       if subject
         return [{ 
-          "subject" => sanitize(hsh["__content__"]), 
+          "subject" => sanitize(hsh["__content__"] || hsh["subject"]), 
           "subjectScheme" => hsh["subjectScheme"],
           "schemeUri" => hsh["schemeURI"], 
           "valueUri" => hsh["valueURI"], 
@@ -1200,19 +1200,19 @@ module Bolognese
 
       # try to extract forId
       if hsh["subjectScheme"] == "FOR"
-        for_id = hsh["__content__"].split(" ").first
+        for_id = hsh["__content__"].to_s.split(" ").first || hsh["subject"].to_s.split(" ").first
         for_id = for_id.rjust(6, "0")        
         
         subject = for_fields.find { |l| l["forId"] == for_id } ||
                   for_disciplines.find { |l| l["forId"] == for_id[0..3] }
       else
-        subject = for_fields.find { |l| l["forLabel"] == hsh["__content__"] } ||
-                  for_disciplines.find { |l| l["forLabel"] == hsh["__content__"] }
+        subject = for_fields.find { |l| l["forLabel"] == hsh["__content__"] || l["forLabel"] == hsh["subject"] } ||
+                  for_disciplines.find { |l| l["forLabel"] == hsh["__content__"] || l["forLabel"] == hsh["subject"] }
       end
 
       if subject
         [{ 
-          "subject" => sanitize(hsh["__content__"]), 
+          "subject" => sanitize(hsh["__content__"] || hsh["subject"]), 
           "subjectScheme" => hsh["subjectScheme"],
           "schemeUri" => hsh["schemeURI"], 
           "valueUri" => hsh["valueURI"], 
@@ -1224,7 +1224,7 @@ module Bolognese
         }]
       else
         [{ 
-          "subject" => sanitize(hsh["__content__"]), 
+          "subject" => sanitize(hsh["__content__"] || hsh["subject"]), 
           "subjectScheme" => hsh["subjectScheme"],
           "schemeUri" => hsh["schemeURI"], 
           "valueUri" => hsh["valueURI"], 

data/lib/bolognese/version.rb

@@ -1,3 +1,3 @@
 module Bolognese
-  VERSION = "1.6.9"
+  VERSION = "1.7.2"
 end

data/lib/bolognese/writers/codemeta_writer.rb

@@ -17,7 +17,7 @@ module Bolognese
           "description" => parse_attributes(descriptions, content: "description", first: true),
           "version" => version_info,
           "tags" => subjects.present? ? Array.wrap(subjects).map { |k| parse_attributes(k, content: "subject", first: true) } : nil,
-          "datePublished" => get_date(dates, "Issued"),
+          "datePublished" => get_date(dates, "Issued") || publication_year,
           "dateModified" => get_date(dates, "Updated"),
           "publisher" => publisher,
           "license" => Array.wrap(rights_list).map { |l| l["rightsUri"] }.compact.unwrap,