bolognese 1.5.16 → 1.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e25803b02c683fa2d39bce713e1f1db91bb5231e5f357a121aa718e920e03fe
-  data.tar.gz: 614f8f6c57c4b5efff60553a8dbd65935c5d37680ec238945b5f8fdb94160905
+  metadata.gz: 49f153c7bf6a7dd69a2cf35844b9fe303574ebeb6f515a00581cea60955dc6d6
+  data.tar.gz: ebc11d65b789fb08e97dafa25c2e9e0b2ddfb6085011478498f2e619ebdd605c
 SHA512:
-  metadata.gz: eae192302662c5d77e0b66aa4799349972baada6e941fe2a250d92fbaddb945e4547ff9a2409aae1acb35136f9b971c567ee9aba1f07e9f69c3b8a96f7b1a4cb
-  data.tar.gz: ebe67deb2039574f9ba98489f1e8e990346fc15ae29418a3e40bb758db14ab4c72593acaff3fa0e131f253e75cbc7b9d6c93efc7c972b79db8a41fc60354b6f2
+  metadata.gz: d30e6e90b55e93bf4febdef8cf5fe3aa17e1821e2ee317f521957cdee0e811f8f94ed07f929b974ea967a8e5e4b7f0c8e7c08f47bc9bee681e72702f263ac80e
+  data.tar.gz: e30113a9e7ceddb8772a8e26310c7bc9ecdb349c93a33596c4d2ff0167c12650631e7dee7860c32a2e4dd8323453ed6acd7315c292060aef5a03954fbca61682

data/.gitignore

@@ -55,3 +55,5 @@ coverage/
 .env.*
 !.env.example
 !.env.travis
+
+.vscode

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    bolognese (1.5.16)
+    bolognese (1.6.2)
       activesupport (>= 4.2.5)
       benchmark_methods (~> 0.7)
       bibtex-ruby (>= 5.1.0)
@@ -30,12 +30,12 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.2.2)
+    activesupport (6.0.3.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-      zeitwerk (~> 2.2)
+      zeitwerk (~> 2.2, >= 2.2.2)
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     benchmark_methods (0.7)
@@ -105,7 +105,7 @@ GEM
       oj (>= 2.8.3)
       oj_mimic_json (~> 1.0, >= 1.0.1)
     mini_portile2 (2.4.0)
-    minitest (5.14.0)
+    minitest (5.14.1)
     multi_json (1.14.1)
     multipart-post (2.1.1)
     namae (1.0.1)
@@ -113,7 +113,7 @@ GEM
       mini_portile2 (~> 2.4.0)
     oj (3.10.6)
     oj_mimic_json (1.0.1)
-    optimist (3.0.0)
+    optimist (3.0.1)
     postrank-uri (1.0.24)
       addressable (>= 2.4.0)
       nokogiri (>= 1.8.0)
@@ -123,40 +123,43 @@ GEM
     rack-test (0.8.3)
       rack (>= 1.0, < 3)
     rake (12.3.3)
-    rdf (3.1.1)
+    rdf (3.1.2)
       hamster (~> 3.0)
       link_header (~> 0.0, >= 0.0.8)
     rdf-aggregate-repo (3.1.0)
       rdf (~> 3.1)
-    rdf-rdfa (3.1.0)
+    rdf-rdfa (3.1.1)
       haml (~> 5.1)
       htmlentities (~> 4.3)
-      rdf (~> 3.1)
+      rdf (~> 3.1, >= 3.1.2)
       rdf-aggregate-repo (~> 3.1)
+      rdf-vocab (~> 3.1, >= 3.1.5)
       rdf-xsd (~> 3.1)
     rdf-rdfxml (3.1.0)
       htmlentities (~> 4.3)
       rdf (~> 3.1)
       rdf-rdfa (~> 3.1)
       rdf-xsd (~> 3.1)
-    rdf-turtle (3.1.0)
+    rdf-turtle (3.1.1)
       ebnf (~> 1.2)
-      rdf (~> 3.1)
+      rdf (~> 3.1, >= 3.1.2)
+    rdf-vocab (3.1.5)
+      rdf (~> 3.1, >= 3.1.2)
     rdf-xsd (3.1.0)
       rdf (~> 3.1)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
       rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.1)
-      rspec-support (~> 3.9.1)
-    rspec-expectations (3.9.1)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
-    rspec-support (3.9.2)
+    rspec-support (3.9.3)
     rspec-xsd (0.1.0)
       nokogiri (~> 1.6)
       rspec (~> 3)

data/lib/bolognese/datacite_utils.rb

@@ -28,12 +28,14 @@ module Bolognese
       insert_publisher(xml)
       insert_publication_year(xml)
       insert_resource_type(xml)
-      insert_alternate_identifiers(xml)
       insert_subjects(xml)
-      insert_language(xml)
       insert_contributors(xml)
       insert_dates(xml)
+      insert_language(xml)
+      insert_alternate_identifiers(xml)
       insert_related_identifiers(xml)
+      insert_sizes(xml)
+      insert_formats(xml)
       insert_version(xml)
       insert_rights_list(xml)
       insert_descriptions(xml)
@@ -170,9 +172,9 @@ module Bolognese
     end
 
     def insert_version(xml)
-      return xml unless version_info.present?
+      return xml unless version.present?
 
-      xml.version(version_info)
+      xml.version(version)
     end
 
 
@@ -201,6 +203,22 @@ module Bolognese
       end
     end
 
+    def insert_sizes(xml)
+      xml.sizes do
+        Array.wrap(sizes).each do |s|
+          xml.size(s)
+        end
+      end
+    end
+
+    def insert_formats(xml)
+      xml.formats do
+        Array.wrap(formats).each do |f|
+          xml.format(f)
+        end
+      end
+    end
+
     def insert_rights_list(xml)
       return xml unless rights_list.present?
 

data/lib/bolognese/metadata.rb

@@ -9,7 +9,7 @@ module Bolognese
     attr_accessor :string, :from, :sandbox, :meta, :regenerate, :issue, :show_errors
     attr_reader :doc, :page_start, :page_end
     attr_writer :id, :provider_id, :client_id, :doi, :identifiers, :creators, :contributors, :titles, :publisher, 
-                :rights_list, :dates, :publication_year, :volume, :url, :version_info,
+                :rights_list, :dates, :publication_year, :volume, :url, :version,
                 :subjects, :contributor, :descriptions, :language, :sizes,
                 :formats, :schema_version, :meta, :container, :agency,
                 :format, :funding_references, :state, :geo_locations,
@@ -36,6 +36,7 @@ module Bolognese
         # generate name for method to call dynamically
         hsh = @from.present? ? send("get_" + @from, id: id, sandbox: options[:sandbox]) : {}
         string = hsh.fetch("string", nil)
+
       elsif input.present? && File.exist?(input)
         filename = File.basename(input)
         ext = File.extname(input)
@@ -105,7 +106,7 @@ module Bolognese
         :publication_year,
         :descriptions,
         :rights_list,
-        :version_info,
+        :version,
         :subjects,
         :language,
         :geo_locations,
@@ -115,7 +116,6 @@ module Bolognese
       ).compact
 
       @regenerate = options[:regenerate] || read_options.present?
-
       # generate name for method to call dynamically
       @meta = @from.present? ? send("read_" + @from, { string: string, sandbox: options[:sandbox], doi: options[:doi], id: id, ra: ra }.merge(read_options)) : {}
     end
@@ -189,10 +189,13 @@ module Bolognese
       @url ||= meta.fetch("url", nil)
     end
 
-    def version_info
-      @version_info ||= meta.fetch("version_info", nil)
+    def version
+      @version ||= meta.fetch("version", nil)
     end
 
+    # for backwards compatibility
+    alias_attribute :version_info, :version
+
     def publication_year
       @publication_year ||= meta.fetch("publication_year", nil)
     end

data/lib/bolognese/metadata_utils.rb

@@ -158,7 +158,7 @@ module Bolognese
         "publisher" => publisher,
         "title" => parse_attributes(titles, content: "title", first: true),
         "URL" => url,
-        "version" => version_info
+        "version" => version
       }.compact.symbolize_keys
     end
 
@@ -180,7 +180,7 @@ module Bolognese
         "identifiers" => identifiers,
         "sizes" => sizes,
         "formats" => formats,
-        "version" => version_info,
+        "version" => version,
         "rights_list" => rights_list,
         "descriptions" => descriptions,
         "geo_locations" => geo_locations,

data/lib/bolognese/readers/citeproc_reader.rb

@@ -100,8 +100,10 @@ module Bolognese
         doi = Array.wrap(identifiers).find { |r| r["identifierType"] == "DOI" }.to_h.fetch("identifier", nil)
           
         state = id.present? || read_options.present? ? "findable" : "not_found"
-        subjects = Array.wrap(meta.fetch("categories", nil)).map do |s|
-          { "subject" => s }
+        subjects = Array.wrap(meta.fetch("categories", nil)).reduce([]) do |sum, subject|
+          sum += name_to_fos(subject)
+
+          sum
         end
 
         { "id" => id,
@@ -119,7 +121,7 @@ module Bolognese
           "publication_year" => publication_year,
           "descriptions" => meta.fetch("abstract", nil).present? ? [{ "description" => sanitize(meta.fetch("abstract")), "descriptionType" => "Abstract" }] : [],
           "rights_list" => rights_list,
-          "version_info" => meta.fetch("version", nil),
+          "version" => meta.fetch("version", nil),
           "subjects" => subjects,
           "state" => state
         }.merge(read_options)

data/lib/bolognese/readers/codemeta_reader.rb

@@ -57,8 +57,10 @@ module Bolognese
           "bibtex" => Bolognese::Utils::SO_TO_BIB_TRANSLATIONS[schema_org] || "misc",
           "ris" => Bolognese::Utils::SO_TO_RIS_TRANSLATIONS[schema_org] || "GEN"
         }.compact
-        subjects = Array.wrap(meta.fetch("tags", nil)).map do |s|
-          { "subject" => s }
+        subjects = Array.wrap(meta.fetch("tags", nil)).reduce([]) do |sum, subject|
+          sum += name_to_fos(subject)
+
+          sum
         end
 
         has_title = meta.fetch("title", nil)
@@ -79,7 +81,7 @@ module Bolognese
           "publication_year" => publication_year,
           "descriptions" => meta.fetch("description", nil).present? ? [{ "description" => sanitize(meta.fetch("description")), "descriptionType" => "Abstract" }] : nil,
           "rights_list" => [{ "rightsUri" => meta.fetch("license", nil) }.compact],
-          "version_info" => meta.fetch("version", nil),
+          "version" => meta.fetch("version", nil),
           "subjects" => subjects,
           "state" => state
         }.merge(read_options)

data/lib/bolognese/readers/crossref_reader.rb

@@ -117,11 +117,8 @@ module Bolognese
         date_updated = { "date" => date_updated.fetch("__content__", nil), "dateType" => "Updated" } if date_updated.present?
 
         date_registered = Array.wrap(query.to_h["crm_item"]).find { |cr| cr["name"] == "deposit-timestamp" }
-        if date_registered && date_registered.fetch("__content__", nil).to_i > 15000000000000 # check for valid input string that includes seconds
-          date_registered = DateTime.strptime(date_registered.fetch("__content__", ""), "%Y%m%d%H%M%S").strftime('%Y-%m-%dT%H:%M:%SZ')
-        else
-          date_registered = nil
-        end
+        date_registered = get_datetime_from_time(date_registered.fetch("__content__", nil)) if date_registered.present?
+
         # check that date is valid iso8601 date
         date_published = nil unless Date.edtf(date_published.to_h["date"]).present?
         date_updated = nil unless Date.edtf(date_updated.to_h["date"]).present?
@@ -175,7 +172,7 @@ module Bolognese
           "publication_year" => publication_year,
           "descriptions" => crossref_description(bibliographic_metadata),
           "rights_list" => crossref_license(program_metadata),
-          "version_info" => nil,
+          "version" => nil,
           "subjects" => nil,
           "language" => nil,
           "sizes" => nil,

data/lib/bolognese/readers/datacite_reader.rb

@@ -93,7 +93,7 @@ module Bolognese
           "bibtex" => Bolognese::Utils::CR_TO_BIB_TRANSLATIONS[resource_type.to_s.underscore.camelcase] || Bolognese::Utils::SO_TO_BIB_TRANSLATIONS[schema_org] || "misc",
           "ris" => Bolognese::Utils::CR_TO_RIS_TRANSLATIONS[resource_type.to_s.underscore.camelcase] || Bolognese::Utils::DC_TO_RIS_TRANSLATIONS[resource_type_general.to_s.dasherize] || "GEN"
         }.compact
-        
+
         titles = Array.wrap(meta.dig("titles", "title")).map do |r|
           if r.blank?
             nil
@@ -122,15 +122,17 @@ module Bolognese
             { "rights" => r["__content__"], "rightsUri" => normalize_url(r["rightsURI"]), "lang" => r["lang"] }.compact
           end
         end.compact
-        subjects = Array.wrap(meta.dig("subjects", "subject")).map do |k|
-          if k.blank?
-            nil
-          elsif k.is_a?(String)
-            { "subject" => sanitize(k) }
-          elsif k.is_a?(Hash)
-            { "subject" => sanitize(k["__content__"]), "subjectScheme" => k["subjectScheme"], "schemeUri" => k["schemeURI"], "valueUri" => k["valueURI"], "lang" => k["lang"] }.compact
+        
+        subjects = Array.wrap(meta.dig("subjects", "subject")).reduce([]) do |sum, subject|
+          if subject.is_a?(String)
+            sum += name_to_fos(subject)
+          elsif subject.is_a?(Hash)
+            sum += hsh_to_fos(subject)
           end
-        end.compact
+
+          sum
+        end.uniq
+
         dates = Array.wrap(meta.dig("dates", "date")).map do |r|
           if r.is_a?(Hash) && date = sanitize(r["__content__"]).presence
             if Date.edtf(date).present? || Bolognese::Utils::UNKNOWN_INFORMATION.key?(date)
@@ -164,12 +166,15 @@ module Bolognese
         funding_references = Array.wrap(meta.dig("fundingReferences", "fundingReference")).compact.map do |fr|
           scheme_uri = parse_attributes(fr["funderIdentifier"], content: "schemeURI")
           funder_identifier = parse_attributes(fr["funderIdentifier"])
-          funder_identifier = !funder_identifier.to_s.start_with?("https://","http://") && scheme_uri.present? ? normalize_id(scheme_uri + funder_identifier) : normalize_id(funder_identifier)
-          
+          funder_identifier_type = parse_attributes(fr["funderIdentifier"], content: "funderIdentifierType")
+          if funder_identifier_type != "Other"
+            funder_identifier = !funder_identifier.to_s.start_with?("https://","http://") && scheme_uri.present? ? normalize_id(scheme_uri + funder_identifier) : normalize_id(funder_identifier)
+          end
+
           {
             "funderName" => fr["funderName"],
             "funderIdentifier" => funder_identifier,
-            "funderIdentifierType" => parse_attributes(fr["funderIdentifier"], content: "funderIdentifierType"),
+            "funderIdentifierType" => funder_identifier_type,
             "awardNumber" => parse_attributes(fr["awardNumber"]),
             "awardUri" => parse_attributes(fr["awardNumber"], content: "awardURI"),
             "awardTitle" => fr["awardTitle"] }.compact
@@ -181,7 +186,7 @@ module Bolognese
             rid = ri["__content__"]
           end
 
-          { 
+          {
             "relatedIdentifier" => rid,
             "relatedIdentifierType" => ri["relatedIdentifierType"],
             "relationType" => ri["relationType"],
@@ -230,7 +235,7 @@ module Bolognese
           "publication_year" => parse_attributes(meta.fetch("publicationYear", nil), first: true).to_s.strip.presence,
           "descriptions" => descriptions,
           "rights_list" => Array.wrap(rights_list),
-          "version_info" => meta.fetch("version", nil).to_s.presence,
+          "version" => meta.fetch("version", nil).to_s.presence,
           "subjects" => subjects,
           "language" => parse_attributes(meta.fetch("language", nil), first: true).to_s.strip.presence,
           "geo_locations" => geo_locations,

data/lib/bolognese/readers/npm_reader.rb

@@ -105,7 +105,7 @@ module Bolognese
           #"publication_year" => publication_year,
           "descriptions" => meta.fetch("description", nil).present? ? [{ "description" => sanitize(meta.fetch("description")), "descriptionType" => "Abstract" }] : [],
           "rights_list" => rights_list,
-          "version_info" => meta.fetch("version", nil),
+          "version" => meta.fetch("version", nil),
           "subjects" => subjects
           #"state" => state
         }.merge(read_options)

data/lib/bolognese/readers/ris_reader.rb

@@ -82,8 +82,10 @@ module Bolognese
           nil
         end
         state = meta.fetch("DO", nil).present? || read_options.present? ? "findable" : "not_found"
-        subjects = Array.wrap(meta.fetch("KW", nil)).map do |s|
-          { "subject" => s }
+        subjects = Array.wrap(meta.fetch("KW", nil)).reduce([]) do |sum, subject|
+          sum += name_to_fos(subject)
+
+          sum
         end
 
         { "id" => id,

data/lib/bolognese/readers/schema_org_reader.rb

@@ -150,8 +150,13 @@ module Bolognese
             "geoLocationBox" => geo_location_box
           }.compact
         end
-        subjects = Array.wrap(meta.fetch("keywords", nil).to_s.split(", ")).map do |s|
-          { "subject" => s }
+
+        # handle keywords as array and as comma-separated string
+        subjects = meta.fetch("keywords", nil)
+        subjects = subjects.to_s.split(", ") if subjects.is_a?(String)
+        subjects = Array.wrap(subjects).reduce([]) do |sum, subject|
+          sum += name_to_fos(subject)
+          sum
         end
 
         { "id" => id,
@@ -173,7 +178,7 @@ module Bolognese
           "dates" => dates,
           "descriptions" => meta.fetch("description", nil).present? ? [{ "description" => sanitize(meta.fetch("description")), "descriptionType" => "Abstract" }] : nil,
           "rights_list" => rights_list,
-          "version_info" => meta.fetch("version", nil).to_s.presence,
+          "version" => meta.fetch("version", nil).to_s.presence,
           "subjects" => subjects,
           "state" => state,
           "schema_version" => meta.fetch("schemaVersion", nil).to_s.presence,

data/lib/bolognese/utils.rb

@@ -974,6 +974,14 @@ module Bolognese
       nil
     end
 
+    # iso8601 datetime without hyphens and colons, used by Crossref
+    # return nil if invalid
+    def get_datetime_from_time(time)
+      DateTime.strptime(time.to_s, "%Y%m%d%H%M%S").strftime('%Y-%m-%dT%H:%M:%SZ')
+    rescue ArgumentError
+      nil
+    end
+
     def get_date(dates, date_type)
       dd = Array.wrap(dates).find { |d| d["dateType"] == date_type } || {}
       dd.fetch("date", nil)
@@ -1048,5 +1056,100 @@ module Bolognese
       error_array
     end
 
+    def name_to_fos(name)
+      # first find subject in Fields of Science (OECD)
+      fos = JSON.load(File.read(File.expand_path('../../../resources/oecd/fos-mappings.json', __FILE__))).fetch("fosFields")
+      
+      subject = fos.find { |l| l["fosLabel"] == name || "FOS: " + l["fosLabel"] == name }
+
+      if subject
+        return [{ 
+          "subject" => sanitize(name) },
+        {
+          "subject" => "FOS: " + subject["fosLabel"],
+          "subjectScheme" => "Fields of Science and Technology (FOS)",
+          "schemeUri" => "http://www.oecd.org/science/inno/38235147.pdf"
+        }]
+      end
+
+      # if not found, look in Fields of Research (Australian and New Zealand Standard Research Classification)
+      # and map to Fields of Science. Add an extra entry for the latter
+      fores = JSON.load(File.read(File.expand_path('../../../resources/oecd/for-mappings.json', __FILE__)))
+      for_fields = fores.fetch("forFields")
+      for_disciplines = fores.fetch("forDisciplines")
+      
+      subject = for_fields.find { |l| l["forLabel"] == name } ||
+                for_disciplines.find { |l| l["forLabel"] == name }
+
+      if subject
+        [{ 
+          "subject" => sanitize(name) },
+        {
+          "subject" => "FOS: " + subject["fosLabel"],
+          "subjectScheme" => "Fields of Science and Technology (FOS)",
+          "schemeUri" => "http://www.oecd.org/science/inno/38235147.pdf"
+        }]
+      else
+        [{ "subject" => sanitize(name) }]
+      end
+    end
+
+    def hsh_to_fos(hsh)
+      # first find subject in Fields of Science (OECD)
+      fos = JSON.load(File.read(File.expand_path('../../../resources/oecd/fos-mappings.json', __FILE__))).fetch("fosFields")
+      subject = fos.find { |l| l["fosLabel"] == hsh["__content__"] || "FOS: " + l["fosLabel"] == hsh["__content__"] }
+
+      if subject
+        return [{ 
+          "subject" => sanitize(hsh["__content__"]), 
+          "subjectScheme" => hsh["subjectScheme"],
+          "schemeUri" => hsh["schemeURI"], 
+          "valueUri" => hsh["valueURI"], 
+          "lang" => hsh["lang"] }.compact,
+        {  
+          "subject" => "FOS: " + subject["fosLabel"],
+          "subjectScheme" => "Fields of Science and Technology (FOS)",
+          "schemeUri" => "http://www.oecd.org/science/inno/38235147.pdf" }.compact] 
+      end
+
+      # if not found, look in Fields of Research (Australian and New Zealand Standard Research Classification)
+      # and map to Fields of Science. Add an extra entry for the latter
+      fores = JSON.load(File.read(File.expand_path('../../../resources/oecd/for-mappings.json', __FILE__)))
+      for_fields = fores.fetch("forFields")
+      for_disciplines = fores.fetch("forDisciplines")
+
+      # try to extract forId
+      if hsh["subjectScheme"] == "FOR"
+        for_id = hsh["__content__"].split(" ").first
+        for_id = for_id.rjust(6, "0")        
+        
+        subject = for_fields.find { |l| l["forId"] == for_id } ||
+                  for_disciplines.find { |l| l["forId"] == for_id[0..3] }
+      else
+        subject = for_fields.find { |l| l["forLabel"] == hsh["__content__"] } ||
+                  for_disciplines.find { |l| l["forLabel"] == hsh["__content__"] }
+      end
+
+      if subject
+        [{ 
+          "subject" => sanitize(hsh["__content__"]), 
+          "subjectScheme" => hsh["subjectScheme"],
+          "schemeUri" => hsh["schemeURI"], 
+          "valueUri" => hsh["valueURI"], 
+          "lang" => hsh["lang"] }.compact,
+        {
+          "subject" => "FOS: " + subject["fosLabel"],
+          "subjectScheme" => "Fields of Science and Technology (FOS)",
+          "schemeUri" => "http://www.oecd.org/science/inno/38235147.pdf"
+        }]
+      else
+        [{ 
+          "subject" => sanitize(hsh["__content__"]), 
+          "subjectScheme" => hsh["subjectScheme"],
+          "schemeUri" => hsh["schemeURI"], 
+          "valueUri" => hsh["valueURI"], 
+          "lang" => hsh["lang"] }.compact]  
+      end
+    end
   end
 end