bolognese 1.3.1 → 2.3.9

data/lib/bolognese/utils.rb

@@ -2,46 +2,112 @@
 
 module Bolognese
   module Utils
-    LICENSE_NAMES = {
-      "http://creativecommons.org/publicdomain/zero/1.0/" => "Public Domain (CC0 1.0)",
-      "http://creativecommons.org/licenses/by/3.0/" => "Creative Commons Attribution 3.0 (CC-BY 3.0)",
-      "http://creativecommons.org/licenses/by/4.0/" => "Creative Commons Attribution 4.0 (CC-BY 4.0)",
-      "http://creativecommons.org/licenses/by-nc/4.0/" => "Creative Commons Attribution Noncommercial 4.0 (CC-BY-NC 4.0)",
-      "http://creativecommons.org/licenses/by-sa/4.0/" => "Creative Commons Attribution Share Alike 4.0 (CC-BY-SA 4.0)",
-      "http://creativecommons.org/licenses/by-nc-nd/4.0/" => "Creative Commons Attribution Noncommercial No Derivatives 4.0 (CC-BY-NC-ND 4.0)"
+    class << self
+      include Utils
+    end
+
+    NORMALIZED_LICENSES = {
+      "https://creativecommons.org/licenses/by/1.0" => "https://creativecommons.org/licenses/by/1.0/legalcode",
+      "https://creativecommons.org/licenses/by/2.0" => "https://creativecommons.org/licenses/by/2.0/legalcode",
+      "https://creativecommons.org/licenses/by/2.5" => "https://creativecommons.org/licenses/by/2.5/legalcode",
+      "https://creativecommons.org/licenses/by/3.0" => "https://creativecommons.org/licenses/by/3.0/legalcode",
+      "https://creativecommons.org/licenses/by/3.0/us" => "https://creativecommons.org/licenses/by/3.0/legalcode",
+      "https://creativecommons.org/licenses/by/4.0" => "https://creativecommons.org/licenses/by/4.0/legalcode",
+      "https://creativecommons.org/licenses/by-nc/1.0" => "https://creativecommons.org/licenses/by-nc/1.0/legalcode",
+      "https://creativecommons.org/licenses/by-nc/2.0" => "https://creativecommons.org/licenses/by-nc/2.0/legalcode",
+      "https://creativecommons.org/licenses/by-nc/2.5" => "https://creativecommons.org/licenses/by-nc/2.5/legalcode",
+      "https://creativecommons.org/licenses/by-nc/3.0" => "https://creativecommons.org/licenses/by-nc/3.0/legalcode",
+      "https://creativecommons.org/licenses/by-nc/4.0" => "https://creativecommons.org/licenses/by-nc/4.0/legalcode",
+      "https://creativecommons.org/licenses/by-nd-nc/1.0" => "https://creativecommons.org/licenses/by-nd-nc/1.0/legalcode",
+      "https://creativecommons.org/licenses/by-nd-nc/2.0" => "https://creativecommons.org/licenses/by-nd-nc/2.0/legalcode",
+      "https://creativecommons.org/licenses/by-nd-nc/2.5" => "https://creativecommons.org/licenses/by-nd-nc/2.5/legalcode",
+      "https://creativecommons.org/licenses/by-nd-nc/3.0" => "https://creativecommons.org/licenses/by-nd-nc/3.0/legalcode",
+      "https://creativecommons.org/licenses/by-nd-nc/4.0" => "https://creativecommons.org/licenses/by-nd-nc/4.0/legalcode",
+      "https://creativecommons.org/licenses/by-nc-sa/1.0" => "https://creativecommons.org/licenses/by-nc-sa/1.0/legalcode",
+      "https://creativecommons.org/licenses/by-nc-sa/2.0" => "https://creativecommons.org/licenses/by-nc-sa/2.0/legalcode",
+      "https://creativecommons.org/licenses/by-nc-sa/2.5" => "https://creativecommons.org/licenses/by-nc-sa/2.5/legalcode",
+      "https://creativecommons.org/licenses/by-nc-sa/3.0" => "https://creativecommons.org/licenses/by-nc-sa/3.0/legalcode",
+      "https://creativecommons.org/licenses/by-nc-sa/4.0" => "https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode",
+      "https://creativecommons.org/licenses/by-nd/1.0" => "https://creativecommons.org/licenses/by-nd/1.0/legalcode",
+      "https://creativecommons.org/licenses/by-nd/2.0" => "https://creativecommons.org/licenses/by-nd/2.0/legalcode",
+      "https://creativecommons.org/licenses/by-nd/2.5" => "https://creativecommons.org/licenses/by-nd/2.5/legalcode",
+      "https://creativecommons.org/licenses/by-nd/3.0" => "https://creativecommons.org/licenses/by-nd/3.0/legalcode",
+      "https://creativecommons.org/licenses/by-nd/4.0" => "https://creativecommons.org/licenses/by-nd/4.0/legalcode",
+      "https://creativecommons.org/licenses/by-sa/1.0" => "https://creativecommons.org/licenses/by-sa/1.0/legalcode",
+      "https://creativecommons.org/licenses/by-sa/2.0" => "https://creativecommons.org/licenses/by-sa/2.0/legalcode",
+      "https://creativecommons.org/licenses/by-sa/2.5" => "https://creativecommons.org/licenses/by-sa/2.5/legalcode",
+      "https://creativecommons.org/licenses/by-sa/3.0" => "https://creativecommons.org/licenses/by-sa/3.0/legalcode",
+      "https://creativecommons.org/licenses/by-sa/4.0" => "https://creativecommons.org/licenses/by-sa/4.0/legalcode",
+      "https://creativecommons.org/licenses/by-nc-nd/1.0" => "https://creativecommons.org/licenses/by-nc-nd/1.0/legalcode",
+      "https://creativecommons.org/licenses/by-nc-nd/2.0" => "https://creativecommons.org/licenses/by-nc-nd/2.0/legalcode",
+      "https://creativecommons.org/licenses/by-nc-nd/2.5" => "https://creativecommons.org/licenses/by-nc-nd/2.5/legalcode",
+      "https://creativecommons.org/licenses/by-nc-nd/3.0" => "https://creativecommons.org/licenses/by-nc-nd/3.0/legalcode",
+      "https://creativecommons.org/licenses/by-nc-nd/4.0" => "https://creativecommons.org/licenses/by-nc-nd/4.0/legalcode",
+      "https://creativecommons.org/licenses/publicdomain" => "https://creativecommons.org/licenses/publicdomain/",
+      "https://creativecommons.org/publicdomain/zero/1.0" => "https://creativecommons.org/publicdomain/zero/1.0/legalcode",
     }
 
     DC_TO_SO_TRANSLATIONS = {
       "Audiovisual" => "MediaObject",
+      "Book" => "Book",
+      "BookChapter" => "Chapter",
       "Collection" => "Collection",
+      "ComputationalNotebook" => "SoftwareSourceCode",
+      "ConferencePaper" => "Article",
+      "ConferenceProceeding" => "Periodical",
+      "DataPaper" => "Article",
       "Dataset" => "Dataset",
+      "Dissertation" => "Thesis",
       "Event" => "Event",
       "Image" => "ImageObject",
       "InteractiveResource" => nil,
+      "Journal" => "Periodical",
+      "JournalArticle" => "ScholarlyArticle",
       "Model" => nil,
+      "OutputManagementPlan" => nil,
+      "PeerReview" => "Review",
       "PhysicalObject" => nil,
+      "Preprint" => nil,
+      "Report" => "Report",
       "Service" => "Service",
       "Software" => "SoftwareSourceCode",
       "Sound" => "AudioObject",
+      "Standard" => nil,
       "Text" => "ScholarlyArticle",
       "Workflow" => nil,
       "Other" => "CreativeWork",
       # not part of DataCite schema, but used internally
       "Periodical" => "Periodical",
-      "DataCatalog" => "DataCatalog"
+      "DataCatalog" => "DataCatalog",
+      "Award" => "Grant",
+      "Project" => "Project"
     }
 
     DC_TO_CP_TRANSLATIONS = {
       "Audiovisual" => "motion_picture",
+      "Book" => "book",
+      "BookChapter" => "chapter",
       "Collection" => nil,
+      "ComputationalNotebook" => nil,
+      "ConferencePaper" => "paper_conference",
+      "ConferenceProceeding" => nil,
+      "DataPaper" => "report",
       "Dataset" => "dataset",
+      "Dissertation" => nil,
       "Event" => nil,
       "Image" => "graphic",
       "InteractiveResource" => nil,
+      "Journal" => nil,
+      "JournalArticle" => "article-journal",
       "Model" => nil,
+      "OutputManagementPlan" => nil,
+      "PeerReview" => "review",
       "PhysicalObject" => nil,
+      "Preprint" => nil,
+      "Report" => "report",
       "Service" => nil,
       "Sound" => "song",
+      "Standard" => nil,
       "Text" => "report",
       "Workflow" => nil,
       "Other" => nil
@@ -86,7 +152,7 @@ module Bolognese
       "EditedBook" => "Book",
       "JournalArticle" => "ScholarlyArticle",
       "Journal" => nil,
-      "Report" => nil,
+      "Report" => "Report",
       "BookSeries" => nil,
       "ReportSeries" => nil,
       "BookTrack" => nil,
@@ -101,7 +167,8 @@ module Bolognese
       "ReferenceEntry" => nil,
       "JournalVolume" => "PublicationVolume",
       "BookSet" => nil,
-      "PostedContent" => "ScholarlyArticle"
+      "PostedContent" => "ScholarlyArticle",
+      "PeerReview" => "Review"
     }
 
     CR_TO_BIB_TRANSLATIONS = {
@@ -115,7 +182,7 @@ module Bolognese
       "EditedBook" => "book",
       "JournalArticle" => "article",
       "Journal" => nil,
-      "Report" => nil,
+      "Report" => "techreport",
       "BookSeries" => nil,
       "ReportSeries" => nil,
       "BookTrack" => nil,
@@ -169,12 +236,44 @@ module Bolognese
       "BookSet" => "book"
     }
 
+    CR_TO_DC_TRANSLATIONS = {
+      "Proceedings" => nil,
+      "ReferenceBook" => nil,
+      "JournalIssue" => "Text",
+      "ProceedingsArticle" => "ConferencePaper",
+      "Other" => "Other",
+      "Dissertation" => "Dissertation",
+      "Dataset" => "Dataset",
+      "EditedBook" => "Book",
+      "JournalArticle" => "JournalArticle",
+      "Journal" => "Journal",
+      "Report" => "Report",
+      "BookSeries" => nil,
+      "ReportSeries" => nil,
+      "BookTrack" => nil,
+      "Standard" => "Standard",
+      "BookSection" => "BookChapter",
+      "BookPart" => nil,
+      "Book" => "Book",
+      "BookChapter" => "BookChapter",
+      "SaComponent" => "Text",
+      "StandardSeries" => "Standard",
+      "Monograph" => "book",
+      "Component" => nil,
+      "ReferenceEntry" => nil,
+      "JournalVolume" => nil,
+      "BookSet" => nil,
+      "PostedContent" => "JournalArticle",
+      "PeerReview" => "PeerReview"
+    }
+
     SO_TO_DC_TRANSLATIONS = {
       "Article" => "Text",
       "AudioObject" => "Sound",
       "Blog" => "Text",
       "BlogPosting" => "Text",
-      "Chapter" => "Text",
+      "Book" => "Book",
+      "Chapter" => "BookChapter",
       "Collection" => "Collection",
       "DataCatalog" => "Dataset",
       "Dataset" => "Dataset",
@@ -182,9 +281,11 @@ module Bolognese
       "ImageObject" => "Image",
       "Movie" => "Audiovisual",
       "PublicationIssue" => "Text",
+      "Report" => "Report",
       "ScholarlyArticle" => "Text",
       "Thesis" => "Text",
       "Service" => "Service",
+      "Review" => "PeerReview",
       "SoftwareSourceCode" => "Software",
       "VideoObject" => "Audiovisual",
       "WebPage" => "Text",
@@ -226,6 +327,7 @@ module Bolognese
       "ImageObject" => "graphic",
       "Movie" => "motion_picture",
       "PublicationIssue" => nil,
+      "Report" => "report",
       "ScholarlyArticle" => "article-journal",
       "Service" => nil,
       "Thesis" => "thesis",
@@ -246,6 +348,7 @@ module Bolognese
       "Event" => nil,
       "ImageObject" => "FIGURE",
       "Movie" => "MPCT",
+      "Report" => "RPRT",
       "PublicationIssue" => nil,
       "ScholarlyArticle" => "JOUR",
       "Service" => nil,
@@ -266,11 +369,11 @@ module Bolognese
       "EditedBook" => "BOOK",
       "JournalArticle" => "JOUR",
       "Journal" => nil,
-      "Report" => nil,
+      "Report" => "RPRT",
       "BookSeries" => nil,
       "ReportSeries" => nil,
       "BookTrack" => nil,
-      "Standard" => nil,
+      "Standard" => "STAND",
       "BookSection" => "CHAP",
       "BookPart" => "CHAP",
       "Book" => "BOOK",
@@ -285,21 +388,73 @@ module Bolognese
 
     DC_TO_RIS_TRANSLATIONS = {
       "Audiovisual" => "MPCT",
+      "Book" => "BOOK",
+      "BookChapter" => "CHAP",
       "Collection" => nil,
+      "ComputationalNotebook" => "COMP",
+      "ConferencePaper" => "CPAPER",
+      "ConferenceProceeding" => "CONF",
+      "DataPaper" => nil,
       "Dataset" => "DATA",
+      "Dissertation" => "THES",
       "Event" => nil,
       "Image" => "FIGURE",
       "InteractiveResource" => nil,
+      "Journal" => nil,
+      "JournalArticle" => "JOUR",
       "Model" => nil,
+      "OutputManagementPlan" => nil,
+      "PeerReview" => nil,
       "PhysicalObject" => nil,
+      "Preprint" => nil,
+      "Report" => "RRPT",
       "Service" => nil,
       "Software" => "COMP",
       "Sound" => "SOUND",
+      "Standard" => nil,
       "Text" => "RPRT",
       "Workflow" => nil,
       "Other" => nil
     }
 
+    RIS_TO_DC_TRANSLATIONS = {
+      "BLOG" => "Text",
+      "GEN" => "Text",
+      "CTLG" => "Collection",
+      "DATA" => "Dataset",
+      "FIGURE" => "Image",
+      "THES" => "Dissertation",
+      "MPCT" => "Audiovisual",
+      "JOUR" => "JournalArticle",
+      "COMP" => "Software",
+      "VIDEO" => "Audiovisual",
+      "ELEC" => "Text"
+    }
+
+    BIB_TO_DC_TRANSLATIONS = {
+      "article" => "JournalArticle",
+      "book" => "Book",
+      "inbook" => "BookChapter",
+      "inproceedings" => nil,
+      "manual" => nil,
+      "misc" => "Other",
+      "phdthesis" => "Dissertation",
+      "proceedings" => "ConferenceProceeding",
+      "techreport" => "Report",
+      "unpublished" => nil
+    }
+
+    CP_TO_DC_TRANSLATIONS = {
+      "song" => "Audiovisual",
+      "post-weblog" => "Text",
+      "dataset" => "Dataset",
+      "graphic" => "Image",
+      "motion_picture" => "Audiovisual",
+      "article-journal" => "JournalArticle",
+      "broadcast" => "Audiovisual",
+      "webpage" => "Text"
+    }
+
     SO_TO_BIB_TRANSLATIONS = {
       "Article" => "article",
       "AudioObject" => "misc",
@@ -322,11 +477,46 @@ module Bolognese
       "WebSite" => "misc"
     }
 
-    def find_from_format(id: nil, string: nil, ext: nil)
+    UNKNOWN_INFORMATION = {
+      ":unac" => "temporarily inaccessible",
+      ":unal" => "unallowed, suppressed intentionally",
+      ":unap" => "not applicable, makes no sense",
+      ":unas" => "value unassigned (e.g., Untitled)",
+      ":unav" => "value unavailable, possibly unknown",
+      ":unkn" => "known to be unknown (e.g., Anonymous, Inconnue)",
+      ":none" => "never had a value, never will",
+      ":null" => "explicitly and meaningfully empty",
+      ":tba" => "to be assigned or announced later",
+      ":etal" => "too numerous to list (et alia)"
+    }
+
+    RESOURCE_PATHS = {
+      spdx: 'spdx/licenses.json',
+      fos: 'oecd/fos-mappings.json',
+      for: 'oecd/for-mappings.json',
+      dfg: 'oecd/dfg-mappings.json'
+    }
+
+    def resources_dir_path
+      File.expand_path('../../../resources', __FILE__) + '/'
+    end
+
+    def resource_file( extra_path )
+      File.read(resources_dir_path + extra_path)
+    end
+
+    def resource_json( resource_symbol )
+      if RESOURCE_PATHS.keys().include?(resource_symbol)
+        JSON.load(resource_file(RESOURCE_PATHS[resource_symbol]))
+      end
+    end
+
+
+    def find_from_format(id: nil, string: nil, ext: nil, filename: nil)
       if id.present?
         find_from_format_by_id(id)
       elsif ext.present?
-        find_from_format_by_ext(string, ext: ext)
+        find_from_format_by_filename(filename) || find_from_format_by_ext(string, ext: ext)
       elsif string.present?
         find_from_format_by_string(string)
       else
@@ -337,11 +527,13 @@ module Bolognese
     def find_from_format_by_id(id)
       id = normalize_id(id)
 
-      if /\A(?:(http|https):\/(\/)?(dx\.)?(doi.org|handle.test.datacite.org)\/)?(doi:)?(10\.\d{4,5}\/.+)\z/.match(id)
+      if /\A(?:(http|https):\/(\/)?(dx\.)?(doi.org|handle.stage.datacite.org)\/)?(doi:)?(10\.\d{4,5}\/.+)\z/.match(id)
         ra = get_doi_ra(id)
         %w(DataCite Crossref mEDRA KISTI JaLC OP).include?(ra) ? ra.downcase : nil
       elsif /\A(?:(http|https):\/(\/)?orcid\.org\/)?(\d{4}-\d{4}-\d{4}-\d{3}[0-9X]+)\z/.match(id)
         "orcid"
+      elsif /\A(http|https):\/(\/)?github\.com\/(.+)\/package.json\z/.match(id)
+        "npm"
       elsif /\A(http|https):\/(\/)?github\.com\/(.+)\z/.match(id)
         "codemeta"
       else
@@ -349,6 +541,12 @@ module Bolognese
       end
     end
 
+    def find_from_format_by_filename(filename)
+      if filename == "package.json"
+        "npm"
+      end
+    end
+
     def find_from_format_by_ext(string, options={})
       if options[:ext] == ".bib"
         "bibtex"
@@ -356,7 +554,7 @@ module Bolognese
         "ris"
       elsif options[:ext] == ".xml" && Maremma.from_xml(string).to_h.dig("crossref_result", "query_result", "body", "query", "doi_record", "crossref")
         "crossref"
-      elsif options[:ext] == ".xml" && Nokogiri::XML(string, nil, 'UTF-8', &:noblanks).collect_namespaces.find { |k, v| v.start_with?("http://datacite.org/schema/kernel") }      
+      elsif options[:ext] == ".xml" && Nokogiri::XML(string, nil, 'UTF-8', &:noblanks).collect_namespaces.find { |k, v| v.start_with?("http://datacite.org/schema/kernel") }
         "datacite"
       elsif options[:ext] == ".json" && Maremma.from_json(string).to_h.dig("@context").to_s.start_with?("http://schema.org", "https://schema.org")
         "schema_org"
@@ -364,7 +562,7 @@ module Bolognese
         "codemeta"
       elsif options[:ext] == ".json" && Maremma.from_json(string).to_h.dig("schemaVersion").to_s.start_with?("http://datacite.org/schema/kernel")
         "datacite_json"
-      elsif options[:ext] == ".json" && Maremma.from_json(string).to_h.dig("types")
+      elsif options[:ext] == ".json" && Maremma.from_json(string).to_h.dig("types") && Maremma.from_json(string).to_h.dig("publication_year").present?
         "crosscite"
       elsif options[:ext] == ".json" && Maremma.from_json(string).to_h.dig("issued", "date-parts").present?
         "citeproc"
@@ -374,7 +572,7 @@ module Bolognese
     def find_from_format_by_string(string)
       if Maremma.from_xml(string).to_h.dig("crossref_result", "query_result", "body", "query", "doi_record", "crossref").present?
         "crossref"
-      elsif Nokogiri::XML(string, nil, 'UTF-8', &:noblanks).collect_namespaces.find { |k, v| v.start_with?("http://datacite.org/schema/kernel") }  
+      elsif Nokogiri::XML(string, nil, 'UTF-8', &:noblanks).collect_namespaces.find { |k, v| v.start_with?("http://datacite.org/schema/kernel") }
         "datacite"
       elsif Maremma.from_json(string).to_h.dig("@context").to_s.start_with?("http://schema.org", "https://schema.org")
         "schema_org"
@@ -382,7 +580,7 @@ module Bolognese
         "codemeta"
       elsif Maremma.from_json(string).to_h.dig("schema-version").to_s.start_with?("http://datacite.org/schema/kernel")
         "datacite_json"
-      elsif Maremma.from_json(string).to_h.dig("types").present?
+      elsif Maremma.from_json(string).to_h.dig("types").present? && Maremma.from_json(string).to_h.dig("publication_year").present?
         "crosscite"
       elsif Maremma.from_json(string).to_h.dig("issued", "date-parts").present?
         "citeproc"
@@ -404,10 +602,14 @@ module Bolognese
     end
 
     def validate_orcid(orcid)
-      orcid = Array(/\A(?:(http|https):\/\/(www\.)?orcid\.org\/)?(\d{4}[[:space:]-]\d{4}[[:space:]-]\d{4}[[:space:]-]\d{3}[0-9X]+)\z/.match(orcid)).last
+      orcid = Array(/\A(?:(?:http|https):\/\/(?:(?:www|sandbox)?\.)?orcid\.org\/)?(\d{4}[[:space:]-]\d{4}[[:space:]-]\d{4}[[:space:]-]\d{3}[0-9X]+)\/{0,1}\z/.match(orcid)).last
       orcid.gsub(/[[:space:]]/, "-") if orcid.present?
     end
 
+    def validate_ror(ror)
+      Array(/^(?:(?:(?:http|https):\/\/)?ror\.org\/)?(0\w{6}\d{2})\/{0,1}$/.match(ror)).last
+    end
+
     def validate_orcid_scheme(orcid_scheme)
       Array(/\A(http|https):\/\/(www\.)?(orcid\.org)/.match(orcid_scheme)).last
     end
@@ -430,7 +632,7 @@ module Bolognese
       elsif element.is_a?(Hash)
         element.fetch( CGI.unescapeHTML(content), nil)
       elsif element.is_a?(Array)
-        a = element.map { |e| e.is_a?(Hash) ? e.fetch( CGI.unescapeHTML(content), nil) : e }.uniq
+        a = element.map { |e| e.is_a?(Hash) ? e.fetch(CGI.unescapeHTML(content), nil) : e }.uniq
         a = options[:first] ? a.first : a.unwrap
       else
         nil
@@ -441,7 +643,7 @@ module Bolognese
       return nil unless id.present?
 
       # check for valid DOI
-      doi = normalize_doi(id, options)
+      doi = DoiUtils::normalize_doi(id, options)
       return doi if doi.present?
 
       # check for valid HTTP uri
@@ -454,7 +656,7 @@ module Bolognese
       nil
     end
 
-    def normalize_url(id)
+    def normalize_url(id, options={})
       return nil unless id.present?
 
       # handle info URIs
@@ -465,12 +667,22 @@ module Bolognese
 
       return nil unless uri && uri.host && %w(http https ftp).include?(uri.scheme)
 
+      # optionally turn into https URL
+      uri.scheme = "https" if options[:https]
+
       # clean up URL
-      PostRank::URI.clean(id)
+      uri.path = PostRank::URI.clean(uri.path)
+
+      uri.to_s
     rescue Addressable::URI::InvalidURIError
       nil
     end
 
+    def normalize_cc_url(id)
+      id = normalize_url(id, https: true)
+      NORMALIZED_LICENSES.fetch(id, id)
+    end
+
     def normalize_orcid(orcid)
       orcid = validate_orcid(orcid)
       return nil unless orcid.present?
@@ -479,11 +691,19 @@ module Bolognese
       "https://orcid.org/" + Addressable::URI.encode(orcid)
     end
 
+    def normalize_ror(ror)
+      ror = validate_ror(ror)
+      return nil unless ror.present?
+
+      # turn ROR into URL
+      "https://ror.org/" + Addressable::URI.encode(ror)
+    end
+
     def normalize_ids(ids: nil, relation_type: nil)
       Array.wrap(ids).select { |idx| idx["@id"].present? }.map do |idx|
         id = normalize_id(idx["@id"])
-        related_identifier_type = doi_from_url(id).present? ? "DOI" : "URL"
-        id = doi_from_url(id) || id
+        related_identifier_type = DoiUtils::doi_from_url(id).present? ? "DOI" : "URL"
+        id = DoiUtils::doi_from_url(id) || id
 
         { "relatedIdentifier" => id,
           "relationType" => relation_type,
@@ -555,6 +775,14 @@ module Bolognese
       nil
     end
 
+    def normalize_publisher(publisher)
+      if publisher.respond_to?(:to_hash)
+        publisher
+      elsif publisher.respond_to?(:to_str)
+        { "name" => publisher }
+      end
+    end
+
     def to_datacite_json(element, options={})
       a = Array.wrap(element).map do |e|
         e.inject({}) {|h, (k,v)| h[k.dasherize] = v; h }
@@ -579,15 +807,15 @@ module Bolognese
         c["affiliation"] = Array.wrap(c["affiliation"]).map do |a|
           if a.is_a?(String)
             name = a
-            id = nil
+            affiliation_identifier = nil
           else
             name = a["name"]
-            id = a["id"]
+            affiliation_identifier = a["affiliationIdentifier"]
           end
 
-          { 
-            "@type" => "Organization", 
-            "@id" => id,
+          {
+            "@type" => "Organization",
+            "@id" => affiliation_identifier,
             "name" => name }.compact
         end.unwrap
         c["@type"] = c["nameType"].present? ? c["nameType"][0..-3] : nil
@@ -599,31 +827,32 @@ module Bolognese
 
     def to_schema_org_contributors(element)
       element = Array.wrap(element).map do |c|
-        c["affiliation"] = Array.wrap(c["affiliation"]).map do |a|
+        transformed_c = c.dup
+        transformed_c["affiliation"] = Array.wrap(c["affiliation"]).map do |a|
           if a.is_a?(String)
             name = a
-            id = nil
+            affiliation_identifier = nil
           else
             name = a["name"]
-            id = a["id"]
+            affiliation_identifier = a["affiliationIdentifier"]
           end
 
-          { 
-            "@type" => "Organization", 
-            "@id" => id,
+          {
+            "@type" => "Organization",
+            "@id" => affiliation_identifier,
             "name" => name }.compact
         end.unwrap
-        c["@type"] = c["nameType"].present? ? c["nameType"][0..-3] : nil
-        c["@id"] = Array.wrap(c["nameIdentifiers"]).first.to_h.fetch("nameIdentifier", nil)
-        c["name"] = c["familyName"].present? ? [c["givenName"], c["familyName"]].join(" ") : c["name"]
-        c.except("nameIdentifiers", "nameType").compact
+        transformed_c["@type"] = c["nameType"].present? ? c["nameType"][0..-3] : nil
+        transformed_c["@id"] = Array.wrap(c["nameIdentifiers"]).first.to_h.fetch("nameIdentifier", nil)
+        transformed_c["name"] = c["familyName"].present? ? [c["givenName"], c["familyName"]].join(" ") : c["name"]
+        transformed_c.except("nameIdentifiers", "nameType").compact
       end.unwrap
     end
 
     def to_schema_org_container(element, options={})
       return nil unless (element.is_a?(Hash) || (element.nil? && options[:container_title].present?))
 
-      { 
+      {
         "@id" => element["identifier"],
         "@type" => (options[:type] == "Dataset") ? "DataCatalog" : "Periodical",
         "name" => element["title"] || options[:container_title] }.compact
@@ -631,7 +860,7 @@ module Bolognese
 
     def to_schema_org_identifiers(element, options={})
       Array.wrap(element).map do |ai|
-        { 
+        {
           "@type" => "PropertyValue",
           "propertyID" => ai["identifierType"],
           "value" => ai["identifier"] }
@@ -641,7 +870,7 @@ module Bolognese
     def to_schema_org_relation(related_identifiers: nil, relation_type: nil)
       return nil unless related_identifiers.present? && relation_type.present?
 
-      relation_type = relation_type == "References" ? ["References", "Cites", "Documents"] : [relation_type] 
+      relation_type = relation_type == "References" ? ["References", "Cites", "Documents"] : [relation_type]
 
       Array.wrap(related_identifiers).select { |ri| relation_type.include?(ri["relationType"]) }.map do |r|
         if r["relatedIdentifierType"] == "ISSN" && r["relationType"] == "IsPartOf"
@@ -672,7 +901,7 @@ module Bolognese
 
       Array.wrap(geo_location).reduce([]) do |sum, gl|
         if gl.fetch("geoLocationPoint", nil)
-          sum << { 
+          sum << {
             "@type" => "Place",
             "geo" => {
               "@type" => "GeoCoordinates",
@@ -683,7 +912,7 @@ module Bolognese
         end
 
         if gl.fetch("geoLocationBox", nil)
-          sum << { 
+          sum << {
             "@type" => "Place",
             "geo" => {
               "@type" => "GeoShape",
@@ -696,19 +925,24 @@ module Bolognese
         end
 
         if gl.fetch("geoLocationPolygon", nil)
-          sum << { 
+          sum << {
             "@type" => "Place",
             "geo" => {
               "@type" => "GeoShape",
               "address" => gl["geoLocationPlace"],
-              "polygon" => Array.wrap(gl.dig("geoLocationPolygon")).map do |glp| 
-                [glp.dig("polygonPoint", "pointLongitude"), glp.dig("polygonPoint", "pointLatitude")].compact
-              end.compact }
+              "polygon" => Array.wrap(gl.dig("geoLocationPolygon")).map do |glp|
+                Array.wrap(glp).map do |glpp|
+                  if glpp.dig("polygonPoint")
+                    [glpp.dig("polygonPoint", "pointLongitude"), glpp.dig("polygonPoint", "pointLatitude")].compact
+                  end
+                end.compact.presence
+              end.compact.presence,
+              }
           }
         end
 
         if gl.fetch("geoLocationPlace", nil) && !gl.fetch("geoLocationPoint", nil) && !gl.fetch("geoLocationBox", nil) && !gl.fetch("geoLocationPolygon", nil)
-          sum << { 
+          sum << {
             "@type" => "Place",
             "geo" => {
               "@type" => "GeoCoordinates",
@@ -728,18 +962,50 @@ module Bolognese
 
     def from_schema_org_creators(element)
       element = Array.wrap(element).map do |c|
+        if c["affiliation"].is_a?(String)
+          c["affiliation"] = { "name" => c["affiliation"] }
+          affiliation_identifier_scheme = nil
+          scheme_uri = nil
+        elsif c.dig("affiliation", "@id").to_s.starts_with?("https://ror.org")
+          affiliation_identifier_scheme = "ROR"
+          scheme_uri = "https://ror.org/"
+        elsif c.dig("affiliation", "@id").to_s.starts_with?("https://isni.org")
+          affiliation_identifier_scheme = "ISNI"
+          scheme_uri = "https://isni.org/isni/"
+        else
+          affiliation_identifier_scheme = nil
+          scheme_uri = nil
+        end
+
         c["nameIdentifier"] = [{ "__content__" => c["@id"], "nameIdentifierScheme" => "ORCID", "schemeUri" => "https://orcid.org" }] if normalize_orcid(c["@id"])
         c["@type"] = c["@type"].find { |t| %w(Person Organization).include?(t) } if c["@type"].is_a?(Array)
         c["creatorName"] = { "nameType" => c["@type"].present? ? c["@type"].titleize + "al" : nil, "__content__" => c["name"] }.compact
-        c.except("@id", "@type", "name") 
+        c["affiliation"] = { "__content__" => c.dig("affiliation", "name"), "affiliationIdentifier" => c.dig("affiliation", "@id"), "affiliationIdentifierScheme" => affiliation_identifier_scheme, "schemeUri" => scheme_uri }.compact.presence
+        c.except("@id", "@type", "name").compact
       end
     end
 
     def from_schema_org_contributors(element)
       element = Array.wrap(element).map do |c|
+        if c["affiliation"].is_a?(String)
+          c["affiliation"] = { "name" => c["affiliation"] }
+          affiliation_identifier_scheme = nil
+          scheme_uri = nil
+        elsif c.dig("affiliation", "@id").to_s.starts_with?("https://ror.org")
+          affiliation_identifier_scheme = "ROR"
+          scheme_uri = "https://ror.org/"
+        elsif c.dig("affiliation", "@id").to_s.starts_with?("https://isni.org")
+          affiliation_identifier_scheme = "ISNI"
+          scheme_uri = "https://isni.org/isni/"
+        else
+          affiliation_identifier_scheme = nil
+          scheme_uri = nil
+        end
+
         c["nameIdentifier"] = [{ "__content__" => c["@id"], "nameIdentifierScheme" => "ORCID", "schemeUri" => "https://orcid.org" }] if normalize_orcid(c["@id"])
         c["contributorName"] = { "nameType" => c["@type"].present? ? c["@type"].titleize + "al" : nil, "__content__" => c["name"] }.compact
-        c.except("@id", "@type", "name") 
+        c["affiliation"] = { "__content__" => c.dig("affiliation", "name"), "affiliationIdentifier" => c.dig("affiliation", "@id"), "affiliationIdentifierScheme" => affiliation_identifier_scheme, "schemeUri" => scheme_uri }.compact.presence
+        c.except("@id", "@type", "name").compact
       end
     end
 
@@ -768,7 +1034,7 @@ module Bolognese
       Array.wrap(element).map do |a|
         if a["literal"].present?
           a["@type"] = "Organization"
-          a["name"] = a["literal"]
+          a["creatorName"] = a["literal"]
         else
           a["@type"] = "Person"
           a["name"] = [a["given"], a["family"]].compact.join(" ")
@@ -804,12 +1070,16 @@ module Bolognese
       custom_scrubber = Bolognese::WhitelistScrubber.new(options)
 
       if text.is_a?(String)
-        # remove excessive internal whitespace with squish
-        Loofah.scrub_fragment(text, custom_scrubber).to_s.squish
+        if options[:new_line]
+          # Remove multiple spaces, tabs, and other whitespace characters while preserving single spaces and new lines
+          Loofah.scrub_fragment(text, custom_scrubber).to_s.gsub(/[ \t]+/, ' ').strip
+        else
+          Loofah.scrub_fragment(text, custom_scrubber).to_s.squish
+        end
       elsif text.is_a?(Hash)
-        sanitize(text.fetch(content, nil))
+        sanitize(text.fetch(content, nil), new_line: options[:new_line])
       elsif text.is_a?(Array)
-        a = text.map { |e| e.is_a?(Hash) ? sanitize(e.fetch(content, nil)) : sanitize(e) }.uniq
+        a = text.map { |e| e.is_a?(Hash) ? sanitize(e.fetch(content, nil), new_line: options[:new_line]) : sanitize(e, new_line: options[:new_line]) }.uniq
         a = options[:first] ? a.first : a.unwrap
       else
         nil
@@ -871,6 +1141,8 @@ module Bolognese
       month = iso8601_time[5..6].to_i
       day = iso8601_time[8..9].to_i
       { 'date-parts' => [[year, month, day].reject { |part| part == 0 }] }
+    rescue TypeError
+      nil
     end
 
     def get_date_from_date_parts(date_as_parts)
@@ -889,7 +1161,7 @@ module Bolognese
 
     def get_iso8601_date(iso8601_time)
       return nil if iso8601_time.nil?
-      
+
       iso8601_time[0..9]
     end
 
@@ -921,6 +1193,14 @@ module Bolognese
       nil
     end
 
+    # iso8601 datetime without hyphens and colons, used by Crossref
+    # return nil if invalid
+    def get_datetime_from_time(time)
+      DateTime.strptime(time.to_s, "%Y%m%d%H%M%S").strftime('%Y-%m-%dT%H:%M:%SZ')
+    rescue ArgumentError
+      nil
+    end
+
     def get_date(dates, date_type)
       dd = Array.wrap(dates).find { |d| d["dateType"] == date_type } || {}
       dd.fetch("date", nil)
@@ -959,7 +1239,9 @@ module Bolognese
         "urn" => "URN",
         "md5" => "md5",
         "minid" => "minid",
-        "dataguid" => "dataguid"
+        "dataguid" => "dataguid",
+        "cstr" => "CSTR",
+        "rrid" => "RRID"
       }
 
       identifierTypes[identifier_type.downcase] || identifier_type
@@ -978,7 +1260,7 @@ module Bolognese
       first_page = pages.present? ? pages.split("-").map(&:strip)[0] : nil
       last_page = pages.present? ? pages.split("-").map(&:strip)[1] : nil
 
-      { 
+      {
         "title" => title,
         "volume" => volume,
         "issue" => issue,
@@ -995,5 +1277,162 @@ module Bolognese
       error_array
     end
 
+    def name_to_spdx(name)
+      spdx = resource_json(:spdx).fetch("licenses")
+      license = spdx.find { |l| l["name"] == name || l["licenseId"] == name || l["seeAlso"].first == normalize_cc_url(name) }
+
+      if license
+        {
+          "rights" => license["name"],
+          "rightsUri" => license["seeAlso"].first,
+          "rightsIdentifier" => license["licenseId"].downcase,
+          "rightsIdentifierScheme" => "SPDX",
+          "schemeUri" => "https://spdx.org/licenses/" }.compact
+      else
+        { "rights" => name }
+      end
+    end
+
+    def hsh_to_spdx(hsh)
+      spdx = resource_json(:spdx).fetch("licenses")
+      license = spdx.find { |l| l["licenseId"].casecmp?(hsh["rightsIdentifier"]) || l["seeAlso"].first == normalize_cc_url(hsh["rightsURI"]) || l["name"] == hsh["rights"] || l["seeAlso"].first == normalize_cc_url(hsh["rights"]) }
+
+      if license
+        {
+          "rights" => license["name"],
+          "rightsUri" => license["seeAlso"].first,
+          "rightsIdentifier" => license["licenseId"].downcase,
+          "rightsIdentifierScheme" => "SPDX",
+          "schemeUri" => "https://spdx.org/licenses/",
+          "lang" => hsh["lang"] }.compact
+      else
+        {
+          "rights" => hsh["__content__"] || hsh["rights"],
+          "rightsUri" => hsh["rightsURI"] || hsh["rightsUri"],
+          "rightsIdentifier" => hsh["rightsIdentifier"].present? ? hsh["rightsIdentifier"].downcase : nil,
+          "rightsIdentifierScheme" => hsh["rightsIdentifierScheme"],
+          "schemeUri" => hsh["schemeUri"],
+          "lang" => hsh["lang"] }.compact
+      end
+    end
+
+    def name_to_fos(name)
+      # first find subject in Fields of Science (OECD)
+      fos = resource_json(:fos).fetch("fosFields")
+
+      subject = fos.find { |l| l["fosLabel"] == name || "FOS: " + l["fosLabel"] == name }
+
+      if subject
+        return [{
+          "subject" => sanitize(name) },
+        {
+          "subject" => "FOS: " + subject["fosLabel"],
+          "subjectScheme" => "Fields of Science and Technology (FOS)",
+          "schemeUri" => "http://www.oecd.org/science/inno/38235147.pdf"
+        }]
+      end
+
+      # if not found, look in Fields of Research (Australian and New Zealand Standard Research Classification)
+      # and map to Fields of Science. Add an extra entry for the latter
+      fores = resource_json(:for)
+      for_fields = fores.fetch("forFields")
+      for_disciplines = fores.fetch("forDisciplines")
+
+      subject = for_fields.find { |l| l["forLabel"] == name } ||
+                for_disciplines.find { |l| l["forLabel"] == name }
+
+      if subject
+        [{
+          "subject" => sanitize(name) },
+        {
+          "subject" => "FOS: " + subject["fosLabel"],
+          "subjectScheme" => "Fields of Science and Technology (FOS)",
+          "schemeUri" => "http://www.oecd.org/science/inno/38235147.pdf"
+        }]
+      else
+        [{ "subject" => sanitize(name) }]
+      end
+    end
+
+    def hsh_to_fos(hsh)
+      # first find subject in Fields of Science (OECD)
+      fos = resource_json(:fos).fetch("fosFields")
+      subject = fos.find { |l| l["fosLabel"] == hsh["__content__"] || "FOS: " + l["fosLabel"] == hsh["__content__"] || l["fosLabel"] == hsh["subject"]}
+
+      if subject
+        return [{
+          "subject" => sanitize(hsh["__content__"] || hsh["subject"]),
+          "subjectScheme" => hsh["subjectScheme"],
+          "schemeUri" => hsh["schemeURI"] || hsh["schemeUri"],
+          "valueUri" => hsh["valueURI"] || hsh["valueUri"],
+          "classificationCode" => hsh["classificationCode"],
+          "lang" => hsh["lang"] }.compact,
+        {
+          "subject" => "FOS: " + subject["fosLabel"],
+          "subjectScheme" => "Fields of Science and Technology (FOS)",
+          "schemeUri" => "http://www.oecd.org/science/inno/38235147.pdf" }.compact]
+      end
+
+      # if not found, look in Fields of Research (Australian and New Zealand Standard Research Classification)
+      # and map to Fields of Science. Add an extra entry for the latter
+      fores = resource_json(:for)
+      for_fields = fores.fetch("forFields")
+      for_disciplines = fores.fetch("forDisciplines")
+
+      # try to extract forId
+      if hsh["subjectScheme"] == "FOR"
+        for_id = hsh["__content__"].to_s.split(" ").first || hsh["subject"].to_s.split(" ").first
+        for_id = for_id.rjust(6, "0")
+
+        subject = for_fields.find { |l| l["forId"] == for_id } ||
+                  for_disciplines.find { |l| l["forId"] == for_id[0..3] }
+      else
+        subject = for_fields.find { |l| l["forLabel"] == hsh["__content__"] || l["forLabel"] == hsh["subject"] } ||
+                  for_disciplines.find { |l| l["forLabel"] == hsh["__content__"] || l["forLabel"] == hsh["subject"] }
+      end
+
+      if subject
+        [{
+          "subject" => sanitize(hsh["__content__"] || hsh["subject"]),
+          "subjectScheme" => hsh["subjectScheme"],
+          "classificationCode" => hsh["classificationCode"],
+          "schemeUri" => hsh["schemeURI"] || hsh["schemeUri"],
+          "valueUri" => hsh["valueURI"] || hsh["valueUri"],
+          "lang" => hsh["lang"] }.compact,
+        {
+          "subject" => "FOS: " + subject["fosLabel"],
+          "subjectScheme" => "Fields of Science and Technology (FOS)",
+          "schemeUri" => "http://www.oecd.org/science/inno/38235147.pdf"
+        }]
+      else
+        [{
+          "subject" => sanitize(hsh["__content__"] || hsh["subject"]),
+          "subjectScheme" => hsh["subjectScheme"],
+          "classificationCode" => hsh["classificationCode"],
+          "schemeUri" => hsh["schemeURI"] || hsh["schemeUri"],
+          "valueUri" => hsh["valueURI"] || hsh["valueUri"],
+          "lang" => hsh["lang"] }.compact]
+      end
+    end
+
+    def dfg_ids_to_fos(dfg_ids)
+      dfgs = resource_json(:dfg).fetch("dfgFields")
+      ids = Array.wrap(dfg_ids)
+
+      subjects = dfgs.select { |l| ids.include?(l["dfgId"])}
+      subjects.map do |subject|
+        {
+          "classificationCode" => subject["fosId"],
+          "subject" =>  subject["fosLabel"],
+          "subjectScheme" => "Fields of Science and Technology (FOS)",
+          "schemeUri" => "http://www.oecd.org/science/inno/38235147.pdf"
+        }
+      end
+    end
+
+    def abstract_description
+      # Fetch the first description with descriptionType "Abstract"
+      Array.wrap(descriptions)&.find { |d| d["descriptionType"] == "Abstract" }
+    end
   end
 end