commonmeta-ruby 3.0.0

data/lib/commonmeta/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Commonmeta
+  VERSION = '3.0.0'
+end

data/lib/commonmeta/whitelist_scrubber.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# modified from https://gist.github.com/ivan-kolmychek/ee2fdc53f3e2c637271d
+
+module Commonmeta
+  class WhitelistScrubber < Loofah::Scrubber
+    def initialize(options = {})
+      @direction = :bottom_up
+      @tags = options[:tags]
+      @attributes = options[:attributes]
+    end
+
+    def scrub(node)
+      scrub_node_attributes(node) and return CONTINUE if node_allowed?(node)
+
+      node.before node.children
+      node.remove
+    end
+
+    private
+
+    def scrub_node_attributes(node)
+      unless @attributes.present? && @attributes.respond_to?(:include?)
+        fallback_scrub_node_attributes(node) and return true
+      end
+
+      node.attribute_nodes.each do |attr_node|
+        attr_node.remove unless @attributes.include?(attr_node.name)
+      end
+    end
+
+    def allowed_not_element_node_types
+      [Nokogiri::XML::Node::TEXT_NODE, Nokogiri::XML::Node::CDATA_SECTION_NODE]
+    end
+
+    def fallback_scrub_node_attributes(node)
+      Loofah::HTML5::Scrub.scrub_attributes(node)
+    end
+
+    def fallback_allowed_element_detection(node)
+      Loofah::HTML5::Scrub.allowed_element?(node.name)
+    end
+
+    def node_allowed?(node)
+      unless @tags.present? && @tags.respond_to?(:include?)
+        return fallback_allowed_element_detection(node)
+      end
+      return true if allowed_not_element_node_types.include?(node.type)
+      return false unless node.type == Nokogiri::XML::Node::ELEMENT_NODE
+
+      @tags.include? node.name
+    end
+  end
+end

data/lib/commonmeta/writers/bibtex_writer.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Commonmeta
+  module Writers
+    module BibtexWriter
+      def bibtex
+        pages = if container.to_h['firstPage'].present?
+                  [container['firstPage'], container['lastPage']].compact.join('-')
+                end
+
+        bib = {
+          bibtex_type: Commonmeta::Utils::CM_TO_BIB_TRANSLATIONS.fetch(type, 'misc'),
+          bibtex_key: normalize_id(id),
+          doi: doi_from_url(id),
+          url: url,
+          author: authors_as_string(creators),
+          keywords: if subjects.present?
+                      Array.wrap(subjects).map do |k|
+                        parse_attributes(k, content: 'subject', first: true)
+                      end.join(', ')
+                    end,
+          language: language,
+          title: parse_attributes(titles, content: 'title', first: true),
+          journal: container && container['title'],
+          volume: container.to_h['volume'],
+          issue: container.to_h['issue'],
+          pages: pages,
+          publisher: publisher.to_h['name'],
+          year: date.to_h['published'] && date['published'].split('-').first,
+          copyright: license.to_h['id']
+        }.compact
+        BibTeX::Entry.new(bib).to_s
+      end
+    end
+  end
+end

data/lib/commonmeta/writers/cff_writer.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Commonmeta
+  module Writers
+    module CffWriter
+      def cff
+        # return nil unless valid? || show_errors
+
+        # only use CFF for software
+        return nil unless %w[Software Collection].include?(type)
+
+        title = parse_attributes(titles, content: 'title', first: true)
+
+        hsh = {
+          'cff-version' => '1.2.0',
+          'message' => "If you use #{title} in your research, please cite it using the following metadata",
+          'doi' => normalize_doi(id),
+          'repository-code' => url,
+          'title' => title,
+          'authors' => write_cff_creators(creators),
+          'abstract' => parse_attributes(descriptions, content: 'description', first: true),
+          'version' => version,
+          'keywords' => if subjects.present?
+                          Array.wrap(subjects).map do |k|
+                            parse_attributes(k, content: 'subject', first: true)
+                          end
+                        end,
+          'date-released' => date['published'],
+          'license' => license.to_h['id'],
+          'references' => { 'identifiers' => Array.wrap(references).map do |r|
+                                               write_cff_reference(r)
+                                             end }
+        }.compact
+        hsh.to_yaml
+      end
+
+      def write_cff_creators(creators)
+        Array.wrap(creators).map do |a|
+          if a['givenName'].present? || a['id'].present?
+            { 'given-names' => a['givenName'],
+              'family-names' => a['familyName'],
+              'orcid' => a['id'],
+              'affiliation' => parse_attributes(a['affiliation'], content: 'name',
+                                                                  first: true) }.compact
+          else
+            { 'name' => a['name'] }
+          end
+        end
+      end
+
+      def write_cff_reference(reference)
+        return nil if reference.blank?
+
+        url = reference['url']
+        doi = reference['doi']
+        value = doi.present? ? doi_from_url(doi) : url
+        type = doi.present? ? 'doi' : 'url'
+
+        { 'type' => type, 'value' => value }.compact
+      end
+    end
+  end
+end

data/lib/commonmeta/writers/citation_writer.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Commonmeta
+  module Writers
+    module CitationWriter
+      def citation
+        cp = CiteProc::Processor.new(style: style, locale: locale, format: 'html')
+        cp.import Array.wrap(csl_hsh)
+        bibliography = cp.render :bibliography, id: normalize_doi(id)
+        bibliography.first
+      end
+    end
+  end
+end

data/lib/commonmeta/writers/codemeta_writer.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Commonmeta
+  module Writers
+    module CodemetaWriter
+      def codemeta
+        return nil unless valid? || show_errors
+
+        hsh = {
+          '@context' => id.present? ? 'https://raw.githubusercontent.com/codemeta/codemeta/master/codemeta.jsonld' : nil,
+          '@type' => Commonmeta::Utils::CM_TO_SO_TRANSLATIONS.fetch(type, 'SoftwareSourceCode'),
+          '@id' => normalize_id(id),
+          'identifier' => to_schema_org_identifiers(alternate_identifiers),
+          'codeRepository' => url,
+          'name' => parse_attributes(titles, content: 'title', first: true),
+          'authors' => creators,
+          'description' => parse_attributes(descriptions, content: 'description', first: true),
+          'version' => version,
+          'tags' => if subjects.present?
+                      Array.wrap(subjects).map do |k|
+                        parse_attributes(k, content: 'subject', first: true)
+                      end
+                    end,
+          'datePublished' => date['published'],
+          'dateModified' => date['updated'],
+          'publisher' => publisher['name'],
+          'license' => license.to_h['id']
+        }.compact
+        JSON.pretty_generate hsh.presence
+      end
+    end
+  end
+end

data/lib/commonmeta/writers/crossref_xml_writer.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Commonmeta
+  module Writers
+    module CrossrefXmlWriter
+      def crossref_xml
+        should_passthru ? raw : write_crossref_xml
+      end
+    end
+  end
+end

data/lib/commonmeta/writers/csl_writer.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Commonmeta
+  module Writers
+    module CslWriter
+      def csl
+        JSON.pretty_generate csl_hsh.presence
+      end
+    end
+  end
+end

data/lib/commonmeta/writers/csv_writer.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Commonmeta
+  module Writers
+    module CsvWriter
+      require 'csv'
+
+      def csv
+        return nil unless valid?
+
+        bib = {
+          doi: doi_from_url(id),
+          url: url,
+          registered: date['published'],
+          state: state,
+          type: Commonmeta::Utils::CM_TO_BIB_TRANSLATIONS.fetch(type, 'misc'),
+          title: parse_attributes(titles, content: 'title', first: true),
+          author: authors_as_string(creators),
+          publisher: publisher['name']
+        }.values
+
+        CSV.generate { |csv| csv << bib }
+      end
+    end
+  end
+end

data/lib/commonmeta/writers/datacite_writer.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Commonmeta
+  module Writers
+    module DataciteWriter
+      def datacite
+        resource_type_general = Commonmeta::Utils::CM_TO_DC_TRANSLATIONS.fetch(type, 'Other')
+        types = { 'resourceTypeGeneral' => resource_type_general,
+                  'resourceType' => type == resource_type_general ? nil : type,
+                  'bibtex' => Commonmeta::Utils::CM_TO_BIB_TRANSLATIONS.fetch(type, 'misc'),
+                  'citeproc' => Commonmeta::Utils::CM_TO_CSL_TRANSLATIONS.fetch(type, 'article'),
+                  'ris' => Commonmeta::Utils::CM_TO_RIS_TRANSLATIONS.fetch(type, 'GEN'),
+                  'schemaOrg' => Commonmeta::Utils::CM_TO_SO_TRANSLATIONS.fetch(type,
+                                                                                'CreativeWork') }.compact
+
+        dates = get_dates_from_date(date)
+        rights_list = spdx_to_hsh(license)
+        related_identifiers = Array.wrap(references).map { |r| datacite_reference(r) }
+
+        hsh = {
+          'id' => id,
+          'doi' => doi_from_url(id),
+          'url' => url,
+          'types' => types,
+          'creators' => Array.wrap(creators).map { |c| datacite_contributor(c) },
+          'titles' => titles,
+          'publisher' => publisher.to_h['name'],
+          'container' => container,
+          'subjects' => subjects,
+          'contributors' => Array.wrap(contributors).map { |c| datacite_contributor(c) },
+          'dates' => dates,
+          'language' => language,
+          'alternate_identifiers' => alternate_identifiers,
+          'sizes' => sizes,
+          'formats' => formats,
+          'version' => version,
+          'rights_list' => rights_list,
+          'descriptions' => descriptions,
+          'geo_locations' => geo_locations,
+          'funding_references' => funding_references,
+          'related_identifiers' => related_identifiers,
+          'schema_version' => schema_version,
+          'provider_id' => provider_id,
+          'client_id' => client_id,
+          'agency' => provider,
+          'state' => state
+        }.compact
+
+        JSON.pretty_generate hsh.transform_keys! { |key|
+          key.camelcase(uppercase_first_letter = :lower)
+        }
+      end
+
+      def datacite_contributor(contributor)
+        type = contributor.fetch('type', nil)
+        type += 'al' if type.present?
+
+        if type == 'Personal'
+          contributor['name'] = [contributor['familyName'], contributor['givenName']].join(', ')
+        end
+        contributor['nameIdentifiers'] = author_name_identifiers(contributor['id'])
+
+        { 'name' => contributor.fetch('name', nil),
+          'nameType' => type,
+          'givenName' => contributor.fetch('givenName', nil),
+          'familyName' => contributor.fetch('familyName', nil),
+          'nameIdentifiers' => contributor.fetch('nameIdentifiers', nil).presence,
+          'affiliation' => contributor.fetch('affiliation', nil).presence,
+          'contributorType' => contributor.fetch('contributorType', nil) }.compact
+      end
+
+      def datacite_reference(reference)
+        return nil unless reference.present? || !reference.is_a?(Hash)
+
+        key = reference['relatedIdentifier']
+        doi = nil
+        url = nil
+
+        if key.present?
+          if key.start_with?('http')
+            url = key
+          else
+            doi = key
+          end
+        end
+
+        { 'relatedIdentifier' => doi,
+          'relatedIdentifierType' => 'DOI',
+          'relationType' => 'References',
+          'relatedMetadataScheme' => 'citeproc+json' }.compact
+      end
+    end
+  end
+end

data/lib/commonmeta/writers/jats_writer.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module Commonmeta
+  module Writers
+    module JatsWriter
+      def jats
+        @jats ||= Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml|
+          xml.send(:'element-citation', publication_type) do
+            insert_citation(xml)
+          end
+        end.to_xml
+      end
+
+      def insert_citation(xml)
+        insert_authors(xml)
+        insert_editors(xml)
+        insert_citation_title(xml) if is_article? || is_data? || is_chapter?
+        insert_source(xml)
+        insert_publisher_name(xml) if publisher['name'].present? && !is_data?
+        insert_publication_date(xml)
+        insert_volume(xml) if container.to_h['volume'].present?
+        insert_issue(xml) if container.to_h['issue'].present?
+        insert_fpage(xml) if container.to_h['firstPage'].present?
+        insert_lpage(xml) if container.to_h['lastPage'].present?
+        insert_version(xml) if version.present?
+        insert_pub_id(xml)
+      end
+
+      def is_article?
+        publication_type.fetch('publication-type', nil) == 'journal'
+      end
+
+      def is_data?
+        publication_type.fetch('publication-type', nil) == 'data'
+      end
+
+      def is_chapter?
+        publication_type.fetch('publication-type', nil) == 'chapter'
+      end
+
+      def insert_authors(xml)
+        return unless creators.present?
+
+        xml.send(:'person-group', 'person-group-type' => 'author') do
+          Array.wrap(creators).each do |au|
+            xml.name do
+              insert_contributor(xml, au)
+            end
+          end
+        end
+      end
+
+      def insert_editors(xml)
+        return unless contributors.present?
+
+        xml.send(:'person-group', 'person-group-type' => 'editor') do
+          Array.wrap(contributors).each do |con|
+            xml.name do
+              insert_contributor(xml, con)
+            end
+          end
+        end
+      end
+
+      def insert_contributor(xml, person)
+        xml.surname(person['familyName']) if person['familyName'].present?
+        xml.send(:'given-names', person['givenName']) if person['givenName'].present?
+      end
+
+      def insert_citation_title(xml)
+        case publication_type.fetch('publication-type', nil)
+        when 'data' then xml.send(:'data-title',
+                                  parse_attributes(titles, content: 'title', first: true))
+        when 'journal' then xml.send(:'article-title',
+                                     parse_attributes(titles, content: 'title', first: true))
+        when 'chapter' then xml.send(:'chapter-title',
+                                     parse_attributes(titles, content: 'title', first: true))
+        end
+      end
+
+      def insert_source(xml)
+        if is_chapter?
+          xml.source(publisher['name'])
+        elsif is_article? || is_data?
+          xml.source((container && container['title']) || publisher['name'])
+        else
+          xml.source(parse_attributes(titles, content: 'title', first: true))
+        end
+      end
+
+      def insert_publisher_name(xml)
+        xml.send(:'publisher-name', publisher['name'])
+      end
+
+      def insert_publication_date(xml)
+        year, month, day = get_date_parts(date['published']).to_h.fetch('date-parts', []).first
+
+        xml.year(year, 'iso-8601-date' => date['published'])
+        xml.month(month.to_s.rjust(2, '0')) if month.present?
+        xml.day(day.to_s.rjust(2, '0')) if day.present?
+      end
+
+      def insert_volume(xml)
+        xml.volume(container['volume'])
+      end
+
+      def insert_issue(xml)
+        xml.issue(container['issue'])
+      end
+
+      def insert_fpage(xml)
+        xml.fpage(container['firstPage'])
+      end
+
+      def insert_lpage(xml)
+        xml.lpage(container['lastPage'])
+      end
+
+      def insert_version(xml)
+        xml.version(version)
+      end
+
+      def insert_pub_id(xml)
+        return nil unless doi_from_url(id).present?
+
+        xml.send(:'pub-id', doi_from_url(id), 'pub-id-type' => 'doi')
+      end
+
+      def date
+        get_date_parts(date['published'])
+      end
+
+      def publication_type
+        { 'publication-type' => Commonmeta::Utils::CM_TO_JATS_TRANSLATIONS.fetch(type, nil) }
+      end
+    end
+  end
+end

data/lib/commonmeta/writers/rdf_xml_writer.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Commonmeta
+  module Writers
+    module RdfXmlWriter
+      def rdf_xml
+        graph.dump(:rdfxml, prefixes: { schema: 'http://schema.org/' })
+      end
+    end
+  end
+end

data/lib/commonmeta/writers/ris_writer.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Commonmeta
+  module Writers
+    module RisWriter
+      def ris
+        sn = container.to_h['identifier']
+        sn = sn.downcase if sn.present? && container.to_h['identifierType'] == 'DOI'
+        {
+          'TY' => Commonmeta::Utils::CM_TO_RIS_TRANSLATIONS.fetch(type, 'GEN'),
+          'T1' => parse_attributes(titles, content: 'title', first: true),
+          'T2' => container.to_h['title'],
+          'AU' => to_ris(creators),
+          'DO' => doi_from_url(id),
+          'UR' => url,
+          'AB' => parse_attributes(descriptions, content: 'description', first: true),
+          'KW' => Array.wrap(subjects).map do |k|
+            parse_attributes(k, content: 'subject', first: true)
+          end.presence,
+          'PY' => date['published'] && date['published'].split('-').first,
+          'PB' => publisher['name'],
+          'LA' => language,
+          'VL' => container.to_h['volume'],
+          'IS' => container.to_h['issue'],
+          'SP' => container.to_h['firstPage'],
+          'EP' => container.to_h['lastPage'],
+          'SN' => sn,
+          'ER' => ''
+        }.compact.map do |k, v|
+          if v.is_a?(Array)
+            v.map do |vi|
+              "#{k}  - #{vi}"
+            end.join("\r\n")
+          else
+            "#{k}  - #{v}"
+          end
+        end.join("\r\n")
+      end
+    end
+  end
+end

data/lib/commonmeta/writers/schema_org_writer.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Commonmeta
+  module Writers
+    module SchemaOrgWriter
+      def schema_hsh
+        { '@context' => 'http://schema.org',
+          '@type' => Commonmeta::Utils::CM_TO_SO_TRANSLATIONS.fetch(type, 'CreativeWork'),
+          '@id' => id,
+          'identifier' => to_schema_org_identifiers(alternate_identifiers),
+          'url' => url,
+          'additionalType' => additional_type,
+          'name' => parse_attributes(titles, content: 'title', first: true),
+          'author' => to_schema_org(creators),
+          'editor' => to_schema_org(contributors),
+          'description' => parse_attributes(descriptions, content: 'description', first: true),
+          'license' => license.to_h['url'],
+          'version' => version,
+          'keywords' => if subjects.present?
+                          Array.wrap(subjects).map do |k|
+                            parse_attributes(k, content: 'subject', first: true)
+                          end.join(', ')
+                        end,
+          'inLanguage' => language,
+          'contentSize' => Array.wrap(sizes).unwrap,
+          'encodingFormat' => Array.wrap(formats).unwrap,
+          'dateCreated' => date['created'],
+          'datePublished' => date['published'],
+          'dateModified' => date['updated'],
+          'pageStart' => container.to_h['firstPage'],
+          'pageEnd' => container.to_h['lastPage'],
+          'spatialCoverage' => to_schema_org_spatial_coverage(geo_locations),
+          'citation' => Array.wrap(references).map { |r| to_schema_org_citation(r) },
+          '@reverse' => reverse.presence,
+          'contentUrl' => Array.wrap(content_url).unwrap,
+          'schemaVersion' => schema_version,
+          'periodical' => if type == 'Dataset'
+                            nil
+                          else
+                            to_schema_org_container(container.to_h, type: 'Periodical')
+                          end,
+          'includedInDataCatalog' => if type == 'Dataset'
+                                       to_schema_org_container(container.to_h, type: 'DataCatalog')
+                                     end,
+          'publisher' => if publisher.present?
+                           { '@type' => 'Organization',
+                             'name' => publisher['name'] }
+                         end,
+          'funder' => to_schema_org_funder(funding_references),
+          'provider' => if provider.present?
+                          { '@type' => 'Organization',
+                            'name' => provider }
+                        end }.compact.presence
+      end
+
+      def schema_org
+        JSON.pretty_generate schema_hsh
+      end
+    end
+  end
+end

data/lib/commonmeta/writers/turtle_writer.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Commonmeta
+  module Writers
+    module TurtleWriter
+      def turtle
+        graph.dump(:ttl, prefixes: { schema: 'http://schema.org/' })
+      end
+    end
+  end
+end

data/lib/commonmeta/xml_converter.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# modified from http://stackoverflow.com/a/29431089
+# preserve attributes in xml
+
+module ActiveSupport
+  class XMLConverter
+    private
+
+    def become_content?(value)
+      value['type'] == 'file' || value['type'] == 'string' || (value['__content__'] && (value.keys.size == 1 && value['__content__'].present?))
+    end
+  end
+end

data/lib/commonmeta.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'dotenv/load'
+require 'active_support/all'
+require 'nokogiri'
+ActiveSupport::XmlMini.backend = 'Nokogiri'
+require 'http'
+require 'multi_json'
+require 'postrank-uri'
+require 'bibtex'
+require 'colorize'
+require 'loofah'
+require 'json/ld'
+require 'rdf/turtle'
+require 'rdf/rdfxml'
+require 'logger'
+require 'iso8601'
+require 'jsonlint'
+require 'benchmark_methods'
+require 'gender_detector'
+require 'citeproc/ruby'
+require 'citeproc'
+require 'csl/styles'
+require 'edtf'
+require 'base32/url'
+
+require 'commonmeta/version'
+require 'commonmeta/metadata'
+require 'commonmeta/cli'
+require 'commonmeta/string'
+require 'commonmeta/array'
+require 'commonmeta/whitelist_scrubber'
+require 'commonmeta/xml_converter'
+
+ENV['USER_AGENT'] ||= 'Mozilla/5.0 (compatible; mailto:info@front-matter.io)'