relaton-nist 1.8.0 → 1.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 708aed7e58956fbc6f13f3762f643c8dcd894d71d27019e640adf63b8b8b5215
-  data.tar.gz: 55e8c9f4bfd97442aebb7a4ee782f5620fba41b4f89e33f44cf79f60df6c7f95
+  metadata.gz: ea210109d9e306b8c9c6b4395812b52454dad6b8fdbd87cc08ec750e9dd2d4ac
+  data.tar.gz: e2dad0af3e66b42b2b5d5f2ecf48653ea21b2662d9bed9f6003a5b7a6d953f1d
 SHA512:
-  metadata.gz: 1a13a4b39a41bce75a8ce7d79f741b38954539701d5f2788b0beee850b4ff54e50055c9ab23a79e0df3b40e6e2dbded6cd4155ea227589d34147b1a0524ce99b
-  data.tar.gz: 477d2f7d2cd8a3870a469e8e2c5ac12a8c68c3ff96fe0c6fea8bdf903d50c1287d63e0a10b7ca81987db3718330b687f3ebedfed149e6a8ee4e973bd0e03a3cb
+  metadata.gz: '08a75dfaf1616874bab1c6edd2eb9a1f80c64a2d8d052dd66a3f4ca4f5b8aac1ee520c397f62327d718865fbd5f476db0e9a4ebef445ae3ecbe6f0366ee75e66'
+  data.tar.gz: cae6a5c20af9c77ead564ee5337d3092356a1786337406c260e75a387cc0446bd9937a23b2a24d1eafcc5b0bb1e98a621288e091634b7d950d112d96f13019cf

data/.github/workflows/rake.yml

@@ -16,19 +16,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.7', '2.6', '2.5', '2.4' ]
+        ruby: [ '3.0', '2.7', '2.6', '2.5' ]
         os: [ ubuntu-latest, windows-latest, macos-latest ]
         experimental: [ false ]
-        include:
-          - ruby: '3.0'
-            os: 'ubuntu-latest'
-            experimental: true
-          - ruby: '3.0'
-            os: 'windows-latest'
-            experimental: true
-          - ruby: '3.0'
-            os: 'macos-latest'
-            experimental: true
     steps:
       - uses: actions/checkout@v2
         with:

data/.gitignore

@@ -7,6 +7,7 @@
 /spec/reports/
 /tmp/
 .vscode/
+/data/
 .rubocop-https---raw-githubusercontent-com-riboseinc-oss-guides-master-ci-rubocop-yml
 
 # rspec failure tracking

data/.rubocop.yml

@@ -5,6 +5,6 @@
 inherit_from:
   - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
 AllCops:
-  TargetRubyVersion: 2.4
+  TargetRubyVersion: 2.5
 Rails:
   Enabled: false

data/README.adoc

@@ -147,6 +147,17 @@ item.docidentifier.first.id
 => "SP 800-38A-Add"
 ----
 
+=== Typed links
+
+NIST documents may have `src` and `doi` link types.
+
+[source,ruby]
+----
+item.link
+=> [#<RelatonBib::TypedUri:0x00007f901971dc10 @content=#<Addressable::URI:0x62c URI:https://csrc.nist.gov/publications/detail/sp/800-67/rev-2/final>, @type="src">,
+ #<RelatonBib::TypedUri:0x00007f901971d6e8 @content=#<Addressable::URI:0x640 URI:https://doi.org/10.6028/NIST.SP.800-67r2>, @type="doi">]
+----
+
 === Create bibliographic item from YAML
 [source,ruby]
 ----
@@ -159,6 +170,25 @@ RelatonNist::NistBibliographicItem.from_hash hash
 ...
 ----
 
+=== Fetch data
+
+This gem uses the https://raw.githubusercontent.com/usnistgov/NIST-Tech-Pubs/nist-pages/xml/allrecords.xml dataset as one of data sources.
+
+The method `RelatonNist::DataFetcher.fetch(output: "data", format: "yaml")` fetches all the documents from the datast and save them to the `./data` folder in YAML format.
+Arguments:
+
+- `output` - folder to save documents (default './data').
+- `format` - format in which the documents are saved. Possimle formats are: `yaml`, `xml`, `bibxxml` (default `yaml`).
+
+[source,ruby]
+----
+RelatonNist::DataFetcher.fetch
+Started at: 2021-09-01 18:01:01 +0200
+Stopped at: 2021-09-01 18:01:43 +0200
+Done in: 42 sec.
+=> nil
+----
+
 == Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/bin/rspec

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rspec' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rspec-core", "rspec")

data/grammars/biblio.rng

@@ -787,6 +787,7 @@
       <value>adapted</value>
       <value>vote-started</value>
       <value>vote-ended</value>
+      <value>announced</value>
     </choice>
   </define>
   <define name="bdate">

data/grammars/isodoc.rng

@@ -45,6 +45,11 @@
         <optional>
           <attribute name="alt"/>
         </optional>
+        <optional>
+          <attribute name="updatetype">
+            <data type="boolean"/>
+          </attribute>
+        </optional>
         <text/>
       </element>
     </define>
@@ -199,6 +204,18 @@
         </zeroOrMore>
       </element>
     </define>
+    <define name="dt">
+      <element name="dt">
+        <optional>
+          <attribute name="id">
+            <data type="ID"/>
+          </attribute>
+        </optional>
+        <zeroOrMore>
+          <ref name="TextElement"/>
+        </zeroOrMore>
+      </element>
+    </define>
     <define name="example">
       <element name="example">
         <attribute name="id">
@@ -543,6 +560,9 @@
     </define>
     <define name="BibDataExtensionType">
       <ref name="doctype"/>
+      <optional>
+        <ref name="docsubtype"/>
+      </optional>
       <optional>
         <ref name="editorialgroup"/>
       </optional>
@@ -890,6 +910,14 @@
     </define>
   </include>
   <!-- end overrides -->
+  <define name="docsubtype">
+    <element name="subdoctype">
+      <ref name="DocumentSubtype"/>
+    </element>
+  </define>
+  <define name="DocumentSubtype">
+    <text/>
+  </define>
   <define name="colgroup">
     <element name="colgroup">
       <oneOrMore>
@@ -939,7 +967,34 @@
   <define name="concept">
     <element name="concept">
       <optional>
-        <attribute name="term"/>
+        <attribute name="ital">
+          <data type="boolean"/>
+        </attribute>
+      </optional>
+      <optional>
+        <attribute name="ref">
+          <data type="boolean"/>
+        </attribute>
+      </optional>
+      <optional>
+        <element name="refterm">
+          <zeroOrMore>
+            <choice>
+              <ref name="PureTextElement"/>
+              <ref name="stem"/>
+            </choice>
+          </zeroOrMore>
+        </element>
+      </optional>
+      <optional>
+        <element name="renderterm">
+          <zeroOrMore>
+            <choice>
+              <ref name="PureTextElement"/>
+              <ref name="stem"/>
+            </choice>
+          </zeroOrMore>
+        </element>
       </optional>
       <choice>
         <ref name="eref"/>
@@ -965,6 +1020,9 @@
       </attribute>
       <attribute name="name"/>
       <attribute name="action"/>
+      <optional>
+        <attribute name="class"/>
+      </optional>
       <zeroOrMore>
         <choice>
           <ref name="TextElement"/>
@@ -1191,13 +1249,17 @@
   </define>
   <define name="IsoWorkgroup">
     <optional>
-      <attribute name="number">
-        <data type="int"/>
-      </attribute>
+      <attribute name="number"/>
     </optional>
     <optional>
       <attribute name="type"/>
     </optional>
+    <optional>
+      <attribute name="identifier"/>
+    </optional>
+    <optional>
+      <attribute name="prefix"/>
+    </optional>
     <text/>
   </define>
   <define name="ics">
@@ -1459,26 +1521,26 @@
     <optional>
       <ref name="section-title"/>
     </optional>
-    <group>
+    <choice>
       <choice>
         <group>
-          <zeroOrMore>
+          <oneOrMore>
             <ref name="BasicBlock"/>
-          </zeroOrMore>
+          </oneOrMore>
           <zeroOrMore>
             <ref name="note"/>
           </zeroOrMore>
         </group>
         <ref name="amend"/>
       </choice>
-      <zeroOrMore>
+      <oneOrMore>
         <choice>
           <ref name="clause-subsection"/>
           <ref name="terms"/>
           <ref name="definitions"/>
         </choice>
-      </zeroOrMore>
-    </group>
+      </oneOrMore>
+    </choice>
   </define>
   <define name="Annex-Section">
     <optional>

data/grammars/nist.rng

@@ -65,6 +65,9 @@
       <optional>
         <ref name="doctype"/>
       </optional>
+      <optional>
+        <ref name="docsubtype"/>
+      </optional>
       <optional>
         <ref name="editorialgroup"/>
       </optional>

data/lib/relaton_nist/data_fetcher.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module RelatonNist
+  class DataFetcher
+    RELATION_TYPES = {
+      "replaces" => "obsoletes",
+      "isVersionOf" => "editionOf",
+      "hasTranslation" => "hasTranslation",
+      "isTranslationOf" => "translatedFrom",
+      "hasPreprint" => "hasReprint",
+      "isSupplementTo" => "complements",
+    }.freeze
+    URL = "https://raw.githubusercontent.com/usnistgov/NIST-Tech-Pubs/nist-pages/xml/allrecords.xml"
+
+    def initialize(output, format)
+      @output = output
+      @format = format
+      @ext = format.sub(/^bib/, "")
+    end
+
+    def parse_docid(doc)
+      doi = doc.at("doi_data/doi").text
+      id = doc.at("publisher_item/item_number", "publisher_item/identifier").text.sub(%r{^/}, "")
+      case doi
+      when "10.6028/NBS.CIRC.12e2revjune" then id.sub!("13e", "12e")
+      when "10.6028/NBS.CIRC.36e2" then id.sub!("46e", "36e")
+      when "10.6028/NBS.HB.67suppJune1967" then id.sub!("1965", "1967")
+      when "10.6028/NBS.HB.105-1r1990" then id.sub!("105-1-1990", "105-1r1990")
+      when "10.6028/NIST.HB.150-10-1995" then id.sub!(/150-10$/, "150-10-1995")
+      end
+      [{ type: "NIST", id: id }, { type: "DOI", id: doi }]
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::DocumentIdentifier>]
+    def fetch_docid(doc)
+      parse_docid(doc).map do |id|
+        RelatonBib::DocumentIdentifier.new(type: id[:type], id: id[:id])
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [RelatonBib::TypedTitleStringCollection, Array]
+    def fetch_title(doc)
+      t = doc.xpath("titles/title|titles/subtitle")
+      return [] unless t.any?
+
+      RelatonBib::TypedTitleString.from_string t.map(&:text).join(" "), "en", "Latn"
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::BibliographicDate>]
+    def fetch_date(doc)
+      doc.xpath("publication_date|approval_date").map do |dt|
+        on = dt.at("year").text
+        if (m = dt.at "month")
+          on += "-#{m.text}"
+          d = dt.at "day"
+          on += "-#{d.text}" if d
+        end
+        type = dt.name == "publication_date" ? "published" : "confirmed"
+        RelatonBib::BibliographicDate.new(type: type, on: on)
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [String]
+    def fetch_edition(doc)
+      doc.at("edition_number")&.text
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<Hash>]
+    def fetch_relation(doc)
+      ns = "http://www.crossref.org/relations.xsd"
+      doc.xpath("./ns:program/ns:related_item", ns: ns).map do |rel|
+        doi = rel.at_xpath("ns:intra_work_relation|ns:inter_work_relation", ns: ns)
+        # ref = doi_to_id doi.text
+        # ref, = parse_docid doc
+        fref = RelatonBib::FormattedRef.new content: doi.text
+        bibitem = RelatonBib::BibliographicItem.new formattedref: fref
+        type = RELATION_TYPES[doi["relationship-type"]]
+        { type: type, bibitem: bibitem }
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::TypedUri>]
+    def fetch_link(doc)
+      url = doc.at("doi_data/resource").text
+      [RelatonBib::TypedUri.new(type: "doi", content: url)]
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::FormattedString>]
+    def fetch_abstract(doc)
+      doc.xpath("jats:abstract/jats:p", "jats" => "http://www.ncbi.nlm.nih.gov/JATS1").map do |a|
+        RelatonBib::FormattedString.new(content: a.text, language: doc["language"], script: "Latn")
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<Hash>]
+    def fetch_contributor(doc) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      contribs = doc.xpath("contributors/person_name").map do |p|
+        forename = []
+        initial = []
+        p.at("given_name")&.text&.split&.each do |fn|
+          if /^(?<init>\w)\.?$/ =~ fn
+            initial << RelatonBib::LocalizedString.new(init, doc["language"], "Latn")
+          else
+            forename << RelatonBib::LocalizedString.new(fn, doc["language"], "Latn")
+          end
+        end
+        sname = p.at("surname").text
+        surname = RelatonBib::LocalizedString.new sname, doc["language"], "Latn"
+        initial = []
+        ident = p.xpath("ORCID").map do |id|
+          RelatonBib::PersonIdentifier.new "orcid", id.text
+        end
+        fullname = RelatonBib::FullName.new(
+          surname: surname, forename: forename, initial: initial, identifier: ident,
+        )
+        person = RelatonBib::Person.new name: fullname
+        { entity: person, role: [{ type: p["contributor_role"] }] }
+      end
+      contribs + doc.xpath("publisher").map do |p|
+        abbr = p.at("../institution/institution_acronym")&.text
+        org = RelatonBib::Organization.new(name: p.at("publisher_name").text, abbreviation: abbr)
+        { entity: org, role: [{ type: "publisher" }] }
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<String>]
+    def fetch_place(doc)
+      doc.xpath("institution/institution_place").map(&:text)
+    end
+
+    #
+    # Save document
+    #
+    # @param bib [RelatonNist::NistBibliographicItem]
+    #
+    def write_file(bib) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      id = bib.docidentifier[0].id.gsub(%r{[/\s:.]}, "_").upcase.sub(/^NIST_IR/, "NISTIR")
+      file = File.join(@output, "#{id}.#{@ext}")
+      if File.exist? file
+        warn "File #{file} exists. Docid: #{bib.docidentifier[0].id}"
+        # warn "Link: #{bib.link.detect { |l| l.type == 'src' }.content}"
+      else
+        output = case @format
+                 when "yaml" then bib.to_hash.to_yaml
+                 when "xml" then bib.to_xml bibdata: true
+                 else bib.send "to_#{@format}"
+                 end
+        File.write file, output, encoding: "UTF-8"
+      end
+    end
+
+    #
+    # Create a document instance an save it.
+    #
+    # @param doc [Nokogiri::XML::Element]
+    #
+    # @raise [StandardError]
+    #
+    def parse_doc(doc) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+      # mtd = doc.at('doi_record/report-paper/report-paper_metadata')
+      item = RelatonNist::NistBibliographicItem.new(
+        type: "standard", docid: fetch_docid(doc), title: fetch_title(doc),
+        link: fetch_link(doc), abstract: fetch_abstract(doc),
+        date: fetch_date(doc), edition: fetch_edition(doc),
+        contributor: fetch_contributor(doc), relation: fetch_relation(doc),
+        place: fetch_place(doc),
+        language: [doc["language"]], script: ["Latn"], doctype: "standard"
+      )
+      write_file item
+    rescue StandardError => e
+      warn "Document: #{doc.at('doi').text}"
+      warn e.message
+      raise e
+    end
+
+    #
+    # Fetch all the documnts from dataset
+    #
+    def fetch # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      t1 = Time.now
+      puts "Started at: #{t1}"
+
+      docs = Nokogiri::XML OpenURI.open_uri URL
+      FileUtils.mkdir @output unless Dir.exist? @output
+      FileUtils.rm Dir[File.join(@output, "*.#{@ext}")]
+      docs.xpath("/body/query/doi_record/report-paper/report-paper_metadata")
+        .each { |doc| parse_doc doc }
+
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    rescue StandardError => e
+      warn e.message
+    end
+
+    #
+    # Fetch all the documnts from dataset
+    #
+    # @param [String] output foldet name to save the documents
+    # @param [String] format format to save the documents (yaml, xml, bibxml)
+    #
+    def self.fetch(output: "data", format: "yaml")
+      new(output, format).fetch
+    end
+  end
+end

data/lib/relaton_nist/hit.rb

@@ -3,6 +3,8 @@
 module RelatonNist
   # Hit.
   class Hit < RelatonBib::Hit
+    attr_writer :fetch
+
     # Parse page.
     # @return [RelatonNist::NistBliographicItem]
     def fetch
@@ -10,7 +12,7 @@ module RelatonNist
     end
 
     # @return [Iteger]
-    def sort_value
+    def sort_value # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
       @sort_value ||= begin
         sort_phrase = [hit[:serie], hit[:code], hit[:title]].join " "
         corr = hit_collection&.text&.split&.map do |w|