relaton-nist 1.9.1 → 1.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d440504b749834c27875dc8a1ee84346451578db5dd1588b75eafdb31087b64c
-  data.tar.gz: ecfb305d172a8afa171fa4493b1ffec2dd458fd413143a7f197034a392ddf451
+  metadata.gz: 76cfec1954dfd58bf622a6a47e34bdbcfe4cf471106eb7ad19582654078a6d38
+  data.tar.gz: 9769f406d5b98840b5bc099d78ab811388170d1fae5872211a7a0c9ed26d869f
 SHA512:
-  metadata.gz: 7af96feb236bddbabd7e90982637b608813d5c2163c7453f217c658035cf34b0ef80adc6441d6be213b77e62b5625d95fb10a30914fc94a1b6cf966b1c921bf6
-  data.tar.gz: 9b9daa02fc2ec7df33ad5c8cdf768c711b1bac0122191e7cfc79ee7c8ed2b7de00fbe6509b3347f387cf4868b24448f91f1a687e45b80fe65ce31d3f7a46cad5
+  metadata.gz: 0e3deefff699c77103afb631a03283697dc05f5c952b60b3602961df37df2095102b16c32e229aeeceb76da12cfda09e43e6b0788b6a4c1f02b74dea8a49f818
+  data.tar.gz: 3244707d2595b5b45bab973399a04b8cc791d3b008e2ef05fcc99d9355cec6915556aea8528e5f992e4ba0452af82bda09fb93737741a72c08a64e7a0039d161

data/.gitignore

@@ -7,6 +7,7 @@
 /spec/reports/
 /tmp/
 .vscode/
+/data/
 .rubocop-https---raw-githubusercontent-com-riboseinc-oss-guides-master-ci-rubocop-yml
 
 # rspec failure tracking

data/README.adoc

@@ -159,6 +159,25 @@ RelatonNist::NistBibliographicItem.from_hash hash
 ...
 ----
 
+=== Fetch data
+
+This gem uses the https://raw.githubusercontent.com/usnistgov/NIST-Tech-Pubs/nist-pages/xml/allrecords.xml dataset as one of data sources.
+
+The method `RelatonNist::DataFetcher.fetch(output: "data", format: "yaml")` fetches all the documents from the datast and save them to the `./data` folder in YAML format.
+Arguments:
+
+- `output` - folder to save documents (default './data').
+- `format` - format in which the documents are saved. Possimle formats are: `yaml`, `xml` (default `yaml`).
+
+[source,ruby]
+----
+RelatonNist::DataFetcher.fetch
+Started at: 2021-09-01 18:01:01 +0200
+Stopped at: 2021-09-01 18:01:43 +0200
+Done in: 42 sec.
+=> nil
+----
+
 == Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/relaton_nist/data_fetcher.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+# require 'English'
+# require 'mechanize'
+# require "fileutils"
+require "yaml"
+# require "open-uri"
+# require "nokogiri"
+# require "relaton_nist"
+
+module RelatonNist
+  class DataFetcher
+    RELATION_TYPES = {
+      "replaces" => "obsoletes",
+      "isVersionOf" => "editionOf",
+      "hasTranslation" => "hasTranslation",
+      "isTranslationOf" => "translatedFrom",
+      "hasPreprint" => "hasReprint",
+      "isSupplementTo" => "complements",
+    }.freeze
+    URL = "https://raw.githubusercontent.com/usnistgov/NIST-Tech-Pubs/nist-pages/xml/allrecords.xml"
+
+    def initialize(output, format)
+      @output = output
+      @format = format
+    end
+
+    def parse_docid(doc)
+      doi = doc.at("doi_data/doi").text
+      id = doc.at("publisher_item/item_number", "publisher_item/identifier").text.sub(%r{^/}, "")
+      case doi
+      when "10.6028/NBS.CIRC.12e2revjune" then id.sub!("13e", "12e")
+      when "10.6028/NBS.CIRC.36e2" then id.sub!("46e", "36e")
+      when "10.6028/NBS.HB.67suppJune1967" then id.sub!("1965", "1967")
+      when "10.6028/NBS.HB.105-1r1990" then id.sub!("105-1-1990", "105-1r1990")
+      when "10.6028/NIST.HB.150-10-1995" then id.sub!(/150-10$/, "150-10-1995")
+      end
+      [{ type: "NIST", id: id }, { type: "DOI", id: doi }]
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::DocumentIdentifier>]
+    def fetch_docid(doc)
+      parse_docid(doc).map do |id|
+        RelatonBib::DocumentIdentifier.new(type: id[:type], id: id[:id])
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [RelatonBib::TypedTitleStringCollection, Array]
+    def fetch_title(doc)
+      t = doc.xpath("titles/title|titles/subtitle")
+      return [] unless t.any?
+
+      RelatonBib::TypedTitleString.from_string t.map(&:text).join(" "), "en", "Latn"
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::BibliographicDate>]
+    def fetch_date(doc)
+      doc.xpath("publication_date|approval_date").map do |dt|
+        on = dt.at("year").text
+        if (m = dt.at "month")
+          on += "-#{m.text}"
+          d = dt.at "day"
+          on += "-#{d.text}" if d
+        end
+        type = dt.name == "publication_date" ? "published" : "confirmed"
+        RelatonBib::BibliographicDate.new(type: type, on: on)
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [String]
+    def fetch_edition(doc)
+      doc.at("edition_number")&.text
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<Hash>]
+    def fetch_relation(doc)
+      ns = "http://www.crossref.org/relations.xsd"
+      doc.xpath("./ns:program/ns:related_item", ns: ns).map do |rel|
+        doi = rel.at_xpath("ns:intra_work_relation|ns:inter_work_relation", ns: ns)
+        # ref = doi_to_id doi.text
+        # ref, = parse_docid doc
+        fref = RelatonBib::FormattedRef.new content: doi.text
+        bibitem = RelatonBib::BibliographicItem.new formattedref: fref
+        type = RELATION_TYPES[doi["relationship-type"]]
+        { type: type, bibitem: bibitem }
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::TypedUri>]
+    def fetch_link(doc)
+      url = doc.at("doi_data/resource").text
+      [RelatonBib::TypedUri.new(type: "doi", content: url)]
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::FormattedString>]
+    def fetch_abstract(doc)
+      doc.xpath("jats:abstract/jats:p", "jats" => "http://www.ncbi.nlm.nih.gov/JATS1").map do |a|
+        RelatonBib::FormattedString.new(content: a.text, language: doc["language"], script: "Latn")
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<Hash>]
+    def fetch_contributor(doc) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      contribs = doc.xpath("contributors/person_name").map do |p|
+        forename = []
+        initial = []
+        p.at("given_name")&.text&.split&.each do |fn|
+          if /^(?<init>\w)\.?$/ =~ fn
+            initial << RelatonBib::LocalizedString.new(init, doc["language"], "Latn")
+          else
+            forename << RelatonBib::LocalizedString.new(fn, doc["language"], "Latn")
+          end
+        end
+        sname = p.at("surname").text
+        surname = RelatonBib::LocalizedString.new sname, doc["language"], "Latn"
+        initial = []
+        ident = p.xpath("ORCID").map do |id|
+          RelatonBib::PersonIdentifier.new "orcid", id.text
+        end
+        fullname = RelatonBib::FullName.new(
+          surname: surname, forename: forename, initial: initial, identifier: ident,
+        )
+        person = RelatonBib::Person.new name: fullname
+        { entity: person, role: [{ type: p["contributor_role"] }] }
+      end
+      contribs + doc.xpath("publisher").map do |p|
+        abbr = p.at("../institution/institution_acronym")&.text
+        org = RelatonBib::Organization.new(name: p.at("publisher_name").text, abbreviation: abbr)
+        { entity: org, role: [{ type: "publisher" }] }
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<String>]
+    def fetch_place(doc)
+      doc.xpath("institution/institution_place").map(&:text)
+    end
+
+    #
+    # Save document
+    #
+    # @param bib [RelatonNist::NistBibliographicItem]
+    #
+    def write_file(bib) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      id = bib.docidentifier[0].id.gsub(%r{[/\s:.]}, "_").upcase.sub(/^NIST_IR/, "NISTIR")
+      file = File.join(@output, "#{id}.#{@format}")
+      if File.exist? file
+        warn "File #{file} exists. Docid: #{bib.docidentifier[0].id}"
+        # warn "Link: #{bib.link.detect { |l| l.type == 'src' }.content}"
+      else
+        output = case @format
+                 when "yaml" then bib.to_hash.to_yaml
+                 when "xml" then bib.to_xml bibdata: true
+                 end
+        File.write file, output, encoding: "UTF-8"
+      end
+    end
+
+    #
+    # Create a document instance an save it.
+    #
+    # @param doc [Nokogiri::XML::Element]
+    #
+    # @raise [StandardError]
+    #
+    def parse_doc(doc) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+      # mtd = doc.at('doi_record/report-paper/report-paper_metadata')
+      item = RelatonNist::NistBibliographicItem.new(
+        type: "standard", docid: fetch_docid(doc), title: fetch_title(doc),
+        link: fetch_link(doc), abstract: fetch_abstract(doc),
+        date: fetch_date(doc), edition: fetch_edition(doc),
+        contributor: fetch_contributor(doc), relation: fetch_relation(doc),
+        place: fetch_place(doc),
+        language: [doc["language"]], script: ["Latn"], doctype: "standard"
+      )
+      write_file item
+    rescue StandardError => e
+      warn "Document: #{doc.at('doi').text}"
+      warn e.message
+      raise e
+    end
+
+    #
+    # Fetch all the documnts from dataset
+    #
+    def fetch # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      t1 = Time.now
+      puts "Started at: #{t1}"
+
+      docs = Nokogiri::XML OpenURI.open_uri URL
+      FileUtils.mkdir @output unless Dir.exist? @output
+      FileUtils.rm Dir[File.join(@output, "*.#{@format}")]
+      docs.xpath("/body/query/doi_record/report-paper/report-paper_metadata")
+        .each { |doc| parse_doc doc }
+
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    rescue StandardError => e
+      warn e.message
+    end
+
+    #
+    # Fetch all the documnts from dataset
+    #
+    # @param [String] output foldet name to save the documents
+    # @param [String] format format to save the documents
+    #
+    def self.fetch(output: "data", format: "yaml")
+      new(output, format).fetch
+    end
+  end
+end

data/lib/relaton_nist/hit_collection.rb

@@ -15,17 +15,19 @@ module RelatonNist
     DATAFILE = File.expand_path "pubs-export.zip", DATAFILEDIR
     GHNISTDATA = "https://raw.githubusercontent.com/relaton/relaton-data-nist/main/data/"
 
-    # @param ref_nbr [String]
-    # @param year [String]
-    # @param opts [Hash]
-    # @option opts [String] :stage
-    def initialize(ref_nbr, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
-      super ref_nbr, year
-
-      # /(?<docid>(?:SP|FIPS)\s[0-9-]+)/ =~ text
+    def self.search(text, year = nil, opts = {})
+      new(text, year).search(opts)
+    end
+
+    def search(opts)
       @array = from_json(**opts)
       @array = from_ga unless @array.any?
+      sort_hits!
+    end
 
+    private
+
+    def sort_hits!
       @array.sort! do |a, b|
         if a.sort_value == b.sort_value
           (b.hit[:release_date] - a.hit[:release_date]).to_i
@@ -33,10 +35,9 @@ module RelatonNist
           b.sort_value - a.sort_value
         end
       end
+      self
     end
 
-    private
-
     def from_ga # rubocop:disable Metrics/AbcSize
       fn = text.gsub(%r{[/\s:.]}, "_").upcase
       yaml = OpenURI.open_uri "#{GHNISTDATA}#{fn}.yaml"

data/lib/relaton_nist/nist_bibliography.rb

@@ -14,7 +14,7 @@ module RelatonNist
       # @param text [String]
       # @return [RelatonNist::HitCollection]
       def search(text, year = nil, opts = {})
-        HitCollection.new text, year, opts
+        HitCollection.search text, year, opts
       rescue OpenURI::HTTPError, SocketError, OpenSSL::SSL::SSLError => e
         raise RelatonBib::RequestError, e.message
       end

data/lib/relaton_nist/processor.rb

@@ -7,6 +7,7 @@ module RelatonNist
       @prefix = "NIST"
       @defaultprefix = %r{^(NIST|NISTGCR|ITL Bulletin|JPCRD|NISTIR|CSRC|FIPS)(/[^\s])?\s}
       @idtype = "NIST"
+      @datasets = %w[nist-tech-pubs]
     end
 
     # @param code [String]
@@ -17,6 +18,10 @@ module RelatonNist
       ::RelatonNist::NistBibliography.get(code, date, opts)
     end
 
+    def fetch_data(_source, opts)
+      DataFetcher.fetch(**opts)
+    end
+
     # @param xml [String]
     # @return [RelatonNist::GbBibliographicItem]
     def from_xml(xml)

data/lib/relaton_nist/version.rb

@@ -1,3 +1,3 @@
 module RelatonNist
-  VERSION = "1.9.1".freeze
+  VERSION = "1.9.2".freeze
 end

data/lib/relaton_nist.rb

@@ -1,5 +1,6 @@
 require "relaton_nist/version"
 require "relaton_nist/nist_bibliography"
+require "relaton_nist/data_fetcher"
 
 # if defined? Relaton
 #   require_relative "relaton/processor"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-nist
 version: !ruby/object:Gem::Version
-  version: 1.9.1
+  version: 1.9.2
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-08-30 00:00:00.000000000 Z
+date: 2021-09-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -175,6 +175,7 @@ files:
 - grammars/reqt.rng
 - lib/relaton_nist.rb
 - lib/relaton_nist/comment_period.rb
+- lib/relaton_nist/data_fetcher.rb
 - lib/relaton_nist/document_relation.rb
 - lib/relaton_nist/document_status.rb
 - lib/relaton_nist/hash_converter.rb