relaton-nist 1.7.4 → 1.9.2

data/grammars/nist.rng

@@ -65,6 +65,9 @@
       <optional>
         <ref name="doctype"/>
       </optional>
+      <optional>
+        <ref name="docsubtype"/>
+      </optional>
       <optional>
         <ref name="editorialgroup"/>
       </optional>
@@ -194,6 +197,9 @@
       <zeroOrMore>
         <ref name="termdocsource"/>
       </zeroOrMore>
+      <optional>
+        <ref name="misccontainer"/>
+      </optional>
       <optional>
         <ref name="boilerplate"/>
       </optional>
@@ -207,6 +213,9 @@
       <optional>
         <ref name="bibliography"/>
       </optional>
+      <zeroOrMore>
+        <ref name="indexsect"/>
+      </zeroOrMore>
     </element>
   </define>
 </grammar>

data/grammars/reqt.rng

@@ -30,15 +30,34 @@
         <data type="boolean"/>
       </attribute>
     </optional>
+    <optional>
+      <attribute name="number"/>
+    </optional>
     <optional>
       <attribute name="subsequence"/>
     </optional>
+    <optional>
+      <attribute name="keep-with-next">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="keep-lines-together">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
     <attribute name="id">
       <data type="ID"/>
     </attribute>
     <optional>
       <attribute name="filename"/>
     </optional>
+    <optional>
+      <attribute name="model"/>
+    </optional>
+    <optional>
+      <attribute name="type"/>
+    </optional>
     <optional>
       <ref name="reqtitle"/>
     </optional>
@@ -48,9 +67,9 @@
     <optional>
       <ref name="subject"/>
     </optional>
-    <optional>
+    <zeroOrMore>
       <ref name="reqinherit"/>
-    </optional>
+    </zeroOrMore>
     <zeroOrMore>
       <ref name="classification"/>
     </zeroOrMore>
@@ -135,6 +154,16 @@
         <data type="boolean"/>
       </attribute>
     </optional>
+    <optional>
+      <attribute name="keep-with-next">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="keep-lines-together">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
     <oneOrMore>
       <ref name="BasicBlock"/>
     </oneOrMore>

data/lib/relaton_nist/data_fetcher.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+# require 'English'
+# require 'mechanize'
+# require "fileutils"
+require "yaml"
+# require "open-uri"
+# require "nokogiri"
+# require "relaton_nist"
+
+module RelatonNist
+  class DataFetcher
+    RELATION_TYPES = {
+      "replaces" => "obsoletes",
+      "isVersionOf" => "editionOf",
+      "hasTranslation" => "hasTranslation",
+      "isTranslationOf" => "translatedFrom",
+      "hasPreprint" => "hasReprint",
+      "isSupplementTo" => "complements",
+    }.freeze
+    URL = "https://raw.githubusercontent.com/usnistgov/NIST-Tech-Pubs/nist-pages/xml/allrecords.xml"
+
+    def initialize(output, format)
+      @output = output
+      @format = format
+    end
+
+    def parse_docid(doc)
+      doi = doc.at("doi_data/doi").text
+      id = doc.at("publisher_item/item_number", "publisher_item/identifier").text.sub(%r{^/}, "")
+      case doi
+      when "10.6028/NBS.CIRC.12e2revjune" then id.sub!("13e", "12e")
+      when "10.6028/NBS.CIRC.36e2" then id.sub!("46e", "36e")
+      when "10.6028/NBS.HB.67suppJune1967" then id.sub!("1965", "1967")
+      when "10.6028/NBS.HB.105-1r1990" then id.sub!("105-1-1990", "105-1r1990")
+      when "10.6028/NIST.HB.150-10-1995" then id.sub!(/150-10$/, "150-10-1995")
+      end
+      [{ type: "NIST", id: id }, { type: "DOI", id: doi }]
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::DocumentIdentifier>]
+    def fetch_docid(doc)
+      parse_docid(doc).map do |id|
+        RelatonBib::DocumentIdentifier.new(type: id[:type], id: id[:id])
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [RelatonBib::TypedTitleStringCollection, Array]
+    def fetch_title(doc)
+      t = doc.xpath("titles/title|titles/subtitle")
+      return [] unless t.any?
+
+      RelatonBib::TypedTitleString.from_string t.map(&:text).join(" "), "en", "Latn"
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::BibliographicDate>]
+    def fetch_date(doc)
+      doc.xpath("publication_date|approval_date").map do |dt|
+        on = dt.at("year").text
+        if (m = dt.at "month")
+          on += "-#{m.text}"
+          d = dt.at "day"
+          on += "-#{d.text}" if d
+        end
+        type = dt.name == "publication_date" ? "published" : "confirmed"
+        RelatonBib::BibliographicDate.new(type: type, on: on)
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [String]
+    def fetch_edition(doc)
+      doc.at("edition_number")&.text
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<Hash>]
+    def fetch_relation(doc)
+      ns = "http://www.crossref.org/relations.xsd"
+      doc.xpath("./ns:program/ns:related_item", ns: ns).map do |rel|
+        doi = rel.at_xpath("ns:intra_work_relation|ns:inter_work_relation", ns: ns)
+        # ref = doi_to_id doi.text
+        # ref, = parse_docid doc
+        fref = RelatonBib::FormattedRef.new content: doi.text
+        bibitem = RelatonBib::BibliographicItem.new formattedref: fref
+        type = RELATION_TYPES[doi["relationship-type"]]
+        { type: type, bibitem: bibitem }
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::TypedUri>]
+    def fetch_link(doc)
+      url = doc.at("doi_data/resource").text
+      [RelatonBib::TypedUri.new(type: "doi", content: url)]
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::FormattedString>]
+    def fetch_abstract(doc)
+      doc.xpath("jats:abstract/jats:p", "jats" => "http://www.ncbi.nlm.nih.gov/JATS1").map do |a|
+        RelatonBib::FormattedString.new(content: a.text, language: doc["language"], script: "Latn")
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<Hash>]
+    def fetch_contributor(doc) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      contribs = doc.xpath("contributors/person_name").map do |p|
+        forename = []
+        initial = []
+        p.at("given_name")&.text&.split&.each do |fn|
+          if /^(?<init>\w)\.?$/ =~ fn
+            initial << RelatonBib::LocalizedString.new(init, doc["language"], "Latn")
+          else
+            forename << RelatonBib::LocalizedString.new(fn, doc["language"], "Latn")
+          end
+        end
+        sname = p.at("surname").text
+        surname = RelatonBib::LocalizedString.new sname, doc["language"], "Latn"
+        initial = []
+        ident = p.xpath("ORCID").map do |id|
+          RelatonBib::PersonIdentifier.new "orcid", id.text
+        end
+        fullname = RelatonBib::FullName.new(
+          surname: surname, forename: forename, initial: initial, identifier: ident,
+        )
+        person = RelatonBib::Person.new name: fullname
+        { entity: person, role: [{ type: p["contributor_role"] }] }
+      end
+      contribs + doc.xpath("publisher").map do |p|
+        abbr = p.at("../institution/institution_acronym")&.text
+        org = RelatonBib::Organization.new(name: p.at("publisher_name").text, abbreviation: abbr)
+        { entity: org, role: [{ type: "publisher" }] }
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<String>]
+    def fetch_place(doc)
+      doc.xpath("institution/institution_place").map(&:text)
+    end
+
+    #
+    # Save document
+    #
+    # @param bib [RelatonNist::NistBibliographicItem]
+    #
+    def write_file(bib) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      id = bib.docidentifier[0].id.gsub(%r{[/\s:.]}, "_").upcase.sub(/^NIST_IR/, "NISTIR")
+      file = File.join(@output, "#{id}.#{@format}")
+      if File.exist? file
+        warn "File #{file} exists. Docid: #{bib.docidentifier[0].id}"
+        # warn "Link: #{bib.link.detect { |l| l.type == 'src' }.content}"
+      else
+        output = case @format
+                 when "yaml" then bib.to_hash.to_yaml
+                 when "xml" then bib.to_xml bibdata: true
+                 end
+        File.write file, output, encoding: "UTF-8"
+      end
+    end
+
+    #
+    # Create a document instance an save it.
+    #
+    # @param doc [Nokogiri::XML::Element]
+    #
+    # @raise [StandardError]
+    #
+    def parse_doc(doc) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+      # mtd = doc.at('doi_record/report-paper/report-paper_metadata')
+      item = RelatonNist::NistBibliographicItem.new(
+        type: "standard", docid: fetch_docid(doc), title: fetch_title(doc),
+        link: fetch_link(doc), abstract: fetch_abstract(doc),
+        date: fetch_date(doc), edition: fetch_edition(doc),
+        contributor: fetch_contributor(doc), relation: fetch_relation(doc),
+        place: fetch_place(doc),
+        language: [doc["language"]], script: ["Latn"], doctype: "standard"
+      )
+      write_file item
+    rescue StandardError => e
+      warn "Document: #{doc.at('doi').text}"
+      warn e.message
+      raise e
+    end
+
+    #
+    # Fetch all the documnts from dataset
+    #
+    def fetch # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      t1 = Time.now
+      puts "Started at: #{t1}"
+
+      docs = Nokogiri::XML OpenURI.open_uri URL
+      FileUtils.mkdir @output unless Dir.exist? @output
+      FileUtils.rm Dir[File.join(@output, "*.#{@format}")]
+      docs.xpath("/body/query/doi_record/report-paper/report-paper_metadata")
+        .each { |doc| parse_doc doc }
+
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    rescue StandardError => e
+      warn e.message
+    end
+
+    #
+    # Fetch all the documnts from dataset
+    #
+    # @param [String] output foldet name to save the documents
+    # @param [String] format format to save the documents
+    #
+    def self.fetch(output: "data", format: "yaml")
+      new(output, format).fetch
+    end
+  end
+end

data/lib/relaton_nist/hit.rb

@@ -3,6 +3,8 @@
 module RelatonNist
   # Hit.
   class Hit < RelatonBib::Hit
+    attr_writer :fetch
+
     # Parse page.
     # @return [RelatonNist::NistBliographicItem]
     def fetch
@@ -10,7 +12,7 @@ module RelatonNist
     end
 
     # @return [Iteger]
-    def sort_value
+    def sort_value # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
       @sort_value ||= begin
         sort_phrase = [hit[:serie], hit[:code], hit[:title]].join " "
         corr = hit_collection&.text&.split&.map do |w|

data/lib/relaton_nist/hit_collection.rb

@@ -13,75 +13,90 @@ module RelatonNist
     PUBS_EXPORT = URI.join(DOMAIN, "/CSRC/media/feeds/metanorma/pubs-export")
     DATAFILEDIR = File.expand_path ".relaton/nist", Dir.home
     DATAFILE = File.expand_path "pubs-export.zip", DATAFILEDIR
+    GHNISTDATA = "https://raw.githubusercontent.com/relaton/relaton-data-nist/main/data/"
 
-    # @param ref_nbr [String]
-    # @param year [String]
-    # @param opts [Hash]
-    # @option opts [String] :stage
-    def initialize(ref_nbr, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize
-      super ref_nbr, year
+    def self.search(text, year = nil, opts = {})
+      new(text, year).search(opts)
+    end
+
+    def search(opts)
+      @array = from_json(**opts)
+      @array = from_ga unless @array.any?
+      sort_hits!
+    end
 
-      /(?<docid>(SP|FIPS)\s[0-9-]+)/ =~ text
-      @array = docid ? from_json(docid, **opts) : from_csrc(**opts)
-      @array = from_csrc(**opts) unless @array.any?
+    private
 
+    def sort_hits!
       @array.sort! do |a, b|
-        if a.sort_value != b.sort_value
-          b.sort_value - a.sort_value
-        else
+        if a.sort_value == b.sort_value
           (b.hit[:release_date] - a.hit[:release_date]).to_i
+        else
+          b.sort_value - a.sort_value
         end
       end
+      self
     end
 
-    private
+    def from_ga # rubocop:disable Metrics/AbcSize
+      fn = text.gsub(%r{[/\s:.]}, "_").upcase
+      yaml = OpenURI.open_uri "#{GHNISTDATA}#{fn}.yaml"
+      hash = YAML.safe_load yaml
+      bib = RelatonNist::NistBibliographicItem.from_hash hash
+      hit = Hit.new({ code: text }, self)
+      hit.fetch = bib
+      [hit]
+    rescue OpenURI::HTTPError => e
+      return [] if e.io.status[0] == "404"
+
+      raise e
+    end
 
     # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
 
     # @param stage [String]
     # @return [Array<RelatonNist::Hit>]
-    def from_csrc(**opts)
-      from, to = nil
-      if year
-        d    = Date.strptime year, "%Y"
-        from = d.strftime "%m/%d/%Y"
-        to   = d.next_year.prev_day.strftime "%m/%d/%Y"
-      end
-      url = "#{DOMAIN}/publications/search?keywords-lg=#{text}"\
-        "&sortBy-lg=relevence"
-      url += "&dateFrom-lg=#{from}" if from
-      url += "&dateTo-lg=#{to}" if to
-      url += if /PD/.match? opts[:stage]
-               "&status-lg=Draft,Retired Draft,Withdrawn"
-             else
-               "&status-lg=Final,Withdrawn"
-             end
-
-      doc = Nokogiri::HTML OpenURI.open_uri(::Addressable::URI.parse(url).normalize)
-      doc.css("table.publications-table > tbody > tr").map do |h|
-        link  = h.at("td/div/strong/a")
-        serie = h.at("td[1]").text.strip
-        code  = h.at("td[2]").text.strip
-        title = link.text
-        doc_url = DOMAIN + link[:href]
-        status = h.at("td[4]").text.strip.downcase
-        release_date = Date.strptime h.at("td[5]").text.strip, "%m/%d/%Y"
-        Hit.new(
-          {
-            code: code, serie: serie, title: title, url: doc_url,
-            status: status, release_date: release_date
-          }, self
-        )
-      end
-    end
+    # def from_csrc(**opts)
+    #   from, to = nil
+    #   if year
+    #     d    = Date.strptime year, "%Y"
+    #     from = d.strftime "%m/%d/%Y"
+    #     to   = d.next_year.prev_day.strftime "%m/%d/%Y"
+    #   end
+    #   url = "#{DOMAIN}/publications/search?keywords-lg=#{text}"\
+    #         "&sortBy-lg=relevence"
+    #   url += "&dateFrom-lg=#{from}" if from
+    #   url += "&dateTo-lg=#{to}" if to
+    #   url += if /PD/.match? opts[:stage]
+    #            "&status-lg=Draft,Retired Draft,Withdrawn"
+    #          else
+    #            "&status-lg=Final,Withdrawn"
+    #          end
+
+    #   doc = Nokogiri::HTML OpenURI.open_uri(::Addressable::URI.parse(url).normalize)
+    #   doc.css("table.publications-table > tbody > tr").map do |h|
+    #     link  = h.at("td/div/strong/a")
+    #     serie = h.at("td[1]").text.strip
+    #     code  = h.at("td[2]").text.strip
+    #     title = link.text
+    #     doc_url = DOMAIN + link[:href]
+    #     status = h.at("td[4]").text.strip.downcase
+    #     release_date = Date.strptime h.at("td[5]").text.strip, "%m/%d/%Y"
+    #     Hit.new(
+    #       {
+    #         code: code, serie: serie, title: title, url: doc_url,
+    #         status: status, release_date: release_date
+    #       }, self
+    #     )
+    #   end
+    # end
     # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
     # Fetches data form json
-    # @param docid [String]
     # @param stage [String]
     # @return [Array<RelatonNist::Hit>]
-    def from_json(docid, **opts)
-      select_data(docid, **opts).map do |h|
+    def from_json(**opts)
+      select_data(**opts).map do |h|
         /(?<serie>(?<=-)\w+$)/ =~ h["series"]
         title = [h["title-main"], h["title-sub"]].compact.join " - "
         release_date = RelatonBib.parse_date h["published-date"], false
@@ -91,22 +106,20 @@ module RelatonNist
       end
     end
 
-    # @param docid [String]
     # @param stage [String]
     # @return [Array<Hach>]
-    def select_data(docid, **opts) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength,Metrics/PerceivedComplexity
-      # ref = docid.sub(/(?<=\d{3}-\d{2})r(\d+)/, ' Rev. \1')
+    def select_data(**opts) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength,Metrics/PerceivedComplexity
       d = Date.strptime year, "%Y" if year
-      # didrx = Regexp.new(docid)
+      statuses = %w[draft-public draft-prelim]
       data.select do |doc|
         next unless match_year?(doc, d)
 
         if /PD/.match? opts[:stage]
-          next unless %w[draft-public draft-prelim].include? doc["status"]
+          next unless statuses.include? doc["status"]
         else
           next unless doc["status"] == "final"
         end
-        doc["docidentifier"].include? docid
+        doc["docidentifier"].include? text
       end
     end
 
@@ -134,8 +147,8 @@ module RelatonNist
     #
     # @prarm ctime [Time, NilClass]
     def fetch_data(ctime)
-      resp = OpenURI.open_uri("#{PUBS_EXPORT}.meta")
-      if !ctime || ctime < resp.last_modified
+      # resp = OpenURI.open_uri("#{PUBS_EXPORT}.meta")
+      if !ctime || ctime < OpenURI.open_uri("#{PUBS_EXPORT}.meta").last_modified
         @data = nil
         uri_open = URI.method(:open) || Kernel.method(:open)
         FileUtils.mkdir_p DATAFILEDIR unless Dir.exist? DATAFILEDIR