relaton-nist 1.9.2 → 1.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 76cfec1954dfd58bf622a6a47e34bdbcfe4cf471106eb7ad19582654078a6d38
-  data.tar.gz: 9769f406d5b98840b5bc099d78ab811388170d1fae5872211a7a0c9ed26d869f
+  metadata.gz: ea210109d9e306b8c9c6b4395812b52454dad6b8fdbd87cc08ec750e9dd2d4ac
+  data.tar.gz: e2dad0af3e66b42b2b5d5f2ecf48653ea21b2662d9bed9f6003a5b7a6d953f1d
 SHA512:
-  metadata.gz: 0e3deefff699c77103afb631a03283697dc05f5c952b60b3602961df37df2095102b16c32e229aeeceb76da12cfda09e43e6b0788b6a4c1f02b74dea8a49f818
-  data.tar.gz: 3244707d2595b5b45bab973399a04b8cc791d3b008e2ef05fcc99d9355cec6915556aea8528e5f992e4ba0452af82bda09fb93737741a72c08a64e7a0039d161
+  metadata.gz: '08a75dfaf1616874bab1c6edd2eb9a1f80c64a2d8d052dd66a3f4ca4f5b8aac1ee520c397f62327d718865fbd5f476db0e9a4ebef445ae3ecbe6f0366ee75e66'
+  data.tar.gz: cae6a5c20af9c77ead564ee5337d3092356a1786337406c260e75a387cc0446bd9937a23b2a24d1eafcc5b0bb1e98a621288e091634b7d950d112d96f13019cf

data/README.adoc

@@ -147,6 +147,17 @@ item.docidentifier.first.id
 => "SP 800-38A-Add"
 ----
 
+=== Typed links
+
+NIST documents may have `src` and `doi` link types.
+
+[source,ruby]
+----
+item.link
+=> [#<RelatonBib::TypedUri:0x00007f901971dc10 @content=#<Addressable::URI:0x62c URI:https://csrc.nist.gov/publications/detail/sp/800-67/rev-2/final>, @type="src">,
+ #<RelatonBib::TypedUri:0x00007f901971d6e8 @content=#<Addressable::URI:0x640 URI:https://doi.org/10.6028/NIST.SP.800-67r2>, @type="doi">]
+----
+
 === Create bibliographic item from YAML
 [source,ruby]
 ----
@@ -167,7 +178,7 @@ The method `RelatonNist::DataFetcher.fetch(output: "data", format: "yaml")` fetc
 Arguments:
 
 - `output` - folder to save documents (default './data').
-- `format` - format in which the documents are saved. Possimle formats are: `yaml`, `xml` (default `yaml`).
+- `format` - format in which the documents are saved. Possimle formats are: `yaml`, `xml`, `bibxxml` (default `yaml`).
 
 [source,ruby]
 ----

data/lib/relaton_nist/data_fetcher.rb

@@ -1,12 +1,6 @@
 # frozen_string_literal: true
 
-# require 'English'
-# require 'mechanize'
-# require "fileutils"
 require "yaml"
-# require "open-uri"
-# require "nokogiri"
-# require "relaton_nist"
 
 module RelatonNist
   class DataFetcher
@@ -23,6 +17,7 @@ module RelatonNist
     def initialize(output, format)
       @output = output
       @format = format
+      @ext = format.sub(/^bib/, "")
     end
 
     def parse_docid(doc)
@@ -151,7 +146,7 @@ module RelatonNist
     #
     def write_file(bib) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
       id = bib.docidentifier[0].id.gsub(%r{[/\s:.]}, "_").upcase.sub(/^NIST_IR/, "NISTIR")
-      file = File.join(@output, "#{id}.#{@format}")
+      file = File.join(@output, "#{id}.#{@ext}")
       if File.exist? file
         warn "File #{file} exists. Docid: #{bib.docidentifier[0].id}"
         # warn "Link: #{bib.link.detect { |l| l.type == 'src' }.content}"
@@ -159,6 +154,7 @@ module RelatonNist
         output = case @format
                  when "yaml" then bib.to_hash.to_yaml
                  when "xml" then bib.to_xml bibdata: true
+                 else bib.send "to_#{@format}"
                  end
         File.write file, output, encoding: "UTF-8"
       end
@@ -197,7 +193,7 @@ module RelatonNist
 
       docs = Nokogiri::XML OpenURI.open_uri URL
       FileUtils.mkdir @output unless Dir.exist? @output
-      FileUtils.rm Dir[File.join(@output, "*.#{@format}")]
+      FileUtils.rm Dir[File.join(@output, "*.#{@ext}")]
       docs.xpath("/body/query/doi_record/report-paper/report-paper_metadata")
         .each { |doc| parse_doc doc }
 
@@ -212,7 +208,7 @@ module RelatonNist
     # Fetch all the documnts from dataset
     #
     # @param [String] output foldet name to save the documents
-    # @param [String] format format to save the documents
+    # @param [String] format format to save the documents (yaml, xml, bibxml)
     #
     def self.fetch(output: "data", format: "yaml")
       new(output, format).fetch

data/lib/relaton_nist/processor.rb

@@ -2,7 +2,7 @@ require "relaton/processor"
 
 module RelatonNist
   class Processor < Relaton::Processor
-    def initialize
+    def initialize # rubocop:disable Lint/MissingSuper
       @short = :relaton_nist
       @prefix = "NIST"
       @defaultprefix = %r{^(NIST|NISTGCR|ITL Bulletin|JPCRD|NISTIR|CSRC|FIPS)(/[^\s])?\s}
@@ -18,6 +18,14 @@ module RelatonNist
       ::RelatonNist::NistBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the documents from a source
+    #
+    # @param [String] _source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
     def fetch_data(_source, opts)
       DataFetcher.fetch(**opts)
     end

data/lib/relaton_nist/scrapper.rb

@@ -62,44 +62,10 @@ module RelatonNist
       # Fetch status.
       # @param doc [Hash]
       # @return [RelatonNist::DocumentStatus]
-      def fetch_status(doc) # , status)
-        # if doc.is_a? Hash
+      def fetch_status(doc)
         stage = doc["status"]
         subst = doc["substage"]
         iter = doc["iteration"] == "initial" ? 1 : doc["iteration"]
-        # else
-        #   case status
-        #   when "draft (obsolete)"
-        #     stage = "draft-public"
-        #     subst = "withdrawn"
-        #   when "retired draft"
-        #     stage = "draft-public"
-        #     subst = "retired"
-        #   when "withdrawn"
-        #     stage = "final"
-        #     subst = "withdrawn"
-        #   when /^draft/
-        #     stage = "draft-public"
-        #     subst = "active"
-        #   else
-        #     stage = status
-        #     subst = "active"
-        #   end
-
-        #   iter = nil
-        #   if stage.include? "draft"
-        #     iter = 1
-        #     history = doc.xpath("//span[@id='pub-history-container']/a"\
-        #                         "|//span[@id='pub-history-container']/span")
-        #     history.each_with_index do |h, idx|
-        #       next if h.name == "a"
-
-        #       iter = idx + 1 if idx.positive?
-        #       break
-        #     end
-        #   end
-        # end
-
         RelatonNist::DocumentStatus.new stage: stage, substage: subst, iteration: iter.to_s
       end
 
@@ -132,7 +98,6 @@ module RelatonNist
         dates
       end
 
-      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       # @param doc [Hash]
       # @return [Array<RelatonBib::ContributionInfo>]
       def fetch_contributors(doc)
@@ -144,23 +109,12 @@ module RelatonNist
         contribs + contributors_json(
           doc["editors"], "editor", doc["language"], doc["script"]
         )
-        # else
-        #   name = "National Institute of Standards and Technology"
-        #   org = RelatonBib::Organization.new(
-        #     name: name, url: "www.nist.gov", abbreviation: "NIST",
-        #   )
-        #   contribs << RelatonBib::ContributionInfo.new(entity: org, role: [type: "publisher"])
-        #   authors = doc.at('//h4[.="Author(s)"]/following-sibling::p')
-        #   contribs += contributors(authors, "author")
-        #   editors = doc.at('//h4[.="Editor(s)"]/following-sibling::p')
-        #   contribs + contributors(editors, "editor")
-        # end
       end
 
       # @param doc [Array<Hash>]
       # @param role [String]
       # @return [Array<RelatonBib::ContributionInfo>]
-      def contributors_json(doc, role, lang = "en", script = "Latn")
+      def contributors_json(doc, role, lang = "en", script = "Latn") # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
         doc.map do |contr|
           if contr["affiliation"]
             if contr["affiliation"]["acronym"]
@@ -185,43 +139,6 @@ module RelatonNist
         end.compact
       end
 
-      # rubocop:disable Metrics/CyclomaticComplexity
-      # @param doc [Nokogiri::HTML::Element, Array<Hash>]
-      # @param role [String]
-      # @return [Array<RelatonBib::ContributionInfo>]
-      # def contributors(doc, role, lang = "en", script = "Latn")
-      #   return [] if doc.nil?
-
-      #   doc.text.split(", ").map do |contr|
-      #     /(?<an>.+?)(\s+\((?<abbrev>.+?)\))?$/ =~ contr.strip
-      #     if abbrev && an.downcase !~ /(task|force|group)/ && an.split.size.between?(2, 3)
-      #       fullname = RelatonBib::FullName.new(
-      #         completename: RelatonBib::LocalizedString.new(an, lang, script)
-      #       )
-      #       case abbrev
-      #       when "NIST"
-      #         org_name = "National Institute of Standards and Technology"
-      #         url = "www.nist.gov"
-      #       when "MITRE"
-      #         org_name = abbrev
-      #         url = "www.mitre.org"
-      #       else
-      #         org_name = abbrev
-      #         url = nil
-      #       end
-      #       org = RelatonBib::Organization.new name: org_name, url: url, abbreviation: abbrev
-      #       affiliation = RelatonBib::Affiliation.new organization: org
-      #       entity = RelatonBib::Person.new(
-      #         name: fullname, affiliation: [affiliation],
-      #       )
-      #     else
-      #       entity = RelatonBib::Organization.new name: an, abbreviation: abbrev
-      #     end
-      #     RelatonBib::ContributionInfo.new entity: entity, role: [type: role]
-      #   end
-      # end
-      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/AbcSize, Metrics/MethodLength
-
       # @param name [Hash]
       # @param lang [Strong]
       # @param script [String]
@@ -253,80 +170,30 @@ module RelatonNist
         return unless doc["edition"]
 
         rev = doc["edition"]
-        # else
-        #   return unless /(?<=Rev\.\s)(?<rev>\d+)/ =~ doc
-        # end
-
         "Revision #{rev}"
       end
 
-      # Fetch abstracts.
-      # @param doc [Nokigiri::HTML::Document]
-      # @return [Array<Hash>]
-      # def fetch_abstract(doc)
-      #   abstract_content = doc.xpath(
-      #     '//div[contains(@class, "pub-abstract-callout")]/div[1]/p',
-      #   ).text
-      #   [{
-      #     content: abstract_content,
-      #     language: "en",
-      #     script: "Latn",
-      #     format: "text/plain",
-      #   }]
-      # end
-
       # Fetch copyright.
       # @param doc [Nokogiri::HTL::Document, String]
       # @return [Array<Hash>]
       def fetch_copyright(doc)
         name = "National Institute of Standards and Technology"
         url = "www.nist.gov"
-        # d = if doc.is_a? String then doc
-        #     else
-        #       doc.at("//span[@id='pub-release-date']")&.text&.strip
-        #     end
         from = doc&.match(/\d{4}/)&.to_s
         [{ owner: [{ name: name, abbreviation: "NIST", url: url }], from: from }]
       end
 
-      # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-
       # Fetch links.
       # @param doc [Hash]
       # @return [Array<Hash>]
       def fetch_link(doc)
         links = []
-        # if doc.is_a? Hash
-        links << { type: "uri", content: doc["uri"] } if doc["uri"]
-        doi = "https://doi.org/" + doc["doi"] if doc["doi"]
-        # else
-        #   pub = doc.at "//p/strong[contains(., 'Publication:')]"
-        #   pdf = pub&.at "./following-sibling::a[.=' Local Download']"
-        #   doi = pub&.at("./following-sibling::a[contains(.,'(DOI)')]")&.attr :href
-        #   links << { type: "pdf", content: pdf[:href] } if pdf
-        # end
-        links << { type: "doi", content: doi } if doi
+        links << { type: "src", content: doc["uri"] } if doc["uri"]
+        if doc["doi"]
+          links << { type: "doi", content: "https://doi.org/#{doc['doi']}" }
+        end
         links
       end
-      # rubocop:enable Metrics/MethodLength
-
-      # Fetch relations.
-      # @param doc [Nokogiri::HTML::Document]
-      # @return [Array<RelatonNist::DocumentRelation>]
-      # def fetch_relations(doc)
-      #   relations = doc.xpath('//span[@id="pub-supersedes-container"]/a').map do |r|
-      #     doc_relation "supersedes", r.text, DOMAIN + r[:href]
-      #   end
-
-      #   relations += doc.xpath('//span[@id="pub-part-container"]/a').map do |r|
-      #     doc_relation "partOf", r.text, DOMAIN + r[:href]
-      #   end
-
-      #   relations + doc.xpath('//span[@id="pub-related-container"]/a').map do |r|
-      #     doc_relation "updates", r.text, DOMAIN + r[:href]
-      #   end
-      # end
-      # rubocop:enable Metrics/AbcSize
 
       def fetch_relations_json(doc)
         relations = doc["supersedes"].map do |r|
@@ -354,68 +221,12 @@ module RelatonNist
         )
       end
 
-      # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-
-      # @param doc [Nokogiri::HTML::Document]
-      # @return [Array<RelatonBib::Series>]
-      # def fetch_series(doc)
-      #   series = doc.xpath "//span[@id='pub-history-container']/a"\
-      #     "|//span[@id='pub-history-container']/span"
-      #   series.map.with_index do |s, idx|
-      #     next if s.name == "span"
-
-      #     iter = if idx.zero? then "I"
-      #            else idx + 1
-      #            end
-
-      #     content = s.text.match(/^[^\(]+/).to_s.strip.squeeze " "
-
-      #     ref = case s.text
-      #           when /^Draft/
-      #             content.match(/(?<=Draft\s).+/).to_s + " (#{iter}PD)"
-      #           when /\(Draft\)/ then content + " (#{iter}PD)"
-      #           else content
-      #           end
-
-      #     fref = RelatonBib::FormattedRef.new(
-      #       content: ref, language: "en", script: "Latn", format: "text/plain",
-      #     )
-      #     RelatonBib::Series.new(formattedref: fref)
-      #   end.select { |s| s }
-      # end
-      # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
-
       # @param doc [Hash]
       # @return [Array<RelatonNist::Keyword>]
       def fetch_keywords(doc)
-        # kws = if doc.is_a? Hash
-        #         doc["keywords"]
-        #       else
-        #         doc.xpath "//span[@id='pub-keywords-container']/span"
-        #       end
         doc["keywords"].map { |kw| kw.is_a?(String) ? kw : kw.text }
       end
 
-      # rubocop:disable Metrics/AbcSize
-      # @param doc [Nokogiri::HTML::Document]
-      # @return [RelatonNist::CommentPeriod, NilClass]
-      # def fetch_commentperiod(doc)
-      #   cp = doc.at "//span[@id='pub-comments-due']"
-      #   return unless cp
-
-      #   to = Date.strptime cp.text.strip, "%B %d, %Y"
-
-      #   d = doc.at("//span[@id='pub-release-date']").text.strip
-      #   from = Date.strptime(d, "%B %Y").to_s
-
-      #   ex = doc.at "//strong[contains(.,'The comment closing date has been "\
-      #   "extended to')]"
-      #   ext = ex&.text&.match(/\w+\s\d{2},\s\d{4}/).to_s
-      #   extended = ext.empty? ? nil : Date.strptime(ext, "%B %d, %Y")
-      #   CommentPeriod.new from: from, to: to, extended: extended
-      # end
-      # rubocop:enable Metrics/AbcSize
-
       # @param json [Hash]
       # @return [RelatonNist::CommentPeriod, NilClass]
       def fetch_commentperiod_json(json)

data/lib/relaton_nist/version.rb

@@ -1,3 +1,3 @@
 module RelatonNist
-  VERSION = "1.9.2".freeze
+  VERSION = "1.9.3".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-nist
 version: !ruby/object:Gem::Version
-  version: 1.9.2
+  version: 1.9.3
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-09-03 00:00:00.000000000 Z
+date: 2021-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml