nistbib 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 93e42e10363b0cf5ace10ea2540e20a7d96bacfa
-  data.tar.gz: 958b66ae4601efb35dbd366a80b5c09ea22a50bd
+  metadata.gz: 896a6c8fd4132200e685aa768a3bab6f089f9f4b
+  data.tar.gz: f399bbd50fcbe7cb1f70d4dee3fb37619f5b051a
 SHA512:
-  metadata.gz: 4fb26a328169066da7e921df16b525a67baa6fc5b8ddebb14fbcb08af41decddd6e27b40a173aa9015972e9a22f844ceea4ee7dd711444600efac36a240a49eb
-  data.tar.gz: e71f295627f4f640f81432508783365ccbc868cc98f277ecb153e920a9e62339d6cf79f81f64d8ee731b09ede32414d3322a9ab162dce416b543fc040da0055b
+  metadata.gz: 28bde9d7d3a30d628a53e0f06a4444859b870aea0a65fcab7ca9315e2330f0a86017b1bce5ea74636ea6b8480e26b294816319f338c1a2081cdaceb0d56b833a
+  data.tar.gz: 5ba05e4c1ecb61b89a05a67c042ec4ce32119f23a139aca620c2715c1177d1eb75c4482a433e2dc5913d910f53c12483fad737b4b1c4b3e8aeea694dfa367fb2

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    nistbib (0.1.5)
-      relaton-bib (~> 0.1.1)
+    nistbib (0.1.6)
+      relaton-bib (~> 0.1.5)
 
 GEM
   remote: https://rubygems.org/
@@ -34,7 +34,7 @@ GEM
       pry (~> 0.10)
     public_suffix (3.0.3)
     rake (10.5.0)
-    relaton-bib (0.1.3)
+    relaton-bib (0.1.5)
       addressable
       nokogiri (~> 1.8.4)
     rspec (3.8.0)

data/README.adoc

@@ -48,13 +48,35 @@ hit_collection = NistBib::NistBibliography.search("8200")
 [source,ruby]
 ----
 item.to_xml
-=> "<nist-standard>
+=> "<bibitem id="NISTIR8200(DRAFT)" type="standard">
       <fetched>2019-05-03</fetched>
       <title format="text/plain" language="en" script="Latn">
         Interagency Report on Status of International Cybersecurity Standardization for the Internet of Things (IoT)
       </title>
       ...
-    </nist-standard>"
+    <bibitem>"
+----
+With argument `bibdata: true` it ouputs XML wrapped by `bibdata` element and adds flavour `ext` element.
+[source,ruby]
+----
+item.to_xml bibdata: true
+=> "<bibdata type="standard">
+    <fetched>2019-05-17</fetched>
+    <title format="text/plain" language="en" script="Latn">
+      Interagency Report on Status of International Cybersecurity Standardization for the Internet of Things (IoT)
+    </title>
+    ...
+    <ext>
+      <doctype>stadard</doctype>
+      <keyword>cybersecurity</keyword>
+      <keyword>cybersecurity objectives</keyword>
+      ...
+      <commentperiod>
+        <from>2018-02-01</from>
+        <to>2018-04-18</to>
+      </commentperiod>
+    </ext>
+  </bibdata>"
 ----
 
 === Get code, and year

data/lib/nistbib/document_status.rb

@@ -1,20 +1,11 @@
 module NistBib
-  class DocumentStatus
+  class DocumentStatus < RelatonBib::DocumentStatus
     STAGES = %w[
       draft-internal draft-wip draft-prelim draft-public final final-review
     ].freeze
 
     SUBSTAGES = %w[active retired withdrawn].freeze
 
-    # @return [String]
-    attr_reader :stage
-
-    # @return [String, NilClass]
-    attr_reader :substage
-
-    # @return [String, NilClass]
-    attr_reader :iteration
-
     # @param stage [String]
     # @param substage [String, NilClass]
     # @param iteration [String, NilClass]
@@ -27,18 +18,7 @@ module NistBib
         raise ArgumentError, "invalid argument: substage (#{substage})"
       end
 
-      @stage = stage
-      @substage = substage
-      @iteration = iteration
-    end
-
-    # @param builder [Nokogiri::XML::Builder]
-    def to_xml(builder)
-      builder.status do
-        builder.stage stage
-        builder.substage substage if substage
-        builder.iteration iteration if iteration
-      end
+      super
     end
   end
 end

data/lib/nistbib/hit.rb

@@ -37,8 +37,8 @@ module NistBib
     end
 
     # @return [String]
-    def to_xml
-      fetch.to_xml
+    def to_xml(**opts)
+      fetch.to_xml **opts
     end
   end
 end

data/lib/nistbib/hit_collection.rb

@@ -23,6 +23,7 @@ module NistBib
     # @param ref_nbr [String]
     # @param year [String]
     # @param opts [Hash]
+    # @option opts [String] :stage
     def initialize(ref_nbr, year = nil, opts)
       @text = ref_nbr
       @year = year
@@ -35,7 +36,7 @@ module NistBib
       url  = "#{DOMAIN}/publications/search?keywords-lg=#{ref_nbr}"
       url += "&dateFrom-lg=#{from}" if from
       url += "&dateTo-lg=#{to}" if to
-      url += "&status-lg=Draft" if /PD/ =~ opts[:stage]
+      url += "&status-lg=Draft,Withdrawn" if /PD/ =~ opts[:stage]
       doc  = Nokogiri::HTML OpenURI.open_uri(::Addressable::URI.parse(url).normalize)
       hits = doc.css("table.publications-table > tbody > tr").map do |h|
         link  = h.at("td/div/strong/a")

data/lib/nistbib/nist_bibliographic_item.rb

@@ -1,7 +1,7 @@
 module NistBib
   class NistBibliographicItem < RelatonBib::BibliographicItem
-    # @return [NistBib::NistSeries, NilClass]
-    attr_reader :nistseries
+    # @return [String]
+    attr_reader :doctype
 
     # @return [Array<NistBib::Keyword>]
     attr_reader :keyword
@@ -29,6 +29,7 @@ module NistBib
     # @param classification [RelatonBib::Classification, NilClass]
     # @param validity [RelatonBib:Validity, NilClass]
     # @param fetched [Date, NilClass] default nil
+    # @param doctype [String]
     # @param keyword [Array<NistBib::Keyword>]
     # @param commentperiod [NistBib::CommentPeriod]
     #
@@ -54,21 +55,23 @@ module NistBib
     # @option relations [String] :type
     # @option relations [RelatonBib::BibliographicItem] :bibitem
     # @option relations [Array<RelatonBib::BibItemLocality>] :bib_locality
-    #
-    # @param nistseries [Nist::NistSeries, NilClass]
     def initialize(**args)
-      @nistseries = args.delete :nistseries
+      @doctype = "stadard"
       @keyword = args.delete(:keyword) || []
       @commentperiod = args.delete :commentperiod
       super
     end
 
     # @param builder [Nokogiri::XML::Builder]
-    def to_xml(builder = nil)
-      super builder, root: :bibitem, date_format: :short do |b|
-        nistseries&.to_xml b
-        keyword.each { |kw| kw.to_xml b }
-        commentperiod&.to_xml b
+    def to_xml(builder = nil, **opts)
+      super builder, date_format: :short, **opts do |b|
+        if opts[:bibdata]
+          b.ext do
+            b.doctype doctype
+            keyword.each { |kw| kw.to_xml b }
+            commentperiod&.to_xml b
+          end
+        end
       end
     end
   end

data/lib/nistbib/nist_bibliography.rb

@@ -2,7 +2,6 @@ require "relaton_bib"
 require "nistbib/nist_bibliographic_item"
 require "nistbib/scrapper"
 require "nistbib/hit_collection"
-require "nistbib/nist_series"
 require "nistbib/xml_parser"
 require "nistbib/keyword"
 require "nistbib/comment_period"
@@ -22,10 +21,15 @@ module NistBib
 
       # @param code [String] the NIST standard Code to look up (e..g "8200")
       # @param year [String] the year the standard was published (optional)
-      # @param opts [Hash] options; restricted to :all_parts if all-parts reference is required
+      #
+      # @param opts [Hash] options
+      # @option opts [TrueClass, FalseClass] :all_parts restricted to all parts
+      #   if all-parts reference is required
+      # @option opts [TrueClass, FalseClass] :bibdata
+      #
       # @return [String] Relaton XML serialisation of reference
       def get(code, year = nil, opts = {})
-        /^(?<code2>[^\(]+)\((?<date2>\w+\s(\d{2},\s)?\d{4})\)(\s\()?(?<stage>[^\)]+)?/ =~ code
+        /^(?<code2>[^\(]+)(\((?<date2>\w+\s(\d{2},\s)?\d{4})\))?\s?\(?((?<=\()(?<stage>[^\)]+))?/ =~ code
         if code2
           code = code2.strip
           if date2
@@ -57,7 +61,7 @@ module NistBib
       private
 
       def nistbib_get1(code, year, opts)
-        result = nistbib_search_filter(code, year, opts) or return nil
+        result = nistbib_search_filter(code, year, opts) || (return nil)
         ret = nistbib_results_filter(result, year, opts)
         return ret[:ret] if ret[:ret]
 
@@ -70,19 +74,35 @@ module NistBib
       # Only expects the first page of results to be populated.
       # Does not match corrigenda etc (e.g. ISO 3166-1:2006/Cor 1:2007)
       # If no match, returns any years which caused mismatch, for error reporting
+      #
+      # @param opts [Hash] options
+      # @option opts [Time] :issued_date
+      # @option opts [Time] :issued_date
+      # @option opts [String] :stage
+      #
+      # @retur [Hash]
       def nistbib_results_filter(result, year, opts)
         missed_years = []
         result.each_slice(3) do |s| # ISO website only allows 3 connections
           fetch_pages(s, 3).each_with_index do |r, _i|
             if opts[:issued_date]
               r.dates.select { |d| d.type == "issued" }.each do |d|
-                return { ret: r } if opts[:issued_date] == d.on
+                next unless opts[:issued_date] == d.on
               end
             elsif opts[:updated_date]
               r.dates.select { |d| d.type == "published" }.each do |d|
-                return { ret: r } if opts[:updated_date] == d.on
+                next unless opts[:updated_date] == d.on
               end
             end
+            if opts[:stage]
+              iter = opts[:stage][-3]
+              iteration = case iter
+                          when "I" then 1
+                          when "F" then "final"
+                          else iter.to_i
+                          end
+              next if iter && r.status.iteration != iteration
+            end
             return { ret: r } if !year
 
             r.dates.select { |d| d.type == "published" }.each do |d|

data/lib/nistbib/scrapper.rb

@@ -15,20 +15,20 @@ module NistBib
         NistBibliographicItem.new(
           fetched: Date.today.to_s,
           type: "standard",
-          id: fetch_id(doc),
+          # id: fetch_id(doc),
           titles: fetch_titles(hit_data),
           link: fetch_link(doc),
           docid: fetch_docid(doc),
           dates: fetch_dates(doc, hit_data[:release_date]),
           contributors: fetch_contributors(doc),
-          edition: nil,
+          edition: fetch_edition(hit_data[:code]),
           language: ["en"],
           script: ["Latn"],
           abstract: fetch_abstract(doc),
           docstatus: fetch_status(doc, hit_data[:status]),
           copyright: fetch_copyright(doc),
           relations: fetch_relations(doc),
-          nistseries: fetch_nistseries(doc),
+          series: fetch_series(doc),
           keyword: fetch_keywords(doc),
           commentperiod: fetch_commentperiod(doc),
         )
@@ -47,7 +47,7 @@ module NistBib
 
       # Fetch docid.
       # @param doc [Nokogiri::HTML::Document]
-      # @return [Hash]
+      # @return [Array<RelatonBib::DocumentIdentifier>]
       def fetch_docid(doc)
         item_ref = doc.at("//div[contains(@class, 'publications-detail')]/h3").
           text.strip
@@ -59,10 +59,10 @@ module NistBib
       # Fetch id.
       # @param doc [Nokogiri::HTML::Document]
       # @return [String]
-      def fetch_id(doc)
-        doc.at("//div[contains(@class, 'publications-detail')]/h3").text.
-          strip.gsub(/\s/, "")
-      end
+      # def fetch_id(doc)
+      #   doc.at("//div[contains(@class, 'publications-detail')]/h3").text.
+      #     strip.gsub(/\s/, "")
+      # end
 
       # Fetch status.
       # @param doc [Nokogiri::HTML::Document]
@@ -72,19 +72,36 @@ module NistBib
         case status
         when "draft (withdrawn)"
           stage = "draft-public"
-          substage = "withdrawn"
+          subst = "withdrawn"
         when "retired draft"
           stage = "draft-public"
-          substage = "retired"
+          subst = "retired"
         when "withdrawn"
           stage = "final"
-          substage = "withdrawn"
+          subst = "withdrawn"
         when "draft"
           stage = "draft-public"
-          substage = "active"
+          subst = "active"
         else
           stage = status
-          substage = "active"
+          subst = "active"
+        end
+
+        iter = nil
+        if stage.include? "draft"
+          iter = 1
+          history = doc.xpath("//span[@id='pub-history-container']/a"\
+            "|//span[@id='pub-history-container']/span")
+          history.each_with_index do |h, idx|
+            next if h.name == "a"
+
+            iter = idx + 1 if idx.positive?
+            # iter = if lsif idx < (history.size - 1) && !history.last.text.include?("Draft")
+            #          "final"
+            #        elsif idx.positive? then idx + 1
+            #        end
+            break
+          end
         end
 
         # if doc.at "//p/strong[text()='Withdrawn:']"
@@ -97,7 +114,7 @@ module NistBib
         #   wip = item_ref.match(/(?<=\()\w+/).to_s
         #   stage = "draft-public" if wip == "DRAFT"
         # end
-        NistBib::DocumentStatus.new(stage: stage, substage: substage)
+        NistBib::DocumentStatus.new stage: stage, substage: subst, iteration: iter
       end
 
       # Fetch titles.
@@ -140,6 +157,7 @@ module NistBib
         contribs + contributors(editors, "editor")
       end
 
+      # rubocop:disable Metrics/CyclomaticComplexity
       def contributors(doc, role)
         return [] if doc.nil?
 
@@ -171,6 +189,13 @@ module NistBib
           RelatonBib::ContributionInfo.new entity: entity, role: [role]
         end
       end
+      # rubocop:enable Metrics/CyclomaticComplexity
+
+      def fetch_edition(code)
+        return unless /(?<=Rev\.\s)(?<rev>\d+)/ =~ code
+
+        "Revision #{rev}"
+      end
 
       # Fetch abstracts.
       # @param doc [Nokigiri::HTML::Document]
@@ -238,13 +263,28 @@ module NistBib
         )
       end
 
-      def fetch_nistseries(doc)
-        doc.xpath("//p/strong[.='Document History:']/following-sibling::*[not(self::br)]").each do |s|
-          NistSeries::ABBREVIATIONS.each do |k, _v|
-            return NistSeries.new k if s.text.include? k
-          end
-        end
-        nil
+      def fetch_series(doc)
+        series = doc.xpath "//span[@id='pub-history-container']/a"\
+          "|//span[@id='pub-history-container']/span"
+        series.map.with_index do |s, idx|
+          next if s.name == "span"
+
+          iter = if idx.zero? then "I"
+                #  elsif status == "final" && idx == (series.size - 1) then "F"
+                 else idx + 1
+                 end
+
+          content = s.text.match(/^[^\(]+/).to_s.strip.gsub "  ", " "
+
+          ref = case content.match(/\w+/).to_s
+                when "Draft" then content.match(/(?<=Draft\s).+/).to_s + " (#{iter}PD)"
+                end
+
+          fref = RelatonBib::FormattedRef.new(
+            content: ref, language: "en", script: "Latn", format: "text/plain",
+          )
+          RelatonBib::Series.new(formattedref: fref)
+        end.select { |s| s }
       end
 
       def fetch_keywords(doc)

data/lib/nistbib/version.rb

@@ -1,3 +1,3 @@
 module NistBib
-  VERSION = "0.1.5".freeze
+  VERSION = "0.1.6".freeze
 end

data/lib/nistbib/xml_parser.rb

@@ -3,7 +3,7 @@ module NistBib
     class << self
       def from_xml(xml)
         doc = Nokogiri::XML xml
-        nistitem = doc.at("/bibitem") || doc.at("/bibdata")
+        nistitem = doc.at("/bibitem|/bibdata")
         NistBibliographicItem.new(item_data(nistitem))
       end
 
@@ -11,10 +11,11 @@ module NistBib
 
       def item_data(nistitem)
         data = super
-        data.delete :series
-        data[:nistseries] = fetch_nistseries(nistitem)
-        data[:keyword] = fetch_keyword(nistitem)
-        data[:commentperiod] = fetch_commentperiod(nistitem)
+        ext = nistitem.at "./ext"
+        return data unless ext
+
+        data[:keyword] = fetch_keyword(ext)
+        data[:commentperiod] = fetch_commentperiod(ext)
         data
       end
 
@@ -25,7 +26,7 @@ module NistBib
         DocumentStatus.new(
           stage: status.at("stage")&.text,
           substage: status.at("substage")&.text,
-          iteration: status.at("iteraton")&.text,
+          iteration: status.at("iteration")&.text,
         )
       end
 
@@ -41,13 +42,6 @@ module NistBib
           Keyword.new kw.children.first.to_xml
         end
       end
-
-      def fetch_nistseries(item)
-        series = item.at "./series"
-        return unless series
-
-        NistSeries.new series.at("./abbreviation").text.upcase.gsub("-", " ")
-      end
     end
   end
 end

data/lib/relaton/processor.rb

@@ -11,7 +11,7 @@ module Relaton
         @idtype = "NIST"
       end
 
-      def get(code, date, opts)
+      def get(code, date = nil, opts = {})
         ::NistBib::NistBibliography.get(code, date, opts)
       end
 

data/nistbib.gemspec

@@ -34,5 +34,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "vcr"
   spec.add_development_dependency "webmock"
 
-  spec.add_dependency "relaton-bib", "~> 0.1.1"
+  spec.add_dependency "relaton-bib", "~> 0.1.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nistbib
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-05-13 00:00:00.000000000 Z
+date: 2019-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -170,14 +170,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: 0.1.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: 0.1.5
 description: 'NistBib: retrive NIST standards.'
 email:
 - open.source@ribose.com
@@ -205,7 +205,6 @@ files:
 - lib/nistbib/keyword.rb
 - lib/nistbib/nist_bibliographic_item.rb
 - lib/nistbib/nist_bibliography.rb
-- lib/nistbib/nist_series.rb
 - lib/nistbib/scrapper.rb
 - lib/nistbib/version.rb
 - lib/nistbib/workers_pool.rb

data/lib/nistbib/nist_series.rb

@@ -1,35 +0,0 @@
-module NistBib
-  class NistSeries
-    ABBREVIATIONS = {
-      "NISTIR" => "NIST Interagency/Internal Report",
-      "NIST SP" => "NIST Special Publication",
-      "NIST FIPS" => "NIST Federal Information Processing Standards",
-    }.freeze
-
-    # @return [String]
-    attr_reader :type
-
-    # @return [String]
-    attr_reader :title
-
-    # @return [RalatonBib::LocalizedString, NilClass]
-    attr_reader :abbreviation
-
-    # @param abbreviation [RelatonBib::LocalizedString, NilClass]
-    def initialize(abbreviation)
-      @type = "main"
-      @abbreviation = RelatonBib::LocalizedString.new(
-        abbreviation.downcase.gsub(" ", "-"), "en", "Latn"
-      )
-      @title = ABBREVIATIONS[abbreviation]
-    end
-
-    # @param builder [Nokogiri::XML::Builder]
-    def to_xml(builder)
-      builder.series type: type do
-        builder.title title
-        builder.abbreviation { abbreviation.to_xml builder } if abbreviation
-      end
-    end
-  end
-end