nistbib 0.1.2 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 434eb7054df007c7604456b7770b728f5396b6bba71fd0f9e922147d0522962d
-  data.tar.gz: b486651f96863f6d4b5c3b3e8e731fbe16d056188a1fe072108e9c368b057d4a
+SHA1:
+  metadata.gz: 93e42e10363b0cf5ace10ea2540e20a7d96bacfa
+  data.tar.gz: 958b66ae4601efb35dbd366a80b5c09ea22a50bd
 SHA512:
-  metadata.gz: f6b366f616b619aaa927a4312af17ca9eb9c2928ec7483acc4dcb3b6f4c5ddc858656ebbe72ea49e98cff7c42e3476aa4afcc5fb934bce5a7c21b3ca3a56ab34
-  data.tar.gz: 040e6331167a4c779c61d348fb230947c145ddb064d2359e3ded3dafd898d46d443cbbe44fac0d3f092d2e47c0151c4a468a0558ee1706076aaaec259ece416c
+  metadata.gz: 4fb26a328169066da7e921df16b525a67baa6fc5b8ddebb14fbcb08af41decddd6e27b40a173aa9015972e9a22f844ceea4ee7dd711444600efac36a240a49eb
+  data.tar.gz: e71f295627f4f640f81432508783365ccbc868cc98f277ecb153e920a9e62339d6cf79f81f64d8ee731b09ede32414d3322a9ab162dce416b543fc040da0055b

data/.gitignore

@@ -7,6 +7,7 @@
 /spec/reports/
 /tmp/
 .vscode/
+.rubocop-https---raw-githubusercontent-com-riboseinc-oss-guides-master-ci-rubocop-yml
 
 # rspec failure tracking
 .rspec_status

data/.rubocop.yml

@@ -0,0 +1,10 @@
+# This project follows the Ribose OSS style guide.
+# https://github.com/riboseinc/oss-guides
+# All project-specific additions and overrides should be specified in this file.
+
+inherit_from:
+  - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
+AllCops:
+  TargetRubyVersion: 2.3
+Rails:
+  Enabled: true

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    nistbib (0.1.2)
-      iso-bib-item (~> 0.4.2)
+    nistbib (0.1.5)
+      relaton-bib (~> 0.1.1)
 
 GEM
   remote: https://rubygems.org/
@@ -20,12 +20,7 @@ GEM
     docile (1.3.1)
     equivalent-xml (0.6.0)
       nokogiri (>= 1.4.3)
-    hashdiff (0.3.8)
-    iso-bib-item (0.4.5)
-      isoics (~> 0.1.6)
-      nokogiri (~> 1.8.4)
-      ruby_deep_clone (~> 0.8.0)
-    isoics (0.1.7)
+    hashdiff (0.3.9)
     json (2.2.0)
     method_source (0.9.2)
     mini_portile2 (2.3.0)
@@ -39,13 +34,16 @@ GEM
       pry (~> 0.10)
     public_suffix (3.0.3)
     rake (10.5.0)
+    relaton-bib (0.1.3)
+      addressable
+      nokogiri (~> 1.8.4)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
       rspec-mocks (~> 3.8.0)
     rspec-core (3.8.0)
       rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.2)
+    rspec-expectations (3.8.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-mocks (3.8.0)
@@ -54,7 +52,6 @@ GEM
     rspec-support (3.8.0)
     ruby-debug-ide (0.6.1)
       rake (>= 0.8.1)
-    ruby_deep_clone (0.8.0)
     safe_yaml (1.0.5)
     simplecov (0.16.1)
       docile (~> 1.1)

data/README.adoc

@@ -40,7 +40,7 @@ hit_collection = NistBib::NistBibliography.search("8200")
  <NistBib::Hit:0x007f82b9b2a1e0 @text="8200" @fetched="false" @fullIdentifier="" @title="8200">]
 
  item = hit_collection[1].fetch
- => #<IsoBibItem::IsoBibliographicItem:0x007f82b986e668
+ => #<NistBib::NistBibliographicItem:0x007f82b986e668
  ...
 ----
 
@@ -48,13 +48,13 @@ hit_collection = NistBib::NistBibliography.search("8200")
 [source,ruby]
 ----
 item.to_xml
-=> "<bibitem type="" id="NISTIR8200(DRAFT)">
-      <fetched>2019-02-11</fetched>
+=> "<nist-standard>
+      <fetched>2019-05-03</fetched>
       <title format="text/plain" language="en" script="Latn">
         Interagency Report on Status of International Cybersecurity Standardization for the Internet of Things (IoT)
       </title>
       ...
-    </bibitem>"
+    </nist-standard>"
 ----
 
 === Get code, and year
@@ -62,7 +62,32 @@ item.to_xml
 ----
 NistBib::NistBibliography.get("8200", "2018", {})
 fetching 8200...
-=> #<IsoBibItem::IsoBibliographicItem:0x007f82bb3370a8
+=> #<NistBib::NistBibliographicItem:0x007f82bb3370a8
+...
+----
+
+=== Get short citation
+Short citation is a convetion about a citation's format. The format for NIST publications is:
+----
+NIST {abbrev(series)} {docnumber} {(edition), optional} {(stage), optioal}
+# or
+{abbrev(series)} {docnumber} {(edition), optional} {(stage), optioal}
+----
+- `(stage)` is empty if the state is "final" (published)
+- `(edition)` is the date of publication or update
+- `docnumber` is the full NIST number, including revision, e.g., 800-52
+
+The format for FIPS publications is:
+----
+FIPS {docnumber}
+# or
+NIST FIPS {docnumber}
+----
+[source,ruby]
+----
+NistBib::NistBibliography.get("8200", "2018", {})
+fetching 8200...
+=> #<NistBib::NistBibliographicItem:0x007f82bb3370a8
 ...
 ----
 

data/lib/nistbib.rb

@@ -1,8 +1,8 @@
 require "nistbib/version"
-require 'nistbib/nist_bibliography'
+require "nistbib/nist_bibliography"
 
 if defined? Relaton
-  require_relative 'relaton/processor'
+  require_relative "relaton/processor"
   Relaton::Registry.instance.register(Relaton::NistBib::Processor)
 end
 

data/lib/nistbib/comment_period.rb

@@ -0,0 +1,30 @@
+module NistBib
+  class CommentPeriod
+    # @return [Date]
+    attr_reader :from
+
+    # @rerurn [Date, NilClass]
+    attr_reader :to
+
+    # @return [Date, NilClass]
+    attr_reader :extended
+
+    # @param from [Date]
+    # @param to [Date, NilClass]
+    # @param extended [Date, NilClass]
+    def initialize(from, to = nil, extended = nil)
+      @from = from
+      @to = to
+      @extended = extended
+    end
+
+    # @param [Nokogiri::XML::Builder]
+    def to_xml(builder)
+      builder.commentperiod do
+        builder.from from.to_s
+        builder.to to.to_s if to
+        builder.extended extended.to_s if extended
+      end
+    end
+  end
+end

data/lib/nistbib/document_status.rb

@@ -0,0 +1,44 @@
+module NistBib
+  class DocumentStatus
+    STAGES = %w[
+      draft-internal draft-wip draft-prelim draft-public final final-review
+    ].freeze
+
+    SUBSTAGES = %w[active retired withdrawn].freeze
+
+    # @return [String]
+    attr_reader :stage
+
+    # @return [String, NilClass]
+    attr_reader :substage
+
+    # @return [String, NilClass]
+    attr_reader :iteration
+
+    # @param stage [String]
+    # @param substage [String, NilClass]
+    # @param iteration [String, NilClass]
+    def initialize(stage:, substage: nil, iteration: nil)
+      unless STAGES.include? stage
+        raise ArgumentError, "invalid argument: stage (#{stage})"
+      end
+
+      if substage && !SUBSTAGES.include?(substage)
+        raise ArgumentError, "invalid argument: substage (#{substage})"
+      end
+
+      @stage = stage
+      @substage = substage
+      @iteration = iteration
+    end
+
+    # @param builder [Nokogiri::XML::Builder]
+    def to_xml(builder)
+      builder.status do
+        builder.stage stage
+        builder.substage substage if substage
+        builder.iteration iteration if iteration
+      end
+    end
+  end
+end

data/lib/nistbib/hit.rb

@@ -3,21 +3,21 @@
 module NistBib
   # Hit.
   class Hit
-    # @return [Isobib::HitCollection]
+    # @return [NistBib::HitCollection]
     attr_reader :hit_collection
 
     # @return [Array<Hash>]
     attr_reader :hit
 
     # @param hit [Hash]
-    # @param hit_collection [Isobib:HitCollection]
+    # @param hit_collection [NistBib:HitCollection]
     def initialize(hit, hit_collection = nil)
       @hit            = hit
       @hit_collection = hit_collection
     end
 
     # Parse page.
-    # @return [Isobib::IsoBibliographicItem]
+    # @return [NistBib::NistBliographicItem]
     def fetch
       @fetch ||= Scrapper.parse_page @hit
     end
@@ -37,15 +37,8 @@ module NistBib
     end
 
     # @return [String]
-    def to_xml(opts = {})
-      #if builder
-        #fetch.to_xml builder, opts
-      #else
-        builder = Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml|
-          fetch.to_xml xml, opts
-        end
-        builder.doc.root.to_xml
-      #end
+    def to_xml
+      fetch.to_xml
     end
   end
 end

data/lib/nistbib/hit_collection.rb

@@ -22,23 +22,35 @@ module NistBib
 
     # @param ref_nbr [String]
     # @param year [String]
-    def initialize(ref_nbr, year = nil) #(text, hit_pages = nil)
+    # @param opts [Hash]
+    def initialize(ref_nbr, year = nil, opts)
       @text = ref_nbr
       @year = year
       from, to = nil
       if year
-        d = Date.strptime year, '%Y'
-        from = d.strftime '%m/%d/%Y'
-        to   = d.next_year.prev_day.strftime '%m/%d/%Y'
+        d = Date.strptime year, "%Y"
+        from = d.strftime "%m/%d/%Y"
+        to   = d.next_year.prev_day.strftime "%m/%d/%Y"
       end
-      url  = "#{DOMAIN}/publications/search?keywords-lg=#{ref_nbr}&dateFrom-lg=#{from}&dateTo-lg=#{to}"
+      url  = "#{DOMAIN}/publications/search?keywords-lg=#{ref_nbr}"
+      url += "&dateFrom-lg=#{from}" if from
+      url += "&dateTo-lg=#{to}" if to
+      url += "&status-lg=Draft" if /PD/ =~ opts[:stage]
       doc  = Nokogiri::HTML OpenURI.open_uri(::Addressable::URI.parse(url).normalize)
-      hits = doc.css('table.publications-table > tbody > tr').map do |h|
-        link  = h.at('td/div/strong/a')
-        code  = h.at('td[2]').text.strip
+      hits = doc.css("table.publications-table > tbody > tr").map do |h|
+        link  = h.at("td/div/strong/a")
+        serie = h.at("td[1]").text.strip
+        code  = h.at("td[2]").text.strip
         title = link.text
         url   = DOMAIN + link[:href]
-        Hit.new({ code: code, title: title, url: url }, self)
+        status = h.at("td[4]").text.strip.downcase
+        release_date = Date.strptime h.at("td[5]").text.strip, "%m/%d/%Y"
+        Hit.new(
+          {
+            code: code, serie: serie, title: title, url: url, status: status,
+            release_date: release_date
+          }, self
+        )
       end
       concat hits
       # concat(hits.map { |h| Hit.new(h, self) })

data/lib/nistbib/keyword.rb

@@ -0,0 +1,16 @@
+module NistBib
+  class Keyword
+    # @return [Nokogiri::XML::DocumentFragment]
+    attr_reader :element
+
+    # @param element [String]
+    def initialize(element)
+      @element = Nokogiri::XML.fragment element
+    end
+
+    # @param builder [Nokogiri::XML::Builder]
+    def to_xml(builder)
+      builder.keyword element.to_xml
+    end
+  end
+end

data/lib/nistbib/nist_bibliographic_item.rb

@@ -0,0 +1,75 @@
+module NistBib
+  class NistBibliographicItem < RelatonBib::BibliographicItem
+    # @return [NistBib::NistSeries, NilClass]
+    attr_reader :nistseries
+
+    # @return [Array<NistBib::Keyword>]
+    attr_reader :keyword
+
+    # @return [NistBib::CommentPeriod]
+    attr_reader :commentperiod
+
+    # @param id [String, NilClass]
+    # @param title [Array<RelatonBib::TypedTitleString>]
+    # @param formattedref [RelatonBib::FormattedRef, NilClass]
+    # @param type [String, NilClass]
+    # @param docid [Array<RelatonBib::DocumentIdentifier>]
+    # @param docnumber [String, NilClass]
+    # @param language [Arra<String>]
+    # @param script [Array<String>]
+    # @param docstatus [NistBib::DocumentStatus, NilClass]
+    # @param edition [String, NilClass]
+    # @param version [RelatonBib::BibliographicItem::Version, NilClass]
+    # @param biblionote [Array<RelatonBib::FormattedStrong>]
+    # @param series [Array<RelatonBib::Series>]
+    # @param medium [RelatonBib::Medium, NilClas]
+    # @param place [Array<String>]
+    # @param extent [Array<Relaton::BibItemLocality>]
+    # @param accesslocation [Array<String>]
+    # @param classification [RelatonBib::Classification, NilClass]
+    # @param validity [RelatonBib:Validity, NilClass]
+    # @param fetched [Date, NilClass] default nil
+    # @param keyword [Array<NistBib::Keyword>]
+    # @param commentperiod [NistBib::CommentPeriod]
+    #
+    # @param dates [Array<Hash>]
+    # @option dates [String] :type
+    # @option dates [String] :from
+    # @option dates [String] :to
+    #
+    # @param contributors [Array<Hash>]
+    # @option contributors [String] :type
+    # @option contributors [String] :from
+    # @option contributirs [String] :to
+    # @option contributors [String] :abbreviation
+    # @option contributors [Array<String>] :roles
+    #
+    # @param abstract [Array<Hash>]
+    # @option abstract [String] :content
+    # @option abstract [String] :language
+    # @option abstract [String] :script
+    # @option abstract [String] :type
+    #
+    # @param relations [Array<Hash>]
+    # @option relations [String] :type
+    # @option relations [RelatonBib::BibliographicItem] :bibitem
+    # @option relations [Array<RelatonBib::BibItemLocality>] :bib_locality
+    #
+    # @param nistseries [Nist::NistSeries, NilClass]
+    def initialize(**args)
+      @nistseries = args.delete :nistseries
+      @keyword = args.delete(:keyword) || []
+      @commentperiod = args.delete :commentperiod
+      super
+    end
+
+    # @param builder [Nokogiri::XML::Builder]
+    def to_xml(builder = nil)
+      super builder, root: :bibitem, date_format: :short do |b|
+        nistseries&.to_xml b
+        keyword.each { |kw| kw.to_xml b }
+        commentperiod&.to_xml b
+      end
+    end
+  end
+end

data/lib/nistbib/nist_bibliography.rb

@@ -1,16 +1,23 @@
-require 'nistbib/scrapper'
-require 'nistbib/hit_collection'
+require "relaton_bib"
+require "nistbib/nist_bibliographic_item"
+require "nistbib/scrapper"
+require "nistbib/hit_collection"
+require "nistbib/nist_series"
+require "nistbib/xml_parser"
+require "nistbib/keyword"
+require "nistbib/comment_period"
+require "nistbib/document_status"
 
 module NistBib
   class NistBibliography
     class << self
       # @param text [String]
       # @return [NistBib::HitCollection]
-      def search(text, year = nil)
-        HitCollection.new text, year
+      def search(text, year = nil, opts = {})
+        HitCollection.new text, year, opts
       rescue
         warn "Could not access https://www.nist.gov"
-        []       
+        []
       end
 
       # @param code [String] the NIST standard Code to look up (e..g "8200")
@@ -18,6 +25,19 @@ module NistBib
       # @param opts [Hash] options; restricted to :all_parts if all-parts reference is required
       # @return [String] Relaton XML serialisation of reference
       def get(code, year = nil, opts = {})
+        /^(?<code2>[^\(]+)\((?<date2>\w+\s(\d{2},\s)?\d{4})\)(\s\()?(?<stage>[^\)]+)?/ =~ code
+        if code2
+          code = code2.strip
+          if date2
+            if /\w+\s\d{4}/ =~ date2
+              opts[:issued_date] = Time.strptime date2, "%B %Y"
+            elsif /\w+\s\d{2},\s\d{4}/ =~ date2
+              opts[:updated_date] = Time.strptime date2, "%B %d, %Y"
+            end
+          end
+          opts[:stage] = stage if stage
+        end
+
         if year.nil?
           /^(?<code1>[^:]+):(?<year1>[^:]+)$/ =~ code
           unless code1.nil?
@@ -26,36 +46,48 @@ module NistBib
           end
         end
 
-        code += '-1' if opts[:all_parts]
+        code += "-1" if opts[:all_parts]
         ret = nistbib_get1(code, year, opts)
-        return nil if ret.nil?
-        ret.to_most_recent_reference unless year || opts[:keep_year]
-        ret.to_all_parts if opts[:all_parts]
+        # return nil if ret.nil?
+        # ret.to_most_recent_reference unless year || opts[:keep_year]
+        # ret.to_all_parts if opts[:all_parts]
         ret
       end
 
       private
 
       def nistbib_get1(code, year, opts)
-        result = nistbib_search_filter(code, year) or return nil
-        ret = nistbib_results_filter(result, year)
+        result = nistbib_search_filter(code, year, opts) or return nil
+        ret = nistbib_results_filter(result, year, opts)
         return ret[:ret] if ret[:ret]
+
         fetch_ref_err(code, year, ret[:years])
       end
 
-      # Sort through the results from Isobib, fetching them three at a time,
+      # Sort through the results from NistBib, fetching them three at a time,
       # and return the first result that matches the code,
       # matches the year (if provided), and which # has a title (amendments do not).
       # Only expects the first page of results to be populated.
       # Does not match corrigenda etc (e.g. ISO 3166-1:2006/Cor 1:2007)
       # If no match, returns any years which caused mismatch, for error reporting
-      def nistbib_results_filter(result, year)
+      def nistbib_results_filter(result, year, opts)
         missed_years = []
         result.each_slice(3) do |s| # ISO website only allows 3 connections
-          fetch_pages(s, 3).each_with_index do |r, i|
+          fetch_pages(s, 3).each_with_index do |r, _i|
+            if opts[:issued_date]
+              r.dates.select { |d| d.type == "issued" }.each do |d|
+                return { ret: r } if opts[:issued_date] == d.on
+              end
+            elsif opts[:updated_date]
+              r.dates.select { |d| d.type == "published" }.each do |d|
+                return { ret: r } if opts[:updated_date] == d.on
+              end
+            end
             return { ret: r } if !year
+
             r.dates.select { |d| d.type == "published" }.each do |d|
               return { ret: r } if year.to_i == d.on.year
+
               missed_years << d.on.year
             end
           end
@@ -71,16 +103,13 @@ module NistBib
         workers.result.sort { |x, y| x[:i] <=> y[:i] }.map { |x| x[:hit] }
       end
 
-      def nistbib_search_filter(code, year)
-        # docidrx = %r{^(ISO|IEC)[^0-9]*\s[0-9-]+}
-        # corrigrx = %r{^(ISO|IEC)[^0-9]*\s[0-9-]+:[0-9]+/}
+      def nistbib_search_filter(code, year, opts)
+        docid = code.match(%r{[0-9-]{3,}}).to_s
+        serie = code.match(%r{(FISP|SP|NISTIR)(?=\s)})
         warn "fetching #{code}..."
-        result = search(code, year)
+        result = search(code, year, opts)
         result.select do |i|
-          !i.hit[:code].empty?
-        #   i.hit[:code] &&
-        #     i.hit[:code].match(docidrx).to_s == code &&
-        #     corrigrx !~ i.hit[:code]
+          i.hit[:code]&.include?(docid) && (!serie || i.hit[:serie] == serie.to_s)
         end
       end
 

data/lib/nistbib/nist_series.rb

@@ -0,0 +1,35 @@
+module NistBib
+  class NistSeries
+    ABBREVIATIONS = {
+      "NISTIR" => "NIST Interagency/Internal Report",
+      "NIST SP" => "NIST Special Publication",
+      "NIST FIPS" => "NIST Federal Information Processing Standards",
+    }.freeze
+
+    # @return [String]
+    attr_reader :type
+
+    # @return [String]
+    attr_reader :title
+
+    # @return [RalatonBib::LocalizedString, NilClass]
+    attr_reader :abbreviation
+
+    # @param abbreviation [RelatonBib::LocalizedString, NilClass]
+    def initialize(abbreviation)
+      @type = "main"
+      @abbreviation = RelatonBib::LocalizedString.new(
+        abbreviation.downcase.gsub(" ", "-"), "en", "Latn"
+      )
+      @title = ABBREVIATIONS[abbreviation]
+    end
+
+    # @param builder [Nokogiri::XML::Builder]
+    def to_xml(builder)
+      builder.series type: type do
+        builder.title title
+        builder.abbreviation { abbreviation.to_xml builder } if abbreviation
+      end
+    end
+  end
+end

data/lib/nistbib/scrapper.rb

@@ -1,9 +1,9 @@
-require 'iso_bib_item'
+require "relaton_bib"
 
 module NistBib
-  module Scrapper
+  class Scrapper
     class << self
-      DOMAIN = 'https://csrc.nist.gov'
+      DOMAIN = "https://csrc.nist.gov".freeze
 
       # Parse page.
       # @param hit_data [Hash]
@@ -12,22 +12,25 @@ module NistBib
       def parse_page(hit_data)
         doc = get_page hit_data[:url]
 
-        IsoBibItem::IsoBibliographicItem.new(
-          docid:        fetch_docid(doc),
-          edition:      nil,
-          language:     ['en'],
-          script:       ['Latn'],
-          titles:       fetch_titles(hit_data),
-          type:         nil,
-          docstatus:    fetch_status(doc),
-          ics:          [],
-          dates:        fetch_dates(doc),
-          contributors: fetch_contributors,
-          workgroup:    nil,
-          abstract:     fetch_abstract(doc),
-          copyright:    fetch_copyright(doc),
-          link:         fetch_link(doc, hit_data[:url]),
-          relations:    fetch_relations(doc)
+        NistBibliographicItem.new(
+          fetched: Date.today.to_s,
+          type: "standard",
+          id: fetch_id(doc),
+          titles: fetch_titles(hit_data),
+          link: fetch_link(doc),
+          docid: fetch_docid(doc),
+          dates: fetch_dates(doc, hit_data[:release_date]),
+          contributors: fetch_contributors(doc),
+          edition: nil,
+          language: ["en"],
+          script: ["Latn"],
+          abstract: fetch_abstract(doc),
+          docstatus: fetch_status(doc, hit_data[:status]),
+          copyright: fetch_copyright(doc),
+          relations: fetch_relations(doc),
+          nistseries: fetch_nistseries(doc),
+          keyword: fetch_keywords(doc),
+          commentperiod: fetch_commentperiod(doc),
         )
       end
 
@@ -38,7 +41,7 @@ module NistBib
       # @return [Array<Nokogiri::HTML::Document, String>]
       def get_page(url)
         uri = URI url
-        resp = Net::HTTP.get_response(uri)#.encode("UTF-8")
+        resp = Net::HTTP.get_response(uri) # .encode("UTF-8")
         Nokogiri::HTML(resp.body)
       end
 
@@ -46,88 +49,127 @@ module NistBib
       # @param doc [Nokogiri::HTML::Document]
       # @return [Hash]
       def fetch_docid(doc)
-        item_ref = doc.at("//div[contains(@class, 'publications-detail')]/h3").text.strip
-        unless item_ref
-          return { project_number: '?', part_number: '', prefix: nil, type: 'NIST', id: '?' }
-        end
-        m = item_ref.match(/(?<=\s)(?<project>[\d-]+)(?<vol>\sVol\.\s)?(?<part>(?(<vol>)\d+|))/)
-        {
-          project_number: m[:project],
-          part_number: m[:part],
-          subpart_number: nil,
-          prefix: nil,
-          type: 'NIST',
-          id: item_ref
-        }
+        item_ref = doc.at("//div[contains(@class, 'publications-detail')]/h3").
+          text.strip
+        return [RelatonBib::DocumentIdentifier.new(type: "nist", id: "?")] unless item_ref
+
+        [RelatonBib::DocumentIdentifier.new(id: item_ref, type: "nist")]
+      end
+
+      # Fetch id.
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [String]
+      def fetch_id(doc)
+        doc.at("//div[contains(@class, 'publications-detail')]/h3").text.
+          strip.gsub(/\s/, "")
       end
 
       # Fetch status.
       # @param doc [Nokogiri::HTML::Document]
       # @param status [String]
       # @return [Hash]
-      def fetch_status(doc)
-        withdrawn = doc.at "//p/strong[text()='Withdrawn:']"
-        return { status: 'withdrawn', stage: '95', substage: '99' } if withdrawn
-
-        item_ref = doc.at("//div[contains(@class, 'publications-detail')]/h3").text.strip
-        wip = item_ref.match(/(?<=\()\w+/).to_s
-        case wip
-        when 'DRAFT'
-          { status: 'working-draft', stage: '20', substage: '20' }
-        # when 'RETIRED DRAFT'
-        #   { status: 'committee-draft', stage: '30', substage: '00' }
+      def fetch_status(doc, status)
+        case status
+        when "draft (withdrawn)"
+          stage = "draft-public"
+          substage = "withdrawn"
+        when "retired draft"
+          stage = "draft-public"
+          substage = "retired"
+        when "withdrawn"
+          stage = "final"
+          substage = "withdrawn"
+        when "draft"
+          stage = "draft-public"
+          substage = "active"
         else
-          { status: 'published', stage: '60', substage: '60' }
+          stage = status
+          substage = "active"
         end
+
+        # if doc.at "//p/strong[text()='Withdrawn:']"
+        #   substage = "withdrawn"
+        # else
+        #   substage = "active"
+        #   item_ref = doc.at(
+        #     "//div[contains(@class, 'publications-detail')]/h3",
+        #   ).text.strip
+        #   wip = item_ref.match(/(?<=\()\w+/).to_s
+        #   stage = "draft-public" if wip == "DRAFT"
+        # end
+        NistBib::DocumentStatus.new(stage: stage, substage: substage)
       end
 
       # Fetch titles.
       # @param hit_data [Hash]
       # @return [Array<Hash>]
       def fetch_titles(hit_data)
-        titles = hit_data[:title].split ' - '
-        # case titles.size
-        # when 0
-        #   intro, main, part = nil, "", nil
-        # when 1
-          intro, main, part = nil, titles[0], nil
-        # when 2
-        #   if /^(Part|Partie) \d+:/ =~ titles[1]
-        #     intro, main, part = nil, titles[0], titles[1]
-        #   else
-        #     intro, main, part = titles[0], titles[1], nil
-        #   end
-        # when 3
-        #   intro, main, part = titles[0], titles[1], titles[2]
-        # else
-        #   intro, main, part = titles[0], titles[1], titles[2..-1]&.join(" -- ")
-        # end
-        [{
-          title_intro: intro,
-          title_main:  main,
-          title_part:  part,
-          language:    'en',
-          script:      'Latn'
-        }]
+        [{ content: hit_data[:title], language: "en", script: "Latn", format: "text/plain" }]
       end
 
       # Fetch dates
       # @param doc [Nokogiri::HTML::Document]
       # @return [Array<Hash>]
-      def fetch_dates(doc)
-        dates = []
-        d = doc.at("//strong[text()='Date Published:']/../text()[2]").text.strip
-        publish_date = Date.strptime(d, '%B %Y').to_s
-        unless publish_date.empty?
-          dates << { type: 'published', on: publish_date }
-        end
+      def fetch_dates(doc, release_date)
+        dates = [{ type: "published", on: release_date.to_s }]
+
+        d = doc.at("//span[@id='pub-release-date']").text.strip
+        date = if /(?<date>\w+\s\d{4})/ =~ d
+                 Date.strptime(date, "%B %Y")
+               elsif /(?<date>\w+\s\d{2},\s\d{4})/ =~ d
+                 Date.strptime(date, "%B %d, %Y")
+               end
+        dates << { type: "issued", on: date.to_s }
+
         dates
       end
 
-      def fetch_contributors
-        name = 'National Institute of Standards and Technology'
-        url = 'www.nist.gov'
-        [{ entity: { name: name, url: url, abbreviation: 'NIST' }, roles: ['publisher'] }]
+      def fetch_contributors(doc)
+        name = "National Institute of Standards and Technology"
+        org = RelatonBib::Organization.new(
+          name: name, url: "www.nist.gov", abbreviation: "NIST",
+        )
+        contribs = [
+          RelatonBib::ContributionInfo.new(entity: org, role: ["publisher"]),
+        ]
+
+        authors = doc.at('//h4[.="Author(s)"]/following-sibling::p')
+        contribs += contributors(authors, "author")
+
+        editors = doc.at('//h4[.="Editor(s)"]/following-sibling::p')
+        contribs + contributors(editors, "editor")
+      end
+
+      def contributors(doc, role)
+        return [] if doc.nil?
+
+        doc.text.split(", ").map do |contr|
+          /(?<an>.+?)(\s+\((?<abbrev>.+?)\))?$/ =~ contr
+          if abbrev && an.downcase !~ /(task|force|group)/ && an.split.size.between?(2, 3)
+            fullname = RelatonBib::FullName.new(
+              completename: RelatonBib::LocalizedString.new(an, "en", "Latn"),
+            )
+            case abbrev
+            when "NIST"
+              org_name = "National Institute of Standards and Technology"
+              url = "www.nist.gov"
+            when "MITRE"
+              org_name = abbrev
+              url = "www.mitre.org"
+            else
+              org_name = abbrev
+              url = nil
+            end
+            org = RelatonBib::Organization.new name: org_name, url: url, abbreviation: abbrev
+            affiliation = RelatonBib::Affilation.new org
+            entity = RelatonBib::Person.new(
+              name: fullname, affiliation: [affiliation], contacts: [],
+            )
+          else
+            entity = RelatonBib::Organization.new name: an, abbreviation: abbrev
+          end
+          RelatonBib::ContributionInfo.new entity: entity, role: [role]
+        end
       end
 
       # Fetch abstracts.
@@ -136,9 +178,10 @@ module NistBib
       def fetch_abstract(doc)
         abstract_content = doc.xpath('//div[contains(@class, "pub-abstract-callout")]/div[1]/p').text
         [{
-          content:  abstract_content,
-          language: 'en',
-          script:   'Latn'
+          content: abstract_content,
+          language: "en",
+          script: "Latn",
+          format: "text/plain",
         }]
       end
 
@@ -146,39 +189,83 @@ module NistBib
       # @param title [String]
       # @return [Hash]
       def fetch_copyright(doc)
-        name = 'National Institute of Standards and Technology'
-        url = 'www.nist.gov'
-        d = doc.at("//strong[text()='Date Published:']/../text()[2]").text.strip
-        from = d.match(/\d{4}$/).to_s
-        { owner: { name: name, abbreviation: 'NIST', url: url }, from: from }
+        name = "National Institute of Standards and Technology"
+        url = "www.nist.gov"
+        d = doc.at("//span[@id='pub-release-date']").text.strip
+        from = d.match(/\d{4}/).to_s
+        { owner: { name: name, abbreviation: "NIST", url: url }, from: from }
       end
 
       # Fetch links.
       # @param doc [Nokogiri::HTML::Document]
-      # @param url [String]
       # @return [Array<Hash>]
-      def fetch_link(doc, url)
-        links = [{ type: 'src', content: url }]
-        obp_elms = doc.at_css('a[data-identity="first-link-display"]')
-        links << { type: 'obp', content: obp_elms[:href] } if obp_elms
+      def fetch_link(doc)
+        pub = doc.at "//p/strong[.='Publication:']"
+        links = []
+        pdf = pub.at "./following-sibling::a[.=' Local Download']"
+        links << { type: "pdf", content: pdf[:href] } if pdf
+        doi = pub.at("./following-sibling::a[contains(.,'(DOI)')]")
+        links << { type: "doi", content: doi[:href] } if doi
         links
       end
 
       # Fetch relations.
       # @param doc [Nokogiri::HTML::Document]
       # @return [Array<Hash>]
-      # rubocop:disable Metrics/MethodLength
       def fetch_relations(doc)
-        relations = doc.xpath('//strong[text()="Other Parts of this Publication:"]/following-sibling::a').map do |r|
-          { type: 'partOf', identifier: r.text, url: DOMAIN + r[:href] }
+        relations = doc.xpath('//span[@id="pub-supersedes-container"]/a').map do |r|
+          doc_relation "supersedes", r
         end
 
-        doc.xpath('//strong[text()="Related NIST Publications:"]/following-sibling::a').map do |r|
-          relations << { type: 'updates', identifier: r.text, url: DOMAIN + r[:href] }
+        relations += doc.xpath('//span[@id="pub-part-container"]/a').map do |r|
+          doc_relation "partOf", r
+        end
+
+        relations + doc.xpath('//span[@id="pub-related-container"]/a').map do |r|
+          doc_relation "updates", r
+        end
+      end
+
+      def doc_relation(type, ref)
+        RelatonBib::DocumentRelation.new(
+          type: type,
+          bibitem: RelatonBib::BibliographicItem.new(
+            formattedref: RelatonBib::FormattedRef.new(
+              content: ref.text, language: "en", script: "Latn", format: "text/plain",
+            ),
+            link: [RelatonBib::TypedUri.new(type: "src", content: DOMAIN + ref[:href])],
+          ),
+        )
+      end
+
+      def fetch_nistseries(doc)
+        doc.xpath("//p/strong[.='Document History:']/following-sibling::*[not(self::br)]").each do |s|
+          NistSeries::ABBREVIATIONS.each do |k, _v|
+            return NistSeries.new k if s.text.include? k
+          end
         end
+        nil
+      end
+
+      def fetch_keywords(doc)
+        kws = doc.xpath "//span[@id='pub-keywords-container']/span"
+        kws.map { |kw| Keyword.new kw.text }
+      end
+
+      def fetch_commentperiod(doc)
+        cp = doc.at "//span[@id='pub-comments-due']"
+        return unless cp
+
+        to = Date.strptime cp.text.strip, "%B %d, %Y"
+
+        d = doc.at("//span[@id='pub-release-date']").text.strip
+        from = Date.strptime(d, "%B %Y").to_s
 
-        relations
+        ex = doc.at "//strong[contains(.,'The comment closing date has been extended to')]"
+        ext = ex&.text&.match(/\w+\s\d{2},\s\d{4}/).to_s
+        extended = ext.empty? ? nil : Date.strptime(ext, "%B %d, %Y")
+        CommentPeriod.new from, to, extended
       end
     end
   end
-end
+end

data/lib/nistbib/version.rb

@@ -1,3 +1,3 @@
 module NistBib
-  VERSION = "0.1.2"
+  VERSION = "0.1.5".freeze
 end

data/lib/nistbib/xml_parser.rb

@@ -0,0 +1,53 @@
+module NistBib
+  class XMLParser < RelatonBib::XMLParser
+    class << self
+      def from_xml(xml)
+        doc = Nokogiri::XML xml
+        nistitem = doc.at("/bibitem") || doc.at("/bibdata")
+        NistBibliographicItem.new(item_data(nistitem))
+      end
+
+      private
+
+      def item_data(nistitem)
+        data = super
+        data.delete :series
+        data[:nistseries] = fetch_nistseries(nistitem)
+        data[:keyword] = fetch_keyword(nistitem)
+        data[:commentperiod] = fetch_commentperiod(nistitem)
+        data
+      end
+
+      def fetch_status(item)
+        status = item.at "./status"
+        return unless status
+
+        DocumentStatus.new(
+          stage: status.at("stage")&.text,
+          substage: status.at("substage")&.text,
+          iteration: status.at("iteraton")&.text,
+        )
+      end
+
+      def fetch_commentperiod(item)
+        cp = item.at "./commentperiod"
+        return unless cp
+
+        CommentPeriod.new cp.at("from").text, cp.at("to")&.text, cp.at("extended")&.text
+      end
+
+      def fetch_keyword(item)
+        item.xpath("./keyword").map do |kw|
+          Keyword.new kw.children.first.to_xml
+        end
+      end
+
+      def fetch_nistseries(item)
+        series = item.at "./series"
+        return unless series
+
+        NistSeries.new series.at("./abbreviation").text.upcase.gsub("-", " ")
+      end
+    end
+  end
+end

data/lib/relaton/processor.rb

@@ -16,7 +16,7 @@ module Relaton
       end
 
       def from_xml(xml)
-        IsoBibItem::XMLParser.from_xml xml
+        ::NistBib::XMLParser.from_xml xml
       end
     end
   end

data/nistbib.gemspec

@@ -1,4 +1,3 @@
-
 lib = File.expand_path("../lib", __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "nistbib/version"
@@ -9,14 +8,14 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Ribose Inc."]
   spec.email         = ["open.source@ribose.com"]
 
-  spec.summary       = %q{NistBib: retrive NIST standards.}
-  spec.description   = %q{NistBib: retrive NIST standards.}
+  spec.summary       = "NistBib: retrive NIST standards."
+  spec.description   = "NistBib: retrive NIST standards."
   spec.homepage      = "https://github.com/metanorma/nistbib"
   spec.license       = "MIT"
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+  spec.files         = Dir.chdir(File.expand_path( __dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
   spec.bindir        = "exe"
@@ -24,16 +23,16 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "bundler", "~> 2.0"
-  spec.add_development_dependency 'equivalent-xml', '~> 0.6'
-  spec.add_development_dependency "pry-byebug"
   spec.add_development_dependency "byebug"
+  spec.add_development_dependency "debase"
+  spec.add_development_dependency "equivalent-xml", "~> 0.6"
+  spec.add_development_dependency "pry-byebug"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency 'simplecov'
-  spec.add_development_dependency 'vcr'
-  spec.add_development_dependency 'webmock'
-  spec.add_development_dependency 'ruby-debug-ide'
-  spec.add_development_dependency 'debase'
+  spec.add_development_dependency "ruby-debug-ide"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "vcr"
+  spec.add_development_dependency "webmock"
 
-  spec.add_dependency 'iso-bib-item', '~> 0.4.2'
+  spec.add_dependency "relaton-bib", "~> 0.1.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nistbib
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.5
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-04-17 00:00:00.000000000 Z
+date: 2019-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -25,21 +25,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: equivalent-xml
+  name: byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry-byebug
+  name: debase
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -53,7 +53,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: byebug
+  name: equivalent-xml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -95,21 +109,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: vcr
+  name: ruby-debug-ide
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -123,7 +123,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: webmock
+  name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -137,7 +137,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: ruby-debug-ide
+  name: vcr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -151,7 +151,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: debase
+  name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -165,19 +165,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: iso-bib-item
+  name: relaton-bib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.2
+        version: 0.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.2
+        version: 0.1.1
 description: 'NistBib: retrive NIST standards.'
 email:
 - open.source@ribose.com
@@ -187,6 +187,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
 - ".travis.yml"
 - Gemfile
 - Gemfile.lock
@@ -197,12 +198,18 @@ files:
 - bin/console
 - bin/setup
 - lib/nistbib.rb
+- lib/nistbib/comment_period.rb
+- lib/nistbib/document_status.rb
 - lib/nistbib/hit.rb
 - lib/nistbib/hit_collection.rb
+- lib/nistbib/keyword.rb
+- lib/nistbib/nist_bibliographic_item.rb
 - lib/nistbib/nist_bibliography.rb
+- lib/nistbib/nist_series.rb
 - lib/nistbib/scrapper.rb
 - lib/nistbib/version.rb
 - lib/nistbib/workers_pool.rb
+- lib/nistbib/xml_parser.rb
 - lib/relaton/processor.rb
 - nistbib.gemspec
 homepage: https://github.com/metanorma/nistbib
@@ -225,7 +232,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
 summary: 'NistBib: retrive NIST standards.'