relaton-nist 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b6d99784844761b6cde3c399bc02e8b34308c562
+  data.tar.gz: 1a267971e69723396eabb57fc8f49d745a5eb014
+SHA512:
+  metadata.gz: d5d9ebf4cb69757b72bb32a12144e7512cde10de1d0fcd9ad6bc2422f095a07f45f5a3ddf3e4e68e980acf63bc359e9d87ee495f3f637f2a6e61ef5d63fbde1d
+  data.tar.gz: 7d478dbb6e7721ae6e018a3765f3b2e19f3fb803fe9581efb25472010d6bc650db2dd5def983782508d0b3fd659a6924e017e3b7dec7b078356ae8c52daff587

data/.gitignore

@@ -0,0 +1,13 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.vscode/
+.rubocop-https---raw-githubusercontent-com-riboseinc-oss-guides-master-ci-rubocop-yml
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,10 @@
+# This project follows the Ribose OSS style guide.
+# https://github.com/riboseinc/oss-guides
+# All project-specific additions and overrides should be specified in this file.
+
+inherit_from:
+  - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
+AllCops:
+  TargetRubyVersion: 2.3
+Rails:
+  Enabled: true

data/.travis.yml

@@ -0,0 +1,17 @@
+# Auto-generated !!! Do not edit it manually
+# use ci-master https://github.com/metanorma/metanorma-build-scripts
+language: ruby
+cache: bundler
+os:
+- linux
+- osx
+rvm:
+- 2.5
+- 2.4
+- ruby-head
+before_install: 
+- gem install bundler -v 2.0.1
+- bundle update
+matrix:
+  allow_failures:
+    - rvm: ruby-head

data/Gemfile

@@ -0,0 +1,7 @@
+Encoding.default_external = Encoding::UTF_8
+Encoding.default_internal = Encoding::UTF_8
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in relaton_nist.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,85 @@
+PATH
+  remote: .
+  specs:
+    relaton-nist (0.1.0)
+      relaton-bib (~> 0.1.6)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.6.0)
+      public_suffix (>= 2.0.2, < 4.0)
+    byebug (11.0.1)
+    coderay (1.1.2)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    debase (0.2.2)
+      debase-ruby_core_source (>= 0.10.2)
+    debase-ruby_core_source (0.10.4)
+    diff-lcs (1.3)
+    docile (1.3.1)
+    equivalent-xml (0.6.0)
+      nokogiri (>= 1.4.3)
+    hashdiff (0.4.0)
+    json (2.2.0)
+    method_source (0.9.2)
+    mini_portile2 (2.3.0)
+    nokogiri (1.8.5)
+      mini_portile2 (~> 2.3.0)
+    pry (0.12.2)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    pry-byebug (3.7.0)
+      byebug (~> 11.0)
+      pry (~> 0.10)
+    public_suffix (3.1.0)
+    rake (10.5.0)
+    relaton-bib (0.1.6)
+      addressable
+      nokogiri (~> 1.8.4)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    ruby-debug-ide (0.6.1)
+      rake (>= 0.8.1)
+    safe_yaml (1.0.5)
+    simplecov (0.16.1)
+      docile (~> 1.1)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.2)
+    vcr (5.0.0)
+    webmock (3.5.1)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  byebug
+  debase
+  equivalent-xml (~> 0.6)
+  pry-byebug
+  rake (~> 10.0)
+  relaton-nist!
+  rspec (~> 3.0)
+  ruby-debug-ide
+  simplecov
+  vcr
+  webmock
+
+BUNDLED WITH
+   2.0.1

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Andrei Kislichenko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.adoc

@@ -0,0 +1,128 @@
+= RelatonNist: retrieve NIST Standards for bibliographic use using the BibliographicItem model
+
+image:https://img.shields.io/gem/v/relaton-nist.svg["Gem Version", link="https://rubygems.org/gems/relaton-nist"]
+image:https://img.shields.io/travis/metanorma/relaton-nist/master.svg["Build Status", link="https://travis-ci.com/metanorma/relaton-nist"]
+image:https://ci.appveyor.com/api/projects/status/vk85u3df4f3kertr?svg=true["Appveyor Build Status", link="https://ci.appveyor.com/project/ribose/relaton-nist"]
+image:https://codeclimate.com/github/metanorma/relaton-nist/badges/gpa.svg["Code Climate", link="https://codeclimate.com/github/metanorma/relaton-nist"]
+
+
+RelatonNist is a Ruby gem that implements the https://github.com/metanorma/metanorma-model-iso#iso-bibliographic-item[IsoBibliographicItem model].
+
+You can use it to retrieve metadata of NIST Standards from https://csrc.nist.gov, and access such metadata through the `IsoBibliographicItem` object.
+
+== Installation
+
+Add this line to your application's Gemfile:
+
+[source,ruby]
+----
+gem 'relaton_nist'
+----
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install relaton_nist
+
+== Usage
+
+=== Search for a standard using keywords
+
+[source,ruby]
+----
+require 'relaton_nist'
+
+hit_collection = RelatonNist::NistBibliography.search("8200")
+=> [<RelatonNist::Hit:0x007f82b9b2b298 @text="8200" @fetched="false" @fullIdentifier="" @title="8200">,
+ <RelatonNist::Hit:0x007f82b9b2a1e0 @text="8200" @fetched="false" @fullIdentifier="" @title="8200">]
+
+ item = hit_collection[1].fetch
+ => #<RelatonNist::NistBibliographicItem:0x007f82b986e668
+ ...
+----
+
+=== XML serialization
+[source,ruby]
+----
+item.to_xml
+=> "<bibitem id="NISTIR8200(DRAFT)" type="standard">
+      <fetched>2019-05-03</fetched>
+      <title format="text/plain" language="en" script="Latn">
+        Interagency Report on Status of International Cybersecurity Standardization for the Internet of Things (IoT)
+      </title>
+      ...
+    <bibitem>"
+----
+With argument `bibdata: true` it ouputs XML wrapped by `bibdata` element and adds flavour `ext` element.
+[source,ruby]
+----
+item.to_xml bibdata: true
+=> "<bibdata type="standard">
+    <fetched>2019-05-17</fetched>
+    <title format="text/plain" language="en" script="Latn">
+      Interagency Report on Status of International Cybersecurity Standardization for the Internet of Things (IoT)
+    </title>
+    ...
+    <ext>
+      <doctype>stadard</doctype>
+      <keyword>cybersecurity</keyword>
+      <keyword>cybersecurity objectives</keyword>
+      ...
+      <commentperiod>
+        <from>2018-02-01</from>
+        <to>2018-04-18</to>
+      </commentperiod>
+    </ext>
+  </bibdata>"
+----
+
+=== Get code, and year
+[source,ruby]
+----
+RelatonNist::NistBibliography.get("8200", "2018", {})
+fetching 8200...
+=> #<RelatonNist::NistBibliographicItem:0x007f82bb3370a8
+...
+----
+
+=== Get short citation
+Short citation is a convetion about a citation's format. The format for NIST publications is:
+----
+NIST {abbrev(series)} {docnumber} {(edition), optional} {(stage), optioal}
+# or
+{abbrev(series)} {docnumber} {(edition), optional} {(stage), optioal}
+----
+- `(stage)` is empty if the state is "final" (published)
+- `(edition)` is the date of publication or update
+- `docnumber` is the full NIST number, including revision, e.g., 800-52
+
+The format for FIPS publications is:
+----
+FIPS {docnumber}
+# or
+NIST FIPS {docnumber}
+----
+[source,ruby]
+----
+RelatonNist::NistBibliography.get("8200", "2018", {})
+fetching 8200...
+=> #<RelatonNist::NistBibliographicItem:0x007f82bb3370a8
+...
+----
+
+== Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+== Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/metanorma/relaton-nist.
+
+== License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/appveyor.yml

@@ -0,0 +1,30 @@
+version: '{build}'
+
+environment:
+  matrix:
+    - RUBY_VERSION: 25
+    - RUBY_VERSION: 24
+    - RUBY_VERSION: 23
+    - RUBY_VERSION: _trunk
+
+matrix:
+  allow_failures:
+    - RUBY_VERSION: _trunk
+
+install:
+  - ps: . { iwr -useb https://raw.githubusercontent.com/metanorma/metanorma-build-scripts/master/appveyor.ps1 } | iex
+  - refreshenv
+
+build_script:
+  - set PATH=C:\Ruby%RUBY_VERSION%\bin;%PATH%
+  - bundle update
+  - bundle install
+
+before_test:
+  - ruby -v
+  - gem -v
+  - bundle -v
+
+test_script:
+  - bundle exec rake
+

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "relaton_nist"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/relaton/processor.rb

@@ -0,0 +1,23 @@
+require "relaton/processor"
+
+module Relaton
+  module RelatonNist
+    class Processor < Relaton::Processor
+
+      def initialize
+        @short = :relaton_nist
+        @prefix = "NIST"
+        @defaultprefix = %r{^(NIST|NISTGCR|ITL Bulletin|JPCRD|NISTIR|CSRC)[ /]}
+        @idtype = "NIST"
+      end
+
+      def get(code, date = nil, opts = {})
+        ::RelatonNist::NistBibliography.get(code, date, opts)
+      end
+
+      def from_xml(xml)
+        ::RelatonNist::XMLParser.from_xml xml
+      end
+    end
+  end
+end

data/lib/relaton_nist/comment_period.rb

@@ -0,0 +1,30 @@
+module RelatonNist
+  class CommentPeriod
+    # @return [Date]
+    attr_reader :from
+
+    # @rerurn [Date, NilClass]
+    attr_reader :to
+
+    # @return [Date, NilClass]
+    attr_reader :extended
+
+    # @param from [Date]
+    # @param to [Date, NilClass]
+    # @param extended [Date, NilClass]
+    def initialize(from, to = nil, extended = nil)
+      @from = from
+      @to = to
+      @extended = extended
+    end
+
+    # @param [Nokogiri::XML::Builder]
+    def to_xml(builder)
+      builder.commentperiod do
+        builder.from from.to_s
+        builder.to to.to_s if to
+        builder.extended extended.to_s if extended
+      end
+    end
+  end
+end

data/lib/relaton_nist/document_status.rb

@@ -0,0 +1,24 @@
+module RelatonNist
+  class DocumentStatus < RelatonBib::DocumentStatus
+    STAGES = %w[
+      draft-internal draft-wip draft-prelim draft-public final final-review
+    ].freeze
+
+    SUBSTAGES = %w[active retired withdrawn].freeze
+
+    # @param stage [String]
+    # @param substage [String, NilClass]
+    # @param iteration [String, NilClass]
+    def initialize(stage:, substage: nil, iteration: nil)
+      # unless STAGES.include? stage
+      #   raise ArgumentError, "invalid argument: stage (#{stage})"
+      # end
+
+      # if substage && !SUBSTAGES.include?(substage)
+      #   raise ArgumentError, "invalid argument: substage (#{substage})"
+      # end
+
+      super
+    end
+  end
+end

data/lib/relaton_nist/hit.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module RelatonNist
+  # Hit.
+  class Hit
+    # @return [RelatonNist::HitCollection]
+    attr_reader :hit_collection
+
+    # @return [Array<Hash>]
+    attr_reader :hit
+
+    # @param hit [Hash]
+    # @param hit_collection [RelatonNist:HitCollection]
+    def initialize(hit, hit_collection = nil)
+      @hit            = hit
+      @hit_collection = hit_collection
+    end
+
+    # Parse page.
+    # @return [RelatonNist::NistBliographicItem]
+    def fetch
+      @fetch ||= Scrapper.parse_page @hit
+    end
+
+    # @return [String]
+    def to_s
+      inspect
+    end
+
+    # @return [String]
+    def inspect
+      "<#{self.class}:#{format('%#.14x', object_id << 1)} "\
+      "@text=\"#{@hit_collection&.text}\" "\
+      "@fetched=\"#{!@fetch.nil?}\" "\
+      "@fullIdentifier=\"#{@fetch&.shortref(nil)}\" "\
+      "@title=\"#{@hit[:code]}\">"
+    end
+
+    # @return [String]
+    def to_xml(**opts)
+      fetch.to_xml **opts
+    end
+
+    # @return [Iteger]
+    def sort_value
+      case hit[:status]
+      when "final" then 1
+      when "withdrawn" then 2
+      when "draft (withdrawn)" then 3
+      else 4
+      end
+    end
+  end
+end

data/lib/relaton_nist/hit_collection.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require "relaton_nist/hit"
+require "addressable/uri"
+require "open-uri"
+
+module RelatonNist
+  # Page of hit collection.
+  class HitCollection < Array
+
+    DOMAIN = "https://csrc.nist.gov"
+
+    # @return [TrueClass, FalseClass]
+    attr_reader :fetched
+
+    # @return [String]
+    attr_reader :text
+
+    # @return [String]
+    attr_reader :year
+
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+
+    # @param ref_nbr [String]
+    # @param year [String]
+    # @param opts [Hash]
+    # @option opts [String] :stage
+    def initialize(ref_nbr, year = nil, opts = {})
+      @text = ref_nbr
+      @year = year
+      from, to = nil
+      if year
+        d = Date.strptime year, "%Y"
+        from = d.strftime "%m/%d/%Y"
+        to   = d.next_year.prev_day.strftime "%m/%d/%Y"
+      end
+      url  = "#{DOMAIN}/publications/search?keywords-lg=#{ref_nbr}"
+      url += "&dateFrom-lg=#{from}" if from
+      url += "&dateTo-lg=#{to}" if to
+      url += if /PD/ =~ opts[:stage]
+               "&status-lg=Draft,Retired Draft,Withdrawn"
+             else
+               "&status-lg=Final,Withdrawn"
+             end
+
+      doc  = Nokogiri::HTML OpenURI.open_uri(::Addressable::URI.parse(url).normalize)
+      hits = doc.css("table.publications-table > tbody > tr").map do |h|
+        link  = h.at("td/div/strong/a")
+        serie = h.at("td[1]").text.strip
+        code  = h.at("td[2]").text.strip
+        title = link.text
+        url   = DOMAIN + link[:href]
+        status = h.at("td[4]").text.strip.downcase
+        release_date = Date.strptime h.at("td[5]").text.strip, "%m/%d/%Y"
+        Hit.new(
+          {
+            code: code, serie: serie, title: title, url: url, status: status,
+            release_date: release_date
+          }, self
+        )
+      end
+      concat(hits.sort { |a, b| a.sort_value - b.sort_value })
+      # concat(hits.map { |h| Hit.new(h, self) })
+      @fetched = false
+      # @hit_pages = hit_pages
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+    # @return [Iecbib::HitCollection]
+    def fetch
+      workers = RelatonBib::WorkersPool.new 4
+      workers.worker(&:fetch)
+      each do |hit|
+        workers << hit
+      end
+      workers.end
+      workers.result
+      @fetched = true
+      self
+    end
+
+    def to_s
+      inspect
+    end
+
+    def inspect
+      "<#{self.class}:#{format('%#.14x', object_id << 1)} @fetched=#{@fetched}>"
+    end
+  end
+end

data/lib/relaton_nist/keyword.rb

@@ -0,0 +1,16 @@
+module RelatonNist
+  class Keyword
+    # @return [Nokogiri::XML::DocumentFragment]
+    attr_reader :element
+
+    # @param element [String]
+    def initialize(element)
+      @element = Nokogiri::XML.fragment element
+    end
+
+    # @param builder [Nokogiri::XML::Builder]
+    def to_xml(builder)
+      builder.keyword element.to_xml
+    end
+  end
+end

data/lib/relaton_nist/nist_bibliographic_item.rb

@@ -0,0 +1,78 @@
+module RelatonNist
+  class NistBibliographicItem < RelatonBib::BibliographicItem
+    # @return [String]
+    attr_reader :doctype
+
+    # @return [Array<RelatonNist::Keyword>]
+    attr_reader :keyword
+
+    # @return [RelatonNist::CommentPeriod]
+    attr_reader :commentperiod
+
+    # @param id [String, NilClass]
+    # @param title [Array<RelatonBib::TypedTitleString>]
+    # @param formattedref [RelatonBib::FormattedRef, NilClass]
+    # @param type [String, NilClass]
+    # @param docid [Array<RelatonBib::DocumentIdentifier>]
+    # @param docnumber [String, NilClass]
+    # @param language [Arra<String>]
+    # @param script [Array<String>]
+    # @param docstatus [RelatonNist::DocumentStatus, NilClass]
+    # @param edition [String, NilClass]
+    # @param version [RelatonBib::BibliographicItem::Version, NilClass]
+    # @param biblionote [Array<RelatonBib::FormattedStrong>]
+    # @param series [Array<RelatonBib::Series>]
+    # @param medium [RelatonBib::Medium, NilClas]
+    # @param place [Array<String>]
+    # @param extent [Array<Relaton::BibItemLocality>]
+    # @param accesslocation [Array<String>]
+    # @param classification [RelatonBib::Classification, NilClass]
+    # @param validity [RelatonBib:Validity, NilClass]
+    # @param fetched [Date, NilClass] default nil
+    # @param doctype [String]
+    # @param keyword [Array<RelatonNist::Keyword>]
+    # @param commentperiod [RelatonNist::CommentPeriod]
+    #
+    # @param dates [Array<Hash>]
+    # @option dates [String] :type
+    # @option dates [String] :from
+    # @option dates [String] :to
+    #
+    # @param contributors [Array<Hash>]
+    # @option contributors [String] :type
+    # @option contributors [String] :from
+    # @option contributirs [String] :to
+    # @option contributors [String] :abbreviation
+    # @option contributors [Array<String>] :roles
+    #
+    # @param abstract [Array<Hash>]
+    # @option abstract [String] :content
+    # @option abstract [String] :language
+    # @option abstract [String] :script
+    # @option abstract [String] :type
+    #
+    # @param relations [Array<Hash>]
+    # @option relations [String] :type
+    # @option relations [RelatonBib::BibliographicItem] :bibitem
+    # @option relations [Array<RelatonBib::BibItemLocality>] :bib_locality
+    def initialize(**args)
+      @doctype = args.delete(:doctype) || "standard"
+      @keyword = args.delete(:keyword) || []
+      @commentperiod = args.delete :commentperiod
+      super
+    end
+
+    # @param builder [Nokogiri::XML::Builder]
+    def to_xml(builder = nil, **opts)
+      super builder, date_format: :short, **opts do |b|
+        if opts[:bibdata]
+          b.ext do
+            b.doctype doctype
+            keyword.each { |kw| kw.to_xml b }
+            commentperiod&.to_xml b
+          end
+        end
+      end
+    end
+  end
+end