pubid-nist 0.1.8

data/exe/pubid-nist

@@ -0,0 +1,97 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require_relative "../lib/pubid-nist"
+require "thor"
+require "csv"
+require "lightly"
+
+def render_report_doc(doc)
+  [doc[:finalPubId] != doc[:id],
+   doc[:finalPubId],
+   doc[:id],
+   doc[:mr] != doc[:doi],
+   doc[:mr],
+   doc[:doi],
+   doc[:title]]
+end
+
+class Pubid::Nist::CLI < Thor
+  desc "report", "Create report for NIST Tech Pubs database (fetches from GitHub)"
+  option :csv, aliases: "-c", type: :boolean, desc: "Export to CSV format"
+  option :updated, aliases: "-u", type: :boolean,
+                   desc: "Return only updated identifiers",
+                   default: false
+  option :pull, aliases: "-p", type: :boolean,
+                desc: "Update cache from NIST Tech Pubs database",
+                default: false
+  def report
+    heading = %w(
+      ID\ changed?
+      New\ PubID
+      Document\ ID
+      DOI\ changed?
+      New\ PubID-MR
+      DOI
+      Title
+    )
+
+    Lightly.clear "documents" if options[:pull]
+    # Pubid::Nist::NistTechPubs.fetch
+    if Lightly.cached? "documents"
+      warn "Using nist-tech-pubs.xml file from local cache"
+    else
+      warn "Cached nist-tech-pubs.xml not present, downloading from GitHub..."
+    end
+
+    puts options[:csv] && heading.to_csv || heading.join(" | ")
+
+    documents = Pubid::Nist::NistTechPubs.status
+
+    documents = documents.reject { |doc| doc[:finalPubId] == doc[:id] } if options[:updated]
+
+    documents.each do |doc|
+      if options[:csv]
+        puts render_report_doc(doc).to_csv
+      else
+        puts (render_report_doc(doc).map do |v|
+          case v
+          when false
+            " -"
+          when true
+            "✅"
+          else
+            v
+          end
+        end).join(" | ")
+      end
+    end
+  end
+
+  desc "convert", "Convert legacy NIST Tech Pubs ID to NIST PubID"
+  option :style, aliases: "-s", type: :string,
+                 desc: "Convert to PubID style (short|long|mr|abbrev)",
+                 default: "short"
+  option :format, aliases: "-f", type: :string,
+                  desc: "Render in format (json|string)",
+                  default: "string"
+  def convert(code)
+    unless %w[mr long short abbrev].include?(options[:style].downcase)
+      raise "Invalid PubID style"
+    end
+
+    raise "Invalid render format" unless %w[string json].include? options[:format].downcase
+
+    unless code.empty?
+      if options[:format] == "string"
+        puts Pubid::Nist::Document.parse(code).to_s(options[:style].to_sym)
+      else
+        puts Pubid::Nist::Document.parse(code).to_json
+      end
+    end
+  rescue Pubid::Nist::Errors::ParseError
+    puts "[Error] This does not seem to be a valid NIST Tech Pubs legacy identifier"
+  end
+end
+
+Pubid::Nist::CLI.start(ARGV)

data/lib/pubid/nist/document.rb

@@ -0,0 +1,286 @@
+# frozen_string_literal: true
+
+require "json"
+
+UPDATE_CODES = YAML.load_file(File.join(File.dirname(__FILE__), "../../../update_codes.yaml"))
+
+REVISION_DESC = {
+  long: ", Revision ",
+  abbrev: ", Rev. ",
+  short: "r",
+  mr: "r",
+}.freeze
+
+VERSION_DESC = {
+  long: ", Version ",
+  abbrev: ", Ver. ",
+  short: "ver",
+  mr: "ver",
+}.freeze
+
+VOLUME_DESC = {
+  long: ", Volume ",
+  abbrev: ", Vol. ",
+  short: "v",
+  mr: "v",
+}.freeze
+
+PART_DESC = {
+  long: " Part ",
+  abbrev: " Pt. ",
+  short: "pt",
+  mr: "pt",
+}.freeze
+
+EDITION_DESC = {
+  long: " Edition ",
+  abbrev: " Ed. ",
+  short: "e",
+  mr: "e",
+}.freeze
+
+SUPPLEMENT_DESC = {
+  long: " Supplement ",
+  abbrev: " Suppl. ",
+  short: "sup",
+  mr: "sup",
+}.freeze
+
+SECTION_DESC = {
+  long: " Section ",
+  abbrev: " Sec. ",
+  short: "sec",
+  mr: "sec",
+}.freeze
+
+APPENDIX_DESC = {
+  long: " Appendix ",
+  abbrev: " App. ",
+  short: "app",
+  mr: "app",
+}.freeze
+
+ERRATA_DESC = {
+  long: " Errata ",
+  abbrev: " Err. ",
+  short: "err",
+  mr: "err",
+}.freeze
+
+INDEX_DESC = {
+  long: " Index ",
+  abbrev: " Index. ",
+  short: "indx",
+  mr: "indx",
+}.freeze
+
+INSERT_DESC = {
+  long: " Insert ",
+  abbrev: " Ins. ",
+  short: "ins",
+  mr: "ins",
+}.freeze
+
+module Pubid::Nist
+  class Document
+    attr_accessor :serie, :code, :revision, :publisher, :version, :volume,
+                  :part, :addendum, :stage, :translation, :update_number,
+                  :edition, :supplement, :update_year, :update_month,
+                  :section, :appendix, :errata, :index, :insert
+
+    def initialize(publisher:, serie:, docnumber:, stage: nil, supplement: nil,
+                   edition_month: nil, edition_year: nil, edition_day: nil, **opts)
+      @publisher = publisher
+      @serie = serie
+      @code = docnumber
+      @stage = Stage.new(stage.to_s) if stage
+      @supplement = (supplement.is_a?(Array) && "") || supplement
+      @edition = parse_edition(edition_month, edition_year, edition_day) if edition_month || edition_year
+      opts.each { |key, value| send("#{key}=", value.to_s) }
+    end
+
+    def parse_edition(edition_month, edition_year, edition_day)
+      if edition_month
+        date = Date.parse("#{edition_day || '01'}/#{edition_month}/#{edition_year}")
+        if edition_day
+          Edition.new(month: date.month, year: date.year, day: date.day)
+        else
+          Edition.new(month: date.month, year: date.year)
+        end
+      else
+        Edition.new(year: edition_year.to_i)
+      end
+    end
+
+    # returns weight based on amount of defined attributes
+    def weight
+      instance_variables.inject(0) do |sum, var|
+        sum + (instance_variable_get(var).nil? ? 0 : 1)
+      end
+    end
+
+    def ==(other)
+      other.instance_variables.each do |var|
+        return false if instance_variable_get(var) != other.instance_variable_get(var)
+      end
+      true
+    end
+
+    def merge(document)
+      document.instance_variables.each do |var|
+        val = document.instance_variable_get(var)
+        current_val = instance_variable_get(var)
+        if [:@serie, :@publisher].include?(var) ||
+            (val && current_val.nil?) ||
+            (val && current_val.to_s.length < val.to_s.length)
+          instance_variable_set(var, val)
+        end
+      end
+
+      self
+    end
+
+    def self.update_old_code(code)
+      UPDATE_CODES.each do |from, to|
+        code = code.gsub(from, to)
+      end
+      code
+    end
+
+    def self.parse(code)
+      code = update_old_code(code)
+      DocumentTransform.new.apply(DocumentParser.new.parse(code))
+    rescue Parslet::ParseFailed => failure
+      raise Pubid::Nist::Errors::ParseError, "#{failure.message}\ncause: #{failure.parse_failure_cause.ascii_tree}"
+    end
+
+    def to_s(format = :short)
+      result = render_serie(format)
+      result += " " unless format == :short || stage.nil?
+      result += "#{stage&.to_s(format)}"\
+                " #{code}#{render_part(format)}#{render_edition(format)}"\
+                "#{render_localities(format)}"\
+                "#{render_update(format)}#{render_translation(format)}"
+      result = render_addendum(result, format)
+
+      return result.gsub(" ", ".") if format == :mr
+
+      result
+    end
+
+    def to_json(*args)
+      result = {
+        styles: {
+          short: to_s(:short),
+          abbrev: to_s(:abbrev),
+          long: to_s(:long),
+          mr: to_s(:mr),
+        }
+      }
+
+      instance_variables.each do |var|
+        val = instance_variable_get(var)
+        result[var.to_s.gsub('@', '')] = val unless val.nil?
+      end
+      result.to_json(*args)
+    end
+
+    def render_serie(format)
+      if serie.to_s(format).include?(publisher.to_s(format))
+        return serie.to_s(format)
+      end
+
+      "#{publisher.to_s(format)} #{serie.to_s(format)}"
+    end
+
+    def render_part(format)
+      result = ""
+      result += "#{VOLUME_DESC[format]}#{volume}" unless volume.nil?
+      result += "#{PART_DESC[format]}#{part}" unless part.nil?
+      result
+    end
+
+    def render_edition(format)
+      result = ""
+
+      result += "#{EDITION_DESC[format]}#{edition.to_s}" unless edition.nil?
+      result += "#{REVISION_DESC[format]}#{revision == '' ? '1' : revision}" if revision
+      result += "#{VERSION_DESC[format]}#{version}" unless version.nil?
+      result
+    end
+
+    def render_localities(format)
+      result = ""
+      result += "#{SUPPLEMENT_DESC[format]}#{supplement}" unless supplement.nil?
+      result += "#{SECTION_DESC[format]}#{section}" unless section.nil?
+      result += "#{APPENDIX_DESC[format]}" unless appendix.nil?
+      result += "#{ERRATA_DESC[format]}" unless errata.nil?
+      result += INDEX_DESC[format] unless index.nil?
+      result += INSERT_DESC[format] unless insert.nil?
+
+      result
+    end
+
+    def render_update(format)
+      return "" if update_year.nil?
+
+      if update_month && update_number.nil?
+        @update_number = "1"
+      end
+
+      if update_year&.length == 2
+        @update_year = "19#{update_year}"
+      end
+
+      if update_number.match?(/\d+/)
+        update_text = update_number
+        update_text += "-#{update_year}" if update_year && !update_year.empty?
+        if update_month
+          date = Date.parse("01/#{update_month}/#{update_year}")
+          update_text += sprintf("%02d", date.month)
+        end
+      else
+        update_text = "1"
+      end
+
+      case format
+      when :long
+        " Update #{update_text}"
+      when :abbrev
+        " Upd. #{update_text}"
+      when :short
+        "/Upd#{update_text}"
+      when :mr
+        ".u#{update_text}"
+      end
+    end
+
+    def render_translation(format)
+      return "" if translation.nil?
+
+      case format
+      when :long, :abbrev
+        " (#{translation.upcase})"
+      when :mr
+        ".#{translation}"
+      when :short
+        "(#{translation})"
+      end
+    end
+
+    def render_addendum(input, format)
+      return input unless addendum
+
+      case format
+      when :long
+        "Addendum to #{input}"
+      when :abbrev
+        "Add. to #{input}"
+      when :short
+        "#{input} Add."
+      when :mr
+        "#{input}.add-1"
+      end
+    end
+  end
+end

data/lib/pubid/nist/document_parser.rb

@@ -0,0 +1,34 @@
+module Pubid::Nist
+  class DocumentParser < Parslet::Parser
+    attr_accessor :parsed
+
+    rule(:series) do
+      ((SERIES["long"].keys
+                     .sort_by(&:length).reverse
+                     .flatten
+                     .reduce do |acc, s|
+          (acc.is_a?(String) ? str(acc) : acc) | str(s)
+        end).as(:series) |
+        (SERIES["mr"].values.reduce do |acc, s|
+          (acc.is_a?(String) ? str(acc) : acc) | str(s)
+        end).as(:series_mr)) >> any.repeat.as(:remaining)
+    end
+
+    root(:series)
+
+    def parse(code)
+      parsed = super(code)
+      series = if parsed[:series]
+                 parsed[:series].to_s
+               else
+                 SERIES["mr"].key(parsed[:series_mr].to_s)
+               end
+      parser = find_parser(series)
+      parser.new.parse(parsed[:remaining].to_s).merge({ series: series })
+    end
+
+    def find_parser(series)
+      PARSERS_CLASSES[series] || Pubid::Nist::Parsers::Default
+    end
+  end
+end

data/lib/pubid/nist/document_transform.rb

@@ -0,0 +1,21 @@
+module Pubid::Nist
+  class DocumentTransform
+    def apply(tree, context = nil)
+      series = tree[:series].to_s.gsub(".", " ")
+      document_parameters = tree.reject do |k, _|
+        %i[report_number first_report_number second_report_number series parts].include?(k)
+      end
+      tree[:parts]&.each { |part| document_parameters.merge!(part) }
+      report_number = tree.values_at(:first_report_number,
+                                     :second_report_number).compact.join("-").upcase
+
+      # using :report_number when need to keep original words case
+      report_number = tree[:report_number] if report_number.empty?
+
+      Document.new(publisher: Publisher.parse(series),
+                   serie: Serie.new(serie: series),
+                   docnumber: report_number,
+                   **document_parameters)
+    end
+  end
+end

data/lib/pubid/nist/edition.rb

@@ -0,0 +1,26 @@
+module Pubid::Nist
+  class Edition
+    attr_accessor :year, :month, :day, :parsed, :sequence
+
+    def initialize(parsed: nil, year: nil, month: nil, day: nil, sequence: nil)
+      @parsed = parsed
+      @year = year
+      @month = month
+      @day = day
+      @sequence = sequence
+    end
+
+    def to_s
+      result = (@sequence && [@sequence]) || []
+      if @day
+        result << Date.new(@year, @month, @day).strftime("%Y%m%d")
+      elsif @month
+        result << Date.new(@year, @month).strftime("%Y%m")
+      elsif @year
+        result << Date.new(@year).strftime("%Y")
+      end
+
+      result.join("-")
+    end
+  end
+end

data/lib/pubid/nist/errors.rb

@@ -0,0 +1,5 @@
+module Pubid::Nist
+  module Errors
+    class ParseError < StandardError; end
+  end
+end

data/lib/pubid/nist/nist_tech_pubs.rb

@@ -0,0 +1,110 @@
+require "relaton_nist/data_fetcher"
+require "nokogiri"
+require "open-uri"
+require "lightly"
+
+Lightly.life = "24h"
+
+module Pubid::Nist
+  class NistTechPubs
+    URL = "https://raw.githubusercontent.com/usnistgov/NIST-Tech-Pubs/nist-pages/xml/allrecords.xml".freeze
+
+    @converted_id = @converted_doi = {}
+
+    class << self
+
+      attr_accessor :documents, :converted_id, :converted_doi
+
+      def fetch
+        Lightly.prune
+        @documents ||= Lightly.get "documents" do
+          Nokogiri::XML(URI.open(URL))
+            .xpath("/body/query/doi_record/report-paper/report-paper_metadata")
+            .map { |doc| parse_docid doc }
+        end
+      rescue StandardError => e
+        warn e.message
+        []
+      end
+
+      def convert(doc)
+        id = @converted_id[doc[:id]] ||= Pubid::Nist::Document.parse(doc[:id])
+        return id unless doc.key?(:doi)
+
+        begin
+          doi = @converted_doi[doc[:doi]] ||=
+            Pubid::Nist::Document.parse(doc[:doi])
+        rescue Errors::ParseError
+          return id
+        end
+        # return more complete pubid
+        id.merge(doi)
+      rescue Errors::ParseError
+        @converted_doi[doc[:doi]] ||= Pubid::Nist::Document.parse(doc[:doi])
+      end
+
+      def parse_docid(doc)
+        id = doc.at("publisher_item/item_number", "publisher_item/identifier")
+          .text.sub(%r{^/}, "")
+        doi = doc.at("doi_data/doi").text.gsub("10.6028/", "")
+        title = doc.at("titles/title").text
+        title += " #{doc.at('titles/subtitle').text}" if doc.at("titles/subtitle")
+        case doi
+        when "10.6028/NBS.CIRC.12e2revjune" then id.sub!("13e", "12e")
+        when "10.6028/NBS.CIRC.36e2" then id.sub!("46e", "36e")
+        when "10.6028/NBS.HB.67suppJune1967" then id.sub!("1965", "1967")
+        when "10.6028/NBS.HB.105-1r1990" then id.sub!("105-1-1990", "105-1r1990")
+        when "10.6028/NIST.HB.150-10-1995" then id.sub!(/150-10$/, "150-10-1995")
+        end
+
+        { id: id, doi: doi, title: title }
+      end
+
+      def comply_with_pubid
+        fetch.select do |doc|
+          convert(doc).to_s == doc[:id]
+        rescue Errors::ParseError
+          false
+        end
+      end
+
+      def different_with_pubid
+        fetch.reject do |doc|
+          convert(doc).to_s == doc[:id]
+        rescue Errors::ParseError
+          true
+        end
+      end
+
+      def parse_fail_with_pubid
+        fetch.select do |doc|
+          convert(doc).to_s && false
+        rescue Errors::ParseError
+          true
+        end
+      end
+
+      # returning current document id, doi, title and final PubID
+      def status
+        fetch.lazy.map do |doc|
+          final_doc = convert(doc)
+          {
+            id: doc[:id],
+            doi: doc[:doi],
+            title: doc[:title],
+            finalPubId: final_doc.to_s,
+            mr: final_doc.to_s(:mr),
+          }
+        rescue Errors::ParseError
+          {
+            id: doc[:id],
+            doi: doc[:doi],
+            title: doc[:title],
+            finalPubId: "parse error",
+            mr: "parse_error",
+          }
+        end
+      end
+    end
+  end
+end

data/lib/pubid/nist/parsers/default.rb

@@ -0,0 +1,107 @@
+module Pubid::Nist
+  module Parsers
+    class Default < Parslet::Parser
+      rule(:identifier) do
+        stage.maybe >> (str(" ") | str(".")) >> report_number >> parts.repeat.as(:parts)
+      end
+
+      rule(:digits) { match('\d').repeat(1) }
+      rule(:letters) { match('[A-Za-z]').repeat(1) }
+      rule(:year_digits) { match('\d').repeat(4, 4) }
+      rule(:month_letters) { match('[A-Za-z]').repeat(3, 3) }
+      rule(:number_suffix) { match("[aA-Z]") }
+      # rule(:small)
+
+      rule(:parts) do
+        (edition | revision | version | volume | part | update | addendum | translation |
+           supplement | errata | index | insert | section | appendix)
+      end
+
+      rule(:appendix) do
+        str("app").as(:appendix)
+      end
+
+      rule(:supplement) do
+        (str("supp") | str("sup")) >> match('\d').repeat.as(:supplement)
+      end
+
+      rule(:errata) do
+        str("-").maybe >> str("errata").as(:errata)
+      end
+
+      rule(:index) do
+        (str("index") | str("indx")).as(:index)
+      end
+
+      rule(:insert) do
+        (str("insert") | str("ins")).as(:insert)
+      end
+
+      rule(:stage) do
+        (str("(") >> (STAGES.keys.reduce do |acc, s|
+          (acc.is_a?(String) ? str(acc) : acc) | str(s)
+        end).as(:stage) >> str(")"))
+      end
+
+      rule(:digits_with_suffix) do
+        digits >> # do not match with 428P1
+          (number_suffix >> match('\d').absent?).maybe
+      end
+
+      rule(:first_report_number) do
+        digits_with_suffix.as(:first_report_number)
+      end
+
+      rule(:second_report_number) do
+        digits_with_suffix.as(:second_report_number)
+      end
+
+      rule(:report_number) do
+        first_report_number >> (str("-") >> second_report_number).maybe
+      end
+
+      rule(:part_prefixes) { str("pt") | str("p") }
+
+      rule(:part) do
+        part_prefixes >> digits.as(:part)
+      end
+
+      rule(:revision) do
+        str("r") >> ((digits >> match("[a-z]").maybe).maybe).as(:revision)
+      end
+
+      rule(:volume) do
+        str("v") >> digits.as(:volume)
+      end
+
+      rule(:version) do
+        str("ver") >> digits.as(:version)
+      end
+
+      rule(:update) do
+        (str("/Upd") | str("/upd")) >> digits.as(:update_number) >> str("-") >> digits.as(:update_year)
+      end
+
+      rule(:translation) do
+        (str("(") >> match('\w').repeat(3, 3).as(:translation) >> str(")")) |
+          (str(".") >> match('\w').repeat(3, 3).as(:translation))
+      end
+
+      rule(:edition_prefixes) { str("e") }
+
+      rule(:edition) do
+        edition_prefixes >> digits.as(:edition)
+      end
+
+      rule(:addendum) do
+        (str("-add") | str(".add-1")).as(:addendum)
+      end
+
+      rule(:section) do
+        str("sec") >> digits.as(:section)
+      end
+
+      root(:identifier)
+    end
+  end
+end

data/lib/pubid/nist/parsers/fips_pub.rb

@@ -0,0 +1,8 @@
+require_relative "nbs_fips"
+
+module Pubid::Nist
+  module Parsers
+    class FipsPub < NbsFips
+    end
+  end
+end

data/lib/pubid/nist/parsers/nbs_bh.rb

@@ -0,0 +1,6 @@
+module Pubid::Nist
+  module Parsers
+    class NbsBh < Default
+    end
+  end
+end