relaton 2.1.0 → 3.0.0.pre.alpha.1

data/lib/relaton/nist/processor.rb

@@ -0,0 +1,68 @@
+require "relaton/core/processor"
+
+module Relaton
+  module Nist
+    class Processor < Core::Processor
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_nist
+        @prefix = "NIST"
+        @defaultprefix = %r{^(NIST|NBS|NISTGCR|ITL Bulletin|JPCRD|NISTIR|CSRC|FIPS)(/[^\s])?\s}
+        @idtype = "NIST"
+        @datasets = %w[nist-tech-pubs]
+      end
+
+      # @param code [String]
+      # @param date [String, NilClass] year
+      # @param opts [Hash]
+      # @return [Relaton::Nist::ItemData]
+      def get(code, date = nil, opts = {})
+        require_relative "../nist"
+        Bibliography.get(code, date, opts)
+      end
+
+      #
+      # Fetch all the documents from a source
+      #
+      # @param [String] _source source name
+      # @param [Hash] opts
+      # @option opts [String] :output directory to output documents
+      # @option opts [String] :format
+      #
+      def fetch_data(_source, opts)
+        require_relative "data_fetcher"
+        DataFetcher.fetch(**opts)
+      end
+
+      # @param xml [String]
+      # @return [Relaton::Nist::ItemData]
+      def from_xml(xml)
+        require_relative "../nist"
+        Item.from_xml(xml)
+      end
+
+      # @param yaml [String]
+      # @return [Relaton::Nist::ItemData]
+      def from_yaml(yaml)
+        require_relative "../nist"
+        Item.from_yaml(yaml)
+      end
+
+      # Returns hash of XML grammar
+      # @return [String]
+      def grammar_hash
+        require_relative "../nist"
+        @grammar_hash ||= Nist.grammar_hash
+      end
+
+      #
+      # Remove index file
+      #
+      def remove_index_file
+        require_relative "../nist"
+        Relaton::Index.find_or_create(
+          :nist, url: true, file: "#{INDEXFILE}.yaml", pubid_class: ::Pubid::Nist::Identifier
+        ).remove_file
+      end
+    end
+  end
+end

data/lib/relaton/nist/pubs_export.rb

@@ -0,0 +1,69 @@
+require "singleton"
+
+module Relaton
+  module Nist
+    class PubsExport
+      include Singleton
+
+      DOMAIN = "https://csrc.nist.gov".freeze
+      PUBS_EXPORT = URI.join(DOMAIN, "/CSRC/media/feeds/metanorma/pubs-export")
+      DATAFILEDIR = File.expand_path ".relaton/nist", Dir.home
+      DATAFILE = File.expand_path "pubs-export.zip", DATAFILEDIR
+
+      def initialize
+        @mutex = Mutex.new
+      end
+
+      #
+      # Fetches json data form server
+      #
+      # @return [Array<Hash>] json data
+      #
+      def data
+        @mutex.synchronize do
+          @data ||= begin
+            ctime = File.ctime DATAFILE if File.exist? DATAFILE
+            if !ctime || ctime.to_date < ::Date.today || File.size(DATAFILE).zero?
+              fetch_data(ctime)
+            end
+            unzip
+          end
+        end
+      end
+
+      private
+
+      #
+      # Fetch data form server and save it to file
+      #
+      # @param ctime [Time, nil] file creation time
+      #
+      def fetch_data(ctime)
+        if !ctime || ctime < last_modified
+          @data = nil
+          uri_open = URI.method(:open) || Kernel.method(:open)
+          FileUtils.mkdir_p DATAFILEDIR
+          IO.copy_stream(uri_open.call("#{PUBS_EXPORT}.zip"), DATAFILE)
+        end
+      end
+
+      def last_modified
+        uri = URI("#{PUBS_EXPORT}.meta")
+        Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
+          Time.httpdate(http.head(uri.request_uri)["last-modified"])
+        end
+      end
+
+      #
+      # upack zip file
+      #
+      # @return [Array<Hash>] json data
+      #
+      def unzip
+        Zip::File.open(DATAFILE) do |zf|
+          JSON.parse zf.first.get_input_stream.read
+        end
+      end
+    end
+  end
+end

data/lib/relaton/nist/relation.rb

@@ -0,0 +1,7 @@
+module Relaton
+  module Nist
+    class Relation
+      attribute :type, :string, values: %w[obsoletedBy supersedes supersededBy]
+    end
+  end
+end

data/lib/relaton/nist/scraper.rb

@@ -0,0 +1,290 @@
+require "yaml"
+require "net/http"
+
+module Relaton
+  module Nist
+    class Scraper
+      extend Core::DateParser
+
+      class << self
+        DOMAIN = "https://csrc.nist.gov".freeze
+
+        # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+
+        # Parse page.
+        # @param hit_data [Hash]
+        # @return [Relaton::Nist::ItemData]
+        def parse_page(hit_data)
+          hit_data[:path] ? fetch_gh(hit_data) : parse_json(hit_data)
+        end
+
+        def fetch_gh(hit_data)
+          uri = URI.parse "#{HitCollection::GHNISTDATA}#{hit_data[:path]}"
+          yaml = Net::HTTP.get(uri)
+          Item.from_yaml(yaml)
+        end
+
+        def parse_json(hit_data)
+          item_data = from_json hit_data
+          titles = fetch_titles(hit_data)
+          item_data[:fetched] = ::Date.today.to_s
+          item_data[:type] = "standard"
+          item_data[:title] = titles
+          ItemData.new(**item_data)
+        end
+
+        private
+
+        def from_json(hit_data)
+          json = hit_data[:json]
+          {
+            source: fetch_link(json),
+            docidentifier: fetch_docid(hit_data),
+            date: fetch_dates(json, hit_data[:release_date]),
+            contributor: fetch_contributors(json),
+            edition: fetch_edition(json),
+            language: [json["language"]],
+            script: [json["script"]],
+            status: fetch_status(json),
+            copyright: fetch_copyright(json["published-date"]),
+            relation: fetch_relations_json(hit_data),
+            place: fetch_place,
+            keyword: fetch_keywords(json),
+            ext: Ext.new(
+              doctype: Doctype.new(content: "standard"),
+              flavor: "nist",
+              commentperiod: fetch_commentperiod_json(json),
+            ),
+          }
+        end
+
+        # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+        # Fetch docid.
+        # @param hit [Hash]
+        # @return [Array<Bib::Docidentifier>]
+        def fetch_docid(hit)
+          ids = [Bib::Docidentifier.new(content: hit[:code], type: "NIST", primary: true)]
+          doi = hit[:json]["doi"]&.split("/")&.last
+          ids << Bib::Docidentifier.new(content: doi, type: "DOI") if doi
+          ids
+        end
+
+        # Fetch status.
+        # @param doc [Hash]
+        # @return [Bib::Status]
+        def fetch_status(doc)
+          stage = doc["status"]
+          subst = doc["substage"]
+          iter = iteration(doc["iteration"])
+          Bib::Status.new(
+            stage: Bib::Status::Stage.new(content: stage),
+            substage: subst ? Bib::Status::Stage.new(content: subst) : nil,
+            iteration: iter.to_s,
+          )
+        end
+
+        def iteration(iter)
+          case iter
+          when "initial", "ipd" then 1
+          when /(\d)pd/i then $1
+          else iter
+          end
+        end
+
+        # Fetch titles.
+        # @param hit_data [Hash]
+        # @return [Array<Bib::Title>]
+        def fetch_titles(hit_data)
+          [Bib::Title.new(content: hit_data[:title], language: "en", script: "Latn")]
+        end
+
+        # Fetch dates
+        # @param doc [Hash]
+        # @param release_date [Date]
+        # @return [Array<Bib::Date>]
+        def fetch_dates(doc, release_date) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+          dates = [Bib::Date.new(type: "published", at: release_date.to_s)]
+
+          issued = safe_parse_date doc["issued-date"]
+          updated = safe_parse_date doc["updated-date"]
+          dates << Bib::Date.new(type: "updated", at: updated.to_s) if updated
+          obsoleted = safe_parse_date doc["obsoleted-date"]
+          dates << Bib::Date.new(type: "obsoleted", at: obsoleted.to_s) if obsoleted
+          dates << Bib::Date.new(type: "issued", at: issued.to_s) if issued
+          dates
+        end
+
+        def safe_parse_date(str)
+          return unless str
+
+          parse_date(str, str: false)
+        end
+
+        # @param doc [Hash]
+        # @return [Array<Bib::Contributor>]
+        def fetch_contributors(doc)
+          contribs = []
+          contribs += contributors_json(
+            doc["authors"], "author", doc["language"], doc["script"],
+          )
+          contribs + contributors_json(
+            doc["editors"], "editor", doc["language"], doc["script"],
+          )
+        end
+
+        # @param doc [Array<Hash>]
+        # @param role [String]
+        # @return [Array<Bib::Contributor>]
+        def contributors_json(doc, role, lang = "en", script = "Latn") # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
+          doc.map do |contr|
+            if contr["affiliation"]
+              abbrev = if contr["affiliation"]["acronym"]
+                         Bib::LocalizedString.new(content: contr["affiliation"]["acronym"])
+                       end
+              org = Bib::Organization.new(
+                name: [Bib::TypedLocalizedString.new(content: contr["affiliation"]["name"])],
+                abbreviation: abbrev,
+              )
+            end
+            if contr["surname"]
+              affiliation = []
+              affiliation << Bib::Affiliation.new(organization: org) if org
+              entity = Bib::Person.new(
+                name: full_name(contr, lang, script), affiliation: affiliation,
+              )
+              Bib::Contributor.new(
+                person: entity,
+                role: [Bib::Contributor::Role.new(type: role)],
+              )
+            elsif org
+              Bib::Contributor.new(
+                organization: org,
+                role: [Bib::Contributor::Role.new(type: role)],
+              )
+            end
+          end.compact
+        end
+
+        # @param name [Hash]
+        # @param lang [String]
+        # @param script [String]
+        # @return [Bib::FullName]
+        def full_name(name, lang, script)
+          Bib::FullName.new(
+            surname: Bib::LocalizedString.new(content: name["surname"], language: lang, script: script),
+            forename: name_parts_forename(name["givenName"], lang, script),
+            addition: name_parts(name["suffix"], lang, script),
+            prefix: name_parts(name["title"], lang, script),
+            completename: Bib::LocalizedString.new(content: name["fullName"], language: lang, script: script),
+          )
+        end
+
+        # @param part [String, NilClass]
+        # @param lang [String]
+        # @param script [String]
+        # @return [Array<Bib::LocalizedString>]
+        def name_parts(part, lang, script)
+          return [] unless part
+
+          [Bib::LocalizedString.new(content: part, language: lang, script: script)]
+        end
+
+        # @param part [String, NilClass]
+        # @param lang [String]
+        # @param script [String]
+        # @return [Array<Bib::FullNameType::Forename>]
+        def name_parts_forename(part, lang, script)
+          return [] unless part
+
+          [Bib::FullNameType::Forename.new(content: part, language: lang, script: script)]
+        end
+
+        # @param doc [Hash]
+        # @return [String, NilClass]
+        def fetch_edition(doc)
+          return unless doc["edition"] || doc["revision"]
+
+          rev = doc["edition"] || doc["revision"]
+          Bib::Edition.new(content: "Revision #{rev}")
+        end
+
+        # Fetch copyright.
+        # @param doc [String]
+        # @return [Array<Bib::Copyright>]
+        def fetch_copyright(doc)
+          from = doc&.match(/\d{4}/)&.to_s
+          owner = Bib::ContributionInfo.new(
+            organization: Bib::Organization.new(
+              name: [Bib::TypedLocalizedString.new(content: "National Institute of Standards and Technology")],
+              abbreviation: Bib::LocalizedString.new(content: "NIST"),
+              uri: [Bib::Uri.new(content: "www.nist.gov")],
+            ),
+          )
+          [Bib::Copyright.new(owner: [owner], from: from)]
+        end
+
+        # Fetch links.
+        # @param doc [Hash]
+        # @return [Array<Bib::Uri>]
+        def fetch_link(doc)
+          links = []
+          links << Bib::Uri.new(type: "src", content: doc["uri"]) if doc["uri"]
+          if doc["doi"]
+            links << Bib::Uri.new(type: "doi", content: "https://doi.org/#{doc['doi']}")
+          end
+          links
+        end
+
+        # @return [Array<Bib::Place>]
+        def fetch_place
+          [Bib::Place.new(city: "Gaithersburg", region: [Bib::Place::RegionType.new(iso: "MD")])]
+        end
+
+        def fetch_relations_json(hit) # rubocop:disable Metrics/AbcSize
+          doc = hit[:json]
+          relations = doc["supersedes"].map do |r|
+            doc_relation "supersedes", hit[:code], r["uri"]
+          end
+
+          relations + doc["superseded-by"].map do |r|
+            doc_relation "updates", hit[:code], r["uri"]
+          end
+        end
+
+        # @param type [String]
+        # @param ref [String]
+        # @param uri [String]
+        # @return [Nist::Relation]
+        def doc_relation(type, ref, uri, lang = "en", script = "Latn") # rubocop:disable Metrics/MethodLength
+          if type == "supersedes"
+            descr = Bib::LocalizedMarkedUpString.new(content: "supersedes", language: lang, script: script)
+            t = "obsoletes"
+          else t = type
+          end
+          ids = [Bib::Docidentifier.new(content: ref, type: "NIST", primary: true)]
+          link = [Bib::Uri.new(type: "src", content: uri)]
+          bib = ItemData.new(formattedref: Bib::Formattedref.new(content: ref), source: link, docidentifier: ids)
+          Relation.new(type: t, description: descr, bibitem: bib)
+        end
+
+        # @param doc [Hash]
+        # @return [Array<Bib::Keyword>]
+        def fetch_keywords(doc)
+          doc["keywords"].map do |kw|
+            text = kw.is_a?(String) ? kw : kw.text
+            Bib::Keyword.new(vocab: Bib::LocalizedString.new(content: text))
+          end
+        end
+
+        # @param json [Hash]
+        # @return [Nist::CommentPeriod, NilClass]
+        def fetch_commentperiod_json(json)
+          return unless json["comment-from"]
+
+          CommentPeriod.new from: json["comment-from"], to: json["comment-to"]
+        end
+      end
+    end
+  end
+end

data/lib/relaton/nist/series.yaml

@@ -0,0 +1,49 @@
+---
+AMS: Advanced Manufacturing Series
+BRPD: Basic Radio Propagation Predictions Series
+BH: Building and Housing Reports
+BMS: Building Materials and Structures Reports
+BSS: Building Science Series
+CRPL: CRPL Solar-Geophysical Data
+IP: CRPL Ionospheric Predictions
+CIRC: Circulars
+CIS: Consumer Information Series
+CS: Commercial Standards
+CSM: Commercial Standards Monthly
+CSWP: Cybersecurity White Papers
+EAB: Economic Analysis Briefs
+FIPS: Federal Information Processing Standards Publications
+GCR: Grant/Contractor Reports
+HB: Handbooks
+HR: Hydraulic Research in the United States
+IRPL: Interservice Radio Propagation Laboratory
+LCIRC: Letter Circular
+MONO: Monographs
+MP: Miscellaneous Publications
+NCSTAR: National Construction Safety Team Act Reports
+NSRDS: National Standard Reference Data Series
+IR: IR (Interagency/Internal Reports)
+OWMWP: Office of Weights and Measures White Papers
+PC: Photographic Circulars
+RPT: NBS Reports
+SIBS: Special Interior Ballistics Studies
+SP: Special Publications (General)
+SP250: 'SP 250: Calibration Services'
+SP260: 'SP 260: Standard Reference Materials'
+SP300: 'SP 300: Precision Measurement and Calibration'
+SP400: 'SP 400: Semiconductor Measurement Technology'
+SP480: 'SP 480: Law Enforcement Technology'
+SP500: 'SP 500: Computer Systems Technology'
+SP700: 'SP 700: Industrial Measurement Series'
+SP800: 'SP 800: Computer Security Series'
+SP823: 'SP 823: Integrated Services Digital Network Series'
+SP960: 'SP 960: NIST Recommended Practice Guides'
+SP1200: 'SP 1200: Protocols'
+SP1500: 'SP 1500: Working Group Papers'
+SP1800: 'SP 1800: NIST Cybersecurity Practice Guides'
+SP1900: 'SP 1900: Cyber-Physical Systems'
+SP2000: 'SP 2000: Standards Coordination'
+SP2100: 'SP 2100: Conference Proceedings'
+TIBM: Technical Information on Building Materials
+TN: Technical Notes
+TTB: Technology Transfer Brief

data/lib/relaton/nist/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Nist
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-nist".freeze
+    end
+  end
+end

data/lib/relaton/nist.rb

@@ -0,0 +1,34 @@
+require "pubid"
+require "relaton/index"
+require "relaton/bib"
+require_relative "version"
+require_relative "nist/util"
+require_relative "nist/item_data"
+require_relative "nist/item"
+require_relative "nist/relation"
+require_relative "nist/bibitem"
+require_relative "nist/bibdata"
+require_relative "nist/pubs_export"
+require_relative "nist/hit"
+require_relative "nist/hit_collection"
+require_relative "nist/scraper"
+require_relative "nist/bibliography"
+# require_relative "nist/tech_pubs_parser"
+
+module Relaton
+  module Nist
+    # Pubid-based index (index-v2): `:id` is a Pubid::Nist::Identifier hash, so
+    # search narrows by number via binary search. The legacy string index-v1
+    # (for older gem versions) is rebuilt separately by the relaton-data-nist
+    # crawler, not by this gem's DataFetcher.
+    INDEXFILE = "index-v2"
+
+    class Error < StandardError; end
+
+    # Returns hash of XML grammar
+    # @return [String]
+    def self.grammar_hash
+      Digest::MD5.hexdigest Relaton::VERSION
+    end
+  end
+end

data/lib/relaton/oasis/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Oasis
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/oasis/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Oasis
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/oasis/bibliography.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "net/http"
+
+module Relaton
+  module Oasis
+    module Bibliography
+      ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-oasis/refs/heads/v2/"
+
+      class << self
+        def search(text, _year = nil, _opts = {}) # rubocop:disable Metrics/MethodLength
+          Util.info "Fetching from Relaton repository ...", key: text
+          /^(?:OASIS\s)?(?<code>.+)/ =~ text
+          row = find_index_entry(code)
+          unless row
+            Util.info "Not found.", key: text
+            return
+          end
+
+          uri = URI("#{ENDPOINT}#{row[:file]}")
+          parse_item(fetch_yaml(uri), text)
+        rescue SocketError, Errno::EINVAL, Errno::ECONNRESET, EOFError,
+               Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+               Net::ProtocolError, Net::ReadTimeout,
+               OpenSSL::SSL::SSLError, Errno::ETIMEDOUT => e
+          raise Relaton::RequestError,
+                "Could not access #{uri}: #{e.message}"
+        end
+
+        def get(ref, year = nil, opts = {})
+          search(ref, year, opts)
+        end
+
+        private
+
+        def find_index_entry(code)
+          index = Relaton::Index.find_or_create(
+            :oasis, url: "#{ENDPOINT}#{INDEXFILE}.zip"
+          )
+          index.search(code).min_by { |r| r[:id] }
+        end
+
+        def fetch_yaml(uri)
+          resp = Net::HTTP.get_response uri
+          unless resp.code == "200"
+            raise Relaton::RequestError,
+                  "Could not access #{uri}: HTTP #{resp.code}"
+          end
+          resp.body
+        end
+
+        def parse_item(yaml, text)
+          item = Item.from_yaml yaml
+          Util.info "Found: `#{item.docidentifier.first.content}`", key: text
+          item.tap { |i| i.fetched = Date.today.to_s }
+        end
+      end
+    end
+  end
+end

data/lib/relaton/oasis/browser_agent.rb

@@ -0,0 +1,71 @@
+require "ferrum"
+require "nokogiri"
+
+module Relaton
+  module Oasis
+    # Thin Ferrum-backed agent that drives headless Chrome with stealth tweaks
+    # so the Cloudflare-protected oasis-open.org host serves real HTML instead
+    # of a "Just a moment..." challenge. Mirrors the pattern used by
+    # Relaton::Cie::BrowserAgent.
+    class BrowserAgent
+      UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " \
+           "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
+      CHALLENGE_MARKERS = ["Just a moment", "challenge-platform"].freeze
+      MAX_CHALLENGE_WAIT = 30
+
+      def initialize
+        @browser = Ferrum::Browser.new(
+          headless: true,
+          timeout: 90,
+          process_timeout: 90,
+          window_size: [1366, 768],
+          browser_options: {
+            "disable-blink-features" => "AutomationControlled",
+            "disable-quic" => nil,
+            "no-sandbox" => nil,
+          },
+        )
+        @browser.headers.set(
+          "Accept-Language" => "en-US,en;q=0.9",
+          "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9," \
+                      "image/webp,*/*;q=0.8",
+          "User-Agent" => UA,
+        )
+        @browser.evaluate_on_new_document(<<~JS)
+          Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+          Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
+          Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5] });
+          window.chrome = { runtime: {} };
+        JS
+      end
+
+      def get(url)
+        @browser.go_to(url)
+        wait_for_challenge
+        Nokogiri::HTML(@browser.body)
+      end
+
+      # HTTP status code of the most recent navigation's main resource,
+      # or nil if no navigation has happened yet.
+      def last_status
+        @browser&.network&.status
+      end
+
+      def quit
+        @browser&.quit
+      ensure
+        @browser = nil
+      end
+
+      private
+
+      def wait_for_challenge
+        MAX_CHALLENGE_WAIT.times do
+          return unless CHALLENGE_MARKERS.any? { |m| @browser.body.include?(m) }
+
+          sleep 1
+        end
+      end
+    end
+  end
+end