relaton 3.0.8 → 3.1.0

data/lib/relaton/sae/bibliography.rb

@@ -0,0 +1,128 @@
+require "net/http"
+
+module Relaton
+  module Sae
+    module Bibliography
+      # Transport failures of the document fetch. They become
+      # `Relaton::RequestError`, the one class `Relaton::Db#net_retry` retries.
+      NET_ERRORS = [
+        SocketError, Errno::EINVAL, Errno::ECONNRESET, Errno::ECONNREFUSED,
+        Errno::ETIMEDOUT, EOFError, Net::HTTPBadResponse,
+        Net::HTTPHeaderSyntaxError, Net::ProtocolError, Net::OpenTimeout,
+        Net::ReadTimeout, OpenSSL::SSL::SSLError
+      ].freeze
+
+      class << self
+        #
+        # Fetch an SAE standard.
+        #
+        # @param ref [String, Pubid::Sae::Identifier] `SAE J3016`,
+        #   `SAE J3016:2021`, or SAE's native edition code `SAE J3016_202104`
+        # @param year [String, nil]
+        # @param _opts [Hash]
+        # @return [Relaton::Sae::ItemData, nil]
+        #
+        def get(ref, year = nil, _opts = {})
+          Util.info "Fetching from Relaton repository ...", key: ref.to_s
+          pubid = parse_ref ref, year
+          return not_found(ref) unless pubid
+
+          row = best_match pubid
+          item = row && fetch_doc(row[:file])
+          return not_found(ref) unless item
+
+          Util.info "Found: `#{primary_id item}`", key: ref.to_s
+          item
+        end
+
+        private
+
+        def not_found(ref)
+          Util.info "Not found.", key: ref.to_s
+          nil
+        end
+
+        #
+        # The query as a `Pubid::Sae::Identifier`.
+        #
+        # A parsed pubid comes from `Relaton::Db` (relaton#205): it is not
+        # mutated, so a year argument goes on a copy. A reference pubid cannot
+        # read answers nil, so `get` answers not-found with a log line instead
+        # of raising (relaton#235): `Db` passes such a reference as a String.
+        #
+        # @return [Pubid::Sae::Identifier, nil]
+        #
+        def parse_ref(ref, year)
+          ref = parse_text(ref.to_s.strip) unless ref.is_a?(::Pubid::Sae::Identifier)
+          return ref unless ref && year
+
+          copy = ::Pubid::Sae::Identifier.from_hash ref.to_hash
+          copy.date = ::Pubid::Sae::Components::Date.new(year: year.to_i)
+          copy
+        end
+
+        def parse_text(text)
+          ::Pubid::Sae::Identifier.parse text
+        rescue ::Pubid::Errors::Error, ArgumentError => e
+          Util.info "Not an SAE identifier pubid reads: #{e.message}", key: text
+          nil
+        end
+
+        #
+        # The index row for a query: the newest edition among its matches.
+        #
+        # Passing the pubid (not a String) is what makes `Index::Type#search`
+        # narrow to the `number` bucket, and with no block it selects with
+        # pubid's subset match `query === row`: a revision, a year or a month
+        # the query omits matches any value, and the type (`J`, `AMS`, …), a
+        # part and an amendment must agree.
+        #
+        # Of the matches, the highest revision wins (aerospace standards), then
+        # the latest date (J-series editions). `r[:file]` breaks a tie, because
+        # the index sort is not stable.
+        #
+        # @param pubid [Pubid::Sae::Identifier]
+        # @return [Hash, nil]
+        #
+        def best_match(pubid)
+          index.search(pubid).max_by do |r|
+            id = r[:id]
+            [revision_rank(id.revision), id.year.to_i, id.date&.month.to_i,
+             r[:file]]
+          end
+        end
+
+        # `B` follows `A`, and a two-letter revision (`AA`) follows `Z`.
+        def revision_rank(revision)
+          [revision.to_s.size, revision.to_s]
+        end
+
+        # @param file [String] the row's path in the data repo
+        # @return [Relaton::Sae::ItemData, nil] nil when the file is missing
+        def fetch_doc(file)
+          uri = URI("#{ENDPOINT}#{file}")
+          resp = Net::HTTP.get_response uri
+          return if resp.code == "404"
+          unless resp.code == "200"
+            raise Relaton::RequestError,
+                  "Could not access #{uri}: HTTP #{resp.code}"
+          end
+
+          Item.from_yaml(resp.body).tap { |i| i.fetched = Date.today.to_s }
+        rescue *NET_ERRORS => e
+          raise Relaton::RequestError, "Could not access #{uri}: #{e.message}"
+        end
+
+        def primary_id(item)
+          (item.docidentifier.detect(&:primary) || item.docidentifier.first)&.content
+        end
+
+        def index
+          Relaton::Index.find_or_create(
+            :sae, pages_url: PAGES_URL, pubid_class: ::Pubid::Sae::Identifier
+          )
+        end
+      end
+    end
+  end
+end

data/lib/relaton/sae/data_fetcher.rb

@@ -0,0 +1,219 @@
+require "net/http"
+require_relative "../core"
+require_relative "../json"
+require_relative "../sae"
+require_relative "../doi"
+
+module Relaton
+  module Sae
+    #
+    # Crawl every SAE standard from Crossref into a data repo.
+    #
+    # sae.org answers a script with a CAPTCHA, and Crossref's text search does
+    # not find a standard by its identifier, but SAE deposits each edition
+    # with a DOI under prefix `10.4271` and type `standard`. This walks that
+    # set with Crossref's deep-paging cursor (1000 records a request, about 48
+    # requests), maps each record with `Relaton::Doi::Parser`, and writes one
+    # document per edition and the pubid `index-v2`.
+    #
+    class DataFetcher < Core::DataFetcher
+      SOURCES = %w[sae-crossref].freeze
+      URL = "https://api.crossref.org/prefixes/10.4271/works".freeze
+      ROWS = 1000
+      # Crossref's public pool allows far more; one request a second is polite
+      # and still finishes the crawl in about a minute of waits.
+      GAP = 1
+      # Attempts for one page before the crawl gives up.
+      ATTEMPTS = 5
+      RETRY_CODES = %w[429 500 502 503 504].freeze
+
+      def fetch(source = nil)
+        unless SOURCES.include? source
+          raise ArgumentError,
+                "Unknown source `#{source}`, expected one of #{SOURCES}"
+        end
+
+        @newest = {}
+        each_page { |items| items.each { |work| parse_work work } }
+        index.save
+        report_errors
+      end
+
+      def index
+        # `pubid_class:` on the producer too: FileIO#save calls `to_hash` only
+        # for instances of it, so without it the crawl writes v1-shaped rows
+        # under a v2 name.
+        @index ||= Relaton::Index.find_or_create(
+          :sae, file: "#{INDEXFILE}.yaml", pubid_class: ::Pubid::Sae::Identifier
+        )
+      end
+
+      def log_error(msg)
+        Util.error msg
+      end
+
+      private
+
+      def each_page
+        cursor = "*"
+        loop do
+          message = get_page(cursor)
+          items = message["items"]
+          break if items.nil? || items.empty?
+
+          yield items
+          cursor = message["next-cursor"]
+          break unless cursor
+        end
+      end
+
+      #
+      # One page of the cursor walk. A throttle or a server error is retried
+      # with a growing wait; anything else, or a page that keeps failing,
+      # stops the crawl, because a partial crawl would publish a partial index.
+      #
+      # @param cursor [String]
+      # @return [Hash] the `message` of the response
+      #
+      def get_page(cursor) # rubocop:disable Metrics/MethodLength
+        uri = page_uri cursor
+        ATTEMPTS.times do |n|
+          pacer.wait
+          resp, failure = attempt uri
+          return Relaton::Json.parse_external(resp.body)["message"] if resp&.code == "200"
+          if resp && !RETRY_CODES.include?(resp.code)
+            raise Relaton::RequestError, "Crossref HTTP #{resp.code} for #{uri}"
+          end
+
+          break if n == ATTEMPTS - 1
+
+          Util.warn "Crossref #{failure || "HTTP #{resp.code}"}, " \
+                    "retry #{n + 1} of #{ATTEMPTS - 1}"
+          sleep backoff(resp, n + 1)
+        end
+        raise Relaton::RequestError,
+              "Crossref keeps failing for #{uri} after #{ATTEMPTS} attempts"
+      end
+
+      # A transport error is retried like a 5xx: one timeout in ~48 requests
+      # must not fail the crawl.
+      #
+      # @return [Array(Net::HTTPResponse|nil, String|nil)] the response, or nil
+      #   and the error message
+      def attempt(uri)
+        [http_get(uri), nil]
+      rescue *Bibliography::NET_ERRORS => e
+        [nil, "#{e.class}: #{e.message}"]
+      end
+
+      def page_uri(cursor)
+        uri = URI URL
+        uri.query = URI.encode_www_form(filter: "type:standard", rows: ROWS,
+                                        cursor: cursor)
+        uri
+      end
+
+      def http_get(uri)
+        Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
+                                            open_timeout: 30, read_timeout: 120) do |http|
+          http.get uri.request_uri, "User-Agent" => Relaton::Doi::Crossref::USER_AGENT
+        end
+      end
+
+      # Crossref sends `X-Rate-Limit-Interval: 1s`; a throttled response can
+      # carry no such header, so the wait is never less than a second.
+      def backoff(resp, num)
+        interval = resp&.[]("x-rate-limit-interval").to_s[/\d+/].to_i
+        [interval, 1].max * num * 2
+      end
+
+      def pacer
+        @pacer ||= Core::Pacer.new(gap: GAP)
+      end
+
+      #
+      # Write one Crossref record and index it.
+      #
+      # @param work [Hash] a Crossref work
+      #
+      def parse_work(work)
+        code = DoiCode.parse work["DOI"], approved_year: approved_year(work)
+        return Util.warn("Not an SAE DOI: `#{work['DOI']}`") unless code
+
+        item = to_item work, code
+        file = write_unique code.native, serialize(item)
+        add_to_index code, work, file
+      end
+
+      def approved_year(work)
+        (work["approved"] || work["issued"])&.dig("date-parts", 0, 0)
+      end
+
+      #
+      # The `Relaton::Doi::Parser` item, with SAE's edition code (as SAE
+      # prints it) as its primary identifier and the DOI kept beside it.
+      #
+      # @return [Relaton::Sae::ItemData]
+      #
+      def to_item(work, code)
+        bib = Relaton::Doi::Parser.parse work
+        bib.id = code.native.delete(" /").gsub(/\W/, "")
+        bib.docidentifier = [
+          Bib::Docidentifier.new(type: "SAE", content: code.printed,
+                                 primary: true),
+          *bib.docidentifier.map do |d|
+            Bib::Docidentifier.new(type: d.type, content: d.content)
+          end,
+        ]
+        bib.ext = Bib::Ext.new(doctype: Bib::Doctype.new(content: "standard"),
+                               flavor: "sae")
+        Item.from_yaml Bib::Item.to_yaml(bib)
+      end
+
+      #
+      # Index the edition under its pubid, or record why it cannot be.
+      #
+      # An id pubid cannot read is recorded in `@errors` (the inherited
+      # `report_errors` logs it and opens the crawl's GitHub issue) and the row
+      # is skipped: `Relaton::Index` rejects the WHOLE index when one row fails
+      # to deserialize. The data file is already written, so the document is
+      # unindexed, never lost.
+      #
+      # The key carries the month, so two editions of one year are two rows.
+      # Two DOIs that spell one code (`ams-4121j`, `ams4121j`) share a key;
+      # the one with the later DOI date, then approval date, keeps the row.
+      #
+      def add_to_index(code, work, file)
+        pubid = parse_pubid code
+        unless pubid
+          @errors[code.native] =
+            "Unparseable SAE id `#{code.native}` was not indexed (#{file})"
+          return
+        end
+
+        key = pubid.to_s
+        stamp = [code.date.to_s, approved_stamp(work)]
+        return if @newest[key] && (@newest[key] <=> stamp) >= 0
+
+        @newest[key] = stamp
+        index.add_or_update pubid, file
+      end
+
+      def parse_pubid(code)
+        ::Pubid::Sae::Identifier.parse code.reference
+      rescue ::Pubid::Errors::Error, ArgumentError
+        nil
+      end
+
+      def approved_stamp(work)
+        Array(work.dig("approved", "date-parts", 0)).map do |d|
+          d.to_i.to_s.rjust(2, "0")
+        end.join("-")
+      end
+
+      def to_yaml(bib) = bib.to_yaml
+      def to_xml(bib) = Bibdata.to_xml(bib)
+      def to_bibxml(bib) = bib.to_rfcxml
+    end
+  end
+end

data/lib/relaton/sae/doi_code.rb

@@ -0,0 +1,83 @@
+module Relaton
+  module Sae
+    #
+    # The SAE codes a Crossref DOI names.
+    #
+    # SAE deposits one DOI per edition, and its suffix is SAE's own edition
+    # code in lowercase: `10.4271/j3016_202104` is `SAE J3016_202104`, the
+    # April 2021 edition of J3016. Aerospace standards are told apart by a
+    # revision letter instead and carry no date (`10.4271/ams4121j`).
+    #
+    # Three strings come out of it:
+    #
+    # * `#native` — the code in the DOI's spelling (`SAE AMSQQA250/4B`). The
+    #   output file is named after it, so a file name never depends on pubid.
+    # * `#printed` — the code as SAE prints it (`SAE AMS-QQ-A-250/4B`), the
+    #   document's primary identifier.
+    # * `#reference` — the form the index row is parsed from. It is the
+    #   native code when the DOI has a date; an undated code gets the year of
+    #   the record's approval date, so a dated reference also reaches an
+    #   aerospace revision (`SAE AMS4121J:2017`).
+    #
+    class DoiCode
+      PREFIX = "10.4271/".freeze
+      # A code, then an optional `_YYYYMM` edition date. The code takes letters,
+      # digits, the `/` of a part (`j2337/2`) and any `_` that is not that date
+      # (`as5259/7a_a1`, `j1939da_4q2012`: 1,163 of the 2026-11 deposits); what
+      # it means is pubid's business, so nothing else is decided here.
+      SUFFIX = %r{\A(?<code>[a-z0-9][a-z0-9/._-]*?)(?:_(?<date>\d{6}))?\z}i
+
+      attr_reader :code, :date, :year
+
+      #
+      # @param doi [String, nil] e.g. `10.4271/j3016_202104`
+      # @param approved_year [String, Integer, nil] the record's approval year
+      # @return [Relaton::Sae::DoiCode, nil] nil for a DOI that is not SAE's
+      #
+      def self.parse(doi, approved_year: nil)
+        suffix = doi.to_s.strip
+        return unless suffix.downcase.start_with? PREFIX
+
+        match = SUFFIX.match suffix[PREFIX.size..]
+        return unless match
+
+        new(match[:code].upcase, match[:date], approved_year)
+      end
+
+      def initialize(code, date, approved_year)
+        @code = code
+        @date = date
+        @year = date ? date[0, 4] : approved_year&.to_s
+      end
+
+      # @return [String] SAE's edition code, e.g. `SAE J3016_202104`
+      def native
+        date ? "SAE #{code}_#{date}" : "SAE #{code}"
+      end
+
+      # @return [String] the code SAE prints, or the native code when pubid
+      #   cannot read it
+      def printed
+        pubid&.to_s || native
+      end
+
+      # @return [String] the reference the index row is parsed from
+      def reference
+        return native if date || !year
+
+        "SAE #{code}:#{year}"
+      end
+
+      # @return [Pubid::Sae::Identifier, nil] the native code, parsed
+      def pubid
+        return @pubid if defined? @pubid
+
+        @pubid = begin
+          ::Pubid::Sae::Identifier.parse native
+        rescue ::Pubid::Errors::Error, ArgumentError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/relaton/sae/item.rb

@@ -0,0 +1,10 @@
+module Relaton
+  module Sae
+    # An SAE bibliographic item. SAE adds no element to the shared model: the
+    # Crossref record maps onto `Bib` as it is, so the only flavor-specific
+    # value is `ext/flavor`.
+    class Item < Bib::Item
+      model ItemData
+    end
+  end
+end

data/lib/relaton/sae/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Sae
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/sae/processor.rb

@@ -0,0 +1,62 @@
+require_relative "../core/processor"
+
+module Relaton
+  module Sae
+    class Processor < Relaton::Core::Processor
+      attr_reader :idtype
+
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_sae
+        @prefix = "SAE"
+        @defaultprefix = /^SAE\s/
+        @idtype = "SAE"
+        @datasets = %w[sae-crossref]
+        # Db cache key: `Pubid::Sae` reads the reference and SAE's native
+        # edition code (`SAE J3016_202104`) alike.
+        @pubid_flavor = :Sae
+      end
+
+      # @param code [String, Pubid::Sae::Identifier]
+      # @param date [String, nil] year
+      # @param opts [Hash]
+      # @return [Relaton::Sae::ItemData, nil]
+      def get(code, date, opts)
+        require_relative "../sae"
+        Bibliography.get(code, date, opts)
+      end
+
+      # @param source [String] `sae-crossref`
+      # @param opts [Hash] `output:`, `format:`
+      def fetch_data(source, opts)
+        require_relative "data_fetcher"
+        DataFetcher.fetch(source, **opts)
+      end
+
+      # @param xml [String]
+      # @return [Relaton::Sae::ItemData]
+      def from_xml(xml)
+        require_relative "../sae"
+        Item.from_xml xml
+      end
+
+      # @param yaml [String]
+      # @return [Relaton::Sae::ItemData]
+      def from_yaml(yaml)
+        require_relative "../sae"
+        Item.from_yaml yaml
+      end
+
+      # @return [String]
+      def grammar_hash
+        require_relative "../sae"
+        @grammar_hash ||= Sae.grammar_hash
+      end
+
+      def remove_index_file
+        require_relative "../sae"
+        Relaton::Index.find_or_create(:sae, url: true,
+                                            file: "#{INDEXFILE}.yaml").remove_file
+      end
+    end
+  end
+end

data/lib/relaton/sae/util.rb

@@ -0,0 +1,9 @@
+module Relaton
+  module Sae
+    module Util
+      extend Relaton::Bib::Util
+
+      PROGNAME = "relaton-sae".freeze
+    end
+  end
+end

data/lib/relaton/sae.rb

@@ -0,0 +1,37 @@
+require "digest/md5"
+require "pubid"
+require_relative "bib"
+require_relative "index"
+require_relative "version"
+
+# Relaton::Sae is the SAE International flavor (relaton/relaton-sae#1).
+#
+# sae.org answers a script with a CAPTCHA, so the flavor reads SAE's Crossref
+# deposits instead: every SAE standard edition has its own DOI
+# (`10.4271/j3016_202104`). `DataFetcher` crawls them into the
+# `relaton-data-sae` data repo with a pubid `index-v2`, and `Bibliography`
+# reads that index. See lib/relaton/sae/CLAUDE.md.
+module Relaton
+  module Sae
+    INDEXFILE = "index-v2".freeze
+    # The machine index on the data repo's Pages site (relaton#206).
+    PAGES_URL = "https://relaton.github.io/relaton-data-sae/".freeze
+    # Where the documents the index names are read from.
+    ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-sae/main/".freeze
+
+    autoload :Util,         "relaton/sae/util"
+    autoload :DoiCode,      "relaton/sae/doi_code"
+    autoload :ItemData,     "relaton/sae/item_data"
+    autoload :Item,         "relaton/sae/item"
+    autoload :Bibitem,      "relaton/sae/bibitem"
+    autoload :Bibdata,      "relaton/sae/bibdata"
+    autoload :Bibliography, "relaton/sae/bibliography"
+    autoload :Processor,    "relaton/sae/processor"
+
+    # Returns hash of XML grammar
+    # @return [String]
+    def self.grammar_hash
+      Digest::MD5.hexdigest Relaton::VERSION
+    end
+  end
+end

data/lib/relaton/un/processor.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "relaton/core/processor"
+require_relative "../core/processor"
 
 module Relaton
   module Un

data/lib/relaton/un.rb

@@ -1,5 +1,5 @@
 require "addressable"
-require "relaton/bib"
+require_relative "bib"
 require_relative "version"
 require_relative "un/util"
 require_relative "un/item_data"

data/lib/relaton/version.rb

@@ -1,3 +1,3 @@
 module Relaton
-  VERSION = "3.0.8".freeze
+  VERSION = "3.1.0".freeze
 end

data/lib/relaton/w3c/bibliography.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require "net/http"
-require "relaton/bib/hash_parser_v1"
+require_relative "../bib/hash_parser_v1"
 
 module Relaton
   module W3c

data/lib/relaton/w3c/data_fetcher.rb

@@ -1,4 +1,4 @@
-require "relaton/core"
+require_relative "../core"
 require "w3c_api"
 require_relative "../w3c"
 require_relative "safe_realize"

data/lib/relaton/w3c/processor.rb

@@ -1,4 +1,4 @@
-require "relaton/core/processor"
+require_relative "../core/processor"
 
 module Relaton
   module W3c

data/lib/relaton/w3c.rb

@@ -1,7 +1,7 @@
 require "pubid"
 require "pubid/w3c"
-require "relaton/bib"
-require "relaton/index"
+require_relative "bib"
+require_relative "index"
 require_relative "version"
 require_relative "w3c/util"
 require_relative "w3c/item"

data/lib/relaton/xsf/data_fetcher.rb

@@ -1,4 +1,4 @@
-require "relaton/core"
+require_relative "../core"
 
 module Relaton
   module Xsf

data/lib/relaton/xsf/processor.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-require "relaton/core/processor"
+require_relative "../core/processor"
 
 module Relaton
   module Xsf

data/lib/relaton/xsf.rb

@@ -5,8 +5,8 @@ require "mechanize"
 # index code names. Processor#remove_index_file names no pubid class: the
 # delete never reads the index (see lib/relaton/index/CLAUDE.md).
 require "pubid"
-require "relaton/index"
-require "relaton/bib"
+require_relative "index"
+require_relative "bib"
 require_relative "version"
 require_relative "xsf/util"
 require_relative "xsf/item_data"

data/lib/relaton-cli.rb

@@ -2,8 +2,8 @@
 # `index_name_for`/`pubid_class_for` resolve `Relaton::<Flavor>::INDEXFILE`, so
 # without the umbrella every `--pubid-flavor` fails with "no relaton flavor".
 # Declaring autoloads is cheap: no flavor is loaded until it is referenced.
-require "relaton"
-require "relaton/db"
+require_relative "relaton"
+require_relative "relaton/db"
 
 module Relaton
   module Cli

data/lib/relaton-render.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
-require "relaton/render/version"
-require "relaton/render/iso690"
+require_relative "relaton/render/iso690"
 
 module Relaton
   module Render