relaton 3.0.7 → 3.1.0

data/lib/relaton/sae/data_fetcher.rb

@@ -0,0 +1,219 @@
+require "net/http"
+require_relative "../core"
+require_relative "../json"
+require_relative "../sae"
+require_relative "../doi"
+
+module Relaton
+  module Sae
+    #
+    # Crawl every SAE standard from Crossref into a data repo.
+    #
+    # sae.org answers a script with a CAPTCHA, and Crossref's text search does
+    # not find a standard by its identifier, but SAE deposits each edition
+    # with a DOI under prefix `10.4271` and type `standard`. This walks that
+    # set with Crossref's deep-paging cursor (1000 records a request, about 48
+    # requests), maps each record with `Relaton::Doi::Parser`, and writes one
+    # document per edition and the pubid `index-v2`.
+    #
+    class DataFetcher < Core::DataFetcher
+      SOURCES = %w[sae-crossref].freeze
+      URL = "https://api.crossref.org/prefixes/10.4271/works".freeze
+      ROWS = 1000
+      # Crossref's public pool allows far more; one request a second is polite
+      # and still finishes the crawl in about a minute of waits.
+      GAP = 1
+      # Attempts for one page before the crawl gives up.
+      ATTEMPTS = 5
+      RETRY_CODES = %w[429 500 502 503 504].freeze
+
+      def fetch(source = nil)
+        unless SOURCES.include? source
+          raise ArgumentError,
+                "Unknown source `#{source}`, expected one of #{SOURCES}"
+        end
+
+        @newest = {}
+        each_page { |items| items.each { |work| parse_work work } }
+        index.save
+        report_errors
+      end
+
+      def index
+        # `pubid_class:` on the producer too: FileIO#save calls `to_hash` only
+        # for instances of it, so without it the crawl writes v1-shaped rows
+        # under a v2 name.
+        @index ||= Relaton::Index.find_or_create(
+          :sae, file: "#{INDEXFILE}.yaml", pubid_class: ::Pubid::Sae::Identifier
+        )
+      end
+
+      def log_error(msg)
+        Util.error msg
+      end
+
+      private
+
+      def each_page
+        cursor = "*"
+        loop do
+          message = get_page(cursor)
+          items = message["items"]
+          break if items.nil? || items.empty?
+
+          yield items
+          cursor = message["next-cursor"]
+          break unless cursor
+        end
+      end
+
+      #
+      # One page of the cursor walk. A throttle or a server error is retried
+      # with a growing wait; anything else, or a page that keeps failing,
+      # stops the crawl, because a partial crawl would publish a partial index.
+      #
+      # @param cursor [String]
+      # @return [Hash] the `message` of the response
+      #
+      def get_page(cursor) # rubocop:disable Metrics/MethodLength
+        uri = page_uri cursor
+        ATTEMPTS.times do |n|
+          pacer.wait
+          resp, failure = attempt uri
+          return Relaton::Json.parse_external(resp.body)["message"] if resp&.code == "200"
+          if resp && !RETRY_CODES.include?(resp.code)
+            raise Relaton::RequestError, "Crossref HTTP #{resp.code} for #{uri}"
+          end
+
+          break if n == ATTEMPTS - 1
+
+          Util.warn "Crossref #{failure || "HTTP #{resp.code}"}, " \
+                    "retry #{n + 1} of #{ATTEMPTS - 1}"
+          sleep backoff(resp, n + 1)
+        end
+        raise Relaton::RequestError,
+              "Crossref keeps failing for #{uri} after #{ATTEMPTS} attempts"
+      end
+
+      # A transport error is retried like a 5xx: one timeout in ~48 requests
+      # must not fail the crawl.
+      #
+      # @return [Array(Net::HTTPResponse|nil, String|nil)] the response, or nil
+      #   and the error message
+      def attempt(uri)
+        [http_get(uri), nil]
+      rescue *Bibliography::NET_ERRORS => e
+        [nil, "#{e.class}: #{e.message}"]
+      end
+
+      def page_uri(cursor)
+        uri = URI URL
+        uri.query = URI.encode_www_form(filter: "type:standard", rows: ROWS,
+                                        cursor: cursor)
+        uri
+      end
+
+      def http_get(uri)
+        Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
+                                            open_timeout: 30, read_timeout: 120) do |http|
+          http.get uri.request_uri, "User-Agent" => Relaton::Doi::Crossref::USER_AGENT
+        end
+      end
+
+      # Crossref sends `X-Rate-Limit-Interval: 1s`; a throttled response can
+      # carry no such header, so the wait is never less than a second.
+      def backoff(resp, num)
+        interval = resp&.[]("x-rate-limit-interval").to_s[/\d+/].to_i
+        [interval, 1].max * num * 2
+      end
+
+      def pacer
+        @pacer ||= Core::Pacer.new(gap: GAP)
+      end
+
+      #
+      # Write one Crossref record and index it.
+      #
+      # @param work [Hash] a Crossref work
+      #
+      def parse_work(work)
+        code = DoiCode.parse work["DOI"], approved_year: approved_year(work)
+        return Util.warn("Not an SAE DOI: `#{work['DOI']}`") unless code
+
+        item = to_item work, code
+        file = write_unique code.native, serialize(item)
+        add_to_index code, work, file
+      end
+
+      def approved_year(work)
+        (work["approved"] || work["issued"])&.dig("date-parts", 0, 0)
+      end
+
+      #
+      # The `Relaton::Doi::Parser` item, with SAE's edition code (as SAE
+      # prints it) as its primary identifier and the DOI kept beside it.
+      #
+      # @return [Relaton::Sae::ItemData]
+      #
+      def to_item(work, code)
+        bib = Relaton::Doi::Parser.parse work
+        bib.id = code.native.delete(" /").gsub(/\W/, "")
+        bib.docidentifier = [
+          Bib::Docidentifier.new(type: "SAE", content: code.printed,
+                                 primary: true),
+          *bib.docidentifier.map do |d|
+            Bib::Docidentifier.new(type: d.type, content: d.content)
+          end,
+        ]
+        bib.ext = Bib::Ext.new(doctype: Bib::Doctype.new(content: "standard"),
+                               flavor: "sae")
+        Item.from_yaml Bib::Item.to_yaml(bib)
+      end
+
+      #
+      # Index the edition under its pubid, or record why it cannot be.
+      #
+      # An id pubid cannot read is recorded in `@errors` (the inherited
+      # `report_errors` logs it and opens the crawl's GitHub issue) and the row
+      # is skipped: `Relaton::Index` rejects the WHOLE index when one row fails
+      # to deserialize. The data file is already written, so the document is
+      # unindexed, never lost.
+      #
+      # The key carries the month, so two editions of one year are two rows.
+      # Two DOIs that spell one code (`ams-4121j`, `ams4121j`) share a key;
+      # the one with the later DOI date, then approval date, keeps the row.
+      #
+      def add_to_index(code, work, file)
+        pubid = parse_pubid code
+        unless pubid
+          @errors[code.native] =
+            "Unparseable SAE id `#{code.native}` was not indexed (#{file})"
+          return
+        end
+
+        key = pubid.to_s
+        stamp = [code.date.to_s, approved_stamp(work)]
+        return if @newest[key] && (@newest[key] <=> stamp) >= 0
+
+        @newest[key] = stamp
+        index.add_or_update pubid, file
+      end
+
+      def parse_pubid(code)
+        ::Pubid::Sae::Identifier.parse code.reference
+      rescue ::Pubid::Errors::Error, ArgumentError
+        nil
+      end
+
+      def approved_stamp(work)
+        Array(work.dig("approved", "date-parts", 0)).map do |d|
+          d.to_i.to_s.rjust(2, "0")
+        end.join("-")
+      end
+
+      def to_yaml(bib) = bib.to_yaml
+      def to_xml(bib) = Bibdata.to_xml(bib)
+      def to_bibxml(bib) = bib.to_rfcxml
+    end
+  end
+end

data/lib/relaton/sae/doi_code.rb

@@ -0,0 +1,83 @@
+module Relaton
+  module Sae
+    #
+    # The SAE codes a Crossref DOI names.
+    #
+    # SAE deposits one DOI per edition, and its suffix is SAE's own edition
+    # code in lowercase: `10.4271/j3016_202104` is `SAE J3016_202104`, the
+    # April 2021 edition of J3016. Aerospace standards are told apart by a
+    # revision letter instead and carry no date (`10.4271/ams4121j`).
+    #
+    # Three strings come out of it:
+    #
+    # * `#native` — the code in the DOI's spelling (`SAE AMSQQA250/4B`). The
+    #   output file is named after it, so a file name never depends on pubid.
+    # * `#printed` — the code as SAE prints it (`SAE AMS-QQ-A-250/4B`), the
+    #   document's primary identifier.
+    # * `#reference` — the form the index row is parsed from. It is the
+    #   native code when the DOI has a date; an undated code gets the year of
+    #   the record's approval date, so a dated reference also reaches an
+    #   aerospace revision (`SAE AMS4121J:2017`).
+    #
+    class DoiCode
+      PREFIX = "10.4271/".freeze
+      # A code, then an optional `_YYYYMM` edition date. The code takes letters,
+      # digits, the `/` of a part (`j2337/2`) and any `_` that is not that date
+      # (`as5259/7a_a1`, `j1939da_4q2012`: 1,163 of the 2026-11 deposits); what
+      # it means is pubid's business, so nothing else is decided here.
+      SUFFIX = %r{\A(?<code>[a-z0-9][a-z0-9/._-]*?)(?:_(?<date>\d{6}))?\z}i
+
+      attr_reader :code, :date, :year
+
+      #
+      # @param doi [String, nil] e.g. `10.4271/j3016_202104`
+      # @param approved_year [String, Integer, nil] the record's approval year
+      # @return [Relaton::Sae::DoiCode, nil] nil for a DOI that is not SAE's
+      #
+      def self.parse(doi, approved_year: nil)
+        suffix = doi.to_s.strip
+        return unless suffix.downcase.start_with? PREFIX
+
+        match = SUFFIX.match suffix[PREFIX.size..]
+        return unless match
+
+        new(match[:code].upcase, match[:date], approved_year)
+      end
+
+      def initialize(code, date, approved_year)
+        @code = code
+        @date = date
+        @year = date ? date[0, 4] : approved_year&.to_s
+      end
+
+      # @return [String] SAE's edition code, e.g. `SAE J3016_202104`
+      def native
+        date ? "SAE #{code}_#{date}" : "SAE #{code}"
+      end
+
+      # @return [String] the code SAE prints, or the native code when pubid
+      #   cannot read it
+      def printed
+        pubid&.to_s || native
+      end
+
+      # @return [String] the reference the index row is parsed from
+      def reference
+        return native if date || !year
+
+        "SAE #{code}:#{year}"
+      end
+
+      # @return [Pubid::Sae::Identifier, nil] the native code, parsed
+      def pubid
+        return @pubid if defined? @pubid
+
+        @pubid = begin
+          ::Pubid::Sae::Identifier.parse native
+        rescue ::Pubid::Errors::Error, ArgumentError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/relaton/sae/item.rb

@@ -0,0 +1,10 @@
+module Relaton
+  module Sae
+    # An SAE bibliographic item. SAE adds no element to the shared model: the
+    # Crossref record maps onto `Bib` as it is, so the only flavor-specific
+    # value is `ext/flavor`.
+    class Item < Bib::Item
+      model ItemData
+    end
+  end
+end

data/lib/relaton/sae/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Sae
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/sae/processor.rb

@@ -0,0 +1,62 @@
+require_relative "../core/processor"
+
+module Relaton
+  module Sae
+    class Processor < Relaton::Core::Processor
+      attr_reader :idtype
+
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_sae
+        @prefix = "SAE"
+        @defaultprefix = /^SAE\s/
+        @idtype = "SAE"
+        @datasets = %w[sae-crossref]
+        # Db cache key: `Pubid::Sae` reads the reference and SAE's native
+        # edition code (`SAE J3016_202104`) alike.
+        @pubid_flavor = :Sae
+      end
+
+      # @param code [String, Pubid::Sae::Identifier]
+      # @param date [String, nil] year
+      # @param opts [Hash]
+      # @return [Relaton::Sae::ItemData, nil]
+      def get(code, date, opts)
+        require_relative "../sae"
+        Bibliography.get(code, date, opts)
+      end
+
+      # @param source [String] `sae-crossref`
+      # @param opts [Hash] `output:`, `format:`
+      def fetch_data(source, opts)
+        require_relative "data_fetcher"
+        DataFetcher.fetch(source, **opts)
+      end
+
+      # @param xml [String]
+      # @return [Relaton::Sae::ItemData]
+      def from_xml(xml)
+        require_relative "../sae"
+        Item.from_xml xml
+      end
+
+      # @param yaml [String]
+      # @return [Relaton::Sae::ItemData]
+      def from_yaml(yaml)
+        require_relative "../sae"
+        Item.from_yaml yaml
+      end
+
+      # @return [String]
+      def grammar_hash
+        require_relative "../sae"
+        @grammar_hash ||= Sae.grammar_hash
+      end
+
+      def remove_index_file
+        require_relative "../sae"
+        Relaton::Index.find_or_create(:sae, url: true,
+                                            file: "#{INDEXFILE}.yaml").remove_file
+      end
+    end
+  end
+end

data/lib/relaton/sae/util.rb

@@ -0,0 +1,9 @@
+module Relaton
+  module Sae
+    module Util
+      extend Relaton::Bib::Util
+
+      PROGNAME = "relaton-sae".freeze
+    end
+  end
+end

data/lib/relaton/sae.rb

@@ -0,0 +1,37 @@
+require "digest/md5"
+require "pubid"
+require_relative "bib"
+require_relative "index"
+require_relative "version"
+
+# Relaton::Sae is the SAE International flavor (relaton/relaton-sae#1).
+#
+# sae.org answers a script with a CAPTCHA, so the flavor reads SAE's Crossref
+# deposits instead: every SAE standard edition has its own DOI
+# (`10.4271/j3016_202104`). `DataFetcher` crawls them into the
+# `relaton-data-sae` data repo with a pubid `index-v2`, and `Bibliography`
+# reads that index. See lib/relaton/sae/CLAUDE.md.
+module Relaton
+  module Sae
+    INDEXFILE = "index-v2".freeze
+    # The machine index on the data repo's Pages site (relaton#206).
+    PAGES_URL = "https://relaton.github.io/relaton-data-sae/".freeze
+    # Where the documents the index names are read from.
+    ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-sae/main/".freeze
+
+    autoload :Util,         "relaton/sae/util"
+    autoload :DoiCode,      "relaton/sae/doi_code"
+    autoload :ItemData,     "relaton/sae/item_data"
+    autoload :Item,         "relaton/sae/item"
+    autoload :Bibitem,      "relaton/sae/bibitem"
+    autoload :Bibdata,      "relaton/sae/bibdata"
+    autoload :Bibliography, "relaton/sae/bibliography"
+    autoload :Processor,    "relaton/sae/processor"
+
+    # Returns hash of XML grammar
+    # @return [String]
+    def self.grammar_hash
+      Digest::MD5.hexdigest Relaton::VERSION
+    end
+  end
+end

data/lib/relaton/un/CLAUDE.md

@@ -88,4 +88,4 @@ Tests are round-trip serialization tests: load a fixture, serialize to XML/YAML,
 - **relaton-bib** (~> 2.0.0-alpha.1) — base bibliographic model with LutaML-Model integration
 - **faraday** — HTTP client
 - **ruby-jing** (dev) — RelaxNG schema validation in tests
-- **equivalent-xml** (dev) — XML comparison in tests
+- **canon** (dev) — XML comparison in tests

data/lib/relaton/un/processor.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "relaton/core/processor"
+require_relative "../core/processor"
 
 module Relaton
   module Un

data/lib/relaton/un.rb

@@ -1,5 +1,5 @@
 require "addressable"
-require "relaton/bib"
+require_relative "bib"
 require_relative "version"
 require_relative "un/util"
 require_relative "un/item_data"

data/lib/relaton/version.rb

@@ -1,3 +1,3 @@
 module Relaton
-  VERSION = "3.0.7".freeze
+  VERSION = "3.1.0".freeze
 end

data/lib/relaton/w3c/bibliography.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require "net/http"
-require "relaton/bib/hash_parser_v1"
+require_relative "../bib/hash_parser_v1"
 
 module Relaton
   module W3c

data/lib/relaton/w3c/data_fetcher.rb

@@ -1,4 +1,4 @@
-require "relaton/core"
+require_relative "../core"
 require "w3c_api"
 require_relative "../w3c"
 require_relative "safe_realize"

data/lib/relaton/w3c/processor.rb

@@ -1,4 +1,4 @@
-require "relaton/core/processor"
+require_relative "../core/processor"
 
 module Relaton
   module W3c

data/lib/relaton/w3c.rb

@@ -1,7 +1,7 @@
 require "pubid"
 require "pubid/w3c"
-require "relaton/bib"
-require "relaton/index"
+require_relative "bib"
+require_relative "index"
 require_relative "version"
 require_relative "w3c/util"
 require_relative "w3c/item"

data/lib/relaton/xsf/data_fetcher.rb

@@ -1,4 +1,4 @@
-require "relaton/core"
+require_relative "../core"
 
 module Relaton
   module Xsf

data/lib/relaton/xsf/processor.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-require "relaton/core/processor"
+require_relative "../core/processor"
 
 module Relaton
   module Xsf

data/lib/relaton/xsf.rb

@@ -5,8 +5,8 @@ require "mechanize"
 # index code names. Processor#remove_index_file names no pubid class: the
 # delete never reads the index (see lib/relaton/index/CLAUDE.md).
 require "pubid"
-require "relaton/index"
-require "relaton/bib"
+require_relative "index"
+require_relative "bib"
 require_relative "version"
 require_relative "xsf/util"
 require_relative "xsf/item_data"

data/lib/relaton-cli.rb

@@ -0,0 +1,17 @@
+# The per-flavor `autoload` lines live in `relaton.rb`, not `relaton/db`.
+# `index_name_for`/`pubid_class_for` resolve `Relaton::<Flavor>::INDEXFILE`, so
+# without the umbrella every `--pubid-flavor` fails with "no relaton flavor".
+# Declaring autoloads is cheap: no flavor is loaded until it is referenced.
+require_relative "relaton"
+require_relative "relaton/db"
+
+module Relaton
+  module Cli
+  end
+end
+
+require_relative "relaton/bibcollection"
+require_relative "relaton/bibdata"
+require_relative "relaton/element_finder"
+require_relative "relaton/cli/yaml_convertor"
+require_relative "relaton/cli"

data/lib/relaton-render.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require_relative "relaton/render/iso690"
+
+module Relaton
+  module Render
+    # A bibliographic item whose style template resolves to nothing: the
+    # single-item API raises, the batch API skips
+    class Unrenderable < StandardError; end
+
+  end
+end

data/lib/relaton.rb

@@ -4,7 +4,7 @@
 # autoloaded: referencing it (or any nested constant) the first time triggers
 # `require "relaton/<flavor>"`, loading that flavor's code on demand rather than
 # at `require "relaton"`. When adding a flavor, add an autoload line here.
-require "relaton/version"
+require_relative "relaton/version"
 require "moxml"
 
 # YAML through stdlib Psych. The former yeptris drop-in rebind
@@ -50,6 +50,7 @@ module Relaton
   autoload :Oasis, "relaton/oasis"
   autoload :Oiml, "relaton/oiml"
   autoload :Omg, "relaton/omg"
+  autoload :Sae, "relaton/sae"
   autoload :Un, "relaton/un"
   autoload :W3c, "relaton/w3c"
   autoload :Xsf, "relaton/xsf"
@@ -100,4 +101,4 @@ module Relaton
   end
 end
 
-require "relaton/db"
+require_relative "relaton/db"