relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/w3c/safe_realize.rb

@@ -0,0 +1,148 @@
+require "concurrent/map"
+require_relative "governor"
+
+module Relaton
+  module W3c
+    # Thin wrapper over lutaml-hal's `realize`. Successful objects are cached by
+    # w3c_api (it caches realized objects keyed by URL), so this only remembers
+    # resources that failed terminally and returns nil for them — so one broken
+    # link doesn't abort the crawl and isn't re-fetched on every reference.
+    #
+    # Transient failures are retried upstream first: w3c_api retries 403 (the
+    # W3C rate-limit signal) and connection/timeout errors, and lutaml-hal
+    # retries 429 and 5xx. By the time an error surfaces here the resource is
+    # either genuinely broken (skipped for the rest of the run) or we are still
+    # being rate-limited (handed to the Governor).
+    #
+    # That distinction is the point: a 429 or 403 means "come back later", not
+    # "this document does not exist". Recording one as a permanent skip is what
+    # silently cost the Aug-2026 relaton-data-w3c crawl 1,412 documents.
+    module SafeRealize
+      # How many times a rate-limited realize is re-attempted, each behind the
+      # governor's shared cooldown, before the resource is written off for this
+      # run. Three spans the escalating cooldowns without letting one unlucky
+      # href monopolise a worker.
+      THROTTLE_ATTEMPTS = 3
+
+      # Hrefs that failed terminally — one map shared by every includer
+      # (DataFetcher and DataParser) since a broken resource is broken for the
+      # whole crawl. Initialized eagerly (at load, single-threaded) so the
+      # parallel fetcher's first concurrent access can't race a lazy `||=`;
+      # Concurrent::Map then handles the concurrent reads/writes.
+      @skipped = Concurrent::Map.new
+
+      # Hrefs lost to rate limiting rather than to a broken resource. Kept apart
+      # from @skipped because they measure how incomplete the crawl is, not a
+      # property of the resource: DataFetcher refuses to save an index once too
+      # many pile up.
+      @throttled = Concurrent::Map.new
+
+      # Shared back-pressure for the whole crawl (see Governor).
+      @governor = Governor.new
+
+      class << self
+        attr_reader :skipped, :throttled
+        attr_accessor :governor
+
+        # Start a crawl (or a spec example) from a clean slate. The governor is
+        # reset in place, so a reference taken before the reset stays live.
+        def reset!
+          @skipped.clear
+          @throttled.clear
+          @governor.reset!
+        end
+      end
+
+      # @param parent_resource [Object, nil] the index/page the link came from.
+      #   When the page was fetched with `embed: true`, its inlined `_embedded`
+      #   payload lets the link realize from memory instead of issuing an HTTP
+      #   request. nil (the default) preserves the plain remote-fetch behavior.
+      def realize(obj, parent_resource: nil)
+        # Once the governor has latched its give-up the run is already lost
+        # (DataFetcher#guard_rate_limited raises and nothing is saved) and the
+        # host has just banned us, so every further request only extends the
+        # ban into the next run. #wait alone stops WAITING, not asking: the
+        # 2026-08-31 crawl spent its last 11 minutes firing at a banned host
+        # because the per-spec fan-out had no back-pressure of its own.
+        #
+        # This is the one choke point every request in the crawl passes
+        # through, so one check covers DataFetcher#fetch_versions and all
+        # seven of DataParser's realize sites. They already tolerate nil —
+        # that is the SafeRealize contract.
+        return nil if SafeRealize.governor.exhausted?
+
+        href = resolve_href(obj)
+        return nil if SafeRealize.skipped.key?(href)
+
+        attempt = 0
+        begin
+          attempt += 1
+          SafeRealize.governor.wait
+          realized = obj.realize(parent_resource: parent_resource)
+          # Only a realize that actually went to the network says anything about
+          # the limiter. With a parent_resource, lutaml-hal serves the object
+          # straight out of the page's `_embedded` payload and never issues a
+          # request (link.rb: "Priority 1: check embedded content first") — and
+          # that is the common case, one per specification. Reporting those as
+          # successes would clear the cooldown and reset the escalation ladder
+          # between every pair of version-history 429s, so the governor could
+          # never reach its give-up threshold: exactly the 60s re-trip loop this
+          # class exists to break.
+          SafeRealize.governor.succeeded! unless parent_resource
+          # The document made it into the dataset after all, so it must stop
+          # counting against the crawl's throttle budget.
+          SafeRealize.throttled.delete(href)
+          realized
+        rescue *Governor::THROTTLE_ERRORS => e
+          retry if record_throttle(href, e, attempt)
+          nil
+        rescue Lutaml::Hal::ConnectionError, Lutaml::Hal::TimeoutError, Faraday::Error, Net::OpenTimeout => e
+          # Network-level failure (already retried by w3c_api). The resource itself
+          # is fine, so don't skip it permanently — a later reference can try again.
+          Util.warn "Failed to realize object: #{href}, error: #{e.message}"
+          nil
+        rescue Lutaml::Hal::NotFoundError
+          Util.warn "Object not found: #{href}"
+          SafeRealize.skipped[href] = true
+          nil
+        rescue Lutaml::Hal::Error => e
+          # Definitive upstream error (5xx, 400/401, …) already retried by
+          # w3c_api / lutaml-hal. Skip the broken/unavailable resource rather than
+          # re-hitting it for every link that references it. 5xx stays here
+          # deliberately: the handful of persistent per-resource 500s a healthy
+          # crawl sees are broken records, not throttling, and routing them
+          # through the governor would open a pool-wide cooldown for each one.
+          Util.warn "Skipping #{href}, upstream error after retries: #{e.message}"
+          SafeRealize.skipped[href] = true
+          nil
+        end
+      end
+
+      private
+
+      #
+      # Record a rate-limited attempt and decide whether to try again.
+      #
+      # @return [Boolean] true to retry — the caller re-enters #realize's begin
+      #   block, whose #wait sits out the shared cooldown first
+      #
+      def record_throttle(href, error, attempt)
+        gave_up = SafeRealize.governor.throttled!(
+          retry_after: Governor.retry_after(error),
+        )
+        # Once the governor has declared the crawl rate-limited there is nothing
+        # to gain from more attempts — the run is about to abort anyway.
+        return true if !gave_up && attempt < THROTTLE_ATTEMPTS
+
+        Util.warn "Throttled out: #{href}, still rate-limited after " \
+                  "#{attempt} attempt(s): #{error.message}"
+        SafeRealize.throttled[href] = true
+        false
+      end
+
+      def resolve_href(obj)
+        obj.href || obj.links.self.href
+      end
+    end
+  end
+end

data/lib/relaton/w3c/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module W3c
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-w3c".freeze
+    end
+  end
+end

data/lib/relaton/w3c.rb

@@ -0,0 +1,36 @@
+require "pubid"
+require "pubid/w3c"
+require "relaton/bib"
+require "relaton/index"
+require_relative "version"
+require_relative "w3c/util"
+require_relative "w3c/item"
+require_relative "w3c/bibitem"
+require_relative "w3c/bibdata"
+require_relative "w3c/bibliography"
+# Not used by this gem any more; loaded so relaton-data-w3c's crawler can
+# require "relaton/w3c" and still reach the legacy index-v1 row shape.
+require_relative "w3c/pubid"
+
+module Relaton
+  module W3c
+    # The runtime index is the pubid-backed `index-v2`, whose rows are
+    # `Pubid::W3c::Identifier` leaves serialized as `_type: pubid:w3c:*`, built
+    # and read via `pubid_class: ::Pubid::W3c::Identifier`. The bespoke
+    # `index-v1` is relaton-v2 legacy and is no longer produced here —
+    # `relaton-data-w3c`'s crawler emits it from the retained public
+    # `Relaton::W3c::PubId`.
+    INDEXFILE = "index-v2".freeze
+
+    class Error < StandardError; end
+
+    # Returns hash of XML reammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/xsf/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Xsf
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/xsf/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Xsf
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/xsf/bibliography.rb

@@ -0,0 +1,84 @@
+module Relaton
+  module Xsf
+    module Bibliography
+      extend self
+
+      #
+      # Search the index for a reference.
+      #
+      # The reference is parsed into a `Pubid::Xsf::Identifier` here and the
+      # identifier -- never the string -- is what reaches `HitCollection`.
+      # `Index::Type#search_candidates` narrows only when its argument is not a
+      # `String`, so parsing at the entry point is what lets the lookup binary
+      # search; `Core::HitCollection` takes a `String` or a pubid by design.
+      #
+      # An unrecognized reference **raises**; like ISO, ETSI and 3GPP we let it
+      # propagate. relaton-cli rescues `Pubid::Errors::Error` and renders
+      # `"..." is not a recognized standards identifier`
+      # (`gems/relaton-cli/lib/relaton/cli/command.rb:324`,
+      # `subcommand_collection.rb:134`), and `Db#fetch` logs it via the
+      # `StandardError` arm at `lib/relaton/db.rb:122`. Rescuing here would
+      # collapse "this identifier is malformed" into "no such document", and a
+      # caller could no longer tell them apart.
+      #
+      # There is also deliberately no substring fallback: the old
+      # `index.search(ref)` compared a substring of the rendered id, so a bare
+      # `001` answered with 11 documents and `#get` took `.first` -- a truncated
+      # reference silently resolved to whichever sorted first.
+      #
+      # @param ref [String] e.g. "XEP 0001", "XEP-0001", "0001"
+      #
+      # @return [Relaton::Xsf::HitCollection]
+      # @raise [Pubid::Errors::ParseError] if the reference is not an XEP id
+      #
+      def search(ref)
+        HitCollection.new(parse_ref(ref)).search
+      end
+
+      def get(code, _year = nil, _opts = {})
+        Util.info "Fetching from Relaton repository ...", key: code
+        result = search(code)
+        if result.empty?
+          Util.info "Not found.", key: code
+          return
+        end
+
+        bib = result.first.item
+        Util.info "Found: `#{bib.docidentifier.first.content}`", key: code
+        bib
+      end
+
+      #
+      # Parse a user reference into a `Pubid::Xsf::Identifier`, or nil.
+      #
+      # `Pubid::Xsf` accepts only the canonical `XEP 0001` spelling, so two
+      # forms are normalized first:
+      #
+      # - **`XEP-0001`**, the spelling xmpp.org itself uses everywhere. It did
+      #   not resolve before either (the substring match compared against
+      #   `XEP 0001`, which has a space), so this is new support rather than a
+      #   preserved behaviour.
+      # - **A bare `0001`**, which the substring match did resolve, so it has to
+      #   keep working. The publisher token is added, the W3C idiom.
+      #
+      # The token match is case-insensitive, so `xep 0001` resolves too.
+      #
+      # Anything else raises -- see `.search`. `Pubid::Errors::ParseError`
+      # includes `Pubid::Errors::Error`, which is the class relaton-cli rescues.
+      #
+      # @param ref [String]
+      # @return [Pubid::Xsf::Identifier]
+      # @raise [Pubid::Errors::ParseError]
+      #
+      def parse_ref(ref)
+        ::Pubid::Xsf::Identifier.parse normalize_ref(ref)
+      end
+
+      private
+
+      def normalize_ref(ref)
+        "XEP #{ref.to_s.strip.sub(/\AXEP[-\s]+/i, '')}"
+      end
+    end
+  end
+end

data/lib/relaton/xsf/data_fetcher.rb

@@ -0,0 +1,106 @@
+require "relaton/core"
+
+module Relaton
+  module Xsf
+    class DataFetcher < Relaton::Core::DataFetcher
+      # `Core::DataFetcher#log_error` raises unless the flavor overrides it, so
+      # `report_errors` is unusable without this. (The ECMA precedent.)
+      def log_error(msg)
+        Util.error msg
+      end
+
+      def index
+        # `pubid_class:` on the producer too: FileIO#save only calls `to_hash`
+        # when the value is an instance of it, so without it the crawl writes
+        # v1-shaped rows under a v2 name, silently.
+        @index ||= Relaton::Index.find_or_create(
+          :xsf, file: "#{INDEXFILE}.yaml", pubid_class: ::Pubid::Xsf::Identifier
+        )
+      end
+
+      def fetch(_source = nil)
+        agent = Mechanize.new
+        resp = agent.get "https://xmpp.org/extensions/refs/"
+        resp.xpath("//a[contains(@href, 'XEP-')]").each do |link|
+          doc = agent.get link[:href]
+          bib = Relaton::Bib::Converter::BibXml.to_item doc.body
+          save_doc bib
+        rescue StandardError => e
+          Util.warn "Failed to parse #{link[:href]}: #{e.message}"
+        end
+        index.save
+        report_errors
+      end
+
+      def save_doc(bib)
+        return unless bib
+
+        bib.ext ||= Relaton::Bib::Ext.new
+        bib.ext.flavor = "xsf"
+
+        docid = bib.docidentifier.detect(&:primary) || bib.docidentifier.first
+        id = docid&.content
+        return unless id
+
+        # Distinct docids can sanitize to one filename; take a path of our own
+        # rather than overwriting the other document (Core#unique_output_file).
+        file = unique_output_file id
+        if @files.include? file
+          # Same reserved path == same id: a genuine duplicate. Checked FIRST,
+          # because a disambiguated path stays != output_file forever.
+          Util.warn "File #{file} already exists"
+        elsif file != output_file(id)
+          Util.warn "File #{output_file id} already exists; writing #{file} instead"
+        end
+        @files << file
+        File.write file, serialize(bib), encoding: "UTF-8"
+        add_to_index id, file
+      end
+
+      #
+      # Index a document under its pubid, skipping one pubid cannot parse.
+      #
+      # This guard is not optional here. One unparseable row does not fail that
+      # row: `Relaton::Index` declares the **whole file** corrupt, deletes it,
+      # and hands back an **empty** index. Measured — 519 good rows plus one
+      # unparseable id loads as 0 rows, and the only trace is two INFO lines
+      # ("Wrong structure of file …", "Considering … corrupt, removing it"). So
+      # indexing one bad row silently breaks every XSF lookup, not just its own.
+      #
+      # A rejection is recorded in `@errors`, which `report_errors` turns into a
+      # GitHub issue at the end of the crawl (the 3GPP/ECMA precedent). Nothing
+      # in the published corpus trips it today: pubid parses all 520 rows,
+      # including the two entries that are pages rather than XEPs — the XEP
+      # repository's `README` and its `xep-xxxx` template, which pubid accepts
+      # as the literal numbers `README` and `xxxx` while still rejecting
+      # anything else non-numeric (`XEP banana`, or a typo like `XEP 00O1`).
+      # So a recorded error now means something genuinely new appeared, which
+      # is exactly when an issue is worth filing.
+      #
+      # The data file is written either way, so a document that cannot be
+      # indexed is unindexed, never lost.
+      #
+      # @param docid [String]
+      # @param file [String]
+      #
+      def add_to_index(docid, file)
+        index.add_or_update ::Pubid::Xsf::Identifier.parse(docid), file
+      rescue StandardError => e
+        @errors[docid] =
+          "Unparseable primary id `#{docid}` was not indexed (#{e.message})"
+      end
+
+      def to_yaml(bib)
+        bib.to_yaml
+      end
+
+      def to_xml(bib)
+        bib.to_xml bibdata: true
+      end
+
+      def to_bibxml(bib)
+        bib.to_rfcxml
+      end
+    end
+  end
+end

data/lib/relaton/xsf/docidentifier.rb

@@ -0,0 +1,46 @@
+module Relaton
+  module Xsf
+    # An XSF document identifier, backed by `Pubid::Xsf`.
+    #
+    # Parses its `content` into a `Pubid::Xsf::Identifier` kept in `@pubid`,
+    # while the lutaml `content` attribute stays a plain **string** for
+    # serialization. Parsing is **soft**: `content=` lazily requires pubid and
+    # rescues `LoadError`/`StandardError`, so a missing gem or non-XSF content
+    # leaves `@pubid` nil rather than raising during deserialization.
+    #
+    # **It deliberately implements none of `remove_part!` / `remove_date!` /
+    # `to_all_parts!`.** An XEP identifier is a publisher and a number and
+    # nothing else — no part, no edition, no date — so there is genuinely
+    # nothing for any of them to strip, and `Bib::Docidentifier` already
+    # defaults all three to no-ops for exactly that case. Empty overrides here
+    # would assert a flavor-specific rule that does not exist. Compare
+    # `Relaton::Ogc::Docidentifier`, which overrides `remove_date!` because OGC
+    # really does carry a revision.
+    #
+    # So this class exists for `#pubid`: it gives consumers the structured
+    # identifier off a docidentifier, the way every other pubid-backed flavor
+    # does, without changing what is serialized.
+    class Docidentifier < Bib::Docidentifier
+      attr_reader :pubid
+
+      def initialize(attrs = {}, options = {})
+        pubid = attrs.is_a?(Hash) ? attrs.delete(:pubid) : nil
+        attrs[:content] ||= pubid.to_s if pubid
+        super
+        @pubid = pubid if pubid
+      end
+
+      def content=(value)
+        super
+        return unless value
+
+        @pubid = begin
+          require "pubid"
+          ::Pubid::Xsf::Identifier.parse(value)
+        rescue LoadError, StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/relaton/xsf/hit.rb

@@ -0,0 +1,17 @@
+module Relaton
+  module Xsf
+    class Hit < Relaton::Core::Hit
+      def item
+        return @doc if @doc
+
+        agent = Mechanize.new
+        resp = agent.get hit[:url]
+        hash = YAML.safe_load resp.body
+        hash["fetched"] = Date.today.to_s
+        @doc = Relaton::Xsf::Item.from_yaml hash.to_yaml
+      rescue StandardError => e
+        raise Relaton::RequestError, e.message
+      end
+    end
+  end
+end

data/lib/relaton/xsf/hit_collection.rb

@@ -0,0 +1,52 @@
+module Relaton
+  module Xsf
+    class HitCollection < Relaton::Core::HitCollection
+      GHDATA_URL = "https://raw.githubusercontent.com/relaton/relaton-data-xsf/v2/".freeze
+
+      #
+      # Find the index rows for the identifier this collection was built with.
+      #
+      # `ref` is a `Pubid::Xsf::Identifier` -- `Bibliography.parse_ref` does that
+      # step, and raises rather than returning nil for an unrecognized
+      # reference. Passing the identifier rather than a string is what enables
+      # the binary search on `id.root.number`. The nil guard in `#rows` is
+      # defensive only: `Core::HitCollection` permits a nil `ref`, and an empty
+      # result beats a NoMethodError if some other caller constructs one.
+      #
+      # An XEP identifier is only a publisher and a number: no edition, no date,
+      # no part. So the reference states everything, and `Type#search`'s default
+      # subset match (`Pubid::SubsetMatch`) is a plain equality -- which is the
+      # point. Ids are unique, one row per XEP, so there is no selection order
+      # to apply either.
+      #
+      # @return [self]
+      #
+      def search
+        @array = rows.map { |row| Hit.new url: "#{GHDATA_URL}#{row[:file]}" }
+        self
+      rescue Relaton::RequestError
+        raise
+      rescue StandardError => e
+        raise Relaton::RequestError, e.message
+      end
+
+      def index
+        @index ||= Relaton::Index.find_or_create(
+          :xsf,
+          url: "#{GHDATA_URL}#{INDEXFILE}.zip",
+          file: "#{INDEXFILE}.yaml",
+          pubid_class: ::Pubid::Xsf::Identifier,
+        )
+      end
+
+      private
+
+      def rows
+        return [] unless ref
+
+        index.search(ref)
+          .sort_by { |row| row[:id].to_s }
+      end
+    end
+  end
+end

data/lib/relaton/xsf/item.rb

@@ -0,0 +1,13 @@
+require_relative "docidentifier"
+
+module Relaton
+  module Xsf
+    class Item < Bib::Item
+      model ItemData
+
+      # Narrow the inherited `docidentifier` to the flavor's own class, so a
+      # deserialized item exposes `#pubid` on each id. (The OGC/ECMA shape.)
+      attribute :docidentifier, Docidentifier, collection: true, initialize_empty: true
+    end
+  end
+end

data/lib/relaton/xsf/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Xsf
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/xsf/processor.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+require "relaton/core/processor"
+
+module Relaton
+  module Xsf
+    class Processor < Relaton::Core::Processor
+      attr_reader :idtype
+
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_xsf
+        @prefix = "XEP"
+        @defaultprefix = %r{^XEP\s}
+        @idtype = "XEP"
+        @datasets = %w[xep-xmpp]
+        @pubid_flavor = :Xsf
+      end
+
+      def get(code, date, opts)
+        require_relative "../xsf"
+        Relaton::Xsf::Bibliography.get(code, date, opts)
+      end
+
+      def fetch_data(_source, opts)
+        require_relative "data_fetcher"
+        Relaton::Xsf::DataFetcher.fetch(**opts)
+      end
+
+      def from_xml(xml)
+        require_relative "../xsf"
+        Relaton::Xsf::Item.from_xml(xml)
+      end
+
+      def from_yaml(yaml)
+        require_relative "../xsf"
+        Relaton::Xsf::Item.from_yaml(yaml)
+      end
+
+      def grammar_hash
+        require_relative "../xsf"
+        Relaton::Xsf.grammar_hash
+      end
+
+      def remove_index_file
+        require_relative "../xsf"
+        Relaton::Index.find_or_create(
+          :xsf, url: true, file: "#{INDEXFILE}.yaml"
+        ).remove_file
+      end
+    end
+  end
+end

data/lib/relaton/xsf/util.rb

@@ -0,0 +1,9 @@
+module Relaton
+  module Xsf
+    module Util
+      extend Relaton::Bib::Util
+
+      PROGNAME = "relaton-xsf".freeze
+    end
+  end
+end

data/lib/relaton/xsf.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "mechanize"
+# pubid loads with the flavor, for the ::Pubid::Xsf::Identifier that the
+# index code names. Processor#remove_index_file names no pubid class: the
+# delete never reads the index (see lib/relaton/index/CLAUDE.md).
+require "pubid"
+require "relaton/index"
+require "relaton/bib"
+require_relative "version"
+require_relative "xsf/util"
+require_relative "xsf/item_data"
+require_relative "xsf/item"
+require_relative "xsf/bibitem"
+require_relative "xsf/bibdata"
+require_relative "xsf/hit"
+require_relative "xsf/hit_collection"
+require_relative "xsf/bibliography"
+
+module Relaton
+  module Xsf
+    INDEXFILE = "index-v2".freeze
+
+    class Error < StandardError; end
+
+    # Your code goes here...
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton-core.rb

@@ -0,0 +1 @@
+require_relative "relaton/core"