relaton 3.0.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/core/governor.rb

@@ -0,0 +1,320 @@
+# frozen_string_literal: true
+
+require "time" # Time.parse, for an HTTP-date Retry-After
+
+module Relaton
+  module Core
+    #
+    # Process-wide, thread-safe rate-limit back-pressure for a crawl.
+    #
+    # Promoted here from `Relaton::W3c::Governor`, which is now a thin binding
+    # of it, when the ITU-R crawler became the second flavor to need
+    # cross-thread throttling. What stays with a flavor is only which errors
+    # mean "throttled" (THROTTLE_ERRORS / .throttle?) and which env vars tune it
+    # (ENV_PREFIX); the ladder, the jitter, the give-up latch and the
+    # Retry-After parsing are shared.
+    #
+    # The problem it solves, as first met on api.w3.org: that host sits behind
+    # Cloudflare, whose rate limiting is bimodal — a run either never trips it
+    # or trips it and stays banned for hours. Before this
+    # existed, each of the crawl's workers reacted to its own 429s in isolation
+    # with a backoff capped at ~31s — far too short to outlast the ban, and the
+    # retries themselves kept the limiter engaged. The Aug-2026 crawl spent 4h56m
+    # doing nothing but re-tripping the limiter, permanently discarding 1,412
+    # documents, before it aborted.
+    #
+    # The governor gives the whole worker pool one shared cooldown. When any
+    # thread sees a 429 it opens (or extends) that cooldown; every other thread
+    # observes it in #wait. Cooldowns escalate per *round* — 60s, 120s, ... to a
+    # 15-minute cap — and after GIVE_UP_AFTER consecutive rounds without a single
+    # success the crawl is declared rate-limited and gives up (~15 min) so CI
+    # fails fast instead of grinding for five hours.
+    #
+    # `clock:` and `sleeper:` are injectable so specs exercise the real
+    # arithmetic without wall-clock waits.
+    #
+    class Governor
+      # First cooldown, in seconds. Deliberately minute-scale: a Cloudflare ban
+      # window outlasts any second-scale backoff.
+      BASE_COOLDOWN = 60
+
+      # Ceiling for a single cooldown, including a server-supplied Retry-After
+      # (which must not be able to park a worker for an hour).
+      MAX_COOLDOWN = 900
+
+      # Consecutive throttle rounds with no intervening success before the crawl
+      # is considered banned. With the ladder above that is ~15 min of cooldown
+      # (60+120+240+480 s; the fifth round latches before its own wait).
+      GIVE_UP_AFTER = 5
+
+      # Fraction of the cooldown added as random jitter, so the workers do not
+      # all resume on the same instant and immediately re-trip the limiter.
+      JITTER_FRACTION = 0.1
+
+      # Which errors mean "you are being rate-limited", as opposed to "this
+      # resource is broken". Core ships none: the distinction is the one thing
+      # only the flavor knows, and getting it wrong in either direction is
+      # expensive — a broken resource routed here opens a pool-wide cooldown
+      # for one bad record, and a throttle routed the other way gets recorded
+      # as permanent data loss (which is exactly what the W3C crawl did before
+      # this class existed). Subclasses override the constant, or #throttle?
+      # when the signal is a status code rather than an exception class.
+      THROTTLE_ERRORS = [].freeze
+
+      # Prefix for the three duration knobs, so two flavors crawling in one
+      # process cannot share a cooldown ladder by accident. A subclass sets its
+      # own ("RELATON_W3C", "RELATON_ITU"); the suffixes are fixed.
+      ENV_PREFIX = "RELATON"
+
+      # Whether this error means the server is rate-limiting us.
+      #
+      # Class membership is the common case. A flavor whose server signals
+      # throttling with a *status code* on a shared exception class (ITU: 503 on
+      # /rec) overrides this instead of the constant.
+      #
+      # @param error [StandardError]
+      # @return [Boolean]
+      def self.throttle?(error)
+        self::THROTTLE_ERRORS.any? { |klass| error.is_a? klass }
+      end
+
+      attr_reader :base, :max, :give_up_after, :env_prefix
+
+      def initialize(base: nil, max: nil, give_up_after: nil,
+                     clock: nil, sleeper: nil, jitter: nil, env_prefix: nil)
+        @env_prefix = env_prefix || self.class::ENV_PREFIX
+        # Remembered so #reset! can re-read the env knobs for the next crawl
+        # without discarding values a caller passed explicitly.
+        @overrides = { base: base, max: max, give_up_after: give_up_after }
+        load_limits
+        @clock = clock || -> { Time.now }
+        @sleeper = sleeper || ->(secs) { sleep secs }
+        @jitter = jitter || ->(secs) { Kernel.rand * secs * self.class::JITTER_FRACTION }
+        @mutex = Mutex.new
+        @cooldown_until = nil
+        @penalty = nil
+        @rounds = 0
+        @events = 0
+        @abandoned = false
+        @abandoned_rounds = nil
+      end
+
+      #
+      # Block until the shared cooldown expires. Called before every request.
+      #
+      # The mutex is never held across the sleep — the deadline is read under
+      # the lock and the sleeping happens outside it, so a cooling-down worker
+      # cannot block another thread from recording its own throttle.
+      #
+      # @return [Numeric] seconds actually slept (0.0 when there was nothing to
+      #   wait for, including once the crawl has given up)
+      #
+      def wait
+        slept = 0.0
+        waited_for = nil
+        loop do
+          # Once we have given up there is nothing left to wait for, and stalling
+          # here would delay the shutdown by up to a full cooldown.
+          deadline, remaining = @mutex.synchronize do
+            gave_up? ? [nil, 0.0] : [@cooldown_until, cooldown_remaining]
+          end
+          break unless remaining.positive?
+          # Re-check after sleeping so a cooldown *extended* mid-round (a later
+          # Retry-After asking for longer) also holds back the workers already
+          # sleeping — they would otherwise wake on the stale deadline and
+          # re-trip the limiter. Looping only on a later deadline keeps a stalled
+          # clock from spinning here.
+          break if waited_for && deadline && deadline <= waited_for
+
+          waited_for = deadline
+          delay = remaining + @jitter.call(remaining)
+          @sleeper.call(delay)
+          slept += delay
+        end
+        slept
+      end
+
+      #
+      # Record a 429 and open/extend the shared cooldown.
+      #
+      # @param retry_after [Integer, nil] seconds from the response's Retry-After
+      #
+      # @return [Boolean] true when the crawl should give up
+      #
+      def throttled!(retry_after: nil)
+        @mutex.synchronize do
+          now = @clock.call
+          @events += 1
+          if @cooldown_until.nil? || now >= @cooldown_until
+            open_round(now, retry_after)
+          elsif retry_after
+            # Mid-round, the server may still ask for longer than we planned.
+            extend_round(now + capped(retry_after))
+          end
+          # Giving up latches. Otherwise a straggler worker succeeding after the
+          # producer already stopped paginating would clear @rounds and let the
+          # crawl save the truncated index it just decided to abandon.
+          #
+          # @rounds is snapshotted at the same moment, because #succeeded!
+          # resets it: read afterwards it names whatever has re-accumulated
+          # since, not the threshold that was crossed. The abort message is
+          # the only place an operator learns why the crawl stopped.
+          unless @abandoned
+            @abandoned = @rounds >= @give_up_after
+            @abandoned_rounds = @rounds if @abandoned
+          end
+          @abandoned
+        end
+      end
+
+      #
+      # Record a successful request: the limiter has released us.
+      #
+      def succeeded!
+        @mutex.synchronize do
+          @rounds = 0
+          @cooldown_until = nil
+          # Decay rather than reset, so a limiter that keeps flapping still
+          # climbs the ladder instead of restarting at `base` every time.
+          @penalty = @penalty && @penalty > @base ? [@penalty / 2.0, @base].max : nil
+        end
+      end
+
+      # @return [Boolean] give_up_after consecutive rounds passed with no success
+      def exhausted?
+        @mutex.synchronize { gave_up? }
+      end
+
+      # Clear all state for a fresh crawl. Resets in place rather than handing
+      # back a new object so callers (and specs) holding this instance keep
+      # observing the live governor.
+      def reset!
+        @mutex.synchronize do
+          load_limits
+          @cooldown_until = nil
+          @penalty = nil
+          @rounds = 0
+          @events = 0
+          @abandoned = false
+          @abandoned_rounds = nil
+        end
+      end
+
+      # @return [Integer] consecutive throttle rounds since the last success
+      def throttle_rounds
+        @mutex.synchronize { @rounds }
+      end
+
+      # The consecutive-round count as it stood when the give-up latched, for
+      # the abort message. Distinct from #throttle_rounds, which stays LIVE
+      # because a crawler logs it per retry as the current round (see
+      # Relaton::Itu::DataCrawlerR) and #succeeded! resets it.
+      #
+      # @return [Integer, nil] nil while the crawl is still running
+      def give_up_rounds
+        @mutex.synchronize { @abandoned_rounds }
+      end
+
+      # @return [Integer] every 429 observed, for reporting
+      def throttle_count
+        @mutex.synchronize { @events }
+      end
+
+      #
+      # Seconds requested by a rate-limited response's Retry-After header.
+      #
+      # lutaml-hal >= 0.2.5 carries a `{ status:, headers: }` hash on every error
+      # mapped from a response (nil for locally raised ones), so anything without
+      # one yields nil.
+      #
+      # @param error [StandardError]
+      #
+      # @return [Integer, nil]
+      #
+      def self.retry_after(error)
+        return nil unless error.respond_to?(:response)
+
+        response = error.response
+        headers = response.is_a?(Hash) ? (response[:headers] || response["headers"]) : nil
+        return nil unless headers.respond_to?(:[])
+
+        seconds_from headers["retry-after"] || headers["Retry-After"]
+      end
+
+      #
+      # Retry-After is either delta-seconds or an HTTP-date (RFC 9110). The date
+      # form is parsed properly rather than by scanning digits, which would pick
+      # up the day-of-month and back off for a wildly wrong duration; a date
+      # already in the past yields 0, i.e. "no extra floor".
+      #
+      def self.seconds_from(value)
+        str = value.to_s.strip
+        return nil if str.empty?
+        return str.to_i if str.match?(/\A\d+\z/)
+
+        begin
+          [(Time.parse(str) - Time.now).ceil, 0].max
+        rescue ArgumentError
+          nil
+        end
+      end
+
+      # A blank or non-positive override is treated as unset.
+      def self.env_int(name, default)
+        value = ENV[name].to_s.strip
+        return default if value.empty?
+
+        parsed = value.to_i
+        parsed.positive? ? parsed : default
+      end
+
+      private
+
+      # "<PREFIX>_THROTTLE_<SUFFIX>", the one thing ENV_PREFIX exists for.
+      def env_name(suffix)
+        "#{@env_prefix}_THROTTLE_#{suffix}"
+      end
+
+      # Read the duration knobs, letting explicit constructor args win over the
+      # environment. Re-read on #reset! so the env vars behave like a flavor's
+      # sibling knobs, which are class methods evaluated per crawl rather than
+      # frozen when the flavor was autoloaded.
+      def load_limits
+        @base = @overrides[:base] || self.class.env_int(env_name("BASE"), self.class::BASE_COOLDOWN)
+        @max = @overrides[:max] || self.class.env_int(env_name("MAX"), self.class::MAX_COOLDOWN)
+        @give_up_after = @overrides[:give_up_after] ||
+          self.class.env_int(env_name("GIVEUP"), self.class::GIVE_UP_AFTER)
+      end
+
+      # Start a new escalation step. Retry-After raises the floor but never
+      # lowers our own ladder — the server's hint is a minimum, not a discount.
+      def open_round(now, retry_after)
+        @rounds += 1
+        ladder = @penalty ? [@penalty * 2, @max].min : @base
+        hint = retry_after ? capped(retry_after) : 0
+        @penalty = [ladder, hint].max
+        @cooldown_until = now + @penalty
+      end
+
+      def extend_round(deadline)
+        @cooldown_until = deadline if deadline > @cooldown_until
+      end
+
+      def cooldown_remaining
+        return 0.0 unless @cooldown_until
+
+        [@cooldown_until - @clock.call, 0.0].max
+      end
+
+      def capped(seconds)
+        [seconds, @max].min
+      end
+
+      # Callers must already hold the mutex. Latched by #throttled! — never
+      # recomputed from @rounds, which #succeeded! resets.
+      def gave_up?
+        @abandoned
+      end
+    end
+  end
+end

data/lib/relaton/core/pacer.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Core
+    #
+    # A process-wide, thread-safe request pacer: the whole worker pool between
+    # them starts **at most one request per `gap` seconds**.
+    #
+    # It exists because the two natural alternatives are both wrong for a
+    # crawler that has to be polite to one host:
+    #
+    # * `sleep delay` before each request (what `Relaton::Itu::DataCrawlerR`
+    #   did) **adds** the delay to the server's own latency. At ITU's measured
+    #   ~1.1 s response time a 1 s delay yields one request every ~2.1 s, so a
+    #   ~7k-request crawl spends ~2 h of a ~4 h run doing nothing at all — and
+    #   the crawl is pinned to one thread, because the pacing *is* the loop.
+    # * a per-worker gap (`Relaton::Cie::DataFetcher::Pacing`) lets N workers
+    #   issue N requests per gap. That is right for CIE, where the point is to
+    #   let one struggling worker slow itself without dragging the others down,
+    #   and wrong here, where the budget belongs to the host.
+    #
+    # This reserves a *slot* instead: each caller takes the next free instant on
+    # a shared timeline and sleeps only until it arrives. The server's latency
+    # therefore counts toward the gap rather than being added on top, so the
+    # rate the host sees is exactly `1 / gap` no matter how many workers there
+    # are — and workers exist only to hide latency, never to raise the rate.
+    #
+    # The mutex is held for the reservation and **never across the sleep**, so a
+    # waiting worker cannot stop another from reserving.
+    #
+    # Idle time does not bank up slots: a reservation never starts earlier than
+    # "now", so a stall elsewhere in the crawl (a governor cooldown, a slow
+    # merge) cannot be repaid afterwards as a burst — which is exactly the burst
+    # that would re-trip the limiter that caused the stall.
+    #
+    # `clock:` and `sleeper:` are injectable so specs exercise the arithmetic
+    # without wall-clock waits (mirroring `Relaton::Core::Governor`).
+    #
+    class Pacer
+      # How the gap is spent.
+      #
+      #   :slot   (default) the gap is the minimum interval between request
+      #           *starts*, shared by the pool, so the host's own latency counts
+      #           toward it. Peak rate 1/gap regardless of worker count.
+      #   :fixed  sleep the whole gap before every request, whatever else is
+      #           happening. This is what a plain `sleep delay` loop did, kept
+      #           as a rollback knob: with one worker it reproduces the old
+      #           request pattern exactly — one request per (gap + latency).
+      DEFAULT_MODE = :slot
+
+      # @param gap [Numeric] minimum seconds between request *starts*. Zero
+      #   disables pacing entirely (used by specs and by cassette replay).
+      # @param mode [Symbol] :slot or :fixed, see DEFAULT_MODE
+      # @param clock [#call, nil] returns monotonic seconds
+      # @param sleeper [#call, nil] receives seconds to sleep
+      def initialize(gap:, mode: DEFAULT_MODE, clock: nil, sleeper: nil)
+        @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+        @sleeper = sleeper || ->(secs) { sleep secs }
+        @mutex = Mutex.new
+        @gap = coerce_gap(gap)
+        @mode = mode == :fixed ? :fixed : DEFAULT_MODE
+        @next = nil
+        @requests = 0
+        @slept = 0.0
+      end
+
+      attr_reader :gap, :mode
+
+      # @return [Integer] requests paced so far, for the run summary
+      def requests
+        @mutex.synchronize { @requests }
+      end
+
+      # @return [Float] total seconds spent waiting, for the run summary
+      def slept
+        @mutex.synchronize { @slept }
+      end
+
+      # @param secs [Numeric] the new minimum gap; takes effect on the next
+      #   reservation. A negative value is clamped to zero.
+      def gap=(secs)
+        @mutex.synchronize { @gap = coerce_gap(secs) }
+      end
+
+      # Clear the timeline and the counters, so the next caller goes out
+      # immediately. For specs, and for a crawl starting a fresh phase.
+      def reset!
+        @mutex.synchronize do
+          @next = nil
+          @requests = 0
+          @slept = 0.0
+        end
+      end
+
+      #
+      # Reserve the next slot and block until it arrives.
+      #
+      # @return [Float] seconds actually slept (0.0 when the slot was already
+      #   due, which is the common case once the host is slower than the gap)
+      #
+      def wait
+        delay = @mode == :fixed ? @mutex.synchronize { @gap } : (reserve - @clock.call)
+        @mutex.synchronize { @requests += 1 }
+        return 0.0 unless delay.positive?
+
+        @sleeper.call delay
+        @mutex.synchronize { @slept += delay }
+        delay
+      end
+
+      private
+
+      # Claim the next instant on the shared timeline and advance it. Held under
+      # the lock; the waiting happens outside.
+      #
+      # @return [Float] the instant this caller may start its request
+      def reserve
+        @mutex.synchronize do
+          now = @clock.call
+          # `max(now)` is what stops an idle stretch from banking up slots: the
+          # timeline catches up to the present rather than paying out a burst.
+          @next = @next.nil? || @next < now ? now : @next
+          reserved = @next
+          @next += @gap
+          reserved
+        end
+      end
+
+      def coerce_gap(secs)
+        [secs.to_f, 0.0].max
+      end
+    end
+  end
+end

data/lib/relaton/core/processor.rb

@@ -13,6 +13,25 @@ module Relaton
       # @return [Array<String>]
       attr_reader :datasets
 
+      # Global document-ID prefixes this flavor owns, e.g. BSI => %w[BS BSI DD …].
+      # The source of truth is **pubid**: a flavor sets @pubid_flavor to its Pubid
+      # module name (e.g. :Iso) in #initialize, and the prefixes are read from
+      # `Pubid::<Flavor>.prefixes` — the SDO's own leading identifier tokens,
+      # including non-obvious ones (BSI `DD`) and joint forms (`ISO/IEC`). Flavors
+      # with no pubid backing fall back to the single canonical @prefix. Loaded
+      # lazily (pubid is only required on first call) and memoized. Feeds the
+      # global prefix register (Relaton.prefix_flavor). (relaton-db#103)
+      #
+      # @return [Array<String>]
+      def prefixes
+        @prefixes ||= if @pubid_flavor
+                        require "pubid"
+                        ::Pubid.const_get(@pubid_flavor).prefixes
+                      else
+                        Array(prefix)
+                      end
+      end
+
       def initialize
         raise "This is an abstract class!"
       end

data/lib/relaton/core/request_error.rb

@@ -0,0 +1,14 @@
+module Relaton
+  # Transport failure while fetching remote data; `Relaton::Db#net_retry`
+  # retries it. It depends on nothing, so `relaton/index` requires this file
+  # alone and does not load the rest of `relaton/core`.
+  #
+  # The class is named `Relaton::RequestError`, not `Relaton::Core::...`: an
+  # error raised without a message uses the class name as its message, and
+  # relaton-cli prints that message.
+  class RequestError < StandardError; end
+
+  module Core
+    RequestError = Relaton::RequestError
+  end
+end

data/lib/relaton/core.rb

@@ -1,5 +1,6 @@
 require "relaton/logger"
 require_relative "version"
+require_relative "core/request_error"
 require_relative "core/array_wrapper"
 require_relative "core/hash_keys_sybolizer"
 require_relative "core/date_parser"
@@ -7,3 +8,5 @@ require_relative "core/processor"
 require_relative "core/data_fetcher"
 require_relative "core/hit_collection"
 require_relative "core/workers_pool"
+require_relative "core/pacer"
+require_relative "core/governor"

data/lib/relaton/db/registry.rb

@@ -11,7 +11,8 @@ module Relaton
         relaton/bsi relaton/cen relaton/iana relaton/3gpp
         relaton/oasis relaton/doi relaton/jis relaton/xsf
         relaton/ccsds relaton/etsi relaton/isbn relaton/plateau
-        relaton/oiml
+        relaton/oiml relaton/jcgm relaton/easc relaton/gost relaton/adobe
+        relaton/iala
       ].freeze
 
       include Singleton
@@ -107,8 +108,47 @@ module Relaton
         nil
       end
 
+      #
+      # Global prefix register (relaton-db#103): all processors that own the
+      # given global prefix, matched case-insensitively and exactly (unlike
+      # #class_by_ref, which matches a prefix at the start of a full reference).
+      # Ordered by registration order (SUPPORTED_GEMS), so results are
+      # deterministic. Lazy: never dereferences a flavor constant.
+      #
+      # @param prefix [String]
+      # @return [Array<Relaton::Core::Processor>]
+      #
+      def processors_by_prefix(prefix)
+        key = prefix.to_s.strip.upcase
+        processors.values.select do |processor|
+          processor.prefixes.any? { |pref| pref.upcase == key }
+        end
+      end
+
+      #
+      # Flavor modules (e.g. Relaton::Iso) that own the given global prefix.
+      # NOTE: dereferencing the returned module forces that flavor's lazy load.
+      # Callers that must stay lazy should use #processors_by_prefix instead.
+      #
+      # @param prefix [String]
+      # @return [Array<Module>]
+      #
+      def flavors_by_prefix(prefix)
+        processors_by_prefix(prefix).map { |processor| flavor_module(processor) }
+      end
+
       private
 
+      # The flavor namespace for a processor: the module enclosing its class.
+      # Relaton::Iso::Processor -> Relaton::Iso. Derived from the class name so
+      # there's no @short string-munging and no 3gpp special case.
+      #
+      # @param processor [Relaton::Core::Processor]
+      # @return [Module]
+      def flavor_module(processor)
+        Object.const_get processor.class.name.split("::")[0..-2].join("::")
+      end
+
       def gem_to_module_path(gem_name)
         gem_name.split("/").map do |part|
           part.capitalize.sub("3gpp", "ThreeGpp")

data/lib/relaton/doi/crossref.rb

@@ -49,7 +49,7 @@ module Relaton
           end
 
           n += 1
-          sleep resp.response["x-rate-limit-interval"].to_i * n
+          sleep backoff(resp.response, n)
         rescue Mechanize::ResponseCodeError => e
           return nil if e.response_code == "404"
 
@@ -58,10 +58,27 @@ module Relaton
           end
 
           n += 1
-          sleep e.page.response["x-rate-limit-interval"].to_i * n
+          sleep backoff(e.page.response, n)
         end
       end
 
+      #
+      # Seconds to wait before retry n. Crossref sends X-Rate-Limit-Interval as
+      # "1s", but a throttled response can carry no rate-limit headers at all —
+      # the 429s seen here had only Date/Content-Length/Connection. Without the
+      # floor that degenerates to `sleep 0`, i.e. hammering the endpoint that
+      # just asked us to slow down.
+      #
+      # @param [Hash, nil] headers The response headers.
+      # @param [Integer] num The attempt number.
+      #
+      # @return [Integer] Delay in seconds, at least 1.
+      #
+      def backoff(headers, num)
+        interval = headers.to_h["x-rate-limit-interval"].to_s[/\d+/].to_i
+        [interval, 1].max * num
+      end
+
       def agent
         @agent ||= Mechanize.new do |a|
           a.user_agent = USER_AGENT