relaton 3.0.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/w3c/data_fetcher.rb

@@ -3,7 +3,6 @@ require "w3c_api"
 require_relative "../w3c"
 require_relative "safe_realize"
 require_relative "data_parser"
-require_relative "pubid"
 
 module Relaton
   module W3c
@@ -27,6 +26,48 @@ module Relaton
       # (rate-limit/connection) before giving up and aborting the crawl.
       PAGE_FETCH_ATTEMPTS = 3
 
+      # How many resources may be lost to rate limiting before the dataset is
+      # considered too incomplete to save. Deliberately low: crawls are bimodal
+      # — a healthy run loses none (Aug 17 2026: 0), a rate-limited one loses
+      # hundreds (Aug 19 2026: 1,412) — so there is no middle ground to protect.
+      # Genuinely broken resources (permanent skips from 404s/5xx) are counted
+      # separately and stay tolerated.
+      DEFAULT_MAX_THROTTLED_LOSSES = 25
+
+      # Retry policy pushed into w3c_api. Its defaults spend ~31s (1+2+4+8+16)
+      # inside lutaml-hal before a 429 ever reaches us, which just delays the
+      # shared back-pressure that actually works; this compresses the same chain
+      # to ~11s (0.5+1+2+4+4) so the Governor can open its pool-wide cooldown
+      # while the ban window is still young.
+      #
+      # `max_retries` deliberately stays at 5. lutaml-hal applies one retry count
+      # to both 429 and 5xx, and unlike a 429 a 5xx is still a *permanent* skip
+      # here — cutting its attempts would blacklist a burst of resources on a
+      # brief upstream wobble that the old policy rode out, and `skipped` is not
+      # counted by guard_throttle_budget, so that loss would be invisible.
+      #
+      # `max_retry_after` matches `max_delay` because a server-instructed wait
+      # must not outlive the compressed chain either. Left unset, lutaml-hal's
+      # DEFAULT_MAX_RETRY_AFTER (300 s) applies, and a large Retry-After on a
+      # 429 parks one thread inside the HTTP layer for up to 5 x 300 s with the
+      # governor hearing nothing — the very thing the compression prevents. The
+      # hint is not discarded, it moves to where it belongs: Governor.retry_after
+      # reads it off the same error and uses it as the FLOOR of the pool-wide
+      # cooldown, under the governor's own 900 s cap.
+      UPSTREAM_RATE_LIMITING = {
+        max_retries: 5, base_delay: 0.5, max_delay: 4.0, backoff_factor: 2.0,
+        max_retry_after: 4.0
+      }.freeze
+
+      # Ceiling on resources dropped to rate limiting before the crawl refuses
+      # to save. Tunable via env var for a deliberately partial run; 0 is a
+      # meaningful setting (tolerate nothing), so it is accepted here unlike in
+      # the governor's duration knobs.
+      def self.max_throttled_losses
+        value = ENV["RELATON_W3C_MAX_THROTTLED"].to_s.strip
+        value.match?(/\A\d+\z/) ? value.to_i : DEFAULT_MAX_THROTTLED_LOSSES
+      end
+
       # Number of fetch_spec worker threads. Tunable via env var so CI or
       # local runs can dial it up for speed or down to lighten load on
       # api.w3.org (or for debugging).
@@ -52,8 +93,14 @@ module Relaton
         @interrupted = false
       end
 
+      # The index is pubid-backed: `pubid_class:` makes Relaton::Index sort the
+      # rows by `id.root.number` on save and serialize each id to its
+      # `_type: pubid:w3c:*` hash. Without it a v1-shaped file would be written
+      # under a v2 name, with no error and no narrowing.
       def index
-        @index ||= Relaton::Index.find_or_create(:W3C, file: "#{INDEXFILE}.yaml")
+        @index ||= Relaton::Index.find_or_create(
+          :W3C, file: "#{INDEXFILE}.yaml", pubid_class: ::Pubid::W3c::Identifier
+        )
       end
 
       def log_error(msg)
@@ -75,6 +122,8 @@ module Relaton
       # of everything fetched so far is saved rather than the run being lost.
       #
       def fetch(_source = nil)
+        SafeRealize.reset!
+        configure_upstream_rate_limiting
         n_workers = self.class.concurrency
         queue = SizedQueue.new(n_workers * 4)
         workers = Array.new(n_workers) { spawn_worker(queue) }
@@ -89,13 +138,52 @@ module Relaton
             n_workers.times { queue << nil } # poison pills
             workers.each(&:join)
           end
-          Util.warn "Crawl interrupted — saving progress collected so far." if @interrupted
+          if @interrupted
+            # Ctrl-C means "give me what you have", so the completeness guards
+            # below are deliberately not applied to an interrupted crawl.
+            Util.warn "Crawl interrupted — saving progress collected so far."
+          else
+            guard_rate_limited
+            guard_throttle_budget
+          end
           index.save
         end
 
         report_errors
       end
 
+      # Abort when the governor concluded we are banned rather than briefly
+      # throttled. Everything fetched so far is discarded on purpose: a partial
+      # dataset must never be saved (crawler.rb wipes data/ first, so saving one
+      # commits mass deletions).
+      def guard_rate_limited
+        return unless SafeRealize.governor.exhausted?
+
+        # #give_up_rounds, not #throttle_rounds: the latter is live and a
+        # straggler success resets it, which is how a give-up at round 5 got
+        # reported as "after 2 consecutive backoffs".
+        rounds = SafeRealize.governor.give_up_rounds ||
+          SafeRealize.governor.throttle_rounds
+        raise CrawlIncompleteError,
+              "crawl is rate-limited: gave up after " \
+              "#{rounds} consecutive backoffs " \
+              "(#{SafeRealize.governor.throttle_count} rate-limited responses); " \
+              "refusing to save a partial dataset"
+      end
+
+      # The per-resource counterpart to guard_complete_pagination: pagination can
+      # complete cleanly while rate limiting has quietly hollowed out the
+      # documents behind it.
+      def guard_throttle_budget
+        lost = SafeRealize.throttled.size
+        budget = self.class.max_throttled_losses
+        return if lost <= budget
+
+        raise CrawlIncompleteError,
+              "#{lost} resources were dropped to rate limiting (budget #{budget}); " \
+              "refusing to save a partial dataset"
+      end
+
       #
       # Page through the specifications index, feeding each spec (paired with
       # its embedded page) to the worker queue. Returns early when interrupted.
@@ -113,11 +201,11 @@ module Relaton
         loop do
           page = specs
           page.links.specifications.each do |spec|
-            break if @interrupted
+            break if stopping?
 
             queue << [spec, page]
           end
-          break if @interrupted
+          break if stopping?
 
           last_page = page.page
           break unless page.next?
@@ -133,6 +221,11 @@ module Relaton
           # `break`ing prevents a rate-limit blip from silently truncating the
           # dataset: a partial crawl must never be saved/committed.
           unless next_page
+            # When the governor gave up, the page didn't fail — we stopped
+            # asking. Let guard_rate_limited report that with the accurate
+            # reason instead of dressing it up as a pagination fault.
+            break if SafeRealize.governor.exhausted?
+
             raise CrawlIncompleteError,
                   "specifications pagination stopped at page #{page.page}: " \
                   "failed to fetch page #{page.page + 1}"
@@ -141,11 +234,29 @@ module Relaton
           specs = next_page
         end
 
-        return if @interrupted
+        return if stopping?
 
         guard_complete_pagination(last_page, expected_pages)
       end
 
+      # Reasons to stop feeding the queue: a Ctrl-C (save what we have) or the
+      # governor concluding the crawl is banned (abort). Both need the producer
+      # to stop; #fetch decides which of the two it was.
+      def stopping?
+        @interrupted || SafeRealize.governor.exhausted?
+      end
+
+      # The mid-spec back-out, and deliberately NOT #stopping?. The two stop
+      # reasons want opposite things of a spec already in flight:
+      #
+      # - Ctrl-C means "give me what you have". The index IS saved, so the
+      #   in-flight spec must finish, or a half-parsed document is written.
+      # - A latched give-up means the run is lost — guard_rate_limited raises
+      #   and nothing is saved — and the host has banned us. Abandon at once.
+      def abandoned?
+        SafeRealize.governor.exhausted?
+      end
+
       # Defense in depth: even when no page fetch raised, make sure pagination
       # actually reached the last page the API advertised. Catches truncation
       # modes other than a failed fetch (e.g. a `next` link that goes missing).
@@ -164,6 +275,10 @@ module Relaton
         # parent_resource serves the spec from embedded data (no HTTP request).
         spec = realize(unrealized_spec, parent_resource: page)
         return unless spec
+        # The latch can land while this spec is in flight. Nothing this crawl
+        # produces is saved any more, so stop before the parse and the write
+        # too, not only before the requests.
+        return if abandoned?
 
         local_errors = Hash.new(true)
         save_doc DataParser.parse(spec, local_errors)
@@ -209,14 +324,41 @@ module Relaton
           if @files.include?(file)
             Util.warn "File #{file} already exists. Document: #{bib.docnumber}" if warn_duplicate
           else
-            pubid = PubId.parse bib.docnumber
-            index.add_or_update pubid.to_hash, file
+            index_primary bib.docidentifier&.detect(&:primary), file
             @files << file
           end
           File.write file, serialize(bib), encoding: "UTF-8"
         end
       end
 
+      #
+      # Add a document's primary id to the index.
+      #
+      # Every W3C id is expected to parse. One that does not is recorded in
+      # `@errors`, so the inherited `report_errors` logs it at ERROR and raises
+      # a tracked GitHub issue at the end of the crawl, and the row is skipped
+      # rather than indexed unparsed: `Relaton::Index` rejects the WHOLE index
+      # if a single row fails to deserialize, and its sort calls
+      # `.root.number` on every id. The data file is still written, so the
+      # document is unindexed, never lost.
+      #
+      # Keyed by the ID rather than the output path: the id is the defect, the
+      # file is only where it landed, and one broken id reaching several files
+      # must still report once.
+      #
+      # @param [Docidentifier, nil] docid the document's primary identifier
+      # @param [String] file path the document was written to
+      #
+      def index_primary(docid, file)
+        if docid&.pubid
+          index.add_or_update docid.pubid, file
+          return
+        end
+
+        id = docid&.content || file
+        @errors[id.to_s] = "Unparseable primary id `#{id}` was not indexed (#{file})"
+      end
+
       def to_xml(bib)
         bib.to_xml(bibdata: true)
       end
@@ -262,15 +404,30 @@ module Relaton
       # embedded_data is populated. Transient 403/5xx/connection failures are
       # already retried upstream (w3c_api/lutaml-hal), but losing an index page
       # drops every spec on it, so retry a few more times here with backoff to
-      # ride out a brief rate-limit window. Returns nil only once the attempts
-      # are exhausted; the caller turns that into a CrawlIncompleteError so the
-      # crawl aborts instead of committing a truncated dataset.
+      # ride out a brief wobble. Rate limiting is handled separately, by the
+      # governor. Returns nil once the attempts are exhausted (or the crawl is
+      # declared banned); the caller turns that into a CrawlIncompleteError so
+      # the crawl aborts instead of committing a truncated dataset.
       def fetch_specifications_page(number)
         attempt = 0
         begin
-          attempt += 1
-          client.specifications(embed: true, page: number)
+          SafeRealize.governor.wait
+          page = client.specifications(embed: true, page: number)
+          SafeRealize.governor.succeeded!
+          page
+        rescue *Governor::THROTTLE_ERRORS => e
+          # How long to keep trying a rate-limited page is the governor's call,
+          # not this counter's: PAGE_FETCH_ATTEMPTS would give up after ~3
+          # minutes and report a *pagination* fault, hiding the real reason and
+          # pre-empting the escalation ladder. Retry until the governor declares
+          # the crawl banned; #wait above paces each attempt, and returns
+          # immediately once it has, so this cannot spin.
+          log_error "Rate-limited fetching specifications page #{number}: #{e.message}"
+          return nil if SafeRealize.governor.throttled!(retry_after: Governor.retry_after(e))
+
+          retry
         rescue Lutaml::Hal::Error, Faraday::Error => e
+          attempt += 1
           log_error "Failed to fetch specifications page #{number} " \
                     "(attempt #{attempt}/#{PAGE_FETCH_ATTEMPTS}): " \
                     "#{e.class}: #{e.message}"
@@ -282,12 +439,33 @@ module Relaton
         end
       end
 
+      # Point w3c_api's rate limiter at UPSTREAM_RATE_LIMITING. This mutates
+      # w3c_api's process-wide singleton and is not restored afterwards — the
+      # crawler is a standalone script, and a shorter inner retry is the right
+      # setting for any caller that has the governor in front of it anyway.
+      # Best-effort: the setter is w3c_api's only public knob and is not part of
+      # a stable contract, so losing it must not fail the run.
+      def configure_upstream_rate_limiting
+        hal = W3cApi::Hal.instance
+        return unless hal.respond_to?(:configure_rate_limiting)
+
+        # w3c_api >= 0.3.3 cascades this through reset_client -> rebuild_register
+        # itself, so the register no longer goes on serving the old policy.
+        # Do NOT add a reset_register call after this: rebuild_register
+        # re-registers eagerly on purpose (Link#realize raises when the name is
+        # absent from lutaml-hal's GlobalRegister), and tearing it back down
+        # would break realize for every model fetched before this point.
+        hal.configure_rate_limiting(UPSTREAM_RATE_LIMITING)
+      rescue StandardError => e
+        Util.warn "Could not configure w3c_api rate limiting: #{e.class}: #{e.message}"
+      end
+
       def spawn_worker(queue)
         Thread.new do
           while (item = queue.pop)
-            # Once interrupted, drain the queue without processing so the
-            # producer unblocks and the pool reaches its poison pills quickly.
-            next if @interrupted
+            # Once stopping, drain the queue without processing so the producer
+            # unblocks and the pool reaches its poison pills quickly.
+            next if stopping?
 
             spec, page = item
             begin

data/lib/relaton/w3c/data_parser.rb

@@ -126,11 +126,11 @@ module Relaton
       #
       # Parse docidentifier
       #
-      # @return [Array<Bib::Docidentifier>] docidentifier
+      # @return [Array<Docidentifier>] docidentifier
       #
       def parse_docid
         id = pub_id(doc_uri)
-        [Bib::Docidentifier.new(type: "W3C", content: id, primary: true)]
+        [Docidentifier.new(type: "W3C", content: id, primary: true)]
       end
 
       #
@@ -371,7 +371,7 @@ module Relaton
         Bib::Organization.new(
           name: [Bib::TypedLocalizedString.new(content: "World Wide Web Consortium")],
           abbreviation: Bib::LocalizedString.new(content: "W3C"),
-          uri: Bib::Uri.new(content: "https://www.w3.org"),
+          uri: [Bib::Uri.new(content: "https://www.w3.org")],
         )
       end
     end

data/lib/relaton/w3c/docidentifier.rb

@@ -0,0 +1,48 @@
+module Relaton
+  module W3c
+    # A W3C document identifier that carries its parsed pubid alongside the
+    # source string.
+    #
+    # `DataParser#pub_id` builds `"W3C REC-xml-names-20091208"` — publisher
+    # prefix included — which is exactly the form
+    # `Pubid::W3c::Identifier.parse` accepts, so nothing has to synthesize or
+    # strip a prefix. `DataFetcher#index_primary` takes `#pubid` straight from
+    # here to build the `index-v2` rows.
+    #
+    # Follows the IHO shape rather than the ISO one: `content=` calls `super`
+    # first, so `content` keeps the source string verbatim and serialization is
+    # unchanged. ISO re-renders `content` from the pubid, which W3C has no need
+    # of and which would alter its published output.
+    class Docidentifier < Bib::Docidentifier
+      attr_reader :pubid
+
+      def content=(value)
+        super
+        @pubid = value && parse(value)
+      end
+
+      # W3C identifiers carry no part, date-as-component or all-parts notion, so
+      # the three mutators `Bib::ItemData` broadcasts over every docidentifier
+      # are no-ops here rather than the inherited NotImplementedError.
+      def remove_part!; end
+
+      def remove_date!; end
+
+      def to_all_parts!; end
+
+      private
+
+      # An identifier that does not parse is a data defect, so it is reported at
+      # ERROR — never at WARN. It does not raise: an already-published record
+      # still has to deserialize and render. The crawl escalates the same
+      # failure into a tracked GitHub issue (see
+      # `DataFetcher#index_primary` and `Core::DataFetcher#report_errors`).
+      def parse(value)
+        ::Pubid::W3c::Identifier.parse value.to_s
+      rescue StandardError => e
+        Util.error "Failed to parse pubid `#{value}`: #{e.message}"
+        nil
+      end
+    end
+  end
+end

data/lib/relaton/w3c/governor.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "lutaml/hal"
+require_relative "../core/governor"
+
+module Relaton
+  module W3c
+    #
+    # W3C binding of {Relaton::Core::Governor}.
+    #
+    # The ladder (60 s → 900 s), the per-round escalation, the jitter, the
+    # latched give-up and the Retry-After parsing all live in core, where the
+    # ITU-R crawler shares them. What is W3C-specific is exactly two things:
+    # which errors mean "rate-limited", and which env vars tune the durations.
+    #
+    class Governor < Relaton::Core::Governor
+      # Errors that mean "you are being rate-limited", as opposed to "this
+      # resource is broken". api.w3.org signals throttling with **403 as well as
+      # 429** — w3c_api's whole faraday-retry layer is built around 403 being the
+      # W3C rate-limit signal — so a 403 must never be recorded as a permanently
+      # broken resource either. Requires lutaml-hal >= 0.2.5, which gave 403 its
+      # own ForbiddenError class (before that it arrived as a bare Error).
+      THROTTLE_ERRORS = [
+        ::Lutaml::Hal::TooManyRequestsError,
+        ::Lutaml::Hal::ForbiddenError,
+      ].freeze
+
+      # RELATON_W3C_THROTTLE_BASE / _MAX / _GIVEUP.
+      ENV_PREFIX = "RELATON_W3C"
+    end
+  end
+end

data/lib/relaton/w3c/item.rb

@@ -1,5 +1,6 @@
 require_relative "item_data"
 require_relative "ext"
+require_relative "docidentifier"
 
 module Relaton
   module W3c
@@ -7,6 +8,8 @@ module Relaton
       model ItemData
 
       attribute :ext, Ext
+      attribute :docidentifier, Docidentifier, collection: true,
+                                               initialize_empty: true
     end
   end
 end

data/lib/relaton/w3c/pubid.rb

@@ -1,5 +1,17 @@
 module Relaton
   module W3c
+    # The legacy `index-v1` row shape.
+    #
+    # This is no longer on any lookup or crawl path in this gem: the flavor
+    # indexes and searches on `Pubid::W3c::Identifier` (see
+    # `Relaton::W3c::Docidentifier` and `INDEXFILE`). It is retained public
+    # only so `relaton-data-w3c`'s crawler can keep emitting the bespoke
+    # `index-v1` that released relaton v2 consumers read — the same reason
+    # `Relaton::Bipm::Id` was retained.
+    #
+    # Do not add call sites. Its decomposition differs from pubid's: it splits
+    # `year` from `date` and keeps a `/`-suffix in `suff`, where pubid folds
+    # both into the slug.
     class PubId
       PARTS = %i[code stage type year date suff].freeze
 

data/lib/relaton/w3c/safe_realize.rb

@@ -1,4 +1,5 @@
 require "concurrent/map"
+require_relative "governor"
 
 module Relaton
   module W3c
@@ -7,10 +8,22 @@ module Relaton
     # resources that failed terminally and returns nil for them — so one broken
     # link doesn't abort the crawl and isn't re-fetched on every reference.
     #
-    # Transient failures are retried upstream: w3c_api retries HTTP 403 (the
+    # Transient failures are retried upstream first: w3c_api retries 403 (the
     # W3C rate-limit signal) and connection/timeout errors, and lutaml-hal
-    # retries 429 and 5xx. By the time an error surfaces here it is terminal.
+    # retries 429 and 5xx. By the time an error surfaces here the resource is
+    # either genuinely broken (skipped for the rest of the run) or we are still
+    # being rate-limited (handed to the Governor).
+    #
+    # That distinction is the point: a 429 or 403 means "come back later", not
+    # "this document does not exist". Recording one as a permanent skip is what
+    # silently cost the Aug-2026 relaton-data-w3c crawl 1,412 documents.
     module SafeRealize
+      # How many times a rate-limited realize is re-attempted, each behind the
+      # governor's shared cooldown, before the resource is written off for this
+      # run. Three spans the escalating cooldowns without letting one unlucky
+      # href monopolise a worker.
+      THROTTLE_ATTEMPTS = 3
+
       # Hrefs that failed terminally — one map shared by every includer
       # (DataFetcher and DataParser) since a broken resource is broken for the
       # whole crawl. Initialized eagerly (at load, single-threaded) so the
@@ -18,8 +31,26 @@ module Relaton
       # Concurrent::Map then handles the concurrent reads/writes.
       @skipped = Concurrent::Map.new
 
-      def self.skipped
-        @skipped
+      # Hrefs lost to rate limiting rather than to a broken resource. Kept apart
+      # from @skipped because they measure how incomplete the crawl is, not a
+      # property of the resource: DataFetcher refuses to save an index once too
+      # many pile up.
+      @throttled = Concurrent::Map.new
+
+      # Shared back-pressure for the whole crawl (see Governor).
+      @governor = Governor.new
+
+      class << self
+        attr_reader :skipped, :throttled
+        attr_accessor :governor
+
+        # Start a crawl (or a spec example) from a clean slate. The governor is
+        # reset in place, so a reference taken before the reset stays live.
+        def reset!
+          @skipped.clear
+          @throttled.clear
+          @governor.reset!
+        end
       end
 
       # @param parent_resource [Object, nil] the index/page the link came from.
@@ -27,30 +58,88 @@ module Relaton
       #   payload lets the link realize from memory instead of issuing an HTTP
       #   request. nil (the default) preserves the plain remote-fetch behavior.
       def realize(obj, parent_resource: nil)
+        # Once the governor has latched its give-up the run is already lost
+        # (DataFetcher#guard_rate_limited raises and nothing is saved) and the
+        # host has just banned us, so every further request only extends the
+        # ban into the next run. #wait alone stops WAITING, not asking: the
+        # 2026-08-31 crawl spent its last 11 minutes firing at a banned host
+        # because the per-spec fan-out had no back-pressure of its own.
+        #
+        # This is the one choke point every request in the crawl passes
+        # through, so one check covers DataFetcher#fetch_versions and all
+        # seven of DataParser's realize sites. They already tolerate nil —
+        # that is the SafeRealize contract.
+        return nil if SafeRealize.governor.exhausted?
+
         href = resolve_href(obj)
         return nil if SafeRealize.skipped.key?(href)
 
-        obj.realize(parent_resource: parent_resource)
-      rescue Lutaml::Hal::ConnectionError, Lutaml::Hal::TimeoutError, Faraday::Error, Net::OpenTimeout => e
-        # Network-level failure (already retried by w3c_api). The resource itself
-        # is fine, so don't skip it permanently — a later reference can try again.
-        Util.warn "Failed to realize object: #{href}, error: #{e.message}"
-        nil
-      rescue Lutaml::Hal::NotFoundError
-        Util.warn "Object not found: #{href}"
-        SafeRealize.skipped[href] = true
-        nil
-      rescue Lutaml::Hal::Error => e
-        # Definitive upstream error (403 rate-limit, 5xx, 429) already retried by
-        # w3c_api / lutaml-hal. Skip the broken/unavailable resource rather than
-        # re-hitting it for every link that references it.
-        Util.warn "Skipping #{href}, upstream error after retries: #{e.message}"
-        SafeRealize.skipped[href] = true
-        nil
+        attempt = 0
+        begin
+          attempt += 1
+          SafeRealize.governor.wait
+          realized = obj.realize(parent_resource: parent_resource)
+          # Only a realize that actually went to the network says anything about
+          # the limiter. With a parent_resource, lutaml-hal serves the object
+          # straight out of the page's `_embedded` payload and never issues a
+          # request (link.rb: "Priority 1: check embedded content first") — and
+          # that is the common case, one per specification. Reporting those as
+          # successes would clear the cooldown and reset the escalation ladder
+          # between every pair of version-history 429s, so the governor could
+          # never reach its give-up threshold: exactly the 60s re-trip loop this
+          # class exists to break.
+          SafeRealize.governor.succeeded! unless parent_resource
+          # The document made it into the dataset after all, so it must stop
+          # counting against the crawl's throttle budget.
+          SafeRealize.throttled.delete(href)
+          realized
+        rescue *Governor::THROTTLE_ERRORS => e
+          retry if record_throttle(href, e, attempt)
+          nil
+        rescue Lutaml::Hal::ConnectionError, Lutaml::Hal::TimeoutError, Faraday::Error, Net::OpenTimeout => e
+          # Network-level failure (already retried by w3c_api). The resource itself
+          # is fine, so don't skip it permanently — a later reference can try again.
+          Util.warn "Failed to realize object: #{href}, error: #{e.message}"
+          nil
+        rescue Lutaml::Hal::NotFoundError
+          Util.warn "Object not found: #{href}"
+          SafeRealize.skipped[href] = true
+          nil
+        rescue Lutaml::Hal::Error => e
+          # Definitive upstream error (5xx, 400/401, …) already retried by
+          # w3c_api / lutaml-hal. Skip the broken/unavailable resource rather than
+          # re-hitting it for every link that references it. 5xx stays here
+          # deliberately: the handful of persistent per-resource 500s a healthy
+          # crawl sees are broken records, not throttling, and routing them
+          # through the governor would open a pool-wide cooldown for each one.
+          Util.warn "Skipping #{href}, upstream error after retries: #{e.message}"
+          SafeRealize.skipped[href] = true
+          nil
+        end
       end
 
       private
 
+      #
+      # Record a rate-limited attempt and decide whether to try again.
+      #
+      # @return [Boolean] true to retry — the caller re-enters #realize's begin
+      #   block, whose #wait sits out the shared cooldown first
+      #
+      def record_throttle(href, error, attempt)
+        gave_up = SafeRealize.governor.throttled!(
+          retry_after: Governor.retry_after(error),
+        )
+        # Once the governor has declared the crawl rate-limited there is nothing
+        # to gain from more attempts — the run is about to abort anyway.
+        return true if !gave_up && attempt < THROTTLE_ATTEMPTS
+
+        Util.warn "Throttled out: #{href}, still rate-limited after " \
+                  "#{attempt} attempt(s): #{error.message}"
+        SafeRealize.throttled[href] = true
+        false
+      end
+
       def resolve_href(obj)
         obj.href || obj.links.self.href
       end

data/lib/relaton/w3c.rb

@@ -1,3 +1,5 @@
+require "pubid"
+require "pubid/w3c"
 require "relaton/bib"
 require "relaton/index"
 require_relative "version"
@@ -6,10 +8,19 @@ require_relative "w3c/item"
 require_relative "w3c/bibitem"
 require_relative "w3c/bibdata"
 require_relative "w3c/bibliography"
+# Not used by this gem any more; loaded so relaton-data-w3c's crawler can
+# require "relaton/w3c" and still reach the legacy index-v1 row shape.
+require_relative "w3c/pubid"
 
 module Relaton
   module W3c
-    INDEXFILE = "index-v1".freeze
+    # The runtime index is the pubid-backed `index-v2`, whose rows are
+    # `Pubid::W3c::Identifier` leaves serialized as `_type: pubid:w3c:*`, built
+    # and read via `pubid_class: ::Pubid::W3c::Identifier`. The bespoke
+    # `index-v1` is relaton-v2 legacy and is no longer produced here —
+    # `relaton-data-w3c`'s crawler emits it from the retained public
+    # `Relaton::W3c::PubId`.
+    INDEXFILE = "index-v2".freeze
 
     class Error < StandardError; end