tebako-release 0.5.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 70f6056da28a7c62a6025b10993b3e1998ab8b4284a5a6ee86bfe799fd6608f7
-  data.tar.gz: 877d474e2f19e6af7c6d3b49f366ea2040c6fef4e06275d0858409069f82a1e3
+  metadata.gz: cc98dded830844b0afdda4fdba2ede81155f424bb58eaab89601a9588e0251c2
+  data.tar.gz: c471ef97c8de46dc8ff5d8dfb5868a20288cf41aace0a91af8d9efbc5dbbfe2a
 SHA512:
-  metadata.gz: 1f63cb53bc85a2fb99df775a471f799b1c70ae139ab1525d87458b5bb052bf7e6e32175c990abcccf1f2bc35a0ef69e1a7709a0cb0075c91a6c401a04be49010
-  data.tar.gz: 02a288b0ac6155e10429137d5d7e2d8e4e0fc1afb3f98b6315e15fd763e4cd97de6e8e063c3d0f01f6384b0a83253c2b0e3b2b6a756d8dac7fae296aea6682c7
+  metadata.gz: 68ba9ef2fde5b4f6fa9522673f2e9480b87fe9ceb2fb6a39a72560bd7511f90c3d3f7e44ac5772fe32aac8c24cdd54205bce2a4e955eb8eb304907cbc9607d84
+  data.tar.gz: b1e26cefc882eb129067448ce5eb5dd2cd8320675bfd336c49ef17876717b6e59572e4081847f55b4389dc206b3a112752f20efbcab48106de7a37aec9b1ad77

data/README.md

@@ -77,3 +77,25 @@ the 422 wedge; tebako-runtime-ruby#189). A forced republication recreates the
 (`gh release delete <tag> --cleanup-tag false` … the first leg's
 race-safe create re-makes it), turning every leg's publish into pure creates.
 Operator-run re-uploads follow the same rule by hand.
+
+## Convergence and rate-limit discipline
+
+Every GitHub API call in the gem rides one internal wrapper
+(`TebakoRelease::Convergence`), with per-operation-class semantics.
+Unconditional calls (plain reads, idempotent writes) ride quota responses
+out: a 403/429 rate-limit answer is never fatal — the wrapper sleeps to the
+response's named window (`x-ratelimit-reset`, else `Retry-After`), or backs
+off exponentially from 30 s up to a 10-minute cap when the response names no
+window — and transport drops retry with escalating waits. Read-after-create
+calls poll for the fresh release object's visibility on a bounded delay
+list; delete-then-upload replaces poll the single-asset endpoint until the
+deletion is visibly absent (plus a short name-release grace) before
+re-uploading, so the 422 propagation race never wedges a leg. The patience
+is bounded twice — eight absorbed quota responses per call, and a two-window
+(~2 h) wall-clock ceiling per process — and then the leg fails with a named,
+resumable error; the metadata replace loop additionally carries a hard
+15-minute deadline. Convergence polls ride the single-asset endpoint (one
+request per poll, never a re-paginated listing), so a catalog fleet stays
+far inside the token's hourly request window. Every retry logs a stderr line
+with the attempt, the wait, and the call's target, so a ride-out is visible
+in the CI log instead of looking like a hang.

data/lib/tebako_release/convergence.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2026 [Ribose Inc](https://www.ribose.com).
+# All rights reserved.
+# This file is a part of tamatebako
+
+require "octokit"
+
+module TebakoRelease
+  # The ONE wrapper every GitHub API call in the gem rides (uploader and
+  # signer alike). A catalog run's ~90 publish/sign legs share one token's
+  # 5,000-request hourly window, and the release backend's reads are only
+  # eventually consistent against its writes — a leg must ride both
+  # physics out, bounded, and fail NAMED and resumable when a budget is
+  # genuinely spent (the 2026-10-03 catalog publish lost 64 of ~90 legs to
+  # terminal 403s; the 0.16.32 smoke wedged two legs for their whole job
+  # timeout in an unpropagated-delete cycle).
+  #
+  # The operation classes (each call site picks its own):
+  # * unconditional calls — plain GETs and idempotent writes: quota
+  #   responses ride the window out, transport drops retry, everything
+  #   else keeps its semantics (a 404/422 is information for the caller,
+  #   never retried here).
+  # * read-after-create reads — a release object a concurrent leg just
+  #   created: poll for visibility on RELEASE_VISIBILITY_DELAYS before
+  #   the named failure (the call sites own the poll loop; the rides
+  #   below keep a quota storm from burning the polls).
+  # * delete-then-upload replaces — poll the SINGLE-ASSET endpoint until
+  #   the delete is visible, pay the name-release grace, then re-upload;
+  #   the poll interval/deadline/grace live here so the uploader and the
+  #   signer share one propagation discipline.
+  #
+  # Retry lines go to stderr (CI-visible) with the attempt, the wait, and
+  # the call's target label; the narrative stream stays on stdout.
+  module Convergence
+    # A ride-out budget genuinely spent: the named, resumable failure —
+    # the leg re-run converges on quota, never on code.
+    class RateLimitBudgetExhausted < StandardError; end
+
+    # GET/DELETE/PUT calls share the same transient network failure modes;
+    # retry them (they are idempotent). A quota response is not one of
+    # those modes: it rides the window out below and never consumes the
+    # transient attempts. Transport-level drops (SSL EOF, TCP reset)
+    # belong here too: a stale keep-alive connection answered with an SSL
+    # EOF is indistinguishable from a fresh one that works — net-http
+    # retries those on EOFError but NOT on OpenSSL::SSL::SSLError, so we
+    # do (the 2026-08-30 backfill crashed on exactly that, on the
+    # verification GET right after a long upload POST).
+    TRANSIENT_ERRORS = [
+      Net::WriteTimeout, Net::ReadTimeout,
+      Faraday::TimeoutError, Faraday::ConnectionFailed,
+      OpenSSL::SSL::SSLError, EOFError, SystemCallError
+    ].freeze
+    TRANSIENT_ATTEMPTS = 4
+
+    # A quota response (403 with a rate-limit body, or a bare 429) must
+    # never kill a leg: the gem is one serialized actor per leg and
+    # sleeping until the window resets is the CORRECT behavior (the 0.16.6
+    # publish burned the tebako-ci token's hourly window in ~30 minutes
+    # and died at the finalize; run 37061404844 lost 64 legs the same
+    # way). Two bounds, whichever spends first:
+    RATE_LIMIT_SETTLE = 5
+    # The response names no window (no reset, no Retry-After — the
+    # secondary-limit shape): capped exponential backoff, doubling from
+    # 30 s to a 10-minute ceiling.
+    RATE_LIMIT_BACKOFF_BASE = 30
+    RATE_LIMIT_BACKOFF_CAP = 600
+    # Per-call quota patience: eight absorbed responses (~45 minutes of
+    # headerless backoff; header-named windows ride their own advice),
+    # then the named error — a fast red leg re-run converges for free,
+    # while a leg that keeps eating quota starves the fleet.
+    RATE_LIMIT_ATTEMPTS = 8
+    # …and the wall-clock ceiling: two full hourly windows waited in one
+    # process means something is systemically wrong — give up loudly
+    # instead of blocking the runner forever.
+    RATE_LIMIT_BUDGET = (2 * 3600) + 300
+
+    # Read-after-create visibility: a release object created by a
+    # concurrent leg (the race-safe create's loser, a sign leg ahead of
+    # its publisher) is not immediately readable — the 2026-10-02/03
+    # force_rebuild fan-out died on exactly this. Bounded polls, stretched
+    # past a short secondary-limit window the winner's own create may be
+    # riding out; each poll rides the wrapper, so a quota storm stretches
+    # the wait INSIDE the call instead of burning polls.
+    RELEASE_VISIBILITY_DELAYS = [5, 10, 15, 20, 30, 30, 30].freeze
+
+    # GitHub asset deletion is only eventually consistent: a same-name
+    # re-upload 422s until the delete propagates (the v0.16.1 windows
+    # publish lost SHA256SUMS.txt to exactly this — four retries inside
+    # ~20 s never saw the absence). Poll for the absence on the
+    # single-asset endpoint (the listing and the authoritative store
+    # replicate independently — the 0.16.32 smoke night flapped the
+    # listing while the store never moved), SLEEPING between polls, under
+    # a wall-clock deadline. The deadline matches the OBSERVED propagation
+    # window: the 2026-08-29 publish watched a deleted name stay
+    # 422-blocked well past the old 60 s. And the listing's truth frees
+    # the name BEFORE the upload validator's replica does (0.16.8: ~15 s
+    # past the visible absence), so every confirmed absence pays the grace.
+    DELETION_PROPAGATION_POLL_INTERVAL = 2
+    DELETION_PROPAGATION_DEADLINE = 180
+    DELETION_PROPAGATION_GRACE = 15
+
+    # Class (a): an unconditional call. Rate-limit responses ride out
+    # (never consuming the transient attempts); transport drops retry with
+    # escalating waits; every other error propagates to the caller's own
+    # semantics untouched.
+    def with_transient_retries(target = "GitHub call", attempts: TRANSIENT_ATTEMPTS) # rubocop:disable Metrics/MethodLength
+      budget = attempts
+      with_rate_limit_rideout(target) do
+        yield
+      rescue *TRANSIENT_ERRORS => e
+        attempts -= 1
+        raise if attempts <= 0
+
+        delay = (5 * (budget - attempts)) + rand(5)
+        warn "github #{target}: #{e.class}; retrying in #{delay}s (#{attempts} attempt(s) left)"
+        sleep delay
+        retry
+      end
+    end
+
+    # The quota ride-out, bounded twice (RATE_LIMIT_ATTEMPTS per call,
+    # RATE_LIMIT_BUDGET wall-clock per process). Octokit 7 maps a 403 with
+    # a quota body — primary and secondary limits alike — to
+    # TooManyRequests, while a bare 429 arrives as a plain ClientError;
+    # both are the quota class here, and every other client error keeps
+    # its semantics (never swallowed, never retried).
+    def with_rate_limit_rideout(target = "GitHub call") # rubocop:disable Metrics/MethodLength
+      attempts = 0
+      begin
+        yield
+      rescue Octokit::ClientError => e
+        raise unless rate_limit_error?(e)
+
+        attempts += 1
+        wait = rate_limit_wait(e, target, attempts)
+        warn "github #{target}: #{e.class} — rate-limited; sleeping #{wait}s " \
+             "(quota response #{attempts}/#{RATE_LIMIT_ATTEMPTS})"
+        sleep wait
+        retry
+      end
+    end
+
+    def rate_limit_error?(error)
+      return true if error.is_a?(Octokit::TooManyRequests)
+      return false unless error.is_a?(Octokit::ClientError)
+
+      error.response_status.to_i == 429
+    rescue NoMethodError
+      # A bare-raised error carries no response — and no quota status.
+      false
+    end
+
+    # The seconds to sleep before the next call, budget-checked: one full
+    # hourly window is a legitimate wait; a ninth absorbed quota response,
+    # or a wait that would push the process past two windows, means
+    # something is systemically wrong — give up loudly instead of
+    # blocking the runner forever.
+    def rate_limit_wait(error, target, attempts) # rubocop:disable Metrics/MethodLength
+      if attempts > RATE_LIMIT_ATTEMPTS
+        raise RateLimitBudgetExhausted,
+              "github #{target}: absorbed #{RATE_LIMIT_ATTEMPTS} rate-limit responses without the window opening — " \
+              "the per-call budget is exhausted; re-run the leg (the token's quota resets hourly)"
+      end
+
+      wait = rate_limit_seconds(error, attempts)
+      return wait if monotonic_now + wait <= rate_limit_deadline
+
+      raise RateLimitBudgetExhausted,
+            "github #{target}: the next rate-limit window is #{wait}s out but this process has a " \
+            "#{RATE_LIMIT_BUDGET}s ride-out budget — two full windows spent; " \
+            "giving up loudly instead of blocking forever"
+    end
+
+    def rate_limit_deadline
+      @rate_limit_deadline ||= monotonic_now + RATE_LIMIT_BUDGET
+    end
+
+    # The reset header names the window's end as a wall-clock epoch (plus
+    # a small settle); a stale or absent reset falls back to Retry-After,
+    # then to the capped exponential backoff. Faraday's real headers are
+    # case-insensitive; read both spellings so a plain hash (the spec
+    # fake) serves the same values.
+    def rate_limit_seconds(error, attempts) # rubocop:disable Metrics/AbcSize
+      headers = begin
+        error.response_headers || {}
+      rescue NoMethodError
+        {} # a bare-raised error carries no response headers
+      end
+      reset = (headers["x-ratelimit-reset"] || headers["X-RateLimit-Reset"]).to_i
+      return reset - Time.now.to_i + RATE_LIMIT_SETTLE if reset > Time.now.to_i
+
+      retry_after = (headers["retry-after"] || headers["Retry-After"]).to_i
+      return retry_after + RATE_LIMIT_SETTLE if retry_after.positive?
+
+      [RATE_LIMIT_BACKOFF_BASE * (2**(attempts - 1)), RATE_LIMIT_BACKOFF_CAP].min
+    end
+
+    # Wall-clock reads for the deadline accounting — monotonic, immune to
+    # clock smear on the runner.
+    def monotonic_now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/tebako_release/signer.rb

@@ -65,6 +65,12 @@ module TebakoRelease
   # declared through TebakoRelease.configure (or TEBAKO_RELEASE_REPO),
   # never by editing a copy.
   class Signer # rubocop:disable Metrics/ClassLength
+    # Every GitHub call below rides the shared convergence wrapper (quota
+    # ride-outs, transient retries, the release-visibility and
+    # deletion-propagation budgets) — tebako-release#14's sign legs died
+    # on terminal 403s mid-convergence; no call escapes the wrapper now.
+    include Convergence
+
     # Armed-but-cannot, provenance, and coverage failures: the pass never
     # ships a partially signed release silently.
     class SigningGateError < StandardError; end
@@ -97,6 +103,16 @@ module TebakoRelease
     # "no assets"); bounded re-asks, then the named failure stands.
     SERVED_BYTES_DELAYS = [5, 10, 20, 40, 80].freeze
 
+    # The young-release wordings the download re-ask loop retries (the
+    # operation class b): the name came FROM the release listing (or the
+    # leg just published it), so gh's "no assets to download" — and a fresh
+    # release object's own "release not found" (the 2026-10-02/03
+    # force_rebuild's sidecar-fetch deaths, tebako-release#13) — are the
+    # read path trailing the write, never absence. Every other named
+    # failure (auth, usage, a genuinely gone release past the budget)
+    # raises at once.
+    YOUNG_RELEASE_WORDINGS = ["no assets to download", "release not found"].freeze
+
     def initialize(client: nil, executor: nil, env: ENV, config: nil)
       @env = env
       @config = config || TebakoRelease.config
@@ -123,7 +139,7 @@ module TebakoRelease
         work = Pathname.new(dir)
         tool = fetch_verified_tool(work)
         key_file = materialize_key(work)
-        assets = @client.release_assets(release.url)
+        assets = with_transient_retries("release assets") { @client.release_assets(release.url) }
         targets = signature_targets(assets.map(&:name))
         stale = stale_targets(targets, assets)
         puts "#{@tag}: #{targets.size} signature targets, #{stale.size} need (re)signing"
@@ -136,7 +152,7 @@ module TebakoRelease
                   "signing needs the listing's sha256 to prove the signed bytes are the served bytes"
           end
 
-          sign_one(work, key_file, tool, release, name, digest)
+          sign_one(work, key_file, tool, release, name, digest, by_name["#{name}.asc"])
         end
         assert_coverage!(release, targets)
       end
@@ -207,9 +223,30 @@ module TebakoRelease
       /\Atebako-pkg-\d+\.\d+\.\d+-#{Regexp.escape(tool_host_id)}#{Regexp.escape(suffix)}\z/
     end
 
+    # Read-after-create convergence (the operation class b): on a fresh
+    # per-line release object a sign leg can schedule AHEAD of the publish
+    # leg's create becoming readable — the 2026-10-02/03 force_rebuild
+    # fan-out died on exactly this. Poll for visibility on the bounded
+    # budget (each poll rides the wrapper, so a quota burst stretches the
+    # wait inside the call), then the named failure stands.
     def find_release
-      @client.release_for_tag(@config.repo, @tag)
+      find_release_once || await_release_visibility
+    end
+
+    def find_release_once
+      with_transient_retries("release for tag #{@tag}") { @client.release_for_tag(@config.repo, @tag) }
     rescue Octokit::NotFound
+      nil
+    end
+
+    def await_release_visibility
+      RELEASE_VISIBILITY_DELAYS.each do |pause|
+        puts "#{@tag}: the release object is not visible yet (a concurrent leg just created it) — " \
+             "re-asking in #{pause}s"
+        sleep pause
+        release = find_release_once
+        return release if release
+      end
       raise SigningGateError, "NAMED FAILURE: no release found for tag #{@tag} — nothing to sign"
     end
 
@@ -234,8 +271,10 @@ module TebakoRelease
     # provenance-pinned: downloaded with its .sha256 sidecar and executed
     # only when the digest matches.
     def fetch_verified_tool(work) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-      latest = @client.latest_release(@config.tool_repo)
-      names = @client.release_assets(latest.url).map(&:name)
+      latest = with_transient_retries("latest #{@config.tool_repo} release") do
+        @client.latest_release(@config.tool_repo)
+      end
+      names = with_transient_retries("tool release assets") { @client.release_assets(latest.url) }.map(&:name)
       tool_name = names.find { |name| name.match?(tool_asset_pattern) }
       unless tool_name
         raise SigningGateError,
@@ -265,7 +304,7 @@ module TebakoRelease
     # verify against the freshly registered key, then converge the .asc onto
     # the release. The digest is the no-fold rule's provenance: the signed
     # bytes are provably the bytes the release serves.
-    def sign_one(work, key_file, tool, release, name, digest) # rubocop:disable Metrics/AbcSize, Metrics/ParameterLists
+    def sign_one(work, key_file, tool, release, name, digest, existing_asc) # rubocop:disable Metrics/AbcSize, Metrics/ParameterLists
       local = Pathname.new(local_packages_dir).join(name)
       target = if local.exist? && Digest::SHA256.file(local).hexdigest == digest
                  local
@@ -274,7 +313,7 @@ module TebakoRelease
                end
       @executor.run(tool, "sign", "--key-file", key_file.to_s, "--no-sums", name, chdir: File.dirname(target.to_s))
       @executor.run(tool, "verify", name, chdir: File.dirname(target.to_s))
-      converge_asc(release, Pathname.new(File.join(File.dirname(target.to_s), "#{name}.asc")))
+      converge_asc(release, Pathname.new(File.join(File.dirname(target.to_s), "#{name}.asc")), existing_asc)
       puts "#{name}: signed and converged"
     end
 
@@ -293,32 +332,40 @@ module TebakoRelease
             "#{name} downloaded with sha256 #{actual}, the listing says #{digest}"
     end
 
-    # The bounded re-ask for the young-release-object lag: the name came FROM
-    # the release listing, so gh's "no assets to download" is the byte store
-    # trailing the listing, never absence — it retries; every other named
-    # failure (auth, usage, a genuinely gone release) raises at once.
+    # The bounded re-ask for the young-release-object lag: the wordings
+    # above retry; everything else raises at once.
     def download_when_served(dir, name)
       pauses = SERVED_BYTES_DELAYS.dup
       begin
         @executor.run("gh", "release", "download", @tag, "--repo", @config.repo,
                       "--pattern", name, "--dir", dir.to_s, "--clobber")
       rescue SigningGateError => e
-        raise unless e.message.include?("no assets to download") && (pause = pauses.shift)
+        raise unless young_release_failure?(e) && (pause = pauses.shift)
 
-        puts "#{name} is listed but not served yet (young release object) — re-asking in #{pause}s"
+        puts "#{name}: the release read path has not converged on the fresh object yet — re-asking in #{pause}s"
         sleep pause
         retry
       end
     end
 
+    def young_release_failure?(error)
+      YOUNG_RELEASE_WORDINGS.any? { |wording| error.message.include?(wording) }
+    end
+
     # A tiny metadata upload, converged: replace whatever the name serves,
-    # then poll until the listing's digest is our bytes (the edge cache
-    # lesson of the uploader, bounded).
-    def converge_asc(release, asc_file)
+    # then poll until the served record's digest is our bytes. The poll
+    # rides the SINGLE-ASSET endpoint (one request per cycle — never a
+    # re-paginated listing: at catalog size the full listing is ~5 pages,
+    # and ~90 legs polling it every cycle is what drained the shared
+    # token's hourly budget mid-fleet, tebako-release#14). The held record
+    # flows between cycles: the pass-start listing's stale record first,
+    # then each upload's landed record.
+    def converge_asc(release, asc_file, existing) # rubocop:disable Metrics/MethodLength
       sha = Digest::SHA256.file(asc_file).hexdigest
+      asset = existing
       converged = false
       CONVERGENCE_DELAYS.each do |pause|
-        converged = asc_converged?(release, asc_file, sha)
+        converged, asset = converge_asc_cycle(release, asc_file, sha, asset)
         break if converged
 
         wait = Signer.jittered(pause)
@@ -328,30 +375,94 @@ module TebakoRelease
       raise SigningGateError, "NAMED FAILURE: #{asc_file.basename} did not converge on #{@tag}" unless converged
     end
 
-    # One convergence cycle: the listing already serving our bytes is done;
-    # anything else is deleted/replaced and re-uploaded for the next poll.
-    # A 422 mid-replace is the deletion-propagation race (the wedge
-    # lesson): the name unblocks within a cycle, so it rides along as
-    # not-yet-converged instead of crashing the pass.
-    def asc_converged?(release, asc_file, sha) # rubocop:disable Metrics/AbcSize
-      existing = @client.release_assets(release.url).find { |asset| asset.name == asc_file.basename.to_s }
-      return true if existing && listed_sha(existing) == sha
-
-      @client.delete_release_asset(existing.id) if existing
-      @client.upload_asset(release.url, asc_file.to_s,
-                           content_type: "text/plain",
-                           name: asc_file.basename.to_s)
-      false
+    # One convergence cycle: the held record already serving our bytes is
+    # done; a stale record is deleted — its absence awaited on the same
+    # single-asset endpoint before the re-upload (the delete-then-upload
+    # class, tebako-release#12's wedge lesson) — and the upload's fresh
+    # record feeds the next cycle's poll. A 422 mid-replace carrying
+    # already_exists is the deletion-propagation race (or a concurrent
+    # leg's landed upload): re-list ONCE to learn the conflicting record,
+    # then ride it as not-yet-converged. Any other 422 is a validation
+    # error, not the race — fail fast and named, never grind.
+    # Returns [converged, record-to-hold].
+    def converge_asc_cycle(release, asc_file, sha, asset) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity
+      name = asc_file.basename.to_s
+      digest = asset && served_sha(asset, name)
+      return [true, asset] if digest == sha
+
+      asset = nil if asset && digest.nil? # the held record left the store — the name is free
+      if asset
+        with_transient_retries("delete #{name}") { @client.delete_release_asset(asset.id) }
+        await_asc_absence(asset, name)
+      end
+      asset = with_transient_retries("upload #{name}") do
+        @client.upload_asset(release.url, asc_file.to_s,
+                             content_type: "text/plain",
+                             name: name)
+      end
+      [false, asset]
     rescue Octokit::UnprocessableEntity => e
-      puts "#{asc_file.basename}: replace raced the 422 propagation window (#{e.class}) — cycling"
+      unless e.message.include?("already_exists")
+        raise SigningGateError,
+              "NAMED FAILURE: the #{name} upload was rejected (#{e.message}) — not the deletion-propagation " \
+              "race; refusing to grind on a validation error"
+      end
+      puts "#{name}: replace raced the 422 propagation window (#{e.class}) — cycling"
+      [false, find_asc_asset(release, name)]
+    end
+
+    # The record's currently-served digest, read on the single-asset
+    # endpoint; nil when the record is gone (our delete propagated, or it
+    # never committed).
+    def served_sha(asset, name)
+      record = with_transient_retries("asset #{name} state") { @client.release_asset(asset.url) }
+      listed_sha(record)
+    rescue Octokit::NotFound
+      nil
+    end
+
+    # The delete is visible only when the single-asset read 404s — the
+    # listing flaps independently of the authoritative store (#12's night),
+    # so the listing is never probed here. Over the deadline the leg fails
+    # fast and named: a resumable red beats a job-timeout wedge.
+    def await_asc_absence(asset, name) # rubocop:disable Metrics/MethodLength
+      deadline = monotonic_now + DELETION_PROPAGATION_DEADLINE
+      until asc_absent?(asset)
+        if monotonic_now >= deadline
+          raise SigningGateError,
+                "NAMED FAILURE: the deletion of #{name} has not propagated within " \
+                "#{DELETION_PROPAGATION_DEADLINE}s — the asset name stays 422-blocked server-side; " \
+                "re-run the sign leg"
+        end
+
+        puts "Waiting for the deletion of #{name} to propagate..."
+        sleep DELETION_PROPAGATION_POLL_INTERVAL
+      end
+      puts "#{name} left the listing; giving the name #{DELETION_PROPAGATION_GRACE}s to free up server-side"
+      sleep DELETION_PROPAGATION_GRACE
+    end
+
+    def asc_absent?(asset)
+      with_transient_retries("asset #{asset.id} existence") { @client.release_asset(asset.url) }
       false
+    rescue Octokit::NotFound
+      true
+    end
+
+    # The recovery read behind a raced 422: ONE re-listing to learn the
+    # conflicting record (our own lagging delete or a concurrent leg's
+    # landed upload) so the next cycle's poll rides the single-asset
+    # endpoint again.
+    def find_asc_asset(release, name)
+      with_transient_retries("release assets") { @client.release_assets(release.url) }
+        .find { |asset| asset.name == name }
     end
 
     # The coverage assertion: after the pass, every target has a .asc on
     # the release — a partially signed release is a named failure, never a
     # quiet state.
     def assert_coverage!(release, targets)
-      names = @client.release_assets(release.url).map(&:name)
+      names = with_transient_retries("release assets") { @client.release_assets(release.url) }.map(&:name)
       missing = targets.reject { |name| names.include?("#{name}.asc") }
       return if missing.empty?
 

data/lib/tebako_release/uploader.rb

@@ -45,16 +45,17 @@ module TebakoRelease
   # language version key/grammar, capability gates, DLL naming — flow
   # through the Config/Adapter.
   class Uploader # rubocop:disable Metrics/ClassLength
+    # Every GitHub call below rides the shared convergence wrapper (quota
+    # ride-outs, transient retries, the release-visibility and
+    # deletion-propagation budgets; RateLimitBudgetExhausted and the
+    # RATE_LIMIT_*/DELETION_PROPAGATION_* constants resolve from there).
+    include Convergence
+
     # Named error (spec 00: named errors, never silent fallbacks): a release
     # asset's deletion did not stop the name from being listed within the
     # propagation deadline — the name stays 422-blocked server-side.
     class DeletionPropagationTimeout < StandardError; end
 
-    # The rate-limit ride-out has a bound: two full hourly windows waited in
-    # one process means something is systemically wrong — give up loudly
-    # instead of blocking the runner forever.
-    class RateLimitBudgetExhausted < StandardError; end
-
     # The era-2 release card (spec 18 C2): every runtime package carries a
     # builder-emitted `<package>.contract.yaml` sidecar (contract_era,
     # image_layout, mount_root, built_from) that manifest_entry folds into
@@ -232,7 +233,7 @@ module TebakoRelease
     def read_previous_manifest
       release = find_release
       asset = release && find_asset(release, "manifest.json")
-      asset && with_transient_retries { download_asset_json(asset) }
+      asset && with_transient_retries("previous manifest.json") { download_asset_json(asset) }
     end
 
     def deep_symbolize(value)
@@ -244,7 +245,7 @@ module TebakoRelease
     end
 
     def download_asset_json(asset)
-      JSON.parse(with_transient_retries { @client.get(asset.browser_download_url) }.to_s)
+      JSON.parse(with_transient_retries("download #{asset.name}") { @client.get(asset.browser_download_url) }.to_s)
     end
 
     def expected_package_names # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
@@ -278,28 +279,46 @@ module TebakoRelease
     # (422s resolve by content inside perform_upload), then verify what the
     # edge serves. The loop repeats until the edge converges or the budget
     # runs out — a metadata rewrite never dies on the first bad cycle.
-    # The convergence cycle sleeps, ~46 min of patience. Tonight's backend
-    # (2026-08-03) blocked a deleted name's re-upload for 4.5+ HOURS.
     # Since the de-rendezvous (spec 13 §2a) a leg rewrites only
     # its OWN packages' shards/sidecars — every name it touches is one it
     # owns, so a convergence grind can never rendezvous with another leg;
     # an incident night is an incident night, and the legs run concurrently.
     METADATA_CONVERGENCE_DELAYS = [5, 15, 30, 60, 120, 240, 480, 600, 600, 600].freeze
 
-    def force_upload(release, file, delays: METADATA_CONVERGENCE_DELAYS)
+    # The TOTAL wall-clock bound on one replace (tebako-release#12): on a
+    # pathological backend night a deleted name never frees and every
+    # re-upload 422s — each cycle's delete-wait and backoff re-arm per
+    # iteration, so without a deadline the loop grinds into the CI job
+    # timeout (the 0.16.32 smoke burned both macos legs' full 60 minutes
+    # on one shard). Fifteen minutes covers a healthy night's propagation
+    # (the 0.16.8 wedge freed in ~15 s; 2026-08-03 took minutes); past it
+    # the replace cannot land tonight and the named error says so — a fast
+    # red leg is resumable for free, a job-timeout death is not.
+    METADATA_CONVERGENCE_DEADLINE = 900
+
+    def force_upload(release, file, delays: METADATA_CONVERGENCE_DELAYS) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       filename = file.basename.to_s
       sha = Digest::SHA256.file(file).hexdigest
+      deadline = monotonic_now + METADATA_CONVERGENCE_DEADLINE
       converged = false
       delays.each do |pause|
         converged = metadata_converged?(release, file, filename, sha)
         break if converged
 
+        raise metadata_deadline_error(filename) if monotonic_now >= deadline
+
         puts "#{filename} has not converged on the release yet; cycling in #{pause}s"
-        sleep pause
+        sleep [pause, deadline - monotonic_now].min
       end
       raise "could not converge #{filename} on the release within the metadata budget" unless converged
     end
 
+    def metadata_deadline_error(filename)
+      "could not converge #{filename} on the release within the #{METADATA_CONVERGENCE_DEADLINE}s " \
+        "convergence deadline — the release backend is wedged on this name tonight (an unpropagated " \
+        "delete holds it 422-blocked); re-run this leg, a healthy backend converges in one cycle"
+    end
+
     # One convergence cycle: read-first (the edge already serves our bytes
     # → done), replace when listed, upload with the short budget (422s
     # resolve by content inside perform_upload), then verify what the edge
@@ -353,7 +372,7 @@ module TebakoRelease
     # correct accept (content is content); absent/stale bytes only mean
     # "take the mutation path".
     def served_content(filename)
-      with_transient_retries { @client.get(download_url(filename)) }.to_s
+      with_transient_retries("served bytes #{filename}") { @client.get(download_url(filename)) }.to_s
     rescue Octokit::NotFound
       nil
     end
@@ -401,27 +420,32 @@ module TebakoRelease
     # matrix legs of one publish run race here (spec 13 §2a's
     # de-rendezvous: no leg waits on another). The loser's create 422s on
     # the taken tag; it polls for the winner's release to become visible
-    # and rides it. Creation is write-once, never a wedge.
-    RELEASE_CREATE_POLL_DELAYS = [5, 5, 5, 5, 5, 5].freeze
-
+    # and rides it (RELEASE_VISIBILITY_DELAYS — each poll rides the
+    # wrapper, so a quota burst stretches the wait inside the call instead
+    # of burning polls). Creation is write-once, never a wedge.
     def get_or_create_release # rubocop:disable Naming/AccessorMethodName
       puts "Looking for release with tag: #{@tag}"
-      @client.release_for_tag(@config.repo, @tag)
-    rescue Octokit::NotFound
-      create_release_race_safe
+      find_release || create_release_race_safe
     end
 
     # A rescue clause's own exceptions never re-enter the sibling rescues —
-    # the create's 422 handling lives in its own method.
+    # the create's 422 handling lives in its own method. The POST itself
+    # rides the wrapper: a quota response mid-fan-out (the 2026-10-03
+    # fleet's terminal `POST /releases: 403`s) sleeps the window out, and
+    # a timed-out create that LANDED server-side 422s the retry — the
+    # race-safe poll below then rides the landed release, exactly the
+    # lost-race shape.
     def create_release_race_safe # rubocop:disable Metrics/MethodLength
       puts "Creating new release for tag: #{@tag}"
-      @client.create_release(@config.repo, @tag,
-                             name: @release_title,
-                             body: release_notes)
+      with_transient_retries("create release #{@tag}") do
+        @client.create_release(@config.repo, @tag,
+                               name: @release_title,
+                               body: release_notes)
+      end
     rescue Octokit::UnprocessableEntity
       # A concurrent leg won the create. Poll for its release to become
       # visible, then ride it — never re-attempt the create.
-      RELEASE_CREATE_POLL_DELAYS.each do |pause|
+      RELEASE_VISIBILITY_DELAYS.each do |pause|
         sleep pause
         release = find_release
         return release if release
@@ -433,7 +457,7 @@ module TebakoRelease
     # The read-only lookup (the previous-entry reads + the audit read the
     # existing release): nil when the tag has no release, never creates one.
     def find_release
-      with_transient_retries { @client.release_for_tag(@config.repo, @tag) }
+      with_transient_retries("release for tag #{@tag}") { @client.release_for_tag(@config.repo, @tag) }
     rescue Octokit::NotFound
       nil
     end
@@ -765,7 +789,7 @@ module TebakoRelease
       end
 
       puts "#{filename}: the landed asset's content disagrees — deleting the partial/stale asset before the retry"
-      with_transient_retries { @client.delete_release_asset(asset.id) }
+      with_transient_retries("delete #{filename}") { @client.delete_release_asset(asset.id) }
       drop_asset_from_memo(asset.id) # the deleted asset leaves the listing
       # The name stays 422-blocked until the delete propagates — poll for
       # the absence so the retry's POST actually lands (the v0.16.3 gnu
@@ -813,7 +837,7 @@ module TebakoRelease
 
     def landed_content(release, filename)
       asset = find_asset(release, filename)
-      return with_transient_retries { @client.get(asset.browser_download_url) }.to_s if asset
+      return with_transient_retries("download #{filename}") { @client.get(asset.browser_download_url) }.to_s if asset
 
       served_content(filename)
     rescue Octokit::NotFound
@@ -831,7 +855,7 @@ module TebakoRelease
     # 2026-08-30) stay with perform_upload (escalating delays, 422-by-content
     # resolution, the per-asset budget) — ONE retry layer, not two.
     def upload_once(release, package, filename)
-      asset = with_rate_limit_rideout do
+      asset = with_rate_limit_rideout("upload #{filename}") do
         @client.upload_asset(release.url, package.to_s,
                              content_type: "application/octet-stream",
                              name: filename)
@@ -1118,7 +1142,8 @@ module TebakoRelease
     end
 
     def download_shard_entry(asset)
-      deep_symbolize(JSON.parse(with_transient_retries { @client.get(asset.browser_download_url) }.to_s))
+      body = with_transient_retries("shard #{asset.name}") { @client.get(asset.browser_download_url) }.to_s
+      deep_symbolize(JSON.parse(body))
     end
 
     # The package stems a set of entries covers (exe + .tfs/.dll facets).
@@ -1413,35 +1438,19 @@ module TebakoRelease
       existing = find_asset(release, filename)
       return unless existing
 
-      with_transient_retries { @client.delete_release_asset(existing.id) }
+      with_transient_retries("delete #{filename}") { @client.delete_release_asset(existing.id) }
       drop_asset_from_memo(existing.id) # the deleted asset leaves the listing
       wait_for_absence_best_effort(release, filename, existing)
     end
 
     # GitHub asset deletion is only eventually consistent: a same-name
-    # re-upload 422s until the delete propagates (the v0.16.1 windows
-    # publish lost SHA256SUMS.txt to exactly this — four retries inside
-    # ~20 s never saw the absence). Poll for the absence, SLEEPING between
-    # polls, under an overall wall-clock deadline; when propagation outlasts
-    # the deadline the named error fires — the wait never spins and never
-    # silently gives up (the 2026-08-20 publish burned its whole job timeout
-    # on a wait whose outcome nobody could act on).
-    # The deadline matches the OBSERVED propagation window: the 2026-08-29
-    # publish watched a deleted name stay 422-blocked well past the old
-    # 60 s — past the per-asset 300 s budget once. Three minutes of polling
-    # (single-asset reads, never a re-listing) covers the incident class; a
-    # name still held after that is wedged, and the caller's
-    # retry/convergence budget rides it out.
-    DELETION_PROPAGATION_POLL_INTERVAL = 2
-    DELETION_PROPAGATION_DEADLINE = 180
-    # The listing's truth frees the name BEFORE the upload validator's
-    # replica does: on the 0.16.8 publish a same-name POST kept 422ing
-    # (Validation Failed / code: already_exists / field: name) ~15 s PAST
-    # the deletion's visible absence — the manual repair that worked was
-    # "gone, grace, then upload". Every confirmed absence pays this grace
-    # before the next POST.
-    DELETION_PROPAGATION_GRACE = 15
-
+    # re-upload 422s until the delete propagates. Poll for the absence,
+    # SLEEPING between polls, under an overall wall-clock deadline; when
+    # propagation outlasts the deadline the named error fires — the wait
+    # never spins and never silently gives up (the 2026-08-20 publish
+    # burned its whole job timeout on a wait whose outcome nobody could
+    # act on). The poll interval, deadline, and grace constants live in
+    # Convergence — the signer shares the discipline.
     def wait_for_absence(release, filename, asset, deadline: DELETION_PROPAGATION_DEADLINE) # rubocop:disable Metrics/MethodLength
       started = monotonic_now
       loop do
@@ -1474,7 +1483,9 @@ module TebakoRelease
     # request window mid-publish. The real API always hands listed assets an
     # api url; the fallback rebuilds it from the release url.
     def asset_deleted?(release, asset)
-      with_transient_retries { @client.release_asset(asset.url || "#{release.url}/assets/#{asset.id}") }
+      with_transient_retries("asset #{asset.id} existence") do
+        @client.release_asset(asset.url || "#{release.url}/assets/#{asset.id}")
+      end
       false
     rescue Octokit::NotFound
       true
@@ -1490,12 +1501,6 @@ module TebakoRelease
       puts "::warning::#{e.message}"
     end
 
-    # Wall-clock reads for the deadline accounting — monotonic, immune to
-    # clock smear on the runner.
-    def monotonic_now
-      Process.clock_gettime(Process::CLOCK_MONOTONIC)
-    end
-
     # release.assets is an embedded array capped at 30 entries, and a raw
     # rels[:assets].get is NOT auto-paginated either (auto_paginate covers
     # client methods, not Sawyer rel gets) — with 100+ assets most lookups
@@ -1509,10 +1514,10 @@ module TebakoRelease
     # exist within a run.
     def all_assets(release)
       @all_assets ||= begin
-        page = with_transient_retries { release.rels[:assets].get }
+        page = with_transient_retries("asset listing") { release.rels[:assets].get }
         assets = page.data
         while (nxt = page.rels[:next])
-          page = with_transient_retries { nxt.get }
+          page = with_transient_retries("asset listing page") { nxt.get }
           assets += page.data
         end
         assets
@@ -1533,89 +1538,6 @@ module TebakoRelease
       all_assets(release).find { |a| a.name == filename }
     end
 
-    # GET/DELETE/PUT calls other than the asset upload share the same
-    # transient network failure modes; retry them (they are idempotent).
-    # A 403 rate-limit response is not one of those modes: it rides the
-    # window out and never consumes the transient attempts. Transport-level
-    # drops (SSL EOF, TCP reset) belong here too: a stale keep-alive
-    # connection answered with an SSL EOF is indistinguishable from a fresh
-    # one that works — net-http retries those on EOFError but NOT on
-    # OpenSSL::SSL::SSLError, so we do (the 2026-08-30 backfill crashed on
-    # exactly that, on the verification GET right after a long upload POST).
-    TRANSIENT_ERRORS = [
-      Net::WriteTimeout, Net::ReadTimeout,
-      Faraday::TimeoutError, Faraday::ConnectionFailed,
-      OpenSSL::SSL::SSLError, EOFError, SystemCallError
-    ].freeze
-
-    def with_transient_retries(attempts: 4)
-      with_rate_limit_rideout do
-        yield
-      rescue *TRANSIENT_ERRORS => e
-        attempts -= 1
-        raise if attempts <= 0
-
-        delay = (5 * (4 - attempts)) + rand(5)
-        puts "#{e.class}; retrying in #{delay}s (#{attempts} attempt(s) left)"
-        sleep delay
-        retry
-      end
-    end
-
-    # A 403 rate-limit response must never kill the publish: the uploader
-    # is one serialized actor, and sleeping until the window resets is the
-    # CORRECT behavior (the 0.16.6 publish burned the tebako-ci token's
-    # 5000-request hourly window in ~30 minutes and died at the finalize —
-    # GET .../assets 403 — after all 334 payload assets had landed). The
-    # upload POST rides this out too; its transient retries (escalating
-    # delays, 422-by-content resolution, the per-asset budget) stay with
-    # perform_upload.
-    RATE_LIMIT_SETTLE = 5
-    RATE_LIMIT_DEFAULT_WAIT = 60
-    RATE_LIMIT_BUDGET = (2 * 3600) + 300
-
-    def with_rate_limit_rideout
-      yield
-    rescue Octokit::TooManyRequests => e
-      wait = rate_limit_wait(e)
-      puts "#{e.class}; rate-limited — sleeping #{wait}s until the window resets"
-      sleep wait
-      retry
-    end
-
-    # The seconds to sleep before the next call, budget-checked: one full
-    # hourly window is a legitimate wait; a wait that would push the process
-    # past two windows means something is systemically wrong — give up
-    # loudly instead of blocking the runner forever.
-    def rate_limit_wait(error)
-      wait = rate_limit_seconds(error)
-      return wait if monotonic_now + wait <= rate_limit_deadline
-
-      raise RateLimitBudgetExhausted,
-            "the next GitHub rate-limit window is #{wait}s out but this publish has a #{RATE_LIMIT_BUDGET}s " \
-            "ride-out budget — two full windows spent; giving up loudly instead of blocking forever"
-    end
-
-    def rate_limit_deadline
-      @rate_limit_deadline ||= monotonic_now + RATE_LIMIT_BUDGET
-    end
-
-    # The reset header names the window's end as a wall-clock epoch (plus a
-    # small settle); a stale or absent reset falls back to Retry-After,
-    # then to a default minute. Faraday's real headers are case-insensitive;
-    # read both spellings so a plain hash (the spec fake) serves the same
-    # values.
-    def rate_limit_seconds(error)
-      headers = error.response_headers || {}
-      reset = (headers["x-ratelimit-reset"] || headers["X-RateLimit-Reset"]).to_i
-      return reset - Time.now.to_i + RATE_LIMIT_SETTLE if reset > Time.now.to_i
-
-      retry_after = (headers["retry-after"] || headers["Retry-After"]).to_i
-      return retry_after + RATE_LIMIT_SETTLE if retry_after.positive?
-
-      RATE_LIMIT_DEFAULT_WAIT
-    end
-
     def run
       process_release
     rescue StandardError => e
@@ -1745,7 +1667,7 @@ module TebakoRelease
         raise Error, "the served image #{image_name} is not on the release — cannot derive its blksum sidecar"
       end
 
-      bytes = with_transient_retries { @client.get(asset.browser_download_url) }.to_s
+      bytes = with_transient_retries("download #{image_name}") { @client.get(asset.browser_download_url) }.to_s
       Pathname.new(Dir.mktmpdir).join(image_name).tap { |path| path.binwrite(bytes) }
     end
 
@@ -1790,7 +1712,8 @@ module TebakoRelease
     end
 
     def extract_bundle(_release, bundle_asset)
-      bytes = with_transient_retries { @client.get(bundle_asset.browser_download_url) }.to_s
+      url = bundle_asset.browser_download_url
+      bytes = with_transient_retries("bundle #{bundle_asset.name}") { @client.get(url) }.to_s
       dir = Pathname.new(Dir.mktmpdir("tebako-release-bundle-"))
       Zlib::GzipReader.wrap(StringIO.new(bytes)) { |gzip| extract_tar_members(bundle_asset, gzip, dir) }
     end

data/lib/tebako_release/version.rb

@@ -5,5 +5,5 @@
 # This file is a part of tamatebako
 
 module TebakoRelease
-  VERSION = "0.5.0"
+  VERSION = "0.5.1"
 end

data/lib/tebako_release.rb

@@ -11,6 +11,7 @@ require_relative "tebako_release/config"
 require_relative "tebako_release/platform"
 require_relative "tebako_release/bundler"
 require_relative "tebako_release/blksum"
+require_relative "tebako_release/convergence"
 require_relative "tebako_release/uploader"
 require_relative "tebako_release/signer"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tebako-release
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Ribose Inc
@@ -42,6 +42,7 @@ files:
 - lib/tebako_release/blksum.rb
 - lib/tebako_release/bundler.rb
 - lib/tebako_release/config.rb
+- lib/tebako_release/convergence.rb
 - lib/tebako_release/error.rb
 - lib/tebako_release/platform.rb
 - lib/tebako_release/signer.rb