tebako-release 0.4.2 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +22 -0
- data/lib/tebako_release/adapter.rb +10 -0
- data/lib/tebako_release/convergence.rb +205 -0
- data/lib/tebako_release/signer.rb +144 -33
- data/lib/tebako_release/uploader.rb +103 -153
- data/lib/tebako_release/version.rb +1 -1
- data/lib/tebako_release.rb +1 -0
- metadata +2 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: cc98dded830844b0afdda4fdba2ede81155f424bb58eaab89601a9588e0251c2
|
|
4
|
+
data.tar.gz: c471ef97c8de46dc8ff5d8dfb5868a20288cf41aace0a91af8d9efbc5dbbfe2a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 68ba9ef2fde5b4f6fa9522673f2e9480b87fe9ceb2fb6a39a72560bd7511f90c3d3f7e44ac5772fe32aac8c24cdd54205bce2a4e955eb8eb304907cbc9607d84
|
|
7
|
+
data.tar.gz: b1e26cefc882eb129067448ce5eb5dd2cd8320675bfd336c49ef17876717b6e59572e4081847f55b4389dc206b3a112752f20efbcab48106de7a37aec9b1ad77
|
data/README.md
CHANGED
|
@@ -77,3 +77,25 @@ the 422 wedge; tebako-runtime-ruby#189). A forced republication recreates the
|
|
|
77
77
|
(`gh release delete <tag> --cleanup-tag false` … the first leg's
|
|
78
78
|
race-safe create re-makes it), turning every leg's publish into pure creates.
|
|
79
79
|
Operator-run re-uploads follow the same rule by hand.
|
|
80
|
+
|
|
81
|
+
## Convergence and rate-limit discipline
|
|
82
|
+
|
|
83
|
+
Every GitHub API call in the gem rides one internal wrapper
|
|
84
|
+
(`TebakoRelease::Convergence`), with per-operation-class semantics.
|
|
85
|
+
Unconditional calls (plain reads, idempotent writes) ride quota responses
|
|
86
|
+
out: a 403/429 rate-limit answer is never fatal — the wrapper sleeps to the
|
|
87
|
+
response's named window (`x-ratelimit-reset`, else `Retry-After`), or backs
|
|
88
|
+
off exponentially from 30 s up to a 10-minute cap when the response names no
|
|
89
|
+
window — and transport drops retry with escalating waits. Read-after-create
|
|
90
|
+
calls poll for the fresh release object's visibility on a bounded delay
|
|
91
|
+
list; delete-then-upload replaces poll the single-asset endpoint until the
|
|
92
|
+
deletion is visibly absent (plus a short name-release grace) before
|
|
93
|
+
re-uploading, so the 422 propagation race never wedges a leg. The patience
|
|
94
|
+
is bounded twice — eight absorbed quota responses per call, and a two-window
|
|
95
|
+
(~2 h) wall-clock ceiling per process — and then the leg fails with a named,
|
|
96
|
+
resumable error; the metadata replace loop additionally carries a hard
|
|
97
|
+
15-minute deadline. Convergence polls ride the single-asset endpoint (one
|
|
98
|
+
request per poll, never a re-paginated listing), so a catalog fleet stays
|
|
99
|
+
far inside the token's hourly request window. Every retry logs a stderr line
|
|
100
|
+
with the attempt, the wait, and the call's target, so a ride-out is visible
|
|
101
|
+
in the CI log instead of looking like a hang.
|
|
@@ -47,6 +47,16 @@ module TebakoRelease
|
|
|
47
47
|
"\\d+\\.\\d+\\.\\d+"
|
|
48
48
|
end
|
|
49
49
|
|
|
50
|
+
# The language segment of the package-name grammar
|
|
51
|
+
# (`tebako-runtime-<ver>-<lang>-<lv>-<triplet>` — tebako#716):
|
|
52
|
+
# declared by factories publishing the post-#716 spelling. Default
|
|
53
|
+
# nil — the pre-#716 spelling carried no language segment, and this
|
|
54
|
+
# machinery keeps composing/parsing it for the immutable releases
|
|
55
|
+
# that already carry it.
|
|
56
|
+
def lang_name
|
|
57
|
+
nil
|
|
58
|
+
end
|
|
59
|
+
|
|
50
60
|
# Spec 36's publish shape: when true, each leg publishes ONE bundle
|
|
51
61
|
# (<stem>.tar.gz — exe + env image + DLLs + in-bundle SHA256SUMS) plus
|
|
52
62
|
# its sidecar and shard, instead of the per-file asset enumeration.
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Copyright (c) 2026 [Ribose Inc](https://www.ribose.com).
|
|
4
|
+
# All rights reserved.
|
|
5
|
+
# This file is a part of tamatebako
|
|
6
|
+
|
|
7
|
+
require "octokit"
|
|
8
|
+
|
|
9
|
+
module TebakoRelease
|
|
10
|
+
# The ONE wrapper every GitHub API call in the gem rides (uploader and
|
|
11
|
+
# signer alike). A catalog run's ~90 publish/sign legs share one token's
|
|
12
|
+
# 5,000-request hourly window, and the release backend's reads are only
|
|
13
|
+
# eventually consistent against its writes — a leg must ride both
|
|
14
|
+
# physics out, bounded, and fail NAMED and resumable when a budget is
|
|
15
|
+
# genuinely spent (the 2026-10-03 catalog publish lost 64 of ~90 legs to
|
|
16
|
+
# terminal 403s; the 0.16.32 smoke wedged two legs for their whole job
|
|
17
|
+
# timeout in an unpropagated-delete cycle).
|
|
18
|
+
#
|
|
19
|
+
# The operation classes (each call site picks its own):
|
|
20
|
+
# * unconditional calls — plain GETs and idempotent writes: quota
|
|
21
|
+
# responses ride the window out, transport drops retry, everything
|
|
22
|
+
# else keeps its semantics (a 404/422 is information for the caller,
|
|
23
|
+
# never retried here).
|
|
24
|
+
# * read-after-create reads — a release object a concurrent leg just
|
|
25
|
+
# created: poll for visibility on RELEASE_VISIBILITY_DELAYS before
|
|
26
|
+
# the named failure (the call sites own the poll loop; the rides
|
|
27
|
+
# below keep a quota storm from burning the polls).
|
|
28
|
+
# * delete-then-upload replaces — poll the SINGLE-ASSET endpoint until
|
|
29
|
+
# the delete is visible, pay the name-release grace, then re-upload;
|
|
30
|
+
# the poll interval/deadline/grace live here so the uploader and the
|
|
31
|
+
# signer share one propagation discipline.
|
|
32
|
+
#
|
|
33
|
+
# Retry lines go to stderr (CI-visible) with the attempt, the wait, and
|
|
34
|
+
# the call's target label; the narrative stream stays on stdout.
|
|
35
|
+
module Convergence
|
|
36
|
+
# A ride-out budget genuinely spent: the named, resumable failure —
|
|
37
|
+
# the leg re-run converges on quota, never on code.
|
|
38
|
+
class RateLimitBudgetExhausted < StandardError; end
|
|
39
|
+
|
|
40
|
+
# GET/DELETE/PUT calls share the same transient network failure modes;
|
|
41
|
+
# retry them (they are idempotent). A quota response is not one of
|
|
42
|
+
# those modes: it rides the window out below and never consumes the
|
|
43
|
+
# transient attempts. Transport-level drops (SSL EOF, TCP reset)
|
|
44
|
+
# belong here too: a stale keep-alive connection answered with an SSL
|
|
45
|
+
# EOF is indistinguishable from a fresh one that works — net-http
|
|
46
|
+
# retries those on EOFError but NOT on OpenSSL::SSL::SSLError, so we
|
|
47
|
+
# do (the 2026-08-30 backfill crashed on exactly that, on the
|
|
48
|
+
# verification GET right after a long upload POST).
|
|
49
|
+
TRANSIENT_ERRORS = [
|
|
50
|
+
Net::WriteTimeout, Net::ReadTimeout,
|
|
51
|
+
Faraday::TimeoutError, Faraday::ConnectionFailed,
|
|
52
|
+
OpenSSL::SSL::SSLError, EOFError, SystemCallError
|
|
53
|
+
].freeze
|
|
54
|
+
TRANSIENT_ATTEMPTS = 4
|
|
55
|
+
|
|
56
|
+
# A quota response (403 with a rate-limit body, or a bare 429) must
|
|
57
|
+
# never kill a leg: the gem is one serialized actor per leg and
|
|
58
|
+
# sleeping until the window resets is the CORRECT behavior (the 0.16.6
|
|
59
|
+
# publish burned the tebako-ci token's hourly window in ~30 minutes
|
|
60
|
+
# and died at the finalize; run 37061404844 lost 64 legs the same
|
|
61
|
+
# way). Two bounds, whichever spends first:
|
|
62
|
+
RATE_LIMIT_SETTLE = 5
|
|
63
|
+
# The response names no window (no reset, no Retry-After — the
|
|
64
|
+
# secondary-limit shape): capped exponential backoff, doubling from
|
|
65
|
+
# 30 s to a 10-minute ceiling.
|
|
66
|
+
RATE_LIMIT_BACKOFF_BASE = 30
|
|
67
|
+
RATE_LIMIT_BACKOFF_CAP = 600
|
|
68
|
+
# Per-call quota patience: eight absorbed responses (~45 minutes of
|
|
69
|
+
# headerless backoff; header-named windows ride their own advice),
|
|
70
|
+
# then the named error — a fast red leg re-run converges for free,
|
|
71
|
+
# while a leg that keeps eating quota starves the fleet.
|
|
72
|
+
RATE_LIMIT_ATTEMPTS = 8
|
|
73
|
+
# …and the wall-clock ceiling: two full hourly windows waited in one
|
|
74
|
+
# process means something is systemically wrong — give up loudly
|
|
75
|
+
# instead of blocking the runner forever.
|
|
76
|
+
RATE_LIMIT_BUDGET = (2 * 3600) + 300
|
|
77
|
+
|
|
78
|
+
# Read-after-create visibility: a release object created by a
|
|
79
|
+
# concurrent leg (the race-safe create's loser, a sign leg ahead of
|
|
80
|
+
# its publisher) is not immediately readable — the 2026-10-02/03
|
|
81
|
+
# force_rebuild fan-out died on exactly this. Bounded polls, stretched
|
|
82
|
+
# past a short secondary-limit window the winner's own create may be
|
|
83
|
+
# riding out; each poll rides the wrapper, so a quota storm stretches
|
|
84
|
+
# the wait INSIDE the call instead of burning polls.
|
|
85
|
+
RELEASE_VISIBILITY_DELAYS = [5, 10, 15, 20, 30, 30, 30].freeze
|
|
86
|
+
|
|
87
|
+
# GitHub asset deletion is only eventually consistent: a same-name
|
|
88
|
+
# re-upload 422s until the delete propagates (the v0.16.1 windows
|
|
89
|
+
# publish lost SHA256SUMS.txt to exactly this — four retries inside
|
|
90
|
+
# ~20 s never saw the absence). Poll for the absence on the
|
|
91
|
+
# single-asset endpoint (the listing and the authoritative store
|
|
92
|
+
# replicate independently — the 0.16.32 smoke night flapped the
|
|
93
|
+
# listing while the store never moved), SLEEPING between polls, under
|
|
94
|
+
# a wall-clock deadline. The deadline matches the OBSERVED propagation
|
|
95
|
+
# window: the 2026-08-29 publish watched a deleted name stay
|
|
96
|
+
# 422-blocked well past the old 60 s. And the listing's truth frees
|
|
97
|
+
# the name BEFORE the upload validator's replica does (0.16.8: ~15 s
|
|
98
|
+
# past the visible absence), so every confirmed absence pays the grace.
|
|
99
|
+
DELETION_PROPAGATION_POLL_INTERVAL = 2
|
|
100
|
+
DELETION_PROPAGATION_DEADLINE = 180
|
|
101
|
+
DELETION_PROPAGATION_GRACE = 15
|
|
102
|
+
|
|
103
|
+
# Class (a): an unconditional call. Rate-limit responses ride out
|
|
104
|
+
# (never consuming the transient attempts); transport drops retry with
|
|
105
|
+
# escalating waits; every other error propagates to the caller's own
|
|
106
|
+
# semantics untouched.
|
|
107
|
+
def with_transient_retries(target = "GitHub call", attempts: TRANSIENT_ATTEMPTS) # rubocop:disable Metrics/MethodLength
|
|
108
|
+
budget = attempts
|
|
109
|
+
with_rate_limit_rideout(target) do
|
|
110
|
+
yield
|
|
111
|
+
rescue *TRANSIENT_ERRORS => e
|
|
112
|
+
attempts -= 1
|
|
113
|
+
raise if attempts <= 0
|
|
114
|
+
|
|
115
|
+
delay = (5 * (budget - attempts)) + rand(5)
|
|
116
|
+
warn "github #{target}: #{e.class}; retrying in #{delay}s (#{attempts} attempt(s) left)"
|
|
117
|
+
sleep delay
|
|
118
|
+
retry
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# The quota ride-out, bounded twice (RATE_LIMIT_ATTEMPTS per call,
|
|
123
|
+
# RATE_LIMIT_BUDGET wall-clock per process). Octokit 7 maps a 403 with
|
|
124
|
+
# a quota body — primary and secondary limits alike — to
|
|
125
|
+
# TooManyRequests, while a bare 429 arrives as a plain ClientError;
|
|
126
|
+
# both are the quota class here, and every other client error keeps
|
|
127
|
+
# its semantics (never swallowed, never retried).
|
|
128
|
+
def with_rate_limit_rideout(target = "GitHub call") # rubocop:disable Metrics/MethodLength
|
|
129
|
+
attempts = 0
|
|
130
|
+
begin
|
|
131
|
+
yield
|
|
132
|
+
rescue Octokit::ClientError => e
|
|
133
|
+
raise unless rate_limit_error?(e)
|
|
134
|
+
|
|
135
|
+
attempts += 1
|
|
136
|
+
wait = rate_limit_wait(e, target, attempts)
|
|
137
|
+
warn "github #{target}: #{e.class} — rate-limited; sleeping #{wait}s " \
|
|
138
|
+
"(quota response #{attempts}/#{RATE_LIMIT_ATTEMPTS})"
|
|
139
|
+
sleep wait
|
|
140
|
+
retry
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def rate_limit_error?(error)
|
|
145
|
+
return true if error.is_a?(Octokit::TooManyRequests)
|
|
146
|
+
return false unless error.is_a?(Octokit::ClientError)
|
|
147
|
+
|
|
148
|
+
error.response_status.to_i == 429
|
|
149
|
+
rescue NoMethodError
|
|
150
|
+
# A bare-raised error carries no response — and no quota status.
|
|
151
|
+
false
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# The seconds to sleep before the next call, budget-checked: one full
|
|
155
|
+
# hourly window is a legitimate wait; a ninth absorbed quota response,
|
|
156
|
+
# or a wait that would push the process past two windows, means
|
|
157
|
+
# something is systemically wrong — give up loudly instead of
|
|
158
|
+
# blocking the runner forever.
|
|
159
|
+
def rate_limit_wait(error, target, attempts) # rubocop:disable Metrics/MethodLength
|
|
160
|
+
if attempts > RATE_LIMIT_ATTEMPTS
|
|
161
|
+
raise RateLimitBudgetExhausted,
|
|
162
|
+
"github #{target}: absorbed #{RATE_LIMIT_ATTEMPTS} rate-limit responses without the window opening — " \
|
|
163
|
+
"the per-call budget is exhausted; re-run the leg (the token's quota resets hourly)"
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
wait = rate_limit_seconds(error, attempts)
|
|
167
|
+
return wait if monotonic_now + wait <= rate_limit_deadline
|
|
168
|
+
|
|
169
|
+
raise RateLimitBudgetExhausted,
|
|
170
|
+
"github #{target}: the next rate-limit window is #{wait}s out but this process has a " \
|
|
171
|
+
"#{RATE_LIMIT_BUDGET}s ride-out budget — two full windows spent; " \
|
|
172
|
+
"giving up loudly instead of blocking forever"
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def rate_limit_deadline
|
|
176
|
+
@rate_limit_deadline ||= monotonic_now + RATE_LIMIT_BUDGET
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
# The reset header names the window's end as a wall-clock epoch (plus
|
|
180
|
+
# a small settle); a stale or absent reset falls back to Retry-After,
|
|
181
|
+
# then to the capped exponential backoff. Faraday's real headers are
|
|
182
|
+
# case-insensitive; read both spellings so a plain hash (the spec
|
|
183
|
+
# fake) serves the same values.
|
|
184
|
+
def rate_limit_seconds(error, attempts) # rubocop:disable Metrics/AbcSize
|
|
185
|
+
headers = begin
|
|
186
|
+
error.response_headers || {}
|
|
187
|
+
rescue NoMethodError
|
|
188
|
+
{} # a bare-raised error carries no response headers
|
|
189
|
+
end
|
|
190
|
+
reset = (headers["x-ratelimit-reset"] || headers["X-RateLimit-Reset"]).to_i
|
|
191
|
+
return reset - Time.now.to_i + RATE_LIMIT_SETTLE if reset > Time.now.to_i
|
|
192
|
+
|
|
193
|
+
retry_after = (headers["retry-after"] || headers["Retry-After"]).to_i
|
|
194
|
+
return retry_after + RATE_LIMIT_SETTLE if retry_after.positive?
|
|
195
|
+
|
|
196
|
+
[RATE_LIMIT_BACKOFF_BASE * (2**(attempts - 1)), RATE_LIMIT_BACKOFF_CAP].min
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
# Wall-clock reads for the deadline accounting — monotonic, immune to
|
|
200
|
+
# clock smear on the runner.
|
|
201
|
+
def monotonic_now
|
|
202
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
end
|
|
@@ -65,6 +65,12 @@ module TebakoRelease
|
|
|
65
65
|
# declared through TebakoRelease.configure (or TEBAKO_RELEASE_REPO),
|
|
66
66
|
# never by editing a copy.
|
|
67
67
|
class Signer # rubocop:disable Metrics/ClassLength
|
|
68
|
+
# Every GitHub call below rides the shared convergence wrapper (quota
|
|
69
|
+
# ride-outs, transient retries, the release-visibility and
|
|
70
|
+
# deletion-propagation budgets) — tebako-release#14's sign legs died
|
|
71
|
+
# on terminal 403s mid-convergence; no call escapes the wrapper now.
|
|
72
|
+
include Convergence
|
|
73
|
+
|
|
68
74
|
# Armed-but-cannot, provenance, and coverage failures: the pass never
|
|
69
75
|
# ships a partially signed release silently.
|
|
70
76
|
class SigningGateError < StandardError; end
|
|
@@ -97,6 +103,16 @@ module TebakoRelease
|
|
|
97
103
|
# "no assets"); bounded re-asks, then the named failure stands.
|
|
98
104
|
SERVED_BYTES_DELAYS = [5, 10, 20, 40, 80].freeze
|
|
99
105
|
|
|
106
|
+
# The young-release wordings the download re-ask loop retries (the
|
|
107
|
+
# operation class b): the name came FROM the release listing (or the
|
|
108
|
+
# leg just published it), so gh's "no assets to download" — and a fresh
|
|
109
|
+
# release object's own "release not found" (the 2026-10-02/03
|
|
110
|
+
# force_rebuild's sidecar-fetch deaths, tebako-release#13) — are the
|
|
111
|
+
# read path trailing the write, never absence. Every other named
|
|
112
|
+
# failure (auth, usage, a genuinely gone release past the budget)
|
|
113
|
+
# raises at once.
|
|
114
|
+
YOUNG_RELEASE_WORDINGS = ["no assets to download", "release not found"].freeze
|
|
115
|
+
|
|
100
116
|
def initialize(client: nil, executor: nil, env: ENV, config: nil)
|
|
101
117
|
@env = env
|
|
102
118
|
@config = config || TebakoRelease.config
|
|
@@ -123,7 +139,7 @@ module TebakoRelease
|
|
|
123
139
|
work = Pathname.new(dir)
|
|
124
140
|
tool = fetch_verified_tool(work)
|
|
125
141
|
key_file = materialize_key(work)
|
|
126
|
-
assets = @client.release_assets(release.url)
|
|
142
|
+
assets = with_transient_retries("release assets") { @client.release_assets(release.url) }
|
|
127
143
|
targets = signature_targets(assets.map(&:name))
|
|
128
144
|
stale = stale_targets(targets, assets)
|
|
129
145
|
puts "#{@tag}: #{targets.size} signature targets, #{stale.size} need (re)signing"
|
|
@@ -136,7 +152,7 @@ module TebakoRelease
|
|
|
136
152
|
"signing needs the listing's sha256 to prove the signed bytes are the served bytes"
|
|
137
153
|
end
|
|
138
154
|
|
|
139
|
-
sign_one(work, key_file, tool, release, name, digest)
|
|
155
|
+
sign_one(work, key_file, tool, release, name, digest, by_name["#{name}.asc"])
|
|
140
156
|
end
|
|
141
157
|
assert_coverage!(release, targets)
|
|
142
158
|
end
|
|
@@ -207,9 +223,30 @@ module TebakoRelease
|
|
|
207
223
|
/\Atebako-pkg-\d+\.\d+\.\d+-#{Regexp.escape(tool_host_id)}#{Regexp.escape(suffix)}\z/
|
|
208
224
|
end
|
|
209
225
|
|
|
226
|
+
# Read-after-create convergence (the operation class b): on a fresh
|
|
227
|
+
# per-line release object a sign leg can schedule AHEAD of the publish
|
|
228
|
+
# leg's create becoming readable — the 2026-10-02/03 force_rebuild
|
|
229
|
+
# fan-out died on exactly this. Poll for visibility on the bounded
|
|
230
|
+
# budget (each poll rides the wrapper, so a quota burst stretches the
|
|
231
|
+
# wait inside the call), then the named failure stands.
|
|
210
232
|
def find_release
|
|
211
|
-
|
|
233
|
+
find_release_once || await_release_visibility
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
def find_release_once
|
|
237
|
+
with_transient_retries("release for tag #{@tag}") { @client.release_for_tag(@config.repo, @tag) }
|
|
212
238
|
rescue Octokit::NotFound
|
|
239
|
+
nil
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
def await_release_visibility
|
|
243
|
+
RELEASE_VISIBILITY_DELAYS.each do |pause|
|
|
244
|
+
puts "#{@tag}: the release object is not visible yet (a concurrent leg just created it) — " \
|
|
245
|
+
"re-asking in #{pause}s"
|
|
246
|
+
sleep pause
|
|
247
|
+
release = find_release_once
|
|
248
|
+
return release if release
|
|
249
|
+
end
|
|
213
250
|
raise SigningGateError, "NAMED FAILURE: no release found for tag #{@tag} — nothing to sign"
|
|
214
251
|
end
|
|
215
252
|
|
|
@@ -234,8 +271,10 @@ module TebakoRelease
|
|
|
234
271
|
# provenance-pinned: downloaded with its .sha256 sidecar and executed
|
|
235
272
|
# only when the digest matches.
|
|
236
273
|
def fetch_verified_tool(work) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
237
|
-
latest =
|
|
238
|
-
|
|
274
|
+
latest = with_transient_retries("latest #{@config.tool_repo} release") do
|
|
275
|
+
@client.latest_release(@config.tool_repo)
|
|
276
|
+
end
|
|
277
|
+
names = with_transient_retries("tool release assets") { @client.release_assets(latest.url) }.map(&:name)
|
|
239
278
|
tool_name = names.find { |name| name.match?(tool_asset_pattern) }
|
|
240
279
|
unless tool_name
|
|
241
280
|
raise SigningGateError,
|
|
@@ -265,7 +304,7 @@ module TebakoRelease
|
|
|
265
304
|
# verify against the freshly registered key, then converge the .asc onto
|
|
266
305
|
# the release. The digest is the no-fold rule's provenance: the signed
|
|
267
306
|
# bytes are provably the bytes the release serves.
|
|
268
|
-
def sign_one(work, key_file, tool, release, name, digest) # rubocop:disable Metrics/AbcSize, Metrics/ParameterLists
|
|
307
|
+
def sign_one(work, key_file, tool, release, name, digest, existing_asc) # rubocop:disable Metrics/AbcSize, Metrics/ParameterLists
|
|
269
308
|
local = Pathname.new(local_packages_dir).join(name)
|
|
270
309
|
target = if local.exist? && Digest::SHA256.file(local).hexdigest == digest
|
|
271
310
|
local
|
|
@@ -274,7 +313,7 @@ module TebakoRelease
|
|
|
274
313
|
end
|
|
275
314
|
@executor.run(tool, "sign", "--key-file", key_file.to_s, "--no-sums", name, chdir: File.dirname(target.to_s))
|
|
276
315
|
@executor.run(tool, "verify", name, chdir: File.dirname(target.to_s))
|
|
277
|
-
converge_asc(release, Pathname.new(File.join(File.dirname(target.to_s), "#{name}.asc")))
|
|
316
|
+
converge_asc(release, Pathname.new(File.join(File.dirname(target.to_s), "#{name}.asc")), existing_asc)
|
|
278
317
|
puts "#{name}: signed and converged"
|
|
279
318
|
end
|
|
280
319
|
|
|
@@ -293,32 +332,40 @@ module TebakoRelease
|
|
|
293
332
|
"#{name} downloaded with sha256 #{actual}, the listing says #{digest}"
|
|
294
333
|
end
|
|
295
334
|
|
|
296
|
-
# The bounded re-ask for the young-release-object lag: the
|
|
297
|
-
#
|
|
298
|
-
# trailing the listing, never absence — it retries; every other named
|
|
299
|
-
# failure (auth, usage, a genuinely gone release) raises at once.
|
|
335
|
+
# The bounded re-ask for the young-release-object lag: the wordings
|
|
336
|
+
# above retry; everything else raises at once.
|
|
300
337
|
def download_when_served(dir, name)
|
|
301
338
|
pauses = SERVED_BYTES_DELAYS.dup
|
|
302
339
|
begin
|
|
303
340
|
@executor.run("gh", "release", "download", @tag, "--repo", @config.repo,
|
|
304
341
|
"--pattern", name, "--dir", dir.to_s, "--clobber")
|
|
305
342
|
rescue SigningGateError => e
|
|
306
|
-
raise unless
|
|
343
|
+
raise unless young_release_failure?(e) && (pause = pauses.shift)
|
|
307
344
|
|
|
308
|
-
puts "#{name}
|
|
345
|
+
puts "#{name}: the release read path has not converged on the fresh object yet — re-asking in #{pause}s"
|
|
309
346
|
sleep pause
|
|
310
347
|
retry
|
|
311
348
|
end
|
|
312
349
|
end
|
|
313
350
|
|
|
351
|
+
def young_release_failure?(error)
|
|
352
|
+
YOUNG_RELEASE_WORDINGS.any? { |wording| error.message.include?(wording) }
|
|
353
|
+
end
|
|
354
|
+
|
|
314
355
|
# A tiny metadata upload, converged: replace whatever the name serves,
|
|
315
|
-
# then poll until the
|
|
316
|
-
#
|
|
317
|
-
|
|
356
|
+
# then poll until the served record's digest is our bytes. The poll
|
|
357
|
+
# rides the SINGLE-ASSET endpoint (one request per cycle — never a
|
|
358
|
+
# re-paginated listing: at catalog size the full listing is ~5 pages,
|
|
359
|
+
# and ~90 legs polling it every cycle is what drained the shared
|
|
360
|
+
# token's hourly budget mid-fleet, tebako-release#14). The held record
|
|
361
|
+
# flows between cycles: the pass-start listing's stale record first,
|
|
362
|
+
# then each upload's landed record.
|
|
363
|
+
def converge_asc(release, asc_file, existing) # rubocop:disable Metrics/MethodLength
|
|
318
364
|
sha = Digest::SHA256.file(asc_file).hexdigest
|
|
365
|
+
asset = existing
|
|
319
366
|
converged = false
|
|
320
367
|
CONVERGENCE_DELAYS.each do |pause|
|
|
321
|
-
converged =
|
|
368
|
+
converged, asset = converge_asc_cycle(release, asc_file, sha, asset)
|
|
322
369
|
break if converged
|
|
323
370
|
|
|
324
371
|
wait = Signer.jittered(pause)
|
|
@@ -328,30 +375,94 @@ module TebakoRelease
|
|
|
328
375
|
raise SigningGateError, "NAMED FAILURE: #{asc_file.basename} did not converge on #{@tag}" unless converged
|
|
329
376
|
end
|
|
330
377
|
|
|
331
|
-
# One convergence cycle: the
|
|
332
|
-
#
|
|
333
|
-
#
|
|
334
|
-
#
|
|
335
|
-
#
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
378
|
+
# One convergence cycle: the held record already serving our bytes is
|
|
379
|
+
# done; a stale record is deleted — its absence awaited on the same
|
|
380
|
+
# single-asset endpoint before the re-upload (the delete-then-upload
|
|
381
|
+
# class, tebako-release#12's wedge lesson) — and the upload's fresh
|
|
382
|
+
# record feeds the next cycle's poll. A 422 mid-replace carrying
|
|
383
|
+
# already_exists is the deletion-propagation race (or a concurrent
|
|
384
|
+
# leg's landed upload): re-list ONCE to learn the conflicting record,
|
|
385
|
+
# then ride it as not-yet-converged. Any other 422 is a validation
|
|
386
|
+
# error, not the race — fail fast and named, never grind.
|
|
387
|
+
# Returns [converged, record-to-hold].
|
|
388
|
+
def converge_asc_cycle(release, asc_file, sha, asset) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity
|
|
389
|
+
name = asc_file.basename.to_s
|
|
390
|
+
digest = asset && served_sha(asset, name)
|
|
391
|
+
return [true, asset] if digest == sha
|
|
392
|
+
|
|
393
|
+
asset = nil if asset && digest.nil? # the held record left the store — the name is free
|
|
394
|
+
if asset
|
|
395
|
+
with_transient_retries("delete #{name}") { @client.delete_release_asset(asset.id) }
|
|
396
|
+
await_asc_absence(asset, name)
|
|
397
|
+
end
|
|
398
|
+
asset = with_transient_retries("upload #{name}") do
|
|
399
|
+
@client.upload_asset(release.url, asc_file.to_s,
|
|
400
|
+
content_type: "text/plain",
|
|
401
|
+
name: name)
|
|
402
|
+
end
|
|
403
|
+
[false, asset]
|
|
345
404
|
rescue Octokit::UnprocessableEntity => e
|
|
346
|
-
|
|
405
|
+
unless e.message.include?("already_exists")
|
|
406
|
+
raise SigningGateError,
|
|
407
|
+
"NAMED FAILURE: the #{name} upload was rejected (#{e.message}) — not the deletion-propagation " \
|
|
408
|
+
"race; refusing to grind on a validation error"
|
|
409
|
+
end
|
|
410
|
+
puts "#{name}: replace raced the 422 propagation window (#{e.class}) — cycling"
|
|
411
|
+
[false, find_asc_asset(release, name)]
|
|
412
|
+
end
|
|
413
|
+
|
|
414
|
+
# The record's currently-served digest, read on the single-asset
|
|
415
|
+
# endpoint; nil when the record is gone (our delete propagated, or it
|
|
416
|
+
# never committed).
|
|
417
|
+
def served_sha(asset, name)
|
|
418
|
+
record = with_transient_retries("asset #{name} state") { @client.release_asset(asset.url) }
|
|
419
|
+
listed_sha(record)
|
|
420
|
+
rescue Octokit::NotFound
|
|
421
|
+
nil
|
|
422
|
+
end
|
|
423
|
+
|
|
424
|
+
# The delete is visible only when the single-asset read 404s — the
|
|
425
|
+
# listing flaps independently of the authoritative store (#12's night),
|
|
426
|
+
# so the listing is never probed here. Over the deadline the leg fails
|
|
427
|
+
# fast and named: a resumable red beats a job-timeout wedge.
|
|
428
|
+
def await_asc_absence(asset, name) # rubocop:disable Metrics/MethodLength
|
|
429
|
+
deadline = monotonic_now + DELETION_PROPAGATION_DEADLINE
|
|
430
|
+
until asc_absent?(asset)
|
|
431
|
+
if monotonic_now >= deadline
|
|
432
|
+
raise SigningGateError,
|
|
433
|
+
"NAMED FAILURE: the deletion of #{name} has not propagated within " \
|
|
434
|
+
"#{DELETION_PROPAGATION_DEADLINE}s — the asset name stays 422-blocked server-side; " \
|
|
435
|
+
"re-run the sign leg"
|
|
436
|
+
end
|
|
437
|
+
|
|
438
|
+
puts "Waiting for the deletion of #{name} to propagate..."
|
|
439
|
+
sleep DELETION_PROPAGATION_POLL_INTERVAL
|
|
440
|
+
end
|
|
441
|
+
puts "#{name} left the listing; giving the name #{DELETION_PROPAGATION_GRACE}s to free up server-side"
|
|
442
|
+
sleep DELETION_PROPAGATION_GRACE
|
|
443
|
+
end
|
|
444
|
+
|
|
445
|
+
def asc_absent?(asset)
|
|
446
|
+
with_transient_retries("asset #{asset.id} existence") { @client.release_asset(asset.url) }
|
|
347
447
|
false
|
|
448
|
+
rescue Octokit::NotFound
|
|
449
|
+
true
|
|
450
|
+
end
|
|
451
|
+
|
|
452
|
+
# The recovery read behind a raced 422: ONE re-listing to learn the
|
|
453
|
+
# conflicting record (our own lagging delete or a concurrent leg's
|
|
454
|
+
# landed upload) so the next cycle's poll rides the single-asset
|
|
455
|
+
# endpoint again.
|
|
456
|
+
def find_asc_asset(release, name)
|
|
457
|
+
with_transient_retries("release assets") { @client.release_assets(release.url) }
|
|
458
|
+
.find { |asset| asset.name == name }
|
|
348
459
|
end
|
|
349
460
|
|
|
350
461
|
# The coverage assertion: after the pass, every target has a .asc on
|
|
351
462
|
# the release — a partially signed release is a named failure, never a
|
|
352
463
|
# quiet state.
|
|
353
464
|
def assert_coverage!(release, targets)
|
|
354
|
-
names = @client.release_assets(release.url).map(&:name)
|
|
465
|
+
names = with_transient_retries("release assets") { @client.release_assets(release.url) }.map(&:name)
|
|
355
466
|
missing = targets.reject { |name| names.include?("#{name}.asc") }
|
|
356
467
|
return if missing.empty?
|
|
357
468
|
|
|
@@ -45,16 +45,17 @@ module TebakoRelease
|
|
|
45
45
|
# language version key/grammar, capability gates, DLL naming — flow
|
|
46
46
|
# through the Config/Adapter.
|
|
47
47
|
class Uploader # rubocop:disable Metrics/ClassLength
|
|
48
|
+
# Every GitHub call below rides the shared convergence wrapper (quota
|
|
49
|
+
# ride-outs, transient retries, the release-visibility and
|
|
50
|
+
# deletion-propagation budgets; RateLimitBudgetExhausted and the
|
|
51
|
+
# RATE_LIMIT_*/DELETION_PROPAGATION_* constants resolve from there).
|
|
52
|
+
include Convergence
|
|
53
|
+
|
|
48
54
|
# Named error (spec 00: named errors, never silent fallbacks): a release
|
|
49
55
|
# asset's deletion did not stop the name from being listed within the
|
|
50
56
|
# propagation deadline — the name stays 422-blocked server-side.
|
|
51
57
|
class DeletionPropagationTimeout < StandardError; end
|
|
52
58
|
|
|
53
|
-
# The rate-limit ride-out has a bound: two full hourly windows waited in
|
|
54
|
-
# one process means something is systemically wrong — give up loudly
|
|
55
|
-
# instead of blocking the runner forever.
|
|
56
|
-
class RateLimitBudgetExhausted < StandardError; end
|
|
57
|
-
|
|
58
59
|
# The era-2 release card (spec 18 C2): every runtime package carries a
|
|
59
60
|
# builder-emitted `<package>.contract.yaml` sidecar (contract_era,
|
|
60
61
|
# image_layout, mount_root, built_from) that manifest_entry folds into
|
|
@@ -232,7 +233,7 @@ module TebakoRelease
|
|
|
232
233
|
def read_previous_manifest
|
|
233
234
|
release = find_release
|
|
234
235
|
asset = release && find_asset(release, "manifest.json")
|
|
235
|
-
asset && with_transient_retries { download_asset_json(asset) }
|
|
236
|
+
asset && with_transient_retries("previous manifest.json") { download_asset_json(asset) }
|
|
236
237
|
end
|
|
237
238
|
|
|
238
239
|
def deep_symbolize(value)
|
|
@@ -244,7 +245,7 @@ module TebakoRelease
|
|
|
244
245
|
end
|
|
245
246
|
|
|
246
247
|
def download_asset_json(asset)
|
|
247
|
-
JSON.parse(with_transient_retries { @client.get(asset.browser_download_url) }.to_s)
|
|
248
|
+
JSON.parse(with_transient_retries("download #{asset.name}") { @client.get(asset.browser_download_url) }.to_s)
|
|
248
249
|
end
|
|
249
250
|
|
|
250
251
|
def expected_package_names # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
@@ -256,7 +257,7 @@ module TebakoRelease
|
|
|
256
257
|
next unless @config.adapter.capable_pair?(env["os"], env["arch"], version)
|
|
257
258
|
|
|
258
259
|
platform = Platform.host_id_for(env["os"], env["arch"])
|
|
259
|
-
"tebako-runtime-#{@version}-#{version}-#{platform}"
|
|
260
|
+
"tebako-runtime-#{@version}-#{lang_infix}#{version}-#{platform}"
|
|
260
261
|
end
|
|
261
262
|
rescue JSON::ParserError => e
|
|
262
263
|
puts "::warning::Could not compute expected package list: #{e.message}"
|
|
@@ -278,28 +279,46 @@ module TebakoRelease
|
|
|
278
279
|
# (422s resolve by content inside perform_upload), then verify what the
|
|
279
280
|
# edge serves. The loop repeats until the edge converges or the budget
|
|
280
281
|
# runs out — a metadata rewrite never dies on the first bad cycle.
|
|
281
|
-
# The convergence cycle sleeps, ~46 min of patience. Tonight's backend
|
|
282
|
-
# (2026-08-03) blocked a deleted name's re-upload for 4.5+ HOURS.
|
|
283
282
|
# Since the de-rendezvous (spec 13 §2a) a leg rewrites only
|
|
284
283
|
# its OWN packages' shards/sidecars — every name it touches is one it
|
|
285
284
|
# owns, so a convergence grind can never rendezvous with another leg;
|
|
286
285
|
# an incident night is an incident night, and the legs run concurrently.
|
|
287
286
|
METADATA_CONVERGENCE_DELAYS = [5, 15, 30, 60, 120, 240, 480, 600, 600, 600].freeze
|
|
288
287
|
|
|
289
|
-
|
|
288
|
+
# The TOTAL wall-clock bound on one replace (tebako-release#12): on a
|
|
289
|
+
# pathological backend night a deleted name never frees and every
|
|
290
|
+
# re-upload 422s — each cycle's delete-wait and backoff re-arm per
|
|
291
|
+
# iteration, so without a deadline the loop grinds into the CI job
|
|
292
|
+
# timeout (the 0.16.32 smoke burned both macos legs' full 60 minutes
|
|
293
|
+
# on one shard). Fifteen minutes covers a healthy night's propagation
|
|
294
|
+
# (the 0.16.8 wedge freed in ~15 s; 2026-08-03 took minutes); past it
|
|
295
|
+
# the replace cannot land tonight and the named error says so — a fast
|
|
296
|
+
# red leg is resumable for free, a job-timeout death is not.
|
|
297
|
+
METADATA_CONVERGENCE_DEADLINE = 900
|
|
298
|
+
|
|
299
|
+
def force_upload(release, file, delays: METADATA_CONVERGENCE_DELAYS) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
290
300
|
filename = file.basename.to_s
|
|
291
301
|
sha = Digest::SHA256.file(file).hexdigest
|
|
302
|
+
deadline = monotonic_now + METADATA_CONVERGENCE_DEADLINE
|
|
292
303
|
converged = false
|
|
293
304
|
delays.each do |pause|
|
|
294
305
|
converged = metadata_converged?(release, file, filename, sha)
|
|
295
306
|
break if converged
|
|
296
307
|
|
|
308
|
+
raise metadata_deadline_error(filename) if monotonic_now >= deadline
|
|
309
|
+
|
|
297
310
|
puts "#{filename} has not converged on the release yet; cycling in #{pause}s"
|
|
298
|
-
sleep pause
|
|
311
|
+
sleep [pause, deadline - monotonic_now].min
|
|
299
312
|
end
|
|
300
313
|
raise "could not converge #{filename} on the release within the metadata budget" unless converged
|
|
301
314
|
end
|
|
302
315
|
|
|
316
|
+
def metadata_deadline_error(filename)
|
|
317
|
+
"could not converge #{filename} on the release within the #{METADATA_CONVERGENCE_DEADLINE}s " \
|
|
318
|
+
"convergence deadline — the release backend is wedged on this name tonight (an unpropagated " \
|
|
319
|
+
"delete holds it 422-blocked); re-run this leg, a healthy backend converges in one cycle"
|
|
320
|
+
end
|
|
321
|
+
|
|
303
322
|
# One convergence cycle: read-first (the edge already serves our bytes
|
|
304
323
|
# → done), replace when listed, upload with the short budget (422s
|
|
305
324
|
# resolve by content inside perform_upload), then verify what the edge
|
|
@@ -353,7 +372,7 @@ module TebakoRelease
|
|
|
353
372
|
# correct accept (content is content); absent/stale bytes only mean
|
|
354
373
|
# "take the mutation path".
|
|
355
374
|
def served_content(filename)
|
|
356
|
-
with_transient_retries { @client.get(download_url(filename)) }.to_s
|
|
375
|
+
with_transient_retries("served bytes #{filename}") { @client.get(download_url(filename)) }.to_s
|
|
357
376
|
rescue Octokit::NotFound
|
|
358
377
|
nil
|
|
359
378
|
end
|
|
@@ -401,27 +420,32 @@ module TebakoRelease
|
|
|
401
420
|
# matrix legs of one publish run race here (spec 13 §2a's
|
|
402
421
|
# de-rendezvous: no leg waits on another). The loser's create 422s on
|
|
403
422
|
# the taken tag; it polls for the winner's release to become visible
|
|
404
|
-
# and rides it
|
|
405
|
-
|
|
406
|
-
|
|
423
|
+
# and rides it (RELEASE_VISIBILITY_DELAYS — each poll rides the
|
|
424
|
+
# wrapper, so a quota burst stretches the wait inside the call instead
|
|
425
|
+
# of burning polls). Creation is write-once, never a wedge.
|
|
407
426
|
def get_or_create_release # rubocop:disable Naming/AccessorMethodName
|
|
408
427
|
puts "Looking for release with tag: #{@tag}"
|
|
409
|
-
|
|
410
|
-
rescue Octokit::NotFound
|
|
411
|
-
create_release_race_safe
|
|
428
|
+
find_release || create_release_race_safe
|
|
412
429
|
end
|
|
413
430
|
|
|
414
431
|
# A rescue clause's own exceptions never re-enter the sibling rescues —
|
|
415
|
-
# the create's 422 handling lives in its own method.
|
|
432
|
+
# the create's 422 handling lives in its own method. The POST itself
|
|
433
|
+
# rides the wrapper: a quota response mid-fan-out (the 2026-10-03
|
|
434
|
+
# fleet's terminal `POST /releases: 403`s) sleeps the window out, and
|
|
435
|
+
# a timed-out create that LANDED server-side 422s the retry — the
|
|
436
|
+
# race-safe poll below then rides the landed release, exactly the
|
|
437
|
+
# lost-race shape.
|
|
416
438
|
def create_release_race_safe # rubocop:disable Metrics/MethodLength
|
|
417
439
|
puts "Creating new release for tag: #{@tag}"
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
440
|
+
with_transient_retries("create release #{@tag}") do
|
|
441
|
+
@client.create_release(@config.repo, @tag,
|
|
442
|
+
name: @release_title,
|
|
443
|
+
body: release_notes)
|
|
444
|
+
end
|
|
421
445
|
rescue Octokit::UnprocessableEntity
|
|
422
446
|
# A concurrent leg won the create. Poll for its release to become
|
|
423
447
|
# visible, then ride it — never re-attempt the create.
|
|
424
|
-
|
|
448
|
+
RELEASE_VISIBILITY_DELAYS.each do |pause|
|
|
425
449
|
sleep pause
|
|
426
450
|
release = find_release
|
|
427
451
|
return release if release
|
|
@@ -433,7 +457,7 @@ module TebakoRelease
|
|
|
433
457
|
# The read-only lookup (the previous-entry reads + the audit read the
|
|
434
458
|
# existing release): nil when the tag has no release, never creates one.
|
|
435
459
|
def find_release
|
|
436
|
-
with_transient_retries { @client.release_for_tag(@config.repo, @tag) }
|
|
460
|
+
with_transient_retries("release for tag #{@tag}") { @client.release_for_tag(@config.repo, @tag) }
|
|
437
461
|
rescue Octokit::NotFound
|
|
438
462
|
nil
|
|
439
463
|
end
|
|
@@ -656,15 +680,42 @@ module TebakoRelease
|
|
|
656
680
|
"#{package.basename.to_s.sub(/\.exe\z/, "")}.dll"
|
|
657
681
|
end
|
|
658
682
|
|
|
683
|
+
# The language infix of the package-name grammar: "ruby-" on
|
|
684
|
+
# post-#716 publishes, empty on the pre-#716 spelling.
|
|
685
|
+
def lang_infix
|
|
686
|
+
name = @config.adapter.lang_name
|
|
687
|
+
name.nil? || name.empty? ? "" : "#{name}-"
|
|
688
|
+
end
|
|
689
|
+
|
|
690
|
+
# The dual-era package-name parse (tebako#716): the pre-#716 spelling
|
|
691
|
+
# is `tebako-runtime-<ver>-<lv>-<triplet>`; the post-#716 spelling
|
|
692
|
+
# inserts the factory's language segment. A present segment that
|
|
693
|
+
# disagrees with the adapter's lang_name is a foreign file, never
|
|
694
|
+
# this factory's package.
|
|
659
695
|
def parse_package_filename(filename)
|
|
660
696
|
grammar = @config.adapter.version_grammar_source
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
end
|
|
697
|
+
pattern = /\Atebako-runtime-#{Regexp.escape(@version)}-(?:([a-z0-9]+)-)?(#{grammar})-(.+?)(?:\.exe)?\z/
|
|
698
|
+
match = pattern.match(filename)
|
|
699
|
+
return warn_unparseable_name(filename) unless match
|
|
700
|
+
return [match[2], match[3]] unless foreign_lang_segment?(match[1])
|
|
666
701
|
|
|
667
|
-
|
|
702
|
+
warn_foreign_segment(filename, match[1])
|
|
703
|
+
end
|
|
704
|
+
|
|
705
|
+
# The optional language capture vs the adapter's declaration.
|
|
706
|
+
def foreign_lang_segment?(lang)
|
|
707
|
+
lang && @config.adapter.lang_name && lang != @config.adapter.lang_name
|
|
708
|
+
end
|
|
709
|
+
|
|
710
|
+
def warn_unparseable_name(filename)
|
|
711
|
+
puts "::warning::Cannot infer version/platform from package filename: #{filename}"
|
|
712
|
+
[nil, nil]
|
|
713
|
+
end
|
|
714
|
+
|
|
715
|
+
def warn_foreign_segment(filename, lang)
|
|
716
|
+
puts "::warning::package filename carries a foreign language segment " \
|
|
717
|
+
"(#{lang}, this factory is #{@config.adapter.lang_name}): #{filename}"
|
|
718
|
+
[nil, nil]
|
|
668
719
|
end
|
|
669
720
|
|
|
670
721
|
# The upload retry budget: escalating delays, ~4.5 minutes of patience.
|
|
@@ -738,7 +789,7 @@ module TebakoRelease
|
|
|
738
789
|
end
|
|
739
790
|
|
|
740
791
|
puts "#{filename}: the landed asset's content disagrees — deleting the partial/stale asset before the retry"
|
|
741
|
-
with_transient_retries { @client.delete_release_asset(asset.id) }
|
|
792
|
+
with_transient_retries("delete #{filename}") { @client.delete_release_asset(asset.id) }
|
|
742
793
|
drop_asset_from_memo(asset.id) # the deleted asset leaves the listing
|
|
743
794
|
# The name stays 422-blocked until the delete propagates — poll for
|
|
744
795
|
# the absence so the retry's POST actually lands (the v0.16.3 gnu
|
|
@@ -786,7 +837,7 @@ module TebakoRelease
|
|
|
786
837
|
|
|
787
838
|
def landed_content(release, filename)
|
|
788
839
|
asset = find_asset(release, filename)
|
|
789
|
-
return with_transient_retries { @client.get(asset.browser_download_url) }.to_s if asset
|
|
840
|
+
return with_transient_retries("download #{filename}") { @client.get(asset.browser_download_url) }.to_s if asset
|
|
790
841
|
|
|
791
842
|
served_content(filename)
|
|
792
843
|
rescue Octokit::NotFound
|
|
@@ -804,7 +855,7 @@ module TebakoRelease
|
|
|
804
855
|
# 2026-08-30) stay with perform_upload (escalating delays, 422-by-content
|
|
805
856
|
# resolution, the per-asset budget) — ONE retry layer, not two.
|
|
806
857
|
def upload_once(release, package, filename)
|
|
807
|
-
asset = with_rate_limit_rideout do
|
|
858
|
+
asset = with_rate_limit_rideout("upload #{filename}") do
|
|
808
859
|
@client.upload_asset(release.url, package.to_s,
|
|
809
860
|
content_type: "application/octet-stream",
|
|
810
861
|
name: filename)
|
|
@@ -1091,7 +1142,8 @@ module TebakoRelease
|
|
|
1091
1142
|
end
|
|
1092
1143
|
|
|
1093
1144
|
def download_shard_entry(asset)
|
|
1094
|
-
|
|
1145
|
+
body = with_transient_retries("shard #{asset.name}") { @client.get(asset.browser_download_url) }.to_s
|
|
1146
|
+
deep_symbolize(JSON.parse(body))
|
|
1095
1147
|
end
|
|
1096
1148
|
|
|
1097
1149
|
# The package stems a set of entries covers (exe + .tfs/.dll facets).
|
|
@@ -1386,35 +1438,19 @@ module TebakoRelease
|
|
|
1386
1438
|
existing = find_asset(release, filename)
|
|
1387
1439
|
return unless existing
|
|
1388
1440
|
|
|
1389
|
-
with_transient_retries { @client.delete_release_asset(existing.id) }
|
|
1441
|
+
with_transient_retries("delete #{filename}") { @client.delete_release_asset(existing.id) }
|
|
1390
1442
|
drop_asset_from_memo(existing.id) # the deleted asset leaves the listing
|
|
1391
1443
|
wait_for_absence_best_effort(release, filename, existing)
|
|
1392
1444
|
end
|
|
1393
1445
|
|
|
1394
1446
|
# GitHub asset deletion is only eventually consistent: a same-name
|
|
1395
|
-
# re-upload 422s until the delete propagates
|
|
1396
|
-
#
|
|
1397
|
-
#
|
|
1398
|
-
#
|
|
1399
|
-
#
|
|
1400
|
-
#
|
|
1401
|
-
#
|
|
1402
|
-
# The deadline matches the OBSERVED propagation window: the 2026-08-29
|
|
1403
|
-
# publish watched a deleted name stay 422-blocked well past the old
|
|
1404
|
-
# 60 s — past the per-asset 300 s budget once. Three minutes of polling
|
|
1405
|
-
# (single-asset reads, never a re-listing) covers the incident class; a
|
|
1406
|
-
# name still held after that is wedged, and the caller's
|
|
1407
|
-
# retry/convergence budget rides it out.
|
|
1408
|
-
DELETION_PROPAGATION_POLL_INTERVAL = 2
|
|
1409
|
-
DELETION_PROPAGATION_DEADLINE = 180
|
|
1410
|
-
# The listing's truth frees the name BEFORE the upload validator's
|
|
1411
|
-
# replica does: on the 0.16.8 publish a same-name POST kept 422ing
|
|
1412
|
-
# (Validation Failed / code: already_exists / field: name) ~15 s PAST
|
|
1413
|
-
# the deletion's visible absence — the manual repair that worked was
|
|
1414
|
-
# "gone, grace, then upload". Every confirmed absence pays this grace
|
|
1415
|
-
# before the next POST.
|
|
1416
|
-
DELETION_PROPAGATION_GRACE = 15
|
|
1417
|
-
|
|
1447
|
+
# re-upload 422s until the delete propagates. Poll for the absence,
|
|
1448
|
+
# SLEEPING between polls, under an overall wall-clock deadline; when
|
|
1449
|
+
# propagation outlasts the deadline the named error fires — the wait
|
|
1450
|
+
# never spins and never silently gives up (the 2026-08-20 publish
|
|
1451
|
+
# burned its whole job timeout on a wait whose outcome nobody could
|
|
1452
|
+
# act on). The poll interval, deadline, and grace constants live in
|
|
1453
|
+
# Convergence — the signer shares the discipline.
|
|
1418
1454
|
def wait_for_absence(release, filename, asset, deadline: DELETION_PROPAGATION_DEADLINE) # rubocop:disable Metrics/MethodLength
|
|
1419
1455
|
started = monotonic_now
|
|
1420
1456
|
loop do
|
|
@@ -1447,7 +1483,9 @@ module TebakoRelease
|
|
|
1447
1483
|
# request window mid-publish. The real API always hands listed assets an
|
|
1448
1484
|
# api url; the fallback rebuilds it from the release url.
|
|
1449
1485
|
def asset_deleted?(release, asset)
|
|
1450
|
-
with_transient_retries
|
|
1486
|
+
with_transient_retries("asset #{asset.id} existence") do
|
|
1487
|
+
@client.release_asset(asset.url || "#{release.url}/assets/#{asset.id}")
|
|
1488
|
+
end
|
|
1451
1489
|
false
|
|
1452
1490
|
rescue Octokit::NotFound
|
|
1453
1491
|
true
|
|
@@ -1463,12 +1501,6 @@ module TebakoRelease
|
|
|
1463
1501
|
puts "::warning::#{e.message}"
|
|
1464
1502
|
end
|
|
1465
1503
|
|
|
1466
|
-
# Wall-clock reads for the deadline accounting — monotonic, immune to
|
|
1467
|
-
# clock smear on the runner.
|
|
1468
|
-
def monotonic_now
|
|
1469
|
-
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
1470
|
-
end
|
|
1471
|
-
|
|
1472
1504
|
# release.assets is an embedded array capped at 30 entries, and a raw
|
|
1473
1505
|
# rels[:assets].get is NOT auto-paginated either (auto_paginate covers
|
|
1474
1506
|
# client methods, not Sawyer rel gets) — with 100+ assets most lookups
|
|
@@ -1482,10 +1514,10 @@ module TebakoRelease
|
|
|
1482
1514
|
# exist within a run.
|
|
1483
1515
|
def all_assets(release)
|
|
1484
1516
|
@all_assets ||= begin
|
|
1485
|
-
page = with_transient_retries { release.rels[:assets].get }
|
|
1517
|
+
page = with_transient_retries("asset listing") { release.rels[:assets].get }
|
|
1486
1518
|
assets = page.data
|
|
1487
1519
|
while (nxt = page.rels[:next])
|
|
1488
|
-
page = with_transient_retries { nxt.get }
|
|
1520
|
+
page = with_transient_retries("asset listing page") { nxt.get }
|
|
1489
1521
|
assets += page.data
|
|
1490
1522
|
end
|
|
1491
1523
|
assets
|
|
@@ -1506,89 +1538,6 @@ module TebakoRelease
|
|
|
1506
1538
|
all_assets(release).find { |a| a.name == filename }
|
|
1507
1539
|
end
|
|
1508
1540
|
|
|
1509
|
-
# GET/DELETE/PUT calls other than the asset upload share the same
|
|
1510
|
-
# transient network failure modes; retry them (they are idempotent).
|
|
1511
|
-
# A 403 rate-limit response is not one of those modes: it rides the
|
|
1512
|
-
# window out and never consumes the transient attempts. Transport-level
|
|
1513
|
-
# drops (SSL EOF, TCP reset) belong here too: a stale keep-alive
|
|
1514
|
-
# connection answered with an SSL EOF is indistinguishable from a fresh
|
|
1515
|
-
# one that works — net-http retries those on EOFError but NOT on
|
|
1516
|
-
# OpenSSL::SSL::SSLError, so we do (the 2026-08-30 backfill crashed on
|
|
1517
|
-
# exactly that, on the verification GET right after a long upload POST).
|
|
1518
|
-
TRANSIENT_ERRORS = [
|
|
1519
|
-
Net::WriteTimeout, Net::ReadTimeout,
|
|
1520
|
-
Faraday::TimeoutError, Faraday::ConnectionFailed,
|
|
1521
|
-
OpenSSL::SSL::SSLError, EOFError, SystemCallError
|
|
1522
|
-
].freeze
|
|
1523
|
-
|
|
1524
|
-
def with_transient_retries(attempts: 4)
|
|
1525
|
-
with_rate_limit_rideout do
|
|
1526
|
-
yield
|
|
1527
|
-
rescue *TRANSIENT_ERRORS => e
|
|
1528
|
-
attempts -= 1
|
|
1529
|
-
raise if attempts <= 0
|
|
1530
|
-
|
|
1531
|
-
delay = (5 * (4 - attempts)) + rand(5)
|
|
1532
|
-
puts "#{e.class}; retrying in #{delay}s (#{attempts} attempt(s) left)"
|
|
1533
|
-
sleep delay
|
|
1534
|
-
retry
|
|
1535
|
-
end
|
|
1536
|
-
end
|
|
1537
|
-
|
|
1538
|
-
# A 403 rate-limit response must never kill the publish: the uploader
|
|
1539
|
-
# is one serialized actor, and sleeping until the window resets is the
|
|
1540
|
-
# CORRECT behavior (the 0.16.6 publish burned the tebako-ci token's
|
|
1541
|
-
# 5000-request hourly window in ~30 minutes and died at the finalize —
|
|
1542
|
-
# GET .../assets 403 — after all 334 payload assets had landed). The
|
|
1543
|
-
# upload POST rides this out too; its transient retries (escalating
|
|
1544
|
-
# delays, 422-by-content resolution, the per-asset budget) stay with
|
|
1545
|
-
# perform_upload.
|
|
1546
|
-
RATE_LIMIT_SETTLE = 5
|
|
1547
|
-
RATE_LIMIT_DEFAULT_WAIT = 60
|
|
1548
|
-
RATE_LIMIT_BUDGET = (2 * 3600) + 300
|
|
1549
|
-
|
|
1550
|
-
def with_rate_limit_rideout
|
|
1551
|
-
yield
|
|
1552
|
-
rescue Octokit::TooManyRequests => e
|
|
1553
|
-
wait = rate_limit_wait(e)
|
|
1554
|
-
puts "#{e.class}; rate-limited — sleeping #{wait}s until the window resets"
|
|
1555
|
-
sleep wait
|
|
1556
|
-
retry
|
|
1557
|
-
end
|
|
1558
|
-
|
|
1559
|
-
# The seconds to sleep before the next call, budget-checked: one full
|
|
1560
|
-
# hourly window is a legitimate wait; a wait that would push the process
|
|
1561
|
-
# past two windows means something is systemically wrong — give up
|
|
1562
|
-
# loudly instead of blocking the runner forever.
|
|
1563
|
-
def rate_limit_wait(error)
|
|
1564
|
-
wait = rate_limit_seconds(error)
|
|
1565
|
-
return wait if monotonic_now + wait <= rate_limit_deadline
|
|
1566
|
-
|
|
1567
|
-
raise RateLimitBudgetExhausted,
|
|
1568
|
-
"the next GitHub rate-limit window is #{wait}s out but this publish has a #{RATE_LIMIT_BUDGET}s " \
|
|
1569
|
-
"ride-out budget — two full windows spent; giving up loudly instead of blocking forever"
|
|
1570
|
-
end
|
|
1571
|
-
|
|
1572
|
-
def rate_limit_deadline
|
|
1573
|
-
@rate_limit_deadline ||= monotonic_now + RATE_LIMIT_BUDGET
|
|
1574
|
-
end
|
|
1575
|
-
|
|
1576
|
-
# The reset header names the window's end as a wall-clock epoch (plus a
|
|
1577
|
-
# small settle); a stale or absent reset falls back to Retry-After,
|
|
1578
|
-
# then to a default minute. Faraday's real headers are case-insensitive;
|
|
1579
|
-
# read both spellings so a plain hash (the spec fake) serves the same
|
|
1580
|
-
# values.
|
|
1581
|
-
def rate_limit_seconds(error)
|
|
1582
|
-
headers = error.response_headers || {}
|
|
1583
|
-
reset = (headers["x-ratelimit-reset"] || headers["X-RateLimit-Reset"]).to_i
|
|
1584
|
-
return reset - Time.now.to_i + RATE_LIMIT_SETTLE if reset > Time.now.to_i
|
|
1585
|
-
|
|
1586
|
-
retry_after = (headers["retry-after"] || headers["Retry-After"]).to_i
|
|
1587
|
-
return retry_after + RATE_LIMIT_SETTLE if retry_after.positive?
|
|
1588
|
-
|
|
1589
|
-
RATE_LIMIT_DEFAULT_WAIT
|
|
1590
|
-
end
|
|
1591
|
-
|
|
1592
1541
|
def run
|
|
1593
1542
|
process_release
|
|
1594
1543
|
rescue StandardError => e
|
|
@@ -1718,7 +1667,7 @@ module TebakoRelease
|
|
|
1718
1667
|
raise Error, "the served image #{image_name} is not on the release — cannot derive its blksum sidecar"
|
|
1719
1668
|
end
|
|
1720
1669
|
|
|
1721
|
-
bytes = with_transient_retries { @client.get(asset.browser_download_url) }.to_s
|
|
1670
|
+
bytes = with_transient_retries("download #{image_name}") { @client.get(asset.browser_download_url) }.to_s
|
|
1722
1671
|
Pathname.new(Dir.mktmpdir).join(image_name).tap { |path| path.binwrite(bytes) }
|
|
1723
1672
|
end
|
|
1724
1673
|
|
|
@@ -1763,7 +1712,8 @@ module TebakoRelease
|
|
|
1763
1712
|
end
|
|
1764
1713
|
|
|
1765
1714
|
def extract_bundle(_release, bundle_asset)
|
|
1766
|
-
|
|
1715
|
+
url = bundle_asset.browser_download_url
|
|
1716
|
+
bytes = with_transient_retries("bundle #{bundle_asset.name}") { @client.get(url) }.to_s
|
|
1767
1717
|
dir = Pathname.new(Dir.mktmpdir("tebako-release-bundle-"))
|
|
1768
1718
|
Zlib::GzipReader.wrap(StringIO.new(bytes)) { |gzip| extract_tar_members(bundle_asset, gzip, dir) }
|
|
1769
1719
|
end
|
data/lib/tebako_release.rb
CHANGED
|
@@ -11,6 +11,7 @@ require_relative "tebako_release/config"
|
|
|
11
11
|
require_relative "tebako_release/platform"
|
|
12
12
|
require_relative "tebako_release/bundler"
|
|
13
13
|
require_relative "tebako_release/blksum"
|
|
14
|
+
require_relative "tebako_release/convergence"
|
|
14
15
|
require_relative "tebako_release/uploader"
|
|
15
16
|
require_relative "tebako_release/signer"
|
|
16
17
|
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: tebako-release
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.5.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc
|
|
@@ -42,6 +42,7 @@ files:
|
|
|
42
42
|
- lib/tebako_release/blksum.rb
|
|
43
43
|
- lib/tebako_release/bundler.rb
|
|
44
44
|
- lib/tebako_release/config.rb
|
|
45
|
+
- lib/tebako_release/convergence.rb
|
|
45
46
|
- lib/tebako_release/error.rb
|
|
46
47
|
- lib/tebako_release/platform.rb
|
|
47
48
|
- lib/tebako_release/signer.rb
|