@botiverse/k-carrier 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/src/artifact/download.ts +43 -72
- package/core/src/artifact/transferPolicy.ts +82 -0
- package/core/src/createUpgrader.ts +40 -7
- package/core/src/index.ts +1 -0
- package/core/src/operation.ts +3 -0
- package/core/src/txn/state.ts +77 -8
- package/core/src/upgrade/drive.ts +12 -1
- package/docs/integration.md +23 -0
- package/harness/src/artifact/downloadHoles.ts +1 -1
- package/package.json +1 -1
|
@@ -32,42 +32,9 @@ import { ArtifactError } from "./errors.ts";
|
|
|
32
32
|
import type { Release } from "./source.ts";
|
|
33
33
|
import { collectStream } from "./collectStream.ts";
|
|
34
34
|
import { partialPathFor } from "./partialPath.ts";
|
|
35
|
+
import type { DownloadOptions } from "./transferPolicy.ts";
|
|
35
36
|
export { partialPathFor } from "./partialPath.ts";
|
|
36
|
-
|
|
37
|
-
export interface DownloadOptions {
|
|
38
|
-
/**
|
|
39
|
-
* Byte progress. `downloaded` counts bytes ON DISK including any resumed
|
|
40
|
-
* prefix, so a resumed download never appears to restart at zero.
|
|
41
|
-
*/
|
|
42
|
-
onProgress?: (downloaded: number, total: number) => void;
|
|
43
|
-
clock?: Clock;
|
|
44
|
-
/** Abort the download after this many ms (0 = no timeout). Default 10000. */
|
|
45
|
-
timeoutMs?: number;
|
|
46
|
-
/**
|
|
47
|
-
* Abort after this many ms with NO bytes arriving (0 = off). Default 0.
|
|
48
|
-
*
|
|
49
|
-
* Distinct from `timeoutMs`, and for large artifacts the more useful of the
|
|
50
|
-
* two: a total budget must either be big enough for the slowest acceptable
|
|
51
|
-
* download of a 150MB binary -- in which case a wedged connection holds for
|
|
52
|
-
* just as long -- or small enough to kill a slow one that was making steady
|
|
53
|
-
* progress. Bounding SILENCE instead follows liveness, so the limit does not
|
|
54
|
-
* have to encode a guess about size or bandwidth.
|
|
55
|
-
*/
|
|
56
|
-
stallTimeoutMs?: number;
|
|
57
|
-
/**
|
|
58
|
-
* HTTP client for the artifact bytes. `staticManifestSource` already takes
|
|
59
|
-
* one; without the same seam here an adopter can point K at their own server
|
|
60
|
-
* for the MANIFEST but not for the BYTES, which is half a seam and surprising
|
|
61
|
-
* in exactly the place it matters (proxies, custom agents, and an adopter's
|
|
62
|
-
* own integration tests all need both).
|
|
63
|
-
*/
|
|
64
|
-
fetchImpl?: typeof fetch;
|
|
65
|
-
/**
|
|
66
|
-
* Directory for partial-download state. When set, an interrupted download
|
|
67
|
-
* leaves its prefix here and the next attempt resumes via Range.
|
|
68
|
-
*/
|
|
69
|
-
resumeDir?: string;
|
|
70
|
-
}
|
|
37
|
+
export type { DownloadOptions } from "./transferPolicy.ts";
|
|
71
38
|
|
|
72
39
|
/** Fetch and verify one release's bytes, resuming from a partial if present. */
|
|
73
40
|
export async function downloadVerified(
|
|
@@ -92,7 +59,9 @@ export async function downloadVerified(
|
|
|
92
59
|
|
|
93
60
|
const bytes = await fetchAndAppend(
|
|
94
61
|
url, partialPath, partialSize, clock, timeoutMs, opts.onProgress, release.size,
|
|
95
|
-
opts.
|
|
62
|
+
opts.responseTimeoutMs ?? opts.stallTimeoutMs ?? 0,
|
|
63
|
+
opts.idleTimeoutMs ?? opts.stallTimeoutMs ?? 0,
|
|
64
|
+
opts.fetchImpl ?? fetch,
|
|
96
65
|
);
|
|
97
66
|
|
|
98
67
|
const sha = sha256Hex(bytes);
|
|
@@ -129,26 +98,24 @@ async function fetchAndAppend(
|
|
|
129
98
|
timeoutMs: number,
|
|
130
99
|
onProgress?: (downloaded: number, total: number) => void,
|
|
131
100
|
total?: number,
|
|
132
|
-
|
|
101
|
+
responseTimeoutMs = 0,
|
|
102
|
+
idleTimeoutMs = 0,
|
|
133
103
|
doFetch: typeof fetch = fetch,
|
|
134
104
|
): Promise<Uint8Array> {
|
|
135
105
|
const controller = new AbortController();
|
|
136
|
-
//
|
|
137
|
-
//
|
|
138
|
-
|
|
139
|
-
let
|
|
140
|
-
let
|
|
106
|
+
// Body liveness is rearmed on every chunk. `timeoutKind` records WHY we
|
|
107
|
+
// aborted because the abort signal cannot distinguish the three budgets.
|
|
108
|
+
let timeoutKind: "overall" | "response" | "idle" | null = null;
|
|
109
|
+
let responseTimer: (() => void) | undefined;
|
|
110
|
+
let idleTimer: (() => void) | undefined;
|
|
141
111
|
// Invoked when a deadline fires so the pending fetch cannot outlive it.
|
|
142
112
|
let aborted: (() => void) | undefined;
|
|
143
|
-
// Which phase we were in when the deadline fired. Reporting "awaiting
|
|
144
|
-
// response" for a stall that happened mid-body sends the reader looking at
|
|
145
|
-
// the wrong end of the transfer.
|
|
146
|
-
let responded = false;
|
|
147
113
|
const abortReason = (u: string): string =>
|
|
148
|
-
|
|
149
|
-
? `download
|
|
150
|
-
|
|
151
|
-
|
|
114
|
+
timeoutKind === "response"
|
|
115
|
+
? `download response timed out after ${responseTimeoutMs}ms (awaiting response): ${u}`
|
|
116
|
+
: timeoutKind === "idle"
|
|
117
|
+
? `download stalled: nothing received for ${idleTimeoutMs}ms (mid-body): ${u}`
|
|
118
|
+
: `download timed out after ${timeoutMs}ms: ${u}`;
|
|
152
119
|
// Declared AFTER `aborted`: an immediate/virtual clock fires this callback
|
|
153
120
|
// synchronously inside `clock.after`, so a timer created earlier would reach
|
|
154
121
|
// `aborted` in its temporal dead zone. Real clocks hide that ordering; the
|
|
@@ -156,15 +123,25 @@ async function fetchAndAppend(
|
|
|
156
123
|
const cancel =
|
|
157
124
|
timeoutMs > 0
|
|
158
125
|
? clock.after(timeoutMs, () => {
|
|
126
|
+
timeoutKind = "overall";
|
|
159
127
|
controller.abort();
|
|
160
128
|
aborted?.();
|
|
161
129
|
})
|
|
162
130
|
: undefined;
|
|
163
|
-
const
|
|
164
|
-
if (
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
131
|
+
const armResponse = (): void => {
|
|
132
|
+
if (responseTimeoutMs <= 0) return;
|
|
133
|
+
responseTimer?.();
|
|
134
|
+
responseTimer = clock.after(responseTimeoutMs, () => {
|
|
135
|
+
timeoutKind = "response";
|
|
136
|
+
controller.abort();
|
|
137
|
+
aborted?.();
|
|
138
|
+
});
|
|
139
|
+
};
|
|
140
|
+
const armIdle = (): void => {
|
|
141
|
+
if (idleTimeoutMs <= 0) return;
|
|
142
|
+
idleTimer?.();
|
|
143
|
+
idleTimer = clock.after(idleTimeoutMs, () => {
|
|
144
|
+
timeoutKind = "idle";
|
|
168
145
|
controller.abort();
|
|
169
146
|
aborted?.();
|
|
170
147
|
});
|
|
@@ -173,7 +150,7 @@ async function fetchAndAppend(
|
|
|
173
150
|
const headers: Record<string, string> = {};
|
|
174
151
|
if (partialSize > 0) headers["Range"] = `bytes=${partialSize}-`;
|
|
175
152
|
let res: Response;
|
|
176
|
-
|
|
153
|
+
armResponse();
|
|
177
154
|
try {
|
|
178
155
|
// Race the abort, do not merely signal it. `AbortSignal` only works if the
|
|
179
156
|
// fetch implementation honours it, and `fetchImpl` is an adopter-supplied
|
|
@@ -193,11 +170,7 @@ async function fetchAndAppend(
|
|
|
193
170
|
const timedOut = controller.signal.aborted;
|
|
194
171
|
throw new ArtifactError(
|
|
195
172
|
"DOWNLOAD_FAILED",
|
|
196
|
-
|
|
197
|
-
? `download stalled: nothing received for ${stallTimeoutMs}ms (awaiting response): ${url}`
|
|
198
|
-
: timedOut
|
|
199
|
-
? `download timed out after ${timeoutMs}ms: ${url}`
|
|
200
|
-
: `fetch failed: ${url}`,
|
|
173
|
+
timedOut ? abortReason(url) : `fetch failed: ${url}`,
|
|
201
174
|
{ cause: err },
|
|
202
175
|
);
|
|
203
176
|
}
|
|
@@ -209,8 +182,9 @@ async function fetchAndAppend(
|
|
|
209
182
|
// server that answers slowly and then streams normally is killed by a
|
|
210
183
|
// deadline that started before the request was even answered -- and the
|
|
211
184
|
// failure reports "awaiting response" while bytes were on their way.
|
|
212
|
-
|
|
213
|
-
|
|
185
|
+
responseTimer?.();
|
|
186
|
+
responseTimer = undefined;
|
|
187
|
+
armIdle();
|
|
214
188
|
|
|
215
189
|
if (partialPath === null) {
|
|
216
190
|
// Stream even with nowhere to resume to. `res.arrayBuffer()` is one
|
|
@@ -219,7 +193,7 @@ async function fetchAndAppend(
|
|
|
219
193
|
// silently had no byte progress and no stall detection at all, while
|
|
220
194
|
// both looked configured.
|
|
221
195
|
try {
|
|
222
|
-
return await collectStream(url, res.body, total, onProgress,
|
|
196
|
+
return await collectStream(url, res.body, total, onProgress, armIdle);
|
|
223
197
|
} catch (err) {
|
|
224
198
|
// Same classification as the resume path. Without this the in-memory
|
|
225
199
|
// branch reported a bare stream error, so an abort we ourselves caused
|
|
@@ -248,10 +222,10 @@ async function fetchAndAppend(
|
|
|
248
222
|
// reads as "it lost my download" to the person watching it.
|
|
249
223
|
let onDisk = partialSize;
|
|
250
224
|
onProgress?.(onDisk, total ?? 0);
|
|
251
|
-
|
|
225
|
+
armIdle();
|
|
252
226
|
await streamToFile(res.body, fh, (chunk) => {
|
|
253
227
|
onDisk += chunk;
|
|
254
|
-
|
|
228
|
+
armIdle();
|
|
255
229
|
onProgress?.(onDisk, total ?? 0);
|
|
256
230
|
});
|
|
257
231
|
await fh.sync();
|
|
@@ -262,11 +236,7 @@ async function fetchAndAppend(
|
|
|
262
236
|
const timedOut = controller.signal.aborted;
|
|
263
237
|
throw new ArtifactError(
|
|
264
238
|
"DOWNLOAD_FAILED",
|
|
265
|
-
|
|
266
|
-
? `download stalled: nothing received for ${stallTimeoutMs}ms (mid-body): ${url}`
|
|
267
|
-
: timedOut
|
|
268
|
-
? `download timed out after ${timeoutMs}ms: ${url}`
|
|
269
|
-
: `download interrupted: ${url}`,
|
|
239
|
+
timedOut ? abortReason(url) : `download interrupted: ${url}`,
|
|
270
240
|
{ cause: err },
|
|
271
241
|
);
|
|
272
242
|
} finally {
|
|
@@ -275,7 +245,8 @@ async function fetchAndAppend(
|
|
|
275
245
|
return new Uint8Array(await fs.readFile(partialPath));
|
|
276
246
|
} finally {
|
|
277
247
|
cancel?.();
|
|
278
|
-
|
|
248
|
+
responseTimer?.();
|
|
249
|
+
idleTimer?.();
|
|
279
250
|
}
|
|
280
251
|
}
|
|
281
252
|
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Production artifact-transfer budgets.
|
|
3
|
+
*
|
|
4
|
+
* Response latency, progress liveness, and total duration answer different
|
|
5
|
+
* questions. Keeping them separate prevents a healthy large transfer from
|
|
6
|
+
* being killed by the same deadline that bounds an unreachable server.
|
|
7
|
+
*/
|
|
8
|
+
import type { Clock } from "../clock.ts";
|
|
9
|
+
|
|
10
|
+
export interface DownloadOptions {
|
|
11
|
+
/** Byte progress, including any prefix already present on disk. */
|
|
12
|
+
onProgress?: (downloaded: number, total: number) => void;
|
|
13
|
+
clock?: Clock;
|
|
14
|
+
/** Whole-download bound (0 = off). Legacy direct-call default is 10000. */
|
|
15
|
+
timeoutMs?: number;
|
|
16
|
+
/** Maximum wait for response headers (0 = off). */
|
|
17
|
+
responseTimeoutMs?: number;
|
|
18
|
+
/** Maximum silence between response-body chunks (0 = off). */
|
|
19
|
+
idleTimeoutMs?: number;
|
|
20
|
+
/** Legacy alias for both responseTimeoutMs and idleTimeoutMs. */
|
|
21
|
+
stallTimeoutMs?: number;
|
|
22
|
+
fetchImpl?: typeof fetch;
|
|
23
|
+
/** Directory holding a resumable partial artifact. */
|
|
24
|
+
resumeDir?: string;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface ArtifactTransferPolicy {
|
|
28
|
+
/** Maximum wait for response headers. */
|
|
29
|
+
responseTimeoutMs: number;
|
|
30
|
+
/** Maximum silence between response-body chunks. */
|
|
31
|
+
idleTimeoutMs: number;
|
|
32
|
+
/** Slowest sustained body rate accepted when deriving the size budget. */
|
|
33
|
+
minimumBytesPerSecond: number;
|
|
34
|
+
/** Absolute ceiling even when the declared artifact is very large. */
|
|
35
|
+
maximumOverallTimeoutMs: number;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface ArtifactTransferTimeouts {
|
|
39
|
+
responseTimeoutMs: number;
|
|
40
|
+
idleTimeoutMs: number;
|
|
41
|
+
overallTimeoutMs: number;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export const DEFAULT_ARTIFACT_TRANSFER_POLICY: ArtifactTransferPolicy = {
|
|
45
|
+
responseTimeoutMs: 30_000,
|
|
46
|
+
idleTimeoutMs: 30_000,
|
|
47
|
+
minimumBytesPerSecond: 64 * 1024,
|
|
48
|
+
maximumOverallTimeoutMs: 30 * 60_000,
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
function requirePositiveInteger(name: string, value: number): void {
|
|
52
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
53
|
+
throw new Error(`ARTIFACT_TRANSFER_POLICY_INVALID: ${name} must be a positive safe integer`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Derive the bounded total budget from the release authority's exact size. */
|
|
58
|
+
export function artifactTransferTimeouts(
|
|
59
|
+
artifactSize: number,
|
|
60
|
+
policy: ArtifactTransferPolicy = DEFAULT_ARTIFACT_TRANSFER_POLICY,
|
|
61
|
+
): ArtifactTransferTimeouts {
|
|
62
|
+
requirePositiveInteger("artifactSize", artifactSize);
|
|
63
|
+
requirePositiveInteger("responseTimeoutMs", policy.responseTimeoutMs);
|
|
64
|
+
requirePositiveInteger("idleTimeoutMs", policy.idleTimeoutMs);
|
|
65
|
+
requirePositiveInteger("minimumBytesPerSecond", policy.minimumBytesPerSecond);
|
|
66
|
+
requirePositiveInteger("maximumOverallTimeoutMs", policy.maximumOverallTimeoutMs);
|
|
67
|
+
if (policy.maximumOverallTimeoutMs < policy.responseTimeoutMs) {
|
|
68
|
+
throw new Error(
|
|
69
|
+
"ARTIFACT_TRANSFER_POLICY_INVALID: maximumOverallTimeoutMs must cover responseTimeoutMs",
|
|
70
|
+
);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const bodyBudgetMs = Math.ceil((artifactSize * 1_000) / policy.minimumBytesPerSecond);
|
|
74
|
+
return {
|
|
75
|
+
responseTimeoutMs: policy.responseTimeoutMs,
|
|
76
|
+
idleTimeoutMs: policy.idleTimeoutMs,
|
|
77
|
+
overallTimeoutMs: Math.min(
|
|
78
|
+
policy.maximumOverallTimeoutMs,
|
|
79
|
+
policy.responseTimeoutMs + bodyBudgetMs,
|
|
80
|
+
),
|
|
81
|
+
};
|
|
82
|
+
}
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* rollback is rare, not routine.
|
|
8
8
|
*/
|
|
9
9
|
import type { Upgrader, UpgraderConfig, UpgradeOutcome, ProvenanceIdentity } from "./upgrader.ts";
|
|
10
|
-
import { phaseAtRest, type TxnState } from "./txn/state.ts";
|
|
10
|
+
import { assertNever, buildTxnState, phaseAtRest, type TxnState } from "./txn/state.ts";
|
|
11
11
|
import type { ProcessEvidence } from "./lifecycle/hostAdapter.ts";
|
|
12
12
|
import { UpgradeEngine } from "./txn/engine.ts";
|
|
13
13
|
import { fileEffects } from "./txn/fileEffects.ts";
|
|
@@ -26,6 +26,7 @@ import { recoverUpgrade } from "./upgrade/recover.ts";
|
|
|
26
26
|
import type { OperationDescriptor } from "./operation.ts";
|
|
27
27
|
import { createOperationLifecycle } from "./operationLifecycle.ts";
|
|
28
28
|
import { driveUpgrade } from "./upgrade/drive.ts";
|
|
29
|
+
import type { ArtifactTransferPolicy } from "./artifact/transferPolicy.ts";
|
|
29
30
|
|
|
30
31
|
export interface CreateUpgraderOptions extends UpgraderConfig {
|
|
31
32
|
clock?: Clock;
|
|
@@ -58,6 +59,12 @@ export interface CreateUpgraderOptions extends UpgraderConfig {
|
|
|
58
59
|
provenance?: ProvenanceJournal;
|
|
59
60
|
/** Identity recorded for reconciles that carry none (local auto-update). */
|
|
60
61
|
provenanceIdentity?: ProvenanceIdentity;
|
|
62
|
+
/**
|
|
63
|
+
* Independent response, idle, and size-derived total budgets for artifact
|
|
64
|
+
* bytes. Defaults are safe for Computer-sized binaries; adopters may make
|
|
65
|
+
* them stricter, but cannot disable every bound.
|
|
66
|
+
*/
|
|
67
|
+
artifactTransferPolicy?: ArtifactTransferPolicy;
|
|
61
68
|
}
|
|
62
69
|
|
|
63
70
|
export function createUpgrader(opts: CreateUpgraderOptions): Upgrader {
|
|
@@ -111,12 +118,35 @@ export function createUpgrader(opts: CreateUpgraderOptions): Upgrader {
|
|
|
111
118
|
async function readState(): Promise<TxnState> {
|
|
112
119
|
const slots = await effects.slots.slotVersions();
|
|
113
120
|
const intents = (await effects.journal.readAll()).map((e) => e.intent);
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
121
|
+
const phase = intents.at(-1) ?? "idle";
|
|
122
|
+
const stableVersion = slots.stable ?? "0.0.0";
|
|
123
|
+
const experimentVersion = slots.experiment;
|
|
124
|
+
switch (phase) {
|
|
125
|
+
case "idle":
|
|
126
|
+
case "promoted":
|
|
127
|
+
return buildTxnState({ phase, stableVersion });
|
|
128
|
+
case "rolled-back":
|
|
129
|
+
return buildTxnState({ phase, stableVersion });
|
|
130
|
+
case "staged":
|
|
131
|
+
case "handing-over":
|
|
132
|
+
case "running-experiment":
|
|
133
|
+
case "readback": {
|
|
134
|
+
// In-flight phases always carry an experiment. A persisted world that
|
|
135
|
+
// says an in-flight phase has an empty experiment slot is
|
|
136
|
+
// inconsistent, and we fail closed rather than fabricate a value that
|
|
137
|
+
// would be a type lie at this (the only runtime) boundary.
|
|
138
|
+
if (experimentVersion === null) {
|
|
139
|
+
throw new Error(
|
|
140
|
+
`TXN_STATE_INCONSISTENT: phase ${phase} is in-flight but the experiment slot is empty`,
|
|
141
|
+
);
|
|
142
|
+
}
|
|
143
|
+
return buildTxnState({ phase, stableVersion, experimentVersion });
|
|
144
|
+
}
|
|
145
|
+
default:
|
|
146
|
+
// Matches phaseAtRest's exhaustive discipline: an unknown phase from a
|
|
147
|
+
// newer core is not silently reinterpreted.
|
|
148
|
+
return assertNever(phase);
|
|
149
|
+
}
|
|
120
150
|
}
|
|
121
151
|
|
|
122
152
|
const operationLifecycle = createOperationLifecycle(opts.stateDir, clock, readState);
|
|
@@ -142,6 +172,9 @@ export function createUpgrader(opts: CreateUpgraderOptions): Upgrader {
|
|
|
142
172
|
},
|
|
143
173
|
...(opts.provenance ? { provenanceJournal: opts.provenance } : {}),
|
|
144
174
|
...(opts.provenanceIdentity ? { provenanceIdentity: opts.provenanceIdentity } : {}),
|
|
175
|
+
...(opts.artifactTransferPolicy
|
|
176
|
+
? { artifactTransferPolicy: opts.artifactTransferPolicy }
|
|
177
|
+
: {}),
|
|
145
178
|
}, request);
|
|
146
179
|
|
|
147
180
|
return {
|
package/core/src/index.ts
CHANGED
|
@@ -18,6 +18,7 @@ export * from "./operation.ts";
|
|
|
18
18
|
// The release-source boundary applications implement and the durable
|
|
19
19
|
// provenance journal they wire into createUpgrader.
|
|
20
20
|
export * from "./artifact/source.ts";
|
|
21
|
+
export * from "./artifact/transferPolicy.ts";
|
|
21
22
|
export * from "./provenance/journal.ts";
|
|
22
23
|
|
|
23
24
|
// The host boundary an adopter implements: HostAdapter, Slot, ProcessEvidence.
|
package/core/src/operation.ts
CHANGED
|
@@ -170,6 +170,9 @@ export async function acknowledgeOperation(
|
|
|
170
170
|
if (current.kind === "unreadable") throw new Error(current.reason);
|
|
171
171
|
if (current.operation.id !== operationId) return "changed";
|
|
172
172
|
if (current.operation.outcome === null) return "not-terminal";
|
|
173
|
+
// Exact replay is idempotent. The first delivery time is part of the audit
|
|
174
|
+
// receipt; a retry must not rewrite it or manufacture a later delivery.
|
|
175
|
+
if (current.operation.acknowledgedAtMs !== null) return "acknowledged";
|
|
173
176
|
await persistOperation(stateDir, {
|
|
174
177
|
...current.operation,
|
|
175
178
|
updatedAtMs: acknowledgedAtMs,
|
package/core/src/txn/state.ts
CHANGED
|
@@ -30,7 +30,7 @@ export type TxnPhase =
|
|
|
30
30
|
* default, never the exclusion-list trap where every future terminal
|
|
31
31
|
* phase silently becomes "in-flight" = allowed to modify.
|
|
32
32
|
*/
|
|
33
|
-
function assertNever(x: never): never {
|
|
33
|
+
export function assertNever(x: never): never {
|
|
34
34
|
throw new Error(`unknown TxnPhase ${String(x)}; refusing to classify — a phase a newer core wrote must not be touched`);
|
|
35
35
|
}
|
|
36
36
|
|
|
@@ -53,13 +53,34 @@ export function phaseAtRest(phase: TxnPhase): boolean {
|
|
|
53
53
|
}
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
56
|
+
/**
|
|
57
|
+
* A discriminated union so that an "illegal" combination is not representable:
|
|
58
|
+
*
|
|
59
|
+
* - terminal / at-rest phases (`idle`, `promoted`, `rolled-back`) NEVER carry
|
|
60
|
+
* an experiment — an experiment slot that outlives a terminal phase is a
|
|
61
|
+
* leaked transaction (k.terminal-leaves-no-experiment would fire at runtime;
|
|
62
|
+
* here it is a compile error instead).
|
|
63
|
+
* - in-flight phases (`staged`..`readback`) ALWAYS carry an experiment — that
|
|
64
|
+
* is exactly what "in flight" means.
|
|
65
|
+
* - only `rolled-back` may carry a `rollbackReason`.
|
|
66
|
+
*
|
|
67
|
+
* Construction must choose a member consistent with `phase`; a `switch` over
|
|
68
|
+
* `TxnState` narrows each member and forces every phase to make a decision
|
|
69
|
+
* (the same discipline as `phaseAtRest`, lifted to the whole record).
|
|
70
|
+
*/
|
|
71
|
+
export type TxnState =
|
|
72
|
+
| { phase: "idle"; stableVersion: string; experimentVersion: null; rollbackReason: null }
|
|
73
|
+
| {
|
|
74
|
+
phase: "staged" | "handing-over" | "running-experiment" | "readback";
|
|
75
|
+
stableVersion: string;
|
|
76
|
+
experimentVersion: string;
|
|
77
|
+
rollbackReason: null;
|
|
78
|
+
}
|
|
79
|
+
| { phase: "promoted"; stableVersion: string; experimentVersion: null; rollbackReason: null }
|
|
80
|
+
| { phase: "rolled-back"; stableVersion: string; experimentVersion: null; rollbackReason: string | null };
|
|
81
|
+
|
|
82
|
+
/** The in-flight, experiment-carrying phases (a helper for exhaustive narrowing). */
|
|
83
|
+
export const IN_FLIGHT_PHASES = ["staged", "handing-over", "running-experiment", "readback"] as const;
|
|
63
84
|
|
|
64
85
|
/** Append-only, write-ahead journal entry. fsync'd before the action runs. */
|
|
65
86
|
export interface JournalEntry {
|
|
@@ -75,3 +96,51 @@ export interface JournalEntry {
|
|
|
75
96
|
* silently reinterpret. Encoded as a version field checked before replay.
|
|
76
97
|
*/
|
|
77
98
|
export const STATE_FORMAT_VERSION = 1;
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* The strongly-typed input to `buildTxnState`. The argument is itself a
|
|
102
|
+
* discriminated union over the phase, so an illegal input combination does
|
|
103
|
+
* not type-check: a terminal/at-rest phase simply cannot carry an experiment,
|
|
104
|
+
* only `rolled-back` may carry a reason, and an in-flight phase must carry
|
|
105
|
+
* one. This is the "impossible states" discipline applied to the constructor's
|
|
106
|
+
* arguments, not just its result.
|
|
107
|
+
*/
|
|
108
|
+
export type TxnStateInput =
|
|
109
|
+
| { phase: "idle" | "promoted"; stableVersion: string }
|
|
110
|
+
| { phase: "rolled-back"; stableVersion: string; rollbackReason?: string | null }
|
|
111
|
+
| {
|
|
112
|
+
phase: "staged" | "handing-over" | "running-experiment" | "readback";
|
|
113
|
+
stableVersion: string;
|
|
114
|
+
experimentVersion: string;
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Soundly construct a `TxnState` member from a discriminated input.
|
|
119
|
+
*
|
|
120
|
+
* The input type already forbids the illegal combinations (an in-flight phase
|
|
121
|
+
* cannot be given a missing experiment, a terminal phase cannot be given an
|
|
122
|
+
* experiment, only `rolled-back` may carry a reason), so the result is sound
|
|
123
|
+
* by construction — there is no per-phase coercion to get wrong here.
|
|
124
|
+
*
|
|
125
|
+
* The one place an inconsistent persisted world can slip in is the runtime
|
|
126
|
+
* boundary that reads the journal and slots but only knows the phase at
|
|
127
|
+
* runtime (see `createUpgrader.readState`); that boundary must decide the
|
|
128
|
+
* phase before calling this, failing closed if a phase is in flight but the
|
|
129
|
+
* experiment slot is empty.
|
|
130
|
+
*/
|
|
131
|
+
export function buildTxnState(input: TxnStateInput): TxnState {
|
|
132
|
+
switch (input.phase) {
|
|
133
|
+
case "idle":
|
|
134
|
+
case "promoted":
|
|
135
|
+
return { phase: input.phase, stableVersion: input.stableVersion, experimentVersion: null, rollbackReason: null };
|
|
136
|
+
case "rolled-back":
|
|
137
|
+
return { phase: input.phase, stableVersion: input.stableVersion, experimentVersion: null, rollbackReason: input.rollbackReason ?? null };
|
|
138
|
+
case "staged":
|
|
139
|
+
case "handing-over":
|
|
140
|
+
case "running-experiment":
|
|
141
|
+
case "readback":
|
|
142
|
+
return { phase: input.phase, stableVersion: input.stableVersion, experimentVersion: input.experimentVersion, rollbackReason: null };
|
|
143
|
+
default:
|
|
144
|
+
return assertNever(input);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import * as path from "node:path";
|
|
2
2
|
import type { Release } from "../artifact/source.ts";
|
|
3
3
|
import { downloadVerified } from "../artifact/download.ts";
|
|
4
|
+
import {
|
|
5
|
+
artifactTransferTimeouts,
|
|
6
|
+
type ArtifactTransferPolicy,
|
|
7
|
+
type DownloadOptions,
|
|
8
|
+
} from "../artifact/transferPolicy.ts";
|
|
4
9
|
import { ArtifactError } from "../artifact/errors.ts";
|
|
5
10
|
import { materializeArtifact } from "../txn/fileEffects.ts";
|
|
6
11
|
import { acquireUpgradeLock } from "../txn/lock.ts";
|
|
@@ -37,6 +42,8 @@ export interface UpgradeDriveDeps {
|
|
|
37
42
|
persistConvergenceReport: (report: ConvergenceReport) => Promise<void>;
|
|
38
43
|
provenanceJournal?: ProvenanceJournal;
|
|
39
44
|
provenanceIdentity?: ProvenanceIdentity;
|
|
45
|
+
artifactTransferPolicy?: ArtifactTransferPolicy;
|
|
46
|
+
downloadArtifact?: (release: Release, opts: DownloadOptions) => Promise<Uint8Array>;
|
|
40
47
|
}
|
|
41
48
|
|
|
42
49
|
export interface UpgradeDriveRequest {
|
|
@@ -115,9 +122,13 @@ export async function driveUpgrade(
|
|
|
115
122
|
|
|
116
123
|
await deps.operation.transition({ phase: "downloading" });
|
|
117
124
|
progress({ stage: "downloading", version: release.version });
|
|
118
|
-
const
|
|
125
|
+
const transfer = artifactTransferTimeouts(release.size, deps.artifactTransferPolicy);
|
|
126
|
+
const bytes = await (deps.downloadArtifact ?? downloadVerified)(release, {
|
|
119
127
|
clock: deps.clock,
|
|
120
128
|
resumeDir: path.join(deps.stateDir, "incoming"),
|
|
129
|
+
timeoutMs: transfer.overallTimeoutMs,
|
|
130
|
+
responseTimeoutMs: transfer.responseTimeoutMs,
|
|
131
|
+
idleTimeoutMs: transfer.idleTimeoutMs,
|
|
121
132
|
onProgress: (downloaded, total) =>
|
|
122
133
|
progress({ stage: "downloading", version: release.version, downloaded, total }),
|
|
123
134
|
});
|
package/docs/integration.md
CHANGED
|
@@ -237,6 +237,29 @@ Three things worth knowing before you draw a bar with it:
|
|
|
237
237
|
and discards anything it throws. An observation surface must never become
|
|
238
238
|
a failure mode — if your renderer breaks, the upgrade still completes.
|
|
239
239
|
|
|
240
|
+
Artifact transfer has three independent fail-closed budgets. Response headers
|
|
241
|
+
must arrive promptly, body progress must not go silent, and the full transfer
|
|
242
|
+
has a hard ceiling derived from the release source's declared byte size. The
|
|
243
|
+
defaults accept a Computer-sized binary that takes longer than ten seconds
|
|
244
|
+
while still bounding an unreachable server and a wedged mid-body stream. An
|
|
245
|
+
adopter with stricter network requirements may provide all four policy fields:
|
|
246
|
+
|
|
247
|
+
```ts
|
|
248
|
+
createUpgrader({
|
|
249
|
+
...,
|
|
250
|
+
artifactTransferPolicy: {
|
|
251
|
+
responseTimeoutMs: 20_000,
|
|
252
|
+
idleTimeoutMs: 30_000,
|
|
253
|
+
minimumBytesPerSecond: 128 * 1024,
|
|
254
|
+
maximumOverallTimeoutMs: 20 * 60_000,
|
|
255
|
+
},
|
|
256
|
+
});
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
The total budget is `responseTimeoutMs + size / minimumBytesPerSecond`, capped
|
|
260
|
+
by `maximumOverallTimeoutMs`. Invalid, zero, or effectively unbounded policies
|
|
261
|
+
are rejected before the byte request starts.
|
|
262
|
+
|
|
240
263
|
The stages are not a parallel state machine: they are derived from the L1
|
|
241
264
|
transaction phases (`stageForPhase`), so a progress display can never show a
|
|
242
265
|
state the transaction does not have.
|
|
@@ -39,7 +39,7 @@ function releaseFor(url: string): Release {
|
|
|
39
39
|
async function serve(chunks: number, gapMs: number, hang = false, stallAfterFirst = false) {
|
|
40
40
|
const server = http.createServer((_req, res) => {
|
|
41
41
|
res.writeHead(200, { "Content-Length": String(BODY.length) });
|
|
42
|
-
if (hang) return; // headers sent, bytes never follow
|
|
42
|
+
if (hang) { res.flushHeaders(); return; } // headers sent, bytes never follow
|
|
43
43
|
const size = Math.ceil(BODY.length / chunks);
|
|
44
44
|
let sent = 0;
|
|
45
45
|
const push = (): void => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@botiverse/k-carrier",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.7",
|
|
4
4
|
"packageManager": "pnpm@11.18.0",
|
|
5
5
|
"description": "A fail-closed upgrade carrier for long-running managed services: two-slot upgrade transactions (promote/rollback), host-driven quiesce/resume handoff, and post-upgrade convergence read-back.",
|
|
6
6
|
"repository": {
|