@boardwalk-labs/runner 0.1.10 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +12 -0
- package/dist/runtime/agent/budget.d.ts +5 -5
- package/dist/runtime/agent/budget.js +8 -8
- package/dist/runtime/agent/model_rates.js +4 -5
- package/dist/runtime/agent/secret_redactor.js +1 -1
- package/dist/runtime/cancel_watcher.js +1 -1
- package/dist/runtime/credit_watcher.d.ts +1 -1
- package/dist/runtime/credit_watcher.js +5 -5
- package/dist/runtime/freeze_coordinator.d.ts +124 -0
- package/dist/runtime/freeze_coordinator.js +276 -0
- package/dist/runtime/identity_relay.d.ts +45 -2
- package/dist/runtime/identity_relay.js +137 -14
- package/dist/runtime/index.d.ts +11 -6
- package/dist/runtime/index.js +119 -39
- package/dist/runtime/leaf_executor.d.ts +2 -2
- package/dist/runtime/program_runner.d.ts +1 -1
- package/dist/runtime/program_runner.js +1 -1
- package/dist/runtime/program_worker.d.ts +3 -3
- package/dist/runtime/program_worker.js +7 -7
- package/dist/runtime/recording_secret_resolver.js +1 -1
- package/dist/runtime/run_abort.js +3 -3
- package/dist/runtime/runner_control_client.d.ts +34 -4
- package/dist/runtime/runner_control_client.js +85 -33
- package/dist/runtime/runtime_flusher.d.ts +9 -0
- package/dist/runtime/runtime_flusher.js +17 -3
- package/dist/runtime/support/index.js +5 -6
- package/dist/runtime/tools/artifacts.js +1 -1
- package/dist/runtime/tools/types.d.ts +5 -5
- package/dist/runtime/tools/types.js +7 -7
- package/dist/runtime/tools/web_search.js +5 -6
- package/dist/runtime/uniqueness_reseed.d.ts +7 -0
- package/dist/runtime/uniqueness_reseed.js +65 -0
- package/dist/runtime/wire/artifact_storage.js +2 -2
- package/dist/runtime/wire/inference_proxy.d.ts +1 -1
- package/dist/runtime/wire/inference_proxy.js +2 -2
- package/dist/runtime/workflow_host.d.ts +50 -1
- package/dist/runtime/workflow_host.js +185 -31
- package/native/reseed.c +52 -0
- package/package.json +12 -5
- package/prebuilds/linux-x64/@boardwalk-labs+runner.node +0 -0
package/binding.gyp
ADDED
|
@@ -74,7 +74,7 @@ export declare class BudgetMeter {
|
|
|
74
74
|
constructor(opts: BudgetMeterOptions);
|
|
75
75
|
/**
|
|
76
76
|
* Add a usage delta to the accumulator + recompute USD. `realCostUsd`, when provided, is the EXACT
|
|
77
|
-
* upstream cost the broker observed for this turn (
|
|
77
|
+
* upstream cost the broker observed for this turn (the managed provider's per-request cost) — used
|
|
78
78
|
* verbatim so the `max_usd` cap tracks ACTUAL spend (already cache-discounted, model-correct).
|
|
79
79
|
* Omitted for a BYO turn (no upstream price) or a managed turn whose cost the broker couldn't read,
|
|
80
80
|
* which fall back to the representative-rate {@link costFor} estimate. Token counts accumulate on
|
|
@@ -83,7 +83,7 @@ export declare class BudgetMeter {
|
|
|
83
83
|
addUsage(delta: UsageDelta, realCostUsd?: number): void;
|
|
84
84
|
/**
|
|
85
85
|
* THIS SESSION's accumulator snapshot (excludes prior-session usage). Read by per-session token
|
|
86
|
-
* metering — which reports token deltas to
|
|
86
|
+
* metering — which reports token deltas to the platform (deferred; not yet wired into the brokered
|
|
87
87
|
* loop) — and by audit/post-run cost rollups. Cap enforcement uses {@link cumulative} instead.
|
|
88
88
|
*/
|
|
89
89
|
snapshot(): BudgetSnapshot;
|
|
@@ -92,7 +92,7 @@ export declare class BudgetMeter {
|
|
|
92
92
|
* construction. Cap enforcement ({@link assertWithinCaps}) and checkpoint persistence use
|
|
93
93
|
* this so a run that resumes after a sleep/wait/crash is bounded by its declared caps across
|
|
94
94
|
* the whole run — not once per session. `snapshot()` stays session-local because
|
|
95
|
-
* per-session token metering reports token deltas; seeding it would double-report
|
|
95
|
+
* per-session token metering reports token deltas; seeding it would double-report usage.
|
|
96
96
|
*/
|
|
97
97
|
cumulative(): BudgetSnapshot;
|
|
98
98
|
/**
|
|
@@ -116,8 +116,8 @@ export declare class BudgetMeter {
|
|
|
116
116
|
/**
|
|
117
117
|
* USD cost for a single delta — the representative-rate ESTIMATE used only when the broker reports
|
|
118
118
|
* no real upstream cost for the turn (a BYO-provider turn, or a managed turn whose cost tap missed).
|
|
119
|
-
* Managed turns instead carry
|
|
120
|
-
* the loop can stamp the per-step
|
|
119
|
+
* Managed turns instead carry the managed provider's exact per-request cost through to {@link addUsage}. Public so
|
|
120
|
+
* the loop can stamp the per-step cost without re-deriving the rate table.
|
|
121
121
|
*/
|
|
122
122
|
costFor(delta: UsageDelta): number;
|
|
123
123
|
}
|
|
@@ -5,12 +5,12 @@
|
|
|
5
5
|
// `AppError(BUDGET_EXCEEDED)` and the loop tears down with a `turn_ended
|
|
6
6
|
// reason='error'` event.
|
|
7
7
|
//
|
|
8
|
-
// USD here tracks the bill closely: for a MANAGED turn the meter is fed
|
|
9
|
-
//
|
|
8
|
+
// USD here tracks the bill closely: for a MANAGED turn the meter is fed the managed provider's exact
|
|
9
|
+
// per-request cost (already cache-discounted + model-correct), so the `max_usd` cap reflects real spend —
|
|
10
10
|
// not a hand-rolled token estimate. Only a BYO turn (no upstream price) or a managed turn whose cost
|
|
11
11
|
// the broker couldn't read falls back to a single flat representative rate
|
|
12
12
|
// (`model_rates.ts::BUDGET_GUARDRAIL_RATE`), since a per-`agent()`-model run has no one model. Actual
|
|
13
|
-
// billing
|
|
13
|
+
// billing is metered by the platform, not by the runner. The meter takes
|
|
14
14
|
// the fallback rate as a constructor arg so tests inject a fixed rate.
|
|
15
15
|
import { AppError, ErrorCode } from "../support/index.js";
|
|
16
16
|
const ZERO_PRIOR = {
|
|
@@ -44,7 +44,7 @@ export class BudgetMeter {
|
|
|
44
44
|
}
|
|
45
45
|
/**
|
|
46
46
|
* Add a usage delta to the accumulator + recompute USD. `realCostUsd`, when provided, is the EXACT
|
|
47
|
-
* upstream cost the broker observed for this turn (
|
|
47
|
+
* upstream cost the broker observed for this turn (the managed provider's per-request cost) — used
|
|
48
48
|
* verbatim so the `max_usd` cap tracks ACTUAL spend (already cache-discounted, model-correct).
|
|
49
49
|
* Omitted for a BYO turn (no upstream price) or a managed turn whose cost the broker couldn't read,
|
|
50
50
|
* which fall back to the representative-rate {@link costFor} estimate. Token counts accumulate on
|
|
@@ -59,7 +59,7 @@ export class BudgetMeter {
|
|
|
59
59
|
}
|
|
60
60
|
/**
|
|
61
61
|
* THIS SESSION's accumulator snapshot (excludes prior-session usage). Read by per-session token
|
|
62
|
-
* metering — which reports token deltas to
|
|
62
|
+
* metering — which reports token deltas to the platform (deferred; not yet wired into the brokered
|
|
63
63
|
* loop) — and by audit/post-run cost rollups. Cap enforcement uses {@link cumulative} instead.
|
|
64
64
|
*/
|
|
65
65
|
snapshot() {
|
|
@@ -78,7 +78,7 @@ export class BudgetMeter {
|
|
|
78
78
|
* construction. Cap enforcement ({@link assertWithinCaps}) and checkpoint persistence use
|
|
79
79
|
* this so a run that resumes after a sleep/wait/crash is bounded by its declared caps across
|
|
80
80
|
* the whole run — not once per session. `snapshot()` stays session-local because
|
|
81
|
-
* per-session token metering reports token deltas; seeding it would double-report
|
|
81
|
+
* per-session token metering reports token deltas; seeding it would double-report usage.
|
|
82
82
|
*/
|
|
83
83
|
cumulative() {
|
|
84
84
|
const s = this.snapshot();
|
|
@@ -159,8 +159,8 @@ export class BudgetMeter {
|
|
|
159
159
|
/**
|
|
160
160
|
* USD cost for a single delta — the representative-rate ESTIMATE used only when the broker reports
|
|
161
161
|
* no real upstream cost for the turn (a BYO-provider turn, or a managed turn whose cost tap missed).
|
|
162
|
-
* Managed turns instead carry
|
|
163
|
-
* the loop can stamp the per-step
|
|
162
|
+
* Managed turns instead carry the managed provider's exact per-request cost through to {@link addUsage}. Public so
|
|
163
|
+
* the loop can stamp the per-step cost without re-deriving the rate table.
|
|
164
164
|
*/
|
|
165
165
|
costFor(delta) {
|
|
166
166
|
const inputCost = ((delta.inputTokens ?? 0) * this.rate.inputPerMillion) / 1_000_000;
|
|
@@ -1,17 +1,16 @@
|
|
|
1
1
|
// The flat budget-guardrail rate — the FALLBACK basis for the in-run `max_usd` cap (BudgetMeter).
|
|
2
2
|
//
|
|
3
|
-
// A MANAGED turn now caps on
|
|
3
|
+
// A MANAGED turn now caps on the managed provider's EXACT per-request cost, forwarded from the broker to
|
|
4
4
|
// the worker (inference_proxy result frame → BudgetMeter.addUsage `realCostUsd`), so the cap tracks
|
|
5
5
|
// real spend — already cache-discounted + model-correct. This flat rate is used ONLY when a turn has
|
|
6
6
|
// no upstream cost: a BYO-provider turn (the org pays its own key), or a managed turn whose cost the
|
|
7
|
-
// broker couldn't read. It bounds runaway loops; it is NOT the bill. Actual billing is
|
|
8
|
-
//
|
|
9
|
-
// run_usage_service). the platform spec
|
|
7
|
+
// broker couldn't read. It bounds runaway loops; it is NOT the bill. Actual billing is metered by the
|
|
8
|
+
// platform, not by the runner.
|
|
10
9
|
//
|
|
11
10
|
// Deliberately NOT a per-model table with a silent fallback: a lookup that returns a Sonnet-class
|
|
12
11
|
// default for any unknown id only created the ILLUSION of precision. The real per-request cost (above)
|
|
13
12
|
// is the precise path; this rate is just the model-agnostic backstop. Un-metered managed models are
|
|
14
|
-
// rejected fail-closed at resolution
|
|
13
|
+
// rejected fail-closed at resolution by the broker, so the cap never has to price
|
|
15
14
|
// a model we don't support.
|
|
16
15
|
/**
|
|
17
16
|
* The flat representative rate (USD / million tokens) the BudgetMeter applies to the `max_usd` cap
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// SecretRedactor — scrubs known secret values from everything bound for an LLM
|
|
1
|
+
// SecretRedactor — scrubs known secret values from everything bound for an LLM.
|
|
2
2
|
//
|
|
3
3
|
// The architectural guarantee is structural: secrets live ONLY in the workflow PROGRAM (the trusted
|
|
4
4
|
// deterministic tool layer), which holds them via `secrets.get` / `ctx.secrets.resolve`. The
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// CancelWatcher — stops a run when the user cancels it mid-flight
|
|
1
|
+
// CancelWatcher — stops a run when the user cancels it mid-flight.
|
|
2
2
|
//
|
|
3
3
|
// The user-cancel counterpart to CreditWatcher. One per run session. On a timer it asks — through the
|
|
4
4
|
// broker (`GET /cancel`, since the runner holds no DB/Redis credential) — whether the run has been
|
|
@@ -4,7 +4,7 @@ export interface CreditWatcherDeps {
|
|
|
4
4
|
/** The run being watched (for correlation/logging). */
|
|
5
5
|
runId: string;
|
|
6
6
|
/** Resolves true while the org can keep spending; false once it's out of credit. Brokered
|
|
7
|
-
* (`RunnerControlClient.checkCredit` → `GET /credit`), so the runner never reads
|
|
7
|
+
* (`RunnerControlClient.checkCredit` → `GET /credit`), so the runner never reads billing state directly. */
|
|
8
8
|
isFunded: () => Promise<boolean>;
|
|
9
9
|
/** Fired exactly once when the org is first seen to be out of credit. */
|
|
10
10
|
onExhausted: () => void;
|
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
// CreditWatcher — stops a run when its org runs out of prepaid credit mid-flight
|
|
1
|
+
// CreditWatcher — stops a run when its org runs out of prepaid credit mid-flight.
|
|
2
2
|
//
|
|
3
3
|
// One per run session. On a timer it asks — through the broker (`GET /credit`, since the runner holds
|
|
4
|
-
// no
|
|
4
|
+
// no billing credential) — whether the org is still funded; the FIRST time it isn't, it fires
|
|
5
5
|
// `onExhausted` once (the orchestrator wires this to `AbortController.abort`, which the WorkflowHost
|
|
6
6
|
// honors cooperatively) and stops checking. This is the org-level counterpart to the per-run
|
|
7
|
-
// BudgetMeter cap: the run can't see
|
|
7
|
+
// BudgetMeter cap: the run can't see its billing balance, so funding is enforced here, out of band,
|
|
8
8
|
// against the live balance that incremental token metering keeps fresh.
|
|
9
9
|
//
|
|
10
|
-
// "Prompt, not instant":
|
|
11
|
-
// abort at the next hook boundary, so a run stops within ~one check interval (plus
|
|
10
|
+
// "Prompt, not instant": the platform's usage metering is eventually-consistent and the host honors the
|
|
11
|
+
// abort at the next hook boundary, so a run stops within ~one check interval (plus metering lag) of
|
|
12
12
|
// going unfunded — bounding overshoot, not eliminating it. Applies to MANAGED and BYOK runs alike,
|
|
13
13
|
// since runtime always burns credit (the gate requires the runtime floor regardless of token billing).
|
|
14
14
|
import { createLogger } from "./support/index.js";
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import type { RelayChannel } from "./identity_relay.js";
|
|
3
|
+
import type { SuspendSignal } from "./suspension.js";
|
|
4
|
+
/** First retry delay after a suspend_abort (doubles per attempt, capped below). */
|
|
5
|
+
export declare const ABORT_RETRY_INITIAL_MS = 30000;
|
|
6
|
+
export declare const ABORT_RETRY_MAX_MS: number;
|
|
7
|
+
/** The wake value inside a wake injection — `kind` echoes the parked seam's reason; the
|
|
8
|
+
* per-kind fields are opaque to the coordinator and interpreted by the seam. */
|
|
9
|
+
export declare const wakeValueSchema: z.ZodObject<{
|
|
10
|
+
kind: z.ZodEnum<{
|
|
11
|
+
sleep: "sleep";
|
|
12
|
+
human_input: "human_input";
|
|
13
|
+
workflow_call: "workflow_call";
|
|
14
|
+
}>;
|
|
15
|
+
answers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
16
|
+
child: z.ZodOptional<z.ZodObject<{
|
|
17
|
+
run_id: z.ZodString;
|
|
18
|
+
status: z.ZodString;
|
|
19
|
+
output: z.ZodUnknown;
|
|
20
|
+
}, z.core.$strip>>;
|
|
21
|
+
}, z.core.$strip>;
|
|
22
|
+
export type WakeValue = z.infer<typeof wakeValueSchema>;
|
|
23
|
+
/** The wake-injection payload as init relays it (snake_case, the platform env contract's
|
|
24
|
+
* sibling): fresh tokens — the frozen ones expired while suspended — plus the
|
|
25
|
+
* authoritative wall clock (the guest's own clock was stopped) and the wake value. */
|
|
26
|
+
export declare const wakePayloadSchema: z.ZodObject<{
|
|
27
|
+
run_token: z.ZodString;
|
|
28
|
+
api_token: z.ZodOptional<z.ZodString>;
|
|
29
|
+
wall_clock_ms: z.ZodNumber;
|
|
30
|
+
wake: z.ZodObject<{
|
|
31
|
+
kind: z.ZodEnum<{
|
|
32
|
+
sleep: "sleep";
|
|
33
|
+
human_input: "human_input";
|
|
34
|
+
workflow_call: "workflow_call";
|
|
35
|
+
}>;
|
|
36
|
+
answers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
37
|
+
child: z.ZodOptional<z.ZodObject<{
|
|
38
|
+
run_id: z.ZodString;
|
|
39
|
+
status: z.ZodString;
|
|
40
|
+
output: z.ZodUnknown;
|
|
41
|
+
}, z.core.$strip>>;
|
|
42
|
+
}, z.core.$strip>;
|
|
43
|
+
}, z.core.$strip>;
|
|
44
|
+
export type WakePayload = z.infer<typeof wakePayloadSchema>;
|
|
45
|
+
/** What a suspending wait resolves to: the wake (the normal path), or — for a `sleep` seam
|
|
46
|
+
* only — the abort that tells it to hold its remainder in-process (other reasons retry the
|
|
47
|
+
* freeze internally and never surface an abort). */
|
|
48
|
+
export type FreezeOutcome = {
|
|
49
|
+
kind: "wake";
|
|
50
|
+
wake: WakeValue;
|
|
51
|
+
} | {
|
|
52
|
+
kind: "aborted";
|
|
53
|
+
reason: string;
|
|
54
|
+
}
|
|
55
|
+
/** The caller withdrew the wait via its abort signal BEFORE it froze (register-without-release:
|
|
56
|
+
* a held gate got its answer during the hold, so it resolves in-process instead of freezing). */
|
|
57
|
+
| {
|
|
58
|
+
kind: "withdrawn";
|
|
59
|
+
};
|
|
60
|
+
export interface FreezeCoordinatorHooks {
|
|
61
|
+
/** Runs at quiescence, immediately before the freeze is requested: flush billable runtime
|
|
62
|
+
* (suspended time must never appear billed) and persist the workspace. Its own failure
|
|
63
|
+
* aborts THIS freeze attempt (the seam holds/retries) — never the run. */
|
|
64
|
+
onBeforeFreeze?: () => Promise<void>;
|
|
65
|
+
/** Runs when a wake lands, before the seam resolves: swap the run/api tokens onto the
|
|
66
|
+
* broker client and rebase the runtime meter past the frozen window. */
|
|
67
|
+
onAfterWake?: (wake: WakePayload) => void | Promise<void>;
|
|
68
|
+
}
|
|
69
|
+
export interface FreezeCoordinatorDeps {
|
|
70
|
+
channel: RelayChannel;
|
|
71
|
+
now?: () => number;
|
|
72
|
+
/** Injected delay (tests). Defaults to a real timer. */
|
|
73
|
+
delay?: (ms: number) => Promise<void>;
|
|
74
|
+
}
|
|
75
|
+
export declare class FreezeCoordinator {
|
|
76
|
+
private readonly deps;
|
|
77
|
+
private readonly now;
|
|
78
|
+
private readonly delay;
|
|
79
|
+
private hooks;
|
|
80
|
+
/** Non-suspending runtime work in flight (the quiescence gate's count). */
|
|
81
|
+
private inFlight;
|
|
82
|
+
private quiescenceWaiters;
|
|
83
|
+
/** True from freeze request until wake/abort — new runtime work queues behind it. */
|
|
84
|
+
private freezePending;
|
|
85
|
+
private gateWaiters;
|
|
86
|
+
/** The parked suspending wait (at most one — waits serialize). */
|
|
87
|
+
private parked;
|
|
88
|
+
/** Serializes suspending waits. */
|
|
89
|
+
private chain;
|
|
90
|
+
constructor(deps: FreezeCoordinatorDeps);
|
|
91
|
+
/** Late-bound per-run hooks (the flusher/broker/redactor exist only once a run is claimed). */
|
|
92
|
+
setHooks(hooks: FreezeCoordinatorHooks): void;
|
|
93
|
+
/**
|
|
94
|
+
* Run one unit of non-suspending runtime work under the gate: queue while a freeze is
|
|
95
|
+
* pending (the work never starts, so nothing can be torn by the pause), then count it
|
|
96
|
+
* in-flight so a suspending wait holds until it drains. The gate check and the count
|
|
97
|
+
* increment share one synchronous segment — a freeze requested in between cannot miss us.
|
|
98
|
+
*/
|
|
99
|
+
trackWork<T>(fn: () => Promise<T>): Promise<T>;
|
|
100
|
+
/** A suspending seam's own wait is NOT "work" — it decrements around its park so the gate
|
|
101
|
+
* sees true quiescence. (The host wraps every hook in trackWork; the suspending seams
|
|
102
|
+
* call these around their freeze wait.) */
|
|
103
|
+
beginWork(): void;
|
|
104
|
+
endWork(): void;
|
|
105
|
+
/**
|
|
106
|
+
* The suspending wait: hold until quiescence, run the pre-freeze hook, request the
|
|
107
|
+
* freeze, and park. The next thing this promise sees is a wake (possibly epochs later,
|
|
108
|
+
* through a restored heap) or a suspend_abort. Returns the wake, or — for `sleep` only —
|
|
109
|
+
* the abort (the seam then holds in-process); other reasons retry the freeze after a
|
|
110
|
+
* backoff. Concurrent suspending waits serialize through an internal chain.
|
|
111
|
+
*/
|
|
112
|
+
suspendingWait(signal: SuspendSignal, abort?: AbortSignal): Promise<FreezeOutcome>;
|
|
113
|
+
/** Relay handler: a wake injection landed. Validates, runs the after-wake hook (token
|
|
114
|
+
* swap + meter rebase), confirms to init, and resolves the parked seam. A wake with no
|
|
115
|
+
* parked seam is a duplicate delivery — re-confirm (idempotent), never crash. */
|
|
116
|
+
onWake(payload: unknown): void;
|
|
117
|
+
/** Relay handler: the snapshot attempt failed; the parked seam falls back to holding. */
|
|
118
|
+
onSuspendAbort(payload: unknown): void;
|
|
119
|
+
private awaitQuiescence;
|
|
120
|
+
private releaseGate;
|
|
121
|
+
/** The host-readable wake summary on the wire (logs/metrics/placement) — the opaque
|
|
122
|
+
* broker_signal beside it carries the full condition. */
|
|
123
|
+
private wakeConditionOf;
|
|
124
|
+
}
|
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
// FreezeCoordinator — the runtime's half of snapshot suspension on the microVM substrate.
|
|
2
|
+
//
|
|
3
|
+
// On this substrate the worker process NEVER exits to suspend: a suspending seam (a long
|
|
4
|
+
// sleep, a human-input gate, a child-run wait) BLOCKS on a real promise, the platform
|
|
5
|
+
// freezes the whole VM into a snapshot, and a later restore resolves that same promise with
|
|
6
|
+
// the wake value — the heap is the literal heap, nothing replays. This coordinator owns the
|
|
7
|
+
// two policies that make freezing safe and predictable:
|
|
8
|
+
//
|
|
9
|
+
// 1. THE QUIESCENCE GATE: never freeze while non-suspending runtime work (an agent leaf, a
|
|
10
|
+
// tool call, an artifact write) is in flight — a snapshot must not capture a live
|
|
11
|
+
// platform stream that would be dead on restore. A suspending wait created during such
|
|
12
|
+
// work HOLDS until the work drains; new runtime work that arrives while a freeze is
|
|
13
|
+
// pending QUEUES (it never started, so nothing is torn) and runs after the wake.
|
|
14
|
+
// 2. SNAPSHOT-FIRST FALLBACK: a `suspend_abort` from the platform (snapshot/store/broker
|
|
15
|
+
// failure) means the seam falls back to HOLDING — a sleep waits in-process for its
|
|
16
|
+
// remainder; a human-input or child wait retries the freeze after a backoff (the host
|
|
17
|
+
// may throttle repeated attempts).
|
|
18
|
+
//
|
|
19
|
+
// Concurrent suspending waits SERIALIZE: the first to reach quiescence freezes with its own
|
|
20
|
+
// wake condition; after its wake, the next takes its turn. (A compound wake condition —
|
|
21
|
+
// one freeze covering several waits — is a future optimization, not a correctness need:
|
|
22
|
+
// the heap survives every cycle.)
|
|
23
|
+
//
|
|
24
|
+
// The wire below this is the guest init's relay (identity_relay `openChannel`); the policy
|
|
25
|
+
// above it is decided platform-side. The coordinator never interprets the wake VALUE — it
|
|
26
|
+
// validates the envelope and hands `wake` to the seam that parked.
|
|
27
|
+
import { z } from "zod";
|
|
28
|
+
import { createLogger } from "./support/index.js";
|
|
29
|
+
const log = createLogger("FreezeCoordinator");
|
|
30
|
+
/** First retry delay after a suspend_abort (doubles per attempt, capped below). */
|
|
31
|
+
export const ABORT_RETRY_INITIAL_MS = 30_000;
|
|
32
|
+
export const ABORT_RETRY_MAX_MS = 5 * 60_000;
|
|
33
|
+
/** The wake value inside a wake injection — `kind` echoes the parked seam's reason; the
|
|
34
|
+
* per-kind fields are opaque to the coordinator and interpreted by the seam. */
|
|
35
|
+
export const wakeValueSchema = z.object({
|
|
36
|
+
kind: z.enum(["sleep", "human_input", "workflow_call"]),
|
|
37
|
+
/** human_input: EVERY gate this suspension raised, keyed by gate key / tool-call id. */
|
|
38
|
+
answers: z.record(z.string(), z.unknown()).optional(),
|
|
39
|
+
/** workflow_call: the finalized child. */
|
|
40
|
+
child: z
|
|
41
|
+
.object({
|
|
42
|
+
run_id: z.string().min(1),
|
|
43
|
+
status: z.string().min(1),
|
|
44
|
+
output: z.unknown(),
|
|
45
|
+
})
|
|
46
|
+
.optional(),
|
|
47
|
+
});
|
|
48
|
+
/** The wake-injection payload as init relays it (snake_case, the platform env contract's
|
|
49
|
+
* sibling): fresh tokens — the frozen ones expired while suspended — plus the
|
|
50
|
+
* authoritative wall clock (the guest's own clock was stopped) and the wake value. */
|
|
51
|
+
export const wakePayloadSchema = z.object({
|
|
52
|
+
run_token: z.string().min(1),
|
|
53
|
+
api_token: z.string().optional(),
|
|
54
|
+
wall_clock_ms: z.number(),
|
|
55
|
+
wake: wakeValueSchema,
|
|
56
|
+
});
|
|
57
|
+
const suspendAbortSchema = z.object({ reason: z.string().optional() });
|
|
58
|
+
export class FreezeCoordinator {
|
|
59
|
+
deps;
|
|
60
|
+
now;
|
|
61
|
+
delay;
|
|
62
|
+
hooks = {};
|
|
63
|
+
/** Non-suspending runtime work in flight (the quiescence gate's count). */
|
|
64
|
+
inFlight = 0;
|
|
65
|
+
quiescenceWaiters = [];
|
|
66
|
+
/** True from freeze request until wake/abort — new runtime work queues behind it. */
|
|
67
|
+
freezePending = false;
|
|
68
|
+
gateWaiters = [];
|
|
69
|
+
/** The parked suspending wait (at most one — waits serialize). */
|
|
70
|
+
parked = null;
|
|
71
|
+
/** Serializes suspending waits. */
|
|
72
|
+
chain = Promise.resolve();
|
|
73
|
+
constructor(deps) {
|
|
74
|
+
this.deps = deps;
|
|
75
|
+
this.now = deps.now ?? Date.now;
|
|
76
|
+
// A pending retry delay legitimately keeps the process alive — the run isn't done.
|
|
77
|
+
this.delay =
|
|
78
|
+
deps.delay ??
|
|
79
|
+
((ms) => new Promise((resolve) => {
|
|
80
|
+
setTimeout(resolve, ms);
|
|
81
|
+
}));
|
|
82
|
+
}
|
|
83
|
+
/** Late-bound per-run hooks (the flusher/broker/redactor exist only once a run is claimed). */
|
|
84
|
+
setHooks(hooks) {
|
|
85
|
+
this.hooks = hooks;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Run one unit of non-suspending runtime work under the gate: queue while a freeze is
|
|
89
|
+
* pending (the work never starts, so nothing can be torn by the pause), then count it
|
|
90
|
+
* in-flight so a suspending wait holds until it drains. The gate check and the count
|
|
91
|
+
* increment share one synchronous segment — a freeze requested in between cannot miss us.
|
|
92
|
+
*/
|
|
93
|
+
async trackWork(fn) {
|
|
94
|
+
while (this.freezePending) {
|
|
95
|
+
await new Promise((resolve) => {
|
|
96
|
+
this.gateWaiters.push(resolve);
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
this.beginWork();
|
|
100
|
+
try {
|
|
101
|
+
return await fn();
|
|
102
|
+
}
|
|
103
|
+
finally {
|
|
104
|
+
this.endWork();
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/** A suspending seam's own wait is NOT "work" — it decrements around its park so the gate
|
|
108
|
+
* sees true quiescence. (The host wraps every hook in trackWork; the suspending seams
|
|
109
|
+
* call these around their freeze wait.) */
|
|
110
|
+
beginWork() {
|
|
111
|
+
this.inFlight += 1;
|
|
112
|
+
}
|
|
113
|
+
endWork() {
|
|
114
|
+
this.inFlight -= 1;
|
|
115
|
+
if (this.inFlight === 0) {
|
|
116
|
+
const waiters = this.quiescenceWaiters;
|
|
117
|
+
this.quiescenceWaiters = [];
|
|
118
|
+
for (const w of waiters)
|
|
119
|
+
w();
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* The suspending wait: hold until quiescence, run the pre-freeze hook, request the
|
|
124
|
+
* freeze, and park. The next thing this promise sees is a wake (possibly epochs later,
|
|
125
|
+
* through a restored heap) or a suspend_abort. Returns the wake, or — for `sleep` only —
|
|
126
|
+
* the abort (the seam then holds in-process); other reasons retry the freeze after a
|
|
127
|
+
* backoff. Concurrent suspending waits serialize through an internal chain.
|
|
128
|
+
*/
|
|
129
|
+
suspendingWait(signal, abort) {
|
|
130
|
+
const turn = this.chain;
|
|
131
|
+
let release = () => undefined;
|
|
132
|
+
this.chain = new Promise((resolve) => {
|
|
133
|
+
release = resolve;
|
|
134
|
+
});
|
|
135
|
+
return (async () => {
|
|
136
|
+
await turn;
|
|
137
|
+
try {
|
|
138
|
+
let backoff = ABORT_RETRY_INITIAL_MS;
|
|
139
|
+
for (;;) {
|
|
140
|
+
// Withdraw window: the abort can cancel the wait any time BEFORE it sends
|
|
141
|
+
// suspend_request (once sent, the VM freezes and the abort is moot — the process is
|
|
142
|
+
// paused). This is register-without-release: a held gate answered during the hold.
|
|
143
|
+
// awaitQuiescence short-circuits when already aborted, so one check after it covers
|
|
144
|
+
// both loop entry and a mid-hold abort.
|
|
145
|
+
await this.awaitQuiescence(abort);
|
|
146
|
+
if (abort?.aborted === true)
|
|
147
|
+
return { kind: "withdrawn" };
|
|
148
|
+
try {
|
|
149
|
+
await this.hooks.onBeforeFreeze?.();
|
|
150
|
+
}
|
|
151
|
+
catch (err) {
|
|
152
|
+
// A failed pre-freeze flush must not strand the seam: treat it like an abort.
|
|
153
|
+
log.warn("freeze_prepare_failed", {
|
|
154
|
+
error: err instanceof Error ? err.message : String(err),
|
|
155
|
+
});
|
|
156
|
+
if (signal.reason === "sleep")
|
|
157
|
+
return { kind: "aborted", reason: "prepare_failed" };
|
|
158
|
+
await this.delay(backoff);
|
|
159
|
+
backoff = Math.min(backoff * 2, ABORT_RETRY_MAX_MS);
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
this.freezePending = true;
|
|
163
|
+
const outcome = await new Promise((resolve) => {
|
|
164
|
+
this.parked = resolve;
|
|
165
|
+
this.deps.channel.sendSuspendRequest({
|
|
166
|
+
reason: signal.reason,
|
|
167
|
+
wake: this.wakeConditionOf(signal),
|
|
168
|
+
broker_signal: signal,
|
|
169
|
+
});
|
|
170
|
+
// ← the VM freezes while this promise is pending; the wake resolves it with the
|
|
171
|
+
// heap (and this very closure) restored.
|
|
172
|
+
});
|
|
173
|
+
this.freezePending = false;
|
|
174
|
+
this.releaseGate();
|
|
175
|
+
if (outcome.kind === "wake")
|
|
176
|
+
return outcome;
|
|
177
|
+
// The park only ever resolves wake or aborted (never withdrawn — that returns before the
|
|
178
|
+
// park). Narrow explicitly so `.reason` is safe and a stray outcome propagates.
|
|
179
|
+
if (outcome.kind !== "aborted")
|
|
180
|
+
return outcome;
|
|
181
|
+
if (signal.reason === "sleep")
|
|
182
|
+
return outcome;
|
|
183
|
+
log.warn("suspend_aborted_retrying", {
|
|
184
|
+
reason: outcome.reason,
|
|
185
|
+
seamReason: signal.reason,
|
|
186
|
+
backoffMs: backoff,
|
|
187
|
+
});
|
|
188
|
+
await this.delay(backoff);
|
|
189
|
+
backoff = Math.min(backoff * 2, ABORT_RETRY_MAX_MS);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
finally {
|
|
193
|
+
release();
|
|
194
|
+
}
|
|
195
|
+
})();
|
|
196
|
+
}
|
|
197
|
+
/** Relay handler: a wake injection landed. Validates, runs the after-wake hook (token
|
|
198
|
+
* swap + meter rebase), confirms to init, and resolves the parked seam. A wake with no
|
|
199
|
+
* parked seam is a duplicate delivery — re-confirm (idempotent), never crash. */
|
|
200
|
+
onWake(payload) {
|
|
201
|
+
const parsed = wakePayloadSchema.safeParse(payload);
|
|
202
|
+
if (!parsed.success) {
|
|
203
|
+
// Unanswerable garbage: init will retry, time out, and the platform's crash path owns
|
|
204
|
+
// recovery. Log loudly — this is a control-plane/init bug, not author code.
|
|
205
|
+
log.error("wake_payload_invalid", { issues: parsed.error.message });
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
if (this.parked === null) {
|
|
209
|
+
log.warn("duplicate_wake_ignored", {});
|
|
210
|
+
this.deps.channel.sendWakeAccepted();
|
|
211
|
+
return;
|
|
212
|
+
}
|
|
213
|
+
const resolve = this.parked;
|
|
214
|
+
this.parked = null;
|
|
215
|
+
void (async () => {
|
|
216
|
+
try {
|
|
217
|
+
await this.hooks.onAfterWake?.(parsed.data);
|
|
218
|
+
}
|
|
219
|
+
catch (err) {
|
|
220
|
+
// Token swap / meter rebase failing is survivable only loudly — the run continues
|
|
221
|
+
// on the old token and fails fast if it truly expired.
|
|
222
|
+
log.error("after_wake_hook_failed", {
|
|
223
|
+
error: err instanceof Error ? err.message : String(err),
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
this.deps.channel.sendWakeAccepted();
|
|
227
|
+
resolve({ kind: "wake", wake: parsed.data.wake });
|
|
228
|
+
})();
|
|
229
|
+
}
|
|
230
|
+
/** Relay handler: the snapshot attempt failed; the parked seam falls back to holding. */
|
|
231
|
+
onSuspendAbort(payload) {
|
|
232
|
+
const parsed = suspendAbortSchema.safeParse(payload);
|
|
233
|
+
const reason = parsed.success ? (parsed.data.reason ?? "unknown") : "unknown";
|
|
234
|
+
if (this.parked === null) {
|
|
235
|
+
log.warn("suspend_abort_without_parked_seam", { reason });
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
const resolve = this.parked;
|
|
239
|
+
this.parked = null;
|
|
240
|
+
resolve({ kind: "aborted", reason });
|
|
241
|
+
}
|
|
242
|
+
awaitQuiescence(abort) {
|
|
243
|
+
if (this.inFlight === 0 || abort?.aborted === true)
|
|
244
|
+
return Promise.resolve();
|
|
245
|
+
return new Promise((resolve) => {
|
|
246
|
+
// Resolve on quiescence OR abort (a held wait wakes immediately to withdraw). Resolving
|
|
247
|
+
// twice is harmless; endWork may still call the queued waiter later.
|
|
248
|
+
this.quiescenceWaiters.push(resolve);
|
|
249
|
+
abort?.addEventListener("abort", () => resolve(), { once: true });
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
releaseGate() {
|
|
253
|
+
const waiters = this.gateWaiters;
|
|
254
|
+
this.gateWaiters = [];
|
|
255
|
+
for (const w of waiters)
|
|
256
|
+
w();
|
|
257
|
+
}
|
|
258
|
+
/** The host-readable wake summary on the wire (logs/metrics/placement) — the opaque
|
|
259
|
+
* broker_signal beside it carries the full condition. */
|
|
260
|
+
wakeConditionOf(signal) {
|
|
261
|
+
switch (signal.reason) {
|
|
262
|
+
case "sleep":
|
|
263
|
+
return { kind: "sleep", wake_at_ms: this.now() + (signal.durationMs ?? 0) };
|
|
264
|
+
case "human_input":
|
|
265
|
+
return {
|
|
266
|
+
kind: "human_input",
|
|
267
|
+
...(signal.humanInput !== undefined ? { request_keys: [signal.humanInput.key] } : {}),
|
|
268
|
+
};
|
|
269
|
+
case "workflow_call":
|
|
270
|
+
return {
|
|
271
|
+
kind: "workflow_call",
|
|
272
|
+
...(signal.childRunId !== undefined ? { child_run_id: signal.childRunId } : {}),
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
}
|
|
@@ -3,6 +3,11 @@ import { z } from "zod";
|
|
|
3
3
|
/** Env var naming the inherited relay fd. Set by the guest init; absent everywhere else
|
|
4
4
|
* (Fargate, self-hosted daemon), where the worker env-boots as always. */
|
|
5
5
|
export declare const RELAY_FD_ENV = "BOARDWALK_IDENTITY_RELAY_FD";
|
|
6
|
+
/** Max bytes of one relay line — the wire protocol's 32 MiB frame cap, mirrored in-guest.
|
|
7
|
+
* An oversized line costs the connection (LF framing cannot resynchronize past it), which
|
|
8
|
+
* for this relay means a hard bootstrap failure. Measured in UTF-16 code units, which for
|
|
9
|
+
* this ASCII-JSON wire is the byte count; the cap is a guard, not exact accounting. */
|
|
10
|
+
export declare const MAX_RELAY_LINE_BYTES: number;
|
|
6
11
|
/** The identity payload — the env contract's fields, as JSON instead of env vars. */
|
|
7
12
|
export declare const relayIdentitySchema: z.ZodObject<{
|
|
8
13
|
run_id: z.ZodString;
|
|
@@ -14,6 +19,19 @@ export declare const relayIdentitySchema: z.ZodObject<{
|
|
|
14
19
|
env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
15
20
|
}, z.core.$strip>;
|
|
16
21
|
export type RelayIdentity = z.infer<typeof relayIdentitySchema>;
|
|
22
|
+
/** The init→worker halves of the post-identity relay: `wake` resolves a frozen suspending
|
|
23
|
+
* seam (payload: the wake-inject body, verbatim), `suspend_abort` tells a parked seam the
|
|
24
|
+
* snapshot attempt failed and it should fall back to holding. */
|
|
25
|
+
export interface RelayChannelHandlers {
|
|
26
|
+
onWake(payload: unknown): void;
|
|
27
|
+
onSuspendAbort(payload: unknown): void;
|
|
28
|
+
}
|
|
29
|
+
/** The worker→init halves: ask to be frozen (payload: the suspend-request body — reason,
|
|
30
|
+
* wake summary, opaque broker signal), and confirm a wake landed (init then acks its host). */
|
|
31
|
+
export interface RelayChannel {
|
|
32
|
+
sendSuspendRequest(payload: unknown): void;
|
|
33
|
+
sendWakeAccepted(): void;
|
|
34
|
+
}
|
|
17
35
|
/**
|
|
18
36
|
* Read the relay fd from `env`, deleting the key (bootstrap-only plumbing; run code has no
|
|
19
37
|
* business seeing it). Returns null when unset — the normal env-boot path.
|
|
@@ -24,15 +42,30 @@ export declare function relayFdFromEnv(env: NodeJS.ProcessEnv): number | null;
|
|
|
24
42
|
* the platform keys LAST, so nothing user-supplied can shadow a platform value.
|
|
25
43
|
*/
|
|
26
44
|
export declare function applyIdentityToEnv(identity: RelayIdentity, env: NodeJS.ProcessEnv): void;
|
|
45
|
+
/** The worker_ready diagnostics — supplied by the worker (init cannot know these),
|
|
46
|
+
* forwarded verbatim by the guest init. Best-effort, never load-bearing. */
|
|
47
|
+
export interface WorkerDiagnostics {
|
|
48
|
+
worker_version?: string;
|
|
49
|
+
node_version: string;
|
|
50
|
+
sdk_version?: string;
|
|
51
|
+
}
|
|
52
|
+
/** Collect the worker_ready diagnostics. Version lookups are best-effort: the runner's own
|
|
53
|
+
* package.json sits two levels above this module in both the src and published dist
|
|
54
|
+
* layouts; the SDK's is reachable only if its exports expose ./package.json. */
|
|
55
|
+
export declare function workerDiagnostics(): WorkerDiagnostics;
|
|
27
56
|
/** One end of the init↔worker relay. Wraps any Duplex so tests run over in-memory streams. */
|
|
28
57
|
export declare class IdentityRelay {
|
|
29
58
|
private readonly stream;
|
|
30
59
|
private buffer;
|
|
31
60
|
private closed;
|
|
61
|
+
private failure;
|
|
32
62
|
private wake;
|
|
63
|
+
private readonly onData;
|
|
33
64
|
constructor(stream: Duplex);
|
|
34
|
-
|
|
35
|
-
|
|
65
|
+
private fail;
|
|
66
|
+
/** Announce the pre-identity park (with the diagnostics payload when provided).
|
|
67
|
+
* Init forwards this as the base-snapshot gate. */
|
|
68
|
+
announceReady(diagnostics?: WorkerDiagnostics): void;
|
|
36
69
|
/**
|
|
37
70
|
* Block until init relays the run identity. THIS is the point the base snapshot freezes:
|
|
38
71
|
* everything before it is generic warm-up shared by every run; everything after belongs
|
|
@@ -43,6 +76,16 @@ export declare class IdentityRelay {
|
|
|
43
76
|
awaitIdentity(): Promise<RelayIdentity>;
|
|
44
77
|
/** Confirm capture. Init acks its host only after this arrives. */
|
|
45
78
|
acceptIdentity(): void;
|
|
79
|
+
/**
|
|
80
|
+
* Attach the post-identity consumer: the suspend/wake channel (the same fd, the same JSON
|
|
81
|
+
* lines). From here the relay dispatches init→worker messages to the handlers —
|
|
82
|
+
* `wake` (resolve a frozen seam) and `suspend_abort` (the seam falls back to holding) —
|
|
83
|
+
* and the returned channel sends the worker→init halves (`suspend_request`,
|
|
84
|
+
* `wake_accepted`). Unknown types are ignored (forward-compatible); malformed lines are
|
|
85
|
+
* logged and skipped, exactly as pre-identity. Call once, after {@link acceptIdentity}.
|
|
86
|
+
*/
|
|
87
|
+
openChannel(handlers: RelayChannelHandlers): RelayChannel;
|
|
88
|
+
private pumpChannel;
|
|
46
89
|
private writeLine;
|
|
47
90
|
private readLine;
|
|
48
91
|
}
|