@relayflows/sdk 2.0.12 → 2.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/authored-budget.d.ts.map +1 -1
- package/dist/authored-budget.js +16 -7
- package/dist/authored-budget.js.map +1 -1
- package/dist/authored-flow-executor.d.ts.map +1 -1
- package/dist/authored-flow-executor.js +5 -0
- package/dist/authored-flow-executor.js.map +1 -1
- package/dist/budget-preflight.d.ts +17 -6
- package/dist/budget-preflight.d.ts.map +1 -1
- package/dist/budget-preflight.js +9 -6
- package/dist/budget-preflight.js.map +1 -1
- package/dist/cli/build.d.ts.map +1 -1
- package/dist/cli/build.js +8 -0
- package/dist/cli/build.js.map +1 -1
- package/dist/cli/deploy.d.ts.map +1 -1
- package/dist/cli/deploy.js +5 -0
- package/dist/cli/deploy.js.map +1 -1
- package/dist/cli/direct-run.d.ts.map +1 -1
- package/dist/cli/direct-run.js +28 -0
- package/dist/cli/direct-run.js.map +1 -1
- package/dist/cli/run.d.ts +8 -0
- package/dist/cli/run.d.ts.map +1 -1
- package/dist/cli/run.js +34 -11
- package/dist/cli/run.js.map +1 -1
- package/dist/cli/step-failure.d.ts +40 -0
- package/dist/cli/step-failure.d.ts.map +1 -0
- package/dist/cli/step-failure.js +150 -0
- package/dist/cli/step-failure.js.map +1 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +4 -4
- package/dist/cli.js.map +1 -1
- package/dist/daemon-connection.d.ts +7 -0
- package/dist/daemon-connection.d.ts.map +1 -1
- package/dist/daemon-connection.js +7 -0
- package/dist/daemon-connection.js.map +1 -1
- package/dist/failure-kinds.d.ts +25 -3
- package/dist/failure-kinds.d.ts.map +1 -1
- package/dist/failure-kinds.js +5 -2
- package/dist/failure-kinds.js.map +1 -1
- package/dist/journal-client.d.ts +2 -6
- package/dist/journal-client.d.ts.map +1 -1
- package/dist/journal-client.js.map +1 -1
- package/dist/model-pricing.d.ts +7 -5
- package/dist/model-pricing.d.ts.map +1 -1
- package/dist/model-pricing.js +7 -5
- package/dist/model-pricing.js.map +1 -1
- package/dist/preflight.d.ts +5 -0
- package/dist/preflight.d.ts.map +1 -1
- package/dist/preflight.js.map +1 -1
- package/dist/protocol.d.ts +22 -5
- package/dist/protocol.d.ts.map +1 -1
- package/dist/spec.d.ts +22 -7
- package/dist/spec.d.ts.map +1 -1
- package/dist/worker-spend.d.ts +13 -9
- package/dist/worker-spend.d.ts.map +1 -1
- package/dist/worker-spend.js +22 -8
- package/dist/worker-spend.js.map +1 -1
- package/package.json +2 -2
- package/src/authored-budget.ts +35 -9
- package/src/authored-flow-executor.ts +5 -0
- package/src/budget-preflight.ts +17 -6
- package/src/cli/build.ts +7 -0
- package/src/cli/deploy.ts +4 -0
- package/src/cli/direct-run.ts +28 -0
- package/src/cli/run.ts +43 -11
- package/src/cli/step-failure.ts +160 -0
- package/src/cli.ts +4 -4
- package/src/daemon-connection.ts +8 -0
- package/src/failure-kinds.ts +25 -3
- package/src/journal-client.ts +2 -1
- package/src/model-pricing.ts +7 -5
- package/src/preflight.ts +5 -0
- package/src/protocol.ts +15 -2
- package/src/spec.ts +23 -1
- package/src/worker-spend.ts +25 -9
- package/dist/cli/deterministic-failure.d.ts +0 -5
- package/dist/cli/deterministic-failure.d.ts.map +0 -1
- package/dist/cli/deterministic-failure.js +0 -66
- package/dist/cli/deterministic-failure.js.map +0 -1
- package/src/cli/deterministic-failure.ts +0 -69
package/src/authored-budget.ts
CHANGED
|
@@ -2,14 +2,31 @@ import { BudgetSyntaxError, parseBudget, toKernelBudget } from './budget.js';
|
|
|
2
2
|
import { AuthoredFlowExecutionError } from './authored-flow-error.js';
|
|
3
3
|
import type { JournalClient } from './journal-client.js';
|
|
4
4
|
import type { RunOutcome } from './protocol.js';
|
|
5
|
-
import type { KernelBudgetSpec, KernelRunSpec } from './spec.js';
|
|
5
|
+
import type { KernelBudgetSpec, KernelPriorSpend, KernelRunSpec } from './spec.js';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* One journaled charge, in the accumulator's exact integer form.
|
|
9
|
+
*
|
|
10
|
+
* `unmetered` mirrors the journal's `budget.dollars_unmetered`: the charge
|
|
11
|
+
* spent tokens whose dollar cost is unknown, so its `micro` is a lower bound
|
|
12
|
+
* and not a measured amount. It is carried, not derived from `micro`, because
|
|
13
|
+
* an unmetered charge and a genuinely free charge both report zero dollars.
|
|
14
|
+
*/
|
|
15
|
+
interface Charge {
|
|
16
|
+
input: bigint;
|
|
17
|
+
output: bigint;
|
|
18
|
+
micro: bigint;
|
|
19
|
+
ms: bigint;
|
|
20
|
+
day: number;
|
|
21
|
+
unmetered: boolean;
|
|
22
|
+
}
|
|
6
23
|
|
|
7
24
|
/** Serialized admission for the internal authored runner's separate step runs. */
|
|
8
25
|
export class AuthoredBudget {
|
|
9
26
|
private readonly limit: KernelBudgetSpec | undefined;
|
|
10
27
|
private failed = false;
|
|
11
28
|
private tail: Promise<unknown> = Promise.resolve();
|
|
12
|
-
private charges:
|
|
29
|
+
private charges: Charge[] = [];
|
|
13
30
|
|
|
14
31
|
constructor(header: unknown) {
|
|
15
32
|
try {
|
|
@@ -32,14 +49,22 @@ export class AuthoredBudget {
|
|
|
32
49
|
// Window selection follows journal timestamps, never the SDK host clock.
|
|
33
50
|
const day = this.charges.at(-1)?.day;
|
|
34
51
|
const total = this.charges.filter(c => this.limit!.window !== 'day' || c.day === day)
|
|
35
|
-
.reduce((s, c) => ({ input: s.input + c.input, output: s.output + c.output, micro: s.micro + c.micro, ms: s.ms + c.ms
|
|
36
|
-
|
|
52
|
+
.reduce((s, c) => ({ input: s.input + c.input, output: s.output + c.output, micro: s.micro + c.micro, ms: s.ms + c.ms,
|
|
53
|
+
// Sticky, exactly as the kernel's running total is: once any charge
|
|
54
|
+
// in the window was unmetered, the carried dollars are a lower bound.
|
|
55
|
+
unmetered: s.unmetered || c.unmetered }),
|
|
56
|
+
{ input: 0n, output: 0n, micro: 0n, ms: 0n, unmetered: false });
|
|
37
57
|
const exactNumber = (n: bigint) => { if (n > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error('budget counter overflow'); return Number(n); };
|
|
38
|
-
|
|
58
|
+
// One explicit conversion to the shared carried-spend shape, so a new
|
|
59
|
+
// accounting field is added here rather than silently dropped inline.
|
|
60
|
+
const priorSpend: KernelPriorSpend = {
|
|
39
61
|
tokens_in: exactNumber(total.input), tokens_out: exactNumber(total.output),
|
|
40
62
|
dollars: `${total.micro / 1_000_000n}.${String(total.micro % 1_000_000n).padStart(6, '0')}`,
|
|
41
|
-
wallclock_ms: exactNumber(total.ms),
|
|
42
|
-
|
|
63
|
+
wallclock_ms: exactNumber(total.ms),
|
|
64
|
+
...(this.limit.window === 'day' && day !== undefined ? { day } : {}),
|
|
65
|
+
...(total.unmetered ? { dollars_unmetered: true as const } : {}),
|
|
66
|
+
};
|
|
67
|
+
const outcome = await journal.runStart({ ...spec, budget: { ...this.limit, prior_spend: priorSpend } }, undefined, admissionKey);
|
|
43
68
|
try {
|
|
44
69
|
if (outcome.completion_reason === 'budget_exceeded') throw new AuthoredFlowExecutionError('step_failed', 'Flow budget exceeded before the next step.', 'budget_exceeded', outcome.run_id);
|
|
45
70
|
return await consume(outcome);
|
|
@@ -49,7 +74,7 @@ export class AuthoredBudget {
|
|
|
49
74
|
const { entries } = await journal.journalRead(outcome.run_id, seq);
|
|
50
75
|
if (entries.length === 0) break;
|
|
51
76
|
for (const raw of entries) {
|
|
52
|
-
const e = raw as {seq: number; entry_type: string; at_ms: number; payload: {budget?: {tokens_in: number; tokens_out: number; dollars: string}; spend?: {wallclock_ms: number}}};
|
|
77
|
+
const e = raw as {seq: number; entry_type: string; at_ms: number; payload: {budget?: {tokens_in: number; tokens_out: number; dollars: string; dollars_unmetered?: boolean}; spend?: {wallclock_ms: number}}};
|
|
53
78
|
seq = e.seq + 1;
|
|
54
79
|
if (!['step.completed', 'memory.injected'].includes(e.entry_type)) continue;
|
|
55
80
|
const b = e.payload.budget;
|
|
@@ -58,7 +83,8 @@ export class AuthoredBudget {
|
|
|
58
83
|
if (fraction.length > 6 && /[1-9]/.test(fraction.slice(6))) throw new Error('budget accounting requires microdollar precision');
|
|
59
84
|
this.charges.push({input: BigInt(b.tokens_in), output: BigInt(b.tokens_out),
|
|
60
85
|
micro: BigInt(whole!) * 1_000_000n + BigInt(fraction.slice(0, 6).padEnd(6, '0')),
|
|
61
|
-
ms: BigInt(e.payload.spend?.wallclock_ms ?? 0), day: Math.floor(e.at_ms / 86_400_000)
|
|
86
|
+
ms: BigInt(e.payload.spend?.wallclock_ms ?? 0), day: Math.floor(e.at_ms / 86_400_000),
|
|
87
|
+
unmetered: b.dollars_unmetered === true});
|
|
62
88
|
}
|
|
63
89
|
}
|
|
64
90
|
}
|
|
@@ -148,6 +148,11 @@ export async function executeAuthoredFlow<Input = undefined>(
|
|
|
148
148
|
const onProgress = options.onProgress;
|
|
149
149
|
const flowPath = options.flowPath ?? join(process.cwd(), 'flow.ts');
|
|
150
150
|
const waitOptions: RunLifecycleOptions = {
|
|
151
|
+
// Carried so a failed `f.agent` can name the journal that holds its
|
|
152
|
+
// evidence. Each authored worker call runs as its own kernel run, and
|
|
153
|
+
// without the data dir the diagnostic can name the run id but not where
|
|
154
|
+
// on disk to read it.
|
|
155
|
+
...(options.dataDir !== undefined ? { dataDir: options.dataDir } : {}),
|
|
151
156
|
...(options.signal !== undefined ? { signal: options.signal } : {}),
|
|
152
157
|
...(options.onWait !== undefined ? { onWait: options.onWait } : {}),
|
|
153
158
|
};
|
package/src/budget-preflight.ts
CHANGED
|
@@ -3,18 +3,29 @@ import type { PreflightWarning } from './preflight.js';
|
|
|
3
3
|
import { hasPricing } from './model-pricing.js';
|
|
4
4
|
import { resolveAdapterKind } from './adapters/index.js';
|
|
5
5
|
|
|
6
|
+
/**
|
|
7
|
+
* The resolved CLI/model pair budget pricing needs, deliberately narrower than
|
|
8
|
+
* `ResolvedCliModel`: pricing depends only on *which* model will run (priced or
|
|
9
|
+
* not) and on the CLI for the Codex wording — not on whether the model came
|
|
10
|
+
* from authoring or an adapter default, which `ResolvedCliModel.source` records
|
|
11
|
+
* for `flows check` reporting. Keeping `source` out means a future resolution
|
|
12
|
+
* rung cannot change pricing by provenance alone.
|
|
13
|
+
*/
|
|
6
14
|
export interface BudgetStepResolution {
|
|
7
15
|
readonly cli?: string;
|
|
8
16
|
readonly model?: string;
|
|
9
17
|
}
|
|
10
18
|
|
|
11
19
|
/**
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
20
|
+
* A missing price never refuses a run. Under a frozen dollar budget:
|
|
21
|
+
* - a priced step journals exact dollars and a crossed `maxDollars` stops the
|
|
22
|
+
* run in the kernel;
|
|
23
|
+
* - an unmetered step (no model, or no frozen price) runs and journals its
|
|
24
|
+
* tokens with `dollars_unmetered: true` (see `workerSpend`), so its unknown
|
|
25
|
+
* cost cannot cross `maxDollars` while its tokens still count toward token
|
|
26
|
+
* ceilings.
|
|
27
|
+
* This warning names each unmetered step so the gap is reported, not silent.
|
|
28
|
+
* Preflight `ok` stays true: warnings never refuse.
|
|
18
29
|
*
|
|
19
30
|
* Codex selects its own model when none is declared, so a Codex step is
|
|
20
31
|
* expected to be unmetered and says so rather than asking for a fake price.
|
package/src/cli/build.ts
CHANGED
|
@@ -62,6 +62,13 @@ export async function runBuild(args: BuildArgs, io: CliIo): Promise<0 | 2> {
|
|
|
62
62
|
emitBuildCheckReport(gate.report, args.json, io);
|
|
63
63
|
return 2;
|
|
64
64
|
}
|
|
65
|
+
// `ok` means "no refusal", not "no diagnostics". Build defers environment
|
|
66
|
+
// probes, so probe-shaped warnings (`command_unprovable`, ...) are expected
|
|
67
|
+
// here and stay in preflight.json; `budget_unmetered` does not depend on a
|
|
68
|
+
// probe and changes what the dollar budget enforces, so it is printed.
|
|
69
|
+
for (const diagnostic of gate.report.diagnostics) {
|
|
70
|
+
if (diagnostic.kind === 'budget_unmetered') io.stderr(`WARNING [${diagnostic.kind}] ${diagnostic.message}`);
|
|
71
|
+
}
|
|
65
72
|
io.stdout(await buildFlow(args.value, args.out ?? 'dist/flows', io.stderr));
|
|
66
73
|
return 0;
|
|
67
74
|
} catch (error) {
|
package/src/cli/deploy.ts
CHANGED
|
@@ -31,6 +31,10 @@ export async function runDeploy(args: DeployArgs, io: CliIo): Promise<0 | 1 | 2>
|
|
|
31
31
|
for (const diagnostic of checked.report.diagnostics) io.stderr(`${diagnostic.severity.toUpperCase()} [${diagnostic.kind}] ${diagnostic.message}`);
|
|
32
32
|
return 2;
|
|
33
33
|
}
|
|
34
|
+
// `ok` may still carry warnings (e.g. `budget_unmetered`); report them.
|
|
35
|
+
for (const diagnostic of checked.report.diagnostics) {
|
|
36
|
+
if (diagnostic.severity === 'warning') io.stderr(`WARNING [${diagnostic.kind}] ${diagnostic.message}`);
|
|
37
|
+
}
|
|
34
38
|
const target = bucketDirectory(args.to, ref);
|
|
35
39
|
if (await exists(target)) {
|
|
36
40
|
await verifyDigest(target, ref.digest);
|
package/src/cli/direct-run.ts
CHANGED
|
@@ -190,6 +190,34 @@ export async function runDirectFlow(
|
|
|
190
190
|
},
|
|
191
191
|
};
|
|
192
192
|
}
|
|
193
|
+
// A step that ran and failed is a run failure, not a protocol failure.
|
|
194
|
+
// Routing it through `protocolFailure` reported `relayflowd could not
|
|
195
|
+
// complete the run request` — which says the daemon broke — and left the
|
|
196
|
+
// report with no `status`, so the summary line printed `RUN <id> unknown`
|
|
197
|
+
// about a run whose outcome was known exactly. The diagnostic carried up
|
|
198
|
+
// from `classifyOutcome` already names the step, its exit code and its
|
|
199
|
+
// output tail; this branch is what lets it reach the terminal. Mirrors
|
|
200
|
+
// the `McpStepError` branch above, which had this shape all along.
|
|
201
|
+
if (error instanceof AuthoredFlowExecutionError && error.code === 'step_failed') {
|
|
202
|
+
return {
|
|
203
|
+
exitCode: 1,
|
|
204
|
+
report: {
|
|
205
|
+
...base,
|
|
206
|
+
ok: false,
|
|
207
|
+
runId: error.runId,
|
|
208
|
+
socketPath,
|
|
209
|
+
status: 'failed',
|
|
210
|
+
completionReason: 'step_failed',
|
|
211
|
+
diagnostics: [...base.diagnostics, {
|
|
212
|
+
severity: 'failure',
|
|
213
|
+
kind: 'step_failed',
|
|
214
|
+
// The `code: ` prefix `AuthoredFlowExecutionError` adds is
|
|
215
|
+
// redundant once the diagnostic is labelled `[step_failed]`.
|
|
216
|
+
message: error.message.replace(/^step_failed: /, ''),
|
|
217
|
+
}],
|
|
218
|
+
},
|
|
219
|
+
};
|
|
220
|
+
}
|
|
193
221
|
const runId = error instanceof AuthoredFlowExecutionError ? error.runId : undefined;
|
|
194
222
|
return protocolFailure('run', base, socketPath, error, runId);
|
|
195
223
|
} finally {
|
package/src/cli/run.ts
CHANGED
|
@@ -11,7 +11,7 @@ import { ensureDaemon, type EnsureDaemonOptions } from '../daemon-lifecycle.js';
|
|
|
11
11
|
import { isAuthoredFlowPath } from '../direct-input.js';
|
|
12
12
|
import { daemonRefusal } from './daemon-refusal.js';
|
|
13
13
|
import type { RunFailureKind, RunWarningKind, StepFailedDetails } from '../failure-kinds.js';
|
|
14
|
-
import {
|
|
14
|
+
import { inspectionHint, stepFailureDetails } from './step-failure.js';
|
|
15
15
|
import { JournalClient, JournalProtocolError } from '../journal-client.js';
|
|
16
16
|
import { attachLocalAgent } from '../local-agent.js';
|
|
17
17
|
import { LlmWorker } from '../llm-worker.js';
|
|
@@ -80,6 +80,14 @@ export interface RunLifecycleOptions {
|
|
|
80
80
|
localAgent?: boolean;
|
|
81
81
|
signal?: AbortSignal;
|
|
82
82
|
onWait?: (progress: RunProgress) => void;
|
|
83
|
+
/**
|
|
84
|
+
* The daemon data dir, carried so a failure diagnostic can name the journal
|
|
85
|
+
* holding the evidence (`<dataDir>/runs/<runId>.sqlite3`) and emit a
|
|
86
|
+
* `flows replay` invocation that will actually resolve. Each verb sets it
|
|
87
|
+
* from its own `--data-dir`; absent only where no data dir exists, and the
|
|
88
|
+
* diagnostic then omits the path rather than guessing one.
|
|
89
|
+
*/
|
|
90
|
+
dataDir?: string;
|
|
83
91
|
/**
|
|
84
92
|
* Attach-or-spawn policy for the daemon this command needs
|
|
85
93
|
* (kernel/DAEMON-LIFECYCLE.md §3). `{ spawn: false }` is `--no-spawn`:
|
|
@@ -123,7 +131,7 @@ async function executeCheckedFlow(
|
|
|
123
131
|
// advertises its existing pins; the daemon still owns surface matching.
|
|
124
132
|
if (options.localAgent) localAgent = await attachLocalAgent(client, dataDir, options.onPtyReady);
|
|
125
133
|
const outcome = await client.runStart(spec, options.reuseFromRunId);
|
|
126
|
-
const execution = await classifyOutcome(client, 'run', outcome, base, socketPath, options);
|
|
134
|
+
const execution = await classifyOutcome(client, 'run', outcome, base, socketPath, { ...options, dataDir });
|
|
127
135
|
if (options.reuseFromRunId !== undefined) {
|
|
128
136
|
execution.report.reuse = await reuseSummary(client, outcome.run_id, options.reuseFromRunId);
|
|
129
137
|
}
|
|
@@ -207,7 +215,7 @@ export async function resumeFlow(
|
|
|
207
215
|
if (await resumeHelperEffect(client, runId, dataDir)) {
|
|
208
216
|
outcome = await client.runResume(runId, options.allowHumanInfluenced);
|
|
209
217
|
}
|
|
210
|
-
return await classifyOutcome(client, 'resume', outcome, base, socketPath, options);
|
|
218
|
+
return await classifyOutcome(client, 'resume', outcome, base, socketPath, { ...options, dataDir });
|
|
211
219
|
} catch (error) {
|
|
212
220
|
if (error instanceof JournalProtocolError && error.code === 'human_influenced_run') {
|
|
213
221
|
return { exitCode: 2, report: { ...base, runId, socketPath,
|
|
@@ -409,18 +417,24 @@ export async function classifyOutcome(
|
|
|
409
417
|
message: `Run "${current.run_id}" failed with completionReason: ${current.completion_reason}.`,
|
|
410
418
|
};
|
|
411
419
|
if (current.completion_reason === 'step_failed') {
|
|
420
|
+
let details: StepFailedDetails | undefined;
|
|
412
421
|
try {
|
|
413
|
-
|
|
414
|
-
if (details !== undefined) {
|
|
415
|
-
Object.assign(diagnostic, details);
|
|
416
|
-
diagnostic.message += ` Step ${JSON.stringify(details.stepId)} exit=${details.exitCode}.`
|
|
417
|
-
+ (details.stderrTail ? `\nStderr (last 1,024 bytes):\n${details.stderrTail}` : '')
|
|
418
|
-
+ `\nInspect: ${details.hint}`;
|
|
419
|
-
}
|
|
422
|
+
details = await stepFailureDetails(client, current.run_id);
|
|
420
423
|
} catch (error) {
|
|
421
424
|
// Inspection must not erase the already known run failure.
|
|
422
|
-
diagnostic.message += ` Could not inspect
|
|
425
|
+
diagnostic.message += ` Could not inspect the failed step: ${errorMessage(error)}`;
|
|
423
426
|
}
|
|
427
|
+
if (details !== undefined) {
|
|
428
|
+
Object.assign(diagnostic, details);
|
|
429
|
+
diagnostic.message += renderStepEvidence(details);
|
|
430
|
+
}
|
|
431
|
+
// Appended whatever the inspection found — including nothing. A failure
|
|
432
|
+
// shape this reader does not recognise, or a journal it could not read,
|
|
433
|
+
// must still end with somewhere to go rather than with a dead end.
|
|
434
|
+
const where = inspectionHint(current.run_id, details?.stepId, options.dataDir);
|
|
435
|
+
Object.assign(diagnostic, where);
|
|
436
|
+
diagnostic.message += `\nInspect: ${where.hint}`
|
|
437
|
+
+ (where.journalPath === undefined ? '' : `\nJournal: ${where.journalPath}`);
|
|
424
438
|
}
|
|
425
439
|
return {
|
|
426
440
|
exitCode: 1,
|
|
@@ -609,6 +623,24 @@ function errorMessage(error: unknown): string {
|
|
|
609
623
|
return error instanceof Error ? error.message : 'unknown protocol error';
|
|
610
624
|
}
|
|
611
625
|
|
|
626
|
+
/**
|
|
627
|
+
* The evidence half of a `step_failed` diagnostic; `inspectionHint` adds the
|
|
628
|
+
* rest. Each field is printed only when the journal actually carried it — an
|
|
629
|
+
* agent step has no exit code to report, and inventing `exit=undefined` (which
|
|
630
|
+
* is what the deterministic-only version printed for one) is worse than
|
|
631
|
+
* silence on that field.
|
|
632
|
+
*/
|
|
633
|
+
function renderStepEvidence(details: StepFailedDetails): string {
|
|
634
|
+
return ` Step ${JSON.stringify(details.stepId)}`
|
|
635
|
+
+ (details.stepType === undefined ? '' : ` (${details.stepType})`)
|
|
636
|
+
+ ` completionReason: ${details.completionReason}`
|
|
637
|
+
+ (details.exitCode === undefined ? '' : ` exit=${details.exitCode}`)
|
|
638
|
+
+ '.'
|
|
639
|
+
+ (details.detail === undefined ? '' : `\nDetail: ${details.detail}`)
|
|
640
|
+
+ (details.stdoutTail ? `\nStdout (last 1,024 bytes):\n${details.stdoutTail}` : '')
|
|
641
|
+
+ (details.stderrTail ? `\nStderr (last 1,024 bytes):\n${details.stderrTail}` : '');
|
|
642
|
+
}
|
|
643
|
+
|
|
612
644
|
function throwIfCanceled(signal: AbortSignal | undefined, stepId: string): void {
|
|
613
645
|
if (signal?.aborted === true) throw new Error(`waiting for running step "${stepId}" was canceled`);
|
|
614
646
|
}
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { DEFAULT_DATA_DIR } from '../daemon-connection.js';
|
|
3
|
+
import type { StepFailedDetails } from '../failure-kinds.js';
|
|
4
|
+
import type { JournalClient } from '../journal-client.js';
|
|
5
|
+
|
|
6
|
+
const TAIL_BYTES = 1_024;
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Read what a failed step left in the journal — for any step type.
|
|
10
|
+
*
|
|
11
|
+
* This was `deterministicFailureDetails`, and its first act was to return
|
|
12
|
+
* `undefined` for any run without a `deterministic` step. That is every
|
|
13
|
+
* `f.agent` and `f.llm` step, because each authored worker call runs as its
|
|
14
|
+
* own single-step kernel run (authored-worker-step.ts). So a failed agent
|
|
15
|
+
* reached the terminal carrying nothing but its taxonomy label — the run said
|
|
16
|
+
* `step_failed` and discarded every account of why, which is the whole reason
|
|
17
|
+
* a local agent failure was undiagnosable.
|
|
18
|
+
*
|
|
19
|
+
* The two step families leave their evidence in different fields, because the
|
|
20
|
+
* kernel preserves `output` on a failed completion only for deterministic
|
|
21
|
+
* steps (`preserve_failure_output`, relayflowd-core/src/machine.rs). For an
|
|
22
|
+
* agent or llm step the worker's `{exit_code, stdout_tail, stderr_tail}` is
|
|
23
|
+
* nulled out of `output` and survives only as the bounded render the daemon
|
|
24
|
+
* captured into `verification.detail` (`worker_failure_detail`,
|
|
25
|
+
* relayflowd/src/engine/remote.rs). Both are read, in that order, and the
|
|
26
|
+
* daemon's render is re-parsed when it carries that same shape: an exit code
|
|
27
|
+
* the daemon stringified on its way into the journal is still an exit code,
|
|
28
|
+
* and printing it as one is the difference between a diagnosis and a blob.
|
|
29
|
+
*/
|
|
30
|
+
export async function stepFailureDetails(
|
|
31
|
+
client: JournalClient,
|
|
32
|
+
runId: string,
|
|
33
|
+
): Promise<StepFailedDetails | undefined> {
|
|
34
|
+
const snapshot = await client.runGet(runId);
|
|
35
|
+
let fromSeq = 1;
|
|
36
|
+
const failures = new Map<string, StepFailedDetails>();
|
|
37
|
+
while (true) {
|
|
38
|
+
const { entries } = await client.journalRead(runId, fromSeq, 100);
|
|
39
|
+
if (entries.length === 0) break;
|
|
40
|
+
for (const raw of entries) {
|
|
41
|
+
const entry = record(raw);
|
|
42
|
+
const seq = entry?.['seq'];
|
|
43
|
+
if (typeof seq !== 'number' || !Number.isSafeInteger(seq) || seq < fromSeq) {
|
|
44
|
+
throw new Error('invalid journal sequence in step failure inspection');
|
|
45
|
+
}
|
|
46
|
+
fromSeq = seq + 1;
|
|
47
|
+
const stepId = entry?.['step_id'];
|
|
48
|
+
if (entry?.['entry_type'] !== 'step.completed' || typeof stepId !== 'string') continue;
|
|
49
|
+
// A later completion supersedes an earlier failed attempt.
|
|
50
|
+
failures.delete(stepId);
|
|
51
|
+
const payload = record(entry['payload']);
|
|
52
|
+
if (payload === undefined) continue;
|
|
53
|
+
const completionReason = payload['completionReason'];
|
|
54
|
+
// A terminal completion that is not a success is the failure, whatever
|
|
55
|
+
// its step type. The old predicate also demanded a non-zero `exit_code`,
|
|
56
|
+
// which no agent completion carries and which a deterministic step that
|
|
57
|
+
// exits 0 and then fails its gate does not carry either — both were
|
|
58
|
+
// silently skipped.
|
|
59
|
+
if (payload['disposition'] !== 'step_done'
|
|
60
|
+
|| typeof completionReason !== 'string' || completionReason === 'success') continue;
|
|
61
|
+
const stepType = snapshot.steps[stepId]?.type;
|
|
62
|
+
failures.set(stepId, {
|
|
63
|
+
stepId,
|
|
64
|
+
completionReason,
|
|
65
|
+
...(stepType === undefined ? {} : { stepType }),
|
|
66
|
+
...evidence(payload),
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return [...failures.values()].at(-1);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Where to look, derived from the run id and data dir ALONE.
|
|
75
|
+
*
|
|
76
|
+
* Deliberately independent of the journal read: a failure whose evidence could
|
|
77
|
+
* not be read, or a failure shape nothing here recognises, must still end with
|
|
78
|
+
* somewhere to go rather than with a dead end. `flows replay` is that command —
|
|
79
|
+
* it already exists and already prints the full journal; nothing ever named it
|
|
80
|
+
* at the moment of failure, which is why the surface looked like it had no way
|
|
81
|
+
* to inspect a finished run.
|
|
82
|
+
*/
|
|
83
|
+
export function inspectionHint(
|
|
84
|
+
runId: string,
|
|
85
|
+
stepId: string | undefined,
|
|
86
|
+
dataDir: string | undefined,
|
|
87
|
+
): { hint: string; journalPath?: string } {
|
|
88
|
+
const at = stepId === undefined ? '' : ` --at ${shellQuote(stepId)}`;
|
|
89
|
+
// Only name a non-default data dir: repeating the default back at an
|
|
90
|
+
// operator who never typed it is noise, and `flows replay` defaults to the
|
|
91
|
+
// same value (cli.ts).
|
|
92
|
+
const dir = dataDir === undefined || dataDir === DEFAULT_DATA_DIR
|
|
93
|
+
? '' : ` --data-dir ${shellQuote(dataDir)}`;
|
|
94
|
+
return {
|
|
95
|
+
hint: `flows replay ${shellQuote(runId)}${at}${dir}`,
|
|
96
|
+
// Left as the operator spelled it rather than resolved: `.relayflowd/...`
|
|
97
|
+
// is what they will recognise in their own working directory.
|
|
98
|
+
...(dataDir === undefined ? {} : { journalPath: join(dataDir, 'runs', `${runId}.sqlite3`) }),
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Pull the process-shaped fields out of whichever field carried them.
|
|
104
|
+
*
|
|
105
|
+
* `verification.detail` is last because it is the daemon's own render rather
|
|
106
|
+
* than the worker's structured report — but for an agent step it is the only
|
|
107
|
+
* thing that survives, so it is parsed when it parses and kept verbatim when
|
|
108
|
+
* it does not. A truncated render (the daemon caps at 2,000 chars and appends
|
|
109
|
+
* a truncation note) will not parse; that falls through to the raw string,
|
|
110
|
+
* which is still the account of what went wrong.
|
|
111
|
+
*/
|
|
112
|
+
function evidence(payload: Record<string, unknown>): Partial<StepFailedDetails> {
|
|
113
|
+
const detail = record(payload['verification'])?.['detail'];
|
|
114
|
+
const structured = record(payload['output'])
|
|
115
|
+
?? record(payload['trajectory_tail'])
|
|
116
|
+
?? (typeof detail === 'string' ? parsed(detail) : undefined);
|
|
117
|
+
const exitCode = structured?.['exit_code'];
|
|
118
|
+
const stdout = structured?.['stdout_tail'];
|
|
119
|
+
const stderr = structured?.['stderr_tail'];
|
|
120
|
+
const structuredShape = typeof exitCode === 'number'
|
|
121
|
+
|| typeof stdout === 'string' || typeof stderr === 'string';
|
|
122
|
+
return {
|
|
123
|
+
...(typeof exitCode === 'number' && Number.isSafeInteger(exitCode) ? { exitCode } : {}),
|
|
124
|
+
...(typeof stdout === 'string' && stdout.length > 0 ? { stdoutTail: tail(stdout) } : {}),
|
|
125
|
+
...(typeof stderr === 'string' ? { stderrTail: tail(stderr) } : {}),
|
|
126
|
+
// Keep the daemon's account only when it was NOT just a render of the
|
|
127
|
+
// fields above — otherwise the same bytes print twice.
|
|
128
|
+
...(typeof detail === 'string' && detail.length > 0 && !structuredShape
|
|
129
|
+
? { detail: tail(detail) } : {}),
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function parsed(value: string): Record<string, unknown> | undefined {
|
|
134
|
+
try {
|
|
135
|
+
return record(JSON.parse(value));
|
|
136
|
+
} catch {
|
|
137
|
+
return undefined;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function record(value: unknown): Record<string, unknown> | undefined {
|
|
142
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
143
|
+
? value as Record<string, unknown> : undefined;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function tail(value: string): string {
|
|
147
|
+
const bytes = Buffer.from(value, 'utf8');
|
|
148
|
+
let start = Math.max(0, bytes.length - TAIL_BYTES);
|
|
149
|
+
// Drop a partial leading code point, avoiding replacement-byte expansion.
|
|
150
|
+
while (start < bytes.length && (bytes[start]! & 0xc0) === 0x80) start += 1;
|
|
151
|
+
// Preserve tabs/newlines; replace binary controls (including ESC and CR),
|
|
152
|
+
// C1 controls and Unicode formatting controls without growing the excerpt.
|
|
153
|
+
return bytes.subarray(start).toString('utf8')
|
|
154
|
+
.replace(/[\p{Cc}\p{Cf}]/gu, character => character === '\n' || character === '\t' ? character : '?');
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function shellQuote(value: string): string {
|
|
158
|
+
return /^[A-Za-z0-9_-]+$/.test(value) && !value.startsWith('-')
|
|
159
|
+
? value : `'${value.replace(/'/g, "'\\''")}'`;
|
|
160
|
+
}
|
package/src/cli.ts
CHANGED
|
@@ -30,6 +30,7 @@ import { parseDigestReference } from './bundle-transport.js';
|
|
|
30
30
|
import { parseBuildArgs, runBuild, type BuildArgs } from './cli/build.js';
|
|
31
31
|
import { runHnMonitor } from './cli/hn-monitor.js';
|
|
32
32
|
import { runTickRunner } from './cli/tick-runner.js';
|
|
33
|
+
import { DEFAULT_DATA_DIR } from './daemon-connection.js';
|
|
33
34
|
import {
|
|
34
35
|
mintObserverUrl,
|
|
35
36
|
resolveObserverLinkEnv,
|
|
@@ -60,7 +61,6 @@ type ParsedArgs =
|
|
|
60
61
|
intervalMs: number; epochMs: number | undefined; maxCatchUp: number | undefined;
|
|
61
62
|
pollIntervalMs: number | undefined };
|
|
62
63
|
|
|
63
|
-
const DEFAULT_DATA_DIR = '.relayflowd';
|
|
64
64
|
const USAGE = [
|
|
65
65
|
'Usage:',
|
|
66
66
|
'flows add <helper-name|@flows/helper-name>',
|
|
@@ -299,7 +299,7 @@ async function observerUrlFrom(
|
|
|
299
299
|
if (mint === undefined) return undefined;
|
|
300
300
|
const outcome = await mint;
|
|
301
301
|
if (outcome.warning !== undefined) {
|
|
302
|
-
io.stderr(`[observer] token mint failed: ${outcome.warning};
|
|
302
|
+
io.stderr(`[observer] token mint failed: ${outcome.warning}; continuing without an observer link (the run is unaffected)`);
|
|
303
303
|
}
|
|
304
304
|
return outcome.observerUrl;
|
|
305
305
|
}
|
|
@@ -394,11 +394,11 @@ export async function finalizeObserverLine(
|
|
|
394
394
|
const outcome = await Promise.race([mint, timeout]);
|
|
395
395
|
if (timer !== undefined) clearTimeout(timer);
|
|
396
396
|
if (outcome === TIMED_OUT) {
|
|
397
|
-
io.stderr('[observer] mint did not complete in time;
|
|
397
|
+
io.stderr('[observer] mint did not complete in time; continuing without an observer link (the run is unaffected)');
|
|
398
398
|
return;
|
|
399
399
|
}
|
|
400
400
|
if (outcome.warning !== undefined) {
|
|
401
|
-
io.stderr(`[observer] token mint failed: ${outcome.warning};
|
|
401
|
+
io.stderr(`[observer] token mint failed: ${outcome.warning}; continuing without an observer link (the run is unaffected)`);
|
|
402
402
|
return;
|
|
403
403
|
}
|
|
404
404
|
if (outcome.observerUrl !== undefined) io.stdout(`Observer: ${outcome.observerUrl}`);
|
package/src/daemon-connection.ts
CHANGED
|
@@ -41,6 +41,14 @@ export const CONNECTION_FILE = 'connection.json';
|
|
|
41
41
|
export const SOCKET_FILE = 'relayflowd.sock';
|
|
42
42
|
export const DAEMON_LOG_FILE = 'relayflowd.log';
|
|
43
43
|
|
|
44
|
+
/**
|
|
45
|
+
* The data dir every verb defaults to. It lives here, next to the other paths
|
|
46
|
+
* derived from it, so the CLI's `--data-dir` default and the failure
|
|
47
|
+
* diagnostic that decides whether to echo `--data-dir` back at an operator
|
|
48
|
+
* cannot drift apart.
|
|
49
|
+
*/
|
|
50
|
+
export const DEFAULT_DATA_DIR = '.relayflowd';
|
|
51
|
+
|
|
44
52
|
/** `<data-dir>/connection.json`, exactly the shape in §1. */
|
|
45
53
|
export interface DaemonConnection {
|
|
46
54
|
socket_path: string;
|
package/src/failure-kinds.ts
CHANGED
|
@@ -62,8 +62,11 @@ export const CHECK_FAILURE_KINDS = [
|
|
|
62
62
|
* accepting every output must not be reported as if it constrained one.
|
|
63
63
|
*
|
|
64
64
|
* `budget_unmetered` names an LLM/agent step under a dollar budget whose model
|
|
65
|
-
* has no frozen price (including Codex, which selects its own model).
|
|
66
|
-
*
|
|
65
|
+
* has no frozen price (including Codex, which selects its own model). A
|
|
66
|
+
* missing price never refuses: the step runs, journals its tokens with
|
|
67
|
+
* `dollars_unmetered: true`, and cannot cross `maxDollars`; token limits still
|
|
68
|
+
* apply. It replaced the `budget_missing_price` refusal (#421), which older
|
|
69
|
+
* `flows check` reports may still show.
|
|
67
70
|
*/
|
|
68
71
|
export const PREFLIGHT_WARNING_KINDS = [
|
|
69
72
|
'unprovable_effects',
|
|
@@ -122,13 +125,32 @@ export type PreflightWarningKind = (typeof PREFLIGHT_WARNING_KINDS)[number];
|
|
|
122
125
|
export type RunFailureKind = (typeof RUN_FAILURE_KINDS)[number];
|
|
123
126
|
export type RunWarningKind = (typeof RUN_WARNING_KINDS)[number];
|
|
124
127
|
|
|
125
|
-
/**
|
|
128
|
+
/**
|
|
129
|
+
* Optional evidence on the existing step_failed diagnostic, not a new kind.
|
|
130
|
+
*
|
|
131
|
+
* Every field is optional because a failure must be reportable on whatever it
|
|
132
|
+
* left behind. A deterministic step leaves `exitCode` plus output tails; an
|
|
133
|
+
* agent or llm step leaves `completionReason` and whatever the daemon captured
|
|
134
|
+
* into `detail` (see cli/step-failure.ts). Absent means "not journaled", never
|
|
135
|
+
* "zero" — an exit code is only ever reported when one was actually recorded.
|
|
136
|
+
*/
|
|
126
137
|
export interface StepFailedDetails {
|
|
127
138
|
stepId?: string;
|
|
139
|
+
/** `deterministic` | `llm` | `agent`, when the run snapshot named one. */
|
|
140
|
+
stepType?: string;
|
|
141
|
+
/** The kernel's per-step reason, e.g. `worker_error`, `retries_exhausted`. */
|
|
142
|
+
completionReason?: string;
|
|
128
143
|
exitCode?: number;
|
|
129
144
|
/** Terminal-safe UTF-8 excerpt, at most 1,024 bytes. */
|
|
145
|
+
stdoutTail?: string;
|
|
146
|
+
/** Terminal-safe UTF-8 excerpt, at most 1,024 bytes. */
|
|
130
147
|
stderrTail?: string;
|
|
148
|
+
/** The daemon's own account, when it was not a render of the fields above. */
|
|
149
|
+
detail?: string;
|
|
150
|
+
/** A runnable `flows replay` invocation for this run. */
|
|
131
151
|
hint?: string;
|
|
152
|
+
/** The on-disk journal for this run, when the data dir is known. */
|
|
153
|
+
journalPath?: string;
|
|
132
154
|
}
|
|
133
155
|
|
|
134
156
|
const CHECK_FAILURE_KIND_SET: ReadonlySet<string> = new Set(CHECK_FAILURE_KINDS);
|
package/src/journal-client.ts
CHANGED
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
type Request,
|
|
23
23
|
type Response,
|
|
24
24
|
type ServerEvent,
|
|
25
|
+
type StepUsage,
|
|
25
26
|
} from './protocol.js';
|
|
26
27
|
import type { KernelRunSpec, StepType } from './spec.js';
|
|
27
28
|
|
|
@@ -366,7 +367,7 @@ export class JournalClient extends EventEmitter {
|
|
|
366
367
|
completionReason: CompletionReason,
|
|
367
368
|
extra: {
|
|
368
369
|
output?: unknown;
|
|
369
|
-
usage?:
|
|
370
|
+
usage?: StepUsage;
|
|
370
371
|
started_pins?: Pins;
|
|
371
372
|
end_pins?: Pins;
|
|
372
373
|
effects?: EffectRef[];
|
package/src/model-pricing.ts
CHANGED
|
@@ -14,11 +14,13 @@ export function hasPricing(model: string | undefined): boolean {
|
|
|
14
14
|
/**
|
|
15
15
|
* Cost accounting for a step's declared model.
|
|
16
16
|
*
|
|
17
|
-
* Returns `undefined` for unpriced models
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
* dollars
|
|
21
|
-
*
|
|
17
|
+
* Returns `undefined` for unpriced models, meaning "no known dollar cost" —
|
|
18
|
+
* never a zero price. Workers do not omit usage on that signal: `workerSpend`
|
|
19
|
+
* falls back to unmetered usage (reported tokens plus `dollars_unmetered: true`,
|
|
20
|
+
* no `dollars`), so token ceilings still count the step and the journal records
|
|
21
|
+
* its dollars as unknown. An unpriced step is unmetered, not refused:
|
|
22
|
+
* `budgetDiagnostics` warns about it at preflight and it cannot cross
|
|
23
|
+
* `maxDollars`. Codex model ids need no entry here; Codex selects its own model.
|
|
22
24
|
*/
|
|
23
25
|
export function pricedUsage(model: string | undefined, input = 0, output = 0):
|
|
24
26
|
| { tokens_in: number; tokens_out: number; dollars: string }
|
package/src/preflight.ts
CHANGED
|
@@ -117,6 +117,11 @@ export type PreflightDiagnostic = PreflightRefusal | PreflightWarning;
|
|
|
117
117
|
export interface PreflightResult {
|
|
118
118
|
plugins?: readonly LoadedPlugin[];
|
|
119
119
|
mcpTools?: Readonly<Record<string, readonly string[]>>;
|
|
120
|
+
/**
|
|
121
|
+
* True when no diagnostic is a refusal. `ok: true` may still carry warnings
|
|
122
|
+
* (e.g. `budget_unmetered`); callers that report to a person must surface
|
|
123
|
+
* `diagnostics`, not just branch on `ok`.
|
|
124
|
+
*/
|
|
120
125
|
ok: boolean;
|
|
121
126
|
gates: StepGateInspection[];
|
|
122
127
|
resolutions: CliResolution[];
|
package/src/protocol.ts
CHANGED
|
@@ -72,10 +72,23 @@ export interface HelloResult {
|
|
|
72
72
|
export interface StepSpend {
|
|
73
73
|
tokens_input: number;
|
|
74
74
|
tokens_output: number;
|
|
75
|
+
/** Metered dollars; a lower bound when `dollars_unmetered` is set. */
|
|
75
76
|
dollars: number;
|
|
77
|
+
/** Present (true) only when the step spent tokens of unknown dollar cost. */
|
|
78
|
+
dollars_unmetered?: true;
|
|
76
79
|
wallclock_ms: number;
|
|
77
80
|
}
|
|
78
81
|
|
|
82
|
+
/**
|
|
83
|
+
* Worker-reported `step.complete` usage. A priced step reports exact
|
|
84
|
+
* `dollars`; a step whose model has no frozen price reports its tokens with
|
|
85
|
+
* `dollars_unmetered: true` and no dollar amount, so unknown cost is never
|
|
86
|
+
* journaled as a measured $0. See `workerSpend` for the full contract.
|
|
87
|
+
*/
|
|
88
|
+
export type StepUsage =
|
|
89
|
+
| { tokens_in: number; tokens_out: number; dollars: string; dollars_unmetered?: never }
|
|
90
|
+
| { tokens_in: number; tokens_out: number; dollars_unmetered: true; dollars?: never };
|
|
91
|
+
|
|
79
92
|
export interface RunStartParams {
|
|
80
93
|
/** Caller-owned retry identity. Reuse with a different spec is refused. */
|
|
81
94
|
admission_key?: string;
|
|
@@ -130,7 +143,7 @@ export interface RunGetResult {
|
|
|
130
143
|
run_id: string;
|
|
131
144
|
status: RunStatus;
|
|
132
145
|
steps: Record<string, StepSnapshot>;
|
|
133
|
-
budget: { tokens_in: number; tokens_out: number; dollars: string };
|
|
146
|
+
budget: { tokens_in: number; tokens_out: number; dollars: string; dollars_unmetered?: true };
|
|
134
147
|
}
|
|
135
148
|
|
|
136
149
|
export interface RunWatchParams {
|
|
@@ -307,7 +320,7 @@ export interface StepCompleteParams {
|
|
|
307
320
|
idempotency_key: string;
|
|
308
321
|
completionReason: CompletionReason;
|
|
309
322
|
output?: unknown;
|
|
310
|
-
usage?:
|
|
323
|
+
usage?: StepUsage;
|
|
311
324
|
started_pins?: Pins;
|
|
312
325
|
end_pins?: Pins;
|
|
313
326
|
effects?: EffectRef[];
|