@relayflows/sdk 2.0.37 → 2.0.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-reaper.d.ts +1 -1
- package/dist/agent-reaper.d.ts.map +1 -1
- package/dist/agent-reaper.js +6 -4
- package/dist/agent-reaper.js.map +1 -1
- package/dist/authored-budget.d.ts +26 -1
- package/dist/authored-budget.d.ts.map +1 -1
- package/dist/authored-budget.js +72 -2
- package/dist/authored-budget.js.map +1 -1
- package/dist/authored-flow-error.d.ts +7 -1
- package/dist/authored-flow-error.d.ts.map +1 -1
- package/dist/authored-flow-error.js.map +1 -1
- package/dist/authored-node-runner.d.ts.map +1 -1
- package/dist/authored-node-runner.js +3 -2
- package/dist/authored-node-runner.js.map +1 -1
- package/dist/authored-root.d.ts.map +1 -1
- package/dist/authored-root.js +141 -6
- package/dist/authored-root.js.map +1 -1
- package/dist/authored-worker-step.js +1 -0
- package/dist/authored-worker-step.js.map +1 -1
- package/dist/child-stop.d.ts +3 -1
- package/dist/child-stop.d.ts.map +1 -1
- package/dist/child-stop.js +42 -2
- package/dist/child-stop.js.map +1 -1
- package/dist/cli/add-extension.js +1 -1
- package/dist/cli/add-extension.js.map +1 -1
- package/dist/cli/check.d.ts.map +1 -1
- package/dist/cli/check.js +25 -7
- package/dist/cli/check.js.map +1 -1
- package/dist/cli/cli-probe.d.ts +1 -0
- package/dist/cli/cli-probe.d.ts.map +1 -1
- package/dist/cli/cli-probe.js +72 -36
- package/dist/cli/cli-probe.js.map +1 -1
- package/dist/cli/direct-run.d.ts.map +1 -1
- package/dist/cli/direct-run.js +4 -1
- package/dist/cli/direct-run.js.map +1 -1
- package/dist/cli/pinned-cli-alias.d.ts +13 -0
- package/dist/cli/pinned-cli-alias.d.ts.map +1 -0
- package/dist/cli/pinned-cli-alias.js +58 -0
- package/dist/cli/pinned-cli-alias.js.map +1 -0
- package/dist/cli/run.d.ts +8 -0
- package/dist/cli/run.d.ts.map +1 -1
- package/dist/cli/run.js +91 -8
- package/dist/cli/run.js.map +1 -1
- package/dist/cli/step-failure.d.ts.map +1 -1
- package/dist/cli/step-failure.js +18 -0
- package/dist/cli/step-failure.js.map +1 -1
- package/dist/cloud-run.d.ts.map +1 -1
- package/dist/cloud-run.js +9 -2
- package/dist/cloud-run.js.map +1 -1
- package/dist/communication/worker.d.ts.map +1 -1
- package/dist/communication/worker.js +24 -6
- package/dist/communication/worker.js.map +1 -1
- package/dist/compile.d.ts.map +1 -1
- package/dist/compile.js +16 -7
- package/dist/compile.js.map +1 -1
- package/dist/failure-kinds.d.ts +9 -1
- package/dist/failure-kinds.d.ts.map +1 -1
- package/dist/failure-kinds.js +6 -0
- package/dist/failure-kinds.js.map +1 -1
- package/dist/flow-extension-loader.d.ts +1 -1
- package/dist/flow-extension-loader.d.ts.map +1 -1
- package/dist/flow-extension-loader.js +143 -14
- package/dist/flow-extension-loader.js.map +1 -1
- package/dist/flow-extension-manifest.d.ts +1 -0
- package/dist/flow-extension-manifest.d.ts.map +1 -1
- package/dist/flow-extension-manifest.js +12 -3
- package/dist/flow-extension-manifest.js.map +1 -1
- package/dist/hosted-extension-manifest.d.ts.map +1 -1
- package/dist/hosted-extension-manifest.js +25 -4
- package/dist/hosted-extension-manifest.js.map +1 -1
- package/dist/hosted-extension-runtime.js +1 -1
- package/dist/hosted-extension-sandbox.js +2 -2
- package/dist/llm-worker.d.ts.map +1 -1
- package/dist/llm-worker.js +4 -2
- package/dist/llm-worker.js.map +1 -1
- package/dist/mcp-stdio.d.ts +2 -1
- package/dist/mcp-stdio.d.ts.map +1 -1
- package/dist/mcp-stdio.js +38 -21
- package/dist/mcp-stdio.js.map +1 -1
- package/dist/preflight.d.ts +2 -0
- package/dist/preflight.d.ts.map +1 -1
- package/dist/preflight.js.map +1 -1
- package/dist/protocol.d.ts +9 -0
- package/dist/protocol.d.ts.map +1 -1
- package/dist/resolved-cli-identity.d.ts +14 -0
- package/dist/resolved-cli-identity.d.ts.map +1 -0
- package/dist/resolved-cli-identity.js +19 -0
- package/dist/resolved-cli-identity.js.map +1 -0
- package/dist/worker-cli.d.ts +3 -1
- package/dist/worker-cli.d.ts.map +1 -1
- package/dist/worker-cli.js +116 -40
- package/dist/worker-cli.js.map +1 -1
- package/dist/worker-lease.d.ts +3 -1
- package/dist/worker-lease.d.ts.map +1 -1
- package/dist/worker-lease.js +47 -9
- package/dist/worker-lease.js.map +1 -1
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +4 -2
- package/dist/worker.js.map +1 -1
- package/dist/wrapper-session.d.ts +3 -1
- package/dist/wrapper-session.d.ts.map +1 -1
- package/dist/wrapper-session.js +93 -53
- package/dist/wrapper-session.js.map +1 -1
- package/package.json +2 -2
- package/src/agent-reaper.ts +7 -4
- package/src/authored-budget.ts +73 -1
- package/src/authored-flow-error.ts +6 -0
- package/src/authored-node-runner.ts +3 -2
- package/src/authored-root.ts +150 -7
- package/src/authored-worker-step.ts +1 -0
- package/src/child-stop.ts +40 -1
- package/src/cli/add-extension.ts +1 -1
- package/src/cli/check.ts +25 -4
- package/src/cli/cli-probe.ts +73 -37
- package/src/cli/direct-run.ts +4 -1
- package/src/cli/pinned-cli-alias.ts +74 -0
- package/src/cli/run.ts +103 -7
- package/src/cli/step-failure.ts +18 -0
- package/src/cloud-run.ts +9 -2
- package/src/communication/worker.ts +24 -9
- package/src/compile.ts +28 -13
- package/src/failure-kinds.ts +14 -0
- package/src/flow-extension-loader.ts +154 -14
- package/src/flow-extension-manifest.ts +6 -3
- package/src/hosted-extension-manifest.ts +28 -5
- package/src/hosted-extension-runtime.ts +1 -1
- package/src/hosted-extension-sandbox.ts +2 -2
- package/src/llm-worker.ts +6 -4
- package/src/mcp-stdio.ts +34 -20
- package/src/preflight.ts +2 -0
- package/src/protocol.ts +9 -0
- package/src/resolved-cli-identity.ts +42 -0
- package/src/worker-cli.ts +123 -37
- package/src/worker-lease.ts +50 -9
- package/src/worker.ts +6 -4
- package/src/wrapper-session.ts +89 -56
package/src/cli/run.ts
CHANGED
|
@@ -408,6 +408,9 @@ export async function resumeFlow(
|
|
|
408
408
|
if (error instanceof AuthoredFlowExecutionError && (error.code === 'step_failed' || error.code === 'gate_failed')) {
|
|
409
409
|
return authoredStepFailure('resume', base, socketPath, error, runId);
|
|
410
410
|
}
|
|
411
|
+
if (error instanceof AuthoredFlowExecutionError && error.code === 'root_lease_lost') {
|
|
412
|
+
return rootLeaseLostReport('resume', base, socketPath, error, runId);
|
|
413
|
+
}
|
|
411
414
|
// An authored resume that parks for want of a worker is a park, reported
|
|
412
415
|
// like the run path's — and with the same remedy, because an authored root
|
|
413
416
|
// admits its worker at run start: resuming again cannot attach one, so the
|
|
@@ -522,6 +525,38 @@ function authoredInputArgument(metadata: AuthoredRootMetadata | undefined): Auth
|
|
|
522
525
|
* drove correctly, and it produces a report with no `status`, which is the
|
|
523
526
|
* whole of what `RUN <id> unknown` ever meant.
|
|
524
527
|
*/
|
|
528
|
+
/**
|
|
529
|
+
* The authored root's lease was lost and not re-dispatched to this process
|
|
530
|
+
* (authored-root.ts driveRootFollowingRetries). Exit 1 because this CLI did
|
|
531
|
+
* not finish the run, but the report says what is true of it: still
|
|
532
|
+
* `running` in the journal, and resumable -- never `protocol_error` with the
|
|
533
|
+
* generic "worker wait canceled" and no run id, which is what it was.
|
|
534
|
+
*/
|
|
535
|
+
export function rootLeaseLostReport(
|
|
536
|
+
command: RunCommand,
|
|
537
|
+
base: CheckReport | RunReport,
|
|
538
|
+
socketPath: string,
|
|
539
|
+
error: AuthoredFlowExecutionError,
|
|
540
|
+
fallbackRunId?: string,
|
|
541
|
+
): RunExecution {
|
|
542
|
+
const rootRunId = error.rootRunId ?? fallbackRunId;
|
|
543
|
+
return {
|
|
544
|
+
exitCode: 1,
|
|
545
|
+
report: {
|
|
546
|
+
...fromBase(command, base),
|
|
547
|
+
ok: false,
|
|
548
|
+
...(rootRunId === undefined ? {} : { runId: rootRunId, rootRunId }),
|
|
549
|
+
socketPath,
|
|
550
|
+
status: 'running',
|
|
551
|
+
diagnostics: [...base.diagnostics, {
|
|
552
|
+
severity: 'failure',
|
|
553
|
+
kind: 'root_lease_lost',
|
|
554
|
+
message: error.message.replace(/^root_lease_lost: /, ''),
|
|
555
|
+
}],
|
|
556
|
+
},
|
|
557
|
+
};
|
|
558
|
+
}
|
|
559
|
+
|
|
525
560
|
export function authoredStepFailure(
|
|
526
561
|
command: RunCommand,
|
|
527
562
|
base: CheckReport | RunReport,
|
|
@@ -798,13 +833,27 @@ export async function classifyOutcome(
|
|
|
798
833
|
let parkedStep: ParkedStep | undefined;
|
|
799
834
|
let needsHuman = false;
|
|
800
835
|
let unclassifiedPolls = 0;
|
|
801
|
-
|
|
836
|
+
// `running` too: an authored body re-driven after a lost root lease re-admits
|
|
837
|
+
// its child by admission key and gets the EXISTING run back, mid-attempt.
|
|
838
|
+
// Only `parked` was followed, so that child -- on the attempt 2 the kernel
|
|
839
|
+
// retried it to -- fell straight through to "status running without a
|
|
840
|
+
// classifiable completion" and failed the flow (customer rw_3a0fcb71).
|
|
841
|
+
while (current.status === 'parked' || current.status === 'running') {
|
|
802
842
|
const inspection = await inspectOutOfBandStep(client, current.run_id);
|
|
803
843
|
// A runnable sibling may only be waiting for capacity held by a live
|
|
804
|
-
// attempt. Keep driving until that
|
|
805
|
-
|
|
844
|
+
// attempt or by a retry in backoff. Keep driving until that active work
|
|
845
|
+
// settles before declaring a park.
|
|
846
|
+
if (inspection?.parkedStep !== undefined && (inspection.needsHuman
|
|
847
|
+
|| (inspection.runningStep === undefined && inspection.backoffStep === undefined))) {
|
|
806
848
|
parkedStep = inspection.parkedStep;
|
|
807
849
|
needsHuman = inspection.needsHuman;
|
|
850
|
+
// `run.get` reports the live snapshot as `running` for an ordinary
|
|
851
|
+
// runnable worker wait, even when `run.start`/`run.resume` already
|
|
852
|
+
// classified the outcome as parked. Conversely, a re-admitted run can
|
|
853
|
+
// arrive here with a stale `running` outcome after its worker vanished.
|
|
854
|
+
// The step-level inspection is authoritative that this is a park; do
|
|
855
|
+
// not copy its broader run status and lose that classification.
|
|
856
|
+
current = { ...current, status: 'parked' };
|
|
808
857
|
break;
|
|
809
858
|
}
|
|
810
859
|
if (inspection?.runningStep !== undefined) {
|
|
@@ -812,6 +861,20 @@ export async function classifyOutcome(
|
|
|
812
861
|
current = await client.runResume(current.run_id, command === 'run' || options.allowHumanInfluenced);
|
|
813
862
|
continue;
|
|
814
863
|
}
|
|
864
|
+
// A step in `backoff` is between attempts: the kernel has scheduled the
|
|
865
|
+
// next one (a dead worker's retry is due at once, but waits for the
|
|
866
|
+
// worker slot the dead attempt held; a failed attempt waits out the
|
|
867
|
+
// retry policy's delay, bounded by its max_backoff_ms). That is progress,
|
|
868
|
+
// not an unclassifiable state -- treating it as one gave up after 2s on a
|
|
869
|
+
// child whose attempt 2 then succeeded (customer rw_3a0fcb71). Once due,
|
|
870
|
+
// the step is dispatched (`running`, followed above) or, with no worker,
|
|
871
|
+
// left `runnable` and reported as a park.
|
|
872
|
+
if (inspection?.backoffStep !== undefined && inspection.runningStep === undefined) {
|
|
873
|
+
throwIfCanceled(options.signal, inspection.backoffStep.id);
|
|
874
|
+
await delay(EXPIRED_LEASE_POLL_MS, options.signal);
|
|
875
|
+
current = await client.runResume(current.run_id, command === 'run' || options.allowHumanInfluenced);
|
|
876
|
+
continue;
|
|
877
|
+
}
|
|
815
878
|
if (inspection?.status === 'completed' || inspection?.status === 'failed') {
|
|
816
879
|
current = await client.runResume(current.run_id, command === 'run' || options.allowHumanInfluenced);
|
|
817
880
|
continue;
|
|
@@ -954,6 +1017,8 @@ interface OutOfBandInspection {
|
|
|
954
1017
|
parkedStep?: ParkedStep;
|
|
955
1018
|
needsHuman: boolean;
|
|
956
1019
|
runningStep?: RunningStep;
|
|
1020
|
+
/** A step waiting out a retry backoff: its next attempt is scheduled. */
|
|
1021
|
+
backoffStep?: ParkedStep;
|
|
957
1022
|
}
|
|
958
1023
|
|
|
959
1024
|
interface RunningStep extends ParkedStep {
|
|
@@ -981,6 +1046,9 @@ async function inspectOutOfBandStep(
|
|
|
981
1046
|
const runningEntry = entries.find(([, step]) =>
|
|
982
1047
|
step.type !== 'deterministic' && step.state === 'running',
|
|
983
1048
|
);
|
|
1049
|
+
const backoffEntry = entries.find(([, step]) =>
|
|
1050
|
+
step.type !== 'deterministic' && step.state === 'backoff',
|
|
1051
|
+
);
|
|
984
1052
|
const parkedEntry = humanEntry ?? runnableEntry;
|
|
985
1053
|
const parkedStep = parkedEntry === undefined ? undefined : {
|
|
986
1054
|
id: parkedEntry[0],
|
|
@@ -996,11 +1064,21 @@ async function inspectOutOfBandStep(
|
|
|
996
1064
|
needsHuman: humanEntry !== undefined,
|
|
997
1065
|
...(parkedStep !== undefined ? { parkedStep } : {}),
|
|
998
1066
|
...(runningStep !== undefined ? { runningStep } : {}),
|
|
1067
|
+
...(backoffEntry !== undefined ? { backoffStep: {
|
|
1068
|
+
id: backoffEntry[0],
|
|
1069
|
+
type: backoffEntry[1].type as Extract<StepType, 'llm' | 'agent'>,
|
|
1070
|
+
} } : {}),
|
|
999
1071
|
};
|
|
1000
1072
|
}
|
|
1001
1073
|
|
|
1002
1074
|
// Match kernel/relayflowd/src/server/client.rs: allow the lease sweep to dispatch a retry.
|
|
1003
1075
|
export const LEASE_SWEEP_GRACE_MS = 5_000;
|
|
1076
|
+
// How often to re-ask the daemon about a step whose lease looks expired here.
|
|
1077
|
+
const EXPIRED_LEASE_POLL_MS = 250;
|
|
1078
|
+
// A deadline unchanged for a whole lease (relayflowd LEASE_RENEWAL_MS, 30s)
|
|
1079
|
+
// plus the sweep grace was renewed by nobody: the attempt is dead and the
|
|
1080
|
+
// daemon never swept it. Fail rather than hang.
|
|
1081
|
+
const STALE_LEASE_MS = 30_000 + LEASE_SWEEP_GRACE_MS;
|
|
1004
1082
|
|
|
1005
1083
|
async function waitForRunningStep(
|
|
1006
1084
|
client: JournalClient,
|
|
@@ -1018,20 +1096,29 @@ async function waitForRunningStep(
|
|
|
1018
1096
|
stepType: runningStep.type,
|
|
1019
1097
|
leaseDeadlineMs,
|
|
1020
1098
|
});
|
|
1099
|
+
// When the lease deadline last changed, on this process's monotonic clock.
|
|
1100
|
+
// A live worker renews every ~10s, moving the deadline; a dead lease the
|
|
1101
|
+
// daemon has not swept keeps it. That -- not a comparison of the daemon's
|
|
1102
|
+
// deadline with our wall clock -- is what separates the two under skew.
|
|
1103
|
+
let deadlineSeenAt = performance.now();
|
|
1021
1104
|
while (true) {
|
|
1022
1105
|
throwIfCanceled(options.signal, runningStep.id);
|
|
1023
1106
|
const remainingMs = leaseDeadlineMs + LEASE_SWEEP_GRACE_MS - Date.now();
|
|
1024
|
-
if (
|
|
1107
|
+
if (performance.now() - deadlineSeenAt > STALE_LEASE_MS) {
|
|
1025
1108
|
throw new Error(
|
|
1026
1109
|
`worker lease for step "${runningStep.id}" expired at ${leaseDeadlineMs} without completion`,
|
|
1027
1110
|
);
|
|
1028
1111
|
}
|
|
1029
|
-
|
|
1112
|
+
// Past the deadline by THIS clock alone is not an expiry: the daemon may
|
|
1113
|
+
// still hold the lease (skew) or be sweeping it into a retry. Keep
|
|
1114
|
+
// following the step, at a slower poll, until the daemon says otherwise.
|
|
1115
|
+
await delay(remainingMs <= 0 ? EXPIRED_LEASE_POLL_MS : Math.min(50, remainingMs), options.signal);
|
|
1030
1116
|
const snapshot = await client.runGet(runId);
|
|
1031
1117
|
const step = snapshot.steps[runningStep.id];
|
|
1032
1118
|
if (step?.state !== 'running') return;
|
|
1033
1119
|
if (step.lease_deadline_ms !== undefined && step.lease_deadline_ms !== leaseDeadlineMs) {
|
|
1034
1120
|
leaseDeadlineMs = step.lease_deadline_ms;
|
|
1121
|
+
deadlineSeenAt = performance.now();
|
|
1035
1122
|
options.onWait?.({
|
|
1036
1123
|
runId,
|
|
1037
1124
|
stepId: runningStep.id,
|
|
@@ -1098,7 +1185,14 @@ function errorMessage(error: unknown): string {
|
|
|
1098
1185
|
}
|
|
1099
1186
|
|
|
1100
1187
|
function throwIfCanceled(signal: AbortSignal | undefined, stepId: string): void {
|
|
1101
|
-
if (signal?.aborted === true)
|
|
1188
|
+
if (signal?.aborted === true) {
|
|
1189
|
+
throw new Error(`waiting for running step "${stepId}" was canceled${cancelCause(signal)}`, { cause: signal.reason });
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
/** Why a wait was canceled, when whoever aborted it said (a lost lease does). */
|
|
1194
|
+
function cancelCause(signal: AbortSignal): string {
|
|
1195
|
+
return signal.reason instanceof Error && signal.reason.name !== 'AbortError' ? `: ${signal.reason.message}` : '';
|
|
1102
1196
|
}
|
|
1103
1197
|
|
|
1104
1198
|
export function delay(ms: number, signal?: AbortSignal): Promise<void> {
|
|
@@ -1112,7 +1206,9 @@ export function delay(ms: number, signal?: AbortSignal): Promise<void> {
|
|
|
1112
1206
|
const cancel = (): void => {
|
|
1113
1207
|
clearTimeout(timer);
|
|
1114
1208
|
signal.removeEventListener('abort', cancel);
|
|
1115
|
-
|
|
1209
|
+
// Carries the abort reason: a bare "worker wait canceled" was all a
|
|
1210
|
+
// lost root lease ever reported (customer rw_3a0fcb71).
|
|
1211
|
+
rejectDelay(new Error(`worker wait canceled${cancelCause(signal)}`, { cause: signal.reason }));
|
|
1116
1212
|
};
|
|
1117
1213
|
if (signal.aborted) cancel();
|
|
1118
1214
|
else signal.addEventListener('abort', cancel, { once: true });
|
package/src/cli/step-failure.ts
CHANGED
|
@@ -63,6 +63,8 @@ export async function stepFailureDetails(
|
|
|
63
63
|
// Keyed by step, so interleaved steps never pool their attempts and a page
|
|
64
64
|
// boundary never splits one step's history.
|
|
65
65
|
const histories = new Map<string, AttemptHistory>();
|
|
66
|
+
// Attempts the kernel itself declared dead (see `unchargedAttempts`).
|
|
67
|
+
const uncharged = new Map<string, number>();
|
|
66
68
|
while (true) {
|
|
67
69
|
const { entries } = await client.journalRead(runId, fromSeq, 100);
|
|
68
70
|
if (entries.length === 0) break;
|
|
@@ -93,6 +95,7 @@ export async function stepFailureDetails(
|
|
|
93
95
|
// holds every entry; only this diagnostic's candidate is cleared.
|
|
94
96
|
if (payload === undefined || typeof completionReason !== 'string' || completionReason === 'success') {
|
|
95
97
|
histories.delete(stepId);
|
|
98
|
+
uncharged.delete(stepId);
|
|
96
99
|
continue;
|
|
97
100
|
}
|
|
98
101
|
const history = histories.get(stepId) ?? { records: [], causes: [] };
|
|
@@ -101,6 +104,10 @@ export async function stepFailureDetails(
|
|
|
101
104
|
history.records.push(attemptFailure(entry['attempt'], completionReason, payload['disposition'], payload));
|
|
102
105
|
history.causes.push(failureCause(completionReason, payload));
|
|
103
106
|
histories.set(stepId, history);
|
|
107
|
+
if (payload['disposition'] !== 'step_done' && payload['completed_by'] === 'kernel'
|
|
108
|
+
&& (completionReason === 'crashed' || completionReason === 'lease_expired')) {
|
|
109
|
+
uncharged.set(stepId, (uncharged.get(stepId) ?? 0) + 1);
|
|
110
|
+
}
|
|
104
111
|
// A terminal completion that is not a success is the failure, whatever
|
|
105
112
|
// its step type. The old predicate also demanded a non-zero `exit_code`,
|
|
106
113
|
// which no agent completion carries and which a deterministic step that
|
|
@@ -116,6 +123,7 @@ export async function stepFailureDetails(
|
|
|
116
123
|
...(stepType === undefined ? {} : { stepType }),
|
|
117
124
|
...(typeof attempt === 'number' && Number.isSafeInteger(attempt) && attempt > 0 ? { attempt } : {}),
|
|
118
125
|
...(maxIterations === undefined ? {} : { maxIterations }),
|
|
126
|
+
...((uncharged.get(stepId) ?? 0) > 0 ? { unchargedAttempts: uncharged.get(stepId) } : {}),
|
|
119
127
|
...terminalEvidence(payload),
|
|
120
128
|
// A single failed attempt is already fully described by the scalars
|
|
121
129
|
// above; repeating it as a one-element history would add a clause to
|
|
@@ -169,6 +177,16 @@ export function renderStepEvidence(details: StepFailedDetails): string {
|
|
|
169
177
|
|
|
170
178
|
function renderAttempt(details: StepFailedDetails): string {
|
|
171
179
|
if (details.attempt === undefined) return '';
|
|
180
|
+
const uncharged = details.unchargedAttempts ?? 0;
|
|
181
|
+
// `attempt=N/M` compares like with like only when every attempt was charged.
|
|
182
|
+
// A worker that died (crashed / lease_expired) costs no iteration, so a
|
|
183
|
+
// one-iteration step can legitimately run attempt 2 -- printed as
|
|
184
|
+
// `attempt=2/1` it read as the budget being overrun. Say which is which.
|
|
185
|
+
if (uncharged > 0 && uncharged < details.attempt) {
|
|
186
|
+
return ` attempt=${details.attempt}`
|
|
187
|
+
+ ` (iteration ${details.attempt - uncharged}${details.maxIterations === undefined ? '' : `/${details.maxIterations}`};`
|
|
188
|
+
+ ` ${uncharged} earlier attempt${uncharged === 1 ? '' : 's'} lost to a dead worker, not charged)`;
|
|
189
|
+
}
|
|
172
190
|
return ` attempt=${details.attempt}`
|
|
173
191
|
+ (details.maxIterations === undefined ? '' : `/${details.maxIterations}`);
|
|
174
192
|
}
|
package/src/cloud-run.ts
CHANGED
|
@@ -98,6 +98,7 @@ export async function prepareCloudSubmission(
|
|
|
98
98
|
options: { input?: JsonValue; signal?: AbortSignal } = {},
|
|
99
99
|
): Promise<CloudSubmission> {
|
|
100
100
|
let spec: FlowSpec | undefined;
|
|
101
|
+
let compiledKernelInput = false;
|
|
101
102
|
let authored: { source: string; authority: CloudAuthoredAuthority; name: string; schedules: ScheduleTriggerSource[]; extensions: readonly FlowExtensionSubmission[] } | undefined;
|
|
102
103
|
const inputPresent = Object.prototype.hasOwnProperty.call(options, 'input');
|
|
103
104
|
let authoredInput: JsonValue | undefined;
|
|
@@ -150,7 +151,11 @@ export async function prepareCloudSubmission(
|
|
|
150
151
|
// compatibility. If both fail, retain the original authoring diagnostic.
|
|
151
152
|
try {
|
|
152
153
|
spec = compileSpec(kernelToAuthoring(parsed));
|
|
153
|
-
|
|
154
|
+
compiledKernelInput = true;
|
|
155
|
+
} catch (kernelError) {
|
|
156
|
+
if (kernelError instanceof CompileError && kernelError.kind === 'untrusted_cli_identity') {
|
|
157
|
+
throw new CloudFlowError('invalid_input', kernelError.message);
|
|
158
|
+
}
|
|
154
159
|
throw error;
|
|
155
160
|
}
|
|
156
161
|
}
|
|
@@ -176,7 +181,9 @@ export async function prepareCloudSubmission(
|
|
|
176
181
|
const kernel = toKernelSpec(spec!);
|
|
177
182
|
// JSON is a YAML subset. Sending canonical data preserves the exact spec
|
|
178
183
|
// while using the server's existing YAML-to-config admission path.
|
|
179
|
-
|
|
184
|
+
// A compiled input stays in the kernel dialect. kernelToAuthoring refuses
|
|
185
|
+
// serialized cli_identity: only local preflight may mint that proof.
|
|
186
|
+
return { workflow: canonicalize(compiledKernelInput ? kernel : spec), fileType: 'yaml', inputPresent: false, specHash: specHash(kernel),
|
|
180
187
|
name: spec!.name ?? "flow", schedules: [] };
|
|
181
188
|
}
|
|
182
189
|
return {
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { communicationHistory } from './history.js';
|
|
2
|
-
import {
|
|
2
|
+
import { mkdir, mkdtemp, rm, symlink } from 'node:fs/promises';
|
|
3
|
+
import { basename, join } from 'node:path';
|
|
3
4
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
4
5
|
import type { JournalClient } from '../journal-client.js';
|
|
5
6
|
import type { StepDispatchEvent } from '../protocol.js';
|
|
6
|
-
import type {
|
|
7
|
+
import type { ResolvedKernelAgentStep } from '../resolved-cli-identity.js';
|
|
7
8
|
import { withWorkerLease } from '../worker-lease.js';
|
|
8
9
|
import { workerInstruction } from '../worker-input.js';
|
|
9
10
|
import { resolveCliModel } from '../cli-adapter.js';
|
|
@@ -20,7 +21,7 @@ export function requireCommunicationCli(cli: string | undefined): void {
|
|
|
20
21
|
export async function completeCommunicationDispatch(client: JournalClient, dispatch: StepDispatchEvent,
|
|
21
22
|
instruction: CommunicationInstruction, dataDir: string, runRoot?: string,
|
|
22
23
|
environment?: NodeJS.ProcessEnv): Promise<void> {
|
|
23
|
-
const spec = dispatch.spec as
|
|
24
|
+
const spec = dispatch.spec as ResolvedKernelAgentStep;
|
|
24
25
|
requireCommunicationCli(spec.cli);
|
|
25
26
|
let output: unknown;
|
|
26
27
|
let completionReason: 'success' | 'worker_error' = 'success';
|
|
@@ -36,7 +37,7 @@ export async function completeCommunicationDispatch(client: JournalClient, dispa
|
|
|
36
37
|
started_pins: dispatch.pins, end_pins: dispatch.pins });
|
|
37
38
|
}
|
|
38
39
|
async function run(client: JournalClient, dispatch: StepDispatchEvent, instruction: CommunicationInstruction,
|
|
39
|
-
spec:
|
|
40
|
+
spec: ResolvedKernelAgentStep, dataDir: string, lease: AbortSignal, runRoot?: string,
|
|
40
41
|
environment?: NodeJS.ProcessEnv): Promise<unknown> {
|
|
41
42
|
// Same contract as the CLI worker: a declared directory is resolved and held
|
|
42
43
|
// inside the same run root the CLI worker measures against, before anything
|
|
@@ -53,6 +54,7 @@ async function run(client: JournalClient, dispatch: StepDispatchEvent, instructi
|
|
|
53
54
|
let pumping: Promise<void> | undefined;
|
|
54
55
|
let receipts = Promise.resolve();
|
|
55
56
|
let unsubscribe: (() => void) | undefined;
|
|
57
|
+
let cliLinkDirectory: string | undefined;
|
|
56
58
|
try {
|
|
57
59
|
let resolve!: (value: unknown) => void;
|
|
58
60
|
let reject!: (reason: unknown) => void;
|
|
@@ -80,10 +82,20 @@ async function run(client: JournalClient, dispatch: StepDispatchEvent, instructi
|
|
|
80
82
|
void receipts.catch(reject);
|
|
81
83
|
});
|
|
82
84
|
// Relay supplies each CLI's launch flags and injection behavior.
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
85
|
+
const cliIdentity = spec.cli_identity ?? spec.cli!;
|
|
86
|
+
const argv0 = basename(cliIdentity);
|
|
87
|
+
let command = spec.cli!;
|
|
88
|
+
if (basename(command) !== argv0) {
|
|
89
|
+
const linkRoot = join(dataDir, 'communication');
|
|
90
|
+
await mkdir(linkRoot, { recursive: true, mode: 0o700 });
|
|
91
|
+
cliLinkDirectory = await mkdtemp(join(linkRoot, 'cli-'));
|
|
92
|
+
command = join(cliLinkDirectory, argv0);
|
|
93
|
+
await symlink(spec.cli!, command, 'file');
|
|
94
|
+
}
|
|
95
|
+
handle = await relay.broker.spawnPty({ name, cli: basename(cliIdentity).replace(/\.exe$/i, ''), task: prompt, channels: [], skipRelayPrompt: true,
|
|
96
|
+
model: resolveCliModel(cliIdentity, spec.model), cwd: directory,
|
|
97
|
+
harnessConfig: { runtime: 'pty', command: quote(command), args: [],
|
|
98
|
+
cwd: directory, env: { ...agentEnvironment(cliIdentity, environment ?? process.env),
|
|
87
99
|
RELAYFLOW_COMMUNICATION_SOCKET: tools.path, RELAYFLOW_COMMUNICATION_TOKEN: tools.token },
|
|
88
100
|
delivery: { mode: 'pty-injection', format: 'relay-block' } } });
|
|
89
101
|
const ready = await handle.waitForReady(Math.min(instruction.timeoutMs, 90_000));
|
|
@@ -100,7 +112,10 @@ async function run(client: JournalClient, dispatch: StepDispatchEvent, instructi
|
|
|
100
112
|
finally {
|
|
101
113
|
try { await receipts; }
|
|
102
114
|
finally { try { await handle?.release('Flow communication attempt ended', { deleteIdentity: true }); }
|
|
103
|
-
finally { try { await tools?.close(); } finally {
|
|
115
|
+
finally { try { await tools?.close(); } finally {
|
|
116
|
+
try { await relay.close(); }
|
|
117
|
+
finally { if (cliLinkDirectory) await rm(cliLinkDirectory, { recursive: true, force: true }); }
|
|
118
|
+
} } }
|
|
104
119
|
}
|
|
105
120
|
}
|
|
106
121
|
}
|
package/src/compile.ts
CHANGED
|
@@ -24,10 +24,7 @@ import type {
|
|
|
24
24
|
AgentStepSpec,
|
|
25
25
|
DeterministicStepSpec,
|
|
26
26
|
FlowSpec,
|
|
27
|
-
KernelAgentStep,
|
|
28
27
|
KernelRunSpec,
|
|
29
|
-
KernelStepCommon,
|
|
30
|
-
KernelStepSpec,
|
|
31
28
|
KernelTriggerSpec,
|
|
32
29
|
KernelVerificationSpec,
|
|
33
30
|
LlmStepSpec,
|
|
@@ -45,6 +42,13 @@ import { validateSpec, type ValidationResult } from './validate.js';
|
|
|
45
42
|
import { snapshotJsonValue } from './json-value.js';
|
|
46
43
|
import { expandYamlHelpers } from './yaml-helpers.js';
|
|
47
44
|
import { expandCommunication, validateCommunicationTopology } from './communication/spec.js';
|
|
45
|
+
import {
|
|
46
|
+
inheritResolvedCliIdentities,
|
|
47
|
+
resolvedCliIdentities,
|
|
48
|
+
type ResolvedKernelAgentStep,
|
|
49
|
+
type ResolvedKernelStepCommon,
|
|
50
|
+
type ResolvedKernelStepSpec,
|
|
51
|
+
} from './resolved-cli-identity.js';
|
|
48
52
|
|
|
49
53
|
export class CompileError extends Error {
|
|
50
54
|
readonly errors: string[];
|
|
@@ -173,7 +177,7 @@ export function compileSpec(spec: unknown): CompiledFlowSpec {
|
|
|
173
177
|
...(input.workspace !== undefined ? { workspace: input.workspace } : {}),
|
|
174
178
|
...(input.tools !== undefined ? { tools: input.tools } : {}),
|
|
175
179
|
};
|
|
176
|
-
return flow;
|
|
180
|
+
return inheritResolvedCliIdentities(spec, flow);
|
|
177
181
|
}
|
|
178
182
|
|
|
179
183
|
function compileStep(step: StepSpec): StepSpec {
|
|
@@ -325,6 +329,7 @@ export function toKernelSpec(flow: FlowSpec): KernelRunSpec {
|
|
|
325
329
|
// This public boundary is callable without compileSpec. Compile again so
|
|
326
330
|
// runtime casts are validated and all returned schema data is snapshotted.
|
|
327
331
|
const compiled = compileSpec(flow);
|
|
332
|
+
const cliIdentities = resolvedCliIdentities(compiled);
|
|
328
333
|
|
|
329
334
|
return {
|
|
330
335
|
version: compiled.version,
|
|
@@ -347,7 +352,10 @@ export function toKernelSpec(flow: FlowSpec): KernelRunSpec {
|
|
|
347
352
|
// reverse order would hand the green gate a source whose barrier it does
|
|
348
353
|
// not wait for.
|
|
349
354
|
steps: lowerNamedGates(lowerStepsGreenGates(compiled.steps))
|
|
350
|
-
.map((step) => toKernelStep(
|
|
355
|
+
.map((step) => toKernelStep(
|
|
356
|
+
resolveNamedAgent(step, compiled.agents),
|
|
357
|
+
cliIdentities?.get(step.id),
|
|
358
|
+
)),
|
|
351
359
|
...(compiled.budget !== undefined ? { budget: toKernelBudget(compiled.budget) } : {}),
|
|
352
360
|
};
|
|
353
361
|
}
|
|
@@ -374,7 +382,7 @@ export function kernelToAuthoring(value: unknown): unknown {
|
|
|
374
382
|
const steps = requireKernelArray(root['steps'], 'spec.steps')
|
|
375
383
|
.map((step, index) => kernelStepToAuthoring(step, `spec.steps[${index}]`));
|
|
376
384
|
const triggers = root['triggers'];
|
|
377
|
-
|
|
385
|
+
const authoring: FlowSpec = {
|
|
378
386
|
...copyDefined(root, ['version', 'name', 'description', 'cli']),
|
|
379
387
|
...(triggers !== undefined
|
|
380
388
|
? {
|
|
@@ -386,7 +394,8 @@ export function kernelToAuthoring(value: unknown): unknown {
|
|
|
386
394
|
...(root['budget'] !== undefined
|
|
387
395
|
? { budget: kernelBudgetToAuthoring(root['budget'], 'spec.budget') }
|
|
388
396
|
: {}),
|
|
389
|
-
};
|
|
397
|
+
} as FlowSpec;
|
|
398
|
+
return authoring;
|
|
390
399
|
}
|
|
391
400
|
|
|
392
401
|
/**
|
|
@@ -443,17 +452,22 @@ function kernelStepToAuthoring(value: unknown, at: string): unknown {
|
|
|
443
452
|
const unionKeys = [
|
|
444
453
|
'id', 'type', 'depends_on', 'max_iterations', 'retry', 'verification', 'memory', 'requirements', 'input',
|
|
445
454
|
'command', 'timeout_ms', 'lease_ms', 'on_non_zero', 'prompt', 'model', 'cli', 'instruction',
|
|
446
|
-
'cwd', 'recovery_mode', 'surfaces', 'permissions',
|
|
455
|
+
'cwd', 'recovery_mode', 'surfaces', 'permissions', 'cli_identity',
|
|
447
456
|
] as const;
|
|
448
457
|
const step = requireKernelObject(value, unionKeys, at);
|
|
458
|
+
if (step['cli_identity'] !== undefined) {
|
|
459
|
+
throw new CompileError([
|
|
460
|
+
`${at}.cli_identity: host-proved adapter identity is not accepted from serialized input`,
|
|
461
|
+
], 'untrusted_cli_identity');
|
|
462
|
+
}
|
|
449
463
|
const type = step['type'];
|
|
450
464
|
const commonKeys = ['id', 'type', 'depends_on', 'max_iterations', 'retry', 'verification', 'memory', 'requirements', 'input'] as const;
|
|
451
465
|
const typeKeys = type === 'deterministic'
|
|
452
466
|
? ['command', 'timeout_ms', 'lease_ms', 'on_non_zero'] as const
|
|
453
467
|
: type === 'llm'
|
|
454
|
-
? ['prompt', 'model', 'cli'] as const
|
|
468
|
+
? ['prompt', 'model', 'cli', 'cli_identity'] as const
|
|
455
469
|
: type === 'agent'
|
|
456
|
-
? ['instruction', 'cli', 'model', 'cwd', 'recovery_mode', 'surfaces', 'permissions'] as const
|
|
470
|
+
? ['instruction', 'cli', 'model', 'cwd', 'recovery_mode', 'surfaces', 'permissions', 'cli_identity'] as const
|
|
457
471
|
: [];
|
|
458
472
|
assertKernelKeys(step, [...commonKeys, ...typeKeys], at);
|
|
459
473
|
if (step['retry'] !== undefined) validateAuthoringRetryDefaults(step['retry'], `${at}.retry`);
|
|
@@ -596,8 +610,8 @@ function isObject(value: unknown): value is Record<string, unknown> {
|
|
|
596
610
|
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
597
611
|
}
|
|
598
612
|
|
|
599
|
-
function toKernelStep(step: StepSpec):
|
|
600
|
-
const common:
|
|
613
|
+
function toKernelStep(step: StepSpec, cliIdentity?: string): ResolvedKernelStepSpec {
|
|
614
|
+
const common: ResolvedKernelStepCommon = {
|
|
601
615
|
id: step.id,
|
|
602
616
|
depends_on: step.input === undefined ? step.dependsOn ?? []
|
|
603
617
|
: [...new Set([...(step.dependsOn ?? []), ...bindingDependencies(step.input)])],
|
|
@@ -605,6 +619,7 @@ function toKernelStep(step: StepSpec): KernelStepSpec {
|
|
|
605
619
|
max_iterations: step.maxIterations ?? 1,
|
|
606
620
|
retry: { ...KERNEL_RETRY_DEFAULTS },
|
|
607
621
|
verification: toKernelVerification(step),
|
|
622
|
+
...(cliIdentity !== undefined ? { cli_identity: cliIdentity } : {}),
|
|
608
623
|
...(step.requirements !== undefined ? { requirements: {
|
|
609
624
|
...Object.fromEntries(Object.entries(step.requirements).filter(([key]) => key !== 'expectedDurationMs')),
|
|
610
625
|
...(step.requirements.expectedDurationMs !== undefined ? { expected_duration_ms: step.requirements.expectedDurationMs } : {}),
|
|
@@ -643,7 +658,7 @@ function toKernelStep(step: StepSpec): KernelStepSpec {
|
|
|
643
658
|
};
|
|
644
659
|
}
|
|
645
660
|
case 'agent': {
|
|
646
|
-
const out:
|
|
661
|
+
const out: ResolvedKernelAgentStep = {
|
|
647
662
|
...common,
|
|
648
663
|
type: 'agent',
|
|
649
664
|
instruction: step.instruction,
|
package/src/failure-kinds.ts
CHANGED
|
@@ -144,6 +144,12 @@ export const RUN_FAILURE_KINDS = [
|
|
|
144
144
|
* attaches and before the resume touches the journal.
|
|
145
145
|
*/
|
|
146
146
|
'local_agent_unavailable',
|
|
147
|
+
/**
|
|
148
|
+
* The authored root's worker lease was lost and the kernel's retry did not
|
|
149
|
+
* reach this process. The run is NOT failed (`status: running`): its
|
|
150
|
+
* completed steps are journaled and `flows resume <rootRunId>` continues it.
|
|
151
|
+
*/
|
|
152
|
+
'root_lease_lost',
|
|
147
153
|
] as const;
|
|
148
154
|
|
|
149
155
|
/**
|
|
@@ -271,6 +277,14 @@ export interface StepFailedDetails {
|
|
|
271
277
|
* `retries_exhausted`.
|
|
272
278
|
*/
|
|
273
279
|
maxIterations?: number;
|
|
280
|
+
/**
|
|
281
|
+
* Attempts before this one that died without a result -- the kernel
|
|
282
|
+
* journaled them `crashed` or `lease_expired` itself -- and so, by design,
|
|
283
|
+
* consumed none of `maxIterations` (relayflowd-core recovery.rs). Present
|
|
284
|
+
* only when non-zero. Without it `attempt` can exceed `maxIterations`
|
|
285
|
+
* (`attempt=2/1`) and read as the kernel overrunning its own budget.
|
|
286
|
+
*/
|
|
287
|
+
unchargedAttempts?: number;
|
|
274
288
|
exitCode?: number;
|
|
275
289
|
/**
|
|
276
290
|
* Terminal-safe UTF-8 excerpt of the captured stdout, at most
|