@uchibeke/gallivant 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +26 -0
- package/LICENSE +204 -0
- package/README.md +201 -2
- package/SECURITY.md +19 -0
- package/bin/gallivant.mjs +257 -0
- package/bin/package-plugin.mjs +16 -0
- package/bin/release-live.mjs +29 -0
- package/bin/release-package.mjs +23 -0
- package/bin/session-runtime.mjs +13 -0
- package/bin/task-delivery-state.mjs +12 -0
- package/docs/README.md +70 -0
- package/docs/agent-setup.md +214 -0
- package/docs/app.md +111 -0
- package/docs/architecture.md +265 -0
- package/docs/chains.md +102 -0
- package/docs/codex-cloud.md +284 -0
- package/docs/commands.md +620 -0
- package/docs/config.md +430 -0
- package/docs/execution.md +796 -0
- package/docs/guardrails.md +311 -0
- package/docs/install.md +338 -0
- package/docs/launch.md +42 -0
- package/docs/learning.md +369 -0
- package/docs/network-recovery.md +101 -0
- package/docs/openai-agents.md +457 -0
- package/docs/prd.md +246 -0
- package/docs/preparation.md +110 -0
- package/docs/release-qualification.md +196 -0
- package/docs/releasing.md +140 -0
- package/docs/runtime-environment.md +113 -0
- package/docs/task-delivery.md +476 -0
- package/docs/tasks.md +739 -0
- package/lib/aport.mjs +95 -0
- package/lib/attribution.mjs +95 -0
- package/lib/block.mjs +89 -0
- package/lib/capabilities.mjs +3 -0
- package/lib/chains.mjs +222 -0
- package/lib/codex_cloud.mjs +160 -0
- package/lib/codex_cloud_config.mjs +20 -0
- package/lib/codex_cloud_input.mjs +49 -0
- package/lib/codex_cloud_result.mjs +74 -0
- package/lib/commands/doctor.mjs +457 -0
- package/lib/commands/init.mjs +34 -0
- package/lib/commands/learn.mjs +54 -0
- package/lib/commands/new.mjs +47 -0
- package/lib/commands/ready.mjs +45 -0
- package/lib/commands/register.mjs +195 -0
- package/lib/commands/resume.mjs +224 -0
- package/lib/commands/serve.mjs +349 -0
- package/lib/commands/start.mjs +247 -0
- package/lib/commands/start_services.mjs +22 -0
- package/lib/commands/status.mjs +150 -0
- package/lib/commands/task.mjs +73 -0
- package/lib/commands/up.mjs +228 -0
- package/lib/commands/why.mjs +143 -0
- package/lib/config.mjs +461 -0
- package/lib/context.mjs +100 -0
- package/lib/control.mjs +202 -0
- package/lib/control_contract.mjs +16 -0
- package/lib/coordinator.mjs +19 -0
- package/lib/credential.mjs +161 -0
- package/lib/daemon.mjs +156 -0
- package/lib/deliverable.mjs +357 -0
- package/lib/events.mjs +261 -0
- package/lib/executable.mjs +31 -0
- package/lib/execution.mjs +631 -0
- package/lib/execution_contract.mjs +15 -0
- package/lib/execution_input.mjs +4 -0
- package/lib/execution_io.mjs +94 -0
- package/lib/execution_journal.mjs +69 -0
- package/lib/execution_preparation.mjs +29 -0
- package/lib/execution_result.mjs +266 -0
- package/lib/fs.mjs +133 -0
- package/lib/gh.mjs +289 -0
- package/lib/git.mjs +434 -0
- package/lib/harness.mjs +521 -0
- package/lib/harness_log.mjs +432 -0
- package/lib/hash.mjs +16 -0
- package/lib/identity.mjs +98 -0
- package/lib/learning.mjs +654 -0
- package/lib/learning_contract.mjs +2 -0
- package/lib/learning_data.mjs +135 -0
- package/lib/learning_input.mjs +111 -0
- package/lib/learning_qualification.mjs +67 -0
- package/lib/learning_signer.mjs +151 -0
- package/lib/learning_store.mjs +88 -0
- package/lib/log.mjs +53 -0
- package/lib/network_diagnostics.mjs +64 -0
- package/lib/network_recovery.mjs +42 -0
- package/lib/notify.mjs +58 -0
- package/lib/openai_agents.mjs +373 -0
- package/lib/openai_agents_config.mjs +14 -0
- package/lib/openai_agents_http.mjs +87 -0
- package/lib/openai_agents_input.mjs +15 -0
- package/lib/openai_agents_sources.mjs +112 -0
- package/lib/owner_delivery.mjs +443 -0
- package/lib/plugin_package.mjs +340 -0
- package/lib/prd.mjs +635 -0
- package/lib/preparation_agents.mjs +33 -0
- package/lib/preparation_codex.mjs +212 -0
- package/lib/probe.mjs +126 -0
- package/lib/prompt.mjs +238 -0
- package/lib/provider_health.mjs +55 -0
- package/lib/public-format.mjs +31 -0
- package/lib/public_output.mjs +49 -0
- package/lib/release_package.mjs +154 -0
- package/lib/repository_files.mjs +86 -0
- package/lib/run.mjs +385 -0
- package/lib/run_log.mjs +94 -0
- package/lib/runtime-environment.mjs +87 -0
- package/lib/runtime_dependencies.mjs +15 -0
- package/lib/runtime_node.mjs +107 -0
- package/lib/runtime_state.mjs +174 -0
- package/lib/sources/index.mjs +27 -0
- package/lib/sources/prd.mjs +50 -0
- package/lib/start_contract.mjs +37 -0
- package/lib/state.mjs +246 -0
- package/lib/task_allowance.mjs +31 -0
- package/lib/task_completion.mjs +61 -0
- package/lib/task_contract.mjs +450 -0
- package/lib/task_delivery.mjs +219 -0
- package/lib/task_delivery_cancel.mjs +34 -0
- package/lib/task_delivery_contract.mjs +61 -0
- package/lib/task_delivery_mcp.mjs +40 -0
- package/lib/task_delivery_store.mjs +43 -0
- package/lib/task_delivery_view.mjs +88 -0
- package/lib/task_environment.mjs +281 -0
- package/lib/task_prepare.mjs +224 -0
- package/lib/task_store.mjs +38 -0
- package/lib/tasks.mjs +513 -0
- package/lib/tick.mjs +1635 -0
- package/lib/time.mjs +36 -0
- package/lib/tracker/assets.mjs +43 -0
- package/lib/tracker/pages.mjs +320 -0
- package/lib/tracker/runs.mjs +102 -0
- package/lib/tracker/stream.mjs +74 -0
- package/lib/tracker_process.mjs +107 -0
- package/lib/verification_sandbox.mjs +181 -0
- package/node_modules/smol-toml/LICENSE +24 -0
- package/node_modules/smol-toml/README.md +605 -0
- package/node_modules/smol-toml/dist/date.d.ts +41 -0
- package/node_modules/smol-toml/dist/date.js +156 -0
- package/node_modules/smol-toml/dist/error.d.ts +37 -0
- package/node_modules/smol-toml/dist/error.js +67 -0
- package/node_modules/smol-toml/dist/extract.js +289 -0
- package/node_modules/smol-toml/dist/index.cjs +770 -0
- package/node_modules/smol-toml/dist/index.d.ts +44 -0
- package/node_modules/smol-toml/dist/index.js +34 -0
- package/node_modules/smol-toml/dist/parse.d.ts +73 -0
- package/node_modules/smol-toml/dist/parse.js +164 -0
- package/node_modules/smol-toml/dist/primitive.js +151 -0
- package/node_modules/smol-toml/dist/stringify.d.ts +32 -0
- package/node_modules/smol-toml/dist/stringify.js +219 -0
- package/node_modules/smol-toml/dist/struct.js +156 -0
- package/node_modules/smol-toml/dist/util.d.ts +43 -0
- package/node_modules/smol-toml/dist/util.js +57 -0
- package/node_modules/smol-toml/package.json +68 -0
- package/npm-shrinkwrap.json +1823 -0
- package/package.json +105 -4
- package/templates/claude-settings.json +18 -0
- package/templates/passport.gallivant.json +83 -0
- package/templates/plugin/bundled-runtime.json +26 -0
- package/templates/plugin/schemas/mcp.schema.json +120 -0
- package/templates/plugin/schemas/plugin.schema.json +65 -0
- package/templates/plugin/task-delivery.SKILL.md +60 -0
- package/templates/prd.md +32 -0
- package/templates/tracker/tracker.css +301 -0
- package/templates/tracker/tracker.js +721 -0
|
@@ -0,0 +1,631 @@
|
|
|
1
|
+
// Versioned durable transport boundary. No transport implements a review loop.
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { existsSync, rmSync } from "node:fs";
|
|
4
|
+
import { randomUUID, createHash } from "node:crypto";
|
|
5
|
+
import { ensureDir, readJson, writeJsonAtomic } from "./fs.mjs";
|
|
6
|
+
import { acquireProcessLock } from "./daemon.mjs";
|
|
7
|
+
import { capabilitiesFor, adapterFor, preparationReceipt, isPreparationReceipt } from "./harness.mjs";
|
|
8
|
+
import { executionLive, PROVIDER_STATES, providerTerminal as terminal, ownedRemoteExecution as ownedRemote } from "./execution_contract.mjs";
|
|
9
|
+
export { executionLive, PROVIDER_STATES } from "./execution_contract.mjs";
|
|
10
|
+
import { prepareLocalExecution } from "./execution_preparation.mjs";
|
|
11
|
+
import { EXECUTION_LIMITS, boundedCall, nextAttempt, executionError, assertExecutionCursor, ownsProviderSession, assertTerminalOutcome, redact, redactResultMetadata } from "./execution_io.mjs";
|
|
12
|
+
import { journalFor } from "./execution_journal.mjs";
|
|
13
|
+
import { requestRunCancellation } from "./run.mjs";
|
|
14
|
+
|
|
15
|
+
export const EXECUTION_VERSION = 1;
|
|
16
|
+
const exactIdentity = id => typeof id === "string" && !!id.trim();
|
|
17
|
+
const freshSessionMatches = (s, sessionId) => !s.learningContinuity?.pending || exactIdentity(sessionId) && sessionId !== s.learningContinuity.sessionId;
|
|
18
|
+
const maximumCost = (...amounts) => amounts.reduce((known, amount) =>
|
|
19
|
+
Number.isFinite(amount) && amount >= 0 ? known === null ? amount : Math.max(known, amount) : known, null);
|
|
20
|
+
const completionTime = (value, startedAt, latestAt, code) => {
|
|
21
|
+
if (value === null || value === undefined) return null;
|
|
22
|
+
if (!Number.isFinite(value) || !Number.isFinite(startedAt) || !Number.isFinite(latestAt) || value < Math.max(0, startedAt) || value > latestAt) throw executionError(code, "invalid owned root completion time; reconcile its authoritative timestamps");
|
|
23
|
+
return value;
|
|
24
|
+
};
|
|
25
|
+
const runCost = (s, id) => maximumCost(s.current?.id === id ? s.current.execution.costUsd : null,
|
|
26
|
+
Object.values(s.operations).find(op => op.runId === id)?.costUsd);
|
|
27
|
+
const digest = v => createHash("sha256").update(JSON.stringify(v)).digest("hex");
|
|
28
|
+
const preparationFingerprint = input => {
|
|
29
|
+
const { receipt: _receipt, ...prepared } = input.prepared || {};
|
|
30
|
+
return digest({ environment: input.environment, ...(input.spec ? { spec: input.spec } : { prompt: input.prompt, prepared }) });
|
|
31
|
+
};
|
|
32
|
+
export const executionRunner = (ctx, id, provider = ctx.state.get(id)?.harness) => ctx.executionFor && provider ? ctx.executionFor(id, provider) : ctx.runnerFor(id);
|
|
33
|
+
|
|
34
|
+
// A sessionless resume is allowed only for an owned, proven unsubmitted first
|
|
35
|
+
// launch. Older refusal entries can recover from the private transport proof;
|
|
36
|
+
// a registry reason or an absent session alone never authorizes a new session.
|
|
37
|
+
export function unsubmittedInitialDispatch(entry, execution) {
|
|
38
|
+
if (execution.capabilities?.transport !== "remote" || entry.session || entry.pr || entry.last?.run || entry.state !== "needs-human" || !["launch-preparation", "dispatch-refused"].includes(entry.reason)) return null;
|
|
39
|
+
const saved = execution.state(), key = entry.start_token;
|
|
40
|
+
const marker = entry.initial_dispatch ?? (entry.reason === "dispatch-refused" && saved.dispatchRefusals?.some(refusal => refusal.operationId === key)
|
|
41
|
+
? { version: 1, operationId: key, status: "unsubmitted" } : null);
|
|
42
|
+
if (!marker || marker.version !== 1 || marker.status !== "unsubmitted" || typeof key !== "string" || !key || key.length > 256 || marker.operationId !== key || entry.execution?.operationId !== key) return null;
|
|
43
|
+
if (entry.dispatch_stopped || saved.stopped) throw executionError("EXECUTION_STOPPED", "initial dispatch stopped; no new session is authorized");
|
|
44
|
+
if (saved.current || Object.keys(saved.operations).length) throw executionError("EXECUTION_UNKNOWN", "initial launch already has an execution intent or run; reconcile its ownership before resume; no new session is authorized");
|
|
45
|
+
return marker;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function assertExecutionDependency(taskService) {
|
|
49
|
+
if (typeof taskService !== "function") throw executionError("EXECUTION_DEPENDENCY", "intake task service is missing; merge the natural-language intake predecessor into the selected base before implementation; do not recreate it");
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** transport: prepare, submit, inspect, continue, collect, cancel; optional lookup(intent).
|
|
53
|
+
* Methods accept scoped immutable input plus AbortSignal and return bounded JSON.
|
|
54
|
+
* A remote adapter never needs a local command, PID, or inherited environment.
|
|
55
|
+
*/
|
|
56
|
+
export function makeExecution({ root, taskId, provider, account = "operator", transport = null, runner = null, legacySessionId = null, capabilities = capabilitiesFor(provider), clock = Date, limits = {}, secrets = [], jitter = Math.random }) {
|
|
57
|
+
const bounds = { ...EXECUTION_LIMITS, ...limits };
|
|
58
|
+
const file = join(root, "execution.json");
|
|
59
|
+
const collectionRejection = s => {
|
|
60
|
+
const job = s.current, ex = job?.execution;
|
|
61
|
+
if (!transport || !ownedRemote(ex) || !["succeeded", "failed"].includes(ex.terminalState)) return null;
|
|
62
|
+
const checkpoint = readJson(join(dirOf(job.id), "result-application.json"));
|
|
63
|
+
return checkpoint?.phase === "blocked" && checkpoint.reason === "result-collection-validation" &&
|
|
64
|
+
checkpoint.taskId === taskId && checkpoint.runId === job.id && checkpoint.providerJobId === ex.providerJobId &&
|
|
65
|
+
checkpoint.sessionId === ex.sessionId && checkpoint.turnId === ex.turnId && checkpoint.inputHead === ex.inputHead ? checkpoint : null;
|
|
66
|
+
};
|
|
67
|
+
const settleCollectionRejection = s => {
|
|
68
|
+
const job = s.current, ex = job.execution;
|
|
69
|
+
Object.assign(ex, { collected: true, resultRejected: true, providerState: ex.terminalState,
|
|
70
|
+
recoveryAction: "result collection rejected; inspect result-application.json and explicitly resume the same session for correction" });
|
|
71
|
+
// Settle provider liveness without inventing a valid result or rewriting its
|
|
72
|
+
// root outcome. The blocked checkpoint prevents implementation completion.
|
|
73
|
+
job.exit = ex.terminalState === "succeeded" ? 0 : 1;
|
|
74
|
+
job.endedAt = ex.completedAt ?? ex.terminalObservedAt ?? clock.now();
|
|
75
|
+
job.completionInferred = ex.completedAt == null;
|
|
76
|
+
};
|
|
77
|
+
const read = () => {
|
|
78
|
+
const s = readJson(file, { version: EXECUTION_VERSION, taskId, provider, account, operations: {}, current: null, stopped: null });
|
|
79
|
+
if (s.version !== EXECUTION_VERSION) throw executionError("EXECUTION_VERSION", `unsupported execution version ${s.version}; stop the coordinator, back up ${root}, and use a compatible package or restore its matching backup; do not redispatch`);
|
|
80
|
+
if (s.taskId !== taskId || s.provider !== provider || s.account !== account) throw executionError("EXECUTION_OWNERSHIP", "execution ownership differs; restore the recorded provider/account; sessions cannot be transplanted");
|
|
81
|
+
if (transport) {
|
|
82
|
+
assertExecutionCursor(s.current?.execution.cursor, { persisted: true });
|
|
83
|
+
for (const op of Object.values(s.operations)) assertExecutionCursor(op.job?.execution?.cursor, { persisted: true });
|
|
84
|
+
}
|
|
85
|
+
// Older/incomplete remote acknowledgements cannot qualify a root turn.
|
|
86
|
+
if (transport && s.current && !ownedRemote(s.current.execution)) {
|
|
87
|
+
const ex = s.current.execution;
|
|
88
|
+
if (ex.acceptance !== "unknown" || !ex.recoveryAction) ex.recoveryAction = "reconcile an exact provider job and root turn using the recorded key";
|
|
89
|
+
Object.assign(ex, { acceptance: "unknown", collected: false,
|
|
90
|
+
providerState: s.stopped ? "cancellation-requested" : "status-unknown",
|
|
91
|
+
terminalState: null, terminalObservedAt: null, completedAt: null });
|
|
92
|
+
const intent = s.operations[s.current.execution.operationId];
|
|
93
|
+
if (intent) intent.acceptance = "unknown";
|
|
94
|
+
}
|
|
95
|
+
// Repair pre-fix failure records that claimed collection without a result.
|
|
96
|
+
const ex = s.current?.execution;
|
|
97
|
+
if (ex) ex.costUsd = runCost(s, s.current.id);
|
|
98
|
+
// The checkpoint is written first. Recover a crash before execution.json or
|
|
99
|
+
// job.json settlement without contacting or redownloading from the provider.
|
|
100
|
+
if (!s.stopped && !ex?.quarantined && collectionRejection(s)) settleCollectionRejection(s);
|
|
101
|
+
if (transport && ex?.providerState === "failed" && ex.collected && !ex.resultRejected && !s.stopped && !ex.quarantined &&
|
|
102
|
+
!existsSync(join(dirOf(s.current.id), "result.json"))) {
|
|
103
|
+
Object.assign(ex, { providerState: "retrieving-results", terminalState: ex.terminalState ?? "failed", collected: false });
|
|
104
|
+
s.current.exit = null; s.current.endedAt = null;
|
|
105
|
+
}
|
|
106
|
+
// Pre-fix collection could overwrite a retained terminal observation.
|
|
107
|
+
// Keep the private cache for inspection but require reconciliation again.
|
|
108
|
+
if (transport && ex?.collected && !s.stopped && !ex.quarantined &&
|
|
109
|
+
["succeeded", "failed"].includes(ex.terminalState) && ex.providerState !== ex.terminalState) {
|
|
110
|
+
Object.assign(ex, { providerState: "retrieving-results", collected: false,
|
|
111
|
+
recoveryAction: "result outcome contradicted the authoritative terminal root; reconcile the saved result" });
|
|
112
|
+
s.current.exit = null; s.current.endedAt = null;
|
|
113
|
+
}
|
|
114
|
+
return s;
|
|
115
|
+
};
|
|
116
|
+
const save = s => writeJsonAtomic(file, s);
|
|
117
|
+
const charge = (s, amount) => {
|
|
118
|
+
const intent = s.operations[s.current.execution.operationId];
|
|
119
|
+
if (intent) intent.costUsd = maximumCost(intent.costUsd, amount);
|
|
120
|
+
if (s.allowance) s.allowance.chargedUsd = Math.max(s.allowance.reservedUsd,
|
|
121
|
+
Object.values(s.operations).reduce((sum, op) => sum + Math.max(op.reservedUsd ?? 0, op.costUsd ?? 0), 0));
|
|
122
|
+
};
|
|
123
|
+
const locked = async fn => {
|
|
124
|
+
ensureDir(root);
|
|
125
|
+
const lock = acquireProcessLock(join(root, "execution.lock"), "execution");
|
|
126
|
+
try { return await fn(read()); } finally { lock.release(); }
|
|
127
|
+
};
|
|
128
|
+
const call = (method, input) => boundedCall(signal => transport[method]({ ...input, signal }), bounds);
|
|
129
|
+
const dirOf = id => {
|
|
130
|
+
if (!/^[A-Za-z0-9_-]+$/.test(id)) throw executionError("EXECUTION_INPUT", "invalid run identity");
|
|
131
|
+
return join(root, id);
|
|
132
|
+
};
|
|
133
|
+
const project = s => {
|
|
134
|
+
if (!s.current) return;
|
|
135
|
+
const dir = ensureDir(dirOf(s.current.id));
|
|
136
|
+
writeJsonAtomic(join(dir, "job.json"), { id: s.current.id, name: s.current.name, startedAt: s.current.startedAt, exit: s.current.exit ?? null, endedAt: s.current.endedAt ?? null, ...(s.current.completionInferred !== undefined ? { completionInferred: s.current.completionInferred } : {}), execution: s.current.execution });
|
|
137
|
+
};
|
|
138
|
+
const cancellationIntent = s => {
|
|
139
|
+
const ex = s.current.execution;
|
|
140
|
+
ex.cancellation ??= { status: capabilities.controls?.cancel ? "pending" : "unsupported",
|
|
141
|
+
reason: s.stopped, requestedAt: clock.now(), idempotencyKey: digest([taskId, provider, account, s.current.id, "cancel"]),
|
|
142
|
+
attempts: 0, failures: 0, nextAttemptAt: clock.now() };
|
|
143
|
+
return ex.cancellation;
|
|
144
|
+
};
|
|
145
|
+
const confirmCancellation = s => {
|
|
146
|
+
Object.assign(s.current.execution, { providerState: "cancelled", collected: true, quarantined: true, recoveryAction: null });
|
|
147
|
+
if (s.current.execution.cancellation) Object.assign(s.current.execution.cancellation, { status: "confirmed", confirmedAt: clock.now() });
|
|
148
|
+
s.current.exit = 137; s.current.endedAt ??= clock.now();
|
|
149
|
+
};
|
|
150
|
+
const collectedLocalOutcome = s => !transport && s.current?.execution.collected && ["succeeded", "failed"].includes(s.current.execution.providerState);
|
|
151
|
+
const completedLocalJob = (s, job) => !transport && job?.id === s.current?.id && job?.running === false && job.exit != null &&
|
|
152
|
+
!job.killed && s.current.execution.acceptance === "accepted" && !s.current.execution.quarantined;
|
|
153
|
+
const localSession = (s, id, reported) => {
|
|
154
|
+
const intent = Object.values(s.operations).find(op => op.runId === id);
|
|
155
|
+
const ex = s.current?.id === id ? s.current.execution : null;
|
|
156
|
+
// The acknowledgement retains a preassigned session even if an older
|
|
157
|
+
// settlement replaced the current identity. Anonymous first turns may
|
|
158
|
+
// establish their session; the legacy registry applies only to old runs.
|
|
159
|
+
const owned = intent?.job?.execution?.sessionId ?? ex?.sessionId ?? (ex || intent ? null : legacySessionId);
|
|
160
|
+
if (s.learningContinuity?.nextRun === id && reported != null && reported === s.learningContinuity.sessionId) throw executionError("EXECUTION_RESULT", "local result reused retired provider memory");
|
|
161
|
+
if ((ex && !ownsProviderSession(owned, ex.sessionId)) || !ownsProviderSession(owned, reported)) {
|
|
162
|
+
throw executionError("EXECUTION_RESULT", "local result does not identify the recorded provider session; reconcile the exact owned session before continuing");
|
|
163
|
+
}
|
|
164
|
+
return reported ?? owned ?? null;
|
|
165
|
+
};
|
|
166
|
+
const settleLocalResult = (s, job) => {
|
|
167
|
+
const ex = s.current.execution, { running: _running, ...record } = job;
|
|
168
|
+
const redactions = [...secrets, ...(readJson(join(dirOf(job.id), "sensitive-redactions.json"))?.secrets || [])];
|
|
169
|
+
const lines = runner.lines(job.id), outcome = redact(adapterFor(provider).outcome(lines), redactions);
|
|
170
|
+
const sessionId = localSession(s, job.id, adapterFor(provider).sessionFrom(lines));
|
|
171
|
+
s.current = { ...record, execution: ex };
|
|
172
|
+
writeJsonAtomic(join(dirOf(job.id), "result.json"), { version: EXECUTION_VERSION, taskId, runId: job.id, providerJobId: ex.providerJobId,
|
|
173
|
+
turnId: ex.turnId, inputHead: ex.inputHead, outcome, sessionId });
|
|
174
|
+
Object.assign(ex, { collected: true, providerState: outcome.ok && job.exit === 0 ? "succeeded" : "failed",
|
|
175
|
+
costUsd: maximumCost(ex.costUsd, outcome.cost), sessionId });
|
|
176
|
+
charge(s, ex.costUsd); save(s); runner.annotate?.(job.id, { execution: ex });
|
|
177
|
+
};
|
|
178
|
+
const confirmLocalStop = (s, job) => {
|
|
179
|
+
// A completed snapshot saved before the stop fences collection survives a
|
|
180
|
+
// crash or result-cache write failure. Active stops have no such snapshot.
|
|
181
|
+
if (!s.current.execution.collected && s.current.exit != null && !s.current.killed &&
|
|
182
|
+
s.current.execution.providerState === "retrieving-results" && completedLocalJob(s, job)) settleLocalResult(s, job);
|
|
183
|
+
// A task stop cannot rewrite a run that was settled before the request.
|
|
184
|
+
// Reconciliation reaches this helper again after the fence is persisted.
|
|
185
|
+
if (collectedLocalOutcome(s)) return;
|
|
186
|
+
Object.assign(s.current.execution, { providerState: "cancelled", collected: true, quarantined: true, recoveryAction: null });
|
|
187
|
+
s.current.exit = 137; s.current.endedAt ??= clock.now();
|
|
188
|
+
// A local implementation timeout stops this run, while the canonical
|
|
189
|
+
// lifecycle still permits an explicit same-session resume after shutdown.
|
|
190
|
+
// Operator, preparation and remote cancellations retain their task fence.
|
|
191
|
+
if (s.stopped === "timeout" && s.current.execution.kind !== "intake") s.stopped = null;
|
|
192
|
+
};
|
|
193
|
+
// Called only under the existing ownership lock and coordinator cadence.
|
|
194
|
+
const sendCancellation = async s => {
|
|
195
|
+
const ex = s.current.execution, request = cancellationIntent(s);
|
|
196
|
+
if (terminal(ex.providerState) && ex.collected) {
|
|
197
|
+
ex.quarantined = true; ex.recoveryAction = null;
|
|
198
|
+
request.status = ex.providerState === "cancelled" ? "confirmed" : "terminal";
|
|
199
|
+
return ex.providerState === "cancelled" ? { confirmed: true } : { confirmed: false, terminal: true };
|
|
200
|
+
}
|
|
201
|
+
if (request.status === "unsupported") { ex.recoveryAction = "remote cancellation unsupported; execution may continue at the retained task link; no future dispatch, late results quarantined"; save(s); project(s); return { confirmed: false, unsupported: true }; }
|
|
202
|
+
if (request.status !== "pending" || ex.acceptance !== "accepted" || !ownedRemote(ex) || clock.now() < request.nextAttemptAt) return { confirmed: false };
|
|
203
|
+
request.attempts++; request.lastAttemptAt = clock.now();
|
|
204
|
+
save(s); project(s); // Persist exact recovered ownership before the stop I/O.
|
|
205
|
+
try {
|
|
206
|
+
const answer = await call("cancel", { taskId, provider, account, runId: s.current.id, providerJobId: ex.providerJobId,
|
|
207
|
+
sessionId: ex.sessionId, turnId: ex.turnId, idempotencyKey: request.idempotencyKey });
|
|
208
|
+
if (typeof answer?.confirmed !== "boolean") throw executionError("EXECUTION_PROTOCOL", "cancellation lacks an acknowledgement; reconcile or retry the recorded key");
|
|
209
|
+
if (answer.confirmed === true) confirmCancellation(s);
|
|
210
|
+
else { request.status = "acknowledged"; ex.recoveryAction = "confirm provider cancellation; no more dispatch"; }
|
|
211
|
+
save(s); project(s);
|
|
212
|
+
return { confirmed: answer?.confirmed === true };
|
|
213
|
+
} catch (error) {
|
|
214
|
+
request.failures++; request.nextAttemptAt = nextAttempt(clock.now(), request.failures, error.retryAfterMs, bounds, jitter);
|
|
215
|
+
ex.recoveryAction = "cancellation reply unavailable; retry the recorded cancellation key";
|
|
216
|
+
ex.nextAttemptAt = Math.max(ex.nextAttemptAt, request.nextAttemptAt);
|
|
217
|
+
save(s); project(s); throw error;
|
|
218
|
+
}
|
|
219
|
+
};
|
|
220
|
+
const authorize = input => {
|
|
221
|
+
if (!capabilities.available) throw executionError("EXECUTION_UNAVAILABLE", `${provider}: ${capabilities.reason}; ${capabilities.fix || "install its implemented transport before selecting it"}`);
|
|
222
|
+
if (capabilities.transport === "remote" && !transport) throw executionError("EXECUTION_TRANSPORT", "remote provider requires its implemented transport; no local executable fallback");
|
|
223
|
+
for (const policy of input.mandatoryPolicy || []) if (!capabilities.policy?.includes(policy)) throw executionError("EXECUTION_POLICY", `provider cannot enforce mandatory policy ${policy}; select a backend that can`);
|
|
224
|
+
if (input.model && !capabilities.models?.includes(input.model)) throw executionError("EXECUTION_INPUT", "unsupported model selection; no fallback is allowed");
|
|
225
|
+
const environments = input.kind === "intake" && capabilities.preparation?.environment ? [capabilities.preparation.environment] : capabilities.environments;
|
|
226
|
+
if (input.environment && !environments?.includes(input.environment)) throw executionError("EXECUTION_INPUT", "unsupported environment selection");
|
|
227
|
+
};
|
|
228
|
+
const authorizePreparation = s => {
|
|
229
|
+
if (s.stopped) throw executionError("EXECUTION_STOPPED", `dispatch stopped: ${s.stopped}; no new preparation`);
|
|
230
|
+
if ((s.current && executionLive(s.current)) || (!transport && runner.current()?.running)) throw executionError("BUSY", "previous preparation is not settled");
|
|
231
|
+
};
|
|
232
|
+
const dispatch = (method, input) => locked(async s => {
|
|
233
|
+
authorize(input);
|
|
234
|
+
if (input.kind === "intake") {
|
|
235
|
+
const fingerprint = preparationFingerprint(input), issued = s.preparations?.[fingerprint];
|
|
236
|
+
if (!isPreparationReceipt(provider, input.preparation, capabilities, account) || !issued || digest(issued) !== digest(input.preparation)) throw executionError("TASK_PREPARATION_BOUNDARY", "dispatch requires this provider/account's issued receipt for the exact prepared input");
|
|
237
|
+
}
|
|
238
|
+
if (typeof input.operationId !== "string" || !input.operationId || input.operationId.length > 256) throw executionError("EXECUTION_INPUT", "a bounded dispatch operationId is required");
|
|
239
|
+
const signature = digest({ method, ...input });
|
|
240
|
+
const prior = Object.hasOwn(s.operations, input.operationId) ? s.operations[input.operationId] : null;
|
|
241
|
+
if (prior) {
|
|
242
|
+
if (prior.signature !== signature) throw executionError("EXECUTION_CONFLICT", "operation identity already owns different input");
|
|
243
|
+
if (prior.acceptance !== "accepted" || (transport && !ownedRemote(prior.job?.execution))) throw executionError("EXECUTION_UNKNOWN", `submission ${prior.runId} acceptance is unknown; reconcile its recorded idempotency key; no new dispatch`);
|
|
244
|
+
return { ...prior.job };
|
|
245
|
+
}
|
|
246
|
+
if (s.learningContinuity?.pending && (method !== "submit" || input.sessionId)) throw executionError("EXECUTION_CONTINUATION", "retired advice requires a new provider session, not continuation");
|
|
247
|
+
if (s.stopped) throw executionError("EXECUTION_STOPPED", `dispatch stopped: ${s.stopped}; late results remain quarantined`);
|
|
248
|
+
if (s.current && executionLive(s.current)) throw executionError("BUSY", "previous provider work is not settled");
|
|
249
|
+
const legacy = !transport && !s.current ? runner.current() : null;
|
|
250
|
+
if (legacy?.running) throw executionError("BUSY", "legacy local writer is still running");
|
|
251
|
+
const ownedSession = transport ? s.current?.execution.sessionId : s.current
|
|
252
|
+
? localSession(s, s.current.id, s.current.execution.sessionId)
|
|
253
|
+
: legacy ? localSession(s, legacy.id, adapterFor(provider).sessionFrom(runner.lines(legacy.id))) : null;
|
|
254
|
+
if (method === "continue" && (!ownedSession || input.sessionId !== ownedSession || !["native-session", "successor-task"].includes(capabilities.continuation))) throw executionError("EXECUTION_CONTINUATION", "continuation requires this provider's recorded logical session and supported continuity");
|
|
255
|
+
if (input.allowance && (!Number.isFinite(input.allowance.reservedUsd) || input.allowance.reservedUsd < 0 || !Number.isFinite(input.allowance.remainingUsd) || input.allowance.reservedUsd > input.allowance.remainingUsd)) throw executionError("EXECUTION_ALLOWANCE", "unknown or exhausted allowance prevents dispatch");
|
|
256
|
+
if (transport && !input.allowance) throw executionError("EXECUTION_ALLOWANCE", "remote dispatch requires a bounded reservation and remaining allowance");
|
|
257
|
+
if (transport && (input.allowance.reservedUsd <= 0 || input.allowance.remainingUsd <= 0)) throw executionError("EXECUTION_ALLOWANCE", "remote dispatch requires a positive allowance and reservation; unknown usage is not zero");
|
|
258
|
+
if (!transport) runner.prepareStart?.();
|
|
259
|
+
const beforeDispatch = structuredClone(s);
|
|
260
|
+
if (input.allowance) {
|
|
261
|
+
s.allowance ??= { totalUsd: input.allowance.remainingUsd, reservedUsd: 0 };
|
|
262
|
+
if (Math.max(s.allowance.reservedUsd, s.allowance.chargedUsd ?? 0) + input.allowance.reservedUsd > s.allowance.totalUsd) throw executionError("EXECUTION_ALLOWANCE", "persisted task allowance exhausted; unknown usage retains its reservation");
|
|
263
|
+
s.allowance.reservedUsd += input.allowance.reservedUsd;
|
|
264
|
+
}
|
|
265
|
+
const id = `run-${randomUUID()}`;
|
|
266
|
+
const intent = { operationId: input.operationId, signature, runId: id, idempotencyKey: digest([taskId, provider, account, input.operationId]), acceptance: "unknown", at: clock.now(), inputHead: input.inputHead ?? null, reservedUsd: input.allowance?.reservedUsd ?? null, costUsd: null };
|
|
267
|
+
s.operations = { ...s.operations, [input.operationId]: intent };
|
|
268
|
+
s.current = { id, name: input.name || method, startedAt: clock.now(), exit: null, execution: {
|
|
269
|
+
version: EXECUTION_VERSION, taskId, provider, account, transport: transport ? "remote" : "cli", runId: id, operationId: input.operationId,
|
|
270
|
+
providerJobId: null, sessionId: input.sessionId ?? (!transport ? input.recordedSessionId : null) ?? null, turnId: null, inputHead: input.inputHead ?? null,
|
|
271
|
+
environment: input.environment ?? null, model: input.model ?? null, providerState: "status-unknown", acceptance: "unknown",
|
|
272
|
+
observedAt: null, reconciledAt: null, nextAttemptAt: clock.now(), failures: 0, costUsd: null, collected: false,
|
|
273
|
+
recoveryAction: "reconcile dispatch acceptance using the recorded key", allowance: input.allowance ?? null,
|
|
274
|
+
controls: capabilities.controls ?? {}, continuation: capabilities.continuation ?? null, mandatoryPolicy: input.mandatoryPolicy || [],
|
|
275
|
+
workflow: input.workflow ?? null,
|
|
276
|
+
kind: input.kind ?? null, preparation: input.kind === "intake" ? input.preparation : null,
|
|
277
|
+
...(input.learning ? { learning: input.learning, managedInputDigest: input.managedInputDigest } : {}),
|
|
278
|
+
} };
|
|
279
|
+
save(s); // Flush intent before a potentially paid operation.
|
|
280
|
+
if (transport) project(s);
|
|
281
|
+
let accepted;
|
|
282
|
+
try {
|
|
283
|
+
accepted = transport ? await call(method, { ...input, taskId, provider, account, runId: id, idempotencyKey: intent.idempotencyKey })
|
|
284
|
+
: await runner.start(input.name, input.spec.cmd, input.spec.args, { ...input.spec.options, runId: id, privateOutput: { taskId, runId: id, provider } });
|
|
285
|
+
} catch (e) {
|
|
286
|
+
if (transport && e.dispatchNotSent === true) {
|
|
287
|
+
// Restore the previous owned root and reservation, while retaining a
|
|
288
|
+
// bounded private refusal record. No provider identity was dispatched;
|
|
289
|
+
// the same authorized operation may be retried after host/input repair.
|
|
290
|
+
beforeDispatch.dispatchRefusals = [...(beforeDispatch.dispatchRefusals || []).slice(-19), { operationId: input.operationId, runId: id, code: String(e.code || "EXECUTION_INPUT").slice(0, 128), at: clock.now() }];
|
|
291
|
+
save(beforeDispatch);
|
|
292
|
+
rmSync(dirOf(id), { recursive: true, force: true });
|
|
293
|
+
throw e;
|
|
294
|
+
}
|
|
295
|
+
s.current.execution.recoveryAction = e.code === "EXECUTION_DEADLINE" ? "deadline exceeded; reconcile acceptance" : "submission reply lost; reconcile acceptance";
|
|
296
|
+
save(s); if (transport) project(s); throw e;
|
|
297
|
+
}
|
|
298
|
+
if (transport && (!ownedRemote(accepted) || !ownsProviderSession(s.current.execution.sessionId, accepted?.sessionId) || !freshSessionMatches(s, accepted?.sessionId))) {
|
|
299
|
+
if (exactIdentity(accepted?.providerJobId)) s.current.execution.providerJobId = accepted.providerJobId;
|
|
300
|
+
s.current.execution.recoveryAction = "submission lacks exact job/root-turn/session ownership; reconcile the recorded key";
|
|
301
|
+
save(s); project(s);
|
|
302
|
+
throw executionError("EXECUTION_UNKNOWN", "submission did not acknowledge exact provider job and root-turn IDs or the owned session; reconcile before retrying");
|
|
303
|
+
}
|
|
304
|
+
const ex = s.current.execution;
|
|
305
|
+
Object.assign(ex, { acceptance: "accepted", providerJobId: accepted.providerJobId ?? accepted.id, sessionId: accepted.sessionId ?? ex.sessionId, turnId: accepted.turnId ?? null, model: accepted.model ?? ex.model, environment: accepted.environment ?? ex.environment, ...(accepted.effective ? { effective: accepted.effective } : {}), providerState: "queued", recoveryAction: null });
|
|
306
|
+
if (!transport) s.current = { ...accepted, execution: { ...ex, runId: accepted.id } };
|
|
307
|
+
if (s.learningContinuity?.pending) s.learningContinuity = { ...s.learningContinuity, pending: false, nextRun: s.current.id, nextSession: ex.sessionId };
|
|
308
|
+
intent.acceptance = "accepted"; intent.job = s.current;
|
|
309
|
+
save(s); if (transport) project(s); else runner.annotate?.(s.current.id, { execution: s.current.execution });
|
|
310
|
+
return { ...s.current };
|
|
311
|
+
});
|
|
312
|
+
const current = () => {
|
|
313
|
+
const s = read();
|
|
314
|
+
if (!transport) {
|
|
315
|
+
const job = runner.current();
|
|
316
|
+
if (job && (!s.current || s.current.id === job.id)) return { ...job, ...(s.current ? { execution: s.current.execution,
|
|
317
|
+
...(s.current.execution.quarantined ? { exit: s.current.exit, endedAt: s.current.endedAt } : {}) } : {}) };
|
|
318
|
+
if (!s.current) return job; // Legacy local runs retain their exact identity.
|
|
319
|
+
}
|
|
320
|
+
return s.current ? { ...s.current, running: executionLive(s.current) } : null;
|
|
321
|
+
};
|
|
322
|
+
async function reconcile({ force = false } = {}) {
|
|
323
|
+
return locked(async s => {
|
|
324
|
+
if (!transport) runner.observe?.();
|
|
325
|
+
if (!s.current) return current();
|
|
326
|
+
if (!transport) {
|
|
327
|
+
let job = runner.current();
|
|
328
|
+
if (!job || job.id !== s.current.id) return current();
|
|
329
|
+
if (s.current.execution.acceptance === "unknown") {
|
|
330
|
+
// Only an exact launch name supplied by the boundary proves local ownership.
|
|
331
|
+
const intent = s.operations[s.current.execution.operationId];
|
|
332
|
+
s.current = { ...job, execution: { ...s.current.execution, runId: job.id, acceptance: "accepted", providerJobId: job.id } };
|
|
333
|
+
if (s.current.execution.sessionId && !freshSessionMatches(s, s.current.execution.sessionId)) throw executionError("EXECUTION_CONTINUATION", "recovered local run reused retired memory");
|
|
334
|
+
if (s.learningContinuity?.pending) s.learningContinuity = { ...s.learningContinuity, pending: false, nextRun: s.current.id, nextSession: s.current.execution.sessionId };
|
|
335
|
+
intent.acceptance = "accepted"; intent.job = s.current;
|
|
336
|
+
}
|
|
337
|
+
if (s.stopped && job.running && capabilities.controls?.cancel && s.current.execution.acceptance === "accepted") {
|
|
338
|
+
// Recover acceptance before initiating a requested stop or resuming
|
|
339
|
+
// the runner's persisted deadline and exact process identities.
|
|
340
|
+
save(s);
|
|
341
|
+
try { requestRunCancellation(runner, job.id); job = runner.current(); s.current.execution.recoveryAction = null; }
|
|
342
|
+
catch (error) { s.current.execution.recoveryAction = error.code || "local shutdown remains unverified"; }
|
|
343
|
+
if (!job || job.id !== s.current.id) { save(s); return current(); }
|
|
344
|
+
}
|
|
345
|
+
Object.assign(s.current.execution, { providerState: job.running ? s.stopped || job.killed ? "cancellation-requested" : "running" : s.current.execution.collected ? s.current.execution.providerState : "retrieving-results", observedAt: clock.now(), reconciledAt: clock.now() });
|
|
346
|
+
if (s.stopped && !job.running) confirmLocalStop(s, job);
|
|
347
|
+
const { running: _running, ...record } = job;
|
|
348
|
+
s.current = { ...record, ...(s.current.execution.quarantined ? { exit: s.current.exit, endedAt: s.current.endedAt } : {}), execution: s.current.execution };
|
|
349
|
+
save(s); runner.annotate?.(s.current.id, { execution: s.current.execution }); return current();
|
|
350
|
+
}
|
|
351
|
+
const ex = s.current.execution;
|
|
352
|
+
if (ex.collected && terminal(ex.providerState)) return current();
|
|
353
|
+
if (s.stopped) cancellationIntent(s);
|
|
354
|
+
if (!force && clock.now() < ex.nextAttemptAt) return current();
|
|
355
|
+
try {
|
|
356
|
+
const observation = ex.acceptance === "unknown"
|
|
357
|
+
? transport.lookup ? await call("lookup", { intent: s.operations[ex.operationId], taskId, account }) : null
|
|
358
|
+
: await call("inspect", { taskId, account, providerJobId: ex.providerJobId, sessionId: ex.sessionId, turnId: ex.turnId, cursor: ex.cursor ?? null });
|
|
359
|
+
if (!observation) { ex.providerState = "status-unknown"; ex.recoveryAction = "provider cannot reconcile acceptance; inspect the recorded key manually"; }
|
|
360
|
+
else {
|
|
361
|
+
assertExecutionCursor(observation.cursor);
|
|
362
|
+
if (!ownedRemote(observation) || (ex.providerJobId && observation.providerJobId !== ex.providerJobId) || (ex.turnId && observation.turnId !== ex.turnId) || !ownsProviderSession(ex.sessionId, observation.sessionId) || !freshSessionMatches(s, observation.sessionId)) throw executionError("EXECUTION_OWNERSHIP", "observation does not identify the recorded provider job/root turn/session");
|
|
363
|
+
if (!PROVIDER_STATES.includes(observation.state)) throw executionError("EXECUTION_PROTOCOL", "unsupported provider state");
|
|
364
|
+
// A stale authoritative read cannot regress newer progress or settle this turn.
|
|
365
|
+
if (!Number.isFinite(observation.observedAt) || observation.observedAt < s.current.startedAt || observation.observedAt < (ex.observedAt ?? 0)) throw executionError("EXECUTION_STALE", "stale provider observation; retrieve current authoritative state");
|
|
366
|
+
if (observation.observedAt > clock.now()) throw executionError("EXECUTION_PROTOCOL", "future provider observation; return current UTC epoch milliseconds before retrying");
|
|
367
|
+
if (ex.terminalState && observation.state !== ex.terminalState) throw executionError("EXECUTION_PROTOCOL", "observation contradicts the recorded terminal root; reconcile its authoritative state");
|
|
368
|
+
const completedAt = completionTime(observation.completedAt, s.current.startedAt, observation.observedAt, "EXECUTION_PROTOCOL");
|
|
369
|
+
if (completedAt !== null && (!terminal(observation.state) || (ex.completedAt != null && completedAt !== ex.completedAt))) throw executionError("EXECUTION_PROTOCOL", "completion time differs from the recorded terminal root");
|
|
370
|
+
if (observation.events?.length > bounds.events || Buffer.byteLength(JSON.stringify(observation.events || [])) > bounds.eventBytes) throw executionError("EXECUTION_PROTOCOL", "provider event page exceeds the documented limits; paginate it");
|
|
371
|
+
const journal = journalFor(dirOf(s.current.id), { taskId, runId: s.current.id, provider, secrets });
|
|
372
|
+
for (const ev of observation.events || []) journal.append({ ...ev, providerJobId: observation.providerJobId, sessionId: observation.sessionId ?? ex.sessionId, turnId: observation.turnId ?? ex.turnId });
|
|
373
|
+
if (observation.gap) {
|
|
374
|
+
const gapId = `gap-${observation.cursor}`;
|
|
375
|
+
journal.append({ eventId: gapId.length <= 512 ? gapId : `gap:${digest([observation.cursor])}`, type: "gap", data: observation.gap, timestamp: clock.now() });
|
|
376
|
+
}
|
|
377
|
+
Object.assign(ex, { providerJobId: observation.providerJobId, sessionId: observation.sessionId ?? ex.sessionId, turnId: observation.turnId, providerState: s.stopped && !terminal(observation.state) ? "cancellation-requested" : observation.state, acceptance: "accepted", observedAt: observation.observedAt, reconciledAt: clock.now(), cursor: observation.cursor ?? ex.cursor, ...(observation.environment ? { environment: observation.environment } : {}), ...(observation.effective ? { effective: observation.effective } : {}), ...(observation.usage !== undefined ? { usage: observation.usage } : {}), ...(observation.environmentState ? { environmentState: observation.environmentState } : {}), failures: 0, recoveryAction: null, costUsd: maximumCost(ex.costUsd, observation.costUsd) });
|
|
378
|
+
if (s.learningContinuity?.pending) s.learningContinuity = { ...s.learningContinuity, pending: false, nextRun: s.current.id, nextSession: ex.sessionId };
|
|
379
|
+
const intent = s.operations[ex.operationId]; intent.acceptance = "accepted"; intent.job = s.current;
|
|
380
|
+
charge(s, ex.costUsd);
|
|
381
|
+
if (terminal(ex.providerState)) {
|
|
382
|
+
ex.completedAt = completedAt ?? ex.completedAt ?? null;
|
|
383
|
+
ex.terminalObservedAt ??= observation.observedAt;
|
|
384
|
+
if (s.stopped) {
|
|
385
|
+
ex.quarantined = true; ex.collected = true; s.current.exit = ex.providerState === "cancelled" ? 137 : 1;
|
|
386
|
+
if (ex.providerState === "cancelled") confirmCancellation(s);
|
|
387
|
+
else ex.cancellation.status = "terminal";
|
|
388
|
+
}
|
|
389
|
+
else if (ex.providerState === "cancelled") { ex.collected = true; s.current.exit = 1; }
|
|
390
|
+
else { ex.terminalState = ex.providerState; ex.providerState = "retrieving-results"; }
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
if (s.stopped) await sendCancellation(s);
|
|
394
|
+
ex.nextAttemptAt = clock.now() + bounds.progressMs;
|
|
395
|
+
} catch (e) {
|
|
396
|
+
if (e.providerObservationError === true) journalFor(dirOf(s.current.id), { taskId, runId: s.current.id, provider, secrets }).append({ eventId: `inspection-error:${clock.now()}:${ex.failures}`, type: "error", timestamp: clock.now(), provenance: "coordinator", data: { code: e.code || "provider reconciliation failed" } });
|
|
397
|
+
ex.providerState = s.stopped ? "cancellation-requested" : "status-unknown"; ex.failures++; ex.recoveryAction = e.code || "provider reconciliation failed"; ex.nextAttemptAt = Math.max(ex.cancellation?.nextAttemptAt ?? 0, nextAttempt(clock.now(), ex.failures, e.retryAfterMs, bounds, jitter)); }
|
|
398
|
+
save(s); project(s); return current();
|
|
399
|
+
});
|
|
400
|
+
}
|
|
401
|
+
const applyCollectedResult = (s, result) => {
|
|
402
|
+
const job = s.current, ex = job.execution, cost = result.outcome.cost;
|
|
403
|
+
Object.assign(ex, { collected: true, providerState: result.outcome.ok ? "succeeded" : "failed",
|
|
404
|
+
sessionId: result.sessionId ?? ex.sessionId,
|
|
405
|
+
costUsd: maximumCost(ex.costUsd, cost), ...(transport ? { completedAt: result.completedAt ?? ex.completedAt ?? null } : {}) });
|
|
406
|
+
charge(s, ex.costUsd);
|
|
407
|
+
job.exit = result.outcome.ok ? 0 : 1;
|
|
408
|
+
if (transport) {
|
|
409
|
+
job.endedAt = ex.completedAt ?? ex.terminalObservedAt ?? result.collectedAt ?? clock.now();
|
|
410
|
+
job.completionInferred = ex.completedAt === null;
|
|
411
|
+
} else job.endedAt ??= result.collectedAt ?? clock.now();
|
|
412
|
+
save(s); if (transport) project(s);
|
|
413
|
+
};
|
|
414
|
+
async function collect() {
|
|
415
|
+
return locked(async s => {
|
|
416
|
+
const job = s.current;
|
|
417
|
+
if (!job) return null;
|
|
418
|
+
const path = join(dirOf(job.id), "result.json");
|
|
419
|
+
const saved = readJson(path);
|
|
420
|
+
const ex = job.execution;
|
|
421
|
+
if (s.stopped || ex.quarantined) throw executionError("EXECUTION_QUARANTINED", "late results are quarantined after a stop");
|
|
422
|
+
if (transport && !ownedRemote(ex)) throw executionError("EXECUTION_RESULT", "exact provider job/root-turn ownership is unresolved; reconcile the recorded key");
|
|
423
|
+
if (transport && !["succeeded", "failed"].includes(ex.terminalState ?? ex.providerState)) throw executionError("EXECUTION_RESULT", "root turn has not authoritatively completed");
|
|
424
|
+
if (collectionRejection(s)) throw executionError("EXECUTION_RESULT_BLOCKED", "result collection is blocked; inspect result-application.json and explicitly resume the same session for correction");
|
|
425
|
+
const validResult = (result, cached = false) => {
|
|
426
|
+
if (result?.outcome?.finished !== true || typeof result.outcome.ok !== "boolean" ||
|
|
427
|
+
(transport && (result.providerJobId !== ex.providerJobId || result.turnId !== ex.turnId || result.inputHead !== ex.inputHead ||
|
|
428
|
+
!ownsProviderSession(ex.sessionId, result.sessionId)))) return false;
|
|
429
|
+
if (!transport) localSession(s, job.id, result.sessionId);
|
|
430
|
+
if (transport) {
|
|
431
|
+
assertTerminalOutcome(ex, result.outcome);
|
|
432
|
+
const latestAt = Math.min(ex.terminalObservedAt ?? clock.now(), cached ? result.collectedAt ?? clock.now() : clock.now());
|
|
433
|
+
completionTime(ex.completedAt, job.startedAt, latestAt, "EXECUTION_RESULT");
|
|
434
|
+
const completedAt = completionTime(result.completedAt, job.startedAt, latestAt, "EXECUTION_RESULT");
|
|
435
|
+
if (completedAt !== null && ex.completedAt != null && completedAt !== ex.completedAt) throw executionError("EXECUTION_RESULT", "result completion time differs from the recorded terminal root");
|
|
436
|
+
}
|
|
437
|
+
return true;
|
|
438
|
+
};
|
|
439
|
+
if (saved) {
|
|
440
|
+
if (!validResult(saved, true) || (transport && (saved.taskId !== taskId || saved.runId !== job.id))) throw executionError("EXECUTION_RESULT", "saved result does not own the exact completed turn");
|
|
441
|
+
applyCollectedResult(s, saved); return saved;
|
|
442
|
+
}
|
|
443
|
+
if (!transport && runner.current()?.running) throw executionError("BUSY", "local writer is still running");
|
|
444
|
+
if (transport && ex.providerState !== "retrieving-results") throw executionError("EXECUTION_RESULT", "root turn has not authoritatively completed");
|
|
445
|
+
let result;
|
|
446
|
+
try {
|
|
447
|
+
result = transport ? await call("collect", { taskId, providerJobId: ex.providerJobId, sessionId: ex.sessionId, turnId: ex.turnId, inputHead: ex.inputHead }) : { outcome: adapterFor(provider).outcome(runner.lines(job.id)), sessionId: adapterFor(provider).sessionFrom(runner.lines(job.id)) };
|
|
448
|
+
} catch (error) {
|
|
449
|
+
if (transport && error.code === "EXECUTION_RESULT" && error.collectionValidation === true) {
|
|
450
|
+
writeJsonAtomic(join(dirOf(job.id), "result-application.json"), { version: 1, taskId, runId: job.id,
|
|
451
|
+
providerJobId: ex.providerJobId, sessionId: ex.sessionId, turnId: ex.turnId, inputHead: ex.inputHead,
|
|
452
|
+
phase: "blocked", reason: "result-collection-validation", blockedAt: new Date(clock.now()).toISOString() });
|
|
453
|
+
settleCollectionRejection(s); charge(s, ex.costUsd); save(s); project(s);
|
|
454
|
+
}
|
|
455
|
+
throw error;
|
|
456
|
+
}
|
|
457
|
+
if (result && Buffer.byteLength(JSON.stringify(result)) > 1_048_576) throw executionError("EXECUTION_RESULT", "result exceeds the bounded manifest; retrieve paginated artifacts before collecting");
|
|
458
|
+
if (!validResult(result)) throw executionError("EXECUTION_RESULT", "missing or mismatched results; reconcile the exact completed turn");
|
|
459
|
+
if (!transport) result.sessionId = localSession(s, job.id, result.sessionId);
|
|
460
|
+
const clean = redactResultMetadata({ ...result, version: EXECUTION_VERSION, taskId, runId: job.id, collectedAt: clock.now() }, secrets);
|
|
461
|
+
writeJsonAtomic(path, clean);
|
|
462
|
+
applyCollectedResult(s, clean);
|
|
463
|
+
return clean;
|
|
464
|
+
});
|
|
465
|
+
}
|
|
466
|
+
async function cancel(reason = "operator-cancelled") {
|
|
467
|
+
return locked(async s => {
|
|
468
|
+
const alreadyStopped = s.stopped;
|
|
469
|
+
// A timeout retry must not downgrade an operator or preparation stop.
|
|
470
|
+
if (!s.stopped || reason !== "timeout") s.stopped = reason;
|
|
471
|
+
// An upgrade may have an owned local writer but no boundary submission.
|
|
472
|
+
// Preserve its identity for stop recovery without inventing a paid intent.
|
|
473
|
+
if (!s.current && !transport) {
|
|
474
|
+
const job = runner.current();
|
|
475
|
+
// Completed pre-boundary runs need only the future-dispatch fence.
|
|
476
|
+
// Adopt live/unfinished writers for the existing shutdown recovery.
|
|
477
|
+
if (job?.running === false && job.exit != null && !job.killed) { save(s); return { confirmed: true }; }
|
|
478
|
+
if (job) {
|
|
479
|
+
const { running: _running, ...record } = job;
|
|
480
|
+
s.current = { ...record, execution: {
|
|
481
|
+
version: EXECUTION_VERSION, taskId, provider, account, runId: job.id,
|
|
482
|
+
operationId: null, legacy: true, transport: "cli", providerJobId: job.id,
|
|
483
|
+
sessionId: legacySessionId, turnId: null, inputHead: null,
|
|
484
|
+
environment: null, model: null, acceptance: "accepted",
|
|
485
|
+
observedAt: clock.now(), reconciledAt: null, nextAttemptAt: clock.now(),
|
|
486
|
+
failures: 0, costUsd: null, collected: false, allowance: null,
|
|
487
|
+
controls: capabilities.controls ?? {}, continuation: capabilities.continuation ?? null,
|
|
488
|
+
mandatoryPolicy: [], workflow: null, kind: null, preparation: null,
|
|
489
|
+
} };
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
if (!s.current) { save(s); return { confirmed: true }; }
|
|
493
|
+
const ex = s.current.execution;
|
|
494
|
+
if (collectedLocalOutcome(s)) { save(s); return { confirmed: true }; }
|
|
495
|
+
if (!transport && (!alreadyStopped || (ex.providerState === "retrieving-results" && s.current.exit != null && !s.current.killed))) {
|
|
496
|
+
const job = runner.current();
|
|
497
|
+
if (completedLocalJob(s, job)) {
|
|
498
|
+
const { running: _running, ...record } = job;
|
|
499
|
+
s.current = { ...record, execution: { ...ex, providerState: "retrieving-results" } };
|
|
500
|
+
save(s); // Fence dispatch and retain authoritative completion before collecting.
|
|
501
|
+
settleLocalResult(s, job); return { confirmed: true };
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
if (transport && !ex.collected && terminal(ex.terminalState)) {
|
|
505
|
+
Object.assign(ex, { providerState: ex.terminalState, collected: true, quarantined: true });
|
|
506
|
+
s.current.exit = 1;
|
|
507
|
+
}
|
|
508
|
+
if (transport && ex.collected && terminal(ex.providerState)) {
|
|
509
|
+
cancellationIntent(s); const answer = await sendCancellation(s); save(s); project(s); return answer;
|
|
510
|
+
}
|
|
511
|
+
ex.providerState = "cancellation-requested"; ex.recoveryAction = "confirm provider cancellation; no more dispatch";
|
|
512
|
+
if (transport) cancellationIntent(s);
|
|
513
|
+
save(s); if (transport) project(s);
|
|
514
|
+
if (transport) return sendCancellation(s);
|
|
515
|
+
if (!capabilities.controls?.cancel || ex.acceptance === "unknown") return { confirmed: false, unsupported: !capabilities.controls?.cancel };
|
|
516
|
+
const answer = requestRunCancellation(runner, s.current.id);
|
|
517
|
+
if (answer?.confirmed) { confirmLocalStop(s, runner.current()); save(s); }
|
|
518
|
+
return { confirmed: answer?.confirmed === true };
|
|
519
|
+
});
|
|
520
|
+
}
|
|
521
|
+
return {
|
|
522
|
+
root, provider, account, capabilities, state: read, current, reconcile, collect, cancel,
|
|
523
|
+
recover: async ({ operationId, authorized = false } = {}) => {
|
|
524
|
+
const pending=await locked(async s => {
|
|
525
|
+
const job=s.current,ex=job?.execution;
|
|
526
|
+
if (authorized!==true || typeof operationId!=="string" || !operationId || operationId.length>256) throw executionError("EXECUTION_CONTROL", "recovery requires explicit authorization and a bounded request identity");
|
|
527
|
+
if (s.stopped || ex?.quarantined) throw executionError("EXECUTION_STOPPED", "recovery cannot clear a stop or release quarantined results");
|
|
528
|
+
if (!transport || !job || !transport.resetRecovery || ex.providerState!=="status-unknown") return null;
|
|
529
|
+
const intent=s.operations[ex.operationId];
|
|
530
|
+
if (!intent || intent.runId!==job.id) throw executionError("EXECUTION_OWNERSHIP", "recovery cannot replace the saved dispatch intent");
|
|
531
|
+
const key=digest([taskId,provider,account,job.id,ex.operationId,operationId]);
|
|
532
|
+
if (ex.recoveryRequests?.[key]==="acknowledged") return null;
|
|
533
|
+
if (!Object.hasOwn(ex.recoveryRequests || {},key) && Object.keys(ex.recoveryRequests || {}).length>=32) throw executionError("EXECUTION_CONTROL", "recovery receipt capacity reached; preserve the saved operation");
|
|
534
|
+
ex.recoveryRequests={...ex.recoveryRequests,[key]:"intent"};save(s);
|
|
535
|
+
await call("resetRecovery",{intent,providerJobId:ex.providerJobId,sessionId:ex.sessionId,turnId:ex.turnId,operationId:key,authorized:true});
|
|
536
|
+
ex.recoveryRequests[key]="reset";ex.failures=0;ex.nextAttemptAt=clock.now();
|
|
537
|
+
ex.recoveryAction="explicit recovery authorized; reconcile the saved operation without resending input";
|
|
538
|
+
save(s);project(s);return {key,runId:job.id};
|
|
539
|
+
});
|
|
540
|
+
if(pending) {
|
|
541
|
+
// Observe the authoritative completion before the coordinator's parallel
|
|
542
|
+
// timeout lane. A result completed within allowance remains collectable
|
|
543
|
+
// even when recovery happens later; no input or continuation is sent.
|
|
544
|
+
await reconcile({force:true});
|
|
545
|
+
await locked(s=>{if(s.current?.id!==pending.runId)throw executionError("EXECUTION_OWNERSHIP","recovered run changed before acknowledgement");s.current.execution.recoveryRequests[pending.key]="acknowledged";save(s);});
|
|
546
|
+
}
|
|
547
|
+
return current();
|
|
548
|
+
},
|
|
549
|
+
result: id => readJson(join(dirOf(id), "result.json")),
|
|
550
|
+
sanitize: (value, id, extraSecrets = []) => redact(value, [...secrets, ...(readJson(join(dirOf(id), "sensitive-redactions.json"))?.secrets || []), ...extraSecrets]),
|
|
551
|
+
prepare: async (input, { build } = {}) => {
|
|
552
|
+
authorize(input);
|
|
553
|
+
if (input.kind !== "intake") return transport ? call("prepare", input) : input;
|
|
554
|
+
authorizePreparation(read());
|
|
555
|
+
const receipt = preparationReceipt(provider, capabilities, account);
|
|
556
|
+
let prepared;
|
|
557
|
+
if (!transport) prepared = await prepareLocalExecution(provider, capabilities, account, input, build, bounds);
|
|
558
|
+
else {
|
|
559
|
+
if (typeof input.environment !== "string" || input.environment !== capabilities.preparation.environment) throw executionError("TASK_PREPARATION_BOUNDARY", "transport preparation requires its declared limited environment");
|
|
560
|
+
const { kind, environment, prompt, allowance, mandatoryPolicy, model } = input;
|
|
561
|
+
prepared = await call("prepare", { kind, environment, prompt, allowance, mandatoryPolicy, model, taskId, provider, account, preparation: receipt });
|
|
562
|
+
if (!prepared || prepared.boundary !== receipt.boundary || prepared.environment !== input.environment || Buffer.byteLength(JSON.stringify(prepared)) > bounds.eventBytes) throw executionError("TASK_PREPARATION_BOUNDARY", "transport preparation did not confirm the selected bounded authority/environment");
|
|
563
|
+
}
|
|
564
|
+
const fingerprint = preparationFingerprint(transport ? { environment: input.environment, prompt: input.prompt, prepared } : prepared);
|
|
565
|
+
const issued = { ...receipt, taskId, fingerprint };
|
|
566
|
+
await locked(s => { authorizePreparation(s); s.preparations = { ...s.preparations, [fingerprint]: issued }; save(s); });
|
|
567
|
+
return { ...prepared, receipt: issued };
|
|
568
|
+
},
|
|
569
|
+
respond: input => locked(async s => {
|
|
570
|
+
if (s.stopped || s.current?.execution.quarantined) throw executionError("EXECUTION_STOPPED", "answers are fenced after a stop");
|
|
571
|
+
const ex = s.current?.execution;
|
|
572
|
+
if (!transport?.respond || !ex || ex.acceptance !== "accepted" || input.sessionId !== ex.sessionId || input.turnId !== ex.turnId || input.authorized !== true) throw executionError("EXECUTION_ACTION", "answer requires authorization and the current owned session/root turn");
|
|
573
|
+
if (typeof input.operationId !== "string" || !input.operationId || input.operationId.length > 256) throw executionError("EXECUTION_INPUT", "answer requires a bounded operation identity");
|
|
574
|
+
const signature = digest(input), key = digest([taskId, provider, account, ex.turnId, input.operationId]);
|
|
575
|
+
const prior = s.answers?.[key];
|
|
576
|
+
if (prior && prior.signature !== signature) throw executionError("EXECUTION_CONFLICT", "answer identity owns different input");
|
|
577
|
+
if (prior?.accepted) return { accepted: true };
|
|
578
|
+
if (prior && typeof transport.recoverResponse !== "function") throw executionError("EXECUTION_UNKNOWN", "answer acceptance unknown; this transport cannot safely recover it; inspect the owned turn without changing the answer identity");
|
|
579
|
+
// Explicitly repeating the identical authorized answer delegates recovery
|
|
580
|
+
// to the provider's authoritative pending-call/idempotency boundary.
|
|
581
|
+
if (!prior) { s.answers = { ...s.answers, [key]: { signature, accepted: false } }; save(s); }
|
|
582
|
+
const result = await call(prior ? "recoverResponse" : "respond", { ...input, taskId, account, providerJobId: ex.providerJobId, idempotencyKey: key });
|
|
583
|
+
s.answers[key].accepted = result?.accepted === true; save(s); return result;
|
|
584
|
+
}),
|
|
585
|
+
freshLearningSession: ({ runId, sessionId }) => locked(s => {
|
|
586
|
+
const job = s.current, ex = job?.execution;
|
|
587
|
+
if (!job || job.id !== runId || ex.sessionId !== sessionId || ex.acceptance !== "accepted" || executionLive(job)) throw executionError("EXECUTION_CONTINUATION", "fresh learning continuity requires the exact settled provider run and session");
|
|
588
|
+
if (s.stopped && s.stopped !== "learned advice retired") throw executionError("EXECUTION_STOPPED", "fresh learning continuity cannot clear an unrelated stop");
|
|
589
|
+
if (s.learningContinuity?.pending) return s.learningContinuity;
|
|
590
|
+
// Preserve old results/quarantine, operations and cumulative allowance.
|
|
591
|
+
// The persisted fence permits only submission of a new provider session.
|
|
592
|
+
s.learningContinuity = { runId, sessionId, pending: true };
|
|
593
|
+
s.stopped = null; save(s);
|
|
594
|
+
return s.learningContinuity;
|
|
595
|
+
}),
|
|
596
|
+
submit: input => dispatch("submit", input), continue: input => dispatch("continue", input),
|
|
597
|
+
inspect: () => current(), // Read-only: never reaches a transport or creates a polling loop.
|
|
598
|
+
start: (name, cmd, args, options) => {
|
|
599
|
+
const input = options.execution || {};
|
|
600
|
+
return dispatch(input.sessionId ? "continue" : "submit", { ...input, name, operationId: input.operationId || digest([name, cmd, args, options.cwd]), spec: { cmd, args, options } });
|
|
601
|
+
},
|
|
602
|
+
lines: id => runner ? runner.lines(id) : readJson(join(dirOf(id), "result.json"))?.lines || [],
|
|
603
|
+
read: id => runner ? runner.read(id) : readJson(join(dirOf(id), "job.json")),
|
|
604
|
+
outcome: id => {
|
|
605
|
+
const s = read(), dir = dirOf(id);
|
|
606
|
+
const reported = readJson(join(dir, "result.json"))?.outcome ?? (runner ? adapterFor(provider).outcome(runner.lines(id)) :
|
|
607
|
+
{ finished: s.current?.id === id && terminal(s.current.execution.providerState), ok: false, cost: null, denials: [], error: s.current?.id === id && s.current.execution.resultRejected ? "result collection rejected; inspect result-application.json" : "results unavailable" });
|
|
608
|
+
return redact({ ...reported, cost: maximumCost(reported.cost, runCost(s, id)) },
|
|
609
|
+
[...secrets, ...(readJson(join(dir, "sensitive-redactions.json"))?.secrets || [])]);
|
|
610
|
+
},
|
|
611
|
+
sessionFrom: id => {
|
|
612
|
+
const s = read(), saved = readJson(join(dirOf(id), "result.json"));
|
|
613
|
+
return runner ? localSession(s, id, saved?.sessionId ?? adapterFor(provider).sessionFrom(runner.lines(id)))
|
|
614
|
+
: saved?.sessionId ?? s.current?.execution.sessionId;
|
|
615
|
+
},
|
|
616
|
+
settle: (id, inferExit) => {
|
|
617
|
+
if (!runner) return current();
|
|
618
|
+
ensureDir(root);
|
|
619
|
+
const lock = acquireProcessLock(join(root, "execution.lock"), "execution");
|
|
620
|
+
try {
|
|
621
|
+
const s = read();
|
|
622
|
+
if (s.current?.id === id && s.current.execution.quarantined) return current();
|
|
623
|
+
const job = runner.settle(id, inferExit);
|
|
624
|
+
if (s.current?.id === id && job?.id === id && !job.running && !s.stopped && !s.current.execution.quarantined) settleLocalResult(s, job);
|
|
625
|
+
return job;
|
|
626
|
+
} finally { lock.release(); }
|
|
627
|
+
},
|
|
628
|
+
kill: id => { if (transport) throw executionError("EXECUTION_CONTROL", "remote cancellation must be requested and confirmed by the coordinator"); return runner.kill(id); },
|
|
629
|
+
waitForKill: id => runner?.waitForKill(id), dirOf,
|
|
630
|
+
};
|
|
631
|
+
}
|