@deksden-com/dd-flow-cli 0.9.0-beta.156 → 0.9.0-beta.159
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/dist/build-info.json +8 -8
- package/dist/cli/error-output.js +11 -1
- package/dist/cli/input-preparation.js +32 -4
- package/dist/cli/run-cli.js +24 -7
- package/dist/harness-runtime/lib/daemon-operations.mjs +46 -11
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +19 -6
- package/dist/harness-runtime/lib/dd-agy.mjs +19 -0
- package/dist/harness-runtime/lib/dd-codex-daemon.mjs +1 -1
- package/dist/harness-runtime/lib/dd-droid-daemon.mjs +1 -1
- package/dist/harness-runtime/lib/dd-grok-daemon.mjs +2 -2
- package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +1 -1
- package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +1 -1
- package/dist/harness-runtime/lib/delegation-instructions.d.mts +1 -0
- package/dist/harness-runtime/lib/delegation-instructions.mjs +3 -0
- package/dist/schemas/code-work-batch.schema.json +1 -1
- package/dist/schemas/vnext-protocol-plan.schema.json +1 -1
- package/dist/services/codex-capacity.js +367 -0
- package/dist/services/continuation-outcome.js +11 -6
- package/dist/services/controller-child-repair.js +46 -24
- package/dist/services/controller-fanout.js +62 -21
- package/dist/services/controller-result-authority.js +13 -8
- package/dist/services/controller-work-failure.js +198 -0
- package/dist/services/lifecycle-invocations.js +280 -96
- package/dist/services/managed-daemon-binding.js +40 -0
- package/dist/services/run-control.js +18 -36
- package/dist/services/run-controller-state.js +8 -0
- package/dist/services/run-controller.js +136 -27
- package/dist/services/vnext-code.js +3 -1
- package/dist/services/vnext-plan-review.js +2 -2
- package/dist/services/vnext-plan.js +3 -3
- package/dist/services/work-registry.js +59 -12
- package/dist/services/work-retry-context.js +55 -0
- package/dist/verification-components.json +3 -0
- package/package.json +1 -1
|
@@ -12,8 +12,42 @@ import { requireProjectByRoot } from "./projects.js";
|
|
|
12
12
|
import { stageAttempt, stageWorkIds } from "./stage-work-graph.js";
|
|
13
13
|
import { runtimeCommand } from "./runtime-command.js";
|
|
14
14
|
import { promptDeliveryInstruction } from "./prompt-delivery.js";
|
|
15
|
-
import { currentStageFinishAuthority } from "./lifecycle-invocations.js";
|
|
15
|
+
import { committedSuccessor, currentStageFinishAuthority } from "./lifecycle-invocations.js";
|
|
16
16
|
import { acceptedChildWork } from "./controller-result-authority.js";
|
|
17
|
+
import { explicitWorkFailure } from "./controller-work-failure.js";
|
|
18
|
+
import { commandOption, parseLifecycleCommand } from "./lifecycle-command.js";
|
|
19
|
+
/** A new transport ID is not progress. Read only the current assignment's
|
|
20
|
+
* committed transition and its check inputs; never inspect mutable files here. */
|
|
21
|
+
export function controllerSemanticCheckpoint(context, input) {
|
|
22
|
+
const scopeJson = context.env.DD_FLOW_INVOCATION_SCOPE;
|
|
23
|
+
const scope = scopeJson ? JSON.parse(scopeJson) : null;
|
|
24
|
+
const project = scope && context.db.get("SELECT id FROM projects WHERE root=?", [scope.projectRoot]);
|
|
25
|
+
const attempt = project && scope ? stageAttempt(context, project.id, scope.runId, input.stage) : null;
|
|
26
|
+
const receipt = (id) => typeof id === "string" && project && scope
|
|
27
|
+
? context.db.get("SELECT declaration_id,input_hash,status FROM check_receipts WHERE id=? AND project_id=? AND run_id=?", [id, project.id, scope.runId]) ?? null : null;
|
|
28
|
+
const stableSet = (values) => [...new Set(values.map(value => JSON.stringify(value)))].sort().map(value => JSON.parse(value));
|
|
29
|
+
const parsed = input.command ? parseLifecycleCommand(input.command) : null;
|
|
30
|
+
const operation = parsed && parsed.kind !== "none" ? {
|
|
31
|
+
operation: parsed.invocation.operation, positional: parsed.invocation.args.positional,
|
|
32
|
+
options: [...parsed.invocation.args.options].filter(([key]) => !["invocation-id", "hook-event-id", "json", "progress-jsonl", "response-file", "reason"].includes(key))
|
|
33
|
+
.sort(([a], [b]) => a.localeCompare(b)).map(([key, values]) => [key, key === "retry-check" ? values.map(receipt) : values])
|
|
34
|
+
} : null;
|
|
35
|
+
const transitions = scopeJson && input.invocationId ? stableSet(context.db.all("SELECT * FROM lifecycle_invocations WHERE scope_json=? AND status='settled' AND json_extract(outcome_json,'$.successor_invocation_id')=?", [scopeJson, input.invocationId]).flatMap(row => {
|
|
36
|
+
const prior = parseLifecycleCommand(row.command);
|
|
37
|
+
if (prior.kind === "none" || commandOption(prior.invocation, "stage") !== input.stage)
|
|
38
|
+
return [];
|
|
39
|
+
const outcome = JSON.parse(row.outcome_json ?? "null");
|
|
40
|
+
if (committedSuccessor(context, row, outcome)?.id !== input.invocationId)
|
|
41
|
+
return [];
|
|
42
|
+
const result = outcome.result, error = outcome.error;
|
|
43
|
+
const references = result?.receipts ?? error?.details?.failures ?? [];
|
|
44
|
+
return [{ outcome: result?.outcome ?? null, verification: result?.verification?.verdict ?? null,
|
|
45
|
+
error: error ? { code: error.code, rejection_signature: error.details?.rejection_signature ?? null, input_key: error.details?.input_key ?? null } : null,
|
|
46
|
+
receipts: stableSet((Array.isArray(references) ? references : []).map(value => receipt(value?.id))) }];
|
|
47
|
+
})) : [];
|
|
48
|
+
const memoryChecks = project && scope && input.reviewInputSha256 ? stableSet(context.db.all("SELECT payload_json FROM audit_events WHERE project_id=? AND event_type='memory.correction_checks' AND json_extract(payload_json,'$.run_id')=? AND json_extract(payload_json,'$.wave')=? AND json_extract(payload_json,'$.review_input_sha256')=?", [project.id, scope.runId, input.wave ?? 0, input.reviewInputSha256]).map(row => { const value = JSON.parse(row.payload_json); return { corrected_input: value.corrected_input_sha256, completed_correction: value.completed_correction_sha256, receipts: stableSet((value.checks ?? []).map((check) => receipt(check.id))) }; })) : [];
|
|
49
|
+
return { phase: input.phase, ...(input.wave === undefined ? {} : { wave: input.wave }), attempt, generation: scope?.generation ?? null, operation, transitions, ...(input.reviewInputSha256 ? { memory_checks: memoryChecks } : {}) };
|
|
50
|
+
}
|
|
17
51
|
function semanticCompletionPrompt(stage, completed, command) {
|
|
18
52
|
const task = stage === "plan-review"
|
|
19
53
|
? "Classify the reviewer findings, apply every accepted correction in the current PLAN-REVIEW Work, and write the authoritative review decision."
|
|
@@ -93,27 +127,34 @@ export async function nextControllerFanout(context, input) {
|
|
|
93
127
|
if (fanout.state === "dispatch_required" || fanout.state === "capacity_required")
|
|
94
128
|
return { kind: "wait", reason: "qualified_capacity_required" };
|
|
95
129
|
}
|
|
96
|
-
const prompt = (action, text) => ({ kind: "prompt", action, prompt: action === "dispatch_work" ? text : `${text}\n\n${lifecycleCommandCompletionInstruction}`, fingerprint: JSON.stringify({ stage: input.stage, parent: fanout.parent_work_id, state: fanout.state, graph: fanout.graph_revision }) });
|
|
130
|
+
const prompt = (action, text, checkpoint) => ({ kind: "prompt", action, prompt: action === "dispatch_work" ? text : `${text}\n\n${lifecycleCommandCompletionInstruction}`, ...(checkpoint ? { checkpoint } : {}), fingerprint: JSON.stringify({ stage: input.stage, action, parent: fanout.parent_work_id, state: fanout.state, graph: fanout.graph_revision, ...(checkpoint ? { semantic: checkpoint } : {}) }) });
|
|
97
131
|
if (fanout.state === "coordinator_required")
|
|
98
132
|
return prompt("materialize_graph", `Continue the current ${input.stage} Stage using its authoritative packet. Materialize only its required Work graph, declaring real dependencies at registration. If the packet's CLI command registers the graph, use its returned Work graph unchanged: do not add or alter dependencies after registration. Accepted predecessor Stages already fence their completed producers; do not add cross-stage edges merely to restate input provenance. Do not launch children, finish the Stage, or start a successor. Return after registering the graph; the runtime will dispatch its ready Work.`);
|
|
99
133
|
if (fanout.works.failed || fanout.works.cancelled) {
|
|
100
134
|
const projectId = requireProjectByRoot(context, input.projectRoot).id;
|
|
101
135
|
const members = stageWorkIds(context, projectId, input.runId, input.stage);
|
|
102
|
-
const
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
136
|
+
const failedIds = context.db.all("SELECT work_id FROM works WHERE project_id=? AND run_id=? AND parent_work_id=? AND status='failed' ORDER BY work_id", [projectId, input.runId, fanout.parent_work_id]).filter(work => members.has(work.work_id));
|
|
137
|
+
const explicit = !input.dispatchFence && !fanout.works.cancelled ? failedIds.map(work => explicitWorkFailure(context, projectId, input.runId, work.work_id)) : [];
|
|
138
|
+
const failure = explicit.length === failedIds.length && explicit.length ? explicit[0] : null;
|
|
139
|
+
if (failure && !(input.deferredFailure === failure.work_id && fanout.works.ready.length))
|
|
140
|
+
return { kind: "failure", failure, ready: fanout.works.ready.length > 0 };
|
|
141
|
+
if (!failure) {
|
|
142
|
+
const failures = context.db.all("SELECT work_id,status,result FROM works WHERE project_id = ? AND run_id = ? AND parent_work_id = ? AND status IN ('failed','cancelled')", [projectId, input.runId, fanout.parent_work_id]).filter(work => members.has(work.work_id)).map(work => {
|
|
143
|
+
let receipt = {};
|
|
144
|
+
try {
|
|
145
|
+
const value = work.result ? JSON.parse(work.result) : null;
|
|
146
|
+
if (value && typeof value === "object" && !Array.isArray(value))
|
|
147
|
+
receipt = value;
|
|
148
|
+
}
|
|
149
|
+
catch { /* Missing legacy metadata is unknown. */ }
|
|
150
|
+
const providerError = safeNativeFailure(receipt.provider_error);
|
|
151
|
+
const nativeBinding = verifiedNativeWorkFailure(context, { projectRoot: input.projectRoot, runId: input.runId, stage: input.stage, parentSessionId: input.sessionId, parentWorkId: fanout.parent_work_id, workId: work.work_id, receipt });
|
|
152
|
+
return { work_id: work.work_id, status: work.status, ...(providerError ? { provider_error: providerError } : {}),
|
|
153
|
+
...(nativeBinding ? { native_binding: nativeBinding } : {}),
|
|
154
|
+
...Object.fromEntries(["terminal_turn_id", "terminal_observed_at"].filter(key => typeof receipt[key] === "string" && receipt[key].length <= 256).map(key => [key, receipt[key]])) };
|
|
155
|
+
});
|
|
156
|
+
throw new AppError("fanout_child_failed", "A child Work failed or was cancelled; no retry authority has been issued", 1, { stage: input.stage, run_id: input.runId, parent_work_id: fanout.parent_work_id, failed: fanout.works.failed, cancelled: fanout.works.cancelled, failures, effect: "unknown" });
|
|
157
|
+
}
|
|
117
158
|
}
|
|
118
159
|
if (fanout.works.ready.length) {
|
|
119
160
|
const choices = fanout.works.ready.map(work => {
|
|
@@ -190,7 +231,7 @@ export async function nextControllerFanout(context, input) {
|
|
|
190
231
|
const run = context.db.get("SELECT flow_kind,run_root,run_dir FROM runs WHERE project_id=? AND id=?", [projectId, input.runId]);
|
|
191
232
|
if (run?.flow_kind === "mb-init" || run?.flow_kind === "mb-upgrade") {
|
|
192
233
|
const attempt = stageAttempt(context, projectId, input.runId, "review");
|
|
193
|
-
const repair = context.db.get(`SELECT CASE event_type WHEN 'memory.second_repair_registration' THEN 2 ELSE 1 END AS wave
|
|
234
|
+
const repair = context.db.get(`SELECT CASE event_type WHEN 'memory.second_repair_registration' THEN 2 ELSE 1 END AS wave,json_extract(payload_json,'$.scope.input_sha256') AS input_sha256
|
|
194
235
|
FROM audit_events WHERE project_id=? AND event_type IN ('memory.repair_registration','memory.second_repair_registration')
|
|
195
236
|
AND json_extract(payload_json,'$.run_id')=? AND json_extract(payload_json,'$.scope.attempt')=?
|
|
196
237
|
AND json_extract(payload_json,'$.scope.parent_work_id')=? ORDER BY wave DESC LIMIT 1`, [projectId, input.runId, attempt, fanout.parent_work_id]);
|
|
@@ -198,7 +239,7 @@ export async function nextControllerFanout(context, input) {
|
|
|
198
239
|
AND json_extract(payload_json,'$.run_id')=? AND json_extract(payload_json,'$.scope.attempt')=?
|
|
199
240
|
AND json_extract(payload_json,'$.scope.parent_work_id')=? AND json_extract(payload_json,'$.wave')=?`, [projectId, input.runId, attempt, fanout.parent_work_id, repair.wave])) {
|
|
200
241
|
const args = ["--run", input.runId, "--project-root", input.projectRoot, "--wave", String(repair.wave), "--json"];
|
|
201
|
-
return prompt("semantic_completion", `Correction wave ${repair.wave} Work has settled. Before Stage finish, run ${runtimeCommand(context, ["memory", "corrections", "checks", ...args])}, then ${runtimeCommand(context, ["memory", "review", "selective-register", ...args])}. Read both complete responses, then end this Turn so the controller can dispatch the independent selective reviewers. Do not finish the Stage or treat repair Work completion as independent acceptance
|
|
242
|
+
return prompt("semantic_completion", `Correction wave ${repair.wave} Work has settled. Before Stage finish, run ${runtimeCommand(context, ["memory", "corrections", "checks", ...args])}, then ${runtimeCommand(context, ["memory", "review", "selective-register", ...args])}. Read both complete responses, then end this Turn so the controller can dispatch the independent selective reviewers. Do not finish the Stage or treat repair Work completion as independent acceptance.`, controllerSemanticCheckpoint(context, { stage: input.stage, phase: "memory_correction_checks_register", wave: repair.wave, reviewInputSha256: repair.input_sha256 }));
|
|
202
243
|
}
|
|
203
244
|
const wave = repair?.wave ?? 0;
|
|
204
245
|
const runHome = run.run_root ?? run.run_dir;
|
|
@@ -214,7 +255,7 @@ export async function nextControllerFanout(context, input) {
|
|
|
214
255
|
? `If the outcome is needs_repair, use the returned canonical findings and correction_plan_binding to author substantive repair tasks with exact owned write_paths. Register that plan with the packet's issued memory corrections register command for wave ${wave + 1}, then end this Turn so the controller can dispatch it. Do not invoke Stage finish before corrections, checks and independent selective review complete.`
|
|
215
256
|
: "If the outcome is needs_repair or blocked, the two correction waves are exhausted. Use the issued pause command for the concrete unresolved blocker; do not mark it accepted.",
|
|
216
257
|
"For blocked collection, use the issued pause command. Do not calculate finding hashes, merge or drop findings, edit project files in the coordinator, create speculative Work or start a successor Stage."
|
|
217
|
-
].join("\n\n"));
|
|
258
|
+
].join("\n\n"), controllerSemanticCheckpoint(context, { stage: input.stage, phase: "memory_collection", wave, command: collect }));
|
|
218
259
|
}
|
|
219
260
|
}
|
|
220
261
|
const finish = currentStageFinishAuthority(context, input.stage);
|
|
@@ -222,7 +263,7 @@ export async function nextControllerFanout(context, input) {
|
|
|
222
263
|
throw new AppError("controller_finish_authority_missing", "Settled child Work has no current Stage finish authority", 1, { stage: input.stage, reconciliation_required: true });
|
|
223
264
|
if (finish.status !== "issued" || !finish.command)
|
|
224
265
|
throw new AppError("controller_finish_authority_pending", "Observe the original Stage finish before another productive dispatch", 1, { stage: input.stage, invocation_id: finish.id, status: finish.status, reconciliation_required: true });
|
|
225
|
-
return prompt("semantic_completion", semanticCompletionPrompt(input.stage, fanout.works.completed, finish.command));
|
|
266
|
+
return { ...prompt("semantic_completion", semanticCompletionPrompt(input.stage, fanout.works.completed, finish.command), controllerSemanticCheckpoint(context, { stage: input.stage, phase: "stage_finish", command: finish.command, invocationId: finish.id })), authority: { id: finish.id, status: finish.status } };
|
|
226
267
|
}
|
|
227
268
|
/** Shared pinned runtime normalization; preserve structured errors at the TS boundary. */
|
|
228
269
|
export function controllerNativeChildren(receipt, rootSessionId) {
|
|
@@ -5,6 +5,7 @@ import { isObservationLoss } from "../harness-runtime/lib/operation-errors.mjs";
|
|
|
5
5
|
import { terminalProviderRecovery } from "../harness-runtime/lib/provider-overload-policy.mjs";
|
|
6
6
|
import { getVnextFanoutStatus } from "./vnext-fanout.js";
|
|
7
7
|
import { normalizeNativeChildren, nativeTreeReadiness } from "../harness-runtime/lib/native-children.mjs";
|
|
8
|
+
import { failureDecisionReady } from "./controller-work-failure.js";
|
|
8
9
|
/** Disappearance from a truncated child list is not retirement evidence. */
|
|
9
10
|
export function acceptedChildrenRetired(expected, receipt, sessionId, operationId) {
|
|
10
11
|
const children = normalizeNativeChildren(receipt, sessionId);
|
|
@@ -37,16 +38,18 @@ export function acceptedTechnicalWorkControl(context, row, controlId) {
|
|
|
37
38
|
const guard = context.db.get("SELECT recovery_id,generation,status,capture_path FROM run_recovery_guards WHERE project_id=? AND run_id=? ORDER BY generation DESC LIMIT 1", [row.project_id, row.run_id]);
|
|
38
39
|
if (!guard || guard.recovery_id !== control.recovery_id || guard.status !== "sealed" || !guard.capture_path || guard.generation <= row.generation)
|
|
39
40
|
return false;
|
|
40
|
-
const events = context.db.all("SELECT data_json FROM run_controller_events WHERE controller_id=? AND event_type='technical_dependency_paused' AND json_extract(data_json,'$.control_id')=?", [row.controller_id, controlId]);
|
|
41
|
-
if (events.length !== 1)
|
|
42
|
-
return false;
|
|
43
|
-
const event = JSON.parse(events[0].data_json);
|
|
44
|
-
if (event.schema_id !== "dd-flow/controller-dependency@1" || event.request_id !== control.request_id || event.generation !== row.generation || typeof event.source_operation_id !== "string")
|
|
45
|
-
return false;
|
|
41
|
+
const events = context.db.all("SELECT data_json FROM run_controller_events WHERE controller_id=? AND event_type='technical_dependency_paused' AND json_extract(data_json,'$.control_id')=?", [row.controller_id, controlId]).map(item => JSON.parse(item.data_json));
|
|
46
42
|
const state = JSON.parse(row.state_json), session = state.current_session === null ? null : state.sessions?.[state.current_session];
|
|
47
|
-
if (!session?.id || !session.owner?.owner_id || event.
|
|
43
|
+
if (!session?.id || !session.owner?.owner_id || !events.length || events.some(event => event.schema_id !== "dd-flow/controller-dependency@1"
|
|
44
|
+
|| event.request_id !== control.request_id || event.generation !== row.generation || event.state_dir !== session.state_dir
|
|
45
|
+
|| event.source_operation_id !== undefined && (typeof event.source_operation_id !== "string" || !event.source_operation_id)))
|
|
46
|
+
return false;
|
|
47
|
+
// Concurrent observers may add unbound notes to the same immutable control.
|
|
48
|
+
// They do not revoke one exact accepted source; conflicting sources still do.
|
|
49
|
+
const sources = [...new Set(events.flatMap(event => event.source_operation_id ? [event.source_operation_id] : []))];
|
|
50
|
+
if (sources.length !== 1)
|
|
48
51
|
return false;
|
|
49
|
-
const op = context.db.get("SELECT generation,status,input_json FROM run_controller_operations WHERE controller_id=? AND operation_id=?", [row.controller_id,
|
|
52
|
+
const op = context.db.get("SELECT generation,status,input_json FROM run_controller_operations WHERE controller_id=? AND operation_id=?", [row.controller_id, sources[0]]);
|
|
50
53
|
if (!op || op.generation !== row.generation || !["outcome_unknown", "completed"].includes(op.status))
|
|
51
54
|
return false;
|
|
52
55
|
const input = JSON.parse(op.input_json), expected = input.obligation;
|
|
@@ -84,6 +87,8 @@ export function controllerObligationAccepted(context, row, expected) {
|
|
|
84
87
|
const stage = [...stages].reverse().find(item => item.stage === expected.stage && (item.attempt ?? "try-001") === expected.attempt);
|
|
85
88
|
if (!stage || stageCycle(context, row.project_id, row.run_id, expected.stage) !== expected.cycle)
|
|
86
89
|
return false;
|
|
90
|
+
if (expected.action === "failure_resolution")
|
|
91
|
+
return Boolean(expected.failure_resolution && failureDecisionReady(expected.failure_resolution));
|
|
87
92
|
if (["done", "skipped"].includes(stage.status))
|
|
88
93
|
return true;
|
|
89
94
|
if (expected.action === "resume")
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { AppError } from "../shared/errors.js";
|
|
4
|
+
import { publishImmutableInput } from "../storage/immutable-input.js";
|
|
5
|
+
import { prepareAgentOutputFile } from "../storage/agent-output.js";
|
|
6
|
+
import { parseLifecycleCommand } from "./lifecycle-command.js";
|
|
7
|
+
import { stageAttempt, stageCycle, stageWorkIds } from "./stage-work-graph.js";
|
|
8
|
+
import { assertRunMutationAllowed } from "./run-recovery.js";
|
|
9
|
+
import { appendEvent, digest } from "./run-controller-state.js";
|
|
10
|
+
import { retryWork } from "./work-registry.js";
|
|
11
|
+
import { blockStageForRuntime } from "./stage-blocker.js";
|
|
12
|
+
import { pauseStageForUser } from "./stage-pause.js";
|
|
13
|
+
/** SQL settlement provenance, never the worker's prose, distinguishes a semantic refusal. */
|
|
14
|
+
export function explicitWorkFailure(context, projectId, runId, workId) {
|
|
15
|
+
const work = context.db.get("SELECT w.status,w.result,w.parent_work_id,w.payload_json,r.run_root FROM works w JOIN runs r ON r.project_id=w.project_id AND r.id=w.run_id WHERE w.project_id=? AND w.run_id=? AND w.work_id=?", [projectId, runId, workId]);
|
|
16
|
+
if (!work || work.status !== "failed" || !work.parent_work_id || work.result === null || !work.run_root)
|
|
17
|
+
return null;
|
|
18
|
+
const payload = JSON.parse(work.payload_json ?? "null");
|
|
19
|
+
if (typeof payload?.kind === "string" && /^(memory|interactive)/.test(payload.kind))
|
|
20
|
+
return null;
|
|
21
|
+
if (!context.db.get("SELECT 1 FROM sqlite_schema WHERE name='audit_events'") || !context.db.get("SELECT 1 FROM sqlite_schema WHERE name='lifecycle_invocations'"))
|
|
22
|
+
return null;
|
|
23
|
+
const resultHash = digestBytes(work.result);
|
|
24
|
+
const receipts = context.db.all(`SELECT a.payload_json,l.command,l.status AS invocation_status FROM audit_events a JOIN lifecycle_invocations l ON l.id=json_extract(a.payload_json,'$.invocation_id')
|
|
25
|
+
WHERE a.project_id=? AND a.event_type='lifecycle.work_settlement' AND json_extract(a.payload_json,'$.run_id')=? AND json_extract(a.payload_json,'$.work_id')=? ORDER BY a.rowid DESC`, [projectId, runId, workId]);
|
|
26
|
+
for (const row of receipts) {
|
|
27
|
+
const receipt = JSON.parse(row.payload_json), parsed = parseLifecycleCommand(row.command);
|
|
28
|
+
if (row.invocation_status !== "settled" || parsed.kind !== "standalone" || parsed.invocation.operation !== "work_fail" || receipt.result_sha256 !== resultHash || receipt.response?.status !== "failed")
|
|
29
|
+
continue;
|
|
30
|
+
if (!context.db.get("SELECT 1 FROM work_sessions WHERE id=? AND work_id=? AND status='failed'", [receipt.work_session_id, workId]))
|
|
31
|
+
continue;
|
|
32
|
+
return { work_id: workId, work_session_id: receipt.work_session_id, parent_work_id: work.parent_work_id, result_sha256: resultHash, primary_reason: work.result, result_ref: path.join(work.run_root, "works", workId, "result.json") };
|
|
33
|
+
}
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
import { createHash } from "node:crypto";
|
|
37
|
+
const digestBytes = (text) => createHash("sha256").update(text).digest("hex");
|
|
38
|
+
function events(context, row, type) {
|
|
39
|
+
return context.db.all(`SELECT e.data_json FROM run_controller_events e JOIN run_controllers c ON c.controller_id=e.controller_id WHERE c.project_id=? AND c.run_id=? AND e.event_type=? ORDER BY e.rowid`, [row.project_id, row.run_id, type]).map(e => JSON.parse(e.data_json));
|
|
40
|
+
}
|
|
41
|
+
export function prepareWorkFailureResolution(context, row, stage, failure) {
|
|
42
|
+
if (!stageWorkIds(context, row.project_id, row.run_id, stage).has(failure.work_id))
|
|
43
|
+
throw new AppError("work_failure_scope_changed", "Failed Work no longer belongs to this Stage", 1);
|
|
44
|
+
const current = explicitWorkFailure(context, row.project_id, row.run_id, failure.work_id);
|
|
45
|
+
if (!current || current.work_session_id !== failure.work_session_id || current.result_sha256 !== failure.result_sha256)
|
|
46
|
+
throw new AppError("work_failure_scope_changed", "Failed Work receipt changed before resolution", 1);
|
|
47
|
+
const attempt = stageAttempt(context, row.project_id, row.run_id, stage), cycle = stageCycle(context, row.project_id, row.run_id, stage);
|
|
48
|
+
const episode = digest([row.project_id, row.run_id, stage, attempt, cycle, failure.work_id, failure.work_session_id, failure.result_sha256]);
|
|
49
|
+
const run = context.db.get("SELECT run_root FROM runs WHERE project_id=? AND id=?", [row.project_id, row.run_id]);
|
|
50
|
+
const round = events(context, row, "work_failure_action_applied").filter(e => e.episode_id === episode).length;
|
|
51
|
+
const root = path.join(run.run_root, "work-failure-resolutions", episode, `round-${round}`);
|
|
52
|
+
const resolution = { ...failure, episode_id: episode, stage, attempt, cycle, decision_file: path.join(root, "decision-input.json"), accepted_file: path.join(root, "decision.json"), diagnostic_file: path.join(root, "failure.json"),
|
|
53
|
+
retry_used: events(context, row, "work_failure_action_reserved").some(e => e.action === "retry" && e.work_id === failure.work_id) };
|
|
54
|
+
fs.mkdirSync(root, { recursive: true });
|
|
55
|
+
publishImmutableInput(resolution.diagnostic_file, Buffer.from(JSON.stringify({ schema_id: "work-failure-context@1", ...failure, episode_id: episode, stage, attempt, cycle, packet: path.join(run.run_root, "works", failure.work_id, "starts") }, null, 2)), "work_failure_context_changed");
|
|
56
|
+
if (!fs.existsSync(resolution.decision_file)) {
|
|
57
|
+
prepareAgentOutputFile(resolution.decision_file);
|
|
58
|
+
fs.writeFileSync(resolution.decision_file, "", { flag: "wx", mode: 0o600 });
|
|
59
|
+
}
|
|
60
|
+
return resolution;
|
|
61
|
+
}
|
|
62
|
+
const text = (value) => typeof value === "string" && value.trim().length > 0;
|
|
63
|
+
export function readFailureDecision(resolution) {
|
|
64
|
+
const file = fs.existsSync(resolution.accepted_file) ? resolution.accepted_file : resolution.decision_file;
|
|
65
|
+
const stat = fs.lstatSync(file);
|
|
66
|
+
if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 1024 * 1024)
|
|
67
|
+
throw new AppError("work_failure_decision_invalid", "Expected a regular decision JSON file", 2);
|
|
68
|
+
let value;
|
|
69
|
+
try {
|
|
70
|
+
value = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
throw new AppError("work_failure_decision_invalid", "Write one complete decision JSON object", 2);
|
|
74
|
+
}
|
|
75
|
+
const keys = ["schema_id", "episode_id", "action", "reason", "evidence_refs", "correction", "blocker", "question"];
|
|
76
|
+
if (!value || Array.isArray(value) || typeof value !== "object" || Object.keys(value).some(k => !keys.includes(k)) || value.schema_id !== "work-failure-decision@1" || value.episode_id !== resolution.episode_id || !["retry", "block", "pause", "fail"].includes(value.action) || !text(value.reason) || !Array.isArray(value.evidence_refs) || !value.evidence_refs.length || !value.evidence_refs.every(text)
|
|
77
|
+
|| value.action === "retry" && !text(value.correction) || value.action === "pause" && !text(value.question)
|
|
78
|
+
|| value.action === "block" && (!value.blocker || !["engine", "harness", "environment"].includes(value.blocker.kind) || !text(value.blocker.code) || !text(value.blocker.summary) || typeof value.blocker.retryable !== "boolean" || Object.keys(value.blocker).some(k => !["kind", "code", "summary", "retryable"].includes(k)))
|
|
79
|
+
|| value.action !== "retry" && value.correction !== undefined || value.action !== "pause" && value.question !== undefined || value.action !== "block" && value.blocker !== undefined)
|
|
80
|
+
throw new AppError("work_failure_decision_invalid", "Decision must match its episode and selected action", 2, { episode_id: resolution.episode_id });
|
|
81
|
+
return value;
|
|
82
|
+
}
|
|
83
|
+
export function failureDecisionReady(resolution) { try {
|
|
84
|
+
readFailureDecision(resolution);
|
|
85
|
+
return true;
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
return false;
|
|
89
|
+
} }
|
|
90
|
+
export function workFailureResolutionPrompt(resolution, diagnostic) {
|
|
91
|
+
return [`Resolve only the retained explicit worker refusal in ${resolution.diagnostic_file}. Read that complete file and its Work packet/results.`,
|
|
92
|
+
"Current implementation is a baseline, not a requirement. Necessary project-local cross-layer corrections are allowed within accepted scope; planned write areas are hints. A worker-authored stop condition cannot silently cancel accepted intent. Do not edit product files, call lifecycle commands, spawn children or run checks in this decision Turn.",
|
|
93
|
+
`Write the decision to this precreated absolute file: ${resolution.decision_file}. No other writes. Return after writing it.`,
|
|
94
|
+
`Use schema_id=work-failure-decision@1, episode_id=${resolution.episode_id}, action=retry|block|pause|fail, nonempty reason and evidence_refs[]. For retry include concrete correction; for block include blocker {kind:engine|harness|environment,code,summary,retryable:boolean}; for pause include question. No extra fields.`,
|
|
95
|
+
resolution.retry_used ? "The one automatic corrective attempt has already been consumed. Select block/pause/fail; another retry cannot be applied automatically." : "Select retry if the Work can be completed in accepted scope; a fresh worker receives your correction. Select pause only for genuinely missing user intent, block for a proven technical dependency, fail only for an explained terminal outcome.",
|
|
96
|
+
"An unsupported exact browser key sequence does not prove all UI work is impossible and selectOption is not keyboard evidence. Preserve unverified mandatory criteria and original requirements.", diagnostic ? `Correct only the rejected decision: ${diagnostic}` : ""].filter(Boolean).join("\n\n");
|
|
97
|
+
}
|
|
98
|
+
export function acceptFailureDecision(context, row, resolution, assertOwner) {
|
|
99
|
+
assertOwner();
|
|
100
|
+
const decision = readFailureDecision(resolution), bytes = Buffer.from(JSON.stringify(decision));
|
|
101
|
+
publishImmutableInput(resolution.accepted_file, bytes, "work_failure_decision_changed");
|
|
102
|
+
context.db.writeTransaction(() => { assertOwner(); if (!events(context, row, "work_failure_decision_accepted").some(e => e.decision_ref === resolution.accepted_file))
|
|
103
|
+
appendEvent(context, row.controller_id, "work_failure_decision_accepted", { episode_id: resolution.episode_id, work_id: resolution.work_id, decision_ref: resolution.accepted_file, decision }); });
|
|
104
|
+
return decision;
|
|
105
|
+
}
|
|
106
|
+
export function workFailureDiagnostic(resolution, decision) {
|
|
107
|
+
return { source: "explicit_work_failure", episode_id: resolution.episode_id, work_id: resolution.work_id, work_session_id: resolution.work_session_id, result_ref: resolution.diagnostic_file, primary_reason: resolution.primary_reason, ...(decision ? { disposition: decision } : {}) };
|
|
108
|
+
}
|
|
109
|
+
/** Repair file-before-event and mutation-before-ack crash windows before HITL delivery. */
|
|
110
|
+
export async function reconcilePendingWorkFailure(context, row, state, continuationKind, projectRoot, assertOwner, writerSettled) {
|
|
111
|
+
const resolution = state.work_failure;
|
|
112
|
+
if (!resolution || resolution.deferred || !fs.existsSync(resolution.accepted_file) || continuationKind === "terminal")
|
|
113
|
+
return "none";
|
|
114
|
+
const session = state.current_session === null ? undefined : state.sessions[state.current_session];
|
|
115
|
+
if (!session)
|
|
116
|
+
throw new AppError("work_failure_scope_changed", "Failure resolution lost its coordinator", 1);
|
|
117
|
+
if (!await writerSettled(session))
|
|
118
|
+
return "waiting";
|
|
119
|
+
const decision = acceptFailureDecision(context, row, resolution, assertOwner);
|
|
120
|
+
applyWorkFailureDecision(context, row, resolution, decision, projectRoot, assertOwner);
|
|
121
|
+
delete state.work_failure;
|
|
122
|
+
state.last_continuation = null;
|
|
123
|
+
return "applied";
|
|
124
|
+
}
|
|
125
|
+
/** Reserve first; a committed retry is recovered through its start authority, never replayed blindly. */
|
|
126
|
+
export function applyWorkFailureDecision(context, row, resolution, decision, projectRoot, assertOwner) {
|
|
127
|
+
assertOwner();
|
|
128
|
+
assertRunMutationAllowed(context, row.project_id, row.run_id, row.generation);
|
|
129
|
+
const index = JSON.parse(context.db.get("SELECT index_json FROM runs WHERE project_id=? AND id=?", [row.project_id, row.run_id])?.index_json ?? "{}");
|
|
130
|
+
if (resolution.stage !== index.current_stage
|
|
131
|
+
|| resolution.attempt !== stageAttempt(context, row.project_id, row.run_id, resolution.stage) || resolution.cycle !== stageCycle(context, row.project_id, row.run_id, resolution.stage)
|
|
132
|
+
|| !stageWorkIds(context, row.project_id, row.run_id, resolution.stage).has(resolution.work_id))
|
|
133
|
+
throw new AppError("work_failure_scope_changed", "Failure resolution belongs to a stale Stage attempt", 1);
|
|
134
|
+
const applied = events(context, row, "work_failure_action_applied").find(e => e.decision_ref === resolution.accepted_file);
|
|
135
|
+
if (applied)
|
|
136
|
+
return applied;
|
|
137
|
+
const reserved = events(context, row, "work_failure_action_reserved").find(e => e.decision_ref === resolution.accepted_file);
|
|
138
|
+
const current = context.db.get("SELECT status,current_start_invocation_id FROM works WHERE project_id=? AND run_id=? AND work_id=?", [row.project_id, row.run_id, resolution.work_id]);
|
|
139
|
+
if (!current)
|
|
140
|
+
throw new AppError("work_failure_scope_changed", "Work disappeared before resolution", 1);
|
|
141
|
+
let action = decision.action;
|
|
142
|
+
const used = events(context, row, "work_failure_action_reserved").some(e => e.action === "retry" && e.work_id === resolution.work_id && e.decision_ref !== resolution.accepted_file);
|
|
143
|
+
const retainedDecision = events(context, row, "work_failure_decision_accepted").find(e => e.decision_ref === resolution.accepted_file);
|
|
144
|
+
if (!retainedDecision || JSON.stringify(retainedDecision.decision) !== JSON.stringify(decision))
|
|
145
|
+
throw new AppError("work_failure_scope_changed", "Failure decision was not accepted or changed after acceptance", 1);
|
|
146
|
+
const exhausted = action === "retry" && used;
|
|
147
|
+
if (exhausted)
|
|
148
|
+
action = "block";
|
|
149
|
+
if (!reserved) {
|
|
150
|
+
if (current.status !== "failed" || explicitWorkFailure(context, row.project_id, row.run_id, resolution.work_id)?.work_session_id !== resolution.work_session_id)
|
|
151
|
+
throw new AppError("work_failure_scope_changed", "Failure changed before action reservation", 1);
|
|
152
|
+
context.db.writeTransaction(() => { assertOwner(); appendEvent(context, row.controller_id, "work_failure_action_reserved", { episode_id: resolution.episode_id, work_id: resolution.work_id, work_session_id: resolution.work_session_id, action, decision_ref: resolution.accepted_file }); });
|
|
153
|
+
}
|
|
154
|
+
else if (reserved.action !== action)
|
|
155
|
+
throw new AppError("work_failure_scope_changed", "Reserved failure action differs", 1);
|
|
156
|
+
let receipt;
|
|
157
|
+
if (action === "retry") {
|
|
158
|
+
receipt = retryWork(context, resolution.work_id, decision.reason, { episode_id: resolution.episode_id, decision_ref: resolution.accepted_file, correction: decision.correction, failed_result_sha256: resolution.result_sha256 });
|
|
159
|
+
}
|
|
160
|
+
else if (action === "fail")
|
|
161
|
+
throw new AppError("work_failure_unresolved", decision.reason, 1, workFailureDiagnostic(resolution, decision));
|
|
162
|
+
else {
|
|
163
|
+
const coordinator = context.db.get("SELECT status FROM works WHERE project_id=? AND run_id=? AND work_id=?", [row.project_id, row.run_id, resolution.parent_work_id]);
|
|
164
|
+
const blocker = exhausted ? { kind: "engine", code: "work_resolution_exhausted", summary: JSON.stringify(workFailureDiagnostic(resolution, decision)), retryable: false } : decision.blocker;
|
|
165
|
+
const boundBlocker = blocker ? { ...blocker, summary: `${blocker.summary}\n[work-failure:${resolution.episode_id}:${digest(resolution.accepted_file)}]` } : undefined;
|
|
166
|
+
if (coordinator?.status === "paused" && reserved) {
|
|
167
|
+
const stage = index.stage_runs?.find((s) => s.stage === resolution.stage);
|
|
168
|
+
const matches = action === "pause" ? stage?.pause?.work_id === resolution.parent_work_id && typeof stage.pause.question_path === "string" && fs.readFileSync(stage.pause.question_path, "utf8") === decision.question
|
|
169
|
+
: stage?.blocker?.work_id === resolution.parent_work_id && stage.blocker.code === boundBlocker?.code && stage.blocker.kind === boundBlocker?.kind && stage.blocker.summary === boundBlocker?.summary;
|
|
170
|
+
if (!matches)
|
|
171
|
+
throw new AppError("work_failure_scope_changed", "Coordinator pause belongs to another failure action", 1);
|
|
172
|
+
receipt = { ok: true, reconciled: true, outcome: action === "pause" ? "paused" : "blocked", run_id: row.run_id, stage: resolution.stage, work_id: resolution.parent_work_id, ...(boundBlocker ? { blocker: boundBlocker } : {}) };
|
|
173
|
+
}
|
|
174
|
+
else {
|
|
175
|
+
if (coordinator?.status !== "running")
|
|
176
|
+
throw new AppError("work_failure_scope_changed", "Failure resolution needs the running coordinator Work", 1);
|
|
177
|
+
const scope = { projectRoot, runId: row.run_id, stage: resolution.stage, workId: resolution.parent_work_id };
|
|
178
|
+
receipt = action === "pause" ? pauseStageForUser(context, { ...scope, question: decision.question }) : blockStageForRuntime(context, { ...scope, ...boundBlocker });
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
const outcome = { episode_id: resolution.episode_id, decision_ref: resolution.accepted_file, stage: resolution.stage, attempt: resolution.attempt, cycle: resolution.cycle, work_id: resolution.work_id, work_session_id: resolution.work_session_id, action, receipt };
|
|
182
|
+
context.db.writeTransaction(() => { assertOwner(); appendEvent(context, row.controller_id, "work_failure_action_applied", outcome); });
|
|
183
|
+
return outcome;
|
|
184
|
+
}
|
|
185
|
+
/** Match only this current bound block; an old unrelated pause must not be hidden. */
|
|
186
|
+
export function workFailureBlockActive(context, row, index) {
|
|
187
|
+
const stage = index.current_stage;
|
|
188
|
+
const blocker = index.stage_runs?.find(s => s.stage === stage)?.blocker;
|
|
189
|
+
if (!stage || !blocker)
|
|
190
|
+
return false;
|
|
191
|
+
return events(context, row, "work_failure_action_applied").some(e => {
|
|
192
|
+
const receipt = e.receipt;
|
|
193
|
+
if (!receipt)
|
|
194
|
+
return false;
|
|
195
|
+
return e.action === "block" && e.stage === stage && e.attempt === stageAttempt(context, row.project_id, row.run_id, stage) && e.cycle === stageCycle(context, row.project_id, row.run_id, stage)
|
|
196
|
+
&& receipt?.work_id === blocker.work_id && receipt.blocker?.summary === blocker.summary && receipt.blocker?.code === blocker.code && receipt.blocker?.kind === blocker.kind;
|
|
197
|
+
});
|
|
198
|
+
}
|