@deksden-com/dd-flow-cli 0.9.0-beta.138 → 0.9.0-beta.139
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/build-info.json +6 -6
- package/dist/cli/runtime-process.js +1 -1
- package/dist/cli.js +2 -0
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +49 -20
- package/dist/harness-runtime/lib/dd-agy.mjs +16 -5
- package/dist/harness-runtime/lib/dd-codex-daemon.mjs +1 -1
- package/dist/harness-runtime/lib/dd-codex.mjs +2 -2
- package/dist/harness-runtime/lib/dd-droid.mjs +30 -8
- package/dist/harness-runtime/lib/dd-grok-daemon.mjs +7 -1
- package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +95 -12
- package/dist/harness-runtime/lib/dd-opencode.mjs +51 -20
- package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +13 -1
- package/dist/harness-runtime/lib/dd-zcode.mjs +17 -4
- package/dist/harness-runtime/lib/delegation-instructions.d.mts +1 -0
- package/dist/harness-runtime/lib/delegation-instructions.mjs +11 -0
- package/dist/harness-runtime/lib/lease-renewal.mjs +2 -1
- package/dist/harness-runtime/lib/native-children.d.mts +2 -0
- package/dist/harness-runtime/lib/native-children.mjs +20 -1
- package/dist/harness-runtime/lib/opencode-plugin.mjs +30 -9
- package/dist/harness-runtime/lib/provider-overload-policy.d.mts +13 -0
- package/dist/harness-runtime/lib/provider-overload-policy.mjs +138 -0
- package/dist/schemas/flow-run.schema.json +4 -0
- package/dist/services/controller-child-capacity.js +13 -0
- package/dist/services/controller-child-observer.js +18 -2
- package/dist/services/controller-execution.js +6 -6
- package/dist/services/external-work-launch.js +2 -4
- package/dist/services/frozen-engine.js +16 -8
- package/dist/services/managed-processes.js +31 -10
- package/dist/services/memory-init-native-capacity.js +65 -10
- package/dist/services/memory-init-preflight.js +21 -3
- package/dist/services/memory-init-project-bootstrap.js +3 -3
- package/dist/services/memory-init-start.js +6 -0
- package/dist/services/memory-init-worktrunk-capability.js +9 -3
- package/dist/services/merge-server.js +2 -4
- package/dist/services/{codex-capacity.js → provider-capacity.js} +34 -16
- package/dist/services/recovery-snapshot-database.js +1 -1
- package/dist/services/run-controller-adapter.js +1 -1
- package/dist/services/run-controller-recovery.js +2 -4
- package/dist/services/run-controller.js +59 -26
- package/dist/services/run-recovery-guard.js +3 -3
- package/dist/services/run-recovery-runtime.js +3 -3
- package/dist/services/runs.js +102 -13
- package/dist/services/vnext-code-review.js +1 -1
- package/dist/services/vnext-code.js +3 -3
- package/dist/services/vnext-fanout.js +3 -2
- package/dist/services/vnext-plan-review.js +1 -1
- package/dist/services/vnext-plan.js +3 -3
- package/dist/services/vnext-protocolize.js +20 -6
- package/dist/services/vnext-workspace-policy.js +7 -2
- package/package.json +1 -1
- package/dist/harness-runtime/lib/codex-capacity-policy.d.mts +0 -10
- package/dist/harness-runtime/lib/codex-capacity-policy.mjs +0 -102
|
@@ -12,6 +12,8 @@ import { readBoundedReleaseFile } from "./release-source.js";
|
|
|
12
12
|
import { acquireInstallationLock } from "./installation-lock.js";
|
|
13
13
|
import { inspectDaemonOperation } from "../harness-runtime/lib/daemon-operations.mjs";
|
|
14
14
|
import { nativeCommandFingerprint } from "../harness-runtime/lib/native-request-proof.mjs";
|
|
15
|
+
import { continueProviderCapacity } from "./provider-capacity.js";
|
|
16
|
+
import { CAPACITY_POLICY, MAX_OVERLOAD_RETRIES, refusalObservation, terminalProviderOverload } from "../harness-runtime/lib/provider-overload-policy.mjs";
|
|
15
17
|
const sha = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
16
18
|
function refuse(reason, effect = "no_effect") {
|
|
17
19
|
throw new AppError("memory_init_native_capacity_required", "Native startup requires its exact qualified probe", 1, { reason, effect, next_action: "reconcile_exact_native_probe" });
|
|
@@ -71,17 +73,40 @@ export async function produceMemoryInitNativeCapacity(input) {
|
|
|
71
73
|
const rootSessionId = adapterSessionId(read(path.join(root, "session-create.json")));
|
|
72
74
|
const inspected = read(path.join(root, "session-inspect.json"));
|
|
73
75
|
const children = rootSessionId && inspected ? normalizeNativeChildren(inspected, rootSessionId) : [];
|
|
74
|
-
|
|
75
|
-
|
|
76
|
+
const promptRoot = `memory-native:${hybridDigest(identity)}:session-prompt`;
|
|
77
|
+
const intents = fs.readdirSync(root).filter(name => /^capacity-intent-\d+\.json$/.test(name)).map(name => read(path.join(root, name))).sort((a, b) => a.ordinal - b.ordinal);
|
|
78
|
+
if (intents.length > MAX_OVERLOAD_RETRIES)
|
|
79
|
+
refuse("probe_capacity_budget_changed", "unknown");
|
|
80
|
+
const promptPhases = ["session-prompt", ...intents.map(intent => `session-prompt-capacity-${intent.ordinal}`)];
|
|
81
|
+
for (let index = 0; index < intents.length; index++) {
|
|
82
|
+
const intent = intents[index];
|
|
83
|
+
if (intent.policy !== CAPACITY_POLICY || intent.root_operation_id !== promptRoot || intent.session_id !== rootSessionId
|
|
84
|
+
|| intent.ordinal !== index + 1 || intent.operation_id !== `${promptRoot}:capacity:${index + 1}:session.prompt`
|
|
85
|
+
|| intent.predecessor !== (index ? intents[index - 1].operation_id : promptRoot))
|
|
86
|
+
refuse("probe_capacity_chain_changed", "unknown");
|
|
87
|
+
}
|
|
88
|
+
for (const phase of ["daemon-start", "session-create", ...promptPhases, "daemon-stop"]) {
|
|
89
|
+
const continuation = /^session-prompt-capacity-(\d+)$/.exec(phase);
|
|
90
|
+
const operationId = continuation ? `${promptRoot}:capacity:${continuation[1]}:session.prompt` : `memory-native:${hybridDigest(identity)}:${phase}`;
|
|
76
91
|
const observed = await inspectDaemonOperation(path.join(root, "daemon"), operationId).catch(() => null);
|
|
77
92
|
const fingerprint = read(path.join(root, `${phase}.request-proof.json`));
|
|
78
93
|
const intent = read(path.join(root, `${phase}.intent.json`));
|
|
79
94
|
const expected = read(path.join(root, `${phase}.json`));
|
|
80
95
|
if (!observed || !fingerprint || fingerprint.command_sha256 !== intent?.command_sha256
|
|
81
96
|
|| observed.invocation_sha256 !== fingerprint.command_sha256 || observed.params_sha256 !== fingerprint.params_sha256
|
|
82
|
-
|| observed.operation_id !== operationId
|
|
83
|
-
|| phase !== "daemon-start" && observed.daemon_id !== start?.daemon_id
|
|
84
|
-
|
|
97
|
+
|| observed.operation_id !== operationId
|
|
98
|
+
|| phase !== "daemon-start" && observed.daemon_id !== start?.daemon_id)
|
|
99
|
+
refuse("probe_provider_custody_changed", "unknown");
|
|
100
|
+
const promptIndex = promptPhases.indexOf(phase);
|
|
101
|
+
if (promptIndex >= 0 && promptIndex < promptPhases.length - 1) {
|
|
102
|
+
const intent = intents[promptIndex];
|
|
103
|
+
if (observed.state !== "failed" || !terminalProviderOverload(observed.error, rootSessionId ?? undefined, intent.turn_id)
|
|
104
|
+
|| intent.refusal?.turn_id !== intent.turn_id || intent.refusal?.session_id !== rootSessionId)
|
|
105
|
+
refuse("probe_capacity_predecessor_changed", "unknown");
|
|
106
|
+
if (hybridDigest(intent.refusal) !== hybridDigest(refusalObservation(observed.error, promptIndex, intent.refusal.first_observed_at)))
|
|
107
|
+
refuse("probe_capacity_refusal_changed", "unknown");
|
|
108
|
+
}
|
|
109
|
+
else if (observed.state !== "completed" || hybridDigest(publicReply(observed.result)) !== hybridDigest(expected))
|
|
85
110
|
refuse("probe_provider_custody_changed", "unknown");
|
|
86
111
|
}
|
|
87
112
|
if (hybridDigest(proof.input) !== hybridDigest(identity) || proof.failure || !rootSessionId
|
|
@@ -112,9 +137,8 @@ export async function produceMemoryInitNativeCapacity(input) {
|
|
|
112
137
|
delete env.DD_FLOW_HOME;
|
|
113
138
|
delete env.DD_FLOW_BIN;
|
|
114
139
|
delete env.DD_FLOW_ENGINE_HOME;
|
|
115
|
-
const call = async (phase, args, timeoutMs = 120_000) => {
|
|
140
|
+
const call = async (phase, args, timeoutMs = 120_000, operationId = `memory-native:${hybridDigest(identity)}:${phase}`) => {
|
|
116
141
|
lease.assertOwnership();
|
|
117
|
-
const operationId = `memory-native:${hybridDigest(identity)}:${phase}`;
|
|
118
142
|
const command = nativeCommandFingerprint({ executable, args: [...args, ...common], cwd: workspace, env, operationId });
|
|
119
143
|
const received = read(path.join(root, `${phase}.json`));
|
|
120
144
|
const intent = read(path.join(root, `${phase}.intent.json`));
|
|
@@ -132,6 +156,12 @@ export async function produceMemoryInitNativeCapacity(input) {
|
|
|
132
156
|
refuse(`unresolved_${phase}`, "unknown");
|
|
133
157
|
const observed = await inspectDaemonOperation(stateDir, operationId).catch(() => null);
|
|
134
158
|
const start = read(path.join(root, "daemon-start.json"));
|
|
159
|
+
if (args[0] === "session" && args[1] === "prompt" && witness.operation_id === operationId
|
|
160
|
+
&& observed?.operation_id === operationId && observed.operation === "session.prompt"
|
|
161
|
+
&& witness.command_sha256 === command && observed.invocation_sha256 === command
|
|
162
|
+
&& observed.params_sha256 === witness.params_sha256 && observed.daemon_id === start?.daemon_id
|
|
163
|
+
&& observed.state === "failed" && observed.error)
|
|
164
|
+
throw observed.error;
|
|
135
165
|
if (observed?.operation_id !== operationId || observed.operation !== args.slice(0, 2).join(".")
|
|
136
166
|
|| witness.command_sha256 !== command || observed.invocation_sha256 !== command
|
|
137
167
|
|| observed.params_sha256 !== witness.params_sha256 || !/^[a-f0-9]{64}$/.test(String(observed.params_sha256))
|
|
@@ -194,8 +224,33 @@ export async function produceMemoryInitNativeCapacity(input) {
|
|
|
194
224
|
rootSessionId = adapterSessionId(created);
|
|
195
225
|
if (!rootSessionId)
|
|
196
226
|
refuse("root_session_identity_missing", "unknown");
|
|
197
|
-
|
|
198
|
-
|
|
227
|
+
const promptArgs = (text) => ["session", "prompt", "--session-id", rootSessionId, "--model", input.profile.model, "--reasoning", input.profile.reasoning, "--prompt", text];
|
|
228
|
+
try {
|
|
229
|
+
await call("session-prompt", promptArgs(renderCapacityInstructions({ harness: input.profile.harness, maximum: input.maximum })), 180_000);
|
|
230
|
+
}
|
|
231
|
+
catch (error) {
|
|
232
|
+
const rootId = `memory-native:${hybridDigest(identity)}:session-prompt`;
|
|
233
|
+
const retainedIntents = () => fs.readdirSync(root).filter(name => /^capacity-intent-\d+\.json$/.test(name)).map(name => read(path.join(root, name))).sort((a, b) => a.ordinal - b.ordinal);
|
|
234
|
+
const retainedFailures = () => fs.readdirSync(root).filter(name => /^capacity-failure-\d+\.json$/.test(name)).map(name => read(path.join(root, name))).sort((a, b) => a.ordinal - b.ordinal);
|
|
235
|
+
const resumed = await continueProviderCapacity({ error, sessionId: rootSessionId, operationId: rootId,
|
|
236
|
+
continuationText: "Continue the same technical capacity probe in this Session. Inspect existing direct children first; do not launch replacements or a second child wave. Complete only the originally requested capacity markers.",
|
|
237
|
+
allowSettledDescendants: true, assertCurrent: () => lease.assertOwnership(),
|
|
238
|
+
inspectOperation: async (id) => await inspectDaemonOperation(stateDir, id),
|
|
239
|
+
inspectSession: async () => runHarnessAdapter({ executable, args: ["session", "inspect", "--session-id", rootSessionId, ...common], env, harness: input.profile.harness, cwd: workspace, operationId: `${rootId}:capacity-inspect`, timeoutMs: 30_000 }),
|
|
240
|
+
retainedIntents, retainedFailures,
|
|
241
|
+
retainIntent: intent => retain(path.join(root, `capacity-intent-${intent.ordinal}.json`), intent),
|
|
242
|
+
retainFailure: (_id, refusal) => {
|
|
243
|
+
const file = path.join(root, `capacity-failure-${refusal.ordinal}.json`);
|
|
244
|
+
const prior = read(file);
|
|
245
|
+
if (prior && hybridDigest(prior) !== hybridDigest(refusal))
|
|
246
|
+
refuse("probe_capacity_failure_changed", "unknown");
|
|
247
|
+
if (!prior)
|
|
248
|
+
retain(file, refusal);
|
|
249
|
+
},
|
|
250
|
+
prompt: (id, text, ordinal) => call(`session-prompt-capacity-${ordinal}`, promptArgs(text), 180_000, id) });
|
|
251
|
+
if (!resumed)
|
|
252
|
+
throw error;
|
|
253
|
+
}
|
|
199
254
|
inspected = await call("session-inspect", ["session", "inspect", "--session-id", rootSessionId]);
|
|
200
255
|
}
|
|
201
256
|
catch (error) {
|
|
@@ -230,7 +285,7 @@ export async function produceMemoryInitNativeCapacity(input) {
|
|
|
230
285
|
inspected, cleanup, children, capacity: children.length, qualified, ...(failure ? { failure } : {}) };
|
|
231
286
|
if (failure) {
|
|
232
287
|
fs.appendFileSync(path.join(root, "observations.jsonl"), `${JSON.stringify(proof)}\n`, { mode: 0o600 });
|
|
233
|
-
|
|
288
|
+
throw new AppError("memory_init_native_capacity_required", "Native startup probe failed or could not be reconciled", 1, { reason: "probe_operation_unresolved", effect: "unknown", next_action: "reconcile_exact_native_probe", primary_error: failure });
|
|
234
289
|
}
|
|
235
290
|
retain(path.join(root, "capacity.json"), proof);
|
|
236
291
|
return await publish(proof);
|
|
@@ -243,6 +243,24 @@ function inspect(input, env, ownedPublication = false, qualifyTools = true, obse
|
|
|
243
243
|
...(generatedLint ? { generated_lint_config_sha256: generatedLint } : {}),
|
|
244
244
|
...(toolProbes ? { tool_probes: toolProbes } : {}) };
|
|
245
245
|
}
|
|
246
|
+
/** Check the prepared executable identities, not another version invocation. */
|
|
247
|
+
function reinspect(facts, input, env, ownedPublication, lintReservation) {
|
|
248
|
+
const tools = { git: executable("git", env), wt: executable("wt", env), "mb-lint": executable("mb-lint", env) };
|
|
249
|
+
if (facts.tool_probes)
|
|
250
|
+
for (const name of ["git", "wt", "mb-lint"]) {
|
|
251
|
+
let checksum;
|
|
252
|
+
try {
|
|
253
|
+
checksum = createHash("sha256").update(readBoundedReleaseFile(tools[name], 64 * 1024 * 1024)).digest("hex");
|
|
254
|
+
}
|
|
255
|
+
catch {
|
|
256
|
+
return refuse("target_changed_after_preparation");
|
|
257
|
+
}
|
|
258
|
+
if (tools[name] !== facts.tools[name] || checksum !== facts.tool_probes[name]?.executable_sha256)
|
|
259
|
+
refuse("target_changed_after_preparation");
|
|
260
|
+
}
|
|
261
|
+
const current = inspect(input, env, ownedPublication, false, undefined, lintReservation, tools);
|
|
262
|
+
return facts.tool_probes ? { ...current, tool_probes: facts.tool_probes } : current;
|
|
263
|
+
}
|
|
246
264
|
/** Read-only inventory of the independently selected bootstrap baseline. The
|
|
247
265
|
* publisher/bootstrap must establish owned-publication custody first. This
|
|
248
266
|
* changes no legacy target fact/checksum and never follows a source symlink.
|
|
@@ -289,7 +307,7 @@ export class PreparedMemoryInitTarget {
|
|
|
289
307
|
}
|
|
290
308
|
facts() { return structuredClone(this.#facts); }
|
|
291
309
|
revalidate() {
|
|
292
|
-
if (hybridDigest(
|
|
310
|
+
if (hybridDigest(reinspect(this.#facts, this.#input, this.#env, false, this.#facts.generated_lint_config_sha256 ? true : undefined)) !== this.sha256)
|
|
293
311
|
refuse("target_changed_after_preparation");
|
|
294
312
|
}
|
|
295
313
|
/** Only the publisher may use this after independently verifying its CLI-home owner journal.
|
|
@@ -297,8 +315,8 @@ export class PreparedMemoryInitTarget {
|
|
|
297
315
|
* lint policy/owned temporary. User sources, scope, HEAD and tools stay pinned.
|
|
298
316
|
*/
|
|
299
317
|
static revalidateOwnedPublication(facts, expectedSha256, env) {
|
|
300
|
-
if (hybridDigest(facts) !== expectedSha256 || hybridDigest(
|
|
301
|
-
workspaceRoot: facts.workspace_root, requestId: facts.request_id }, env, true, facts.
|
|
318
|
+
if (hybridDigest(facts) !== expectedSha256 || hybridDigest(reinspect(facts, { projectRoot: facts.project_root,
|
|
319
|
+
workspaceRoot: facts.workspace_root, requestId: facts.request_id }, env, true, facts.generated_lint_config_sha256)) !== expectedSha256)
|
|
302
320
|
refuse("target_changed_after_preparation");
|
|
303
321
|
}
|
|
304
322
|
/** Restore only after the publication service checks its independent journal and root owner.
|
|
@@ -2,7 +2,7 @@ import fs from "node:fs";
|
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { createHash } from "node:crypto";
|
|
4
4
|
import semver from "semver";
|
|
5
|
-
import { AppError } from "../shared/errors.js";
|
|
5
|
+
import { AppError, errorRecord } from "../shared/errors.js";
|
|
6
6
|
import { hybridDigest } from "./hybrid-contracts.js";
|
|
7
7
|
import { FrozenGenericExecution } from "./generic-execution.js";
|
|
8
8
|
import { PreparedMemoryInitTarget } from "./memory-init-preflight.js";
|
|
@@ -509,7 +509,7 @@ export class PreparedMemoryInitProjectBootstrap {
|
|
|
509
509
|
}
|
|
510
510
|
catch (error) {
|
|
511
511
|
throw new AppError("memory_init_engine_bootstrap_failed", "Engine-bound bootstrap retained an incomplete request outcome", 1, { reason: error instanceof AppError ? error.details.reason ?? error.code : "engine_bootstrap_operation_failed",
|
|
512
|
-
effect: "partial", next_action: "reconcile_exact_request" });
|
|
512
|
+
effect: "partial", next_action: "reconcile_exact_request", cause: errorRecord(error) });
|
|
513
513
|
}
|
|
514
514
|
}
|
|
515
515
|
apply(context, onCheckpoint) {
|
|
@@ -593,7 +593,7 @@ export class PreparedMemoryInitProjectBootstrap {
|
|
|
593
593
|
}
|
|
594
594
|
catch (error) {
|
|
595
595
|
throw new AppError("memory_init_project_bootstrap_failed", "Project bootstrap retained an incomplete request outcome", 1, { reason: error instanceof AppError ? error.details?.reason ?? error.code : "bootstrap_operation_failed",
|
|
596
|
-
effect: retained ? "partial" : "no_effect", next_action: "reconcile_exact_request" });
|
|
596
|
+
effect: retained ? "partial" : "no_effect", next_action: "reconcile_exact_request", cause: errorRecord(error) });
|
|
597
597
|
}
|
|
598
598
|
}
|
|
599
599
|
}
|
|
@@ -22,6 +22,7 @@ import { openMemoryInitStores } from "./memory-init-stores.js";
|
|
|
22
22
|
import { prepareMemoryInitRequestCandidate } from "./memory-init-request-candidate.js";
|
|
23
23
|
import { prepareMemoryInitHarnessCommands } from "./memory-init-project-bootstrap.js";
|
|
24
24
|
import { readMemoryInitRequestInput } from "./memory-init-request-input.js";
|
|
25
|
+
import { publishSetup } from "../harness-runtime/lib/setup-progress.mjs";
|
|
25
26
|
function refuse(reason) {
|
|
26
27
|
throw new AppError("memory_init_start_invalid", "Memory init requires its exact startup request", 1, { reason, effect: "no_effect", next_action: "reconcile_exact_request" });
|
|
27
28
|
}
|
|
@@ -99,10 +100,13 @@ export function prepareMemoryInitFreshRequest(env, scope, input, rawIntake = Buf
|
|
|
99
100
|
const candidate = prepareMemoryInitRequestCandidate(resolveDdFlowHome(env), scope, materialBytes, rawIntake);
|
|
100
101
|
if (candidate.objective !== raw.objective)
|
|
101
102
|
refuse("raw_intake_objective_mismatch");
|
|
103
|
+
publishSetup("probe_completed", "memory-init:request-candidate");
|
|
102
104
|
const selection = prepareMemoryInitSelection({ ddFlowHome: resolveDdFlowHome(env), env, now: () => new Date().toISOString() }, scope, raw);
|
|
105
|
+
publishSetup("probe_completed", "memory-init:selection");
|
|
103
106
|
// A known incompatible delivered reader refuses before candidate/store
|
|
104
107
|
// effects. Historical SDK requests still use their original @1/@2 contract.
|
|
105
108
|
selection.engine.admitMemoryInitFlow(selection.source, "dd-flow/memory-init-startup-input@3");
|
|
109
|
+
publishSetup("probe_completed", "memory-init:flow-admission");
|
|
106
110
|
const retained = candidate.retain();
|
|
107
111
|
let stores;
|
|
108
112
|
try {
|
|
@@ -113,10 +117,12 @@ export function prepareMemoryInitFreshRequest(env, scope, input, rawIntake = Buf
|
|
|
113
117
|
throw error;
|
|
114
118
|
throw new AppError("memory_init_fresh_preparation_incomplete", "Request candidate remains retained; store admission did not complete", 1, { effect: "partial", request_id: scope.requestId, cause: errorRecord(error), next_action: "reconcile_exact_request" });
|
|
115
119
|
}
|
|
120
|
+
publishSetup("probe_completed", "memory-init:stores");
|
|
116
121
|
let admitted = false;
|
|
117
122
|
return withStoreCleanup(() => {
|
|
118
123
|
try {
|
|
119
124
|
const prepared = PreparedMemoryInitStartupInput.prepare(stores.context, selection, candidate);
|
|
125
|
+
publishSetup("probe_completed", "memory-init:startup-input");
|
|
120
126
|
admitted = true;
|
|
121
127
|
return { ...stores, prepared };
|
|
122
128
|
}
|
|
@@ -8,8 +8,8 @@ import { hybridDigest } from "./hybrid-contracts.js";
|
|
|
8
8
|
import { PreparedMemoryInitTarget } from "./memory-init-preflight.js";
|
|
9
9
|
import { readBoundedReleaseFile } from "./release-source.js";
|
|
10
10
|
const hash = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
11
|
-
function refuse(reason) {
|
|
12
|
-
throw new AppError("memory_init_worktrunk_capability_failed", "Selected Worktrunk cannot create the required workspace", 1, { reason, effect: "no_effect", next_action: "repair_prerequisite_then_retry" });
|
|
11
|
+
function refuse(reason, diagnostics) {
|
|
12
|
+
throw new AppError("memory_init_worktrunk_capability_failed", "Selected Worktrunk cannot create the required workspace", 1, { reason, effect: "no_effect", next_action: "repair_prerequisite_then_retry", ...(diagnostics ? { tool_probe: diagnostics } : {}) });
|
|
13
13
|
}
|
|
14
14
|
/** Historical custody shape only; it does not run or qualify an executable. */
|
|
15
15
|
export function isMemoryInitWorktrunkCapabilityFacts(value) {
|
|
@@ -83,9 +83,15 @@ export class PreparedMemoryInitWorktrunkCapability {
|
|
|
83
83
|
const run = (file, args) => {
|
|
84
84
|
ownRoot();
|
|
85
85
|
unchanged();
|
|
86
|
+
const started = performance.now();
|
|
86
87
|
const result = spawnSync(file, args, { cwd: repo, env: probeEnv, shell: false, windowsHide: true, timeout: 5000, maxBuffer: 256 * 1024 });
|
|
87
88
|
if (result.error || result.signal || result.status !== 0)
|
|
88
|
-
refuse("worktrunk_probe_execution_failed"
|
|
89
|
+
refuse("worktrunk_probe_execution_failed", {
|
|
90
|
+
name: path.basename(file), phase: "workspace_probe", timeout_ms: 5000, elapsed_ms: Math.round(performance.now() - started),
|
|
91
|
+
status: result.status, signal: result.signal, error_code: result.error?.code ?? null,
|
|
92
|
+
timed_out: result.error?.code === "ETIMEDOUT",
|
|
93
|
+
stdout_bytes: result.stdout?.length ?? null, stderr_bytes: result.stderr?.length ?? null
|
|
94
|
+
});
|
|
89
95
|
ownRoot();
|
|
90
96
|
unchanged();
|
|
91
97
|
return new TextDecoder("utf-8", { fatal: true }).decode(result.stdout);
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { managedInvocationContext, managedLifecycleCommand } from "./lifecycle-invocations.js";
|
|
2
2
|
import { stageLifecycleInstruction } from "../harness-runtime/lib/delegation-instructions.mjs";
|
|
3
|
-
import {
|
|
3
|
+
import { continueProviderCapacity } from "./provider-capacity.js";
|
|
4
4
|
import { errorRecord } from "./run-controller-state.js";
|
|
5
5
|
import crypto from "node:crypto";
|
|
6
6
|
import fs from "node:fs";
|
|
@@ -164,11 +164,9 @@ async function dispatch(context, input) {
|
|
|
164
164
|
prompted = await adapterCall(["session", "prompt", "--session-id", sessionId, "--prompt-file", promptFile, ...common], null, `MERGE ${input.request.merge_request_id} Session ${sessionId} is still running`);
|
|
165
165
|
}
|
|
166
166
|
catch (error) {
|
|
167
|
-
if (!["codex-desktop", "opencode-server"].includes(runtimeHarness(input.profile)))
|
|
168
|
-
throw error;
|
|
169
167
|
const capacityIntents = (retained.capacity_intents ?? []);
|
|
170
168
|
const capacityFailures = (retained.capacity_failures ?? []);
|
|
171
|
-
const resumed = await
|
|
169
|
+
const resumed = await continueProviderCapacity({ error, sessionId, operationId: String(retained.prompt_operation_id), allowSettledDescendants: true,
|
|
172
170
|
signal: ownership.signal,
|
|
173
171
|
continuationText: `Continue only MERGE ${input.request.merge_request_id}, Work ${input.request.executor_work_id}, RUN ${input.request.run_id} in this same Session and target ${input.request.target_workspace}. First inspect current durable Stage/Work/MERGE receipts. Never repeat Stage start, accepted answer, completed apply or checks. Follow only the current exact issued successor and stop at the original boundary.`,
|
|
174
172
|
retainedIntents: () => capacityIntents,
|
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import { setTimeout as delay } from "node:timers/promises";
|
|
2
2
|
import crypto from "node:crypto";
|
|
3
|
-
import { CAPACITY_POLICY, capacityBackoff, nativeItemsSettled,
|
|
3
|
+
import { CAPACITY_POLICY, MAX_OVERLOAD_RETRIES, capacityBackoff, nativeItemsSettled, normalizeProviderFailure, overloadExhausted, parseCapacityOperation, refusalObservation, terminalProviderOverload } from "../harness-runtime/lib/provider-overload-policy.mjs";
|
|
4
4
|
import { AppError } from "../shared/errors.js";
|
|
5
5
|
import { isObservationLoss } from "../harness-runtime/lib/operation-errors.mjs";
|
|
6
|
-
export function
|
|
6
|
+
export function providerChildFailure(child) {
|
|
7
|
+
return child.provider_failure ?? { code: child.native_source ? "provider_capacity_transient" : "turn_interrupted", details: { native_source: child.native_source, provider_session_id: child.session_id, turn_id: child.terminal_turn_id, native_turn_id: child.terminal_turn_id, terminal_status: child.status, observed_at: child.terminal_observed_at ?? null, provider_error: child.provider_error, native_turn_items: child.native_turn_items } };
|
|
8
|
+
}
|
|
9
|
+
export function eligibleProviderChildOverload(child, parentSessionId) {
|
|
7
10
|
return typeof child.session_id === "string" && child.session_id !== parentSessionId && child.parent_session_id === parentSessionId && child.provenance !== "parent_mismatch" && child.status === "failed" && typeof child.terminal_turn_id === "string" && nativeItemsSettled(child.native_turn_items)
|
|
8
|
-
&&
|
|
11
|
+
&& terminalProviderOverload(providerChildFailure(child), child.session_id, child.terminal_turn_id);
|
|
9
12
|
}
|
|
10
13
|
/** A terminal root is not a settled tree. Only the daemon's exact owned tree
|
|
11
14
|
* with a settled root and positively live descendants permits waiting. */
|
|
@@ -37,7 +40,7 @@ export function assertCapacityOwnerAdmission(context, record, operationId, bindi
|
|
|
37
40
|
return;
|
|
38
41
|
const root = parseCapacityOperation(operationId).root;
|
|
39
42
|
const childOrdinal = /:child:[a-f0-9]+:followup:(\d+)$/.exec(root)?.[1];
|
|
40
|
-
if (childOrdinal && (!Number.isSafeInteger(Number(childOrdinal)) || Number(childOrdinal) < 1 || String(Number(childOrdinal)) !== childOrdinal))
|
|
43
|
+
if (childOrdinal && (!Number.isSafeInteger(Number(childOrdinal)) || Number(childOrdinal) < 1 || Number(childOrdinal) > MAX_OVERLOAD_RETRIES || String(Number(childOrdinal)) !== childOrdinal))
|
|
41
44
|
throw new AppError("capacity_continuation_conflict", "Invalid native child continuation ordinal", 1);
|
|
42
45
|
if (root === operationId && !childOrdinal)
|
|
43
46
|
return;
|
|
@@ -71,7 +74,9 @@ export function assertCapacityOwnerAdmission(context, record, operationId, bindi
|
|
|
71
74
|
if (intent)
|
|
72
75
|
break;
|
|
73
76
|
}
|
|
74
|
-
if (
|
|
77
|
+
if (intent?.policy && intent.policy !== CAPACITY_POLICY)
|
|
78
|
+
throw new AppError("capacity_policy_mismatch", "Continuation is pinned to another retry policy", 1);
|
|
79
|
+
if (!intent || parsed.ordinal > MAX_OVERLOAD_RETRIES || intent.policy !== CAPACITY_POLICY || intent.root_operation_id !== root || intent.ordinal !== parsed.ordinal
|
|
75
80
|
|| intent.predecessor !== predecessor || intent.operation_id !== operationId || intent.refusal?.turn_id !== intent.turn_id
|
|
76
81
|
|| intent.refusal?.ordinal !== parsed.ordinal - 1 || intent.refusal?.session_id !== intent.session_id
|
|
77
82
|
|| !Number.isFinite(Date.parse(intent.not_before)) || Date.parse(intent.not_before) > Date.parse(context.now())
|
|
@@ -115,9 +120,13 @@ export function capacityDelay(error, ordinal, now = Date.now()) {
|
|
|
115
120
|
return capacityBackoff(error, ordinal, now);
|
|
116
121
|
}
|
|
117
122
|
/** Continue a confirmed terminal Turn, never replay its completed effects. */
|
|
118
|
-
export async function
|
|
123
|
+
export async function continueProviderCapacity(input) {
|
|
119
124
|
let { error, operationId } = input;
|
|
120
125
|
const history = [...(input.retainedIntents?.() ?? [])];
|
|
126
|
+
if (history.some(intent => intent.policy !== CAPACITY_POLICY) || input.retainedFailures?.().some(refusal => refusal.policy !== CAPACITY_POLICY))
|
|
127
|
+
throw new AppError("capacity_policy_mismatch", "Retained continuation uses another retry policy", 1);
|
|
128
|
+
if (history.length > MAX_OVERLOAD_RETRIES)
|
|
129
|
+
throw new AppError("capacity_continuation_conflict", "Retained episode exceeds its retry budget", 1);
|
|
121
130
|
const parsed = parseCapacityOperation(input.operationId);
|
|
122
131
|
const rootId = parsed.root;
|
|
123
132
|
const text = input.continuationText ?? "Continue the same assigned task in this Session. Check current durable RUN, Work and Stage state first. Never repeat an observed lifecycle command, accepted HITL answer, check or merge apply; use only an exact issued successor. Stop at the assigned boundary.";
|
|
@@ -217,9 +226,9 @@ export async function continueCodexCapacity(input) {
|
|
|
217
226
|
throw new AppError("capacity_continuation_blocked", "Retained completion cannot be reconciled with its owner boundary", 1);
|
|
218
227
|
return boundary === "paused" || boundary === "completed" ? { provider_session_id: input.sessionId, capacity_decision: boundary } : observed.result;
|
|
219
228
|
}
|
|
220
|
-
if (observed.state === "failed" && observed.error && !
|
|
229
|
+
if (observed.state === "failed" && observed.error && !terminalProviderOverload(observed.error, input.sessionId))
|
|
221
230
|
throw observed.error;
|
|
222
|
-
if (observed.state !== "failed" || !
|
|
231
|
+
if (observed.state !== "failed" || !terminalProviderOverload(observed.error, input.sessionId))
|
|
223
232
|
throw new AppError("operation_observation_lost", "Retained successor outcome is not proven; do not resend", 1, { operation_id: intent.operation_id, primary_error: error, observed });
|
|
224
233
|
operationId = intent.operation_id;
|
|
225
234
|
error = observed.error;
|
|
@@ -240,7 +249,7 @@ export async function continueCodexCapacity(input) {
|
|
|
240
249
|
for (;; ordinal += 1) {
|
|
241
250
|
if (!Number.isSafeInteger(ordinal))
|
|
242
251
|
conflict("Continuation ordinal overflow");
|
|
243
|
-
if (!
|
|
252
|
+
if (!terminalProviderOverload(error, input.sessionId)) {
|
|
244
253
|
if (ordinal === 1)
|
|
245
254
|
return null;
|
|
246
255
|
throw error;
|
|
@@ -250,14 +259,14 @@ export async function continueCodexCapacity(input) {
|
|
|
250
259
|
return completed;
|
|
251
260
|
const prior = await observe(() => input.inspectOperation(operationId));
|
|
252
261
|
const failure = prior.error;
|
|
253
|
-
const confirmed =
|
|
262
|
+
const confirmed = normalizeProviderFailure(failure, input.sessionId);
|
|
254
263
|
const turnId = confirmed?.turn_id;
|
|
255
264
|
input.signal?.throwIfAborted();
|
|
256
265
|
input.assertCurrent();
|
|
257
266
|
if (prior.primary_error)
|
|
258
267
|
throw prior.primary_error;
|
|
259
268
|
if (prior.operation_id !== operationId || prior.operation !== "session.prompt" || prior.session_id !== input.sessionId
|
|
260
|
-
|| prior.state !== "failed" || prior.primary_error || !turnId || !
|
|
269
|
+
|| prior.state !== "failed" || prior.primary_error || !turnId || !terminalProviderOverload(failure, input.sessionId, turnId)) {
|
|
261
270
|
throw blocked("Retained predecessor does not prove the exact terminal overloaded Turn");
|
|
262
271
|
}
|
|
263
272
|
const readiness = async () => {
|
|
@@ -272,7 +281,7 @@ export async function continueCodexCapacity(input) {
|
|
|
272
281
|
const ready = await readiness();
|
|
273
282
|
if (ready === "unproven")
|
|
274
283
|
throw blocked("Native predecessor readiness is unproven; do not cancel or replay to manufacture proof");
|
|
275
|
-
const native =
|
|
284
|
+
const native = normalizeProviderFailure(failure, input.sessionId, turnId);
|
|
276
285
|
let refusal = refusals.find(item => item.ordinal === ordinal - 1);
|
|
277
286
|
if (refusal && (refusal.turn_id !== turnId || refusal.session_id !== input.sessionId || refusal.policy !== CAPACITY_POLICY))
|
|
278
287
|
conflict("Failure ordinal identifies a different native Turn");
|
|
@@ -293,17 +302,26 @@ export async function continueCodexCapacity(input) {
|
|
|
293
302
|
input.retainFailure?.(operationId, refusal);
|
|
294
303
|
refusals.push(refusal);
|
|
295
304
|
}
|
|
296
|
-
const
|
|
297
|
-
if (
|
|
305
|
+
const exhausted = overloadExhausted(refusals);
|
|
306
|
+
if (exhausted) {
|
|
298
307
|
if (input.atomicIntent)
|
|
299
308
|
input.retainFailure?.(operationId, refusal);
|
|
300
|
-
throw new AppError("
|
|
309
|
+
throw new AppError("provider_unavailable", "Provider overload retry episode exhausted", 1, { ...exhausted, root_operation_id: rootId, provider_session_id: input.sessionId, continuation_count: ordinal - 1, primary_error: refusals.length ? input.error : failure, last_error: failure });
|
|
301
310
|
}
|
|
302
311
|
const nextId = `${rootId}:capacity:${ordinal}:session.prompt`;
|
|
303
312
|
const existing = history.find(item => item.operation_id === nextId);
|
|
304
313
|
if (existing && (existing.predecessor !== operationId || existing.turn_id !== turnId || existing.prompt_sha256 !== promptSha || existing.ordinal !== ordinal))
|
|
305
314
|
throw error;
|
|
306
|
-
|
|
315
|
+
let wait;
|
|
316
|
+
try {
|
|
317
|
+
wait = existing ? Math.max(0, Date.parse(existing.not_before) - Date.now()) : capacityDelay(failure, ordinal);
|
|
318
|
+
}
|
|
319
|
+
catch (cause) {
|
|
320
|
+
if (input.atomicIntent)
|
|
321
|
+
input.retainFailure?.(operationId, refusal);
|
|
322
|
+
const metadata = cause;
|
|
323
|
+
throw new AppError(metadata.code ?? "capacity_retry_metadata_invalid", metadata.message ?? "Invalid provider retry metadata", 1, { ...metadata.details, policy: CAPACITY_POLICY, attempts: refusals.length, root_operation_id: rootId, primary_error: input.error, last_error: failure });
|
|
324
|
+
}
|
|
307
325
|
if (!Number.isFinite(wait) || Date.now() + wait > 8.64e15 || input.deadline !== undefined && Date.now() + wait >= input.deadline) {
|
|
308
326
|
if (input.atomicIntent)
|
|
309
327
|
input.retainFailure?.(operationId, refusal);
|
|
@@ -64,7 +64,7 @@ export function captureRecoveryDatabase(source, output, projectId, runId, protoc
|
|
|
64
64
|
rules[table] = `${ownProject} AND protocol_id IN (${protocols})`;
|
|
65
65
|
// Relocation proofs are RUN-owned provenance needed to map unchanged context
|
|
66
66
|
// bytes through subsequent recovery generations, not portable live authority.
|
|
67
|
-
rules.audit_events = `${ownProject} AND (protocol_id IN (${protocols}) OR (event_type
|
|
67
|
+
rules.audit_events = `${ownProject} AND (protocol_id IN (${protocols}) OR (event_type IN ('flow_run.publication_relocation', 'flow_run.workspace_rebound') AND json_extract(payload_json, '$.run_id') IN (SELECT run_id FROM capture_scope)))`;
|
|
68
68
|
// Bootstrap precedes RUN/project allocation: some owning receipts have
|
|
69
69
|
// NULL project_id. Select only the captured MB bundle's independently
|
|
70
70
|
// anchored request/roots, never all bootstrap receipts for this project.
|
|
@@ -6,7 +6,7 @@ import { adapterSessionId, runHarnessAdapter } from "./harness-adapter.js";
|
|
|
6
6
|
import { runtimeHarness } from "./execution-policy.js";
|
|
7
7
|
import { appendEvent, digest, digestBytes, errorRecord, requireRun } from "./run-controller-state.js";
|
|
8
8
|
import { managedProcessStatus } from "./managed-processes.js";
|
|
9
|
-
import { CAPACITY_POLICY, parseCapacityOperation } from "../harness-runtime/lib/
|
|
9
|
+
import { CAPACITY_POLICY, parseCapacityOperation } from "../harness-runtime/lib/provider-overload-policy.mjs";
|
|
10
10
|
import { recoveryObservationBudget } from "./recovery-observation-budget.js";
|
|
11
11
|
import { canonicalNativeOperation } from "../harness-runtime/lib/adapter-timeouts.mjs";
|
|
12
12
|
import { retainedDaemonReply } from "../harness-runtime/lib/daemon-observation.mjs";
|
|
@@ -15,7 +15,7 @@ import { managedProcessBinding, managedProcessRuntimeHome, managedProcessStatus,
|
|
|
15
15
|
import { requireRun, requireController, controllerReceipt, appendEvent, saveState, digest, errorRecord } from "./run-controller-state.js";
|
|
16
16
|
import { controllerAdapter, controllerEnvironment, sessionArguments } from "./run-controller-adapter.js";
|
|
17
17
|
import { runHarnessAdapter } from "./harness-adapter.js";
|
|
18
|
-
import {
|
|
18
|
+
import { continueProviderCapacity } from "./provider-capacity.js";
|
|
19
19
|
import { spawnController, recordControllerLauncher } from "./run-controller-process.js";
|
|
20
20
|
import { readNativeDaemonState, requireManagedDaemonBinding } from "./managed-daemon-binding.js";
|
|
21
21
|
import { managedInvocationContext } from "./lifecycle-invocations.js";
|
|
@@ -213,12 +213,10 @@ export async function resumeControllerSession(context, row, manifest, state, ass
|
|
|
213
213
|
receipt = await sendAck(dispatch.operation_id, dispatch.prompt);
|
|
214
214
|
}
|
|
215
215
|
catch (error) {
|
|
216
|
-
if (!["codex-desktop", "opencode-server"].includes(runtimeHarness(session.profile)))
|
|
217
|
-
throw error;
|
|
218
216
|
const continuationText = `Continue the same retained Session and recovery generation. First inspect the current RUN recovery state; only if the exact ACK still awaits acceptance, execute the exact standalone acceptance command below once. Never deliver it again after acceptance. Then stop at the original Stage or Work-graph boundary.\n\n${dispatch.prompt}`;
|
|
219
217
|
const inspect = (args, operationId) => runHarnessAdapter({ executable: manifest.adapters[runtimeHarness(session.profile)].executable,
|
|
220
218
|
args, env: controllerEnvironment(executionContext, manifest, session), runtimeOwner: session.owner, operationId, timeoutMs: 30_000 });
|
|
221
|
-
const resumed = await
|
|
219
|
+
const resumed = await continueProviderCapacity({ error, sessionId: session.id, operationId: dispatch.operation_id, allowSettledDescendants: true, continuationText,
|
|
222
220
|
retainedIntents: () => {
|
|
223
221
|
const binding = context.db.get("SELECT runtime_dispatch_json FROM run_recovery_bindings WHERE recovery_id = ?", [recovery.recovery_id]);
|
|
224
222
|
return binding?.runtime_dispatch_json ? JSON.parse(binding.runtime_dispatch_json).capacity_intents ?? [] : [];
|