@deksden-com/dd-flow-cli 0.9.0-beta.138 → 0.9.0-beta.139

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/CHANGELOG.md +15 -0
  2. package/dist/build-info.json +6 -6
  3. package/dist/cli/runtime-process.js +1 -1
  4. package/dist/cli.js +2 -0
  5. package/dist/harness-runtime/lib/dd-agy-daemon.mjs +49 -20
  6. package/dist/harness-runtime/lib/dd-agy.mjs +16 -5
  7. package/dist/harness-runtime/lib/dd-codex-daemon.mjs +1 -1
  8. package/dist/harness-runtime/lib/dd-codex.mjs +2 -2
  9. package/dist/harness-runtime/lib/dd-droid.mjs +30 -8
  10. package/dist/harness-runtime/lib/dd-grok-daemon.mjs +7 -1
  11. package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +95 -12
  12. package/dist/harness-runtime/lib/dd-opencode.mjs +51 -20
  13. package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +13 -1
  14. package/dist/harness-runtime/lib/dd-zcode.mjs +17 -4
  15. package/dist/harness-runtime/lib/delegation-instructions.d.mts +1 -0
  16. package/dist/harness-runtime/lib/delegation-instructions.mjs +11 -0
  17. package/dist/harness-runtime/lib/lease-renewal.mjs +2 -1
  18. package/dist/harness-runtime/lib/native-children.d.mts +2 -0
  19. package/dist/harness-runtime/lib/native-children.mjs +20 -1
  20. package/dist/harness-runtime/lib/opencode-plugin.mjs +30 -9
  21. package/dist/harness-runtime/lib/provider-overload-policy.d.mts +13 -0
  22. package/dist/harness-runtime/lib/provider-overload-policy.mjs +138 -0
  23. package/dist/schemas/flow-run.schema.json +4 -0
  24. package/dist/services/controller-child-capacity.js +13 -0
  25. package/dist/services/controller-child-observer.js +18 -2
  26. package/dist/services/controller-execution.js +6 -6
  27. package/dist/services/external-work-launch.js +2 -4
  28. package/dist/services/frozen-engine.js +16 -8
  29. package/dist/services/managed-processes.js +31 -10
  30. package/dist/services/memory-init-native-capacity.js +65 -10
  31. package/dist/services/memory-init-preflight.js +21 -3
  32. package/dist/services/memory-init-project-bootstrap.js +3 -3
  33. package/dist/services/memory-init-start.js +6 -0
  34. package/dist/services/memory-init-worktrunk-capability.js +9 -3
  35. package/dist/services/merge-server.js +2 -4
  36. package/dist/services/{codex-capacity.js → provider-capacity.js} +34 -16
  37. package/dist/services/recovery-snapshot-database.js +1 -1
  38. package/dist/services/run-controller-adapter.js +1 -1
  39. package/dist/services/run-controller-recovery.js +2 -4
  40. package/dist/services/run-controller.js +59 -26
  41. package/dist/services/run-recovery-guard.js +3 -3
  42. package/dist/services/run-recovery-runtime.js +3 -3
  43. package/dist/services/runs.js +102 -13
  44. package/dist/services/vnext-code-review.js +1 -1
  45. package/dist/services/vnext-code.js +3 -3
  46. package/dist/services/vnext-fanout.js +3 -2
  47. package/dist/services/vnext-plan-review.js +1 -1
  48. package/dist/services/vnext-plan.js +3 -3
  49. package/dist/services/vnext-protocolize.js +20 -6
  50. package/dist/services/vnext-workspace-policy.js +7 -2
  51. package/package.json +1 -1
  52. package/dist/harness-runtime/lib/codex-capacity-policy.d.mts +0 -10
  53. package/dist/harness-runtime/lib/codex-capacity-policy.mjs +0 -102
@@ -1,17 +1,35 @@
1
1
  export const NATIVE_CHILD_CONTRACT_VERSION = "native-children@1";
2
2
  import { errorRecord } from "./operation-errors.mjs";
3
+ import { safeProviderError } from "./provider-overload-policy.mjs";
3
4
 
4
5
  function nativeFailure(code, message, exitCode, details) { return Object.assign(new Error(message), { code, exitCode, details }); }
5
6
 
7
+ /** Project a positively settled native tree, never infer it from missing activity. */
8
+ export function settledNativeTree(root, children, settled, rootSettled = false) {
9
+ if (settled !== true && rootSettled !== true || typeof root !== "string" || !root || !Array.isArray(children)) return {};
10
+ const ids = children.map(child => child.provider_session_id ?? child.session_id ?? child.childSessionId ?? child.sessionId);
11
+ if (ids.some(id => typeof id !== "string" || !id || id === root) || new Set(ids).size !== ids.length) return {};
12
+ const observations = [{ provider_session_id: root, status: "idle", settled: true }, ...children.map((child, index) => {
13
+ const status = String(child.status ?? child.state ?? "unknown").toLowerCase();
14
+ const terminal = settled === true || ["completed", "success", "succeeded", "failed", "cancelled", "canceled", "settled_by_root"].includes(status);
15
+ return { provider_session_id: ids[index], status: terminal ? "idle" : ["running", "active", "inprogress", "pending"].includes(status) ? "running" : "unknown", settled: terminal };
16
+ })];
17
+ return { tree_session_ids: [root, ...ids], unsettled_sessions: observations.filter(item => !item.settled).map(item => item.provider_session_id), tree_observations: observations };
18
+ }
19
+
6
20
  /** Retain typed failure facts, never provider headers, request bodies or free-form messages. */
7
21
  export function safeNativeFailure(value, depth = 0) {
8
22
  if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
9
23
  const result = {};
10
- for (const key of ["type", "code", "source", "reason", "errorPhase", "exceptionKind", "providerId", "modelId", "providerKind", "transport", "retryable", "willRetry", "statusCode", "resets_at", "retryAfter", "retry_after", "retryAfterMs", "resetAt"]) {
24
+ for (const key of ["type", "code", "source", "reason", "errorPhase", "exceptionKind", "providerId", "modelId", "providerKind", "transport", "retryable", "willRetry", "statusCode", "resets_at", "retryAfterMs", "resetAt"]) {
11
25
  const scalar = value[key];
12
26
  if (typeof scalar === "boolean" || typeof scalar === "number" && Number.isFinite(scalar)
13
27
  || typeof scalar === "string" && scalar.length <= 256 && /^[\w.:/ -]+$/.test(scalar)) result[key] = scalar;
14
28
  }
29
+ // Timing metadata is not an identifier: HTTP dates contain commas and an
30
+ // invalid value must reach the policy as invalid, not disappear into default backoff.
31
+ const timing = safeProviderError(value);
32
+ if (Object.hasOwn(timing, "retryAfter")) result.retryAfter = timing.retryAfter;
15
33
  if (typeof value.codexErrorInfo === "string" && /^[\w-]{1,128}$/.test(value.codexErrorInfo)) result.codexErrorInfo = value.codexErrorInfo;
16
34
  else if (value.codexErrorInfo && typeof value.codexErrorInfo.code === "string" && /^[\w-]{1,128}$/.test(value.codexErrorInfo.code)) result.codexErrorInfo = { code: value.codexErrorInfo.code };
17
35
  const attribution = depth === 0 && value.attribution && safeNativeFailure(value.attribution, 1);
@@ -55,6 +73,7 @@ export function normalizeNativeChildren(receipt, rootSessionId) {
55
73
  ...(text(value, ["terminal_turn_id"]) ? { terminal_turn_id: text(value, ["terminal_turn_id"]) } : {}),
56
74
  ...(text(value, ["terminal_observed_at"]) ? { terminal_observed_at: text(value, ["terminal_observed_at"]) } : {}),
57
75
  ...(value.provider_error !== undefined ? { provider_error: value.provider_error } : {}),
76
+ ...(["opencode.v2.session.execution", "agy.result", "acp.error_response", "droid.agent_turn_outcome", "grok.subagent.event"].includes(value.native_source) ? { native_source: value.native_source } : {}),
58
77
  ...(value.native_turn_items && typeof value.native_turn_items === "object" ? { native_turn_items: value.native_turn_items } : {}) });
59
78
  };
60
79
  for (const value of list(receipt.descendants)) add(value, "adapter.descendants");
@@ -1,20 +1,25 @@
1
1
  import path from "node:path";
2
2
  import { managedFlowEnv } from './managed-flow-env.mjs';
3
3
  import * as Plugin from "../vendor/opencode-plugin.mjs";
4
- import { hookBudget, invokeNativeHook } from "./native-hook-command.mjs";
4
+ import { hookBudget, invokeNativeHook, writeNativeHookFailure } from "./native-hook-command.mjs";
5
+ import { callDaemon } from "./dd-opencode-daemon.mjs";
6
+ import { errorRecord } from "./operation-errors.mjs";
5
7
 
6
8
  /** Launch-bound server plugin; never uses a process-global 'last Session'. */
7
- export function makeOpenCodePlugin(binding, invoke = invokeNativeHook) {
9
+ export function makeOpenCodePlugin(binding, invoke = invokeNativeHook, resolve = callDaemon) {
8
10
  return Plugin.define({ id: binding.ddFlowBin ? "dd-flow" : "dd-flow-security", async setup(ctx) {
9
11
  const participating = new Map();
10
12
  const key = event => JSON.stringify([binding.daemonId, event.sessionID, event.messageID, event.id]);
11
13
  const envelope = async (phase, event, budget, outcome) => {
12
- const info = await budget.run(() => ctx.session.get({ sessionID: event.sessionID }), "identity");
13
- if (!info?.id || info.id !== event.sessionID || !info.location?.directory) throw new Error("OpenCode Session identity is unavailable");
14
14
  const input = event.input;
15
15
  if (!input || typeof input !== "object" || typeof input.command !== "string") throw new Error("OpenCode shell input is unavailable");
16
- const cwd = path.resolve(info.location.directory, input.workdir ?? ".");
16
+ const identity = await budget.run(() => resolve(binding.stateDir, "hook.resolve", { sessionId: event.sessionID,
17
+ payload: { phase, message_id: event.messageID, tool_call_id: event.id, input } }, budget.remaining("identity")), "identity");
18
+ const info = identity?.nativeSession;
19
+ if (!identity?.nativeHookBinding || info?.id !== event.sessionID || !info.directory) throw Object.assign(new Error("OpenCode Session identity is unavailable"), { code: "native_hook_unproven" });
20
+ const cwd = path.resolve(info.directory, input.workdir ?? ".");
17
21
  return { schema_id: "dd-flow/opencode-tool-event@1", native_protocol: 2,
22
+ native_hook_binding: identity.nativeHookBinding,
18
23
  event_id: [binding.daemonId, event.sessionID, event.messageID, event.id].join(":"), phase,
19
24
  daemon_id: binding.daemonId, server_version: binding.version, plugin_sha256: process.env.DD_OPENCODE_PLUGIN_SHA256,
20
25
  session: { provider_session_id: info.id, parent_provider_session_id: info.parentID ?? null,
@@ -23,9 +28,26 @@ export function makeOpenCodePlugin(binding, invoke = invokeNativeHook) {
23
28
  profile: { provider: info.model?.providerID, model: info.model?.id, variant: info.model?.variant ?? "default", agent: event.agent },
24
29
  input: { ...input, workdir: cwd }, ...(outcome ? { outcome } : {}) };
25
30
  };
26
- const deliver = (payload, budget) => invoke({ bin: binding.ddFlowBin, nodeBin: binding.nodeBin,
27
- args: ["opencode", "event", "handle", "--project-root", binding.projectRoot, "--json"],
28
- payload, home: binding.ddFlowHome, timeoutMs: budget.remaining("admission", 15000) });
31
+ const deliver = async (payload, budget) => {
32
+ const identity = { nativeHookBinding: payload.native_hook_binding };
33
+ try {
34
+ const result = await invoke({ bin: binding.ddFlowBin, nodeBin: binding.nodeBin,
35
+ args: ["opencode", "event", "handle", "--project-root", binding.projectRoot, "--json"],
36
+ payload: { ...payload, native_hook_binding: identity.nativeHookBinding }, home: binding.ddFlowHome, timeoutMs: budget.remaining("admission", 15000) });
37
+ await budget.run(() => resolve(binding.stateDir, "hook.verify", { binding: identity.nativeHookBinding }, budget.remaining("generation")), "generation");
38
+ if (result?.permissionDecision === "deny" || result?.hookSpecificOutput?.permissionDecision === "deny") throw Object.assign(new Error("dd-flow denied native shell admission"), { code: "native_hook_rejected" });
39
+ return result;
40
+ } catch (error) {
41
+ error.details = { ...error.details, native_hook_binding: identity.nativeHookBinding, ...(payload.phase === "after" ? { effect: "unknown", tool_outcome: payload.outcome } : {}) };
42
+ try { await writeNativeHookFailure(binding.stateDir, identity.nativeHookBinding, error, hookBudget(1_000, null)); await resolve(binding.stateDir, "hook.failed", {}, 1_000); }
43
+ catch (storage) {
44
+ error.details.receipt_error = { code: storage.code ?? "hook_receipt_write_failed", message: storage.message };
45
+ try { await resolve(binding.stateDir, "hook.failed", { binding: identity.nativeHookBinding, failure: errorRecord(error) }, 1_000); }
46
+ catch (notify) { error.details.receipt_error.notification_error = notify.code ?? "hook_notification_failed"; }
47
+ }
48
+ throw error;
49
+ }
50
+ };
29
51
  await ctx.shell.hook("create.before", event => {
30
52
  Object.assign(event.env, managedFlowEnv(binding));
31
53
  delete event.env.OPENCODE_PASSWORD; delete event.env.OPENCODE_SERVER_PASSWORD;
@@ -37,7 +59,6 @@ export function makeOpenCodePlugin(binding, invoke = invokeNativeHook) {
37
59
  const budget = hookBudget(), payload = await envelope("before", event, budget);
38
60
  event.input = { ...event.input, workdir: payload.session.directory };
39
61
  const result = await deliver({ ...payload, input: event.input }, budget);
40
- if (result?.permissionDecision === "deny" || result?.hookSpecificOutput?.permissionDecision === "deny") throw new Error("dd-flow denied native shell admission");
41
62
  if (result?.observed) {
42
63
  participating.set(key(event), payload);
43
64
  await ctx.storage.set(key(event), JSON.parse(JSON.stringify(payload)));
@@ -0,0 +1,13 @@
1
+ export const CAPACITY_POLICY: "provider-overload-episode@1";
2
+ export const MAX_OVERLOAD_RETRIES: 2;
3
+ export const MAX_RETRY_AFTER_MS: 30000;
4
+ export function classifyProviderError(native: unknown): string;
5
+ export function safeProviderError(native: unknown): Record<string, unknown>;
6
+ export type CapacityRefusal = { policy: string; ordinal: number; session_id: string; turn_id: string; first_observed_at: string; native_terminal_at?: string; native_terminal_basis?: string };
7
+ export function parseCapacityOperation(id: string): { root: string; ordinal: number };
8
+ export function normalizeProviderFailure(error: unknown, sessionId?: string, turnId?: string): null | { code: string; session_id: string; turn_id: string; native_error: Record<string, unknown>; observed_at: unknown; terminal_at: unknown; native_turn_items: unknown; primary_error: { details?: Record<string, unknown> } };
9
+ export function terminalProviderOverload(error: unknown, sessionId?: string, turnId?: string): boolean;
10
+ export function nativeItemsSettled(items: unknown): boolean;
11
+ export function capacityBackoff(error: unknown, ordinal: number, now?: number, random?: () => number): number;
12
+ export function refusalObservation(error: unknown, ordinal: number, firstObservedAt: string): CapacityRefusal;
13
+ export function overloadExhausted(refusals: CapacityRefusal[], maxRetries?: number): null | { policy: string; reason: string; attempts: number; refusals: CapacityRefusal[] };
@@ -0,0 +1,138 @@
1
+ /** Pure policy shared by the controller and pinned-engine EVAL consumers. */
2
+ export const CAPACITY_POLICY = "provider-overload-episode@1";
3
+ export const MAX_OVERLOAD_RETRIES = 2;
4
+ export const MAX_RETRY_AFTER_MS = 30_000;
5
+
6
+ function policyError(code, message, details = {}) { return Object.assign(new Error(message), { code, details }); }
7
+
8
+ /** Only call on a native error envelope, never prompt or tool output. */
9
+ export function classifyProviderError(native) {
10
+ if (!native || typeof native !== "object") return "provider_failed";
11
+ const code = [native.code, native.type, native.codexErrorInfo?.code ?? native.codexErrorInfo].filter(value => typeof value === "string").join(" ");
12
+ const message = String(native.message ?? "");
13
+ const text = `${code} ${message}`;
14
+ const status = Number(native.status ?? native.statusCode ?? native.status_code ?? native.http_status);
15
+ if (status === 402 || /quota|usageLimitExceeded|sessionBudgetExceeded|billing|payment.required|balance.exhausted|insufficient.credit|resource_exhausted.*limit/i.test(text)) return "provider_quota_exhausted";
16
+ if (status === 401 || status === 403 || /provider_auth_failed|invalid.api.key|authentication|unauthorized|eligib|forbidden|access.denied|rejected.this.account/i.test(text)) return "provider_auth_failed";
17
+ if (/provider_capacity_transient|serverOverloaded|model.?capacity|at capacity|server.overload|model.overload|server.?busy/i.test(text) || /(?:^|\s)overloaded_error(?:\s|$)/i.test(code) || status === 503 || status === 529) return "provider_capacity_transient";
18
+ if (/rateLimitExceeded|rate.limit/i.test(text) && (native.retryAfter !== undefined || native.retry_after !== undefined || Object.keys(native.headers ?? {}).some(key => key.toLowerCase() === "retry-after"))) return "provider_rate_limited";
19
+ return status === 429 || /provider_limit_unknown|\b429\b|rateLimitExceeded|rate.limit/i.test(text) ? "provider_limit_unknown" : "provider_failed";
20
+ }
21
+
22
+ /** Public retry evidence excludes response bodies, credentials and other headers. */
23
+ export function safeProviderError(native) {
24
+ const result = { code: classifyProviderError(native) };
25
+ if (!native || typeof native !== "object") return result;
26
+ const status = native.status ?? native.statusCode ?? native.status_code ?? native.http_status;
27
+ if (Number.isInteger(Number(status))) result.status = Number(status);
28
+ const header = Object.entries(native.headers ?? {}).find(([key]) => key.toLowerCase() === "retry-after")?.[1];
29
+ const retry = header ?? native.retryAfter ?? native.retry_after;
30
+ if (retry !== undefined && retry !== null) result.retryAfter = typeof retry === "string" && retry.length <= 128 || typeof retry === "number" && Number.isFinite(retry) ? retry : "invalid-retry-after";
31
+ if (typeof native.willRetry === "boolean") result.willRetry = native.willRetry;
32
+ for (const key of ["reset_at", "resets_at", "resetAt", "resetsAt", "reset_time"]) {
33
+ const value = native[key];
34
+ if (typeof value === "string" && value.length <= 128 || typeof value === "number" && Number.isFinite(value)) result[key] = value;
35
+ }
36
+ return result;
37
+ }
38
+
39
+ export function parseCapacityOperation(id) {
40
+ if (typeof id !== "string" || !id) throw policyError("capacity_continuation_conflict", "Missing capacity operation identity");
41
+ const match = /^(.*):capacity:(\d+):session\.prompt$/.exec(id);
42
+ if (!match) return { root: id, ordinal: 0 };
43
+ const ordinal = Number(match[2]);
44
+ if (!Number.isSafeInteger(ordinal) || ordinal < 1 || String(ordinal) !== match[2]) throw policyError("capacity_continuation_conflict", "Capacity ordinal is not a canonical positive safe integer", { operation_id: id });
45
+ return { root: match[1], ordinal };
46
+ }
47
+
48
+ export function normalizeProviderFailure(error, expectedSessionId, expectedTurnId) {
49
+ let current = error;
50
+ for (let depth = 0; depth < 4 && current && typeof current === "object"; depth++) {
51
+ // The same continuation authority applies to a proven v2 execution
52
+ // refusal. Never classify transport loss or an active native retry here.
53
+ if (current.code === "provider_capacity_transient" || current.code === "provider_rate_limited") {
54
+ const details = current.details ?? {};
55
+ if (!["opencode.v2.session.execution", "agy.result", "acp.error_response", "droid.agent_turn_outcome", "grok.subagent.event"].includes(details.native_source) || details.terminal_status !== "failed"
56
+ || details.will_retry === true || details.native_retry_active === true || details.provider_error?.willRetry === true
57
+ || details.provider_error && !["provider_capacity_transient", "provider_rate_limited"].includes(classifyProviderError(details.provider_error))
58
+ || typeof details.provider_session_id !== "string" || !details.provider_session_id
59
+ || typeof details.turn_id !== "string" || !details.turn_id || details.native_turn_id !== details.turn_id
60
+ || expectedSessionId !== undefined && details.provider_session_id !== expectedSessionId
61
+ || expectedTurnId !== undefined && details.turn_id !== expectedTurnId) return null;
62
+ return { code: "serverOverloaded", session_id: details.provider_session_id, turn_id: details.turn_id,
63
+ native_error: details.provider_error ?? { type: details.native_error_type, status: details.http_status }, observed_at: details.observed_at ?? null,
64
+ terminal_at: null, terminal_basis: null, native_turn_items: details.native_turn_items ?? null, primary_error: current };
65
+ }
66
+ if (current.code === "turn_interrupted") {
67
+ const details = current.details ?? {}, native = details.provider_error;
68
+ const code = typeof native?.codexErrorInfo === "string" ? native.codexErrorInfo : native?.codexErrorInfo?.code;
69
+ if (native?.willRetry === true || details.terminal_status !== "failed" || typeof details.provider_session_id !== "string" || !details.provider_session_id
70
+ || expectedSessionId !== undefined && details.provider_session_id !== expectedSessionId
71
+ || typeof details.turn_id !== "string" || !details.turn_id || details.native_turn_id !== details.turn_id
72
+ || expectedTurnId !== undefined && details.turn_id !== expectedTurnId || typeof code !== "string") return null;
73
+ return { code, session_id: details.provider_session_id, turn_id: details.turn_id, native_error: native,
74
+ observed_at: details.observed_at ?? null, terminal_at: details.terminal_at ?? null, terminal_basis: details.terminal_basis ?? null,
75
+ native_turn_items: details.native_turn_items ?? null, primary_error: current };
76
+ }
77
+ // No arbitrary recursive cause search: an ownership/hook/lifecycle error wins.
78
+ if (current.code !== "harness_adapter_failed") return null;
79
+ current = current.details?.cause ?? current.cause;
80
+ }
81
+ return null;
82
+ }
83
+
84
+ export function terminalProviderOverload(error, expectedSessionId, expectedTurnId) {
85
+ const normalized = normalizeProviderFailure(error, expectedSessionId, expectedTurnId);
86
+ return normalized !== null && ["provider_capacity_transient", "provider_rate_limited"].includes(classifyProviderError(normalized.native_error));
87
+ }
88
+
89
+ export function nativeItemsSettled(items) {
90
+ return items?.observed === true && typeof items.possible_effects === "boolean" && items.pending !== true
91
+ && (items.possible_effects === false || items.pending === false);
92
+ }
93
+
94
+ export function capacityBackoff(error, ordinal, now = Date.now(), random = Math.random) {
95
+ if (!Number.isSafeInteger(ordinal) || ordinal < 1 || !Number.isFinite(now)) throw policyError("capacity_continuation_conflict", "Invalid capacity timing identity");
96
+ const native = normalizeProviderFailure(error)?.native_error ?? error?.details?.provider_error ?? error;
97
+ const header = Object.entries(native?.headers ?? {}).find(([key]) => key.toLowerCase() === "retry-after")?.[1];
98
+ const value = header ?? native?.retryAfter ?? native?.retry_after;
99
+ let retry = 0;
100
+ if (value !== undefined && value !== null) {
101
+ retry = typeof value === "number" || typeof value === "string" && /^\d+(\.\d+)?$/.test(value.trim()) ? Number(value) * 1000 : typeof value === "string" && /^\w{3}, \d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2} GMT$/.test(value.trim()) ? Math.max(0, Date.parse(value) - now) : NaN;
102
+ if (!Number.isFinite(retry) || retry < 0 || !Number.isFinite(now + retry) || now + retry > 8.64e15)
103
+ throw policyError("capacity_retry_metadata_invalid", "Retry-After cannot be represented safely", { retry_after: value });
104
+ }
105
+ const jitter = random();
106
+ if (!Number.isFinite(jitter) || jitter < 0 || jitter >= 1) throw policyError("capacity_retry_metadata_invalid", "Invalid retry jitter");
107
+ const wait = Math.max((ordinal === 1 ? 5_000 : 15_000) + Math.floor(jitter * 251), retry);
108
+ if (wait > MAX_RETRY_AFTER_MS) throw policyError("provider_unavailable", "Provider requested a delay beyond the retry budget", { policy: CAPACITY_POLICY, reason: "retry_after_exceeds_budget", retry_after_ms: retry, suggested_retry_at: new Date(now + retry).toISOString() });
109
+ return wait;
110
+ }
111
+
112
+ export function refusalObservation(error, ordinal, firstObservedAt) {
113
+ const failure = normalizeProviderFailure(error);
114
+ const observed = Date.parse(firstObservedAt);
115
+ if (!failure || !Number.isSafeInteger(ordinal) || ordinal < 0 || !Number.isFinite(observed))
116
+ throw policyError("capacity_continuation_conflict", "Refusal lacks an immutable native identity/time");
117
+ if (failure.observed_at !== null && (typeof failure.observed_at !== "string" || !Number.isFinite(Date.parse(failure.observed_at)) || Date.parse(failure.observed_at) !== observed))
118
+ throw policyError("capacity_continuation_conflict", "Native refusal observation time is invalid or changed");
119
+ const native = failure.terminal_at && Date.parse(failure.terminal_at);
120
+ return { policy: CAPACITY_POLICY, ordinal, session_id: failure.session_id, turn_id: failure.turn_id,
121
+ first_observed_at: firstObservedAt, ...(Number.isFinite(native) && typeof failure.terminal_basis === "string" && failure.terminal_basis ? { native_terminal_at: failure.terminal_at, native_terminal_basis: failure.terminal_basis } : {}) };
122
+ }
123
+
124
+ export function overloadExhausted(refusals, maxRetries = MAX_OVERLOAD_RETRIES) {
125
+ if (!Number.isSafeInteger(maxRetries) || maxRetries < 0 || maxRetries > MAX_OVERLOAD_RETRIES) throw policyError("capacity_continuation_conflict", "Invalid retry budget");
126
+ for (const [index, refusal] of refusals.entries()) {
127
+ const previous = refusals[index - 1];
128
+ if (typeof refusal.first_observed_at !== "string" || !Number.isFinite(Date.parse(refusal.first_observed_at))
129
+ || previous && Date.parse(refusal.first_observed_at) < Date.parse(previous.first_observed_at))
130
+ throw policyError("capacity_continuation_conflict", "Refusal observation clock reversed or is invalid");
131
+ }
132
+ for (const [index, refusal] of refusals.entries()) {
133
+ if (refusal.policy !== CAPACITY_POLICY) throw policyError("capacity_policy_mismatch", "Retained episode uses another retry policy");
134
+ const previous = refusals[index - 1];
135
+ if (refusal.ordinal !== index || previous && (refusal.session_id !== previous.session_id || refusals.slice(0, index).some(item => item.turn_id === refusal.turn_id))) throw policyError("capacity_continuation_conflict", "Refusal chain identity is inconsistent");
136
+ }
137
+ return refusals.length > maxRetries ? { policy: CAPACITY_POLICY, reason: "overload_retries_exhausted", attempts: refusals.length, refusals } : null;
138
+ }
@@ -23,6 +23,10 @@
23
23
  "publication_sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}
24
24
  }
25
25
  },
26
+ "workspace_route": {
27
+ "type": "object", "additionalProperties": false, "required": ["revision", "sha256"],
28
+ "properties": {"revision": {"type": "integer", "minimum": 1}, "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}}
29
+ },
26
30
  "execution_bundle": {
27
31
  "type": "object", "additionalProperties": false,
28
32
  "required": ["schema_id", "manifest_sha256", "descriptor_sha256", "admission_sha256", "flow_id"],
@@ -0,0 +1,13 @@
1
+ /** Only a proven new successful terminal Turn closes a child's retry episode. */
2
+ export function closeSuccessfulChildCapacity(state, children, parentId) {
3
+ const closed = [];
4
+ for (const child of children) {
5
+ const retained = state.child_capacity?.[child.session_id];
6
+ if (!retained || child.parent_session_id !== parentId || child.provenance === "parent_mismatch"
7
+ || child.status !== "completed" || !child.terminal_turn_id || child.terminal_turn_id === retained.turn_id)
8
+ continue;
9
+ delete state.child_capacity[child.session_id];
10
+ closed.push(child.session_id);
11
+ }
12
+ return closed;
13
+ }
@@ -12,6 +12,12 @@ export function retainedChildObserver(input) {
12
12
  const provider_error = safeNativeFailure(child.provider_error);
13
13
  return { session_id: child.session_id, parent_session_id: child.parent_session_id, status: "failed", source: child.source, terminal_turn_id: child.terminal_turn_id,
14
14
  ...(typeof child.terminal_observed_at === "string" && Number.isFinite(Date.parse(child.terminal_observed_at)) ? { terminal_observed_at: child.terminal_observed_at } : {}),
15
+ ...(child.native_source ? { native_source: child.native_source } : {}),
16
+ ...(child.native_turn_items ? { native_turn_items: {
17
+ ...(typeof child.native_turn_items.observed === "boolean" ? { observed: child.native_turn_items.observed } : {}),
18
+ ...(typeof child.native_turn_items.possible_effects === "boolean" ? { possible_effects: child.native_turn_items.possible_effects } : {}),
19
+ ...(typeof child.native_turn_items.pending === "boolean" ? { pending: child.native_turn_items.pending } : {})
20
+ } } : {}),
15
21
  ...(provider_error ? { provider_error } : {}) };
16
22
  };
17
23
  const accept = (children) => {
@@ -26,6 +32,8 @@ export function retainedChildObserver(input) {
26
32
  if (sameScope(observation.scope)) {
27
33
  if (Array.isArray(observation.children))
28
34
  accept(observation.children);
35
+ for (const key of observation.resolved ?? [])
36
+ retained.delete(key);
29
37
  if (Object.hasOwn(observation, "gap"))
30
38
  gap = observation.gap ?? null;
31
39
  }
@@ -35,23 +43,31 @@ export function retainedChildObserver(input) {
35
43
  return;
36
44
  input.assertOwner();
37
45
  const failed = children.filter(child => child.parent_session_id === input.scope.root_session_id && child.status === "failed" && child.terminal_turn_id && !retained.has(`${child.session_id}:${child.terminal_turn_id}`)).map(normalized);
38
- if (!failed.length && !gap)
46
+ // This is an authoritative current native snapshot. Older failed Turns
47
+ // remain in the journal/episode history, not in the current-outcome fence.
48
+ const resolved = [...retained].filter(([, old]) => children.some(child => child.parent_session_id === input.scope.root_session_id
49
+ && child.provenance !== "parent_mismatch" && child.session_id === old.session_id && ["completed", "failed"].includes(child.status)
50
+ && child.terminal_turn_id && child.terminal_turn_id !== old.terminal_turn_id)).map(([key]) => key);
51
+ if (!failed.length && !resolved.length && !gap)
39
52
  return;
40
53
  const fd = fs.openSync(input.file, "a", 0o600);
41
54
  try {
42
- fs.writeSync(fd, `${JSON.stringify({ scope: input.scope, children: failed, gap: null })}\n`);
55
+ fs.writeSync(fd, `${JSON.stringify({ scope: input.scope, children: failed, ...(resolved.length ? { resolved } : {}), gap: null })}\n`);
43
56
  fs.fsyncSync(fd);
44
57
  }
45
58
  finally {
46
59
  fs.closeSync(fd);
47
60
  }
48
61
  accept(failed);
62
+ for (const key of resolved)
63
+ retained.delete(key);
49
64
  gap = null;
50
65
  };
51
66
  return {
52
67
  failed: () => [...retained.values()],
53
68
  gap: () => gap,
54
69
  drain: () => pending ?? Promise.resolve(),
70
+ observeSettledChildren: complete,
55
71
  completePostTurn(receipt) {
56
72
  const evidence = receipt.evidence;
57
73
  const tree = evidence?.subagents;
@@ -27,13 +27,13 @@ export function genericControllerRouting(execution) {
27
27
  Object.freeze(policy);
28
28
  return {
29
29
  kind: "descriptor", policy, profiles, mergeMode: null,
30
- transition(stage, fromStage) {
30
+ transition(stage, fromStage, _currentRun) {
31
31
  const selected = execution.stage(stage);
32
32
  if (fromStage)
33
33
  execution.stage(fromStage);
34
34
  return { ...resolveExecutionPolicy({ ddFlowHome: "", policy, profiles, stage }), session_mode: selected.session_mode };
35
35
  },
36
- cwd(stage) { execution.stage(stage); return path.resolve(snapshot.scope.workspace); }
36
+ cwd(stage, _mergeTargetWorkspace, _currentRun) { execution.stage(stage); return path.resolve(snapshot.scope.workspace); }
37
37
  };
38
38
  }
39
39
  /** Restore execution from RUN authority, never from a generic manifest copy.
@@ -77,12 +77,12 @@ export function selectControllerExecution(context, run, manifest) {
77
77
  return {
78
78
  kind: "sdlc", policy, profiles, bundle: bundle ?? undefined, binding: undefined, legacy,
79
79
  mergeMode: legacy.settings.merge_mode,
80
- transition(stage, fromStage) {
80
+ transition(stage, fromStage, currentRun = run) {
81
81
  return resolveStageTransition({ ddFlowHome: context.ddFlowHome, policy, profiles, ...(fromStage ? { fromStage } : {}), stage,
82
- projectRoot: run.project_root, workspaceRoot: run.workspace_root, stageSessionMode: legacy.settings.stage_session_mode, mergeMode: legacy.settings.merge_mode });
82
+ projectRoot: currentRun.project_root, workspaceRoot: currentRun.workspace_root, stageSessionMode: legacy.settings.stage_session_mode, mergeMode: legacy.settings.merge_mode });
83
83
  },
84
- cwd(stage, mergeTargetWorkspace) {
85
- return stageExecutionCwd({ stage, projectRoot: run.project_root, workspaceRoot: run.workspace_root, ...(mergeTargetWorkspace ? { mergeTargetWorkspace } : {}) });
84
+ cwd(stage, mergeTargetWorkspace, currentRun = run) {
85
+ return stageExecutionCwd({ stage, projectRoot: currentRun.project_root, workspaceRoot: currentRun.workspace_root, ...(mergeTargetWorkspace ? { mergeTargetWorkspace } : {}) });
86
86
  }
87
87
  };
88
88
  }
@@ -16,7 +16,7 @@ import { reviewExecutionCopyHash, reviewSourceFilter, reviewSourceHash, reviewSo
16
16
  import { workspaceGitFiles } from "./workspace-files.js";
17
17
  import { qualifyAgyHooks } from "../harness-runtime/lib/dd-agy-daemon.mjs";
18
18
  import { lifecycleRetryInstruction } from "../harness-runtime/lib/delegation-instructions.mjs";
19
- import { continueCodexCapacity } from "./codex-capacity.js";
19
+ import { continueProviderCapacity } from "./provider-capacity.js";
20
20
  import { errorRecord } from "./run-controller-state.js";
21
21
  import { readFlowRunExecutionBundle } from "./runs.js";
22
22
  import { stageWorkIds } from "./stage-work-graph.js";
@@ -190,8 +190,6 @@ export async function launchExternalWork(context, input) {
190
190
  prompted = await call(["session", "prompt", "--session-id", sessionId, "--prompt-file", promptFile, ...common], null);
191
191
  }
192
192
  catch (error) {
193
- if (!["codex-desktop", "opencode-server"].includes(runtimeHarness(resolved.worker)))
194
- throw error;
195
193
  const retained = context.db.get("SELECT adapter_receipt_json FROM work_launches WHERE operation_id = ?", [operationId]);
196
194
  const capacityIntents = retained?.adapter_receipt_json ? JSON.parse(retained.adapter_receipt_json).capacity_intents ?? [] : [];
197
195
  const capacityFailures = retained?.adapter_receipt_json ? JSON.parse(retained.adapter_receipt_json).capacity_failures ?? [] : [];
@@ -200,7 +198,7 @@ export async function launchExternalWork(context, input) {
200
198
  if (changed.changes !== 1)
201
199
  throw new AppError("capacity_continuation_blocked", "External Work launch owner changed", 1);
202
200
  };
203
- const resumed = await continueCodexCapacity({ error, sessionId, operationId: promptOperationId, allowSettledDescendants: true,
201
+ const resumed = await continueProviderCapacity({ error, sessionId, operationId: promptOperationId, allowSettledDescendants: true,
204
202
  ...(input.signal ? { signal: input.signal } : {}),
205
203
  continuationText: `Continue only Work ${work.work_id} in RUN ${work.run_id}, Stage ${stage ?? "current"}, in the same Session. Inspect its current durable state and the retained Work packet. Never repeat Work start or a completed tool, accepted answer, check or lifecycle command. Use only the current exact issued successor and finish at the original Work boundary.`,
206
204
  retainedIntents: () => capacityIntents,
@@ -10,8 +10,14 @@ import { shellArgument } from "./runtime-command.js";
10
10
  import { hybridDigest } from "./hybrid-contracts.js";
11
11
  import { isMemoryInitResourceStorageFacts } from "./memory-init-resource-storage.js";
12
12
  import { memoryInitStartupInputSchemas } from "./memory-init-startup-custody.js";
13
- function storageProbeRefuse(reason) {
14
- throw new AppError("frozen_engine_storage_incompatible", "Selected frozen engine cannot admit memory-init storage", 1, { reason, effect: "no_effect" });
13
+ import { SETUP_STARTUP_MS } from "../harness-runtime/lib/setup-progress.mjs";
14
+ function storageProbeRefuse(reason, probe) {
15
+ throw new AppError("frozen_engine_storage_incompatible", "Selected frozen engine cannot admit memory-init storage", 1, { reason, effect: "no_effect", ...(probe ? { probe } : {}) });
16
+ }
17
+ function probeFailure(result) {
18
+ const code = result.error?.code ?? null;
19
+ return { status: result.status, signal: result.signal, error_code: code, timed_out: code === "ETIMEDOUT",
20
+ timeout_ms: SETUP_STARTUP_MS, stdout_bytes: result.stdout?.length ?? null, stderr_bytes: result.stderr?.length ?? null };
15
21
  }
16
22
  const hash = (bytes) => crypto.createHash("sha256").update(bytes).digest("hex");
17
23
  function refuse(reason) {
@@ -113,8 +119,8 @@ export class AdmittedFrozenEngine {
113
119
  /** Qualify actual descriptor handlers in this immutable executable, not the
114
120
  * invoking CLI catalog. No storage or productive operation is admitted. */
115
121
  admitMemoryInitFlow(source, startupInputSchema) {
116
- const fail = (reason) => {
117
- throw new AppError("frozen_engine_flow_incompatible", "Selected frozen engine cannot admit memory-init handlers", 1, { reason, phase: "prepare", effect: "no_effect" });
122
+ const fail = (reason, probe) => {
123
+ throw new AppError("frozen_engine_flow_incompatible", "Selected frozen engine cannot admit memory-init handlers", 1, { reason, phase: "prepare", effect: "no_effect", ...(probe ? { probe } : {}) });
118
124
  };
119
125
  const revalidate = () => {
120
126
  try {
@@ -147,9 +153,11 @@ export class AdmittedFrozenEngine {
147
153
  revalidate();
148
154
  const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !["NODE_OPTIONS", "NODE_PATH"].includes(key) && !key.startsWith("DD_FLOW_")));
149
155
  const result = spawnSync(this.nodeExecutable, [this.entrypoint, "runtime", "flow", "admit", "--operation", "memory-init", "--json"], { input, cwd: this.#manifest.snapshot_root, env: { ...env, DD_FLOW_ENGINE_MODE: "1", DD_FLOW_ENGINE_HOME: this.#manifest.snapshot_root },
150
- timeout: 15000, maxBuffer: 1024 * 1024, shell: false, windowsHide: true });
156
+ // This is cold startup of a selected CLI, not an already-loaded handler.
157
+ // Admission is bounded local validation; no provider operation is started.
158
+ timeout: SETUP_STARTUP_MS, maxBuffer: 1024 * 1024, shell: false, windowsHide: true });
151
159
  if (result.error || result.signal || result.status !== 0)
152
- fail("selected_engine_probe_failed");
160
+ fail("selected_engine_probe_failed", probeFailure(result));
153
161
  revalidate();
154
162
  let response;
155
163
  try {
@@ -182,9 +190,9 @@ export class AdmittedFrozenEngine {
182
190
  revalidateEngine();
183
191
  const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !["NODE_OPTIONS", "NODE_PATH"].includes(key)));
184
192
  const result = spawnSync(this.nodeExecutable, [this.entrypoint, "runtime", "storage", "admit", "--operation", "memory-init", "--json"], { env: { ...env, DD_FLOW_HOME: home, DD_FLOW_RESOURCE_HOME: resourceHome, DD_FLOW_ENGINE_MODE: "1", DD_FLOW_ENGINE_HOME: this.#manifest.snapshot_root },
185
- timeout: 15000, maxBuffer: 1024 * 1024, shell: false, windowsHide: true });
193
+ timeout: SETUP_STARTUP_MS, maxBuffer: 1024 * 1024, shell: false, windowsHide: true });
186
194
  if (result.error || result.signal || result.status !== 0)
187
- storageProbeRefuse("selected_engine_probe_failed");
195
+ storageProbeRefuse("selected_engine_probe_failed", probeFailure(result));
188
196
  revalidateEngine();
189
197
  let response;
190
198
  try {
@@ -5,6 +5,7 @@ import { spawnSync } from "node:child_process";
5
5
  import net from "node:net";
6
6
  import { getResourceDatabase } from "../storage/database.js";
7
7
  import { AppError, errorRecord } from "../shared/errors.js";
8
+ import { maintenanceRetryable } from "../harness-runtime/lib/lease-renewal.mjs";
8
9
  export function managedProcessBinding(record, home) {
9
10
  return { resource_home: home, process_id: record.id, registration_sha256: crypto.createHash("sha256").update(JSON.stringify([home, record.id, record.kind, record.owner_id, record.lease_token, record.pid, record.pid_started_at, record.project_id, record.run_id, record.work_id, record.check_id, record.operation_id, record.metadata_json])).digest("hex") };
10
11
  }
@@ -203,17 +204,28 @@ export function heartbeatManagedProcess(context, input) {
203
204
  }
204
205
  export function renewManagedProcess(context, input) {
205
206
  prepareManagedProcessHeartbeat(context, input);
206
- if (context.resourceDatabaseMode === "prepared_existing")
207
- registry(context);
208
- const reader = getResourceDatabase(resourceHome(context), "read_existing", context.databaseDeadlineAt);
207
+ let db;
209
208
  try {
210
- if (!reader.get("SELECT 1 FROM managed_processes WHERE id=? AND lease_token=? AND state IN ('starting','running','stopping')", [input.id, input.leaseToken]))
211
- return { owned: false, process_id: input.id };
209
+ if (context.resourceDatabaseMode === "prepared_existing")
210
+ registry(context);
211
+ const reader = getResourceDatabase(resourceHome(context), "read_existing", context.databaseDeadlineAt);
212
+ try {
213
+ if (!reader.get("SELECT 1 FROM managed_processes WHERE id=? AND lease_token=? AND state IN ('starting','running','stopping')", [input.id, input.leaseToken]))
214
+ return { owned: false, process_id: input.id };
215
+ }
216
+ finally {
217
+ reader.close?.();
218
+ }
219
+ db = registry(context);
212
220
  }
213
- finally {
214
- reader.close?.();
221
+ catch (error) {
222
+ // No lease mutation has started. The storage helper cannot know that
223
+ // operation boundary and deliberately defaults to an unknown effect.
224
+ if (error instanceof AppError && error.code === "process_heartbeat_timeout"
225
+ && maintenanceRetryable({ ...error, details: { ...error.details, effect: "no_effect" } }))
226
+ throw new AppError(error.code, error.message, error.exitCode, { ...error.details, effect: "no_effect", maintenance_phase: "renewal_prepare" });
227
+ throw error;
215
228
  }
216
- const db = registry(context);
217
229
  return db.writeTransaction(() => {
218
230
  const now = context.now();
219
231
  if (!db.get("SELECT 1 FROM managed_processes WHERE id=? AND lease_token=? AND state IN ('starting','running','stopping')", [input.id, input.leaseToken]))
@@ -228,8 +240,9 @@ export function renewManagedProcess(context, input) {
228
240
  /** Read-only lease check for a caller already holding a RUN writer. Renewal
229
241
  * must happen before entering that writer, never from this assertion. */
230
242
  export function assertManagedProcessLease(context, input) {
231
- const reader = getResourceDatabase(resourceHome(context), "read_isolated", context.databaseDeadlineAt);
243
+ let reader;
232
244
  try {
245
+ reader = getResourceDatabase(resourceHome(context), "read_isolated", context.databaseDeadlineAt);
233
246
  const current = reader.get("SELECT pid, lease_expires_at, state FROM managed_processes WHERE id = ? AND lease_token = ?", [input.id, input.leaseToken]);
234
247
  if (!current || current.pid !== input.pid || !["starting", "running"].includes(current.state)) {
235
248
  throw new AppError("controller_lease_lost", "Managed owner lease changed before productive dispatch", 1, { process_id: input.id });
@@ -237,8 +250,16 @@ export function assertManagedProcessLease(context, input) {
237
250
  if (!Number.isFinite(Date.parse(current.lease_expires_at)) || Date.parse(current.lease_expires_at) <= Date.now())
238
251
  throw new AppError("controller_lease_unconfirmed", "Managed owner requires renewal before productive dispatch", 1, { process_id: input.id, reason: "renewal_required" });
239
252
  }
253
+ catch (error) {
254
+ const failure = errorRecord(error);
255
+ // This assertion only reads. Unavailable observation is not evidence of
256
+ // owner loss; the caller must reconfirm outside its RUN writer.
257
+ if (maintenanceRetryable({ ...failure, details: { ...failure.details, effect: "no_effect" } }))
258
+ throw new AppError("controller_lease_unconfirmed", "Managed owner observation requires confirmation before productive dispatch", 1, { process_id: input.id, reason: "observation_unavailable", cause: failure });
259
+ throw error;
260
+ }
240
261
  finally {
241
- reader.close?.();
262
+ reader?.close?.();
242
263
  }
243
264
  }
244
265
  export function prepareManagedProcessFinish(context, input) {