@deksden-com/dd-flow-cli 0.9.0-beta.138 → 0.9.0-beta.139
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/build-info.json +6 -6
- package/dist/cli/runtime-process.js +1 -1
- package/dist/cli.js +2 -0
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +49 -20
- package/dist/harness-runtime/lib/dd-agy.mjs +16 -5
- package/dist/harness-runtime/lib/dd-codex-daemon.mjs +1 -1
- package/dist/harness-runtime/lib/dd-codex.mjs +2 -2
- package/dist/harness-runtime/lib/dd-droid.mjs +30 -8
- package/dist/harness-runtime/lib/dd-grok-daemon.mjs +7 -1
- package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +95 -12
- package/dist/harness-runtime/lib/dd-opencode.mjs +51 -20
- package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +13 -1
- package/dist/harness-runtime/lib/dd-zcode.mjs +17 -4
- package/dist/harness-runtime/lib/delegation-instructions.d.mts +1 -0
- package/dist/harness-runtime/lib/delegation-instructions.mjs +11 -0
- package/dist/harness-runtime/lib/lease-renewal.mjs +2 -1
- package/dist/harness-runtime/lib/native-children.d.mts +2 -0
- package/dist/harness-runtime/lib/native-children.mjs +20 -1
- package/dist/harness-runtime/lib/opencode-plugin.mjs +30 -9
- package/dist/harness-runtime/lib/provider-overload-policy.d.mts +13 -0
- package/dist/harness-runtime/lib/provider-overload-policy.mjs +138 -0
- package/dist/schemas/flow-run.schema.json +4 -0
- package/dist/services/controller-child-capacity.js +13 -0
- package/dist/services/controller-child-observer.js +18 -2
- package/dist/services/controller-execution.js +6 -6
- package/dist/services/external-work-launch.js +2 -4
- package/dist/services/frozen-engine.js +16 -8
- package/dist/services/managed-processes.js +31 -10
- package/dist/services/memory-init-native-capacity.js +65 -10
- package/dist/services/memory-init-preflight.js +21 -3
- package/dist/services/memory-init-project-bootstrap.js +3 -3
- package/dist/services/memory-init-start.js +6 -0
- package/dist/services/memory-init-worktrunk-capability.js +9 -3
- package/dist/services/merge-server.js +2 -4
- package/dist/services/{codex-capacity.js → provider-capacity.js} +34 -16
- package/dist/services/recovery-snapshot-database.js +1 -1
- package/dist/services/run-controller-adapter.js +1 -1
- package/dist/services/run-controller-recovery.js +2 -4
- package/dist/services/run-controller.js +59 -26
- package/dist/services/run-recovery-guard.js +3 -3
- package/dist/services/run-recovery-runtime.js +3 -3
- package/dist/services/runs.js +102 -13
- package/dist/services/vnext-code-review.js +1 -1
- package/dist/services/vnext-code.js +3 -3
- package/dist/services/vnext-fanout.js +3 -2
- package/dist/services/vnext-plan-review.js +1 -1
- package/dist/services/vnext-plan.js +3 -3
- package/dist/services/vnext-protocolize.js +20 -6
- package/dist/services/vnext-workspace-policy.js +7 -2
- package/package.json +1 -1
- package/dist/harness-runtime/lib/codex-capacity-policy.d.mts +0 -10
- package/dist/harness-runtime/lib/codex-capacity-policy.mjs +0 -102
|
@@ -1,17 +1,35 @@
|
|
|
1
1
|
export const NATIVE_CHILD_CONTRACT_VERSION = "native-children@1";
|
|
2
2
|
import { errorRecord } from "./operation-errors.mjs";
|
|
3
|
+
import { safeProviderError } from "./provider-overload-policy.mjs";
|
|
3
4
|
|
|
4
5
|
function nativeFailure(code, message, exitCode, details) { return Object.assign(new Error(message), { code, exitCode, details }); }
|
|
5
6
|
|
|
7
|
+
/** Project a positively settled native tree, never infer it from missing activity. */
|
|
8
|
+
export function settledNativeTree(root, children, settled, rootSettled = false) {
|
|
9
|
+
if (settled !== true && rootSettled !== true || typeof root !== "string" || !root || !Array.isArray(children)) return {};
|
|
10
|
+
const ids = children.map(child => child.provider_session_id ?? child.session_id ?? child.childSessionId ?? child.sessionId);
|
|
11
|
+
if (ids.some(id => typeof id !== "string" || !id || id === root) || new Set(ids).size !== ids.length) return {};
|
|
12
|
+
const observations = [{ provider_session_id: root, status: "idle", settled: true }, ...children.map((child, index) => {
|
|
13
|
+
const status = String(child.status ?? child.state ?? "unknown").toLowerCase();
|
|
14
|
+
const terminal = settled === true || ["completed", "success", "succeeded", "failed", "cancelled", "canceled", "settled_by_root"].includes(status);
|
|
15
|
+
return { provider_session_id: ids[index], status: terminal ? "idle" : ["running", "active", "inprogress", "pending"].includes(status) ? "running" : "unknown", settled: terminal };
|
|
16
|
+
})];
|
|
17
|
+
return { tree_session_ids: [root, ...ids], unsettled_sessions: observations.filter(item => !item.settled).map(item => item.provider_session_id), tree_observations: observations };
|
|
18
|
+
}
|
|
19
|
+
|
|
6
20
|
/** Retain typed failure facts, never provider headers, request bodies or free-form messages. */
|
|
7
21
|
export function safeNativeFailure(value, depth = 0) {
|
|
8
22
|
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
9
23
|
const result = {};
|
|
10
|
-
for (const key of ["type", "code", "source", "reason", "errorPhase", "exceptionKind", "providerId", "modelId", "providerKind", "transport", "retryable", "willRetry", "statusCode", "resets_at", "
|
|
24
|
+
for (const key of ["type", "code", "source", "reason", "errorPhase", "exceptionKind", "providerId", "modelId", "providerKind", "transport", "retryable", "willRetry", "statusCode", "resets_at", "retryAfterMs", "resetAt"]) {
|
|
11
25
|
const scalar = value[key];
|
|
12
26
|
if (typeof scalar === "boolean" || typeof scalar === "number" && Number.isFinite(scalar)
|
|
13
27
|
|| typeof scalar === "string" && scalar.length <= 256 && /^[\w.:/ -]+$/.test(scalar)) result[key] = scalar;
|
|
14
28
|
}
|
|
29
|
+
// Timing metadata is not an identifier: HTTP dates contain commas and an
|
|
30
|
+
// invalid value must reach the policy as invalid, not disappear into default backoff.
|
|
31
|
+
const timing = safeProviderError(value);
|
|
32
|
+
if (Object.hasOwn(timing, "retryAfter")) result.retryAfter = timing.retryAfter;
|
|
15
33
|
if (typeof value.codexErrorInfo === "string" && /^[\w-]{1,128}$/.test(value.codexErrorInfo)) result.codexErrorInfo = value.codexErrorInfo;
|
|
16
34
|
else if (value.codexErrorInfo && typeof value.codexErrorInfo.code === "string" && /^[\w-]{1,128}$/.test(value.codexErrorInfo.code)) result.codexErrorInfo = { code: value.codexErrorInfo.code };
|
|
17
35
|
const attribution = depth === 0 && value.attribution && safeNativeFailure(value.attribution, 1);
|
|
@@ -55,6 +73,7 @@ export function normalizeNativeChildren(receipt, rootSessionId) {
|
|
|
55
73
|
...(text(value, ["terminal_turn_id"]) ? { terminal_turn_id: text(value, ["terminal_turn_id"]) } : {}),
|
|
56
74
|
...(text(value, ["terminal_observed_at"]) ? { terminal_observed_at: text(value, ["terminal_observed_at"]) } : {}),
|
|
57
75
|
...(value.provider_error !== undefined ? { provider_error: value.provider_error } : {}),
|
|
76
|
+
...(["opencode.v2.session.execution", "agy.result", "acp.error_response", "droid.agent_turn_outcome", "grok.subagent.event"].includes(value.native_source) ? { native_source: value.native_source } : {}),
|
|
58
77
|
...(value.native_turn_items && typeof value.native_turn_items === "object" ? { native_turn_items: value.native_turn_items } : {}) });
|
|
59
78
|
};
|
|
60
79
|
for (const value of list(receipt.descendants)) add(value, "adapter.descendants");
|
|
@@ -1,20 +1,25 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
2
|
import { managedFlowEnv } from './managed-flow-env.mjs';
|
|
3
3
|
import * as Plugin from "../vendor/opencode-plugin.mjs";
|
|
4
|
-
import { hookBudget, invokeNativeHook } from "./native-hook-command.mjs";
|
|
4
|
+
import { hookBudget, invokeNativeHook, writeNativeHookFailure } from "./native-hook-command.mjs";
|
|
5
|
+
import { callDaemon } from "./dd-opencode-daemon.mjs";
|
|
6
|
+
import { errorRecord } from "./operation-errors.mjs";
|
|
5
7
|
|
|
6
8
|
/** Launch-bound server plugin; never uses a process-global 'last Session'. */
|
|
7
|
-
export function makeOpenCodePlugin(binding, invoke = invokeNativeHook) {
|
|
9
|
+
export function makeOpenCodePlugin(binding, invoke = invokeNativeHook, resolve = callDaemon) {
|
|
8
10
|
return Plugin.define({ id: binding.ddFlowBin ? "dd-flow" : "dd-flow-security", async setup(ctx) {
|
|
9
11
|
const participating = new Map();
|
|
10
12
|
const key = event => JSON.stringify([binding.daemonId, event.sessionID, event.messageID, event.id]);
|
|
11
13
|
const envelope = async (phase, event, budget, outcome) => {
|
|
12
|
-
const info = await budget.run(() => ctx.session.get({ sessionID: event.sessionID }), "identity");
|
|
13
|
-
if (!info?.id || info.id !== event.sessionID || !info.location?.directory) throw new Error("OpenCode Session identity is unavailable");
|
|
14
14
|
const input = event.input;
|
|
15
15
|
if (!input || typeof input !== "object" || typeof input.command !== "string") throw new Error("OpenCode shell input is unavailable");
|
|
16
|
-
const
|
|
16
|
+
const identity = await budget.run(() => resolve(binding.stateDir, "hook.resolve", { sessionId: event.sessionID,
|
|
17
|
+
payload: { phase, message_id: event.messageID, tool_call_id: event.id, input } }, budget.remaining("identity")), "identity");
|
|
18
|
+
const info = identity?.nativeSession;
|
|
19
|
+
if (!identity?.nativeHookBinding || info?.id !== event.sessionID || !info.directory) throw Object.assign(new Error("OpenCode Session identity is unavailable"), { code: "native_hook_unproven" });
|
|
20
|
+
const cwd = path.resolve(info.directory, input.workdir ?? ".");
|
|
17
21
|
return { schema_id: "dd-flow/opencode-tool-event@1", native_protocol: 2,
|
|
22
|
+
native_hook_binding: identity.nativeHookBinding,
|
|
18
23
|
event_id: [binding.daemonId, event.sessionID, event.messageID, event.id].join(":"), phase,
|
|
19
24
|
daemon_id: binding.daemonId, server_version: binding.version, plugin_sha256: process.env.DD_OPENCODE_PLUGIN_SHA256,
|
|
20
25
|
session: { provider_session_id: info.id, parent_provider_session_id: info.parentID ?? null,
|
|
@@ -23,9 +28,26 @@ export function makeOpenCodePlugin(binding, invoke = invokeNativeHook) {
|
|
|
23
28
|
profile: { provider: info.model?.providerID, model: info.model?.id, variant: info.model?.variant ?? "default", agent: event.agent },
|
|
24
29
|
input: { ...input, workdir: cwd }, ...(outcome ? { outcome } : {}) };
|
|
25
30
|
};
|
|
26
|
-
const deliver = (payload, budget) =>
|
|
27
|
-
|
|
28
|
-
|
|
31
|
+
const deliver = async (payload, budget) => {
|
|
32
|
+
const identity = { nativeHookBinding: payload.native_hook_binding };
|
|
33
|
+
try {
|
|
34
|
+
const result = await invoke({ bin: binding.ddFlowBin, nodeBin: binding.nodeBin,
|
|
35
|
+
args: ["opencode", "event", "handle", "--project-root", binding.projectRoot, "--json"],
|
|
36
|
+
payload: { ...payload, native_hook_binding: identity.nativeHookBinding }, home: binding.ddFlowHome, timeoutMs: budget.remaining("admission", 15000) });
|
|
37
|
+
await budget.run(() => resolve(binding.stateDir, "hook.verify", { binding: identity.nativeHookBinding }, budget.remaining("generation")), "generation");
|
|
38
|
+
if (result?.permissionDecision === "deny" || result?.hookSpecificOutput?.permissionDecision === "deny") throw Object.assign(new Error("dd-flow denied native shell admission"), { code: "native_hook_rejected" });
|
|
39
|
+
return result;
|
|
40
|
+
} catch (error) {
|
|
41
|
+
error.details = { ...error.details, native_hook_binding: identity.nativeHookBinding, ...(payload.phase === "after" ? { effect: "unknown", tool_outcome: payload.outcome } : {}) };
|
|
42
|
+
try { await writeNativeHookFailure(binding.stateDir, identity.nativeHookBinding, error, hookBudget(1_000, null)); await resolve(binding.stateDir, "hook.failed", {}, 1_000); }
|
|
43
|
+
catch (storage) {
|
|
44
|
+
error.details.receipt_error = { code: storage.code ?? "hook_receipt_write_failed", message: storage.message };
|
|
45
|
+
try { await resolve(binding.stateDir, "hook.failed", { binding: identity.nativeHookBinding, failure: errorRecord(error) }, 1_000); }
|
|
46
|
+
catch (notify) { error.details.receipt_error.notification_error = notify.code ?? "hook_notification_failed"; }
|
|
47
|
+
}
|
|
48
|
+
throw error;
|
|
49
|
+
}
|
|
50
|
+
};
|
|
29
51
|
await ctx.shell.hook("create.before", event => {
|
|
30
52
|
Object.assign(event.env, managedFlowEnv(binding));
|
|
31
53
|
delete event.env.OPENCODE_PASSWORD; delete event.env.OPENCODE_SERVER_PASSWORD;
|
|
@@ -37,7 +59,6 @@ export function makeOpenCodePlugin(binding, invoke = invokeNativeHook) {
|
|
|
37
59
|
const budget = hookBudget(), payload = await envelope("before", event, budget);
|
|
38
60
|
event.input = { ...event.input, workdir: payload.session.directory };
|
|
39
61
|
const result = await deliver({ ...payload, input: event.input }, budget);
|
|
40
|
-
if (result?.permissionDecision === "deny" || result?.hookSpecificOutput?.permissionDecision === "deny") throw new Error("dd-flow denied native shell admission");
|
|
41
62
|
if (result?.observed) {
|
|
42
63
|
participating.set(key(event), payload);
|
|
43
64
|
await ctx.storage.set(key(event), JSON.parse(JSON.stringify(payload)));
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export const CAPACITY_POLICY: "provider-overload-episode@1";
|
|
2
|
+
export const MAX_OVERLOAD_RETRIES: 2;
|
|
3
|
+
export const MAX_RETRY_AFTER_MS: 30000;
|
|
4
|
+
export function classifyProviderError(native: unknown): string;
|
|
5
|
+
export function safeProviderError(native: unknown): Record<string, unknown>;
|
|
6
|
+
export type CapacityRefusal = { policy: string; ordinal: number; session_id: string; turn_id: string; first_observed_at: string; native_terminal_at?: string; native_terminal_basis?: string };
|
|
7
|
+
export function parseCapacityOperation(id: string): { root: string; ordinal: number };
|
|
8
|
+
export function normalizeProviderFailure(error: unknown, sessionId?: string, turnId?: string): null | { code: string; session_id: string; turn_id: string; native_error: Record<string, unknown>; observed_at: unknown; terminal_at: unknown; native_turn_items: unknown; primary_error: { details?: Record<string, unknown> } };
|
|
9
|
+
export function terminalProviderOverload(error: unknown, sessionId?: string, turnId?: string): boolean;
|
|
10
|
+
export function nativeItemsSettled(items: unknown): boolean;
|
|
11
|
+
export function capacityBackoff(error: unknown, ordinal: number, now?: number, random?: () => number): number;
|
|
12
|
+
export function refusalObservation(error: unknown, ordinal: number, firstObservedAt: string): CapacityRefusal;
|
|
13
|
+
export function overloadExhausted(refusals: CapacityRefusal[], maxRetries?: number): null | { policy: string; reason: string; attempts: number; refusals: CapacityRefusal[] };
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
/** Pure policy shared by the controller and pinned-engine EVAL consumers. */
|
|
2
|
+
export const CAPACITY_POLICY = "provider-overload-episode@1";
|
|
3
|
+
export const MAX_OVERLOAD_RETRIES = 2;
|
|
4
|
+
export const MAX_RETRY_AFTER_MS = 30_000;
|
|
5
|
+
|
|
6
|
+
function policyError(code, message, details = {}) { return Object.assign(new Error(message), { code, details }); }
|
|
7
|
+
|
|
8
|
+
/** Only call on a native error envelope, never prompt or tool output. */
|
|
9
|
+
export function classifyProviderError(native) {
|
|
10
|
+
if (!native || typeof native !== "object") return "provider_failed";
|
|
11
|
+
const code = [native.code, native.type, native.codexErrorInfo?.code ?? native.codexErrorInfo].filter(value => typeof value === "string").join(" ");
|
|
12
|
+
const message = String(native.message ?? "");
|
|
13
|
+
const text = `${code} ${message}`;
|
|
14
|
+
const status = Number(native.status ?? native.statusCode ?? native.status_code ?? native.http_status);
|
|
15
|
+
if (status === 402 || /quota|usageLimitExceeded|sessionBudgetExceeded|billing|payment.required|balance.exhausted|insufficient.credit|resource_exhausted.*limit/i.test(text)) return "provider_quota_exhausted";
|
|
16
|
+
if (status === 401 || status === 403 || /provider_auth_failed|invalid.api.key|authentication|unauthorized|eligib|forbidden|access.denied|rejected.this.account/i.test(text)) return "provider_auth_failed";
|
|
17
|
+
if (/provider_capacity_transient|serverOverloaded|model.?capacity|at capacity|server.overload|model.overload|server.?busy/i.test(text) || /(?:^|\s)overloaded_error(?:\s|$)/i.test(code) || status === 503 || status === 529) return "provider_capacity_transient";
|
|
18
|
+
if (/rateLimitExceeded|rate.limit/i.test(text) && (native.retryAfter !== undefined || native.retry_after !== undefined || Object.keys(native.headers ?? {}).some(key => key.toLowerCase() === "retry-after"))) return "provider_rate_limited";
|
|
19
|
+
return status === 429 || /provider_limit_unknown|\b429\b|rateLimitExceeded|rate.limit/i.test(text) ? "provider_limit_unknown" : "provider_failed";
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** Public retry evidence excludes response bodies, credentials and other headers. */
|
|
23
|
+
export function safeProviderError(native) {
|
|
24
|
+
const result = { code: classifyProviderError(native) };
|
|
25
|
+
if (!native || typeof native !== "object") return result;
|
|
26
|
+
const status = native.status ?? native.statusCode ?? native.status_code ?? native.http_status;
|
|
27
|
+
if (Number.isInteger(Number(status))) result.status = Number(status);
|
|
28
|
+
const header = Object.entries(native.headers ?? {}).find(([key]) => key.toLowerCase() === "retry-after")?.[1];
|
|
29
|
+
const retry = header ?? native.retryAfter ?? native.retry_after;
|
|
30
|
+
if (retry !== undefined && retry !== null) result.retryAfter = typeof retry === "string" && retry.length <= 128 || typeof retry === "number" && Number.isFinite(retry) ? retry : "invalid-retry-after";
|
|
31
|
+
if (typeof native.willRetry === "boolean") result.willRetry = native.willRetry;
|
|
32
|
+
for (const key of ["reset_at", "resets_at", "resetAt", "resetsAt", "reset_time"]) {
|
|
33
|
+
const value = native[key];
|
|
34
|
+
if (typeof value === "string" && value.length <= 128 || typeof value === "number" && Number.isFinite(value)) result[key] = value;
|
|
35
|
+
}
|
|
36
|
+
return result;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function parseCapacityOperation(id) {
|
|
40
|
+
if (typeof id !== "string" || !id) throw policyError("capacity_continuation_conflict", "Missing capacity operation identity");
|
|
41
|
+
const match = /^(.*):capacity:(\d+):session\.prompt$/.exec(id);
|
|
42
|
+
if (!match) return { root: id, ordinal: 0 };
|
|
43
|
+
const ordinal = Number(match[2]);
|
|
44
|
+
if (!Number.isSafeInteger(ordinal) || ordinal < 1 || String(ordinal) !== match[2]) throw policyError("capacity_continuation_conflict", "Capacity ordinal is not a canonical positive safe integer", { operation_id: id });
|
|
45
|
+
return { root: match[1], ordinal };
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function normalizeProviderFailure(error, expectedSessionId, expectedTurnId) {
|
|
49
|
+
let current = error;
|
|
50
|
+
for (let depth = 0; depth < 4 && current && typeof current === "object"; depth++) {
|
|
51
|
+
// The same continuation authority applies to a proven v2 execution
|
|
52
|
+
// refusal. Never classify transport loss or an active native retry here.
|
|
53
|
+
if (current.code === "provider_capacity_transient" || current.code === "provider_rate_limited") {
|
|
54
|
+
const details = current.details ?? {};
|
|
55
|
+
if (!["opencode.v2.session.execution", "agy.result", "acp.error_response", "droid.agent_turn_outcome", "grok.subagent.event"].includes(details.native_source) || details.terminal_status !== "failed"
|
|
56
|
+
|| details.will_retry === true || details.native_retry_active === true || details.provider_error?.willRetry === true
|
|
57
|
+
|| details.provider_error && !["provider_capacity_transient", "provider_rate_limited"].includes(classifyProviderError(details.provider_error))
|
|
58
|
+
|| typeof details.provider_session_id !== "string" || !details.provider_session_id
|
|
59
|
+
|| typeof details.turn_id !== "string" || !details.turn_id || details.native_turn_id !== details.turn_id
|
|
60
|
+
|| expectedSessionId !== undefined && details.provider_session_id !== expectedSessionId
|
|
61
|
+
|| expectedTurnId !== undefined && details.turn_id !== expectedTurnId) return null;
|
|
62
|
+
return { code: "serverOverloaded", session_id: details.provider_session_id, turn_id: details.turn_id,
|
|
63
|
+
native_error: details.provider_error ?? { type: details.native_error_type, status: details.http_status }, observed_at: details.observed_at ?? null,
|
|
64
|
+
terminal_at: null, terminal_basis: null, native_turn_items: details.native_turn_items ?? null, primary_error: current };
|
|
65
|
+
}
|
|
66
|
+
if (current.code === "turn_interrupted") {
|
|
67
|
+
const details = current.details ?? {}, native = details.provider_error;
|
|
68
|
+
const code = typeof native?.codexErrorInfo === "string" ? native.codexErrorInfo : native?.codexErrorInfo?.code;
|
|
69
|
+
if (native?.willRetry === true || details.terminal_status !== "failed" || typeof details.provider_session_id !== "string" || !details.provider_session_id
|
|
70
|
+
|| expectedSessionId !== undefined && details.provider_session_id !== expectedSessionId
|
|
71
|
+
|| typeof details.turn_id !== "string" || !details.turn_id || details.native_turn_id !== details.turn_id
|
|
72
|
+
|| expectedTurnId !== undefined && details.turn_id !== expectedTurnId || typeof code !== "string") return null;
|
|
73
|
+
return { code, session_id: details.provider_session_id, turn_id: details.turn_id, native_error: native,
|
|
74
|
+
observed_at: details.observed_at ?? null, terminal_at: details.terminal_at ?? null, terminal_basis: details.terminal_basis ?? null,
|
|
75
|
+
native_turn_items: details.native_turn_items ?? null, primary_error: current };
|
|
76
|
+
}
|
|
77
|
+
// No arbitrary recursive cause search: an ownership/hook/lifecycle error wins.
|
|
78
|
+
if (current.code !== "harness_adapter_failed") return null;
|
|
79
|
+
current = current.details?.cause ?? current.cause;
|
|
80
|
+
}
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export function terminalProviderOverload(error, expectedSessionId, expectedTurnId) {
|
|
85
|
+
const normalized = normalizeProviderFailure(error, expectedSessionId, expectedTurnId);
|
|
86
|
+
return normalized !== null && ["provider_capacity_transient", "provider_rate_limited"].includes(classifyProviderError(normalized.native_error));
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function nativeItemsSettled(items) {
|
|
90
|
+
return items?.observed === true && typeof items.possible_effects === "boolean" && items.pending !== true
|
|
91
|
+
&& (items.possible_effects === false || items.pending === false);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export function capacityBackoff(error, ordinal, now = Date.now(), random = Math.random) {
|
|
95
|
+
if (!Number.isSafeInteger(ordinal) || ordinal < 1 || !Number.isFinite(now)) throw policyError("capacity_continuation_conflict", "Invalid capacity timing identity");
|
|
96
|
+
const native = normalizeProviderFailure(error)?.native_error ?? error?.details?.provider_error ?? error;
|
|
97
|
+
const header = Object.entries(native?.headers ?? {}).find(([key]) => key.toLowerCase() === "retry-after")?.[1];
|
|
98
|
+
const value = header ?? native?.retryAfter ?? native?.retry_after;
|
|
99
|
+
let retry = 0;
|
|
100
|
+
if (value !== undefined && value !== null) {
|
|
101
|
+
retry = typeof value === "number" || typeof value === "string" && /^\d+(\.\d+)?$/.test(value.trim()) ? Number(value) * 1000 : typeof value === "string" && /^\w{3}, \d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2} GMT$/.test(value.trim()) ? Math.max(0, Date.parse(value) - now) : NaN;
|
|
102
|
+
if (!Number.isFinite(retry) || retry < 0 || !Number.isFinite(now + retry) || now + retry > 8.64e15)
|
|
103
|
+
throw policyError("capacity_retry_metadata_invalid", "Retry-After cannot be represented safely", { retry_after: value });
|
|
104
|
+
}
|
|
105
|
+
const jitter = random();
|
|
106
|
+
if (!Number.isFinite(jitter) || jitter < 0 || jitter >= 1) throw policyError("capacity_retry_metadata_invalid", "Invalid retry jitter");
|
|
107
|
+
const wait = Math.max((ordinal === 1 ? 5_000 : 15_000) + Math.floor(jitter * 251), retry);
|
|
108
|
+
if (wait > MAX_RETRY_AFTER_MS) throw policyError("provider_unavailable", "Provider requested a delay beyond the retry budget", { policy: CAPACITY_POLICY, reason: "retry_after_exceeds_budget", retry_after_ms: retry, suggested_retry_at: new Date(now + retry).toISOString() });
|
|
109
|
+
return wait;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export function refusalObservation(error, ordinal, firstObservedAt) {
|
|
113
|
+
const failure = normalizeProviderFailure(error);
|
|
114
|
+
const observed = Date.parse(firstObservedAt);
|
|
115
|
+
if (!failure || !Number.isSafeInteger(ordinal) || ordinal < 0 || !Number.isFinite(observed))
|
|
116
|
+
throw policyError("capacity_continuation_conflict", "Refusal lacks an immutable native identity/time");
|
|
117
|
+
if (failure.observed_at !== null && (typeof failure.observed_at !== "string" || !Number.isFinite(Date.parse(failure.observed_at)) || Date.parse(failure.observed_at) !== observed))
|
|
118
|
+
throw policyError("capacity_continuation_conflict", "Native refusal observation time is invalid or changed");
|
|
119
|
+
const native = failure.terminal_at && Date.parse(failure.terminal_at);
|
|
120
|
+
return { policy: CAPACITY_POLICY, ordinal, session_id: failure.session_id, turn_id: failure.turn_id,
|
|
121
|
+
first_observed_at: firstObservedAt, ...(Number.isFinite(native) && typeof failure.terminal_basis === "string" && failure.terminal_basis ? { native_terminal_at: failure.terminal_at, native_terminal_basis: failure.terminal_basis } : {}) };
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export function overloadExhausted(refusals, maxRetries = MAX_OVERLOAD_RETRIES) {
|
|
125
|
+
if (!Number.isSafeInteger(maxRetries) || maxRetries < 0 || maxRetries > MAX_OVERLOAD_RETRIES) throw policyError("capacity_continuation_conflict", "Invalid retry budget");
|
|
126
|
+
for (const [index, refusal] of refusals.entries()) {
|
|
127
|
+
const previous = refusals[index - 1];
|
|
128
|
+
if (typeof refusal.first_observed_at !== "string" || !Number.isFinite(Date.parse(refusal.first_observed_at))
|
|
129
|
+
|| previous && Date.parse(refusal.first_observed_at) < Date.parse(previous.first_observed_at))
|
|
130
|
+
throw policyError("capacity_continuation_conflict", "Refusal observation clock reversed or is invalid");
|
|
131
|
+
}
|
|
132
|
+
for (const [index, refusal] of refusals.entries()) {
|
|
133
|
+
if (refusal.policy !== CAPACITY_POLICY) throw policyError("capacity_policy_mismatch", "Retained episode uses another retry policy");
|
|
134
|
+
const previous = refusals[index - 1];
|
|
135
|
+
if (refusal.ordinal !== index || previous && (refusal.session_id !== previous.session_id || refusals.slice(0, index).some(item => item.turn_id === refusal.turn_id))) throw policyError("capacity_continuation_conflict", "Refusal chain identity is inconsistent");
|
|
136
|
+
}
|
|
137
|
+
return refusals.length > maxRetries ? { policy: CAPACITY_POLICY, reason: "overload_retries_exhausted", attempts: refusals.length, refusals } : null;
|
|
138
|
+
}
|
|
@@ -23,6 +23,10 @@
|
|
|
23
23
|
"publication_sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}
|
|
24
24
|
}
|
|
25
25
|
},
|
|
26
|
+
"workspace_route": {
|
|
27
|
+
"type": "object", "additionalProperties": false, "required": ["revision", "sha256"],
|
|
28
|
+
"properties": {"revision": {"type": "integer", "minimum": 1}, "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}}
|
|
29
|
+
},
|
|
26
30
|
"execution_bundle": {
|
|
27
31
|
"type": "object", "additionalProperties": false,
|
|
28
32
|
"required": ["schema_id", "manifest_sha256", "descriptor_sha256", "admission_sha256", "flow_id"],
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/** Only a proven new successful terminal Turn closes a child's retry episode. */
|
|
2
|
+
export function closeSuccessfulChildCapacity(state, children, parentId) {
|
|
3
|
+
const closed = [];
|
|
4
|
+
for (const child of children) {
|
|
5
|
+
const retained = state.child_capacity?.[child.session_id];
|
|
6
|
+
if (!retained || child.parent_session_id !== parentId || child.provenance === "parent_mismatch"
|
|
7
|
+
|| child.status !== "completed" || !child.terminal_turn_id || child.terminal_turn_id === retained.turn_id)
|
|
8
|
+
continue;
|
|
9
|
+
delete state.child_capacity[child.session_id];
|
|
10
|
+
closed.push(child.session_id);
|
|
11
|
+
}
|
|
12
|
+
return closed;
|
|
13
|
+
}
|
|
@@ -12,6 +12,12 @@ export function retainedChildObserver(input) {
|
|
|
12
12
|
const provider_error = safeNativeFailure(child.provider_error);
|
|
13
13
|
return { session_id: child.session_id, parent_session_id: child.parent_session_id, status: "failed", source: child.source, terminal_turn_id: child.terminal_turn_id,
|
|
14
14
|
...(typeof child.terminal_observed_at === "string" && Number.isFinite(Date.parse(child.terminal_observed_at)) ? { terminal_observed_at: child.terminal_observed_at } : {}),
|
|
15
|
+
...(child.native_source ? { native_source: child.native_source } : {}),
|
|
16
|
+
...(child.native_turn_items ? { native_turn_items: {
|
|
17
|
+
...(typeof child.native_turn_items.observed === "boolean" ? { observed: child.native_turn_items.observed } : {}),
|
|
18
|
+
...(typeof child.native_turn_items.possible_effects === "boolean" ? { possible_effects: child.native_turn_items.possible_effects } : {}),
|
|
19
|
+
...(typeof child.native_turn_items.pending === "boolean" ? { pending: child.native_turn_items.pending } : {})
|
|
20
|
+
} } : {}),
|
|
15
21
|
...(provider_error ? { provider_error } : {}) };
|
|
16
22
|
};
|
|
17
23
|
const accept = (children) => {
|
|
@@ -26,6 +32,8 @@ export function retainedChildObserver(input) {
|
|
|
26
32
|
if (sameScope(observation.scope)) {
|
|
27
33
|
if (Array.isArray(observation.children))
|
|
28
34
|
accept(observation.children);
|
|
35
|
+
for (const key of observation.resolved ?? [])
|
|
36
|
+
retained.delete(key);
|
|
29
37
|
if (Object.hasOwn(observation, "gap"))
|
|
30
38
|
gap = observation.gap ?? null;
|
|
31
39
|
}
|
|
@@ -35,23 +43,31 @@ export function retainedChildObserver(input) {
|
|
|
35
43
|
return;
|
|
36
44
|
input.assertOwner();
|
|
37
45
|
const failed = children.filter(child => child.parent_session_id === input.scope.root_session_id && child.status === "failed" && child.terminal_turn_id && !retained.has(`${child.session_id}:${child.terminal_turn_id}`)).map(normalized);
|
|
38
|
-
|
|
46
|
+
// This is an authoritative current native snapshot. Older failed Turns
|
|
47
|
+
// remain in the journal/episode history, not in the current-outcome fence.
|
|
48
|
+
const resolved = [...retained].filter(([, old]) => children.some(child => child.parent_session_id === input.scope.root_session_id
|
|
49
|
+
&& child.provenance !== "parent_mismatch" && child.session_id === old.session_id && ["completed", "failed"].includes(child.status)
|
|
50
|
+
&& child.terminal_turn_id && child.terminal_turn_id !== old.terminal_turn_id)).map(([key]) => key);
|
|
51
|
+
if (!failed.length && !resolved.length && !gap)
|
|
39
52
|
return;
|
|
40
53
|
const fd = fs.openSync(input.file, "a", 0o600);
|
|
41
54
|
try {
|
|
42
|
-
fs.writeSync(fd, `${JSON.stringify({ scope: input.scope, children: failed, gap: null })}\n`);
|
|
55
|
+
fs.writeSync(fd, `${JSON.stringify({ scope: input.scope, children: failed, ...(resolved.length ? { resolved } : {}), gap: null })}\n`);
|
|
43
56
|
fs.fsyncSync(fd);
|
|
44
57
|
}
|
|
45
58
|
finally {
|
|
46
59
|
fs.closeSync(fd);
|
|
47
60
|
}
|
|
48
61
|
accept(failed);
|
|
62
|
+
for (const key of resolved)
|
|
63
|
+
retained.delete(key);
|
|
49
64
|
gap = null;
|
|
50
65
|
};
|
|
51
66
|
return {
|
|
52
67
|
failed: () => [...retained.values()],
|
|
53
68
|
gap: () => gap,
|
|
54
69
|
drain: () => pending ?? Promise.resolve(),
|
|
70
|
+
observeSettledChildren: complete,
|
|
55
71
|
completePostTurn(receipt) {
|
|
56
72
|
const evidence = receipt.evidence;
|
|
57
73
|
const tree = evidence?.subagents;
|
|
@@ -27,13 +27,13 @@ export function genericControllerRouting(execution) {
|
|
|
27
27
|
Object.freeze(policy);
|
|
28
28
|
return {
|
|
29
29
|
kind: "descriptor", policy, profiles, mergeMode: null,
|
|
30
|
-
transition(stage, fromStage) {
|
|
30
|
+
transition(stage, fromStage, _currentRun) {
|
|
31
31
|
const selected = execution.stage(stage);
|
|
32
32
|
if (fromStage)
|
|
33
33
|
execution.stage(fromStage);
|
|
34
34
|
return { ...resolveExecutionPolicy({ ddFlowHome: "", policy, profiles, stage }), session_mode: selected.session_mode };
|
|
35
35
|
},
|
|
36
|
-
cwd(stage) { execution.stage(stage); return path.resolve(snapshot.scope.workspace); }
|
|
36
|
+
cwd(stage, _mergeTargetWorkspace, _currentRun) { execution.stage(stage); return path.resolve(snapshot.scope.workspace); }
|
|
37
37
|
};
|
|
38
38
|
}
|
|
39
39
|
/** Restore execution from RUN authority, never from a generic manifest copy.
|
|
@@ -77,12 +77,12 @@ export function selectControllerExecution(context, run, manifest) {
|
|
|
77
77
|
return {
|
|
78
78
|
kind: "sdlc", policy, profiles, bundle: bundle ?? undefined, binding: undefined, legacy,
|
|
79
79
|
mergeMode: legacy.settings.merge_mode,
|
|
80
|
-
transition(stage, fromStage) {
|
|
80
|
+
transition(stage, fromStage, currentRun = run) {
|
|
81
81
|
return resolveStageTransition({ ddFlowHome: context.ddFlowHome, policy, profiles, ...(fromStage ? { fromStage } : {}), stage,
|
|
82
|
-
projectRoot:
|
|
82
|
+
projectRoot: currentRun.project_root, workspaceRoot: currentRun.workspace_root, stageSessionMode: legacy.settings.stage_session_mode, mergeMode: legacy.settings.merge_mode });
|
|
83
83
|
},
|
|
84
|
-
cwd(stage, mergeTargetWorkspace) {
|
|
85
|
-
return stageExecutionCwd({ stage, projectRoot:
|
|
84
|
+
cwd(stage, mergeTargetWorkspace, currentRun = run) {
|
|
85
|
+
return stageExecutionCwd({ stage, projectRoot: currentRun.project_root, workspaceRoot: currentRun.workspace_root, ...(mergeTargetWorkspace ? { mergeTargetWorkspace } : {}) });
|
|
86
86
|
}
|
|
87
87
|
};
|
|
88
88
|
}
|
|
@@ -16,7 +16,7 @@ import { reviewExecutionCopyHash, reviewSourceFilter, reviewSourceHash, reviewSo
|
|
|
16
16
|
import { workspaceGitFiles } from "./workspace-files.js";
|
|
17
17
|
import { qualifyAgyHooks } from "../harness-runtime/lib/dd-agy-daemon.mjs";
|
|
18
18
|
import { lifecycleRetryInstruction } from "../harness-runtime/lib/delegation-instructions.mjs";
|
|
19
|
-
import {
|
|
19
|
+
import { continueProviderCapacity } from "./provider-capacity.js";
|
|
20
20
|
import { errorRecord } from "./run-controller-state.js";
|
|
21
21
|
import { readFlowRunExecutionBundle } from "./runs.js";
|
|
22
22
|
import { stageWorkIds } from "./stage-work-graph.js";
|
|
@@ -190,8 +190,6 @@ export async function launchExternalWork(context, input) {
|
|
|
190
190
|
prompted = await call(["session", "prompt", "--session-id", sessionId, "--prompt-file", promptFile, ...common], null);
|
|
191
191
|
}
|
|
192
192
|
catch (error) {
|
|
193
|
-
if (!["codex-desktop", "opencode-server"].includes(runtimeHarness(resolved.worker)))
|
|
194
|
-
throw error;
|
|
195
193
|
const retained = context.db.get("SELECT adapter_receipt_json FROM work_launches WHERE operation_id = ?", [operationId]);
|
|
196
194
|
const capacityIntents = retained?.adapter_receipt_json ? JSON.parse(retained.adapter_receipt_json).capacity_intents ?? [] : [];
|
|
197
195
|
const capacityFailures = retained?.adapter_receipt_json ? JSON.parse(retained.adapter_receipt_json).capacity_failures ?? [] : [];
|
|
@@ -200,7 +198,7 @@ export async function launchExternalWork(context, input) {
|
|
|
200
198
|
if (changed.changes !== 1)
|
|
201
199
|
throw new AppError("capacity_continuation_blocked", "External Work launch owner changed", 1);
|
|
202
200
|
};
|
|
203
|
-
const resumed = await
|
|
201
|
+
const resumed = await continueProviderCapacity({ error, sessionId, operationId: promptOperationId, allowSettledDescendants: true,
|
|
204
202
|
...(input.signal ? { signal: input.signal } : {}),
|
|
205
203
|
continuationText: `Continue only Work ${work.work_id} in RUN ${work.run_id}, Stage ${stage ?? "current"}, in the same Session. Inspect its current durable state and the retained Work packet. Never repeat Work start or a completed tool, accepted answer, check or lifecycle command. Use only the current exact issued successor and finish at the original Work boundary.`,
|
|
206
204
|
retainedIntents: () => capacityIntents,
|
|
@@ -10,8 +10,14 @@ import { shellArgument } from "./runtime-command.js";
|
|
|
10
10
|
import { hybridDigest } from "./hybrid-contracts.js";
|
|
11
11
|
import { isMemoryInitResourceStorageFacts } from "./memory-init-resource-storage.js";
|
|
12
12
|
import { memoryInitStartupInputSchemas } from "./memory-init-startup-custody.js";
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
import { SETUP_STARTUP_MS } from "../harness-runtime/lib/setup-progress.mjs";
|
|
14
|
+
function storageProbeRefuse(reason, probe) {
|
|
15
|
+
throw new AppError("frozen_engine_storage_incompatible", "Selected frozen engine cannot admit memory-init storage", 1, { reason, effect: "no_effect", ...(probe ? { probe } : {}) });
|
|
16
|
+
}
|
|
17
|
+
function probeFailure(result) {
|
|
18
|
+
const code = result.error?.code ?? null;
|
|
19
|
+
return { status: result.status, signal: result.signal, error_code: code, timed_out: code === "ETIMEDOUT",
|
|
20
|
+
timeout_ms: SETUP_STARTUP_MS, stdout_bytes: result.stdout?.length ?? null, stderr_bytes: result.stderr?.length ?? null };
|
|
15
21
|
}
|
|
16
22
|
const hash = (bytes) => crypto.createHash("sha256").update(bytes).digest("hex");
|
|
17
23
|
function refuse(reason) {
|
|
@@ -113,8 +119,8 @@ export class AdmittedFrozenEngine {
|
|
|
113
119
|
/** Qualify actual descriptor handlers in this immutable executable, not the
|
|
114
120
|
* invoking CLI catalog. No storage or productive operation is admitted. */
|
|
115
121
|
admitMemoryInitFlow(source, startupInputSchema) {
|
|
116
|
-
const fail = (reason) => {
|
|
117
|
-
throw new AppError("frozen_engine_flow_incompatible", "Selected frozen engine cannot admit memory-init handlers", 1, { reason, phase: "prepare", effect: "no_effect" });
|
|
122
|
+
const fail = (reason, probe) => {
|
|
123
|
+
throw new AppError("frozen_engine_flow_incompatible", "Selected frozen engine cannot admit memory-init handlers", 1, { reason, phase: "prepare", effect: "no_effect", ...(probe ? { probe } : {}) });
|
|
118
124
|
};
|
|
119
125
|
const revalidate = () => {
|
|
120
126
|
try {
|
|
@@ -147,9 +153,11 @@ export class AdmittedFrozenEngine {
|
|
|
147
153
|
revalidate();
|
|
148
154
|
const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !["NODE_OPTIONS", "NODE_PATH"].includes(key) && !key.startsWith("DD_FLOW_")));
|
|
149
155
|
const result = spawnSync(this.nodeExecutable, [this.entrypoint, "runtime", "flow", "admit", "--operation", "memory-init", "--json"], { input, cwd: this.#manifest.snapshot_root, env: { ...env, DD_FLOW_ENGINE_MODE: "1", DD_FLOW_ENGINE_HOME: this.#manifest.snapshot_root },
|
|
150
|
-
|
|
156
|
+
// This is cold startup of a selected CLI, not an already-loaded handler.
|
|
157
|
+
// Admission is bounded local validation; no provider operation is started.
|
|
158
|
+
timeout: SETUP_STARTUP_MS, maxBuffer: 1024 * 1024, shell: false, windowsHide: true });
|
|
151
159
|
if (result.error || result.signal || result.status !== 0)
|
|
152
|
-
fail("selected_engine_probe_failed");
|
|
160
|
+
fail("selected_engine_probe_failed", probeFailure(result));
|
|
153
161
|
revalidate();
|
|
154
162
|
let response;
|
|
155
163
|
try {
|
|
@@ -182,9 +190,9 @@ export class AdmittedFrozenEngine {
|
|
|
182
190
|
revalidateEngine();
|
|
183
191
|
const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !["NODE_OPTIONS", "NODE_PATH"].includes(key)));
|
|
184
192
|
const result = spawnSync(this.nodeExecutable, [this.entrypoint, "runtime", "storage", "admit", "--operation", "memory-init", "--json"], { env: { ...env, DD_FLOW_HOME: home, DD_FLOW_RESOURCE_HOME: resourceHome, DD_FLOW_ENGINE_MODE: "1", DD_FLOW_ENGINE_HOME: this.#manifest.snapshot_root },
|
|
185
|
-
timeout:
|
|
193
|
+
timeout: SETUP_STARTUP_MS, maxBuffer: 1024 * 1024, shell: false, windowsHide: true });
|
|
186
194
|
if (result.error || result.signal || result.status !== 0)
|
|
187
|
-
storageProbeRefuse("selected_engine_probe_failed");
|
|
195
|
+
storageProbeRefuse("selected_engine_probe_failed", probeFailure(result));
|
|
188
196
|
revalidateEngine();
|
|
189
197
|
let response;
|
|
190
198
|
try {
|
|
@@ -5,6 +5,7 @@ import { spawnSync } from "node:child_process";
|
|
|
5
5
|
import net from "node:net";
|
|
6
6
|
import { getResourceDatabase } from "../storage/database.js";
|
|
7
7
|
import { AppError, errorRecord } from "../shared/errors.js";
|
|
8
|
+
import { maintenanceRetryable } from "../harness-runtime/lib/lease-renewal.mjs";
|
|
8
9
|
export function managedProcessBinding(record, home) {
|
|
9
10
|
return { resource_home: home, process_id: record.id, registration_sha256: crypto.createHash("sha256").update(JSON.stringify([home, record.id, record.kind, record.owner_id, record.lease_token, record.pid, record.pid_started_at, record.project_id, record.run_id, record.work_id, record.check_id, record.operation_id, record.metadata_json])).digest("hex") };
|
|
10
11
|
}
|
|
@@ -203,17 +204,28 @@ export function heartbeatManagedProcess(context, input) {
|
|
|
203
204
|
}
|
|
204
205
|
export function renewManagedProcess(context, input) {
|
|
205
206
|
prepareManagedProcessHeartbeat(context, input);
|
|
206
|
-
|
|
207
|
-
registry(context);
|
|
208
|
-
const reader = getResourceDatabase(resourceHome(context), "read_existing", context.databaseDeadlineAt);
|
|
207
|
+
let db;
|
|
209
208
|
try {
|
|
210
|
-
if (
|
|
211
|
-
|
|
209
|
+
if (context.resourceDatabaseMode === "prepared_existing")
|
|
210
|
+
registry(context);
|
|
211
|
+
const reader = getResourceDatabase(resourceHome(context), "read_existing", context.databaseDeadlineAt);
|
|
212
|
+
try {
|
|
213
|
+
if (!reader.get("SELECT 1 FROM managed_processes WHERE id=? AND lease_token=? AND state IN ('starting','running','stopping')", [input.id, input.leaseToken]))
|
|
214
|
+
return { owned: false, process_id: input.id };
|
|
215
|
+
}
|
|
216
|
+
finally {
|
|
217
|
+
reader.close?.();
|
|
218
|
+
}
|
|
219
|
+
db = registry(context);
|
|
212
220
|
}
|
|
213
|
-
|
|
214
|
-
|
|
221
|
+
catch (error) {
|
|
222
|
+
// No lease mutation has started. The storage helper cannot know that
|
|
223
|
+
// operation boundary and deliberately defaults to an unknown effect.
|
|
224
|
+
if (error instanceof AppError && error.code === "process_heartbeat_timeout"
|
|
225
|
+
&& maintenanceRetryable({ ...error, details: { ...error.details, effect: "no_effect" } }))
|
|
226
|
+
throw new AppError(error.code, error.message, error.exitCode, { ...error.details, effect: "no_effect", maintenance_phase: "renewal_prepare" });
|
|
227
|
+
throw error;
|
|
215
228
|
}
|
|
216
|
-
const db = registry(context);
|
|
217
229
|
return db.writeTransaction(() => {
|
|
218
230
|
const now = context.now();
|
|
219
231
|
if (!db.get("SELECT 1 FROM managed_processes WHERE id=? AND lease_token=? AND state IN ('starting','running','stopping')", [input.id, input.leaseToken]))
|
|
@@ -228,8 +240,9 @@ export function renewManagedProcess(context, input) {
|
|
|
228
240
|
/** Read-only lease check for a caller already holding a RUN writer. Renewal
|
|
229
241
|
* must happen before entering that writer, never from this assertion. */
|
|
230
242
|
export function assertManagedProcessLease(context, input) {
|
|
231
|
-
|
|
243
|
+
let reader;
|
|
232
244
|
try {
|
|
245
|
+
reader = getResourceDatabase(resourceHome(context), "read_isolated", context.databaseDeadlineAt);
|
|
233
246
|
const current = reader.get("SELECT pid, lease_expires_at, state FROM managed_processes WHERE id = ? AND lease_token = ?", [input.id, input.leaseToken]);
|
|
234
247
|
if (!current || current.pid !== input.pid || !["starting", "running"].includes(current.state)) {
|
|
235
248
|
throw new AppError("controller_lease_lost", "Managed owner lease changed before productive dispatch", 1, { process_id: input.id });
|
|
@@ -237,8 +250,16 @@ export function assertManagedProcessLease(context, input) {
|
|
|
237
250
|
if (!Number.isFinite(Date.parse(current.lease_expires_at)) || Date.parse(current.lease_expires_at) <= Date.now())
|
|
238
251
|
throw new AppError("controller_lease_unconfirmed", "Managed owner requires renewal before productive dispatch", 1, { process_id: input.id, reason: "renewal_required" });
|
|
239
252
|
}
|
|
253
|
+
catch (error) {
|
|
254
|
+
const failure = errorRecord(error);
|
|
255
|
+
// This assertion only reads. Unavailable observation is not evidence of
|
|
256
|
+
// owner loss; the caller must reconfirm outside its RUN writer.
|
|
257
|
+
if (maintenanceRetryable({ ...failure, details: { ...failure.details, effect: "no_effect" } }))
|
|
258
|
+
throw new AppError("controller_lease_unconfirmed", "Managed owner observation requires confirmation before productive dispatch", 1, { process_id: input.id, reason: "observation_unavailable", cause: failure });
|
|
259
|
+
throw error;
|
|
260
|
+
}
|
|
240
261
|
finally {
|
|
241
|
-
reader
|
|
262
|
+
reader?.close?.();
|
|
242
263
|
}
|
|
243
264
|
}
|
|
244
265
|
export function prepareManagedProcessFinish(context, input) {
|