@deksden-com/dd-flow-cli 0.9.0-beta.124 → 0.9.0-beta.127
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +10 -1
- package/dist/build-info.json +7 -7
- package/dist/cli/argument-values.js +38 -0
- package/dist/cli/command-inputs.js +4 -3
- package/dist/cli/help.js +1 -0
- package/dist/cli/hook-ingress.js +1 -1
- package/dist/cli/run-cli.js +126 -193
- package/dist/cli/runtime-maintenance-diagnostics.js +46 -0
- package/dist/cli/runtime-maintenance-ingress.js +51 -0
- package/dist/cli/runtime-process.js +128 -0
- package/dist/cli.js +8 -5
- package/dist/harness-runtime/bin/dd-agy.mjs +3 -2
- package/dist/harness-runtime/bin/dd-codex.mjs +3 -2
- package/dist/harness-runtime/bin/dd-droid.mjs +3 -2
- package/dist/harness-runtime/bin/dd-grok.mjs +16 -6
- package/dist/harness-runtime/bin/dd-opencode.mjs +3 -2
- package/dist/harness-runtime/bin/dd-zcode.mjs +20 -16
- package/dist/harness-runtime/lib/adapter-timeouts.d.mts +5 -0
- package/dist/harness-runtime/lib/adapter-timeouts.mjs +34 -0
- package/dist/harness-runtime/lib/daemon-observation.d.mts +2 -0
- package/dist/harness-runtime/lib/daemon-observation.mjs +163 -0
- package/dist/harness-runtime/lib/daemon-operations.mjs +60 -13
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +151 -72
- package/dist/harness-runtime/lib/dd-agy.mjs +11 -23
- package/dist/harness-runtime/lib/dd-codex-daemon.mjs +9 -14
- package/dist/harness-runtime/lib/dd-codex.mjs +252 -80
- package/dist/harness-runtime/lib/dd-droid-daemon.mjs +14 -19
- package/dist/harness-runtime/lib/dd-droid.mjs +117 -31
- package/dist/harness-runtime/lib/dd-grok-daemon.mjs +56 -30
- package/dist/harness-runtime/lib/dd-grok.mjs +15 -13
- package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +18 -27
- package/dist/harness-runtime/lib/dd-opencode.mjs +44 -14
- package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +56 -53
- package/dist/harness-runtime/lib/dd-zcode.mjs +200 -111
- package/dist/harness-runtime/lib/lease-renewal.d.mts +8 -0
- package/dist/harness-runtime/lib/lease-renewal.mjs +23 -0
- package/dist/harness-runtime/lib/lifecycle-command.d.mts +3 -3
- package/dist/harness-runtime/lib/lifecycle-command.mjs +74 -20
- package/dist/harness-runtime/lib/managed-daemon.d.mts +10 -1
- package/dist/harness-runtime/lib/managed-daemon.mjs +228 -40
- package/dist/harness-runtime/lib/native-children.d.mts +13 -0
- package/dist/harness-runtime/lib/native-children.mjs +78 -0
- package/dist/harness-runtime/lib/native-hook-command.d.mts +4 -0
- package/dist/harness-runtime/lib/native-hook-command.mjs +115 -11
- package/dist/harness-runtime/lib/native-streams.mjs +45 -0
- package/dist/harness-runtime/lib/observation-clock.d.mts +18 -0
- package/dist/harness-runtime/lib/observation-clock.mjs +73 -17
- package/dist/harness-runtime/lib/operation-errors.d.mts +14 -0
- package/dist/harness-runtime/lib/operation-errors.mjs +105 -8
- package/dist/harness-runtime/lib/operation-progress.d.mts +7 -0
- package/dist/harness-runtime/lib/operation-progress.mjs +112 -0
- package/dist/harness-runtime/lib/process-json.mjs +136 -38
- package/dist/harness-runtime/lib/tool-observations.d.mts +1 -0
- package/dist/harness-runtime/lib/tool-observations.mjs +5 -2
- package/dist/harness-runtime/lib/zcode-usage.mjs +3 -2
- package/dist/runtime/context.js +8 -4
- package/dist/schemas/run-control-receipt.schema.json +19 -0
- package/dist/services/code-checks.js +79 -9
- package/dist/services/codex-capacity.js +117 -41
- package/dist/services/codex-hook-delivery.js +2 -2
- package/dist/services/compatibility-preflight.js +4 -1
- package/dist/services/controller-fanout.js +9 -76
- package/dist/services/engines.js +70 -20
- package/dist/services/eval-snapshots.js +179 -40
- package/dist/services/external-work-launch.js +13 -8
- package/dist/services/harness-adapter.js +175 -11
- package/dist/services/hooks.js +65 -21
- package/dist/services/lifecycle-invocations.js +404 -32
- package/dist/services/managed-lease-monitor.js +128 -0
- package/dist/services/managed-processes.js +103 -47
- package/dist/services/merge-server.js +58 -21
- package/dist/services/recovery-observation-budget.js +67 -7
- package/dist/services/recovery-snapshot-database.js +7 -4
- package/dist/services/run-control-receipt.js +1 -1
- package/dist/services/run-control-worker.js +177 -118
- package/dist/services/run-control.js +99 -8
- package/dist/services/run-controller-adapter.js +169 -46
- package/dist/services/run-controller-capture.js +14 -2
- package/dist/services/run-controller-process.js +11 -4
- package/dist/services/run-controller-state.js +68 -7
- package/dist/services/run-controller.js +166 -94
- package/dist/services/run-fork.js +8 -3
- package/dist/services/run-recovery-guard.js +75 -0
- package/dist/services/run-recovery-runtime.js +1 -56
- package/dist/services/run-recovery.js +17 -13
- package/dist/services/runs.js +39 -6
- package/dist/services/runtime-budget.js +32 -15
- package/dist/services/runtime-scope-control.js +8 -0
- package/dist/services/runtime-scope-resume.js +30 -8
- package/dist/services/runtime-scope-worker.js +118 -73
- package/dist/services/runtime-service.js +6 -7
- package/dist/services/scope-writer-cleanup.js +3 -2
- package/dist/services/stage-context.js +135 -0
- package/dist/services/stage-lifecycle.js +3 -15
- package/dist/services/stage-pause.js +3 -2
- package/dist/services/usage.js +14 -3
- package/dist/services/vnext-code-review.js +10 -9
- package/dist/services/vnext-code.js +10 -8
- package/dist/services/vnext-contracts.js +10 -1
- package/dist/services/vnext-execution-profile.js +75 -2
- package/dist/services/vnext-fanout.js +5 -0
- package/dist/services/vnext-merge.js +10 -9
- package/dist/services/vnext-plan-review.js +21 -17
- package/dist/services/vnext-plan.js +9 -7
- package/dist/services/vnext-protocolize.js +15 -9
- package/dist/services/vnext-specify.js +6 -6
- package/dist/services/work-registry.js +34 -4
- package/dist/services/workspace-bootstrap.js +14 -3
- package/dist/shared/errors.js +11 -6
- package/dist/storage/agent-output.js +26 -0
- package/dist/storage/database.js +80 -21
- package/package.json +7 -4
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export const NATIVE_CHILD_CONTRACT_VERSION: "native-children@1";
|
|
2
|
+
export interface NativeChild {
|
|
3
|
+
session_id: string;
|
|
4
|
+
parent_session_id: string;
|
|
5
|
+
status: "running" | "unknown" | "completed" | "failed" | "cancelled" | "settled_by_root";
|
|
6
|
+
source: string;
|
|
7
|
+
provenance?: "root_child" | "parent_mismatch";
|
|
8
|
+
terminal_turn_id?: string;
|
|
9
|
+
terminal_observed_at?: string;
|
|
10
|
+
provider_error?: unknown;
|
|
11
|
+
native_turn_items?: { observed?: boolean; possible_effects?: boolean; pending?: boolean };
|
|
12
|
+
}
|
|
13
|
+
export function normalizeNativeChildren(receipt: Record<string, unknown>, rootSessionId: string): NativeChild[];
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
export const NATIVE_CHILD_CONTRACT_VERSION = "native-children@1";
|
|
2
|
+
import { errorRecord } from "./operation-errors.mjs";
|
|
3
|
+
|
|
4
|
+
function nativeFailure(code, message, exitCode, details) { return Object.assign(new Error(message), { code, exitCode, details }); }
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
/** Normalize provider-observed topology only; never parse assistant prose. */
|
|
8
|
+
export function normalizeNativeChildren(receipt, rootSessionId) {
|
|
9
|
+
const candidates = [];
|
|
10
|
+
const object = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
11
|
+
receipt = object(receipt);
|
|
12
|
+
const list = (value) => Array.isArray(value) ? value : [];
|
|
13
|
+
const text = (value, keys) => { for (const key of keys) if (typeof value[key] === "string" && value[key]) return value[key]; return null; };
|
|
14
|
+
const settledTree = new Set(
|
|
15
|
+
list(receipt.tree_observations)
|
|
16
|
+
.filter(value => object(value).settled === true)
|
|
17
|
+
.map(value => text(object(value), ["provider_session_id", "session_id"]))
|
|
18
|
+
.filter(id => id !== null)
|
|
19
|
+
);
|
|
20
|
+
const add = (raw, source, scopedParent = null, fallback = "unknown") => {
|
|
21
|
+
const value = object(raw);
|
|
22
|
+
const id = text(value, ["provider_session_id", "session_id", "sessionId", "childSessionId", "conversation_id", "conversationId", "id", "subagentId"]);
|
|
23
|
+
const parent = text(value, ["parent_provider_session_id", "parent_session_id", "parentSessionId", "parentID", "parentId"]) ?? scopedParent;
|
|
24
|
+
if (!id) return;
|
|
25
|
+
const outcomeError = object(value.outcome_error);
|
|
26
|
+
if (typeof outcomeError.code === "string") {
|
|
27
|
+
const record = errorRecord(outcomeError);
|
|
28
|
+
throw Object.assign(nativeFailure(record.code, record.message, 1, record.details), record, { details: { ...object(record.details), session_id: id, source } });
|
|
29
|
+
}
|
|
30
|
+
const state = String(value.status ?? value.state ?? fallback).toLowerCase();
|
|
31
|
+
if (state === "lost") throw nativeFailure("native_child_outcome_lost", "Native child ended without a recoverable outcome", 1, { session_id: id, parent_session_id: parent, source, native_status: state });
|
|
32
|
+
// Provider idle means no current activity, not a Work result. Only an
|
|
33
|
+
// explicit terminal result or a separately proven root settlement may be
|
|
34
|
+
// terminal here; unknown states stay unknown for reconciliation.
|
|
35
|
+
const status = ["completed", "complete", "done", "success", "succeeded"].includes(state) ? "completed" : state === "settled_by_root" ? "settled_by_root" : ["failed", "error"].includes(state) ? "failed" : ["cancelled", "canceled", "interrupted"].includes(state) ? "cancelled" : ["running", "active", "pending"].includes(state) ? "running" : fallback;
|
|
36
|
+
const rootChild = parent === rootSessionId;
|
|
37
|
+
const settled = rootChild && status === "unknown" && settledTree.has(id) ? "settled_by_root" : status;
|
|
38
|
+
candidates.push({ session_id: id, parent_session_id: parent ?? "", status: settled, source, ...(rootChild ? {} : { provenance: "parent_mismatch" }),
|
|
39
|
+
...(text(value, ["terminal_turn_id"]) ? { terminal_turn_id: text(value, ["terminal_turn_id"]) } : {}),
|
|
40
|
+
...(text(value, ["terminal_observed_at"]) ? { terminal_observed_at: text(value, ["terminal_observed_at"]) } : {}),
|
|
41
|
+
...(value.provider_error !== undefined ? { provider_error: value.provider_error } : {}),
|
|
42
|
+
...(value.native_turn_items && typeof value.native_turn_items === "object" ? { native_turn_items: value.native_turn_items } : {}) });
|
|
43
|
+
};
|
|
44
|
+
for (const value of list(receipt.descendants)) add(value, "adapter.descendants");
|
|
45
|
+
for (const value of list(receipt.children)) add(value, "opencode.session.children");
|
|
46
|
+
const subagents = object(object(receipt.evidence).subagents ?? receipt.subagents);
|
|
47
|
+
for (const value of list(subagents.running)) add(value, "zcode/session/subagents", rootSessionId, "running");
|
|
48
|
+
for (const value of list(subagents.completed)) add(value, "zcode/session/subagents", rootSessionId, "completed");
|
|
49
|
+
for (const value of list(object(subagents.ended).items)) {
|
|
50
|
+
const status = String(object(value).status ?? "unknown");
|
|
51
|
+
if (!["success", "failed", "cancelled", "lost"].includes(status)) throw nativeFailure("native_child_outcome_unavailable", "ZCode ended child has no recognized native outcome", 1, { child: value });
|
|
52
|
+
add(value, "zcode/session/subagents", rootSessionId);
|
|
53
|
+
}
|
|
54
|
+
const unique = new Map();
|
|
55
|
+
const rank = { unknown: 0, running: 1, settled_by_root: 2, completed: 3, failed: 3, cancelled: 3 };
|
|
56
|
+
for (const child of candidates) {
|
|
57
|
+
const previous = unique.get(child.session_id);
|
|
58
|
+
if (previous && previous.parent_session_id !== child.parent_session_id) {
|
|
59
|
+
// Keep the contradictory parent visible to the existing provenance gate,
|
|
60
|
+
// independently of input order or which source happened to say success.
|
|
61
|
+
const foreign = [previous, child].filter(item => item.parent_session_id !== rootSessionId).sort((a, b) => a.parent_session_id.localeCompare(b.parent_session_id))[0];
|
|
62
|
+
unique.set(child.session_id, { ...foreign, provenance: "parent_mismatch" });
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
if (previous && rank[previous.status] === 3 && rank[child.status] === 3 && previous.status !== child.status) throw nativeFailure("native_child_outcome_conflict", "Native child has contradictory terminal outcomes", 1, { session_id: child.session_id, observations: [previous, child] });
|
|
66
|
+
if (previous && previous.status === child.status && previous.terminal_turn_id && child.terminal_turn_id && previous.terminal_turn_id !== child.terminal_turn_id)
|
|
67
|
+
throw nativeFailure("native_child_turn_conflict", "Native child has contradictory terminal Turn identities", 1, { session_id: child.session_id, observations: [previous, child] });
|
|
68
|
+
if (previous && previous.status === child.status) {
|
|
69
|
+
unique.set(child.session_id, { ...previous, ...child,
|
|
70
|
+
...(child.terminal_turn_id ?? previous.terminal_turn_id ? { terminal_turn_id: child.terminal_turn_id ?? previous.terminal_turn_id } : {}),
|
|
71
|
+
...(child.provider_error ?? previous.provider_error ? { provider_error: child.provider_error ?? previous.provider_error } : {}),
|
|
72
|
+
...(child.native_turn_items ?? previous.native_turn_items ? { native_turn_items: child.native_turn_items ?? previous.native_turn_items } : {}) });
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
if (!previous || rank[child.status] > rank[previous.status]) unique.set(child.session_id, child);
|
|
76
|
+
}
|
|
77
|
+
return [...unique.values()].sort((left, right) => left.session_id.localeCompare(right.session_id));
|
|
78
|
+
}
|
|
@@ -1,3 +1,7 @@
|
|
|
1
1
|
export function invokeNativeHook(input: { bin: string; args: string[]; payload: unknown; home?: string | undefined; cwd?: string; env?: NodeJS.ProcessEnv; timeoutMs?: number }): Promise<Record<string, unknown> & { timing?: Record<string, number> }>;
|
|
2
2
|
export function hookBudget(timeoutMs?: number, deadlineAt?: number | null): { deadlineAt: number; remaining(phase: string, maximum?: number): number; run<T>(action: () => T | Promise<T>, phase: string): Promise<T> };
|
|
3
3
|
export function readNativeHookInput(stream: NodeJS.ReadStream, budget: { remaining(phase: string): number }, maxBytes?: number): Promise<string>;
|
|
4
|
+
export function acceptNativeHookRequest(stateDir: string, binding: Record<string, unknown>, current?: { requests: Map<string, { binding: Record<string, unknown> }>; accepting?: Map<string, Promise<Record<string, unknown>>> } | null): Promise<Record<string, unknown>>;
|
|
5
|
+
export function writeNativeHookFailure(stateDir: string, binding: Record<string, unknown> | undefined, error: unknown, budget: ReturnType<typeof hookBudget>): Promise<void>;
|
|
6
|
+
export function readNativeHookFailures(stateDir: string, requests: Map<string, { current: unknown; binding: Record<string, unknown> }>, current: unknown, reject: (error: Error & { code?: string; details?: Record<string, unknown> }) => void): Promise<void>;
|
|
7
|
+
export function runBoundedCommand(input: { bin: string; args: string[]; input?: string | null; cwd?: string; env?: NodeJS.ProcessEnv; timeoutMs?: number; timeoutCode?: string; inheritDeadline?: boolean; cleanupWithinBudget?: boolean; maintenanceProgress?: { action: string; processId: string | null; inactivityMs: number } | null; signal?: AbortSignal }): Promise<string>;
|
|
@@ -1,5 +1,65 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
3
|
+
import { readFile } from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
2
5
|
import { reportedFailure } from "./process-json.mjs";
|
|
6
|
+
import { errorRecord, reportedError } from "./operation-errors.mjs";
|
|
7
|
+
import { writeJsonAtomic } from "./runner-events.mjs";
|
|
8
|
+
|
|
9
|
+
/** Retain accepted native requests in the existing operation hook ledger. */
|
|
10
|
+
export async function acceptNativeHookRequest(stateDir, binding, current = null) {
|
|
11
|
+
if (["operation_id", "daemon_id", "tool_call_id", "root_provider_session_id", "provider_session_id"].some(key => typeof binding[key] !== "string" || !binding[key])
|
|
12
|
+
|| !Number.isSafeInteger(binding.turn_generation) || binding.turn_generation < 0
|
|
13
|
+
|| !/^[a-f0-9]{64}$/.test(binding.command_sha256 ?? "")) throw Object.assign(new Error("Native hook has no current owned operation"), { code: "native_hook_unproven" });
|
|
14
|
+
const key = JSON.stringify([binding.provider_session_id, binding.tool_call_id, binding.event]);
|
|
15
|
+
const existing = current && [...current.requests.values()].find(value => JSON.stringify([value.binding.provider_session_id, value.binding.tool_call_id, value.binding.event]) === key)?.binding;
|
|
16
|
+
const same = value => {
|
|
17
|
+
if (value.command_sha256 !== binding.command_sha256 || value.native_cwd !== binding.native_cwd) throw Object.assign(new Error("Native tool identity was reused with changed hook input"), { code: "native_hook_identity_conflict" });
|
|
18
|
+
return value;
|
|
19
|
+
};
|
|
20
|
+
if (existing) return same(existing);
|
|
21
|
+
current && (current.accepting ??= new Map());
|
|
22
|
+
if (current?.accepting.has(key)) return same(await current.accepting.get(key));
|
|
23
|
+
const accepted = (async () => {
|
|
24
|
+
const directory = path.join(stateDir, "operations", createHash("sha256").update(binding.operation_id).digest("hex"));
|
|
25
|
+
const requested = JSON.parse(await readFile(path.join(directory, "requested.json"), "utf8"));
|
|
26
|
+
if (requested.operation_id !== binding.operation_id || requested.daemon_id !== binding.daemon_id || !["session.prompt", "session.create", "session.start"].includes(requested.operation)
|
|
27
|
+
|| requested.operation === "session.prompt" && requested.session_id && requested.session_id !== binding.root_provider_session_id) throw Object.assign(new Error("Native hook operation does not match its retained owner"), { code: "native_hook_unproven" });
|
|
28
|
+
const receipt = { ...binding, schema_id: "dd-flow/hook-request@1", request_id: randomUUID() };
|
|
29
|
+
await writeJsonAtomic(path.join(directory, "hooks", `${receipt.request_id}.json`), receipt);
|
|
30
|
+
return receipt;
|
|
31
|
+
})();
|
|
32
|
+
current?.accepting.set(key, accepted);
|
|
33
|
+
try { return await accepted; } finally { current?.accepting.delete(key); }
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export async function writeNativeHookFailure(stateDir, binding, error, budget) {
|
|
37
|
+
if (!binding?.request_id || !/^[A-Za-z0-9-]{1,128}$/.test(binding.request_id)) return;
|
|
38
|
+
await budget.run(() => writeJsonAtomic(path.join(stateDir, "hook-failures", `${binding.request_id}.json`), {
|
|
39
|
+
...errorRecord(error), request_id: binding.request_id, observed_at: new Date().toISOString(), native_hook_binding: binding,
|
|
40
|
+
}), "failure_receipt");
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** Read only requests accepted for this exact live operation, never scan a mailbox. */
|
|
44
|
+
export async function readNativeHookFailures(stateDir, requests, current, reject) {
|
|
45
|
+
for (const [id, accepted] of requests) {
|
|
46
|
+
if (accepted.current !== current) continue;
|
|
47
|
+
let value;
|
|
48
|
+
try { value = JSON.parse(await readFile(path.join(stateDir, "hook-failures", `${id}.json`), "utf8")); }
|
|
49
|
+
catch (error) { if (error.code === "ENOENT") continue; value = { code: "native_hook_receipt_invalid", message: error.message }; }
|
|
50
|
+
if (value.request_id !== id || !value.native_hook_binding || JSON.stringify(value.native_hook_binding) !== JSON.stringify(accepted.binding) || typeof value.code !== "string" || typeof value.message !== "string") {
|
|
51
|
+
if (!accepted.diagnostic) {
|
|
52
|
+
accepted.diagnostic = { code: "native_hook_receipt_invalid", message: "Accepted hook request has an unreadable or unbound failure receipt", request_id: id, operation_id: accepted.binding.operation_id, effect: "unknown", receipt_bound: false };
|
|
53
|
+
process.stderr.write(`${JSON.stringify({ native_hook_diagnostic: accepted.diagnostic })}\n`);
|
|
54
|
+
}
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
requests.delete(id);
|
|
58
|
+
const error = reportedError(value, "Native hook failed");
|
|
59
|
+
error.details = { ...error.details, native_hook_binding: accepted.binding };
|
|
60
|
+
reject(error);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
3
63
|
|
|
4
64
|
/** The caller supplies its qualified native limit; phases share that deadline. */
|
|
5
65
|
export function hookBudget(timeoutMs = 25_000, deadlineAt = process.env.DD_FLOW_HOOK_DEADLINE_AT === undefined ? null : Number(process.env.DD_FLOW_HOOK_DEADLINE_AT)) {
|
|
@@ -9,7 +69,7 @@ export function hookBudget(timeoutMs = 25_000, deadlineAt = process.env.DD_FLOW_
|
|
|
9
69
|
const remaining = (phase, maximum = duration) => {
|
|
10
70
|
const value = duration - (performance.now() - started);
|
|
11
71
|
if (value <= 0) throw Object.assign(new Error(`Native hook deadline exceeded during ${phase}`), { code: "native_hook_timeout", details: { phase, elapsed_ms: Math.round(performance.now() - started), effect: "unknown" } });
|
|
12
|
-
return Math.max(1, Math.min(maximum, value));
|
|
72
|
+
return Math.max(1, Math.floor(Math.min(maximum, value)));
|
|
13
73
|
};
|
|
14
74
|
return { deadlineAt: Math.floor(Date.now() + duration), remaining, async run(action, phase) {
|
|
15
75
|
const timeoutMs = remaining(phase);
|
|
@@ -57,26 +117,35 @@ export async function terminateCommandTree(child, { platform = process.platform,
|
|
|
57
117
|
}
|
|
58
118
|
|
|
59
119
|
/** Preserve the native hook protocol across the CLI process boundary. */
|
|
60
|
-
export async function runBoundedCommand({ bin, args, input = null, cwd, env = {}, timeoutMs = 15_000, timeoutCode = "native_hook_timeout" }) {
|
|
61
|
-
|
|
120
|
+
export async function runBoundedCommand({ bin, args, input = null, cwd, env = {}, timeoutMs = 15_000, timeoutCode = "native_hook_timeout", inheritDeadline = true, cleanupWithinBudget = false, maintenanceProgress = null, signal }) {
|
|
121
|
+
if (signal?.aborted) throw Object.assign(new Error("Bounded CLI request cancelled before launch"), { code: "process_maintenance_cancelled", details: { effect: "no_effect" } });
|
|
122
|
+
const inherited = inheritDeadline ? env.DD_FLOW_HOOK_DEADLINE_AT ?? process.env.DD_FLOW_HOOK_DEADLINE_AT : undefined;
|
|
62
123
|
if (inherited !== undefined) {
|
|
63
124
|
const deadline = Number(inherited);
|
|
64
125
|
if (!Number.isSafeInteger(deadline) || deadline < 0) throw Object.assign(new Error("Invalid native hook deadline"), { code: "hook_deadline_invalid", details: { effect: "no_effect" } });
|
|
65
126
|
timeoutMs = Math.min(timeoutMs, deadline - Date.now());
|
|
66
127
|
}
|
|
67
128
|
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) throw Object.assign(new Error("Native hook budget exhausted before subprocess launch"), { code: timeoutCode, details: { phase: "subprocess", effect: "no_effect" } });
|
|
129
|
+
if (maintenanceProgress && (!Number.isFinite(maintenanceProgress.inactivityMs) || maintenanceProgress.inactivityMs <= 0
|
|
130
|
+
|| typeof maintenanceProgress.action !== "string" || !(maintenanceProgress.processId === null || typeof maintenanceProgress.processId === "string")))
|
|
131
|
+
throw Object.assign(new Error("Invalid maintenance progress budget"), { code: "process_maintenance_timeout", details: { effect: "no_effect", native_dispatch_started: false } });
|
|
68
132
|
const started = performance.now();
|
|
69
133
|
// Transport/physical cleanup has its own small tail inside the outer limit.
|
|
70
|
-
const
|
|
134
|
+
const cleanupMs = Math.min(500, timeoutMs / 5);
|
|
135
|
+
const commandMs = cleanupWithinBudget ? Math.max(1, timeoutMs - cleanupMs) : timeoutMs;
|
|
136
|
+
const deadlineAt = Math.floor(Date.now() + Math.max(0, commandMs - Math.min(250, commandMs / 10)));
|
|
71
137
|
const script = /\.[cm]?js$/.test(bin);
|
|
72
138
|
const output = await new Promise((resolve, reject) => {
|
|
73
139
|
const child = spawn(script ? process.execPath : bin, [...(script ? [bin] : []), ...args], {
|
|
74
140
|
cwd, env: { ...process.env, ...env, DD_FLOW_HOOK_DEADLINE_AT: String(deadlineAt) }, detached: process.platform !== "win32", stdio: ["pipe", "pipe", "pipe"]
|
|
75
141
|
});
|
|
76
142
|
let stdout = "", stderr = "", failure, cleanupTimer, settled = false, termination;
|
|
143
|
+
let timer, phaseBuffer = "", lastProgress = started;
|
|
144
|
+
const progressPhases = new Set();
|
|
145
|
+
const commandDeadline = started + commandMs;
|
|
77
146
|
const finish = (error, value) => {
|
|
78
147
|
if (settled) return;
|
|
79
|
-
settled = true; clearTimeout(timer); clearTimeout(cleanupTimer);
|
|
148
|
+
settled = true; clearTimeout(timer); clearTimeout(cleanupTimer); signal?.removeEventListener("abort", abort);
|
|
80
149
|
if (error) {
|
|
81
150
|
const diagnostics = hookDiagnostics(stderr);
|
|
82
151
|
if (diagnostics.length) error.details = { ...error.details, hook_diagnostics: diagnostics };
|
|
@@ -86,7 +155,8 @@ export async function runBoundedCommand({ bin, args, input = null, cwd, env = {}
|
|
|
86
155
|
const stop = error => {
|
|
87
156
|
if (termination || settled) return;
|
|
88
157
|
failure ??= error;
|
|
89
|
-
|
|
158
|
+
clearTimeout(timer);
|
|
159
|
+
failure.details = { ...failure.details, elapsed_ms: Math.round(performance.now() - started), effect: "unknown", subprocess_pid: child.pid ?? null };
|
|
90
160
|
termination = terminateCommandTree(child).catch(cleanup => {
|
|
91
161
|
if (cleanup.code === "ESRCH") return;
|
|
92
162
|
failure.details.cleanup_error = { code: cleanup.code, message: cleanup.message };
|
|
@@ -97,11 +167,45 @@ export async function runBoundedCommand({ bin, args, input = null, cwd, env = {}
|
|
|
97
167
|
failure.details.cleanup_unconfirmed = true;
|
|
98
168
|
child.stdin.destroy(); child.stdout.destroy(); child.stderr.destroy(); child.unref();
|
|
99
169
|
finish(failure);
|
|
100
|
-
}, 500);
|
|
170
|
+
}, cleanupWithinBudget ? cleanupMs : 500);
|
|
101
171
|
};
|
|
102
|
-
|
|
172
|
+
// A useful, request-bound phase advances the silence window, not ownership
|
|
173
|
+
// authority. The child storage deadline and parent uncertainty horizon stay
|
|
174
|
+
// fixed; repeated phases, logging and polling cannot keep a writer alive.
|
|
175
|
+
const arm = () => {
|
|
176
|
+
clearTimeout(timer);
|
|
177
|
+
const now = performance.now();
|
|
178
|
+
const remaining = Math.min(commandDeadline - now, maintenanceProgress ? lastProgress + maintenanceProgress.inactivityMs - now : Infinity);
|
|
179
|
+
if (remaining <= 0) { stop(Object.assign(new Error("Bounded CLI request timed out"), { code: timeoutCode })); return; }
|
|
180
|
+
timer = setTimeout(arm, Math.ceil(remaining));
|
|
181
|
+
};
|
|
182
|
+
arm();
|
|
183
|
+
const abort = () => stop(Object.assign(new Error("Bounded CLI request cancelled"), { code: "process_maintenance_cancelled" }));
|
|
184
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
185
|
+
if (signal?.aborted) abort();
|
|
103
186
|
child.stdout.setEncoding("utf8").on("data", chunk => { stdout += chunk; if (stdout.length > 1_048_576) stop(new Error("Native hook response exceeds protocol limit")); });
|
|
104
|
-
|
|
187
|
+
// Match the existing response bound: a legal bounded error tree may exceed
|
|
188
|
+
// 64KiB when pretty-printed, plus maintenance phases preceding the receipt.
|
|
189
|
+
child.stderr.setEncoding("utf8").on("data", chunk => {
|
|
190
|
+
stderr = (stderr + chunk).slice(-1_048_576);
|
|
191
|
+
if (!maintenanceProgress || termination || settled) return;
|
|
192
|
+
phaseBuffer += chunk;
|
|
193
|
+
const lines = phaseBuffer.split("\n"); phaseBuffer = lines.pop().slice(-16_384);
|
|
194
|
+
for (const line of lines) {
|
|
195
|
+
if (line.length > 16_384) continue;
|
|
196
|
+
let phase; try { phase = JSON.parse(line); } catch { continue; }
|
|
197
|
+
if (phase?.kind !== "process_maintenance_phase" || phase.action !== maintenanceProgress.action
|
|
198
|
+
|| (phase.process_id ?? null) !== maintenanceProgress.processId || phase.native_dispatch_started !== false
|
|
199
|
+
|| !["request", "store_open", "store_validation", "lookup", "writer_reservation", "writer_reserved", "write", "commit", "committed", "ack"].includes(phase.phase)
|
|
200
|
+
|| phase.store_kind !== undefined && !["run", "resource"].includes(phase.store_kind)) continue;
|
|
201
|
+
const key = `${phase.store_kind ?? ""}:${phase.phase}`;
|
|
202
|
+
if (progressPhases.has(key)) continue;
|
|
203
|
+
// Delayed observation cannot revive an already-expired silence window.
|
|
204
|
+
const now = performance.now();
|
|
205
|
+
if (now >= commandDeadline || now - lastProgress >= maintenanceProgress.inactivityMs) { arm(); return; }
|
|
206
|
+
progressPhases.add(key); lastProgress = now; arm();
|
|
207
|
+
}
|
|
208
|
+
});
|
|
105
209
|
child.stdin.on("error", error => { failure ??= error; });
|
|
106
210
|
child.on("error", error => finish(failure ?? error));
|
|
107
211
|
child.on("close", code => {
|
|
@@ -121,10 +225,10 @@ function hookDiagnostics(stderr) {
|
|
|
121
225
|
return stderr.split("\n").flatMap(line => {
|
|
122
226
|
try {
|
|
123
227
|
const value = JSON.parse(line);
|
|
124
|
-
if (value?.kind
|
|
228
|
+
if (!["native_hook_phase", "process_maintenance_phase"].includes(value?.kind)) return [];
|
|
125
229
|
const phase = typeof value.phase === "string" && /^[a-z_]{1,48}$/.test(value.phase) ? value.phase : null;
|
|
126
230
|
if (!phase) return [];
|
|
127
|
-
return [{ phase, ...Object.fromEntries(["elapsed_ms", "remaining_ms", "sqlite_extended_code"].filter(key => Number.isFinite(value[key])).map(key => [key, value[key]])) }];
|
|
231
|
+
return [{ phase, ...Object.fromEntries(["elapsed_ms", "remaining_ms", "sqlite_extended_code"].filter(key => Number.isFinite(value[key])).map(key => [key, value[key]])), ...(value.kind === "process_maintenance_phase" ? { ...Object.fromEntries(["action", "process_id", "operation_id", "lease_expires_at", "store_kind", "maintenance_effect", "attempt_kind", "role"].filter(key => typeof value[key] === "string" && /^[A-Za-z0-9:._-]{1,128}$/.test(value[key])).map(key => [key, value[key]])), ...(typeof value.native_dispatch_started === "boolean" ? { native_dispatch_started: value.native_dispatch_started } : {}) } : {}) }];
|
|
128
232
|
} catch { return []; }
|
|
129
233
|
}).slice(-16);
|
|
130
234
|
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// Async iteration backpressures the native pipe while its existing evidence
|
|
2
|
+
// sink drains. Bound one JSON reply, never lifetime diagnostics.
|
|
3
|
+
export const MAX_NATIVE_LINE_BYTES = 8 * 1024 * 1024;
|
|
4
|
+
export async function readNativeLines(stream, onLine, onError) {
|
|
5
|
+
let buffer = "";
|
|
6
|
+
stream.setEncoding("utf8");
|
|
7
|
+
try {
|
|
8
|
+
for await (const chunk of stream.iterator({ destroyOnReturn: false })) {
|
|
9
|
+
buffer += chunk;
|
|
10
|
+
let at;
|
|
11
|
+
while ((at = buffer.indexOf("\n")) >= 0) {
|
|
12
|
+
const line = buffer.slice(0, at); buffer = buffer.slice(at + 1);
|
|
13
|
+
assertLine(line);
|
|
14
|
+
if (line.trim()) await onLine(line);
|
|
15
|
+
}
|
|
16
|
+
assertLine(buffer);
|
|
17
|
+
}
|
|
18
|
+
if (buffer.trim()) { assertLine(buffer); await onLine(buffer); }
|
|
19
|
+
} catch (cause) { stream.pause(); await observationFailure(cause, onError); }
|
|
20
|
+
}
|
|
21
|
+
export function assertNativeLine(line, phase = "reply") {
|
|
22
|
+
if (Buffer.byteLength(line) > MAX_NATIVE_LINE_BYTES) throw Object.assign(new Error(`Native JSON ${phase} exceeds the bounded capture limit`), { code: phase === "request" ? "native_request_limit" : "operation_observation_lost", details: { reason: `native_${phase}_limit`, limit_bytes: MAX_NATIVE_LINE_BYTES, ...(phase === "request" ? { effect: "no_effect" } : { native_outcome: "unknown" }) } });
|
|
23
|
+
}
|
|
24
|
+
const assertLine = assertNativeLine;
|
|
25
|
+
export function admitNativeRequestBuffer(buffer, socket, schema) {
|
|
26
|
+
try { assertNativeLine(buffer, "request"); return true; }
|
|
27
|
+
catch (error) { socket.pause(); socket.end(`${JSON.stringify({ schema_id: schema, id: null, ok: false, error: { code: error.code, message: error.message, details: error.details } })}\n`, () => socket.destroy()); return false; }
|
|
28
|
+
}
|
|
29
|
+
export function retainNativeReplyBuffer(buffer, finish) {
|
|
30
|
+
try { assertNativeLine(buffer); return true; }
|
|
31
|
+
catch (error) { finish(error); return false; }
|
|
32
|
+
}
|
|
33
|
+
export async function streamNativeDiagnostics(stream, onChunk, onError) {
|
|
34
|
+
stream.setEncoding("utf8");
|
|
35
|
+
try { for await (const chunk of stream.iterator({ destroyOnReturn: false })) await onChunk(String(chunk)); }
|
|
36
|
+
catch (cause) { stream.pause(); await observationFailure(cause, onError); }
|
|
37
|
+
}
|
|
38
|
+
async function observationFailure(cause, onError) {
|
|
39
|
+
const error = nativeObservationError(cause);
|
|
40
|
+
try { await onError(error); }
|
|
41
|
+
catch (publication) { error.cleanup_error ??= { code: publication.code ?? null, message: publication.message }; }
|
|
42
|
+
}
|
|
43
|
+
export function nativeObservationError(cause, details = {}) {
|
|
44
|
+
return !cause.code || /^[A-Z_]+$/.test(cause.code) ? Object.assign(new Error("Native evidence reader failed", { cause }), { code: "operation_observation_lost", details: { native_outcome: "unknown", observation_error: { code: cause.code ?? null, message: cause.message }, ...details } }) : cause;
|
|
45
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export const OBSERVATION_CLOCK_CONTRACT_VERSION: "operation-progress@1";
|
|
2
|
+
export function validateObservationDuration(value: number, label?: string): number;
|
|
3
|
+
export interface ObservationState {
|
|
4
|
+
policy: string; timeout_ms: number; started_at: number; cursor?: unknown;
|
|
5
|
+
observed_at: number | null; elapsed_ms: number; uncertainty_elapsed_ms: number;
|
|
6
|
+
gaps: number; observation_lost: boolean;
|
|
7
|
+
}
|
|
8
|
+
export class ObservationClock {
|
|
9
|
+
constructor(options: { timeoutMs: number; gapMs?: number; uncertaintyMs?: number;
|
|
10
|
+
wall?: () => number; monotonic?: () => number; onGap?: (value: Record<string, unknown>) => void;
|
|
11
|
+
saved?: ObservationState | null });
|
|
12
|
+
observationLost: boolean;
|
|
13
|
+
sample(progress?: unknown, progressAt?: number): boolean;
|
|
14
|
+
state(): ObservationState;
|
|
15
|
+
}
|
|
16
|
+
export function observedTimeout(callback: (clock: ObservationClock) => void, timeoutMs: number,
|
|
17
|
+
options?: { gapMs?: number; uncertaintyMs?: number; wall?: () => number; monotonic?: () => number;
|
|
18
|
+
onGap?: (value: Record<string, unknown>) => void; progress?: () => unknown; saved?: ObservationState | null }): ReturnType<typeof setInterval>;
|
|
@@ -1,36 +1,92 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
export const OBSERVATION_CLOCK_CONTRACT_VERSION = "operation-progress@1";
|
|
2
|
+
export function validateObservationDuration(value, label = "observation duration") {
|
|
3
|
+
if (!Number.isSafeInteger(value) || value < 1 || value > 2_147_483_647)
|
|
4
|
+
throw new RangeError(`${label} must be integer milliseconds in 1..2147483647`);
|
|
5
|
+
return value;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
/** Only observed monotonic time is silence. A clock gap is uncertainty, never
|
|
9
|
+
* progress or proof of provider failure. Durable snapshots retain that episode. */
|
|
3
10
|
export class ObservationClock {
|
|
4
|
-
constructor({ timeoutMs, gapMs = 60_000,
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
this
|
|
11
|
+
constructor({ timeoutMs, gapMs = 60_000, uncertaintyMs = 120_000, wall = Date.now,
|
|
12
|
+
monotonic = () => performance.now(), onGap = () => {}, saved = null }) {
|
|
13
|
+
validateObservationDuration(timeoutMs); validateObservationDuration(gapMs, "gap duration");
|
|
14
|
+
validateObservationDuration(uncertaintyMs, "observation recovery duration");
|
|
15
|
+
Object.assign(this, { timeoutMs, gapMs, uncertaintyMs, wall, monotonic, onGap });
|
|
16
|
+
this.lastWall = wall(); this.lastMono = monotonic(); this.startedWall = this.lastWall;
|
|
17
|
+
this.elapsed = 0; this.progress = undefined; this.progressAt = null;
|
|
18
|
+
this.uncertain = false; this.uncertaintyElapsed = 0; this.gaps = 0; this.observationLost = false;
|
|
19
|
+
if (saved) {
|
|
20
|
+
if (saved.policy !== OBSERVATION_CLOCK_CONTRACT_VERSION || saved.timeout_ms !== timeoutMs
|
|
21
|
+
|| !Number.isFinite(saved.started_at) || !Number.isSafeInteger(saved.gaps) || saved.gaps < 0
|
|
22
|
+
|| saved.observed_at !== null && !Number.isFinite(saved.observed_at)
|
|
23
|
+
|| typeof saved.observation_lost !== "boolean"
|
|
24
|
+
|| !Number.isFinite(saved.elapsed_ms) || saved.elapsed_ms < 0
|
|
25
|
+
|| !Number.isFinite(saved.uncertainty_elapsed_ms) || saved.uncertainty_elapsed_ms < 0)
|
|
26
|
+
throw new RangeError("invalid retained observation clock");
|
|
27
|
+
this.startedWall = saved.started_at; this.elapsed = saved.elapsed_ms;
|
|
28
|
+
this.progress = saved.cursor; this.progressAt = saved.observed_at;
|
|
29
|
+
// A reconstructed observer cannot prove silence during its absence.
|
|
30
|
+
this.uncertain = true; this.gaps = saved.gaps + 1;
|
|
31
|
+
this.uncertaintyElapsed = saved.uncertainty_elapsed_ms;
|
|
32
|
+
this.observationLost = saved.observation_lost || this.gaps > 1 || this.uncertaintyElapsed >= uncertaintyMs;
|
|
33
|
+
}
|
|
9
34
|
}
|
|
10
35
|
sample(progress, progressAt) {
|
|
36
|
+
if (progress && typeof progress === "object" && Object.hasOwn(progress, "cursor")) {
|
|
37
|
+
progressAt = progress.observed_at; progress = progress.cursor;
|
|
38
|
+
}
|
|
11
39
|
const wall = this.wall(), mono = this.monotonic();
|
|
12
40
|
const delta = mono - this.lastMono, wallDelta = wall - this.lastWall;
|
|
13
|
-
const gap = delta
|
|
41
|
+
const gap = !Number.isFinite(delta) || !Number.isFinite(wallDelta)
|
|
42
|
+
|| delta > this.gapMs || wallDelta > this.gapMs || delta < 0 || wallDelta < 0;
|
|
14
43
|
if (gap) {
|
|
44
|
+
this.uncertain = true; this.gaps++;
|
|
15
45
|
this.onGap({ source: "observer_clock", classification: "observation_gap", confirmed_sleep: false,
|
|
16
|
-
started_at:
|
|
46
|
+
started_at: Number.isFinite(this.lastWall) ? this.lastWall : null, observed_at: Number.isFinite(wall) ? wall : null,
|
|
17
47
|
wall_delta_ms: wallDelta, monotonic_delta_ms: delta });
|
|
18
|
-
|
|
19
|
-
|
|
48
|
+
} else {
|
|
49
|
+
this.elapsed += delta;
|
|
50
|
+
if (this.uncertain) this.uncertaintyElapsed += delta;
|
|
51
|
+
}
|
|
20
52
|
this.lastWall = wall; this.lastMono = mono;
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
53
|
+
const validCursor = typeof progress === "string" || typeof progress === "boolean"
|
|
54
|
+
|| typeof progress === "number" && Number.isFinite(progress);
|
|
55
|
+
const changed = validCursor && progress !== this.progress;
|
|
56
|
+
const timestamped = progressAt !== undefined;
|
|
57
|
+
const validTime = !timestamped ? wall >= this.startedWall && (this.progressAt === null || wall >= this.progressAt)
|
|
58
|
+
: Number.isFinite(progressAt) && progressAt >= this.startedWall
|
|
59
|
+
&& progressAt <= wall && (this.progressAt === null || progressAt >= this.progressAt);
|
|
60
|
+
if (changed && validTime && !this.observationLost) {
|
|
61
|
+
const eventAt = timestamped ? progressAt : wall;
|
|
62
|
+
// A delayed pre-gap event cannot establish that observation has recovered.
|
|
63
|
+
const recovers = !this.uncertain || eventAt >= wall - Math.max(0, gap ? 0 : delta);
|
|
64
|
+
this.progress = progress; this.progressAt = eventAt;
|
|
65
|
+
this.elapsed = Math.max(0, wall - eventAt);
|
|
66
|
+
if (recovers) { this.uncertain = false; this.gaps = 0; this.uncertaintyElapsed = 0; }
|
|
67
|
+
}
|
|
68
|
+
if (changed && timestamped && Number.isFinite(progressAt) && !validTime && progressAt > wall) {
|
|
69
|
+
if (!this.uncertain) this.onGap({ source: "observer_clock", classification: "clock_skew",
|
|
70
|
+
confirmed_sleep: false, observed_at: wall, signal_at: progressAt });
|
|
71
|
+
this.uncertain = true;
|
|
24
72
|
}
|
|
25
|
-
|
|
73
|
+
if (this.uncertain && (this.gaps > 1 || this.uncertaintyElapsed >= this.uncertaintyMs)) this.observationLost = true;
|
|
74
|
+
return this.observationLost || !this.uncertain && this.elapsed >= this.timeoutMs;
|
|
75
|
+
}
|
|
76
|
+
state() {
|
|
77
|
+
return { policy: OBSERVATION_CLOCK_CONTRACT_VERSION, timeout_ms: this.timeoutMs,
|
|
78
|
+
started_at: this.startedWall, cursor: this.progress, observed_at: this.progressAt,
|
|
79
|
+
elapsed_ms: this.elapsed, uncertainty_elapsed_ms: this.uncertaintyElapsed,
|
|
80
|
+
gaps: this.gaps, observation_lost: this.observationLost };
|
|
26
81
|
}
|
|
27
82
|
}
|
|
28
83
|
|
|
29
|
-
/** Node
|
|
84
|
+
/** Node timer handle, compatible with existing clearTimeout cleanup. */
|
|
30
85
|
export function observedTimeout(callback, timeoutMs, options = {}) {
|
|
31
86
|
const clock = new ObservationClock({ timeoutMs, ...options });
|
|
32
87
|
const timer = setInterval(() => {
|
|
33
|
-
|
|
88
|
+
const snapshot = options.progress?.();
|
|
89
|
+
if (clock.sample(snapshot)) { clearInterval(timer); callback(clock); }
|
|
34
90
|
}, Math.max(1, Math.min(1_000, timeoutMs / 4)));
|
|
35
91
|
return timer;
|
|
36
92
|
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export const OPERATION_ERROR_CONTRACT_VERSION: "operation-errors@2";
|
|
2
|
+
export interface ErrorRecord {
|
|
3
|
+
code: string;
|
|
4
|
+
message: string;
|
|
5
|
+
retryable?: boolean;
|
|
6
|
+
exitCode?: number;
|
|
7
|
+
details?: unknown;
|
|
8
|
+
cause?: ErrorRecord;
|
|
9
|
+
cleanup_error?: ErrorRecord;
|
|
10
|
+
}
|
|
11
|
+
export function errorRecord(error: unknown, depth?: number): ErrorRecord;
|
|
12
|
+
export function isObservationLoss(error: unknown): boolean;
|
|
13
|
+
export function isProvenShellCorrection(error: unknown): boolean;
|
|
14
|
+
export function reportedError(value: Record<string, unknown>, fallback: string): Error & ErrorRecord;
|
|
@@ -1,16 +1,113 @@
|
|
|
1
1
|
// An observer losing its response does not establish that a provider Turn failed.
|
|
2
|
+
export const OPERATION_ERROR_CONTRACT_VERSION = "operation-errors@2";
|
|
3
|
+
// Only use after the native delivery/operation owner has been verified.
|
|
4
|
+
export function isProvenShellCorrection(error) {
|
|
5
|
+
const details = error?.details;
|
|
6
|
+
return error?.code === "lifecycle_shell_syntax_invalid" && details?.phase === "prepare"
|
|
7
|
+
&& details.effect === "no_effect" && details.recoverable === true
|
|
8
|
+
&& details.disposition === "pre_cli_shell_refusal" && details.cli_admission_barrier === true
|
|
9
|
+
&& typeof details.issued_invocation_id === "string" && details.issued_invocation_id.trim().length > 0
|
|
10
|
+
&& typeof details.event_key === "string" && details.event_key.trim().length > 0;
|
|
11
|
+
}
|
|
2
12
|
export function isObservationLoss(error) {
|
|
3
|
-
return ["rpc_timeout", "native_timeout", "native_outcome_unknown", "native_backend_dead", "native_backend_pipe_broken", "bridge_exited", "bridge_pipe_broken", "daemon_timeout", "turn_timeout", "operation_observation_lost", "daemon_connection_closed"].includes(error?.code);
|
|
13
|
+
return ["rpc_timeout", "native_timeout", "native_outcome_unknown", "native_backend_dead", "native_backend_pipe_broken", "bridge_exited", "bridge_pipe_broken", "daemon_timeout", "turn_timeout", "operation_observation_lost", "daemon_connection_closed", "harness_adapter_timeout", "harness_adapter_output_limit", "harness_adapter_invalid", "subject_liveness_timeout", "command_observation_lost", "command_output_limit", "operation_output_limit"].includes(error?.code);
|
|
4
14
|
}
|
|
5
15
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
16
|
+
function field(value, key) { try { return value?.[key]; } catch { return "[unreadable]"; } }
|
|
17
|
+
function text(value) { try { return String(value); } catch { return "[unreadable]"; } }
|
|
18
|
+
function boundedText(value, budget, limit = 16000) {
|
|
19
|
+
let result = value.slice(0, limit);
|
|
20
|
+
if (Buffer.byteLength(JSON.stringify(result)) > budget.remaining) result = result.slice(0, Math.max(0, Math.floor(budget.remaining / 6)));
|
|
21
|
+
budget.remaining -= Buffer.byteLength(JSON.stringify(result));
|
|
22
|
+
return result;
|
|
23
|
+
}
|
|
24
|
+
// Lifecycle controls are not optional diagnostics. Reserve them before bulk
|
|
25
|
+
// messages/details, including nested typed causes, without copying authority
|
|
26
|
+
// that was truncated into a different command or continuation receipt.
|
|
27
|
+
const operationalDetailKeys = ["phase", "effect", "recoverable", "retryable_no_effect", "handoff_required", "storage_retry", "repeated_without_progress", "cleanup_unconfirmed", "native_dispatch_started", "disposition", "cli_admission_barrier", "native_pre_cli_refusal", "issued_invocation_id", "invocation_id", "event_key", "refusal_event_key", "hook_event_id", "work_id", "hook_retry_signature", "rejection_signature", "retry_command", "repair_command", "continuation", "retry_instruction", "recovery_instruction", "authority_truncated"];
|
|
28
|
+
function operationalDetails(value, budget) {
|
|
29
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
30
|
+
const entries = [];
|
|
31
|
+
let authorityTruncated = false;
|
|
32
|
+
for (const key of operationalDetailKeys) {
|
|
33
|
+
try { if (!Object.hasOwn(value, key)) continue; } catch { continue; }
|
|
34
|
+
const raw = field(value, key), reserved = { ...budget };
|
|
35
|
+
const name = boundedText(key, reserved), result = jsonDiagnostic(raw, 0, new Set(), reserved);
|
|
36
|
+
if (name !== key || typeof raw === "string" && result !== raw
|
|
37
|
+
|| key === "continuation" && JSON.stringify(result).includes("[truncated]")) {
|
|
38
|
+
authorityTruncated ||= ["retry_command", "repair_command", "continuation"].includes(key);
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
Object.assign(budget, reserved);
|
|
42
|
+
entries.push([key, result]);
|
|
43
|
+
}
|
|
44
|
+
return { ...Object.fromEntries(entries), ...(authorityTruncated ? { recoverable: false, authority_truncated: true } : {}) };
|
|
45
|
+
}
|
|
46
|
+
function jsonDiagnostic(value, depth, seen, budget, excludedKeys = []) {
|
|
47
|
+
if (depth >= 12 || budget.remaining <= 0 || budget.nodes-- <= 0) return "[truncated]";
|
|
48
|
+
if (typeof value === "string") { const result = boundedText(value, budget); return result.length === value.length ? result : `${result}[truncated]`; }
|
|
49
|
+
if (value === null || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value)) { budget.remaining -= String(value).length + 1; return value; }
|
|
50
|
+
if (typeof value !== "object") return "[unavailable]";
|
|
51
|
+
if (seen.has(value)) return "[circular]";
|
|
52
|
+
seen.add(value); budget.remaining -= 2;
|
|
53
|
+
let result;
|
|
54
|
+
try {
|
|
55
|
+
if (Array.isArray(value)) {
|
|
56
|
+
result = [];
|
|
57
|
+
for (let i = 0; i < Math.min(value.length, 256); i++) {
|
|
58
|
+
if (budget.remaining <= 0 || budget.nodes <= 0) { result.push("[truncated]"); break; }
|
|
59
|
+
result.push(jsonDiagnostic(field(value, i), depth + 1, seen, budget));
|
|
60
|
+
}
|
|
61
|
+
if (value.length > 256) result.push("[truncated]");
|
|
62
|
+
} else {
|
|
63
|
+
const entries = [];
|
|
64
|
+
// Error.message and the standard Error.cause are non-enumerable.
|
|
65
|
+
// Keep the original identity and shared bounds for nested diagnostics.
|
|
66
|
+
const errorFields = value instanceof Error ? ["message", "cause"].filter(key => field(value, key) !== undefined) : [];
|
|
67
|
+
for (const key of errorFields) {
|
|
68
|
+
if (budget.remaining <= 0 || budget.nodes <= 0) { entries.push(["[truncated]", true]); break; }
|
|
69
|
+
entries.push([boundedText(key, budget), jsonDiagnostic(field(value, key), depth + 1, seen, budget)]);
|
|
70
|
+
}
|
|
71
|
+
for (const key in value) {
|
|
72
|
+
if (!Object.hasOwn(value, key) || errorFields.includes(key) || excludedKeys.includes(key)) continue;
|
|
73
|
+
if (entries.length >= 256 || budget.remaining <= 0 || budget.nodes <= 0) { entries.push(["[truncated]", true]); break; }
|
|
74
|
+
const name = boundedText(key, budget);
|
|
75
|
+
entries.push([name, jsonDiagnostic(field(value, key), depth + 1, seen, budget)]);
|
|
76
|
+
}
|
|
77
|
+
result = Object.fromEntries(entries);
|
|
78
|
+
}
|
|
79
|
+
} catch { result = "[unreadable]"; }
|
|
80
|
+
seen.delete(value); return result;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function record(error, depth, seen, budget, pending) {
|
|
84
|
+
if (error && typeof error === "object") {
|
|
85
|
+
if (seen.has(error)) return { code: "diagnostic_cycle", message: "[circular]" };
|
|
86
|
+
seen.add(error);
|
|
87
|
+
}
|
|
88
|
+
const value = Object.fromEntries(["code", "message", "retryable", "exitCode", "details", "cause", "cleanup_error"].map(key => [key, field(error, key)]));
|
|
89
|
+
const result = {
|
|
90
|
+
code: boundedText(typeof value.code === "string" ? value.code : "operation_failed", budget),
|
|
91
|
+
...(typeof value.retryable === "boolean" ? { retryable: value.retryable } : {}),
|
|
92
|
+
...(typeof value.exitCode === "number" && Number.isFinite(value.exitCode) ? { exitCode: value.exitCode } : {}),
|
|
13
93
|
};
|
|
94
|
+
pending.push([result, typeof value.message === "string" ? value.message : text(error), value.details]);
|
|
95
|
+
if (value.cause && value.cause !== "[unreadable]" && depth < 3) result.cause = record(value.cause, depth + 1, new Set(seen), budget, pending);
|
|
96
|
+
if (value.cleanup_error && value.cleanup_error !== "[unreadable]" && depth < 3) result.cleanup_error = record(value.cleanup_error, depth + 1, new Set(seen), budget, pending);
|
|
97
|
+
return result;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export function errorRecord(error, depth = 0, seen = new Set()) {
|
|
101
|
+
const budget = { remaining: 65536, nodes: 2048 }, pending = [];
|
|
102
|
+
// Reserve every typed code before messages and optional bulk details.
|
|
103
|
+
const result = record(error, depth, seen, budget, pending);
|
|
104
|
+
const controls = pending.map(([, , value]) => operationalDetails(value, budget));
|
|
105
|
+
for (const [target, message] of pending) target.message = boundedText(message, budget);
|
|
106
|
+
for (const [index, [target, , value]] of pending.entries()) if (value !== undefined) {
|
|
107
|
+
const details = jsonDiagnostic(value, 0, new Set(), budget, controls[index] ? operationalDetailKeys : []);
|
|
108
|
+
target.details = controls[index] ? { ...controls[index], ...(details && typeof details === "object" ? details : { "[truncated]": true }) } : details;
|
|
109
|
+
}
|
|
110
|
+
return result;
|
|
14
111
|
}
|
|
15
112
|
|
|
16
113
|
export function reportedError(value, fallback) {
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export const OPERATION_PROGRESS_CONTRACT: "dd-flow/operation-progress@1";
|
|
2
|
+
export function operationDirectory(root: string, id: string): string;
|
|
3
|
+
export function withOperationProgress<T>(root: string, id: string, action: () => T): T;
|
|
4
|
+
export function readOperationRecord(root: string, id: string, name: string): Promise<Record<string, any> | null>;
|
|
5
|
+
export function readOperationProgress(root: string, id: string, expected?: Record<string, unknown>): Promise<(Record<string, unknown> & { sequence: number; observed_at: string }) | null>;
|
|
6
|
+
export function readOperationWaiting(root: string, id: string, expected?: Record<string, unknown>): Promise<boolean>;
|
|
7
|
+
export function suspendOperationClock(clock: import("./observation-clock.mjs").ObservationClock): void;
|