@deksden-com/dd-flow-cli 0.9.0-beta.9 → 0.9.0-beta.90

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/CHANGELOG.md +460 -0
  2. package/README.md +65 -0
  3. package/dist/build-info.json +8 -8
  4. package/dist/cli/command-inputs.js +304 -0
  5. package/dist/cli/help.js +89 -13
  6. package/dist/cli/hook-ingress.js +82 -0
  7. package/dist/cli/input-preparation.js +115 -0
  8. package/dist/cli/run-cli.js +1409 -382
  9. package/dist/cli.js +6 -2
  10. package/dist/harness-runtime/bin/dd-agy.mjs +37 -0
  11. package/dist/harness-runtime/bin/dd-codex.mjs +25 -0
  12. package/dist/harness-runtime/bin/dd-droid.mjs +33 -0
  13. package/dist/harness-runtime/bin/dd-grok.mjs +30 -0
  14. package/dist/harness-runtime/bin/dd-opencode.mjs +21 -0
  15. package/dist/harness-runtime/bin/dd-zcode.mjs +83 -0
  16. package/dist/harness-runtime/lib/daemon-operations.mjs +215 -0
  17. package/dist/harness-runtime/lib/dd-agy-daemon.mjs +423 -0
  18. package/dist/harness-runtime/lib/dd-agy.mjs +59 -0
  19. package/dist/harness-runtime/lib/dd-codex-daemon.d.mts +1 -0
  20. package/dist/harness-runtime/lib/dd-codex-daemon.mjs +146 -0
  21. package/dist/harness-runtime/lib/dd-codex.mjs +562 -0
  22. package/dist/harness-runtime/lib/dd-droid-daemon.mjs +125 -0
  23. package/dist/harness-runtime/lib/dd-droid.mjs +471 -0
  24. package/dist/harness-runtime/lib/dd-grok-daemon.mjs +274 -0
  25. package/dist/harness-runtime/lib/dd-grok.mjs +174 -0
  26. package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +198 -0
  27. package/dist/harness-runtime/lib/dd-opencode.mjs +98 -0
  28. package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +644 -0
  29. package/dist/harness-runtime/lib/dd-zcode.mjs +910 -0
  30. package/dist/harness-runtime/lib/delegation-instructions.d.mts +10 -0
  31. package/dist/harness-runtime/lib/delegation-instructions.mjs +130 -0
  32. package/dist/harness-runtime/lib/dispatch-fence.mjs +14 -0
  33. package/dist/harness-runtime/lib/driver-recovery.mjs +129 -0
  34. package/dist/harness-runtime/lib/droid-observation.mjs +66 -0
  35. package/dist/harness-runtime/lib/managed-daemon.mjs +260 -0
  36. package/dist/harness-runtime/lib/model-observations.mjs +77 -0
  37. package/dist/harness-runtime/lib/native-hook-command.d.mts +1 -0
  38. package/dist/harness-runtime/lib/native-hook-command.mjs +34 -0
  39. package/dist/harness-runtime/lib/observation-clock.mjs +36 -0
  40. package/dist/harness-runtime/lib/operation-context.mjs +5 -0
  41. package/dist/harness-runtime/lib/operation-errors.mjs +19 -0
  42. package/dist/harness-runtime/lib/process-json.mjs +70 -0
  43. package/dist/harness-runtime/lib/process-snapshot.mjs +10 -0
  44. package/dist/harness-runtime/lib/runner-events.mjs +200 -0
  45. package/dist/harness-runtime/lib/runner-lock.mjs +42 -0
  46. package/dist/harness-runtime/lib/session-settlement.mjs +30 -0
  47. package/dist/harness-runtime/lib/tool-observations.d.mts +17 -0
  48. package/dist/harness-runtime/lib/tool-observations.mjs +153 -0
  49. package/dist/schemas/agent-profile.schema.json +1 -1
  50. package/dist/schemas/code-review-result.schema.json +2 -2
  51. package/dist/schemas/code-work-batch.schema.json +4 -3
  52. package/dist/schemas/code-work-result.schema.json +4 -4
  53. package/dist/schemas/plan-review-result.schema.json +2 -2
  54. package/dist/schemas/run-control-receipt.schema.json +99 -0
  55. package/dist/schemas/run-control-request.schema.json +36 -0
  56. package/dist/schemas/vnext-protocol-plan.schema.json +2 -2
  57. package/dist/services/canon.js +11 -8
  58. package/dist/services/cleanup.js +57 -13
  59. package/dist/services/cli-operation-classifier.js +22 -24
  60. package/dist/services/code-checks.js +543 -96
  61. package/dist/services/codex-hook-delivery.js +28 -0
  62. package/dist/services/command-context.js +80 -0
  63. package/dist/services/config.js +11 -8
  64. package/dist/services/continuation-outcome.js +20 -0
  65. package/dist/services/controller-fanout.js +132 -0
  66. package/dist/services/dashboard.js +103 -31
  67. package/dist/services/engines.js +121 -76
  68. package/dist/services/eval-snapshots.js +750 -73
  69. package/dist/services/execution-policy.js +170 -0
  70. package/dist/services/external-work-launch.js +110 -0
  71. package/dist/services/harness-adapter.js +147 -24
  72. package/dist/services/harness-config.js +2 -0
  73. package/dist/services/hooks.js +587 -299
  74. package/dist/services/lanes.js +61 -53
  75. package/dist/services/lifecycle-command.js +118 -7
  76. package/dist/services/lifecycle-invocations.js +1055 -0
  77. package/dist/services/managed-daemon-binding.js +38 -0
  78. package/dist/services/managed-processes.js +283 -27
  79. package/dist/services/merge-queue.js +179 -104
  80. package/dist/services/merge-server.js +73 -28
  81. package/dist/services/migrations.js +14 -9
  82. package/dist/services/native-daemon-history.js +49 -0
  83. package/dist/services/native-session-control.js +39 -0
  84. package/dist/services/plan-runtime.js +19 -8
  85. package/dist/services/plans.js +28 -25
  86. package/dist/services/projects.js +18 -5
  87. package/dist/services/prompts.js +23 -16
  88. package/dist/services/protocols.js +64 -20
  89. package/dist/services/recovery-observation-budget.js +61 -0
  90. package/dist/services/recovery-snapshot-database.js +107 -0
  91. package/dist/services/repair-intents.js +40 -0
  92. package/dist/services/run-control-receipt.js +56 -0
  93. package/dist/services/run-control-worker.js +363 -0
  94. package/dist/services/run-control.js +876 -0
  95. package/dist/services/run-controller-adapter.js +222 -0
  96. package/dist/services/run-controller-capture.js +134 -0
  97. package/dist/services/run-controller-process.js +189 -0
  98. package/dist/services/run-controller-recovery.js +222 -0
  99. package/dist/services/run-controller-state.js +36 -0
  100. package/dist/services/run-controller.js +789 -0
  101. package/dist/services/run-engine-bindings.js +1 -1
  102. package/dist/services/run-fork.js +138 -0
  103. package/dist/services/run-observations.js +112 -0
  104. package/dist/services/run-recovery-runtime.js +69 -0
  105. package/dist/services/run-recovery.js +343 -0
  106. package/dist/services/runs.js +368 -74
  107. package/dist/services/runtime-budget.js +434 -0
  108. package/dist/services/runtime-command.js +43 -0
  109. package/dist/services/runtime-scope-capture.js +50 -0
  110. package/dist/services/runtime-scope-control.js +450 -0
  111. package/dist/services/runtime-scope-resume.js +565 -0
  112. package/dist/services/runtime-scope-stop.js +99 -0
  113. package/dist/services/runtime-scope-worker.js +243 -0
  114. package/dist/services/runtime-service.js +97 -0
  115. package/dist/services/schema-validation.js +10 -8
  116. package/dist/services/sessions.js +35 -69
  117. package/dist/services/stage-blocker.js +17 -7
  118. package/dist/services/stage-context.js +44 -18
  119. package/dist/services/stage-lifecycle.js +99 -102
  120. package/dist/services/stage-pause.js +84 -55
  121. package/dist/services/stage-work-graph.js +35 -0
  122. package/dist/services/usage.js +37 -93
  123. package/dist/services/vnext-code-review.js +195 -88
  124. package/dist/services/vnext-code.js +285 -252
  125. package/dist/services/vnext-execution-profile.js +5 -3
  126. package/dist/services/vnext-fanout.js +160 -17
  127. package/dist/services/vnext-merge.js +291 -104
  128. package/dist/services/vnext-plan-review.js +139 -81
  129. package/dist/services/vnext-plan.js +138 -69
  130. package/dist/services/vnext-protocolize.js +129 -56
  131. package/dist/services/vnext-specify.js +88 -65
  132. package/dist/services/work-registry.js +716 -168
  133. package/dist/services/workspace-bootstrap.js +76 -0
  134. package/dist/services/worktrees.js +85 -35
  135. package/dist/shared/entity-references.js +8 -0
  136. package/dist/shared/errors.js +12 -0
  137. package/dist/storage/database.js +459 -32
  138. package/dist/storage/paths.js +17 -4
  139. package/dist/storage/work-references.js +23 -0
  140. package/dist/storage/writer-contract.js +77 -0
  141. package/dist/storage/writer-migration.js +102 -0
  142. package/package.json +24 -13
  143. package/tools/repair-paused-run-status.mjs +59 -0
@@ -0,0 +1,77 @@
1
+ import { createHash } from "node:crypto";
2
+ import { mkdir, open, readFile } from "node:fs/promises";
3
+ import path from "node:path";
4
+ import { withRunnerLock } from "./runner-lock.mjs";
5
+
6
+ const text = value => typeof value === "string" && value.trim() ? value.trim() : null;
7
+ const fields = ["provider", "model", "reasoning", "mode", "permission_mode"];
8
+ export const observedFields = value => Object.fromEntries(fields.map(key => [key, text(value?.[key])]));
9
+ export const modelObservationFile = journal => path.join(path.dirname(journal), "model-observations.jsonl");
10
+
11
+ /** Routing is allowed. Permission/autonomy changes remain integrity failures.
12
+ * Missing native fields are unknown, never proof that the request was applied. */
13
+ export function checkObservedProfile(requested, observed, { strict = false, required = [] } = {}) {
14
+ const unknown = fields.filter(key => requested?.[key] && !observed?.[key]);
15
+ const changes = fields.filter(key => requested?.[key] && observed?.[key] && requested[key] !== observed[key]);
16
+ const violations = changes.filter(key => strict || ["mode", "permission_mode"].includes(key));
17
+ const missing = required.filter(key => !observed?.[key]);
18
+ if (violations.length || missing.length) throw Object.assign(new Error("native execution profile integrity is not confirmed"), {
19
+ code: "profile_integrity_violation", details: { requested: observedFields(requested), observed: observedFields(observed), violations, missing }
20
+ });
21
+ return { requested: observedFields(requested), observed: observedFields(observed), status: unknown.length ? "incomplete" : changes.length ? "mixed" : "matched", matched: !unknown.length && !changes.length, unknown, changes };
22
+ }
23
+
24
+ export async function readModelObservations(journal) {
25
+ try { const text = await readFile(modelObservationFile(journal), "utf8"); return text.slice(0, text.lastIndexOf("\n") + 1).split("\n").filter(Boolean).map(JSON.parse); }
26
+ catch (error) { if (error.code === "ENOENT") return []; throw error; }
27
+ }
28
+
29
+ /** Only adapter-verified native facts enter this file. It records changes,
30
+ * not every tool or token; reading it never requires replaying native history. */
31
+ export async function observeModel({ journal, harness, sessionId, parentSessionId = null, requested = {}, observed = {}, source, evidence = "configured", reason = null, nativeTimestamp = null, previousModel = null }) {
32
+ if (!journal || !sessionId) return null;
33
+ if (!source || !["configured", "request", "response", "unavailable"].includes(evidence)) throw new Error("model observation requires a native source and evidence level");
34
+ const file = modelObservationFile(journal), profile = observedFields(observed);
35
+ await mkdir(path.dirname(file), { recursive: true, mode: 0o700 });
36
+ return withRunnerLock(file, async () => {
37
+ const events = await readModelObservations(journal);
38
+ const replay = source?.event_sha256 ? events.find(item => item.session_id === sessionId && item.source?.event_sha256 === source.event_sha256) : null;
39
+ if (replay) return replay;
40
+ const channel = source?.channel ?? evidence;
41
+ const previous = events.findLast(item => item.harness === harness && item.session_id === sessionId && (item.source?.channel ?? item.evidence) === channel);
42
+ const normalizedReason = text(reason)?.slice(0, 200) ?? null;
43
+ if (previous && JSON.stringify(previous.observed) === JSON.stringify(profile) && previous.reason === normalizedReason && previous.evidence === evidence) return previous;
44
+ const previousKnown = events.findLast(item => item.harness === harness && item.session_id === sessionId && item.evidence === evidence && item.observed?.model);
45
+ const fromModel = text(previousModel) ?? previousKnown?.observed.model ?? null;
46
+ const changed = Boolean(profile.model && fromModel && profile.model !== fromModel);
47
+ const value = { schema_id: "dd-flow/model-observation@1", sequence: events.length + 1,
48
+ harness, session_id: sessionId, parent_session_id: parentSessionId,
49
+ kind: changed ? "model_changed" : !profile.model ? "model_observation_unavailable" : "model_observed",
50
+ previous_model: fromModel, requested: observedFields(requested), observed: profile,
51
+ source, evidence, reason: normalizedReason, native_timestamp: text(nativeTimestamp), observed_at: new Date().toISOString() };
52
+ value.id = createHash("sha256").update(JSON.stringify(value)).digest("hex");
53
+ const handle = await open(file, "a", 0o600);
54
+ try {
55
+ const bytes = await readFile(file); const committed = bytes.lastIndexOf(10) + 1;
56
+ if (committed < bytes.length) await handle.truncate(committed);
57
+ await handle.write(`${JSON.stringify(value)}\n`); await handle.sync();
58
+ } finally { await handle.close(); }
59
+ const directory = await open(path.dirname(file), "r");
60
+ try { await directory.sync(); } finally { await directory.close(); }
61
+ return value;
62
+ });
63
+ }
64
+
65
+ export function modelAttribution(events) {
66
+ const sessions = new Map();
67
+ for (const event of events) {
68
+ const key = `${event.harness}:${event.session_id}`;
69
+ const session = sessions.get(key) ?? { harness: event.harness, session_id: event.session_id, parent_session_id: event.parent_session_id, requested: event.requested, observations: [] };
70
+ session.observations.push(event); sessions.set(key, session);
71
+ }
72
+ const models = [...new Set(events.map(event => event.observed?.model).filter(Boolean))];
73
+ const incomplete = !events.length || [...sessions.values()].some(session => session.observations.some(event => event.evidence === "unavailable" || !event.observed?.model));
74
+ return { models, mixed: models.length > 1 || events.some(event => event.observed?.model && event.requested?.model && event.observed.model !== event.requested.model),
75
+ observation_completeness: incomplete ? "incomplete" : "available_native_sources", transitions: events.filter(event => event.kind === "model_changed"),
76
+ sessions: [...sessions.values()], usage_attribution: "session_totals_unattributed_to_model", note: "Native settings/events do not prove undisclosed server routing or per-model token costs." };
77
+ }
@@ -0,0 +1 @@
1
+ export function invokeNativeHook(input: { bin: string; args: string[]; payload: unknown; home?: string | undefined; timeoutMs?: number }): Promise<Record<string, unknown> & { timing?: Record<string, number> }>;
@@ -0,0 +1,34 @@
1
+ import { spawn } from "node:child_process";
2
+
3
+ /** Preserve the native hook protocol across the CLI process boundary. */
4
+ export async function invokeNativeHook({ bin, args, payload, home, timeoutMs = 15_000 }) {
5
+ const script = /\.[cm]?js$/.test(bin);
6
+ const output = await new Promise((resolve, reject) => {
7
+ const child = spawn(script ? process.execPath : bin, [...(script ? [bin] : []), ...args], {
8
+ env: { ...process.env, DD_FLOW_HOME: home }, stdio: ["pipe", "pipe", "pipe"]
9
+ });
10
+ let stdout = "", stderr = "", failure;
11
+ const stop = error => { failure ??= error; child.kill("SIGKILL"); };
12
+ const timer = setTimeout(() => stop(Object.assign(new Error("Native hook admission timed out"), { code: "native_hook_timeout" })), timeoutMs);
13
+ child.stdout.setEncoding("utf8").on("data", chunk => { stdout += chunk; if (stdout.length > 1_048_576) stop(new Error("Native hook response exceeds protocol limit")); });
14
+ child.stderr.setEncoding("utf8").on("data", chunk => { stderr = (stderr + chunk).slice(-65_536); });
15
+ child.stdin.on("error", error => { failure ??= error; });
16
+ child.on("error", error => { clearTimeout(timer); reject(error); });
17
+ child.on("close", code => {
18
+ clearTimeout(timer);
19
+ if (failure) return reject(failure);
20
+ if (code !== 0) {
21
+ let record;
22
+ try { record = JSON.parse(stderr.trim()).error; } catch { /* Non-JSON stderr remains diagnostic evidence. */ }
23
+ return reject(Object.assign(new Error(record?.message ?? (stderr.trim() || "Native hook admission failed")), { code: record?.code ?? "native_hook_failed", details: record?.details }));
24
+ }
25
+ resolve(stdout);
26
+ });
27
+ child.stdin.end(`${JSON.stringify(payload)}\n`);
28
+ });
29
+ let result;
30
+ try { result = JSON.parse(output); } catch { throw Object.assign(new Error("Invalid native hook JSON response"), { code: "native_hook_response_invalid" }); }
31
+ if (result?.ok === false && typeof result.error?.code === "string") throw Object.assign(new Error(result.error.message ?? "Native hook failed"), { code: result.error.code, details: result.error.details });
32
+ if (!result || typeof result !== "object" || Array.isArray(result) || result.ok === false) throw Object.assign(new Error("Native hook did not return a successful protocol object"), { code: "native_hook_response_invalid" });
33
+ return result;
34
+ }
@@ -0,0 +1,36 @@
1
+ /** Only observed time counts toward inactivity. A scheduling/clock gap is not
2
+ * evidence of OS sleep, nor of provider progress. Call sample regularly. */
3
+ export class ObservationClock {
4
+ constructor({ timeoutMs, gapMs = 60_000, wall = Date.now, monotonic = () => performance.now(), onGap = () => {} }) {
5
+ if (!(timeoutMs > 0) || !(gapMs > 0)) throw new RangeError("positive observation timeout and gap required");
6
+ Object.assign(this, { timeoutMs, gapMs, wall, monotonic, onGap });
7
+ this.lastWall = wall(); this.lastMono = monotonic(); this.elapsed = 0;
8
+ this.progress = undefined;
9
+ }
10
+ sample(progress, progressAt) {
11
+ const wall = this.wall(), mono = this.monotonic();
12
+ const delta = mono - this.lastMono, wallDelta = wall - this.lastWall;
13
+ const gap = delta > this.gapMs || wallDelta > this.gapMs || delta < 0 || wallDelta < 0;
14
+ if (gap) {
15
+ this.onGap({ source: "observer_clock", classification: "observation_gap", confirmed_sleep: false,
16
+ started_at: new Date(this.lastWall).toISOString(), observed_at: new Date(wall).toISOString(),
17
+ wall_delta_ms: wallDelta, monotonic_delta_ms: delta });
18
+ this.elapsed = 0;
19
+ } else this.elapsed += delta;
20
+ this.lastWall = wall; this.lastMono = mono;
21
+ if (progress !== undefined && progress !== this.progress) {
22
+ this.progress = progress;
23
+ this.elapsed = !gap && Number.isFinite(progressAt) ? Math.max(0, Math.min(this.timeoutMs, wall - progressAt)) : 0;
24
+ }
25
+ return this.elapsed >= this.timeoutMs;
26
+ }
27
+ }
28
+
29
+ /** Node timeout handle: callers can keep their existing clearTimeout cleanup. */
30
+ export function observedTimeout(callback, timeoutMs, options = {}) {
31
+ const clock = new ObservationClock({ timeoutMs, ...options });
32
+ const timer = setInterval(() => {
33
+ if (clock.sample(options.progress?.())) { clearInterval(timer); callback(); }
34
+ }, Math.max(1, Math.min(1_000, timeoutMs / 4)));
35
+ return timer;
36
+ }
@@ -0,0 +1,5 @@
1
+ import { AsyncLocalStorage } from "node:async_hooks";
2
+
3
+ // The existing runner operation owns its child driver requests, including calls
4
+ // nested in Judge/HITL/fanout. Async context keeps parallel executions separate.
5
+ export const operationContext = new AsyncLocalStorage();
@@ -0,0 +1,19 @@
1
+ // An observer losing its response does not establish that a provider Turn failed.
2
+ export function isObservationLoss(error) {
3
+ return ["rpc_timeout", "native_timeout", "native_outcome_unknown", "native_backend_dead", "native_backend_pipe_broken", "bridge_exited", "bridge_pipe_broken", "daemon_timeout", "turn_timeout", "operation_observation_lost", "daemon_connection_closed"].includes(error?.code);
4
+ }
5
+
6
+ export function errorRecord(error, depth = 0) {
7
+ return {
8
+ code: typeof error?.code === "string" ? error.code : "operation_failed",
9
+ message: typeof error?.message === "string" ? error.message : String(error),
10
+ ...(typeof error?.retryable === "boolean" ? { retryable: error.retryable } : {}),
11
+ ...(error?.details !== undefined ? { details: error.details } : {}),
12
+ ...(error?.cause && depth < 3 ? { cause: errorRecord(error.cause, depth + 1) } : {}),
13
+ };
14
+ }
15
+
16
+ export function reportedError(value, fallback) {
17
+ const record = errorRecord({ ...value, message: value.message ?? fallback });
18
+ return Object.assign(new Error(record.message), record);
19
+ }
@@ -0,0 +1,70 @@
1
+ import { spawn } from "node:child_process";
2
+ import { reportedError } from "./operation-errors.mjs";
3
+
4
+ function failure(message, code) {
5
+ return Object.assign(new Error(message), { code });
6
+ }
7
+
8
+ function reportedFailure(stdout, stderr, fallback) {
9
+ for (const text of [stdout, stderr]) {
10
+ for (const line of [text.trim(), ...text.trim().split(/\n/).reverse()]) {
11
+ try {
12
+ const value = JSON.parse(line);
13
+ const error = value?.error;
14
+ if (typeof error?.code === "string" && error.code) return reportedError(error, fallback);
15
+ } catch { /* Not a structured CLI error line. */ }
16
+ }
17
+ }
18
+ return failure(stderr.trim() || stdout.trim() || fallback, "flow_reconciliation_failed");
19
+ }
20
+
21
+ function processTarget(executable, args) {
22
+ return /\.[cm]?js$/.test(executable) ? { command: process.execPath, args: [executable, ...args] } : { command: executable, args };
23
+ }
24
+
25
+ export async function commandJson(bin, args, { cwd, env = {}, input = null, onProgress = null } = {}) {
26
+ return await new Promise((resolve, reject) => {
27
+ // A canonical runtime may intentionally shadow the ambient `dd-flow`
28
+ // executable with its captured engine. Keep that choice in the supplied
29
+ // environment rather than letting a host-global shim leak into a run.
30
+ const executable = bin === "dd-flow" && typeof env.DD_FLOW_BIN === "string" ? env.DD_FLOW_BIN : bin;
31
+ const target = processTarget(executable, [...args, "--json"]);
32
+ const child = spawn(target.command, target.args, { cwd, env: { ...process.env, ...env }, stdio: [input === null ? "ignore" : "pipe", "pipe", "pipe"] });
33
+ let stdout = ""; let stderr = ""; let pendingStderr = ""; let settled = false; let progressError = null;
34
+ const finish = (error, value) => { if (settled) return; settled = true; error ? reject(error) : resolve(value); };
35
+ const deliverProgress = (line) => {
36
+ let value; try { value = JSON.parse(line); } catch { return; }
37
+ if (typeof onProgress !== "function" || progressError) return;
38
+ try { const returned = onProgress(value); if (returned && typeof returned.then === "function") { Promise.resolve(returned).catch(() => {}); throw failure("commandJson onProgress must be synchronous", "progress_callback_async"); } }
39
+ catch (error) { progressError = error; child.kill("SIGKILL"); }
40
+ };
41
+ child.stdout.setEncoding("utf8").on("data", (data) => { stdout += data; });
42
+ child.stderr.setEncoding("utf8").on("data", (data) => {
43
+ stderr += data; pendingStderr += data;
44
+ for (;;) {
45
+ const end = pendingStderr.indexOf("\n"); if (end < 0) break;
46
+ const line = pendingStderr.slice(0, end); pendingStderr = pendingStderr.slice(end + 1);
47
+ deliverProgress(line);
48
+ }
49
+ });
50
+ child.once("error", (error) => finish(progressError ?? error));
51
+ if (input !== null) child.stdin.end(input);
52
+ child.on("close", (status) => {
53
+ if (pendingStderr) deliverProgress(pendingStderr);
54
+ if (progressError) return finish(progressError);
55
+ if (status !== 0) return finish(reportedFailure(stdout, stderr, `${executable} exited ${status}`));
56
+ try { const value = JSON.parse(stdout.trim()); if (value?.ok === false) { const error = value.error && typeof value.error === "object" ? value.error : value; if (typeof error.code === "string") return finish(reportedError(error, `${bin} returned an error envelope`)); } finish(null, value); }
57
+ catch (error) { finish(failure(`${bin} returned invalid JSON: ${error.message}`, "flow_reconciliation_failed")); }
58
+ });
59
+ });
60
+ }
61
+
62
+ export async function commandText(bin, args, { cwd, env = {} } = {}) {
63
+ return await new Promise((resolve, reject) => {
64
+ const executable = bin === "dd-flow" && typeof env.DD_FLOW_BIN === "string" ? env.DD_FLOW_BIN : bin;
65
+ const target = processTarget(executable, args);
66
+ const child = spawn(target.command, target.args, { cwd, env: { ...process.env, ...env }, stdio: ["ignore", "pipe", "pipe"] }); let stdout = ""; let stderr = "";
67
+ child.stdout.setEncoding("utf8").on("data", (data) => { stdout += data; }); child.stderr.setEncoding("utf8").on("data", (data) => { stderr += data; }); child.on("error", reject);
68
+ child.on("close", (status) => status === 0 ? resolve(stdout.trim()) : reject(failure(stderr.trim() || stdout.trim() || `${executable} exited ${status}`, "command_failed")));
69
+ });
70
+ }
@@ -0,0 +1,10 @@
1
+ import { execFile } from "node:child_process";
2
+ import { promisify } from "node:util";
3
+ const run = promisify(execFile);
4
+
5
+ /** Inspect OS descendants without using process names as ownership. A start-time
6
+ * identity prevents a later unrelated process with a recycled PID being killed. */
7
+ export async function processSnapshot() {
8
+ const { stdout } = await run("ps", ["-axo", "pid=,ppid=,pgid=,lstart=,stat="], { timeout: 5_000 });
9
+ return stdout.trim().split("\n").flatMap(line => { const parts = line.trim().split(/\s+/); return parts.length >= 9 ? [{ pid: Number(parts[0]), ppid: Number(parts[1]), pgid: Number(parts[2]), started: parts.slice(3, 8).join(" "), zombie: parts[8].startsWith("Z") }] : []; });
10
+ }
@@ -0,0 +1,200 @@
1
+ import { createHash, randomUUID } from "node:crypto";
2
+ import { mkdir, open, readFile, rename, unlink } from "node:fs/promises";
3
+ import path from "node:path";
4
+ import { withRunnerLock as withLock } from "./runner-lock.mjs";
5
+ import { isObservationLoss, errorRecord } from "./operation-errors.mjs";
6
+ import { operationContext } from "./operation-context.mjs";
7
+
8
+ const appendQueues = new Map();
9
+
10
+ export function canonicalJson(value) {
11
+ const normalize = (item) => {
12
+ if (Array.isArray(item)) return item.map(normalize);
13
+ if (item && typeof item === "object") return Object.fromEntries(Object.keys(item).sort().map((key) => [key, normalize(item[key])]));
14
+ return item;
15
+ };
16
+ return `${JSON.stringify(normalize(value))}\n`;
17
+ }
18
+
19
+ export function sha256(value) {
20
+ return createHash("sha256").update(value).digest("hex");
21
+ }
22
+
23
+ export function hashJson(value) { return sha256(canonicalJson(value)); }
24
+
25
+ export async function writeJsonAtomic(file, value) {
26
+ await mkdir(path.dirname(file), { recursive: true });
27
+ const temporary = `${file}.${process.pid}.${randomUUID()}.tmp`;
28
+ try {
29
+ const payload = await open(temporary, "wx", 0o600);
30
+ try { await payload.writeFile(`${JSON.stringify(value, null, 2)}\n`); await payload.sync(); }
31
+ finally { await payload.close(); }
32
+ await rename(temporary, file);
33
+ const directory = await open(path.dirname(file), "r");
34
+ try { await directory.sync(); }
35
+ finally { await directory.close(); }
36
+ } finally { await unlink(temporary).catch(error => { if (error.code !== "ENOENT") throw error; }); }
37
+ }
38
+
39
+ export async function readJsonLines(file, label = "JSONL") {
40
+ try {
41
+ const source = await readFile(file, "utf8");
42
+ return source.split("\n").filter(Boolean).map((line, index) => {
43
+ try { return JSON.parse(line); }
44
+ catch { throw new Error(`invalid ${label} record at ${file}:${index + 1}`); }
45
+ });
46
+ } catch (error) {
47
+ if (error?.code === "ENOENT") return [];
48
+ throw error;
49
+ }
50
+ }
51
+
52
+ export async function readEvents(file) {
53
+ const records = await readJsonLines(file, "runner event");
54
+ return records.map((record, index) => {
55
+ try { return validateRunnerEvent(record); }
56
+ catch { throw new Error(`invalid runner event at ${file}:${index + 1}`); }
57
+ });
58
+ }
59
+
60
+ export function validateRunnerEvent(value) {
61
+ if (!value || typeof value !== "object" || Array.isArray(value) || value.specversion !== "1.0" || typeof value.id !== "string" || typeof value.source !== "string" || typeof value.type !== "string" || typeof value.time !== "string" || !value.data || typeof value.data !== "object" || !Number.isInteger(value.data.sequence) || value.data.sequence < 1) {
62
+ throw new Error("invalid normalized runner event");
63
+ }
64
+ return value;
65
+ }
66
+
67
+ export async function appendEvent(file, input) {
68
+ const prior = appendQueues.get(file) ?? Promise.resolve();
69
+ const current = prior.then(() => withLock(file, async () => {
70
+ const events = await readEvents(file);
71
+ // Internal callers can validate a dispatch or bind its generation under
72
+ // the same cross-process lock that orders cancellation and completion.
73
+ if (input.deduplicate && input.id) { const existing = events.find(event => event.id === input.id); if (existing) return existing; }
74
+ if (input.beforeAppend?.(events) === false) return null;
75
+ const sequence = events.length + 1;
76
+ const event = {
77
+ specversion: "1.0",
78
+ id: input.id ?? `EVT-${randomUUID()}`,
79
+ source: input.source,
80
+ type: input.type,
81
+ time: input.time ?? new Date().toISOString(),
82
+ subject: input.subject ?? "runner",
83
+ datacontenttype: "application/json",
84
+ runid: input.runId,
85
+ executionid: input.executionId,
86
+ traceid: input.traceId,
87
+ spanid: input.spanId ?? randomUUID(),
88
+ ...(input.parentSpanId ? { parentspanid: input.parentSpanId } : {}),
89
+ data: { sequence, ...input.data }
90
+ };
91
+ const context = operationContext.getStore();
92
+ if (context?.eventsFile === file && context.executionId === input.executionId && context.operationId && input.executionId) {
93
+ event.data.execution_operation_id ??= context.operationId;
94
+ }
95
+ validateRunnerEvent(event);
96
+ await mkdir(path.dirname(file), { recursive: true });
97
+ const handle = await open(file, "a");
98
+ try { await handle.write(`${JSON.stringify(event)}\n`); await handle.sync(); }
99
+ finally { await handle.close(); }
100
+ return event;
101
+ }));
102
+ const tail = current.catch(() => {}); appendQueues.set(file, tail);
103
+ try { return await current; }
104
+ finally { if (appendQueues.get(file) === tail) appendQueues.delete(file); }
105
+ }
106
+
107
+ export function reduceEvents(events) {
108
+ const operations = new Map();
109
+ let state = "planned";
110
+ for (const event of events) {
111
+ const data = event?.data ?? {};
112
+ if (["dev.dd.eval.planned", "dev.dd.eval.completed"].includes(event.type) && typeof data.state === "string") state = data.state;
113
+ if (typeof data.operation_id !== "string") continue;
114
+ const prior = operations.get(data.operation_id) ?? { id: data.operation_id, requested: false, started: false, terminal: null };
115
+ if (event.type.endsWith(".requested")) prior.requested = true;
116
+ if (event.type.endsWith(".started")) prior.started = true;
117
+ if (event.type.endsWith(".observation_lost")) prior.observation_lost = data.error ?? { code: "observation_lost" };
118
+ if (event.type.endsWith(".completed") || event.type.endsWith(".failed") || event.type.endsWith(".cancelled")) {
119
+ const terminal = event.type.split(".").at(-1);
120
+ const result = terminal === "completed" ? data.result : { error: data.error ?? null };
121
+ const resultHash = hashJson({ terminal, result });
122
+ if (prior.terminal && (prior.terminal !== terminal || prior.result_hash !== resultHash)) {
123
+ throw Object.assign(new Error(`runner operation ${data.operation_id} has conflicting terminal events`), { code: "journal_conflict", operation_id: data.operation_id });
124
+ }
125
+ prior.terminal = terminal;
126
+ prior.result = result;
127
+ prior.result_hash = resultHash;
128
+ }
129
+ operations.set(data.operation_id, prior);
130
+ }
131
+ return { state, operations: Object.fromEntries([...operations].map(([id, value]) => [id, value])) };
132
+ }
133
+
134
+ export async function recordOperation({ eventsFile, source, runId, executionId, traceId, operationId, operation, subject = "runner", action }) {
135
+ const existing = await withLock(`${eventsFile}.${operationId.replace(/[^a-zA-Z0-9._-]/g, "_")}`, async () => {
136
+ const current = reduceEvents(await readEvents(eventsFile)).operations[operationId];
137
+ if (current?.terminal === "completed") return current;
138
+ if (current?.terminal) throw Object.assign(new Error(`operation ${operationId} is already ${current.terminal}; use explicit recovery instead of replaying it`), { code: "operation_terminal", operation_id: operationId, terminal: current.terminal });
139
+ if (current?.started && !current.terminal) {
140
+ const error = new Error(current.observation_lost
141
+ ? `operation ${operationId} has no confirmed outcome; reconcile its provider state before retrying`
142
+ : `operation ${operationId} is already in progress; reconcile before retrying`);
143
+ error.code = current.observation_lost ? "operation_observation_lost" : "operation_in_progress";
144
+ error.operation_id = operationId;
145
+ throw error;
146
+ }
147
+ await appendEvent(eventsFile, { source, runId, executionId, traceId, subject, type: "dev.dd.eval.operation.requested", data: { operation_id: operationId, operation, status: "requested" } });
148
+ await appendEvent(eventsFile, { source, runId, executionId, traceId, subject, type: "dev.dd.eval.operation.started", data: { operation_id: operationId, operation, status: "started" } });
149
+ return null;
150
+ });
151
+ if (existing) return { reused: true, operation_id: operationId, result: existing.result };
152
+ try {
153
+ const result = await operationContext.run({ operationId, eventsFile, runId, executionId }, action);
154
+ await completeOperation({ eventsFile, source, runId, executionId, traceId, subject, operationId, operation, result });
155
+ return { reused: false, operation_id: operationId, result };
156
+ } catch (error) {
157
+ if (isObservationLoss(error)) await observationLost({ eventsFile, source, runId, executionId, traceId, subject, operationId, operation, error });
158
+ else await failOperation({ eventsFile, source, runId, executionId, traceId, subject, operationId, operation, error });
159
+ throw error;
160
+ }
161
+ }
162
+
163
+ export async function completeOperation({ eventsFile, source, runId, executionId, traceId, subject = "runner", operationId, operation, result }) {
164
+ return await settleOperation({ eventsFile, source, runId, executionId, traceId, subject, operationId, operation, terminal: "completed", result });
165
+ }
166
+
167
+ export async function failOperation({ eventsFile, source, runId, executionId, traceId, subject = "runner", operationId, operation, error }) {
168
+ return await settleOperation({ eventsFile, source, runId, executionId, traceId, subject, operationId, operation, terminal: "failed", result: { error: errorRecord(error) } });
169
+ }
170
+
171
+ /** A lost observer response is deliberately non-terminal: replay could duplicate a provider Turn. */
172
+ export async function observationLost({ eventsFile, source, runId, executionId, traceId, subject = "runner", operationId, operation, error }) {
173
+ return await withLock(`${eventsFile}.${operationId.replace(/[^a-zA-Z0-9._-]/g, "_")}`, async () => {
174
+ const existing = reduceEvents(await readEvents(eventsFile)).operations[operationId];
175
+ if (existing?.terminal) return { reused: true, operation_id: operationId, result: existing.result };
176
+ if (!existing?.started) throw Object.assign(new Error(`operation ${operationId} has no started receipt`), { code: "operation_not_started" });
177
+ if (existing.observation_lost) return { reused: true, operation_id: operationId, observation_lost: existing.observation_lost };
178
+ const recorded = errorRecord(error);
179
+ await appendEvent(eventsFile, { source, runId, executionId, traceId, subject, type: "dev.dd.eval.operation.observation_lost", data: { operation_id: operationId, operation, status: "observation_lost", error: recorded } });
180
+ return { reused: false, operation_id: operationId, observation_lost: recorded };
181
+ });
182
+ }
183
+
184
+ async function settleOperation({ eventsFile, source, runId, executionId, traceId, subject, operationId, operation, terminal, result }) {
185
+ return await withLock(`${eventsFile}.${operationId.replace(/[^a-zA-Z0-9._-]/g, "_")}`, async () => {
186
+ const existing = reduceEvents(await readEvents(eventsFile)).operations[operationId];
187
+ const expectedHash = hashJson({ terminal, result });
188
+ if (existing?.terminal) {
189
+ if (existing.terminal === terminal && existing.result_hash === expectedHash) return { reused: true, operation_id: operationId, result: existing.result };
190
+ throw Object.assign(new Error(`runner operation ${operationId} has conflicting terminal result`), { code: "journal_conflict", operation_id: operationId });
191
+ }
192
+ if (!existing?.started) throw Object.assign(new Error(`operation ${operationId} has no started receipt`), { code: "operation_not_started" });
193
+ const type = terminal === "completed" ? "dev.dd.eval.operation.completed" : "dev.dd.eval.operation.failed";
194
+ const data = terminal === "completed"
195
+ ? { operation_id: operationId, operation, status: terminal, result }
196
+ : { operation_id: operationId, operation, status: terminal, error: result.error };
197
+ await appendEvent(eventsFile, { source, runId, executionId, traceId, subject, type, data });
198
+ return { reused: false, operation_id: operationId, result };
199
+ });
200
+ }
@@ -0,0 +1,42 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import { mkdirSync, readdirSync, readFileSync, writeFileSync, unlinkSync, rmdirSync } from "node:fs";
3
+ import path from "node:path";
4
+
5
+ function release(lock, owner) {
6
+ try { unlinkSync(path.join(lock, owner)); } catch (error) { if (error.code === "ENOENT") return; throw error; }
7
+ // Never recursively remove a directory: another owner may already have acquired it.
8
+ try { rmdirSync(lock); } catch (error) { if (!["ENOENT", "ENOTEMPTY", "EEXIST"].includes(error.code)) throw error; }
9
+ }
10
+
11
+ function reclaimDeadOwner(lock) {
12
+ let entries;
13
+ try { entries = readdirSync(lock); } catch (error) { if (error.code === "ENOENT") return; throw error; }
14
+ if (entries.length !== 1 || !/^owner-.*\.json$/.test(entries[0])) return;
15
+ let owner;
16
+ try { owner = JSON.parse(readFileSync(path.join(lock, entries[0]), "utf8")); } catch { return; }
17
+ if (!Number.isInteger(owner.pid) || owner.pid <= 0) return;
18
+ try { process.kill(owner.pid, 0); }
19
+ catch (error) { if (error.code === "ESRCH") release(lock, entries[0]); }
20
+ // A live/reused PID or unknown owner is conservative: age never proves death.
21
+ }
22
+
23
+ export async function withRunnerLock(file, action, { timeoutMs = 30_000 } = {}) {
24
+ const lock = `${file}.lock`;
25
+ const owner = `owner-${process.pid}-${randomUUID()}.json`;
26
+ const deadline = performance.now() + timeoutMs;
27
+ for (;;) {
28
+ let acquired = false;
29
+ try { mkdirSync(lock); acquired = true; }
30
+ catch (error) { if (error.code !== "EEXIST") throw error; }
31
+ if (acquired) {
32
+ // No await between acquisition and identifying the owner.
33
+ try { writeFileSync(path.join(lock, owner), JSON.stringify({ pid: process.pid }), { flag: "wx" }); }
34
+ catch (error) { release(lock, owner); throw error; }
35
+ try { return await action(); }
36
+ finally { release(lock, owner); }
37
+ }
38
+ reclaimDeadOwner(lock);
39
+ if (performance.now() >= deadline) throw Object.assign(new Error(`runner lock has a live or unconfirmed owner: ${lock}; inspect its owner before recovery`), { code: "runner_lock_timeout" });
40
+ await new Promise((resolve) => setTimeout(resolve, 10));
41
+ }
42
+ }
@@ -0,0 +1,30 @@
1
+ import { errorRecord } from "./operation-errors.mjs";
2
+
3
+ /** Diagnostic failure never prevents cancellation of already owned Sessions.
4
+ * Only a fresh successful observation can establish settlement. */
5
+ export async function waitForSettlement({ observe, cancel, knownSessions = () => [], timeoutMs = 5_000 }) {
6
+ const deadline = performance.now() + timeoutMs;
7
+ const cancelled = new Set(), errors = [];
8
+ if (cancel) for (const id of knownSessions()) {
9
+ cancelled.add(id);
10
+ try { await cancel(id); } catch (error) { errors.push({ phase: "cancel", session_id: id, ...errorRecord(error) }); }
11
+ }
12
+ for (;;) {
13
+ let state;
14
+ try { state = await observe(); }
15
+ catch (error) {
16
+ if (!cancel) throw error;
17
+ errors.push({ phase: "observe", ...errorRecord(error) });
18
+ state = { sessions: knownSessions(), active: true, observation_error: errorRecord(error) };
19
+ }
20
+ if (!state.active && state.sessions.length === 0) return { ...state, cleanup_errors: errors };
21
+ // The observation budget must not skip a newly discovered owned target.
22
+ // Attempt its cancellation once, then leave physical settlement pending.
23
+ for (const id of state.sessions) if (cancel && !cancelled.has(id)) {
24
+ cancelled.add(id);
25
+ try { await cancel(id); } catch (error) { errors.push({ phase: "cancel", session_id: id, ...errorRecord(error) }); }
26
+ }
27
+ if (!cancel || performance.now() >= deadline) throw Object.assign(new Error("daemon still owns an unsettled Session tree"), { code: "tree_not_settled", details: { ...state, cleanup_errors: errors } });
28
+ await new Promise(resolve => setTimeout(resolve, 50));
29
+ }
30
+ }
@@ -0,0 +1,17 @@
1
+ export interface ToolEvent {
2
+ session?: { harness_id: string; session_id: string };
3
+ call_id?: string;
4
+ status?: string;
5
+ name?: string;
6
+ reason?: string;
7
+ [key: string]: unknown;
8
+ }
9
+ export class ToolObservations {
10
+ calls: Map<string, ToolEvent>;
11
+ gaps: Set<string>;
12
+ add(event: ToolEvent | null): void;
13
+ summary(sessionId?: string, cursor?: Set<string>): { total: number; failures: number; pending: number; by_tool: Record<string, number>; completeness: string; sessions: Array<{ session: { harness_id: string; session_id: string }; total: number; failures: number; pending: number; by_tool: Record<string, number> }>; reasons: string[] };
14
+ cursor(): Set<string>;
15
+ }
16
+ export function codexToolObservation(record: Record<string, unknown>, sessionId: string): ToolEvent | null;
17
+ export function replayToolJournal(file: string, options: { harness: string; rootSessionId: string; adapterSessionId?: string }, reducer?: ToolObservations): ToolObservations;