@deksden-com/dd-flow-cli 0.9.0-beta.55 → 0.9.0-beta.62

package/dist/services/runtime-scope-control.js

@@ -235,7 +235,7 @@ export async function controlRuntimeScopeNativeOwners(context, scopeId, generati
             const sessions = await Promise.all(ids.map(async (id) => {
                 try {
                     let cancellation;
-                    let cancellationAcknowledged = action !== "stop";
+                    let cancellationError;
                     if (action === "stop") {
                         const operationId = `scope-cancel:${crypto.createHash("sha256").update(JSON.stringify([scopeId, generation, record.id, state.daemon_id, id])).digest("hex")}`;
                         const now = context.now();
@@ -255,22 +255,34 @@ export async function controlRuntimeScopeNativeOwners(context, scopeId, generati
                             if (saved.status === "completed") {
                                 assertNativeCancellationReceipt(saved.receipt, id);
                                 cancellation = saved.receipt;
-                                cancellationAcknowledged = true;
+                            }
+                            else if (saved.status === "failed") {
+                                cancellation = saved.receipt;
+                                try {
+                                    cancellationError = JSON.parse(saved.error ?? "");
+                                }
+                                catch {
+                                    cancellationError = { code: "cancellation_outcome_unknown", message: saved.error ?? "Retained cancellation has no conclusive outcome" };
+                                }
                             }
                             else {
                                 const result = await cancelControlledNativeSession({ executable: metadata.adapter_executable, sessionId: id, stateDir: metadata.state_dir, env, operationId, observe: !inserted, assertCurrent: assertOwner });
                                 registry.run("UPDATE managed_resources SET metadata_json = json_set(metadata_json, '$.status', ?, '$.receipt', json(?)), updated_at = ? WHERE resource_kind = 'scope-control-operation' AND resource_key = ?", [result.status, JSON.stringify(result.receipt), context.now(), operationId]);
                                 cancellation = result.receipt;
-                                cancellationAcknowledged = result.status === "completed";
                             }
                         }
                         catch (error) {
-                            registry.run("UPDATE managed_resources SET metadata_json = json_set(metadata_json, '$.status', 'outcome_unknown', '$.error', ?), updated_at = ? WHERE resource_kind = 'scope-control-operation' AND resource_key = ?", [String(error), context.now(), operationId]);
-                            throw error;
+                            cancellationError = { code: error instanceof AppError ? error.code : "cancellation_outcome_unknown", message: String(error) };
+                            registry.run("UPDATE managed_resources SET metadata_json = json_set(metadata_json, '$.status', 'outcome_unknown', '$.error', ?), updated_at = ? WHERE resource_kind = 'scope-control-operation' AND resource_key = ?", [JSON.stringify(cancellationError), context.now(), operationId]);
                         }
+                        if (cancellationError?.code === "controller_operation_mismatch")
+                            throw new AppError("controller_operation_mismatch", "Cancellation evidence belongs to another operation or Session", 1);
                     }
                     const receipt = await inspectControlledNativeSession({ executable: metadata.adapter_executable, sessionId: id, stateDir: metadata.state_dir, env, operationId: `scope:${scopeId}:${generation}:inspect:${crypto.randomUUID()}`, assertCurrent: assertOwner });
-                    return { session_id: id, settled: receipt.settled === true && cancellationAcknowledged, receipt, ...(cancellation ? { cancellation } : {}) };
+                    // Physical settlement is determined by this fresh inspection. A past
+                    // cancel ACK (or an inconclusive cancel outcome) is retained as
+                    // evidence, never substituted for the state of the native tree.
+                    return { session_id: id, settled: receipt.settled === true, receipt, ...(cancellation ? { cancellation } : {}), ...(cancellationError ? { cancellation_error: cancellationError } : {}) };
                 }
                 catch (error) {
                     return { session_id: id, settled: false, reason: error instanceof AppError ? error.code : "runtime_scope_session_control_failed", message: String(error) };
@@ -294,7 +306,7 @@ export async function controlRuntimeScopeNativeOwners(context, scopeId, generati
     }));
 }
 /** Scope routing only: existing RUN workers own physical drain and capture. */
-export async function controlRuntimeScopeRuns(context, scopeId, requestId, mode) {
+export async function controlRuntimeScopeRuns(context, scopeId, requestId, mode, reconcileId) {
     const control = requestRuntimeScopeControl(context, scopeId, requestId, mode);
     const registry = getResourceDatabase(resourceHome(context));
     const inventory = runtimeBudgetStatus(context, scopeId).processes;
@@ -341,7 +353,7 @@ export async function controlRuntimeScopeRuns(context, scopeId, requestId, mode)
                 throw error;
             }
             seen.add(key);
-            const receipt = await requestRunControl(target, { ...input, waitMs: 0 });
+            const receipt = await requestRunControl(target, { ...input, waitMs: 0, ...(reconcileId ? { reconcileId: crypto.createHash("sha256").update(JSON.stringify([scopeId, control.generation, reconcileId])).digest("hex") } : {}) });
             runs.push({ runtime_home: metadata.dd_flow_home, project_id: record.project_id, run_id: record.run_id, request_id: input.requestId, receipt });
         }
         catch (error) {

package/dist/services/runtime-scope-worker.js

@@ -10,6 +10,7 @@ import { confirmManagedProcess, finishManagedProcess, heartbeatManagedProcess, p
 import { publicRuntimeScopeWorker, requestRuntimeScopeControl, runtimeBudgetStatus } from "./runtime-budget.js";
 import { controlRuntimeScopeRuns } from "./runtime-scope-control.js";
 import { prepareRuntimeScopeResume, recordRuntimeScopeResume } from "./runtime-scope-resume.js";
+import { recoveryObservationBudget, renewRecoveryObservation, withRecoveryObservation } from "./recovery-observation-budget.js";
 export async function launchRuntimeScopeResume(context, input) {
     const resume = recordRuntimeScopeResume(context, input);
     if (!resume.current)
@@ -19,7 +20,7 @@ export async function launchRuntimeScopeResume(context, input) {
     const control = runtimeBudgetStatus(context, input.scopeId).control;
     if (control?.generation !== input.generation)
         throw new AppError("runtime_scope_control_superseded", "Scope resume source changed before owner launch", 1);
-    const owner = await launchRuntimeScopeControl(context, input.scopeId, control.request_id, control.requested_mode);
+    const owner = await launchRuntimeScopeControl(context, input.scopeId, control.request_id, control.requested_mode, undefined, undefined, input.requestId);
     return { ...owner, resume, dispatch_blocked: true, settled: false };
 }
 const keyFor = (scope, generation) => crypto.createHash("sha256").update(JSON.stringify([scope, generation])).digest("hex");
@@ -29,7 +30,14 @@ function birth() {
         throw new AppError("runtime_scope_owner_unknown", "Scope owner birth identity is unavailable", 1);
     return { pid: process.pid, pid_started_at: started };
 }
-export async function launchRuntimeScopeControl(context, scopeId, requestId, mode, manifest) {
+export async function reconcileRuntimeScopeControl(context, scopeId, generation, requestId) {
+    const control = runtimeBudgetStatus(context, scopeId).control;
+    if (!control || control.generation !== generation)
+        throw new AppError("runtime_scope_control_superseded", "Reconciliation requires the current controlled scope generation", 1);
+    renewRecoveryObservation({}, requestId);
+    return launchRuntimeScopeControl(context, scopeId, control.request_id, control.requested_mode, undefined, requestId);
+}
+export async function launchRuntimeScopeControl(context, scopeId, requestId, mode, manifest, reconcileId, resumeId) {
     const control = requestRuntimeScopeControl(context, scopeId, requestId, mode, manifest), db = getResourceDatabase(resourceHome(context)), key = keyFor(scopeId, control.generation);
     let worker;
     db.exec("BEGIN IMMEDIATE");
@@ -50,7 +58,18 @@ export async function launchRuntimeScopeControl(context, scopeId, requestId, mod
             db.exec("COMMIT");
             return { scope_id: scopeId, control, current: true, settled: false, worker: publicRuntimeScopeWorker(existing) };
         }
-        worker = { scope_id: scopeId, generation: control.generation, request_id: requestId, mode, token: crypto.randomUUID(), status: "starting", launcher: birth() };
+        if (existing?.status === "completed" && !reconcileId && !resumeId) {
+            db.exec("COMMIT");
+            return { scope_id: scopeId, control, current: true, settled: false, worker: publicRuntimeScopeWorker(existing) };
+        }
+        const renewal = resumeId ? `resume:${resumeId}` : reconcileId;
+        const snapshot = renewal ? renewRecoveryObservation(existing?.snapshot ?? {}, renewal) : existing?.snapshot;
+        if (existing && recoveryObservationBudget(snapshot).state().remaining_ms === 0) {
+            db.exec("COMMIT");
+            return { scope_id: scopeId, control, current: true, settled: false, worker: publicRuntimeScopeWorker(existing) };
+        }
+        const cleanupRequest = resumeId ? undefined : reconcileId ?? existing?.reconcile_id;
+        worker = { scope_id: scopeId, generation: control.generation, request_id: requestId, mode, token: crypto.randomUUID(), status: "starting", launcher: birth(), ...(snapshot ? { snapshot } : {}), ...(cleanupRequest ? { reconcile_id: cleanupRequest } : {}) };
         const now = context.now();
         db.run("INSERT INTO managed_resources (resource_kind,resource_key,owner_id,lease_token,lease_expires_at,process_id,metadata_json,created_at,updated_at) VALUES ('scope-control-worker',?,?,?,?,NULL,?,?,?) ON CONFLICT(resource_kind,resource_key) DO UPDATE SET lease_token = excluded.lease_token, metadata_json = excluded.metadata_json, updated_at = excluded.updated_at", [key, scopeId, worker.token, now, JSON.stringify(worker), now, now]);
         db.exec("COMMIT");
@@ -107,6 +126,10 @@ export async function serveRuntimeScopeControl(context, scopeId, generation) {
         throw error;
     }
     let heartbeat;
+    // A fence must never be held indefinitely by an observer that cannot obtain
+    // fresh provider evidence. A later operator request continues the retained
+    // budget; this owner records why it stopped without claiming settlement.
+    const observation = recoveryObservationBudget(worker.snapshot);
     try {
         const record = registerManagedProcess(context, { kind: "scope-control", ownerId: key, operationId: `scope-control:${key}:${token}`, ownerPid: process.pid, metadata: { role: "control", scope_id: scopeId, generation } });
         worker.process_id = record.id;
@@ -125,6 +148,19 @@ export async function serveRuntimeScopeControl(context, scopeId, generation) {
         for (;;) {
             if (leaseLost)
                 throw new AppError("runtime_scope_owner_changed", "Scope worker lost its managed lease", 1);
+            if (observation.exhausted()) {
+                worker.status = "failed";
+                worker.error = "recovery_observation_budget_exhausted";
+                worker.snapshot = {
+                    ...(worker.snapshot ?? {}),
+                    status: "recovery_blocked",
+                    settled: false,
+                    pending_reasons: ["recovery_observation_budget_exhausted"],
+                    error: { code: "recovery_observation_budget_exhausted", message: "Scope recovery observation exceeded its 120 second budget" },
+                    recovery_observation: observation.state(),
+                };
+                return { scope_id: scopeId, generation, status: "recovery_blocked", settled: false, error: worker.error };
+            }
             const status = runtimeBudgetStatus(context, scopeId);
             const release = status.release;
             if (release?.current === true && release.generation === generation) {
@@ -137,9 +173,16 @@ export async function serveRuntimeScopeControl(context, scopeId, generation) {
                 break;
             }
             const resume = status.resume;
-            worker.snapshot = await prepareRuntimeScopeResume(context, scopeId, generation, resume?.generation === generation ? resume.request_id : undefined) ?? await controlRuntimeScopeRuns(context, scopeId, worker.request_id, worker.mode);
+            const outcome = await withRecoveryObservation(observation, async () => (worker.reconcile_id ? null : await prepareRuntimeScopeResume(context, scopeId, generation, resume?.generation === generation ? resume.request_id : undefined))
+                ?? await controlRuntimeScopeRuns(context, scopeId, worker.request_id, worker.mode, worker.reconcile_id));
+            observation.exhausted();
+            worker.snapshot = { ...outcome, recovery_observation: observation.state() };
             db.run("UPDATE managed_resources SET metadata_json = ?, updated_at = ? WHERE resource_kind = 'scope-control-worker' AND resource_key = ? AND lease_token = ?", [JSON.stringify(worker), context.now(), key, token]);
-            await delay(1_000);
+            if (worker.snapshot.drain?.physical_settled === true) {
+                worker.status = "completed";
+                break;
+            }
+            await delay(observation.nextDelay());
         }
         return { scope_id: scopeId, generation, status: worker.status };
     }
@@ -158,6 +201,8 @@ export async function serveRuntimeScopeControl(context, scopeId, generation) {
     finally {
         if (heartbeat)
             clearInterval(heartbeat);
+        observation.exhausted();
+        worker.snapshot = { ...(worker.snapshot ?? {}), recovery_observation: observation.state() };
         db.run("UPDATE managed_resources SET metadata_json = ?, updated_at = ? WHERE resource_kind = 'scope-control-worker' AND resource_key = ? AND lease_token = ?", [JSON.stringify(worker), context.now(), key, token]);
         if (worker.process_id && worker.process_lease_token)
             finishManagedProcess(context, { id: worker.process_id, leaseToken: worker.process_lease_token, state: worker.status === "failed" ? "failed" : "stopped", reason: "scope_owner_exited" });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@deksden-com/dd-flow-cli",
-  "version": "0.9.0-beta.55",
+  "version": "0.9.0-beta.62",
   "description": "Mechanical runtime CLI for dd-flow workflows.",
   "type": "module",
   "bin": {

package/tools/repair-paused-run-status.mjs

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+/* global process, console */
+// Explicit repair of legacy fenced RUN projections; no migrations or process control.
+import fs from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { parseArgs } from 'node:util';
+import { DatabaseSync } from 'node:sqlite';
+import { registerStoreWriter } from '../dist/storage/writer-contract.js';
+import { setFlowRunRecoveryPaused, completeFlowRun } from '../dist/services/runs.js';
+
+const { values } = parseArgs({ options: { home: { type: 'string' }, run: { type: 'string' }, project: { type: 'string' }, apply: { type: 'boolean', default: false }, 'cancel-abandoned': { type: 'boolean', default: false } } });
+if (!values.home || !values.run || !values.project) throw new Error('Usage: --home <runtime> --project <project-id> --run <run-id> [--apply]');
+const home = fs.realpathSync(values.home), file = path.join(home, 'db.sqlite');
+const db = new DatabaseSync(file, { readOnly: !values.apply });
+let runtimePath, original, backup, report, changed = false;
+try {
+  db.exec('PRAGMA busy_timeout=4000');
+  if (values.apply) { registerStoreWriter(db); db.exec('BEGIN IMMEDIATE'); }
+  const run = db.prepare('SELECT * FROM runs WHERE project_id=? AND id=?').get(values.project, values.run);
+  if (!run) throw new Error('RUN not found');
+  const guard = db.prepare('SELECT * FROM run_recovery_guards WHERE project_id=? AND run_id=? ORDER BY generation DESC LIMIT 1').get(values.project, values.run);
+  const cancel = values['cancel-abandoned'];
+  if (cancel) {
+    if (fs.existsSync(run.project_root)) throw new Error('Abandoned-fixture cancellation requires an absent project directory');
+    for (const table of ['works', 'sessions', 'run_controllers']) {
+      if (!db.prepare("SELECT 1 FROM sqlite_schema WHERE type='table' AND name=?").get(table)) continue;
+      if (db.prepare(`SELECT 1 FROM ${table} WHERE project_id=? AND run_id=? LIMIT 1`).get(values.project, values.run)) throw new Error('Abandoned-fixture cancellation requires no Work, Session or controller history');
+    }
+    if (guard) throw new Error('A recovery-owned RUN requires recovery reconciliation');
+  }
+  if (!cancel && !["draining", "sealed", "resuming"].includes(guard?.status)) throw new Error("Repair requires the current recovery to fence productive admission");
+  if (run.status !== 'running') {
+    report = { run_id: run.id, status: run.status, changed: false };
+  } else {
+    runtimePath = fs.realpathSync(run.runtime_path);
+    const relative = path.relative(home, runtimePath);
+    if (relative.startsWith('..') || path.isAbsolute(relative)) throw new Error('Runtime file is outside the selected home');
+    original = fs.readFileSync(runtimePath);
+    if (JSON.parse(original).status !== 'running' || JSON.parse(run.index_json).status !== 'running') throw new Error('RUN projections disagree; manual review required');
+    if (values.apply) {
+      backup = `${runtimePath}.status-repair-${crypto.randomUUID()}.json`;
+      fs.writeFileSync(backup, JSON.stringify({ recovery_id: guard?.recovery_id ?? null, row: run, runtime: original.toString('utf8') }), { flag: 'wx', mode: 0o600 });
+      const context = { ddFlowHome: home, env: { ...process.env, DD_FLOW_HOME: home }, now: () => new Date().toISOString(), db: {
+        path: file, writable: true, get: (sql, params = []) => db.prepare(sql).get(...params), all: (sql, params = []) => db.prepare(sql).all(...params), run: (sql, params = []) => db.prepare(sql).run(...params), exec: sql => db.exec(sql)
+      } };
+      changed = true;
+      if (cancel) completeFlowRun(context, { projectRoot: run.project_root, runId: run.id, status: 'cancelled', manualOverrideReason: 'Operator requested cleanup of abandoned fixture: project removed; no Work, Session or controller history.' });
+      else setFlowRunRecoveryPaused(context, values.project, values.run, true);
+    }
+    report = { run_id: run.id, from: 'running', to: cancel ? 'cancelled' : 'paused', recovery_id: guard?.recovery_id ?? null, applied: values.apply, backup };
+  }
+  if (values.apply) db.exec('COMMIT');
+  console.log(JSON.stringify(report));
+} catch (error) {
+  if (values.apply && db.isTransaction) db.exec('ROLLBACK');
+  if (changed && original) { const temporary = `${runtimePath}.repair-rollback`; fs.writeFileSync(temporary, original); fs.renameSync(temporary, runtimePath); }
+  throw error;
+} finally { db.close(); }