@deksden-com/dd-flow-cli 0.9.0-beta.16 → 0.9.0-beta.161

package/dist/harness-runtime/lib/daemon-observation.mjs

@@ -0,0 +1,171 @@
+import net from 'node:net';
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { randomUUID } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { hashJson, writeJsonAtomic } from './runner-events.mjs';
+import { ObservationClock } from './observation-clock.mjs';
+import { canonicalNativeOperation, nativeOperationWait, validateDaemonWait } from './adapter-timeouts.mjs';
+import { assertNativeLine } from './native-streams.mjs';
+import { operationDirectory, readOperationRecord, readOperationProgress, readOperationWaiting, suspendOperationClock } from './operation-progress.mjs';
+
+const productive = new Set(['session.create', 'session.prompt', 'session.start', 'session.fork', 'session.resume']);
+const progressObserver = new AsyncLocalStorage();
+export const withDaemonProgressObserver = (observer, action) => progressObserver.run(observer, action);
+const fault = (code, message, details) => Object.assign(new Error(message), { code, retryable: false, details });
+export async function retainedDaemonReply(root, id, binding = {}) {
+  const requested = await readOperationRecord(root, id, 'requested.json');
+  if (!requested) return null;
+  if (requested.operation_id !== id || Object.entries(binding).some(([key, value]) => (requested[key] ?? null) !== (value ?? null))) throw fault('operation_conflict', 'retained operation differs from dispatched request', { operation_id: id });
+  const terminal = await readOperationRecord(root, id, 'result.json');
+  if (!terminal) return null;
+  if (['operation_id','daemon_id','operation','params_sha256','owner_id','generation'].some(field=>Object.hasOwn(terminal,field)&&(terminal[field] ?? null)!==(requested[field] ?? null))) throw fault('operation_conflict','native outcome binding differs from original operation',{operation_id:id});
+  if (terminal.state !== 'completed') throw Object.assign(new Error(terminal.error?.message ?? 'native operation failed'), terminal.error);
+  if (Number.isSafeInteger(requested.generation)) try {
+    const fence = JSON.parse(await readFile(path.join(root,'operation-fence.json'),'utf8'));
+    if (fence.daemon_id === requested.daemon_id && fence.generation > requested.generation) throw fault('operation_cancelled','continuation is fenced by newer control',{operation_id:id,native_result:terminal.result});
+  } catch(error) { if (error.code !== 'ENOENT') throw error; }
+  const settlement = await readOperationRecord(root, id, 'settlement.json');
+  try {
+    const daemon = JSON.parse(await readFile(path.join(root,'daemon.json'),'utf8'));
+    if ((daemon.daemon_id ?? daemon.config?.daemonId ?? null)!==requested.daemon_id || (daemon.config?.runtime_owner?.owner_id ?? null)!==(requested.owner_id ?? null) || ['stopping','clean','unclean','cleanup_failed','failed'].includes(daemon.shutdown_state)) return {ready:false,outcome:terminal.result,settlement,reason:'daemon_authority_changed'};
+    if (requested.owner_id) try { process.kill(daemon.pid,0); } catch { return {ready:false,outcome:terminal.result,settlement,reason:'daemon_owner_dead'}; }
+  } catch(error) { if(error.code!=='ENOENT') throw error; if(requested.daemon_id || requested.owner_id) return {ready:false,outcome:terminal.result,settlement,reason:'daemon_absent'}; }
+  if (!['settled', 'not_required'].includes(settlement?.state)) return { ready: false, outcome: terminal.result, settlement };
+  return { ready: true, outcome: terminal.result, settlement, value: settlement.state === 'not_required' ? terminal.result : { ...terminal.result, settlement } };
+}
+export async function recoverDaemonReply(root, id, { timeoutMs = 30_000, signal, binding = {}, onProgress = () => {}, onOutcome = () => {}, onPendingSettlement, observationClock } = {}) {
+  const saved = observationClock ? null : await readOperationRecord(root, id, 'observer-recovery.json') ?? await readOperationRecord(root, id, 'observer-transport.json');
+  signal?.throwIfAborted();
+  const clock = observationClock ?? new ObservationClock({ timeoutMs: saved?.timeout_ms ?? timeoutMs, saved });
+  let lastSequence = clock.progress ?? 0, lastSaved = 0, latestProgress, settlementAttempted = false, settlementError;
+  const exhaustedCursor = clock.progress;
+  let exhausted = clock.observationLost || clock.elapsed >= clock.timeoutMs;
+  for (;;) {
+    signal?.throwIfAborted();
+    const retained = await retainedDaemonReply(root, id, binding);
+    signal?.throwIfAborted();
+    if (retained) {
+      onOutcome(retained.outcome);
+      signal?.throwIfAborted();
+      if (retained.ready) return retained.value;
+    }
+    const progress = await readOperationProgress(root, id, binding);
+    signal?.throwIfAborted();
+    if (progress && progress.sequence > lastSequence) {
+      lastSequence = progress.sequence;
+      const observedAt = Date.parse(progress.observed_at);
+      // The outer cursor counts diagnostics, not durable sequences. A handoff
+      // may recognize a higher sequence with the same already-observed event.
+      if (clock.progressAt === null || observedAt > clock.progressAt) {
+        latestProgress={cursor:progress.sequence,observed_at:observedAt}; onProgress(progress);
+      }
+    }
+    const waiting = await readOperationWaiting(root,id,binding);
+    signal?.throwIfAborted();
+    if (waiting) suspendOperationClock(clock);
+    const sampled = waiting ? false : clock.sample(latestProgress);
+    if (clock.progress !== exhaustedCursor && clock.elapsed < clock.timeoutMs && !clock.observationLost) exhausted = false;
+    const expired = !waiting && (exhausted || sampled);
+    if ((expired || Date.now() - lastSaved >= 1_000) && await readOperationRecord(root,id,'requested.json')) { await writeJsonAtomic(path.join(operationDirectory(root,id),'observer-recovery.json'),clock.state()); lastSaved = Date.now(); }
+    if (expired) {
+      const final = await retainedDaemonReply(root, id, binding);
+      signal?.throwIfAborted();
+      if (final?.ready) return final.value;
+      throw fault(final ? 'settlement_reconciliation_required' : 'operation_observation_lost', 'original operation has no dispatch-ready reply', { operation_id: id, native_outcome: final ? 'completed' : 'unknown', native_result: final?.outcome, settlement: final?.settlement, reason:final?.reason, ...(settlementError ? { settlement_error:settlementError } : {}), state_dir: root });
+    }
+    if (retained && onPendingSettlement && !settlementAttempted && !retained.reason) {
+      settlementAttempted = true;
+      try { await onPendingSettlement(retained); } catch (error) { settlementError = {code:error.code,message:error.message}; }
+      signal?.throwIfAborted();
+      continue;
+    }
+    await new Promise(resolve => setTimeout(resolve, Math.min(100, timeoutMs / 4)));
+  }
+}
+export async function callDaemonObserved({ stateDir, socket, request, responseSchema, harness, timeoutMs }) {
+  request = { ...request, operation: canonicalNativeOperation(harness, request.operation) };
+  validateDaemonWait(harness, request.operation, timeoutMs);
+  assertNativeLine(JSON.stringify(request), 'request');
+  const durable = productive.has(request.operation);
+  const verification = request.operation === 'hook.verify' && request.params?.binding?.native_admission === 'assigned_work_start';
+  if (verification) request = { ...request, id: randomUUID() }; // Read-only proof is fresh, never a replay of prior authority.
+  const observable = durable || verification, onProgress = progressObserver.getStore();
+  const binding = { operation: request.operation, params_sha256: hashJson({ operation: request.operation, params: request.params ?? {} }) };
+  let initialDaemon;
+  if (observable) try {
+    const state = JSON.parse(await readFile(path.join(stateDir,'daemon.json'),'utf8'));
+    initialDaemon = state.daemon_id ?? state.config?.daemonId ?? null;
+    binding.owner_id = state.config?.runtime_owner?.owner_id ?? null;
+  } catch (error) { if (error.code !== 'ENOENT') throw error; }
+  // Reject a reused ID with different inputs before opening a transport.
+  // Otherwise its retained progress can win the race with the daemon's
+  // conflict reply and misclassify a valid old receipt as corrupt progress.
+  if (durable) await retainedDaemonReply(stateDir, request.id, {
+    ...binding, ...(initialDaemon === undefined ? {} : { daemon_id: initialDaemon })
+  });
+  const savedClock = durable ? await readOperationRecord(stateDir,request.id,'observer-transport.json') : null;
+  return await new Promise((resolve, reject) => {
+    const client = net.createConnection(socket);
+    let buffer = '', done = false, polling = false, pollFlight, lastSequence = savedClock?.cursor ?? 0, frozenDaemon = initialDaemon, disconnected = false, latestProgress, lastSaved = 0;
+    const bound = () => ({ ...binding, ...(frozenDaemon === undefined ? {} : { daemon_id: frozenDaemon }) });
+    const nativeWork = nativeOperationWait(harness, request.operation) === 'native-work';
+    let clock = nativeWork ? null : new ObservationClock({ timeoutMs, saved: savedClock });
+    const finish = (error, result) => { if (done) return; done = true; clearInterval(timer); client.destroy(); void Promise.resolve(pollFlight).then(() => error ? reject(error) : resolve(result)); };
+    const reconcile = async error => {
+      if (done) return;
+      try {
+        const reply = durable ? await retainedDaemonReply(stateDir, request.id, bound()) : null;
+        if (reply?.ready) return finish(null, reply.value);
+        if (reply) error = fault('settlement_reconciliation_required', 'native outcome retained; dispatch readiness is not confirmed', { operation_id: request.id, native_outcome: 'completed', native_result: reply.outcome, settlement: reply.settlement, reason:reply.reason, state_dir: stateDir });
+        else if (durable) error = fault(error.code, error.message, { ...error.details, operation_id: request.id, native_outcome: 'unknown', state_dir: stateDir });
+        finish(error);
+      } catch (cause) { finish(cause); }
+    };
+    const poll = () => {
+      if (done || polling) return pollFlight; polling = true;
+      pollFlight = (async () => {
+      try {
+        const progress = observable ? await readOperationProgress(stateDir, request.id, { ...binding, ...(frozenDaemon === undefined ? {} : { daemon_id: frozenDaemon }) }) : null;
+        if (done) return;
+        if (progress && progress.sequence > lastSequence) {
+          frozenDaemon ??= progress.daemon_id; lastSequence = progress.sequence;
+          binding.generation ??= progress.generation;
+          latestProgress = {cursor:progress.sequence,observed_at:Date.parse(progress.observed_at)};
+          if (verification && progress.phase === 'verification_progress') onProgress?.(progress);
+          process.stderr.write(`${JSON.stringify({ kind: 'operation_progress', ...progress })}\n`);
+        }
+        if (durable && disconnected) {
+          const reply = await retainedDaemonReply(stateDir, request.id, bound());
+          if (reply?.ready) return finish(null, reply.value);
+        }
+        const waiting = durable && await readOperationWaiting(stateDir,request.id,bound());
+        if (done) return;
+        if (clock && waiting) suspendOperationClock(clock);
+        const expired = !waiting && clock?.sample(latestProgress ?? (durable ? undefined : buffer.length || undefined));
+        if (durable && clock && (expired || Date.now()-lastSaved>=1_000) && await readOperationRecord(stateDir,request.id,'requested.json')) { await writeJsonAtomic(path.join(operationDirectory(stateDir,request.id),'observer-transport.json'),clock.state()); lastSaved=Date.now(); }
+        if (expired) await reconcile(fault('daemon_timeout', `${request.operation} timed out`));
+      } catch (error) { finish(error); } finally { polling = false; }
+      })();
+      return pollFlight;
+    };
+    const timer = setInterval(poll, Math.max(1, Math.min(100, (timeoutMs ?? 400) / 4)));
+    client.setEncoding('utf8'); client.on('connect', () => client.write(`${JSON.stringify(request)}\n`));
+    const lost = error => { if (durable) { disconnected = true; clock ??= new ObservationClock({timeoutMs: timeoutMs ?? 30_000,saved:savedClock}); void poll(); } else void reconcile(error); };
+    client.on('error', error => lost(fault('daemon_not_running', error.message)));
+    client.on('end', () => lost(fault('daemon_connection_closed', 'daemon closed before reply')));
+    client.on('close', () => lost(fault('daemon_connection_closed', 'daemon closed before reply')));
+    client.on('data', chunk => { void (async () => {
+      buffer += chunk;
+      const at = buffer.indexOf('\n'); if (at < 0) return;
+      try {
+        const response = JSON.parse(buffer.slice(0, at));
+        if (response.schema_id !== responseSchema || response.id !== request.id) throw fault('operation_observation_lost', 'daemon reply identity mismatch');
+        if (!response.ok) return finish(Object.assign(new Error(response.error?.message ?? 'daemon operation failed'), response.error));
+        if (durable && (binding.owner_id && !response.result?.settlement || response.result?.settlement && !['settled','not_required'].includes(response.result.settlement.state))) { disconnected = true; clock ??= new ObservationClock({timeoutMs:savedClock?.timeout_ms ?? timeoutMs ?? 30_000,saved:savedClock}); void poll(); return; }
+        if (durable) { const retained=await retainedDaemonReply(stateDir,request.id,bound()); if(retained) { if(retained.ready) return finish(null,retained.value); disconnected=true; clock ??= new ObservationClock({timeoutMs:timeoutMs ?? 30_000,saved:savedClock}); void poll(); return; } return void reconcile(fault('operation_observation_lost','productive native reply has no durable original receipt')); }
+        finish(null, response.result);
+      } catch (error) { void reconcile(error.code ? error : Object.assign(fault('operation_observation_lost', 'daemon reply could not be decoded', {native_outcome:'unknown'}), {cause:error})); }
+    })(); });
+  });
+}

package/dist/harness-runtime/lib/daemon-operations.d.mts

@@ -0,0 +1,2 @@
+export function inspectDaemonOperation(root: string, id: string): Promise<Record<string, unknown>>;
+export function validateNativeLaunchEnvironment(value: unknown): Record<string, string | null>;