@ngockhoale/ukit 3.0.8 → 3.0.9

package/src/core/agentRuntime/supervisor.js

@@ -0,0 +1,565 @@
+/**
+ * agentRuntime/supervisor.js — decision-first-runtime G2 (DR-03), SPEC §4.
+ *
+ * Owned-process supervisor: detached process-group spawn, serialized
+ * per-operation transition chain (journal seq stays gap-free under
+ * exit/cancel races), bounded event appends only on transitions and
+ * observation tiers (never per tick), SIGTERM→grace→SIGKILL group cancel
+ * through an injectable killImpl (only PIDs this supervisor spawned are
+ * ever signalled), the disableLaunches()/enableLaunches() kill switch
+ * (typed refuse, zero fs writes, evidence preserved), and a thin
+ * reconcile() delegate to recovery.js.
+ *
+ * Kill switch: `decisionRuntime.supervisor.stage` — absent/malformed →
+ * 'off'; `config.enabled === true` also enables the launch surface for
+ * embedders that do not carry runtimeConfig stage flags.
+ */
+
+import { spawn } from 'node:child_process';
+import path from 'node:path';
+
+import { resolveConfigStage } from '../runtimeConfig.js';
+import { selectWaitPolicy } from './promotion.js';
+import {
+  CONTRACT_VERSION,
+  SIDE_EFFECT_CLASSES,
+  isTerminal,
+  validateTransition,
+} from './contract.js';
+import {
+  appendEvent,
+  writeOperationState,
+  readJournal,
+} from './eventStore.js';
+import { classifyObservation } from './liveness.js';
+import { createArtifactWriter } from './artifacts.js';
+import { reconcileOwnedOperations } from './recovery.js';
+
+const SUPERVISOR_FLAG = 'decisionRuntime.supervisor.stage';
+// G6 (DR-09): `decisionRuntime.promotion.stage` — absent/malformed → 'off';
+// 'shadow' consults selectWaitPolicy at the caller-wait decision and records a
+// bounded receipt WITHOUT changing served wait semantics (prior path always
+// served; a throw/malformed decision falls back deterministically).
+const PROMOTION_FLAG = 'decisionRuntime.promotion.stage';
+const PRODUCER_VERSION = '3.0.8';
+const PRIVACY_CLASS = 'internal';
+const SIGTERM = 'SIGTERM';
+const SIGKILL = 'SIGKILL';
+
+const DEFAULT_LIMITS = Object.freeze({
+  pollIntervalMs: 250,
+  killGraceMs: 2_000,
+  closeWaitMs: 5_000,
+});
+
+const unsupported = (code) => ({ unsupported: true, code });
+
+// Real FileHandle exposes .sync(), not .fsync(); tolerate both.
+const defaultFsyncImpl = async (handle) => {
+  if (handle && typeof handle.sync === 'function') return handle.sync();
+  if (handle && typeof handle.fsync === 'function') return handle.fsync();
+  return undefined;
+};
+
+const defaultSpawnImpl = (argv, opts) => spawn(argv[0], argv.slice(1), opts);
+const defaultKillImpl = (signal, target) => process.kill(target, signal);
+
+function specIsValid(spec) {
+  if (spec === null || typeof spec !== 'object') return false;
+  if (typeof spec.operationId !== 'string' || spec.operationId === '') return false;
+  if (!Number.isInteger(spec.attempt) || spec.attempt < 1) return false;
+  const hasCommand = typeof spec.command === 'string' && spec.command !== '';
+  const hasArgv = Array.isArray(spec.argv) && spec.argv.length > 0
+    && spec.argv.every((a) => typeof a === 'string' && a !== '');
+  if (!hasCommand && !hasArgv) return false;
+  return SIDE_EFFECT_CLASSES.includes(spec.sideEffectClass);
+}
+
+/**
+ * @param {object} opts
+ * @param {string} opts.runtimeDir eventStore root (events/, state/, artifacts/)
+ * @param {string} [opts.artifactDir] bounded output artifact dir
+ * @param {object} [opts.config] runtimeConfig / `{enabled:true}` gate
+ * @param {object} [opts.clock] injectable `{now:number,setTimeout,clearTimeout}`
+ * @param {object} [opts.processTable] injected process snapshot for reconcile
+ * @param {number} [opts.fencingEpoch] this owner's monotonic epoch
+ * @param {number} [opts.ownerEpoch] epoch floor for transition validation
+ * @param {object} [opts.limits] {pollIntervalMs,killGraceMs,closeWaitMs,artifact caps}
+ * @param {Function} [opts.spawnImpl] (argv, spawnOpts) → child-like proc
+ * @param {Function} [opts.killImpl] (signal, target) → void (target = -pgid)
+ * @param {Function} [opts.fsyncImpl] artifact fsync
+ * @param {Function} [opts.appendEventImpl] journal append (test injection)
+ * @param {object} [opts.artifactDeps] artifact writer dep overrides
+ * @returns {object} Supervisor — SPEC §4 surface.
+ */
+export function createSupervisor(opts = {}) {
+  const runtimeDir = opts.runtimeDir;
+  const artifactDir = opts.artifactDir
+    ?? (typeof runtimeDir === 'string' ? path.join(runtimeDir, 'artifacts') : undefined);
+  const promotionStage = () => resolveConfigStage(opts.config ?? null, PROMOTION_FLAG);
+  const waitPolicyFn = typeof opts.waitPolicyFn === 'function' ? opts.waitPolicyFn : selectWaitPolicy;
+  const durationProfile = opts.durationProfile ?? null;
+  const resourcesFn = typeof opts.resourcesFn === 'function' ? opts.resourcesFn : () => ({});
+  // Bounded receipt log — shadow-stage evidence only, never fs.
+  const WAIT_RECEIPT_CAP = 128;
+  const waitReceipts = [];
+
+  function recordWaitReceipt(entry) {
+    if (waitReceipts.length >= WAIT_RECEIPT_CAP) waitReceipts.shift();
+    waitReceipts.push(Object.freeze(entry));
+  }
+
+  /**
+   * Consult selectWaitPolicy at the caller-wait decision point. Only invoked
+   * when `decisionRuntime.promotion` resolves non-'off'; a policy throw or
+   * malformed decision is still recorded and the prior path is always served.
+   */
+  function consultWaitPolicy(spec, policy) {
+    let decision;
+    let error;
+    try {
+      decision = waitPolicyFn(
+        durationProfile,
+        { class: spec?.sideEffectClass, spec },
+        resourcesFn() ?? {},
+        policy,
+      );
+    } catch (err) {
+      error = String(err?.message ?? err);
+    }
+    const valid = decision !== null && typeof decision === 'object'
+      && (decision.mode === 'foreground' || decision.mode === 'handle');
+    recordWaitReceipt({
+      operationId: spec?.operationId ?? null,
+      stage: promotionStage(),
+      policy: valid ? { mode: decision.mode, reason: decision.reason } : null,
+      error: error ?? null,
+    });
+  }
+
+  const limits = { ...DEFAULT_LIMITS, ...(opts.limits ?? {}) };
+  const stage = () => resolveConfigStage(opts.config ?? null, SUPERVISOR_FLAG);
+  const enabledByConfig = () => stage() !== 'off' || opts.config?.enabled === true;
+
+  let launchesDisabled = !enabledByConfig();
+  const enabledAtCreate = enabledByConfig();
+  let closed = false;
+
+  const spawnImpl = opts.spawnImpl ?? defaultSpawnImpl;
+  const killImpl = opts.killImpl ?? defaultKillImpl;
+  const append = opts.appendEventImpl ?? appendEvent;
+  const clock = opts.clock && typeof opts.clock.setTimeout === 'function'
+    ? opts.clock
+    : null;
+  const nowMs = clock ? () => clock.now : () => Date.now();
+  const setT = clock ? (cb, ms) => clock.setTimeout(cb, ms) : (cb, ms) => setTimeout(cb, ms);
+  const clearT = clock ? (t) => clock.clearTimeout(t) : (t) => clearTimeout(t);
+  const isoNow = () => new Date(nowMs()).toISOString();
+
+  const fencingEpoch = Number.isInteger(opts.fencingEpoch) ? opts.fencingEpoch : 1;
+  const ownerEpoch = Number.isInteger(opts.ownerEpoch) ? opts.ownerEpoch : undefined;
+
+  const artifactDeps = {
+    fsyncImpl: opts.fsyncImpl ?? defaultFsyncImpl,
+    ...(opts.artifactDeps ?? {}),
+  };
+  const artifactWriter = artifactDir
+    ? createArtifactWriter(artifactDir, limits, artifactDeps)
+    : null;
+
+  /** @type {Map<string, object>} live operation records by operationId */
+  const operations = new Map();
+
+  function makeEvent(op, eventType, safePayload, artifactRefs = []) {
+    return {
+      eventId: `supervisor-${op.id}-${op.seq}`,
+      operationId: op.id,
+      seq: op.seq,
+      eventType,
+      observedAt: isoNow(),
+      producerVersion: PRODUCER_VERSION,
+      contractVersion: CONTRACT_VERSION,
+      privacyClass: PRIVACY_CLASS,
+      artifactRefs,
+      safePayload,
+    };
+  }
+
+  /** Serialized per-op journal/state write; seq allocated inside the chain. */
+  function enqueue(op, fn) {
+    op.chain = op.chain.then(fn).catch((err) => {
+      op.chainError = err;
+    });
+    return op.chain;
+  }
+
+  function emitEvent(op, eventType, safePayload, artifactRefs) {
+    return enqueue(op, async () => {
+      op.seq += 1;
+      const event = makeEvent(op, eventType, safePayload, artifactRefs);
+      await append(runtimeDir, event);
+      return event;
+    });
+  }
+
+  function transition(op, to, extra = {}, artifactRefs) {
+    const from = op.state;
+    const verdict = validateTransition(from, to, { fencingEpoch, ownerEpoch });
+    if (!verdict.ok) return Promise.resolve(verdict);
+    op.state = to;
+    return enqueue(op, async () => {
+      op.seq += 1;
+      const event = makeEvent(op, 'operation.transition', { from, to, ...extra }, artifactRefs);
+      await append(runtimeDir, event);
+      await writeOperationState(runtimeDir, op.id, {
+        state: to,
+        fencingEpoch,
+        ownerEpoch,
+      });
+      return event;
+    });
+  }
+
+  function emitObservation(op, cls) {
+    if (op.observedClasses.has(cls)) return;
+    op.observedClasses.add(cls);
+    emitEvent(op, 'operation.observation', { class: cls });
+  }
+
+  function killGroup(op, signal) {
+    try {
+      killImpl(signal, -op.pgid);
+    } catch {
+      // ESRCH: group already gone — nothing owned left to reap.
+    }
+  }
+
+  /** SIGTERM → grace → SIGKILL on the owned process group only. */
+  function performCancel(op, reason) {
+    if (op.cancelRequested) return;
+    op.cancelRequested = true;
+    op.cancelReason = reason;
+    transition(op, 'cancel_pending', { reason });
+    killGroup(op, SIGTERM);
+    op.timers.add(setT(() => {
+      if (!op.exited) killGroup(op, SIGKILL);
+      // Backstop: a child that never reports exit still resolves terminal.
+      const from = op.state;
+      if (validateTransition(from, 'cancelled', { fencingEpoch, ownerEpoch }).ok) {
+        op.state = 'cancelled';
+        enqueue(op, async () => {
+          op.seq += 1;
+          const event = makeEvent(op, 'operation.transition', {
+            from,
+            to: 'cancelled',
+            reason: op.cancelReason ?? 'killed',
+          }, op.artifactRefs);
+          await append(runtimeDir, event);
+          await writeOperationState(runtimeDir, op.id, {
+            state: 'cancelled',
+            fencingEpoch,
+            ownerEpoch,
+          });
+        });
+      }
+    }, limits.killGraceMs));
+  }
+
+  function scheduleTick(op) {
+    op.timers.add(setT(() => tick(op), limits.pollIntervalMs));
+  }
+
+  function tick(op) {
+    if (op.exited || isTerminal(op.state)) return;
+    const now = nowMs();
+    const sample = {
+      processAlive: !op.exited,
+      bytesWritten: op.bytesWritten,
+      lastOutputAt: op.lastOutputAt,
+      startedAt: op.startedAt,
+      now,
+    };
+    const cls = classifyObservation(sample, op.prevSample, {
+      silentAfterMs: op.policy.silentAfterMs,
+      stallTimeoutMs: op.spec.stallTimeoutMs ?? op.policy.stallTimeoutMs,
+      wallTimeoutMs: op.spec.wallTimeoutMs ?? op.policy.wallTimeoutMs,
+      stallAction: op.policy.stallAction,
+    });
+    op.prevSample = sample;
+    switch (cls) {
+      case 'timeout':
+        performCancel(op, 'wall_timeout');
+        break;
+      case 'stalled':
+        emitObservation(op, 'stalled');
+        if (op.policy.stallAction === 'cancel') {
+          performCancel(op, 'stalled');
+        }
+        break;
+      case 'silent_live':
+        emitObservation(op, 'silent_live');
+        break;
+      default:
+        // live_progressing / exited: no persistent write (never per-tick).
+        break;
+    }
+    if (!op.exited && !isTerminal(op.state) && !op.cancelRequested) {
+      scheduleTick(op);
+    }
+  }
+
+  function artifactErrorCode(refs) {
+    for (const ref of refs) {
+      if (ref?.error?.code) return ref.error.code;
+    }
+    return null;
+  }
+
+  /** Runs INSIDE op.chain (via onExit's enqueue) — appends inline, never re-enqueues. */
+  async function finalizeExit(op) {
+    if (op.finalized || isTerminal(op.state)) return;
+    op.finalized = true;
+
+    let refs = [];
+    if (op.artifacts) {
+      try {
+        refs = await op.artifacts.close();
+      } catch {
+        refs = [];
+      }
+    }
+    op.artifactRefs = refs.filter((r) => r && r.bytes >= 0);
+    const artErr = artifactErrorCode(refs);
+
+    let to;
+    const extra = {};
+    if (op.cancelRequested || op.state === 'cancel_pending') {
+      to = 'cancelled';
+      extra.reason = op.cancelReason ?? 'killed';
+    } else {
+      const failed = artErr != null || (op.exitInfo?.code ?? 0) !== 0;
+      to = failed ? 'failed' : 'completed';
+      extra.exitCode = op.exitInfo?.code ?? null;
+      extra.signal = op.exitInfo?.signal ?? null;
+      if (artErr != null) {
+        extra.error = { code: artErr };
+      } else if (failed) {
+        extra.error = {
+          code: 'exit_nonzero',
+          exitCode: op.exitInfo?.code ?? null,
+          signal: op.exitInfo?.signal ?? null,
+        };
+      }
+    }
+
+    const from = op.state;
+    const verdict = validateTransition(from, to, { fencingEpoch, ownerEpoch });
+    if (!verdict.ok) return;
+    op.state = to;
+    op.seq += 1;
+    const event = makeEvent(op, 'operation.transition', { from, to, ...extra }, op.artifactRefs);
+    await append(runtimeDir, event);
+    await writeOperationState(runtimeDir, op.id, { state: to, fencingEpoch, ownerEpoch });
+  }
+
+  function onExit(op, code, signal) {
+    if (op.exited) return;
+    op.exited = true;
+    op.exitInfo = { code, signal };
+    for (const t of op.timers) clearT(t);
+    op.timers.clear();
+    // Drain the transition chain before deciding the terminal path so a
+    // racing cancel_pending write lands first (seq stays linear).
+    enqueue(op, () => finalizeExit(op));
+  }
+
+  const supervisor = {
+    /**
+     * Launch an owned operation. Kill switch (flag off or
+     * disableLaunches()) → `{ unsupported:true, code:'launch_disabled' }`
+     * with zero fs writes; malformed spec → `invalid_spec`; the spawned
+     * child runs detached in its own process group so cancel can reap the
+     * whole tree via `-pgid`.
+     */
+    async start(spec, policy = {}) {
+      if (closed || launchesDisabled || !enabledByConfig()) {
+        return unsupported('launch_disabled');
+      }
+      if (!specIsValid(spec)) {
+        return unsupported('invalid_spec');
+      }
+      // G6 promotion consult — shadow evaluates+records a receipt; served
+      // wait semantics below are always the prior path.
+      if (promotionStage() !== 'off') {
+        consultWaitPolicy(spec, policy);
+      }
+
+      const op = {
+        id: spec.operationId,
+        spec,
+        policy: policy && typeof policy === 'object' ? policy : {},
+        state: 'queued',
+        seq: 0,
+        chain: Promise.resolve(),
+        chainError: null,
+        proc: null,
+        pgid: null,
+        exited: false,
+        finalized: false,
+        cancelRequested: false,
+        cancelReason: null,
+        exitInfo: null,
+        bytesWritten: 0,
+        lastOutputAt: null,
+        startedAt: nowMs(),
+        prevSample: null,
+        observedClasses: new Set(),
+        artifactRefs: [],
+        artifacts: null,
+        timers: new Set(),
+      };
+      operations.set(op.id, op);
+
+      await transition(op, 'starting', {
+        attempt: spec.attempt,
+        sideEffectClass: spec.sideEffectClass,
+      });
+
+      let proc;
+      try {
+        const argv = Array.isArray(spec.argv) && spec.argv.length > 0
+          ? spec.argv
+          : ['/bin/sh', '-c', spec.command];
+        proc = spawnImpl(argv, {
+          detached: true,
+          stdio: ['ignore', 'pipe', 'pipe'],
+          env: spec.env,
+          cwd: spec.cwd,
+        });
+      } catch (err) {
+        await transition(op, 'failed', {
+          error: { code: 'spawn_failed', message: String(err?.message ?? err) },
+        });
+        return { ok: false, code: 'spawn_failed' };
+      }
+
+      op.proc = proc;
+      op.pgid = proc.pid;
+      op.lastOutputAt = nowMs();
+
+      if (artifactWriter) {
+        try {
+          op.artifacts = artifactWriter.open(op.id);
+        } catch {
+          op.artifacts = null;
+        }
+      }
+
+      const onData = (stream) => (chunk) => {
+        op.bytesWritten += chunk?.length ?? 0;
+        op.lastOutputAt = nowMs();
+        try {
+          op.artifacts?.[stream]?.write(chunk);
+        } catch {
+          // Artifact stream errors surface via refs at close.
+        }
+      };
+      proc.stdout?.on('data', onData('stdout'));
+      proc.stderr?.on('data', onData('stderr'));
+      proc.on?.('error', () => onExit(op, proc.exitCode ?? 1, null));
+      proc.on?.('exit', (code, signal) => onExit(op, code, signal));
+
+      await transition(op, 'running', { pid: proc.pid ?? null });
+      if (!op.exited) scheduleTick(op);
+
+      return {
+        operationId: op.id,
+        pid: proc.pid,
+        state: op.state,
+        get currentState() {
+          return op.state;
+        },
+      };
+    },
+
+    /** Async replay of the operation journal (contract v1 events). */
+    async* observe(operationId) {
+      if (typeof runtimeDir !== 'string' || runtimeDir === '') return;
+      try {
+        for await (const e of readJournal(runtimeDir, operationId)) yield e;
+      } catch {
+        return;
+      }
+    },
+
+    /** Descendant-aware cancel: SIGTERM→grace→SIGKILL on the owned group. */
+    async cancel(operationId, reason = 'cancel_requested') {
+      const op = operations.get(operationId);
+      if (!op) return { ok: false, code: 'not_found' };
+      if (isTerminal(op.state)) return { ok: false, code: 'already_terminal' };
+      performCancel(op, reason);
+      return { ok: true };
+    },
+
+    /** One-shot restart reconciliation — thin delegate to recovery.js. */
+    async reconcile() {
+      if (typeof runtimeDir !== 'string' || runtimeDir === '') {
+        return { results: [] };
+      }
+      try {
+        // Dynamic import keeps the delegate resilient when recovery.js is
+        // absent from a partial tree (avoids a static-resolution failure).
+        const mod = await import('./recovery.js').catch(() => null);
+        const fn = mod?.reconcileOwnedOperations ?? reconcileOwnedOperations;
+        if (typeof fn !== 'function') return { results: [] };
+        return await fn({
+          runtimeDir,
+          processTable: opts.processTable ?? {},
+          fencingEpoch,
+          ownerEpoch: opts.ownerEpoch,
+          launchDisabled: launchesDisabled,
+          now: opts.now,
+        });
+      } catch {
+        return { results: [] };
+      }
+    },
+
+    disableLaunches() {
+      launchesDisabled = true;
+    },
+
+    enableLaunches() {
+      if (enabledAtCreate) launchesDisabled = false;
+    },
+
+    /** Bounded frozen receipts of shadow-stage wait-policy consults (G6). */
+    waitPolicyReceipts() {
+      return [...waitReceipts];
+    },
+
+    /**
+     * Reap in-flight owned children and drain journals with a bounded wait
+     * on the REAL clock — the injected clock may never advance on its own.
+     */
+    async close() {
+      closed = true;
+      for (const op of operations.values()) {
+        for (const t of op.timers) clearT(t);
+        op.timers.clear();
+        if (!op.exited && !isTerminal(op.state)) {
+          op.cancelRequested = true;
+          killGroup(op, SIGKILL);
+        }
+      }
+      const deadline = new Promise((r) => setTimeout(r, limits.closeWaitMs));
+      await Promise.race([
+        Promise.allSettled([...operations.values()].map((op) => op.chain)),
+        deadline,
+      ]);
+      operations.clear();
+    },
+  };
+
+  return supervisor;
+}