@ngockhoale/ukit 3.0.8 → 3.0.10

package/src/core/agentRuntime/recovery.js

@@ -0,0 +1,266 @@
+/**
+ * agentRuntime/recovery.js — decision-first-runtime G2 (DR-04), SPEC §5.
+ *
+ * Restart reconciliation: replay journals + guarded state on supervisor start
+ * and resolve every recorded operation exactly once. The journal is truth;
+ * the state file is a cache consulted only for launch metadata the journal
+ * cannot carry yet (PID, processStartTime, exitStatus, fencing/lease epochs).
+ *
+ * Hard invariants (SPEC §5, G2-FR05/06):
+ *  - never spawns: a crash cannot produce a second side-effecting action;
+ *  - never kills: a PID not provably ours (start-time/lease match) is foreign;
+ *  - non-terminal ops with no live owned process → `recovery_required`
+ *    (typed code per crash window) — never silently re-spawned or replayed;
+ *  - a provably exited process whose terminal event was never committed gets
+ *    exactly one terminal append — idempotent on seq (double reconcile = 0);
+ *  - stale fencingEpoch → `stale_owner` reject before any durable write.
+ *
+ * Resolution codes:
+ *  'start_unconfirmed'  journal has no start proof — never re-spawn
+ *  'orphan_possible'    spawn committed but PID record lost — do not kill
+ *  'live_mismatch'      recorded PID live but (pid,startTime) mismatched —
+ *                       PID reuse: foreign process, never signal it
+ *  'exited_unconfirmed' provably gone/exited without outcome → terminal append
+ *                       when exitStatus is known, else recovery_required
+ *  'launch_disabled'    kill switch active at restart — zero launches
+ *  'stale_owner'        fencingEpoch below recorded owner epoch
+ */
+
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+
+import {
+  CONTRACT_VERSION,
+  isTerminal,
+  validateTransition,
+} from './contract.js';
+import {
+  appendEvent,
+  deriveOperationState,
+  writeOperationState,
+} from './eventStore.js';
+
+const JOURNAL_DIR = 'events';
+const STATE_DIR = 'state';
+const PRODUCER_VERSION = '3.0.8';
+
+async function pathExists(p) {
+  try {
+    await fs.access(p);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function readJsonSafe(file) {
+  try {
+    return JSON.parse(await fs.readFile(file, 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
+/** Enumerate operation ids from both journal and guarded-state dirs. */
+async function listOperationIds(runtimeDir) {
+  const ids = new Set();
+  for (const [sub, ext] of [[JOURNAL_DIR, '.jsonl'], [STATE_DIR, '.json']]) {
+    const dir = path.join(runtimeDir, sub);
+    let entries;
+    try {
+      entries = await fs.readdir(dir);
+    } catch {
+      continue;
+    }
+    for (const name of entries) {
+      if (name.endsWith(ext)) ids.add(name.slice(0, -ext.length));
+    }
+  }
+  return [...ids].sort();
+}
+
+function ownerEpochFor(stateRec, ctx) {
+  const epochs = [ctx?.ownerEpoch, stateRec?.fencingEpoch, stateRec?.leaseEpoch]
+    .filter((n) => Number.isInteger(n) && n >= 0);
+  return epochs.length ? Math.max(...epochs) : 0;
+}
+
+function makeEvent(operationId, seq, eventType, safePayload, now) {
+  return {
+    eventId: `recovery-${operationId}-${seq}`,
+    operationId,
+    seq,
+    eventType,
+    observedAt: now(),
+    producerVersion: PRODUCER_VERSION,
+    contractVersion: CONTRACT_VERSION,
+    privacyClass: 'internal',
+    artifactRefs: [],
+    safePayload,
+  };
+}
+
+/**
+ * Reconcile all recorded operations once on supervisor start.
+ *
+ * @param {object} ctx
+ * @param {string} ctx.runtimeDir eventStore root (events/, state/)
+ * @param {object} [ctx.processTable] `{ pid: { alive, startTime, exitStatus? } }`
+ *   injected snapshot — reconcile never probes or signals processes itself
+ * @param {number} ctx.fencingEpoch this owner's monotonic epoch
+ * @param {number} [ctx.ownerEpoch] epoch floor when state records lack one
+ * @param {boolean} [ctx.launchDisabled] kill switch active at restart
+ * @param {() => string} [ctx.now] ISO timestamp source (injectable clock)
+ * @returns {Promise<{results: Array<{operationId, resolution, code}>}>}
+ */
+export async function reconcileOwnedOperations(ctx) {
+  const runtimeDir = ctx?.runtimeDir;
+  if (typeof runtimeDir !== 'string' || runtimeDir === '') {
+    throw new TypeError('reconcileOwnedOperations: runtimeDir is required');
+  }
+  const processTable = ctx?.processTable && typeof ctx.processTable === 'object'
+    ? ctx.processTable
+    : {};
+  const fencingEpoch = Number.isInteger(ctx?.fencingEpoch) ? ctx.fencingEpoch : 0;
+  const launchDisabled = ctx?.launchDisabled === true;
+  const now = typeof ctx?.now === 'function' ? ctx.now : () => new Date().toISOString();
+
+  const results = [];
+  for (const operationId of await listOperationIds(runtimeDir)) {
+    results.push(await reconcileOne({
+      runtimeDir,
+      operationId,
+      processTable,
+      fencingEpoch,
+      ownerEpoch: undefined,
+      launchDisabled,
+      ctxOwnerEpoch: ctx?.ownerEpoch,
+      now,
+    }));
+  }
+  return { results };
+}
+
+async function reconcileOne({
+  runtimeDir,
+  operationId,
+  processTable,
+  fencingEpoch,
+  launchDisabled,
+  ctxOwnerEpoch,
+  now,
+}) {
+  const stateRec = await readJsonSafe(path.join(runtimeDir, STATE_DIR, `${operationId}.json`));
+  const derived = await deriveOperationState(runtimeDir, operationId);
+  const journalState = derived.state; // truth
+  const journalHasEvents = derived.lastSeq > 0;
+  const stateFileState = typeof stateRec?.state === 'string' ? stateRec.state : null;
+  const effectiveState = journalState ?? stateFileState;
+  const lastSeq = derived.lastSeq;
+  const ownerEpoch = ownerEpochFor(stateRec, { ownerEpoch: ctxOwnerEpoch });
+  const fenced = { fencingEpoch, ownerEpoch };
+
+  const result = (resolution, code) => ({ operationId, resolution, code });
+
+  const persistRecovery = async (from, code) => {
+    const t = validateTransition(from, 'recovery_required', fenced);
+    if (!t.ok) return result('stale_owner', t.code);
+    const event = makeEvent(operationId, lastSeq + 1, 'operation.transition', {
+      from,
+      to: 'recovery_required',
+      resolution: 'recovery_required',
+      code,
+    }, now);
+    await appendEvent(runtimeDir, event);
+    await writeOperationState(runtimeDir, operationId, {
+      ...(stateRec ?? {}),
+      state: 'recovery_required',
+      resolution: code,
+    });
+    return result('recovery_required', code);
+  };
+
+  const persistTerminal = async (from, to, code) => {
+    const t = validateTransition(from, to, fenced);
+    if (!t.ok) return result('stale_owner', t.code);
+    const event = makeEvent(operationId, lastSeq + 1, 'operation.transition', {
+      from,
+      to,
+      resolution: 'terminal_appended',
+      code,
+    }, now);
+    await appendEvent(runtimeDir, event);
+    await writeOperationState(runtimeDir, operationId, {
+      ...(stateRec ?? {}),
+      state: to,
+      resolution: code,
+    });
+    return result('terminal_appended', code);
+  };
+
+  // --- resolved already -----------------------------------------------------
+  if (isTerminal(journalState)) return result('already_terminal', journalState);
+  if (journalState === 'recovery_required') {
+    return result('already_recovery_required', 'recovery_required');
+  }
+  if (isTerminal(effectiveState)) {
+    // state cache recorded a terminal outcome but the journal never got the
+    // commit — backfill exactly one terminal event (idempotent on seq).
+    return persistTerminal(journalState, stateFileState, 'exited_unconfirmed');
+  }
+  // --- fencing: never mutate under a stale owner ----------------------------
+  // Any durable resolution below re-validates via validateTransition; check
+  // eagerly so a stale owner rejects before ANY write attempt.
+  if (fencingEpoch < ownerEpoch) {
+    return result('stale_owner', 'stale_owner');
+  }
+
+  // --- crash window: journal has no start proof -----------------------------
+  if (!journalHasEvents) {
+    if (stateRec?.spawned === true || Number.isInteger(stateRec?.pid)) {
+      // spawn committed but the PID/handle record was lost — a live foreign
+      // PID cannot be proven ours; never kill, never re-spawn.
+      return persistRecovery('starting', 'orphan_possible');
+    }
+    return persistRecovery('starting', 'start_unconfirmed');
+  }
+
+  // --- kill switch at restart: zero launches --------------------------------
+  if (launchDisabled) {
+    return persistRecovery(effectiveState ?? 'starting', 'launch_disabled');
+  }
+
+  // --- liveness fence: (pid, processStartTime) must both match --------------
+  const pid = Number.isInteger(stateRec?.pid) ? stateRec.pid : null;
+  const entry = pid != null ? processTable[pid] ?? processTable[String(pid)] : null;
+  const alive = entry?.alive === true;
+
+  if (pid != null && alive) {
+    const recordedStart = stateRec?.processStartTime;
+    const matches = recordedStart != null && entry.startTime === recordedStart;
+    if (!matches) {
+      // PID reuse: the live process is foreign — never signal it.
+      return persistRecovery(effectiveState, 'live_mismatch');
+    }
+    // Provably ours and alive → resume observation only; durable state stays
+    // non-terminal so a second reconcile keeps resuming (idempotent).
+    return result('resumed', 'alive_owned');
+  }
+
+  // --- no live owned process ------------------------------------------------
+  if (Number.isInteger(stateRec?.exitStatus)) {
+    const to = stateRec.exitStatus === 'cancelled'
+      ? 'cancelled'
+      : stateRec.exitStatus === 0
+        ? 'completed'
+        : 'failed';
+    return persistTerminal(effectiveState, to, 'exited_unconfirmed');
+  }
+  // Gone without outcome (or still starting): ambiguous — recovery_required,
+  // never a guessed terminal state, never a re-spawn. A `starting` op whose
+  // PID record never landed may have left an orphan we cannot prove is ours.
+  const tailCode = effectiveState === 'starting' && pid == null
+    ? 'orphan_possible'
+    : 'exited_unconfirmed';
+  return persistRecovery(effectiveState ?? 'starting', tailCode);
+}

package/src/core/agentRuntime/resourcePolicy.js

@@ -0,0 +1,78 @@
+// Deterministic resource-pressure policy for runtime event emission (G6 / TASK-002).
+// Pure module: no I/O, no runtimeConfig reads, no adapter/context imports.
+// Maps {cpuPressure, eventClass, queueDepth} to a frozen ResourceDecision —
+// {action:'emit'|'coalesce'|'defer', maxDeferMs?, reason:string}.
+//
+// DR-FR09 invariant: terminal, failure, permission and user-input events are
+// NEVER dropped or deferred — they always `emit` under any pressure or queue
+// depth. Only `progress`-class observations may be coalesced/deferred, and
+// deferral is always bounded (maxDeferMs) — there is no 'drop' action.
+// Unknown event class or malformed input resolves `emit` (fail open for
+// correctness, not throughput); this function never throws.
+
+export const RESOURCE_ACTIONS = Object.freeze(['emit', 'coalesce', 'defer']);
+
+export const EVENT_CLASSES = Object.freeze([
+  'terminal',
+  'failure',
+  'permission',
+  'user-input',
+  'progress',
+]);
+
+export const PRESSURE_LEVELS = Object.freeze(['normal', 'elevated', 'critical']);
+
+// Bounded deferral ceilings (literal config values, ms). Coalescing may batch
+// for up to this window; deferral may postpone emission by up to this window.
+const MAX_DEFER_MS = Object.freeze({ elevated: 500, critical: 2000 });
+
+// Queue depth below which pressure alone does not justify deferral — coalesce
+// is the cheaper backpressure signal and is preferred first.
+const DEFER_QUEUE_DEPTH = 8;
+
+const CRITICAL_CLASSES = new Set(['terminal', 'failure', 'permission', 'user-input']);
+const PRESSURE_SET = new Set(PRESSURE_LEVELS);
+
+const freeze = (action, reason, extra) =>
+  Object.freeze({ action, reason, ...extra });
+
+/**
+ * selectResourcePolicy({cpuPressure, eventClass, queueDepth}) → frozen
+ * ResourceDecision. Never throws; never returns `drop`.
+ */
+export function selectResourcePolicy(input) {
+  if (input === null || typeof input !== 'object' || Array.isArray(input)) {
+    return freeze('emit', 'malformed');
+  }
+
+  const { cpuPressure, eventClass, queueDepth } = input;
+
+  if (typeof eventClass !== 'string' || !EVENT_CLASSES.includes(eventClass)) {
+    // Any non-string or unrecognized class — including missing fields on a
+    // structurally valid object — fails open.
+    if (typeof eventClass === 'string' && eventClass.length > 0) {
+      return freeze('emit', 'class:unknown');
+    }
+    return freeze('emit', 'malformed');
+  }
+
+  // DR-FR09: critical classes always emit, regardless of pressure/depth.
+  if (CRITICAL_CLASSES.has(eventClass)) {
+    return freeze('emit', 'class:critical');
+  }
+
+  // eventClass === 'progress' from here.
+  if (!PRESSURE_SET.has(cpuPressure) || typeof queueDepth !== 'number' || !Number.isFinite(queueDepth) || queueDepth < 0) {
+    return freeze('emit', 'malformed');
+  }
+
+  if (cpuPressure === 'normal') {
+    return freeze('emit', 'pressure:normal');
+  }
+
+  const maxDeferMs = MAX_DEFER_MS[cpuPressure];
+  if (cpuPressure === 'critical' || queueDepth >= DEFER_QUEUE_DEPTH) {
+    return freeze('defer', 'pressure:defer', { maxDeferMs });
+  }
+  return freeze('coalesce', 'pressure:coalesce', { maxDeferMs });
+}

package/src/core/agentRuntime/runtimeSupport.js

@@ -0,0 +1,237 @@
+/**
+ * agentRuntime/runtimeSupport.js — decision-first-runtime G7 (DR-10),
+ * SPEC §2 G7-FR03, §4, §5.
+ *
+ * buildRuntimeSupportRecords(dir, opts?)
+ *   → { version, records[], coverage, redaction_version }
+ *
+ * Pure reader over the eventStore journals: each operation becomes one
+ * bounded support record carrying counts/codes/timings/artifact digests
+ * ONLY. `safePayload` strings are never copied into records — the only
+ * payload-derived values are state names, typed codes (safePayload.code /
+ * resolution / exit_code), and artifact sha256 digests. Every record is
+ * passed through sanitizeForSupport (default-deny, second privacy
+ * boundary): rejections are drop-and-count, never repaired, never
+ * bypassed.
+ *
+ * Invariants:
+ *   - operationIds are pseudonymized per call (bundle-local salted
+ *     sha256, projector.js convention) — raw IDs never appear in output.
+ *   - no writes to journals/runtime storage; no config reads.
+ *   - corrupt journals and unreadable dirs degrade to typed coverage —
+ *     never throw raw, never fabricate records.
+ *   - bounds: maxRecords / maxBytes caps are drop-and-count.
+ */
+
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+
+import { readJournal } from './eventStore.js';
+import { sanitizeForSupport } from '../observability/privacy/sanitizeForSupport.js';
+import { REDACTION_VERSION } from '../observability/privacy/allowlist.js';
+
+export const SUPPORT_RECORDS_VERSION = 'ukit-runtime-support/1';
+
+const JOURNAL_DIR = 'events';
+
+export const DEFAULT_CAPS = Object.freeze({
+  maxRecords: 2000,
+  maxBytes: 8 * 1024 * 1024,
+});
+
+function sha256Hex(input) {
+  return crypto.createHash('sha256').update(input).digest('hex');
+}
+
+/** Bundle-local pseudonym: deterministic within one call, salted per call. */
+function pseudonymizer(salt) {
+  const cache = new Map();
+  return (value) => {
+    let pseudo = cache.get(value);
+    if (!pseudo) {
+      pseudo = `ref-${sha256Hex(`${salt}:${value}`).slice(0, 16)}`;
+      cache.set(value, pseudo);
+    }
+    return pseudo;
+  };
+}
+
+async function listOperationIds(dir) {
+  let names;
+  try {
+    names = await fs.readdir(path.join(dir, JOURNAL_DIR));
+  } catch {
+    return [];
+  }
+  return names
+    .filter((n) => n.endsWith('.jsonl'))
+    .map((n) => n.slice(0, -'.jsonl'.length))
+    .sort();
+}
+
+/**
+ * Artifact digest refs only — `path` (a raw filesystem path) is never
+ * emitted. Object refs contribute `{name: stream, fingerprint: sha256}`;
+ * opaque string refs contribute a salted digest pseudonym.
+ */
+function artifactDigests(events, pseudo) {
+  const out = [];
+  for (const event of events) {
+    for (const ref of event.artifactRefs ?? []) {
+      if (ref && typeof ref === 'object' && typeof ref.sha256 === 'string') {
+        out.push({
+          name: typeof ref.stream === 'string' ? ref.stream : 'artifact',
+          fingerprint: ref.sha256,
+        });
+      } else if (typeof ref === 'string' && ref.length > 0) {
+        out.push({ name: 'artifact', fingerprint: sha256Hex(`ref:${ref}`).slice(0, 32) });
+      }
+    }
+  }
+  return out.slice(0, 32);
+}
+
+/**
+ * Summarize one operation journal into a support-view payload.
+ * Counts/codes/timings/digests only — no safePayload strings are copied
+ * except state names and typed code fields, which the support gate then
+ * re-verifies (default-deny).
+ */
+function summarize(events, pseudo) {
+  let status = 'unknown';
+  let firstAt = NaN;
+  let lastAt = NaN;
+  const typeCounts = new Map();
+  const errorCodes = new Set();
+  const reasonCodes = new Set();
+
+  for (const event of events) {
+    typeCounts.set(event.eventType, (typeCounts.get(event.eventType) ?? 0) + 1);
+    const t = Date.parse(event.observedAt);
+    if (!Number.isNaN(t)) {
+      if (Number.isNaN(firstAt) || t < firstAt) firstAt = t;
+      if (Number.isNaN(lastAt) || t > lastAt) lastAt = t;
+    }
+    const sp = event.safePayload;
+    if (sp && typeof sp === 'object') {
+      if (event.eventType === 'operation.transition' && typeof sp.to === 'string') {
+        status = sp.to;
+      }
+      if (typeof sp.code === 'string') errorCodes.add(sp.code);
+      if (typeof sp.resolution === 'string') reasonCodes.add(sp.resolution);
+      if (Number.isFinite(sp.exit_code) && sp.exit_code !== 0) {
+        errorCodes.add(`exit_${sp.exit_code}`);
+      }
+    }
+  }
+
+  const payload = {
+    status,
+    count: events.length,
+    duration_ms: Number.isNaN(firstAt) ? 0 : Math.max(0, lastAt - firstAt),
+    type: [...typeCounts.entries()]
+      .sort(([a], [b]) => a.localeCompare(b))
+      .map(([name, count]) => ({ name, count })),
+    error_code: [...errorCodes].sort(),
+    reason_code: [...reasonCodes].sort(),
+    fingerprint: artifactDigests(events, pseudo),
+  };
+  return payload;
+}
+
+/**
+ * Build privacy-sanitized support records for every operation journal
+ * under `dir/events/`.
+ *
+ * @param {string} dir eventStore root
+ * @param {object} [opts]
+ * @param {string} [opts.salt] pseudonym salt (default: random per call)
+ * @param {number} [opts.maxRecords] cap on emitted records (drop-and-count)
+ * @param {number} [opts.maxBytes] cap on total serialized record bytes
+ * @param {(text:string)=>{hasSecret:boolean}} [opts.scanner] gate scanner
+ * @returns {Promise<{version:string, records:object[],
+ *   coverage:{read:number, rejected_records:number,
+ *   rejection_reasons:object, dropped_records:number},
+ *   redaction_version:string}>}
+ */
+export async function buildRuntimeSupportRecords(dir, opts = {}) {
+  const salt = typeof opts.salt === 'string' && opts.salt.length > 0
+    ? opts.salt
+    : crypto.randomBytes(16).toString('hex');
+  const maxRecords = Number.isInteger(opts.maxRecords) && opts.maxRecords >= 0
+    ? opts.maxRecords
+    : DEFAULT_CAPS.maxRecords;
+  const maxBytes = Number.isInteger(opts.maxBytes) && opts.maxBytes > 0
+    ? opts.maxBytes
+    : DEFAULT_CAPS.maxBytes;
+  const gate = typeof opts.scanner === 'function'
+    ? (rec) => sanitizeForSupport(rec, { scanner: opts.scanner })
+    : (rec) => sanitizeForSupport(rec);
+  const pseudo = pseudonymizer(salt);
+
+  const coverage = {
+    read: 0,
+    rejected_records: 0,
+    rejection_reasons: {},
+    dropped_records: 0,
+  };
+  const reject = (reason) => {
+    coverage.rejected_records += 1;
+    coverage.rejection_reasons[reason] =
+      (coverage.rejection_reasons[reason] ?? 0) + 1;
+  };
+
+  const records = [];
+  let bytes = 0;
+  let capped = false;
+
+  for (const operationId of await listOperationIds(dir)) {
+    coverage.read += 1;
+
+    const events = [];
+    try {
+      for await (const event of readJournal(dir, operationId)) {
+        events.push(event);
+      }
+    } catch (err) {
+      // typed store errors only; never propagate raw
+      reject(typeof err?.code === 'string' ? err.code : 'journal_unreadable');
+      continue;
+    }
+
+    const record = {
+      record_type: 'operation_summary',
+      semantic_name: 'agentRuntime.operation.summary',
+      schema_version: '1',
+      record_id: pseudo(operationId),
+      sequence: events.length > 0 ? events[events.length - 1].seq : 0,
+      privacy_class: 'internal',
+      payload: summarize(events, pseudo),
+      origin: 'agentRuntime.eventStore',
+      redaction_version: REDACTION_VERSION,
+    };
+
+    const verdict = gate(record);
+    if (!verdict.ok) {
+      reject(verdict.reason);
+      continue;
+    }
+
+    const size = Buffer.byteLength(JSON.stringify(verdict.record), 'utf8');
+    if (capped || records.length >= maxRecords || bytes + size > maxBytes) {
+      capped = true;
+      coverage.dropped_records += 1;
+      continue;
+    }
+    records.push(verdict.record);
+    bytes += size;
+  }
+
+  return {
+    version: SUPPORT_RECORDS_VERSION,
+    records,
+    coverage,
+    redaction_version: REDACTION_VERSION,
+  };
+}