@ngockhoale/ukit 3.3.3 → 3.4.0

package/src/core/observability/emit/lifecycle.js

@@ -60,8 +60,8 @@
  * No LLM, no daemon, no sync fs on the hot path. The module never throws.
  */
 
+import crypto from 'node:crypto';
 import path from 'node:path';
-
 import { createRecorder } from './recorder.js';
 import { registerCrashCapture } from './crash.js';
 import { resolveStage } from './config.js';
@@ -101,6 +101,73 @@ function isNonEmptyString(value) {
   return typeof value === 'string' && value.length > 0;
 }
 
+// --- FR-003 (BL-005): shared semantic names + record builder --------------
+//
+// The parity-locked pair src/core/observability/emit/sessionBoot.js ↔
+// template_project/.claude/ukit/runtime/observability-emit.mjs emits the
+// same record shape the recorder's startSpan() produces (recorder.js) —
+// these names are the single canonical vocabulary for both emit paths.
+
+export const SEMANTIC_EXECUTION_STARTED = 'execution.started';
+export const SEMANTIC_EXECUTION_COMPLETED = 'execution.completed';
+export const SEMANTIC_EXECUTION_FAILED = 'execution.failed';
+export const SEMANTIC_EXECUTION_BLOCKED = 'execution.blocked';
+export const SEMANTIC_OUTCOME_OBSERVED = 'outcome.observed';
+
+/**
+ * Deterministic session-boot join key. `payload.request_key` on the boot
+ * record is derived (not random) so terminal emitters can recompute the
+ * same key for the session: sha256('<projectRoot>\n<sessionId>'), 24 hex.
+ */
+export function sessionBootRequestKey({ projectRoot, sessionId } = {}) {
+  const digest = crypto
+    .createHash('sha256')
+    .update(`${String(projectRoot ?? '')}\n${String(sessionId ?? '')}`)
+    .digest('hex')
+    .slice(0, 24);
+  return `session-boot-${digest}`;
+}
+
+/**
+ * The ONE session-boot `execution.started` record shape. Same field
+ * ordering contract as the recorder's own span-root emit: fresh
+ * trace/span/execution ids, caller's session/engine stamped in both the
+ * envelope and the payload, wall-clock `ts` echoed inside payload for the
+ * join tooling that never parses envelope timestamps.
+ * Returns null for a missing projectRoot (callers must never fabricate
+ * a storage root) — the emit layer maps that to a typed drop.
+ */
+export function buildSessionBootRecord({
+  projectRoot,
+  engine,
+  sessionId,
+  nowMs = Date.now(),
+} = {}) {
+  if (!isNonEmptyString(projectRoot)) return null;
+  const trace_id = `trace-${crypto.randomUUID()}`;
+  return {
+    semantic_name: SEMANTIC_EXECUTION_STARTED,
+    trace_id,
+    span_id: `span-${crypto.randomUUID()}`,
+    parent_span_id: null,
+    execution_id: `exec-${crypto.randomUUID()}`,
+    session_id: isNonEmptyString(sessionId) ? sessionId : undefined,
+    project_ref: `proj-${crypto
+      .createHash('sha256')
+      .update(String(path.resolve(projectRoot)))
+      .digest('hex')
+      .slice(0, 12)}`,
+    payload: {
+      operation: 'session-boot',
+      duration_ms: null,
+      engine: isNonEmptyString(engine) ? engine : 'unknown',
+      sessionId: isNonEmptyString(sessionId) ? sessionId : 'unknown',
+      ts: new Date(nowMs).toISOString(),
+      request_key: sessionBootRequestKey({ projectRoot, sessionId }),
+    },
+  };
+}
+
 // --- disabled recorder (SPEC §8: missing root → emit→dropped/recorder-disabled)
 
 function makeDisabledRecorder() {

package/src/core/observability/emit/sessionBoot.js

@@ -0,0 +1,393 @@
+/**
+ * sessionBoot.js (TASK-003, SPEC §5 FR-003, §6, §14) — the live session-boot
+ * emit surface: the production caller the recorder layer never had.
+ *
+ *   emitSessionBoot({ projectRoot, engine, sessionId, config?, deps? })
+ *     → { status: 'written', written, elapsed_ms }
+ *       | { status: 'dropped'|'partial'|'error', reason }
+ *   emitOutcomeObserved({ projectRoot, record, config?, deps? }) → same result
+ *   resolveEmitStage(config) → 'off' | 'shadow' | 'canary' | 'default'
+ *   loadEmitConfig(projectRoot) → merged emit config (async, never throws)
+ *
+ * A hook child process lives for milliseconds — `startLifecycle`'s interval
+ * timers and beforeExit flush cannot survive it (SPEC §14 chose the bounded
+ * one-shot emit). One call = one recorder = one record = one flush, each
+ * step racing a deadline, so the emit can never delay session start: it is
+ * fire-and-forget, bounded, and never throws.
+ *
+ * Stage contract: this surface emits by default — the shipped
+ * `.ukit/storage/config.json` sets observability.stage='default' and the
+ * runtime default config agrees — so 'off' is an explicit kill switch the
+ * owner sets; a missing config file or missing stage emits (the same
+ * "default" the rest of the runtime resolves to). A caller-supplied
+ * `config` object is used as the complete merged config, so callers that
+ * want the live stage pass the runtime config themselves; absent config
+ * resolves the 'default' effective stage.
+ *
+ * The parity-locked mirror
+ * template_project/.claude/ukit/runtime/observability-emit.mjs cannot
+ * import package source — it replicates this pipeline (envelope fill →
+ * validate → sanitize → bounded append) with identical semantics. Keep the
+ * record shape changes coordinated with `tests/consistency/
+ * sessionBootEmitParity.test.js`.
+ */
+
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { createRecorder } from './recorder.js';
+import {
+  buildSessionBootRecord,
+  segmentsRoot,
+  sessionBootRequestKey,
+  SEMANTIC_OUTCOME_OBSERVED,
+} from './lifecycle.js';
+import { resolveStage } from './config.js';
+
+/** Hard upper bound for the whole one-shot emit (ms). */
+export const SESSION_BOOT_EMIT_DEADLINE_MS = 250;
+
+const VALID_STAGES = new Set(['off', 'shadow', 'canary', 'default']);
+
+function isPlainObject(value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function isNonEmptyString(value) {
+  return typeof value === 'string' && value.length > 0;
+}
+
+/**
+ * The emit surface's effective stage: the explicit literal 'off' is the
+ * kill switch; any other explicit stage value resolves through
+ * resolveStage verbatim; absent/malformed resolves 'default' (the shipped
+ * + code default), never the recorder's fail-safe 'off'.
+ */
+export function resolveEmitStage(config = null) {
+  const node = isPlainObject(config) ? config.observability : undefined;
+  if (isPlainObject(node) && node.stage === 'off') return 'off';
+  const stage = resolveStage(config);
+  return stage === 'off' ? 'default' : stage;
+}
+
+/**
+ * `.ukit/storage/config.json` on disk → emit config. A missing/corrupt
+ * file means a pre-install or bare project: effective stage 'default'
+ * (the shipped value), mirroring the runtime's own code default.
+ */
+export async function loadEmitConfig(projectRoot) {
+  if (!isNonEmptyString(projectRoot)) return null;
+  try {
+    const raw = await fs.promises.readFile(
+      path.join(projectRoot, '.ukit', 'storage', 'config.json'),
+      'utf8',
+    );
+    const parsed = JSON.parse(raw);
+    return isPlainObject(parsed) ? parsed : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Race a promise against a deadline; the loser keeps running detached
+ * (bounded telemetry accepts a late write over blocking the caller).
+ * Returns { value } | { timedOut: true }. The timer is unref'd so it can
+ * never pin a dying hook process.
+ */
+async function bounded(promise, deadlineMs) {
+  if (!Number.isFinite(deadlineMs) || deadlineMs <= 0) {
+    return { timedOut: true };
+  }
+  let timer;
+  try {
+    const result = await Promise.race([
+      Promise.resolve(promise).then((value) => ({ value })),
+      new Promise((resolve) => {
+        timer = setTimeout(() => resolve({ timedOut: true }), deadlineMs);
+        if (timer.unref) timer.unref();
+      }),
+    ]);
+    return result;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+function toEmitResult(result, elapsed_ms) {
+  const out = { ...result };
+  out.elapsed_ms = Math.max(0, elapsed_ms);
+  return out;
+}
+
+/**
+ * The shared one-shot emit pipeline. `record` is a semantic-record
+ * candidate; the recorder fills envelope fields, validates, sanitizes,
+ * applies the sampling decider, and flush() drains through the segment
+ * writer — the twin never reimplements any of those stages.
+ */
+async function emitOne({ projectRoot, config, record, deps, deadlineMs }) {
+  const started = Date.now();
+  const fail = (reason) => toEmitResult({ status: 'dropped', reason }, Date.now() - started);
+
+  if (!isNonEmptyString(projectRoot)) return fail('missing-project-root');
+  if (!isPlainObject(record)) return fail('invalid-record');
+  if (resolveEmitStage(config) === 'off') return fail('stage-off');
+
+  const injected = isPlainObject(deps) ? deps : {};
+  const clock = {
+    nowMs: typeof injected.nowMs === 'function' ? injected.nowMs : () => injected.nowMs ?? Date.now(),
+    nowNs:
+      typeof injected.nowNs === 'function'
+        ? injected.nowNs
+        : () => injected.nowNs ?? process.hrtime.bigint(),
+    iso:
+      typeof injected.iso === 'function'
+        ? injected.iso
+        : () => injected.iso ?? new Date().toISOString(),
+  };
+  const writer =
+    typeof injected.writer === 'function' ? injected.writer : undefined;
+  const samplingSeed = Number.isFinite(injected.samplingSeed) ? injected.samplingSeed : undefined;
+
+  // resolveEmitStage owns the 'default'-absent asymmetry: the effective
+  // stage written back here is 'off' only when the owner set the literal
+  // 'off' (that case already returned above); an absent or malformed stage
+  // resolves 'default' — the shipped + code default — while a supplied
+  // 'shadow'/'canary' passes through verbatim. Other observability keys
+  // (sampling, seams) ride along untouched for the recorder's own gates.
+  const effectiveConfig = isPlainObject(config)
+    ? {
+        ...config,
+        observability: {
+          ...(isPlainObject(config.observability) ? config.observability : {}),
+          stage: resolveEmitStage(config),
+        },
+      }
+    : { observability: { stage: 'default' } };
+
+  const budget = Number.isFinite(deadlineMs) && deadlineMs > 0
+    ? Math.min(deadlineMs, SESSION_BOOT_EMIT_DEADLINE_MS)
+    : SESSION_BOOT_EMIT_DEADLINE_MS;
+
+  // appendRecord's resolveRoot requires the segments root's PARENT to
+  // already exist (realpath containment), so a fresh project needs the
+  // .ukit/storage/observability tree created here — bounded by the same
+  // deadline so a stalled mount can never pin the hook.
+  const targetRoot = segmentsRoot(path.resolve(projectRoot));
+  const seeded = await bounded(
+    fs.promises.mkdir(targetRoot, { recursive: true }).catch((err) => err),
+    budget,
+  );
+  if (seeded.timedOut) {
+    return toEmitResult({ status: 'partial', reason: 'deadline' }, Date.now() - started);
+  }
+  try {
+    if (seeded.value instanceof Error) throw seeded.value;
+  } catch {
+    // mkdir raced with another process or failed — fall through and let
+    // the write path report the typed io reason.
+  }
+  try {
+    const recorder = createRecorder({
+      root: segmentsRoot(path.resolve(projectRoot)),
+      config: effectiveConfig,
+      clock,
+      ...(writer ? { writer } : {}),
+      ...(samplingSeed !== undefined ? { samplingSeed } : {}),
+    });
+    const emitted = recorder.emit(record);
+    if (emitted.status !== 'accepted') {
+      return fail(emitted.reason || 'dropped');
+    }
+    const flushed = await bounded(recorder.flush({ deadlineMs: budget }), budget);
+    if (flushed.timedOut) {
+      return toEmitResult({ status: 'partial', reason: 'deadline' }, Date.now() - started);
+    }
+    const result = flushed.value;
+    if (result.status === 'ok' && result.written > 0) {
+      return toEmitResult(
+        { status: 'written', written: result.written },
+        Date.now() - started,
+      );
+    }
+    return toEmitResult(
+      { status: result.status === 'partial' ? 'partial' : 'dropped', reason: result.reason || 'write-failed' },
+      Date.now() - started,
+    );
+  } catch {
+    return fail('emit-error');
+  }
+}
+
+/**
+ * `emitSessionBoot({ projectRoot, engine, sessionId, config?, deps? })` —
+ * append ONE `execution.started` record to the project's active segment.
+ * `deps` is the test-injection seam: { nowMs?, nowNs?, iso?, writer?,
+ * samplingSeed? } (values or zero-arg functions; the recorder's clock
+ * contract). Never throws, never exceeds the deadline.
+ */
+export async function emitSessionBoot({
+  projectRoot,
+  engine,
+  sessionId,
+  config,
+  deps,
+  deadlineMs,
+} = {}) {
+  const injected = isPlainObject(deps) ? deps : {};
+  const nowMs = typeof injected.nowMs === 'function' ? injected.nowMs() : injected.nowMs;
+  const record = buildSessionBootRecord({
+    projectRoot,
+    engine: isNonEmptyString(engine) ? engine : 'unknown',
+    sessionId,
+    nowMs: Number.isFinite(nowMs) ? nowMs : Date.now(),
+  });
+  return emitOne({ projectRoot, config, record, deps: injected, deadlineMs });
+}
+
+/**
+ * `emitOutcomeObserved({ projectRoot, record, config?, deps? })` — the
+ * generic terminal-record seam TASK-005's emitters consume. `record` is a
+ * full semantic-record candidate; a missing `semantic_name` defaults to
+ * 'outcome.observed' (the seam's own event).
+ */
+export async function emitOutcomeObserved({
+  projectRoot,
+  record,
+  config,
+  deps,
+  deadlineMs,
+} = {}) {
+  const candidate = isPlainObject(record) ? { ...record } : record;
+  if (isPlainObject(candidate) && candidate.semantic_name === undefined) {
+    candidate.semantic_name = SEMANTIC_OUTCOME_OBSERVED;
+  }
+  return emitOne({
+    projectRoot,
+    config,
+    record: candidate,
+    deps: isPlainObject(deps) ? deps : {},
+    deadlineMs,
+  });
+}
+
+
+// --- TASK-005 (FR-005, BL-007): terminal OutcomeRecord construction ---------
+// The three terminal emitters (record-execution.mjs receipts,
+// stop-coordinator.mjs verdicts, execution-ledger.mjs finalizers) share ONE
+// record builder so every `outcome.observed` record carries the ARCH
+// §Data Contracts OutcomeRecord fields plus the TASK-003 session-boot
+// `request_key` join id — the missing 1-of-3934 emitter surface (TM §2).
+
+export const OUTCOME_VERDICTS = Object.freeze([
+  'VERIFIED', 'FAILED', 'BLOCKED', 'INCONCLUSIVE', 'INTERRUPTED',
+]);
+
+const OUTCOME_ENFORCEMENTS = new Set(['enforced', 'advisory']);
+
+// FR-005/FR-006: `advisory` is the Codex marker — the engine stamp may arrive
+// as an explicit `engine` argument or as `env.UKIT_HARNESS` (the stamp the
+// .sh wrappers and the omp bridge export). Anything else is `enforced`.
+export function deriveEnforcement({ engine = null, env = null } = {}) {
+  const fromArg = isNonEmptyString(engine) ? engine : null;
+  const fromEnv = isPlainObject(env) && isNonEmptyString(env.UKIT_HARNESS) ? env.UKIT_HARNESS : null;
+  return (fromArg === 'codex' || fromEnv === 'codex') ? 'advisory' : 'enforced';
+}
+
+// The run id is the ledger's session identity: `run-<safe session id>` when a
+// session is known, `run-transcript-<sha256[:12]>` for transcript-only
+// payloads (never the raw path — it would be path-redacted anyway), else null.
+export function outcomeRunId(payload = {}) {
+  const sessionId = isNonEmptyString(payload.session_id)
+    ? payload.session_id
+    : (isNonEmptyString(payload.sessionId) ? payload.sessionId : null);
+  if (sessionId) {
+    const safe = sessionId.replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 96);
+    return `run-${safe}`;
+  }
+  const transcript = isNonEmptyString(payload.transcript_path)
+    ? payload.transcript_path
+    : (isNonEmptyString(payload.transcriptPath) ? payload.transcriptPath : null);
+  if (transcript) {
+    const digest = crypto.createHash('sha256').update(String(transcript)).digest('hex').slice(0, 12);
+    return `run-transcript-${digest}`;
+  }
+  return null;
+}
+
+function outcomeStringList(value, cap) {
+  if (!Array.isArray(value)) return [];
+  return value
+    .filter((item) => typeof item === 'string' && item.length > 0)
+    .map((item) => item.slice(0, 160))
+    .slice(0, cap);
+}
+
+function outcomeCost(value) {
+  if (!isPlainObject(value)) return { latencyMs: null, rounds: null };
+  const cost = value;
+  return {
+    latencyMs: Number.isFinite(cost.latencyMs) && cost.latencyMs >= 0 ? cost.latencyMs : null,
+    rounds: Number.isFinite(cost.rounds) && cost.rounds >= 0 ? cost.rounds : null,
+  };
+}
+
+/**
+ * buildOutcomeObservedRecord({ outcome, projectRoot, payload, sessionId,
+ * engine, env }) → semantic-record candidate for `outcome.observed`.
+ *
+ * `outcome` is the ARCH §Data Contracts OutcomeRecord subset the emitter knows:
+ *   { outcomeId?, runId?, routeId?, routeFingerprint?, ledgerKey?, verdict,
+ *     evidenceIds?, cost?, reviewActions?, enforcement?, engine?, source?, ts? }
+ * Every field defaults sanely — a terminal event without runId/route data
+ * still emits with nulls (task case 4) and never blocks the caller's path.
+ * `payload` is the hook payload: it supplies session identity for runId
+ * derivation, the session-boot request_key join, and the envelope session_id.
+ * `enforcement` derives per FR-006 unless the emitter overrode it.
+ */
+export function buildOutcomeObservedRecord({
+  outcome = null,
+  projectRoot = null,
+  payload = {},
+  sessionId = null,
+  engine = null,
+  env = null,
+} = {}) {
+  const safe = isPlainObject(outcome) ? outcome : {};
+  const resolvedSessionId = isNonEmptyString(sessionId)
+    ? sessionId
+    : (isNonEmptyString(payload?.session_id)
+      ? payload.session_id
+      : (isNonEmptyString(payload?.sessionId) ? payload.sessionId : null));
+  const engineName = isNonEmptyString(engine)
+    ? engine
+    : (isNonEmptyString(safe.engine) ? safe.engine : null);
+  const verdict = OUTCOME_VERDICTS.includes(safe.verdict) ? safe.verdict : 'INCONCLUSIVE';
+  const recordPayload = {
+    outcomeId: isNonEmptyString(safe.outcomeId) ? safe.outcomeId : `oc-${crypto.randomUUID()}`,
+    runId: safe.runId !== undefined ? safe.runId : outcomeRunId(payload),
+    routeId: safe.routeId !== undefined ? safe.routeId : null,
+    routeFingerprint: safe.routeFingerprint !== undefined ? safe.routeFingerprint : null,
+    ledgerKey: safe.ledgerKey !== undefined ? safe.ledgerKey : null,
+    verdict,
+    evidenceIds: outcomeStringList(safe.evidenceIds, 32),
+    cost: outcomeCost(safe.cost),
+    reviewActions: outcomeStringList(safe.reviewActions, 32),
+    enforcement: OUTCOME_ENFORCEMENTS.has(safe.enforcement)
+      ? safe.enforcement
+      : deriveEnforcement({ engine: engineName, env }),
+    engine: engineName,
+    source: isNonEmptyString(safe.source) ? safe.source.slice(0, 96) : null,
+    request_key: sessionBootRequestKey({ projectRoot, sessionId: resolvedSessionId }),
+    ts: isNonEmptyString(safe.ts) ? safe.ts : new Date().toISOString(),
+  };
+  return {
+    semantic_name: SEMANTIC_OUTCOME_OBSERVED,
+    session_id: resolvedSessionId ?? undefined,
+    payload: recordPayload,
+  };
+}
+
+export { sessionBootRequestKey };

package/src/core/observability/privacy/allowlist.js

@@ -11,7 +11,7 @@
  * redaction rules change so downstream readers can detect stale records.
  */
 
-export const REDACTION_VERSION = 'df-redact-5';
+export const REDACTION_VERSION = 'df-redact-6';
 
 /**
  * Envelope fields permitted on a persisted record (DF-FR01) plus the
@@ -179,6 +179,15 @@ export const ALLOWED_SUPPORT_PAYLOAD_FIELDS = new Set([
   'digest',
   'policy_version',
   'rate',
+  // TASK-C89-002 (SPEC §3 FR-01): subagent.usage provenance payload — the
+  // fingerprint-deduped component rows, measured byte counts, typed token
+  // estimates, the observed parent run ref and the telemetry-completeness
+  // marker. Metadata only; component text itself is never emitted.
+  'components',
+  'bytes',
+  'tokens',
+  'parent_run_ref',
+  'telemetry_complete',
 ]);
 
 // --- size caps (prototype defaults per SPEC §14; freeze after measurement) ---

package/src/core/observability/schema/registry.js

@@ -15,6 +15,7 @@ import { PRIVACY_CLASSES } from './constants.js';
 
 const V1 = '1';
 const V1_1 = '1.1';
+const V1_2 = '1.2';
 
 export const SEMANTIC_REGISTRY = Object.freeze({
   'execution.started': {
@@ -210,6 +211,15 @@ export const SEMANTIC_REGISTRY = Object.freeze({
     introduced_version: V1_1,
     deprecated: null,
   },
+  // --- schema v1.2 additions (TASK-C89-002, SPEC §3 FR-01, append-only) ---
+  'subagent.usage': {
+    definition: 'One child/subagent run emitted typed usage provenance. Payload carries fingerprint-deduped prompt components with measured bytes and ESTIMATED tokens, provider usage as a typed {value,source} resource, and the observed parent run ref — unknown stays UNKNOWN/null, never 0, never prompt text.',
+    unit: 'event',
+    privacy_class: 'internal',
+    retention_hint: 'retain',
+    introduced_version: V1_2,
+    deprecated: null,
+  },
 });
 
 // Error/wait/reason vocabulary: codes + definitions, no free-text grouping.