@ngockhoale/ukit 3.0.12 → 3.1.0

package/src/core/observability/adapters/ingest.js

@@ -0,0 +1,576 @@
+// ingest.js (TASK-006, SPEC §5 DF2-FR06, §8) — the collect lane that turns
+// already-stored `.ukit/storage/` telemetry into canonical SemanticRecords.
+// Sources (all read-only, never throws):
+//
+//   hook_latency  .ukit/storage/cache/hook-latency/*.jsonl
+//                   rows from hook-telemetry.mjs → adaptHookTelemetryEvent
+//   routes        .ukit/storage/cache/route-audit.json joined to
+//                   cache/exec-ledger/*.json on requestKey
+//                   (same join discipline as collectRouteOutcomes — that
+//                   owner exports aggregates only, so the row join lives
+//                   here) → adaptRouteOutcomes
+//   decisions     .ukit/storage/memory/rollout-receipts.jsonl
+//                   bounded receipts from memoryFlags.appendRolloutReceipt
+//                   → mapped onto the decisionAdapter shape
+//   context       .ukit/storage/cache/retriever-lanes.jsonl
+//                   each event's mergedPaths become injected context items
+//                   → adaptContextItems
+//
+//   ingestStoredTelemetry({ projectRoot, recorder, deadlineMs?, sources?, now? })
+//     → Promise<{ ok, status:'complete'|'partial'|'degraded',
+//                  coverage: { <source>: { read, emitted, rejected, skipped, reason? } },
+//                  last_cursors }>
+//
+// Counters: `read` rows consumed by the adapter lane; `emitted` records
+// accepted (or queued-at-full-queue) by recorder.emit; `rejected` covers
+// malformed lines, null adapter output, and recorder drops — one logical
+// row never lands twice. `skipped` counts rows left unprocessed when the
+// deadline fired mid-source.
+//
+// Cursors live at `.ukit/storage/observability/ingest-cursors.json`
+// (tmp+rename atomic): per-file byte offsets for append-only JSONL, a
+// bounded seen-set for route join rows. Cursor loss or a cursor write
+// failure re-reads rows it already emitted — bounded telemetry accepts a
+// rare duplicate over blocking a collect. `feedback-manual.jsonl` is
+// deliberately NOT read: TASK-013 emits `outcome.observed` at write-time;
+// ingesting that file would double-count the same event.
+
+import crypto from 'node:crypto';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import { adaptHookTelemetryEvent } from './hookTelemetryAdapter.js';
+import { adaptRouteOutcomes } from './routeAdapter.js';
+import { adaptDecisionReceipts } from './decisionAdapter.js';
+import { adaptContextItems } from './contextAdapter.js';
+import { createAdapterContext, isPlainObject } from './common.js';
+import { listLedgerFiles } from '../../../diagnostics/ledgerFiles.js';
+
+const STORAGE_REL = path.join('.ukit', 'storage');
+const CACHE_REL = path.join(STORAGE_REL, 'cache');
+const HOOK_LATENCY_DIR_REL = path.join(CACHE_REL, 'hook-latency');
+const ROUTE_AUDIT_REL = path.join(CACHE_REL, 'route-audit.json');
+const EXEC_LEDGER_DIR_REL = path.join(CACHE_REL, 'exec-ledger');
+const ROLLOUT_RECEIPTS_REL = path.join(STORAGE_REL, 'memory', 'rollout-receipts.jsonl');
+const RETRIEVER_LANES_REL = path.join(CACHE_REL, 'retriever-lanes.jsonl');
+const CURSOR_REL = path.join(STORAGE_REL, 'observability', 'ingest-cursors.json');
+
+// Bounded cursor state: hook-latency keeps ≤200 files by owner sweep and
+// the route seen-set only needs to outlive the audit window plus the
+// ledger dir cap — these caps cover both with headroom.
+const MAX_FILE_CURSORS = 512;
+const MAX_ROUTE_SEEN = 4096;
+const MAX_HOOK_FILES = 256;
+const DEFAULT_LIMIT_LEDGERS = 500;
+
+const LF = 0x0a;
+const UTF8_BOM = '\uFEFF';
+
+function zeroCoverage() {
+  return { read: 0, emitted: 0, rejected: 0, skipped: 0 };
+}
+
+function emptyCursors() {
+  return { version: 1, sources: {} };
+}
+
+function boundedMapSet(map, key, value) {
+  map.delete(key); // refresh insertion order so pruning is oldest-first
+  map.set(key, value);
+  while (map.size > MAX_FILE_CURSORS) map.delete(map.keys().next().value);
+}
+
+async function readJsonSafe(filePath) {
+  try {
+    return { doc: JSON.parse(await fs.readFile(filePath, 'utf8')), error: null };
+  } catch (error) {
+    return { doc: null, error: error?.code === 'ENOENT' ? null : 'malformed_doc' };
+  }
+}
+
+async function loadCursors(projectRoot) {
+  const file = path.join(projectRoot, CURSOR_REL);
+  try {
+    const doc = JSON.parse(await fs.readFile(file, 'utf8'));
+    if (!isPlainObject(doc) || !isPlainObject(doc.sources)) {
+      return { cursors: emptyCursors(), reset: true };
+    }
+    return { cursors: { version: 1, sources: doc.sources }, reset: false };
+  } catch (error) {
+    return { cursors: emptyCursors(), reset: error?.code !== 'ENOENT' };
+  }
+}
+
+async function saveCursors(projectRoot, cursors) {
+  const file = path.join(projectRoot, CURSOR_REL);
+  const tmp = `${file}.tmp-${process.pid}-${Math.random().toString(16).slice(2)}`;
+  await fs.mkdir(path.dirname(file), { recursive: true });
+  await fs.writeFile(tmp, `${JSON.stringify(cursors)}\n`, 'utf8');
+  try {
+    await fs.rename(tmp, file);
+  } catch (error) {
+    await fs.rm(tmp, { force: true }).catch(() => {});
+    throw error;
+  }
+}
+
+// Split a buffer tail into lines with byte-accurate next-offsets. A
+// trailing partial line (torn write) is returned too — like the segment
+// reader it is a permanent reject, so the cursor may advance past it
+// rather than blocking forever on a dead byte tail.
+function splitLines(buf, start) {
+  const lines = [];
+  let pos = start;
+  while (pos < buf.length) {
+    const nl = buf.indexOf(LF, pos);
+    const end = nl === -1 ? buf.length : nl;
+    lines.push({ offset: pos, next: nl === -1 ? buf.length : nl + 1, raw: buf.subarray(pos, end) });
+    pos = lines[lines.length - 1].next;
+  }
+  return lines;
+}
+
+/**
+ * Consume an append-only JSONL file from its stored byte cursor.
+ * `rowFn(line)` adapts+emits one parsed row and updates `cov` itself;
+ * the consumer owns `read`, malformed-line `rejected`, deadline `skipped`,
+ * and cursor advancement. Rotation/truncation (file smaller than the
+ * stored offset) resets the cursor to 0 — the kept lines re-emit as
+ * bounded duplicates rather than silently vanishing.
+ */
+async function consumeJsonlFile(filePath, cursorEntry, cov, rowFn, expired) {
+  let stat;
+  try {
+    stat = await fs.stat(filePath);
+  } catch (error) {
+    return error?.code === 'ENOENT' ? {} : { error: 'io_stat' };
+  }
+  let offset = cursorEntry?.offset ?? 0;
+  if (!Number.isInteger(offset) || offset < 0 || offset > stat.size) offset = 0;
+  if (offset === stat.size) return {}; // no new bytes
+
+  let buf;
+  try {
+    buf = await fs.readFile(filePath);
+  } catch {
+    return { error: 'io_read' };
+  }
+
+  const lines = splitLines(buf, offset);
+  let consumed = offset;
+  let partial = false;
+  for (let i = 0; i < lines.length; i += 1) {
+    const line = lines[i];
+    if (expired()) {
+      cov.skipped += lines.length - i;
+      partial = true;
+      break;
+    }
+    cov.read += 1;
+    let text = line.raw.toString('utf8').trim();
+    if (line.offset === 0 && text.startsWith(UTF8_BOM)) text = text.slice(1);
+    if (text.length > 0) {
+      let parsed = null;
+      try {
+        parsed = JSON.parse(text);
+      } catch {
+        /* malformed line → permanent reject below */
+      }
+      if (parsed === null) cov.rejected += 1;
+      else if (rowFn(parsed) === 'skipped') {
+        cov.skipped += lines.length - i;
+        partial = true;
+        break;
+      }
+    }
+    // Blank and malformed lines are dead space — the cursor advances past
+    // them; only a rowFn('skipped') stop leaves the cursor behind.
+    consumed = line.next;
+  }
+  return { partial, entry: { offset: consumed, size: stat.size } };
+}
+
+// --- source: routes -----------------------------------------------------------
+
+// Join route-audit entries to exec-ledger docs on requestKey — identical
+// discipline to collectRouteOutcomes (last audit entry per key wins,
+// ledgers ordered mtime-desc, non-ledger filenames excluded), but kept at
+// row granularity: adaptRouteOutcomes consumes {audit, ledger} pairs and
+// the owner's public surface returns aggregates only.
+
+// Deterministic serialization for the dedup fingerprint: object keys sort
+// so a same-content write in a different key order cannot masquerade as a
+// mutation. Values are ledger doc fields only — JSON.parse output, never
+// cyclic.
+function stableValue(value) {
+  if (Array.isArray(value)) return value.map(stableValue);
+  if (isPlainObject(value)) {
+    const out = {};
+    for (const key of Object.keys(value).sort()) out[key] = stableValue(value[key]);
+    return out;
+  }
+  return value;
+}
+
+// Ledger docs are mutated in place (execution-ledger.mjs bumps updatedAt on
+// every mutation and appends receipts). The dedup key must reflect the doc
+// version, not just its identity: updatedAt is the fast signal, and a
+// digest of the stabilized doc catches mutations that reuse or lack a
+// distinct timestamp (same-millisecond writes, hand-edited docs). An absent
+// ledger fingerprints as '0'.
+function ledgerFingerprint(doc) {
+  if (!isPlainObject(doc)) return '0';
+  const digest = crypto
+    .createHash('sha1')
+    .update(JSON.stringify(stableValue(doc)))
+    .digest('hex')
+    .slice(0, 12);
+  return `${Number.isFinite(doc.updatedAt) ? doc.updatedAt : 0}:${digest}`;
+}
+
+async function collectRouteRows(projectRoot, { limitLedgers, errors }) {
+  const { doc: auditDoc, error: auditError } = await readJsonSafe(
+    path.join(projectRoot, ROUTE_AUDIT_REL),
+  );
+  if (auditError) errors.push('malformed_audit');
+  const auditEntries = Array.isArray(auditDoc?.entries) ? auditDoc.entries : [];
+
+  const ledgerDir = path.join(projectRoot, EXEC_LEDGER_DIR_REL);
+  const names = await listLedgerFiles(ledgerDir, limitLedgers);
+  const ledgers = [];
+  for (const name of names) {
+    const { doc, error } = await readJsonSafe(path.join(ledgerDir, name));
+    if (error) {
+      errors.push('malformed_ledger');
+      continue;
+    }
+    if (isPlainObject(doc)) ledgers.push({ name, doc });
+  }
+
+  const auditByKey = new Map();
+  for (const entry of auditEntries) {
+    if (isPlainObject(entry) && typeof entry.requestKey === 'string' && entry.requestKey) {
+      auditByKey.set(entry.requestKey, entry);
+    }
+  }
+  const ledgerByKey = new Map();
+  for (const { name, doc } of ledgers) {
+    if (typeof doc.requestKey === 'string' && doc.requestKey && !ledgerByKey.has(doc.requestKey)) {
+      ledgerByKey.set(doc.requestKey, { name, doc });
+    }
+  }
+
+  const rows = [];
+  const matched = new Set();
+  for (const audit of auditByKey.values()) {
+    const hit = ledgerByKey.get(audit.requestKey);
+    if (hit) matched.add(hit.name);
+    // The dedup key embeds the ledger fingerprint (updatedAt + digest of
+    // the doc), so a ledger arriving after its audit — or mutated in place
+    // (execution-ledger.mjs rewrites the same file as the run progresses) —
+    // re-emits as the newer row instead of deduping against the stale key.
+    rows.push({
+      key: `a|${audit.ts ?? ''}|${audit.requestKey}|${hit ? ledgerFingerprint(hit.doc) : 0}`,
+      row: { audit, ledger: hit?.doc ?? {} },
+    });
+  }
+  for (const { name, doc } of ledgers) {
+    if (matched.has(name)) continue;
+    if (typeof doc.requestKey !== 'string' || !doc.requestKey) continue;
+    rows.push({
+      key: `l|${name}|${doc.requestKey}|${ledgerFingerprint(doc)}`,
+      row: { audit: {}, ledger: doc },
+    });
+  }
+  return rows;
+}
+
+function decisionReceiptShape(line) {
+  // rollout-receipts.jsonl rows carry {ts, plane, stage, outcome, code,
+  // latencyBand} (memoryFlags.RECEIPT_FIELDS). Map onto the adapter's
+  // provenance shape: plane → experiment, outcome → outcomeClass,
+  // code → fallbackCode, latencyBand → latencyClass.
+  if (!isPlainObject(line)) return null;
+  return {
+    ts: line.ts,
+    stage: line.stage,
+    experiment: line.plane,
+    outcomeClass: line.outcome,
+    fallbackCode: line.code,
+    latencyClass: line.latencyBand,
+  };
+}
+
+function contextItemShapes(line) {
+  // A retriever-lane event's mergedPaths are the items that entered the
+  // episode context. `referenced` stays false — nothing on disk observes a
+  // later reference, and the adapter marks the record a heuristic signal
+  // (evaluator_caveat), never a verdict.
+  if (!isPlainObject(line) || !Array.isArray(line.mergedPaths)) return [];
+  const items = [];
+  for (const ref of line.mergedPaths.slice(0, 32)) {
+    if (typeof ref !== 'string' || ref.length === 0) continue;
+    items.push({
+      kind: 'retrieval',
+      ref,
+      ts: line.ts,
+      selected: true,
+      injected: true,
+      referenced: false,
+    });
+  }
+  return items;
+}
+
+/**
+ * Emit one adapted record through the recorder and count it. The recorder
+ * contract: 'accepted' queues the record; 'queue-full' still queues the
+ * incoming record (the OLDEST queued one is evicted — the new row lands);
+ * every other status is a drop this lane counts as rejected, keeping the
+ * recorder's typed reason (e.g. 'stage-off', 'sampled-out') on coverage.
+ */
+function emitCounted(recorder, cov, record) {
+  let res = null;
+  try {
+    res = recorder.emit(record);
+  } catch {
+    res = { status: 'dropped', reason: 'emit_threw' };
+  }
+  if (res && (res.status === 'accepted' || res.status === 'queue-full')) {
+    cov.emitted += 1;
+    return true;
+  }
+  cov.rejected += 1;
+  cov.last_drop_reason = res?.reason ?? 'emit_rejected';
+  return false;
+}
+
+export async function ingestStoredTelemetry({
+  projectRoot,
+  recorder,
+  deadlineMs,
+  sources,
+  now,
+  limitLedgers,
+} = {}) {
+  if (
+    typeof projectRoot !== 'string' ||
+    projectRoot.length === 0 ||
+    !recorder ||
+    typeof recorder.emit !== 'function'
+  ) {
+    return { ok: false, reason: 'invalid_args' };
+  }
+
+  const nowFn = typeof now === 'function' ? now : () => Date.now();
+  const start = nowFn();
+  const deadline = Number.isFinite(deadlineMs) ? start + deadlineMs : null;
+  const expired = () => deadline !== null && nowFn() >= deadline;
+  const wanted = Array.isArray(sources) && sources.length > 0 ? new Set(sources) : null;
+  const enabled = (name) => wanted === null || wanted.has(name);
+
+  const { cursors, reset: cursorReset } = await loadCursors(projectRoot);
+  const coverage = {};
+  let degraded = false;
+  let partial = false;
+
+
+  const runSource = async (name, fn) => {
+    const cov = zeroCoverage();
+    coverage[name] = cov;
+    if (!enabled(name)) {
+      cov.reason = 'not_requested';
+      return;
+    }
+    if (expired()) {
+      partial = true;
+      cov.reason = 'deadline';
+      return;
+    }
+    try {
+      await fn(cov);
+    } catch {
+      degraded = true;
+      cov.errors = ['source_error'];
+      cov.reason = 'source_error';
+    }
+    if (cov.deadlineHit) {
+      partial = true;
+      cov.reason = 'deadline';
+    }
+    if (cov.errors && !cov.reason) cov.reason = cov.errors[0];
+    if (cov.last_drop_reason && !cov.reason) cov.reason = cov.last_drop_reason;
+  };
+
+  // --- hook_latency ---
+  await runSource('hook_latency', async (cov) => {
+    const dir = path.join(projectRoot, HOOK_LATENCY_DIR_REL);
+    let entries;
+    try {
+      entries = await fs.readdir(dir, { withFileTypes: true });
+    } catch {
+      return; // missing/unreadable dir → zero coverage, not an error
+    }
+    const files = entries
+      .filter((e) => e.isFile() && e.name.endsWith('.jsonl'))
+      .map((e) => e.name)
+      .sort()
+      .slice(0, MAX_HOOK_FILES);
+    const stored = cursors.sources.hook_latency;
+    const fileCursors = new Map(
+      Object.entries(isPlainObject(stored?.files) ? stored.files : {}),
+    );
+    const ctx = createAdapterContext({ writerId: 'ingest-hook-latency' });
+    for (const name of files) {
+      if (expired()) {
+        cov.deadlineHit = true;
+        break;
+      }
+      const outcome = await consumeJsonlFile(
+        path.join(dir, name),
+        fileCursors.get(name),
+        cov,
+        (row) => {
+          const record = adaptHookTelemetryEvent(row, ctx);
+          if (!record) {
+            cov.rejected += 1;
+            return 'consumed';
+          }
+          emitCounted(recorder, cov, record);
+          return 'consumed';
+        },
+        expired,
+      );
+      if (outcome.error) {
+        degraded = true;
+        (cov.errors ??= []).push(outcome.error);
+      }
+      if (outcome.entry) boundedMapSet(fileCursors, name, outcome.entry);
+      if (outcome.partial) {
+        cov.deadlineHit = true;
+        break;
+      }
+    }
+    cursors.sources.hook_latency = { files: Object.fromEntries(fileCursors) };
+  });
+
+  // --- routes ---
+  await runSource('routes', async (cov) => {
+    const joinErrors = [];
+    const rows = await collectRouteRows(projectRoot, {
+      limitLedgers:
+        Number.isInteger(limitLedgers) && limitLedgers > 0 ? limitLedgers : DEFAULT_LIMIT_LEDGERS,
+      errors: joinErrors,
+    });
+    if (joinErrors.length > 0) {
+      degraded = true;
+      cov.errors = joinErrors;
+    }
+    const stored = cursors.sources.routes;
+    const seen = new Set(Array.isArray(stored?.seen) ? stored.seen : []);
+    const ctxOpts = { writerId: 'ingest-routes' };
+    for (let i = 0; i < rows.length; i += 1) {
+      if (expired()) {
+        cov.skipped += rows.length - i;
+        cov.deadlineHit = true;
+        break;
+      }
+      const { key, row } = rows[i];
+      if (seen.has(key)) continue;
+      cov.read += 1;
+      const records = adaptRouteOutcomes([row], ctxOpts);
+      if (records.length === 0) {
+        cov.rejected += 1;
+      }
+      for (const record of records) {
+        emitCounted(recorder, cov, record);
+      }
+      seen.add(key);
+    }
+    // Bounded tail: dedup only needs to outlive the input window.
+    const all = [...seen];
+    cursors.sources.routes = {
+      seen: all.length > MAX_ROUTE_SEEN ? all.slice(-MAX_ROUTE_SEEN) : all,
+      ...(all.length > MAX_ROUTE_SEEN ? { seen_truncated: true } : {}),
+    };
+  });
+
+  // --- decisions ---
+  await runSource('decisions', async (cov) => {
+    const stored = cursors.sources.decisions;
+    const ctxOpts = { writerId: 'ingest-decisions' };
+    const outcome = await consumeJsonlFile(
+      path.join(projectRoot, ROLLOUT_RECEIPTS_REL),
+      isPlainObject(stored?.entry) ? stored.entry : null,
+      cov,
+      (line) => {
+        const shaped = decisionReceiptShape(line);
+        const records = shaped ? adaptDecisionReceipts([shaped], ctxOpts) : [];
+        if (records.length === 0) {
+          cov.rejected += 1;
+          return 'consumed';
+        }
+        for (const record of records) {
+          emitCounted(recorder, cov, record);
+        }
+        return 'consumed';
+      },
+      expired,
+    );
+    if (outcome.error) {
+      degraded = true;
+      (cov.errors ??= []).push(outcome.error);
+    }
+    if (outcome.entry) cursors.sources.decisions = { entry: outcome.entry };
+    if (outcome.partial) cov.deadlineHit = true;
+  });
+
+  // --- context ---
+  await runSource('context', async (cov) => {
+    const stored = cursors.sources.context;
+    const ctxOpts = { writerId: 'ingest-context' };
+    const outcome = await consumeJsonlFile(
+      path.join(projectRoot, RETRIEVER_LANES_REL),
+      isPlainObject(stored?.entry) ? stored.entry : null,
+      cov,
+      (line) => {
+        const records = adaptContextItems(contextItemShapes(line), ctxOpts);
+        if (records.length === 0) {
+          cov.rejected += 1;
+          return 'consumed';
+        }
+        for (const record of records) {
+          emitCounted(recorder, cov, record);
+        }
+        return 'consumed';
+      },
+      expired,
+    );
+    if (outcome.error) {
+      degraded = true;
+      (cov.errors ??= []).push(outcome.error);
+    }
+    if (outcome.entry) cursors.sources.context = { entry: outcome.entry };
+    if (outcome.partial) cov.deadlineHit = true;
+  });
+
+  // Cursor persist is the atomic commit point (tmp+rename). A failed write
+  // leaves the LAST good cursor on disk — the next collect re-emits those
+  // rows as bounded duplicates rather than claiming delivery it never made.
+  let cursorError = null;
+  try {
+    await saveCursors(projectRoot, cursors);
+  } catch (error) {
+    cursorError = error?.code ?? 'cursor_persist_failed';
+    degraded = true;
+  }
+
+  return {
+    ok: true,
+    status: degraded ? 'degraded' : partial ? 'partial' : 'complete',
+    coverage,
+    last_cursors: cursors,
+    ...(cursorReset ? { cursor_reset: true } : {}),
+    ...(cursorError ? { cursor_error: cursorError } : {}),
+  };
+}