@ngockhoale/ukit 3.4.27 → 3.4.28

package/template_project/.claude/ukit/runtime/usage-capture.mjs

@@ -0,0 +1,338 @@
+#!/usr/bin/env node
+// usage-capture.mjs — C100 MB-1b: SHADOW capture of per-call usage metadata.
+//
+// Attached to the EXISTING Stop event (stop-coordinator.mjs) — no extra hook
+// invocation. Stage-gated by `.ukit/storage/config.json` -> usageCapture.stage;
+// absent / 'off' / anything but 'shadow' emits nothing. Observation only: it
+// never changes a routing, stop or tool decision, makes no network or LLM call.
+//
+// Reads the host transcript INCREMENTALLY: a per-session byte cursor means each
+// Stop reads only bytes appended since the previous Stop, never more than
+// MAX_READ_BYTES. Rows are metadata only — no prompt, response, tool input,
+// credential or credentialId is ever copied.
+//
+// Provenance: every row read from a host transcript is HOST_TRANSCRIPT.
+// providerId / gatewayId are identity metadata copied from explicit record keys;
+// they are NOT evidence of who measured the usage. PROVIDER / GATEWAY would need
+// an authenticated source (none exists yet), so they are never emitted here.
+//
+// Completeness: the capture is bounded, so it can be incomplete. Every gap
+// (skipped history, backlog, truncation/rotation, unparsable usage lines, row
+// cap, log rotation, lock timeout, deadline) is written as a `kind:'coverage'`
+// record and a sticky per-session flag. Consumers must treat a session with any
+// partial coverage as partial and never report its cache hit-rate as complete
+// (see summarizeUsageRows).
+//
+// Delivery is at-least-once with dedupe: a crash between the log append and the
+// cursor write replays the window; rows already present in the bounded log tail
+// are skipped by dedupeKey.
+//
+// Engines: Claude Code transcripts (snake_case usage) are supported; omp's
+// camelCase shape is parsed when handed in but the omp bridge does not call this
+// module yet; Codex is explicitly unsupported.
+
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { withAsyncLock } from './async-lock.mjs';
+
+export const USAGE_ROW_VERSION = 1;
+export const MAX_READ_BYTES = 1024 * 1024;
+export const MAX_ROWS_PER_STOP = 5000;
+export const MAX_LOG_BYTES = 256 * 1024;
+export const DEDUPE_TAIL_BYTES = 64 * 1024;
+export const CAPTURE_DEADLINE_MS = 250;
+const LOCK_DEADLINE_MS = 150;
+
+const CACHE_STATUS = Object.freeze({ ZERO: 'present_zero', POSITIVE: 'present_positive', ABSENT: 'absent', INVALID: 'invalid' });
+const READ_KEYS = ['cache_read_input_tokens', 'cache_read_tokens', 'cacheRead'];
+const WRITE_KEYS = ['cache_creation_input_tokens', 'cache_creation_tokens', 'cacheWrite'];
+const SAFE_ID = /^[A-Za-z0-9._:/@-]{1,128}$/;
+const SECRET_LIKE = /sk-[A-Za-z0-9]{8,}|bearer|secret|passw|token=|key=|api[_-]?key/i;
+const PROVIDER_KEYS = ['providerId', 'provider_id', 'provider', 'apiProvider'];
+const GATEWAY_KEYS = ['gatewayId', 'gateway_id', 'gatewayName', 'gateway'];
+
+export function resolveUsageCaptureStage(config) {
+  return config?.usageCapture?.stage === 'shadow' ? 'shadow' : 'off';
+}
+
+export function classifyCache(usage, keys) {
+  if (!usage || typeof usage !== 'object') return { status: CACHE_STATUS.ABSENT, value: null };
+  for (const key of keys) {
+    const raw = usage[key];
+    if (raw === undefined || raw === null) continue;
+    if (typeof raw !== 'number' || !Number.isFinite(raw) || raw < 0) return { status: CACHE_STATUS.INVALID, value: null };
+    return { status: raw === 0 ? CACHE_STATUS.ZERO : CACHE_STATUS.POSITIVE, value: raw };
+  }
+  return { status: CACHE_STATUS.ABSENT, value: null };
+}
+
+export function cacheMetadataStatus(read, write) {
+  if (read.status === CACHE_STATUS.INVALID || write.status === CACHE_STATUS.INVALID) return CACHE_STATUS.INVALID;
+  if (read.status === CACHE_STATUS.ABSENT) return CACHE_STATUS.ABSENT;
+  return read.status;
+}
+
+function safeId(record, message, keys) {
+  for (const holder of [message, record]) {
+    for (const key of keys) {
+      const value = holder?.[key];
+      if (typeof value !== 'string') continue;
+      return SAFE_ID.test(value) && !SECRET_LIKE.test(value) ? value : null;
+    }
+  }
+  return null;
+}
+
+const count = (v) => (typeof v === 'number' && Number.isFinite(v) && v >= 0 ? v : null);
+const hash = (s) => crypto.createHash('sha1').update(s).digest('hex').slice(0, 16);
+
+export function buildUsageRow(record, { harness = 'claude-code', now = Date.now(), line = null } = {}) {
+  if (!record || (record.type !== 'assistant' && record.role !== 'assistant')) return null;
+  const message = record.message && typeof record.message === 'object' ? record.message : record;
+  const usage = message.usage && typeof message.usage === 'object' && !Array.isArray(message.usage) ? message.usage : null;
+  if (!usage) return null;
+  const read = classifyCache(usage, READ_KEYS);
+  const write = classifyCache(usage, WRITE_KEYS);
+  const id = typeof message.id === 'string' && SAFE_ID.test(message.id) ? message.id : null;
+  return {
+    v: USAGE_ROW_VERSION,
+    kind: 'usage',
+    ts: new Date(now).toISOString(),
+    harness,
+    provenance: 'HOST_TRANSCRIPT',
+    dedupeKey: id ? `id:${id}` : `h:${hash(line ?? JSON.stringify(usage))}`,
+    messageId: id,
+    model: typeof message.model === 'string' && SAFE_ID.test(message.model) ? message.model : null,
+    providerId: safeId(record, message, PROVIDER_KEYS),
+    gatewayId: safeId(record, message, GATEWAY_KEYS),
+    inputTokens: count(usage.input_tokens ?? usage.input),
+    outputTokens: count(usage.output_tokens ?? usage.output),
+    cacheReadTokens: read.value,
+    cacheWriteTokens: write.value,
+    cacheReadStatus: read.status,
+    cacheWriteStatus: write.status,
+    cacheMetadataStatus: cacheMetadataStatus(read, write),
+  };
+}
+
+/**
+ * Consumer helper: hit-rate over rows with KNOWN cache metadata only. A session
+ * with any partial coverage record reports coverage:'partial' and withholds the
+ * headline hitRate (partialHitRate is explicitly labelled).
+ */
+export function summarizeUsageRows(rows) {
+  let partial = false;
+  let known = 0; let unknown = 0; let cacheRead = 0; let inputSide = 0;
+  for (const row of rows || []) {
+    if (row?.kind === 'coverage') { if (row.status === 'partial') partial = true; continue; }
+    if (row?.kind !== 'usage') continue;
+    if (row.cacheMetadataStatus === 'absent' || row.cacheMetadataStatus === 'invalid') { unknown += 1; continue; }
+    known += 1;
+    cacheRead += row.cacheReadTokens ?? 0;
+    inputSide += (row.inputTokens ?? 0) + (row.cacheReadTokens ?? 0) + (row.cacheWriteTokens ?? 0);
+  }
+  const rate = inputSide > 0 ? cacheRead / inputSide : null;
+  return {
+    coverage: partial ? 'partial' : 'complete',
+    knownCalls: known,
+    unknownCalls: unknown,
+    hitRate: partial ? null : rate,
+    partialHitRate: partial ? rate : null,
+  };
+}
+
+async function readConfig(projectRoot) {
+  try {
+    return JSON.parse(await fs.readFile(path.join(projectRoot, '.ukit', 'storage', 'config.json'), 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
+const sessionFile = (value) => crypto.createHash('sha256').update(String(value)).digest('hex').slice(0, 16);
+
+async function readLogTailKeys(file) {
+  const keys = new Set();
+  let handle;
+  try {
+    handle = await fs.open(file, 'r');
+    const { size } = await handle.stat();
+    const start = Math.max(0, size - DEDUPE_TAIL_BYTES);
+    const buf = Buffer.alloc(size - start);
+    await handle.read(buf, 0, buf.length, start);
+    let text = buf.toString('utf8');
+    if (start > 0) text = text.slice(text.indexOf('\n') + 1);
+    for (const line of text.split('\n')) {
+      if (!line.includes('"dedupeKey"')) continue;
+      try { const k = JSON.parse(line)?.dedupeKey; if (typeof k === 'string') keys.add(k); } catch { /* torn line */ }
+    }
+  } catch { /* no log yet */ } finally {
+    await handle?.close().catch(() => {});
+  }
+  return keys;
+}
+
+async function rotateIfNeeded(file, incoming) {
+  try {
+    const { size } = await fs.stat(file);
+    if (size + incoming > MAX_LOG_BYTES) { await fs.rename(file, `${file}.1`); return true; }
+  } catch { /* ENOENT or race: append recreates */ }
+  return false;
+}
+
+async function readCursor(file) {
+  try {
+    const c = JSON.parse(await fs.readFile(file, 'utf8'));
+    if (Number.isInteger(c?.offset) && c.offset >= 0) return c;
+  } catch { /* none/corrupt */ }
+  return null;
+}
+
+async function startsAtLineBoundary(handle, offset) {
+  if (offset === 0) return true;
+  const b = Buffer.alloc(1);
+  const { bytesRead } = await handle.read(b, 0, 1, offset - 1);
+  return bytesRead === 1 && b[0] === 0x0a;
+}
+
+async function captureInner({ projectRoot, payload = {}, harness, now = Date.now() }) {
+  const result = { stage: 'off', rows: 0, bytesRead: 0 };
+  const config = await readConfig(projectRoot);
+  result.stage = resolveUsageCaptureStage(config);
+  if (result.stage !== 'shadow') return result;
+  const engine = harness || process.env.UKIT_HARNESS || 'claude-code';
+  if (engine === 'codex') return { ...result, skipped: 'unsupported-engine' };
+  const transcript = payload?.transcript_path;
+  if (typeof transcript !== 'string' || !transcript) return { ...result, skipped: 'no-transcript-path' };
+
+  const dir = path.join(projectRoot, '.ukit', 'storage', 'cache', 'usage-capture');
+  const key = sessionFile(payload.session_id || transcript);
+  const cursorFile = path.join(dir, `${key}.cursor.json`);
+  const logFile = path.join(dir, `${key}.jsonl`);
+  await fs.mkdir(dir, { recursive: true });
+
+  // The cursor is read, advanced and written under ONE lock: two concurrent
+  // Stops cannot both consume the same window. A lock timeout leaves the cursor
+  // untouched, so the bytes are picked up by the next Stop (delayed, not lost).
+  const lock = await withAsyncLock(logFile, { deadlineMs: LOCK_DEADLINE_MS }, async () => {
+    const handle = await fs.open(transcript, 'r');
+    try {
+      const st = await handle.stat();
+      const size = st.size;
+      const cursor = await readCursor(cursorFile);
+      const reasons = [];
+      let skippedBytes = 0;
+      let offset = 0;
+      if (cursor) {
+        if ((cursor.ino != null && cursor.ino !== st.ino) || cursor.offset > size) {
+          reasons.push('transcript-rotated-or-truncated');
+        } else {
+          offset = cursor.offset;
+        }
+      } else {
+        reasons.push('initial-window');
+      }
+      if (size - offset > MAX_READ_BYTES && (!cursor || reasons.length)) {
+        // No trustworthy cursor: only the newest window is read; older bytes are a declared gap.
+        skippedBytes = size - MAX_READ_BYTES - offset;
+        offset = size - MAX_READ_BYTES;
+        if (!reasons.length) reasons.push('initial-window');
+      } else if (!cursor && size === 0) {
+        reasons.length = 0;
+      } else if (!reasons.includes('transcript-rotated-or-truncated') && reasons[0] === 'initial-window' && skippedBytes === 0) {
+        reasons.length = 0; // whole small transcript read from byte 0: nothing was skipped
+      }
+      const length = Math.min(size - offset, MAX_READ_BYTES);
+      let pendingBytes = size - offset - length;
+      const next = { v: 1, offset, ino: st.ino, partial: Boolean(cursor?.partial) };
+      let lines = [];
+      let consumed = 0;
+      if (length > 0) {
+        const buf = Buffer.alloc(length);
+        const { bytesRead } = await handle.read(buf, 0, length, offset);
+        result.bytesRead = bytesRead;
+        const text = buf.subarray(0, bytesRead).toString('utf8');
+        const lastNl = text.lastIndexOf('\n');
+        if (lastNl >= 0) {
+          consumed = Buffer.byteLength(text.slice(0, lastNl + 1), 'utf8');
+          lines = text.slice(0, lastNl + 1).split('\n');
+          if (!(await startsAtLineBoundary(handle, offset))) lines = lines.slice(1); // window began mid-line
+        }
+      }
+      if (pendingBytes > 0) reasons.push('backlog');
+      next.offset = offset + consumed;
+
+      let unparsedUsageLines = 0;
+      const rows = [];
+      const seen = new Set();
+      for (const line of lines) {
+        if (!line.includes('"usage"')) continue;
+        let record;
+        try { record = JSON.parse(line); } catch { unparsedUsageLines += 1; continue; }
+        const row = buildUsageRow(record, { harness: engine, now, line });
+        if (!row) continue;
+        if (seen.has(row.dedupeKey)) { // streaming chunks of one message: last chunk wins
+          rows[rows.findIndex((r) => r.dedupeKey === row.dedupeKey)] = row;
+          continue;
+        }
+        seen.add(row.dedupeKey);
+        rows.push(row);
+      }
+      let droppedRows = 0;
+      if (rows.length > MAX_ROWS_PER_STOP) { droppedRows = rows.length - MAX_ROWS_PER_STOP; rows.splice(0, droppedRows); reasons.push('row-cap'); }
+      if (unparsedUsageLines > 0) reasons.push('unparsed-usage-lines');
+
+      const known = rows.length ? await readLogTailKeys(logFile) : new Set();
+      const fresh = rows.filter((r) => !known.has(r.dedupeKey));
+      const records = [...fresh];
+      let rotated = false;
+      const partialNow = reasons.length > 0;
+      if (partialNow) {
+        records.push({
+          v: USAGE_ROW_VERSION, kind: 'coverage', ts: new Date(now).toISOString(), harness: engine,
+          status: 'partial', reasons: [...new Set(reasons)], skippedBytes, pendingBytes, droppedRows, unparsedUsageLines,
+        });
+        next.partial = true;
+      }
+      if (records.length) {
+        const body = records.map((r) => JSON.stringify(r)).join('\n') + '\n';
+        rotated = await rotateIfNeeded(logFile, Buffer.byteLength(body));
+        if (rotated) {
+          // Older rows beyond the retention window are gone: say so, and let consumers know.
+          records.unshift({ v: USAGE_ROW_VERSION, kind: 'coverage', ts: new Date(now).toISOString(), harness: engine, status: 'partial', reasons: ['log-rotated'], skippedBytes: 0, pendingBytes: 0, droppedRows: 0, unparsedUsageLines: 0 });
+          next.partial = true;
+        }
+        await fs.appendFile(logFile, records.map((r) => JSON.stringify(r)).join('\n') + '\n', 'utf8');
+      }
+      const tmp = `${cursorFile}.${process.pid}.tmp`;
+      await fs.writeFile(tmp, JSON.stringify(next));
+      await fs.rename(tmp, cursorFile);
+      result.rows = fresh.length;
+      result.coverage = next.partial ? 'partial' : 'complete';
+      if (pendingBytes > 0) result.pendingBytes = pendingBytes;
+    } finally {
+      await handle.close().catch(() => {});
+    }
+  });
+  if (lock?.ok === false) return { ...result, skipped: 'lock-timeout' };
+  return result;
+}
+
+/**
+ * Capture usage rows appended to the transcript since the last Stop.
+ * Never throws and never takes longer than CAPTURE_DEADLINE_MS to return; a
+ * deadline hit returns skipped:'deadline' (the next Stop resumes from the cursor).
+ */
+export async function captureUsageShadow(opts = {}) {
+  let timer;
+  const deadline = new Promise((resolve) => {
+    timer = setTimeout(() => resolve({ stage: 'shadow', rows: 0, bytesRead: 0, skipped: 'deadline' }), opts.deadlineMs ?? CAPTURE_DEADLINE_MS);
+    timer.unref?.();
+  });
+  try {
+    return await Promise.race([captureInner(opts).catch(() => ({ stage: 'off', rows: 0, bytesRead: 0, skipped: 'error' })), deadline]);
+  } finally {
+    clearTimeout(timer);
+  }
+}

package/template_project/.omp/hooks/pre/ukit-bridge.js

@@ -751,6 +751,12 @@ function normalizeToolInput(toolName, input) {
     const paths = Array.isArray(normalized.paths) ? normalized.paths.filter(Boolean) : [];
     normalized.file_path = normalized.path || paths[0] || undefined;
   }
+  // omp's eval tool carries `{language, code}` (no `command`), but block-dangerous and the
+  // secret guard only read tool_input.command. Surface the code as the command so the same
+  // Bash chain scans it; otherwise an eval payload would reach the gate as an empty command.
+  if (toolName === 'Bash' && typeof normalized.command !== 'string' && typeof normalized.code === 'string') {
+    normalized.command = normalized.code;
+  }
   return normalized;
 }
 
@@ -789,8 +795,39 @@ function sendContext(pi, context, deliverAs, { display = false } = {}) {
   pi.sendMessage(hookContextMessage(content, { display }), { deliverAs });
 }
 
-export async function runToolCall(pi, event, { projectRoot, context: extensionContext = {} }) {
+// eval threat model (F2): the Bash chain is a text matcher over a command string. It cannot
+// prove that arbitrary Python/JS is safe (shutil.rmtree, fs.rmSync, pathlib loops, dynamic
+// strings all pass it — verified in a sentinel sandbox). So eval is DENIED by the bridge unless
+// the user opted in with orchestration.allowUnsafeEval=true in .ukit/storage/config.json,
+// independent of .omp/config.yml tools.approval.eval. When opted in, the Bash chain still runs
+// as best-effort defence in depth; it is NOT a safety guarantee for eval code.
+async function readAllowUnsafeEval(projectRoot) {
+  try {
+    const parsed = JSON.parse(
+      await fs.promises.readFile(path.join(projectRoot, '.ukit', 'storage', 'config.json'), 'utf8'),
+    );
+    return parsed?.orchestration?.allowUnsafeEval === true;
+  } catch {
+    return false; // missing/corrupt config → fail closed
+  }
+}
+
+export async function runToolCall(pi, event, { projectRoot, context: extensionContext = {}, allowUnsafeEval } = {}) {
   const rawToolName = event.toolName ?? event.tool;
+  if (String(rawToolName ?? '').trim().toLowerCase() === 'eval') {
+    const allowed = allowUnsafeEval ?? await readAllowUnsafeEval(projectRoot);
+    if (!allowed) {
+      return {
+        block: true,
+        toolName: 'Bash',
+        invoked: [],
+        context: [],
+        reason: 'UKit denies the omp eval tool: arbitrary Python/JS cannot be verified by the shell guard. '
+          + 'Use the bash tool for commands (guarded), or opt in explicitly with orchestration.allowUnsafeEval=true '
+          + 'in .ukit/storage/config.json (unsafe; the guard is best-effort for eval).',
+      };
+    }
+  }
   const mappedToolName = mapToolName(rawToolName);
   // An unmapped tool that LOOKS like a file mutation must not sail past the guards — but
   // hard-blocking it froze the run whenever omp renamed or added a tool, because the model
@@ -811,9 +848,16 @@ export async function runToolCall(pi, event, { projectRoot, context: extensionCo
     toolUseId: event.toolCallId,
     ...metadata,
   });
-  // Only Edit|Write stays fail-closed on a bridge/runner transport failure; Bash and
-  // Read|Grep|Glob fail open so diagnosis/recovery tools are never bricked by a runner outage.
-  const failClosedOnTransportError = matcherGroupFor(toolName) === 'Edit|Write';
+  // F4: Edit|Write and Bash (which also carries omp's `eval`) are security-critical and fail
+  // CLOSED on a bridge/runner transport failure — an unverifiable shell/eval call is not
+  // allowed to proceed. Read|Grep|Glob stay fail-open (documented limit: the secret-read guard
+  // is advisory on a runner outage) so diagnosis tools are not bricked. Recovery for a Bash
+  // outage is a user action, not a model one: the user exports UKIT_OMP_BASH_FAIL_OPEN=1 (or
+  // fixes Node/PATH) and restarts omp. This only affects PreToolUse; the Stop gate is separate,
+  // so a blocked Bash call cannot create a Stop-hook loop.
+  const group = matcherGroupFor(toolName);
+  const bashFailOpenOverride = process.env.UKIT_OMP_BASH_FAIL_OPEN === '1';
+  const failClosedOnTransportError = group === 'Edit|Write' || (group === 'Bash' && !bashFailOpenOverride);
   const result = await runScriptChain(pi, scriptsForToolCall(toolName), payload, {
     projectRoot,
     failClosedOnTransportError,