@ngockhoale/ukit 3.4.45 → 3.4.46

package/scripts/audit/subagent-cost/reader.mjs

@@ -1,332 +0,0 @@
-import fs from 'node:fs';
-import path from 'node:path';
-import readline from 'node:readline';
-import { createHash } from 'node:crypto';
-import { classifyCacheRead, classifyCacheWrite, cacheMetadataStatus, extractIdentity } from './usage-fields.mjs';
-
-const sha1 = (value) => createHash('sha1').update(value).digest('hex').slice(0, 12);
-const timestampMs = (value) => {
-  const result = typeof value === 'number' ? value : Date.parse(value);
-  return Number.isFinite(result) ? result : null;
-};
-const estimateTokens = (text) => (text ? Math.ceil(text.length / 4) : 0);
-const contentText = (content) => {
-  if (typeof content === 'string') return content;
-  if (!Array.isArray(content)) return '';
-  return content.filter((part) => part?.type === 'text' && typeof part.text === 'string')
-    .map((part) => part.text).join('\n');
-};
-
-async function readJsonl(file, stats, consume) {
-  const stream = fs.createReadStream(file, { encoding: 'utf8' });
-  const lines = readline.createInterface({ input: stream, crlfDelay: Infinity });
-  try {
-    for await (const line of lines) {
-      stats.lines += 1;
-      if (!line.trim()) continue;
-      let record;
-      try {
-        record = JSON.parse(line);
-      } catch {
-        stats.malformed += 1;
-        continue;
-      }
-      consume(record);
-    }
-  } finally {
-    lines.close();
-    stream.destroy();
-  }
-}
-
-function transcriptFiles(root) {
-  const files = [];
-  const rootEntries = fs.readdirSync(root, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name));
-  for (const entry of rootEntries) {
-    const entryPath = path.join(root, entry.name);
-    if (entry.isFile() && entry.name.endsWith('.jsonl')) {
-      files.push({ file: entryPath, sessionId: entry.name.slice(0, -'.jsonl'.length), isAgent: false });
-    }
-  }
-  for (const entry of rootEntries) {
-    if (!entry.isDirectory()) continue;
-    const sessionDir = path.join(root, entry.name);
-    const subagentsDir = path.join(sessionDir, 'subagents');
-    let subagentEntries;
-    try {
-      subagentEntries = fs.readdirSync(subagentsDir, { withFileTypes: true });
-    } catch (error) {
-      if (error.code === 'ENOENT' || error.code === 'ENOTDIR') continue;
-      throw error;
-    }
-    for (const agentEntry of subagentEntries.sort((a, b) => a.name.localeCompare(b.name))) {
-      if (!agentEntry.isFile() || !agentEntry.name.endsWith('.jsonl')) continue;
-      const stem = agentEntry.name.slice(0, -'.jsonl'.length);
-      files.push({
-        file: path.join(subagentsDir, agentEntry.name),
-        metaFile: path.join(subagentsDir, `${stem}.meta.json`),
-        sessionId: entry.name,
-        agentId: stem.replace(/^agent-/, ''),
-        isAgent: true,
-      });
-    }
-  }
-  return files;
-}
-
-async function readMeta(file) {
-  try {
-    const value = JSON.parse(await fs.promises.readFile(file, 'utf8'));
-    return value && typeof value === 'object' && !Array.isArray(value) ? value : null;
-  } catch {
-    return null;
-  }
-}
-
-function makeCall(record, sessionId, agentId, agentType, depth, ts, priorTs, meta = null) {
-  const message = record.message ?? {};
-  const usage = message.usage ?? {};
-  const input = Number(usage.input_tokens) || 0;
-  const readMetric = classifyCacheRead(usage);
-  const writeMetric = classifyCacheWrite(usage);
-  const cacheRead = readMetric.value;
-  const cacheWrite = writeMetric.value;
-  const identity = extractIdentity(record, meta);
-  const output = Number(usage.output_tokens) || 0;
-  const key = message.id ?? record.requestId ?? record.uuid;
-  return {
-    id: key == null ? null : String(key),
-    sessionId,
-    turnId: null,
-    agentId: agentId ?? null,
-    agentType: agentType ?? null,
-    model: message.model ?? record.model ?? null,
-    ts,
-    latencyMs: ts != null && priorTs != null && ts >= priorTs ? ts - priorTs : null,
-    input,
-    cacheRead,
-    cacheWrite,
-    cacheReadStatus: readMetric.status,
-    cacheWriteStatus: writeMetric.status,
-    cacheMetadataStatus: cacheMetadataStatus(readMetric, writeMetric),
-    providerId: identity.providerId,
-    gatewayId: identity.gatewayId,
-    taskId: identity.taskId,
-    spawnPrefixHash: identity.spawnPrefixHash,
-    output,
-    zero: input === 0 && cacheRead === 0 && cacheWrite === 0 && output === 0,
-    depth,
-    _dedupeKey: key == null ? null : String(key),
-  };
-}
-
-function hashFinalReferences(finalText, parentText) {
-  if (!finalText || !parentText) return null;
-  const words = (text) => text.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
-  const finalWords = words(finalText);
-  const parentWords = words(parentText);
-  if (finalWords.length < 5 || parentWords.length < 5) return null;
-  const grams = new Set();
-  for (let i = 0; i <= parentWords.length - 5; i += 1) {
-    grams.add(sha1(parentWords.slice(i, i + 5).join(' ')));
-  }
-  let matches = 0;
-  let total = 0;
-  for (let i = 0; i <= finalWords.length - 5; i += 1) {
-    total += 1;
-    if (grams.has(sha1(finalWords.slice(i, i + 5).join(' ')))) matches += 1;
-  }
-  return total ? matches / total : null;
-}
-
-export async function loadCorpus(root, { since = null, until = null } = {}) {
-  let rootStat;
-  try {
-    rootStat = fs.statSync(root);
-  } catch {
-    throw new Error(`root not found: ${root}`);
-  }
-  if (!rootStat.isDirectory()) throw new Error(`root not found: ${root}`);
-
-  const stats = {
-    files: 0, lines: 0, malformed: 0, zeroUsage: 0, dupIds: 0,
-    missingMeta: 0, noParent: 0, unattributed: 0,
-  };
-  const files = transcriptFiles(root);
-  const mainFilesBySession = new Map();
-  const agentFiles = [];
-  const promptRecordsBySession = new Map();
-  const toolUsesBySession = new Map();
-  const calls = [];
-  const agents = [];
-  const turns = [];
-  const sinceMs = since == null ? null : timestampMs(since);
-  const untilMs = until == null ? null : timestampMs(until);
-
-  for (const fileInfo of files) {
-    stats.files += 1;
-    const meta = fileInfo.isAgent ? await readMeta(fileInfo.metaFile) : null;
-    if (fileInfo.isAgent && !meta) stats.missingMeta += 1;
-    const agentType = fileInfo.isAgent ? (meta?.agentType ?? null) : null;
-    const depth = fileInfo.isAgent
-      ? (Number.isFinite(meta?.spawnDepth) ? meta.spawnDepth : 1)
-      : 0;
-    const deduped = new Map();
-    const userRecords = [];
-    let previousTs = null;
-    let finalAssistantText = null;
-
-    await readJsonl(fileInfo.file, stats, (record) => {
-      const ts = timestampMs(record.timestamp);
-      if (!fileInfo.isAgent && record.type === 'user' && !record.isSidechain) {
-        const body = contentText(record.message?.content ?? record.content);
-        if (body.trim()) userRecords.push({ ts, body });
-      }
-      if (record.type !== 'assistant') return;
-
-      const call = makeCall(
-        record,
-        fileInfo.sessionId,
-        fileInfo.isAgent ? fileInfo.agentId : null,
-        fileInfo.isAgent ? (agentType ?? record.attributionAgent ?? 'unknown') : null,
-        depth,
-        ts,
-        previousTs,
-        meta,
-      );
-      previousTs = ts;
-      const key = call._dedupeKey ?? `line:${stats.lines}`;
-      if (deduped.has(key) && call._dedupeKey != null) stats.dupIds += 1;
-      deduped.set(key, { call, record });
-
-      if (fileInfo.isAgent) {
-        const text = contentText(record.message?.content);
-        if (text) finalAssistantText = text;
-      } else {
-        for (const part of record.message?.content ?? []) {
-          if (part?.type !== 'tool_use' || typeof part.id !== 'string') continue;
-          if (!toolUsesBySession.has(fileInfo.sessionId)) toolUsesBySession.set(fileInfo.sessionId, new Map());
-          toolUsesBySession.get(fileInfo.sessionId).set(part.id, { ts, record, tool: part });
-        }
-      }
-    });
-
-    const records = [...deduped.values()];
-    if (fileInfo.isAgent) {
-      agentFiles.push({ fileInfo, meta, agentType, depth, records, finalAssistantText });
-    } else {
-      mainFilesBySession.set(fileInfo.sessionId, records);
-      promptRecordsBySession.set(fileInfo.sessionId, userRecords);
-    }
-  }
-
-  const turnsBySession = new Map();
-  for (const [sessionId, prompts] of promptRecordsBySession) {
-    prompts.sort((a, b) => (a.ts ?? 0) - (b.ts ?? 0));
-    const sessionTurns = prompts.map((prompt, index) => ({
-      turnId: `${sessionId}#${index + 1}`,
-      sessionId,
-      ts: prompt.ts,
-      promptHash: sha1(prompt.body),
-      promptTokensEst: estimateTokens(prompt.body),
-    }));
-    turnsBySession.set(sessionId, sessionTurns);
-    turns.push(...sessionTurns);
-  }
-
-  function turnForTime(sessionId, ts) {
-    const sessionTurns = turnsBySession.get(sessionId) ?? [];
-    let selected = null;
-    for (const turn of sessionTurns) {
-      if (ts == null || turn.ts == null || turn.ts <= ts) selected = turn;
-      else break;
-    }
-    return selected;
-  }
-
-  const parentTextBySessionTurn = new Map();
-  for (const [sessionId, records] of mainFilesBySession) {
-    let latestTurn = null;
-    const assistantTextsByTurn = new Map();
-    for (const { call, record } of records) {
-      const turn = turnForTime(sessionId, call.ts);
-      if (turn) latestTurn = turn;
-      call.turnId = latestTurn?.turnId ?? `${sessionId}#unattributed`;
-      if (record.message?.content) {
-        const text = contentText(record.message.content);
-        if (text) assistantTextsByTurn.set(call.turnId, text);
-      }
-      calls.push(call);
-    }
-    for (const [turnId, text] of assistantTextsByTurn) {
-      parentTextBySessionTurn.set(`${sessionId}\u0000${turnId}`, text);
-    }
-  }
-
-  agentFiles.sort((a, b) => {
-    const aTs = a.records[0]?.call.ts ?? 0;
-    const bTs = b.records[0]?.call.ts ?? 0;
-    return aTs - bTs || a.fileInfo.agentId.localeCompare(b.fileInfo.agentId);
-  });
-  const agentsById = new Map();
-  for (const { fileInfo, meta, agentType, depth, records, finalAssistantText } of agentFiles) {
-    const toolUseId = meta?.toolUseId ?? null;
-    const parent = toolUseId == null ? null : toolUsesBySession.get(fileInfo.sessionId)?.get(toolUseId) ?? null;
-    if (!parent) stats.noParent += 1;
-    const turn = parent ? turnForTime(fileInfo.sessionId, parent.ts) : null;
-    const turnId = turn?.turnId ?? `${fileInfo.sessionId}#unattributed`;
-    if (!turn) stats.unattributed += 1;
-
-    const delegatedPrompt = typeof parent?.tool?.input?.prompt === 'string' ? parent.tool.input.prompt : '';
-    const description = typeof meta?.description === 'string' ? meta.description : '';
-    const segments = delegatedPrompt.split(/\n\s*\n/)
-      .map((segment) => segment.trim())
-      .filter((segment) => segment.length >= 200)
-      .map((segment) => ({ h: sha1(segment), t: estimateTokens(segment) }));
-    const firstTs = records.reduce((value, item) => item.call.ts == null ? value : value == null ? item.call.ts : Math.min(value, item.call.ts), null);
-    const lastTs = records.reduce((value, item) => item.call.ts == null ? value : value == null ? item.call.ts : Math.max(value, item.call.ts), null);
-    const parentText = parentTextBySessionTurn.get(`${fileInfo.sessionId}\u0000${turnId}`) ?? '';
-    const agent = {
-      agentId: fileInfo.agentId,
-      sessionId: fileInfo.sessionId,
-      turnId,
-      agentType: agentType ?? records.find((item) => item.record.attributionAgent)?.record.attributionAgent ?? 'unknown',
-      descriptionHash: description ? sha1(description) : null,
-      toolUseId,
-      depth,
-      parentAgentId: meta?.parentAgentId ?? null,
-      metaModel: meta?.model ?? null,
-      hasMeta: Boolean(meta),
-      promptHash: delegatedPrompt ? sha1(delegatedPrompt) : null,
-      promptTokensEst: delegatedPrompt ? estimateTokens(delegatedPrompt) : null,
-      segments,
-      finalTokensEst: finalAssistantText ? estimateTokens(finalAssistantText) : null,
-      finalRefRatio: hashFinalReferences(finalAssistantText, parentText),
-      firstTs,
-      lastTs,
-    };
-    agents.push(agent);
-    agentsById.set(`${fileInfo.sessionId}\u0000${fileInfo.agentId}`, agent);
-    for (const { call } of records) {
-      call.turnId = turnId;
-      calls.push(call);
-    }
-  }
-
-  // Resolve explicit parent-agent links when the parent transcript is present; retain null for unknown links.
-  for (const agent of agents) {
-    if (agent.parentAgentId != null) {
-      const parent = agentsById.get(`${agent.sessionId}\u0000${agent.parentAgentId}`);
-      if (parent && agent.turnId.endsWith('#unattributed')) agent.turnId = parent.turnId;
-    }
-  }
-
-  stats.zeroUsage = calls.reduce((count, call) => count + (call.zero ? 1 : 0), 0);
-  const filteredCalls = calls.filter((call) =>
-    (sinceMs == null || (call.ts != null && call.ts >= sinceMs)) &&
-    (untilMs == null || (call.ts != null && call.ts <= untilMs)));
-  for (const call of filteredCalls) {
-    delete call._dedupeKey;
-  }
-  return { calls: filteredCalls, turns, agents, stats };
-}

package/scripts/audit/subagent-cost/render.mjs

@@ -1,34 +0,0 @@
-const fmt = (value) => value == null || !Number.isFinite(value) ? 'N/A' : Number(value).toLocaleString('en-US', { maximumFractionDigits: 3 });
-const table = (headers, rows) => [`| ${headers.join(' | ')} |`, `| ${headers.map(() => '---').join(' | ')} |`, ...rows.map((row) => `| ${row.map((cell) => String(cell ?? 'N/A').replaceAll('|', '\\|').replaceAll('\n', ' ')).join(' | ')} |`)].join('\n');
-
-export function renderMarkdown(report) {
-  const m = report.metrics ?? {};
-  const d = report.duplicates ?? {};
-  const totals = m.totals ?? {};
-  const typeRows = Object.entries(m.byType ?? {}).sort(([a], [b]) => a.localeCompare(b))
-    .map(([type, value]) => [type, value.agents, value.calls, fmt(value.inputSide), fmt(value.uncached), fmt(value.output), fmt(value.latencyP50)]);
-  const topRows = (items, key) => (items ?? []).map((item) => [item[key], item.agentType ?? 'N/A', item.calls, fmt(item.uncached), fmt(item.inputSide), fmt(item.output)]);
-  const flowRows = (m.tokenFlow ?? []).flatMap((turn) => turn.calls.map((call) => [turn.turnId, new Date(call.ts).toISOString(), call.agentType ?? 'main', call.depth, fmt(call.input), fmt(call.cacheRead), fmt(call.cacheWrite), fmt(call.output)]));
-  const flags = (value) => (value?.turnIds ?? []).map((turnId) => [turnId]);
-  return [
-    '# Sub-agent cost audit report',
-    '',
-    `Generated: ${report.generatedAt}`, `Root: ${report.root}`, `Window: ${report.window?.since ?? 'all'} — ${report.window?.until ?? 'all'}`,
-    '', '## Totals', '',
-    table(['Group', 'Calls', 'Input', 'Cache read', 'Cache write', 'Output', 'Uncached', 'Input-side'], ['main', 'sub'].map((key) => { const t = totals[key] ?? {}; return [key, t.calls, fmt(t.input), fmt(t.cacheRead), fmt(t.cacheWrite), fmt(t.output), fmt(t.uncached), fmt(t.inputSide)]; })),
-    '', '## Main vs sub-agent', '', table(['Metric', 'Value'], Object.entries(m.share ?? {}).map(([key, value]) => [key, fmt(value)])),
-    '', '## By type', '', table(['Type', 'Agents', 'Calls', 'Input-side', 'Uncached', 'Output', 'Latency p50 ms'], typeRows),
-    '', '## Fan-out', '', table(['Metric', 'Value'], [['Maximum', m.fanout?.max], ['p90', fmt(m.fanout?.p90)], ...Object.entries(m.fanout?.histogram ?? {}).map(([count, turns]) => [`${count} agents`, turns])]),
-    '', '## Cache', '', table(['Group', 'Hit rate', 'Cache read', 'Input-side'], ['main', 'sub'].map((key) => { const c = m.cache?.[key] ?? {}; return [key, fmt(c.hitRate), fmt(c.cacheRead), fmt(c.inputSide)]; })),
-    '', '## Top agents (20)', '', table(['Agent', 'Type', 'Calls', 'Uncached', 'Input-side', 'Output'], topRows(m.topAgents, 'agentId')),
-    '', '## Top turns (20)', '', table(['Turn', 'Type', 'Calls', 'Uncached', 'Input-side', 'Output'], topRows(m.topTurns, 'turnId')),
-    '', '## Token flow (5 costliest turns)', '', table(['Turn', 'Timestamp', 'Type', 'Depth', 'Input', 'Cache read', 'Cache write', 'Output'], flowRows),
-    '', '## N+1 flags', '', table(['Turn'], flags(m.nPlusOne)),
-    '', '## Retry flags', '', table(['Turn'], flags(m.retryLoops)),
-    '', '## Repeated-loop flags', '', table(['Turn'], flags(m.loops)),
-    '', '## Heavy calls and replay hypothesis', '', table(['Metric', 'Value'], [['Heavy calls', m.heavyCalls?.count], ['Heavy share', fmt(m.heavyCalls?.share)], ['Replay verdict', d.replayHypothesis?.verdict], ['Replay share', fmt(d.replayHypothesis?.share)]]),
-    '', '## Context duplication', '', table(['Metric', 'Value'], [['Amplification', fmt(d.amplification)], ['Duplicate context ratio', fmt(d.duplicateRatio)], ['Within-turn ratio', fmt(d.withinTurn?.ratio)], ['Cross-turn ratio', fmt(d.crossTurn?.ratio)]]),
-    '', '## Missing telemetry', '', ...(report.missing ?? []).map((item) => `- ${item}`),
-    '',
-  ].join('\n');
-}

package/scripts/audit/subagent-cost/usage-fields.mjs

@@ -1,82 +0,0 @@
-// MB-1 (C100) — telemetry correctness helpers shared by the sub-agent cost reader.
-// Pure, local, no LLM calls. Rules: a missing/invalid usage field is NEVER a measured zero;
-// an explicit 0 stays 0; provider/gateway/task/prefix identity is read only from explicit
-// allowlisted keys (the model name is never a signal) and unknown stays null.
-
-export const CACHE_STATUS = Object.freeze({
-  PRESENT_ZERO: 'present_zero',
-  PRESENT_POSITIVE: 'present_positive',
-  ABSENT: 'absent',
-  INVALID: 'invalid',
-});
-
-const READ_KEYS = ['cache_read_input_tokens', 'cache_read_tokens'];
-const WRITE_KEYS = ['cache_creation_input_tokens', 'cache_creation_tokens'];
-const PROVIDER_KEYS = ['providerId', 'provider_id', 'provider', 'apiProvider'];
-const GATEWAY_KEYS = ['gatewayId', 'gateway_id', 'gatewayName', 'gateway'];
-const MAX_ID_LENGTH = 128;
-const SAFE_ID = /^[A-Za-z0-9._:/@-]{1,128}$/;
-const SECRET_LIKE = /sk-[A-Za-z0-9]{8,}|bearer|secret|passw|token=|key=|api[_-]?key/i;
-
-function numericOrNull(raw) {
-  if (typeof raw === 'number') return Number.isFinite(raw) ? raw : null;
-  if (typeof raw === 'string' && raw.trim() !== '') {
-    const value = Number(raw);
-    return Number.isFinite(value) ? value : null;
-  }
-  return null;
-}
-
-/** Classify one cache metric. `value` is only the measured number; 0 for absent/invalid is a
- * summing convenience — consumers must read `status`, never infer a miss from value 0. */
-export function classifyCacheMetric(usage, keys) {
-  if (!usage || typeof usage !== 'object') return { status: CACHE_STATUS.ABSENT, value: 0 };
-  for (const key of keys) {
-    const raw = usage[key];
-    if (raw === undefined || raw === null) continue;
-    const value = numericOrNull(raw);
-    if (value === null || value < 0) return { status: CACHE_STATUS.INVALID, value: 0 };
-    return { status: value === 0 ? CACHE_STATUS.PRESENT_ZERO : CACHE_STATUS.PRESENT_POSITIVE, value };
-  }
-  return { status: CACHE_STATUS.ABSENT, value: 0 };
-}
-
-export const classifyCacheRead = (usage) => classifyCacheMetric(usage, READ_KEYS);
-export const classifyCacheWrite = (usage) => classifyCacheMetric(usage, WRITE_KEYS);
-
-/** Combined status for a call: a known read status wins; otherwise the worse of the pair. */
-export function cacheMetadataStatus(read, write) {
-  if (read.status === CACHE_STATUS.INVALID || write.status === CACHE_STATUS.INVALID) return CACHE_STATUS.INVALID;
-  if (read.status === CACHE_STATUS.ABSENT) return CACHE_STATUS.ABSENT;
-  return read.status;
-}
-
-function safeId(raw) {
-  if (typeof raw !== 'string') return null;
-  const value = raw.trim();
-  if (value.length === 0 || value.length > MAX_ID_LENGTH) return null;
-  if (!SAFE_ID.test(value) || SECRET_LIKE.test(value)) return null;
-  return value;
-}
-
-function firstId(sources, keys) {
-  for (const source of sources) {
-    if (!source || typeof source !== 'object') continue;
-    for (const key of keys) {
-      const id = safeId(source[key]);
-      if (id) return id;
-    }
-  }
-  return null;
-}
-
-/** Identity metadata from explicit keys only; null means UNKNOWN (never a guessed value). */
-export function extractIdentity(record, meta = null) {
-  const sources = [record, record?.message, meta];
-  return {
-    providerId: firstId(sources, PROVIDER_KEYS),
-    gatewayId: firstId(sources, GATEWAY_KEYS),
-    taskId: firstId(sources, ['taskId', 'task_id']),
-    spawnPrefixHash: firstId(sources, ['spawnPrefixHash', 'spawn_prefix_hash']),
-  };
-}