@ngockhoale/ukit 3.4.45 → 3.4.46

package/scripts/audit/subagent-cost/duplicates.mjs

@@ -1,195 +0,0 @@
-const HEAVY_INPUT_SIDE_MIN = 250_000;
-const HEAVY_OUTPUT_MAX = 2_000;
-const MIN_HEAVY_CALLS_FOR_VERDICT = 10;
-const TOP_LIMIT = 20;
-
-const isFiniteNumber = Number.isFinite;
-const inputSide = (call) => (isFiniteNumber(call.input) ? call.input : 0)
-  + (isFiniteNumber(call.cacheRead) ? call.cacheRead : 0)
-  + (isFiniteNumber(call.cacheWrite) ? call.cacheWrite : 0);
-const ratio = (numerator, denominator) => denominator > 0 ? numerator / denominator : null;
-const agentType = (agent) => agent.agentType ?? 'unknown';
-const sortText = (a, b) => String(a) < String(b) ? -1 : (String(a) > String(b) ? 1 : 0);
-
-function getAgents(corpus) {
-  return Array.isArray(corpus?.agents) ? corpus.agents.filter((agent) => agent && typeof agent === 'object') : [];
-}
-
-function getCalls(corpus) {
-  return Array.isArray(corpus?.calls) ? corpus.calls.filter((call) => call && typeof call === 'object') : [];
-}
-
-function collectSegments(agents) {
-  const byHash = new Map();
-  for (const agent of agents) {
-    if (!Array.isArray(agent.segments)) continue;
-    for (const segment of agent.segments) {
-      if (!segment || typeof segment !== 'object' || segment.h == null
-        || !isFiniteNumber(segment.t) || segment.t < 0) continue;
-      if (!byHash.has(segment.h)) byHash.set(segment.h, []);
-      byHash.get(segment.h).push({
-        h: segment.h,
-        t: segment.t,
-        turnId: agent.turnId,
-        agentId: agent.agentId,
-        agentType: agentType(agent),
-      });
-    }
-  }
-  return byHash;
-}
-
-function repeatedOccurrences(occurrences) {
-  // A segment can occur more than once inside an agent prompt. Repetition is
-  // measured between agents, so retain that agent's first occurrence only.
-  const firstByAgent = new Map();
-  for (const occurrence of occurrences) {
-    if (occurrence.agentId == null || firstByAgent.has(occurrence.agentId)) continue;
-    firstByAgent.set(occurrence.agentId, occurrence);
-  }
-  const distinctAgents = [...firstByAgent.values()];
-  if (distinctAgents.length < 2) return null;
-  return distinctAgents;
-}
-
-function duplicateBreakdown(occurrences) {
-  const agents = repeatedOccurrences(occurrences);
-  if (!agents) return null;
-
-  let withinTokens = 0;
-  let crossTokens = 0;
-  const first = agents[0];
-  for (const occurrence of agents.slice(1)) {
-    if (occurrence.turnId === first.turnId) withinTokens += occurrence.t;
-    else crossTokens += occurrence.t;
-  }
-  return {
-    withinTokens,
-    crossTokens,
-    count: agents.length,
-    displayTokens: first.t,
-    duplicateTokens: withinTokens + crossTokens,
-  };
-}
-
-function computeAmplification(corpus, agents, calls) {
-  const agentsById = new Map(agents.filter((agent) => agent.agentId != null)
-    .map((agent) => [agent.agentId, agent]));
-  const inputByType = new Map();
-  for (const call of calls) {
-    const agent = agentsById.get(call.agentId);
-    if (!agent || call.zero || (call.depth ?? agent.depth ?? 1) <= 0) continue;
-    const type = agentType(agent);
-    inputByType.set(type, (inputByType.get(type) ?? 0) + inputSide(call));
-  }
-
-  const promptsByType = new Map();
-  for (const agent of agents) {
-    if (agent.agentId == null || (agent.depth ?? 1) <= 0
-      || !isFiniteNumber(agent.promptTokensEst) || agent.promptTokensEst < 0) continue;
-    const type = agentType(agent);
-    promptsByType.set(type, (promptsByType.get(type) ?? 0) + agent.promptTokensEst);
-  }
-
-  const inputTotal = [...inputByType.values()].reduce((sum, value) => sum + value, 0);
-  const promptTotal = [...promptsByType.values()].reduce((sum, value) => sum + value, 0);
-  const types = [...new Set([...inputByType.keys(), ...promptsByType.keys()])].sort(sortText);
-  return {
-    overall: promptTotal === 0 ? null : inputTotal / promptTotal,
-    byType: Object.fromEntries(types.map((type) => [
-      type,
-      (promptsByType.get(type) ?? 0) === 0 ? null : (inputByType.get(type) ?? 0) / promptsByType.get(type),
-    ])),
-  };
-}
-
-function summarizeSegments(byHash) {
-  let totalTokens = 0;
-  let duplicateTokens = 0;
-  let withinTokens = 0;
-  let crossTokens = 0;
-  const repeated = [];
-
-  for (const [hash, occurrences] of byHash) {
-    totalTokens += occurrences.reduce((sum, item) => sum + item.t, 0);
-    const breakdown = duplicateBreakdown(occurrences);
-    if (!breakdown) continue;
-    duplicateTokens += breakdown.duplicateTokens;
-    withinTokens += breakdown.withinTokens;
-    crossTokens += breakdown.crossTokens;
-    repeated.push({
-      h: hash,
-      t: breakdown.displayTokens,
-      count: breakdown.count,
-      agentTypes: [...new Set(occurrences.filter((item) => item.agentId != null)
-        .map((item) => item.agentType))].sort(sortText),
-      duplicateTokens: breakdown.duplicateTokens,
-    });
-  }
-
-  repeated.sort((a, b) => b.duplicateTokens - a.duplicateTokens || sortText(a.h, b.h));
-  return {
-    totalTokens,
-    duplicateTokens,
-    withinTokens,
-    crossTokens,
-    topRepeatedSegments: repeated.slice(0, TOP_LIMIT).map(({ duplicateTokens: _tokens, ...segment }) => segment),
-  };
-}
-
-function computeReplayHypothesis(calls, agents, byHash) {
-  const agentsById = new Map(agents.filter((agent) => agent.agentId != null)
-    .map((agent) => [agent.agentId, agent]));
-  const heavyCalls = calls.filter((call) => {
-    const agent = agentsById.get(call.agentId);
-    if (!agent || call.zero || (call.depth ?? agent.depth ?? 1) <= 0) return false;
-    return inputSide(call) >= HEAVY_INPUT_SIDE_MIN
-      && isFiniteNumber(call.output) && call.output <= HEAVY_OUTPUT_MAX;
-  });
-  const heavyTurns = new Set(heavyCalls.map((call) => call.turnId).filter((turnId) => turnId != null));
-  const sharedHeavyTurns = new Set();
-
-  for (const occurrences of byHash.values()) {
-    const agentIdsByTurn = new Map();
-    for (const occurrence of repeatedOccurrences(occurrences) ?? []) {
-      if (occurrence.turnId == null || !heavyTurns.has(occurrence.turnId)) continue;
-      if (!agentIdsByTurn.has(occurrence.turnId)) agentIdsByTurn.set(occurrence.turnId, new Set());
-      agentIdsByTurn.get(occurrence.turnId).add(occurrence.agentId);
-    }
-    for (const [turnId, ids] of agentIdsByTurn) if (ids.size >= 2) sharedHeavyTurns.add(turnId);
-  }
-
-  return {
-    verdict: heavyCalls.length < MIN_HEAVY_CALLS_FOR_VERDICT
-      ? 'inconclusive'
-      : (sharedHeavyTurns.size > 0 ? 'supported' : 'not_supported'),
-    heavyCalls: heavyCalls.length,
-    heavyTurns: heavyTurns.size,
-    sharedHeavyTurns: sharedHeavyTurns.size,
-    share: ratio(sharedHeavyTurns.size, heavyTurns.size),
-  };
-}
-
-export function computeDuplicates(corpus) {
-  const agents = getAgents(corpus);
-  const calls = getCalls(corpus);
-  const byHash = collectSegments(agents);
-  const segments = summarizeSegments(byHash);
-  const amplification = computeAmplification(corpus, agents, calls);
-
-  return {
-    amplification: amplification.overall,
-    amplificationByType: amplification.byType,
-    duplicateRatio: ratio(segments.duplicateTokens, segments.totalTokens),
-    withinTurn: {
-      tokens: segments.withinTokens,
-      ratio: ratio(segments.withinTokens, segments.totalTokens),
-    },
-    crossTurn: {
-      tokens: segments.crossTokens,
-      ratio: ratio(segments.crossTokens, segments.totalTokens),
-    },
-    topRepeatedSegments: segments.topRepeatedSegments,
-    replayHypothesis: computeReplayHypothesis(calls, agents, byHash),
-  };
-}

package/scripts/audit/subagent-cost/metrics.mjs

@@ -1,193 +0,0 @@
-const N_PLUS_ONE_MIN_AGENTS = 3;
-const N_PLUS_ONE_MAX_PROMPT_TOKENS = 2_000;
-const RETRY_MIN_OCCURRENCES = 2;
-const LOOP_MIN_OCCURRENCES = 5;
-const HEAVY_INPUT_SIDE_MIN = 250_000;
-const HEAVY_OUTPUT_MAX = 2_000;
-const TOP_LIMIT = 20;
-
-export function percentile(values, p) {
-  if (!Array.isArray(values) || values.length === 0) return null;
-  const sorted = values.filter(Number.isFinite).sort((a, b) => a - b);
-  if (sorted.length === 0) return null;
-  const rank = Math.max(1, Math.ceil(p * sorted.length));
-  return sorted[rank - 1];
-}
-
-const sum = (items, field) => items.reduce((total, item) => total + (Number.isFinite(item[field]) ? item[field] : 0), 0);
-const ratio = (n, d) => d > 0 ? n / d : null;
-const inputSide = (call) => (call.input ?? 0) + (call.cacheRead ?? 0) + (call.cacheWrite ?? 0);
-const uncached = (call) => (call.input ?? 0) + (call.cacheWrite ?? 0);
-const callTokens = (call) => inputSide(call) + (call.output ?? 0);
-const validCalls = (corpus) => (corpus.calls ?? []).filter((call) => !call.zero);
-const grouped = (items, keyFn) => {
-  const result = new Map();
-  for (const item of items) {
-    const key = keyFn(item);
-    if (key == null) continue;
-    if (!result.has(key)) result.set(key, []);
-    result.get(key).push(item);
-  }
-  return result;
-};
-const totalFor = (calls) => ({
-  calls: calls.length,
-  input: sum(calls, 'input'),
-  cacheRead: sum(calls, 'cacheRead'),
-  cacheWrite: sum(calls, 'cacheWrite'),
-  output: sum(calls, 'output'),
-  uncached: calls.reduce((n, c) => n + uncached(c), 0),
-  inputSide: calls.reduce((n, c) => n + inputSide(c), 0),
-});
-// Cache metadata is only trustworthy when the read key was recorded (present_zero/present_positive).
-// Absent/invalid calls are excluded from the hit-rate denominator — they are UNKNOWN, not a miss.
-// Calls without a status (historical corpora built before MB-1) are treated as recorded.
-const cacheKnown = (call) => call.cacheMetadataStatus == null
-  || call.cacheMetadataStatus === 'present_zero' || call.cacheMetadataStatus === 'present_positive';
-const cacheFor = (calls) => {
-  const known = calls.filter(cacheKnown);
-  const totals = totalFor(known);
-  return {
-    hitRate: ratio(totals.cacheRead, totals.inputSide),
-    cacheRead: totals.cacheRead,
-    inputSide: totals.inputSide,
-    knownCalls: known.length,
-    unknownCalls: calls.length - known.length,
-  };
-};
-const sortedTop = (items, getValue, getKey) => [...items]
-  .map((item) => ({ ...item, uncached: getValue(item), inputSide: getKey(item) }))
-  .sort((a, b) => b.uncached - a.uncached || b.inputSide - a.inputSide || String(a.agentId ?? a.turnId).localeCompare(String(b.agentId ?? b.turnId)))
-  .slice(0, TOP_LIMIT);
-
-function callPercentiles(calls, selector) {
-  return {
-    p50: percentile(calls.map(selector), 0.5),
-    p90: percentile(calls.map(selector), 0.9),
-    p99: percentile(calls.map(selector), 0.99),
-  };
-}
-
-function computeTasks(corpus, calls) {
-  const tasks = new Map();
-  for (const turn of corpus.turns ?? []) tasks.set(turn.turnId, { turnId: turn.turnId, calls: [] });
-  for (const call of calls) {
-    const key = call.turnId ?? `${call.sessionId}#unattributed`;
-    if (!tasks.has(key)) tasks.set(key, { turnId: key, calls: [] });
-    tasks.get(key).calls.push(call);
-  }
-  const rows = [...tasks.values()];
-  const subAgentCounts = rows.map((task) => new Set((corpus.agents ?? []).filter((a) => a.turnId === task.turnId).map((a) => a.agentId)).size);
-  return {
-    tasks: rows.length,
-    avgSubAgents: ratio(subAgentCounts.reduce((a, b) => a + b, 0), rows.length),
-    avgCalls: ratio(rows.reduce((n, task) => n + task.calls.length, 0), rows.length),
-    ...Object.fromEntries(['Calls', 'Tokens'].flatMap((label) => {
-      const values = rows.map((task) => label === 'Calls' ? task.calls.length : task.calls.reduce((n, c) => n + callTokens(c), 0));
-      return [[`p50${label}`, percentile(values, 0.5)], [`p90${label}`, percentile(values, 0.9)], [`p99${label}`, percentile(values, 0.99)]];
-    })),
-  };
-}
-
-export function computeMetrics(corpus) {
-  const calls = validCalls(corpus);
-  const mainCalls = calls.filter((call) => (call.depth ?? 0) === 0);
-  const subCalls = calls.filter((call) => (call.depth ?? 0) > 0);
-  const main = totalFor(mainCalls);
-  const sub = totalFor(subCalls);
-  const all = totalFor(calls);
-  const totalOutput = all.output;
-  const cacheByType = {};
-  const byType = {};
-  const typeCalls = grouped(subCalls, (call) => call.agentType ?? 'unknown');
-  for (const [type, typeGroup] of typeCalls) {
-    const agents = (corpus.agents ?? []).filter((agent) => (agent.agentType ?? 'unknown') === type);
-    const totals = totalFor(typeGroup);
-    const latencies = typeGroup.map((call) => call.latencyMs).filter(Number.isFinite);
-    const finalTokens = agents.map((agent) => agent.finalTokensEst).filter(Number.isFinite);
-    const refs = agents.map((agent) => agent.finalRefRatio).filter(Number.isFinite);
-    byType[type] = {
-      agents: new Set(agents.map((agent) => agent.agentId)).size,
-      calls: totals.calls,
-      inputSide: totals.inputSide,
-      uncached: totals.uncached,
-      output: totals.output,
-      avgCallsPerAgent: ratio(totals.calls, new Set(agents.map((agent) => agent.agentId)).size),
-      outputPerInput: ratio(totals.output, totals.inputSide),
-      avgFinalTokensEst: ratio(finalTokens.reduce((a, b) => a + b, 0), finalTokens.length),
-      avgFinalRefRatio: ratio(refs.reduce((a, b) => a + b, 0), refs.length),
-      latencyP50: percentile(latencies, 0.5),
-      latencyP90: percentile(latencies, 0.9),
-      valueProxy: { label: 'proxy; not a measure of true agent value', avgFinalTokensEst: ratio(finalTokens.reduce((a, b) => a + b, 0), finalTokens.length), avgFinalRefRatio: ratio(refs.reduce((a, b) => a + b, 0), refs.length), outputPerInput: ratio(totals.output, totals.inputSide) },
-    };
-    cacheByType[type] = cacheFor(typeGroup);
-  }
-  const depthCalls = grouped(calls, (call) => call.depth ?? 0);
-  const byDepth = Object.fromEntries([...depthCalls].map(([depth, group]) => [depth, totalFor(group)]));
-  const modelCalls = grouped(calls, (call) => call.model ?? 'unknown');
-  const byModel = Object.fromEntries([...modelCalls].map(([model, group]) => [model, totalFor(group)]));
-  const turnAgents = grouped(corpus.agents ?? [], (agent) => agent.turnId);
-  const fanoutCounts = [...turnAgents.values()].map((agents) => agents.length);
-  const fanoutHistogram = {};
-  for (const count of fanoutCounts) fanoutHistogram[count] = (fanoutHistogram[count] ?? 0) + 1;
-  const heavy = subCalls.filter((call) => inputSide(call) >= HEAVY_INPUT_SIDE_MIN && call.output <= HEAVY_OUTPUT_MAX);
-  const heavyByType = {};
-  for (const [type, group] of grouped(heavy, (call) => call.agentType ?? 'unknown')) heavyByType[type] = group.length;
-  const heavyBySession = {};
-  for (const [sessionId, group] of grouped(heavy, (call) => call.sessionId ?? 'unknown')) heavyBySession[sessionId] = group.length;
-  const turns = [...new Set([...calls.map((call) => call.turnId), ...(corpus.turns ?? []).map((turn) => turn.turnId)].filter(Boolean))];
-  const turnAgentsAll = grouped(corpus.agents ?? [], (agent) => agent.turnId);
-  const flagged = (predicate) => {
-    const ids = [];
-    for (const [turnId, agents] of turnAgentsAll) if (predicate(agents)) ids.push(turnId);
-    return { count: ids.length, turnIds: ids.slice(0, TOP_LIMIT) };
-  };
-  const nPlusOne = flagged((agents) => {
-    const sorted = [...agents].sort((a, b) => a.firstTs - b.firstTs);
-    let run = 0;
-    let previous = null;
-    for (const agent of sorted) {
-      const small = Number.isFinite(agent.promptTokensEst) && agent.promptTokensEst <= N_PLUS_ONE_MAX_PROMPT_TOKENS;
-      if (small && previous && previous.agentType === agent.agentType && agent.firstTs >= previous.lastTs) run += 1;
-      else run = small ? 1 : 0;
-      previous = agent;
-      if (run >= N_PLUS_ONE_MIN_AGENTS) return true;
-    }
-    return false;
-  });
-  const retryLoops = flagged((agents) => [...grouped(agents.filter((a) => a.promptHash != null), (a) => `${a.agentType}\0${a.promptHash}`).values()].some((group) => group.length >= RETRY_MIN_OCCURRENCES));
-  const loops = flagged((agents) => [...grouped(agents, (a) => a.agentType).values()].some((group) => group.length >= LOOP_MIN_OCCURRENCES));
-  const agentStats = (corpus.agents ?? []).map((agent) => {
-    const group = calls.filter((call) => call.agentId === agent.agentId);
-    return { agentId: agent.agentId, agentType: agent.agentType, turnId: agent.turnId, calls: group.length, uncached: group.reduce((n, c) => n + uncached(c), 0), inputSide: group.reduce((n, c) => n + inputSide(c), 0), output: sum(group, 'output') };
-  });
-  const turnStats = turns.map((turnId) => {
-    const group = calls.filter((call) => call.turnId === turnId);
-    return { turnId, calls: group.length, uncached: group.reduce((n, c) => n + uncached(c), 0), inputSide: group.reduce((n, c) => n + inputSide(c), 0), output: sum(group, 'output') };
-  });
-  const costliest = [...turnStats].sort((a, b) => b.uncached - a.uncached).slice(0, 5);
-  const tokenFlow = costliest.map(({ turnId }) => ({ turnId, calls: calls.filter((call) => call.turnId === turnId).sort((a, b) => a.ts - b.ts).map((call) => ({ ts: call.ts, agentType: call.agentType, depth: call.depth, input: call.input, cacheRead: call.cacheRead, cacheWrite: call.cacheWrite, output: call.output })) }));
-
-  return {
-    totals: { main, sub },
-    share: {
-      subOfInputSide: ratio(sub.inputSide, all.inputSide),
-      subOfUncached: ratio(sub.uncached, all.uncached),
-      subOfOutput: ratio(sub.output, totalOutput),
-      subOfCalls: ratio(sub.calls, all.calls),
-    },
-    cache: { main: cacheFor(mainCalls), sub: cacheFor(subCalls), byType: cacheByType },
-    perTask: computeTasks(corpus, calls),
-    byType,
-    byDepth,
-    byModel,
-    fanout: { histogram: fanoutHistogram, max: fanoutCounts.length ? Math.max(...fanoutCounts) : 0, p90: percentile(fanoutCounts, 0.9) },
-    heavyCalls: { count: heavy.length, share: ratio(heavy.length, subCalls.length), byType: heavyByType, bySession: heavyBySession },
-    nPlusOne,
-    retryLoops,
-    loops,
-    topAgents: sortedTop(agentStats, (agent) => agent.uncached, (agent) => agent.inputSide),
-    topTurns: sortedTop(turnStats, (turn) => turn.uncached, (turn) => turn.inputSide),
-    tokenFlow,
-  };
-}