@tangle-network/traces 0.23.1 → 1.0.0

package/dist/chunk-J2PM5SVN.js

@@ -0,0 +1,503 @@
+import {
+  classifyLiveActions,
+  createPipelineAccumulator,
+  ruleContext,
+  runRules
+} from "./chunk-7CBV5RJ2.js";
+import {
+  conformanceOfSpans,
+  skippedAnalyses
+} from "./chunk-TW43MR5D.js";
+import {
+  metadataOnly
+} from "./chunk-V2OCGX57.js";
+import {
+  exportBytes,
+  openDeterministicTraceStore
+} from "./chunk-3I4PJ5Q7.js";
+import {
+  CONTEXT_BANDS,
+  createSessionFactsReportBuilder,
+  modelCallMeasurement,
+  priceReport
+} from "./chunk-UVE5ANVV.js";
+import {
+  appendAll
+} from "./chunk-NZ5Y7UEL.js";
+
+// src/improvement.ts
+import { createHash } from "crypto";
+import { mkdir, writeFile } from "fs/promises";
+import { join, resolve } from "path";
+import { makeFinding } from "@tangle-network/agent-eval/analyst";
+
+// src/analyze.ts
+import {
+  buildDefaultAnalystRegistry
+} from "@tangle-network/agent-eval/analyst";
+var MIN_TOOL_CALLS_FOR_TOOL_SIGNALS = 10;
+var TOOL_SIGNALS = /* @__PURE__ */ new Set(["single-tool-dependency", "no-self-verification"]);
+var SHELL_TOOLS = /* @__PURE__ */ new Set(["bash", "exec_command", "exec", "shell", "local_shell", "run_shell_command", "terminal"]);
+function evidenceOf(finding, traceId) {
+  const ref = finding.evidence_refs.find((evidence) => evidence.uri.includes(`/trace/${encodeURIComponent(traceId)}/`));
+  if (!ref?.excerpt) return null;
+  try {
+    return JSON.parse(ref.excerpt);
+  } catch {
+    return null;
+  }
+}
+function toolSignalHolds(finding, traceId, checked) {
+  const evidence = evidenceOf(finding, traceId);
+  const calls = evidence?.calls ?? evidence?.tool_calls;
+  if (typeof calls !== "number" || calls < MIN_TOOL_CALLS_FOR_TOOL_SIGNALS) return false;
+  if (finding.subject === "single-tool-dependency") return !(typeof evidence?.tool === "string" && SHELL_TOOLS.has(evidence.tool.toLowerCase()));
+  return !checked.has(traceId);
+}
+function withHeldToolSignals(result, checked) {
+  const findings = result.findings.flatMap((finding) => {
+    if (!finding.subject || !TOOL_SIGNALS.has(finding.subject)) return [finding];
+    const traces = Array.isArray(finding.metadata?.evidence_trace_ids) ? finding.metadata.evidence_trace_ids : [];
+    const kept = traces.filter((traceId) => toolSignalHolds(finding, traceId, checked));
+    if (kept.length === 0) return [];
+    if (kept.length === traces.length) return [finding];
+    return [{
+      ...finding,
+      claim: finding.claim.replace(/^\d+\//u, `${kept.length}/`),
+      evidence_refs: finding.evidence_refs.filter((evidence) => kept.some((traceId) => evidence.uri.includes(`/trace/${encodeURIComponent(traceId)}/`))),
+      metadata: { ...finding.metadata, evidence_trace_ids: kept, observed_trace_count: kept.length }
+    }];
+  });
+  const dropped = result.findings.length - findings.length;
+  return dropped === 0 ? result : {
+    ...result,
+    findings,
+    per_analyst: result.per_analyst.map((summary) => ({
+      ...summary,
+      findings_count: findings.filter((finding) => finding.analyst_id === summary.analyst_id).length
+    }))
+  };
+}
+var BEHAVIORAL_DEFAULT_MAX_TRACES = 1e3;
+function createSpanAnalysis(opts = {}) {
+  const chunks = [];
+  const traces = /* @__PURE__ */ new Set();
+  const checked = /* @__PURE__ */ new Set();
+  return {
+    add(spans) {
+      if (spans.length === 0) return;
+      chunks.push(exportBytes(opts.registry ? spans : metadataOnly(spans).spans));
+      if (opts.registry) return;
+      const byTrace2 = /* @__PURE__ */ new Map();
+      for (const span of spans) {
+        traces.add(span.trace_id);
+        const group = byTrace2.get(span.trace_id);
+        if (group) group.push(span);
+        else byTrace2.set(span.trace_id, [span]);
+      }
+      for (const [traceId, group] of byTrace2) if (classifyLiveActions(group).some((action) => action.kind === "verify")) checked.add(traceId);
+    },
+    async run() {
+      if (chunks.length === 0) throw new Error("analyzeSpans: no spans to analyze");
+      opts.signal?.throwIfAborted();
+      const runId = opts.runId ?? `traces-${Date.now()}`;
+      const store = openDeterministicTraceStore(chunks.length === 1 ? chunks[0] : Buffer.concat(chunks));
+      chunks.length = 0;
+      if (opts.registry) {
+        const result2 = await opts.registry.run(runId, { traceStore: store }, { signal: opts.signal });
+        opts.signal?.throwIfAborted();
+        return { result: result2 };
+      }
+      opts.signal?.throwIfAborted();
+      const registry = buildDefaultAnalystRegistry({
+        registry: { log: opts.log },
+        behavioral: { maxTraces: Math.max(BEHAVIORAL_DEFAULT_MAX_TRACES, traces.size) }
+      });
+      const result = await registry.run(runId, { traceStore: store }, { signal: opts.signal });
+      opts.signal?.throwIfAborted();
+      return { result: withHeldToolSignals(result, checked) };
+    }
+  };
+}
+async function analyzeSpans(spans, opts = {}) {
+  if (spans.length === 0) throw new Error("analyzeSpans: no spans to analyze");
+  opts.signal?.throwIfAborted();
+  const analysis = createSpanAnalysis(opts);
+  analysis.add(spans);
+  return analysis.run();
+}
+
+// src/improvement.ts
+function spanEvidenceUri(traceId, spanId) {
+  return `trace://${encodeURIComponent(traceId)}/span/${encodeURIComponent(spanId)}`;
+}
+var severityRank = {
+  critical: 0,
+  high: 1,
+  medium: 2,
+  low: 3,
+  info: 4
+};
+function rankFindings(findings) {
+  return [...findings].sort((a, b) => severityRank[a.severity] - severityRank[b.severity] || a.area.localeCompare(b.area) || a.claim.localeCompare(b.claim));
+}
+function renderFindingPacket(packet, title = "Trace finding packet") {
+  const lines = [`# ${title}`, ""];
+  lines.push(`${packet.findings.length} evidence-backed finding(s) with actions and checks.`);
+  lines.push("");
+  if (packet.findings.length > 0) {
+    lines.push("## findings");
+    lines.push("");
+    for (const finding of rankFindings(packet.findings)) {
+      lines.push(`### ${finding.severity.toUpperCase()} \u2014 ${finding.claim}`);
+      lines.push("");
+      lines.push(`- **Area:** ${finding.area}`);
+      lines.push(`- **Analyst:** ${finding.analyst_id}`);
+      if (finding.recommended_action) lines.push(`- **Action:** ${finding.recommended_action}`);
+      if (finding.validation_plan) lines.push(`- **Check:** ${finding.validation_plan}`);
+      for (const ref of finding.evidence_refs.slice(0, 3)) {
+        lines.push(`- **Evidence:** ${ref.kind} ${ref.uri}${ref.excerpt ? ` \u2014 \`${ref.excerpt.slice(0, 180)}\`` : ""}`);
+      }
+      lines.push("");
+    }
+  }
+  return lines.join("\n");
+}
+function buildTraceFindingPacket(opts) {
+  const generatedAt = opts.generatedAt ?? (/* @__PURE__ */ new Date()).toISOString();
+  const packet = {
+    schemaVersion: 1,
+    kind: "traces.finding_packet",
+    generatedAt,
+    source: opts.source ?? "traces",
+    findings: opts.findings
+  };
+  return { ...packet, report: renderFindingPacket(packet, opts.title) };
+}
+var RULE_AREA = {
+  "same-failing-command": "tool-use",
+  "verification-without-change": "verification",
+  "high-tool-error-rate": "tool-use"
+};
+var DEMOTED = /* @__PURE__ */ new Set(["no-self-verification"]);
+var LOWER = {
+  critical: "high",
+  high: "medium",
+  medium: "low",
+  low: "low",
+  info: "info"
+};
+function measuredSeverity(finding) {
+  return finding.subject !== void 0 && DEMOTED.has(finding.subject) ? { ...finding, severity: LOWER[finding.severity] } : finding;
+}
+function ruleFinding(live, traceId) {
+  const spanIds = [...new Set(live.evidence.flatMap((item) => item.spanIds ?? []))];
+  return makeFinding({
+    analyst_id: "traces-rules",
+    area: RULE_AREA[live.ruleId] ?? "tool-use",
+    subject: live.ruleId,
+    claim: live.claim,
+    severity: live.severity,
+    rationale: live.title,
+    evidence_refs: [
+      ...spanIds.slice(0, 3).map((spanId) => ({
+        kind: "span",
+        uri: spanEvidenceUri(traceId, spanId),
+        excerpt: live.evidence.find((item) => item.spanIds?.includes(spanId))?.value.slice(0, 240) ?? live.title
+      })),
+      ...live.evidence.filter((item) => item.kind === "metric").map((item) => ({
+        kind: "metric",
+        uri: `metric://trace/${encodeURIComponent(traceId)}/rules/${live.ruleId}/${item.label}`,
+        excerpt: item.value
+      }))
+    ],
+    recommended_action: live.action,
+    validation_plan: live.check,
+    confidence: 1,
+    metadata: { source: "traces.rules", ruleId: live.ruleId, sessionId: live.session.sessionId, traceId, spanIds },
+    id_basis: `${traceId}:${live.fingerprint}`
+  });
+}
+function terminalFailureFinding(pipelines, spans) {
+  const failedRuns = pipelines.failureClusters.totalFailures;
+  if (failedRuns === 0) return void 0;
+  const top = pipelines.failureClusters.clusters[0];
+  const root = spans.find((item) => item.parent_span_id === null && item.status.code === "ERROR" && (!top?.exampleRunId || item.trace_id === top.exampleRunId));
+  return makeFinding({
+    analyst_id: "traces-deterministic",
+    area: "reliability",
+    subject: "terminal-failure",
+    claim: `${failedRuns}/${pipelines.failureClusters.totalRuns} session(s) ended in an error`,
+    severity: "high",
+    rationale: "A session whose root span failed did not finish its work.",
+    evidence_refs: [
+      ...root ? [{ kind: "span", uri: spanEvidenceUri(root.trace_id, root.span_id), excerpt: `${root.name}: ${root.status.message ?? "root span ended with ERROR"}`.slice(0, 240) }] : [],
+      {
+        kind: "metric",
+        uri: `metric://pipelines/failure_cluster/${top?.failureClass ?? "unknown"}`,
+        excerpt: top ? `${top.runCount} session(s); example: ${top.exampleError ?? "error details not captured"}` : `${failedRuns} failed session(s)`
+      }
+    ],
+    recommended_action: "Read the error the session ended on and fix its cause before rerunning the task.",
+    validation_plan: "Rerun the task and require the session to end without an error.",
+    confidence: 1,
+    metadata: { source: "agent-eval.failureClusterView", failedRuns, totalRuns: pipelines.failureClusters.totalRuns, topFailureClass: top?.failureClass }
+  });
+}
+var ERROR_TEXT_CAP = 80;
+var CLUSTERS_KEPT = 5;
+function errorLine(span) {
+  const output = span.attributes["output.value"];
+  const text = span.status.message ?? (typeof output === "string" ? output : "");
+  const line = text.split("\n").map((part) => part.replace(/\s+/gu, " ").trim()).find((part) => part.length > 0) ?? "no error text recorded";
+  return line.length > ERROR_TEXT_CAP ? `${line.slice(0, ERROR_TEXT_CAP - 1)}\u2026` : line;
+}
+function toolErrorsOf(sessionId, tools, errored) {
+  const clusters = /* @__PURE__ */ new Map();
+  for (const span of errored) {
+    const tool = typeof span.attributes["tool.name"] === "string" ? span.attributes["tool.name"] : span.name.replace(/^tool\./u, "");
+    const error = errorLine(span);
+    const key = `${tool}\0${error}`;
+    const cluster = clusters.get(key) ?? { tool, error, count: 0, spanIds: [] };
+    cluster.count += 1;
+    if (cluster.spanIds.length < 3) cluster.spanIds.push(span.span_id);
+    clusters.set(key, cluster);
+  }
+  return {
+    sessionId,
+    calls: tools.length,
+    errors: errored.length,
+    clusters: [...clusters.values()].sort((a, b) => b.count - a.count || a.tool.localeCompare(b.tool)).slice(0, CLUSTERS_KEPT)
+  };
+}
+function byTrace(spans) {
+  const groups = /* @__PURE__ */ new Map();
+  for (const span of spans) {
+    const group = groups.get(span.trace_id);
+    if (group) group.push(span);
+    else groups.set(span.trace_id, [span]);
+  }
+  return groups;
+}
+var FACT_FIELDS = ["cwd", "repo", "toolCalls", "subagents", "pullRequests", "humanTurns", "finalMessages", "changedFiles", "firstRecordAt", "lastRecordAt"];
+function factsCannotAnswer(facts) {
+  const sessionId = facts.sessionId ?? facts.traceId;
+  const out = [];
+  for (const field of FACT_FIELDS) {
+    const fact = facts[field];
+    if (fact.value === null && fact.unavailable) out.push({ analysis: `facts.${field}`, reason: fact.unavailable, sessionId });
+  }
+  if (facts.tokens.input === null && facts.tokens.unavailable) out.push({ analysis: "facts.tokens", reason: facts.tokens.unavailable, sessionId });
+  else if (facts.cost.usd === null && facts.cost.unavailable) out.push({ analysis: "facts.cost", reason: facts.cost.unavailable, sessionId });
+  if (facts.firstFailure.status === "unknown") out.push({ analysis: "facts.firstFailure", reason: facts.firstFailure.reason, sessionId });
+  const waste = facts.measurements.waste;
+  if (waste.minutes === null && waste.unavailable) out.push({ analysis: "facts.measurements.waste", reason: waste.unavailable, sessionId });
+  return out;
+}
+function argumentsCannotAnswer(pipelines) {
+  const calls = pipelines.toolUse.reduce((total, run) => total + run.totalCalls, 0);
+  const captured = pipelines.toolUse.reduce((total, run) => total + run.callsWithCapturedArgs, 0);
+  if (calls === 0 || captured === calls) return [];
+  return [{
+    analysis: "argument comparisons: repeated commands, loops, blind retries and failing streaks",
+    reason: `${calls - captured}/${calls} tool call(s) kept no arguments, so whether two calls were the same cannot be compared`
+  }];
+}
+var CONFORMANCE_SPANS_READ = 25e4;
+var CONFORMANCE_TEXT_KEPT = 256;
+function conformanceView(span) {
+  let attributes;
+  for (const [key, value] of Object.entries(span.attributes)) {
+    if (typeof value !== "string" || value.length <= CONFORMANCE_TEXT_KEPT) continue;
+    attributes ??= { ...span.attributes };
+    attributes[key] = `sha256:${createHash("sha256").update(value).digest("hex")}`;
+  }
+  return attributes ? { ...span, attributes } : span;
+}
+var perThousand = (count, calls) => Math.round(count * 1e3 / calls * 100) / 100;
+function guardRefusalRates(toolCalls, byGuard) {
+  const refusals = [...byGuard.values()].reduce((sum, count) => sum + count, 0);
+  return {
+    toolCalls,
+    refusals,
+    perThousandCalls: toolCalls === 0 ? null : perThousand(refusals, toolCalls),
+    byGuard: [...byGuard].sort(([a, x], [b, y]) => y - x || a.localeCompare(b)).map(([guard, count]) => ({ guard, refusals: count, perThousandCalls: toolCalls === 0 ? 0 : perThousand(count, toolCalls) }))
+  };
+}
+function createTraceInvestigation(opts) {
+  const generatedAt = opts.generatedAt ?? (/* @__PURE__ */ new Date()).toISOString();
+  let selected;
+  const contexts = [];
+  const latencies = Array.from({ length: CONTEXT_BANDS }, () => []);
+  const builder = createSessionFactsReportBuilder({
+    ...opts.facts,
+    harness: opts.harness,
+    generatedAt,
+    onSession: (reading) => {
+      selected = reading.facts;
+      appendAll(contexts, reading.operating.contextTokens);
+      reading.operating.latencyMs.forEach((values, band) => appendAll(latencies[band], values));
+    }
+  });
+  let toolCalls = 0;
+  const byGuard = /* @__PURE__ */ new Map();
+  const limitKinds = /* @__PURE__ */ new Map();
+  let terminalFailures = 0;
+  let terminalAtLimit = 0;
+  const pipelines = createPipelineAccumulator({ minLoopOccurrences: opts.minLoopOccurrences });
+  const analysis = createSpanAnalysis({
+    ...opts.registry ? { registry: opts.registry } : {},
+    runId: `traces-analysis-${Date.parse(generatedAt) || Date.now()}`,
+    ...opts.signal ? { signal: opts.signal } : {},
+    ...opts.log ? { log: opts.log } : {}
+  });
+  const rules = /* @__PURE__ */ new Map();
+  const failedRoots = [];
+  const conformanceSpans = [];
+  let spanCount = 0;
+  let empty = true;
+  return {
+    async add(spans, batch = {}) {
+      opts.signal?.throwIfAborted();
+      if (spans.length === 0) return;
+      empty = false;
+      spanCount += spans.length;
+      const groups = byTrace(spans);
+      for (const [traceId, group] of groups) {
+        selected = void 0;
+        builder.add(group);
+        const sheet = selected;
+        const failed = group.find((item) => item.parent_span_id === null && item.status.code === "ERROR");
+        if (failed) failedRoots.push({ ...failed, attributes: {} });
+        if (!sheet) continue;
+        const { guardRefusals, limitStop } = sheet.measurements;
+        toolCalls += sheet.toolCalls.value ?? 0;
+        for (const [guard, count] of Object.entries(guardRefusals.byGuard)) byGuard.set(guard, (byGuard.get(guard) ?? 0) + count);
+        if (limitStop) limitKinds.set(limitStop.kind, (limitKinds.get(limitStop.kind) ?? 0) + 1);
+        if (failed) {
+          terminalFailures += 1;
+          if (limitStop) terminalAtLimit += 1;
+        }
+        const sessionId = sheet.sessionId ?? sheet.traceId;
+        const context = ruleContext(group, {
+          session: { harness: sheet.harness ?? batch.harness ?? opts.harness, sessionId, cwd: sheet.cwd.value },
+          generatedAt,
+          facts: [sheet]
+        });
+        rules.set(traceId, {
+          findings: runRules(context).map((live) => ruleFinding(live, sheet.traceId)),
+          toolErrors: toolErrorsOf(sessionId, context.tools, context.erroredTools)
+        });
+      }
+      await pipelines.add(spans);
+      analysis.add(spans);
+      if (!opts.conformance) {
+        for (const span of spans) {
+          if (conformanceSpans.length >= CONFORMANCE_SPANS_READ) break;
+          conformanceSpans.push(conformanceView(span));
+        }
+      }
+    },
+    addUnread(entry) {
+      builder.addUnread(entry);
+    },
+    async finish(end = {}) {
+      if (empty) throw new Error("runTraceInvestigation: no spans to analyze");
+      opts.signal?.throwIfAborted();
+      const harness = end.harness ?? opts.harness;
+      for (const entry of opts.unread ?? []) builder.addUnread(entry);
+      const facts = await priceReport({ ...builder.build(), harness });
+      opts.signal?.throwIfAborted();
+      const measures = pipelines.report();
+      const findings = [];
+      const toolErrors = [];
+      for (const sheet of facts.sessions) {
+        const held = rules.get(sheet.traceId);
+        if (!held) continue;
+        appendAll(findings, held.findings);
+        toolErrors.push(held.toolErrors);
+      }
+      const terminal = terminalFailureFinding(measures, failedRoots);
+      if (terminal) findings.push(terminal);
+      const result = await analysis.run();
+      appendAll(findings, result.result.findings);
+      for (const summary of result.result.per_analyst) {
+        if (summary.status !== "ok") opts.log?.(`[analyst] ${summary.status.toUpperCase()} ${summary.analyst_id}`, { error: summary.error?.message ?? summary.reason });
+      }
+      let conformance = end.conformance ?? opts.conformance;
+      if (!conformance) {
+        const views = conformanceSpans;
+        if (spanCount > views.length) views.length = spanCount;
+        conformance = conformanceOfSpans(views);
+      }
+      const cannotAnswer = [
+        ...skippedAnalyses(conformance).map((skipped) => ({ analysis: `${skipped.capability}: ${skipped.analysis}`, reason: skipped.reason })),
+        ...argumentsCannotAnswer(measures),
+        ...result.result.per_analyst.filter((summary) => summary.status !== "ok").map((summary) => ({ analysis: `analyst ${summary.analyst_id}`, reason: summary.error?.message ?? summary.reason ?? summary.status })),
+        ...(facts.selection?.unreadSessions ?? []).map((entry) => ({
+          analysis: "every question about this session",
+          reason: entry.reason,
+          sessionId: entry.sessionId ?? entry.traceId ?? entry.source ?? "unknown"
+        })),
+        ...facts.sessions.flatMap(factsCannotAnswer)
+      ];
+      return {
+        schemaVersion: 1,
+        kind: "traces.analysis",
+        generatedAt,
+        harness,
+        facts,
+        packet: buildTraceFindingPacket({
+          findings: rankFindings(findings.map(measuredSeverity)),
+          generatedAt,
+          source: "traces analyze",
+          title: "traces analyze findings"
+        }),
+        measures: {
+          ...measures,
+          toolErrors,
+          operating: {
+            modelCalls: modelCallMeasurement({ contextTokens: contexts, latencyMs: latencies }),
+            guardRefusals: guardRefusalRates(toolCalls, byGuard),
+            limitStops: {
+              sessions: [...limitKinds.values()].reduce((sum, count) => sum + count, 0),
+              byKind: Object.fromEntries([...limitKinds].sort(([a], [b]) => a.localeCompare(b))),
+              terminalFailures,
+              terminalFailuresAtLimit: terminalAtLimit
+            }
+          }
+        },
+        conformance,
+        cannotAnswer
+      };
+    }
+  };
+}
+async function runTraceInvestigation(opts) {
+  if (opts.spans.length === 0) throw new Error("runTraceInvestigation: no spans to analyze");
+  const { spans, ...rest } = opts;
+  const investigation = createTraceInvestigation(rest);
+  await investigation.add(spans);
+  return investigation.finish();
+}
+async function writeAnalysisArtifacts(report, packet, outDir) {
+  const directory = resolve(outDir);
+  await mkdir(directory, { recursive: true });
+  const paths = { directory, report: join(directory, "report.md"), findings: join(directory, "findings.json") };
+  await Promise.all([
+    writeFile(paths.report, report, "utf8"),
+    writeFile(paths.findings, `${JSON.stringify(packet, null, 2)}
+`, "utf8")
+  ]);
+  return paths;
+}
+
+export {
+  analyzeSpans,
+  spanEvidenceUri,
+  rankFindings,
+  buildTraceFindingPacket,
+  createTraceInvestigation,
+  runTraceInvestigation,
+  writeAnalysisArtifacts
+};