@tangle-network/agent-eval 0.20.12 → 0.21.0

package/dist/chunk-SNUHRBDL.js

@@ -0,0 +1,154 @@
+// src/trace/raw-provider-sink.ts
+import { promises as fs } from "fs";
+import * as path from "path";
+var REDACTED_HEADER_NAMES = /* @__PURE__ */ new Set([
+  "authorization",
+  "x-api-key",
+  "x-auth-token",
+  "cookie",
+  "set-cookie",
+  "proxy-authorization"
+]);
+var REDACTED_BODY_KEY = /^(api[_-]?key|bearer|password|secret|token|access[_-]?token|refresh[_-]?token)$/i;
+function defaultProviderRedactor(event) {
+  const redactedFields = [...event.redactedFields ?? []];
+  const requestHeaders = redactHeaders(event.requestHeaders, "request", redactedFields);
+  const responseHeaders = redactHeaders(event.responseHeaders, "response", redactedFields);
+  const requestBody = redactBody(event.requestBody, "requestBody", redactedFields);
+  const responseBody = redactBody(event.responseBody, "responseBody", redactedFields);
+  return { ...event, requestHeaders, responseHeaders, requestBody, responseBody, redactedFields };
+}
+function redactHeaders(headers, prefix, redactedFields) {
+  if (!headers) return headers;
+  const out = {};
+  for (const [k, v] of Object.entries(headers)) {
+    if (REDACTED_HEADER_NAMES.has(k.toLowerCase())) {
+      redactedFields.push(`${prefix}Headers.${k}`);
+      continue;
+    }
+    out[k] = v;
+  }
+  return out;
+}
+function redactBody(value, pathStr, redactedFields) {
+  if (value == null) return value;
+  if (Array.isArray(value)) return value.map((v, i) => redactBody(v, `${pathStr}[${i}]`, redactedFields));
+  if (typeof value === "object") {
+    const out = {};
+    for (const [k, v] of Object.entries(value)) {
+      if (REDACTED_BODY_KEY.test(k)) {
+        redactedFields.push(`${pathStr}.${k}`);
+        continue;
+      }
+      out[k] = redactBody(v, `${pathStr}.${k}`, redactedFields);
+    }
+    return out;
+  }
+  return value;
+}
+var InMemoryRawProviderSink = class {
+  events = [];
+  redactor;
+  constructor(opts = {}) {
+    this.redactor = opts.redactor ?? defaultProviderRedactor;
+  }
+  async record(event) {
+    this.events.push(this.redactor({ ...event, redactedFields: event.redactedFields ?? [] }));
+  }
+  async list(filter = {}) {
+    return this.events.filter(
+      (e) => (filter.runId === void 0 || e.runId === filter.runId) && (filter.spanId === void 0 || e.spanId === filter.spanId) && (filter.direction === void 0 || e.direction === filter.direction) && (filter.attemptIndex === void 0 || e.attemptIndex === filter.attemptIndex)
+    );
+  }
+  size() {
+    return this.events.length;
+  }
+};
+var NoopRawProviderSink = class {
+  async record() {
+  }
+};
+var FileSystemRawProviderSink = class {
+  dir;
+  fileName;
+  rollAtBytes;
+  redactor;
+  bytesWritten = 0;
+  rollIndex = 0;
+  initPromise = null;
+  constructor(opts) {
+    this.dir = opts.dir;
+    this.fileName = opts.fileName ?? "raw-provider-events.ndjson";
+    this.rollAtBytes = opts.rollAtBytes ?? 32 * 1024 * 1024;
+    this.redactor = opts.redactor ?? defaultProviderRedactor;
+  }
+  async ensureInit() {
+    if (!this.initPromise) {
+      this.initPromise = fs.mkdir(this.dir, { recursive: true }).then(() => void 0);
+    }
+    await this.initPromise;
+  }
+  currentPath() {
+    if (this.rollIndex === 0) return path.join(this.dir, this.fileName);
+    return path.join(this.dir, `${this.fileName}.${this.rollIndex}`);
+  }
+  async record(event) {
+    await this.ensureInit();
+    const redacted = this.redactor({ ...event, redactedFields: event.redactedFields ?? [] });
+    const line = JSON.stringify(redacted) + "\n";
+    if (this.bytesWritten + line.length > this.rollAtBytes && this.bytesWritten > 0) {
+      this.rollIndex += 1;
+      this.bytesWritten = 0;
+    }
+    await fs.appendFile(this.currentPath(), line, "utf8");
+    this.bytesWritten += line.length;
+  }
+  async list(filter = {}) {
+    await this.ensureInit();
+    const out = [];
+    for (let i = 0; i <= this.rollIndex; i++) {
+      const file = i === 0 ? path.join(this.dir, this.fileName) : path.join(this.dir, `${this.fileName}.${i}`);
+      let body;
+      try {
+        body = await fs.readFile(file, "utf8");
+      } catch (err) {
+        if (err.code === "ENOENT") continue;
+        throw err;
+      }
+      for (const line of body.split("\n")) {
+        if (!line) continue;
+        const event = JSON.parse(line);
+        if (filter.runId !== void 0 && event.runId !== filter.runId) continue;
+        if (filter.spanId !== void 0 && event.spanId !== filter.spanId) continue;
+        if (filter.direction !== void 0 && event.direction !== filter.direction) continue;
+        if (filter.attemptIndex !== void 0 && event.attemptIndex !== filter.attemptIndex) continue;
+        out.push(event);
+      }
+    }
+    return out;
+  }
+};
+function providerFromBaseUrl(baseUrl) {
+  const lower = baseUrl.toLowerCase();
+  if (lower.includes("api.openai.com")) return "openai";
+  if (lower.includes("api.anthropic.com")) return "anthropic";
+  if (lower.includes("generativelanguage.googleapis.com")) return "google";
+  if (lower.includes("api.together.ai") || lower.includes("api.together.xyz")) return "together";
+  if (lower.includes("api.deepseek.com")) return "deepseek";
+  if (lower.includes("router.tangle.tools")) return "tangle-router";
+  if (lower.includes("api.litellm") || lower.includes("litellm")) return "litellm";
+  try {
+    return new URL(baseUrl).host;
+  } catch {
+    return baseUrl;
+  }
+}
+
+export {
+  defaultProviderRedactor,
+  InMemoryRawProviderSink,
+  NoopRawProviderSink,
+  FileSystemRawProviderSink,
+  providerFromBaseUrl
+};
+//# sourceMappingURL=chunk-SNUHRBDL.js.map

package/dist/chunk-SNUHRBDL.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/trace/raw-provider-sink.ts"],"sourcesContent":["/**\n * RawProviderSink — first-class persistence for the actual HTTP-level\n * request/response bodies of every LLM provider call.\n *\n * Why this is a separate sink from the structured `LlmSpan`:\n *\n *   - `LlmSpan` records the *intent* — model name, messages, output text,\n *     usage. It's what dashboards read; it's NOT enough for forensics.\n *   - When a downstream consumer reports \"the verifier used the wrong route\"\n *     or \"tokens look right but reasoning was missing,\" the only way to\n *     answer is the raw HTTP body. Span fields can lie (a proxy can echo\n *     a different `model` value than what actually answered); the raw\n *     response is ground truth.\n *\n * Default behaviour: opt-in. Pass `rawSink` to `LlmClientOptions` (or the\n * matrix runner / BuilderSession sets it up automatically) and every\n * request, response, and error is recorded — including retries, with the\n * attempt index attached so a flaky call's full event chain is recoverable.\n *\n * Redaction is enforced at sink time. The default redactor strips\n * `Authorization`, `X-Api-Key`, `X-Auth-Token`, `Cookie` headers and any\n * payload field whose key matches `apiKey | api_key | bearer | password |\n * secret | token` (case-insensitive). Override via the sink constructor or\n * the per-call `redactor`. The `redactedFields` array on the persisted\n * event lets a reviewer see what was stripped without exposing the values.\n */\n\nimport { promises as fs } from 'node:fs'\nimport * as path from 'node:path'\n\nexport type RawProviderDirection = 'request' | 'response' | 'error'\n\nexport interface RawProviderEvent {\n  /** Stable id. Generated by the sink if omitted. */\n  eventId: string\n  /** Trace context populated by `LlmClient` when the call is wrapped in a span. */\n  runId?: string\n  spanId?: string\n  /**\n   * Logical provider name. Free-form so callers can use whatever id matches\n   * their topology (`'openai'`, `'anthropic'`, `'tangle-router'`, …). When\n   * omitted, derived from `baseUrl` in `LlmClientOptions`.\n   */\n  provider: string\n  model: string\n  /** Endpoint path, e.g. `'/v1/chat/completions'`. */\n  endpoint: string\n  /** Base URL used for the call (already-normalised — no trailing slash). */\n  baseUrl: string\n  /** 0-indexed retry attempt. The first attempt is 0; a retried call gets 1, 2, … */\n  attemptIndex: number\n  direction: RawProviderDirection\n  /** Unix ms. */\n  timestamp: number\n  /** Wall-clock duration of the call leg. Set on `response` and `error` events; null on `request`. */\n  durationMs?: number\n  statusCode?: number\n  requestHeaders?: Record<string, string>\n  requestBody?: unknown\n  responseHeaders?: Record<string, string>\n  responseBody?: unknown\n  /** Set on `direction: 'error'` events. */\n  errorMessage?: string\n  /** Field paths the redactor stripped from this event ('header:Authorization', 'body.apiKey', …). */\n  redactedFields: string[]\n}\n\nexport interface RawProviderSinkFilter {\n  runId?: string\n  spanId?: string\n  direction?: RawProviderDirection\n  attemptIndex?: number\n}\n\nexport interface RawProviderSink {\n  record(event: RawProviderEvent): Promise<void>\n  /** Optional listing — implementations that durably persist (file, db) should support this. */\n  list?(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>\n  /** Optional teardown for backed implementations. */\n  close?(): Promise<void>\n}\n\nexport type ProviderRedactor = (event: RawProviderEvent) => RawProviderEvent\n\nconst REDACTED_HEADER_NAMES = new Set([\n  'authorization',\n  'x-api-key',\n  'x-auth-token',\n  'cookie',\n  'set-cookie',\n  'proxy-authorization',\n])\n\nconst REDACTED_BODY_KEY = /^(api[_-]?key|bearer|password|secret|token|access[_-]?token|refresh[_-]?token)$/i\n\n/**\n * Default redactor — strips well-known auth headers and any body field whose\n * key matches the credential pattern. Records every redacted path on\n * `event.redactedFields` so a downstream reviewer can see what was removed.\n */\nexport function defaultProviderRedactor(event: RawProviderEvent): RawProviderEvent {\n  const redactedFields: string[] = [...(event.redactedFields ?? [])]\n  const requestHeaders = redactHeaders(event.requestHeaders, 'request', redactedFields)\n  const responseHeaders = redactHeaders(event.responseHeaders, 'response', redactedFields)\n  const requestBody = redactBody(event.requestBody, 'requestBody', redactedFields)\n  const responseBody = redactBody(event.responseBody, 'responseBody', redactedFields)\n  return { ...event, requestHeaders, responseHeaders, requestBody, responseBody, redactedFields }\n}\n\nfunction redactHeaders(\n  headers: Record<string, string> | undefined,\n  prefix: 'request' | 'response',\n  redactedFields: string[],\n): Record<string, string> | undefined {\n  if (!headers) return headers\n  const out: Record<string, string> = {}\n  for (const [k, v] of Object.entries(headers)) {\n    if (REDACTED_HEADER_NAMES.has(k.toLowerCase())) {\n      redactedFields.push(`${prefix}Headers.${k}`)\n      continue\n    }\n    out[k] = v\n  }\n  return out\n}\n\nfunction redactBody(\n  value: unknown,\n  pathStr: string,\n  redactedFields: string[],\n): unknown {\n  if (value == null) return value\n  if (Array.isArray(value)) return value.map((v, i) => redactBody(v, `${pathStr}[${i}]`, redactedFields))\n  if (typeof value === 'object') {\n    const out: Record<string, unknown> = {}\n    for (const [k, v] of Object.entries(value as Record<string, unknown>)) {\n      if (REDACTED_BODY_KEY.test(k)) {\n        redactedFields.push(`${pathStr}.${k}`)\n        continue\n      }\n      out[k] = redactBody(v, `${pathStr}.${k}`, redactedFields)\n    }\n    return out\n  }\n  return value\n}\n\n// ── In-memory ────────────────────────────────────────────────────────────\n\nexport interface InMemoryRawProviderSinkOptions {\n  redactor?: ProviderRedactor\n}\n\nexport class InMemoryRawProviderSink implements RawProviderSink {\n  private events: RawProviderEvent[] = []\n  private redactor: ProviderRedactor\n\n  constructor(opts: InMemoryRawProviderSinkOptions = {}) {\n    this.redactor = opts.redactor ?? defaultProviderRedactor\n  }\n\n  async record(event: RawProviderEvent): Promise<void> {\n    this.events.push(this.redactor({ ...event, redactedFields: event.redactedFields ?? [] }))\n  }\n\n  async list(filter: RawProviderSinkFilter = {}): Promise<RawProviderEvent[]> {\n    return this.events.filter((e) =>\n      (filter.runId === undefined || e.runId === filter.runId) &&\n      (filter.spanId === undefined || e.spanId === filter.spanId) &&\n      (filter.direction === undefined || e.direction === filter.direction) &&\n      (filter.attemptIndex === undefined || e.attemptIndex === filter.attemptIndex),\n    )\n  }\n\n  size(): number { return this.events.length }\n}\n\nexport class NoopRawProviderSink implements RawProviderSink {\n  async record(): Promise<void> { /* no-op */ }\n}\n\n// ── Filesystem (NDJSON) ──────────────────────────────────────────────────\n\nexport interface FileSystemRawProviderSinkOptions {\n  /** Directory the NDJSON file is written into. Created if missing. */\n  dir: string\n  /** File name; default `'raw-provider-events.ndjson'`. */\n  fileName?: string\n  /** Bytes after which the writer rolls over to a new file (default 32 MiB). */\n  rollAtBytes?: number\n  redactor?: ProviderRedactor\n}\n\nexport class FileSystemRawProviderSink implements RawProviderSink {\n  private dir: string\n  private fileName: string\n  private rollAtBytes: number\n  private redactor: ProviderRedactor\n  private bytesWritten = 0\n  private rollIndex = 0\n  private initPromise: Promise<void> | null = null\n\n  constructor(opts: FileSystemRawProviderSinkOptions) {\n    this.dir = opts.dir\n    this.fileName = opts.fileName ?? 'raw-provider-events.ndjson'\n    this.rollAtBytes = opts.rollAtBytes ?? 32 * 1024 * 1024\n    this.redactor = opts.redactor ?? defaultProviderRedactor\n  }\n\n  private async ensureInit(): Promise<void> {\n    if (!this.initPromise) {\n      this.initPromise = fs.mkdir(this.dir, { recursive: true }).then(() => undefined)\n    }\n    await this.initPromise\n  }\n\n  private currentPath(): string {\n    if (this.rollIndex === 0) return path.join(this.dir, this.fileName)\n    return path.join(this.dir, `${this.fileName}.${this.rollIndex}`)\n  }\n\n  async record(event: RawProviderEvent): Promise<void> {\n    await this.ensureInit()\n    const redacted = this.redactor({ ...event, redactedFields: event.redactedFields ?? [] })\n    const line = JSON.stringify(redacted) + '\\n'\n    if (this.bytesWritten + line.length > this.rollAtBytes && this.bytesWritten > 0) {\n      this.rollIndex += 1\n      this.bytesWritten = 0\n    }\n    await fs.appendFile(this.currentPath(), line, 'utf8')\n    this.bytesWritten += line.length\n  }\n\n  async list(filter: RawProviderSinkFilter = {}): Promise<RawProviderEvent[]> {\n    await this.ensureInit()\n    const out: RawProviderEvent[] = []\n    for (let i = 0; i <= this.rollIndex; i++) {\n      const file = i === 0\n        ? path.join(this.dir, this.fileName)\n        : path.join(this.dir, `${this.fileName}.${i}`)\n      let body: string\n      try {\n        body = await fs.readFile(file, 'utf8')\n      } catch (err) {\n        if ((err as NodeJS.ErrnoException).code === 'ENOENT') continue\n        throw err\n      }\n      for (const line of body.split('\\n')) {\n        if (!line) continue\n        const event = JSON.parse(line) as RawProviderEvent\n        if (filter.runId !== undefined && event.runId !== filter.runId) continue\n        if (filter.spanId !== undefined && event.spanId !== filter.spanId) continue\n        if (filter.direction !== undefined && event.direction !== filter.direction) continue\n        if (filter.attemptIndex !== undefined && event.attemptIndex !== filter.attemptIndex) continue\n        out.push(event)\n      }\n    }\n    return out\n  }\n}\n\n// ── Helpers ──────────────────────────────────────────────────────────────\n\n/**\n * Best-effort provider id from a base URL. Falls back to the URL host when\n * none of the well-known patterns match.\n */\nexport function providerFromBaseUrl(baseUrl: string): string {\n  const lower = baseUrl.toLowerCase()\n  if (lower.includes('api.openai.com')) return 'openai'\n  if (lower.includes('api.anthropic.com')) return 'anthropic'\n  if (lower.includes('generativelanguage.googleapis.com')) return 'google'\n  if (lower.includes('api.together.ai') || lower.includes('api.together.xyz')) return 'together'\n  if (lower.includes('api.deepseek.com')) return 'deepseek'\n  if (lower.includes('router.tangle.tools')) return 'tangle-router'\n  if (lower.includes('api.litellm') || lower.includes('litellm')) return 'litellm'\n  try {\n    return new URL(baseUrl).host\n  } catch {\n    return baseUrl\n  }\n}\n"],"mappings":";AA2BA,SAAS,YAAY,UAAU;AAC/B,YAAY,UAAU;AAwDtB,IAAM,wBAAwB,oBAAI,IAAI;AAAA,EACpC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,oBAAoB;AAOnB,SAAS,wBAAwB,OAA2C;AACjF,QAAM,iBAA2B,CAAC,GAAI,MAAM,kBAAkB,CAAC,CAAE;AACjE,QAAM,iBAAiB,cAAc,MAAM,gBAAgB,WAAW,cAAc;AACpF,QAAM,kBAAkB,cAAc,MAAM,iBAAiB,YAAY,cAAc;AACvF,QAAM,cAAc,WAAW,MAAM,aAAa,eAAe,cAAc;AAC/E,QAAM,eAAe,WAAW,MAAM,cAAc,gBAAgB,cAAc;AAClF,SAAO,EAAE,GAAG,OAAO,gBAAgB,iBAAiB,aAAa,cAAc,eAAe;AAChG;AAEA,SAAS,cACP,SACA,QACA,gBACoC;AACpC,MAAI,CAAC,QAAS,QAAO;AACrB,QAAM,MAA8B,CAAC;AACrC,aAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,OAAO,GAAG;AAC5C,QAAI,sBAAsB,IAAI,EAAE,YAAY,CAAC,GAAG;AAC9C,qBAAe,KAAK,GAAG,MAAM,WAAW,CAAC,EAAE;AAC3C;AAAA,IACF;AACA,QAAI,CAAC,IAAI;AAAA,EACX;AACA,SAAO;AACT;AAEA,SAAS,WACP,OACA,SACA,gBACS;AACT,MAAI,SAAS,KAAM,QAAO;AAC1B,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO,MAAM,IAAI,CAAC,GAAG,MAAM,WAAW,GAAG,GAAG,OAAO,IAAI,CAAC,KAAK,cAAc,CAAC;AACtG,MAAI,OAAO,UAAU,UAAU;AAC7B,UAAM,MAA+B,CAAC;AACtC,eAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,KAAgC,GAAG;AACrE,UAAI,kBAAkB,KAAK,CAAC,GAAG;AAC7B,uBAAe,KAAK,GAAG,OAAO,IAAI,CAAC,EAAE;AACrC;AAAA,MACF;AACA,UAAI,CAAC,IAAI,WAAW,GAAG,GAAG,OAAO,IAAI,CAAC,IAAI,cAAc;AAAA,IAC1D;AACA,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAQO,IAAM,0BAAN,MAAyD;AAAA,EACtD,SAA6B,CAAC;AAAA,EAC9B;AAAA,EAER,YAAY,OAAuC,CAAC,GAAG;AACrD,SAAK,WAAW,KAAK,YAAY;AAAA,EACnC;AAAA,EAEA,MAAM,OAAO,OAAwC;AACnD,SAAK,OAAO,KAAK,KAAK,SAAS,EAAE,GAAG,OAAO,gBAAgB,MAAM,kBAAkB,CAAC,EAAE,CAAC,CAAC;AAAA,EAC1F;AAAA,EAEA,MAAM,KAAK,SAAgC,CAAC,GAAgC;AAC1E,WAAO,KAAK,OAAO;AAAA,MAAO,CAAC,OACxB,OAAO,UAAU,UAAa,EAAE,UAAU,OAAO,WACjD,OAAO,WAAW,UAAa,EAAE,WAAW,OAAO,YACnD,OAAO,cAAc,UAAa,EAAE,cAAc,OAAO,eACzD,OAAO,iBAAiB,UAAa,EAAE,iBAAiB,OAAO;AAAA,IAClE;AAAA,EACF;AAAA,EAEA,OAAe;AAAE,WAAO,KAAK,OAAO;AAAA,EAAO;AAC7C;AAEO,IAAM,sBAAN,MAAqD;AAAA,EAC1D,MAAM,SAAwB;AAAA,EAAc;AAC9C;AAcO,IAAM,4BAAN,MAA2D;AAAA,EACxD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,eAAe;AAAA,EACf,YAAY;AAAA,EACZ,cAAoC;AAAA,EAE5C,YAAY,MAAwC;AAClD,SAAK,MAAM,KAAK;AAChB,SAAK,WAAW,KAAK,YAAY;AACjC,SAAK,cAAc,KAAK,eAAe,KAAK,OAAO;AACnD,SAAK,WAAW,KAAK,YAAY;AAAA,EACnC;AAAA,EAEA,MAAc,aAA4B;AACxC,QAAI,CAAC,KAAK,aAAa;AACrB,WAAK,cAAc,GAAG,MAAM,KAAK,KAAK,EAAE,WAAW,KAAK,CAAC,EAAE,KAAK,MAAM,MAAS;AAAA,IACjF;AACA,UAAM,KAAK;AAAA,EACb;AAAA,EAEQ,cAAsB;AAC5B,QAAI,KAAK,cAAc,EAAG,QAAY,UAAK,KAAK,KAAK,KAAK,QAAQ;AAClE,WAAY,UAAK,KAAK,KAAK,GAAG,KAAK,QAAQ,IAAI,KAAK,SAAS,EAAE;AAAA,EACjE;AAAA,EAEA,MAAM,OAAO,OAAwC;AACnD,UAAM,KAAK,WAAW;AACtB,UAAM,WAAW,KAAK,SAAS,EAAE,GAAG,OAAO,gBAAgB,MAAM,kBAAkB,CAAC,EAAE,CAAC;AACvF,UAAM,OAAO,KAAK,UAAU,QAAQ,IAAI;AACxC,QAAI,KAAK,eAAe,KAAK,SAAS,KAAK,eAAe,KAAK,eAAe,GAAG;AAC/E,WAAK,aAAa;AAClB,WAAK,eAAe;AAAA,IACtB;AACA,UAAM,GAAG,WAAW,KAAK,YAAY,GAAG,MAAM,MAAM;AACpD,SAAK,gBAAgB,KAAK;AAAA,EAC5B;AAAA,EAEA,MAAM,KAAK,SAAgC,CAAC,GAAgC;AAC1E,UAAM,KAAK,WAAW;AACtB,UAAM,MAA0B,CAAC;AACjC,aAAS,IAAI,GAAG,KAAK,KAAK,WAAW,KAAK;AACxC,YAAM,OAAO,MAAM,IACV,UAAK,KAAK,KAAK,KAAK,QAAQ,IAC5B,UAAK,KAAK,KAAK,GAAG,KAAK,QAAQ,IAAI,CAAC,EAAE;AAC/C,UAAI;AACJ,UAAI;AACF,eAAO,MAAM,GAAG,SAAS,MAAM,MAAM;AAAA,MACvC,SAAS,KAAK;AACZ,YAAK,IAA8B,SAAS,SAAU;AACtD,cAAM;AAAA,MACR;AACA,iBAAW,QAAQ,KAAK,MAAM,IAAI,GAAG;AACnC,YAAI,CAAC,KAAM;AACX,cAAM,QAAQ,KAAK,MAAM,IAAI;AAC7B,YAAI,OAAO,UAAU,UAAa,MAAM,UAAU,OAAO,MAAO;AAChE,YAAI,OAAO,WAAW,UAAa,MAAM,WAAW,OAAO,OAAQ;AACnE,YAAI,OAAO,cAAc,UAAa,MAAM,cAAc,OAAO,UAAW;AAC5E,YAAI,OAAO,iBAAiB,UAAa,MAAM,iBAAiB,OAAO,aAAc;AACrF,YAAI,KAAK,KAAK;AAAA,MAChB;AAAA,IACF;AACA,WAAO;AAAA,EACT;AACF;AAQO,SAAS,oBAAoB,SAAyB;AAC3D,QAAM,QAAQ,QAAQ,YAAY;AAClC,MAAI,MAAM,SAAS,gBAAgB,EAAG,QAAO;AAC7C,MAAI,MAAM,SAAS,mBAAmB,EAAG,QAAO;AAChD,MAAI,MAAM,SAAS,mCAAmC,EAAG,QAAO;AAChE,MAAI,MAAM,SAAS,iBAAiB,KAAK,MAAM,SAAS,kBAAkB,EAAG,QAAO;AACpF,MAAI,MAAM,SAAS,kBAAkB,EAAG,QAAO;AAC/C,MAAI,MAAM,SAAS,qBAAqB,EAAG,QAAO;AAClD,MAAI,MAAM,SAAS,aAAa,KAAK,MAAM,SAAS,SAAS,EAAG,QAAO;AACvE,MAAI;AACF,WAAO,IAAI,IAAI,OAAO,EAAE;AAAA,EAC1B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;","names":[]}

package/dist/{chunk-KWUAAIHR.js → chunk-WOK2RTWG.js}

@@ -497,6 +497,121 @@ function runToTraceId(run) {
   return cleaned.slice(0, 32).padEnd(32, "0");
 }
 
+// src/trace/integrity.ts
+var RunIntegrityError = class extends Error {
+  constructor(report) {
+    super(
+      `Run ${report.runId} failed integrity check: ${report.issues.map((i) => i.code).join(", ")}`
+    );
+    this.report = report;
+    this.name = "RunIntegrityError";
+  }
+  report;
+};
+async function assertRunCaptured(store, runId, expectations = {}) {
+  const issues = [];
+  const run = await store.getRun(runId);
+  if (!run) {
+    return {
+      ok: false,
+      runId,
+      llmSpanCount: 0,
+      judgeSpanCount: 0,
+      toolSpanCount: 0,
+      rawProviderEventCount: 0,
+      rawSpanCoverage: { covered: 0, total: 0 },
+      issues: [{ code: "no_run", message: `Run ${runId} not found in store.` }]
+    };
+  }
+  const spans = await store.spans({ runId });
+  const llmSpans2 = spans.filter((s) => s.kind === "llm");
+  const judgeSpans2 = spans.filter((s) => s.kind === "judge");
+  const toolSpans2 = spans.filter((s) => s.kind === "tool");
+  const llmMin = expectations.llmSpansMin ?? 0;
+  const judgeMin = expectations.judgeSpansMin ?? 0;
+  const toolMin = expectations.toolSpansMin ?? 0;
+  if (llmSpans2.length < llmMin) {
+    issues.push({
+      code: "missing_llm_spans",
+      message: `Expected \u2265 ${llmMin} LLM spans, found ${llmSpans2.length}.`,
+      detail: { expected: llmMin, found: llmSpans2.length }
+    });
+  }
+  if (judgeSpans2.length < judgeMin) {
+    issues.push({
+      code: "missing_judge_spans",
+      message: `Expected \u2265 ${judgeMin} judge spans, found ${judgeSpans2.length}.`,
+      detail: { expected: judgeMin, found: judgeSpans2.length }
+    });
+  }
+  if (toolSpans2.length < toolMin) {
+    issues.push({
+      code: "missing_tool_spans",
+      message: `Expected \u2265 ${toolMin} tool spans, found ${toolSpans2.length}.`,
+      detail: { expected: toolMin, found: toolSpans2.length }
+    });
+  }
+  let rawEventCount = 0;
+  let coverage = { covered: 0, total: llmSpans2.length };
+  if (expectations.rawSink) {
+    if (!expectations.rawSink.list) {
+      issues.push({
+        code: "no_raw_sink",
+        message: "Provided rawSink does not implement list(); cannot verify capture."
+      });
+    } else {
+      const events = await expectations.rawSink.list({ runId });
+      rawEventCount = events.length;
+      const rawMin = expectations.rawProviderEventsMin ?? 1;
+      if (rawEventCount < rawMin) {
+        issues.push({
+          code: "missing_raw_events",
+          message: `Expected \u2265 ${rawMin} raw provider events, found ${rawEventCount}.`,
+          detail: { expected: rawMin, found: rawEventCount }
+        });
+      }
+      if (expectations.requireRawCoverageOfLlmSpans) {
+        const requestEventsBySpan = new Set(
+          events.filter((e) => e.direction === "request" && e.spanId).map((e) => e.spanId)
+        );
+        const orphaned = llmSpans2.filter((s) => !requestEventsBySpan.has(s.spanId));
+        coverage = { covered: llmSpans2.length - orphaned.length, total: llmSpans2.length };
+        if (orphaned.length > 0) {
+          issues.push({
+            code: "orphan_llm_span",
+            message: `${orphaned.length} LLM span(s) have no matching raw provider request event.`,
+            detail: { orphanedSpanIds: orphaned.map((s) => s.spanId) }
+          });
+        }
+      }
+    }
+  } else if (expectations.requireRawCoverageOfLlmSpans || expectations.rawProviderEventsMin) {
+    issues.push({
+      code: "no_raw_sink",
+      message: "Raw coverage required but no rawSink supplied to the integrity check."
+    });
+  }
+  if (expectations.requireOutcome && (run.outcome === void 0 || run.outcome === null)) {
+    issues.push({
+      code: "missing_outcome",
+      message: `Run ${runId} has no outcome recorded.`
+    });
+  }
+  return {
+    ok: issues.length === 0,
+    runId,
+    llmSpanCount: llmSpans2.length,
+    judgeSpanCount: judgeSpans2.length,
+    toolSpanCount: toolSpans2.length,
+    rawProviderEventCount: rawEventCount,
+    rawSpanCoverage: coverage,
+    issues
+  };
+}
+function throwIfRunIncomplete(report) {
+  if (!report.ok) throw new RunIntegrityError(report);
+}
+
 // src/trace-analyst/types.ts
 var DEFAULT_TRACE_ANALYST_BUDGETS = {
   perCallByteCeiling: 15e4,
@@ -1468,6 +1583,43 @@ function normalizeRecordArray(value) {
   return value.map((item) => item && typeof item === "object" ? { ...item } : { value: item });
 }
 
+// src/trace-analyst/hook.ts
+var DEFAULT_QUESTION = "Summarise what happened in this run. Surface any failure modes, surprising findings, or evidence that the run's verdict is wrong.";
+function traceAnalystOnRunComplete(opts) {
+  return async (ctx) => {
+    if (opts.shouldRun && !opts.shouldRun(ctx)) return;
+    const source = opts.analyze.source;
+    if (source === void 0) {
+      await ctx.store.appendEvent({
+        eventId: `analyst-skip-${ctx.runId}`,
+        runId: ctx.runId,
+        kind: "log",
+        timestamp: Date.now(),
+        payload: { source: "trace_analyst_hook", reason: "no source configured" }
+      });
+      return;
+    }
+    const result = await analyzeTraces(
+      { question: opts.question ?? DEFAULT_QUESTION },
+      { ...opts.analyze, source }
+    );
+    if (opts.save) await opts.save(result, ctx);
+    if (opts.gateOn && !opts.gateOn(result, ctx)) {
+      await ctx.store.appendEvent({
+        eventId: `analyst-gate-${ctx.runId}`,
+        runId: ctx.runId,
+        kind: "log",
+        timestamp: Date.now(),
+        payload: {
+          source: "trace_analyst_hook",
+          reason: "analyst_gate_failed",
+          findings: result.findings
+        }
+      });
+    }
+  };
+}
+
 // src/trace-analyst/insights.ts
 var DOMAIN_STOP_WORDS = /* @__PURE__ */ new Set([
   "and",
@@ -1739,6 +1891,9 @@ export {
   redactValue,
   OTEL_AGENT_EVAL_SCOPE,
   exportRunAsOtlp,
+  RunIntegrityError,
+  assertRunCaptured,
+  throwIfRunIncomplete,
   DEFAULT_TRACE_ANALYST_BUDGETS,
   TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
   OtlpFileTraceStore,
@@ -1751,6 +1906,7 @@ export {
   buildTraceAnalystTools,
   traceAnalystFunctionGroup,
   analyzeTraces,
+  traceAnalystOnRunComplete,
   tokenizeDomainWords,
   inferDomainKeywords,
   domainEvidencePattern,
@@ -1761,4 +1917,4 @@ export {
   defaultTraceInsightPanel,
   buildTraceInsightPrompt
 };
-//# sourceMappingURL=chunk-KWUAAIHR.js.map
+//# sourceMappingURL=chunk-WOK2RTWG.js.map