@tangle-network/agent-eval 0.23.0 → 0.24.0

package/dist/chunk-SQQLHODJ.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/trace/raw-provider-sink.ts"],"sourcesContent":["/**\n * RawProviderSink — first-class persistence for the actual HTTP-level\n * request/response bodies of every LLM provider call.\n *\n * Why this is a separate sink from the structured `LlmSpan`:\n *\n *   - `LlmSpan` records the *intent* — model name, messages, output text,\n *     usage. It's what dashboards read; it's NOT enough for forensics.\n *   - When a downstream consumer reports \"the verifier used the wrong route\"\n *     or \"tokens look right but reasoning was missing,\" the only way to\n *     answer is the raw HTTP body. Span fields can lie (a proxy can echo\n *     a different `model` value than what actually answered); the raw\n *     response is ground truth.\n *\n * Default behaviour: opt-in. Pass `rawSink` to `LlmClientOptions` (or the\n * matrix runner / BuilderSession sets it up automatically) and every\n * request, response, and error is recorded — including retries, with the\n * attempt index attached so a flaky call's full event chain is recoverable.\n *\n * Redaction is enforced at sink time. The default redactor strips\n * `Authorization`, `X-Api-Key`, `X-Auth-Token`, `Cookie` headers and any\n * payload field whose key matches `apiKey | api_key | bearer | password |\n * secret | token` (case-insensitive). Override via the sink constructor or\n * the per-call `redactor`. The `redactedFields` array on the persisted\n * event lets a reviewer see what was stripped without exposing the values.\n */\n\nimport { promises as fs } from 'node:fs'\nimport * as path from 'node:path'\n\nexport type RawProviderDirection = 'request' | 'response' | 'error'\n\nexport interface RawProviderEvent {\n  /** Stable id. Generated by the sink if omitted. */\n  eventId: string\n  /** Trace context populated by `LlmClient` when the call is wrapped in a span. */\n  runId?: string\n  spanId?: string\n  /**\n   * Logical provider name. Free-form so callers can use whatever id matches\n   * their topology (`'openai'`, `'anthropic'`, `'tangle-router'`, …). When\n   * omitted, derived from `baseUrl` in `LlmClientOptions`.\n   */\n  provider: string\n  model: string\n  /** Endpoint path, e.g. `'/v1/chat/completions'`. */\n  endpoint: string\n  /** Base URL used for the call (already-normalised — no trailing slash). */\n  baseUrl: string\n  /** 0-indexed retry attempt. The first attempt is 0; a retried call gets 1, 2, … */\n  attemptIndex: number\n  direction: RawProviderDirection\n  /** Unix ms. */\n  timestamp: number\n  /** Wall-clock duration of the call leg. Set on `response` and `error` events; null on `request`. */\n  durationMs?: number\n  statusCode?: number\n  requestHeaders?: Record<string, string>\n  requestBody?: unknown\n  responseHeaders?: Record<string, string>\n  responseBody?: unknown\n  /** Set on `direction: 'error'` events. */\n  errorMessage?: string\n  /** Field paths the redactor stripped from this event ('header:Authorization', 'body.apiKey', …). */\n  redactedFields: string[]\n}\n\nexport interface RawProviderSinkFilter {\n  runId?: string\n  spanId?: string\n  direction?: RawProviderDirection\n  attemptIndex?: number\n}\n\nexport interface RawProviderSink {\n  record(event: RawProviderEvent): Promise<void>\n  /** Optional listing — implementations that durably persist (file, db) should support this. */\n  list?(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>\n  /** Optional teardown for backed implementations. */\n  close?(): Promise<void>\n}\n\nexport type ProviderRedactor = (event: RawProviderEvent) => RawProviderEvent\n\nconst REDACTED_HEADER_NAMES = new Set([\n  'authorization',\n  'x-api-key',\n  'x-auth-token',\n  'cookie',\n  'set-cookie',\n  'proxy-authorization',\n])\n\nconst REDACTED_BODY_KEY = /^(api[_-]?key|bearer|password|secret|token|access[_-]?token|refresh[_-]?token)$/i\n\n/**\n * Default redactor — strips well-known auth headers and any body field whose\n * key matches the credential pattern. Records every redacted path on\n * `event.redactedFields` so a downstream reviewer can see what was removed.\n */\nexport function defaultProviderRedactor(event: RawProviderEvent): RawProviderEvent {\n  const redactedFields: string[] = [...(event.redactedFields ?? [])]\n  const requestHeaders = redactHeaders(event.requestHeaders, 'request', redactedFields)\n  const responseHeaders = redactHeaders(event.responseHeaders, 'response', redactedFields)\n  const requestBody = redactBody(event.requestBody, 'requestBody', redactedFields)\n  const responseBody = redactBody(event.responseBody, 'responseBody', redactedFields)\n  return { ...event, requestHeaders, responseHeaders, requestBody, responseBody, redactedFields }\n}\n\nfunction redactHeaders(\n  headers: Record<string, string> | undefined,\n  prefix: 'request' | 'response',\n  redactedFields: string[],\n): Record<string, string> | undefined {\n  if (!headers) return headers\n  const out: Record<string, string> = {}\n  for (const [k, v] of Object.entries(headers)) {\n    if (REDACTED_HEADER_NAMES.has(k.toLowerCase())) {\n      redactedFields.push(`${prefix}Headers.${k}`)\n      continue\n    }\n    out[k] = v\n  }\n  return out\n}\n\nfunction redactBody(\n  value: unknown,\n  pathStr: string,\n  redactedFields: string[],\n): unknown {\n  if (value == null) return value\n  if (Array.isArray(value)) return value.map((v, i) => redactBody(v, `${pathStr}[${i}]`, redactedFields))\n  if (typeof value === 'object') {\n    const out: Record<string, unknown> = {}\n    for (const [k, v] of Object.entries(value as Record<string, unknown>)) {\n      if (REDACTED_BODY_KEY.test(k)) {\n        redactedFields.push(`${pathStr}.${k}`)\n        continue\n      }\n      out[k] = redactBody(v, `${pathStr}.${k}`, redactedFields)\n    }\n    return out\n  }\n  return value\n}\n\n// ── In-memory ────────────────────────────────────────────────────────────\n\nexport interface InMemoryRawProviderSinkOptions {\n  redactor?: ProviderRedactor\n}\n\nexport class InMemoryRawProviderSink implements RawProviderSink {\n  private events: RawProviderEvent[] = []\n  private redactor: ProviderRedactor\n\n  constructor(opts: InMemoryRawProviderSinkOptions = {}) {\n    this.redactor = opts.redactor ?? defaultProviderRedactor\n  }\n\n  async record(event: RawProviderEvent): Promise<void> {\n    this.events.push(this.redactor({ ...event, redactedFields: event.redactedFields ?? [] }))\n  }\n\n  async list(filter: RawProviderSinkFilter = {}): Promise<RawProviderEvent[]> {\n    return this.events.filter((e) =>\n      (filter.runId === undefined || e.runId === filter.runId) &&\n      (filter.spanId === undefined || e.spanId === filter.spanId) &&\n      (filter.direction === undefined || e.direction === filter.direction) &&\n      (filter.attemptIndex === undefined || e.attemptIndex === filter.attemptIndex),\n    )\n  }\n\n  size(): number { return this.events.length }\n}\n\nexport class NoopRawProviderSink implements RawProviderSink {\n  async record(): Promise<void> { /* no-op */ }\n  /**\n   * Returns an empty array. Implemented so `assertRunCaptured` does not\n   * trip the `no_raw_sink` issue when a caller explicitly opts out of\n   * capture by passing this sink — opt-out is a deliberate choice, not a\n   * misconfiguration.\n   */\n  async list(): Promise<RawProviderEvent[]> { return [] }\n}\n\n// ── Filesystem (NDJSON) ──────────────────────────────────────────────────\n\nexport interface FileSystemRawProviderSinkOptions {\n  /** Directory the NDJSON file is written into. Created if missing. */\n  dir: string\n  /** File name; default `'raw-provider-events.ndjson'`. */\n  fileName?: string\n  /** Bytes after which the writer rolls over to a new file (default 32 MiB). */\n  rollAtBytes?: number\n  redactor?: ProviderRedactor\n}\n\nexport class FileSystemRawProviderSink implements RawProviderSink {\n  private dir: string\n  private fileName: string\n  private rollAtBytes: number\n  private redactor: ProviderRedactor\n  private bytesWritten = 0\n  private rollIndex = 0\n  private initPromise: Promise<void> | null = null\n\n  constructor(opts: FileSystemRawProviderSinkOptions) {\n    this.dir = opts.dir\n    this.fileName = opts.fileName ?? 'raw-provider-events.ndjson'\n    this.rollAtBytes = opts.rollAtBytes ?? 32 * 1024 * 1024\n    this.redactor = opts.redactor ?? defaultProviderRedactor\n  }\n\n  private async ensureInit(): Promise<void> {\n    if (!this.initPromise) {\n      this.initPromise = fs.mkdir(this.dir, { recursive: true }).then(() => undefined)\n    }\n    await this.initPromise\n  }\n\n  private currentPath(): string {\n    if (this.rollIndex === 0) return path.join(this.dir, this.fileName)\n    return path.join(this.dir, `${this.fileName}.${this.rollIndex}`)\n  }\n\n  async record(event: RawProviderEvent): Promise<void> {\n    await this.ensureInit()\n    const redacted = this.redactor({ ...event, redactedFields: event.redactedFields ?? [] })\n    const line = JSON.stringify(redacted) + '\\n'\n    if (this.bytesWritten + line.length > this.rollAtBytes && this.bytesWritten > 0) {\n      this.rollIndex += 1\n      this.bytesWritten = 0\n    }\n    await fs.appendFile(this.currentPath(), line, 'utf8')\n    this.bytesWritten += line.length\n  }\n\n  async list(filter: RawProviderSinkFilter = {}): Promise<RawProviderEvent[]> {\n    await this.ensureInit()\n    const out: RawProviderEvent[] = []\n    for (let i = 0; i <= this.rollIndex; i++) {\n      const file = i === 0\n        ? path.join(this.dir, this.fileName)\n        : path.join(this.dir, `${this.fileName}.${i}`)\n      let body: string\n      try {\n        body = await fs.readFile(file, 'utf8')\n      } catch (err) {\n        if ((err as NodeJS.ErrnoException).code === 'ENOENT') continue\n        throw err\n      }\n      for (const line of body.split('\\n')) {\n        if (!line) continue\n        const event = JSON.parse(line) as RawProviderEvent\n        if (filter.runId !== undefined && event.runId !== filter.runId) continue\n        if (filter.spanId !== undefined && event.spanId !== filter.spanId) continue\n        if (filter.direction !== undefined && event.direction !== filter.direction) continue\n        if (filter.attemptIndex !== undefined && event.attemptIndex !== filter.attemptIndex) continue\n        out.push(event)\n      }\n    }\n    return out\n  }\n}\n\n// ── Helpers ──────────────────────────────────────────────────────────────\n\n/**\n * Best-effort provider id from a base URL. Falls back to the URL host when\n * none of the well-known patterns match.\n */\nexport function providerFromBaseUrl(baseUrl: string): string {\n  const lower = baseUrl.toLowerCase()\n  if (lower.includes('api.openai.com')) return 'openai'\n  if (lower.includes('api.anthropic.com')) return 'anthropic'\n  if (lower.includes('generativelanguage.googleapis.com')) return 'google'\n  if (lower.includes('api.together.ai') || lower.includes('api.together.xyz')) return 'together'\n  if (lower.includes('api.deepseek.com')) return 'deepseek'\n  if (lower.includes('router.tangle.tools')) return 'tangle-router'\n  if (lower.includes('api.litellm') || lower.includes('litellm')) return 'litellm'\n  try {\n    return new URL(baseUrl).host\n  } catch {\n    return baseUrl\n  }\n}\n"],"mappings":";AA2BA,SAAS,YAAY,UAAU;AAC/B,YAAY,UAAU;AAwDtB,IAAM,wBAAwB,oBAAI,IAAI;AAAA,EACpC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,oBAAoB;AAOnB,SAAS,wBAAwB,OAA2C;AACjF,QAAM,iBAA2B,CAAC,GAAI,MAAM,kBAAkB,CAAC,CAAE;AACjE,QAAM,iBAAiB,cAAc,MAAM,gBAAgB,WAAW,cAAc;AACpF,QAAM,kBAAkB,cAAc,MAAM,iBAAiB,YAAY,cAAc;AACvF,QAAM,cAAc,WAAW,MAAM,aAAa,eAAe,cAAc;AAC/E,QAAM,eAAe,WAAW,MAAM,cAAc,gBAAgB,cAAc;AAClF,SAAO,EAAE,GAAG,OAAO,gBAAgB,iBAAiB,aAAa,cAAc,eAAe;AAChG;AAEA,SAAS,cACP,SACA,QACA,gBACoC;AACpC,MAAI,CAAC,QAAS,QAAO;AACrB,QAAM,MAA8B,CAAC;AACrC,aAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,OAAO,GAAG;AAC5C,QAAI,sBAAsB,IAAI,EAAE,YAAY,CAAC,GAAG;AAC9C,qBAAe,KAAK,GAAG,MAAM,WAAW,CAAC,EAAE;AAC3C;AAAA,IACF;AACA,QAAI,CAAC,IAAI;AAAA,EACX;AACA,SAAO;AACT;AAEA,SAAS,WACP,OACA,SACA,gBACS;AACT,MAAI,SAAS,KAAM,QAAO;AAC1B,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO,MAAM,IAAI,CAAC,GAAG,MAAM,WAAW,GAAG,GAAG,OAAO,IAAI,CAAC,KAAK,cAAc,CAAC;AACtG,MAAI,OAAO,UAAU,UAAU;AAC7B,UAAM,MAA+B,CAAC;AACtC,eAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,KAAgC,GAAG;AACrE,UAAI,kBAAkB,KAAK,CAAC,GAAG;AAC7B,uBAAe,KAAK,GAAG,OAAO,IAAI,CAAC,EAAE;AACrC;AAAA,MACF;AACA,UAAI,CAAC,IAAI,WAAW,GAAG,GAAG,OAAO,IAAI,CAAC,IAAI,cAAc;AAAA,IAC1D;AACA,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAQO,IAAM,0BAAN,MAAyD;AAAA,EACtD,SAA6B,CAAC;AAAA,EAC9B;AAAA,EAER,YAAY,OAAuC,CAAC,GAAG;AACrD,SAAK,WAAW,KAAK,YAAY;AAAA,EACnC;AAAA,EAEA,MAAM,OAAO,OAAwC;AACnD,SAAK,OAAO,KAAK,KAAK,SAAS,EAAE,GAAG,OAAO,gBAAgB,MAAM,kBAAkB,CAAC,EAAE,CAAC,CAAC;AAAA,EAC1F;AAAA,EAEA,MAAM,KAAK,SAAgC,CAAC,GAAgC;AAC1E,WAAO,KAAK,OAAO;AAAA,MAAO,CAAC,OACxB,OAAO,UAAU,UAAa,EAAE,UAAU,OAAO,WACjD,OAAO,WAAW,UAAa,EAAE,WAAW,OAAO,YACnD,OAAO,cAAc,UAAa,EAAE,cAAc,OAAO,eACzD,OAAO,iBAAiB,UAAa,EAAE,iBAAiB,OAAO;AAAA,IAClE;AAAA,EACF;AAAA,EAEA,OAAe;AAAE,WAAO,KAAK,OAAO;AAAA,EAAO;AAC7C;AAEO,IAAM,sBAAN,MAAqD;AAAA,EAC1D,MAAM,SAAwB;AAAA,EAAc;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAO5C,MAAM,OAAoC;AAAE,WAAO,CAAC;AAAA,EAAE;AACxD;AAcO,IAAM,4BAAN,MAA2D;AAAA,EACxD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,eAAe;AAAA,EACf,YAAY;AAAA,EACZ,cAAoC;AAAA,EAE5C,YAAY,MAAwC;AAClD,SAAK,MAAM,KAAK;AAChB,SAAK,WAAW,KAAK,YAAY;AACjC,SAAK,cAAc,KAAK,eAAe,KAAK,OAAO;AACnD,SAAK,WAAW,KAAK,YAAY;AAAA,EACnC;AAAA,EAEA,MAAc,aAA4B;AACxC,QAAI,CAAC,KAAK,aAAa;AACrB,WAAK,cAAc,GAAG,MAAM,KAAK,KAAK,EAAE,WAAW,KAAK,CAAC,EAAE,KAAK,MAAM,MAAS;AAAA,IACjF;AACA,UAAM,KAAK;AAAA,EACb;AAAA,EAEQ,cAAsB;AAC5B,QAAI,KAAK,cAAc,EAAG,QAAY,UAAK,KAAK,KAAK,KAAK,QAAQ;AAClE,WAAY,UAAK,KAAK,KAAK,GAAG,KAAK,QAAQ,IAAI,KAAK,SAAS,EAAE;AAAA,EACjE;AAAA,EAEA,MAAM,OAAO,OAAwC;AACnD,UAAM,KAAK,WAAW;AACtB,UAAM,WAAW,KAAK,SAAS,EAAE,GAAG,OAAO,gBAAgB,MAAM,kBAAkB,CAAC,EAAE,CAAC;AACvF,UAAM,OAAO,KAAK,UAAU,QAAQ,IAAI;AACxC,QAAI,KAAK,eAAe,KAAK,SAAS,KAAK,eAAe,KAAK,eAAe,GAAG;AAC/E,WAAK,aAAa;AAClB,WAAK,eAAe;AAAA,IACtB;AACA,UAAM,GAAG,WAAW,KAAK,YAAY,GAAG,MAAM,MAAM;AACpD,SAAK,gBAAgB,KAAK;AAAA,EAC5B;AAAA,EAEA,MAAM,KAAK,SAAgC,CAAC,GAAgC;AAC1E,UAAM,KAAK,WAAW;AACtB,UAAM,MAA0B,CAAC;AACjC,aAAS,IAAI,GAAG,KAAK,KAAK,WAAW,KAAK;AACxC,YAAM,OAAO,MAAM,IACV,UAAK,KAAK,KAAK,KAAK,QAAQ,IAC5B,UAAK,KAAK,KAAK,GAAG,KAAK,QAAQ,IAAI,CAAC,EAAE;AAC/C,UAAI;AACJ,UAAI;AACF,eAAO,MAAM,GAAG,SAAS,MAAM,MAAM;AAAA,MACvC,SAAS,KAAK;AACZ,YAAK,IAA8B,SAAS,SAAU;AACtD,cAAM;AAAA,MACR;AACA,iBAAW,QAAQ,KAAK,MAAM,IAAI,GAAG;AACnC,YAAI,CAAC,KAAM;AACX,cAAM,QAAQ,KAAK,MAAM,IAAI;AAC7B,YAAI,OAAO,UAAU,UAAa,MAAM,UAAU,OAAO,MAAO;AAChE,YAAI,OAAO,WAAW,UAAa,MAAM,WAAW,OAAO,OAAQ;AACnE,YAAI,OAAO,cAAc,UAAa,MAAM,cAAc,OAAO,UAAW;AAC5E,YAAI,OAAO,iBAAiB,UAAa,MAAM,iBAAiB,OAAO,aAAc;AACrF,YAAI,KAAK,KAAK;AAAA,MAChB;AAAA,IACF;AACA,WAAO;AAAA,EACT;AACF;AAQO,SAAS,oBAAoB,SAAyB;AAC3D,QAAM,QAAQ,QAAQ,YAAY;AAClC,MAAI,MAAM,SAAS,gBAAgB,EAAG,QAAO;AAC7C,MAAI,MAAM,SAAS,mBAAmB,EAAG,QAAO;AAChD,MAAI,MAAM,SAAS,mCAAmC,EAAG,QAAO;AAChE,MAAI,MAAM,SAAS,iBAAiB,KAAK,MAAM,SAAS,kBAAkB,EAAG,QAAO;AACpF,MAAI,MAAM,SAAS,kBAAkB,EAAG,QAAO;AAC/C,MAAI,MAAM,SAAS,qBAAqB,EAAG,QAAO;AAClD,MAAI,MAAM,SAAS,aAAa,KAAK,MAAM,SAAS,SAAS,EAAG,QAAO;AACvE,MAAI;AACF,WAAO,IAAI,IAAI,OAAO,EAAE;AAAA,EAC1B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;","names":[]}