@tangle-network/agent-eval 0.129.0 → 0.130.0

package/dist/raw-provider-sink-BU29Sh8h.d.ts

@@ -0,0 +1,134 @@
+//#region src/trace/raw-provider-sink.d.ts
+/**
+ * RawProviderSink — first-class persistence for the actual HTTP-level
+ * request/response bodies of every LLM provider call.
+ *
+ * Why this is a separate sink from the structured `LlmSpan`:
+ *
+ *   - `LlmSpan` records the *intent* — model name, messages, output text,
+ *     usage. It's what dashboards read; it's NOT enough for forensics.
+ *   - When a downstream consumer reports "the verifier used the wrong route"
+ *     or "tokens look right but reasoning was missing," the only way to
+ *     answer is the raw HTTP body. Span fields can lie (a proxy can echo
+ *     a different `model` value than what actually answered); the raw
+ *     response is ground truth.
+ *
+ * Default behaviour: opt-in. Pass `rawSink` to `LlmClientOptions` (or the
+ * matrix runner / BuilderSession sets it up automatically) and every
+ * request, response, and error is recorded — including retries, with the
+ * attempt index attached so a flaky call's full event chain is recoverable.
+ *
+ * Redaction is enforced at sink time. The default redactor strips
+ * `Authorization`, `X-Api-Key`, `X-Auth-Token`, `Cookie` headers and any
+ * payload field whose key matches `apiKey | api_key | bearer | password |
+ * secret | token` (case-insensitive). Override via the sink constructor or
+ * the per-call `redactor`. The `redactedFields` array on the persisted
+ * event lets a reviewer see what was stripped without exposing the values.
+ */
+type RawProviderDirection = 'request' | 'response' | 'error';
+interface RawProviderEvent {
+  /** Stable id. Generated by the sink if omitted. */
+  eventId: string;
+  /** Trace context populated by `LlmClient` when the call is wrapped in a span. */
+  runId?: string;
+  spanId?: string;
+  /**
+   * Logical provider name. Free-form so callers can use whatever id matches
+   * their topology (`'openai'`, `'anthropic'`, `'tangle-router'`, …). When
+   * omitted, derived from `baseUrl` in `LlmClientOptions`.
+   */
+  provider: string;
+  model: string;
+  /** Endpoint path, e.g. `'/v1/chat/completions'`. */
+  endpoint: string;
+  /** Base URL used for the call (already-normalised — no trailing slash). */
+  baseUrl: string;
+  /** 0-indexed retry attempt. The first attempt is 0; a retried call gets 1, 2, … */
+  attemptIndex: number;
+  direction: RawProviderDirection;
+  /** Unix ms. */
+  timestamp: number;
+  /** Wall-clock duration of the call leg. Set on `response` and `error` events; null on `request`. */
+  durationMs?: number;
+  statusCode?: number;
+  requestHeaders?: Record<string, string>;
+  requestBody?: unknown;
+  responseHeaders?: Record<string, string>;
+  responseBody?: unknown;
+  /** Set on `direction: 'error'` events. */
+  errorMessage?: string;
+  /** Field paths the redactor stripped from this event ('header:Authorization', 'body.apiKey', …). */
+  redactedFields: string[];
+}
+interface RawProviderSinkFilter {
+  runId?: string;
+  spanId?: string;
+  direction?: RawProviderDirection;
+  attemptIndex?: number;
+}
+interface RawProviderSink {
+  record(event: RawProviderEvent): Promise<void>;
+  /** Optional listing — implementations that durably persist (file, db) should support this. */
+  list?(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>;
+  /** Optional teardown for backed implementations. */
+  close?(): Promise<void>;
+}
+type ProviderRedactor = (event: RawProviderEvent) => RawProviderEvent;
+/**
+ * Default redactor — strips well-known auth headers and any body field whose
+ * key matches the credential pattern. Records every redacted path on
+ * `event.redactedFields` so a downstream reviewer can see what was removed.
+ */
+declare function defaultProviderRedactor(event: RawProviderEvent): RawProviderEvent;
+interface InMemoryRawProviderSinkOptions {
+  redactor?: ProviderRedactor;
+}
+declare class InMemoryRawProviderSink implements RawProviderSink {
+  private events;
+  private redactor;
+  constructor(opts?: InMemoryRawProviderSinkOptions);
+  record(event: RawProviderEvent): Promise<void>;
+  list(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>;
+  size(): number;
+}
+declare class NoopRawProviderSink implements RawProviderSink {
+  record(): Promise<void>;
+  /**
+   * Returns an empty array. Implemented so `assertRunCaptured` does not
+   * trip the `no_raw_sink` issue when a caller explicitly opts out of
+   * capture by passing this sink — opt-out is a deliberate choice, not a
+   * misconfiguration.
+   */
+  list(): Promise<RawProviderEvent[]>;
+}
+interface FileSystemRawProviderSinkOptions {
+  /** Directory the NDJSON file is written into. Created if missing. */
+  dir: string;
+  /** File name; default `'raw-provider-events.ndjson'`. */
+  fileName?: string;
+  /** Bytes after which the writer rolls over to a new file (default 32 MiB). */
+  rollAtBytes?: number;
+  redactor?: ProviderRedactor;
+}
+declare class FileSystemRawProviderSink implements RawProviderSink {
+  private dir;
+  private fileName;
+  private rollAtBytes;
+  private redactor;
+  private bytesWritten;
+  private rollIndex;
+  private initPromise;
+  constructor(opts: FileSystemRawProviderSinkOptions);
+  private ensureInit;
+  private currentPath;
+  record(event: RawProviderEvent): Promise<void>;
+  list(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>;
+}
+/**
+ * Best-effort provider id from a base URL. Falls back to the URL host when
+ * none of the well-known patterns match.
+ */
+declare function providerFromBaseUrl(baseUrl: string): string;
+//#endregion
+export { NoopRawProviderSink as a, RawProviderEvent as c, defaultProviderRedactor as d, providerFromBaseUrl as f, InMemoryRawProviderSinkOptions as i, RawProviderSink as l, FileSystemRawProviderSinkOptions as n, ProviderRedactor as o, InMemoryRawProviderSink as r, RawProviderDirection as s, FileSystemRawProviderSink as t, RawProviderSinkFilter as u };
+//# sourceMappingURL=raw-provider-sink-BU29Sh8h.d.ts.map

package/dist/raw-provider-sink-BU29Sh8h.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"raw-provider-sink-BU29Sh8h.d.ts","names":[],"sources":["../src/trace/raw-provider-sink.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;KA8BY;UAEK;;EAEf;;EAEA;EACA;;;;;;EAMA;EACA;;EAEA;;EAEA;;EAEA;EACA,WAAW;;EAEX;;EAEA;EACA;EACA,iBAAiB;EACjB;EACA,kBAAkB;EAClB;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA,YAAY;EACZ;;UAGe;EACf,OAAO,OAAO,mBAAmB;;EAEjC,MAAM,SAAS,wBAAwB,QAAQ;;EAE/C,UAAU;;KAGA,oBAAoB,OAAO,qBAAqB;;;;;;iBAmB5C,wBAAwB,OAAO,mBAAmB;UA8CjD;EACf,WAAW;;cAGA,mCAAmC;UACtC;UACA;EAER,YAAY,OAAM;EAIZ,OAAO,OAAO,mBAAmB;EAIjC,KAAK,SAAQ,wBAA6B,QAAQ;EAUxD;;cAKW,+BAA+B;EACpC,UAAU;;;;;;;EASV,QAAQ,QAAQ;;UAOP;;EAEf;;EAEA;;EAEA;EACA,WAAW;;cAGA,qCAAqC;UACxC;UACA;UACA;UACA;UACA;UACA;UACA;EAER,YAAY,MAAM;UAOJ;UAON;EAKF,OAAO,OAAO,mBAAmB;EAYjC,KAAK,SAAQ,wBAA6B,QAAQ;;;;;;iBAkC1C,oBAAoB"}

package/dist/redact-7Aq1ukl-.js

@@ -0,0 +1,107 @@
+//#region src/trace/redact.ts
+/** OWASP / common-sense defaults — extend per-domain. */
+const DEFAULT_REDACTION_RULES = [
+	{
+		id: "email",
+		pattern: /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi
+	},
+	{
+		id: "ssn",
+		pattern: /\b\d{3}-\d{2}-\d{4}\b/g
+	},
+	{
+		id: "credit-card",
+		pattern: /\b(?:\d[ -]*?){13,16}\b/g
+	},
+	{
+		id: "phone-us",
+		pattern: /\b(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b/g
+	},
+	{
+		id: "ipv4",
+		pattern: /\b(?:\d{1,3}\.){3}\d{1,3}\b/g
+	},
+	{
+		id: "aws-access-key",
+		pattern: /\bAKIA[0-9A-Z]{16}\b/g
+	},
+	{
+		id: "bearer",
+		pattern: /\bBearer\s+[A-Za-z0-9._~+/=-]{10,}/gi
+	},
+	{
+		id: "sk-key",
+		pattern: /\bsk-[A-Za-z0-9_-]{10,}\b/g
+	},
+	{
+		id: "private-key-block",
+		pattern: /-----BEGIN (?:RSA |EC |OPENSSH |DSA )?PRIVATE KEY-----[\s\S]*?-----END[^-]*-----/g
+	}
+];
+const REDACTION_VERSION = "1.0.0";
+/**
+* Redact a single string. Returns the new string and a per-rule count of
+* how many substitutions fired.
+*/
+function redactString(input, rules = DEFAULT_REDACTION_RULES) {
+	const byRule = {};
+	let redactionCount = 0;
+	let output = input;
+	for (const rule of rules) {
+		let hits = 0;
+		output = output.replace(rule.pattern, () => {
+			hits++;
+			return rule.replacement ?? `[redacted:${rule.id}]`;
+		});
+		if (hits > 0) {
+			byRule[rule.id] = hits;
+			redactionCount += hits;
+		}
+	}
+	return {
+		output,
+		report: {
+			redactionCount,
+			byRule
+		}
+	};
+}
+/**
+* Walk a JSON-ish value applying `redactString` to every string leaf.
+* Arrays and plain objects are recursed; other types pass through
+* untouched. Circular references throw — traces should be tree-shaped.
+*/
+function redactValue(value, rules = DEFAULT_REDACTION_RULES, report = {
+	redactionCount: 0,
+	byRule: {}
+}) {
+	if (typeof value === "string") {
+		const { output, report: r } = redactString(value, rules);
+		report.redactionCount += r.redactionCount;
+		for (const [k, v] of Object.entries(r.byRule)) report.byRule[k] = (report.byRule[k] ?? 0) + v;
+		return {
+			value: output,
+			report
+		};
+	}
+	if (Array.isArray(value)) return {
+		value: value.map((v) => redactValue(v, rules, report).value),
+		report
+	};
+	if (value !== null && typeof value === "object") {
+		const next = {};
+		for (const [k, v] of Object.entries(value)) next[k] = redactValue(v, rules, report).value;
+		return {
+			value: next,
+			report
+		};
+	}
+	return {
+		value,
+		report
+	};
+}
+//#endregion
+export { redactValue as i, REDACTION_VERSION as n, redactString as r, DEFAULT_REDACTION_RULES as t };
+
+//# sourceMappingURL=redact-7Aq1ukl-.js.map

package/dist/redact-7Aq1ukl-.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"redact-7Aq1ukl-.js","names":[],"sources":["../src/trace/redact.ts"],"sourcesContent":["/**\n * Redaction — remove PII / secrets from trace payloads before persist.\n *\n * Pre-persistence rules mean raw traces in storage are already scrubbed.\n * Unredacted variants (for debugging / post-mortems) live in a separate\n * storage layer with stricter access controls; this module only covers\n * the default scrub-then-persist path.\n *\n * Rules compose: pass an array of `RedactionRule`, each is applied in\n * order. Strings that match get replaced with a tagged sentinel so the\n * eval framework can count how many redactions happened per run\n * (surfaced via `redaction_applied` events).\n */\n\nexport interface RedactionRule {\n  id: string\n  pattern: RegExp\n  /** Replacement — e.g. '[PII:email]'. Defaults to `[redacted:{id}]`. */\n  replacement?: string\n}\n\nexport interface RedactionReport {\n  redactionCount: number\n  byRule: Record<string, number>\n}\n\n/** OWASP / common-sense defaults — extend per-domain. */\nexport const DEFAULT_REDACTION_RULES: RedactionRule[] = [\n  { id: 'email', pattern: /\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}\\b/gi },\n  { id: 'ssn', pattern: /\\b\\d{3}-\\d{2}-\\d{4}\\b/g },\n  { id: 'credit-card', pattern: /\\b(?:\\d[ -]*?){13,16}\\b/g },\n  { id: 'phone-us', pattern: /\\b(?:\\+?1[-.\\s]?)?\\(?\\d{3}\\)?[-.\\s]?\\d{3}[-.\\s]?\\d{4}\\b/g },\n  { id: 'ipv4', pattern: /\\b(?:\\d{1,3}\\.){3}\\d{1,3}\\b/g },\n  { id: 'aws-access-key', pattern: /\\bAKIA[0-9A-Z]{16}\\b/g },\n  { id: 'bearer', pattern: /\\bBearer\\s+[A-Za-z0-9._~+/=-]{10,}/gi },\n  { id: 'sk-key', pattern: /\\bsk-[A-Za-z0-9_-]{10,}\\b/g },\n  {\n    id: 'private-key-block',\n    pattern: /-----BEGIN (?:RSA |EC |OPENSSH |DSA )?PRIVATE KEY-----[\\s\\S]*?-----END[^-]*-----/g,\n  },\n]\n\nexport const REDACTION_VERSION = '1.0.0'\n\n/**\n * Redact a single string. Returns the new string and a per-rule count of\n * how many substitutions fired.\n */\nexport function redactString(\n  input: string,\n  rules: RedactionRule[] = DEFAULT_REDACTION_RULES,\n): { output: string; report: RedactionReport } {\n  const byRule: Record<string, number> = {}\n  let redactionCount = 0\n  let output = input\n  for (const rule of rules) {\n    let hits = 0\n    output = output.replace(rule.pattern, () => {\n      hits++\n      return rule.replacement ?? `[redacted:${rule.id}]`\n    })\n    if (hits > 0) {\n      byRule[rule.id] = hits\n      redactionCount += hits\n    }\n  }\n  return { output, report: { redactionCount, byRule } }\n}\n\n/**\n * Walk a JSON-ish value applying `redactString` to every string leaf.\n * Arrays and plain objects are recursed; other types pass through\n * untouched. Circular references throw — traces should be tree-shaped.\n */\nexport function redactValue(\n  value: unknown,\n  rules: RedactionRule[] = DEFAULT_REDACTION_RULES,\n  report: RedactionReport = { redactionCount: 0, byRule: {} },\n): { value: unknown; report: RedactionReport } {\n  if (typeof value === 'string') {\n    const { output, report: r } = redactString(value, rules)\n    report.redactionCount += r.redactionCount\n    for (const [k, v] of Object.entries(r.byRule)) {\n      report.byRule[k] = (report.byRule[k] ?? 0) + v\n    }\n    return { value: output, report }\n  }\n  if (Array.isArray(value)) {\n    return {\n      value: value.map((v) => redactValue(v, rules, report).value),\n      report,\n    }\n  }\n  if (value !== null && typeof value === 'object') {\n    const next: Record<string, unknown> = {}\n    for (const [k, v] of Object.entries(value)) {\n      next[k] = redactValue(v, rules, report).value\n    }\n    return { value: next, report }\n  }\n  return { value, report }\n}\n"],"mappings":";;AA2BA,MAAa,0BAA2C;CACtD;EAAE,IAAI;EAAS,SAAS;CAA8C;CACtE;EAAE,IAAI;EAAO,SAAS;CAAyB;CAC/C;EAAE,IAAI;EAAe,SAAS;CAA2B;CACzD;EAAE,IAAI;EAAY,SAAS;CAA2D;CACtF;EAAE,IAAI;EAAQ,SAAS;CAA+B;CACtD;EAAE,IAAI;EAAkB,SAAS;CAAwB;CACzD;EAAE,IAAI;EAAU,SAAS;CAAuC;CAChE;EAAE,IAAI;EAAU,SAAS;CAA6B;CACtD;EACE,IAAI;EACJ,SAAS;CACX;AACF;AAEA,MAAa,oBAAoB;;;;;AAMjC,SAAgB,aACd,OACA,QAAyB,yBACoB;CAC7C,MAAM,SAAiC,CAAC;CACxC,IAAI,iBAAiB;CACrB,IAAI,SAAS;CACb,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,OAAO;EACX,SAAS,OAAO,QAAQ,KAAK,eAAe;GAC1C;GACA,OAAO,KAAK,eAAe,aAAa,KAAK,GAAG;EAClD,CAAC;EACD,IAAI,OAAO,GAAG;GACZ,OAAO,KAAK,MAAM;GAClB,kBAAkB;EACpB;CACF;CACA,OAAO;EAAE;EAAQ,QAAQ;GAAE;GAAgB;EAAO;CAAE;AACtD;;;;;;AAOA,SAAgB,YACd,OACA,QAAyB,yBACzB,SAA0B;CAAE,gBAAgB;CAAG,QAAQ,CAAC;AAAE,GACb;CAC7C,IAAI,OAAO,UAAU,UAAU;EAC7B,MAAM,EAAE,QAAQ,QAAQ,MAAM,aAAa,OAAO,KAAK;EACvD,OAAO,kBAAkB,EAAE;EAC3B,KAAK,MAAM,CAAC,GAAG,MAAM,OAAO,QAAQ,EAAE,MAAM,GAC1C,OAAO,OAAO,MAAM,OAAO,OAAO,MAAM,KAAK;EAE/C,OAAO;GAAE,OAAO;GAAQ;EAAO;CACjC;CACA,IAAI,MAAM,QAAQ,KAAK,GACrB,OAAO;EACL,OAAO,MAAM,KAAK,MAAM,YAAY,GAAG,OAAO,MAAM,CAAC,CAAC,KAAK;EAC3D;CACF;CAEF,IAAI,UAAU,QAAQ,OAAO,UAAU,UAAU;EAC/C,MAAM,OAAgC,CAAC;EACvC,KAAK,MAAM,CAAC,GAAG,MAAM,OAAO,QAAQ,KAAK,GACvC,KAAK,KAAK,YAAY,GAAG,OAAO,MAAM,CAAC,CAAC;EAE1C,OAAO;GAAE,OAAO;GAAM;EAAO;CAC/B;CACA,OAAO;EAAE;EAAO;CAAO;AACzB"}