@tangle-network/agent-eval 0.117.0 → 0.118.0

package/dist/{chunk-LQUTGLOZ.js → chunk-ZET2UAYW.js}

@@ -1,57 +1,8 @@
-// src/trace/schema.ts
-var TRACE_SCHEMA_VERSION = "1.0.0";
-var FAILURE_CLASSES = [
-  "success",
-  "reasoning_error",
-  "tool_selection_error",
-  "tool_argument_error",
-  "tool_recovery_failure",
-  "hallucination",
-  "instruction_following",
-  "safety_refusal_miss",
-  "policy_violation",
-  "budget_exceeded",
-  "format_drift",
-  "permission_escalation",
-  "pii_leak",
-  "cost_overrun",
-  "timeout",
-  "sandbox_failure",
-  "missing_user_data",
-  "missing_domain_data",
-  "missing_codebase_context",
-  "missing_runtime_context",
-  "missing_credentials",
-  "missing_integration_connection",
-  "missing_integration_scope",
-  "integration_approval_required",
-  "integration_auth_expired",
-  "integration_provider_failure",
-  "bad_integration_manifest",
-  "unsafe_integration_write_denied",
-  "stale_external_data",
-  "bad_retrieval",
-  "insufficient_evidence",
-  "contradictory_evidence",
-  "ambiguous_user_intent",
-  "knowledge_readiness_blocked",
-  "unknown"
-];
-function isLlmSpan(s) {
-  return s.kind === "llm";
-}
-function isToolSpan(s) {
-  return s.kind === "tool";
-}
-function isRetrievalSpan(s) {
-  return s.kind === "retrieval";
-}
-function isJudgeSpan(s) {
-  return s.kind === "judge";
-}
-function isSandboxSpan(s) {
-  return s.kind === "sandbox";
-}
+import {
+  isJudgeSpan,
+  isLlmSpan,
+  isToolSpan
+} from "./chunk-MA6HLL3S.js";
 
 // src/trace/query.ts
 async function runsForScenario(store, scenarioId) {
@@ -103,12 +54,18 @@ function aggregateLlm(spans) {
       inputTokens: acc.inputTokens + (s.inputTokens ?? 0),
       outputTokens: acc.outputTokens + (s.outputTokens ?? 0),
       cachedTokens: acc.cachedTokens + (s.cachedTokens ?? 0),
-      // reasoningTokens is on LlmSpan but was omitted here — reasoning usage was
-      // invisible to any cost/perf analysis reading this aggregate.
+      cacheWriteTokens: acc.cacheWriteTokens + (s.cacheWriteTokens ?? 0),
       reasoningTokens: acc.reasoningTokens + (s.reasoningTokens ?? 0),
       costUsd: acc.costUsd + (s.costUsd ?? 0)
     }),
-    { inputTokens: 0, outputTokens: 0, cachedTokens: 0, reasoningTokens: 0, costUsd: 0 }
+    {
+      inputTokens: 0,
+      outputTokens: 0,
+      cachedTokens: 0,
+      cacheWriteTokens: 0,
+      reasoningTokens: 0,
+      costUsd: 0
+    }
   );
 }
 function runFailureClass(run) {
@@ -119,13 +76,6 @@ function runFailureClass(run) {
 }
 
 export {
-  TRACE_SCHEMA_VERSION,
-  FAILURE_CLASSES,
-  isLlmSpan,
-  isToolSpan,
-  isRetrievalSpan,
-  isJudgeSpan,
-  isSandboxSpan,
   runsForScenario,
   llmSpans,
   toolSpans,
@@ -136,4 +86,4 @@ export {
   aggregateLlm,
   runFailureClass
 };
-//# sourceMappingURL=chunk-LQUTGLOZ.js.map
+//# sourceMappingURL=chunk-ZET2UAYW.js.map

package/dist/chunk-ZET2UAYW.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/trace/query.ts"],"sourcesContent":["/**\n * Typed query helpers over TraceStore.\n *\n * Not a full SQL engine — a minimal, composable set of operators that\n * cover the canned-pipeline use cases. For ad-hoc analytics, persist to\n * NDJSON and point DuckDB at it; the schema is stable so external SQL\n * tooling works out of the box.\n */\n\nimport type { FailureClass, JudgeSpan, LlmSpan, Run, ToolSpan } from './schema'\nimport { isJudgeSpan, isLlmSpan, isToolSpan } from './schema'\nimport type { TraceStore } from './store'\n\nexport async function runsForScenario(store: TraceStore, scenarioId: string): Promise<Run[]> {\n  return store.listRuns({ scenarioId })\n}\n\nexport async function llmSpans(store: TraceStore, runId?: string): Promise<LlmSpan[]> {\n  const spans = await store.spans({ runId, kind: 'llm' })\n  return spans.filter(isLlmSpan)\n}\n\nexport async function toolSpans(\n  store: TraceStore,\n  runId?: string,\n  toolName?: string,\n): Promise<ToolSpan[]> {\n  const spans = await store.spans({ runId, kind: 'tool', toolName })\n  return spans.filter(isToolSpan)\n}\n\n/** Query judge-kind spans from the trace store, optionally scoped to a single run. */\nexport async function judgeSpans(store: TraceStore, runId?: string): Promise<JudgeSpan[]> {\n  const spans = await store.spans({ runId, kind: 'judge' })\n  return spans.filter(isJudgeSpan)\n}\n\n/** Group spans by any key selector. */\nexport function groupBy<T, K extends string | number>(items: T[], key: (t: T) => K): Map<K, T[]> {\n  const map = new Map<K, T[]>()\n  for (const item of items) {\n    const k = key(item)\n    let bucket = map.get(k)\n    if (!bucket) {\n      bucket = []\n      map.set(k, bucket)\n    }\n    bucket.push(item)\n  }\n  return map\n}\n\n/** Hash tool arguments to an orderless-key-stable string for de-duplication. */\nexport function argHash(args: unknown): string {\n  return stableStringify(args)\n}\n\n/** Whether argument-based comparisons are valid for this tool call. */\nexport function hasCapturedToolArgs(span: ToolSpan): boolean {\n  return span.argsCaptured !== false\n}\n\nfunction stableStringify(value: unknown): string {\n  // Must ALWAYS return a string: JSON.stringify(undefined) — and stringify of\n  // functions/symbols — yields the JS value `undefined`, which would make\n  // argHash return a non-string and silently break the de-dup keys behind\n  // stuck-loop and failure-cluster.\n  if (value === undefined) return 'undefined'\n  if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'undefined'\n  if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`\n  // Drop undefined-valued keys to match JSON.stringify semantics, so\n  // `{a:1}` and `{a:1,b:undefined}` hash identically.\n  const obj = value as Record<string, unknown>\n  const keys = Object.keys(obj)\n    .filter((k) => obj[k] !== undefined)\n    .sort()\n  const parts = keys.map((k) => `${JSON.stringify(k)}:${stableStringify(obj[k])}`)\n  return `{${parts.join(',')}}`\n}\n\n/** Sum an LLM-span array into aggregate token + cost. */\nexport function aggregateLlm(spans: LlmSpan[]): {\n  inputTokens: number\n  outputTokens: number\n  cachedTokens: number\n  cacheWriteTokens: number\n  reasoningTokens: number\n  costUsd: number\n} {\n  return spans.reduce(\n    (acc, s) => ({\n      inputTokens: acc.inputTokens + (s.inputTokens ?? 0),\n      outputTokens: acc.outputTokens + (s.outputTokens ?? 0),\n      cachedTokens: acc.cachedTokens + (s.cachedTokens ?? 0),\n      cacheWriteTokens: acc.cacheWriteTokens + (s.cacheWriteTokens ?? 0),\n      reasoningTokens: acc.reasoningTokens + (s.reasoningTokens ?? 0),\n      costUsd: acc.costUsd + (s.costUsd ?? 0),\n    }),\n    {\n      inputTokens: 0,\n      outputTokens: 0,\n      cachedTokens: 0,\n      cacheWriteTokens: 0,\n      reasoningTokens: 0,\n      costUsd: 0,\n    },\n  )\n}\n\n/** Pick the outcome's failure class when present, else derive 'success' from run status. */\nexport function runFailureClass(run: Run): FailureClass {\n  if (run.outcome?.failureClass) return run.outcome.failureClass\n  if (run.status === 'completed' && run.outcome?.pass !== false) return 'success'\n  if (run.status === 'aborted') return 'budget_exceeded'\n  return 'unknown'\n}\n"],"mappings":";;;;;;;AAaA,eAAsB,gBAAgB,OAAmB,YAAoC;AAC3F,SAAO,MAAM,SAAS,EAAE,WAAW,CAAC;AACtC;AAEA,eAAsB,SAAS,OAAmB,OAAoC;AACpF,QAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,OAAO,MAAM,MAAM,CAAC;AACtD,SAAO,MAAM,OAAO,SAAS;AAC/B;AAEA,eAAsB,UACpB,OACA,OACA,UACqB;AACrB,QAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,OAAO,MAAM,QAAQ,SAAS,CAAC;AACjE,SAAO,MAAM,OAAO,UAAU;AAChC;AAGA,eAAsB,WAAW,OAAmB,OAAsC;AACxF,QAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,OAAO,MAAM,QAAQ,CAAC;AACxD,SAAO,MAAM,OAAO,WAAW;AACjC;AAGO,SAAS,QAAsC,OAAY,KAA+B;AAC/F,QAAM,MAAM,oBAAI,IAAY;AAC5B,aAAW,QAAQ,OAAO;AACxB,UAAM,IAAI,IAAI,IAAI;AAClB,QAAI,SAAS,IAAI,IAAI,CAAC;AACtB,QAAI,CAAC,QAAQ;AACX,eAAS,CAAC;AACV,UAAI,IAAI,GAAG,MAAM;AAAA,IACnB;AACA,WAAO,KAAK,IAAI;AAAA,EAClB;AACA,SAAO;AACT;AAGO,SAAS,QAAQ,MAAuB;AAC7C,SAAO,gBAAgB,IAAI;AAC7B;AAGO,SAAS,oBAAoB,MAAyB;AAC3D,SAAO,KAAK,iBAAiB;AAC/B;AAEA,SAAS,gBAAgB,OAAwB;AAK/C,MAAI,UAAU,OAAW,QAAO;AAChC,MAAI,UAAU,QAAQ,OAAO,UAAU,SAAU,QAAO,KAAK,UAAU,KAAK,KAAK;AACjF,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO,IAAI,MAAM,IAAI,eAAe,EAAE,KAAK,GAAG,CAAC;AAGzE,QAAM,MAAM;AACZ,QAAM,OAAO,OAAO,KAAK,GAAG,EACzB,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,MAAS,EAClC,KAAK;AACR,QAAM,QAAQ,KAAK,IAAI,CAAC,MAAM,GAAG,KAAK,UAAU,CAAC,CAAC,IAAI,gBAAgB,IAAI,CAAC,CAAC,CAAC,EAAE;AAC/E,SAAO,IAAI,MAAM,KAAK,GAAG,CAAC;AAC5B;AAGO,SAAS,aAAa,OAO3B;AACA,SAAO,MAAM;AAAA,IACX,CAAC,KAAK,OAAO;AAAA,MACX,aAAa,IAAI,eAAe,EAAE,eAAe;AAAA,MACjD,cAAc,IAAI,gBAAgB,EAAE,gBAAgB;AAAA,MACpD,cAAc,IAAI,gBAAgB,EAAE,gBAAgB;AAAA,MACpD,kBAAkB,IAAI,oBAAoB,EAAE,oBAAoB;AAAA,MAChE,iBAAiB,IAAI,mBAAmB,EAAE,mBAAmB;AAAA,MAC7D,SAAS,IAAI,WAAW,EAAE,WAAW;AAAA,IACvC;AAAA,IACA;AAAA,MACE,aAAa;AAAA,MACb,cAAc;AAAA,MACd,cAAc;AAAA,MACd,kBAAkB;AAAA,MAClB,iBAAiB;AAAA,MACjB,SAAS;AAAA,IACX;AAAA,EACF;AACF;AAGO,SAAS,gBAAgB,KAAwB;AACtD,MAAI,IAAI,SAAS,aAAc,QAAO,IAAI,QAAQ;AAClD,MAAI,IAAI,WAAW,eAAe,IAAI,SAAS,SAAS,MAAO,QAAO;AACtE,MAAI,IAAI,WAAW,UAAW,QAAO;AACrC,SAAO;AACT;","names":[]}