@tangle-network/agent-eval 0.119.0 → 0.119.1

package/dist/chunk-FIUFRXP3.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/trace-analyst/prompts.ts","../src/trace-analyst/analyst.ts"],"sourcesContent":["/** Ax RLM prompt for bounded trace discovery and evidence-backed analysis. */\n\nexport const TRACE_ANALYST_ACTOR_DESCRIPTION = `You answer questions about an OTLP-shaped JSONL trace dataset using the trace tools provided in the \\`traces\\` namespace.\n\nDISCOVERY → NARROW → DEEP-READ protocol — follow exactly:\n\n1. ALWAYS call \\`traces.getDatasetOverview({})\\` FIRST without a regex_pattern. The result tells you total_traces, raw_jsonl_bytes, services, agents, models, and sample_trace_ids (real ids — never fabricate one).\n\n2. Use raw_jsonl_bytes to gauge how expensive raw scans will be. \\`filters.regex_pattern\\` is the one scan-heavy filter on getDatasetOverview / queryTraces / countTraces — narrow with indexed fields (has_errors, model_names, service_names, agent_names, time bounds) BEFORE adding a regex on a large dataset.\n\n3. To list more traces than the sample, call \\`traces.queryTraces({ filters?, limit, offset? })\\`. Each summary carries raw_jsonl_bytes — use it to choose between viewTrace and searchTrace BEFORE calling either.\n\n4. Per-trace inspection:\n   - SMALL trace (raw_jsonl_bytes well under 150_000): call \\`traces.viewTrace({ trace_id })\\`. Returns all spans. Per-attribute payloads are head-capped at ~4KB; large \\`input.value\\` / \\`output.value\\` / \\`llm.input_messages\\` will show a \\`[trace-analyst truncated: N bytes]\\` marker.\n   - LARGE trace (raw_jsonl_bytes near or above 150_000, or you saw an \\`oversized\\` response): use \\`traces.searchTrace({ trace_id, regex_pattern })\\` to get bounded SpanMatchRecords (span metadata + matched text + surrounding context). Then call \\`traces.viewSpans({ trace_id, span_ids: [...] })\\` for surgical reads (~16KB cap, 4× higher than discovery), or \\`traces.searchSpan({ trace_id, span_id, regex_pattern })\\` for one large span. Stays bounded regardless of trace size.\n   - Useful regex patterns: \\`STATUS_CODE_ERROR\\` (failures), tool names like \\`grep\\` or \\`view_trace\\`, error strings like \\`MaxTurnsExceeded\\`, model names, attribute keys.\n\n5. ONLY call viewTrace / viewSpans / searchTrace / searchSpan with trace/span ids you have already seen in sample_trace_ids, a queryTraces page, or a previous search result. Never invent ids.\n\n5a. **Result-shape contract** — searchTrace and searchSpan return \\`{ trace_id, hits, total_matches, has_more }\\`. Iterate \\`result.hits\\` (NOT result.matches). Each hit has \\`{ span_id, span_name, span_kind, attribute_path, matched_text, context_before, context_after, match_offset }\\`. viewTrace returns \\`{ trace_id, spans }\\` (or \\`oversized\\`). viewSpans returns \\`{ trace_id, spans, missing_span_ids, truncated_attribute_count }\\`. Never assume a field name — log the result shape first if unsure.\n\n6. If viewTrace returns an \\`oversized\\` summary instead of \\`spans\\`, DO NOT retry the same call. Read the summary's top_span_names, span_count, span_response_bytes_max, error_span_count to plan a follow-up: switch to searchTrace (or searchSpan for one large span), then viewSpans on a smaller, surgical span_ids set.\n\n7. If searchTrace or searchSpan returns has_more=true, REFINE the regex to be more specific rather than blindly raising max_matches.\n\n8. If a tool errors (invalid regex, range error), STOP and reconsider — don't retry with a guessed id or argument. Use the discovery tools above to recover.\n\n9. If a ~4KB-truncated payload from viewTrace / searchTrace matters for your answer, first try viewSpans on that span id (~16KB cap). If a 16KB-truncated payload from viewSpans still matters, narrow further with searchSpan against a more specific regex rather than asking for the full payload again.\n\n10. If maxDepth > 0 and the question splits into independent semantic branches, delegate well-defined subtasks to subagents using \\`await llmQuery(...)\\`. Pass narrow context and a focused query. Examples:\n\n    const reviews = await llmQuery([\n      { query: 'Drill into trace abc123 — what tool calls preceded the failure?', context: { trace_id: 'abc123' } },\n      { query: 'Drill into trace def456 — same failure mode?', context: { trace_id: 'def456' } },\n    ]);\n\nOBSERVABILITY rules:\n- Each non-final actor turn must emit at least one \\`console.log(...)\\` for evidence. Up to 3 logs per turn is fine when correlating multiple data sources (e.g. one log for findings list, one for source-file content, one for derived analysis).\n- Do NOT combine \\`console.log\\` with \\`final(...)\\` or \\`askClarification(...)\\` in the same turn — finish gathering data first, then call final on its own turn.\n- Reuse runtime variables across turns; don't recompute.\n- When done, call \\`await final(answer)\\` with the fully-formed report. The responder rewrites the answer into output fields; if you only pass a vague summary string the responder has nothing concrete to format.\n\nCRITICAL — \\`final()\\` payload contract for evidence-grounded analysis tasks:\n- Pass a STRUCTURED object as the second arg with the actual data the responder needs to format the answer. Do NOT pass abstract instructions; pass evidence.\n- Example for per-item verdict tasks:\n  \\`\\`\\`js\n  await final(\"Format the per-item verdict report from the evidence below.\", {\n    findings: [\n      { id: 'sub-1-finding-1', claim: '...', verdict: 'TRUE-POSITIVE', evidence: 'lines 42-45 of contracts/X.sol show ...' },\n      ...all items\n    ],\n    systemic_summary: '3 sentences I wrote based on the evidence above'\n  });\n  \\`\\`\\`\n- Calling \\`final(\"answer\", {})\\` with no evidence is a failure mode — the responder will hallucinate or echo back the field names. Always include the gathered data.\n- Premature final after a single viewSpans call is INSUFFICIENT for per-finding analysis tasks. Read the requested attributes (e.g. \\`spans[i].attributes['redteam.finding.title']\\`), and for each one perform the requested cross-reference (e.g. read the source SPAN's \\`attributes['source.content']\\`).\n\nOUTPUT contract — your final answer must include:\n- A clear prose conclusion answering the user's question.\n- Trace ids and span ids cited as evidence for each claim.\n- Failure modes named in the user's domain language, with frequency and concrete examples.\n\nDo NOT invent trace ids, span ids, error messages, or model names. Every fact must be traceable to a tool result.`\n\nexport const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = 'trace-analyst-actor-v5-2026-05-06'\n\n/** Subagent prompt for focused trace-inspection subtasks. */\nexport const TRACE_ANALYST_SUBAGENT_DESCRIPTION = `You are a trace-analyst subagent. Your parent has delegated a focused trace-inspection question. Use the same DISCOVERY → NARROW → DEEP-READ protocol but stay tightly scoped: do exactly what was asked, return a concise compact answer, do NOT spawn further subagents unless the parent's question is genuinely multi-branch.\n\nCite trace ids and span ids for every claim. Do NOT invent ids.`\n","import { type AxActorTurn, type AxAIService, type AxFunction, AxJSRuntime, agent } from '@ax-llm/ax'\nimport {\n  TRACE_ANALYST_ACTOR_DESCRIPTION,\n  TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,\n  TRACE_ANALYST_SUBAGENT_DESCRIPTION,\n} from './prompts'\nimport type { TraceAnalysisStore } from './store'\nimport { OtlpFileTraceStore, TraceFileMissingError } from './store-otlp'\nimport { buildTraceAnalystTools } from './tools'\n\nexport interface AnalyzeTracesInput {\n  /** The user-facing question. Domain framing belongs here, not in the\n   *  actor description. */\n  question: string\n}\n\nexport interface AnalyzeTracesResult {\n  /** The responder's prose answer. */\n  answer: string\n  /** Bulleted findings extracted from the responder's structured output. */\n  findings: string[]\n  /** Per-actor-turn snapshots captured via `actorTurnCallback`. */\n  turns: AnalyzeTracesTurnSnapshot[]\n  /** Total turns the actor took. */\n  turnCount: number\n  /** Token usage by role. */\n  usage: TraceAnalystUsage\n  /** Full system + assistant + tool message log by role. */\n  chatLog: TraceAnalystChatLog\n  /** Prompt version that produced this run. */\n  actorPromptVersion: string\n}\n\nexport interface TraceAnalystUsage {\n  actor: TraceAnalystUsageEntry[]\n  responder: TraceAnalystUsageEntry[]\n}\n\nexport interface TraceAnalystUsageEntry {\n  [key: string]: unknown\n}\n\nexport interface TraceAnalystChatLog {\n  actor: TraceAnalystChatMessage[]\n  responder: TraceAnalystChatMessage[]\n}\n\nexport interface TraceAnalystChatMessage {\n  [key: string]: unknown\n}\n\nexport interface AnalyzeTracesTurnSnapshot {\n  turn: number\n  isError: boolean\n  /** The JS code the actor produced for this turn. */\n  code: string\n  /** The formatted action-log entry the actor sees on the next turn. */\n  output: string\n  /** Provider thought (when `actorOptions.showThoughts` is true and the\n   *  provider returns it). */\n  thought?: string\n}\n\nexport interface AnalyzeTracesOptions {\n  /** Trace data source. Pass either an OTLP-JSONL path or a custom store. */\n  source: string | TraceAnalysisStore\n  /** Caller-provided AxAIService. */\n  ai: AxAIService\n  /** Model id forwarded to actor + responder. */\n  model?: string\n  /** Recursion depth. 0 = no sub-agent dispatch. Default 1. */\n  maxDepth?: number\n  /** Maximum actor turns. Default 12. */\n  maxTurns?: number\n  /** Maximum parallel sub-agent calls in batched llmQuery. Default 2. */\n  maxParallelSubagents?: number\n  /** Override the actor description. */\n  actorDescription?: string\n  /** Override the subagent description. */\n  subagentDescription?: string\n  /** Per-turn observability hook. */\n  onTurn?: (turn: AnalyzeTracesTurnSnapshot) => void | Promise<void>\n  /** Override max runtime characters per turn. Default 6000. */\n  maxRuntimeChars?: number\n  /** When set, every turn's snapshot is appended to this JSONL file\n   *  immediately. If the analyst crashes mid-loop (provider 503,\n   *  network error, validator reject) the partial reasoning is still\n   *  on disk. Replay the file with the responder afterward to recover\n   *  evidence. */\n  progressLogPath?: string\n}\n\n/**\n * Run the trace analyst.\n *\n * Throws:\n *   - `TraceFileMissingError` if `source` is a path and doesn't exist.\n *   - `AxAgentClarificationError` if the analyst asks for clarification.\n *   - Provider errors (auth, rate limits) propagate from the AI service.\n */\nexport async function analyzeTraces(\n  input: AnalyzeTracesInput,\n  options: AnalyzeTracesOptions,\n): Promise<AnalyzeTracesResult> {\n  if (!input.question || typeof input.question !== 'string') {\n    throw new TypeError('analyzeTraces: input.question must be a non-empty string')\n  }\n\n  const store: TraceAnalysisStore =\n    typeof options.source === 'string'\n      ? new OtlpFileTraceStore({ path: options.source })\n      : options.source\n\n  // Pre-warm file stores so missing inputs fail before the RLM starts.\n  if (store instanceof OtlpFileTraceStore) {\n    await store.ensureIndexed()\n  }\n\n  const tools: AxFunction[] = buildTraceAnalystTools({ store })\n  const turns: AnalyzeTracesTurnSnapshot[] = []\n\n  // Persist each turn as JSONL so interrupted analyst runs keep useful evidence.\n  let progressFs: import('node:fs').WriteStream | undefined\n  if (options.progressLogPath) {\n    const { createWriteStream } = await import('node:fs')\n    const { mkdir } = await import('node:fs/promises')\n    const { dirname } = await import('node:path')\n    await mkdir(dirname(options.progressLogPath), { recursive: true })\n    progressFs = createWriteStream(options.progressLogPath, { flags: 'a' })\n  }\n\n  const actorTurnCallback = async (turn: AxActorTurn): Promise<void> => {\n    const snap: AnalyzeTracesTurnSnapshot = {\n      turn: turn.turn,\n      isError: turn.isError,\n      code: turn.code,\n      output: turn.output,\n      thought: turn.thought,\n    }\n    turns.push(snap)\n    if (progressFs) {\n      try {\n        progressFs.write(`${JSON.stringify({ ...snap, ts: Date.now() })}\\n`)\n      } catch {\n        // Progress logging must never fail the analyst.\n      }\n    }\n    if (options.onTurn) await options.onTurn(snap)\n  }\n\n  const maxDepth = options.maxDepth ?? 1\n  const maxTurns = options.maxTurns ?? 12\n  const maxParallelSubagents = options.maxParallelSubagents ?? 2\n  const maxRuntimeChars = options.maxRuntimeChars ?? 6000\n  const functions = tools as unknown as NonNullable<Parameters<typeof agent>[1]>['functions']\n\n  const analyst = agent<{ question: string }, { answer: string; findings: string[] }>(\n    // `reasoning!` is an internal (Ax `!`) scratchpad field: generated first to\n    // force reason-before-conclude, stripped from the returned output — so the\n    // consumed shape stays { answer, findings }. Brings the trace-analyst to the\n    // same prose-first CoT ordering the kind-factory gets from its `report` field.\n    'question:string -> reasoning!:string, answer:string, findings:string[]',\n    {\n      agentIdentity: {\n        name: 'TraceAnalyst',\n        description:\n          'Analyzes OTLP-shaped JSONL traces using bounded discovery tools to identify systemic failure modes.',\n      },\n      contextFields: ['question'],\n      runtime: new AxJSRuntime({\n        permissions: [],\n        blockDynamicImport: true,\n        allowedModules: [],\n        freezeIntrinsics: true,\n        blockShadowRealm: true,\n        // RLM stdout mode relies on runtime bindings persisting across turns.\n        preventGlobalThisExtensions: false,\n      }),\n      mode: maxDepth > 0 ? 'advanced' : 'simple',\n      recursionOptions: maxDepth > 0 ? { maxDepth } : undefined,\n      maxTurns,\n      maxRuntimeChars,\n      maxBatchedLlmQueryConcurrency: maxParallelSubagents,\n      promptLevel: 'detailed',\n      // Trace analysis depends on exact prior tool results and runtime variables.\n      contextPolicy: { preset: 'full', budget: 'balanced' },\n      functions,\n      actorOptions: {\n        description: options.actorDescription ?? TRACE_ANALYST_ACTOR_DESCRIPTION,\n        ...(options.model ? { model: options.model } : {}),\n        // Keep actor messages tool-call/content shaped across reasoning models.\n        showThoughts: false,\n        thinkingTokenBudget: 'none',\n      },\n      responderOptions: {\n        ...(options.model ? { model: options.model } : {}),\n        description: options.subagentDescription ?? TRACE_ANALYST_SUBAGENT_DESCRIPTION,\n        showThoughts: false,\n      },\n      actorTurnCallback,\n      bubbleErrors: [TraceFileMissingError],\n    },\n  )\n\n  let result: { answer: unknown; findings: unknown }\n  try {\n    result = await analyst.forward(options.ai, { question: input.question })\n  } finally {\n    if (progressFs) {\n      await new Promise<void>((resolve) => progressFs!.end(() => resolve()))\n    }\n  }\n\n  return {\n    answer: typeof result.answer === 'string' ? result.answer : String(result.answer ?? ''),\n    findings: Array.isArray(result.findings)\n      ? result.findings.filter((s): s is string => typeof s === 'string')\n      : [],\n    turns,\n    turnCount: turns.length,\n    usage: normalizeRoleArrays(analyst.getUsage()),\n    chatLog: normalizeRoleArrays(analyst.getChatLog()),\n    actorPromptVersion: TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,\n  }\n}\n\nfunction normalizeRoleArrays(value: unknown): {\n  actor: Record<string, unknown>[]\n  responder: Record<string, unknown>[]\n} {\n  const record = value && typeof value === 'object' ? (value as Record<string, unknown>) : {}\n  return {\n    actor: normalizeRecordArray(record.actor),\n    responder: normalizeRecordArray(record.responder),\n  }\n}\n\nfunction normalizeRecordArray(value: unknown): Record<string, unknown>[] {\n  if (!Array.isArray(value)) return []\n  return value.map((item) =>\n    item && typeof item === 'object' ? { ...(item as Record<string, unknown>) } : { value: item },\n  )\n}\n"],"mappings":";;;;;;;AAEO,IAAM,kCAAkC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA8DxC,IAAM,0CAA0C;AAGhD,IAAM,qCAAqC;AAAA;AAAA;;;ACnElD,SAA8D,aAAa,aAAa;AAoGxF,eAAsB,cACpB,OACA,SAC8B;AAC9B,MAAI,CAAC,MAAM,YAAY,OAAO,MAAM,aAAa,UAAU;AACzD,UAAM,IAAI,UAAU,0DAA0D;AAAA,EAChF;AAEA,QAAM,QACJ,OAAO,QAAQ,WAAW,WACtB,IAAI,mBAAmB,EAAE,MAAM,QAAQ,OAAO,CAAC,IAC/C,QAAQ;AAGd,MAAI,iBAAiB,oBAAoB;AACvC,UAAM,MAAM,cAAc;AAAA,EAC5B;AAEA,QAAM,QAAsB,uBAAuB,EAAE,MAAM,CAAC;AAC5D,QAAM,QAAqC,CAAC;AAG5C,MAAI;AACJ,MAAI,QAAQ,iBAAiB;AAC3B,UAAM,EAAE,kBAAkB,IAAI,MAAM,OAAO,IAAS;AACpD,UAAM,EAAE,MAAM,IAAI,MAAM,OAAO,aAAkB;AACjD,UAAM,EAAE,QAAQ,IAAI,MAAM,OAAO,MAAW;AAC5C,UAAM,MAAM,QAAQ,QAAQ,eAAe,GAAG,EAAE,WAAW,KAAK,CAAC;AACjE,iBAAa,kBAAkB,QAAQ,iBAAiB,EAAE,OAAO,IAAI,CAAC;AAAA,EACxE;AAEA,QAAM,oBAAoB,OAAO,SAAqC;AACpE,UAAM,OAAkC;AAAA,MACtC,MAAM,KAAK;AAAA,MACX,SAAS,KAAK;AAAA,MACd,MAAM,KAAK;AAAA,MACX,QAAQ,KAAK;AAAA,MACb,SAAS,KAAK;AAAA,IAChB;AACA,UAAM,KAAK,IAAI;AACf,QAAI,YAAY;AACd,UAAI;AACF,mBAAW,MAAM,GAAG,KAAK,UAAU,EAAE,GAAG,MAAM,IAAI,KAAK,IAAI,EAAE,CAAC,CAAC;AAAA,CAAI;AAAA,MACrE,QAAQ;AAAA,MAER;AAAA,IACF;AACA,QAAI,QAAQ,OAAQ,OAAM,QAAQ,OAAO,IAAI;AAAA,EAC/C;AAEA,QAAM,WAAW,QAAQ,YAAY;AACrC,QAAM,WAAW,QAAQ,YAAY;AACrC,QAAM,uBAAuB,QAAQ,wBAAwB;AAC7D,QAAM,kBAAkB,QAAQ,mBAAmB;AACnD,QAAM,YAAY;AAElB,QAAM,UAAU;AAAA;AAAA;AAAA;AAAA;AAAA,IAKd;AAAA,IACA;AAAA,MACE,eAAe;AAAA,QACb,MAAM;AAAA,QACN,aACE;AAAA,MACJ;AAAA,MACA,eAAe,CAAC,UAAU;AAAA,MAC1B,SAAS,IAAI,YAAY;AAAA,QACvB,aAAa,CAAC;AAAA,QACd,oBAAoB;AAAA,QACpB,gBAAgB,CAAC;AAAA,QACjB,kBAAkB;AAAA,QAClB,kBAAkB;AAAA;AAAA,QAElB,6BAA6B;AAAA,MAC/B,CAAC;AAAA,MACD,MAAM,WAAW,IAAI,aAAa;AAAA,MAClC,kBAAkB,WAAW,IAAI,EAAE,SAAS,IAAI;AAAA,MAChD;AAAA,MACA;AAAA,MACA,+BAA+B;AAAA,MAC/B,aAAa;AAAA;AAAA,MAEb,eAAe,EAAE,QAAQ,QAAQ,QAAQ,WAAW;AAAA,MACpD;AAAA,MACA,cAAc;AAAA,QACZ,aAAa,QAAQ,oBAAoB;AAAA,QACzC,GAAI,QAAQ,QAAQ,EAAE,OAAO,QAAQ,MAAM,IAAI,CAAC;AAAA;AAAA,QAEhD,cAAc;AAAA,QACd,qBAAqB;AAAA,MACvB;AAAA,MACA,kBAAkB;AAAA,QAChB,GAAI,QAAQ,QAAQ,EAAE,OAAO,QAAQ,MAAM,IAAI,CAAC;AAAA,QAChD,aAAa,QAAQ,uBAAuB;AAAA,QAC5C,cAAc;AAAA,MAChB;AAAA,MACA;AAAA,MACA,cAAc,CAAC,qBAAqB;AAAA,IACtC;AAAA,EACF;AAEA,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,QAAQ,QAAQ,QAAQ,IAAI,EAAE,UAAU,MAAM,SAAS,CAAC;AAAA,EACzE,UAAE;AACA,QAAI,YAAY;AACd,YAAM,IAAI,QAAc,CAAC,YAAY,WAAY,IAAI,MAAM,QAAQ,CAAC,CAAC;AAAA,IACvE;AAAA,EACF;AAEA,SAAO;AAAA,IACL,QAAQ,OAAO,OAAO,WAAW,WAAW,OAAO,SAAS,OAAO,OAAO,UAAU,EAAE;AAAA,IACtF,UAAU,MAAM,QAAQ,OAAO,QAAQ,IACnC,OAAO,SAAS,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ,IAChE,CAAC;AAAA,IACL;AAAA,IACA,WAAW,MAAM;AAAA,IACjB,OAAO,oBAAoB,QAAQ,SAAS,CAAC;AAAA,IAC7C,SAAS,oBAAoB,QAAQ,WAAW,CAAC;AAAA,IACjD,oBAAoB;AAAA,EACtB;AACF;AAEA,SAAS,oBAAoB,OAG3B;AACA,QAAM,SAAS,SAAS,OAAO,UAAU,WAAY,QAAoC,CAAC;AAC1F,SAAO;AAAA,IACL,OAAO,qBAAqB,OAAO,KAAK;AAAA,IACxC,WAAW,qBAAqB,OAAO,SAAS;AAAA,EAClD;AACF;AAEA,SAAS,qBAAqB,OAA2C;AACvE,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO,CAAC;AACnC,SAAO,MAAM;AAAA,IAAI,CAAC,SAChB,QAAQ,OAAO,SAAS,WAAW,EAAE,GAAI,KAAiC,IAAI,EAAE,OAAO,KAAK;AAAA,EAC9F;AACF;","names":[]}