@tangle-network/agent-eval 0.137.0 → 0.139.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +78 -0
- package/README.md +34 -0
- package/dist/analyst/index.d.ts +485 -104
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +10 -607
- package/dist/analyst/index.js.map +1 -1
- package/dist/{benchmark-YDrpumqB.js → benchmark-CYtcIF2V.js} +299 -159
- package/dist/benchmark-CYtcIF2V.js.map +1 -0
- package/dist/{benchmark-CHX4orG7.d.ts → benchmark-DDVdWcwA.d.ts} +67 -15
- package/dist/benchmark-DDVdWcwA.d.ts.map +1 -0
- package/dist/benchmark-command-BKfjOBJ5.js +4537 -0
- package/dist/benchmark-command-BKfjOBJ5.js.map +1 -0
- package/dist/benchmarks/index.d.ts +1 -1
- package/dist/benchmarks/index.js +1 -1
- package/dist/{benchmarks-DCLkQOmc.js → benchmarks-zxhy1QV3.js} +4 -3
- package/dist/{benchmarks-DCLkQOmc.js.map → benchmarks-zxhy1QV3.js.map} +1 -1
- package/dist/campaign/index.d.ts +5 -5
- package/dist/campaign/index.js +4 -3
- package/dist/{campaign-lgObcHFC.js → campaign-DrS6_hLd.js} +19 -11
- package/dist/campaign-DrS6_hLd.js.map +1 -0
- package/dist/canonical-D011XM8r.js +86 -0
- package/dist/canonical-D011XM8r.js.map +1 -0
- package/dist/cli.js +10 -3
- package/dist/cli.js.map +1 -1
- package/dist/{client-C8L6h6Wf.d.ts → client-BohnDFBq.d.ts} +4 -4
- package/dist/{client-C8L6h6Wf.d.ts.map → client-BohnDFBq.d.ts.map} +1 -1
- package/dist/{completion-verifier-DSyRNVzU.d.ts → completion-verifier-IPoP4fQO.d.ts} +178 -4
- package/dist/completion-verifier-IPoP4fQO.d.ts.map +1 -0
- package/dist/contract/index.d.ts +10 -10
- package/dist/contract/index.js +9 -8
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +2 -2
- package/dist/{cost-ledger-D-5_-dhi.js → cost-ledger-CZ9diLxY.js} +90 -45
- package/dist/cost-ledger-CZ9diLxY.js.map +1 -0
- package/dist/{cost-ledger-D2o6JOrL.d.ts → cost-ledger-DKgyIWRj.d.ts} +6 -2
- package/dist/cost-ledger-DKgyIWRj.d.ts.map +1 -0
- package/dist/default-registry-B8vf7Rmf.d.ts +118 -0
- package/dist/default-registry-B8vf7Rmf.d.ts.map +1 -0
- package/dist/default-registry-BgJJItGr.js +2364 -0
- package/dist/default-registry-BgJJItGr.js.map +1 -0
- package/dist/dspy-rlm-engine-DTkVyDX-.js +344 -0
- package/dist/dspy-rlm-engine-DTkVyDX-.js.map +1 -0
- package/dist/{eval-campaign-CHqfLnff.js → eval-campaign-BmptJj50.js} +2 -2
- package/dist/{eval-campaign-CHqfLnff.js.map → eval-campaign-BmptJj50.js.map} +1 -1
- package/dist/exact-types-MaaFcllV.d.ts +234 -0
- package/dist/exact-types-MaaFcllV.d.ts.map +1 -0
- package/dist/external-optimizer-contracts-BrxY2Sli.d.ts +32 -0
- package/dist/external-optimizer-contracts-BrxY2Sli.d.ts.map +1 -0
- package/dist/{extract-usage-p-56bh8q.js → extract-usage-DZs601Va.js} +2 -2
- package/dist/{extract-usage-p-56bh8q.js.map → extract-usage-DZs601Va.js.map} +1 -1
- package/dist/{feedback-trajectory-N_F0PwHz.d.ts → feedback-trajectory-BJUWOkJM.d.ts} +3 -2
- package/dist/feedback-trajectory-BJUWOkJM.d.ts.map +1 -0
- package/dist/fuzz.d.ts +1 -1
- package/dist/fuzz.js +1 -1
- package/dist/{hf-dataset-DBJXXoY1.js → hf-dataset-XggBupCr.js} +2 -2
- package/dist/{hf-dataset-DBJXXoY1.js.map → hf-dataset-XggBupCr.js.map} +1 -1
- package/dist/hosted/index.d.ts +3 -3
- package/dist/{index-C-Pr4OWg.d.ts → index-BTm_P9aC.d.ts} +12 -11
- package/dist/index-BTm_P9aC.d.ts.map +1 -0
- package/dist/{index-U3RHOShi.d.ts → index-CWOPCJiw.d.ts} +2 -2
- package/dist/{index-U3RHOShi.d.ts.map → index-CWOPCJiw.d.ts.map} +1 -1
- package/dist/{index-DRNl6g_N.d.ts → index-CtR1xh4V.d.ts} +3 -3
- package/dist/{index-DRNl6g_N.d.ts.map → index-CtR1xh4V.d.ts.map} +1 -1
- package/dist/index-DEb46kc6.d.ts.map +1 -1
- package/dist/{index-BnP1QJUv.d.ts → index-_66rVpwN.d.ts} +5 -5
- package/dist/{index-BnP1QJUv.d.ts.map → index-_66rVpwN.d.ts.map} +1 -1
- package/dist/index.d.ts +35 -55
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +55 -514
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-B9ooYH_g.d.ts → insight-report-Bu5Wi9tG.d.ts} +4 -4
- package/dist/{insight-report-B9ooYH_g.d.ts.map → insight-report-Bu5Wi9tG.d.ts.map} +1 -1
- package/dist/{integrity-CKxosZ5Z.d.ts → integrity-COTh3DTH.d.ts} +2 -2
- package/dist/{integrity-CKxosZ5Z.d.ts.map → integrity-COTh3DTH.d.ts.map} +1 -1
- package/dist/kind-factory-CFxA0JQX.js +2133 -0
- package/dist/kind-factory-CFxA0JQX.js.map +1 -0
- package/dist/ledger-core/index.js +2 -1
- package/dist/{ledger-core-t6sItivm.js → ledger-core-Dxz0Rkwa.js} +210 -99
- package/dist/ledger-core-Dxz0Rkwa.js.map +1 -0
- package/dist/{llm-client-DKB25jV8.js → llm-client-bkztEfIx.js} +5 -5
- package/dist/llm-client-bkztEfIx.js.map +1 -0
- package/dist/meta-eval/index.d.ts +2 -2
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{proposal-findings-DCawte-y.js → proposal-findings-2GIUo1et.js} +2 -68
- package/dist/proposal-findings-2GIUo1et.js.map +1 -0
- package/dist/{release-report-CofgVNZt.d.ts → release-report-fZarvIm-.d.ts} +3 -3
- package/dist/{release-report-CofgVNZt.d.ts.map → release-report-fZarvIm-.d.ts.map} +1 -1
- package/dist/{replay-K8FaC0CB.d.ts → replay-DjG4IG60.d.ts} +34 -143
- package/dist/replay-DjG4IG60.d.ts.map +1 -0
- package/dist/{replay-Bju0T8Ls.js → replay-SA4OB7O7.js} +48 -136
- package/dist/replay-SA4OB7O7.js.map +1 -0
- package/dist/reporting.d.ts +4 -4
- package/dist/{researcher-Da0Wj-bt.d.ts → researcher-BxhtGfKa.d.ts} +5 -5
- package/dist/{researcher-Da0Wj-bt.d.ts.map → researcher-BxhtGfKa.d.ts.map} +1 -1
- package/dist/{reward-hacking-CQ3hTCO3.d.ts → reward-hacking-CqSLiV51.d.ts} +2 -2
- package/dist/{reward-hacking-CQ3hTCO3.d.ts.map → reward-hacking-CqSLiV51.d.ts.map} +1 -1
- package/dist/rl.d.ts +5 -5
- package/dist/rl.js +1 -1
- package/dist/rollout/index.d.ts +1 -1
- package/dist/rollout/index.js +2 -2
- package/dist/{rollout-DQFl0UXA.js → rollout-8nj3mYvx.js} +2 -2
- package/dist/{rollout-DQFl0UXA.js.map → rollout-8nj3mYvx.js.map} +1 -1
- package/dist/{rubric-predictive-validity-C4sztLR3.d.ts → rubric-predictive-validity-DQBQj6uV.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-C4sztLR3.d.ts.map → rubric-predictive-validity-DQBQj6uV.d.ts.map} +1 -1
- package/dist/{run-evidence-BDIircdA.d.ts → run-evidence-C4RcRQT5.d.ts} +3 -3
- package/dist/{run-evidence-BDIircdA.d.ts.map → run-evidence-C4RcRQT5.d.ts.map} +1 -1
- package/dist/{run-record-BPCa2rQ8.d.ts → run-record-CztDMXVF.d.ts} +2 -2
- package/dist/{run-record-BPCa2rQ8.d.ts.map → run-record-CztDMXVF.d.ts.map} +1 -1
- package/dist/{semantic-concept-judge-Bz64IckK.js → semantic-concept-judge-BuIJ9IfB.js} +49 -6
- package/dist/semantic-concept-judge-BuIJ9IfB.js.map +1 -0
- package/dist/{server-KjXZZUDX.js → server-DaCpLfi0.js} +3 -3
- package/dist/{server-KjXZZUDX.js.map → server-DaCpLfi0.js.map} +1 -1
- package/dist/single-run-lock-BTTtPZ9N.js +989 -0
- package/dist/single-run-lock-BTTtPZ9N.js.map +1 -0
- package/dist/{skill-usage-CFDLLlhF.d.ts → skill-usage-B-BFS8M2.d.ts} +65 -40
- package/dist/skill-usage-B-BFS8M2.d.ts.map +1 -0
- package/dist/{skillopt-optimization-method-f4o9sUT4.js → skillopt-optimization-method-BbGnCC53.js} +20 -979
- package/dist/skillopt-optimization-method-BbGnCC53.js.map +1 -0
- package/dist/{skillopt-optimization-method-BpbnlvAZ.d.ts → skillopt-optimization-method-_s0Tub7Y.d.ts} +11 -39
- package/dist/skillopt-optimization-method-_s0Tub7Y.d.ts.map +1 -0
- package/dist/{statistics-_7P642CN.d.ts → statistics-B5d0Zd-z.d.ts} +2 -2
- package/dist/{statistics-_7P642CN.d.ts.map → statistics-B5d0Zd-z.d.ts.map} +1 -1
- package/dist/store-otlp-DX4fGIcf.js +757 -0
- package/dist/store-otlp-DX4fGIcf.js.map +1 -0
- package/dist/{summary-report-DHipz9Kx.d.ts → summary-report-Cg7BifAM.d.ts} +3 -3
- package/dist/{summary-report-DHipz9Kx.d.ts.map → summary-report-Cg7BifAM.d.ts.map} +1 -1
- package/dist/tool-groups-CdYq22lX.d.ts +258 -0
- package/dist/tool-groups-CdYq22lX.d.ts.map +1 -0
- package/dist/traces.d.ts +7 -6
- package/dist/traces.js +5 -4
- package/dist/{types-CTGbIm57.d.ts → types-BBFNHxSK.d.ts} +5 -5
- package/dist/{types-CTGbIm57.d.ts.map → types-BBFNHxSK.d.ts.map} +1 -1
- package/dist/{types-CKswbJGO.d.ts → types-DoEYskCd.d.ts} +5 -5
- package/dist/{types-CKswbJGO.d.ts.map → types-DoEYskCd.d.ts.map} +1 -1
- package/dist/{types-CTvKfr5F.d.ts → types-uPrS6mD-.d.ts} +2 -2
- package/dist/{types-CTvKfr5F.d.ts.map → types-uPrS6mD-.d.ts.map} +1 -1
- package/dist/usage-receipt-CgxMEBZq.js +134 -0
- package/dist/usage-receipt-CgxMEBZq.js.map +1 -0
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.js +1 -1
- package/docs/trace-analysis.md +191 -385
- package/package.json +5 -4
- package/dist/analyze-runs-PVtnfjvA.d.ts +0 -72
- package/dist/analyze-runs-PVtnfjvA.d.ts.map +0 -1
- package/dist/benchmark-CHX4orG7.d.ts.map +0 -1
- package/dist/benchmark-YDrpumqB.js.map +0 -1
- package/dist/campaign-lgObcHFC.js.map +0 -1
- package/dist/completion-verifier-DSyRNVzU.d.ts.map +0 -1
- package/dist/concurrency-MUjT7VjM.js +0 -109
- package/dist/concurrency-MUjT7VjM.js.map +0 -1
- package/dist/cost-ledger-D-5_-dhi.js.map +0 -1
- package/dist/cost-ledger-D2o6JOrL.d.ts.map +0 -1
- package/dist/default-registry-CLXbRt0f.js +0 -2594
- package/dist/default-registry-CLXbRt0f.js.map +0 -1
- package/dist/default-registry-Dc5D_Loc.d.ts +0 -202
- package/dist/default-registry-Dc5D_Loc.d.ts.map +0 -1
- package/dist/feedback-trajectory-N_F0PwHz.d.ts.map +0 -1
- package/dist/index-C-Pr4OWg.d.ts.map +0 -1
- package/dist/ledger-core-t6sItivm.js.map +0 -1
- package/dist/llm-client-DKB25jV8.js.map +0 -1
- package/dist/proposal-findings-DCawte-y.js.map +0 -1
- package/dist/registry-BdM7SuTr.d.ts +0 -124
- package/dist/registry-BdM7SuTr.d.ts.map +0 -1
- package/dist/replay-Bju0T8Ls.js.map +0 -1
- package/dist/replay-K8FaC0CB.d.ts.map +0 -1
- package/dist/semantic-concept-judge-Bz64IckK.js.map +0 -1
- package/dist/skill-usage-CFDLLlhF.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-BpbnlvAZ.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-f4o9sUT4.js.map +0 -1
- package/dist/tools-DZk2Jn64.js +0 -1876
- package/dist/tools-DZk2Jn64.js.map +0 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llm-client-bkztEfIx.js","names":[],"sources":["../src/llm-client.ts"],"sourcesContent":["/**\n * LLM client with graceful degrade.\n *\n * OpenAI-compatible `/v1/chat/completions` client with:\n * - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).\n * - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).\n * - One retry at temperature 1 when a model explicitly requires it.\n * - Graceful json_schema → json_object degrade on 400 with schema-reject body.\n * - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.\n * - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI\n * directly, cli-bridge subscriptions, and any router that speaks the spec.\n *\n * Usage:\n * const { value, result } = await callLlmJson<MyType>(\n * { model: 'gpt-4o', messages: [...], jsonSchema: { name: 'x', schema: {...} } },\n * { baseUrl: 'https://router.tangle.tools/v1', apiKey: process.env.KEY },\n * )\n *\n * `createChatClient` wraps this implementation for provider-neutral package\n * entry points. Direct callers can use `callLlm` or `callLlmJson`.\n */\n\nimport {\n type CostReceiptInput,\n type CustomTokenPricing,\n costForTokenPricing,\n type MaximumCharge,\n} from './cost-ledger'\nimport { AgentEvalError, CaptureIntegrityError } from './errors'\nimport {\n defaultProviderRedactor,\n type ProviderRedactor,\n providerFromBaseUrl,\n type RawProviderEvent,\n type RawProviderSink,\n} from './trace/raw-provider-sink'\n\n// ─── Types ──────────────────────────────────────────────────────────────\n\nexport interface LlmMessage {\n role: 'system' | 'user' | 'assistant'\n /**\n * Either a plain text content string OR a multimodal content array\n * (text + image_url parts) for vision-capable models.\n */\n content:\n | string\n | Array<\n | { type: 'text'; text: string }\n | { type: 'image_url'; image_url: { url: string; detail?: 'auto' | 'low' | 'high' } }\n >\n}\n\nexport type LlmThinkingMode = 'enabled' | 'disabled'\n\nexport interface LlmCallRequest {\n model: string\n messages: LlmMessage[]\n /** Optional JSON-mode response format (response_format: json_object). */\n jsonMode?: boolean\n /** Optional structured output via JSON Schema. Falls back to json_object on 400. */\n jsonSchema?: { name: string; schema: Record<string, unknown> }\n temperature?: number\n maxTokens?: number\n /** OpenAI-compatible reasoning mode. Omitted when the provider default should apply. */\n thinking?: LlmThinkingMode\n /** Per-call timeout, default 300s. */\n timeoutMs?: number\n}\n\n/** Conservative priced bound for the exact text request sent to a provider.\n * Returns undefined when output or multimodal input is not bounded, causing a\n * capped CostLedger to reject the call before execution. Pass\n * `customTokenPricing` when package pricing does not cover the model or endpoint. */\nexport function maximumChargeForLlmRequest(\n request: Pick<LlmCallRequest, 'model' | 'messages' | 'jsonSchema' | 'maxTokens' | 'thinking'>,\n options: LlmClientOptions = {},\n): MaximumCharge | undefined {\n if (request.maxTokens === undefined) return undefined\n if (!Number.isInteger(request.maxTokens) || request.maxTokens <= 0) {\n throw new RangeError(`maximumChargeForLlmRequest: maxTokens must be a positive integer`)\n }\n if (\n request.messages.some(\n (message) =>\n Array.isArray(message.content) && message.content.some((part) => part.type === 'image_url'),\n )\n ) {\n return undefined\n }\n\n const attempts = resolveMaximumAttempts(options.maximumAttempts)\n const forceJsonObject = options.jsonSchemaTransport === 'json-object'\n // A byte-level tokenizer cannot emit more input tokens than request bytes.\n // Pricing the complete body also covers role/schema framing omitted from content-only estimates.\n const requestBytes = new TextEncoder().encode(\n JSON.stringify(buildBody(request, forceJsonObject, options.thinking)),\n ).byteLength\n // A rejected response schema can trigger one JSON-mode batch with the same output limit.\n const batches = request.jsonSchema && !forceJsonObject ? 2 : 1\n const usage = {\n inputTokens: requestBytes * attempts * batches,\n outputTokens: request.maxTokens * attempts * batches,\n }\n return options.customTokenPricing\n ? { customTokenPricing: options.customTokenPricing, ...usage }\n : { model: request.model, ...usage }\n}\n\nexport interface LlmUsage {\n promptTokens: number\n completionTokens: number\n totalTokens: number\n /** False when the provider omitted or malformed prompt/completion usage. */\n captured?: boolean\n /** Reasoning-token subset of completionTokens, when reported. */\n reasoningTokens?: number\n /** Proxies populate this when prompt caching is on. */\n cachedPromptTokens?: number\n}\n\nexport interface LlmCallResult {\n /** The text content of the first choice. Empty string if none. */\n content: string\n usage: LlmUsage\n /**\n * Cost in USD. Uses the provider's reported cost when present, otherwise\n * caller-supplied token pricing. `null` when neither is available.\n */\n costUsd: number | null\n /** Model name actually used (echoed from response). */\n model: string\n /** Wall-clock duration of the HTTP call (last attempt, if retried). */\n durationMs: number\n /**\n * `finish_reason` echoed from the first choice (`stop`, `length`,\n * `content_filter`, `tool_calls`, ...). `null` when the provider omits it.\n * Exposed so a free-form `callLlm` caller CAN detect a truncated answer\n * (`length`) instead of treating a cut-off completion as complete. Note:\n * `callLlm` does not itself reject on it — acting on this signal is the\n * caller's responsibility (in-repo free-form drivers do not yet enforce it).\n */\n finishReason?: string | null\n /**\n * True when `content.trim()` is empty. An empty completion is a silent zero\n * for free-form `callLlm` callers; this flag is the signal a caller can\n * inspect to fail loud rather than proceed on an empty string. `callLlm`\n * surfaces it but does not throw on it.\n */\n contentEmpty?: boolean\n /** Raw response body. */\n raw: Record<string, unknown>\n}\n\nexport type LlmCallMetadata = Pick<LlmCallResult, 'usage' | 'costUsd' | 'model' | 'durationMs'>\n\n/** Convert a provider result into the canonical paid-call receipt input. */\nexport function costReceiptFromLlm(\n result: LlmCallResult,\n customTokenPricing?: CustomTokenPricing,\n): CostReceiptInput {\n const cachedTokens = result.usage.cachedPromptTokens ?? 0\n const inputTokens = Math.max(0, result.usage.promptTokens - cachedTokens)\n const configuredCostUsd =\n result.costUsd === null && customTokenPricing && result.usage.captured !== false\n ? costForTokenPricing(customTokenPricing, {\n inputTokens,\n ...(cachedTokens > 0 ? { cachedTokens } : {}),\n outputTokens: result.usage.completionTokens,\n })\n : undefined\n return {\n model: result.model,\n inputTokens,\n outputTokens: result.usage.completionTokens,\n reasoningTokens: result.usage.reasoningTokens,\n cachedTokens: cachedTokens > 0 ? cachedTokens : undefined,\n actualCostUsd: result.costUsd ?? configuredCostUsd,\n usageUnknown: result.usage.captured === false,\n }\n}\n\n/** Structured-response failures retain their completed provider receipt. */\nexport function costReceiptFromLlmError(\n error: Error,\n customTokenPricing?: CustomTokenPricing,\n): CostReceiptInput | undefined {\n return error instanceof LlmResponseError\n ? costReceiptFromLlm(error.result, customTokenPricing)\n : undefined\n}\n\nexport class LlmCallError extends AgentEvalError {\n constructor(\n message: string,\n public readonly status: number,\n public readonly body: string,\n public readonly model: string,\n ) {\n super('judge', message)\n }\n}\n\n/** A provider response completed and incurred measurable usage, but its content\n * could not satisfy the caller's response contract. The response envelope is\n * retained so accounting can commit the receipt before the error propagates. */\nexport class LlmResponseError extends AgentEvalError {\n constructor(\n message: string,\n public readonly result: LlmCallResult,\n options?: { cause?: unknown },\n ) {\n super('judge', message, options)\n }\n}\n\nexport interface LlmClientOptions {\n /** Base URL (without trailing slash). Must end at the `/v1` prefix. */\n baseUrl?: string\n /** Bearer token — either `apiKey` or `bearer` populates `Authorization: Bearer ...`. */\n apiKey?: string\n bearer?: string\n /** Override for the `Authorization` header (e.g. `X-Auth: ...`). Takes precedence over apiKey/bearer. */\n authHeader?: { name: string; value: string }\n /** Stable provider idempotency key, reused across retries of this logical call. */\n idempotencyKey?: string\n /** Default timeout in ms. Per-call can override. */\n defaultTimeoutMs?: number\n /**\n * Caller-supplied abort signal — e.g. a campaign-wide cancel. Linked to\n * each attempt's per-attempt timeout controller, so aborting it cancels\n * the in-flight fetch. A caller abort is FATAL: it is not retried even\n * though an AbortError otherwise matches the transient patterns.\n */\n signal?: AbortSignal\n /**\n * Cross-attempt wall-clock budget in ms, measured from the first attempt.\n * Before launching each attempt the loop checks the remaining budget and\n * stops retrying once it is exhausted, rather than waiting the full\n * per-attempt timeout on every retry. Bounds total time independent of\n * total attempts × `timeoutMs`.\n */\n deadlineMs?: number\n /** Total provider attempts. Default 3. */\n maximumAttempts?: number\n /** Token rates used when the provider omits cost or package pricing does not cover the model. */\n customTokenPricing?: CustomTokenPricing\n /**\n * Transport for requests that declare `jsonSchema`. `native` sends\n * `response_format: json_schema`; `json-object` sends the broadly supported\n * JSON mode and relies on the caller to include the schema in model-visible\n * instructions. Default: `native`.\n */\n jsonSchemaTransport?: 'native' | 'json-object'\n /**\n * JSON payload parsing policy. `extract` accepts fenced or prose-prefixed JSON.\n * `exact` requires the complete response content to be one JSON value.\n * Default: `extract`.\n */\n jsonPayloadMode?: 'extract' | 'exact'\n /** Default provider reasoning mode. A per-call request value takes precedence. */\n thinking?: LlmThinkingMode\n /** Fetch implementation — defaults to global `fetch`. Override for custom transport (e.g. tests). */\n fetch?: typeof fetch\n /**\n * Optional raw HTTP capture sink. When provided, every request, response,\n * and error (across all retry attempts) is recorded to the sink, with auth\n * headers and credential-shaped body fields redacted by default. This is\n * the layer-1 forensics primitive: structured `LlmSpan`s record intent,\n * raw events record what actually crossed the wire.\n */\n rawSink?: RawProviderSink\n /**\n * Logical provider id attached to raw events. When omitted, derived from\n * `baseUrl` via `providerFromBaseUrl`.\n */\n provider?: string\n /** Trace context attached to raw events; populated by emitter-aware callers. */\n traceContext?: { runId?: string; spanId?: string }\n /** Override the redaction strategy for this call. Defaults to `defaultProviderRedactor`. */\n redactor?: ProviderRedactor\n}\n\n// ─── Internals ──────────────────────────────────────────────────────────\n\nconst DEFAULT_BASE_URL = 'https://router.tangle.tools/v1'\n// Flagship / reasoning models routinely take several minutes on large prompts (a\n// reflection over many failures, a long tool transcript). A tight cap aborts a\n// legitimately-slow but healthy call — and because every retry attempt re-uses\n// the same window, such a model aborts on ALL attempts and the loop throws. The\n// default is generous enough to let those complete, bounded enough that a truly\n// hung call still fails over after retries, and tunable per deployment via\n// TANGLE_LLM_TIMEOUT_MS. Per-call `req.timeoutMs` / `opts.defaultTimeoutMs`\n// still win for callers that know their model's latency.\nconst DEFAULT_TIMEOUT_MS = Number(process.env.TANGLE_LLM_TIMEOUT_MS) || 300_000\nconst DEFAULT_MAXIMUM_ATTEMPTS =\n process.env.TANGLE_LLM_MAXIMUM_ATTEMPTS === undefined\n ? 3\n : Number(process.env.TANGLE_LLM_MAXIMUM_ATTEMPTS)\n\nfunction resolveMaximumAttempts(configured: number | undefined): number {\n const attempts = configured ?? DEFAULT_MAXIMUM_ATTEMPTS\n if (!Number.isInteger(attempts) || attempts <= 0) {\n throw new RangeError('LLM maximum attempts must be a positive integer')\n }\n return attempts\n}\n\nfunction providerTokenCount(value: unknown): number | undefined {\n return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : undefined\n}\n\nconst RETRYABLE_STATUS = new Set([429, 502, 503, 504])\n\n/**\n * Transient transport/network error signatures, matched against an error's\n * name, message, and `code`. Covers fetch/undici network failures, aborts\n * and timeouts, and — critically — HTTP/2 transport faults a keep-alive\n * connection raises mid-response: `terminated`, `NGHTTP2_INTERNAL_ERROR`,\n * `UND_ERR_*`, `other side closed`. Those last ones carry no clean HTTP\n * status; unrecognised, they escape the retry loop and surface as an\n * uncaught rejection.\n */\nconst TRANSIENT_ERROR_PATTERNS: readonly RegExp[] = [\n /AbortError/i,\n /TimeoutError/i,\n /this operation was aborted/i,\n /fetch failed/i,\n /ECONNRESET/i,\n /ETIMEDOUT/i,\n /EAI_AGAIN/i,\n /socket hang up/i,\n /stream.*ended.*unexpectedly/i,\n /terminated/i,\n /other side closed/i,\n /NGHTTP2/i,\n /UND_ERR/i,\n]\n\n/**\n * True when an error is a transient transport/network fault worth retrying,\n * as opposed to a deterministic failure (4xx schema reject, JSON parse) that\n * a retry cannot fix. Inspects `LlmCallError.status`, then the error's\n * name/message/code, then recurses into `error.cause` — undici nests the\n * real socket fault one or more levels under `.cause`.\n *\n * This is the retry classifier for the package: `callLlm` and\n * `withJudgeRetry` both route through it, so connection failures are treated\n * consistently across transports.\n */\nexport function isTransientLlmError(err: unknown): boolean {\n return classifyTransient(err, 0)\n}\n\nfunction classifyTransient(err: unknown, depth: number): boolean {\n if (err instanceof LlmCallError) return RETRYABLE_STATUS.has(err.status)\n if (!(err instanceof Error)) return false\n // Foreign transport errors can carry a numeric HTTP status without being an\n // LlmCallError. A retryable status is decisive.\n const status = (err as { status?: unknown }).status\n if (typeof status === 'number' && RETRYABLE_STATUS.has(status)) return true\n const code = (err as { code?: unknown }).code\n const haystack = `${err.name}\\n${err.message}\\n${typeof code === 'string' ? code : ''}`\n if (TRANSIENT_ERROR_PATTERNS.some((p) => p.test(haystack))) return true\n const cause = (err as { cause?: unknown }).cause\n if (depth < 4 && cause instanceof Error && cause !== err) {\n return classifyTransient(cause, depth + 1)\n }\n return false\n}\n\nfunction parseRetryAfter(headers: Headers): number | null {\n const h = headers.get('retry-after')\n if (!h) return null\n const asNumber = Number(h)\n if (Number.isFinite(asNumber) && asNumber > 0) return asNumber * 1000\n const asDate = Date.parse(h)\n if (Number.isFinite(asDate)) return Math.max(0, asDate - Date.now())\n return null\n}\n\n/** Exponential backoff: 500ms, 1s, 2s, 4s, ... capped at 16s. Attempt is 0-indexed. */\nexport function backoffMs(attempt: number): number {\n return Math.min(500 * 2 ** attempt, 16_000)\n}\n\nfunction buildHeaders(opts: LlmClientOptions): Record<string, string> {\n const headers: Record<string, string> = {\n 'Content-Type': 'application/json',\n Accept: 'application/json',\n }\n if (opts.authHeader) {\n headers[opts.authHeader.name] = opts.authHeader.value\n } else if (opts.bearer || opts.apiKey) {\n headers.Authorization = `Bearer ${opts.bearer ?? opts.apiKey}`\n }\n if (opts.idempotencyKey) headers['Idempotency-Key'] = opts.idempotencyKey\n return headers\n}\n\nfunction isSchemaRejection(status: number, body: string): boolean {\n if (status !== 400) return false\n const lower = body.toLowerCase()\n return (\n lower.includes('response_format') ||\n lower.includes('json_schema') ||\n lower.includes('is unavailable') ||\n lower.includes('not supported')\n )\n}\n\nfunction isTemperatureOneRejection(status: number, body: string): boolean {\n if (status !== 400 || !/temperature/i.test(body)) return false\n return (\n /temperature[^.\\n]{0,120}\\b(?:only|must|should|required|requires?)\\b[^.\\n]{0,40}\\b1(?:\\.0+)?\\b/i.test(\n body,\n ) || /\\bonly\\s+1(?:\\.0+)?\\s+is\\s+allowed\\b[^.\\n]{0,120}\\btemperature\\b/i.test(body)\n )\n}\n\nfunction buildBody(\n req: LlmCallRequest,\n forceJsonObject: boolean,\n defaultThinking?: LlmThinkingMode,\n): Record<string, unknown> {\n const body: Record<string, unknown> = {\n model: req.model,\n messages: req.messages,\n temperature: req.temperature ?? 0,\n }\n if (req.maxTokens != null) {\n if (usesMaxCompletionTokens(req.model)) body.max_completion_tokens = req.maxTokens\n else body.max_tokens = req.maxTokens\n }\n const thinking = req.thinking ?? defaultThinking\n if (thinking !== undefined) {\n body.thinking = { type: thinking }\n }\n\n if (req.jsonSchema && !forceJsonObject) {\n body.response_format = {\n type: 'json_schema',\n json_schema: { name: req.jsonSchema.name, schema: req.jsonSchema.schema, strict: true },\n }\n } else if (req.jsonMode || req.jsonSchema) {\n body.response_format = { type: 'json_object' }\n }\n\n return body\n}\n\nfunction usesMaxCompletionTokens(model: string): boolean {\n return /^gpt-5(?:[.-]|$)/i.test(model)\n}\n\nasync function sleep(ms: number): Promise<void> {\n return new Promise((resolve) => setTimeout(resolve, ms))\n}\n\n/**\n * Combine the per-attempt timeout signal with an optional caller signal into\n * one signal the fetch listens on. Prefers the native `AbortSignal.any`; falls\n * back to manual wiring on runtimes that predate it. The caller signal is also\n * propagated to the timeout controller so aborting it cancels the in-flight\n * fetch immediately.\n */\nfunction linkSignals(timeoutController: AbortController, caller?: AbortSignal): AbortSignal {\n if (!caller) return timeoutController.signal\n if (typeof (AbortSignal as { any?: unknown }).any === 'function') {\n return AbortSignal.any([timeoutController.signal, caller])\n }\n if (caller.aborted) {\n timeoutController.abort()\n } else {\n caller.addEventListener('abort', () => timeoutController.abort(), { once: true })\n }\n return timeoutController.signal\n}\n\n/** True once the cross-attempt wall-clock budget (if any) is exhausted. */\nfunction deadlineExceeded(start: number, deadlineMs: number | undefined): boolean {\n return deadlineMs != null && Date.now() - start >= deadlineMs\n}\n\n// ─── Public API ─────────────────────────────────────────────────────────\n\n/**\n * Strip a ```json / ``` code fence if the model emitted one.\n * Idempotent for naked JSON. Some models (claude-code via router, certain\n * deepseek models) wrap output even under json_object.\n */\nexport function stripFencedJson(raw: string): string {\n const trimmed = raw.trim()\n const m = trimmed.match(/^```(?:json)?\\s*\\n?([\\s\\S]*?)\\n?```\\s*$/)\n return m ? m[1]!.trim() : trimmed\n}\n\nexport function extractJsonPayload(raw: string): string {\n const stripped = stripFencedJson(raw)\n try {\n JSON.parse(stripped)\n return stripped\n } catch {\n // A response that declares a JSON root must parse as that complete root.\n // Scanning onward could turn a truncated object into one of its valid nested\n // arrays or objects and silently change the response schema.\n if (stripped.startsWith('{') || stripped.startsWith('[')) return stripped\n }\n\n // Only prose-leading responses may contain a recoverable JSON payload.\n const starts = [...stripped.matchAll(/[[{]/g)]\n .map((match) => match.index)\n .filter((index) => index != null)\n for (const start of starts) {\n const candidate = extractBalancedJson(stripped, start)\n if (!candidate) continue\n try {\n JSON.parse(candidate)\n return candidate\n } catch {\n // Keep scanning; earlier braces may belong to prose.\n }\n }\n\n return stripped\n}\n\nfunction extractBalancedJson(input: string, start: number): string | null {\n const opener = input[start]\n const closer = opener === '{' ? '}' : opener === '[' ? ']' : null\n if (!closer) return null\n\n const stack: string[] = [closer]\n let isInString = false\n let isEscaped = false\n\n for (let i = start + 1; i < input.length; i++) {\n const char = input[i]!\n if (isEscaped) {\n isEscaped = false\n continue\n }\n if (char === '\\\\') {\n isEscaped = isInString\n continue\n }\n if (char === '\"') {\n isInString = !isInString\n continue\n }\n if (isInString) continue\n\n if (char === '{') stack.push('}')\n else if (char === '[') stack.push(']')\n else if (char === stack[stack.length - 1]) {\n stack.pop()\n if (stack.length === 0) return input.slice(start, i + 1)\n }\n }\n\n return null\n}\n\n/**\n * Low-level call. Returns raw content + usage + cost. Retries on transient\n * failures; does NOT degrade schema here — callers that want graceful\n * degrade use `callLlmJson`.\n */\nexport async function callLlm(\n req: LlmCallRequest,\n opts: LlmClientOptions = {},\n): Promise<LlmCallResult> {\n const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\\/+$/, '')\n const url = `${baseUrl}/chat/completions`\n const endpoint = '/chat/completions'\n const timeoutMs = req.timeoutMs ?? opts.defaultTimeoutMs ?? DEFAULT_TIMEOUT_MS\n const maximumAttempts = resolveMaximumAttempts(opts.maximumAttempts)\n const fetchFn = opts.fetch ?? globalThis.fetch\n const headers = buildHeaders(opts)\n const provider = opts.provider ?? providerFromBaseUrl(baseUrl)\n const sink = opts.rawSink\n const redactor = opts.redactor ?? defaultProviderRedactor\n const traceContext = opts.traceContext\n const callerSignal = opts.signal\n const deadlineMs = opts.deadlineMs\n const deadlineStart = Date.now()\n if (opts.customTokenPricing) {\n costForTokenPricing(opts.customTokenPricing, { inputTokens: 0, outputTokens: 0 })\n }\n\n let lastErr: unknown\n let effectiveRequest = req\n for (let attempt = 0; attempt < maximumAttempts; attempt++) {\n // A caller cancel is fatal — never retried. Checking before each attempt\n // means an already-aborted signal short-circuits without firing fetch.\n if (callerSignal?.aborted) {\n throw new DOMException('callLlm aborted by caller signal', 'AbortError')\n }\n // Stop retrying once the cross-attempt budget is spent rather than burning\n // a full per-attempt timeout on each remaining retry.\n if (attempt > 0 && deadlineExceeded(deadlineStart, deadlineMs)) {\n throw lastErr instanceof Error ? lastErr : new Error(String(lastErr))\n }\n const controller = new AbortController()\n const attemptSignal = linkSignals(controller, callerSignal)\n const timeoutHandle = setTimeout(() => controller.abort(), timeoutMs)\n const started = Date.now()\n const requestBody = buildBody(\n effectiveRequest,\n opts.jsonSchemaTransport === 'json-object',\n opts.thinking,\n )\n let attemptErrorRecorded = false\n if (sink) {\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'request',\n timestamp: started,\n requestHeaders: headers,\n requestBody,\n redactedFields: [],\n })\n }\n\n try {\n const res = await fetchFn(url, {\n method: 'POST',\n headers,\n body: JSON.stringify(requestBody),\n signal: attemptSignal,\n })\n clearTimeout(timeoutHandle)\n const responseHeaders = sink ? headersToObject(res.headers) : undefined\n\n if (!res.ok) {\n const body = await res.text()\n if (sink) {\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'error',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n statusCode: res.status,\n responseHeaders,\n responseBody: body,\n errorMessage: `HTTP ${res.status}`,\n redactedFields: [],\n })\n attemptErrorRecorded = true\n }\n const err = new LlmCallError(\n `LLM call failed with HTTP ${res.status}`,\n res.status,\n body,\n req.model,\n )\n if (\n isTemperatureOneRejection(res.status, body) &&\n effectiveRequest.temperature !== 1 &&\n attempt < maximumAttempts - 1 &&\n !deadlineExceeded(deadlineStart, deadlineMs)\n ) {\n lastErr = err\n effectiveRequest = { ...effectiveRequest, temperature: 1 }\n continue\n }\n if (\n RETRYABLE_STATUS.has(res.status) &&\n attempt < maximumAttempts - 1 &&\n !deadlineExceeded(deadlineStart, deadlineMs)\n ) {\n lastErr = err\n const retryAfter = parseRetryAfter(res.headers)\n await sleep(retryAfter ?? backoffMs(attempt))\n continue\n }\n throw err\n }\n\n const text = await res.text()\n let json: Record<string, unknown>\n try {\n json = JSON.parse(text) as Record<string, unknown>\n } catch (parseErr) {\n if (sink) {\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'error',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n statusCode: res.status,\n responseHeaders,\n responseBody: text,\n errorMessage: `non-JSON response: ${parseErr instanceof Error ? parseErr.message : String(parseErr)}`,\n redactedFields: [],\n })\n attemptErrorRecorded = true\n }\n throw parseErr\n }\n if (sink) {\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'response',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n statusCode: res.status,\n responseHeaders,\n responseBody: json,\n redactedFields: [],\n })\n }\n const choice = (\n json.choices as\n | Array<{ message?: { content?: string }; finish_reason?: string | null }>\n | undefined\n )?.[0]\n const usageRaw =\n json.usage && typeof json.usage === 'object' && !Array.isArray(json.usage)\n ? (json.usage as Record<string, unknown>)\n : undefined\n const promptTokens = providerTokenCount(usageRaw?.prompt_tokens)\n const completionTokens = providerTokenCount(usageRaw?.completion_tokens)\n const totalTokens = providerTokenCount(usageRaw?.total_tokens)\n const completionDetails =\n usageRaw?.completion_tokens_details &&\n typeof usageRaw.completion_tokens_details === 'object' &&\n !Array.isArray(usageRaw.completion_tokens_details)\n ? (usageRaw.completion_tokens_details as Record<string, unknown>)\n : undefined\n const reasoningRaw = completionDetails?.reasoning_tokens\n const reasoningTokens =\n reasoningRaw === undefined ? undefined : providerTokenCount(reasoningRaw)\n const cachedRaw =\n usageRaw?.prompt_tokens_details &&\n typeof usageRaw.prompt_tokens_details === 'object' &&\n !Array.isArray(usageRaw.prompt_tokens_details)\n ? (usageRaw.prompt_tokens_details as Record<string, unknown>).cached_tokens\n : undefined\n const cachedPromptTokens = cachedRaw === undefined ? undefined : providerTokenCount(cachedRaw)\n const usageCaptured =\n promptTokens !== undefined &&\n completionTokens !== undefined &&\n (reasoningRaw === undefined ||\n (reasoningTokens !== undefined && reasoningTokens <= completionTokens)) &&\n (cachedRaw === undefined ||\n (cachedPromptTokens !== undefined && cachedPromptTokens <= promptTokens)) &&\n (totalTokens === undefined || totalTokens === promptTokens + completionTokens)\n const costFromProxy = (json._response_cost ?? json.cost_usd) as number | undefined\n const content = choice?.message?.content ?? ''\n\n const configuredCost =\n typeof costFromProxy !== 'number' && usageCaptured && opts.customTokenPricing\n ? costForTokenPricing(opts.customTokenPricing, {\n inputTokens: promptTokens! - (cachedPromptTokens ?? 0),\n ...(cachedPromptTokens ? { cachedTokens: cachedPromptTokens } : {}),\n outputTokens: completionTokens!,\n })\n : undefined\n\n return {\n content,\n finishReason: choice?.finish_reason ?? null,\n contentEmpty: content.trim().length === 0,\n usage: {\n promptTokens: promptTokens ?? 0,\n completionTokens: completionTokens ?? 0,\n totalTokens: totalTokens ?? (promptTokens ?? 0) + (completionTokens ?? 0),\n captured: usageCaptured,\n reasoningTokens,\n cachedPromptTokens,\n },\n costUsd: typeof costFromProxy === 'number' ? costFromProxy : (configuredCost ?? null),\n model: (json.model as string) ?? req.model,\n durationMs: Date.now() - started,\n raw: json,\n }\n } catch (err) {\n clearTimeout(timeoutHandle)\n lastErr = err\n // A caller cancel is fatal even though an AbortError matches the\n // transient patterns — a cancelled call must surface immediately, not\n // be retried against the same dead intent.\n if (callerSignal?.aborted) {\n if (sink && !attemptErrorRecorded) {\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'error',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n errorMessage: err instanceof Error ? err.message : String(err),\n redactedFields: [],\n })\n }\n throw err\n }\n if (sink && !attemptErrorRecorded) {\n // Record only if neither the !res.ok branch nor the JSON.parse catch\n // already produced an error event for this attempt. Covers network\n // failures, timeouts, and aborts.\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'error',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n errorMessage: err instanceof Error ? err.message : String(err),\n redactedFields: [],\n })\n }\n if (\n attempt < maximumAttempts - 1 &&\n isTransientLlmError(err) &&\n !deadlineExceeded(deadlineStart, deadlineMs)\n ) {\n await sleep(backoffMs(attempt))\n continue\n }\n throw err\n }\n }\n throw lastErr instanceof Error ? lastErr : new Error(String(lastErr))\n}\n\nasync function recordRaw(\n sink: RawProviderSink,\n redactor: ProviderRedactor,\n event: RawProviderEvent,\n): Promise<void> {\n // Errors from sinks must not crash the LLM call. Forensic capture is\n // best-effort; the structured trace is the system of record.\n try {\n await sink.record(redactor(event))\n } catch {\n // Intentionally swallowed.\n }\n}\n\nfunction headersToObject(h: Headers): Record<string, string> {\n const out: Record<string, string> = {}\n h.forEach((value, key) => {\n out[key] = value\n })\n return out\n}\n\nfunction cryptoEventId(): string {\n if (typeof globalThis.crypto?.randomUUID === 'function') return globalThis.crypto.randomUUID()\n return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`\n}\n\n/**\n * Structured-output call. Returns parsed JSON plus the raw result envelope.\n * Degrades `jsonSchema` → `jsonMode` on a 400 that names the schema param —\n * critical for deepseek-v3/v4, kimi-k2.6, and other models that don't accept\n * the `response_format.json_schema` shape but DO accept `json_object`.\n */\nexport async function callLlmJson<T = unknown>(\n req: LlmCallRequest,\n opts: LlmClientOptions = {},\n): Promise<{ value: T; result: LlmCallResult }> {\n const result = await callLlmStructured(req, opts)\n const value = parseJsonResult<T>(result, opts.jsonPayloadMode ?? 'extract')\n return { value, result }\n}\n\n/** Shared schema-to-JSON-mode fallback that preserves the raw result. */\nasync function callLlmStructured(\n req: LlmCallRequest,\n opts: LlmClientOptions = {},\n): Promise<LlmCallResult> {\n try {\n return await callLlm({ ...req, jsonMode: req.jsonMode ?? !req.jsonSchema }, opts)\n } catch (err) {\n if (\n opts.jsonSchemaTransport !== 'json-object' &&\n err instanceof LlmCallError &&\n isSchemaRejection(err.status, err.body) &&\n req.jsonSchema\n ) {\n const degradedReq: LlmCallRequest = { ...req, jsonMode: true, jsonSchema: undefined }\n return await callLlm(degradedReq, opts)\n }\n throw err\n }\n}\n\nfunction parseJsonResult<T>(\n result: LlmCallResult,\n jsonPayloadMode: NonNullable<LlmClientOptions['jsonPayloadMode']>,\n): T {\n try {\n if (result.finishReason === 'length') {\n throw new Error(\n `LLM returned truncated JSON content (model=${result.model}, finishReason=length)`,\n )\n }\n return parseJsonSafely<T>(result.content, result.model, jsonPayloadMode)\n } catch (error) {\n if (error instanceof LlmResponseError) throw error\n const cause = error instanceof Error ? error : new Error(String(error))\n throw new LlmResponseError(cause.message, result, { cause })\n }\n}\n\nfunction parseJsonSafely<T>(\n content: string,\n model: string,\n jsonPayloadMode: NonNullable<LlmClientOptions['jsonPayloadMode']>,\n): T {\n const payload = jsonPayloadMode === 'exact' ? content : extractJsonPayload(content)\n try {\n return JSON.parse(payload) as T\n } catch {\n throw new Error(`LLM returned non-JSON content (model=${model})`)\n }\n}\n\n// ─── Route assertion ────────────────────────────────────────────────────\n\nexport type LlmRouteAssertionReason =\n | 'no_explicit_base_url'\n | 'base_url_blocked'\n | 'base_url_not_allowed'\n | 'no_auth'\n | 'wrong_provider'\n\nexport class LlmRouteAssertionError extends CaptureIntegrityError {\n constructor(\n message: string,\n public readonly reason: LlmRouteAssertionReason,\n public readonly baseUrl: string,\n ) {\n super(message)\n }\n}\n\nexport interface LlmRouteRequirements {\n /**\n * Throw if `opts.baseUrl` is undefined, i.e. the call would fall back to\n * `DEFAULT_BASE_URL`. Set this for evaluation runs where silently using\n * the public/free-tier router is a defect — the launch reviewer needs to\n * know exactly which provider answered.\n */\n requireExplicitBaseUrl?: boolean\n /**\n * Allowlist of acceptable base URLs. Strings match by prefix\n * (case-insensitive); RegExps test against the full base URL.\n */\n allowedBaseUrls?: Array<string | RegExp>\n /** Blocklist that takes precedence over `allowedBaseUrls`. */\n blockedBaseUrls?: Array<string | RegExp>\n /** Throw if no auth header / api key is configured. */\n requireAuth?: boolean\n /**\n * Logical provider id the configured `baseUrl` is expected to match (via\n * `providerFromBaseUrl`). Mainly useful when paired with `requireExplicitBaseUrl`.\n */\n expectedProvider?: string\n}\n\n/**\n * Fail-loud assertion that the configured LLM client points at the route\n * the caller intends. Designed for the matrix-runner preflight: invoke\n * once before any LLM call to catch misconfiguration before a sweep burns\n * dollars on the wrong provider.\n *\n * Throws `LlmRouteAssertionError`. Pure — no I/O — so it's safe to call\n * from constructors and CI gates.\n */\nexport function assertLlmRoute(opts: LlmClientOptions, req: LlmRouteRequirements = {}): void {\n const baseUrlExplicit = opts.baseUrl !== undefined\n const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\\/+$/, '')\n\n if (req.requireExplicitBaseUrl && !baseUrlExplicit) {\n throw new LlmRouteAssertionError(\n `assertLlmRoute: requireExplicitBaseUrl set but opts.baseUrl is undefined; would fall back to ${DEFAULT_BASE_URL}.`,\n 'no_explicit_base_url',\n baseUrl,\n )\n }\n\n if (req.blockedBaseUrls?.some((p) => matchUrl(baseUrl, p))) {\n throw new LlmRouteAssertionError(\n `assertLlmRoute: baseUrl ${baseUrl} matches a blocked pattern.`,\n 'base_url_blocked',\n baseUrl,\n )\n }\n\n if (req.allowedBaseUrls && req.allowedBaseUrls.length > 0) {\n const ok = req.allowedBaseUrls.some((p) => matchUrl(baseUrl, p))\n if (!ok) {\n throw new LlmRouteAssertionError(\n `assertLlmRoute: baseUrl ${baseUrl} is not in the allowed list (${req.allowedBaseUrls.map(describePattern).join(', ')}).`,\n 'base_url_not_allowed',\n baseUrl,\n )\n }\n }\n\n if (req.requireAuth && !opts.apiKey && !opts.bearer && !opts.authHeader) {\n throw new LlmRouteAssertionError(\n `assertLlmRoute: requireAuth set but no apiKey, bearer, or authHeader was supplied.`,\n 'no_auth',\n baseUrl,\n )\n }\n\n if (req.expectedProvider) {\n const actual = opts.provider ?? providerFromBaseUrl(baseUrl)\n if (actual !== req.expectedProvider) {\n throw new LlmRouteAssertionError(\n `assertLlmRoute: expected provider ${req.expectedProvider} but baseUrl ${baseUrl} resolves to ${actual}.`,\n 'wrong_provider',\n baseUrl,\n )\n }\n }\n}\n\nfunction matchUrl(url: string, pattern: string | RegExp): boolean {\n if (pattern instanceof RegExp) return pattern.test(url)\n return url.toLowerCase().startsWith(pattern.toLowerCase())\n}\n\nfunction describePattern(p: string | RegExp): string {\n return p instanceof RegExp ? p.source : p\n}\n\n/**\n * Probe whether a model is reachable. Returns latency + null error on\n * success; `ok=false` + error message on any failure (HTTP, timeout,\n * network, parse). Designed for sweep preflights — fail loud at the\n * boundary before burning a 30-leaf run on a misconfigured router.\n *\n * Sends a tiny `ping` message with `maxTokens=64`. Reasoning models\n * (glm-5.1, deepseek-v4) can burn the entire budget on internal reasoning\n * for short prompts, so don't tighten this further. We don't validate\n * content; HTTP 200 means reachable.\n */\nexport async function probeLlm(\n model: string,\n opts: LlmClientOptions & { timeoutMs?: number } = {},\n): Promise<{ ok: boolean; latencyMs: number; error: string | null }> {\n const start = Date.now()\n try {\n await callLlm(\n {\n model,\n messages: [{ role: 'user', content: 'ping' }],\n maxTokens: 64,\n timeoutMs: opts.timeoutMs ?? 30_000,\n },\n opts,\n )\n return { ok: true, latencyMs: Date.now() - start, error: null }\n } catch (err) {\n return {\n ok: false,\n latencyMs: Date.now() - start,\n error: err instanceof Error ? err.message : String(err),\n }\n }\n}\n\n/**\n * Stateful client — construct once with defaults, call many times.\n * Thin wrapper around the free functions; exists for callers that want\n * to inject a single configured instance into multiple primitives.\n */\nexport class LlmClient {\n readonly maximumAttempts: number\n private readonly opts: LlmClientOptions\n\n constructor(opts: LlmClientOptions = {}) {\n this.opts = opts\n this.maximumAttempts = resolveMaximumAttempts(opts.maximumAttempts)\n }\n\n call(req: LlmCallRequest, per?: LlmClientOptions): Promise<LlmCallResult> {\n const options = { ...this.opts, ...per }\n return req.jsonSchema ? callLlmStructured(req, options) : callLlm(req, options)\n }\n\n callJson<T = unknown>(\n req: LlmCallRequest,\n per?: LlmClientOptions,\n ): Promise<{ value: T; result: LlmCallResult }> {\n return callLlmJson<T>(req, { ...this.opts, ...per })\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0EA,SAAgB,2BACd,SACA,UAA4B,CAAC,GACF;CAC3B,IAAI,QAAQ,cAAc,KAAA,GAAW,OAAO,KAAA;CAC5C,IAAI,CAAC,OAAO,UAAU,QAAQ,SAAS,KAAK,QAAQ,aAAa,GAC/D,MAAM,IAAI,WAAW,kEAAkE;CAEzF,IACE,QAAQ,SAAS,MACd,YACC,MAAM,QAAQ,QAAQ,OAAO,KAAK,QAAQ,QAAQ,MAAM,SAAS,KAAK,SAAS,WAAW,CAC9F,GAEA;CAGF,MAAM,WAAW,uBAAuB,QAAQ,eAAe;CAC/D,MAAM,kBAAkB,QAAQ,wBAAwB;CAGxD,MAAM,eAAe,IAAI,YAAY,CAAC,CAAC,OACrC,KAAK,UAAU,UAAU,SAAS,iBAAiB,QAAQ,QAAQ,CAAC,CACtE,CAAC,CAAC;CAEF,MAAM,UAAU,QAAQ,cAAc,CAAC,kBAAkB,IAAI;CAC7D,MAAM,QAAQ;EACZ,aAAa,eAAe,WAAW;EACvC,cAAc,QAAQ,YAAY,WAAW;CAC/C;CACA,OAAO,QAAQ,qBACX;EAAE,oBAAoB,QAAQ;EAAoB,GAAG;CAAM,IAC3D;EAAE,OAAO,QAAQ;EAAO,GAAG;CAAM;AACvC;;AAkDA,SAAgB,mBACd,QACA,oBACkB;CAClB,MAAM,eAAe,OAAO,MAAM,sBAAsB;CACxD,MAAM,cAAc,KAAK,IAAI,GAAG,OAAO,MAAM,eAAe,YAAY;CACxE,MAAM,oBACJ,OAAO,YAAY,QAAQ,sBAAsB,OAAO,MAAM,aAAa,QACvE,oBAAoB,oBAAoB;EACtC;EACA,GAAI,eAAe,IAAI,EAAE,aAAa,IAAI,CAAC;EAC3C,cAAc,OAAO,MAAM;CAC7B,CAAC,IACD,KAAA;CACN,OAAO;EACL,OAAO,OAAO;EACd;EACA,cAAc,OAAO,MAAM;EAC3B,iBAAiB,OAAO,MAAM;EAC9B,cAAc,eAAe,IAAI,eAAe,KAAA;EAChD,eAAe,OAAO,WAAW;EACjC,cAAc,OAAO,MAAM,aAAa;CAC1C;AACF;;AAGA,SAAgB,wBACd,OACA,oBAC8B;CAC9B,OAAO,iBAAiB,mBACpB,mBAAmB,MAAM,QAAQ,kBAAkB,IACnD,KAAA;AACN;AAEA,IAAa,eAAb,cAAkC,eAAe;CAG7B;CACA;CACA;CAJlB,YACE,SACA,QACA,MACA,OACA;EACA,MAAM,SAAS,OAAO;EAJN,KAAA,SAAA;EACA,KAAA,OAAA;EACA,KAAA,QAAA;CAGlB;AACF;;;;AAKA,IAAa,mBAAb,cAAsC,eAAe;CAGjC;CAFlB,YACE,SACA,QACA,SACA;EACA,MAAM,SAAS,SAAS,OAAO;EAHf,KAAA,SAAA;CAIlB;AACF;AAuEA,MAAM,mBAAmB;AASzB,MAAM,qBAAqB,OAAO,QAAQ,IAAI,qBAAqB,KAAK;AACxE,MAAM,2BACJ,QAAQ,IAAI,gCAAgC,KAAA,IACxC,IACA,OAAO,QAAQ,IAAI,2BAA2B;AAEpD,SAAS,uBAAuB,YAAwC;CACtE,MAAM,WAAW,cAAc;CAC/B,IAAI,CAAC,OAAO,UAAU,QAAQ,KAAK,YAAY,GAC7C,MAAM,IAAI,WAAW,iDAAiD;CAExE,OAAO;AACT;AAEA,SAAS,mBAAmB,OAAoC;CAC9D,OAAO,OAAO,UAAU,YAAY,OAAO,cAAc,KAAK,KAAK,SAAS,IAAI,QAAQ,KAAA;AAC1F;AAEA,MAAM,mCAAmB,IAAI,IAAI;CAAC;CAAK;CAAK;CAAK;AAAG,CAAC;;;;;;;;;;AAWrD,MAAM,2BAA8C;CAClD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;;;;;;;;;;;;AAaA,SAAgB,oBAAoB,KAAuB;CACzD,OAAO,kBAAkB,KAAK,CAAC;AACjC;AAEA,SAAS,kBAAkB,KAAc,OAAwB;CAC/D,IAAI,eAAe,cAAc,OAAO,iBAAiB,IAAI,IAAI,MAAM;CACvE,IAAI,EAAE,eAAe,QAAQ,OAAO;CAGpC,MAAM,SAAU,IAA6B;CAC7C,IAAI,OAAO,WAAW,YAAY,iBAAiB,IAAI,MAAM,GAAG,OAAO;CACvE,MAAM,OAAQ,IAA2B;CACzC,MAAM,WAAW,GAAG,IAAI,KAAK,IAAI,IAAI,QAAQ,IAAI,OAAO,SAAS,WAAW,OAAO;CACnF,IAAI,yBAAyB,MAAM,MAAM,EAAE,KAAK,QAAQ,CAAC,GAAG,OAAO;CACnE,MAAM,QAAS,IAA4B;CAC3C,IAAI,QAAQ,KAAK,iBAAiB,SAAS,UAAU,KACnD,OAAO,kBAAkB,OAAO,QAAQ,CAAC;CAE3C,OAAO;AACT;AAEA,SAAS,gBAAgB,SAAiC;CACxD,MAAM,IAAI,QAAQ,IAAI,aAAa;CACnC,IAAI,CAAC,GAAG,OAAO;CACf,MAAM,WAAW,OAAO,CAAC;CACzB,IAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,GAAG,OAAO,WAAW;CACjE,MAAM,SAAS,KAAK,MAAM,CAAC;CAC3B,IAAI,OAAO,SAAS,MAAM,GAAG,OAAO,KAAK,IAAI,GAAG,SAAS,KAAK,IAAI,CAAC;CACnE,OAAO;AACT;;AAGA,SAAgB,UAAU,SAAyB;CACjD,OAAO,KAAK,IAAI,MAAM,KAAK,SAAS,IAAM;AAC5C;AAEA,SAAS,aAAa,MAAgD;CACpE,MAAM,UAAkC;EACtC,gBAAgB;EAChB,QAAQ;CACV;CACA,IAAI,KAAK,YACP,QAAQ,KAAK,WAAW,QAAQ,KAAK,WAAW;MAC3C,IAAI,KAAK,UAAU,KAAK,QAC7B,QAAQ,gBAAgB,UAAU,KAAK,UAAU,KAAK;CAExD,IAAI,KAAK,gBAAgB,QAAQ,qBAAqB,KAAK;CAC3D,OAAO;AACT;AAEA,SAAS,kBAAkB,QAAgB,MAAuB;CAChE,IAAI,WAAW,KAAK,OAAO;CAC3B,MAAM,QAAQ,KAAK,YAAY;CAC/B,OACE,MAAM,SAAS,iBAAiB,KAChC,MAAM,SAAS,aAAa,KAC5B,MAAM,SAAS,gBAAgB,KAC/B,MAAM,SAAS,eAAe;AAElC;AAEA,SAAS,0BAA0B,QAAgB,MAAuB;CACxE,IAAI,WAAW,OAAO,CAAC,eAAe,KAAK,IAAI,GAAG,OAAO;CACzD,OACE,iGAAiG,KAC/F,IACF,KAAK,oEAAoE,KAAK,IAAI;AAEtF;AAEA,SAAS,UACP,KACA,iBACA,iBACyB;CACzB,MAAM,OAAgC;EACpC,OAAO,IAAI;EACX,UAAU,IAAI;EACd,aAAa,IAAI,eAAe;CAClC;CACA,IAAI,IAAI,aAAa,MACnB,IAAI,wBAAwB,IAAI,KAAK,GAAG,KAAK,wBAAwB,IAAI;MACpE,KAAK,aAAa,IAAI;CAE7B,MAAM,WAAW,IAAI,YAAY;CACjC,IAAI,aAAa,KAAA,GACf,KAAK,WAAW,EAAE,MAAM,SAAS;CAGnC,IAAI,IAAI,cAAc,CAAC,iBACrB,KAAK,kBAAkB;EACrB,MAAM;EACN,aAAa;GAAE,MAAM,IAAI,WAAW;GAAM,QAAQ,IAAI,WAAW;GAAQ,QAAQ;EAAK;CACxF;MACK,IAAI,IAAI,YAAY,IAAI,YAC7B,KAAK,kBAAkB,EAAE,MAAM,cAAc;CAG/C,OAAO;AACT;AAEA,SAAS,wBAAwB,OAAwB;CACvD,OAAO,oBAAoB,KAAK,KAAK;AACvC;AAEA,eAAe,MAAM,IAA2B;CAC9C,OAAO,IAAI,SAAS,YAAY,WAAW,SAAS,EAAE,CAAC;AACzD;;;;;;;;AASA,SAAS,YAAY,mBAAoC,QAAmC;CAC1F,IAAI,CAAC,QAAQ,OAAO,kBAAkB;CACtC,IAAI,OAAQ,YAAkC,QAAQ,YACpD,OAAO,YAAY,IAAI,CAAC,kBAAkB,QAAQ,MAAM,CAAC;CAE3D,IAAI,OAAO,SACT,kBAAkB,MAAM;MAExB,OAAO,iBAAiB,eAAe,kBAAkB,MAAM,GAAG,EAAE,MAAM,KAAK,CAAC;CAElF,OAAO,kBAAkB;AAC3B;;AAGA,SAAS,iBAAiB,OAAe,YAAyC;CAChF,OAAO,cAAc,QAAQ,KAAK,IAAI,IAAI,SAAS;AACrD;;;;;;AASA,SAAgB,gBAAgB,KAAqB;CACnD,MAAM,UAAU,IAAI,KAAK;CACzB,MAAM,IAAI,QAAQ,MAAM,yCAAyC;CACjE,OAAO,IAAI,EAAE,EAAE,CAAE,KAAK,IAAI;AAC5B;AAEA,SAAgB,mBAAmB,KAAqB;CACtD,MAAM,WAAW,gBAAgB,GAAG;CACpC,IAAI;EACF,KAAK,MAAM,QAAQ;EACnB,OAAO;CACT,QAAQ;EAIN,IAAI,SAAS,WAAW,GAAG,KAAK,SAAS,WAAW,GAAG,GAAG,OAAO;CACnE;CAGA,MAAM,SAAS,CAAC,GAAG,SAAS,SAAS,OAAO,CAAC,CAAC,CAC3C,KAAK,UAAU,MAAM,KAAK,CAAC,CAC3B,QAAQ,UAAU,SAAS,IAAI;CAClC,KAAK,MAAM,SAAS,QAAQ;EAC1B,MAAM,YAAY,oBAAoB,UAAU,KAAK;EACrD,IAAI,CAAC,WAAW;EAChB,IAAI;GACF,KAAK,MAAM,SAAS;GACpB,OAAO;EACT,QAAQ,CAER;CACF;CAEA,OAAO;AACT;AAEA,SAAS,oBAAoB,OAAe,OAA8B;CACxE,MAAM,SAAS,MAAM;CACrB,MAAM,SAAS,WAAW,MAAM,MAAM,WAAW,MAAM,MAAM;CAC7D,IAAI,CAAC,QAAQ,OAAO;CAEpB,MAAM,QAAkB,CAAC,MAAM;CAC/B,IAAI,aAAa;CACjB,IAAI,YAAY;CAEhB,KAAK,IAAI,IAAI,QAAQ,GAAG,IAAI,MAAM,QAAQ,KAAK;EAC7C,MAAM,OAAO,MAAM;EACnB,IAAI,WAAW;GACb,YAAY;GACZ;EACF;EACA,IAAI,SAAS,MAAM;GACjB,YAAY;GACZ;EACF;EACA,IAAI,SAAS,MAAK;GAChB,aAAa,CAAC;GACd;EACF;EACA,IAAI,YAAY;EAEhB,IAAI,SAAS,KAAK,MAAM,KAAK,GAAG;OAC3B,IAAI,SAAS,KAAK,MAAM,KAAK,GAAG;OAChC,IAAI,SAAS,MAAM,MAAM,SAAS,IAAI;GACzC,MAAM,IAAI;GACV,IAAI,MAAM,WAAW,GAAG,OAAO,MAAM,MAAM,OAAO,IAAI,CAAC;EACzD;CACF;CAEA,OAAO;AACT;;;;;;AAOA,eAAsB,QACpB,KACA,OAAyB,CAAC,GACF;CACxB,MAAM,WAAW,KAAK,WAAW,iBAAA,CAAkB,QAAQ,QAAQ,EAAE;CACrE,MAAM,MAAM,GAAG,QAAQ;CACvB,MAAM,WAAW;CACjB,MAAM,YAAY,IAAI,aAAa,KAAK,oBAAoB;CAC5D,MAAM,kBAAkB,uBAAuB,KAAK,eAAe;CACnE,MAAM,UAAU,KAAK,SAAS,WAAW;CACzC,MAAM,UAAU,aAAa,IAAI;CACjC,MAAM,WAAW,KAAK,YAAY,oBAAoB,OAAO;CAC7D,MAAM,OAAO,KAAK;CAClB,MAAM,WAAW,KAAK,YAAY;CAClC,MAAM,eAAe,KAAK;CAC1B,MAAM,eAAe,KAAK;CAC1B,MAAM,aAAa,KAAK;CACxB,MAAM,gBAAgB,KAAK,IAAI;CAC/B,IAAI,KAAK,oBACP,oBAAoB,KAAK,oBAAoB;EAAE,aAAa;EAAG,cAAc;CAAE,CAAC;CAGlF,IAAI;CACJ,IAAI,mBAAmB;CACvB,KAAK,IAAI,UAAU,GAAG,UAAU,iBAAiB,WAAW;EAG1D,IAAI,cAAc,SAChB,MAAM,IAAI,aAAa,oCAAoC,YAAY;EAIzE,IAAI,UAAU,KAAK,iBAAiB,eAAe,UAAU,GAC3D,MAAM,mBAAmB,QAAQ,UAAU,IAAI,MAAM,OAAO,OAAO,CAAC;EAEtE,MAAM,aAAa,IAAI,gBAAgB;EACvC,MAAM,gBAAgB,YAAY,YAAY,YAAY;EAC1D,MAAM,gBAAgB,iBAAiB,WAAW,MAAM,GAAG,SAAS;EACpE,MAAM,UAAU,KAAK,IAAI;EACzB,MAAM,cAAc,UAClB,kBACA,KAAK,wBAAwB,eAC7B,KAAK,QACP;EACA,IAAI,uBAAuB;EAC3B,IAAI,MACF,MAAM,UAAU,MAAM,UAAU;GAC9B,SAAS,cAAc;GACvB,OAAO,cAAc;GACrB,QAAQ,cAAc;GACtB;GACA,OAAO,IAAI;GACX;GACA;GACA,cAAc;GACd,WAAW;GACX,WAAW;GACX,gBAAgB;GAChB;GACA,gBAAgB,CAAC;EACnB,CAAC;EAGH,IAAI;GACF,MAAM,MAAM,MAAM,QAAQ,KAAK;IAC7B,QAAQ;IACR;IACA,MAAM,KAAK,UAAU,WAAW;IAChC,QAAQ;GACV,CAAC;GACD,aAAa,aAAa;GAC1B,MAAM,kBAAkB,OAAO,gBAAgB,IAAI,OAAO,IAAI,KAAA;GAE9D,IAAI,CAAC,IAAI,IAAI;IACX,MAAM,OAAO,MAAM,IAAI,KAAK;IAC5B,IAAI,MAAM;KACR,MAAM,UAAU,MAAM,UAAU;MAC9B,SAAS,cAAc;MACvB,OAAO,cAAc;MACrB,QAAQ,cAAc;MACtB;MACA,OAAO,IAAI;MACX;MACA;MACA,cAAc;MACd,WAAW;MACX,WAAW,KAAK,IAAI;MACpB,YAAY,KAAK,IAAI,IAAI;MACzB,YAAY,IAAI;MAChB;MACA,cAAc;MACd,cAAc,QAAQ,IAAI;MAC1B,gBAAgB,CAAC;KACnB,CAAC;KACD,uBAAuB;IACzB;IACA,MAAM,MAAM,IAAI,aACd,6BAA6B,IAAI,UACjC,IAAI,QACJ,MACA,IAAI,KACN;IACA,IACE,0BAA0B,IAAI,QAAQ,IAAI,KAC1C,iBAAiB,gBAAgB,KACjC,UAAU,kBAAkB,KAC5B,CAAC,iBAAiB,eAAe,UAAU,GAC3C;KACA,UAAU;KACV,mBAAmB;MAAE,GAAG;MAAkB,aAAa;KAAE;KACzD;IACF;IACA,IACE,iBAAiB,IAAI,IAAI,MAAM,KAC/B,UAAU,kBAAkB,KAC5B,CAAC,iBAAiB,eAAe,UAAU,GAC3C;KACA,UAAU;KAEV,MAAM,MADa,gBAAgB,IAAI,OAClB,KAAK,UAAU,OAAO,CAAC;KAC5C;IACF;IACA,MAAM;GACR;GAEA,MAAM,OAAO,MAAM,IAAI,KAAK;GAC5B,IAAI;GACJ,IAAI;IACF,OAAO,KAAK,MAAM,IAAI;GACxB,SAAS,UAAU;IACjB,IAAI,MAAM;KACR,MAAM,UAAU,MAAM,UAAU;MAC9B,SAAS,cAAc;MACvB,OAAO,cAAc;MACrB,QAAQ,cAAc;MACtB;MACA,OAAO,IAAI;MACX;MACA;MACA,cAAc;MACd,WAAW;MACX,WAAW,KAAK,IAAI;MACpB,YAAY,KAAK,IAAI,IAAI;MACzB,YAAY,IAAI;MAChB;MACA,cAAc;MACd,cAAc,sBAAsB,oBAAoB,QAAQ,SAAS,UAAU,OAAO,QAAQ;MAClG,gBAAgB,CAAC;KACnB,CAAC;KACD,uBAAuB;IACzB;IACA,MAAM;GACR;GACA,IAAI,MACF,MAAM,UAAU,MAAM,UAAU;IAC9B,SAAS,cAAc;IACvB,OAAO,cAAc;IACrB,QAAQ,cAAc;IACtB;IACA,OAAO,IAAI;IACX;IACA;IACA,cAAc;IACd,WAAW;IACX,WAAW,KAAK,IAAI;IACpB,YAAY,KAAK,IAAI,IAAI;IACzB,YAAY,IAAI;IAChB;IACA,cAAc;IACd,gBAAgB,CAAC;GACnB,CAAC;GAEH,MAAM,SACJ,KAAK,UAGH;GACJ,MAAM,WACJ,KAAK,SAAS,OAAO,KAAK,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK,KAAK,IACpE,KAAK,QACN,KAAA;GACN,MAAM,eAAe,mBAAmB,UAAU,aAAa;GAC/D,MAAM,mBAAmB,mBAAmB,UAAU,iBAAiB;GACvE,MAAM,cAAc,mBAAmB,UAAU,YAAY;GAO7D,MAAM,gBALJ,UAAU,6BACV,OAAO,SAAS,8BAA8B,YAC9C,CAAC,MAAM,QAAQ,SAAS,yBAAyB,IAC5C,SAAS,4BACV,KAAA,EAAA,EACkC;GACxC,MAAM,kBACJ,iBAAiB,KAAA,IAAY,KAAA,IAAY,mBAAmB,YAAY;GAC1E,MAAM,YACJ,UAAU,yBACV,OAAO,SAAS,0BAA0B,YAC1C,CAAC,MAAM,QAAQ,SAAS,qBAAqB,IACxC,SAAS,sBAAkD,gBAC5D,KAAA;GACN,MAAM,qBAAqB,cAAc,KAAA,IAAY,KAAA,IAAY,mBAAmB,SAAS;GAC7F,MAAM,gBACJ,iBAAiB,KAAA,KACjB,qBAAqB,KAAA,MACpB,iBAAiB,KAAA,KACf,oBAAoB,KAAA,KAAa,mBAAmB,sBACtD,cAAc,KAAA,KACZ,uBAAuB,KAAA,KAAa,sBAAsB,kBAC5D,gBAAgB,KAAA,KAAa,gBAAgB,eAAe;GAC/D,MAAM,gBAAiB,KAAK,kBAAkB,KAAK;GACnD,MAAM,UAAU,QAAQ,SAAS,WAAW;GAE5C,MAAM,iBACJ,OAAO,kBAAkB,YAAY,iBAAiB,KAAK,qBACvD,oBAAoB,KAAK,oBAAoB;IAC3C,aAAa,gBAAiB,sBAAsB;IACpD,GAAI,qBAAqB,EAAE,cAAc,mBAAmB,IAAI,CAAC;IACjE,cAAc;GAChB,CAAC,IACD,KAAA;GAEN,OAAO;IACL;IACA,cAAc,QAAQ,iBAAiB;IACvC,cAAc,QAAQ,KAAK,CAAC,CAAC,WAAW;IACxC,OAAO;KACL,cAAc,gBAAgB;KAC9B,kBAAkB,oBAAoB;KACtC,aAAa,gBAAgB,gBAAgB,MAAM,oBAAoB;KACvE,UAAU;KACV;KACA;IACF;IACA,SAAS,OAAO,kBAAkB,WAAW,gBAAiB,kBAAkB;IAChF,OAAQ,KAAK,SAAoB,IAAI;IACrC,YAAY,KAAK,IAAI,IAAI;IACzB,KAAK;GACP;EACF,SAAS,KAAK;GACZ,aAAa,aAAa;GAC1B,UAAU;GAIV,IAAI,cAAc,SAAS;IACzB,IAAI,QAAQ,CAAC,sBACX,MAAM,UAAU,MAAM,UAAU;KAC9B,SAAS,cAAc;KACvB,OAAO,cAAc;KACrB,QAAQ,cAAc;KACtB;KACA,OAAO,IAAI;KACX;KACA;KACA,cAAc;KACd,WAAW;KACX,WAAW,KAAK,IAAI;KACpB,YAAY,KAAK,IAAI,IAAI;KACzB,cAAc,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;KAC7D,gBAAgB,CAAC;IACnB,CAAC;IAEH,MAAM;GACR;GACA,IAAI,QAAQ,CAAC,sBAIX,MAAM,UAAU,MAAM,UAAU;IAC9B,SAAS,cAAc;IACvB,OAAO,cAAc;IACrB,QAAQ,cAAc;IACtB;IACA,OAAO,IAAI;IACX;IACA;IACA,cAAc;IACd,WAAW;IACX,WAAW,KAAK,IAAI;IACpB,YAAY,KAAK,IAAI,IAAI;IACzB,cAAc,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC7D,gBAAgB,CAAC;GACnB,CAAC;GAEH,IACE,UAAU,kBAAkB,KAC5B,oBAAoB,GAAG,KACvB,CAAC,iBAAiB,eAAe,UAAU,GAC3C;IACA,MAAM,MAAM,UAAU,OAAO,CAAC;IAC9B;GACF;GACA,MAAM;EACR;CACF;CACA,MAAM,mBAAmB,QAAQ,UAAU,IAAI,MAAM,OAAO,OAAO,CAAC;AACtE;AAEA,eAAe,UACb,MACA,UACA,OACe;CAGf,IAAI;EACF,MAAM,KAAK,OAAO,SAAS,KAAK,CAAC;CACnC,QAAQ,CAER;AACF;AAEA,SAAS,gBAAgB,GAAoC;CAC3D,MAAM,MAA8B,CAAC;CACrC,EAAE,SAAS,OAAO,QAAQ;EACxB,IAAI,OAAO;CACb,CAAC;CACD,OAAO;AACT;AAEA,SAAS,gBAAwB;CAC/B,IAAI,OAAO,WAAW,QAAQ,eAAe,YAAY,OAAO,WAAW,OAAO,WAAW;CAC7F,OAAO,GAAG,KAAK,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,KAAK,OAAO,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,MAAM,GAAG,EAAE;AAC7E;;;;;;;AAQA,eAAsB,YACpB,KACA,OAAyB,CAAC,GACoB;CAC9C,MAAM,SAAS,MAAM,kBAAkB,KAAK,IAAI;CAEhD,OAAO;EAAE,OADK,gBAAmB,QAAQ,KAAK,mBAAmB,SACpD;EAAG;CAAO;AACzB;;AAGA,eAAe,kBACb,KACA,OAAyB,CAAC,GACF;CACxB,IAAI;EACF,OAAO,MAAM,QAAQ;GAAE,GAAG;GAAK,UAAU,IAAI,YAAY,CAAC,IAAI;EAAW,GAAG,IAAI;CAClF,SAAS,KAAK;EACZ,IACE,KAAK,wBAAwB,iBAC7B,eAAe,gBACf,kBAAkB,IAAI,QAAQ,IAAI,IAAI,KACtC,IAAI,YAGJ,OAAO,MAAM,QAAQ;GADiB,GAAG;GAAK,UAAU;GAAM,YAAY,KAAA;EAC3C,GAAG,IAAI;EAExC,MAAM;CACR;AACF;AAEA,SAAS,gBACP,QACA,iBACG;CACH,IAAI;EACF,IAAI,OAAO,iBAAiB,UAC1B,MAAM,IAAI,MACR,8CAA8C,OAAO,MAAM,uBAC7D;EAEF,OAAO,gBAAmB,OAAO,SAAS,OAAO,OAAO,eAAe;CACzE,SAAS,OAAO;EACd,IAAI,iBAAiB,kBAAkB,MAAM;EAC7C,MAAM,QAAQ,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;EACtE,MAAM,IAAI,iBAAiB,MAAM,SAAS,QAAQ,EAAE,MAAM,CAAC;CAC7D;AACF;AAEA,SAAS,gBACP,SACA,OACA,iBACG;CACH,MAAM,UAAU,oBAAoB,UAAU,UAAU,mBAAmB,OAAO;CAClF,IAAI;EACF,OAAO,KAAK,MAAM,OAAO;CAC3B,QAAQ;EACN,MAAM,IAAI,MAAM,wCAAwC,MAAM,EAAE;CAClE;AACF;AAWA,IAAa,yBAAb,cAA4C,sBAAsB;CAG9C;CACA;CAHlB,YACE,SACA,QACA,SACA;EACA,MAAM,OAAO;EAHG,KAAA,SAAA;EACA,KAAA,UAAA;CAGlB;AACF;;;;;;;;;;AAmCA,SAAgB,eAAe,MAAwB,MAA4B,CAAC,GAAS;CAC3F,MAAM,kBAAkB,KAAK,YAAY,KAAA;CACzC,MAAM,WAAW,KAAK,WAAW,iBAAA,CAAkB,QAAQ,QAAQ,EAAE;CAErE,IAAI,IAAI,0BAA0B,CAAC,iBACjC,MAAM,IAAI,uBACR,gGAAgG,iBAAiB,IACjH,wBACA,OACF;CAGF,IAAI,IAAI,iBAAiB,MAAM,MAAM,SAAS,SAAS,CAAC,CAAC,GACvD,MAAM,IAAI,uBACR,2BAA2B,QAAQ,8BACnC,oBACA,OACF;CAGF,IAAI,IAAI,mBAAmB,IAAI,gBAAgB,SAAS,GAElD;MAAA,CADO,IAAI,gBAAgB,MAAM,MAAM,SAAS,SAAS,CAAC,CACxD,GACJ,MAAM,IAAI,uBACR,2BAA2B,QAAQ,+BAA+B,IAAI,gBAAgB,IAAI,eAAe,CAAC,CAAC,KAAK,IAAI,EAAE,KACtH,wBACA,OACF;CAAA;CAIJ,IAAI,IAAI,eAAe,CAAC,KAAK,UAAU,CAAC,KAAK,UAAU,CAAC,KAAK,YAC3D,MAAM,IAAI,uBACR,sFACA,WACA,OACF;CAGF,IAAI,IAAI,kBAAkB;EACxB,MAAM,SAAS,KAAK,YAAY,oBAAoB,OAAO;EAC3D,IAAI,WAAW,IAAI,kBACjB,MAAM,IAAI,uBACR,qCAAqC,IAAI,iBAAiB,eAAe,QAAQ,eAAe,OAAO,IACvG,kBACA,OACF;CAEJ;AACF;AAEA,SAAS,SAAS,KAAa,SAAmC;CAChE,IAAI,mBAAmB,QAAQ,OAAO,QAAQ,KAAK,GAAG;CACtD,OAAO,IAAI,YAAY,CAAC,CAAC,WAAW,QAAQ,YAAY,CAAC;AAC3D;AAEA,SAAS,gBAAgB,GAA4B;CACnD,OAAO,aAAa,SAAS,EAAE,SAAS;AAC1C;;;;;;;;;;;;AAaA,eAAsB,SACpB,OACA,OAAkD,CAAC,GACgB;CACnE,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI;EACF,MAAM,QACJ;GACE;GACA,UAAU,CAAC;IAAE,MAAM;IAAQ,SAAS;GAAO,CAAC;GAC5C,WAAW;GACX,WAAW,KAAK,aAAa;EAC/B,GACA,IACF;EACA,OAAO;GAAE,IAAI;GAAM,WAAW,KAAK,IAAI,IAAI;GAAO,OAAO;EAAK;CAChE,SAAS,KAAK;EACZ,OAAO;GACL,IAAI;GACJ,WAAW,KAAK,IAAI,IAAI;GACxB,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;EACxD;CACF;AACF;;;;;;AAOA,IAAa,YAAb,MAAuB;CACrB;CACA;CAEA,YAAY,OAAyB,CAAC,GAAG;EACvC,KAAK,OAAO;EACZ,KAAK,kBAAkB,uBAAuB,KAAK,eAAe;CACpE;CAEA,KAAK,KAAqB,KAAgD;EACxE,MAAM,UAAU;GAAE,GAAG,KAAK;GAAM,GAAG;EAAI;EACvC,OAAO,IAAI,aAAa,kBAAkB,KAAK,OAAO,IAAI,QAAQ,KAAK,OAAO;CAChF;CAEA,SACE,KACA,KAC8C;EAC9C,OAAO,YAAe,KAAK;GAAE,GAAG,KAAK;GAAM,GAAG;EAAI,CAAC;CACrD;AACF"}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { f as Run } from "../schema-BtVldJ3T.js";
|
|
2
2
|
import { s as TraceStore } from "../store-CT9YIIve.js";
|
|
3
|
-
import { Ct as GoldenItem, St as ContinuousCalibrationResult, a as CorpusAgreementReport, bt as ContinuousAgreement, vt as CalibrationResult, yt as CandidateScore } from "../statistics-
|
|
3
|
+
import { Ct as GoldenItem, St as ContinuousCalibrationResult, a as CorpusAgreementReport, bt as ContinuousAgreement, vt as CalibrationResult, yt as CandidateScore } from "../statistics-B5d0Zd-z.js";
|
|
4
4
|
import { a as OutcomeFilter, i as InMemoryOutcomeStore, n as FileSystemOutcomeStore, o as OutcomeStore, r as FileSystemOutcomeStoreOptions, t as DeploymentOutcome } from "../outcome-store-BYHIuO0e.js";
|
|
5
5
|
import { n as SeriesConvergenceResult, t as SeriesConvergenceOptions } from "../series-convergence-ofsqPWhs.js";
|
|
6
|
-
import { a as rubricPredictiveValidity, i as RubricRanking, n as RubricPredictiveValidityInput, r as RubricPredictiveValidityReport, t as RubricOutcomePair } from "../rubric-predictive-validity-
|
|
6
|
+
import { a as rubricPredictiveValidity, i as RubricRanking, n as RubricPredictiveValidityInput, r as RubricPredictiveValidityReport, t as RubricOutcomePair } from "../rubric-predictive-validity-DQBQj6uV.js";
|
|
7
7
|
//#region src/meta-eval/correlation-study.d.ts
|
|
8
8
|
interface EvalMetricSpec {
|
|
9
9
|
id: string;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { p as CostProvenance } from "../cost-ledger-
|
|
2
|
-
import { w as JudgeScore } from "../types-
|
|
1
|
+
import { p as CostProvenance } from "../cost-ledger-DKgyIWRj.js";
|
|
2
|
+
import { w as JudgeScore } from "../types-BBFNHxSK.js";
|
|
3
3
|
import { o as MatrixResult } from "../index-DSC51roc.js";
|
|
4
4
|
import { AgentProfile } from "@tangle-network/agent-interface";
|
|
5
5
|
//#region src/multishot/types.d.ts
|
package/dist/openapi.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"openapi": "3.1.0",
|
|
3
3
|
"info": {
|
|
4
4
|
"title": "@tangle-network/agent-eval — wire protocol",
|
|
5
|
-
"version": "0.
|
|
5
|
+
"version": "0.139.0",
|
|
6
6
|
"description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
|
|
7
7
|
"contact": {
|
|
8
8
|
"name": "Tangle Network",
|
|
@@ -1,70 +1,4 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
import { createHash } from "node:crypto";
|
|
3
|
-
//#region src/analyst/types.ts
|
|
4
|
-
/**
|
|
5
|
-
* Analyst contract — the missing orchestration layer over agent-eval's
|
|
6
|
-
* existing analyzers (analyzeTraces, MultiLayerVerifier, RunCritic,
|
|
7
|
-
* SemanticConceptJudge, JudgeFn, ...).
|
|
8
|
-
*
|
|
9
|
-
* Each existing primitive returns its own output shape. The Analyst
|
|
10
|
-
* contract is the single envelope every primitive lifts into, so a
|
|
11
|
-
* registry can run N analysts against a run and a single renderer can
|
|
12
|
-
* compose findings without knowing which analyzer produced them.
|
|
13
|
-
*
|
|
14
|
-
* The contract is intentionally domain-agnostic: nothing here knows
|
|
15
|
-
* about code, voice, RAG, or any particular agent stack. Analysts
|
|
16
|
-
* declare what INPUT KIND they need (a trace store, an artifact dir,
|
|
17
|
-
* a RunRecord, a JudgeInput, or `custom`), and the registry routes
|
|
18
|
-
* the matching input from `AnalystRunInputs`.
|
|
19
|
-
*/
|
|
20
|
-
/**
|
|
21
|
-
* Compute the stable finding_id from the identity-defining fields.
|
|
22
|
-
* Default implementation hashes {analyst_id, area, subject, normalized claim}.
|
|
23
|
-
* Analysts that emit findings whose claim text varies per run (timestamps,
|
|
24
|
-
* counts) SHOULD either: (a) pass an explicit `id_basis` to fix the hash,
|
|
25
|
-
* or (b) move the variable part into `rationale`/`metadata` and keep the
|
|
26
|
-
* `claim` static.
|
|
27
|
-
*/
|
|
28
|
-
function computeFindingId(input) {
|
|
29
|
-
const basis = JSON.stringify({
|
|
30
|
-
a: input.analyst_id,
|
|
31
|
-
r: input.area,
|
|
32
|
-
s: input.subject ?? "",
|
|
33
|
-
c: normalizeClaim(input.id_basis ?? input.claim)
|
|
34
|
-
});
|
|
35
|
-
return `f_${createHash("sha256").update(basis).digest("hex").slice(0, 20)}`;
|
|
36
|
-
}
|
|
37
|
-
function normalizeClaim(c) {
|
|
38
|
-
return c.toLowerCase().replace(/\s+/g, " ").replace(/[.!?;:,]+$/g, "").trim();
|
|
39
|
-
}
|
|
40
|
-
/**
|
|
41
|
-
* Convenience factory: produce a fully-formed AnalystFinding with the
|
|
42
|
-
* id computed automatically. Analyst code stays terse.
|
|
43
|
-
*/
|
|
44
|
-
function makeFinding(init) {
|
|
45
|
-
const { id_basis, produced_at, ...rest } = init;
|
|
46
|
-
return {
|
|
47
|
-
schema_version: "1.0.0",
|
|
48
|
-
finding_id: computeFindingId({
|
|
49
|
-
analyst_id: rest.analyst_id,
|
|
50
|
-
area: rest.area,
|
|
51
|
-
subject: rest.subject,
|
|
52
|
-
claim: rest.claim,
|
|
53
|
-
id_basis
|
|
54
|
-
}),
|
|
55
|
-
produced_at: produced_at ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
56
|
-
...rest
|
|
57
|
-
};
|
|
58
|
-
}
|
|
59
|
-
/** Build a finding whose source is explicitly allowed during candidate generation. */
|
|
60
|
-
function makeProposalFinding(init) {
|
|
61
|
-
const { proposal_origin, ...finding } = init;
|
|
62
|
-
return {
|
|
63
|
-
...makeFinding(finding),
|
|
64
|
-
proposal_origin
|
|
65
|
-
};
|
|
66
|
-
}
|
|
67
|
-
//#endregion
|
|
68
2
|
//#region src/abort-signal.ts
|
|
69
3
|
/** Combine active cancellation sources without wrapping a single source. */
|
|
70
4
|
function combineAbortSignals(...signals) {
|
|
@@ -159,6 +93,6 @@ function findingLabel(finding, index) {
|
|
|
159
93
|
return typeof id === "string" && id.length > 0 ? id : `index ${index}`;
|
|
160
94
|
}
|
|
161
95
|
//#endregion
|
|
162
|
-
export { clamp01 as a,
|
|
96
|
+
export { clamp01 as a, aggregateRunScore as i, isProposalFinding as n, combineAbortSignals as o, DEFAULT_RUN_SCORE_WEIGHTS as r, assertProposalFindings as t };
|
|
163
97
|
|
|
164
|
-
//# sourceMappingURL=proposal-findings-
|
|
98
|
+
//# sourceMappingURL=proposal-findings-2GIUo1et.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"proposal-findings-2GIUo1et.js","names":[],"sources":["../src/abort-signal.ts","../src/run-score.ts","../src/analyst/proposal-findings.ts"],"sourcesContent":["/** Combine active cancellation sources without wrapping a single source. */\nexport function combineAbortSignals(\n ...signals: Array<AbortSignal | undefined>\n): AbortSignal | undefined {\n const active = [\n ...new Set(signals.filter((signal): signal is AbortSignal => signal !== undefined)),\n ]\n if (active.length === 0) return undefined\n if (active.length === 1) return active[0]\n return AbortSignal.any(active)\n}\n","export interface RunScore {\n success: number\n goalProgress: number\n repoGroundedness: number\n driftPenalty: number\n toolUseQuality: number\n patchQuality: number\n testReality: number\n finalGate: number\n reviewerBlockers: number\n costUsd: number\n wallSeconds: number\n notes?: string[]\n}\n\nexport interface RunScoreWeights {\n success: number\n goalProgress: number\n repoGroundedness: number\n driftPenalty: number\n toolUseQuality: number\n patchQuality: number\n testReality: number\n finalGate: number\n reviewerBlockers: number\n costUsd: number\n wallSeconds: number\n}\n\nexport const DEFAULT_RUN_SCORE_WEIGHTS: RunScoreWeights = {\n success: 4,\n goalProgress: 2,\n repoGroundedness: 1.5,\n driftPenalty: -1.5,\n toolUseQuality: 1,\n patchQuality: 1.25,\n testReality: 1.5,\n finalGate: 3,\n reviewerBlockers: -2,\n costUsd: -0.2,\n wallSeconds: -0.1,\n}\n\nexport function aggregateRunScore(score: RunScore, weights: Partial<RunScoreWeights> = {}): number {\n const w = { ...DEFAULT_RUN_SCORE_WEIGHTS, ...weights }\n return (\n w.success * clamp01(score.success) +\n w.goalProgress * clamp01(score.goalProgress) +\n w.repoGroundedness * clamp01(score.repoGroundedness) +\n w.driftPenalty * clamp01(score.driftPenalty) +\n w.toolUseQuality * clamp01(score.toolUseQuality) +\n w.patchQuality * clamp01(score.patchQuality) +\n w.testReality * clamp01(score.testReality) +\n w.finalGate * clamp01(score.finalGate) +\n w.reviewerBlockers * clamp01(score.reviewerBlockers) +\n w.costUsd * Math.max(0, finiteOrZero(score.costUsd)) +\n w.wallSeconds * Math.max(0, finiteOrZero(score.wallSeconds) / 60)\n )\n}\n\nexport function clamp01(value: number): number {\n if (!Number.isFinite(value)) return 0\n return Math.max(0, Math.min(1, value))\n}\n\nfunction finiteOrZero(value: number): number {\n return Number.isFinite(value) ? value : 0\n}\n","import { z } from 'zod'\nimport type { ProposalFinding } from './types'\n\nconst ProposalFindingSchema = z\n .object({\n schema_version: z.literal('1.0.0'),\n finding_id: z.string().min(1),\n analyst_id: z.string().min(1),\n produced_at: z.string().min(1),\n severity: z.enum(['critical', 'high', 'medium', 'low', 'info']),\n area: z.string().min(1),\n claim: z.string().min(1),\n rationale: z.string().optional(),\n evidence_refs: z.array(\n z\n .object({\n kind: z.enum(['span', 'event', 'artifact', 'finding', 'metric']),\n uri: z.string().min(1),\n excerpt: z.string().optional(),\n })\n .strict(),\n ),\n recommended_action: z.string().optional(),\n validation_plan: z.string().optional(),\n confidence: z.number().min(0).max(1),\n subject: z.string().optional(),\n derived_from_judge: z.boolean().optional(),\n metadata: z.record(z.string(), z.unknown()).optional(),\n proposal_origin: z.enum(['search', 'production']),\n })\n .strict() satisfies z.ZodType<ProposalFinding>\n\n/** True when a finding names a source candidate generation may learn from. */\nexport function isProposalFinding(finding: unknown): finding is ProposalFinding {\n return ProposalFindingSchema.safeParse(finding).success\n}\n\n/**\n * Reject findings whose source has not been explicitly admitted for candidate\n * generation. Search feedback and observed production behavior are allowed;\n * final evaluation data has no allowed origin.\n */\nexport function assertProposalFindings(\n findings: unknown,\n context = 'proposal findings',\n): ReadonlyArray<ProposalFinding> {\n if (!Array.isArray(findings)) {\n throw new TypeError(`${context}: expected an array`)\n }\n const rejected = findings.flatMap((finding, index) =>\n isProposalFinding(finding) ? [] : [findingLabel(finding, index)],\n )\n if (rejected.length > 0) {\n throw new Error(\n `${context}: every finding must match AnalystFinding and declare ` +\n `proposal_origin as search or production. ` +\n `Rejected findings: [${rejected.join(', ')}].`,\n )\n }\n return findings as ReadonlyArray<ProposalFinding>\n}\n\nfunction findingLabel(finding: unknown, index: number): string {\n if (typeof finding !== 'object' || finding === null) return `index ${index}`\n const id = (finding as { finding_id?: unknown }).finding_id\n return typeof id === 'string' && id.length > 0 ? id : `index ${index}`\n}\n"],"mappings":";;;AACA,SAAgB,oBACd,GAAG,SACsB;CACzB,MAAM,SAAS,CACb,GAAG,IAAI,IAAI,QAAQ,QAAQ,WAAkC,WAAW,KAAA,CAAS,CAAC,CACpF;CACA,IAAI,OAAO,WAAW,GAAG,OAAO,KAAA;CAChC,IAAI,OAAO,WAAW,GAAG,OAAO,OAAO;CACvC,OAAO,YAAY,IAAI,MAAM;AAC/B;;;ACmBA,MAAa,4BAA6C;CACxD,SAAS;CACT,cAAc;CACd,kBAAkB;CAClB,cAAc;CACd,gBAAgB;CAChB,cAAc;CACd,aAAa;CACb,WAAW;CACX,kBAAkB;CAClB,SAAS;CACT,aAAa;AACf;AAEA,SAAgB,kBAAkB,OAAiB,UAAoC,CAAC,GAAW;CACjG,MAAM,IAAI;EAAE,GAAG;EAA2B,GAAG;CAAQ;CACrD,OACE,EAAE,UAAU,QAAQ,MAAM,OAAO,IACjC,EAAE,eAAe,QAAQ,MAAM,YAAY,IAC3C,EAAE,mBAAmB,QAAQ,MAAM,gBAAgB,IACnD,EAAE,eAAe,QAAQ,MAAM,YAAY,IAC3C,EAAE,iBAAiB,QAAQ,MAAM,cAAc,IAC/C,EAAE,eAAe,QAAQ,MAAM,YAAY,IAC3C,EAAE,cAAc,QAAQ,MAAM,WAAW,IACzC,EAAE,YAAY,QAAQ,MAAM,SAAS,IACrC,EAAE,mBAAmB,QAAQ,MAAM,gBAAgB,IACnD,EAAE,UAAU,KAAK,IAAI,GAAG,aAAa,MAAM,OAAO,CAAC,IACnD,EAAE,cAAc,KAAK,IAAI,GAAG,aAAa,MAAM,WAAW,IAAI,EAAE;AAEpE;AAEA,SAAgB,QAAQ,OAAuB;CAC7C,IAAI,CAAC,OAAO,SAAS,KAAK,GAAG,OAAO;CACpC,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AACvC;AAEA,SAAS,aAAa,OAAuB;CAC3C,OAAO,OAAO,SAAS,KAAK,IAAI,QAAQ;AAC1C;;;AChEA,MAAM,wBAAwB,EAC3B,OAAO;CACN,gBAAgB,EAAE,QAAQ,OAAO;CACjC,YAAY,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;CAC5B,YAAY,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;CAC5B,aAAa,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;CAC7B,UAAU,EAAE,KAAK;EAAC;EAAY;EAAQ;EAAU;EAAO;CAAM,CAAC;CAC9D,MAAM,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;CACtB,OAAO,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;CACvB,WAAW,EAAE,OAAO,CAAC,CAAC,SAAS;CAC/B,eAAe,EAAE,MACf,EACG,OAAO;EACN,MAAM,EAAE,KAAK;GAAC;GAAQ;GAAS;GAAY;GAAW;EAAQ,CAAC;EAC/D,KAAK,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;EACrB,SAAS,EAAE,OAAO,CAAC,CAAC,SAAS;CAC/B,CAAC,CAAC,CACD,OAAO,CACZ;CACA,oBAAoB,EAAE,OAAO,CAAC,CAAC,SAAS;CACxC,iBAAiB,EAAE,OAAO,CAAC,CAAC,SAAS;CACrC,YAAY,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;CACnC,SAAS,EAAE,OAAO,CAAC,CAAC,SAAS;CAC7B,oBAAoB,EAAE,QAAQ,CAAC,CAAC,SAAS;CACzC,UAAU,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS;CACrD,iBAAiB,EAAE,KAAK,CAAC,UAAU,YAAY,CAAC;AAClD,CAAC,CAAC,CACD,OAAO;;AAGV,SAAgB,kBAAkB,SAA8C;CAC9E,OAAO,sBAAsB,UAAU,OAAO,CAAC,CAAC;AAClD;;;;;;AAOA,SAAgB,uBACd,UACA,UAAU,qBACsB;CAChC,IAAI,CAAC,MAAM,QAAQ,QAAQ,GACzB,MAAM,IAAI,UAAU,GAAG,QAAQ,oBAAoB;CAErD,MAAM,WAAW,SAAS,SAAS,SAAS,UAC1C,kBAAkB,OAAO,IAAI,CAAC,IAAI,CAAC,aAAa,SAAS,KAAK,CAAC,CACjE;CACA,IAAI,SAAS,SAAS,GACpB,MAAM,IAAI,MACR,GAAG,QAAQ,qHAEc,SAAS,KAAK,IAAI,EAAE,GAC/C;CAEF,OAAO;AACT;AAEA,SAAS,aAAa,SAAkB,OAAuB;CAC7D,IAAI,OAAO,YAAY,YAAY,YAAY,MAAM,OAAO,SAAS;CACrE,MAAM,KAAM,QAAqC;CACjD,OAAO,OAAO,OAAO,YAAY,GAAG,SAAS,IAAI,KAAK,SAAS;AACjE"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { o as FailureClass } from "./schema-BtVldJ3T.js";
|
|
2
|
-
import { a as RunRecord, s as RunSplitTag } from "./run-record-
|
|
2
|
+
import { a as RunRecord, s as RunSplitTag } from "./run-record-CztDMXVF.js";
|
|
3
3
|
import { a as DatasetScenario, o as DatasetSplit, r as DatasetManifest } from "./dataset-v_Y5902-.js";
|
|
4
|
-
import { x as GateDecision } from "./summary-report-
|
|
4
|
+
import { x as GateDecision } from "./summary-report-Cg7BifAM.js";
|
|
5
5
|
//#region src/release-confidence.d.ts
|
|
6
6
|
/** Severity of an actionable finding attached to a run/trace. */
|
|
7
7
|
type AsiSeverity = 'info' | 'warning' | 'error' | 'critical';
|
|
@@ -241,4 +241,4 @@ interface RenderReleaseReportOptions {
|
|
|
241
241
|
declare function renderReleaseReport(scorecard: ReleaseConfidenceScorecard, options?: RenderReleaseReportOptions): string;
|
|
242
242
|
//#endregion
|
|
243
243
|
export { ReleaseConfidenceStatus as _, JudgeReplayGateArgs as a, assertReleaseConfidence as b, judgeReplayGate as c, ReleaseConfidenceAxis as d, ReleaseConfidenceAxisName as f, ReleaseConfidenceScorecard as g, ReleaseConfidenceMetrics as h, BootstrapResult as i, ActionableSideInfo as l, ReleaseConfidenceIssue as m, renderReleaseReport as n, Verdict as o, ReleaseConfidenceInput as p, BootstrapOptions as r, bootstrapCi as s, RenderReleaseReportOptions as t, AsiSeverity as u, ReleaseConfidenceThresholds as v, evaluateReleaseConfidence as x, ReleaseTraceEvidence as y };
|
|
244
|
-
//# sourceMappingURL=release-report-
|
|
244
|
+
//# sourceMappingURL=release-report-fZarvIm-.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"release-report-
|
|
1
|
+
{"version":3,"file":"release-report-fZarvIm-.d.ts","names":[],"sources":["../src/release-confidence.ts","../src/promotion-gate.ts","../src/release-report.ts"],"mappings":";;;;;;KAsBY;;UAGK;;EAEf;;EAEA;EACA,WAAW;;EAEX;;EAEA;;EAEA;;EAEA;EACA,WAAW;;KAGD;KACA;UAQK;EACf;EACA;EACA,QAAQ;EACR;EACA;EACA;EACA;EACA;;EAEA,eAAe;EACf,MAAM;EACN,WAAW;;UAGI;;EAEf;EACA;EACA;EACA;;EAEA;EACA;EACA;;EAEA;EACA;EACA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA,UAAU;EACV,qBAAqB;EACrB,gBAAgB;EAChB,kBAAkB;EAClB,eAAe;EACf,aAAa;;UAGE;EACf,MAAM;EACN,QAAQ;EACR;EACA;;UAGe;EACf,MAAM;EACN;EACA;EACA;;UAGe;EACf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,aAAa,OAAO;EACpB,cAAc;EACd,oBAAoB,QAAQ,OAAO;EACnC,0BAA0B;;;;;;;EAO1B;;UAGe;EACf;EACA;EACA;EACA,QAAQ;EACR;EACA,MAAM;EACN,QAAQ;EACR,SAAS;EACT,SAAS;EACT,cAAc;EACd;;iBAkBc,0BACd,OAAO,yBACN;iBA4Ha,wBAAwB,OAAO,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;KCxR5D;UAEK;EACf;EACA;;EAEA;;EAEA;;EAEA;;EAEA;EACA;EACA,SAAS;;UAGM;;EAEf;;EAEA;;;;;;EAMA;;EAEA;;;;;;;;;iBAUc,YACd,oBACA,qBACA,UAAS,mBACR;;;;;;;;;;;;;;;;UAsHc,oBAAoB;EACnC,iBAAiB;EACjB,kBAAkB;;EAElB,QAAQ,QAAQ,YAAY;EAC5B;EACA;;EAEA;;EAEA;;;;;iBAMoB,gBAAgB,SACpC,MAAM,oBAAoB,WACzB,QAAQ;EAAoB;EAAyB;;;;UCxMvC;EACf;EACA,gBAAgB;EAChB;EACA;EACA;;iBAGc,oBACd,WAAW,4BACX,UAAS"}
|
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
import { c as ReplayError, l as ValidationError, o as LimitExceededError, r as CaptureIntegrityError, s as NotFoundError, t as AgentEvalError } from "./errors-DkfjIDvD.js";
|
|
2
|
+
import { c as CostLedgerHandle } from "./cost-ledger-DKgyIWRj.js";
|
|
2
3
|
import { S as ToolSpan, f as Run } from "./schema-BtVldJ3T.js";
|
|
3
|
-
import { a as RunRecord, l as RunTerminalOutcome, s as RunSplitTag, u as RunTokenUsage } from "./run-record-
|
|
4
|
-
import { _t as RawProviderSink, gt as RawProviderEvent } from "./types-
|
|
4
|
+
import { a as RunRecord, l as RunTerminalOutcome, s as RunSplitTag, u as RunTokenUsage } from "./run-record-CztDMXVF.js";
|
|
5
|
+
import { _t as RawProviderSink, gt as RawProviderEvent } from "./types-uPrS6mD-.js";
|
|
5
6
|
import { s as TraceStore } from "./store-CT9YIIve.js";
|
|
6
7
|
import { n as RunCompleteHookContext, t as RunCompleteHook } from "./emitter-DGQGoLyj.js";
|
|
7
|
-
import { D as SearchSpanResult, E as QueryTracesPage, F as TraceAnalystSpanStatus, L as ViewSpansResult, M as TraceAnalystFilters, O as SearchTraceResult, P as TraceAnalystSpanKind, S as TraceAnalysisStoreContext, w as DatasetOverview, x as TraceAnalysisStore, y as BoundedTraceAnalysisStoreOptions, z as ViewTraceResult } from "./types-
|
|
8
|
-
import {
|
|
9
|
-
import { r as CorrectnessChecker } from "./completion-verifier-DSyRNVzU.js";
|
|
10
|
-
import { t as AnalyzeRunsOptions } from "./analyze-runs-PVtnfjvA.js";
|
|
11
|
-
import { AxAIService, AxAgentActorTurnCallbackArgs, AxFunction } from "@ax-llm/ax";
|
|
8
|
+
import { D as SearchSpanResult, E as QueryTracesPage, F as TraceAnalystSpanStatus, L as ViewSpansResult, M as TraceAnalystFilters, O as SearchTraceResult, P as TraceAnalystSpanKind, S as TraceAnalysisStoreContext, f as AnalystUsageReceipt, i as AnalystFinding, w as DatasetOverview, x as TraceAnalysisStore, y as BoundedTraceAnalysisStoreOptions, z as ViewTraceResult } from "./types-DoEYskCd.js";
|
|
9
|
+
import { i as TraceAnalysisEngine, o as TraceAnalysisEngineResult, s as TraceAnalystLimits, t as TraceToolGroupName } from "./tool-groups-CdYq22lX.js";
|
|
12
10
|
//#region src/trace/otel.d.ts
|
|
13
11
|
declare const OTEL_AGENT_EVAL_SCOPE: {
|
|
14
12
|
name: string;
|
|
@@ -243,86 +241,38 @@ declare function createBoundedTraceAnalysisStore(source: TraceAnalysisStore, opt
|
|
|
243
241
|
//#endregion
|
|
244
242
|
//#region src/trace-analyst/analyst.d.ts
|
|
245
243
|
interface AnalyzeTracesInput {
|
|
246
|
-
/** The user-facing question. Domain framing belongs here, not in the
|
|
247
|
-
* actor description. */
|
|
248
244
|
question: string;
|
|
245
|
+
id?: string;
|
|
246
|
+
description?: string;
|
|
247
|
+
area?: string;
|
|
249
248
|
}
|
|
250
|
-
|
|
251
|
-
/** The actor's submitted prose answer. */
|
|
252
|
-
answer: string;
|
|
253
|
-
/** Bulleted findings from the actor's structured completion. */
|
|
254
|
-
findings: string[];
|
|
255
|
-
/** Per-turn snapshots captured via `actorTurnCallback`. */
|
|
256
|
-
turns: AnalyzeTracesTurnSnapshot[];
|
|
257
|
-
/** Total turns the actor took. */
|
|
258
|
-
turnCount: number;
|
|
259
|
-
/** Token usage by role. */
|
|
260
|
-
usage: TraceAnalystUsage;
|
|
261
|
-
/** Full system + assistant + tool message log by role. */
|
|
262
|
-
chatLog: TraceAnalystChatLog;
|
|
263
|
-
/** Prompt version that produced this run. */
|
|
264
|
-
actorPromptVersion: string;
|
|
265
|
-
}
|
|
266
|
-
interface TraceAnalystUsage {
|
|
267
|
-
actor: TraceAnalystUsageEntry[];
|
|
268
|
-
responder: TraceAnalystUsageEntry[];
|
|
269
|
-
}
|
|
270
|
-
interface TraceAnalystUsageEntry {
|
|
271
|
-
[key: string]: unknown;
|
|
272
|
-
}
|
|
273
|
-
interface TraceAnalystChatLog {
|
|
274
|
-
actor: TraceAnalystChatMessage[];
|
|
275
|
-
responder: TraceAnalystChatMessage[];
|
|
276
|
-
}
|
|
277
|
-
interface TraceAnalystChatMessage {
|
|
278
|
-
[key: string]: unknown;
|
|
279
|
-
}
|
|
280
|
-
interface AnalyzeTracesTurnSnapshot {
|
|
281
|
-
stage: AxAgentActorTurnCallbackArgs['stage'];
|
|
282
|
-
turn: number;
|
|
283
|
-
isError: boolean;
|
|
284
|
-
/** The JS code the actor produced for this turn. */
|
|
285
|
-
code: string;
|
|
286
|
-
/** The formatted action-log entry the actor sees on the next turn. */
|
|
287
|
-
output: string;
|
|
288
|
-
/** Provider thought (when `executorOptions.showThoughts` is true and the
|
|
289
|
-
* provider returns it). */
|
|
290
|
-
thought?: string;
|
|
291
|
-
}
|
|
249
|
+
type AnalyzeTracesResult = TraceAnalysisEngineResult;
|
|
292
250
|
interface AnalyzeTracesOptions {
|
|
293
|
-
/**
|
|
251
|
+
/** OTLP JSONL path or a caller-owned store. */
|
|
294
252
|
source: string | TraceAnalysisStore;
|
|
295
|
-
/**
|
|
296
|
-
|
|
297
|
-
/**
|
|
298
|
-
|
|
299
|
-
/**
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
253
|
+
/** Recursive research engine. */
|
|
254
|
+
engine: TraceAnalysisEngine;
|
|
255
|
+
/** Override the general trace-research policy. */
|
|
256
|
+
instructions?: string;
|
|
257
|
+
/** Smallest tool set that can answer the question. Default: all. */
|
|
258
|
+
toolGroup?: TraceToolGroupName;
|
|
259
|
+
limits?: Partial<TraceAnalystLimits>;
|
|
260
|
+
runId?: string;
|
|
261
|
+
budgetUsd?: number;
|
|
262
|
+
costLedger?: CostLedgerHandle;
|
|
263
|
+
costPhase?: string;
|
|
264
|
+
priorFindings?: readonly AnalystFinding[];
|
|
265
|
+
upstreamFindings?: readonly AnalystFinding[];
|
|
266
|
+
recordUsage?: (receipt: AnalystUsageReceipt) => void;
|
|
267
|
+
tags?: Record<string, string>;
|
|
268
|
+
log?: (message: string, fields?: Record<string, unknown>) => void;
|
|
306
269
|
signal?: AbortSignal;
|
|
307
|
-
/** Override the actor description. */
|
|
308
|
-
actorDescription?: string;
|
|
309
|
-
/** Per-turn observability hook. */
|
|
310
|
-
onTurn?: (turn: AnalyzeTracesTurnSnapshot) => void | Promise<void>;
|
|
311
|
-
/** Override max runtime characters per turn. Default 6000. */
|
|
312
|
-
maxRuntimeChars?: number;
|
|
313
|
-
/** When set, every turn's snapshot is appended to this JSONL file
|
|
314
|
-
* immediately. If the analyst crashes mid-loop (provider 503,
|
|
315
|
-
* network error, validator reject) the partial reasoning is still
|
|
316
|
-
* on disk for diagnosis and recovery. */
|
|
317
|
-
progressLogPath?: string;
|
|
318
270
|
}
|
|
319
271
|
/**
|
|
320
|
-
*
|
|
272
|
+
* Answer one question by recursively inspecting a trace store.
|
|
321
273
|
*
|
|
322
|
-
*
|
|
323
|
-
*
|
|
324
|
-
* - `AxAgentClarificationError` if the analyst asks for clarification.
|
|
325
|
-
* - Provider errors (auth, rate limits) propagate from the AI service.
|
|
274
|
+
* The returned answer, cited findings, engine steps, call counts, and runtime
|
|
275
|
+
* identity are one audit record. A direct one-shot model call is not used.
|
|
326
276
|
*/
|
|
327
277
|
declare function analyzeTraces(input: AnalyzeTracesInput, options: AnalyzeTracesOptions): Promise<AnalyzeTracesResult>;
|
|
328
278
|
//#endregion
|
|
@@ -678,9 +628,9 @@ declare function otlpToTraceRunRecords(otlpJsonl: string, opts: OtlpToRunRecords
|
|
|
678
628
|
declare function otlpRowsToTraceRunRecords(rows: Iterable<object>, opts: OtlpToRunRecordsOptions): OtlpTraceRunRecord[];
|
|
679
629
|
//#endregion
|
|
680
630
|
//#region src/trace-analyst/prompts.d.ts
|
|
681
|
-
/**
|
|
682
|
-
declare const TRACE_ANALYST_ACTOR_DESCRIPTION = "
|
|
683
|
-
declare const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = "trace-analyst-
|
|
631
|
+
/** General policy for recursive, evidence-backed trace analysis. */
|
|
632
|
+
declare const TRACE_ANALYST_ACTOR_DESCRIPTION = "Answer the question by inspecting the OTLP trace dataset with the available tools.\n\n1. Call getDatasetOverview first. Use its real trace ids and dataset size to plan the investigation.\n2. Narrow with queryTraces and countTraces before scanning large payloads.\n3. For a small trace, use viewTrace. For a large trace, use searchTrace and then viewSpans or searchSpan.\n4. Never invent a trace id, span id, tool result, error, frequency, or final outcome.\n5. When a search reports has_more, refine the query before drawing a conclusion.\n6. Use llm_query only over evidence already loaded. A recursive query cannot inspect traces itself.\n7. Cite exact evidence URIs returned by the tools. Include a short exact excerpt when it supports the claim.\n8. Return no finding when the available evidence cannot support one.\n\nThe prose answer must directly answer the question and state important uncertainty.\nThe findings array contains only actionable or decision-relevant claims supported by inspected evidence.";
|
|
633
|
+
declare const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = "trace-analyst-research-v1-2026-07-30";
|
|
684
634
|
//#endregion
|
|
685
635
|
//#region src/trace-analyst/store-otlp.d.ts
|
|
686
636
|
interface ToolSpansToTraceAnalysisStoreOptions {
|
|
@@ -785,65 +735,6 @@ declare class ToolTraceMissingError extends CaptureIntegrityError {
|
|
|
785
735
|
*/
|
|
786
736
|
declare function toolSpansToTraceAnalysisStore(spans: readonly ToolSpan[] | null | undefined, options?: ToolSpansToTraceAnalysisStoreOptions): TraceAnalysisStore;
|
|
787
737
|
//#endregion
|
|
788
|
-
//#region src/eval-tools.d.ts
|
|
789
|
-
/** One agent-callable tool. `parameters` is a JSON Schema (draft-07+) object. */
|
|
790
|
-
interface EvalToolDef {
|
|
791
|
-
name: string;
|
|
792
|
-
description: string;
|
|
793
|
-
parameters: Record<string, unknown>;
|
|
794
|
-
handler: (args: unknown, ctx?: {
|
|
795
|
-
signal?: AbortSignal;
|
|
796
|
-
}) => Promise<unknown>;
|
|
797
|
-
}
|
|
798
|
-
interface MakeEvalToolsConfig {
|
|
799
|
-
/** Judges available to `run_judges`. Omit to exclude the tool. */
|
|
800
|
-
judges?: Array<JudgeConfig<unknown>>;
|
|
801
|
-
/** Host-side correctness checker for `verify_completion` (the third
|
|
802
|
-
* `verifyCompletion` argument — a function, so it cannot cross the wire).
|
|
803
|
-
* Omit to exclude the tool. */
|
|
804
|
-
completion?: {
|
|
805
|
-
checkCorrectness: CorrectnessChecker;
|
|
806
|
-
};
|
|
807
|
-
/** `analyzeRuns` options minus `runs` (runs arrive as tool args, inline or
|
|
808
|
-
* via `path`). Omit to exclude the tool. */
|
|
809
|
-
analyze?: Omit<AnalyzeRunsOptions, 'runs'>;
|
|
810
|
-
}
|
|
811
|
-
/** OpenAI function-tool wire shape for an `EvalToolDef`. */
|
|
812
|
-
declare function toOpenAiTool(def: EvalToolDef): {
|
|
813
|
-
type: 'function';
|
|
814
|
-
function: {
|
|
815
|
-
name: string;
|
|
816
|
-
description: string;
|
|
817
|
-
parameters: Record<string, unknown>;
|
|
818
|
-
};
|
|
819
|
-
};
|
|
820
|
-
/**
|
|
821
|
-
* Build the eval toolset for the supplied config. Only sections present in
|
|
822
|
-
* `cfg` produce tools, so the agent's tool list mirrors what the host
|
|
823
|
-
* actually wired. Handlers fail loud on malformed args — no silent defaults.
|
|
824
|
-
*/
|
|
825
|
-
declare function makeEvalTools(cfg: MakeEvalToolsConfig): EvalToolDef[];
|
|
826
|
-
//#endregion
|
|
827
|
-
//#region src/trace-analyst/tools.d.ts
|
|
828
|
-
declare const TRACE_ANALYST_TOOL_NAMESPACE: 'traces';
|
|
829
|
-
interface BuildTraceAnalysisToolsOptions extends BoundedTraceAnalysisStoreOptions {
|
|
830
|
-
store: TraceAnalysisStore;
|
|
831
|
-
}
|
|
832
|
-
interface TraceAnalysisToolDescriptor extends EvalToolDef {
|
|
833
|
-
namespace: typeof TRACE_ANALYST_TOOL_NAMESPACE;
|
|
834
|
-
}
|
|
835
|
-
/** Bind all seven trace reads without exposing an agent framework type. */
|
|
836
|
-
declare function buildTraceAnalysisToolDescriptors(options: BuildTraceAnalysisToolsOptions): TraceAnalysisToolDescriptor[];
|
|
837
|
-
/** Adapt the canonical descriptors into Ax functions used by the built-in analyst. */
|
|
838
|
-
declare function buildTraceAnalystTools(options: BuildTraceAnalysisToolsOptions): AxFunction[];
|
|
839
|
-
declare function traceAnalystFunctionGroup(options: BuildTraceAnalysisToolsOptions): {
|
|
840
|
-
namespace: string;
|
|
841
|
-
title: string;
|
|
842
|
-
selectionCriteria: string;
|
|
843
|
-
description: string;
|
|
844
|
-
functions: AxFunction[];
|
|
845
|
-
};
|
|
846
|
-
//#endregion
|
|
847
738
|
//#region src/replay.d.ts
|
|
848
739
|
declare class ReplayCacheMissError extends ReplayError {
|
|
849
740
|
readonly url: string;
|
|
@@ -938,5 +829,5 @@ declare function iterateRawCalls(sink: RawProviderSink, filter?: {
|
|
|
938
829
|
spanId?: string;
|
|
939
830
|
}): AsyncGenerator<ReplayCacheEntry>;
|
|
940
831
|
//#endregion
|
|
941
|
-
export {
|
|
942
|
-
//# sourceMappingURL=replay-
|
|
832
|
+
export { traceAnalystOnRunComplete as $, stringField as A, OtlpSpanRoleInput as At, TraceInsightReadiness as B, exportRunAsOtlp as Bt, ProjectedOtlpSpan as C, createOtelTracingStore as Ct, inferOtlpKind as D, OtelExporter as Dt, firstStringAttr as E, OtelExportConfig as Et, TraceInsightFinding as F, traceSpanKindToOpenInferenceKind as Ft, defaultTraceInsightPanel as G, TraceInsightTask as H, TraceInsightPanelRole as I, OTEL_AGENT_EVAL_SCOPE as It, inferDomainKeywords as J, describeTraceInsightScope as K, TraceInsightPromptInput as L, OtlpExport as Lt, flattenOtlpExportToNdjson as M, applyToolSpanOtlpAttributes as Mt, OtlpFlatLine as N, classifyOtlpSpanRole as Nt, projectOtlpFlatLine as O, createOtelExporter as Ot, TraceInsightContext as P, isOtlpModelCall as Pt, TraceAnalystHookOptions as Q, TraceInsightQualityGate as R, OtlpResourceSpans as Rt, otlpToTraceRunRecords as S, redactValue as St, extractOtlpAttributes as T, ExportableSpan as Tt, buildTraceInsightContext as U, TraceInsightSuite as V, buildTraceInsightPrompt as W, scoreTraceInsightReadiness as X, planTraceInsightQuestions as Y, tokenizeDomainWords as Z, OtlpTraceRunRecord as _, DEFAULT_REDACTION_RULES as _t, ReplayFetchOptions as a, TraceFileMissingError as at, otlpRowsToTraceRunRecords as b, RedactionRule as bt, ToolTraceMissingError as c, AnalyzeTracesInput as ct, OtlpFileTraceStoreOptions as d, analyzeTraces as dt, SpanNotFoundError as et, ToolSpansToTraceAnalysisStoreOptions as f, createBoundedTraceAnalysisStore as ft, OtlpToRunRecordsOptions as g, convertTraceStoresToOtlp as gt, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION as h, TracesToOtlpResult as ht, ReplayCacheStats as i, TraceFileMalformedError as it, FlattenOtlpOptions as j, ToolSpanOtlpInput as jt, readOtlpStatus as k, OtlpSpanRole as kt, toolSpansToTraceAnalysisStore as l, AnalyzeTracesOptions as lt, TRACE_ANALYST_ACTOR_DESCRIPTION as m, TraceStoreToOtlpOptions as mt, ReplayCacheEntry as n, TraceAnalysisStoreContractError as nt, createReplayFetch as o, TraceFileTooLargeError as ot, otlpTextToTraceAnalysisStore as p, TraceStoreSource as pt, domainEvidencePattern as q, ReplayCacheMissError as r, TraceAnalysisValidationError as rt, iterateRawCalls as s, TraceNotFoundError as st, ReplayCache as t, TraceAnalysisLimitError as tt, OtlpFileTraceStore as u, AnalyzeTracesResult as ut, TraceAggregate as v, REDACTION_VERSION as vt, asString as w, otelRunCompleteHook as wt, otlpToRunRecords as x, redactString as xt, otlpRowsToRunRecords as y, RedactionReport as yt, TraceInsightQuestion as z, OtlpSpan as zt };
|
|
833
|
+
//# sourceMappingURL=replay-DjG4IG60.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"replay-DjG4IG60.d.ts","names":[],"sources":["../src/trace/otel.ts","../src/trace/otlp-attributes.ts","../src/trace/otel-export.ts","../src/trace/otel-bridge.ts","../src/trace/redact.ts","../src/trace/store-to-otlp.ts","../src/trace-analyst/store-boundary.ts","../src/trace-analyst/analyst.ts","../src/trace-analyst/errors.ts","../src/trace-analyst/hook.ts","../src/trace-analyst/insights.ts","../src/trace/otlp-flat.ts","../src/trace-analyst/otlp-flatten.ts","../src/trace-analyst/otlp-span.ts","../src/trace-analyst/otlp-to-run-records.ts","../src/trace-analyst/prompts.ts","../src/trace-analyst/store-otlp.ts","../src/trace-analyst/store-tool-spans.ts","../src/replay.ts"],"mappings":";;;;;;;;;;cAoBa;EAA0B;EAAoC;;UAE1D;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA,YAAY;IACV;IACA;MAAS;MAAsB;MAAmB;MAAsB;;;EAE1E,SAAS;IAAQ;IAAsB;IAAc,aAAa;;EAClE;IAAW;IAAc;;;UAGV;EACf;IAAY,YAAY;;EACxB,YAAY;IAAQ,cAAc;IAAuB,OAAO;;;UAGjD;EACf,eAAe;;;iBAIK,gBACpB,OAAO,YACP,eACA,gBAAe,4CACd,QAAQ;;;KC5BC,oBAAoB,KAC9B;KAIU;UAUK;EACf;EACA,YAAY;EACZ;;;;;;;iBAkBc,qBAAqB,OAAO,oBAAoB;iBA2BhD,gBAAgB,OAAO;iBAkBvB,4BACd,YAAY,yBACZ,MAAM;iBAMQ,iCAAiC;;;UC/FhC;;EAEf;;EAEA,UAAU;;EAEV;;EAEA;;EAEA,qBAAqB;;EAErB;;UAGe;;EAEf,WAAW,MAAM;;EAEjB,SAAS;;EAET,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,OAAO;EACP,aAAa;;;;;;iBAOC,mBAAmB,SAAS,mBAAmB;;;;;;;iBC9C/C,oBAAoB,UAAU,eAAe;;;;;;;;;iBAoB7C,uBACd,OAAO,YACP,UAAU,cACV,kBACC;;;;;;;;;;;;;;;;UC/Bc;EACf;EACA,SAAS;;EAET;;UAGe;EACf;EACA,QAAQ;;;cAIG,yBAAyB;cAezB;;;;;iBAMG,aACd,eACA,QAAO;EACJ;EAAgB,QAAQ;;;;;;;iBAuBb,YACd,gBACA,QAAO,iBACP,SAAQ;EACL;EAAgB,QAAQ;;;;UC1CZ;;EAEf;;EAEA;;EAEA;;;EAGA;;;;;;;;;;;;UAae;;EAEf;;EAEA;;EAEA;;;;UAKe;;;EAGf;;;EAGA,sBAAsB,KAAK,QAAQ;;EAEnC,iBAAiB,KAAK,QAAQ;;;;;;;;;;;iBAYhB,yBACd,iBAAiB,4BAA4B,oBAC7C,iBACA,OAAM,0BACL;;;;iBChEa,gCACd,QAAQ,oBACR,UAAS,mCACR;;;UClBc;EACf;EACA;EACA;EACA;;KAGU,sBAAsB;UAEjB;;EAEf,iBAAiB;;EAEjB,QAAQ;;EAER;;EAEA,YAAY;EACZ,SAAS,QAAQ;EACjB;EACA;EACA,aAAa;EACb;EACA,yBAAyB;EACzB,4BAA4B;EAC5B,eAAe,SAAS;EACxB,OAAO;EACP,OAAO,iBAAiB,SAAS;EACjC,SAAS;;;;;;;;iBASW,cACpB,OAAO,oBACP,SAAS,uBACR,QAAQ;;;;cC7CE,qCAAqC;WAC/B;;;cAIN,gCAAgC;WAC1B;WACR;WACA;WACA;EAET,YAAY,mBAAmB,gBAAgB,eAAe;;;cASnD,wCAAwC;WAClC;WACR;EAET,YAAY,mBAAmB,iBAAiB;IAAY;;;cAMjD,8BAA8B;WACxB;WACR;EAET,YAAY;;cAMD,+BAA+B;WACjC;WACA;WACA;EAET,YAAY,cAAc,oBAAoB;;cAcnC,gCAAgC;WAClC;WACA;WACA;EAET,YAAY,cAAc,qBAAqB,qBAAqB;;cAQzD,2BAA2B;WACrB;WACR;EAET,YAAY;;cAMD,0BAA0B;WACpB;WACR;WACA;EAET,YAAY,kBAAkB;;;;UC1Ef;;;;;;EAMf,SAAS,KAAK;IAAoC,SAAS;;;;;;;EAM3D;;;;;;EAMA,QAAQ,QAAQ,qBAAqB,KAAK,2BAA2B;;;;;EAKrE,aAAa,KAAK;;;;;;EAMlB,UAAU,QAAQ,qBAAqB,KAAK;;iBAM9B,0BAA0B,MAAM,0BAA0B;;;UCzDzD;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf,OAAO;EACP,WAAW;EACX,QAAQ;EACR,SAAS;EACT;;UAGe;EACf,OAAO;EACP;EACA;EACA,WAAW;EACX,OAAO;EACP,UAAU;EACV,OAAO;EACP,QAAQ;;UAGO;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA,OAAO;;iBA0BO,oBAAoB;iBAMpB,oBAAoB,OAAO;iBAuB3B,sBAAsB,qBAAqB;iBAS3C,0BAA0B,OAAO;iBAsBjC,0BAA0B,OAAO,0BAA0B;iBAsD3D,yBAAyB,OAAO,0BAA0B;iBAa1D,2BAA2B,SAAS,sBAAsB;iBA2D1D,4BAA4B;iBA6B5B,wBAAwB,OAAO;;;KChTnC;UAEK;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;IACE,MAAM;IACN;;EAEF;IAAY,YAAY;;EACxB,YAAY;EACZ,SAAS;IAAQ;IAAc;IAAuB,aAAa;;;;;UCEpD;;;;EAIf;;EAEA,UAAU,QAAQ;;iBA+DJ,0BACd,YAAY,YACZ,OAAM,qBACL;;;;;;;;;UC1Dc;EACf;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;EACA;EACA,QAAQ;EACR;EACA;EACA;EACA;EACA;;EAEA,YAAY;;;;;;iBAOE,oBAAoB,KAAK,0BAA0B;iBAkEnD,eAAe,KAAK;EAClC,MAAM;EACN;;iBAkBc,cAAc,OAAO,0BAA0B;;;;;;iBAwB/C,sBAAsB,KAAK,0BAA0B;iBAoBrD,YAAY,KAAK,yBAAyB;iBAK1C,SAAS;;iBAKT,gBACd,OAAO,yBACP;;;UCpIe;;EAEf;;;EAGA;;;EAGA,WAAW;;EAEX;;EAEA;;EAEA;;EAEA;;;;;;;EAOA;;;;;;EAMA;;;;;;;;;EASA,wBAAwB;;;;;;;;;EASxB,iBAAiB,eAAe,MAAM;;;;;EAKtC,yBAAyB,kBAAkB;;;;;;UAO5B;EACf,QAAQ;;EAER;;EAEA;;;UAIe;EACf;;EAEA;;EAEA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,YAAY;;EAEZ;EACA;EACA;EACA;EACA;;EAEA,iBAAiB;;;;;;;iBAeH,iBAAiB,mBAAmB,MAAM,0BAA0B;;;;;;iBASpE,qBACd,MAAM,kBACN,MAAM,0BACL;;iBAKa,sBACd,mBACA,MAAM,0BACL;;iBAQa,0BACd,MAAM,kBACN,MAAM,0BACL;;;;cC9MU;cAcA;;;UC0HI;;EAEf;;EAEA;;EAEA;;EAEA;;KAGG,gCAAgC;UAEpB,kCAAkC;;EAEjD;;;;;;;;EAQA;;uBAMa,kCAAkC;mBAC9B;mBACA;mBACA;mBACA;UACT;UACA;UACA;UACA;EAER,YAAY,MAAM;EAyBZ,SAAS,kBAAkB,UAAU,4BAA4B;EAOjE,SACJ;IAAQ;IAAkB;KAC1B,UAAU,4BACT;EAeG,YACJ,UAAU,qBACV,UAAU,4BACT,QAAQ;EAoDL,YACJ;IAAQ,UAAU;IAAqB;IAAe;KACtD,UAAU,4BACT,QAAQ;EAmBL,YACJ,UAAU,qBACV,UAAU,4BACT;EAMG,UACJ;IAAQ;IAAkB;KAC1B,UAAU,4BACT,QAAQ;EAuCL,UACJ;IAAQ;IAAkB;IAA6B;KACvD,UAAU,4BACT,QAAQ;EAsFL,YACJ;IAAQ;IAAkB;IAAuB;KACjD,UAAU,4BACT,QAAQ;EA4CL,WACJ;IAAQ;IAAkB;IAAiB;IAAuB;KAClE,UAAU,4BACT,QAAQ;;;EA6CL,cAAc,UAAU,4BAA4B;UAI5C;qBAmBK,WAAW,SAAS,cAAc,QAAQ;qBAE1C;UAEL;UAmBA;UAsHA;UA+DN;UAkBA;UAwDA;UAmBM;;cA+CH,2BAA2B;mBACrB;mBACA;EAEjB,YAAY,MAAM;;;YAYF,WAAW,SAAS,cAAc,QAAQ;YAkBhD;;;iBAwBI,6BACd,cACA,OAAM,uCACL;;;;cC/7BU,8BAA8B;EACzC;;;;;;iBASc,8BACd,gBAAgB,+BAChB,UAAS,uCACR;;;cCSU,6BAA6B;WAEtB;WACA;EAFlB,YACkB,aACA,oBAChB;;UAMa;EACf,SAAS;EACT,UAAU;;UAGK;EACf;EACA,YAAY;EACZ,SAAS;;EAET;;;;;;;;;;cAWW;UACH;UACA;UACA;UACA;;;;;SAMK,SACX,MAAM,iBACN;IAAU;IAAgB;MACzB,QAAQ;;SASE,WAAW,QAAQ,qBAAqB,QAAQ;;EA4B7D;EAIA,SAAS;;EAUR,WAAW,iBAAiB;;;;;;EASvB,OAAO,uBAAuB,QAAQ;;UAM7B;;;;;;;EAOf;EACA,uBAAuB;;EAEvB,SAAS;IAAQ;IAAa;IAAkB;;;EAEhD,gBAAgB;IAAQ;IAAa;;;;;;;;;;;;;iBAavB,kBAAkB,OAAO,aAAa,OAAM,4BAAiC;;;;;;iBAsDtE,gBACrB,MAAM,iBACN;EAAU;EAAgB;IACzB,eAAe"}
|