@tangle-network/agent-eval 0.94.0 → 0.95.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/CHANGELOG.md +32 -0
  2. package/README.md +44 -30
  3. package/dist/adapters/http.d.ts +8 -7
  4. package/dist/adapters/http.js.map +1 -1
  5. package/dist/adapters/langchain.d.ts +3 -2
  6. package/dist/adapters/otel.d.ts +5 -4
  7. package/dist/analyst/index.d.ts +11 -31
  8. package/dist/analyst/index.js +5 -65
  9. package/dist/analyst/index.js.map +1 -1
  10. package/dist/{analyze-runs-B6Ljo_dI.d.ts → analyze-runs-DtT6F_6T.d.ts} +3 -3
  11. package/dist/belief-state/index.d.ts +4 -3
  12. package/dist/benchmarks/index.d.ts +3 -2
  13. package/dist/campaign/index.d.ts +727 -616
  14. package/dist/campaign/index.js +1863 -1316
  15. package/dist/campaign/index.js.map +1 -1
  16. package/dist/{chunk-2K6UUZ7P.js → chunk-2T4EZACH.js} +1 -1
  17. package/dist/chunk-2T4EZACH.js.map +1 -0
  18. package/dist/{chunk-CTBHKLEU.js → chunk-77T4STFI.js} +59 -86
  19. package/dist/chunk-77T4STFI.js.map +1 -0
  20. package/dist/{chunk-EGPMSBEZ.js → chunk-7QTQKIDD.js} +178 -177
  21. package/dist/chunk-7QTQKIDD.js.map +1 -0
  22. package/dist/{chunk-MIFZUPEK.js → chunk-AQ5WQAIV.js} +21 -6
  23. package/dist/chunk-AQ5WQAIV.js.map +1 -0
  24. package/dist/chunk-DJWX3GVS.js +81 -0
  25. package/dist/chunk-DJWX3GVS.js.map +1 -0
  26. package/dist/{chunk-S6OZEZQK.js → chunk-HMA63UEO.js} +37 -9
  27. package/dist/{chunk-S6OZEZQK.js.map → chunk-HMA63UEO.js.map} +1 -1
  28. package/dist/{chunk-TBDR6PAI.js → chunk-IZCEK2HR.js} +2 -2
  29. package/dist/{chunk-SD2YFWQQ.js → chunk-KKWJD5E6.js} +20 -20
  30. package/dist/chunk-KKWJD5E6.js.map +1 -0
  31. package/dist/{chunk-KWRRMR3J.js → chunk-LO6IOIJ2.js} +10 -10
  32. package/dist/chunk-LO6IOIJ2.js.map +1 -0
  33. package/dist/{chunk-E4GH6USR.js → chunk-NZEQVRH5.js} +2 -2
  34. package/dist/chunk-NZEQVRH5.js.map +1 -0
  35. package/dist/{chunk-MPQWFX6Y.js → chunk-PSWWQXHF.js} +13 -88
  36. package/dist/chunk-PSWWQXHF.js.map +1 -0
  37. package/dist/{chunk-Q5LIB7BC.js → chunk-S4SYLDFX.js} +2 -2
  38. package/dist/chunk-S4SYLDFX.js.map +1 -0
  39. package/dist/{chunk-KW53MSA5.js → chunk-X74V6ESX.js} +2 -2
  40. package/dist/{chunk-QMUEXQJS.js → chunk-YBIGNSCZ.js} +81 -4
  41. package/dist/chunk-YBIGNSCZ.js.map +1 -0
  42. package/dist/{chunk-2KNZHH3P.js → chunk-Z6L6YSU6.js} +2 -2
  43. package/dist/{code-agent-session-BO8nCnv3.d.ts → code-agent-session-CPHRCb4-.d.ts} +1 -1
  44. package/dist/contract/index.d.ts +91 -43
  45. package/dist/contract/index.js +127 -17
  46. package/dist/contract/index.js.map +1 -1
  47. package/dist/{control-D6qwHXIR.d.ts → control-Doncu-B_.d.ts} +2 -2
  48. package/dist/control.d.ts +3 -2
  49. package/dist/control.js +2 -2
  50. package/dist/{corpus-B8A4BDR3.d.ts → corpus-D4YW9UoJ.d.ts} +1 -1
  51. package/dist/{default-registry-6dhErQbs.d.ts → default-registry-GyE8X5SP.d.ts} +3 -3
  52. package/dist/diagnose.d.ts +4 -3
  53. package/dist/diagnose.js +1 -1
  54. package/dist/{run-improvement-loop-DBahB8Ax.d.ts → gepa-C1NCIZ9o.d.ts} +117 -130
  55. package/dist/hosted/index.d.ts +5 -4
  56. package/dist/{index-Bx3gZ8xl.d.ts → index-_Y4oNOOb.d.ts} +1 -1
  57. package/dist/index.d.ts +76 -81
  58. package/dist/index.js +66 -31
  59. package/dist/index.js.map +1 -1
  60. package/dist/{insight-report-DWl3z9tl.d.ts → insight-report-BnRjTibG.d.ts} +1 -1
  61. package/dist/{kind-factory-0BhLSI27.d.ts → kind-factory-X3eDYbKn.d.ts} +2 -3
  62. package/dist/matrix/index.d.ts +1 -1
  63. package/dist/meta-eval/index.d.ts +3 -2
  64. package/dist/multishot/index.d.ts +4 -4
  65. package/dist/multishot/index.js.map +1 -1
  66. package/dist/openapi.json +1 -1
  67. package/dist/{pre-registration-mAnCugl9.d.ts → pre-registration-nfUdc9EQ.d.ts} +2 -42
  68. package/dist/{provenance-P-bCL2Fo.d.ts → provenance-CncDq9qE.d.ts} +26 -41
  69. package/dist/{release-report-BEbWmVYj.d.ts → release-report-pidWUMZ2.d.ts} +2 -2
  70. package/dist/reporting.d.ts +5 -4
  71. package/dist/{researcher-B0C2_fVO.d.ts → researcher-Jr8ME1dZ.d.ts} +2 -2
  72. package/dist/rl.d.ts +516 -515
  73. package/dist/rl.js +612 -612
  74. package/dist/rl.js.map +1 -1
  75. package/dist/{rubric-predictive-validity-Cy_W-hWZ.d.ts → rubric-predictive-validity-C2hDKM8Z.d.ts} +1 -1
  76. package/dist/{run-campaign-7WNXMDSN.js → run-campaign-WXY7KI67.js} +2 -2
  77. package/dist/{run-record-e7vj1uZQ.d.ts → run-record-CP2ObebC.d.ts} +14 -18
  78. package/dist/{runtime-trajectory-BDgfGZSr.d.ts → runtime-trajectory-BOUUjI0y.d.ts} +1 -1
  79. package/dist/{semantic-concept-judge-B9MgmBnM.d.ts → semantic-concept-judge-DSBB2Cfp.d.ts} +2 -2
  80. package/dist/{summary-report-BDOFevaT.d.ts → summary-report-CInXwsza.d.ts} +1 -1
  81. package/dist/testing-C21CHsq2.d.ts +20 -0
  82. package/dist/testing.d.ts +1 -0
  83. package/dist/testing.js +8 -0
  84. package/dist/testing.js.map +1 -0
  85. package/dist/traces.d.ts +26 -10
  86. package/dist/traces.js +41 -11
  87. package/dist/{types-Ce17tDlG.d.ts → types-B5x54y6n.d.ts} +1 -1
  88. package/dist/{types-mn5Aqk7x.d.ts → types-BUxNaJ8c.d.ts} +2 -4
  89. package/dist/{types-BU-7W85F.d.ts → types-DQRY8ZT-.d.ts} +60 -58
  90. package/dist/workflow/index.d.ts +5 -4
  91. package/dist/workflow/index.js +1 -1
  92. package/docs/campaign-proposers.md +170 -0
  93. package/docs/concepts.md +8 -4
  94. package/docs/customer-journeys.md +15 -13
  95. package/docs/design/loop-taxonomy.md +34 -66
  96. package/docs/distributed-driver.md +14 -14
  97. package/docs/feature-guide.md +1 -1
  98. package/docs/hosted-ingest-spec.md +2 -3
  99. package/docs/multi-shot-optimization.md +8 -8
  100. package/docs/product-eval-adoption.md +1 -1
  101. package/docs/self-improvement-map.md +33 -29
  102. package/package.json +8 -14
  103. package/dist/chunk-2K6UUZ7P.js.map +0 -1
  104. package/dist/chunk-CTBHKLEU.js.map +0 -1
  105. package/dist/chunk-E4GH6USR.js.map +0 -1
  106. package/dist/chunk-EGPMSBEZ.js.map +0 -1
  107. package/dist/chunk-KWRRMR3J.js.map +0 -1
  108. package/dist/chunk-MIFZUPEK.js.map +0 -1
  109. package/dist/chunk-MPQWFX6Y.js.map +0 -1
  110. package/dist/chunk-Q5LIB7BC.js.map +0 -1
  111. package/dist/chunk-QMUEXQJS.js.map +0 -1
  112. package/dist/chunk-SD2YFWQQ.js.map +0 -1
  113. package/docs/design/external-agent-wedge.md +0 -89
  114. package/docs/design/phase-d-rfc.md +0 -125
  115. package/docs/design/phase4-consumer-migration.md +0 -70
  116. package/docs/design/primitives-integration-spec.md +0 -393
  117. package/docs/design/product-self-improvement-loop.md +0 -146
  118. package/docs/design/self-improvement-engine.md +0 -140
  119. package/docs/design/self-improvement-protocol.md +0 -223
  120. package/docs/design/self-improvement-roadmap.md +0 -106
  121. package/docs/design/substrate-gaps.md +0 -118
  122. package/docs/phase-b-pairing-kit.md +0 -188
  123. package/docs/phase-b-runbook.md +0 -176
  124. package/docs/pilot/README.md +0 -62
  125. package/docs/pilot/customer-checklist.md +0 -90
  126. package/docs/pilot/integration-foreign-stack.md +0 -296
  127. package/docs/pilot/integration-tangle-stack.md +0 -248
  128. package/docs/pilot/one-pager.md +0 -161
  129. package/docs/pilot/sample-insight-report.json +0 -172
  130. package/docs/quickstart-external.md +0 -229
  131. package/docs/research/belief-state-agent-eval-roadmap.md +0 -593
  132. package/docs/research/research-roadmap.md +0 -205
  133. package/docs/specs/driver-honest-spec.md +0 -251
  134. package/docs/specs/hermes-self-improvement-audit.md +0 -93
  135. package/docs/specs/profile-versioning.md +0 -291
  136. package/docs/three-package-architecture.md +0 -168
  137. /package/dist/{chunk-TBDR6PAI.js.map → chunk-IZCEK2HR.js.map} +0 -0
  138. /package/dist/{chunk-KW53MSA5.js.map → chunk-X74V6ESX.js.map} +0 -0
  139. /package/dist/{chunk-2KNZHH3P.js.map → chunk-Z6L6YSU6.js.map} +0 -0
  140. /package/dist/{run-campaign-7WNXMDSN.js.map → run-campaign-WXY7KI67.js.map} +0 -0
@@ -1,4 +1,4 @@
1
- import { R as RunRecord } from './run-record-e7vj1uZQ.js';
1
+ import { R as RunRecord } from './run-record-CP2ObebC.js';
2
2
  import { b as OutcomeStore } from './outcome-store-rnXLEqSn.js';
3
3
 
4
4
  /**
@@ -1,10 +1,10 @@
1
1
  import {
2
2
  runCampaign
3
- } from "./chunk-2K6UUZ7P.js";
3
+ } from "./chunk-2T4EZACH.js";
4
4
  import "./chunk-56GC6VYO.js";
5
5
  import "./chunk-3BFEG2F6.js";
6
6
  import "./chunk-PZ5AY32C.js";
7
7
  export {
8
8
  runCampaign
9
9
  };
10
- //# sourceMappingURL=run-campaign-7WNXMDSN.js.map
10
+ //# sourceMappingURL=run-campaign-WXY7KI67.js.map
@@ -1,3 +1,4 @@
1
+ import { AgentProfile } from '@tangle-network/agent-interface';
1
2
  import { V as ValidationError } from './errors-CzMUYo7b.js';
2
3
  import { F as FailureClass } from './schema-m0gsnbt3.js';
3
4
 
@@ -7,7 +8,7 @@ type AgentProfileJson = string | number | boolean | null | AgentProfileJson[] |
7
8
  };
8
9
  type AgentProfileDimensionValue = string | number | boolean | null;
9
10
  interface AgentProfileSource {
10
- /** Runtime/profile contract being fingerprinted, e.g. `sandbox-agent-profile`. */
11
+ /** Runtime/profile contract being fingerprinted, e.g. `agent-interface-profile`. */
11
12
  kind: string;
12
13
  /** sha256 over the canonical source profile object. */
13
14
  hash: string;
@@ -74,9 +75,9 @@ declare function groupRunsByAgentProfileCell<T extends {
74
75
  * new kind is a deliberate cross-product schema change. */
75
76
  declare const AGENT_PROFILE_KINDS: {
76
77
  /** A profile declared via `defineAgentProfile(...)` from
77
- * `@tangle-network/sandbox`. The default kind for sandbox-hosted
78
- * products (gtm-agent, blueprint-agent, sandbox, evals). */
79
- readonly SANDBOX_AGENT_PROFILE: "sandbox-agent-profile";
78
+ * `@tangle-network/agent-interface`. The default kind for router-backed
79
+ * and sandbox-backed products. */
80
+ readonly AGENT_INTERFACE_PROFILE: "agent-interface-profile";
80
81
  };
81
82
  type AgentProfileKind = (typeof AGENT_PROFILE_KINDS)[keyof typeof AGENT_PROFILE_KINDS];
82
83
  /** Canonicalize an arbitrary value into `AgentProfileJson` by JSON
@@ -84,26 +85,21 @@ type AgentProfileKind = (typeof AGENT_PROFILE_KINDS)[keyof typeof AGENT_PROFILE_
84
85
  * as JSON (functions, BigInt, cycles) — non-portable profiles fail loud
85
86
  * rather than silently dropping fields. */
86
87
  declare function toAgentProfileJson(value: unknown): AgentProfileJson;
87
- /** Minimal shape required of any sandbox-SDK `AgentProfile` anything
88
- * with a non-empty `name` and `version` plus JSON-serializable contents.
89
- * Compatible with `defineAgentProfile(...)` output from
90
- * `@tangle-network/sandbox`; products that have not yet declared a real
91
- * profile can pass a `{ name, version, ...metadata }` stub. */
92
- interface SandboxAgentProfileLike {
88
+ /** Canonical AgentProfile shape required when deriving a stable cell id. */
89
+ type AgentInterfaceProfileLike = AgentProfile & {
93
90
  name: string;
94
91
  version: string;
95
- [key: string]: unknown;
96
- }
92
+ };
97
93
  /** Higher-level helper that hard-codes the canonical
98
- * `sandbox-agent-profile` kind plus the JSON canonicalization. Equivalent
94
+ * `agent-interface-profile` kind plus the JSON canonicalization. Equivalent
99
95
  * to calling `buildAgentProfileCell` with `profileId = \`${name}@${version}\``
100
- * and `sourceProfile = { kind: SANDBOX_AGENT_PROFILE, profile: <round-tripped> }`.
96
+ * and `sourceProfile = { kind: AGENT_INTERFACE_PROFILE, profile: <round-tripped> }`.
101
97
  *
102
- * Use this from any product consuming a sandbox-SDK `AgentProfile`; the
98
+ * Use this from any product consuming an agent-interface `AgentProfile`; the
103
99
  * manual `buildAgentProfileCell` call is reserved for advanced cases
104
100
  * (custom kinds, pre-computed source hashes, alternate profileId
105
101
  * conventions). */
106
- declare function buildSandboxAgentProfileCell(profile: SandboxAgentProfileLike, input: Omit<AgentProfileCellInput, 'profileId' | 'sourceProfile'>): Promise<AgentProfileCell>;
102
+ declare function buildAgentInterfaceProfileCell(profile: AgentInterfaceProfileLike, input: Omit<AgentProfileCellInput, 'profileId' | 'sourceProfile'>): Promise<AgentProfileCell>;
107
103
 
108
104
  /**
109
105
  * Paper-grade RunRecord schema + runtime validator.
@@ -221,7 +217,7 @@ interface RunOutcome {
221
217
  * model (after any steering bundle merge).
222
218
  * - `configHash` is the sha256 of the effective run config (model,
223
219
  * temperature, tools, judges, splits). The pair (promptHash,
224
- * configHash) uniquely identifies an experimental cell.
220
+ * configHash) uniquely identifies an experiment cell.
225
221
  *
226
222
  * Model snapshot discipline:
227
223
  * - `model` MUST encode a snapshot version. Bare aliases like
@@ -315,4 +311,4 @@ declare function parseRunRecordSafe(input: unknown): {
315
311
  /** Round-trip helper — `JSON.parse(JSON.stringify(record))` then validate. */
316
312
  declare function roundTripRunRecord(record: RunRecord): RunRecord;
317
313
 
318
- export { type AgentProfileCell as A, validateAgentProfileCell as B, validateRunRecord as C, verifyAgentProfileCell as D, type JudgeScoresRecord as J, type RunRecord as R, type SandboxAgentProfileLike as S, type RunSplitTag as a, type RunTokenUsage as b, type RunJudgeMetadata as c, type AgentProfileCellInput as d, AGENT_PROFILE_KINDS as e, type AgentProfileCellSchemaVersion as f, AgentProfileCellValidationError as g, type AgentProfileDimensionValue as h, type AgentProfileHarness as i, type AgentProfileJson as j, type AgentProfileKind as k, type AgentProfileSource as l, type AgentProfileSourceInput as m, type RunOutcome as n, RunRecordValidationError as o, agentProfileCellHashMaterial as p, agentProfileCellKey as q, assertRunAgentProfileCell as r, buildAgentProfileCell as s, buildSandboxAgentProfileCell as t, groupRunsByAgentProfileCell as u, isRunRecord as v, parseRunRecordSafe as w, requireAgentProfileCell as x, roundTripRunRecord as y, toAgentProfileJson as z };
314
+ export { type AgentProfileCell as A, toAgentProfileJson as B, validateAgentProfileCell as C, validateRunRecord as D, verifyAgentProfileCell as E, type JudgeScoresRecord as J, type RunRecord as R, type RunSplitTag as a, type RunTokenUsage as b, type RunJudgeMetadata as c, type AgentProfileCellInput as d, AGENT_PROFILE_KINDS as e, type AgentInterfaceProfileLike as f, type AgentProfileCellSchemaVersion as g, AgentProfileCellValidationError as h, type AgentProfileDimensionValue as i, type AgentProfileHarness as j, type AgentProfileJson as k, type AgentProfileKind as l, type AgentProfileSource as m, type AgentProfileSourceInput as n, type RunOutcome as o, RunRecordValidationError as p, agentProfileCellHashMaterial as q, agentProfileCellKey as r, assertRunAgentProfileCell as s, buildAgentInterfaceProfileCell as t, buildAgentProfileCell as u, groupRunsByAgentProfileCell as v, isRunRecord as w, parseRunRecordSafe as x, requireAgentProfileCell as y, roundTripRunRecord as z };
@@ -1,4 +1,4 @@
1
- import { a as RunSplitTag } from './run-record-e7vj1uZQ.js';
1
+ import { a as RunSplitTag } from './run-record-CP2ObebC.js';
2
2
 
3
3
  interface RuntimeTrajectoryHookEvent {
4
4
  id: string;
@@ -1,7 +1,7 @@
1
1
  import { AxAIService } from '@ax-llm/ax';
2
2
  import { z } from 'zod';
3
- import { c as AnalystFinding, A as Analyst, a as AnalystContext } from './types-Ce17tDlG.js';
4
- import { T as TraceAnalystKindSpec } from './kind-factory-0BhLSI27.js';
3
+ import { c as AnalystFinding, A as Analyst, a as AnalystContext } from './types-B5x54y6n.js';
4
+ import { T as TraceAnalystKindSpec } from './kind-factory-X3eDYbKn.js';
5
5
  import { a as TraceAnalystSpan } from './store-C1YxJDEK.js';
6
6
  import { L as LlmClientOptions } from './llm-client-Bj7g0rqu.js';
7
7
  import { S as Severity } from './multi-layer-verifier-DUZXrPDA.js';
@@ -1,4 +1,4 @@
1
- import { R as RunRecord } from './run-record-e7vj1uZQ.js';
1
+ import { R as RunRecord } from './run-record-CP2ObebC.js';
2
2
  import { F as FailureClusterReport } from './failure-cluster-DH9Flgcf.js';
3
3
 
4
4
  /**
@@ -0,0 +1,20 @@
1
+ /**
2
+ * LockedJsonlAppender — mutex-serialized JSONL append helper for arbitrary
3
+ * payloads. The reference-replay store does the same thing for typed
4
+ * `ReferenceReplayRun` rows; this is the generic version used by
5
+ * `MutationTelemetry`, `TrialTelemetry`, and any other consumer that wants
6
+ * append-only durable telemetry without rolling its own lock.
7
+ *
8
+ * Locks are per absolute file path (process-local). Cross-process
9
+ * concurrency is NOT addressed — that's an fcntl/flock problem.
10
+ */
11
+ declare class LockedJsonlAppender {
12
+ readonly path: string;
13
+ private readonly mutex;
14
+ constructor(path: string);
15
+ append(entry: unknown): Promise<void>;
16
+ }
17
+ /** Reset all internal mutex state — tests only. */
18
+ declare function resetLockedAppendersForTesting(): void;
19
+
20
+ export { LockedJsonlAppender as L, resetLockedAppendersForTesting as r };
@@ -0,0 +1 @@
1
+ export { r as resetLockedAppendersForTesting } from './testing-C21CHsq2.js';
@@ -0,0 +1,8 @@
1
+ import {
2
+ resetLockedAppendersForTesting
3
+ } from "./chunk-DJWX3GVS.js";
4
+ import "./chunk-PZ5AY32C.js";
5
+ export {
6
+ resetLockedAppendersForTesting
7
+ };
8
+ //# sourceMappingURL=testing.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
package/dist/traces.d.ts CHANGED
@@ -14,8 +14,26 @@ import { A as AnalyzeTracesOptions, b as AnalyzeTracesResult } from './analyst-C
14
14
  export { a as AnalyzeTracesInput, c as AnalyzeTracesTurnSnapshot, d as analyzeTraces } from './analyst-C8HHvfJp.js';
15
15
  import { h as TraceAnalystSpanKind, i as TraceAnalystSpanStatus, T as TraceAnalysisStore, g as TraceAnalystFilters, b as DatasetOverview, Q as QueryTracesPage, l as ViewTraceResult, V as ViewSpansResult, c as SearchTraceResult, S as SearchSpanResult } from './store-C1YxJDEK.js';
16
16
  export { D as DEFAULT_TRACE_ANALYST_BUDGETS, E as ErrorCluster, d as SpanMatchRecord, e as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, f as TraceAnalystByteBudgets, a as TraceAnalystSpan, j as TraceAnalystTraceSummary, k as ViewTraceOversized } from './store-C1YxJDEK.js';
17
- import { a as RunSplitTag, b as RunTokenUsage, R as RunRecord } from './run-record-e7vj1uZQ.js';
17
+ import { a as RunSplitTag, b as RunTokenUsage, R as RunRecord } from './run-record-CP2ObebC.js';
18
18
  import { AxFunction } from '@ax-llm/ax';
19
+ import '@tangle-network/agent-interface';
20
+
21
+ /** Canonical OpenInference-over-OTLP attribute vocabulary used at the trace boundary. */
22
+ declare const OPENINFERENCE_SPAN_KIND = "openinference.span.kind";
23
+ declare const LLM_MODEL_NAME = "llm.model_name";
24
+ declare const LLM_INPUT_TOKENS = "llm.token_count.prompt";
25
+ declare const LLM_OUTPUT_TOKENS = "llm.token_count.completion";
26
+ declare const LLM_CACHED_TOKENS = "llm.token_count.prompt_cache_hit";
27
+ declare const LLM_COST_USD = "llm.cost_usd";
28
+ declare const TOOL_NAME = "tool.name";
29
+ declare const SPAN_KIND_ATTR_KEYS: readonly ["openinference.span.kind", "inference.observation_kind"];
30
+ declare const LLM_MODEL_ATTR_KEYS: readonly ["llm.model_name", "inference.llm.model_name", "llm.model", "gen_ai.request.model", "gen_ai.response.model"];
31
+ declare const LLM_INPUT_TOKEN_ATTR_KEYS: readonly ["llm.token_count.prompt", "inference.llm.input_tokens", "llm.input_tokens", "gen_ai.usage.input_tokens", "gen_ai.usage.prompt_tokens"];
32
+ declare const LLM_OUTPUT_TOKEN_ATTR_KEYS: readonly ["llm.token_count.completion", "inference.llm.output_tokens", "llm.output_tokens", "gen_ai.usage.output_tokens", "gen_ai.usage.completion_tokens"];
33
+ declare const LLM_CACHED_TOKEN_ATTR_KEYS: readonly ["llm.token_count.prompt_cache_hit", "inference.llm.cached_tokens", "llm.cached_tokens", "gen_ai.usage.cached_tokens"];
34
+ declare const LLM_COST_ATTR_KEYS: readonly ["llm.cost_usd", "inference.llm.cost.total", "llm.cost.total", "gen_ai.usage.cost"];
35
+ declare const TOOL_NAME_ATTR_KEYS: readonly ["tool.name", "inference.tool.name"];
36
+ declare function traceSpanKindToOpenInferenceKind(kind: string): string;
19
37
 
20
38
  /**
21
39
  * Trace-analyst auto-execution hook.
@@ -232,11 +250,9 @@ interface OtlpFlatLine {
232
250
  }>;
233
251
  }
234
252
  interface FlattenOtlpOptions {
235
- /** `'openinference'` (default) mirrors per-span attributes into the
236
- * OpenInference vocabulary the analyst's `inferKind` reads
237
- * (`llm.model`→`llm.model_name`, `tool.name`→`inference.tool.name`,
238
- * `span.kind`→`openinference.span.kind` uppercased). `'none'` passes
239
- * attributes through untouched. */
253
+ /** `'openinference'` (default) mirrors legacy per-span attributes into the
254
+ * canonical OpenInference vocabulary the analyst readers consume. `'none'`
255
+ * passes attributes through untouched. */
240
256
  attributeVocabulary?: 'openinference' | 'none';
241
257
  /** Override the numeric-kind → otlp-string mapping. */
242
258
  kindMap?: Partial<Record<number, string>>;
@@ -321,9 +337,9 @@ declare function firstStringAttr(attrs: Record<string, unknown>, keys: readonly
321
337
  * same shape `flattenOtlpExportToNdjson` produces) into validated
322
338
  * `RunRecord[]` — one record per `trace_id` (one trace == one task).
323
339
  *
324
- * This is the offline ingestion primitive the AppWorld driver bench and the
340
+ * This is the offline ingestion primitive the AppWorld proposer bench and the
325
341
  * hosted Intelligence product both stand on: traces in, paper-grade rows
326
- * out, ready for `compareDrivers` / `analyzeRuns` / the promotion gate.
342
+ * out, ready for `compareProposers` / `analyzeRuns` / the promotion gate.
327
343
  *
328
344
  * Aggregation per trace:
329
345
  * - tokenUsage: sum LLM-span `input` / `output` (+ `cached` when present)
@@ -354,7 +370,7 @@ interface OtlpToRunRecordsOptions {
354
370
  /** Logical experiment grouping for every produced record. */
355
371
  experimentId: string;
356
372
  /** Candidate (variant) id — the surface these traces exercised. The
357
- * bench passes the driver label here so `compareDrivers` can pair rows. */
373
+ * bench passes the proposer label here so `compareProposers` can pair rows. */
358
374
  candidateId: string;
359
375
  /** Split assignment for every produced record. Default `'holdout'` —
360
376
  * ingested traces are evidence, not the optimizer's training pool. */
@@ -957,4 +973,4 @@ declare function iterateRawCalls(sink: RawProviderSink, filter?: {
957
973
  spanId?: string;
958
974
  }): AsyncGenerator<ReplayCacheEntry>;
959
975
 
960
- export { AnalyzeTracesOptions, AnalyzeTracesResult, type CaptureFetchContext, type CaptureFetchOptions, DatasetOverview, type ExportableSpan, type ExtractedUsage, type FlattenOtlpOptions, OTEL_AGENT_EVAL_SCOPE, type OtelExportConfig, type OtelExporter, type OtlpExport, OtlpFileTraceStore, type OtlpFileTraceStoreOptions, type OtlpFlatLine, type OtlpResourceSpans, type OtlpSpan, type OtlpToRunRecordsOptions, type OtlpTraceRunRecord, type ProjectedOtlpSpan, ProviderRedactor, QueryTracesPage, RawProviderEvent, RawProviderSink, ReplayCache, type ReplayCacheEntry, ReplayCacheMissError, type ReplayCacheStats, type ReplayFetchOptions, Run, RunCompleteHook, RunCompleteHookContext, SearchSpanResult, SearchTraceResult, SpanNotFoundError, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, type TraceAggregate, TraceAnalysisStore, TraceAnalystFilters, type TraceAnalystHookOptions, TraceAnalystSpanKind, TraceAnalystSpanStatus, TraceFileMissingError, type TraceInsightContext, type TraceInsightFinding, type TraceInsightPanelRole, type TraceInsightPromptInput, type TraceInsightQualityGate, type TraceInsightQuestion, type TraceInsightReadiness, type TraceInsightSuite, type TraceInsightTask, TraceNotFoundError, TraceStore, type TraceStoreSource, type TraceStoreToOtlpOptions, type TracesToOtlpResult, ViewSpansResult, ViewTraceResult, asNumber, asString, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, captureFetchToRawSink, convertTraceStoresToOtlp, createOtelExporter, createOtelTracingStore, createReplayFetch, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, extractOtlpAttributes, extractUsage, extractUsageFromResponse, extractUsageFromSse, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, inferDomainKeywords, inferOtlpKind, iterateRawCalls, otelRunCompleteHook, otlpToRunRecords, otlpToTraceRunRecords, planTraceInsightQuestions, projectOtlpFlatLine, readOtlpStatus, scoreTraceInsightReadiness, stringField, tokenizeDomainWords, traceAnalystFunctionGroup, traceAnalystOnRunComplete };
976
+ export { AnalyzeTracesOptions, AnalyzeTracesResult, type CaptureFetchContext, type CaptureFetchOptions, DatasetOverview, type ExportableSpan, type ExtractedUsage, type FlattenOtlpOptions, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, OPENINFERENCE_SPAN_KIND, OTEL_AGENT_EVAL_SCOPE, type OtelExportConfig, type OtelExporter, type OtlpExport, OtlpFileTraceStore, type OtlpFileTraceStoreOptions, type OtlpFlatLine, type OtlpResourceSpans, type OtlpSpan, type OtlpToRunRecordsOptions, type OtlpTraceRunRecord, type ProjectedOtlpSpan, ProviderRedactor, QueryTracesPage, RawProviderEvent, RawProviderSink, ReplayCache, type ReplayCacheEntry, ReplayCacheMissError, type ReplayCacheStats, type ReplayFetchOptions, Run, RunCompleteHook, RunCompleteHookContext, SPAN_KIND_ATTR_KEYS, SearchSpanResult, SearchTraceResult, SpanNotFoundError, TOOL_NAME, TOOL_NAME_ATTR_KEYS, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, type TraceAggregate, TraceAnalysisStore, TraceAnalystFilters, type TraceAnalystHookOptions, TraceAnalystSpanKind, TraceAnalystSpanStatus, TraceFileMissingError, type TraceInsightContext, type TraceInsightFinding, type TraceInsightPanelRole, type TraceInsightPromptInput, type TraceInsightQualityGate, type TraceInsightQuestion, type TraceInsightReadiness, type TraceInsightSuite, type TraceInsightTask, TraceNotFoundError, TraceStore, type TraceStoreSource, type TraceStoreToOtlpOptions, type TracesToOtlpResult, ViewSpansResult, ViewTraceResult, asNumber, asString, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, captureFetchToRawSink, convertTraceStoresToOtlp, createOtelExporter, createOtelTracingStore, createReplayFetch, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, extractOtlpAttributes, extractUsage, extractUsageFromResponse, extractUsageFromSse, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, inferDomainKeywords, inferOtlpKind, iterateRawCalls, otelRunCompleteHook, otlpToRunRecords, otlpToTraceRunRecords, planTraceInsightQuestions, projectOtlpFlatLine, readOtlpStatus, scoreTraceInsightReadiness, stringField, tokenizeDomainWords, traceAnalystFunctionGroup, traceAnalystOnRunComplete, traceSpanKindToOpenInferenceKind };
package/dist/traces.js CHANGED
@@ -28,7 +28,13 @@ import {
28
28
  scoreTraceInsightReadiness,
29
29
  tokenizeDomainWords,
30
30
  traceAnalystOnRunComplete
31
- } from "./chunk-CTBHKLEU.js";
31
+ } from "./chunk-77T4STFI.js";
32
+ import {
33
+ TRACE_ANALYST_ACTOR_DESCRIPTION,
34
+ TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
35
+ TRACE_ANALYST_SUBAGENT_DESCRIPTION,
36
+ analyzeTraces
37
+ } from "./chunk-IZCEK2HR.js";
32
38
  import {
33
39
  DEFAULT_REDACTION_RULES,
34
40
  REDACTION_VERSION,
@@ -54,16 +60,24 @@ import {
54
60
  isSandboxSpan,
55
61
  isToolSpan
56
62
  } from "./chunk-5BKGXME7.js";
57
- import {
58
- TRACE_ANALYST_ACTOR_DESCRIPTION,
59
- TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
60
- TRACE_ANALYST_SUBAGENT_DESCRIPTION,
61
- analyzeTraces
62
- } from "./chunk-TBDR6PAI.js";
63
63
  import {
64
64
  DEFAULT_TRACE_ANALYST_BUDGETS,
65
+ LLM_CACHED_TOKENS,
66
+ LLM_CACHED_TOKEN_ATTR_KEYS,
67
+ LLM_COST_ATTR_KEYS,
68
+ LLM_COST_USD,
69
+ LLM_INPUT_TOKENS,
70
+ LLM_INPUT_TOKEN_ATTR_KEYS,
71
+ LLM_MODEL_ATTR_KEYS,
72
+ LLM_MODEL_NAME,
73
+ LLM_OUTPUT_TOKENS,
74
+ LLM_OUTPUT_TOKEN_ATTR_KEYS,
75
+ OPENINFERENCE_SPAN_KIND,
65
76
  OtlpFileTraceStore,
77
+ SPAN_KIND_ATTR_KEYS,
66
78
  SpanNotFoundError,
79
+ TOOL_NAME,
80
+ TOOL_NAME_ATTR_KEYS,
67
81
  TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
68
82
  TraceFileMissingError,
69
83
  TraceNotFoundError,
@@ -77,8 +91,9 @@ import {
77
91
  projectOtlpFlatLine,
78
92
  readOtlpStatus,
79
93
  stringField,
80
- traceAnalystFunctionGroup
81
- } from "./chunk-QMUEXQJS.js";
94
+ traceAnalystFunctionGroup,
95
+ traceSpanKindToOpenInferenceKind
96
+ } from "./chunk-YBIGNSCZ.js";
82
97
  import {
83
98
  RunIntegrityError,
84
99
  assertRunCaptured,
@@ -91,7 +106,7 @@ import {
91
106
  defaultProviderRedactor,
92
107
  providerFromBaseUrl
93
108
  } from "./chunk-PC4UYEBM.js";
94
- import "./chunk-KWRRMR3J.js";
109
+ import "./chunk-LO6IOIJ2.js";
95
110
  import {
96
111
  TraceEmitter,
97
112
  llmSpanFromProvider
@@ -107,14 +122,28 @@ export {
107
122
  FileSystemTraceStore,
108
123
  InMemoryRawProviderSink,
109
124
  InMemoryTraceStore,
125
+ LLM_CACHED_TOKENS,
126
+ LLM_CACHED_TOKEN_ATTR_KEYS,
127
+ LLM_COST_ATTR_KEYS,
128
+ LLM_COST_USD,
129
+ LLM_INPUT_TOKENS,
130
+ LLM_INPUT_TOKEN_ATTR_KEYS,
131
+ LLM_MODEL_ATTR_KEYS,
132
+ LLM_MODEL_NAME,
133
+ LLM_OUTPUT_TOKENS,
134
+ LLM_OUTPUT_TOKEN_ATTR_KEYS,
110
135
  NoopRawProviderSink,
136
+ OPENINFERENCE_SPAN_KIND,
111
137
  OTEL_AGENT_EVAL_SCOPE,
112
138
  OtlpFileTraceStore,
113
139
  REDACTION_VERSION,
114
140
  ReplayCache,
115
141
  ReplayCacheMissError,
116
142
  RunIntegrityError,
143
+ SPAN_KIND_ATTR_KEYS,
117
144
  SpanNotFoundError,
145
+ TOOL_NAME,
146
+ TOOL_NAME_ATTR_KEYS,
118
147
  TRACE_ANALYST_ACTOR_DESCRIPTION,
119
148
  TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
120
149
  TRACE_ANALYST_SUBAGENT_DESCRIPTION,
@@ -178,6 +207,7 @@ export {
178
207
  tokenizeDomainWords,
179
208
  toolSpans,
180
209
  traceAnalystFunctionGroup,
181
- traceAnalystOnRunComplete
210
+ traceAnalystOnRunComplete,
211
+ traceSpanKindToOpenInferenceKind
182
212
  };
183
213
  //# sourceMappingURL=traces.js.map
@@ -1,4 +1,4 @@
1
- import { R as RunRecord } from './run-record-e7vj1uZQ.js';
1
+ import { R as RunRecord } from './run-record-CP2ObebC.js';
2
2
  import { T as TraceAnalysisStore } from './store-C1YxJDEK.js';
3
3
  import { a as JudgeInput } from './types-C7DGg5ex.js';
4
4
  import { b as LlmCallRequest, c as LlmCallResult } from './llm-client-Bj7g0rqu.js';
@@ -1,13 +1,11 @@
1
1
  import { D as DefaultVerdict } from './verdict-C9MlYujm.js';
2
2
 
3
3
  /**
4
- * @experimental
5
- *
6
4
  * N-axis cartesian matrix over substrate types — types module.
7
5
  *
8
6
  * The matrix is a runner + aggregator. It iterates the cartesian product of
9
- * caller-provided axes (any value type — `AgentProfile` from sandbox, `Driver`
10
- * / `Validator` from agent-runtime, rubric records, thinking levels, anything)
7
+ * caller-provided axes (any value type — `AgentProfile` from agent-interface,
8
+ * `Driver` / `Validator` from agent-runtime, rubric records, thinking levels, anything)
11
9
  * and aggregates per-axis pass/score/cost summaries. Substrate types are
12
10
  * imported at the boundary by JSDoc only; the matrix never wraps them.
13
11
  */