@tangle-network/agent-eval 0.94.0 → 0.95.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/README.md +44 -30
- package/dist/adapters/http.d.ts +8 -7
- package/dist/adapters/http.js.map +1 -1
- package/dist/adapters/langchain.d.ts +3 -2
- package/dist/adapters/otel.d.ts +5 -4
- package/dist/analyst/index.d.ts +11 -31
- package/dist/analyst/index.js +5 -65
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-B6Ljo_dI.d.ts → analyze-runs-DtT6F_6T.d.ts} +3 -3
- package/dist/belief-state/index.d.ts +4 -3
- package/dist/benchmarks/index.d.ts +3 -2
- package/dist/campaign/index.d.ts +727 -616
- package/dist/campaign/index.js +1863 -1316
- package/dist/campaign/index.js.map +1 -1
- package/dist/{chunk-2K6UUZ7P.js → chunk-2T4EZACH.js} +1 -1
- package/dist/chunk-2T4EZACH.js.map +1 -0
- package/dist/{chunk-CTBHKLEU.js → chunk-77T4STFI.js} +59 -86
- package/dist/chunk-77T4STFI.js.map +1 -0
- package/dist/{chunk-EGPMSBEZ.js → chunk-7QTQKIDD.js} +178 -177
- package/dist/chunk-7QTQKIDD.js.map +1 -0
- package/dist/{chunk-MIFZUPEK.js → chunk-AQ5WQAIV.js} +21 -6
- package/dist/chunk-AQ5WQAIV.js.map +1 -0
- package/dist/chunk-DJWX3GVS.js +81 -0
- package/dist/chunk-DJWX3GVS.js.map +1 -0
- package/dist/{chunk-S6OZEZQK.js → chunk-HMA63UEO.js} +37 -9
- package/dist/{chunk-S6OZEZQK.js.map → chunk-HMA63UEO.js.map} +1 -1
- package/dist/{chunk-TBDR6PAI.js → chunk-IZCEK2HR.js} +2 -2
- package/dist/{chunk-SD2YFWQQ.js → chunk-KKWJD5E6.js} +20 -20
- package/dist/chunk-KKWJD5E6.js.map +1 -0
- package/dist/{chunk-KWRRMR3J.js → chunk-LO6IOIJ2.js} +10 -10
- package/dist/chunk-LO6IOIJ2.js.map +1 -0
- package/dist/{chunk-E4GH6USR.js → chunk-NZEQVRH5.js} +2 -2
- package/dist/chunk-NZEQVRH5.js.map +1 -0
- package/dist/{chunk-MPQWFX6Y.js → chunk-PSWWQXHF.js} +13 -88
- package/dist/chunk-PSWWQXHF.js.map +1 -0
- package/dist/{chunk-Q5LIB7BC.js → chunk-S4SYLDFX.js} +2 -2
- package/dist/chunk-S4SYLDFX.js.map +1 -0
- package/dist/{chunk-KW53MSA5.js → chunk-X74V6ESX.js} +2 -2
- package/dist/{chunk-QMUEXQJS.js → chunk-YBIGNSCZ.js} +81 -4
- package/dist/chunk-YBIGNSCZ.js.map +1 -0
- package/dist/{chunk-2KNZHH3P.js → chunk-Z6L6YSU6.js} +2 -2
- package/dist/{code-agent-session-BO8nCnv3.d.ts → code-agent-session-CPHRCb4-.d.ts} +1 -1
- package/dist/contract/index.d.ts +91 -43
- package/dist/contract/index.js +127 -17
- package/dist/contract/index.js.map +1 -1
- package/dist/{control-D6qwHXIR.d.ts → control-Doncu-B_.d.ts} +2 -2
- package/dist/control.d.ts +3 -2
- package/dist/control.js +2 -2
- package/dist/{corpus-B8A4BDR3.d.ts → corpus-D4YW9UoJ.d.ts} +1 -1
- package/dist/{default-registry-6dhErQbs.d.ts → default-registry-GyE8X5SP.d.ts} +3 -3
- package/dist/diagnose.d.ts +4 -3
- package/dist/diagnose.js +1 -1
- package/dist/{run-improvement-loop-DBahB8Ax.d.ts → gepa-C1NCIZ9o.d.ts} +117 -130
- package/dist/hosted/index.d.ts +5 -4
- package/dist/{index-Bx3gZ8xl.d.ts → index-_Y4oNOOb.d.ts} +1 -1
- package/dist/index.d.ts +76 -81
- package/dist/index.js +66 -31
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-DWl3z9tl.d.ts → insight-report-BnRjTibG.d.ts} +1 -1
- package/dist/{kind-factory-0BhLSI27.d.ts → kind-factory-X3eDYbKn.d.ts} +2 -3
- package/dist/matrix/index.d.ts +1 -1
- package/dist/meta-eval/index.d.ts +3 -2
- package/dist/multishot/index.d.ts +4 -4
- package/dist/multishot/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{pre-registration-mAnCugl9.d.ts → pre-registration-nfUdc9EQ.d.ts} +2 -42
- package/dist/{provenance-P-bCL2Fo.d.ts → provenance-CncDq9qE.d.ts} +26 -41
- package/dist/{release-report-BEbWmVYj.d.ts → release-report-pidWUMZ2.d.ts} +2 -2
- package/dist/reporting.d.ts +5 -4
- package/dist/{researcher-B0C2_fVO.d.ts → researcher-Jr8ME1dZ.d.ts} +2 -2
- package/dist/rl.d.ts +516 -515
- package/dist/rl.js +612 -612
- package/dist/rl.js.map +1 -1
- package/dist/{rubric-predictive-validity-Cy_W-hWZ.d.ts → rubric-predictive-validity-C2hDKM8Z.d.ts} +1 -1
- package/dist/{run-campaign-7WNXMDSN.js → run-campaign-WXY7KI67.js} +2 -2
- package/dist/{run-record-e7vj1uZQ.d.ts → run-record-CP2ObebC.d.ts} +14 -18
- package/dist/{runtime-trajectory-BDgfGZSr.d.ts → runtime-trajectory-BOUUjI0y.d.ts} +1 -1
- package/dist/{semantic-concept-judge-B9MgmBnM.d.ts → semantic-concept-judge-DSBB2Cfp.d.ts} +2 -2
- package/dist/{summary-report-BDOFevaT.d.ts → summary-report-CInXwsza.d.ts} +1 -1
- package/dist/testing-C21CHsq2.d.ts +20 -0
- package/dist/testing.d.ts +1 -0
- package/dist/testing.js +8 -0
- package/dist/testing.js.map +1 -0
- package/dist/traces.d.ts +26 -10
- package/dist/traces.js +41 -11
- package/dist/{types-Ce17tDlG.d.ts → types-B5x54y6n.d.ts} +1 -1
- package/dist/{types-mn5Aqk7x.d.ts → types-BUxNaJ8c.d.ts} +2 -4
- package/dist/{types-BU-7W85F.d.ts → types-DQRY8ZT-.d.ts} +60 -58
- package/dist/workflow/index.d.ts +5 -4
- package/dist/workflow/index.js +1 -1
- package/docs/campaign-proposers.md +170 -0
- package/docs/concepts.md +8 -4
- package/docs/customer-journeys.md +15 -13
- package/docs/design/loop-taxonomy.md +34 -66
- package/docs/distributed-driver.md +14 -14
- package/docs/feature-guide.md +1 -1
- package/docs/hosted-ingest-spec.md +2 -3
- package/docs/multi-shot-optimization.md +8 -8
- package/docs/product-eval-adoption.md +1 -1
- package/docs/self-improvement-map.md +33 -29
- package/package.json +8 -14
- package/dist/chunk-2K6UUZ7P.js.map +0 -1
- package/dist/chunk-CTBHKLEU.js.map +0 -1
- package/dist/chunk-E4GH6USR.js.map +0 -1
- package/dist/chunk-EGPMSBEZ.js.map +0 -1
- package/dist/chunk-KWRRMR3J.js.map +0 -1
- package/dist/chunk-MIFZUPEK.js.map +0 -1
- package/dist/chunk-MPQWFX6Y.js.map +0 -1
- package/dist/chunk-Q5LIB7BC.js.map +0 -1
- package/dist/chunk-QMUEXQJS.js.map +0 -1
- package/dist/chunk-SD2YFWQQ.js.map +0 -1
- package/docs/design/external-agent-wedge.md +0 -89
- package/docs/design/phase-d-rfc.md +0 -125
- package/docs/design/phase4-consumer-migration.md +0 -70
- package/docs/design/primitives-integration-spec.md +0 -393
- package/docs/design/product-self-improvement-loop.md +0 -146
- package/docs/design/self-improvement-engine.md +0 -140
- package/docs/design/self-improvement-protocol.md +0 -223
- package/docs/design/self-improvement-roadmap.md +0 -106
- package/docs/design/substrate-gaps.md +0 -118
- package/docs/phase-b-pairing-kit.md +0 -188
- package/docs/phase-b-runbook.md +0 -176
- package/docs/pilot/README.md +0 -62
- package/docs/pilot/customer-checklist.md +0 -90
- package/docs/pilot/integration-foreign-stack.md +0 -296
- package/docs/pilot/integration-tangle-stack.md +0 -248
- package/docs/pilot/one-pager.md +0 -161
- package/docs/pilot/sample-insight-report.json +0 -172
- package/docs/quickstart-external.md +0 -229
- package/docs/research/belief-state-agent-eval-roadmap.md +0 -593
- package/docs/research/research-roadmap.md +0 -205
- package/docs/specs/driver-honest-spec.md +0 -251
- package/docs/specs/hermes-self-improvement-audit.md +0 -93
- package/docs/specs/profile-versioning.md +0 -291
- package/docs/three-package-architecture.md +0 -168
- /package/dist/{chunk-TBDR6PAI.js.map → chunk-IZCEK2HR.js.map} +0 -0
- /package/dist/{chunk-KW53MSA5.js.map → chunk-X74V6ESX.js.map} +0 -0
- /package/dist/{chunk-2KNZHH3P.js.map → chunk-Z6L6YSU6.js.map} +0 -0
- /package/dist/{run-campaign-7WNXMDSN.js.map → run-campaign-WXY7KI67.js.map} +0 -0
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import {
|
|
2
2
|
runCampaign
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-2T4EZACH.js";
|
|
4
4
|
import "./chunk-56GC6VYO.js";
|
|
5
5
|
import "./chunk-3BFEG2F6.js";
|
|
6
6
|
import "./chunk-PZ5AY32C.js";
|
|
7
7
|
export {
|
|
8
8
|
runCampaign
|
|
9
9
|
};
|
|
10
|
-
//# sourceMappingURL=run-campaign-
|
|
10
|
+
//# sourceMappingURL=run-campaign-WXY7KI67.js.map
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { AgentProfile } from '@tangle-network/agent-interface';
|
|
1
2
|
import { V as ValidationError } from './errors-CzMUYo7b.js';
|
|
2
3
|
import { F as FailureClass } from './schema-m0gsnbt3.js';
|
|
3
4
|
|
|
@@ -7,7 +8,7 @@ type AgentProfileJson = string | number | boolean | null | AgentProfileJson[] |
|
|
|
7
8
|
};
|
|
8
9
|
type AgentProfileDimensionValue = string | number | boolean | null;
|
|
9
10
|
interface AgentProfileSource {
|
|
10
|
-
/** Runtime/profile contract being fingerprinted, e.g. `
|
|
11
|
+
/** Runtime/profile contract being fingerprinted, e.g. `agent-interface-profile`. */
|
|
11
12
|
kind: string;
|
|
12
13
|
/** sha256 over the canonical source profile object. */
|
|
13
14
|
hash: string;
|
|
@@ -74,9 +75,9 @@ declare function groupRunsByAgentProfileCell<T extends {
|
|
|
74
75
|
* new kind is a deliberate cross-product schema change. */
|
|
75
76
|
declare const AGENT_PROFILE_KINDS: {
|
|
76
77
|
/** A profile declared via `defineAgentProfile(...)` from
|
|
77
|
-
* `@tangle-network/
|
|
78
|
-
*
|
|
79
|
-
readonly
|
|
78
|
+
* `@tangle-network/agent-interface`. The default kind for router-backed
|
|
79
|
+
* and sandbox-backed products. */
|
|
80
|
+
readonly AGENT_INTERFACE_PROFILE: "agent-interface-profile";
|
|
80
81
|
};
|
|
81
82
|
type AgentProfileKind = (typeof AGENT_PROFILE_KINDS)[keyof typeof AGENT_PROFILE_KINDS];
|
|
82
83
|
/** Canonicalize an arbitrary value into `AgentProfileJson` by JSON
|
|
@@ -84,26 +85,21 @@ type AgentProfileKind = (typeof AGENT_PROFILE_KINDS)[keyof typeof AGENT_PROFILE_
|
|
|
84
85
|
* as JSON (functions, BigInt, cycles) — non-portable profiles fail loud
|
|
85
86
|
* rather than silently dropping fields. */
|
|
86
87
|
declare function toAgentProfileJson(value: unknown): AgentProfileJson;
|
|
87
|
-
/**
|
|
88
|
-
|
|
89
|
-
* Compatible with `defineAgentProfile(...)` output from
|
|
90
|
-
* `@tangle-network/sandbox`; products that have not yet declared a real
|
|
91
|
-
* profile can pass a `{ name, version, ...metadata }` stub. */
|
|
92
|
-
interface SandboxAgentProfileLike {
|
|
88
|
+
/** Canonical AgentProfile shape required when deriving a stable cell id. */
|
|
89
|
+
type AgentInterfaceProfileLike = AgentProfile & {
|
|
93
90
|
name: string;
|
|
94
91
|
version: string;
|
|
95
|
-
|
|
96
|
-
}
|
|
92
|
+
};
|
|
97
93
|
/** Higher-level helper that hard-codes the canonical
|
|
98
|
-
* `
|
|
94
|
+
* `agent-interface-profile` kind plus the JSON canonicalization. Equivalent
|
|
99
95
|
* to calling `buildAgentProfileCell` with `profileId = \`${name}@${version}\``
|
|
100
|
-
* and `sourceProfile = { kind:
|
|
96
|
+
* and `sourceProfile = { kind: AGENT_INTERFACE_PROFILE, profile: <round-tripped> }`.
|
|
101
97
|
*
|
|
102
|
-
* Use this from any product consuming
|
|
98
|
+
* Use this from any product consuming an agent-interface `AgentProfile`; the
|
|
103
99
|
* manual `buildAgentProfileCell` call is reserved for advanced cases
|
|
104
100
|
* (custom kinds, pre-computed source hashes, alternate profileId
|
|
105
101
|
* conventions). */
|
|
106
|
-
declare function
|
|
102
|
+
declare function buildAgentInterfaceProfileCell(profile: AgentInterfaceProfileLike, input: Omit<AgentProfileCellInput, 'profileId' | 'sourceProfile'>): Promise<AgentProfileCell>;
|
|
107
103
|
|
|
108
104
|
/**
|
|
109
105
|
* Paper-grade RunRecord schema + runtime validator.
|
|
@@ -221,7 +217,7 @@ interface RunOutcome {
|
|
|
221
217
|
* model (after any steering bundle merge).
|
|
222
218
|
* - `configHash` is the sha256 of the effective run config (model,
|
|
223
219
|
* temperature, tools, judges, splits). The pair (promptHash,
|
|
224
|
-
* configHash) uniquely identifies an
|
|
220
|
+
* configHash) uniquely identifies an experiment cell.
|
|
225
221
|
*
|
|
226
222
|
* Model snapshot discipline:
|
|
227
223
|
* - `model` MUST encode a snapshot version. Bare aliases like
|
|
@@ -315,4 +311,4 @@ declare function parseRunRecordSafe(input: unknown): {
|
|
|
315
311
|
/** Round-trip helper — `JSON.parse(JSON.stringify(record))` then validate. */
|
|
316
312
|
declare function roundTripRunRecord(record: RunRecord): RunRecord;
|
|
317
313
|
|
|
318
|
-
export { type AgentProfileCell as A,
|
|
314
|
+
export { type AgentProfileCell as A, toAgentProfileJson as B, validateAgentProfileCell as C, validateRunRecord as D, verifyAgentProfileCell as E, type JudgeScoresRecord as J, type RunRecord as R, type RunSplitTag as a, type RunTokenUsage as b, type RunJudgeMetadata as c, type AgentProfileCellInput as d, AGENT_PROFILE_KINDS as e, type AgentInterfaceProfileLike as f, type AgentProfileCellSchemaVersion as g, AgentProfileCellValidationError as h, type AgentProfileDimensionValue as i, type AgentProfileHarness as j, type AgentProfileJson as k, type AgentProfileKind as l, type AgentProfileSource as m, type AgentProfileSourceInput as n, type RunOutcome as o, RunRecordValidationError as p, agentProfileCellHashMaterial as q, agentProfileCellKey as r, assertRunAgentProfileCell as s, buildAgentInterfaceProfileCell as t, buildAgentProfileCell as u, groupRunsByAgentProfileCell as v, isRunRecord as w, parseRunRecordSafe as x, requireAgentProfileCell as y, roundTripRunRecord as z };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { AxAIService } from '@ax-llm/ax';
|
|
2
2
|
import { z } from 'zod';
|
|
3
|
-
import { c as AnalystFinding, A as Analyst, a as AnalystContext } from './types-
|
|
4
|
-
import { T as TraceAnalystKindSpec } from './kind-factory-
|
|
3
|
+
import { c as AnalystFinding, A as Analyst, a as AnalystContext } from './types-B5x54y6n.js';
|
|
4
|
+
import { T as TraceAnalystKindSpec } from './kind-factory-X3eDYbKn.js';
|
|
5
5
|
import { a as TraceAnalystSpan } from './store-C1YxJDEK.js';
|
|
6
6
|
import { L as LlmClientOptions } from './llm-client-Bj7g0rqu.js';
|
|
7
7
|
import { S as Severity } from './multi-layer-verifier-DUZXrPDA.js';
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LockedJsonlAppender — mutex-serialized JSONL append helper for arbitrary
|
|
3
|
+
* payloads. The reference-replay store does the same thing for typed
|
|
4
|
+
* `ReferenceReplayRun` rows; this is the generic version used by
|
|
5
|
+
* `MutationTelemetry`, `TrialTelemetry`, and any other consumer that wants
|
|
6
|
+
* append-only durable telemetry without rolling its own lock.
|
|
7
|
+
*
|
|
8
|
+
* Locks are per absolute file path (process-local). Cross-process
|
|
9
|
+
* concurrency is NOT addressed — that's an fcntl/flock problem.
|
|
10
|
+
*/
|
|
11
|
+
declare class LockedJsonlAppender {
|
|
12
|
+
readonly path: string;
|
|
13
|
+
private readonly mutex;
|
|
14
|
+
constructor(path: string);
|
|
15
|
+
append(entry: unknown): Promise<void>;
|
|
16
|
+
}
|
|
17
|
+
/** Reset all internal mutex state — tests only. */
|
|
18
|
+
declare function resetLockedAppendersForTesting(): void;
|
|
19
|
+
|
|
20
|
+
export { LockedJsonlAppender as L, resetLockedAppendersForTesting as r };
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { r as resetLockedAppendersForTesting } from './testing-C21CHsq2.js';
|
package/dist/testing.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
|
package/dist/traces.d.ts
CHANGED
|
@@ -14,8 +14,26 @@ import { A as AnalyzeTracesOptions, b as AnalyzeTracesResult } from './analyst-C
|
|
|
14
14
|
export { a as AnalyzeTracesInput, c as AnalyzeTracesTurnSnapshot, d as analyzeTraces } from './analyst-C8HHvfJp.js';
|
|
15
15
|
import { h as TraceAnalystSpanKind, i as TraceAnalystSpanStatus, T as TraceAnalysisStore, g as TraceAnalystFilters, b as DatasetOverview, Q as QueryTracesPage, l as ViewTraceResult, V as ViewSpansResult, c as SearchTraceResult, S as SearchSpanResult } from './store-C1YxJDEK.js';
|
|
16
16
|
export { D as DEFAULT_TRACE_ANALYST_BUDGETS, E as ErrorCluster, d as SpanMatchRecord, e as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, f as TraceAnalystByteBudgets, a as TraceAnalystSpan, j as TraceAnalystTraceSummary, k as ViewTraceOversized } from './store-C1YxJDEK.js';
|
|
17
|
-
import { a as RunSplitTag, b as RunTokenUsage, R as RunRecord } from './run-record-
|
|
17
|
+
import { a as RunSplitTag, b as RunTokenUsage, R as RunRecord } from './run-record-CP2ObebC.js';
|
|
18
18
|
import { AxFunction } from '@ax-llm/ax';
|
|
19
|
+
import '@tangle-network/agent-interface';
|
|
20
|
+
|
|
21
|
+
/** Canonical OpenInference-over-OTLP attribute vocabulary used at the trace boundary. */
|
|
22
|
+
declare const OPENINFERENCE_SPAN_KIND = "openinference.span.kind";
|
|
23
|
+
declare const LLM_MODEL_NAME = "llm.model_name";
|
|
24
|
+
declare const LLM_INPUT_TOKENS = "llm.token_count.prompt";
|
|
25
|
+
declare const LLM_OUTPUT_TOKENS = "llm.token_count.completion";
|
|
26
|
+
declare const LLM_CACHED_TOKENS = "llm.token_count.prompt_cache_hit";
|
|
27
|
+
declare const LLM_COST_USD = "llm.cost_usd";
|
|
28
|
+
declare const TOOL_NAME = "tool.name";
|
|
29
|
+
declare const SPAN_KIND_ATTR_KEYS: readonly ["openinference.span.kind", "inference.observation_kind"];
|
|
30
|
+
declare const LLM_MODEL_ATTR_KEYS: readonly ["llm.model_name", "inference.llm.model_name", "llm.model", "gen_ai.request.model", "gen_ai.response.model"];
|
|
31
|
+
declare const LLM_INPUT_TOKEN_ATTR_KEYS: readonly ["llm.token_count.prompt", "inference.llm.input_tokens", "llm.input_tokens", "gen_ai.usage.input_tokens", "gen_ai.usage.prompt_tokens"];
|
|
32
|
+
declare const LLM_OUTPUT_TOKEN_ATTR_KEYS: readonly ["llm.token_count.completion", "inference.llm.output_tokens", "llm.output_tokens", "gen_ai.usage.output_tokens", "gen_ai.usage.completion_tokens"];
|
|
33
|
+
declare const LLM_CACHED_TOKEN_ATTR_KEYS: readonly ["llm.token_count.prompt_cache_hit", "inference.llm.cached_tokens", "llm.cached_tokens", "gen_ai.usage.cached_tokens"];
|
|
34
|
+
declare const LLM_COST_ATTR_KEYS: readonly ["llm.cost_usd", "inference.llm.cost.total", "llm.cost.total", "gen_ai.usage.cost"];
|
|
35
|
+
declare const TOOL_NAME_ATTR_KEYS: readonly ["tool.name", "inference.tool.name"];
|
|
36
|
+
declare function traceSpanKindToOpenInferenceKind(kind: string): string;
|
|
19
37
|
|
|
20
38
|
/**
|
|
21
39
|
* Trace-analyst auto-execution hook.
|
|
@@ -232,11 +250,9 @@ interface OtlpFlatLine {
|
|
|
232
250
|
}>;
|
|
233
251
|
}
|
|
234
252
|
interface FlattenOtlpOptions {
|
|
235
|
-
/** `'openinference'` (default) mirrors per-span attributes into the
|
|
236
|
-
* OpenInference vocabulary the analyst
|
|
237
|
-
*
|
|
238
|
-
* `span.kind`→`openinference.span.kind` uppercased). `'none'` passes
|
|
239
|
-
* attributes through untouched. */
|
|
253
|
+
/** `'openinference'` (default) mirrors legacy per-span attributes into the
|
|
254
|
+
* canonical OpenInference vocabulary the analyst readers consume. `'none'`
|
|
255
|
+
* passes attributes through untouched. */
|
|
240
256
|
attributeVocabulary?: 'openinference' | 'none';
|
|
241
257
|
/** Override the numeric-kind → otlp-string mapping. */
|
|
242
258
|
kindMap?: Partial<Record<number, string>>;
|
|
@@ -321,9 +337,9 @@ declare function firstStringAttr(attrs: Record<string, unknown>, keys: readonly
|
|
|
321
337
|
* same shape `flattenOtlpExportToNdjson` produces) into validated
|
|
322
338
|
* `RunRecord[]` — one record per `trace_id` (one trace == one task).
|
|
323
339
|
*
|
|
324
|
-
* This is the offline ingestion primitive the AppWorld
|
|
340
|
+
* This is the offline ingestion primitive the AppWorld proposer bench and the
|
|
325
341
|
* hosted Intelligence product both stand on: traces in, paper-grade rows
|
|
326
|
-
* out, ready for `
|
|
342
|
+
* out, ready for `compareProposers` / `analyzeRuns` / the promotion gate.
|
|
327
343
|
*
|
|
328
344
|
* Aggregation per trace:
|
|
329
345
|
* - tokenUsage: sum LLM-span `input` / `output` (+ `cached` when present)
|
|
@@ -354,7 +370,7 @@ interface OtlpToRunRecordsOptions {
|
|
|
354
370
|
/** Logical experiment grouping for every produced record. */
|
|
355
371
|
experimentId: string;
|
|
356
372
|
/** Candidate (variant) id — the surface these traces exercised. The
|
|
357
|
-
* bench passes the
|
|
373
|
+
* bench passes the proposer label here so `compareProposers` can pair rows. */
|
|
358
374
|
candidateId: string;
|
|
359
375
|
/** Split assignment for every produced record. Default `'holdout'` —
|
|
360
376
|
* ingested traces are evidence, not the optimizer's training pool. */
|
|
@@ -957,4 +973,4 @@ declare function iterateRawCalls(sink: RawProviderSink, filter?: {
|
|
|
957
973
|
spanId?: string;
|
|
958
974
|
}): AsyncGenerator<ReplayCacheEntry>;
|
|
959
975
|
|
|
960
|
-
export { AnalyzeTracesOptions, AnalyzeTracesResult, type CaptureFetchContext, type CaptureFetchOptions, DatasetOverview, type ExportableSpan, type ExtractedUsage, type FlattenOtlpOptions, OTEL_AGENT_EVAL_SCOPE, type OtelExportConfig, type OtelExporter, type OtlpExport, OtlpFileTraceStore, type OtlpFileTraceStoreOptions, type OtlpFlatLine, type OtlpResourceSpans, type OtlpSpan, type OtlpToRunRecordsOptions, type OtlpTraceRunRecord, type ProjectedOtlpSpan, ProviderRedactor, QueryTracesPage, RawProviderEvent, RawProviderSink, ReplayCache, type ReplayCacheEntry, ReplayCacheMissError, type ReplayCacheStats, type ReplayFetchOptions, Run, RunCompleteHook, RunCompleteHookContext, SearchSpanResult, SearchTraceResult, SpanNotFoundError, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, type TraceAggregate, TraceAnalysisStore, TraceAnalystFilters, type TraceAnalystHookOptions, TraceAnalystSpanKind, TraceAnalystSpanStatus, TraceFileMissingError, type TraceInsightContext, type TraceInsightFinding, type TraceInsightPanelRole, type TraceInsightPromptInput, type TraceInsightQualityGate, type TraceInsightQuestion, type TraceInsightReadiness, type TraceInsightSuite, type TraceInsightTask, TraceNotFoundError, TraceStore, type TraceStoreSource, type TraceStoreToOtlpOptions, type TracesToOtlpResult, ViewSpansResult, ViewTraceResult, asNumber, asString, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, captureFetchToRawSink, convertTraceStoresToOtlp, createOtelExporter, createOtelTracingStore, createReplayFetch, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, extractOtlpAttributes, extractUsage, extractUsageFromResponse, extractUsageFromSse, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, inferDomainKeywords, inferOtlpKind, iterateRawCalls, otelRunCompleteHook, otlpToRunRecords, otlpToTraceRunRecords, planTraceInsightQuestions, projectOtlpFlatLine, readOtlpStatus, scoreTraceInsightReadiness, stringField, tokenizeDomainWords, traceAnalystFunctionGroup, traceAnalystOnRunComplete };
|
|
976
|
+
export { AnalyzeTracesOptions, AnalyzeTracesResult, type CaptureFetchContext, type CaptureFetchOptions, DatasetOverview, type ExportableSpan, type ExtractedUsage, type FlattenOtlpOptions, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, OPENINFERENCE_SPAN_KIND, OTEL_AGENT_EVAL_SCOPE, type OtelExportConfig, type OtelExporter, type OtlpExport, OtlpFileTraceStore, type OtlpFileTraceStoreOptions, type OtlpFlatLine, type OtlpResourceSpans, type OtlpSpan, type OtlpToRunRecordsOptions, type OtlpTraceRunRecord, type ProjectedOtlpSpan, ProviderRedactor, QueryTracesPage, RawProviderEvent, RawProviderSink, ReplayCache, type ReplayCacheEntry, ReplayCacheMissError, type ReplayCacheStats, type ReplayFetchOptions, Run, RunCompleteHook, RunCompleteHookContext, SPAN_KIND_ATTR_KEYS, SearchSpanResult, SearchTraceResult, SpanNotFoundError, TOOL_NAME, TOOL_NAME_ATTR_KEYS, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, type TraceAggregate, TraceAnalysisStore, TraceAnalystFilters, type TraceAnalystHookOptions, TraceAnalystSpanKind, TraceAnalystSpanStatus, TraceFileMissingError, type TraceInsightContext, type TraceInsightFinding, type TraceInsightPanelRole, type TraceInsightPromptInput, type TraceInsightQualityGate, type TraceInsightQuestion, type TraceInsightReadiness, type TraceInsightSuite, type TraceInsightTask, TraceNotFoundError, TraceStore, type TraceStoreSource, type TraceStoreToOtlpOptions, type TracesToOtlpResult, ViewSpansResult, ViewTraceResult, asNumber, asString, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, captureFetchToRawSink, convertTraceStoresToOtlp, createOtelExporter, createOtelTracingStore, createReplayFetch, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, extractOtlpAttributes, extractUsage, extractUsageFromResponse, extractUsageFromSse, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, inferDomainKeywords, inferOtlpKind, iterateRawCalls, otelRunCompleteHook, otlpToRunRecords, otlpToTraceRunRecords, planTraceInsightQuestions, projectOtlpFlatLine, readOtlpStatus, scoreTraceInsightReadiness, stringField, tokenizeDomainWords, traceAnalystFunctionGroup, traceAnalystOnRunComplete, traceSpanKindToOpenInferenceKind };
|
package/dist/traces.js
CHANGED
|
@@ -28,7 +28,13 @@ import {
|
|
|
28
28
|
scoreTraceInsightReadiness,
|
|
29
29
|
tokenizeDomainWords,
|
|
30
30
|
traceAnalystOnRunComplete
|
|
31
|
-
} from "./chunk-
|
|
31
|
+
} from "./chunk-77T4STFI.js";
|
|
32
|
+
import {
|
|
33
|
+
TRACE_ANALYST_ACTOR_DESCRIPTION,
|
|
34
|
+
TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
|
|
35
|
+
TRACE_ANALYST_SUBAGENT_DESCRIPTION,
|
|
36
|
+
analyzeTraces
|
|
37
|
+
} from "./chunk-IZCEK2HR.js";
|
|
32
38
|
import {
|
|
33
39
|
DEFAULT_REDACTION_RULES,
|
|
34
40
|
REDACTION_VERSION,
|
|
@@ -54,16 +60,24 @@ import {
|
|
|
54
60
|
isSandboxSpan,
|
|
55
61
|
isToolSpan
|
|
56
62
|
} from "./chunk-5BKGXME7.js";
|
|
57
|
-
import {
|
|
58
|
-
TRACE_ANALYST_ACTOR_DESCRIPTION,
|
|
59
|
-
TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
|
|
60
|
-
TRACE_ANALYST_SUBAGENT_DESCRIPTION,
|
|
61
|
-
analyzeTraces
|
|
62
|
-
} from "./chunk-TBDR6PAI.js";
|
|
63
63
|
import {
|
|
64
64
|
DEFAULT_TRACE_ANALYST_BUDGETS,
|
|
65
|
+
LLM_CACHED_TOKENS,
|
|
66
|
+
LLM_CACHED_TOKEN_ATTR_KEYS,
|
|
67
|
+
LLM_COST_ATTR_KEYS,
|
|
68
|
+
LLM_COST_USD,
|
|
69
|
+
LLM_INPUT_TOKENS,
|
|
70
|
+
LLM_INPUT_TOKEN_ATTR_KEYS,
|
|
71
|
+
LLM_MODEL_ATTR_KEYS,
|
|
72
|
+
LLM_MODEL_NAME,
|
|
73
|
+
LLM_OUTPUT_TOKENS,
|
|
74
|
+
LLM_OUTPUT_TOKEN_ATTR_KEYS,
|
|
75
|
+
OPENINFERENCE_SPAN_KIND,
|
|
65
76
|
OtlpFileTraceStore,
|
|
77
|
+
SPAN_KIND_ATTR_KEYS,
|
|
66
78
|
SpanNotFoundError,
|
|
79
|
+
TOOL_NAME,
|
|
80
|
+
TOOL_NAME_ATTR_KEYS,
|
|
67
81
|
TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
|
|
68
82
|
TraceFileMissingError,
|
|
69
83
|
TraceNotFoundError,
|
|
@@ -77,8 +91,9 @@ import {
|
|
|
77
91
|
projectOtlpFlatLine,
|
|
78
92
|
readOtlpStatus,
|
|
79
93
|
stringField,
|
|
80
|
-
traceAnalystFunctionGroup
|
|
81
|
-
|
|
94
|
+
traceAnalystFunctionGroup,
|
|
95
|
+
traceSpanKindToOpenInferenceKind
|
|
96
|
+
} from "./chunk-YBIGNSCZ.js";
|
|
82
97
|
import {
|
|
83
98
|
RunIntegrityError,
|
|
84
99
|
assertRunCaptured,
|
|
@@ -91,7 +106,7 @@ import {
|
|
|
91
106
|
defaultProviderRedactor,
|
|
92
107
|
providerFromBaseUrl
|
|
93
108
|
} from "./chunk-PC4UYEBM.js";
|
|
94
|
-
import "./chunk-
|
|
109
|
+
import "./chunk-LO6IOIJ2.js";
|
|
95
110
|
import {
|
|
96
111
|
TraceEmitter,
|
|
97
112
|
llmSpanFromProvider
|
|
@@ -107,14 +122,28 @@ export {
|
|
|
107
122
|
FileSystemTraceStore,
|
|
108
123
|
InMemoryRawProviderSink,
|
|
109
124
|
InMemoryTraceStore,
|
|
125
|
+
LLM_CACHED_TOKENS,
|
|
126
|
+
LLM_CACHED_TOKEN_ATTR_KEYS,
|
|
127
|
+
LLM_COST_ATTR_KEYS,
|
|
128
|
+
LLM_COST_USD,
|
|
129
|
+
LLM_INPUT_TOKENS,
|
|
130
|
+
LLM_INPUT_TOKEN_ATTR_KEYS,
|
|
131
|
+
LLM_MODEL_ATTR_KEYS,
|
|
132
|
+
LLM_MODEL_NAME,
|
|
133
|
+
LLM_OUTPUT_TOKENS,
|
|
134
|
+
LLM_OUTPUT_TOKEN_ATTR_KEYS,
|
|
110
135
|
NoopRawProviderSink,
|
|
136
|
+
OPENINFERENCE_SPAN_KIND,
|
|
111
137
|
OTEL_AGENT_EVAL_SCOPE,
|
|
112
138
|
OtlpFileTraceStore,
|
|
113
139
|
REDACTION_VERSION,
|
|
114
140
|
ReplayCache,
|
|
115
141
|
ReplayCacheMissError,
|
|
116
142
|
RunIntegrityError,
|
|
143
|
+
SPAN_KIND_ATTR_KEYS,
|
|
117
144
|
SpanNotFoundError,
|
|
145
|
+
TOOL_NAME,
|
|
146
|
+
TOOL_NAME_ATTR_KEYS,
|
|
118
147
|
TRACE_ANALYST_ACTOR_DESCRIPTION,
|
|
119
148
|
TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
|
|
120
149
|
TRACE_ANALYST_SUBAGENT_DESCRIPTION,
|
|
@@ -178,6 +207,7 @@ export {
|
|
|
178
207
|
tokenizeDomainWords,
|
|
179
208
|
toolSpans,
|
|
180
209
|
traceAnalystFunctionGroup,
|
|
181
|
-
traceAnalystOnRunComplete
|
|
210
|
+
traceAnalystOnRunComplete,
|
|
211
|
+
traceSpanKindToOpenInferenceKind
|
|
182
212
|
};
|
|
183
213
|
//# sourceMappingURL=traces.js.map
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { R as RunRecord } from './run-record-
|
|
1
|
+
import { R as RunRecord } from './run-record-CP2ObebC.js';
|
|
2
2
|
import { T as TraceAnalysisStore } from './store-C1YxJDEK.js';
|
|
3
3
|
import { a as JudgeInput } from './types-C7DGg5ex.js';
|
|
4
4
|
import { b as LlmCallRequest, c as LlmCallResult } from './llm-client-Bj7g0rqu.js';
|
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
import { D as DefaultVerdict } from './verdict-C9MlYujm.js';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* @experimental
|
|
5
|
-
*
|
|
6
4
|
* N-axis cartesian matrix over substrate types — types module.
|
|
7
5
|
*
|
|
8
6
|
* The matrix is a runner + aggregator. It iterates the cartesian product of
|
|
9
|
-
* caller-provided axes (any value type — `AgentProfile` from
|
|
10
|
-
* / `Validator` from agent-runtime, rubric records, thinking levels, anything)
|
|
7
|
+
* caller-provided axes (any value type — `AgentProfile` from agent-interface,
|
|
8
|
+
* `Driver` / `Validator` from agent-runtime, rubric records, thinking levels, anything)
|
|
11
9
|
* and aggregates per-axis pass/score/cost summaries. Substrate types are
|
|
12
10
|
* imported at the boundary by JSDoc only; the matrix never wraps them.
|
|
13
11
|
*/
|