@tangle-network/agent-eval 0.95.0 → 0.95.1

package/dist/{trajectory.d.ts → trajectory-2TkpSEVh.d.ts}

@@ -1,3 +1,6 @@
+import { S as Span, a as TraceEvent } from './schema-m0gsnbt3.js';
+import { T as TraceStore } from './store-BcFXE6LG.js';
+
 /**
  * Trajectory — ordered, structured view over a run's spans.
  *
@@ -7,9 +10,8 @@
  * analyzers (stuck-loop detection, waste ratio) live in
  * `pipelines/` and consume the trajectory.
  */
-import type { Span, TraceEvent } from './trace/schema';
-import type { TraceStore } from './trace/store';
-export interface TrajectoryStep {
+
+interface TrajectoryStep {
     index: number;
     span: Span;
     /** Depth in the span tree from the root. 0 = top-level. */
@@ -17,7 +19,7 @@ export interface TrajectoryStep {
     /** Events attached to this span. */
     events: TraceEvent[];
 }
-export interface Trajectory {
+interface Trajectory {
     runId: string;
     steps: TrajectoryStep[];
     llmTurns: number;
@@ -26,7 +28,6 @@ export interface Trajectory {
     retrievals: number;
     totalDurationMs: number;
 }
-export declare function buildTrajectory(store: TraceStore, runId: string): Promise<Trajectory>;
-export type { Span, TraceEvent } from './trace/schema';
-export type { TraceStore } from './trace/store';
-//# sourceMappingURL=trajectory.d.ts.map
+declare function buildTrajectory(store: TraceStore, runId: string): Promise<Trajectory>;
+
+export { type Trajectory as T, type TrajectoryStep as a, buildTrajectory as b };

package/dist/{analyst/types.d.ts → types-B5x54y6n.d.ts}

@@ -1,3 +1,98 @@
+import { R as RunRecord } from './run-record-CP2ObebC.js';
+import { T as TraceAnalysisStore } from './store-C1YxJDEK.js';
+import { a as JudgeInput } from './types-C7DGg5ex.js';
+import { b as LlmCallRequest, c as LlmCallResult } from './llm-client-Bj7g0rqu.js';
+
+/**
+ * ChatClient — the single LLM abstraction analysts call.
+ *
+ * agent-eval already ships an `LlmClient` (OpenAI-compatible, retry,
+ * graceful JSON-schema degrade) and judges that talk to `TCloud`. Two
+ * mixed patterns force every analyst author to pick a transport, which
+ * couples analyst code to runtime concerns (cli-bridge vs router vs
+ * sandbox-sdk) it shouldn't know about.
+ *
+ * `ChatClient` is one interface every analyst takes via `AnalystContext.chat`.
+ * The operator decides at the registry boundary which transport binds
+ * to it. Analyst code stays transport-agnostic; swapping production
+ * (sandbox-sdk) for local dev (cli-bridge) or tests (mock) is a one-
+ * line factory call.
+ *
+ * Designed to coexist: existing `LlmClient` callers and existing
+ * `TCloud`-based judges keep working untouched. New analyst code uses
+ * `ChatClient`. When old call sites migrate, they pick up budgeting,
+ * cancellation, and unified telemetry for free.
+ */
+
+/**
+ * Unified chat interface. Mirrors LlmCallRequest/Result so the OpenAI-
+ * compatible mental model stays. Two methods: a one-shot `chat()` and
+ * an `streamChat()` for future agentic loops (not yet exposed).
+ */
+interface ChatClient {
+    /** Display name of the bound transport — included in telemetry. */
+    readonly transport: ChatTransport;
+    /** Default model when caller omits — operators bind this per environment. */
+    readonly defaultModel?: string;
+    chat(req: ChatRequest, opts?: ChatCallOpts): Promise<ChatResponse>;
+}
+type ChatTransport = 'router' | 'sandbox-sdk' | 'cli-bridge' | 'direct-provider' | 'mock';
+interface ChatRequest extends Omit<LlmCallRequest, 'model'> {
+    /** Optional — falls back to ChatClient.defaultModel. */
+    model?: string;
+}
+type ChatResponse = LlmCallResult;
+interface ChatCallOpts {
+    /** Cancel the in-flight request. */
+    signal?: AbortSignal;
+    /** Hard USD ceiling for this single call (informational; the underlying transport may not enforce). */
+    maxCostUsd?: number;
+    /** Correlation tag carried into request headers when the transport allows. */
+    correlationId?: string;
+}
+type CreateChatClientOpts = RouterTransportOpts | CliBridgeTransportOpts | DirectProviderTransportOpts | SandboxSdkTransportOpts | MockTransportOpts;
+interface BaseTransportOpts {
+    defaultModel?: string;
+}
+interface RouterTransportOpts extends BaseTransportOpts {
+    transport: 'router';
+    baseUrl?: string;
+    apiKey: string;
+}
+interface CliBridgeTransportOpts extends BaseTransportOpts {
+    transport: 'cli-bridge';
+    baseUrl?: string;
+    bearer?: string;
+}
+interface DirectProviderTransportOpts extends BaseTransportOpts {
+    transport: 'direct-provider';
+    baseUrl: string;
+    apiKey: string;
+}
+/**
+ * Sandbox-SDK transport. Provided as a thin pass-through: the caller
+ * supplies a callable that mimics LlmClient.chat() against an already-
+ * configured Sandbox handle. We don't import the SDK here to keep
+ * agent-eval dep-free of @tangle-network/sandbox.
+ */
+interface SandboxSdkTransportOpts extends BaseTransportOpts {
+    transport: 'sandbox-sdk';
+    chat: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>;
+}
+/**
+ * Mock transport for tests. The handler receives the request and returns
+ * whatever the test wants. No retries, no JSON-schema degrade.
+ */
+interface MockTransportOpts extends BaseTransportOpts {
+    transport: 'mock';
+    handler: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>;
+}
+/**
+ * Build a ChatClient bound to a specific transport. The returned client
+ * is safe to share across analysts in a single registry run.
+ */
+declare function createChatClient(opts: CreateChatClientOpts): ChatClient;
+
 /**
  * Analyst contract — the missing orchestration layer over agent-eval's
  * existing analyzers (analyzeTraces, MultiLayerVerifier, RunCritic,
@@ -14,15 +109,12 @@
  * a RunRecord, a JudgeInput, or `custom`), and the registry routes
  * the matching input from `AnalystRunInputs`.
  */
-import type { RunRecord } from '../run-record';
-import type { TraceAnalysisStore } from '../trace-analyst/store';
-import type { JudgeInput } from '../types';
-import type { ChatClient } from './chat-client';
+
 /**
  * Unified envelope every analyst emits. Schema-versioned so renderers
  * and time-series diffs survive future field additions.
  */
-export interface AnalystFinding {
+interface AnalystFinding {
     schema_version: '1.0.0';
     /**
      * Stable hash over identity-defining fields (analyst_id + canonical
@@ -66,8 +158,8 @@ export interface AnalystFinding {
     /** Analyst-private extras; renderers ignore unless they know the analyst. */
     metadata?: Record<string, unknown>;
 }
-export type AnalystSeverity = 'critical' | 'high' | 'medium' | 'low' | 'info';
-export interface EvidenceRef {
+type AnalystSeverity = 'critical' | 'high' | 'medium' | 'low' | 'info';
+interface EvidenceRef {
     /**
      * Where the evidence lives. `span` and `event` refer to OTLP trace
      * elements; `artifact` to a file inside the run's artifact tree;
@@ -84,8 +176,8 @@ export interface EvidenceRef {
  * (e.g. an embedding cache, a partner SDK handle) read it from
  * `AnalystRunInputs.custom[<analyst id>]`.
  */
-export type AnalystInputKind = 'trace-store' | 'artifact-dir' | 'run-record' | 'judge-input' | 'custom';
-export interface AnalystCost {
+type AnalystInputKind = 'trace-store' | 'artifact-dir' | 'run-record' | 'judge-input' | 'custom';
+interface AnalystCost {
     /** `deterministic` analysts MUST NOT call the LLM. */
     kind: 'deterministic' | 'llm';
     /** Optional declared upper bound; the registry can enforce a budget. */
@@ -93,7 +185,7 @@ export interface AnalystCost {
     /** Models the analyst expects to use (informational). */
     models?: string[];
 }
-export interface AnalystRequirements {
+interface AnalystRequirements {
     /** Min number of shots / samples the analyst needs to produce signal. */
     min_shots?: number;
     /** Capabilities the runtime must supply (e.g. ['network', 'gpu']). */
@@ -103,7 +195,7 @@ export interface AnalystRequirements {
  * What's passed to every analyst call. The registry resolves which
  * field the analyst's `inputKind` selects and asserts it's present.
  */
-export interface AnalystRunInputs {
+interface AnalystRunInputs {
     traceStore?: TraceAnalysisStore;
     artifactDir?: string;
     runRecord?: RunRecord;
@@ -111,7 +203,7 @@ export interface AnalystRunInputs {
     /** Keyed by analyst id; populated by callers that registered custom analysts. */
     custom?: Record<string, unknown>;
 }
-export interface AnalystContext {
+interface AnalystContext {
     runId: string;
     /** Stable correlation id so logs from a single registry.run() share a tag. */
     correlationId: string;
@@ -149,7 +241,7 @@ export interface AnalystContext {
  * `TraceAnalysisStore`); the registry passes the right field from
  * `AnalystRunInputs` based on `inputKind`.
  */
-export interface Analyst<TInput = unknown> {
+interface Analyst<TInput = unknown> {
     /** Stable identifier — appears in finding_id, telemetry, and registry exclusion lists. */
     readonly id: string;
     /** Human-readable. One sentence. */
@@ -169,7 +261,7 @@ export interface Analyst<TInput = unknown> {
  * or (b) move the variable part into `rationale`/`metadata` and keep the
  * `claim` static.
  */
-export declare function computeFindingId(input: {
+declare function computeFindingId(input: {
     analyst_id: string;
     area: string;
     subject?: string;
@@ -181,11 +273,11 @@ export declare function computeFindingId(input: {
  * Convenience factory: produce a fully-formed AnalystFinding with the
  * id computed automatically. Analyst code stays terse.
  */
-export declare function makeFinding(init: Omit<AnalystFinding, 'schema_version' | 'finding_id' | 'produced_at'> & {
+declare function makeFinding(init: Omit<AnalystFinding, 'schema_version' | 'finding_id' | 'produced_at'> & {
     id_basis?: string;
     produced_at?: string;
 }): AnalystFinding;
-export interface AnalystRunSummary {
+interface AnalystRunSummary {
     analyst_id: string;
     status: 'ok' | 'skipped' | 'failed';
     /** Why skipped — missing input, budget exceeded, capability unmet. */
@@ -199,7 +291,7 @@ export interface AnalystRunSummary {
         message: string;
     };
 }
-export interface AnalystRunResult {
+interface AnalystRunResult {
     run_id: string;
     correlation_id: string;
     started_at: string;
@@ -222,7 +314,7 @@ export interface AnalystRunResult {
  * is the right granularity; subscribers wanting per-finding rendering
  * iterate `event.findings` themselves.
  */
-export type AnalystRunEvent = {
+type AnalystRunEvent = {
     type: 'run-started';
     run_id: string;
     correlation_id: string;
@@ -245,4 +337,5 @@ export type AnalystRunEvent = {
     type: 'run-completed';
     result: AnalystRunResult;
 };
-//# sourceMappingURL=types.d.ts.map
+
+export { type Analyst as A, type ChatClient as C, type DirectProviderTransportOpts as D, type EvidenceRef as E, type MockTransportOpts as M, type RouterTransportOpts as R, type SandboxSdkTransportOpts as S, type AnalystContext as a, type AnalystRunSummary as b, type AnalystFinding as c, type AnalystRunResult as d, type AnalystRunInputs as e, type AnalystRunEvent as f, type AnalystCost as g, type AnalystSeverity as h, type AnalystInputKind as i, type AnalystRequirements as j, type ChatCallOpts as k, type ChatRequest as l, type ChatResponse as m, type ChatTransport as n, type CliBridgeTransportOpts as o, type CreateChatClientOpts as p, computeFindingId as q, createChatClient as r, makeFinding as s };

package/dist/{matrix/types.d.ts → types-BUxNaJ8c.d.ts}

@@ -1,3 +1,5 @@
+import { D as DefaultVerdict } from './verdict-C9MlYujm.js';
+
 /**
  * N-axis cartesian matrix over substrate types — types module.
  *
@@ -7,11 +9,10 @@
  * and aggregates per-axis pass/score/cost summaries. Substrate types are
  * imported at the boundary by JSDoc only; the matrix never wraps them.
  */
-import type { DefaultVerdict } from '../verdict';
-export type { DefaultVerdict } from '../verdict';
+
 /** One axis = one dimension to iterate. `V` is the value type — pass any
  *  substrate type (AgentProfile, Driver, Validator, rubric record). */
-export interface MatrixAxis<V> {
+interface MatrixAxis<V> {
     /** Axis name. Becomes the key in `MatrixResult.byAxis`. */
     name: string;
     /** Stable id per value. Used as the bucket key in aggregation. */
@@ -24,7 +25,7 @@ export interface MatrixAxis<V> {
     label?: (value: V, id: string) => string;
 }
 /** A cell carries one picked value from each axis, keyed by axis name. */
-export interface MatrixCell {
+interface MatrixCell {
     axes: Record<string, {
         id: string;
         value: unknown;
@@ -34,7 +35,7 @@ export interface MatrixCell {
     /** Stable sort key — preserves cartesian order across concurrent execution. */
     ordinal: number;
 }
-export interface CellResult<Output> {
+interface CellResult<Output> {
     output: Output;
     verdict: DefaultVerdict;
     costUsd: number;
@@ -47,7 +48,7 @@ export interface CellResult<Output> {
         kind: string;
     };
 }
-export interface AxisSummary {
+interface AxisSummary {
     axisName: string;
     axisValue: string;
     cells: number;
@@ -58,7 +59,7 @@ export interface AxisSummary {
     totalCostUsd: number;
     meanDurationMs: number;
 }
-export interface MatrixResult<Output> {
+interface MatrixResult<Output> {
     cells: Array<{
         cell: MatrixCell;
         runs: CellResult<Output>[];
@@ -80,7 +81,7 @@ export interface MatrixResult<Output> {
     /** Stable id-like string generated at the end of the run. */
     matrixId: string;
 }
-export interface RunAgentMatrixOptions<Output> {
+interface RunAgentMatrixOptions<Output> {
     axes: MatrixAxis<unknown>[];
     /** User-supplied cell executor. May throw; the matrix captures throws as
      *  `CellResult.error` and continues. */
@@ -103,4 +104,5 @@ export interface RunAgentMatrixOptions<Output> {
      *  and suppresses scheduling of new ones. */
     signal?: AbortSignal;
 }
-//# sourceMappingURL=types.d.ts.map
+
+export type { AxisSummary as A, CellResult as C, MatrixResult as M, RunAgentMatrixOptions as R, MatrixAxis as a, MatrixCell as b };

package/dist/{types.d.ts → types-C7DGg5ex.d.ts}

@@ -1,4 +1,6 @@
-export interface Scenario {
+import { TCloud } from '@tangle-network/tcloud';
+
+interface Scenario {
     id: string;
     persona: string;
     label: string;
@@ -8,37 +10,37 @@ export interface Scenario {
     artifactChecks: ArtifactCheck[];
     systemPromptAppend?: string;
 }
-export interface Turn {
+interface Turn {
     user: string;
     expectedBehaviors: string[];
     adversarial?: boolean;
     feedbackType?: 'correction' | 'rejection' | 'vague' | 'contradictory' | 'escalation';
 }
-export interface ArtifactCheck {
+interface ArtifactCheck {
     type: 'vault_file_exists' | 'vault_file_contains' | 'block_extracted' | 'code_valid' | 'generation_produced' | 'tool_created' | string;
     target: string;
     contains?: string;
     minCount?: number;
     description: string;
 }
-export interface JudgeConfig {
+interface JudgeConfig {
     model: string;
     temperature: number;
     rubric: JudgeRubric;
 }
-export interface JudgeRubric {
+interface JudgeRubric {
     name: string;
     description: string;
     dimensions: RubricDimension[];
 }
-export interface RubricDimension {
+interface RubricDimension {
     name: string;
     description: string;
     anchor_low: string;
     anchor_high: string;
     weight: number;
 }
-export interface ScenarioResult {
+interface ScenarioResult {
     scenarioId: string;
     persona: string;
     turns: TurnResult[];
@@ -49,7 +51,7 @@ export interface ScenarioResult {
     totalDurationMs: number;
     artifacts: CollectedArtifacts;
 }
-export interface TurnResult {
+interface TurnResult {
     turnIndex: number;
     userMessage: string;
     agentResponse: string;
@@ -61,19 +63,19 @@ export interface TurnResult {
     containsCode: boolean;
     containsToolCall: boolean;
 }
-export interface ArtifactResult {
+interface ArtifactResult {
     check: ArtifactCheck;
     passed: boolean;
     detail?: string;
 }
-export interface JudgeScore {
+interface JudgeScore {
     judgeName: string;
     dimension: string;
     score: number;
     reasoning: string;
     evidence?: string;
 }
-export interface CollectedArtifacts {
+interface CollectedArtifacts {
     vaultFiles: {
         path: string;
         content: string;
@@ -88,7 +90,7 @@ export interface CollectedArtifacts {
     }[];
     toolCalls: string[];
 }
-export interface BenchmarkReport {
+interface BenchmarkReport {
     timestamp: string;
     generation: number;
     promptVersion: string;
@@ -117,7 +119,7 @@ export interface BenchmarkReport {
         }[];
     };
 }
-export interface RouteMap {
+interface RouteMap {
     signup?: string;
     login?: string;
     workspaces?: string;
@@ -130,13 +132,13 @@ export interface RouteMap {
     generations?: string;
     [key: string]: string | undefined;
 }
-export interface ProductClientConfig {
+interface ProductClientConfig {
     baseUrl: string;
     routes: RouteMap;
     /** Per-request timeout in ms before the request is aborted. Default 30s. */
     timeoutMs?: number;
 }
-export interface ScenarioFile {
+interface ScenarioFile {
     id: string;
     category: string;
     persona: string;
@@ -153,12 +155,12 @@ export interface ScenarioFile {
     turns: Turn[];
     artifactChecks: ArtifactCheck[];
 }
-export interface CompletionCriterion {
+interface CompletionCriterion {
     name: string;
     check: (state: DriverState) => boolean;
     progress?: (state: DriverState) => number;
 }
-export interface FeedbackPattern {
+interface FeedbackPattern {
     trigger: string;
     response: string;
 }
@@ -170,8 +172,8 @@ export interface FeedbackPattern {
  *   relentless  — senior partner reviewing for a client who will litigate;
  *                 interrogates every claim, accepts nothing undefended.
  */
-export type PersonaRigor = 'cooperative' | 'demanding' | 'relentless';
-export interface PersonaConfig {
+type PersonaRigor = 'cooperative' | 'demanding' | 'relentless';
+interface PersonaConfig {
     id: string;
     role: string;
     goal: string;
@@ -202,7 +204,7 @@ export interface PersonaConfig {
      */
     curveballs?: string[];
 }
-export interface DriverState {
+interface DriverState {
     tasks: number;
     events: number;
     proposals: {
@@ -214,7 +216,7 @@ export interface DriverState {
     codeBlocks: number;
     generations: number;
 }
-export interface TurnMetrics {
+interface TurnMetrics {
     turn: number;
     timestamp: string;
     tasks: number;
@@ -236,7 +238,7 @@ export interface TurnMetrics {
     totalCostUsd: number;
     completionPercent: number;
 }
-export interface DriverResult {
+interface DriverResult {
     personaId: string;
     /** True when the simulated user professionally signed off (driver said DONE). */
     completed: boolean;
@@ -255,7 +257,7 @@ export interface DriverResult {
     totalCostUsd: number;
     finalQualityScore: number | null;
 }
-export interface BenchmarkRunnerConfig {
+interface BenchmarkRunnerConfig {
     scenarios: Scenario[];
     judges: JudgeFn[];
     systemPrompt: string;
@@ -265,32 +267,32 @@ export interface BenchmarkRunnerConfig {
     generation?: number;
     promptVersion?: string;
 }
-export interface JudgeInput {
+interface JudgeInput {
     scenario: Scenario;
     turns: TurnResult[];
     artifacts: CollectedArtifacts;
 }
-export type JudgeFn = (tc: TCloud, input: JudgeInput) => Promise<JudgeScore[]>;
-import type { TCloud } from '@tangle-network/tcloud';
-export type { TCloud };
-export interface TestResult {
+type JudgeFn = (tc: TCloud, input: JudgeInput) => Promise<JudgeScore[]>;
+
+interface TestResult {
     name: string;
     passed: boolean;
     duration: number;
     detail?: string;
     checks: CheckResult[];
 }
-export interface CheckResult {
+interface CheckResult {
     name: string;
     passed: boolean;
     expected: string;
     actual: string;
 }
-export interface EvalResult {
+interface EvalResult {
     scenario: string;
     status: 'pass' | 'fail' | 'skip';
     duration: number;
     detail?: string;
     artifact?: string;
 }
-//# sourceMappingURL=types.d.ts.map
+
+export type { ArtifactCheck as A, BenchmarkRunnerConfig as B, CheckResult as C, DriverResult as D, EvalResult as E, FeedbackPattern as F, JudgeScore as J, ProductClientConfig as P, RouteMap as R, Scenario as S, TestResult as T, JudgeInput as a, JudgeFn as b, BenchmarkReport as c, PersonaConfig as d, DriverState as e, CollectedArtifacts as f, ScenarioResult as g, TurnMetrics as h, ScenarioFile as i, CompletionCriterion as j, ArtifactResult as k, JudgeConfig as l, JudgeRubric as m, PersonaRigor as n, RubricDimension as o, Turn as p, TurnResult as q };