@tangle-network/agent-eval 0.129.0 → 0.130.0

package/dist/types-DGsxbAEd.d.ts

@@ -0,0 +1,387 @@
+import { c as CostLedgerHandle, f as CostLedgerSummary } from "./cost-ledger-Dye6jCgg.js";
+import { i as LlmCallResult, r as LlmCallRequest } from "./llm-client-B_nIBlYo.js";
+//#region src/analyst/chat-client.d.ts
+/**
+ * Unified chat interface using the package's canonical LLM request and result.
+ */
+interface ChatClient {
+  /** Display name of the bound transport, included in telemetry. */
+  readonly transport: ChatTransport;
+  /** Default model when the caller omits one. */
+  readonly defaultModel?: string;
+  /** Total provider attempts this transport can make for one chat call. */
+  readonly maximumAttempts?: number;
+  /** Implementations must enforce `req.maxTokens` when it is present. */
+  chat(req: ChatRequest, opts?: ChatCallOpts): Promise<ChatResponse>;
+}
+type ChatTransport = 'router' | 'sandbox-sdk' | 'cli-bridge' | 'direct-provider' | 'custom' | 'mock';
+interface ChatRequest extends Omit<LlmCallRequest, 'model'> {
+  /** Optional — falls back to ChatClient.defaultModel. */
+  model?: string;
+}
+type ChatResponse = LlmCallResult;
+interface ChatCallOpts {
+  /** Cancel the in-flight request. */
+  signal?: AbortSignal;
+  /** Hard USD ceiling for this single call (informational; the underlying transport may not enforce). */
+  maxCostUsd?: number;
+  /** Correlation tag carried into request headers when the transport allows. */
+  correlationId?: string;
+  /** Stable provider idempotency key for retries/redrives of one paid call. */
+  idempotencyKey?: string;
+}
+type CreateChatClientOpts = RouterTransportOpts | CliBridgeTransportOpts | DirectProviderTransportOpts | SandboxSdkTransportOpts | CustomTransportOpts | MockTransportOpts;
+interface BaseTransportOpts {
+  defaultModel?: string;
+  /** Total provider attempts. Required for opaque transports used in capped runs. */
+  maximumAttempts?: number;
+}
+interface RouterTransportOpts extends BaseTransportOpts {
+  transport: 'router';
+  baseUrl?: string;
+  apiKey: string;
+}
+interface CliBridgeTransportOpts extends BaseTransportOpts {
+  transport: 'cli-bridge';
+  baseUrl?: string;
+  bearer?: string;
+}
+interface DirectProviderTransportOpts extends BaseTransportOpts {
+  transport: 'direct-provider';
+  baseUrl: string;
+  apiKey: string;
+}
+/**
+ * Sandbox-SDK transport. The caller supplies a canonical chat function for an
+ * already-configured Sandbox handle, so agent-eval does not import the SDK.
+ */
+interface SandboxSdkTransportOpts extends BaseTransportOpts {
+  transport: 'sandbox-sdk';
+  chat: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>;
+}
+/** Caller-adapted SDK or transport returning the canonical ChatResponse shape. */
+interface CustomTransportOpts extends BaseTransportOpts {
+  transport: 'custom';
+  chat: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>;
+}
+/**
+ * Mock transport for tests. The handler receives the request and returns
+ * whatever the test wants. No retries, no JSON-schema degrade.
+ */
+interface MockTransportOpts extends BaseTransportOpts {
+  transport: 'mock';
+  handler: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>;
+}
+/**
+ * Build a ChatClient bound to a specific transport. The returned client
+ * is safe to share across analysts in a single registry run.
+ */
+declare function createChatClient(opts: CreateChatClientOpts): ChatClient;
+//#endregion
+//#region src/types.d.ts
+interface Scenario {
+  id: string;
+  persona: string;
+  label: string;
+  thesis: string;
+  dimensions: string[];
+  turns: Turn[];
+  artifactChecks: ArtifactCheck[];
+  systemPromptAppend?: string;
+}
+interface Turn {
+  user: string;
+  expectedBehaviors: string[];
+  adversarial?: boolean;
+  feedbackType?: 'correction' | 'rejection' | 'vague' | 'contradictory' | 'escalation';
+}
+interface ArtifactCheck {
+  type: 'vault_file_exists' | 'vault_file_contains' | 'block_extracted' | 'code_valid' | 'generation_produced' | 'tool_created' | string;
+  target: string;
+  contains?: string;
+  minCount?: number;
+  description: string;
+}
+interface JudgeConfig {
+  model: string;
+  temperature: number;
+  rubric: JudgeRubric;
+}
+interface JudgeRubric {
+  name: string;
+  description: string;
+  dimensions: RubricDimension[];
+}
+interface RubricDimension {
+  name: string;
+  description: string;
+  anchor_low: string;
+  anchor_high: string;
+  weight: number;
+}
+interface ScenarioResult {
+  scenarioId: string;
+  persona: string;
+  turns: TurnResult[];
+  artifactResults: ArtifactResult[];
+  judgeScores: JudgeScore[];
+  judgeErrors: number;
+  overallScore: number;
+  totalDurationMs: number;
+  artifacts: CollectedArtifacts;
+  /** Agent and judge spend attributed to this scenario. */
+  cost?: CostLedgerSummary;
+}
+interface TurnResult {
+  turnIndex: number;
+  userMessage: string;
+  agentResponse: string;
+  durationMs: number;
+  blocksExtracted: {
+    type: string;
+    title: string;
+  }[];
+  containsCode: boolean;
+  containsToolCall: boolean;
+}
+interface ArtifactResult {
+  check: ArtifactCheck;
+  passed: boolean;
+  detail?: string;
+}
+interface JudgeScore {
+  judgeName: string;
+  dimension: string;
+  score: number;
+  reasoning: string;
+  evidence?: string;
+}
+interface CollectedArtifacts {
+  vaultFiles: {
+    path: string;
+    content: string;
+  }[];
+  blocksExtracted: {
+    type: string;
+    fields: Record<string, string>;
+  }[];
+  codeBlocks: {
+    language: string;
+    code: string;
+  }[];
+  toolCalls: string[];
+}
+interface BenchmarkReport {
+  timestamp: string;
+  generation: number;
+  promptVersion: string;
+  scenarioCount: number;
+  results: ScenarioResult[];
+  cost?: CostLedgerSummary;
+  summary: {
+    overallAvg: number;
+    byPersona: Record<string, {
+      avg: number;
+      passed: number;
+      total: number;
+    }>;
+    byDimension: Record<string, {
+      avg: number;
+      scores: number[];
+    }>;
+    weakest: {
+      scenario: string;
+      score: number;
+      reason: string;
+    }[];
+    strongest: {
+      scenario: string;
+      score: number;
+      reason: string;
+    }[];
+  };
+}
+interface RouteMap {
+  signup?: string;
+  login?: string;
+  workspaces?: string;
+  threads?: string;
+  chat?: string;
+  tasks?: string;
+  events?: string;
+  approvals?: string;
+  vault?: string;
+  generations?: string;
+  [key: string]: string | undefined;
+}
+interface ProductClientConfig {
+  baseUrl: string;
+  routes: RouteMap;
+  /** Per-request timeout in ms before the request is aborted. Default 30s. */
+  timeoutMs?: number;
+}
+interface ScenarioFile {
+  id: string;
+  category: string;
+  persona: string;
+  label: string;
+  thesis: string;
+  isControl?: boolean;
+  rubric?: {
+    dimensions: {
+      name: string;
+      description: string;
+      weight: number;
+    }[];
+  };
+  turns: Turn[];
+  artifactChecks: ArtifactCheck[];
+}
+interface CompletionCriterion {
+  name: string;
+  check: (state: DriverState) => boolean;
+  progress?: (state: DriverState) => number;
+}
+interface FeedbackPattern {
+  trigger: string;
+  response: string;
+}
+/**
+ * How hard the simulated user pushes back. The driver LLM scales its tone
+ * and follow-up aggression to this:
+ *   cooperative — forgiving early adopter; accepts reasonable answers.
+ *   demanding   — experienced professional; rejects vague or hedged answers.
+ *   relentless  — senior partner reviewing for a client who will litigate;
+ *                 interrogates every claim, accepts nothing undefended.
+ */
+type PersonaRigor = 'cooperative' | 'demanding' | 'relentless';
+interface PersonaConfig {
+  id: string;
+  role: string;
+  goal: string;
+  completionCriteria: CompletionCriterion[];
+  feedbackPatterns?: FeedbackPattern[];
+  maxTurns: number;
+  driverModel?: string;
+  /** How adversarial the simulated user is. Defaults to 'demanding'. */
+  rigor?: PersonaRigor;
+  /**
+   * Domain expertise the simulated user holds — quoted into the driver
+   * prompt so it challenges the agent with authority instead of vague
+   * dissatisfaction. e.g. "a 15-year M&A partner who knows GAAP
+   * working-capital mechanics cold".
+   */
+  expertise?: string;
+  /**
+   * Substantive issues a senior professional in this role would
+   * interrogate — traps the scenario hides, claims that must be defended.
+   * The driver probes these without revealing them verbatim; the agent
+   * must surface them on its own.
+   */
+  pressurePoints?: string[];
+  /**
+   * Curveballs the driver may inject once the agent is coasting — changed
+   * facts, a hostile counterparty position, a new constraint. Forces the
+   * agent to re-derive rather than recite.
+   */
+  curveballs?: string[];
+}
+interface DriverState {
+  tasks: number;
+  events: number;
+  proposals: {
+    pending: number;
+    approved: number;
+    rejected: number;
+  };
+  vaultFiles: string[];
+  codeBlocks: number;
+  generations: number;
+}
+interface TurnMetrics {
+  turn: number;
+  timestamp: string;
+  tasks: number;
+  events: number;
+  proposals: {
+    pending: number;
+    approved: number;
+    rejected: number;
+  };
+  vaultFiles: number;
+  responseLatencyMs: number;
+  responseChars: number;
+  codeBlocksProduced: number;
+  blocksExtracted: number;
+  qualityScore?: number;
+  inputTokens: number;
+  outputTokens: number;
+  estimatedCostUsd: number;
+  totalCostUsd: number;
+  completionPercent: number;
+}
+interface DriverResult {
+  personaId: string;
+  /** True when the simulated user professionally signed off (driver said DONE). */
+  completed: boolean;
+  /** Turn at which the simulated user signed off, or null if it never did. */
+  turnsToCompletion: number | null;
+  /**
+   * Turn at which nominal completionCriteria were first all met, or null.
+   * Distinct from turnsToCompletion: criteria can be met while the
+   * simulated professional is still unsatisfied with the work's rigor.
+   */
+  criteriaMetAtTurn: number | null;
+  totalTurns: number;
+  metrics: TurnMetrics[];
+  finalState: DriverState;
+  convergenceCurve: number[];
+  totalCostUsd: number;
+  finalQualityScore: number | null;
+}
+interface BenchmarkRunnerConfig {
+  scenarios: Scenario[];
+  judges: JudgeFn[];
+  systemPrompt: string;
+  model?: string;
+  judgeModel?: string;
+  passThreshold?: number;
+  generation?: number;
+  promptVersion?: string;
+  /** Shared ledger for agent and judge calls made by the benchmark. */
+  costLedger?: CostLedgerHandle;
+}
+interface JudgeInput {
+  scenario: Scenario;
+  turns: TurnResult[];
+  artifacts: CollectedArtifacts;
+  /** Shared ledger for paid built-in judges. Direct calls default to an uncapped ledger. */
+  costLedger?: CostLedgerHandle;
+  costPhase?: string;
+  costTags?: Record<string, string>;
+  signal?: AbortSignal;
+}
+type JudgeFn = (chat: ChatClient, input: JudgeInput) => Promise<JudgeScore[]>;
+interface TestResult {
+  name: string;
+  passed: boolean;
+  duration: number;
+  detail?: string;
+  checks: CheckResult[];
+}
+interface CheckResult {
+  name: string;
+  passed: boolean;
+  expected: string;
+  actual: string;
+}
+interface EvalResult {
+  scenario: string;
+  status: 'pass' | 'fail' | 'skip';
+  duration: number;
+  detail?: string;
+  artifact?: string;
+}
+//#endregion
+export { ChatClient as A, SandboxSdkTransportOpts as B, ScenarioFile as C, TurnMetrics as D, Turn as E, CreateChatClientOpts as F, CustomTransportOpts as I, DirectProviderTransportOpts as L, ChatResponse as M, ChatTransport as N, TurnResult as O, CliBridgeTransportOpts as P, MockTransportOpts as R, Scenario as S, TestResult as T, createChatClient as V, PersonaConfig as _, CheckResult as a, RouteMap as b, DriverResult as c, FeedbackPattern as d, JudgeConfig as f, JudgeScore as g, JudgeRubric as h, BenchmarkRunnerConfig as i, ChatRequest as j, ChatCallOpts as k, DriverState as l, JudgeInput as m, ArtifactResult as n, CollectedArtifacts as o, JudgeFn as p, BenchmarkReport as r, CompletionCriterion as s, ArtifactCheck as t, EvalResult as u, PersonaRigor as v, ScenarioResult as w, RubricDimension as x, ProductClientConfig as y, RouterTransportOpts as z };
+//# sourceMappingURL=types-DGsxbAEd.d.ts.map

package/dist/types-DGsxbAEd.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types-DGsxbAEd.d.ts","names":[],"sources":["../src/analyst/chat-client.ts","../src/types.ts"],"mappings":";;;;;;UAkBiB;;WAEN,WAAW;;WAEX;;WAEA;;EAGT,KAAK,KAAK,aAAa,OAAO,eAAe,QAAQ;;KAG3C;UAQK,oBAAoB,KAAK;;EAExC;;KAGU,eAAe;UAEV;;EAEf,SAAS;;EAET;;EAEA;;EAEA;;KAKU,uBACR,sBACA,yBACA,8BACA,0BACA,sBACA;UAEM;EACR;;EAEA;;UAGe,4BAA4B;EAC3C;EACA;EACA;;UAGe,+BAA+B;EAC9C;EACA;EACA;;UAGe,oCAAoC;EACnD;EACA;EACA;;;;;;UAOe,gCAAgC;EAC/C;EACA,OAAO,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;UAI1C,4BAA4B;EAC3C;EACA,OAAO,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;;;;UAO1C,0BAA0B;EACzC;EACA,UAAU,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;;;;iBAO9C,iBAAiB,MAAM,uBAAuB;;;UCjH7C;EACf;EACA;EACA;EACA;EACA;EACA,OAAO;EACP,gBAAgB;EAChB;;UAGe;EACf;EACA;EACA;EACA;;UAKe;EACf;EAQA;EACA;EACA;EACA;;UAKe;EACf;EACA;EACA,QAAQ;;UAGO;EACf;EACA;EACA,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;;UAKe;EACf;EACA;EACA,OAAO;EACP,iBAAiB;EACjB,aAAa;EACb;EACA;EACA;EACA,WAAW;;EAEX,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA;IAAmB;IAAc;;EACjC;EACA;;UAGe;EACf,OAAO;EACP;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;IAAc;IAAc;;EAC5B;IAAmB;IAAc,QAAQ;;EACzC;IAAc;IAAkB;;EAChC;;UAKe;EACf;EACA;EACA;EACA;EACA,SAAS;EACT,OAAO;EACP;IACE;IACA,WAAW;MAAiB;MAAa;MAAgB;;IACzD,aAAa;MAAiB;MAAa;;IAC3C;MAAW;MAAkB;MAAe;;IAC5C;MAAa;MAAkB;MAAe;;;;UAMjC;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;GACC;;UAGc;EACf;EACA,QAAQ;;EAER;;UAKe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;IACE;MACE;MACA;MACA;;;EAGJ,OAAO;EACP,gBAAgB;;UAKD;EACf;EACA,QAAQ,OAAO;EACf,YAAY,OAAO;;UAGJ;EACf;EACA;;;;;;;;;;KAWU;UAEK;EACf;EACA;EACA;EACA,oBAAoB;EACpB,mBAAmB;EACnB;EACA;;EAEA,QAAQ;;;;;;;EAOR;;;;;;;EAOA;;;;;;EAMA;;UAGe;EACf;EACA;EACA;IAAa;IAAiB;IAAkB;;EAChD;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;IAAa;IAAiB;IAAkB;;EAChD;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;;EAEA;;EAEA;;;;;;EAMA;EACA;EACA,SAAS;EACT,YAAY;EACZ;EACA;EACA;;UAKe;EACf,WAAW;EACX,QAAQ;EACR;EACA;EACA;EACA;EACA;EACA;;EAEA,aAAa;;UAGE;EACf,UAAU;EACV,OAAO;EACP,WAAW;;EAEX,aAAa;EACb;EACA,WAAW;EACX,SAAS;;KAGC,WAAW,MAAM,YAAY,OAAO,eAAe,QAAQ;UAItD;EACf;EACA;EACA;EACA;EACA,QAAQ;;UAGO;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA"}