@tangle-network/agent-eval 0.1.0

package/README.md

@@ -0,0 +1,58 @@
+# @tangle-network/agent-eval
+
+Domain-agnostic evaluation framework for Tangle agent apps. Multi-turn scenario execution, multi-judge scoring, agent-driver meta-testing, convergence tracking. Every agent (tax, legal, film, gtm) imports this to get a reproducible quality harness.
+
+## Install
+
+```bash
+npm install @tangle-network/agent-eval
+```
+
+## Usage
+
+```ts
+import { BenchmarkRunner, ProductClient, defaultJudges } from '@tangle-network/agent-eval'
+
+const client = new ProductClient({
+  baseUrl: 'https://my-agent.tangle.tools',
+  routes: {
+    signup: '/api/auth/sign-up/email',
+    chat: '/api/chat',
+    // ...
+  },
+})
+
+const runner = new BenchmarkRunner(client, {
+  scenarios: myScenarios,
+  judges: defaultJudges('film production'),
+  systemPrompt: MY_SYSTEM_PROMPT,
+})
+
+const report = await runner.run()
+```
+
+## What's in the box
+
+- **ProductClient** — configurable HTTP client (routes are config, not code)
+- **ScenarioRegistry** — auto-discovery + filtering
+- **executeScenario** — multi-turn executor with artifact collection
+- **BenchmarkRunner** — orchestrates scenarios + judges + scoring
+- **AgentDriver** — meta-agent that plays personas against a real product
+- **MetricsCollector** — per-turn product state metrics
+- **ConvergenceTracker** — completion% over turns
+- **Reporter** — markdown + console output
+- **Judges** — 4 built-in (domain expert, code execution, coherence, adversarial) + `createCustomJudge` factory
+
+## Tier
+
+Marketplace tier of the [agent-builder](https://github.com/drewstone/tangle-agent-builder) three-tier architecture. Uses [`@tangle-network/tcloud`](https://github.com/tangle-network/tcloud) for judge LLM calls.
+
+## Related
+
+- [`@tangle-network/agent-gateway`](https://github.com/tangle-network/agent-gateway) — the gateway agents published through
+- [`@tangle-network/agent-client`](https://github.com/tangle-network/agent-client) — consumer SDK for those endpoints
+- [`@tangle-network/tcloud`](https://github.com/tangle-network/tcloud) — platform SDK (used internally by judges)
+
+## License
+
+MIT

package/dist/index.d.ts

@@ -0,0 +1,579 @@
+import { TCloud } from '@tangle-network/tcloud';
+
+interface Scenario {
+    id: string;
+    persona: string;
+    label: string;
+    thesis: string;
+    dimensions: string[];
+    turns: Turn[];
+    artifactChecks: ArtifactCheck[];
+    systemPromptAppend?: string;
+}
+interface Turn {
+    user: string;
+    expectedBehaviors: string[];
+    adversarial?: boolean;
+    feedbackType?: 'correction' | 'rejection' | 'vague' | 'contradictory' | 'escalation';
+}
+interface ArtifactCheck {
+    type: 'vault_file_exists' | 'vault_file_contains' | 'block_extracted' | 'code_valid' | 'generation_produced' | 'tool_created' | string;
+    target: string;
+    contains?: string;
+    minCount?: number;
+    description: string;
+}
+interface JudgeConfig {
+    model: string;
+    temperature: number;
+    rubric: JudgeRubric;
+}
+interface JudgeRubric {
+    name: string;
+    description: string;
+    dimensions: RubricDimension[];
+}
+interface RubricDimension {
+    name: string;
+    description: string;
+    anchor_low: string;
+    anchor_high: string;
+    weight: number;
+}
+interface ScenarioResult {
+    scenarioId: string;
+    persona: string;
+    turns: TurnResult[];
+    artifactResults: ArtifactResult[];
+    judgeScores: JudgeScore[];
+    judgeErrors: number;
+    overallScore: number;
+    totalDurationMs: number;
+    artifacts: CollectedArtifacts;
+}
+interface TurnResult {
+    turnIndex: number;
+    userMessage: string;
+    agentResponse: string;
+    durationMs: number;
+    blocksExtracted: {
+        type: string;
+        title: string;
+    }[];
+    containsCode: boolean;
+    containsToolCall: boolean;
+}
+interface ArtifactResult {
+    check: ArtifactCheck;
+    passed: boolean;
+    detail?: string;
+}
+interface JudgeScore {
+    judgeName: string;
+    dimension: string;
+    score: number;
+    reasoning: string;
+    evidence?: string;
+}
+interface CollectedArtifacts {
+    vaultFiles: {
+        path: string;
+        content: string;
+    }[];
+    blocksExtracted: {
+        type: string;
+        fields: Record<string, string>;
+    }[];
+    codeBlocks: {
+        language: string;
+        code: string;
+    }[];
+    toolCalls: string[];
+}
+interface BenchmarkReport {
+    timestamp: string;
+    generation: number;
+    promptVersion: string;
+    scenarioCount: number;
+    results: ScenarioResult[];
+    summary: {
+        overallAvg: number;
+        byPersona: Record<string, {
+            avg: number;
+            passed: number;
+            total: number;
+        }>;
+        byDimension: Record<string, {
+            avg: number;
+            scores: number[];
+        }>;
+        weakest: {
+            scenario: string;
+            score: number;
+            reason: string;
+        }[];
+        strongest: {
+            scenario: string;
+            score: number;
+            reason: string;
+        }[];
+    };
+}
+interface RouteMap {
+    signup?: string;
+    login?: string;
+    workspaces?: string;
+    threads?: string;
+    chat?: string;
+    tasks?: string;
+    events?: string;
+    approvals?: string;
+    vault?: string;
+    generations?: string;
+    [key: string]: string | undefined;
+}
+interface ProductClientConfig {
+    baseUrl: string;
+    routes: RouteMap;
+}
+interface ScenarioFile {
+    id: string;
+    category: string;
+    persona: string;
+    label: string;
+    thesis: string;
+    isControl?: boolean;
+    rubric?: {
+        dimensions: {
+            name: string;
+            description: string;
+            weight: number;
+        }[];
+    };
+    turns: Turn[];
+    artifactChecks: ArtifactCheck[];
+}
+interface CompletionCriterion {
+    name: string;
+    check: (state: DriverState) => boolean;
+    progress?: (state: DriverState) => number;
+}
+interface FeedbackPattern {
+    trigger: string;
+    response: string;
+}
+interface PersonaConfig {
+    id: string;
+    role: string;
+    goal: string;
+    completionCriteria: CompletionCriterion[];
+    feedbackPatterns?: FeedbackPattern[];
+    maxTurns: number;
+    driverModel?: string;
+}
+interface DriverState {
+    tasks: number;
+    events: number;
+    proposals: {
+        pending: number;
+        approved: number;
+        rejected: number;
+    };
+    vaultFiles: string[];
+    codeBlocks: number;
+    generations: number;
+}
+interface TurnMetrics {
+    turn: number;
+    timestamp: string;
+    tasks: number;
+    events: number;
+    proposals: {
+        pending: number;
+        approved: number;
+        rejected: number;
+    };
+    vaultFiles: number;
+    responseLatencyMs: number;
+    responseChars: number;
+    codeBlocksProduced: number;
+    blocksExtracted: number;
+    qualityScore?: number;
+    inputTokens: number;
+    outputTokens: number;
+    estimatedCostUsd: number;
+    totalCostUsd: number;
+    completionPercent: number;
+}
+interface DriverResult {
+    personaId: string;
+    completed: boolean;
+    turnsToCompletion: number | null;
+    totalTurns: number;
+    metrics: TurnMetrics[];
+    finalState: DriverState;
+    convergenceCurve: number[];
+    totalCostUsd: number;
+    finalQualityScore: number | null;
+}
+interface BenchmarkRunnerConfig {
+    scenarios: Scenario[];
+    judges: JudgeFn[];
+    systemPrompt: string;
+    model?: string;
+    judgeModel?: string;
+    passThreshold?: number;
+    generation?: number;
+    promptVersion?: string;
+}
+interface JudgeInput {
+    scenario: Scenario;
+    turns: TurnResult[];
+    artifacts: CollectedArtifacts;
+}
+type JudgeFn = (tc: TCloud, input: JudgeInput) => Promise<JudgeScore[]>;
+
+interface TestResult {
+    name: string;
+    passed: boolean;
+    duration: number;
+    detail?: string;
+    checks: CheckResult[];
+}
+interface CheckResult {
+    name: string;
+    passed: boolean;
+    expected: string;
+    actual: string;
+}
+interface EvalResult {
+    scenario: string;
+    status: 'pass' | 'fail' | 'skip';
+    duration: number;
+    detail?: string;
+    artifact?: string;
+}
+
+/**
+ * ProductClient — configurable HTTP client for exercising any agent's APIs.
+ *
+ * Routes are config, not hardcoded. Each agent provides its own RouteMap.
+ */
+declare class ProductClient {
+    private baseUrl;
+    private routes;
+    private cookies;
+    constructor(config: ProductClientConfig);
+    private route;
+    signup(name: string, email: string, password: string): Promise<{
+        userId: string;
+    }>;
+    login(email: string, password: string): Promise<void>;
+    createWorkspace(name: string, type?: string): Promise<string>;
+    createThread(workspaceId: string): Promise<string>;
+    chat(workspaceId: string, threadId: string, content: string, _opts?: {
+        blockPatterns?: RegExp[];
+    }): Promise<{
+        text: string;
+        blocks: {
+            type: string;
+            title: string;
+        }[];
+    }>;
+    getTasks(workspaceId: string): Promise<{
+        id: string;
+        title: string;
+        status: string;
+        priority: string;
+    }[]>;
+    getEvents(workspaceId: string): Promise<{
+        id: string;
+        title: string;
+        type: string;
+    }[]>;
+    getApprovals(workspaceId: string): Promise<{
+        id: string;
+        title: string;
+        status: string;
+        type: string;
+    }[]>;
+    getVaultTree(workspaceId: string): Promise<string[]>;
+    approveAction(workspaceId: string, id: string): Promise<void>;
+    rejectAction(workspaceId: string, id: string, reason: string): Promise<void>;
+    getGenerations(workspaceId: string): Promise<{
+        id: string;
+        type: string;
+        prompt: string;
+    }[]>;
+    /** Generic GET for custom routes */
+    get(path: string): Promise<Record<string, unknown>>;
+    /** Generic POST for custom routes */
+    post(path: string, body: Record<string, unknown>): Promise<Record<string, unknown>>;
+    /** Generic PATCH for custom routes */
+    patch(path: string, body: Record<string, unknown>): Promise<Record<string, unknown>>;
+}
+/**
+ * Run a full e2e workflow test against a live product.
+ *
+ * The `workflow` callback receives a ProductClient and returns CheckResults.
+ * This is the generic harness — each agent defines its own workflow steps.
+ */
+declare function runE2EWorkflow(client: ProductClient, name: string, workflow: (client: ProductClient) => Promise<CheckResult[]>): Promise<TestResult>;
+
+/**
+ * Create a domain expert judge with a configurable domain.
+ *
+ * The judge evaluates professional accuracy and depth.
+ */
+declare function createDomainExpertJudge(domain: string): JudgeFn;
+/**
+ * Code execution judge — evaluates whether code blocks are valid and runnable.
+ */
+declare const codeExecutionJudge: JudgeFn;
+/**
+ * Coherence judge — evaluates multi-turn consistency and progression.
+ */
+declare const coherenceJudge: JudgeFn;
+/**
+ * Adversarial judge — red-teams agent responses.
+ */
+declare const adversarialJudge: JudgeFn;
+/**
+ * Create a custom judge with a fully custom prompt.
+ */
+declare function createCustomJudge(name: string, systemPrompt: string, opts?: {
+    model?: string;
+    temperature?: number;
+    maxTokens?: number;
+}): JudgeFn;
+/** Default judge set (domain must be provided for domain expert) */
+declare function defaultJudges(domain: string): JudgeFn[];
+
+interface ExecutorConfig {
+    /** System prompt for the agent under test */
+    systemPrompt: string;
+    /** Model to use for the agent */
+    model?: string;
+    /** Judges to run after execution */
+    judges: JudgeFn[];
+    /** Regex patterns for detecting tool/API calls in responses */
+    toolCallPatterns?: RegExp[];
+    /** Block delimiter pattern (default: :::type\n...\n:::) */
+    blockPattern?: RegExp;
+    /** Custom artifact checker for domain-specific checks */
+    artifactChecker?: (check: Scenario['artifactChecks'][0], artifacts: CollectedArtifacts) => {
+        passed: boolean;
+        detail: string;
+    } | null;
+}
+/**
+ * Execute a scenario against an LLM via tcloud.
+ *
+ * Runs multi-turn conversation, extracts artifacts, runs judges.
+ */
+declare function executeScenario(tc: TCloud, scenario: Scenario, config: ExecutorConfig): Promise<ScenarioResult>;
+
+/**
+ * BenchmarkRunner — orchestrates scenarios, executor, judges, and scoring.
+ *
+ * Domain-agnostic. Each agent provides its own scenarios, judges, and system prompt.
+ */
+declare class BenchmarkRunner {
+    private tc;
+    private config;
+    constructor(tc: TCloud, config: BenchmarkRunnerConfig);
+    run(scenarios?: Scenario[]): Promise<BenchmarkReport>;
+}
+
+/** Per-1K token pricing for common models */
+declare const MODEL_PRICING: Record<string, {
+    input: number;
+    output: number;
+}>;
+/** Estimate token count from string length (chars / 4 approximation) */
+declare function estimateTokens(text: string): number;
+/** Calculate cost in USD from token counts and model */
+declare function estimateCost(inputTokens: number, outputTokens: number, model: string): number;
+/**
+ * TokenCounter — accumulates token usage and cost across turns.
+ */
+declare class TokenCounter {
+    private totalInput;
+    private totalOutput;
+    private totalCost;
+    private model;
+    constructor(model?: string);
+    /** Record tokens for a turn, returns per-turn cost */
+    record(inputTokens: number, outputTokens: number): number;
+    /** Estimate and record from raw text */
+    recordFromText(inputText: string, outputText: string): {
+        inputTokens: number;
+        outputTokens: number;
+        cost: number;
+    };
+    getTotalInput(): number;
+    getTotalOutput(): number;
+    getTotalCost(): number;
+}
+/**
+ * MetricsCollector — collects per-turn metrics from the product.
+ *
+ * After each turn, queries the product's APIs to measure state changes.
+ */
+declare class MetricsCollector {
+    private client;
+    private workspaceId;
+    private metrics;
+    constructor(client: ProductClient, workspaceId: string);
+    /** Collect metrics after a turn completes */
+    collect(turn: number, responseLatencyMs: number, responseChars: number, codeBlocksProduced: number, blocksExtracted: number, completionCriteriaMet: number, completionCriteriaTotal: number, qualityScore?: number, inputTokens?: number, outputTokens?: number, estimatedCostUsd?: number): Promise<TurnMetrics>;
+    /** Get current product state */
+    getState(): Promise<DriverState>;
+    /** Get all collected metrics */
+    getMetrics(): TurnMetrics[];
+    /** Get convergence curve (completion% over turns) */
+    getConvergenceCurve(): number[];
+}
+
+/**
+ * Normalize scores so all dimensions follow "higher = better".
+ * Inverted dimensions (hallucination, false_confidence, worst_failure)
+ * already use inverted scoring in the prompt (10 = no hallucination),
+ * but this function ensures consistency if raw scores leak through.
+ */
+declare function normalizeScores(scores: JudgeScore[]): JudgeScore[];
+/** Weighted mean — falls back to uniform weights when omitted */
+declare function weightedMean(scores: {
+    score: number;
+    weight?: number;
+}[]): number;
+/** Bootstrap confidence interval */
+declare function confidenceInterval(scores: number[], confidence?: number): {
+    mean: number;
+    lower: number;
+    upper: number;
+};
+/**
+ * Inter-rater reliability — simplified Krippendorff's alpha.
+ *
+ * Each inner array is one judge's scores for all items.
+ * All arrays must have the same length (same items scored).
+ */
+declare function interRaterReliability(judgeScores: JudgeScore[][]): number;
+/**
+ * Mann-Whitney U test for comparing two independent groups.
+ * Returns U statistic and approximate p-value (normal approximation).
+ */
+declare function mannWhitneyU(a: number[], b: number[]): {
+    u: number;
+    p: number;
+};
+/** Partial credit: returns 0-1 ratio of current toward target */
+declare function partialCredit(current: number, target: number): number;
+
+/**
+ * ConvergenceTracker — tracks completion percentage over turns.
+ *
+ * Produces convergence curves showing how quickly the agent reaches
+ * completion criteria.
+ */
+declare class ConvergenceTracker {
+    private criteria;
+    private history;
+    constructor(criteria: CompletionCriterion[]);
+    /** Evaluate criteria against current state, record result */
+    record(turn: number, state: DriverState): {
+        completionPercent: number;
+        complete: boolean;
+        criteriaStatus: Record<string, boolean | number>;
+    };
+    /** Get convergence curve */
+    getCurve(): number[];
+    /** Get full history with per-criterion status */
+    getHistory(): {
+        turn: number;
+        completionPercent: number;
+        criteriaStatus: Record<string, boolean | number>;
+    }[];
+    /** Find the turn where completion first reached 100% (or null) */
+    getTurnToCompletion(): number | null;
+}
+
+/**
+ * ScenarioRegistry — manages scenario discovery and filtering.
+ *
+ * Each agent registers its scenarios. The registry handles conversion
+ * from ScenarioFile format to the framework's Scenario type.
+ */
+declare class ScenarioRegistry {
+    private scenarios;
+    private scenarioFiles;
+    /** Register scenarios from ScenarioFile format */
+    registerFiles(files: ScenarioFile[]): void;
+    /** Register pre-built Scenario objects directly */
+    register(scenarios: Scenario[]): void;
+    /** Get all scenarios */
+    all(): Scenario[];
+    /** Get scenarios filtered by category */
+    byCategory(category: string): Scenario[];
+    /** List all categories with counts */
+    listCategories(): {
+        category: string;
+        count: number;
+    }[];
+    /** Get scenarios filtered by persona */
+    byPersona(persona: string): Scenario[];
+    /** Get a single scenario by ID */
+    byId(id: string): Scenario | undefined;
+    /** Count total scenarios */
+    get count(): number;
+}
+
+interface AgentDriverConfig {
+    client: ProductClient;
+    driverModel?: string;
+    /** System prompt context for the driver LLM to understand the product */
+    productContext?: string;
+}
+/**
+ * AgentDriver — meta-agent that plays a persona against the real product.
+ *
+ * Uses a driver LLM (Claude/GPT-4o) to decide what to say each turn.
+ * Not scripted — the driver gets the current product state and decides
+ * the next realistic user message.
+ */
+declare class AgentDriver {
+    private tc;
+    private client;
+    private driverModel;
+    private productContext;
+    constructor(tc: TCloud, config: AgentDriverConfig);
+    /**
+     * Run a persona through the product.
+     *
+     * Returns metrics on how many turns to completion, cost curve,
+     * quality curve, and convergence curve.
+     */
+    run(persona: PersonaConfig): Promise<DriverResult>;
+    /** Use the driver LLM to decide what the "user" says next */
+    private decideNextMessage;
+    /** Handle pending approvals based on persona feedback patterns */
+    private handleApprovals;
+    /** Describe which completion criteria are met */
+    private describeCompletion;
+}
+
+/**
+ * Report generation utilities.
+ *
+ * Outputs convergence curves, cost curves, quality curves,
+ * and per-persona summaries in markdown format.
+ */
+/** Generate a markdown report from benchmark results */
+declare function formatBenchmarkReport(report: BenchmarkReport): string;
+/** Generate a markdown report from agent driver results */
+declare function formatDriverReport(results: DriverResult[]): string;
+/** Print a compact summary to console */
+declare function printDriverSummary(results: DriverResult[]): void;
+
+export { AgentDriver, type AgentDriverConfig, type ArtifactCheck, type ArtifactResult, type BenchmarkReport, BenchmarkRunner, type BenchmarkRunnerConfig, type CheckResult, type CollectedArtifacts, type CompletionCriterion, ConvergenceTracker, type DriverResult, type DriverState, type EvalResult, type ExecutorConfig, type FeedbackPattern, type JudgeConfig, type JudgeFn, type JudgeInput, type JudgeRubric, type JudgeScore, MODEL_PRICING, MetricsCollector, type PersonaConfig, ProductClient, type ProductClientConfig, type RouteMap, type RubricDimension, type Scenario, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type TestResult, TokenCounter, type Turn, type TurnMetrics, type TurnResult, adversarialJudge, codeExecutionJudge, coherenceJudge, confidenceInterval, createCustomJudge, createDomainExpertJudge, defaultJudges, estimateCost, estimateTokens, executeScenario, formatBenchmarkReport, formatDriverReport, interRaterReliability, mannWhitneyU, normalizeScores, partialCredit, printDriverSummary, runE2EWorkflow, weightedMean };