@directive-run/ai 1.13.0 → 1.14.0

package/dist/devtools.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"names":[],"mappings":"","file":"devtools.js"}

package/dist/evals.cjs

@@ -0,0 +1,2 @@
+'use strict';var chunkXN5LUOVS_cjs=require('./chunk-XN5LUOVS.cjs');require('./chunk-3WO4MWJM.cjs');Object.defineProperty(exports,"createEvalSuite",{enumerable:true,get:function(){return chunkXN5LUOVS_cjs.k}});Object.defineProperty(exports,"evalAssert",{enumerable:true,get:function(){return chunkXN5LUOVS_cjs.l}});Object.defineProperty(exports,"evalCoherence",{enumerable:true,get:function(){return chunkXN5LUOVS_cjs.j}});Object.defineProperty(exports,"evalCost",{enumerable:true,get:function(){return chunkXN5LUOVS_cjs.a}});Object.defineProperty(exports,"evalFaithfulness",{enumerable:true,get:function(){return chunkXN5LUOVS_cjs.h}});Object.defineProperty(exports,"evalJudge",{enumerable:true,get:function(){return chunkXN5LUOVS_cjs.f}});Object.defineProperty(exports,"evalLatency",{enumerable:true,get:function(){return chunkXN5LUOVS_cjs.b}});Object.defineProperty(exports,"evalMatch",{enumerable:true,get:function(){return chunkXN5LUOVS_cjs.g}});Object.defineProperty(exports,"evalOutputLength",{enumerable:true,get:function(){return chunkXN5LUOVS_cjs.c}});Object.defineProperty(exports,"evalRelevance",{enumerable:true,get:function(){return chunkXN5LUOVS_cjs.i}});Object.defineProperty(exports,"evalSafety",{enumerable:true,get:function(){return chunkXN5LUOVS_cjs.d}});Object.defineProperty(exports,"evalStructure",{enumerable:true,get:function(){return chunkXN5LUOVS_cjs.e}});//# sourceMappingURL=evals.cjs.map
+//# sourceMappingURL=evals.cjs.map

package/dist/evals.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"names":[],"mappings":"","file":"evals.cjs"}

package/dist/evals.d.cts

@@ -0,0 +1,361 @@
+import { D as DebugTimeline } from './debug-timeline-DpnRMnLU.cjs';
+import { q as RunResult, b as AgentLike, c as AgentRunner, R as RunOptions } from './types-DJ09LjZX.cjs';
+import '@directive-run/core';
+
+/**
+ * Evaluation Framework — Constraint-driven agent evaluation.
+ *
+ * Define eval criteria as composable functions. Run agents against datasets
+ * and score their outputs across multiple dimensions. Results integrate with
+ * the debug timeline for DevTools visualization.
+ *
+ * @example
+ * ```typescript
+ * const suite = createEvalSuite({
+ *   criteria: {
+ *     safe: evalSafety({ categories: ["pii"] }),
+ *     costEfficient: evalCost({ maxTokensPerRun: 5000 }),
+ *     fast: evalLatency({ maxMs: 3000 }),
+ *   },
+ *   agents: [researchAgent, writerAgent],
+ *   runner: myRunner,
+ *   dataset: [
+ *     { id: "case-1", input: "What is AI?", expected: "explanation about AI" },
+ *   ],
+ * });
+ *
+ * const results = await suite.run();
+ * // results.summary — pass/fail per criterion per agent
+ * // results.details — per-case breakdown
+ * ```
+ *
+ * @module
+ */
+
+/** Single test case in the eval dataset */
+interface EvalCase {
+    /** Unique identifier for tracking across runs */
+    id?: string;
+    /** Input to feed the agent */
+    input: string;
+    /** Expected output or reference answer (for comparison-based criteria) */
+    expected?: string;
+    /** Reference context for faithfulness evaluation */
+    context?: string;
+    /** Tags for filtering and grouping results */
+    tags?: string[];
+    /** Additional context passed to criteria */
+    metadata?: Record<string, unknown>;
+}
+/** Result of evaluating a single criterion on a single case */
+interface EvalScore {
+    /** Score from 0.0 to 1.0 */
+    score: number;
+    /** Whether this score passes the criterion threshold */
+    passed: boolean;
+    /** Reason for the score */
+    reason?: string;
+    /** Duration of evaluation (ms) */
+    durationMs: number;
+}
+/** Context passed to eval criterion functions */
+interface EvalContext {
+    /** The agent being evaluated */
+    agent: AgentLike;
+    /** The test case */
+    testCase: EvalCase;
+    /** The agent's run result */
+    result: RunResult<unknown>;
+    /** Duration of the agent run (ms) */
+    runDurationMs: number;
+}
+/** Eval criterion function — scores an agent's output */
+type EvalCriterionFn = (context: EvalContext) => EvalScore | Promise<EvalScore>;
+/** Named eval criterion */
+interface EvalCriterion {
+    name: string;
+    fn: EvalCriterionFn;
+    /** Score threshold for passing. Default: 0.5 */
+    threshold?: number;
+    /** Weight for aggregation. Default: 1.0 */
+    weight?: number;
+}
+/** Per-case detail result */
+interface EvalCaseResult {
+    /** Test case that was evaluated */
+    testCase: EvalCase;
+    /** Agent that was evaluated */
+    agentName: string;
+    /** Agent run result */
+    runResult: RunResult<unknown>;
+    /** Score per criterion */
+    scores: Record<string, EvalScore>;
+    /** Overall weighted score (0.0-1.0) */
+    overallScore: number;
+    /** Whether all criteria passed */
+    allPassed: boolean;
+    /** Agent run duration (ms) */
+    runDurationMs: number;
+}
+/** Per-agent summary */
+interface EvalAgentSummary {
+    agentName: string;
+    /** Average score per criterion */
+    criterionAverages: Record<string, number>;
+    /** Pass rate per criterion (0.0-1.0) */
+    criterionPassRates: Record<string, number>;
+    /** Overall weighted average score */
+    overallScore: number;
+    /** Overall pass rate */
+    passRate: number;
+    /** Total tokens consumed */
+    totalTokens: number;
+    /** Average latency per run (ms) */
+    avgLatencyMs: number;
+    /** Total cases evaluated */
+    totalCases: number;
+    /** Cases that passed all criteria */
+    passedCases: number;
+}
+/** Complete eval suite results */
+interface EvalResults {
+    /** Summary per agent */
+    summary: Record<string, EvalAgentSummary>;
+    /** Detailed per-case results */
+    details: EvalCaseResult[];
+    /** Total duration (ms) */
+    durationMs: number;
+    /** Total tokens consumed across all agents and cases */
+    totalTokens: number;
+    /** Timestamp when the eval started */
+    startedAt: number;
+    /** Timestamp when the eval completed */
+    completedAt: number;
+}
+/** Configuration for createEvalSuite */
+interface EvalSuiteConfig {
+    /** Named criteria to evaluate */
+    criteria: Record<string, EvalCriterionFn | EvalCriterion>;
+    /** Agents to evaluate */
+    agents: AgentLike[];
+    /** Agent runner function */
+    runner: AgentRunner;
+    /** Dataset of test cases */
+    dataset: EvalCase[];
+    /** Run options passed to the runner */
+    runOptions?: Omit<RunOptions, "signal">;
+    /** Maximum concurrent agent runs. Default: 5 */
+    concurrency?: number;
+    /** Optional debug timeline for recording eval events */
+    timeline?: DebugTimeline;
+    /** Callback fired on each case completion */
+    onCaseComplete?: (result: EvalCaseResult) => void;
+    /** Callback fired on each agent completion */
+    onAgentComplete?: (summary: EvalAgentSummary) => void;
+    /** Abort signal */
+    signal?: AbortSignal;
+}
+/** Eval suite instance */
+interface EvalSuite {
+    /** Run the full evaluation */
+    run(): Promise<EvalResults>;
+    /** Run evaluation for a specific agent only */
+    runAgent(agentName: string): Promise<EvalAgentSummary>;
+    /** Get the list of agents being evaluated */
+    getAgents(): AgentLike[];
+    /** Get the list of criteria */
+    getCriteria(): string[];
+    /** Get the dataset */
+    getDataset(): EvalCase[];
+}
+/** Options for cost evaluation */
+interface EvalCostOptions {
+    /** Maximum tokens per run */
+    maxTokensPerRun: number;
+}
+/**
+ * Evaluate cost efficiency — scores based on token usage relative to a budget.
+ *
+ * Score = 1.0 when tokens \<= maxTokensPerRun * 0.5,
+ * Score = 0.0 when tokens \>= maxTokensPerRun * 2.
+ * Linear interpolation between.
+ *
+ * @param options - Cost evaluation options including `maxTokensPerRun`.
+ * @returns An eval criterion that scores token usage against the budget.
+ */
+declare function evalCost(options: EvalCostOptions): EvalCriterion;
+/** Options for latency evaluation */
+interface EvalLatencyOptions {
+    /** Maximum acceptable latency (ms) */
+    maxMs: number;
+}
+/**
+ * Evaluate latency — scores based on agent run duration.
+ *
+ * Score = 1.0 when duration \<= maxMs * 0.5,
+ * Score = 0.0 when duration \>= maxMs * 2.
+ * Linear interpolation between.
+ *
+ * @param options - Latency evaluation options including `maxMs`.
+ * @returns An eval criterion that scores run duration against the limit.
+ */
+declare function evalLatency(options: EvalLatencyOptions): EvalCriterion;
+/** Options for output length evaluation */
+interface EvalOutputLengthOptions {
+    /** Minimum output length (chars) */
+    minLength?: number;
+    /** Maximum output length (chars) */
+    maxLength?: number;
+}
+/**
+ * Evaluate output length — ensures output is within an acceptable range.
+ */
+declare function evalOutputLength(options: EvalOutputLengthOptions): EvalCriterion;
+/** Options for safety evaluation */
+interface EvalSafetyOptions {
+    /** Patterns to check for in output (overrides categories) */
+    blockedPatterns?: RegExp[];
+    /** Categories of content to check: "pii", "violence", "self_harm", "illegal" */
+    categories?: Array<"pii" | "violence" | "self_harm" | "illegal">;
+}
+/**
+ * Evaluate safety — checks output for blocked patterns or category-based content.
+ *
+ * When `categories` is provided, uses built-in pattern sets for each category.
+ * When `blockedPatterns` is provided, uses those directly (overrides categories).
+ * When neither is provided, defaults to all safety categories.
+ *
+ * Score = 1.0 when no blocked patterns found.
+ * Score = 0.0 when any blocked pattern matches.
+ */
+declare function evalSafety(options?: EvalSafetyOptions): EvalCriterion;
+/** Options for output structure evaluation */
+interface EvalStructureOptions {
+    /** Expected output type */
+    type?: "json" | "string";
+    /** Required keys if type is "json" */
+    requiredKeys?: string[];
+}
+/**
+ * Evaluate output structure — checks that output matches an expected format.
+ */
+declare function evalStructure(options: EvalStructureOptions): EvalCriterion;
+/**
+ * Evaluate with a custom LLM judge — uses a runner to grade the output.
+ *
+ * The judge agent receives the input, output, and expected answer, and
+ * returns a JSON score.
+ */
+interface EvalJudgeOptions {
+    /** Runner to use for the judge */
+    runner: AgentRunner;
+    /** Judge agent */
+    judge: AgentLike;
+    /** Custom grading prompt template. {{input}}, {{output}}, {{expected}} are replaced. */
+    promptTemplate?: string;
+    /** Optional abort signal */
+    signal?: AbortSignal;
+    /** Timeout for the judge call in ms. Default: 30_000 */
+    timeoutMs?: number;
+}
+/**
+ * Evaluate output quality by delegating to a judge agent that scores from 0.0 to 1.0.
+ *
+ * @param options - Judge evaluation options including `runner`, `judge` agent, and optional `promptTemplate`.
+ * @returns An eval criterion that runs a judge agent and returns its score.
+ */
+declare function evalJudge(options: EvalJudgeOptions): EvalCriterion;
+/**
+ * Evaluate exact or substring match against expected output.
+ */
+interface EvalMatchOptions {
+    /** Match mode. Default: "contains" */
+    mode?: "exact" | "contains" | "regex";
+    /** Case-insensitive matching. Default: true */
+    caseInsensitive?: boolean;
+}
+/**
+ * Evaluate exact or substring match against expected output.
+ *
+ * @param options - Match evaluation options including `mode` and `caseInsensitive`.
+ * @returns An eval criterion that checks output against the expected value.
+ */
+declare function evalMatch(options?: EvalMatchOptions): EvalCriterion;
+/** Options for LLM-based semantic evaluation criteria */
+interface EvalSemanticOptions {
+    /** Runner to use for the judge LLM */
+    runner: AgentRunner;
+    /** Judge agent (model to use for evaluation) */
+    judge: AgentLike;
+    /** Optional abort signal */
+    signal?: AbortSignal;
+    /** Timeout for the judge call in ms. Default: 30_000 */
+    timeoutMs?: number;
+}
+/**
+ * Evaluate faithfulness — whether the output is grounded in the provided context.
+ *
+ * Requires `context` field on the EvalCase. Uses an LLM judge internally
+ * to extract and verify claims against the reference context.
+ */
+declare function evalFaithfulness(options: EvalSemanticOptions): EvalCriterion;
+/**
+ * Evaluate relevance — whether the output directly addresses the input question.
+ *
+ * Uses an LLM judge to assess how well the agent's output answers
+ * the original question.
+ */
+declare function evalRelevance(options: EvalSemanticOptions): EvalCriterion;
+/**
+ * Evaluate coherence — whether the output is logically consistent and well-structured.
+ *
+ * Uses an LLM judge to assess the internal coherence, logical flow,
+ * and consistency of the output.
+ */
+declare function evalCoherence(options: EvalSemanticOptions): EvalCriterion;
+/**
+ * Create an evaluation suite for testing agents against a dataset.
+ *
+ * @example
+ * ```typescript
+ * const suite = createEvalSuite({
+ *   criteria: {
+ *     fast: evalLatency({ maxMs: 3000 }),
+ *     cheap: evalCost({ maxTokensPerRun: 5000 }),
+ *   },
+ *   agents: [researchAgent, writerAgent],
+ *   runner: myRunner,
+ *   dataset: [{ input: "What is AI?" }],
+ * });
+ *
+ * const results = await suite.run();
+ * ```
+ */
+declare function createEvalSuite(config: EvalSuiteConfig): EvalSuite;
+/** Options for eval assertions in CI */
+interface EvalAssertOptions {
+    /** Minimum weighted overall score required (0.0-1.0) */
+    minScore?: number;
+    /** Minimum pass rate required (0.0-1.0) */
+    minPassRate?: number;
+    /** Criteria that must achieve 100% pass rate */
+    failOn?: string[];
+}
+/**
+ * Assert eval results meet requirements — designed for CI pipelines.
+ *
+ * Throws an error with details if any assertion fails.
+ *
+ * @example
+ * ```typescript
+ * const results = await suite.run();
+ * evalAssert(results, {
+ *   minScore: 0.8,
+ *   minPassRate: 0.9,
+ *   failOn: ["safety"],
+ * });
+ * ```
+ */
+declare function evalAssert(results: EvalResults, options: EvalAssertOptions): void;
+
+export { type EvalAgentSummary, type EvalAssertOptions, type EvalCase, type EvalCaseResult, type EvalContext, type EvalCostOptions, type EvalCriterion, type EvalCriterionFn, type EvalJudgeOptions, type EvalLatencyOptions, type EvalMatchOptions, type EvalOutputLengthOptions, type EvalResults, type EvalSafetyOptions, type EvalScore, type EvalSemanticOptions, type EvalStructureOptions, type EvalSuite, type EvalSuiteConfig, createEvalSuite, evalAssert, evalCoherence, evalCost, evalFaithfulness, evalJudge, evalLatency, evalMatch, evalOutputLength, evalRelevance, evalSafety, evalStructure };