@botpress/adk 1.15.3 → 1.16.0

package/dist/eval/loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../src/eval/loader.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAEzD,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAY5E;AAED,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAcjF;AAED,wBAAsB,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAYlG;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,MAAM,CAAC,EAAE,UAAU,GAAG,cAAc,EAAE,CAe1F"}

package/dist/eval/runner.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Eval runner — orchestrates eval execution.
+ */
+import type { EvalDefinition, EvalReport, EvalRunReport, EvalRunnerConfig, EvalFilter, BotConnection } from './types';
+/**
+ * Run a single eval against a bot.
+ */
+export declare function runEval(evalDef: EvalDefinition, connection: BotConnection, options?: {
+    devServerUrl?: string;
+}): Promise<EvalReport>;
+/**
+ * Run a suite of evals.
+ */
+export declare function runEvalSuite(config: EvalRunnerConfig, filter?: EvalFilter): Promise<EvalRunReport>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/eval/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EACV,cAAc,EACd,UAAU,EACV,aAAa,EACb,gBAAgB,EAChB,UAAU,EAIV,aAAa,EACd,MAAM,SAAS,CAAA;AAahB;;GAEG;AACH,wBAAsB,OAAO,CAC3B,OAAO,EAAE,cAAc,EACvB,UAAU,EAAE,aAAa,EACzB,OAAO,GAAE;IAAE,YAAY,CAAC,EAAE,MAAM,CAAA;CAAO,GACtC,OAAO,CAAC,UAAU,CAAC,CAmNrB;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,MAAM,EAAE,gBAAgB,EAAE,MAAM,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC,CAuExG"}

package/dist/eval/store.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Eval results store.
+ * Persists run results as JSON files in .adk/evals/runs/.
+ */
+import type { EvalRunReport } from './types';
+export declare function saveRunResult(agentPath: string, report: EvalRunReport): string;
+export declare function loadRunResult(agentPath: string, runId: string): EvalRunReport | null;
+export declare function listRunResults(agentPath: string, limit?: number): EvalRunReport[];
+export declare function getLatestRun(agentPath: string): EvalRunReport | null;
+//# sourceMappingURL=store.d.ts.map

package/dist/eval/store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/eval/store.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAU5C,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,GAAG,MAAM,CAM9E;AAED,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAYpF;AAED,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,SAAK,GAAG,aAAa,EAAE,CAc7E;AAED,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAGpE"}

package/dist/eval/traces.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Trace data extraction for evals.
+ * Uses the ADK dev server's HTTP API to fetch traces by conversationId.
+ * This approach is used instead of direct TraceReader access because:
+ * 1. The eval engine runs in the ADK package, not the CLI
+ * 2. The dev server is always running when evals execute
+ * 3. HTTP is the established interface for trace queries
+ */
+import type { TraceData } from './types';
+/**
+ * Fetch and extract structured trace data for a conversation.
+ * If previousToolCallCount is provided, only returns tool calls after that index.
+ * When expectNewCalls is true, retries fetching until new tool calls appear.
+ */
+export declare function getTraceData(conversationId: string, devServerUrl: string, options?: {
+    previousToolCallCount?: number;
+    expectNewCalls?: boolean;
+}): Promise<TraceData>;
+//# sourceMappingURL=traces.d.ts.map

package/dist/eval/traces.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"traces.d.ts","sourceRoot":"","sources":["../../src/eval/traces.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAY,SAAS,EAAa,MAAM,SAAS,CAAA;AAoD7D;;;;GAIG;AACH,wBAAsB,YAAY,CAChC,cAAc,EAAE,MAAM,EACtB,YAAY,EAAE,MAAM,EACpB,OAAO,GAAE;IAAE,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,OAAO,CAAA;CAAO,GACzE,OAAO,CAAC,SAAS,CAAC,CAsBpB"}

package/dist/eval/types.d.ts

@@ -0,0 +1,202 @@
+/**
+ * Eval type definitions.
+ * Ported from adk-eval-standalone/src/parser/schema.ts with added report types.
+ */
+export type MatchOperator = string | {
+    equals: unknown;
+} | {
+    contains: string;
+} | {
+    not_contains: string;
+} | {
+    matches: string;
+} | {
+    in: unknown[];
+} | {
+    exists: boolean;
+} | {
+    gte: number;
+} | {
+    lte: number;
+};
+export type ResponseAssertion = {
+    contains: string;
+} | {
+    not_contains: string;
+} | {
+    matches: string;
+} | {
+    llm_judge: string;
+} | {
+    similar_to: string;
+};
+export type ToolAssertion = {
+    called: string;
+    params?: Record<string, MatchOperator>;
+} | {
+    not_called: string;
+} | {
+    call_order: string[];
+};
+export interface StateAssertion {
+    path: string;
+    equals?: unknown;
+    changed?: boolean;
+}
+export type TableAssertion = {
+    table: string;
+    row_exists: Record<string, MatchOperator>;
+} | {
+    table: string;
+    row_count: MatchOperator;
+    where?: Record<string, MatchOperator>;
+};
+export interface WorkflowAssertion {
+    name: string;
+    entered?: boolean;
+    completed?: boolean;
+}
+export interface TurnAssertions {
+    response?: ResponseAssertion[];
+    tools?: ToolAssertion[];
+    state?: StateAssertion[];
+    tables?: TableAssertion[];
+    workflow?: WorkflowAssertion[];
+}
+export interface ConversationTurn {
+    user: string;
+    assert?: TurnAssertions;
+}
+export interface OutcomeAssertions {
+    state?: StateAssertion[];
+    tables?: TableAssertion[];
+    workflow?: WorkflowAssertion[];
+}
+export interface EvalDefinition {
+    name: string;
+    description?: string;
+    tags?: string[];
+    type?: 'capability' | 'regression';
+    conversation: ConversationTurn[];
+    outcome?: OutcomeAssertions;
+}
+/** Type-safe eval definition helper. Provides intellisense for eval files. */
+export declare function defineEval(def: EvalDefinition): EvalDefinition;
+export interface EvalContext {
+    botId: string;
+    userId: string;
+    conversationId: string;
+}
+export interface GraderResult {
+    assertion: string;
+    pass: boolean;
+    expected: string;
+    actual: string;
+}
+export interface ToolCall {
+    name: string;
+    input: Record<string, unknown>;
+    output: string;
+    status: string;
+}
+export interface TraceData {
+    toolCalls: ToolCall[];
+    totalToolCallCount: number;
+    raw: TraceSpan[];
+}
+export interface TraceSpan {
+    t: 'start' | 'end' | 'beat';
+    traceId: string;
+    spanId: string;
+    parentSpanId?: string | null;
+    name?: string;
+    startNs?: number;
+    endNs?: number;
+    durationNs?: number;
+    attrs?: Record<string, unknown>;
+    [key: string]: unknown;
+}
+export interface TurnReport {
+    turnNumber: number;
+    userMessage: string;
+    botResponse: string;
+    assertions: GraderResult[];
+    pass: boolean;
+    /** Time spent waiting for the bot to respond (ms) */
+    botDuration: number;
+    /** Time spent fetching traces + running graders (ms) */
+    evalDuration: number;
+}
+export interface EvalReport {
+    name: string;
+    description?: string;
+    type?: 'capability' | 'regression';
+    tags?: string[];
+    turns: TurnReport[];
+    outcomeAssertions: GraderResult[];
+    pass: boolean;
+    duration: number;
+    error?: string;
+}
+export type EvalProgressEvent = {
+    type: 'suite_start';
+    totalEvals: number;
+} | {
+    type: 'eval_start';
+    evalName: string;
+    index: number;
+} | {
+    type: 'eval_complete';
+    evalName: string;
+    index: number;
+    report: EvalReport;
+} | {
+    type: 'suite_complete';
+    report: EvalRunReport;
+};
+export interface EvalRunReport {
+    id: string;
+    timestamp: string;
+    evals: EvalReport[];
+    passed: number;
+    failed: number;
+    total: number;
+    duration: number;
+    filter?: {
+        names?: string[];
+        tags?: string[];
+        type?: 'capability' | 'regression';
+    };
+}
+export interface BotConnection {
+    webhookId: string;
+    botId: string;
+    token: string;
+    apiUrl: string;
+}
+export interface BotResponse {
+    text: string;
+    raw: unknown;
+}
+export interface TurnResult {
+    conversationId: string;
+    responses: BotResponse[];
+}
+export interface EvalRunnerConfig {
+    credentials: {
+        token: string;
+        apiUrl: string;
+        botId: string;
+        webhookId?: string;
+    };
+    agentPath: string;
+    /** URL of the dev server providing traces and config. Defaults to http://localhost:3001 */
+    devServerUrl?: string;
+    onProgress?: (event: EvalProgressEvent) => void;
+}
+export interface EvalFilter {
+    names?: string[];
+    tags?: string[];
+    type?: 'capability' | 'regression';
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/eval/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,MAAM,MAAM,aAAa,GACrB,MAAM,GACN;IAAE,MAAM,EAAE,OAAO,CAAA;CAAE,GACnB;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,GACpB;IAAE,YAAY,EAAE,MAAM,CAAA;CAAE,GACxB;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,GACnB;IAAE,EAAE,EAAE,OAAO,EAAE,CAAA;CAAE,GACjB;IAAE,MAAM,EAAE,OAAO,CAAA;CAAE,GACnB;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GACf;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,CAAA;AAInB,MAAM,MAAM,iBAAiB,GACzB;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,GACpB;IAAE,YAAY,EAAE,MAAM,CAAA;CAAE,GACxB;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,GACnB;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,GACrB;IAAE,UAAU,EAAE,MAAM,CAAA;CAAE,CAAA;AAI1B,MAAM,MAAM,aAAa,GACrB;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;CAAE,GAC1D;IAAE,UAAU,EAAE,MAAM,CAAA;CAAE,GACtB;IAAE,UAAU,EAAE,MAAM,EAAE,CAAA;CAAE,CAAA;AAI5B,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,OAAO,CAAC,EAAE,OAAO,CAAA;CAClB;AAED,MAAM,MAAM,cAAc,GACtB;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;CAAE,GAC5D;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,aAAa,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;CAAE,CAAA;AAEtF,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,SAAS,CAAC,EAAE,OAAO,CAAA;CACpB;AAID,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAA;IAC9B,KAAK,CAAC,EAAE,aAAa,EAAE,CAAA;IACvB,KAAK,CAAC,EAAE,cAAc,EAAE,CAAA;IACxB,MAAM,CAAC,EAAE,cAAc,EAAE,CAAA;IACzB,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAA;CAC/B;AAID,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,CAAC,EAAE,cAAc,CAAA;CACxB;AAID,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,cAAc,EAAE,CAAA;IACxB,MAAM,CAAC,EAAE,cAAc,EAAE,CAAA;IACzB,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAA;CAC/B;AAID,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,IAAI,CAAC,EAAE,YAAY,GAAG,YAAY,CAAA;IAClC,YAAY,EAAE,gBAAgB,EAAE,CAAA;IAChC,OAAO,CAAC,EAAE,iBAAiB,CAAA;CAC5B;AAED,8EAA8E;AAC9E,wBAAgB,UAAU,CAAC,GAAG,EAAE,cAAc,GAAG,cAAc,CAE9D;AAID,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,cAAc,EAAE,MAAM,CAAA;CACvB;AAID,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAA;IACjB,IAAI,EAAE,OAAO,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;CACf;AAID,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC9B,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;CACf;AAID,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,QAAQ,EAAE,CAAA;IACrB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,GAAG,EAAE,SAAS,EAAE,CAAA;CACjB;AAID,MAAM,WAAW,SAAS;IACxB,CAAC,EAAE,OAAO,GAAG,KAAK,GAAG,MAAM,CAAA;IAC3B,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC/B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACvB;AAID,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,YAAY,EAAE,CAAA;IAC1B,IAAI,EAAE,OAAO,CAAA;IACb,qDAAqD;IACrD,WAAW,EAAE,MAAM,CAAA;IACnB,wDAAwD;IACxD,YAAY,EAAE,MAAM,CAAA;CACrB;AAID,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,IAAI,CAAC,EAAE,YAAY,GAAG,YAAY,CAAA;IAClC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,KAAK,EAAE,UAAU,EAAE,CAAA;IACnB,iBAAiB,EAAE,YAAY,EAAE,CAAA;IACjC,IAAI,EAAE,OAAO,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAID,MAAM,MAAM,iBAAiB,GACzB;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAC3C;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACvD;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,UAAU,CAAA;CAAE,GAC9E;IAAE,IAAI,EAAE,gBAAgB,CAAC;IAAC,MAAM,EAAE,aAAa,CAAA;CAAE,CAAA;AAIrD,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAA;IACV,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,UAAU,EAAE,CAAA;IACnB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE;QACP,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;QAChB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;QACf,IAAI,CAAC,EAAE,YAAY,GAAG,YAAY,CAAA;KACnC,CAAA;CACF;AAID,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;CACf;AAID,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,OAAO,CAAA;CACb;AAID,MAAM,WAAW,UAAU;IACzB,cAAc,EAAE,MAAM,CAAA;IACtB,SAAS,EAAE,WAAW,EAAE,CAAA;CACzB;AAID,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE;QACX,KAAK,EAAE,MAAM,CAAA;QACb,MAAM,EAAE,MAAM,CAAA;QACd,KAAK,EAAE,MAAM,CAAA;QACb,SAAS,CAAC,EAAE,MAAM,CAAA;KACnB,CAAA;IACD,SAAS,EAAE,MAAM,CAAA;IACjB,2FAA2F;IAC3F,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAA;CAChD;AAID,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,IAAI,CAAC,EAAE,YAAY,GAAG,YAAY,CAAA;CACnC"}

package/dist/generators/plugin-action-types.d.ts

@@ -0,0 +1,3 @@
+import { AgentProject } from '../agent-project/agent-project.js';
+export declare function generatePluginActionTypes(project: AgentProject): Promise<void>;
+//# sourceMappingURL=plugin-action-types.d.ts.map

package/dist/generators/plugin-action-types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"plugin-action-types.d.ts","sourceRoot":"","sources":["../../src/generators/plugin-action-types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAA;AAKhE,wBAAsB,yBAAyB,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAqCpF"}

package/dist/generators/plugin-types.d.ts

@@ -0,0 +1,17 @@
+import { ParsedPlugin } from '../plugins/types.js';
+declare const getPluginNames: (plugin: ParsedPlugin) => {
+    typings: {
+        index: string;
+        actions: string;
+    };
+    paths: {
+        index: string;
+        actions: string;
+    };
+};
+export declare function generatePluginTypes(plugin: ParsedPlugin): Promise<{
+    names: ReturnType<typeof getPluginNames>;
+    files: Record<string, string>;
+}>;
+export {};
+//# sourceMappingURL=plugin-types.d.ts.map

package/dist/generators/plugin-types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"plugin-types.d.ts","sourceRoot":"","sources":["../../src/generators/plugin-types.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAA;AA0DlD,QAAA,MAAM,cAAc,GAAI,QAAQ,YAAY;;;;;;;;;CAS1C,CAAA;AAEF,wBAAsB,mBAAmB,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC;IACvE,KAAK,EAAE,UAAU,CAAC,OAAO,cAAc,CAAC,CAAA;IACxC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC9B,CAAC,CAiED"}

package/dist/index.d.ts

@@ -5,13 +5,16 @@ export { workspaceCache } from './workspace/index.js';
 export { getRelativeTime } from './utils/time.js';
 export { stringifyWithOrder, orderKeys, agentInfoKeyOrder, dependenciesKeyOrder, integrationKeyOrder, } from './utils/json-ordering.js';
 export { AgentProject, ValidationErrors, ConfigWriter } from './agent-project/index.js';
-export { ConfigManager } from './config/index.js';
-export type { StoredConfig } from './config/index.js';
+export type { ConfigSchemaFieldUpdate } from './agent-project/index.js';
+export { ConfigManager, coerceConfigValue, getInnerTypeName } from './config/index.js';
+export type { StoredConfig, ConfigFieldDescriptor, SetResult } from './config/index.js';
 export { ValidationErrorCode, ValidationSeverity, ProjectState } from './agent-project/index.js';
 export type { AgentConfig, Dependencies, AgentInfo, ProjectInfo, ValidationResult, ValidationError, BuildOutput, BuildEvent, BuildEventType, FileChangeEvent, FileChangeType, WatchOptions, Integration, IntegrationChange, IntegrationSyncResult, } from './agent-project/index.js';
 export { AgentProjectGenerator } from './agent-init/index.js';
 export { IntegrationManager, IntegrationParser, IntegrationCache, IntegrationOperations, IntegrationChecker, IntegrationSyncManager, HubCache, } from './integrations/index.js';
 export type { IntegrationRef, IntegrationDefinition, ParsedIntegration, IntegrationValidationResult, IntegrationCheckResult, IntegrationInstallStatus, IntegrationRegistrationStatus, AddIntegrationOptions, AddIntegrationResult, UpgradeIntegrationOptions, RemoveIntegrationOptions, IntegrationVersionInfo, HubCacheEntry, HubCacheData, SyncSummary, } from './integrations/index.js';
+export { PluginParser } from './plugins/index.js';
+export type { PluginRef, PluginDependencyMapping, ParsedPlugin } from './plugins/index.js';
 export { InterfaceOperations, InterfaceManager, InterfaceParser, EnhancedInterfaceCache } from './interfaces/index.js';
 export type { AddInterfaceResult, InterfaceOperationsOptions, InterfaceManagerOptions } from './interfaces/index.js';
 export { AssetsManager } from './assets/index.js';
@@ -30,5 +33,8 @@ export type { WatchChangeType, FileChange, FileChangeEvent as FileWatcherChangeE
 export { PreflightChecker, PreflightFormatter } from './preflight/index.js';
 export type { PreflightCheckResult, PendingPreflightResult } from './preflight/index.js';
 export { ScriptRunner, runScript, setupTestRuntime } from './runner/index.js';
-export type { ScriptRunnerOptions, RunScriptOptions, TestRuntimeResult, SetupTestRuntimeOptions } from './runner/index.js';
+export type { ScriptRunnerOptions, RunScriptOptions, TestRuntimeResult, SetupTestRuntimeOptions, } from './runner/index.js';
+export { defineEval, runEval, runEvalSuite, loadEvalsFromDir, loadEvalByName, loadEvalFile, filterEvals, } from './eval/index.js';
+export { saveRunResult, loadRunResult, listRunResults, getLatestRun } from './eval/index.js';
+export type { EvalDefinition, EvalReport, EvalRunReport, EvalRunnerConfig, EvalFilter, EvalProgressEvent, GraderResult, TurnReport, } from './eval/index.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACzE,YAAY,EAAE,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAG9F,cAAc,qBAAqB,CAAA;AAEnC,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AACjD,OAAO,EACL,kBAAkB,EAClB,SAAS,EACT,iBAAiB,EACjB,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,0BAA0B,CAAA;AAGjC,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAA;AAGvF,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAA;AACjD,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AACrD,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAA;AAChG,YAAY,EACV,WAAW,EACX,YAAY,EACZ,SAAS,EACT,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,WAAW,EACX,UAAU,EACV,cAAc,EACd,eAAe,EACf,cAAc,EACd,YAAY,EACZ,WAAW,EACX,iBAAiB,EACjB,qBAAqB,GACtB,MAAM,0BAA0B,CAAA;AAGjC,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAG7D,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,gBAAgB,EAChB,qBAAqB,EACrB,kBAAkB,EAClB,sBAAsB,EACtB,QAAQ,GACT,MAAM,yBAAyB,CAAA;AAChC,YAAY,EACV,cAAc,EACd,qBAAqB,EACrB,iBAAiB,EACjB,2BAA2B,EAC3B,sBAAsB,EACtB,wBAAwB,EACxB,6BAA6B,EAC7B,qBAAqB,EACrB,oBAAoB,EACpB,yBAAyB,EACzB,wBAAwB,EACxB,sBAAsB,EACtB,aAAa,EACb,YAAY,EACZ,WAAW,GACZ,MAAM,yBAAyB,CAAA;AAGhC,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAA;AACtH,YAAY,EAAE,kBAAkB,EAAE,0BAA0B,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAA;AAGpH,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAA;AACjD,YAAY,EACV,SAAS,EACT,cAAc,EACd,aAAa,EACb,eAAe,EACf,aAAa,EACb,kBAAkB,EAClB,gBAAgB,EAChB,WAAW,GACZ,MAAM,mBAAmB,CAAA;AAG1B,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAA;AAC/F,OAAO,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAA;AAC5E,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAA;AAGtE,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAE1G,YAAY,EAAE,mBAAmB,EAAE,YAAY,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAA;AAG9G,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAChD,YAAY,EACV,mBAAmB,EACnB,aAAa,EACb,eAAe,EACf,gBAAgB,EAChB,aAAa,EACb,kBAAkB,EAClB,UAAU,EACV,WAAW,GACZ,MAAM,mBAAmB,CAAA;AAG1B,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AACzF,YAAY,EACV,uBAAuB,EACvB,UAAU,EACV,YAAY,EACZ,aAAa,EACb,UAAU,EACV,kBAAkB,EAClB,gBAAgB,EAChB,WAAW,GACZ,MAAM,sBAAsB,CAAA;AAG7B,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAA;AACrD,YAAY,EACV,eAAe,EACf,UAAU,EACV,eAAe,IAAI,sBAAsB,EACzC,kBAAkB,GACnB,MAAM,yBAAyB,CAAA;AAGhC,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAA;AAC3E,YAAY,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAA;AAGxF,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AAC7E,YAAY,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACzE,YAAY,EAAE,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAG9F,cAAc,qBAAqB,CAAA;AAEnC,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AACjD,OAAO,EACL,kBAAkB,EAClB,SAAS,EACT,iBAAiB,EACjB,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,0BAA0B,CAAA;AAGjC,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAA;AACvF,YAAY,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAA;AAGvE,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACtF,YAAY,EAAE,YAAY,EAAE,qBAAqB,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAA;AACvF,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAA;AAChG,YAAY,EACV,WAAW,EACX,YAAY,EACZ,SAAS,EACT,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,WAAW,EACX,UAAU,EACV,cAAc,EACd,eAAe,EACf,cAAc,EACd,YAAY,EACZ,WAAW,EACX,iBAAiB,EACjB,qBAAqB,GACtB,MAAM,0BAA0B,CAAA;AAGjC,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAG7D,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,gBAAgB,EAChB,qBAAqB,EACrB,kBAAkB,EAClB,sBAAsB,EACtB,QAAQ,GACT,MAAM,yBAAyB,CAAA;AAChC,YAAY,EACV,cAAc,EACd,qBAAqB,EACrB,iBAAiB,EACjB,2BAA2B,EAC3B,sBAAsB,EACtB,wBAAwB,EACxB,6BAA6B,EAC7B,qBAAqB,EACrB,oBAAoB,EACpB,yBAAyB,EACzB,wBAAwB,EACxB,sBAAsB,EACtB,aAAa,EACb,YAAY,EACZ,WAAW,GACZ,MAAM,yBAAyB,CAAA;AAGhC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACjD,YAAY,EAAE,SAAS,EAAE,uBAAuB,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AAG1F,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAA;AACtH,YAAY,EAAE,kBAAkB,EAAE,0BAA0B,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAA;AAGpH,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAA;AACjD,YAAY,EACV,SAAS,EACT,cAAc,EACd,aAAa,EACb,eAAe,EACf,aAAa,EACb,kBAAkB,EAClB,gBAAgB,EAChB,WAAW,GACZ,MAAM,mBAAmB,CAAA;AAG1B,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAA;AAC/F,OAAO,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAA;AAC5E,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAA;AAGtE,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAE1G,YAAY,EAAE,mBAAmB,EAAE,YAAY,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAA;AAG9G,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAChD,YAAY,EACV,mBAAmB,EACnB,aAAa,EACb,eAAe,EACf,gBAAgB,EAChB,aAAa,EACb,kBAAkB,EAClB,UAAU,EACV,WAAW,GACZ,MAAM,mBAAmB,CAAA;AAG1B,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AACzF,YAAY,EACV,uBAAuB,EACvB,UAAU,EACV,YAAY,EACZ,aAAa,EACb,UAAU,EACV,kBAAkB,EAClB,gBAAgB,EAChB,WAAW,GACZ,MAAM,sBAAsB,CAAA;AAG7B,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAA;AACrD,YAAY,EACV,eAAe,EACf,UAAU,EACV,eAAe,IAAI,sBAAsB,EACzC,kBAAkB,GACnB,MAAM,yBAAyB,CAAA;AAGhC,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAA;AAC3E,YAAY,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAA;AAGxF,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AAC7E,YAAY,EACV,mBAAmB,EACnB,gBAAgB,EAChB,iBAAiB,EACjB,uBAAuB,GACxB,MAAM,mBAAmB,CAAA;AAG1B,OAAO,EACL,UAAU,EACV,OAAO,EACP,YAAY,EACZ,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,WAAW,GACZ,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC5F,YAAY,EACV,cAAc,EACd,UAAU,EACV,aAAa,EACb,gBAAgB,EAChB,UAAU,EACV,iBAAiB,EACjB,YAAY,EACZ,UAAU,GACX,MAAM,iBAAiB,CAAA"}