@vauban-org/agent-sdk 0.16.0

package/dist/errors.d.ts

@@ -0,0 +1,102 @@
+/**
+ * Typed port errors (Sprint-459).
+ *
+ * Agent handlers can pattern-match on concrete classes to implement
+ * intelligent retry, HITL escalation, or graceful degradation:
+ *
+ *   try {
+ *     await deps.brain.archiveKnowledge(entry);
+ *   } catch (err) {
+ *     if (err instanceof BrainRateLimit) {
+ *       await sleep(err.retryAfterMs);
+ *       return retry();
+ *     }
+ *     if (err instanceof BrainUnavailable) {
+ *       return { skipped: true, reason: "brain down" };
+ *     }
+ *     throw err;
+ *   }
+ *
+ * All port-error classes extend `PortError` for a single instanceof check
+ * when agents want to distinguish port failures from tool / LLM failures.
+ *
+ * Errors carry a `cause` field (native Error cause) so the underlying
+ * exception remains debuggable; and a `retryable` boolean so orchestrators
+ * can apply generic retry policies without class-specific knowledge.
+ */
+/** Base class for every port-layer error thrown by an SDK-injected port. */
+export declare abstract class PortError extends Error {
+    /** Port identifier that raised the error (e.g. "brain", "db"). */
+    abstract readonly port: string;
+    /** True if the caller MAY retry after some backoff. */
+    readonly retryable: boolean;
+    constructor(message: string, options?: {
+        cause?: unknown;
+        retryable?: boolean;
+    });
+}
+export declare class BrainUnavailable extends PortError {
+    readonly port = "brain";
+    constructor(message?: string, cause?: unknown);
+}
+export declare class BrainRateLimit extends PortError {
+    readonly port = "brain";
+    /** Server-advised wait before retry. Missing if unknown. */
+    readonly retryAfterMs?: number;
+    constructor(opts?: {
+        message?: string;
+        retryAfterMs?: number;
+        cause?: unknown;
+    });
+}
+export declare class BrainValidationError extends PortError {
+    readonly port = "brain";
+    constructor(message: string, cause?: unknown);
+}
+export declare class BrainAuthError extends PortError {
+    readonly port = "brain";
+    constructor(message?: string, cause?: unknown);
+}
+export declare class DbConnectionLost extends PortError {
+    readonly port = "db";
+    constructor(message?: string, cause?: unknown);
+}
+export declare class DbQueryError extends PortError {
+    readonly port = "db";
+    /** The SQL fragment that failed (first 200 chars). */
+    readonly sqlPreview?: string;
+    constructor(opts: {
+        message: string;
+        cause?: unknown;
+        sqlPreview?: string;
+    });
+}
+export declare class OutcomeAttributionFailed extends PortError {
+    readonly port = "outcome";
+    /** agent_run.id that failed to attribute. */
+    readonly runId: string;
+    constructor(runId: string, message: string, cause?: unknown);
+}
+/**
+ * Thrown by `triggerCycle()` when `opts.dryRun` is not explicitly set.
+ *
+ * Audit-ready agents must declare execution intent at every call site.
+ * Passing `{dryRun: true}` for non-destructive runs, `{dryRun: false}` for
+ * live execution, prevents accidental side-effect execution.
+ *
+ * @public
+ */
+export declare class MissingDryRunFlagError extends Error {
+    constructor();
+}
+/**
+ * True if the error is a PortError produced by the named port.
+ * Convenience for narrow error handling without leaking class imports.
+ */
+export declare function isPortError(err: unknown, port?: string): err is PortError;
+/**
+ * True if the error is a PortError and the originating port marked it
+ * retryable. Returns false for unknown errors — caller should rethrow.
+ */
+export declare function isRetryablePortError(err: unknown): err is PortError;
+//# sourceMappingURL=errors.d.ts.map

package/dist/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,4EAA4E;AAC5E,8BAAsB,SAAU,SAAQ,KAAK;IAC3C,kEAAkE;IAClE,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAC/B,uDAAuD;IACvD,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;gBAG1B,OAAO,EAAE,MAAM,EACf,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAO;CAWzD;AAID,qBAAa,gBAAiB,SAAQ,SAAS;IAC7C,QAAQ,CAAC,IAAI,WAAW;gBACZ,OAAO,SAA8B,EAAE,KAAK,CAAC,EAAE,OAAO;CAGnE;AAED,qBAAa,cAAe,SAAQ,SAAS;IAC3C,QAAQ,CAAC,IAAI,WAAW;IACxB,4DAA4D;IAC5D,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;gBAE7B,IAAI,GAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAA;KAAO;CAQ1E;AAED,qBAAa,oBAAqB,SAAQ,SAAS;IACjD,QAAQ,CAAC,IAAI,WAAW;gBACZ,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO;CAG7C;AAED,qBAAa,cAAe,SAAQ,SAAS;IAC3C,QAAQ,CAAC,IAAI,WAAW;gBACZ,OAAO,SAAwB,EAAE,KAAK,CAAC,EAAE,OAAO;CAG7D;AAID,qBAAa,gBAAiB,SAAQ,SAAS;IAC7C,QAAQ,CAAC,IAAI,QAAQ;gBACT,OAAO,SAA6B,EAAE,KAAK,CAAC,EAAE,OAAO;CAGlE;AAED,qBAAa,YAAa,SAAQ,SAAS;IACzC,QAAQ,CAAC,IAAI,QAAQ;IACrB,sDAAsD;IACtD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;gBACjB,IAAI,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE;CAI5E;AAID,qBAAa,wBAAyB,SAAQ,SAAS;IACrD,QAAQ,CAAC,IAAI,aAAa;IAC1B,6CAA6C;IAC7C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;gBACX,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO;CAI5D;AAMD;;;;;;;;GAQG;AACH,qBAAa,sBAAuB,SAAQ,KAAK;;CAOhD;AAID;;;GAGG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,GAAG,IAAI,SAAS,CAIzE;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,OAAO,GAAG,GAAG,IAAI,SAAS,CAEnE"}

package/dist/errors.js

@@ -0,0 +1,135 @@
+/**
+ * Typed port errors (Sprint-459).
+ *
+ * Agent handlers can pattern-match on concrete classes to implement
+ * intelligent retry, HITL escalation, or graceful degradation:
+ *
+ *   try {
+ *     await deps.brain.archiveKnowledge(entry);
+ *   } catch (err) {
+ *     if (err instanceof BrainRateLimit) {
+ *       await sleep(err.retryAfterMs);
+ *       return retry();
+ *     }
+ *     if (err instanceof BrainUnavailable) {
+ *       return { skipped: true, reason: "brain down" };
+ *     }
+ *     throw err;
+ *   }
+ *
+ * All port-error classes extend `PortError` for a single instanceof check
+ * when agents want to distinguish port failures from tool / LLM failures.
+ *
+ * Errors carry a `cause` field (native Error cause) so the underlying
+ * exception remains debuggable; and a `retryable` boolean so orchestrators
+ * can apply generic retry policies without class-specific knowledge.
+ */
+/** Base class for every port-layer error thrown by an SDK-injected port. */
+export class PortError extends Error {
+    /** True if the caller MAY retry after some backoff. */
+    retryable;
+    constructor(message, options = {}) {
+        super(message);
+        this.name = new.target.name;
+        this.retryable = options.retryable ?? false;
+        // Ensure `.cause` is preserved across node versions.
+        if (options.cause !== undefined) {
+            this.cause = options.cause;
+        }
+        Object.setPrototypeOf(this, new.target.prototype);
+    }
+}
+// ─── Brain port errors ────────────────────────────────────────────────────
+export class BrainUnavailable extends PortError {
+    port = "brain";
+    constructor(message = "Brain service unreachable", cause) {
+        super(message, { cause, retryable: true });
+    }
+}
+export class BrainRateLimit extends PortError {
+    port = "brain";
+    /** Server-advised wait before retry. Missing if unknown. */
+    retryAfterMs;
+    constructor(opts = {}) {
+        super(opts.message ?? "Brain rate limit exceeded", {
+            cause: opts.cause,
+            retryable: true,
+        });
+        this.retryAfterMs = opts.retryAfterMs;
+    }
+}
+export class BrainValidationError extends PortError {
+    port = "brain";
+    constructor(message, cause) {
+        super(message, { cause, retryable: false });
+    }
+}
+export class BrainAuthError extends PortError {
+    port = "brain";
+    constructor(message = "Brain auth rejected", cause) {
+        super(message, { cause, retryable: false });
+    }
+}
+// ─── Db port errors ───────────────────────────────────────────────────────
+export class DbConnectionLost extends PortError {
+    port = "db";
+    constructor(message = "Database connection lost", cause) {
+        super(message, { cause, retryable: true });
+    }
+}
+export class DbQueryError extends PortError {
+    port = "db";
+    /** The SQL fragment that failed (first 200 chars). */
+    sqlPreview;
+    constructor(opts) {
+        super(opts.message, { cause: opts.cause, retryable: false });
+        this.sqlPreview = opts.sqlPreview?.slice(0, 200);
+    }
+}
+// ─── Outcome port errors ──────────────────────────────────────────────────
+export class OutcomeAttributionFailed extends PortError {
+    port = "outcome";
+    /** agent_run.id that failed to attribute. */
+    runId;
+    constructor(runId, message, cause) {
+        super(message, { cause, retryable: false });
+        this.runId = runId;
+    }
+}
+// ─── Helpers ──────────────────────────────────────────────────────────────
+// ─── OODA audit errors ────────────────────────────────────────────────────
+/**
+ * Thrown by `triggerCycle()` when `opts.dryRun` is not explicitly set.
+ *
+ * Audit-ready agents must declare execution intent at every call site.
+ * Passing `{dryRun: true}` for non-destructive runs, `{dryRun: false}` for
+ * live execution, prevents accidental side-effect execution.
+ *
+ * @public
+ */
+export class MissingDryRunFlagError extends Error {
+    constructor() {
+        super("triggerCycle() requires explicit dryRun flag (true|false). Audit-ready agents must declare execution intent.");
+        this.name = "MissingDryRunFlagError";
+    }
+}
+// ─── Helpers ──────────────────────────────────────────────────────────────
+/**
+ * True if the error is a PortError produced by the named port.
+ * Convenience for narrow error handling without leaking class imports.
+ */
+export function isPortError(err, port) {
+    if (!(err instanceof PortError))
+        return false;
+    if (port !== undefined && err.port !== port)
+        return false;
+    return true;
+}
+/**
+ * True if the error is a PortError and the originating port marked it
+ * retryable. Returns false for unknown errors — caller should rethrow.
+ */
+export function isRetryablePortError(err) {
+    return err instanceof PortError && err.retryable;
+}
+//# sourceMappingURL=errors.js.map

package/dist/errors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,4EAA4E;AAC5E,MAAM,OAAgB,SAAU,SAAQ,KAAK;IAG3C,uDAAuD;IAC9C,SAAS,CAAU;IAE5B,YACE,OAAe,EACf,UAAoD,EAAE;QAEtD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;QAC5B,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;QAC5C,qDAAqD;QACrD,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC/B,IAA4B,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QACtD,CAAC;QACD,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC;CACF;AAED,6EAA6E;AAE7E,MAAM,OAAO,gBAAiB,SAAQ,SAAS;IACpC,IAAI,GAAG,OAAO,CAAC;IACxB,YAAY,OAAO,GAAG,2BAA2B,EAAE,KAAe;QAChE,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7C,CAAC;CACF;AAED,MAAM,OAAO,cAAe,SAAQ,SAAS;IAClC,IAAI,GAAG,OAAO,CAAC;IACxB,4DAA4D;IACnD,YAAY,CAAU;IAC/B,YACE,OAAqE,EAAE;QAEvE,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,2BAA2B,EAAE;YACjD,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QACH,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;IACxC,CAAC;CACF;AAED,MAAM,OAAO,oBAAqB,SAAQ,SAAS;IACxC,IAAI,GAAG,OAAO,CAAC;IACxB,YAAY,OAAe,EAAE,KAAe;QAC1C,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9C,CAAC;CACF;AAED,MAAM,OAAO,cAAe,SAAQ,SAAS;IAClC,IAAI,GAAG,OAAO,CAAC;IACxB,YAAY,OAAO,GAAG,qBAAqB,EAAE,KAAe;QAC1D,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9C,CAAC;CACF;AAED,6EAA6E;AAE7E,MAAM,OAAO,gBAAiB,SAAQ,SAAS;IACpC,IAAI,GAAG,IAAI,CAAC;IACrB,YAAY,OAAO,GAAG,0BAA0B,EAAE,KAAe;QAC/D,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7C,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,SAAS;IAChC,IAAI,GAAG,IAAI,CAAC;IACrB,sDAAsD;IAC7C,UAAU,CAAU;IAC7B,YAAY,IAA+D;QACzE,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACnD,CAAC;CACF;AAED,6EAA6E;AAE7E,MAAM,OAAO,wBAAyB,SAAQ,SAAS;IAC5C,IAAI,GAAG,SAAS,CAAC;IAC1B,6CAA6C;IACpC,KAAK,CAAS;IACvB,YAAY,KAAa,EAAE,OAAe,EAAE,KAAe;QACzD,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;CACF;AAED,6EAA6E;AAE7E,6EAA6E;AAE7E;;;;;;;;GAQG;AACH,MAAM,OAAO,sBAAuB,SAAQ,KAAK;IAC/C;QACE,KAAK,CACH,8GAA8G,CAC/G,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;IACvC,CAAC;CACF;AAED,6EAA6E;AAE7E;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,GAAY,EAAE,IAAa;IACrD,IAAI,CAAC,CAAC,GAAG,YAAY,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC;IAC9C,IAAI,IAAI,KAAK,SAAS,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC1D,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,GAAY;IAC/C,OAAO,GAAG,YAAY,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC;AACnD,CAAC"}

package/dist/evals/datasets/index.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Synthetic eval dataset — 30 tasks, product-agnostic.
+ * Sprint-579: eval-dataset-30.
+ *
+ * Categories: math (6), code (6), classification (6),
+ *             summarization (6), reasoning (6).
+ * Ground truth labels manually validated.
+ */
+export type EvalCategory = "math" | "code" | "classification" | "summarization" | "reasoning";
+export type EvalDifficulty = "easy" | "medium" | "hard";
+export interface EvalTask {
+    id: string;
+    category: EvalCategory;
+    difficulty: EvalDifficulty;
+    /** Prompt sent to the model under evaluation. */
+    prompt: string;
+    /** Ground truth — exact match or list of accepted values. */
+    groundTruth: string | string[];
+    /** Optional: judge function id for non-exact-match scoring. */
+    scorer?: "exact" | "numeric_tolerance" | "contains" | "code_output";
+    /** Tolerance for numeric_tolerance scorer. */
+    tolerance?: number;
+}
+export declare const EVAL_DATASET_30: EvalTask[];
+export declare function getTasksByCategory(category: EvalCategory): EvalTask[];
+export declare function getTasksByDifficulty(difficulty: EvalDifficulty): EvalTask[];
+//# sourceMappingURL=index.d.ts.map

package/dist/evals/datasets/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/evals/datasets/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,MAAM,MAAM,YAAY,GACpB,MAAM,GACN,MAAM,GACN,gBAAgB,GAChB,eAAe,GACf,WAAW,CAAC;AAEhB,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;AAExD,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,YAAY,CAAC;IACvB,UAAU,EAAE,cAAc,CAAC;IAC3B,iDAAiD;IACjD,MAAM,EAAE,MAAM,CAAC;IACf,6DAA6D;IAC7D,WAAW,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC/B,+DAA+D;IAC/D,MAAM,CAAC,EAAE,OAAO,GAAG,mBAAmB,GAAG,UAAU,GAAG,aAAa,CAAC;IACpE,8CAA8C;IAC9C,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AA0SD,eAAO,MAAM,eAAe,EAAE,QAAQ,EAMrC,CAAC;AAEF,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,YAAY,GAAG,QAAQ,EAAE,CAErE;AAED,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,cAAc,GAAG,QAAQ,EAAE,CAE3E"}

package/dist/evals/datasets/index.js

@@ -0,0 +1,279 @@
+/**
+ * Synthetic eval dataset — 30 tasks, product-agnostic.
+ * Sprint-579: eval-dataset-30.
+ *
+ * Categories: math (6), code (6), classification (6),
+ *             summarization (6), reasoning (6).
+ * Ground truth labels manually validated.
+ */
+// ─── Math (6) ────────────────────────────────────────────────────────────────
+const mathTasks = [
+    {
+        id: "math-001",
+        category: "math",
+        difficulty: "easy",
+        prompt: "What is 17 × 23?",
+        groundTruth: "391",
+        scorer: "exact",
+    },
+    {
+        id: "math-002",
+        category: "math",
+        difficulty: "easy",
+        prompt: "What is the square root of 144?",
+        groundTruth: "12",
+        scorer: "exact",
+    },
+    {
+        id: "math-003",
+        category: "math",
+        difficulty: "medium",
+        prompt: "A train travels at 80 km/h. How many kilometers does it cover in 2 hours and 45 minutes?",
+        groundTruth: "220",
+        scorer: "numeric_tolerance",
+        tolerance: 0.5,
+    },
+    {
+        id: "math-004",
+        category: "math",
+        difficulty: "medium",
+        prompt: "What is the sum of the first 10 positive odd numbers?",
+        groundTruth: "100",
+        scorer: "exact",
+    },
+    {
+        id: "math-005",
+        category: "math",
+        difficulty: "hard",
+        prompt: "If f(x) = 3x² − 2x + 1, what is f(4)?",
+        groundTruth: "41",
+        scorer: "exact",
+    },
+    {
+        id: "math-006",
+        category: "math",
+        difficulty: "hard",
+        prompt: "A rectangle has a perimeter of 56 cm and a length that is twice its width. What is the area in cm²?",
+        groundTruth: ["196", "196 cm²", "196cm²"],
+        scorer: "contains",
+    },
+];
+// ─── Code generation (6) ─────────────────────────────────────────────────────
+const codeTasks = [
+    {
+        id: "code-001",
+        category: "code",
+        difficulty: "easy",
+        prompt: "Write a Python function `is_palindrome(s: str) -> bool` that returns True if the string is a palindrome (case-insensitive, ignore spaces).",
+        groundTruth: "is_palindrome",
+        scorer: "contains",
+    },
+    {
+        id: "code-002",
+        category: "code",
+        difficulty: "easy",
+        prompt: "Write a TypeScript function `sum(arr: number[]): number` that returns the sum of all elements.",
+        groundTruth: "reduce",
+        scorer: "contains",
+    },
+    {
+        id: "code-003",
+        category: "code",
+        difficulty: "medium",
+        prompt: "Write a Python function `fibonacci(n: int) -> list[int]` that returns the first n Fibonacci numbers. fibonacci(8) should return [0,1,1,2,3,5,8,13].",
+        groundTruth: "[0, 1, 1, 2, 3, 5, 8, 13]",
+        scorer: "contains",
+    },
+    {
+        id: "code-004",
+        category: "code",
+        difficulty: "medium",
+        prompt: "Write a TypeScript function `groupBy<T>(arr: T[], keyFn: (item: T) => string): Record<string, T[]>` that groups array elements by a key function.",
+        groundTruth: "Record",
+        scorer: "contains",
+    },
+    {
+        id: "code-005",
+        category: "code",
+        difficulty: "hard",
+        prompt: "Write a Python function `lru_cache_impl(capacity: int)` that returns an LRU cache object with `get(key)` and `put(key, value)` methods. Use an OrderedDict.",
+        groundTruth: "OrderedDict",
+        scorer: "contains",
+    },
+    {
+        id: "code-006",
+        category: "code",
+        difficulty: "hard",
+        prompt: "Implement a TypeScript `debounce<T extends (...args: unknown[]) => void>(fn: T, delay: number): T` function.",
+        groundTruth: "setTimeout",
+        scorer: "contains",
+    },
+];
+// ─── Classification (6) ──────────────────────────────────────────────────────
+const classificationTasks = [
+    {
+        id: "class-001",
+        category: "classification",
+        difficulty: "easy",
+        prompt: "Classify the sentiment of this text as POSITIVE, NEGATIVE, or NEUTRAL:\n\"The product arrived on time and works perfectly.\"",
+        groundTruth: "POSITIVE",
+        scorer: "contains",
+    },
+    {
+        id: "class-002",
+        category: "classification",
+        difficulty: "easy",
+        prompt: "Is the following email spam or not spam?\n\"Congratulations! You've been selected for a $1,000 gift card. Click here to claim.\"",
+        groundTruth: "spam",
+        scorer: "contains",
+    },
+    {
+        id: "class-003",
+        category: "classification",
+        difficulty: "medium",
+        prompt: "Classify this news headline into one of: POLITICS, TECHNOLOGY, SPORTS, FINANCE, HEALTH.\n\"Central bank raises interest rates by 25 basis points amid inflation concerns.\"",
+        groundTruth: "FINANCE",
+        scorer: "contains",
+    },
+    {
+        id: "class-004",
+        category: "classification",
+        difficulty: "medium",
+        prompt: "Is this code comment a TODO, a FIXME, or neither?\n\"// This should be refactored once the API stabilizes.\"",
+        groundTruth: ["TODO", "todo"],
+        scorer: "contains",
+    },
+    {
+        id: "class-005",
+        category: "classification",
+        difficulty: "hard",
+        prompt: "Classify the programming language: `let x: i32 = 42; fn main() { println!(\"{}\", x); }`",
+        groundTruth: "Rust",
+        scorer: "contains",
+    },
+    {
+        id: "class-006",
+        category: "classification",
+        difficulty: "hard",
+        prompt: "Classify the logical fallacy in: \"We should trust Dr. Smith's diet advice — he has a PhD in biochemistry.\"\nChoose from: AD_HOMINEM, APPEAL_TO_AUTHORITY, STRAW_MAN, FALSE_DICHOTOMY",
+        groundTruth: "APPEAL_TO_AUTHORITY",
+        scorer: "contains",
+    },
+];
+// ─── Summarization (6) ───────────────────────────────────────────────────────
+const summarizationTasks = [
+    {
+        id: "sum-001",
+        category: "summarization",
+        difficulty: "easy",
+        prompt: "Summarize in one sentence:\n\"The Eiffel Tower is a wrought-iron lattice tower in Paris, France. It was designed by Gustave Eiffel and built between 1887 and 1889 as the entrance arch to the 1889 World's Fair. It stands 330 metres tall and attracts millions of tourists each year.\"",
+        groundTruth: ["Eiffel", "Paris", "1889"],
+        scorer: "contains",
+    },
+    {
+        id: "sum-002",
+        category: "summarization",
+        difficulty: "easy",
+        prompt: "Extract the key action from this sentence in 5 words or fewer:\n\"The board unanimously voted to approve the merger with Acme Corp after reviewing the Q3 financials.\"",
+        groundTruth: ["approved", "merger", "voted"],
+        scorer: "contains",
+    },
+    {
+        id: "sum-003",
+        category: "summarization",
+        difficulty: "medium",
+        prompt: "Summarize the following code in one sentence describing what it does:\n```python\ndef fn(lst):\n    return [x for x in lst if x % 2 == 0]\n```",
+        groundTruth: ["even", "filter"],
+        scorer: "contains",
+    },
+    {
+        id: "sum-004",
+        category: "summarization",
+        difficulty: "medium",
+        prompt: "Give a one-line TL;DR for this commit message:\n\"fix: prevent null pointer exception in UserService.getProfile when user has no associated organization by adding optional chaining and a default empty object fallback\"",
+        groundTruth: ["null", "UserService", "optional"],
+        scorer: "contains",
+    },
+    {
+        id: "sum-005",
+        category: "summarization",
+        difficulty: "hard",
+        prompt: "Summarize the main thesis of this argument in one sentence:\n\"Critics of universal basic income argue that giving people money without work requirements removes the incentive to be productive, may lead to inflation as more money chases the same goods, and could be prohibitively expensive for governments to sustain in the long term.\"",
+        groundTruth: ["incentive", "inflation", "expensive"],
+        scorer: "contains",
+    },
+    {
+        id: "sum-006",
+        category: "summarization",
+        difficulty: "hard",
+        prompt: "In ≤20 words, capture the decision made in this ADR:\n\"We chose PostgreSQL over MongoDB because our data has clear relational structure, we need ACID guarantees for financial transactions, and the team has strong PostgreSQL expertise.\"",
+        groundTruth: ["PostgreSQL", "MongoDB"],
+        scorer: "contains",
+    },
+];
+// ─── Reasoning multi-step (6) ────────────────────────────────────────────────
+const reasoningTasks = [
+    {
+        id: "reason-001",
+        category: "reasoning",
+        difficulty: "easy",
+        prompt: "Alice is taller than Bob. Bob is taller than Carol. Is Alice taller than Carol? Answer YES or NO.",
+        groundTruth: "YES",
+        scorer: "contains",
+    },
+    {
+        id: "reason-002",
+        category: "reasoning",
+        difficulty: "easy",
+        prompt: "A store sells apples for €1.50 each and gives a 10% discount if you buy 5 or more. How much does it cost to buy exactly 6 apples?",
+        groundTruth: ["8.10", "8,10"],
+        scorer: "contains",
+    },
+    {
+        id: "reason-003",
+        category: "reasoning",
+        difficulty: "medium",
+        prompt: "You have a 3-litre jug and a 5-litre jug. You need exactly 4 litres. Describe the minimum number of steps needed. How many steps?",
+        groundTruth: ["6", "7"],
+        scorer: "contains",
+    },
+    {
+        id: "reason-004",
+        category: "reasoning",
+        difficulty: "medium",
+        prompt: "If all Bloops are Razzies, and all Razzies are Lazzies, are all Bloops definitely Lazzies? Answer YES or NO and give one-line reasoning.",
+        groundTruth: "YES",
+        scorer: "contains",
+    },
+    {
+        id: "reason-005",
+        category: "reasoning",
+        difficulty: "hard",
+        prompt: "A function has O(n log n) time complexity. If it takes 10ms for n=1000, approximately how many milliseconds will it take for n=1,000,000? Round to the nearest integer.",
+        groundTruth: ["20000", "20,000"],
+        scorer: "contains",
+    },
+    {
+        id: "reason-006",
+        category: "reasoning",
+        difficulty: "hard",
+        prompt: "Three switches control three bulbs in the next room. You can flip switches as many times as you like, but you can only enter the room once. Describe a strategy to identify which switch controls which bulb.",
+        groundTruth: ["heat", "warm", "on"],
+        scorer: "contains",
+    },
+];
+// ─── Full dataset ─────────────────────────────────────────────────────────────
+export const EVAL_DATASET_30 = [
+    ...mathTasks,
+    ...codeTasks,
+    ...classificationTasks,
+    ...summarizationTasks,
+    ...reasoningTasks,
+];
+export function getTasksByCategory(category) {
+    return EVAL_DATASET_30.filter((t) => t.category === category);
+}
+export function getTasksByDifficulty(difficulty) {
+    return EVAL_DATASET_30.filter((t) => t.difficulty === difficulty);
+}
+//# sourceMappingURL=index.js.map

package/dist/evals/datasets/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/evals/datasets/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAyBH,gFAAgF;AAEhF,MAAM,SAAS,GAAe;IAC5B;QACE,EAAE,EAAE,UAAU;QACd,QAAQ,EAAE,MAAM;QAChB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,kBAAkB;QAC1B,WAAW,EAAE,KAAK;QAClB,MAAM,EAAE,OAAO;KAChB;IACD;QACE,EAAE,EAAE,UAAU;QACd,QAAQ,EAAE,MAAM;QAChB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,iCAAiC;QACzC,WAAW,EAAE,IAAI;QACjB,MAAM,EAAE,OAAO;KAChB;IACD;QACE,EAAE,EAAE,UAAU;QACd,QAAQ,EAAE,MAAM;QAChB,UAAU,EAAE,QAAQ;QACpB,MAAM,EACJ,0FAA0F;QAC5F,WAAW,EAAE,KAAK;QAClB,MAAM,EAAE,mBAAmB;QAC3B,SAAS,EAAE,GAAG;KACf;IACD;QACE,EAAE,EAAE,UAAU;QACd,QAAQ,EAAE,MAAM;QAChB,UAAU,EAAE,QAAQ;QACpB,MAAM,EACJ,uDAAuD;QACzD,WAAW,EAAE,KAAK;QAClB,MAAM,EAAE,OAAO;KAChB;IACD;QACE,EAAE,EAAE,UAAU;QACd,QAAQ,EAAE,MAAM;QAChB,UAAU,EAAE,MAAM;QAClB,MAAM,EACJ,uCAAuC;QACzC,WAAW,EAAE,IAAI;QACjB,MAAM,EAAE,OAAO;KAChB;IACD;QACE,EAAE,EAAE,UAAU;QACd,QAAQ,EAAE,MAAM;QAChB,UAAU,EAAE,MAAM;QAClB,MAAM,EACJ,qGAAqG;QACvG,WAAW,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC;QACzC,MAAM,EAAE,UAAU;KACnB;CACF,CAAC;AAEF,gFAAgF;AAEhF,MAAM,SAAS,GAAe;IAC5B;QACE,EAAE,EAAE,UAAU;QACd,QAAQ,EAAE,MAAM;QAChB,UAAU,EAAE,MAAM;QAClB,MAAM,EACJ,4IAA4I;QAC9I,WAAW,EAAE,eAAe;QAC5B,MAAM,EAAE,UAAU;KACnB;IACD;QACE,EAAE,EAAE,UAAU;QACd,QAAQ,EAAE,MAAM;QAChB,UAAU,EAAE,MAAM;QAClB,MAAM,EACJ,gGAAgG;QAClG,WAAW,EAAE,QAAQ;QACrB,MAAM,EAAE,UAAU;KACnB;IACD;QACE,EAAE,EAAE,UAAU;QACd,QAAQ,EAAE,MAAM;QAChB,UAAU,EAAE,QAAQ;QACpB,MAAM,EACJ,qJAAqJ;QACvJ,WAAW,EAAE,2BAA2B;QACxC,MAAM,EAAE,UAAU;KACnB;IACD;QACE,EAAE,EAAE,UAAU;QACd,QAAQ,EAAE,MAAM;QAChB,UAAU,EAAE,QAAQ;QACpB,MAAM,EACJ,mJAAmJ;QACrJ,WAAW,EAAE,QAAQ;QACrB,MAAM,EAAE,UAAU;KACnB;IACD;QACE,EAAE,EAAE,UAAU;QACd,QAAQ,EAAE,MAAM;QAChB,UAAU,EAAE,MAAM;QAClB,MAAM,EACJ,6JAA6J;QAC/J,WAAW,EAAE,aAAa;QAC1B,MAAM,EAAE,UAAU;KACnB;IACD;QACE,EAAE,EAAE,UAAU;QACd,QAAQ,EAAE,MAAM;QAChB,UAAU,EAAE,MAAM;QAClB,MAAM,EACJ,8GAA8G;QAChH,WAAW,EAAE,YAAY;QACzB,MAAM,EAAE,UAAU;KACnB;CACF,CAAC;AAEF,gFAAgF;AAEhF,MAAM,mBAAmB,GAAe;IACtC;QACE,EAAE,EAAE,WAAW;QACf,QAAQ,EAAE,gBAAgB;QAC1B,UAAU,EAAE,MAAM;QAClB,MAAM,EACJ,8HAA8H;QAChI,WAAW,EAAE,UAAU;QACvB,MAAM,EAAE,UAAU;KACnB;IACD;QACE,EAAE,EAAE,WAAW;QACf,QAAQ,EAAE,gBAAgB;QAC1B,UAAU,EAAE,MAAM;QAClB,MAAM,EACJ,kIAAkI;QACpI,WAAW,EAAE,MAAM;QACnB,MAAM,EAAE,UAAU;KACnB;IACD;QACE,EAAE,EAAE,WAAW;QACf,QAAQ,EAAE,gBAAgB;QAC1B,UAAU,EAAE,QAAQ;QACpB,MAAM,EACJ,6KAA6K;QAC/K,WAAW,EAAE,SAAS;QACtB,MAAM,EAAE,UAAU;KACnB;IACD;QACE,EAAE,EAAE,WAAW;QACf,QAAQ,EAAE,gBAAgB;QAC1B,UAAU,EAAE,QAAQ;QACpB,MAAM,EACJ,8GAA8G;QAChH,WAAW,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;QAC7B,MAAM,EAAE,UAAU;KACnB;IACD;QACE,EAAE,EAAE,WAAW;QACf,QAAQ,EAAE,gBAAgB;QAC1B,UAAU,EAAE,MAAM;QAClB,MAAM,EACJ,0FAA0F;QAC5F,WAAW,EAAE,MAAM;QACnB,MAAM,EAAE,UAAU;KACnB;IACD;QACE,EAAE,EAAE,WAAW;QACf,QAAQ,EAAE,gBAAgB;QAC1B,UAAU,EAAE,MAAM;QAClB,MAAM,EACJ,wLAAwL;QAC1L,WAAW,EAAE,qBAAqB;QAClC,MAAM,EAAE,UAAU;KACnB;CACF,CAAC;AAEF,gFAAgF;AAEhF,MAAM,kBAAkB,GAAe;IACrC;QACE,EAAE,EAAE,SAAS;QACb,QAAQ,EAAE,eAAe;QACzB,UAAU,EAAE,MAAM;QAClB,MAAM,EACJ,4RAA4R;QAC9R,WAAW,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC;QACxC,MAAM,EAAE,UAAU;KACnB;IACD;QACE,EAAE,EAAE,SAAS;QACb,QAAQ,EAAE,eAAe;QACzB,UAAU,EAAE,MAAM;QAClB,MAAM,EACJ,yKAAyK;QAC3K,WAAW,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC;QAC5C,MAAM,EAAE,UAAU;KACnB;IACD;QACE,EAAE,EAAE,SAAS;QACb,QAAQ,EAAE,eAAe;QACzB,UAAU,EAAE,QAAQ;QACpB,MAAM,EACJ,gJAAgJ;QAClJ,WAAW,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC;QAC/B,MAAM,EAAE,UAAU;KACnB;IACD;QACE,EAAE,EAAE,SAAS;QACb,QAAQ,EAAE,eAAe;QACzB,UAAU,EAAE,QAAQ;QACpB,MAAM,EACJ,4NAA4N;QAC9N,WAAW,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,UAAU,CAAC;QAChD,MAAM,EAAE,UAAU;KACnB;IACD;QACE,EAAE,EAAE,SAAS;QACb,QAAQ,EAAE,eAAe;QACzB,UAAU,EAAE,MAAM;QAClB,MAAM,EACJ,kVAAkV;QACpV,WAAW,EAAE,CAAC,WAAW,EAAE,WAAW,EAAE,WAAW,CAAC;QACpD,MAAM,EAAE,UAAU;KACnB;IACD;QACE,EAAE,EAAE,SAAS;QACb,QAAQ,EAAE,eAAe;QACzB,UAAU,EAAE,MAAM;QAClB,MAAM,EACJ,+OAA+O;QACjP,WAAW,EAAE,CAAC,YAAY,EAAE,SAAS,CAAC;QACtC,MAAM,EAAE,UAAU;KACnB;CACF,CAAC;AAEF,gFAAgF;AAEhF,MAAM,cAAc,GAAe;IACjC;QACE,EAAE,EAAE,YAAY;QAChB,QAAQ,EAAE,WAAW;QACrB,UAAU,EAAE,MAAM;QAClB,MAAM,EACJ,mGAAmG;QACrG,WAAW,EAAE,KAAK;QAClB,MAAM,EAAE,UAAU;KACnB;IACD;QACE,EAAE,EAAE,YAAY;QAChB,QAAQ,EAAE,WAAW;QACrB,UAAU,EAAE,MAAM;QAClB,MAAM,EACJ,mIAAmI;QACrI,WAAW,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;QAC7B,MAAM,EAAE,UAAU;KACnB;IACD;QACE,EAAE,EAAE,YAAY;QAChB,QAAQ,EAAE,WAAW;QACrB,UAAU,EAAE,QAAQ;QACpB,MAAM,EACJ,mIAAmI;QACrI,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;QACvB,MAAM,EAAE,UAAU;KACnB;IACD;QACE,EAAE,EAAE,YAAY;QAChB,QAAQ,EAAE,WAAW;QACrB,UAAU,EAAE,QAAQ;QACpB,MAAM,EACJ,0IAA0I;QAC5I,WAAW,EAAE,KAAK;QAClB,MAAM,EAAE,UAAU;KACnB;IACD;QACE,EAAE,EAAE,YAAY;QAChB,QAAQ,EAAE,WAAW;QACrB,UAAU,EAAE,MAAM;QAClB,MAAM,EACJ,yKAAyK;QAC3K,WAAW,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC;QAChC,MAAM,EAAE,UAAU;KACnB;IACD;QACE,EAAE,EAAE,YAAY;QAChB,QAAQ,EAAE,WAAW;QACrB,UAAU,EAAE,MAAM;QAClB,MAAM,EACJ,+MAA+M;QACjN,WAAW,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC;QACnC,MAAM,EAAE,UAAU;KACnB;CACF,CAAC;AAEF,iFAAiF;AAEjF,MAAM,CAAC,MAAM,eAAe,GAAe;IACzC,GAAG,SAAS;IACZ,GAAG,SAAS;IACZ,GAAG,mBAAmB;IACtB,GAAG,kBAAkB;IACrB,GAAG,cAAc;CAClB,CAAC;AAEF,MAAM,UAAU,kBAAkB,CAAC,QAAsB;IACvD,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC;AAChE,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,UAA0B;IAC7D,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,UAAU,CAAC,CAAC;AACpE,CAAC"}

package/dist/evals/harness.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Eval harness — strategy × dataset → {accuracy, cost, latency, rationale}.
+ *
+ * Sprint-579: eval-harness. Product-agnostic; consumer apps supply their own datasets.
+ *
+ * Usage:
+ *   const result = await evalSuite(singleShotStrategy, EVAL_DATASET_30);
+ *   console.log(result.accuracy); // 0.73
+ */
+import type { EvalTask } from "./datasets/index.js";
+export interface StrategyOutput {
+    output: string;
+    costUsd?: number;
+    latencyMs?: number;
+    candidates?: string[];
+}
+export interface EvalStrategy {
+    name: string;
+    run(prompt: string): Promise<StrategyOutput>;
+}
+export interface EvalTaskResult {
+    taskId: string;
+    category: string;
+    difficulty: string;
+    passed: boolean;
+    output: string;
+    score: number;
+    latencyMs: number;
+    costUsd: number;
+    rationale?: string;
+}
+export interface EvalSuiteResult {
+    strategy: string;
+    accuracy: number;
+    costUsd: number;
+    latencyMs: number;
+    rationale: EvalTaskResult[];
+    meta: {
+        totalTasks: number;
+        passedTasks: number;
+        failedTasks: number;
+        durationMs: number;
+    };
+}
+export declare function scoreTask(task: EvalTask, output: string): number;
+/**
+ * Run a strategy against a dataset of eval tasks.
+ * Returns accuracy (0-1), total cost in USD, median latency in ms,
+ * and per-task rationale.
+ */
+export declare function evalSuite(strategy: EvalStrategy, dataset: EvalTask[]): Promise<EvalSuiteResult>;
+//# sourceMappingURL=harness.d.ts.map

package/dist/evals/harness.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"harness.d.ts","sourceRoot":"","sources":["../../src/evals/harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAIpD,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;CAC9C;AAID,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,cAAc,EAAE,CAAC;IAC5B,IAAI,EAAE;QACJ,UAAU,EAAE,MAAM,CAAC;QACnB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAgCD,wBAAgB,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAchE;AAID;;;;GAIG;AACH,wBAAsB,SAAS,CAC7B,QAAQ,EAAE,YAAY,EACtB,OAAO,EAAE,QAAQ,EAAE,GAClB,OAAO,CAAC,eAAe,CAAC,CA4D1B"}