@rudderjs/ai 1.4.0 → 1.6.0

package/dist/budget-orm/index.js

@@ -0,0 +1,177 @@
+/**
+ * `@rudderjs/ai/budget-orm` — ORM-backed {@link BudgetStorage} for #A6 Phase 4.
+ *
+ * Production-grade replacement for `memoryBudgetStorage()` (which is
+ * single-process only). Persists per-user spend counters in a
+ * `BudgetUsage` table via the registered `@rudderjs/orm` adapter — works
+ * across queue workers, web processes, and horizontally-scaled deployments.
+ *
+ * Wire it into your AI middleware:
+ *
+ * ```ts
+ * import { withBudget } from '@rudderjs/ai'
+ * import { ormBudgetStorage } from '@rudderjs/ai/budget-orm'
+ *
+ * const budgeted = withBudget({
+ *   user:    (ctx) => ctx.context as string,
+ *   budget:  () => ({ daily: 0.50, monthly: 10 }),
+ *   storage: ormBudgetStorage(),
+ * })
+ * ```
+ *
+ * The schema lives at {@link budgetUsagePrismaSchema} — copy it into your
+ * Prisma schema (or a new `prisma/schema/<file>.prisma` if you use the
+ * multi-file setup). The `@@unique([userId, period, periodKey])`
+ * constraint is the one load-bearing index — without it, the
+ * find-or-create path can race and produce duplicate rows.
+ *
+ * # Atomicity caveat
+ *
+ * `checkAndDebit` does a read-then-conditional-increment. The increment
+ * itself is atomic (`UPDATE col = col + n`), but the cap check sits
+ * between the read and the write. Under high concurrency for a single
+ * user (more than ~1 in-flight budgeted request at a time), total spend
+ * can briefly exceed `cap` by up to `costUsd × concurrency`. For typical
+ * apps this is a non-issue.
+ *
+ * Strict guarantees require a database transaction with serializable
+ * isolation or a Redis-backed counter — both planned as follow-ups. File
+ * an issue if you hit this in production.
+ */
+import { Model } from '@rudderjs/orm';
+import { periodKey as buildPeriodKey, } from '../budget/storage.js';
+// ─── ORM Model ────────────────────────────────────────────
+/**
+ * Model row backing {@link OrmBudgetStorage}. Exposed so apps that
+ * want admin views (e.g. "show me top spenders this month") can use
+ * `BudgetUsageRecord.where(...).get()` instead of routing every read
+ * through the {@link BudgetStorage} interface.
+ *
+ * The `@@unique([userId, period, periodKey])` constraint is required —
+ * without it, two concurrent first-writes for the same user/period
+ * create duplicate rows and the cap accounting silently drifts.
+ */
+export class BudgetUsageRecord extends Model {
+    static table = 'budgetUsage';
+    static fillable = ['userId', 'period', 'periodKey', 'spent'];
+}
+// ─── BudgetStorage adapter ────────────────────────────────
+/**
+ * Production `BudgetStorage` backed by the registered `@rudderjs/orm`
+ * adapter. See the module JSDoc for setup + the atomicity caveat.
+ */
+export class OrmBudgetStorage {
+    async checkAndDebit(opts) {
+        if (!Number.isFinite(opts.cap) || opts.cap < 0) {
+            throw new Error(`[RudderJS AI] BudgetStorage: cap must be a non-negative finite number, got ${opts.cap}`);
+        }
+        if (!Number.isFinite(opts.costUsd) || opts.costUsd < 0) {
+            throw new Error(`[RudderJS AI] BudgetStorage: costUsd must be a non-negative finite number, got ${opts.costUsd}`);
+        }
+        const now = opts.now ?? new Date();
+        const key = buildPeriodKey(opts.period, now, opts.timezone);
+        const existing = await BudgetUsageRecord
+            .where('userId', opts.userId)
+            .where('period', opts.period)
+            .where('periodKey', key)
+            .first();
+        // ─── No row yet — first write for this period ─────────
+        if (!existing) {
+            // Pure-read on an empty bucket — still empty after.
+            if (opts.costUsd === 0) {
+                return { allowed: true, spent: 0, cap: opts.cap };
+            }
+            // Single debit larger than cap — refuse before creating the row,
+            // so we don't pollute storage with denied requests.
+            if (opts.costUsd > opts.cap) {
+                return { allowed: false, spent: 0, cap: opts.cap };
+            }
+            try {
+                await BudgetUsageRecord.create({
+                    userId: opts.userId,
+                    period: opts.period,
+                    periodKey: key,
+                    spent: opts.costUsd,
+                });
+                return { allowed: true, spent: opts.costUsd, cap: opts.cap };
+            }
+            catch (e) {
+                // Race: another caller created the row between our `first()` and
+                // `create()`. Re-read and fall through to the increment path.
+                // We deliberately don't sniff the error type — any create failure
+                // means the row may now exist; let the re-read decide.
+                const refetched = await BudgetUsageRecord
+                    .where('userId', opts.userId)
+                    .where('period', opts.period)
+                    .where('periodKey', key)
+                    .first();
+                if (!refetched)
+                    throw e; // not a unique-constraint race; surface the original error
+                return this._applyIncrementPath(refetched, opts);
+            }
+        }
+        return this._applyIncrementPath(existing, opts);
+    }
+    /** Apply the read-then-conditional-increment path on an existing row. */
+    async _applyIncrementPath(row, opts) {
+        const current = Number(row.spent ?? 0);
+        // Pure read.
+        if (opts.costUsd === 0) {
+            return { allowed: true, spent: current, cap: opts.cap };
+        }
+        // Cap check — read-then-decide. Atomic under single-writer; under
+        // concurrent writers, see the module-level atomicity caveat.
+        if (current + opts.costUsd > opts.cap) {
+            return { allowed: false, spent: current, cap: opts.cap };
+        }
+        const updated = await BudgetUsageRecord.increment(row.id, 'spent', opts.costUsd);
+        const newSpent = Number(updated?.spent ?? current + opts.costUsd);
+        return { allowed: true, spent: newSpent, cap: opts.cap };
+    }
+    async reset(userId, period, now, timezone) {
+        const key = buildPeriodKey(period, now ?? new Date(), timezone);
+        await BudgetUsageRecord
+            .where('userId', userId)
+            .where('period', period)
+            .where('periodKey', key)
+            .deleteAll();
+    }
+}
+/**
+ * Convenience factory — returns a fresh {@link OrmBudgetStorage}
+ * instance. Prefer this over `new OrmBudgetStorage()` for symmetry with
+ * `memoryBudgetStorage()`.
+ */
+export function ormBudgetStorage() {
+    return new OrmBudgetStorage();
+}
+// ─── Schema reference ─────────────────────────────────────
+/**
+ * Reference Prisma schema for `OrmBudgetStorage`. Copy into your
+ * `prisma/schema/<file>.prisma` (or paste alongside an existing model).
+ *
+ * The `@@unique([userId, period, periodKey])` constraint is required —
+ * without it the find-or-create path can race and produce duplicate
+ * rows, breaking cap accounting.
+ *
+ * SQLite stores `Float` as `REAL`; Postgres / MySQL as `DOUBLE
+ * PRECISION` / `DOUBLE`. All three give 15+ significant digits — more
+ * than enough for sub-cent budget tracking.
+ */
+export const budgetUsagePrismaSchema = `model BudgetUsage {
+  id        String   @id @default(cuid())
+  userId    String
+  /// 'daily' | 'monthly'
+  period    String
+  /// YYYY-MM-DD (daily) or YYYY-MM (monthly), in the configured timezone
+  periodKey String
+  /// Cumulative USD spend in this period
+  spent     Float    @default(0)
+  createdAt DateTime @default(now())
+  updatedAt DateTime @updatedAt
+
+  @@unique([userId, period, periodKey])
+  @@index([userId])
+}
+`;
+//# sourceMappingURL=index.js.map

package/dist/budget-orm/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/budget-orm/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAA;AACrC,OAAO,EAKL,SAAS,IAAI,cAAc,GAC5B,MAAM,sBAAsB,CAAA;AAE7B,6DAA6D;AAE7D;;;;;;;;;GASG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IAC1C,MAAM,CAAU,KAAK,GAAM,aAAa,CAAA;IACxC,MAAM,CAAU,QAAQ,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,CAAC,CAAA;;AAcvE,6DAA6D;AAE7D;;;GAGG;AACH,MAAM,OAAO,gBAAgB;IAC3B,KAAK,CAAC,aAAa,CAAC,IAAwB;QAC1C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,8EAA8E,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;QAC3G,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACvD,MAAM,IAAI,KAAK,CAAC,kFAAkF,IAAI,CAAC,OAAO,EAAE,CAAC,CAAA;QACnH,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAA;QAClC,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;QAE3D,MAAM,QAAQ,GAAG,MAAM,iBAAiB;aACrC,KAAK,CAAC,QAAQ,EAAK,IAAI,CAAC,MAAM,CAAC;aAC/B,KAAK,CAAC,QAAQ,EAAK,IAAI,CAAC,MAAM,CAAC;aAC/B,KAAK,CAAC,WAAW,EAAE,GAAG,CAAC;aACvB,KAAK,EAAyC,CAAA;QAEjD,yDAAyD;QACzD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,oDAAoD;YACpD,IAAI,IAAI,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC;gBACvB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAA;YACnD,CAAC;YACD,iEAAiE;YACjE,oDAAoD;YACpD,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC5B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAA;YACpD,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,iBAAiB,CAAC,MAAM,CAAC;oBAC7B,MAAM,EAAK,IAAI,CAAC,MAAM;oBACtB,MAAM,EAAK,IAAI,CAAC,MAAM;oBACtB,SAAS,EAAE,GAAG;oBACd,KAAK,EAAM,IAAI,CAAC,OAAO;iBACxB,CAAC,CAAA;gBACF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAA;YAC9D,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,iEAAiE;gBACjE,8DAA8D;gBAC9D,kEAAkE;gBAClE,uDAAuD;gBACvD,MAAM,SAAS,GAAG,MAAM,iBAAiB;qBACtC,KAAK,CAAC,QAAQ,EAAK,IAAI,CAAC,MAAM,CAAC;qBAC/B,KAAK,CAAC,QAAQ,EAAK,IAAI,CAAC,MAAM,CAAC;qBAC/B,KAAK,CAAC,WAAW,EAAE,GAAG,CAAC;qBACvB,KAAK,EAAyC,CAAA;gBACjD,IAAI,CAAC,SAAS;oBAAE,MAAM,CAAC,CAAA,CAAE,2DAA2D;gBACpF,OAAO,IAAI,CAAC,mBAAmB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;YAClD,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;IACjD,CAAC;IAED,yEAAyE;IACjE,KAAK,CAAC,mBAAmB,CAC/B,GAAuB,EACvB,IAAwB;QAExB,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,CAAA;QAEtC,aAAa;QACb,IAAI,IAAI,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAA;QACzD,CAAC;QAED,kEAAkE;QAClE,6DAA6D;QAC7D,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAA;QAC1D,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAiC,CAAA;QAChH,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,EAAE,KAAK,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,CAAA;QACjE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAA;IAC1D,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAAc,EAAE,MAAoB,EAAE,GAAU,EAAE,QAAiB;QAC7E,MAAM,GAAG,GAAG,cAAc,CAAC,MAAM,EAAE,GAAG,IAAI,IAAI,IAAI,EAAE,EAAE,QAAQ,CAAC,CAAA;QAC/D,MAAM,iBAAiB;aACpB,KAAK,CAAC,QAAQ,EAAK,MAAM,CAAC;aAC1B,KAAK,CAAC,QAAQ,EAAK,MAAM,CAAC;aAC1B,KAAK,CAAC,WAAW,EAAE,GAAG,CAAC;aACvB,SAAS,EAAE,CAAA;IAChB,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB;IAC9B,OAAO,IAAI,gBAAgB,EAAE,CAAA;AAC/B,CAAC;AAED,6DAA6D;AAE7D;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG;;;;;;;;;;;;;;;CAetC,CAAA"}

package/dist/commands/ai-eval.d.ts

@@ -0,0 +1,93 @@
+/**
+ * `pnpm rudder ai:eval` — discover `evals/**\/*.eval.ts` suites,
+ * run each, and report. Console reporter by default; `--json` emits
+ * a machine-readable envelope to stdout for CI.
+ *
+ * Registered from the CLI loader (`packages/cli/src/index.ts`)
+ * — the AiProvider doesn't own this so it surfaces even when the
+ * user app fails to boot, matching the `command:list --json`
+ * graceful-degradation pattern from #349.
+ */
+import type { EvalSuite } from '../eval/index.js';
+type Rudder = {
+    command(name: string, handler: (args: string[]) => void | Promise<void>): {
+        description(text: string): unknown;
+    };
+};
+/** CLI flags + positional name filter. */
+export interface AiEvalOptions {
+    /** Substring filter (case-insensitive) applied to suite names. */
+    filter?: string;
+    /** Stop on the first failing suite. */
+    bail: boolean;
+    /** Emit `{ suites: [...] }` JSON to stdout. */
+    json: boolean;
+    /**
+     * Run against the real provider, capture each case's assistant
+     * turns to `evals/__fixtures__/<suite>/<case>.json`. Existing
+     * fixtures are overwritten — diff in your VCS to see what changed.
+     * Default `false`.
+     */
+    record?: boolean;
+    /**
+     * Swap the runtime with `AiFake.fake()` and feed each case its
+     * recorded fixture via `respondWithSequence`. Zero API calls,
+     * deterministic regression tests. Cases without a fixture fall
+     * through to a normal run with a stderr warning. Default `false`.
+     */
+    replay?: boolean;
+    /**
+     * Path for a self-contained HTML report (#A5 Phase 5). Pasteable
+     * into PR comments / Slack threads. Coexists with `--json` (JSON
+     * still goes to stdout, HTML goes to disk).
+     */
+    html?: string;
+}
+/**
+ * Test seam — every external dependency gets an injectable
+ * override. The CLI handler defaults each to its real impl.
+ */
+export interface AiEvalDeps {
+    cwd?: string;
+    stdout?: {
+        write(s: string): boolean | void;
+    };
+    stderr?: {
+        write(s: string): boolean | void;
+    };
+    /** Override the file walk (test harness returns a virtual list). */
+    discover?: (cwd: string, pattern: string) => Promise<string[]>;
+    /** Override file → suite loader (test harness uses an in-memory map). */
+    loadSuite?: (absPath: string) => Promise<EvalSuite | null>;
+    /** Override config lookup (test harness skips `@rudderjs/core`). */
+    configPattern?: () => string | null | Promise<string | null>;
+    /**
+     * Override fixtures directory (defaults to `<cwd>/evals/__fixtures__`).
+     * Tests point to a tmpdir to keep round-trips off the source tree.
+     */
+    fixturesDir?: string;
+}
+/** Register the `ai:eval` command on the rudder runner. */
+export declare function registerAiEvalCommand(rudder: Rudder): void;
+/**
+ * Parse the rest-of-line. Recognizes:
+ *  - boolean flags: `--bail`, `--json`, `--record`, `--replay`
+ *  - value flags  : `--html <path>` or `--html=<path>`
+ *  - one positional name filter (anything not consumed above)
+ */
+export declare function parseArgs(args: string[]): AiEvalOptions;
+/**
+ * Execute the CLI flow. Returns the process exit code (0 = all pass,
+ * 1 = at least one suite had a failure or no suites discovered).
+ *
+ * The handler is `process.exit`-free so tests can drive it directly.
+ */
+export declare function runEvalCli(opts: AiEvalOptions, deps?: AiEvalDeps): Promise<number>;
+/**
+ * Recursive walk constrained to a `<dir>/**\/*<suffix>` shape.
+ * Returns absolute paths sorted lexicographically for stable test
+ * output and predictable `--bail` ordering.
+ */
+export declare function discoverSuiteFiles(cwd: string, pattern: string): Promise<string[]>;
+export {};
+//# sourceMappingURL=ai-eval.d.ts.map

package/dist/commands/ai-eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-eval.d.ts","sourceRoot":"","sources":["../../src/commands/ai-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAMH,OAAO,KAAK,EAAE,SAAS,EAAiC,MAAM,kBAAkB,CAAA;AAUhF,KAAK,MAAM,GAAG;IACZ,OAAO,CACL,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAChD;QAAE,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAA;CAC1C,CAAA;AAED,0CAA0C;AAC1C,MAAM,WAAW,aAAa;IAC5B,kEAAkE;IAClE,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,uCAAuC;IACvC,IAAI,EAAK,OAAO,CAAA;IAChB,+CAA+C;IAC/C,IAAI,EAAK,OAAO,CAAA;IAChB;;;;;OAKG;IACH,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB;;;;;OAKG;IACH,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,GAAG,CAAC,EAAS,MAAM,CAAA;IACnB,MAAM,CAAC,EAAM;QAAE,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,CAAA;KAAE,CAAA;IACjD,MAAM,CAAC,EAAM;QAAE,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,CAAA;KAAE,CAAA;IACjD,oEAAoE;IACpE,QAAQ,CAAC,EAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;IAChE,yEAAyE;IACzE,SAAS,CAAC,EAAG,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAAA;IAC3D,oEAAoE;IACpE,aAAa,CAAC,EAAE,MAAM,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAA;IAC5D;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,2DAA2D;AAC3D,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAO1D;AAMD;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,aAAa,CA8BvD;AAID;;;;;GAKG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,aAAa,EAAE,IAAI,GAAE,UAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAsE5F;AA4JD;;;;GAIG;AACH,wBAAsB,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAMxF"}

package/dist/commands/ai-eval.js

@@ -0,0 +1,378 @@
+/**
+ * `pnpm rudder ai:eval` — discover `evals/**\/*.eval.ts` suites,
+ * run each, and report. Console reporter by default; `--json` emits
+ * a machine-readable envelope to stdout for CI.
+ *
+ * Registered from the CLI loader (`packages/cli/src/index.ts`)
+ * — the AiProvider doesn't own this so it surfaces even when the
+ * user app fails to boot, matching the `command:list --json`
+ * graceful-degradation pattern from #349.
+ */
+import { readdir } from 'node:fs/promises';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { runSuite, reportConsole, evalSuite, stepsFromResponse } from '../eval/index.js';
+import { reportJson } from '../eval/json-reporter.js';
+import { reportHtml } from '../eval/html-reporter.js';
+import { defaultFixturesDir, readFixture, writeFixture } from '../eval/fixtures.js';
+import { AiFake } from '../fake.js';
+/** Register the `ai:eval` command on the rudder runner. */
+export function registerAiEvalCommand(rudder) {
+    rudder.command('ai:eval', async (rawArgs) => {
+        const code = await runEvalCli(parseArgs(rawArgs));
+        if (code !== 0)
+            process.exit(code);
+    }).description('Run eval suites — pnpm rudder ai:eval [name-pattern] [--bail] [--json] [--record|--replay] [--html <path>]');
+}
+// ─── Args parser ─────────────────────────────────────────
+const VALUE_FLAGS = new Set(['--html']);
+/**
+ * Parse the rest-of-line. Recognizes:
+ *  - boolean flags: `--bail`, `--json`, `--record`, `--replay`
+ *  - value flags  : `--html <path>` or `--html=<path>`
+ *  - one positional name filter (anything not consumed above)
+ */
+export function parseArgs(args) {
+    const positional = [];
+    const opts = { bail: false, json: false };
+    for (let i = 0; i < args.length; i++) {
+        const a = args[i];
+        if (!a.startsWith('--')) {
+            positional.push(a);
+            continue;
+        }
+        // `--flag=value` form
+        const eq = a.indexOf('=');
+        const name = eq >= 0 ? a.slice(0, eq) : a;
+        const inline = eq >= 0 ? a.slice(eq + 1) : undefined;
+        if (name === '--bail') {
+            opts.bail = true;
+            continue;
+        }
+        if (name === '--json') {
+            opts.json = true;
+            continue;
+        }
+        if (name === '--record') {
+            opts.record = true;
+            continue;
+        }
+        if (name === '--replay') {
+            opts.replay = true;
+            continue;
+        }
+        if (VALUE_FLAGS.has(name)) {
+            const value = inline ?? args[i + 1];
+            if (!inline)
+                i++; // consumed the next arg
+            if (!value)
+                throw new Error(`[RudderJS AI] ${name} requires a value`);
+            if (name === '--html')
+                opts.html = value;
+            continue;
+        }
+        // unknown flag — surface as positional so the user sees the typo
+        positional.push(a);
+    }
+    if (positional[0])
+        opts.filter = positional[0];
+    return opts;
+}
+// ─── Runner ──────────────────────────────────────────────
+/**
+ * Execute the CLI flow. Returns the process exit code (0 = all pass,
+ * 1 = at least one suite had a failure or no suites discovered).
+ *
+ * The handler is `process.exit`-free so tests can drive it directly.
+ */
+export async function runEvalCli(opts, deps = {}) {
+    const cwd = deps.cwd ?? process.cwd();
+    const stdout = deps.stdout ?? process.stdout;
+    const stderr = deps.stderr ?? process.stderr;
+    if (opts.record && opts.replay) {
+        stderr.write('[ai:eval] --record and --replay are mutually exclusive\n');
+        return 1;
+    }
+    const pattern = await Promise.resolve((deps.configPattern ?? loadConfigPattern)()) ?? 'evals/**/*.eval.ts';
+    const discover = deps.discover ?? discoverSuiteFiles;
+    const files = await discover(cwd, pattern);
+    if (files.length === 0) {
+        stderr.write(`[ai:eval] no suites found matching ${pattern}\n`);
+        return opts.json ? emitJson(stdout, []) : 1;
+    }
+    const loader = deps.loadSuite ?? defaultSuiteLoader;
+    const fixturesDir = deps.fixturesDir ?? defaultFixturesDir(cwd);
+    const reports = [];
+    const fullReports = [];
+    let exitCode = 0;
+    // `--replay` swaps the global runtime once, restored when we're done.
+    // The per-case fixture is set on the AiFake instance inside the
+    // wrapped agent factory just before each case's `agent.prompt()`.
+    let fake = null;
+    if (opts.replay)
+        fake = AiFake.fake();
+    try {
+        for (const file of files) {
+            let suite;
+            try {
+                suite = await loader(file);
+            }
+            catch (err) {
+                stderr.write(`[ai:eval] failed to load ${path.relative(cwd, file)}: ${formatError(err)}\n`);
+                exitCode = 1;
+                if (opts.bail)
+                    break;
+                continue;
+            }
+            if (!suite) {
+                stderr.write(`[ai:eval] ${path.relative(cwd, file)} has no default eval suite — skipping\n`);
+                continue;
+            }
+            if (opts.filter && !suite.name.toLowerCase().includes(opts.filter.toLowerCase()))
+                continue;
+            const decorated = await decorateForMode(suite, opts, { fixturesDir, stderr, fake });
+            const report = await runSuite(decorated);
+            fullReports.push(report);
+            if (opts.json) {
+                reports.push(reportJson(report));
+            }
+            else {
+                reportConsole(report, { log: (s) => stdout.write(`${s}\n`) });
+            }
+            if (report.failed > 0) {
+                exitCode = 1;
+                if (opts.bail)
+                    break;
+            }
+        }
+    }
+    finally {
+        if (fake)
+            fake.restore();
+    }
+    if (opts.json)
+        emitJson(stdout, reports);
+    if (opts.html)
+        await writeHtmlReport(opts.html, fullReports, cwd, stderr);
+    return exitCode;
+}
+async function writeHtmlReport(htmlPath, reports, cwd, stderr) {
+    const { writeFile, mkdir } = await import('node:fs/promises');
+    const abs = path.isAbsolute(htmlPath) ? htmlPath : path.resolve(cwd, htmlPath);
+    try {
+        await mkdir(path.dirname(abs), { recursive: true });
+        await writeFile(abs, reportHtml(reports));
+        stderr.write(`[ai:eval] wrote HTML report → ${path.relative(cwd, abs)}\n`);
+    }
+    catch (err) {
+        stderr.write(`[ai:eval] failed to write HTML report ${abs}: ${formatError(err)}\n`);
+    }
+}
+function emitJson(stdout, suites) {
+    stdout.write(`${JSON.stringify({ suites }, null, 2)}\n`);
+    return 0;
+}
+function formatError(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+/**
+ * Wrap a suite so each case captures the response (`--record`) or
+ * pre-loads the fake's sequence (`--replay`) before running. A
+ * normal run returns the suite untouched.
+ *
+ * Implemented as a per-case `agent` / `assert` decoration so the
+ * runner stays unchanged — `runSuite` doesn't need to know about
+ * the fixture format. The original `agent`/`assert` for each case
+ * are still called; we just slip work in around them.
+ *
+ * For replay, fixtures load up-front (sync factory contract) so the
+ * AiFake is primed before each `agent.prompt()` runs.
+ */
+async function decorateForMode(suite, opts, ctx) {
+    if (!opts.record && !opts.replay)
+        return suite;
+    // Pre-load every fixture for replay so the per-case factory can
+    // call `respondWithSequence` synchronously.
+    const replaySteps = new Map();
+    if (opts.replay) {
+        for (let i = 0; i < suite.spec.cases.length; i++) {
+            const c = suite.spec.cases[i];
+            const caseName = c.name ?? `case-${i}`;
+            try {
+                const fixture = await readFixture(ctx.fixturesDir, suite.name, caseName);
+                if (fixture)
+                    replaySteps.set(caseName, fixture.steps);
+                else
+                    ctx.stderr.write(`[ai:eval] no fixture for ${suite.name}/${caseName} — running against live provider\n`);
+            }
+            catch (err) {
+                ctx.stderr.write(`[ai:eval] fixture load error for ${suite.name}/${caseName}: ${formatError(err)}\n`);
+            }
+        }
+    }
+    const wrapped = suite.spec.cases.map((c, i) => {
+        const caseName = c.name ?? `case-${i}`;
+        const baseFactory = c.agent ?? suite.spec.agent;
+        const baseAssert = c.assert;
+        const factory = opts.replay
+            ? wrapReplayFactory(baseFactory, replaySteps.get(caseName), ctx.fake)
+            : baseFactory;
+        const assert = opts.record
+            ? wrapRecordAssert(baseAssert, suite.name, caseName, c.input, ctx)
+            : baseAssert;
+        const out = {
+            input: c.input,
+            assert,
+            agent: factory,
+        };
+        if (c.name)
+            out.name = c.name;
+        if (c.timeout !== undefined)
+            out.timeout = c.timeout;
+        if (c.skip !== undefined)
+            out.skip = c.skip;
+        return out;
+    });
+    const newSpec = {
+        agent: suite.spec.agent,
+        cases: wrapped,
+    };
+    if (suite.spec.timeout !== undefined)
+        newSpec.timeout = suite.spec.timeout;
+    return evalSuite(suite.name, newSpec);
+}
+/**
+ * Replay path: before each case runs, prime the shared `AiFake`
+ * with the case's recorded steps. When the fixture is missing the
+ * factory still returns the agent — it'll hit whatever the AiFake
+ * is currently scripted to return (typically falling back to the
+ * default ambient response, which surfaces as an obvious diff in
+ * the case's assertion).
+ */
+function wrapReplayFactory(base, steps, fake) {
+    return () => {
+        if (fake && steps)
+            fake.respondWithSequence(steps);
+        return base();
+    };
+}
+/**
+ * Record path: after each case's assertion runs, capture the
+ * agent response's assistant turns to the fixture file. Wrapping
+ * the assert is the cleanest hook — the runner already passes
+ * `response` into it, and the wrapped fn still returns the
+ * original assertion's result.
+ */
+function wrapRecordAssert(base, suite, caseName, input, ctx) {
+    return async (response, mctx) => {
+        try {
+            const file = await writeFixture(ctx.fixturesDir, suite, caseName, {
+                input,
+                steps: stepsFromResponse(response),
+            });
+            ctx.stderr.write(`[ai:eval] recorded ${path.relative(process.cwd(), file)}\n`);
+        }
+        catch (err) {
+            ctx.stderr.write(`[ai:eval] failed to record ${suite}/${caseName}: ${formatError(err)}\n`);
+        }
+        return base(response, mctx);
+    };
+}
+// ─── File discovery ──────────────────────────────────────
+/**
+ * Recursive walk constrained to a `<dir>/**\/*<suffix>` shape.
+ * Returns absolute paths sorted lexicographically for stable test
+ * output and predictable `--bail` ordering.
+ */
+export async function discoverSuiteFiles(cwd, pattern) {
+    const { root, suffix } = parsePattern(pattern);
+    const absRoot = path.resolve(cwd, root);
+    const out = [];
+    await walk(absRoot, suffix, out);
+    return out.sort();
+}
+/**
+ * Tiny pattern parser — supports `<dir>/**\/*<suffix>` and bare
+ * `*<suffix>` (current directory). Anything more elaborate is
+ * deferred to userland (run a custom script that imports `runSuite`).
+ *
+ * Examples:
+ *   `evals/**\/*.eval.ts`     → root=`evals`,    suffix=`.eval.ts`
+ *   `tests/agents/**\/*.ts`   → root=`tests/agents`, suffix=`.ts`
+ *   `*.eval.ts`               → root=`.`,        suffix=`.eval.ts`
+ */
+function parsePattern(pattern) {
+    const doubleStar = pattern.indexOf('**');
+    let prefix;
+    let postfix;
+    if (doubleStar >= 0) {
+        prefix = pattern.slice(0, doubleStar).replace(/\/$/, '');
+        postfix = pattern.slice(doubleStar + 2).replace(/^\//, '');
+    }
+    else {
+        const lastSlash = pattern.lastIndexOf('/');
+        prefix = lastSlash >= 0 ? pattern.slice(0, lastSlash) : '';
+        postfix = lastSlash >= 0 ? pattern.slice(lastSlash + 1) : pattern;
+    }
+    if (!postfix.startsWith('*')) {
+        throw new Error(`[RudderJS AI] Unsupported eval pattern "${pattern}". ` +
+            `Expected <dir>/**/*<suffix> or *<suffix>.`);
+    }
+    return {
+        root: prefix || '.',
+        suffix: postfix.slice(1),
+    };
+}
+async function walk(dir, suffix, out) {
+    let entries;
+    try {
+        entries = await readdir(dir, { withFileTypes: true });
+    }
+    catch (err) {
+        if (err.code === 'ENOENT')
+            return;
+        throw err;
+    }
+    for (const entry of entries) {
+        const p = path.join(dir, entry.name);
+        if (entry.isDirectory()) {
+            if (entry.name === 'node_modules' || entry.name.startsWith('.'))
+                continue;
+            await walk(p, suffix, out);
+        }
+        else if (entry.isFile() && entry.name.endsWith(suffix)) {
+            out.push(p);
+        }
+    }
+}
+// ─── Suite loader ────────────────────────────────────────
+async function defaultSuiteLoader(file) {
+    const mod = await import(pathToFileURL(file).href);
+    const candidate = (mod['default'] ?? mod['suite']);
+    if (!candidate || typeof candidate.name !== 'string' || !candidate.spec)
+        return null;
+    return candidate;
+}
+// ─── Config lookup ───────────────────────────────────────
+/**
+ * Read `config('ai').eval.pattern` from the booted app. Returns
+ * `null` (default pattern) when `@rudderjs/core` isn't loadable
+ * or the app didn't boot — the CLI must still work in
+ * introspective mode (#349).
+ */
+async function loadConfigPattern() {
+    try {
+        // Dynamic import so the static graph doesn't pin `@rudderjs/core`
+        // (optional peer). Falls back to default when core isn't loadable
+        // or the app didn't boot.
+        const core = await import('@rudderjs/core');
+        if (typeof core.config !== 'function')
+            return null;
+        const cfg = core.config('ai');
+        return cfg?.eval?.pattern ?? null;
+    }
+    catch {
+        return null;
+    }
+}
+//# sourceMappingURL=ai-eval.js.map