@moda-ai/cli 1.18.0 → 1.18.1

package/dist/cli.js

@@ -177,6 +177,7 @@ Commands:
   prompts sync --watch             Upload again when prompt files change
   prompts promote <key>            Move a dev/staging/prod label
   prompts ab                       Run a judged prompt A/B replay experiment
+  prompts propose                  Revise a prompt from a run's failures (--gate to auto-A/B, --promote-on-win)
 
   Skills:
   skills gen                       Kick off tenant-wide skill generation

package/dist/prompts-ab.d.ts

@@ -1,5 +1,8 @@
 import type { CommandContext } from './command-context.js';
 import type { ProfileIOOptions } from './profiles.js';
+export declare const DEFAULT_POLL_INTERVAL_MS = 15000;
+export declare const DEFAULT_WAIT_TIMEOUT_MS: number;
+export declare const TERMINAL_RUN_STATUSES: Set<string>;
 export interface PromptArmPayload {
     content: string;
     prompt_version_id?: string;
@@ -15,4 +18,54 @@ export interface PromptAbResult {
     cases: Array<Record<string, unknown>>;
 }
 export declare function runPromptAb(flags: Record<string, string>, profileOptions: ProfileIOOptions, context: CommandContext): Promise<void>;
+export interface ComparisonPayload {
+    run: {
+        runId: string;
+        status: string;
+        caseCount: number;
+        prodPassCount: number;
+        proposedPassCount: number;
+        prodPassRate: number;
+        proposedPassRate: number;
+        notes: string;
+    } | null;
+    cases: Array<{
+        caseId: string;
+        prodPass: boolean | null;
+        proposedPass: boolean | null;
+        prodScore: number | null;
+        proposedScore: number | null;
+    }>;
+}
+export declare function resolvePromptArmSpec(source: string, label: string, opts?: {
+    preferKey?: boolean;
+}): PromptArmPayload;
+export declare function fetchLatestRun(tenantId: string, replaySetId: string, runId: string, profileOptions: ProfileIOOptions): Promise<ComparisonPayload>;
+/**
+ * Strict outcome of a completed comparison for gating: 'candidate' only when the
+ * proposed arm passed strictly more cases than prod on a completed run. A tie,
+ * regression, skip, error, or missing run is never a candidate win.
+ */
+export declare function candidateOutcome(payload: ComparisonPayload): 'candidate' | 'baseline' | 'tie' | 'no-run';
+export declare function formatVerdict(payload: ComparisonPayload): string;
+/**
+ * Enqueue a prod-vs-proposed replay comparison and poll `/runs/latest` (pinned to
+ * OUR run id) until it reaches a terminal status or the deadline. Shared by
+ * `prompts ab` and the `prompts propose --gate` auto-gate. Returns the terminal
+ * comparison payload; throws on enqueue/poll timeout.
+ */
+export declare function enqueueAndPollComparison(args: {
+    tenantId: string;
+    replaySetId: string;
+    promptArms: {
+        prod: PromptArmPayload;
+        proposed: PromptArmPayload;
+    };
+    seedsPerCase?: number;
+    assistantModel?: string;
+    promotePrimary?: boolean;
+    timeoutMs?: number;
+    pollIntervalMs?: number;
+    onWaiting?: (status: string) => void;
+}, profileOptions: ProfileIOOptions): Promise<ComparisonPayload>;
 //# sourceMappingURL=prompts-ab.d.ts.map

package/dist/prompts-ab.js

@@ -4,9 +4,9 @@ import { callControlAPI, callDataAPI } from './api-client.js';
 import { resolveTenantId, validateConfig } from './config.js';
 import { discoverPrompts, parsePromptFileForAb, readPromptLock, runPromptSync, } from './prompts.js';
 const DEFAULT_SEEDS_PER_CASE = 3;
-const DEFAULT_POLL_INTERVAL_MS = 15_000;
-const DEFAULT_WAIT_TIMEOUT_MS = 2 * 60 * 60 * 1000;
-const TERMINAL_RUN_STATUSES = new Set(['completed', 'skipped', 'error']);
+export const DEFAULT_POLL_INTERVAL_MS = 15_000;
+export const DEFAULT_WAIT_TIMEOUT_MS = 2 * 60 * 60 * 1000;
+export const TERMINAL_RUN_STATUSES = new Set(['completed', 'skipped', 'error']);
 export async function runPromptAb(flags, profileOptions, context) {
     validateConfig();
     const tenantId = flags['tenant-id'] || resolveTenantId(profileOptions);
@@ -81,23 +81,40 @@ export async function runPromptAb(flags, profileOptions, context) {
     }
     context.output.writeData(result);
 }
-function resolvePromptArmSpec(source, label) {
+export function resolvePromptArmSpec(source, label, opts = {}) {
     const trimmed = source.trim();
     if (!trimmed) {
         throw new Error(`Empty prompt source for ${label}`);
     }
+    const fromDiscovery = () => {
+        const discovered = discoverPrompts();
+        const match = discovered.find((prompt) => prompt.key === trimmed)
+            ?? discovered.find((prompt) => prompt.sourcePath === trimmed);
+        return match ? { key: match.key, content: match.content } : null;
+    };
     const absPath = isAbsolute(trimmed) ? trimmed : resolve(process.cwd(), trimmed);
     let key;
     let content;
-    if (existsSync(absPath)) {
+    // When the caller knows `source` is a registry key (e.g. `propose --gate`'s
+    // `--prompt-key`), resolve by discovery first so a same-named file/dir in the
+    // cwd can't shadow the intended prompt; fall back to path resolution otherwise.
+    const preferred = opts.preferKey ? fromDiscovery() : null;
+    if (preferred) {
+        key = preferred.key;
+        content = preferred.content;
+    }
+    else if (opts.preferKey) {
+        // Key-first resolution must NOT fall through to path resolution — otherwise a
+        // cwd file named like the key would still shadow an unresolved registry key.
+        throw new Error(`Could not resolve prompt key '${trimmed}' — no discovered prompt matched this key`);
+    }
+    else if (existsSync(absPath)) {
         const parsed = parsePromptFileForAb(absPath);
         key = parsed.key;
         content = parsed.content;
     }
     else {
-        const discovered = discoverPrompts();
-        const match = discovered.find((prompt) => prompt.key === trimmed)
-            ?? discovered.find((prompt) => prompt.sourcePath === trimmed);
+        const match = fromDiscovery();
         if (!match) {
             throw new Error(`Could not resolve prompt '${trimmed}' — pass a .prompt.md path or a discovered prompt key`);
         }
@@ -176,13 +193,29 @@ async function loadScenarioFromConversation(conversationId) {
     }
     return `Replay conversation ${conversationId}: help the user accomplish their original goal using the available tools.`;
 }
-async function fetchLatestRun(tenantId, replaySetId, runId, profileOptions) {
+export async function fetchLatestRun(tenantId, replaySetId, runId, profileOptions) {
     // Pin the poll to OUR run id — otherwise a concurrent comparison on the same
     // set would make `/runs/latest` return the newer run forever and this command
     // would stall until timeout even after its own run finished.
     return callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(replaySetId)}/runs/latest?runId=${encodeURIComponent(runId)}`, { method: 'GET' }, profileOptions, { timeoutMs: 60_000, retries: 2 });
 }
-function formatVerdict(payload) {
+/**
+ * Strict outcome of a completed comparison for gating: 'candidate' only when the
+ * proposed arm passed strictly more cases than prod on a completed run. A tie,
+ * regression, skip, error, or missing run is never a candidate win.
+ */
+export function candidateOutcome(payload) {
+    const run = payload.run;
+    if (!run || run.status !== 'completed')
+        return 'no-run';
+    const delta = run.proposedPassCount - run.prodPassCount;
+    if (delta > 0)
+        return 'candidate';
+    if (delta < 0)
+        return 'baseline';
+    return 'tie';
+}
+export function formatVerdict(payload) {
     const run = payload.run;
     if (!run)
         return 'no run';
@@ -224,6 +257,43 @@ function formatPass(value) {
 function pct(rate) {
     return `${Math.round(Math.max(0, Math.min(1, rate)) * 100)}%`;
 }
+/**
+ * Enqueue a prod-vs-proposed replay comparison and poll `/runs/latest` (pinned to
+ * OUR run id) until it reaches a terminal status or the deadline. Shared by
+ * `prompts ab` and the `prompts propose --gate` auto-gate. Returns the terminal
+ * comparison payload; throws on enqueue/poll timeout.
+ */
+export async function enqueueAndPollComparison(args, profileOptions) {
+    const enqueue = (await callControlAPI(`/tenants/${encodeURIComponent(args.tenantId)}/replay-sets/${encodeURIComponent(args.replaySetId)}/run`, {
+        method: 'POST',
+        body: JSON.stringify({
+            promptArms: args.promptArms,
+            seedsPerCase: args.seedsPerCase ?? DEFAULT_SEEDS_PER_CASE,
+            assistantModel: args.assistantModel || undefined,
+            promotePrimary: args.promotePrimary ?? false,
+        }),
+    }, profileOptions, { timeoutMs: 120_000, retries: 1 }));
+    const timeoutMs = args.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS;
+    const pollIntervalMs = args.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
+    const deadline = Date.now() + timeoutMs;
+    let latest = null;
+    while (Date.now() < deadline) {
+        latest = await fetchLatestRun(args.tenantId, args.replaySetId, enqueue.runId, profileOptions);
+        const status = latest.run?.status ?? '';
+        const runId = latest.run?.runId ?? '';
+        if (TERMINAL_RUN_STATUSES.has(status) && runId === enqueue.runId) {
+            return latest;
+        }
+        args.onWaiting?.(status || 'pending');
+        await sleep(pollIntervalMs);
+    }
+    if (!latest?.run
+        || latest.run.runId !== enqueue.runId
+        || !TERMINAL_RUN_STATUSES.has(latest.run.status)) {
+        throw new Error(`Timed out after ${timeoutMs}ms waiting for replay run ${enqueue.runId} to finish`);
+    }
+    return latest;
+}
 function parsePositiveInt(raw, fallback, max) {
     if (!raw)
         return fallback;

package/dist/prompts-propose.d.ts

@@ -2,9 +2,14 @@ import type { CommandContext } from './command-context.js';
 import type { ProfileIOOptions } from './profiles.js';
 /**
  * `moda prompts propose` — turn a `prompts ab` run's failures into a revised
- * candidate registered as a new (unlabeled) prompt version. Does NOT promote:
- * the operator gates the candidate with a fresh `prompts ab` (ideally on the
- * printed holdout cases the revision never saw) and promotes only if it wins.
+ * candidate registered as a new (unlabeled) prompt version. Does NOT promote by
+ * default: the operator gates the candidate with a fresh `prompts ab` (ideally on
+ * the printed holdout cases the revision never saw) and promotes only if it wins.
+ *
+ * `--gate` chains that gate automatically: it A/Bs the fresh candidate (proposed
+ * arm) vs the on-disk baseline (prod arm) over the same replay set and prints the
+ * verdict. `--promote-on-win` then promotes the candidate to `prod` ONLY on a
+ * strict candidate win. `--model=<slug>` overrides the revision model.
  */
 export declare function runPromptPropose(positionals: string[], flags: Record<string, string>, profileOptions: ProfileIOOptions, context: CommandContext): Promise<void>;
 //# sourceMappingURL=prompts-propose.d.ts.map

package/dist/prompts-propose.js

@@ -1,14 +1,20 @@
 import { writeFileSync } from 'node:fs';
 import { isAbsolute, resolve } from 'node:path';
 import { callControlAPI } from './api-client.js';
-import { validateConfig } from './config.js';
+import { resolveTenantId, validateConfig } from './config.js';
+import { candidateOutcome, enqueueAndPollComparison, formatVerdict, resolvePromptArmSpec, } from './prompts-ab.js';
 // Propose is a single OpenRouter revision behind the backend; give it room.
 const PROPOSE_TIMEOUT_MS = 300_000;
 /**
  * `moda prompts propose` — turn a `prompts ab` run's failures into a revised
- * candidate registered as a new (unlabeled) prompt version. Does NOT promote:
- * the operator gates the candidate with a fresh `prompts ab` (ideally on the
- * printed holdout cases the revision never saw) and promotes only if it wins.
+ * candidate registered as a new (unlabeled) prompt version. Does NOT promote by
+ * default: the operator gates the candidate with a fresh `prompts ab` (ideally on
+ * the printed holdout cases the revision never saw) and promotes only if it wins.
+ *
+ * `--gate` chains that gate automatically: it A/Bs the fresh candidate (proposed
+ * arm) vs the on-disk baseline (prod arm) over the same replay set and prints the
+ * verdict. `--promote-on-win` then promotes the candidate to `prod` ONLY on a
+ * strict candidate win. `--model=<slug>` overrides the revision model.
  */
 export async function runPromptPropose(positionals, flags, profileOptions, context) {
     validateConfig();
@@ -16,21 +22,124 @@ export async function runPromptPropose(positionals, flags, profileOptions, conte
     const fromRunId = flags['from-run'] || flags['from-run-id'] || flags['run-id'];
     const replaySetId = flags['set-id'] || flags['replay-set-id'];
     if (!promptKey || !fromRunId || !replaySetId) {
-        throw new Error('Usage: moda prompts propose --prompt-key=<key> --from-run=<run_id> --set-id=<replay_set_id> [--max-dossiers=8] [--out=path.prompt.md]');
+        throw new Error('Usage: moda prompts propose --prompt-key=<key> --from-run=<run_id> --set-id=<replay_set_id> [--max-dossiers=8] [--model=<slug>] [--out=path.prompt.md] [--gate] [--promote-on-win]');
     }
     const maxDossiers = parsePositiveInt(flags['max-dossiers'], 8, 16);
-    const result = (await callControlAPI(`/prompts/${encodeURIComponent(promptKey)}/propose`, { method: 'POST', body: JSON.stringify({ fromRunId, replaySetId, maxDossiers }) }, profileOptions, { timeoutMs: PROPOSE_TIMEOUT_MS, retries: 0 }));
+    const model = (flags.model ?? flags['revise-model'] ?? '').trim();
+    const result = (await callControlAPI(`/prompts/${encodeURIComponent(promptKey)}/propose`, {
+        method: 'POST',
+        body: JSON.stringify({ fromRunId, replaySetId, maxDossiers, model: model || undefined }),
+    }, profileOptions, { timeoutMs: PROPOSE_TIMEOUT_MS, retries: 0 }));
     let outPath;
     if (result.status === 'proposed' && result.content && flags.out) {
         outPath = isAbsolute(flags.out) ? flags.out : resolve(process.cwd(), flags.out);
         writeFileSync(outPath, result.content, 'utf8');
     }
+    // The gate needs the candidate content in-hand for the proposed arm.
+    const wantGate = flags.gate === 'true';
+    const promoteOnWin = flags['promote-on-win'] === 'true';
+    let gate;
+    if (wantGate) {
+        if (result.status !== 'proposed' || !result.content) {
+            gate = {
+                ran: false,
+                scope: 'full-set',
+                scopeNote: `no candidate to gate (${result.status})`,
+                verdict: 'skipped',
+                outcome: 'no-run',
+                runId: null,
+                promoted: false,
+            };
+        }
+        else {
+            gate = await runGate({
+                result,
+                promptKey,
+                replaySetId,
+                flags,
+                promoteOnWin,
+                profileOptions,
+                context,
+            });
+        }
+    }
+    if (context.outputMode === 'human') {
+        printHuman(result, outPath, gate);
+    }
+    context.output.writeData({
+        ...result,
+        ...(outPath ? { outPath } : {}),
+        ...(gate ? { gate } : {}),
+    });
+}
+async function runGate(args) {
+    const { result, promptKey, replaySetId, flags, promoteOnWin, profileOptions, context } = args;
+    const tenantId = flags['tenant-id'] || resolveTenantId(profileOptions);
+    if (!tenantId) {
+        throw new Error('Missing tenant id for --gate. Run `moda init` or pass --tenant-id=<id>.');
+    }
+    // prod arm = the on-disk/registry baseline. `promptKey` is always a registry
+    // key here, so resolve key-first (`preferKey`) — otherwise a same-named file or
+    // dir in the cwd could shadow it and gate the candidate against wrong content.
+    // proposed arm = the freshly proposed candidate content returned above.
+    const prodArm = resolvePromptArmSpec(promptKey, 'baseline', { preferKey: true });
+    const proposedArm = {
+        content: result.content ?? '',
+        prompt_version_id: result.versionId ?? '',
+        prompt_id: '',
+        prompt_label: 'candidate',
+    };
+    const seedsPerCase = parsePositiveInt(flags.seeds ?? flags['seeds-per-case'], 3, 10);
+    const assistantModel = (flags['assistant-model'] ?? '').trim();
+    const timeoutMs = parsePositiveInt(flags['gate-timeout'] ?? flags.timeout, 2 * 60 * 60 * 1000, 24 * 60 * 60 * 1000);
+    const pollIntervalMs = parsePositiveInt(flags['poll-interval'], 15_000, 120_000);
+    // The /run endpoint scores the WHOLE replay set — there is no per-case subset
+    // knob — so the gate cannot be scoped to just the holdout ids the revision
+    // never saw. Say so plainly so a win isn't read as overfitting-proof.
+    const scopeNote = result.holdoutCaseIds.length
+        ? `gated over the FULL replay set (${result.holdoutCaseIds.length} holdout case(s) could not be isolated — the run endpoint scores every case)`
+        : 'gated over the FULL replay set';
     if (context.outputMode === 'human') {
-        printHuman(result, outPath);
+        process.stderr.write(`\nGating candidate for '${promptKey}' — ${scopeNote}...\n`);
     }
-    context.output.writeData(outPath ? { ...result, outPath } : result);
+    const payload = await enqueueAndPollComparison({
+        tenantId,
+        replaySetId,
+        promptArms: { prod: prodArm, proposed: proposedArm },
+        seedsPerCase,
+        assistantModel: assistantModel || undefined,
+        promotePrimary: false,
+        timeoutMs,
+        pollIntervalMs,
+        onWaiting: context.outputMode === 'human'
+            ? (status) => process.stderr.write(`Waiting for gate run (${status})...\n`)
+            : undefined,
+    }, profileOptions);
+    const outcome = candidateOutcome(payload);
+    const gate = {
+        ran: true,
+        scope: 'full-set',
+        scopeNote,
+        verdict: formatVerdict(payload),
+        outcome,
+        runId: payload.run?.runId ?? null,
+        promoted: false,
+    };
+    if (promoteOnWin) {
+        if (outcome === 'candidate' && result.versionId) {
+            await callControlAPI(`/prompts/${encodeURIComponent(promptKey)}/labels`, { method: 'POST', body: JSON.stringify({ label: 'prod', versionId: result.versionId }) }, profileOptions);
+            gate.promoted = true;
+        }
+        else {
+            gate.promoteSkippedReason =
+                outcome === 'candidate'
+                    ? 'no candidate versionId to promote'
+                    : `candidate did not strictly win (${outcome})`;
+        }
+    }
+    return gate;
 }
-function printHuman(result, outPath) {
+function printHuman(result, outPath, gate) {
     const w = (s) => process.stderr.write(s + '\n');
     if (result.status !== 'proposed') {
         w(`No candidate produced (${result.status}): ${result.note}`);
@@ -46,7 +155,33 @@ function printHuman(result, outPath) {
         w(`  risks:        ${result.risks.map((r) => `- ${r}`).join('\n                ')}`);
     if (outPath)
         w(`  written to:   ${outPath}`);
+    if (result.lowN) {
+        w('');
+        w(`  ⚠ low sample: ${result.lowN}`);
+    }
     w('');
+    if (gate?.ran) {
+        w('Gate (automatic A/B of candidate vs baseline):');
+        w(`  scope:    ${gate.scopeNote}`);
+        w(`  verdict:  ${gate.verdict}`);
+        if (gate.runId)
+            w(`  run id:   ${gate.runId}`);
+        if (gate.promoted) {
+            w(`  promoted: candidate won — promoted version ${result.versionId} to label 'prod'`);
+        }
+        else if (gate.promoteSkippedReason) {
+            w(`  promote:  skipped (${gate.promoteSkippedReason})`);
+        }
+        w('');
+        if (!gate.promoted) {
+            w(`  moda prompts promote ${result.promptKey} --label=prod --version=${result.versionId}`);
+        }
+        return;
+    }
+    if (gate && !gate.ran) {
+        w(`Gate not run: ${gate.scopeNote}`);
+        w('');
+    }
     w('Next — gate on holdout cases the revision never saw, then promote if it wins:');
     if (result.holdoutCaseIds.length) {
         w(`  holdout cases (${result.holdoutCaseIds.length}): ${result.holdoutCaseIds.join(', ')}`);
@@ -65,6 +200,7 @@ function printHuman(result, outPath) {
         w(`  # re-run with --out=<path.prompt.md> to materialize the candidate, then:`);
         w(`  moda prompts ab --baseline-key=${result.promptKey} --candidate=<path.prompt.md> --auto-generate`);
     }
+    w(`  # or: moda prompts propose ... --gate --promote-on-win  (auto-A/B, promote on a strict win)`);
     w(`  moda prompts promote ${result.promptKey} --label=prod --version=${result.versionId}`);
 }
 function parsePositiveInt(raw, fallback, max) {

package/dist/schemas.d.ts

@@ -43,24 +43,24 @@ export declare const ConversationsSchema: z.ZodObject<{
 }, "strip", z.ZodTypeAny, {
     search?: string | undefined;
     environment?: "staging" | "all" | "development" | "production" | undefined;
+    outcome?: "any" | "positive" | "negative" | undefined;
     time_range?: "all" | "1h" | "3d" | "7d" | "24h" | "30d" | "90d" | undefined;
     limit?: number | undefined;
     offset?: number | undefined;
     cluster_id?: string | undefined;
     user_id?: string | undefined;
     world_state?: string | undefined;
-    outcome?: "any" | "positive" | "negative" | undefined;
     include_world_state?: boolean | undefined;
 }, {
     search?: string | undefined;
     environment?: "staging" | "all" | "development" | "production" | undefined;
+    outcome?: "any" | "positive" | "negative" | undefined;
     time_range?: "all" | "1h" | "3d" | "7d" | "24h" | "30d" | "90d" | undefined;
     limit?: number | undefined;
     offset?: number | undefined;
     cluster_id?: string | undefined;
     user_id?: string | undefined;
     world_state?: string | undefined;
-    outcome?: "any" | "positive" | "negative" | undefined;
     include_world_state?: unknown;
 }>;
 /** moda world-state <conversationId> — per-conversation world-state drill-in. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@moda-ai/cli",
-  "version": "1.18.0",
+  "version": "1.18.1",
   "description": "CLI for Moda - AI agent analytics and observability",
   "type": "module",
   "bin": {

package/skills/integration/index.json

@@ -1,7 +1,7 @@
 {
   "schema_version": "moda.skill_index.v1",
-  "bundled_at": "2026-07-09T22:31:35.483Z",
-  "cli_version": "1.18.0",
+  "bundled_at": "2026-07-10T23:46:28.191Z",
+  "cli_version": "1.18.1",
   "skills": [
     {
       "id": "integration-node-anthropic",