@elizaos/plugin-training 2.0.3-beta.5 → 2.0.3-beta.7

package/dist/core/promotion-persist.d.ts

@@ -0,0 +1,116 @@
+/**
+ * Persistence wiring for the A/B promotion gate.
+ *
+ * Pulled out of `training-orchestrator.ts` so it can be unit-tested without
+ * dragging in `@elizaos/agent` (which transitively imports the AI SDK gateway
+ * and breaks bare vitest runs). The orchestrator imports and calls
+ * `gatedPersistNativeResult` from here.
+ *
+ * Contract:
+ *   1. Resolve the incumbent prompt — current artifact via
+ *      `service.getPrompt(task)`, falling back to the baseline template.
+ *   2. Run the promotion gate on (incumbent, candidate, dataset, scorer).
+ *   3. On promote → write via `service.setPrompt(...)` and prune the per-task
+ *      store to the configured retention budget.
+ *   4. On reject → write `candidate_rejected_<timestamp>.json` under
+ *      `<store-root>/<task>/rejected/` and leave the incumbent in place.
+ *
+ * No fallbacks on failure: missing services / missing store roots return
+ * structured notes so the orchestrator can surface them in the run record.
+ */
+import type { OptimizationExample, PromptScorer } from "../optimizers/index.js";
+import type { TrajectoryTrainingTask } from "./trajectory-task-datasets.js";
+export type PromotionOptimizerName = "instruction-search" | "prompt-evolution" | "gepa" | "bootstrap-fewshot" | "dspy-bootstrap-fewshot" | "dspy-copro" | "dspy-mipro";
+export interface PromotionFewShotExample {
+    id?: string;
+    input: {
+        user: string;
+        system?: string;
+    };
+    expectedOutput: string;
+    reward?: number;
+    metadata?: Record<string, unknown>;
+}
+export interface PromotionArtifactInput {
+    task: TrajectoryTrainingTask;
+    optimizer: PromotionOptimizerName;
+    baseline: string;
+    prompt: string;
+    score: number;
+    baselineScore: number;
+    datasetId: string;
+    datasetSize: number;
+    generatedAt: string;
+    lineage: Array<{
+        round: number;
+        variant: number;
+        score: number;
+        notes?: string;
+    }>;
+    fewShotExamples?: PromotionFewShotExample[];
+}
+export interface PromotionServiceLike {
+    setPrompt: (task: TrajectoryTrainingTask, artifact: PromotionArtifactInput) => Promise<string>;
+    /**
+     * Synchronous accessor for the incumbent prompt. Optional because older
+     * builds may not expose it; the gate falls back to the baseline template
+     * when missing.
+     */
+    getPrompt?: (task: TrajectoryTrainingTask) => {
+        prompt: string;
+        optimizerSource: PromotionOptimizerName;
+    } | null;
+    /**
+     * Returns the on-disk root used to store artifacts. Required for rejected /
+     * pruned bookkeeping; missing → gate still runs but rejected files are not
+     * persisted (logged via notes).
+     */
+    getStoreRoot?: () => string;
+}
+export interface PromotionNativeBackendResultLike {
+    optimizer: PromotionOptimizerName;
+    datasetSize: number;
+    score: number;
+    baselineScore: number;
+    result: {
+        optimizedPrompt: string;
+        lineage: Array<{
+            round: number;
+            variant: number;
+            score: number;
+            notes?: string;
+        }>;
+        fewShotExamples?: PromotionFewShotExample[];
+    };
+    /** Full parsed dataset. Fallback target for the gate when no holdout exists. */
+    dataset: OptimizationExample[];
+    /**
+     * Optional held-out subset the optimizer never saw. When present and
+     * non-empty the promotion gate scores against this set instead of
+     * `dataset`, eliminating train-on-test contamination.
+     */
+    holdoutSet?: OptimizationExample[];
+    scorer: PromptScorer;
+}
+export interface GatedPersistInput {
+    task: TrajectoryTrainingTask;
+    datasetPath: string;
+    runId: string;
+    baselinePrompt: string;
+    result: PromotionNativeBackendResultLike;
+    service: PromotionServiceLike;
+    /** Notes already accumulated by the dispatcher; new lines are appended. */
+    notesPrefix: string[];
+}
+export interface GatedPersistResult {
+    invoked: boolean;
+    artifactPath?: string;
+    notes: string[];
+}
+/**
+ * Gate + persist step extracted from the orchestrator's native dispatcher so
+ * it can be tested without spinning up a real optimizer or runtime. Returns
+ * the same shape the dispatcher emits.
+ */
+export declare function gatedPersistNativeResult(input: GatedPersistInput): Promise<GatedPersistResult>;
+//# sourceMappingURL=promotion-persist.d.ts.map

package/dist/core/promotion-persist.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"promotion-persist.d.ts","sourceRoot":"","sources":["../../src/core/promotion-persist.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAOhF,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AAE5E,MAAM,MAAM,sBAAsB,GAC9B,oBAAoB,GACpB,kBAAkB,GAClB,MAAM,GACN,mBAAmB,GACnB,wBAAwB,GACxB,YAAY,GACZ,YAAY,CAAC;AAEjB,MAAM,WAAW,uBAAuB;IACtC,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACzC,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,sBAAsB,CAAC;IAC7B,SAAS,EAAE,sBAAsB,CAAC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,KAAK,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,eAAe,CAAC,EAAE,uBAAuB,EAAE,CAAC;CAC7C;AAED,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,CACT,IAAI,EAAE,sBAAsB,EAC5B,QAAQ,EAAE,sBAAsB,KAC7B,OAAO,CAAC,MAAM,CAAC,CAAC;IACrB;;;;OAIG;IACH,SAAS,CAAC,EAAE,CACV,IAAI,EAAE,sBAAsB,KACzB;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,sBAAsB,CAAA;KAAE,GAAG,IAAI,CAAC;IACxE;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,gCAAgC;IAC/C,SAAS,EAAE,sBAAsB,CAAC;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE;QACN,eAAe,EAAE,MAAM,CAAC;QACxB,OAAO,EAAE,KAAK,CAAC;YACb,KAAK,EAAE,MAAM,CAAC;YACd,OAAO,EAAE,MAAM,CAAC;YAChB,KAAK,EAAE,MAAM,CAAC;YACd,KAAK,CAAC,EAAE,MAAM,CAAC;SAChB,CAAC,CAAC;QACH,eAAe,CAAC,EAAE,uBAAuB,EAAE,CAAC;KAC7C,CAAC;IACF,gFAAgF;IAChF,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B;;;;OAIG;IACH,UAAU,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACnC,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,sBAAsB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,gCAAgC,CAAC;IACzC,OAAO,EAAE,oBAAoB,CAAC;IAC9B,2EAA2E;IAC3E,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;;;GAIG;AACH,wBAAsB,wBAAwB,CAC5C,KAAK,EAAE,iBAAiB,GACvB,OAAO,CAAC,kBAAkB,CAAC,CAmG7B"}

package/dist/core/promotion-persist.js

@@ -0,0 +1,93 @@
+import {
+  DEFAULT_PROMOTED_ARTIFACT_RETENTION,
+  prunePromotedArtifacts,
+  writeRejectedCandidate
+} from "./artifact-store.js";
+import { evaluatePromotion } from "./promotion-gate.js";
+async function gatedPersistNativeResult(input) {
+  const notes = [...input.notesPrefix];
+  const incumbentResolved = typeof input.service.getPrompt === "function" ? input.service.getPrompt(input.task) : null;
+  const incumbentPrompt = incumbentResolved?.prompt ?? input.baselinePrompt;
+  const incumbentSource = incumbentResolved ? "current" : "baseline";
+  const holdoutSet = input.result.holdoutSet;
+  const gateDataset = holdoutSet && holdoutSet.length > 0 ? holdoutSet : input.result.dataset;
+  const gateSource = holdoutSet && holdoutSet.length > 0 ? `holdout(n=${holdoutSet.length})` : `full-dataset(n=${input.result.dataset.length}) [no holdout available]`;
+  const decision = await evaluatePromotion({
+    incumbentPrompt,
+    candidatePrompt: input.result.result.optimizedPrompt,
+    dataset: gateDataset,
+    scorer: input.result.scorer
+  });
+  notes.push(
+    `promotion-gate ${decision.promote ? "PROMOTE" : "REJECT"} incumbent_source=${incumbentSource} gate_dataset=${gateSource} ${decision.reason}`
+  );
+  const generatedAt = (/* @__PURE__ */ new Date()).toISOString();
+  if (!decision.promote) {
+    const storeRoot2 = input.service.getStoreRoot?.();
+    if (!storeRoot2) {
+      notes.push(
+        "OptimizedPromptService does not expose getStoreRoot; rejected candidate not persisted"
+      );
+      return { invoked: true, notes };
+    }
+    const rejectedPath = await writeRejectedCandidate(storeRoot2, input.task, {
+      rejectedAt: generatedAt,
+      task: input.task,
+      optimizer: input.result.optimizer,
+      candidatePrompt: input.result.result.optimizedPrompt,
+      incumbentPrompt,
+      scores: {
+        incumbentMeanScore: decision.incumbentMeanScore,
+        incumbentStdDev: decision.incumbentStdDev,
+        candidateScore: decision.candidateScore,
+        delta: decision.delta,
+        promotionMargin: decision.promotionMargin,
+        noiseThreshold: decision.noiseThreshold,
+        incumbentReseeds: decision.incumbentReseeds,
+        examplesPerPass: decision.examplesPerPass,
+        incumbentScores: decision.incumbentScores
+      },
+      reason: decision.reason,
+      datasetId: input.datasetPath,
+      runId: input.runId
+    });
+    notes.push(`rejected candidate written to ${rejectedPath}`);
+    return { invoked: true, notes };
+  }
+  const writePath = await input.service.setPrompt(input.task, {
+    task: input.task,
+    optimizer: input.result.optimizer,
+    baseline: input.baselinePrompt,
+    prompt: input.result.result.optimizedPrompt,
+    score: input.result.score,
+    baselineScore: input.result.baselineScore,
+    datasetId: input.datasetPath,
+    datasetSize: input.result.datasetSize,
+    generatedAt,
+    lineage: input.result.result.lineage,
+    fewShotExamples: input.result.result.fewShotExamples
+  });
+  notes.push(`artifact written to ${writePath}`);
+  const storeRoot = input.service.getStoreRoot?.();
+  if (storeRoot) {
+    const removed = await prunePromotedArtifacts(
+      storeRoot,
+      input.task,
+      DEFAULT_PROMOTED_ARTIFACT_RETENTION
+    );
+    if (removed.length > 0) {
+      notes.push(
+        `pruned ${removed.length} stale artifact(s); retained ${DEFAULT_PROMOTED_ARTIFACT_RETENTION} most recent`
+      );
+    }
+  }
+  return {
+    invoked: true,
+    artifactPath: writePath,
+    notes
+  };
+}
+export {
+  gatedPersistNativeResult
+};
+//# sourceMappingURL=promotion-persist.js.map

package/dist/core/promotion-persist.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/core/promotion-persist.ts"],"sourcesContent":["/**\n * Persistence wiring for the A/B promotion gate.\n *\n * Pulled out of `training-orchestrator.ts` so it can be unit-tested without\n * dragging in `@elizaos/agent` (which transitively imports the AI SDK gateway\n * and breaks bare vitest runs). The orchestrator imports and calls\n * `gatedPersistNativeResult` from here.\n *\n * Contract:\n *   1. Resolve the incumbent prompt — current artifact via\n *      `service.getPrompt(task)`, falling back to the baseline template.\n *   2. Run the promotion gate on (incumbent, candidate, dataset, scorer).\n *   3. On promote → write via `service.setPrompt(...)` and prune the per-task\n *      store to the configured retention budget.\n *   4. On reject → write `candidate_rejected_<timestamp>.json` under\n *      `<store-root>/<task>/rejected/` and leave the incumbent in place.\n *\n * No fallbacks on failure: missing services / missing store roots return\n * structured notes so the orchestrator can surface them in the run record.\n */\n\nimport type { OptimizationExample, PromptScorer } from \"../optimizers/index.js\";\nimport {\n  DEFAULT_PROMOTED_ARTIFACT_RETENTION,\n  prunePromotedArtifacts,\n  writeRejectedCandidate,\n} from \"./artifact-store.js\";\nimport { evaluatePromotion } from \"./promotion-gate.js\";\nimport type { TrajectoryTrainingTask } from \"./trajectory-task-datasets.js\";\n\nexport type PromotionOptimizerName =\n  | \"instruction-search\"\n  | \"prompt-evolution\"\n  | \"gepa\"\n  | \"bootstrap-fewshot\"\n  | \"dspy-bootstrap-fewshot\"\n  | \"dspy-copro\"\n  | \"dspy-mipro\";\n\nexport interface PromotionFewShotExample {\n  id?: string;\n  input: { user: string; system?: string };\n  expectedOutput: string;\n  reward?: number;\n  metadata?: Record<string, unknown>;\n}\n\nexport interface PromotionArtifactInput {\n  task: TrajectoryTrainingTask;\n  optimizer: PromotionOptimizerName;\n  baseline: string;\n  prompt: string;\n  score: number;\n  baselineScore: number;\n  datasetId: string;\n  datasetSize: number;\n  generatedAt: string;\n  lineage: Array<{\n    round: number;\n    variant: number;\n    score: number;\n    notes?: string;\n  }>;\n  fewShotExamples?: PromotionFewShotExample[];\n}\n\nexport interface PromotionServiceLike {\n  setPrompt: (\n    task: TrajectoryTrainingTask,\n    artifact: PromotionArtifactInput,\n  ) => Promise<string>;\n  /**\n   * Synchronous accessor for the incumbent prompt. Optional because older\n   * builds may not expose it; the gate falls back to the baseline template\n   * when missing.\n   */\n  getPrompt?: (\n    task: TrajectoryTrainingTask,\n  ) => { prompt: string; optimizerSource: PromotionOptimizerName } | null;\n  /**\n   * Returns the on-disk root used to store artifacts. Required for rejected /\n   * pruned bookkeeping; missing → gate still runs but rejected files are not\n   * persisted (logged via notes).\n   */\n  getStoreRoot?: () => string;\n}\n\nexport interface PromotionNativeBackendResultLike {\n  optimizer: PromotionOptimizerName;\n  datasetSize: number;\n  score: number;\n  baselineScore: number;\n  result: {\n    optimizedPrompt: string;\n    lineage: Array<{\n      round: number;\n      variant: number;\n      score: number;\n      notes?: string;\n    }>;\n    fewShotExamples?: PromotionFewShotExample[];\n  };\n  /** Full parsed dataset. Fallback target for the gate when no holdout exists. */\n  dataset: OptimizationExample[];\n  /**\n   * Optional held-out subset the optimizer never saw. When present and\n   * non-empty the promotion gate scores against this set instead of\n   * `dataset`, eliminating train-on-test contamination.\n   */\n  holdoutSet?: OptimizationExample[];\n  scorer: PromptScorer;\n}\n\nexport interface GatedPersistInput {\n  task: TrajectoryTrainingTask;\n  datasetPath: string;\n  runId: string;\n  baselinePrompt: string;\n  result: PromotionNativeBackendResultLike;\n  service: PromotionServiceLike;\n  /** Notes already accumulated by the dispatcher; new lines are appended. */\n  notesPrefix: string[];\n}\n\nexport interface GatedPersistResult {\n  invoked: boolean;\n  artifactPath?: string;\n  notes: string[];\n}\n\n/**\n * Gate + persist step extracted from the orchestrator's native dispatcher so\n * it can be tested without spinning up a real optimizer or runtime. Returns\n * the same shape the dispatcher emits.\n */\nexport async function gatedPersistNativeResult(\n  input: GatedPersistInput,\n): Promise<GatedPersistResult> {\n  const notes = [...input.notesPrefix];\n\n  const incumbentResolved =\n    typeof input.service.getPrompt === \"function\"\n      ? input.service.getPrompt(input.task)\n      : null;\n  const incumbentPrompt = incumbentResolved?.prompt ?? input.baselinePrompt;\n  const incumbentSource = incumbentResolved ? \"current\" : \"baseline\";\n\n  // Prefer the held-out subset (the optimizer never saw it) so the gate is\n  // not a train-on-test pass. Fall back to the full dataset for back-compat\n  // and for tiny datasets where the deterministic split produced no holdout.\n  const holdoutSet = input.result.holdoutSet;\n  const gateDataset =\n    holdoutSet && holdoutSet.length > 0 ? holdoutSet : input.result.dataset;\n  const gateSource =\n    holdoutSet && holdoutSet.length > 0\n      ? `holdout(n=${holdoutSet.length})`\n      : `full-dataset(n=${input.result.dataset.length}) [no holdout available]`;\n\n  const decision = await evaluatePromotion({\n    incumbentPrompt,\n    candidatePrompt: input.result.result.optimizedPrompt,\n    dataset: gateDataset,\n    scorer: input.result.scorer,\n  });\n  notes.push(\n    `promotion-gate ${decision.promote ? \"PROMOTE\" : \"REJECT\"} incumbent_source=${incumbentSource} gate_dataset=${gateSource} ${decision.reason}`,\n  );\n\n  const generatedAt = new Date().toISOString();\n  if (!decision.promote) {\n    const storeRoot = input.service.getStoreRoot?.();\n    if (!storeRoot) {\n      notes.push(\n        \"OptimizedPromptService does not expose getStoreRoot; rejected candidate not persisted\",\n      );\n      return { invoked: true, notes };\n    }\n    const rejectedPath = await writeRejectedCandidate(storeRoot, input.task, {\n      rejectedAt: generatedAt,\n      task: input.task,\n      optimizer: input.result.optimizer,\n      candidatePrompt: input.result.result.optimizedPrompt,\n      incumbentPrompt,\n      scores: {\n        incumbentMeanScore: decision.incumbentMeanScore,\n        incumbentStdDev: decision.incumbentStdDev,\n        candidateScore: decision.candidateScore,\n        delta: decision.delta,\n        promotionMargin: decision.promotionMargin,\n        noiseThreshold: decision.noiseThreshold,\n        incumbentReseeds: decision.incumbentReseeds,\n        examplesPerPass: decision.examplesPerPass,\n        incumbentScores: decision.incumbentScores,\n      },\n      reason: decision.reason,\n      datasetId: input.datasetPath,\n      runId: input.runId,\n    });\n    notes.push(`rejected candidate written to ${rejectedPath}`);\n    return { invoked: true, notes };\n  }\n\n  const writePath = await input.service.setPrompt(input.task, {\n    task: input.task,\n    optimizer: input.result.optimizer,\n    baseline: input.baselinePrompt,\n    prompt: input.result.result.optimizedPrompt,\n    score: input.result.score,\n    baselineScore: input.result.baselineScore,\n    datasetId: input.datasetPath,\n    datasetSize: input.result.datasetSize,\n    generatedAt,\n    lineage: input.result.result.lineage,\n    fewShotExamples: input.result.result.fewShotExamples,\n  });\n  notes.push(`artifact written to ${writePath}`);\n\n  const storeRoot = input.service.getStoreRoot?.();\n  if (storeRoot) {\n    const removed = await prunePromotedArtifacts(\n      storeRoot,\n      input.task,\n      DEFAULT_PROMOTED_ARTIFACT_RETENTION,\n    );\n    if (removed.length > 0) {\n      notes.push(\n        `pruned ${removed.length} stale artifact(s); retained ${DEFAULT_PROMOTED_ARTIFACT_RETENTION} most recent`,\n      );\n    }\n  }\n\n  return {\n    invoked: true,\n    artifactPath: writePath,\n    notes,\n  };\n}\n"],"mappings":"AAsBA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,yBAAyB;AA4GlC,eAAsB,yBACpB,OAC6B;AAC7B,QAAM,QAAQ,CAAC,GAAG,MAAM,WAAW;AAEnC,QAAM,oBACJ,OAAO,MAAM,QAAQ,cAAc,aAC/B,MAAM,QAAQ,UAAU,MAAM,IAAI,IAClC;AACN,QAAM,kBAAkB,mBAAmB,UAAU,MAAM;AAC3D,QAAM,kBAAkB,oBAAoB,YAAY;AAKxD,QAAM,aAAa,MAAM,OAAO;AAChC,QAAM,cACJ,cAAc,WAAW,SAAS,IAAI,aAAa,MAAM,OAAO;AAClE,QAAM,aACJ,cAAc,WAAW,SAAS,IAC9B,aAAa,WAAW,MAAM,MAC9B,kBAAkB,MAAM,OAAO,QAAQ,MAAM;AAEnD,QAAM,WAAW,MAAM,kBAAkB;AAAA,IACvC;AAAA,IACA,iBAAiB,MAAM,OAAO,OAAO;AAAA,IACrC,SAAS;AAAA,IACT,QAAQ,MAAM,OAAO;AAAA,EACvB,CAAC;AACD,QAAM;AAAA,IACJ,kBAAkB,SAAS,UAAU,YAAY,QAAQ,qBAAqB,eAAe,iBAAiB,UAAU,IAAI,SAAS,MAAM;AAAA,EAC7I;AAEA,QAAM,eAAc,oBAAI,KAAK,GAAE,YAAY;AAC3C,MAAI,CAAC,SAAS,SAAS;AACrB,UAAMA,aAAY,MAAM,QAAQ,eAAe;AAC/C,QAAI,CAACA,YAAW;AACd,YAAM;AAAA,QACJ;AAAA,MACF;AACA,aAAO,EAAE,SAAS,MAAM,MAAM;AAAA,IAChC;AACA,UAAM,eAAe,MAAM,uBAAuBA,YAAW,MAAM,MAAM;AAAA,MACvE,YAAY;AAAA,MACZ,MAAM,MAAM;AAAA,MACZ,WAAW,MAAM,OAAO;AAAA,MACxB,iBAAiB,MAAM,OAAO,OAAO;AAAA,MACrC;AAAA,MACA,QAAQ;AAAA,QACN,oBAAoB,SAAS;AAAA,QAC7B,iBAAiB,SAAS;AAAA,QAC1B,gBAAgB,SAAS;AAAA,QACzB,OAAO,SAAS;AAAA,QAChB,iBAAiB,SAAS;AAAA,QAC1B,gBAAgB,SAAS;AAAA,QACzB,kBAAkB,SAAS;AAAA,QAC3B,iBAAiB,SAAS;AAAA,QAC1B,iBAAiB,SAAS;AAAA,MAC5B;AAAA,MACA,QAAQ,SAAS;AAAA,MACjB,WAAW,MAAM;AAAA,MACjB,OAAO,MAAM;AAAA,IACf,CAAC;AACD,UAAM,KAAK,iCAAiC,YAAY,EAAE;AAC1D,WAAO,EAAE,SAAS,MAAM,MAAM;AAAA,EAChC;AAEA,QAAM,YAAY,MAAM,MAAM,QAAQ,UAAU,MAAM,MAAM;AAAA,IAC1D,MAAM,MAAM;AAAA,IACZ,WAAW,MAAM,OAAO;AAAA,IACxB,UAAU,MAAM;AAAA,IAChB,QAAQ,MAAM,OAAO,OAAO;AAAA,IAC5B,OAAO,MAAM,OAAO;AAAA,IACpB,eAAe,MAAM,OAAO;AAAA,IAC5B,WAAW,MAAM;AAAA,IACjB,aAAa,MAAM,OAAO;AAAA,IAC1B;AAAA,IACA,SAAS,MAAM,OAAO,OAAO;AAAA,IAC7B,iBAAiB,MAAM,OAAO,OAAO;AAAA,EACvC,CAAC;AACD,QAAM,KAAK,uBAAuB,SAAS,EAAE;AAE7C,QAAM,YAAY,MAAM,QAAQ,eAAe;AAC/C,MAAI,WAAW;AACb,UAAM,UAAU,MAAM;AAAA,MACpB;AAAA,MACA,MAAM;AAAA,MACN;AAAA,IACF;AACA,QAAI,QAAQ,SAAS,GAAG;AACtB,YAAM;AAAA,QACJ,UAAU,QAAQ,MAAM,gCAAgC,mCAAmC;AAAA,MAC7F;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AAAA,IACL,SAAS;AAAA,IACT,cAAc;AAAA,IACd;AAAA,EACF;AACF;","names":["storeRoot"]}

package/dist/core/prompt-compare.d.ts

@@ -0,0 +1,99 @@
+/**
+ * Prompt A/B comparison harness.
+ *
+ * Given two prompts (baseline + variant) and a dataset of historical
+ * input/expected pairs, run each prompt through the same model and
+ * report mean scores plus per-example deltas. Used to gate prompt
+ * cleanup / compression changes before they ship: a variant that
+ * regresses against the historical reference indicates a behavioral
+ * change, not a pure cosmetic edit.
+ *
+ * The harness is a thin wrapper over the native optimizer scorer
+ * (`createPromptScorer` from optimizers/scoring.ts) — that scorer is
+ * already designed for prompt-vs-dataset evaluation, just with a
+ * single prompt at a time. We invoke it twice and diff the results.
+ *
+ * Two scoring modes are supported:
+ *
+ * - `vs_historical` (default): each prompt is scored against the
+ *   recorded `expectedOutput` (Jaccard token overlap by default,
+ *   action-name match for the action_planner task). Cheap and
+ *   deterministic. Both prompts are scored independently; the delta
+ *   tells you whether the variant reproduces the historical output
+ *   as well as the baseline does.
+ *
+ * - `pairwise`: run baseline on every example to capture v1 outputs,
+ *   then run variant on the same inputs and compare v2 outputs to v1
+ *   outputs directly (pairwise Jaccard). This answers "did the
+ *   variant produce semantically equivalent output?", which is a
+ *   stricter regression test than `vs_historical` because the latter
+ *   is biased — historical outputs were likely produced by a prompt
+ *   close to the baseline.
+ *
+ * No new model abstractions are introduced. Reuses:
+ *   - `parseJsonlDataset()` from backends/native.ts (private — mirrored inline here to avoid exporting the training-backend parser)
+ *   - `createRuntimeAdapter()` from optimizers/scoring.ts
+ *   - `createPromptScorer()` from optimizers/scoring.ts
+ *   - `scoreAgreement()` / `scorePlannerAction()` from optimizers/scoring.ts
+ *
+ * Cost note: N examples × 2 prompts = 2N model calls per run in
+ * `vs_historical` mode; same in `pairwise` mode (baseline outputs are
+ * captured once, variant once). Default temperature 0 for determinism.
+ */
+import { type LlmAdapter, type OptimizationExample, type UseModelHandler } from "../optimizers/index.js";
+import type { TrajectoryTrainingTask } from "./trajectory-task-datasets.js";
+export type ScorerKind = "agreement" | "planner_action";
+export type CompareMode = "vs_historical" | "pairwise";
+export interface PromptComparisonInput {
+    /** System prompt under test as the baseline (often the current canonical prompt). */
+    baselinePrompt: string;
+    /** System prompt under test as the variant (e.g. caveman-compressed). */
+    variantPrompt: string;
+    /** Dataset of `(input, expectedOutput)` rows. Path to a JSONL file produced by `exportTrajectoryTaskDatasets`, or an in-memory array. */
+    dataset: string | OptimizationExample[];
+    /** Task hint — selects the right scorer when `scorer` is omitted. Defaults to `agreement`. */
+    task?: TrajectoryTrainingTask;
+    /** Force a specific scorer regardless of task. */
+    scorer?: ScorerKind;
+    /** Cap how many examples to score (handy for cheap previews). */
+    maxExamples?: number;
+    /** Compare mode: `vs_historical` (default) or `pairwise`. */
+    mode?: CompareMode;
+    /** Temperature passed to the adapter. Defaults to 0 for determinism. */
+    temperature?: number;
+    /** Max tokens per completion. Defaults to 512. */
+    maxTokens?: number;
+    /** Loose runtime shape — only `useModel` is required. Mutually exclusive with `adapter`. */
+    runtime?: {
+        useModel: UseModelHandler;
+    };
+    /** Pre-built LLM adapter (tests, alternative providers). */
+    adapter?: LlmAdapter;
+}
+export interface PromptComparisonResult {
+    baselineScore: number;
+    variantScore: number;
+    /** `variantScore - baselineScore`. Positive means variant is closer to reference. */
+    delta: number;
+    /** Percentage delta, where 0 baseline collapses to 0 to avoid divide-by-zero. */
+    deltaPercent: number;
+    examplesScored: number;
+    scorer: ScorerKind;
+    mode: CompareMode;
+    /** True when the variant did not measurably regress (delta ≥ -tolerance). */
+    passed: boolean;
+    /** Tolerance applied to `passed`. Defaults to 0.02 (2 percentage points). */
+    tolerance: number;
+}
+/** Default tolerance: a variant is considered safe if its score is within
+ *  2 percentage points of the baseline. Tunable per call. */
+export declare const DEFAULT_REGRESSION_TOLERANCE = 0.02;
+/**
+ * Compare two prompts on the same dataset and report mean scores plus
+ * delta. Throws on dataset I/O errors; never throws for "variant is
+ * worse" — read `result.passed` for the gate decision.
+ */
+export declare function comparePrompts(input: PromptComparisonInput): Promise<PromptComparisonResult>;
+/** Render a result as a single-line summary suitable for CLI output. */
+export declare function formatComparisonSummary(result: PromptComparisonResult): string;
+//# sourceMappingURL=prompt-compare.d.ts.map

package/dist/core/prompt-compare.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt-compare.d.ts","sourceRoot":"","sources":["../../src/core/prompt-compare.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAGH,OAAO,EAGL,KAAK,UAAU,EACf,KAAK,mBAAmB,EAGxB,KAAK,eAAe,EACrB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AAE5E,MAAM,MAAM,UAAU,GAAG,WAAW,GAAG,gBAAgB,CAAC;AACxD,MAAM,MAAM,WAAW,GAAG,eAAe,GAAG,UAAU,CAAC;AAEvD,MAAM,WAAW,qBAAqB;IACpC,qFAAqF;IACrF,cAAc,EAAE,MAAM,CAAC;IACvB,yEAAyE;IACzE,aAAa,EAAE,MAAM,CAAC;IACtB,yIAAyI;IACzI,OAAO,EAAE,MAAM,GAAG,mBAAmB,EAAE,CAAC;IACxC,8FAA8F;IAC9F,IAAI,CAAC,EAAE,sBAAsB,CAAC;IAC9B,kDAAkD;IAClD,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,iEAAiE;IACjE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,6DAA6D;IAC7D,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,wEAAwE;IACxE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kDAAkD;IAClD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,4FAA4F;IAC5F,OAAO,CAAC,EAAE;QAAE,QAAQ,EAAE,eAAe,CAAA;KAAE,CAAC;IACxC,4DAA4D;IAC5D,OAAO,CAAC,EAAE,UAAU,CAAC;CACtB;AAED,MAAM,WAAW,sBAAsB;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,qFAAqF;IACrF,KAAK,EAAE,MAAM,CAAC;IACd,iFAAiF;IACjF,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,UAAU,CAAC;IACnB,IAAI,EAAE,WAAW,CAAC;IAClB,6EAA6E;IAC7E,MAAM,EAAE,OAAO,CAAC;IAChB,6EAA6E;IAC7E,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;6DAC6D;AAC7D,eAAO,MAAM,4BAA4B,OAAO,CAAC;AAEjD;;;;GAIG;AACH,wBAAsB,cAAc,CAClC,KAAK,EAAE,qBAAqB,GAC3B,OAAO,CAAC,sBAAsB,CAAC,CAkDjC;AAsND,wEAAwE;AACxE,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,sBAAsB,GAC7B,MAAM,CAWR"}

package/dist/core/prompt-compare.js

@@ -0,0 +1,210 @@
+import { existsSync, readFileSync } from "node:fs";
+import {
+  createPromptScorer,
+  createRuntimeAdapter,
+  scoreAgreement,
+  scorePlannerAction
+} from "../optimizers/index.js";
+const DEFAULT_REGRESSION_TOLERANCE = 0.02;
+async function comparePrompts(input) {
+  const examples = loadDataset(input.dataset);
+  const cap = typeof input.maxExamples === "number" && input.maxExamples > 0 ? Math.min(input.maxExamples, examples.length) : examples.length;
+  const limited = examples.slice(0, cap);
+  if (limited.length === 0) {
+    return emptyResult(input);
+  }
+  const adapter = await resolveAdapter(input);
+  const scorerKind = input.scorer ?? (input.task === "action_planner" ? "planner_action" : "agreement");
+  const compare = scorerKind === "planner_action" ? scorePlannerAction : scoreAgreement;
+  const mode = input.mode ?? "vs_historical";
+  if (mode === "pairwise") {
+    return runPairwise({
+      adapter,
+      baselinePrompt: input.baselinePrompt,
+      variantPrompt: input.variantPrompt,
+      examples: limited,
+      compare,
+      scorerKind,
+      temperature: input.temperature ?? 0,
+      maxTokens: input.maxTokens ?? 512,
+      tolerance: DEFAULT_REGRESSION_TOLERANCE
+    });
+  }
+  const scorer = createPromptScorer(adapter, {
+    compare,
+    temperature: input.temperature ?? 0,
+    maxTokens: input.maxTokens ?? 512
+  });
+  const baselineScore = await scorer(input.baselinePrompt, limited);
+  const variantScore = await scorer(input.variantPrompt, limited);
+  return finalize({
+    baselineScore,
+    variantScore,
+    examplesScored: limited.length,
+    scorerKind,
+    mode,
+    tolerance: DEFAULT_REGRESSION_TOLERANCE
+  });
+}
+function loadDataset(dataset) {
+  if (typeof dataset !== "string") return dataset;
+  if (!existsSync(dataset)) {
+    throw new Error(`[prompt-compare] dataset not found at ${dataset}`);
+  }
+  const raw = readFileSync(dataset, "utf-8");
+  const lines = raw.split("\n").filter((line) => line.trim().length > 0);
+  const examples = [];
+  let index = 0;
+  for (const line of lines) {
+    const example = jsonlLineToExample(line, index);
+    if (example) examples.push(example);
+    index += 1;
+  }
+  return examples;
+}
+function jsonlLineToExample(line, index) {
+  let parsed;
+  try {
+    parsed = JSON.parse(line);
+  } catch {
+    return null;
+  }
+  let system;
+  let user;
+  let expected;
+  if (typeof parsed.request?.system === "string" && parsed.request.system) {
+    system = parsed.request.system;
+  }
+  for (const msg of parsed.request?.messages ?? []) {
+    if (!system && msg.role === "system" && typeof msg.content === "string") {
+      system = msg.content;
+    }
+    if (msg.role === "user" && typeof msg.content === "string") {
+      user = user ? `${user}
+${msg.content}` : msg.content;
+    }
+    if (msg.role === "assistant" && typeof msg.content === "string") {
+      expected = msg.content;
+    }
+  }
+  if (!user && typeof parsed.request?.prompt === "string") {
+    user = parsed.request.prompt;
+  }
+  if (parsed.response) {
+    if (typeof parsed.response.text === "string" && parsed.response.text) {
+      expected = parsed.response.text;
+    } else if (Array.isArray(parsed.response.toolCalls)) {
+      expected = JSON.stringify({ toolCalls: parsed.response.toolCalls });
+    }
+  }
+  if (!user || !expected) return null;
+  return {
+    id: `row-${index}`,
+    input: { system, user },
+    expectedOutput: expected
+  };
+}
+async function resolveAdapter(input) {
+  if (input.adapter) return input.adapter;
+  const trainProvider = process.env.TRAIN_MODEL_PROVIDER?.trim() ?? process.env.TRAINING_PROVIDER?.trim();
+  if (trainProvider === "cerebras") {
+    const { getTrainingUseModelAdapter } = await import("./cerebras-eval-model.js");
+    return createRuntimeAdapter(getTrainingUseModelAdapter());
+  }
+  if (!input.runtime) {
+    throw new Error(
+      "[prompt-compare] either `runtime` or `adapter` must be provided"
+    );
+  }
+  return createRuntimeAdapter(input.runtime.useModel);
+}
+async function runPairwise(input) {
+  let baselineToReference = 0;
+  let variantToReference = 0;
+  let variantToBaseline = 0;
+  for (const example of input.examples) {
+    const baselineOutput = await input.adapter.complete({
+      system: input.baselinePrompt,
+      user: example.input.user,
+      temperature: input.temperature,
+      maxTokens: input.maxTokens
+    });
+    const variantOutput = await input.adapter.complete({
+      system: input.variantPrompt,
+      user: example.input.user,
+      temperature: input.temperature,
+      maxTokens: input.maxTokens
+    });
+    baselineToReference += input.compare(
+      baselineOutput,
+      example.expectedOutput
+    );
+    variantToReference += input.compare(variantOutput, example.expectedOutput);
+    variantToBaseline += input.compare(variantOutput, baselineOutput);
+  }
+  const n = input.examples.length;
+  const baselineScore = baselineToReference / n;
+  const variantScore = variantToReference / n;
+  const result = finalize({
+    baselineScore,
+    variantScore,
+    examplesScored: n,
+    scorerKind: input.scorerKind,
+    mode: "pairwise",
+    tolerance: input.tolerance
+  });
+  const pairwise = variantToBaseline / n;
+  return {
+    ...result,
+    delta: pairwise - 1,
+    deltaPercent: (pairwise - 1) * 100,
+    passed: pairwise + input.tolerance >= 1
+  };
+}
+function finalize(input) {
+  const delta = input.variantScore - input.baselineScore;
+  const deltaPercent = input.baselineScore === 0 ? 0 : delta / input.baselineScore * 100;
+  return {
+    baselineScore: input.baselineScore,
+    variantScore: input.variantScore,
+    delta,
+    deltaPercent,
+    examplesScored: input.examplesScored,
+    scorer: input.scorerKind,
+    mode: input.mode,
+    passed: delta + input.tolerance >= 0,
+    tolerance: input.tolerance
+  };
+}
+function emptyResult(input) {
+  const scorer = input.scorer ?? (input.task === "action_planner" ? "planner_action" : "agreement");
+  return {
+    baselineScore: 0,
+    variantScore: 0,
+    delta: 0,
+    deltaPercent: 0,
+    examplesScored: 0,
+    scorer,
+    mode: input.mode ?? "vs_historical",
+    passed: true,
+    tolerance: DEFAULT_REGRESSION_TOLERANCE
+  };
+}
+function formatComparisonSummary(result) {
+  const sign = result.delta >= 0 ? "+" : "";
+  const verdict = result.passed ? "PASS" : "FAIL";
+  return [
+    `[prompt-compare] ${verdict} mode=${result.mode} scorer=${result.scorer}`,
+    `n=${result.examplesScored}`,
+    `baseline=${result.baselineScore.toFixed(4)}`,
+    `variant=${result.variantScore.toFixed(4)}`,
+    `delta=${sign}${result.delta.toFixed(4)} (${sign}${result.deltaPercent.toFixed(2)}%)`,
+    `tolerance=${result.tolerance}`
+  ].join(" ");
+}
+export {
+  DEFAULT_REGRESSION_TOLERANCE,
+  comparePrompts,
+  formatComparisonSummary
+};
+//# sourceMappingURL=prompt-compare.js.map

package/dist/core/prompt-compare.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/core/prompt-compare.ts"],"sourcesContent":["/**\n * Prompt A/B comparison harness.\n *\n * Given two prompts (baseline + variant) and a dataset of historical\n * input/expected pairs, run each prompt through the same model and\n * report mean scores plus per-example deltas. Used to gate prompt\n * cleanup / compression changes before they ship: a variant that\n * regresses against the historical reference indicates a behavioral\n * change, not a pure cosmetic edit.\n *\n * The harness is a thin wrapper over the native optimizer scorer\n * (`createPromptScorer` from optimizers/scoring.ts) — that scorer is\n * already designed for prompt-vs-dataset evaluation, just with a\n * single prompt at a time. We invoke it twice and diff the results.\n *\n * Two scoring modes are supported:\n *\n * - `vs_historical` (default): each prompt is scored against the\n *   recorded `expectedOutput` (Jaccard token overlap by default,\n *   action-name match for the action_planner task). Cheap and\n *   deterministic. Both prompts are scored independently; the delta\n *   tells you whether the variant reproduces the historical output\n *   as well as the baseline does.\n *\n * - `pairwise`: run baseline on every example to capture v1 outputs,\n *   then run variant on the same inputs and compare v2 outputs to v1\n *   outputs directly (pairwise Jaccard). This answers \"did the\n *   variant produce semantically equivalent output?\", which is a\n *   stricter regression test than `vs_historical` because the latter\n *   is biased — historical outputs were likely produced by a prompt\n *   close to the baseline.\n *\n * No new model abstractions are introduced. Reuses:\n *   - `parseJsonlDataset()` from backends/native.ts (private — mirrored inline here to avoid exporting the training-backend parser)\n *   - `createRuntimeAdapter()` from optimizers/scoring.ts\n *   - `createPromptScorer()` from optimizers/scoring.ts\n *   - `scoreAgreement()` / `scorePlannerAction()` from optimizers/scoring.ts\n *\n * Cost note: N examples × 2 prompts = 2N model calls per run in\n * `vs_historical` mode; same in `pairwise` mode (baseline outputs are\n * captured once, variant once). Default temperature 0 for determinism.\n */\n\nimport { existsSync, readFileSync } from \"node:fs\";\nimport {\n  createPromptScorer,\n  createRuntimeAdapter,\n  type LlmAdapter,\n  type OptimizationExample,\n  scoreAgreement,\n  scorePlannerAction,\n  type UseModelHandler,\n} from \"../optimizers/index.js\";\nimport type { TrajectoryTrainingTask } from \"./trajectory-task-datasets.js\";\n\nexport type ScorerKind = \"agreement\" | \"planner_action\";\nexport type CompareMode = \"vs_historical\" | \"pairwise\";\n\nexport interface PromptComparisonInput {\n  /** System prompt under test as the baseline (often the current canonical prompt). */\n  baselinePrompt: string;\n  /** System prompt under test as the variant (e.g. caveman-compressed). */\n  variantPrompt: string;\n  /** Dataset of `(input, expectedOutput)` rows. Path to a JSONL file produced by `exportTrajectoryTaskDatasets`, or an in-memory array. */\n  dataset: string | OptimizationExample[];\n  /** Task hint — selects the right scorer when `scorer` is omitted. Defaults to `agreement`. */\n  task?: TrajectoryTrainingTask;\n  /** Force a specific scorer regardless of task. */\n  scorer?: ScorerKind;\n  /** Cap how many examples to score (handy for cheap previews). */\n  maxExamples?: number;\n  /** Compare mode: `vs_historical` (default) or `pairwise`. */\n  mode?: CompareMode;\n  /** Temperature passed to the adapter. Defaults to 0 for determinism. */\n  temperature?: number;\n  /** Max tokens per completion. Defaults to 512. */\n  maxTokens?: number;\n  /** Loose runtime shape — only `useModel` is required. Mutually exclusive with `adapter`. */\n  runtime?: { useModel: UseModelHandler };\n  /** Pre-built LLM adapter (tests, alternative providers). */\n  adapter?: LlmAdapter;\n}\n\nexport interface PromptComparisonResult {\n  baselineScore: number;\n  variantScore: number;\n  /** `variantScore - baselineScore`. Positive means variant is closer to reference. */\n  delta: number;\n  /** Percentage delta, where 0 baseline collapses to 0 to avoid divide-by-zero. */\n  deltaPercent: number;\n  examplesScored: number;\n  scorer: ScorerKind;\n  mode: CompareMode;\n  /** True when the variant did not measurably regress (delta ≥ -tolerance). */\n  passed: boolean;\n  /** Tolerance applied to `passed`. Defaults to 0.02 (2 percentage points). */\n  tolerance: number;\n}\n\n/** Default tolerance: a variant is considered safe if its score is within\n *  2 percentage points of the baseline. Tunable per call. */\nexport const DEFAULT_REGRESSION_TOLERANCE = 0.02;\n\n/**\n * Compare two prompts on the same dataset and report mean scores plus\n * delta. Throws on dataset I/O errors; never throws for \"variant is\n * worse\" — read `result.passed` for the gate decision.\n */\nexport async function comparePrompts(\n  input: PromptComparisonInput,\n): Promise<PromptComparisonResult> {\n  const examples = loadDataset(input.dataset);\n  const cap =\n    typeof input.maxExamples === \"number\" && input.maxExamples > 0\n      ? Math.min(input.maxExamples, examples.length)\n      : examples.length;\n  const limited = examples.slice(0, cap);\n\n  if (limited.length === 0) {\n    return emptyResult(input);\n  }\n\n  const adapter = await resolveAdapter(input);\n  const scorerKind: ScorerKind =\n    input.scorer ??\n    (input.task === \"action_planner\" ? \"planner_action\" : \"agreement\");\n  const compare =\n    scorerKind === \"planner_action\" ? scorePlannerAction : scoreAgreement;\n  const mode: CompareMode = input.mode ?? \"vs_historical\";\n\n  if (mode === \"pairwise\") {\n    return runPairwise({\n      adapter,\n      baselinePrompt: input.baselinePrompt,\n      variantPrompt: input.variantPrompt,\n      examples: limited,\n      compare,\n      scorerKind,\n      temperature: input.temperature ?? 0,\n      maxTokens: input.maxTokens ?? 512,\n      tolerance: DEFAULT_REGRESSION_TOLERANCE,\n    });\n  }\n\n  const scorer = createPromptScorer(adapter, {\n    compare,\n    temperature: input.temperature ?? 0,\n    maxTokens: input.maxTokens ?? 512,\n  });\n  const baselineScore = await scorer(input.baselinePrompt, limited);\n  const variantScore = await scorer(input.variantPrompt, limited);\n\n  return finalize({\n    baselineScore,\n    variantScore,\n    examplesScored: limited.length,\n    scorerKind,\n    mode,\n    tolerance: DEFAULT_REGRESSION_TOLERANCE,\n  });\n}\n\nfunction loadDataset(\n  dataset: string | OptimizationExample[],\n): OptimizationExample[] {\n  if (typeof dataset !== \"string\") return dataset;\n  if (!existsSync(dataset)) {\n    throw new Error(`[prompt-compare] dataset not found at ${dataset}`);\n  }\n  const raw = readFileSync(dataset, \"utf-8\");\n  const lines = raw.split(\"\\n\").filter((line) => line.trim().length > 0);\n  const examples: OptimizationExample[] = [];\n  let index = 0;\n  for (const line of lines) {\n    const example = jsonlLineToExample(line, index);\n    if (example) examples.push(example);\n    index += 1;\n  }\n  return examples;\n}\n\ninterface JsonlMessage {\n  role: \"system\" | \"developer\" | \"user\" | \"assistant\" | \"tool\";\n  content: string;\n}\n\ninterface JsonlRow {\n  format?: string;\n  request?: { system?: string; prompt?: string; messages?: JsonlMessage[] };\n  response?: { text?: string; toolCalls?: unknown[] };\n}\n\n/** Parse one `eliza_native_v1` row to an OptimizationExample. Mirrors\n *  `rowToExample()` in backends/native.ts; copied here to avoid an\n *  import cycle and to accept rows that don't carry the `boundary`\n *  field (older exports). */\nfunction jsonlLineToExample(\n  line: string,\n  index: number,\n): OptimizationExample | null {\n  let parsed: JsonlRow;\n  try {\n    parsed = JSON.parse(line) as JsonlRow;\n  } catch {\n    return null;\n  }\n  let system: string | undefined;\n  let user: string | undefined;\n  let expected: string | undefined;\n  if (typeof parsed.request?.system === \"string\" && parsed.request.system) {\n    system = parsed.request.system;\n  }\n  for (const msg of parsed.request?.messages ?? []) {\n    if (!system && msg.role === \"system\" && typeof msg.content === \"string\") {\n      system = msg.content;\n    }\n    if (msg.role === \"user\" && typeof msg.content === \"string\") {\n      user = user ? `${user}\\n${msg.content}` : msg.content;\n    }\n    if (msg.role === \"assistant\" && typeof msg.content === \"string\") {\n      expected = msg.content;\n    }\n  }\n  if (!user && typeof parsed.request?.prompt === \"string\") {\n    user = parsed.request.prompt;\n  }\n  if (parsed.response) {\n    if (typeof parsed.response.text === \"string\" && parsed.response.text) {\n      expected = parsed.response.text;\n    } else if (Array.isArray(parsed.response.toolCalls)) {\n      expected = JSON.stringify({ toolCalls: parsed.response.toolCalls });\n    }\n  }\n  if (!user || !expected) return null;\n  return {\n    id: `row-${index}`,\n    input: { system, user },\n    expectedOutput: expected,\n  };\n}\n\nasync function resolveAdapter(\n  input: PromptComparisonInput,\n): Promise<LlmAdapter> {\n  if (input.adapter) return input.adapter;\n  // Standing direction: training-side comparison runs on Cerebras\n  // gpt-oss-120b unless the operator passes their own adapter.\n  const trainProvider =\n    process.env.TRAIN_MODEL_PROVIDER?.trim() ??\n    process.env.TRAINING_PROVIDER?.trim();\n  if (trainProvider === \"cerebras\") {\n    const { getTrainingUseModelAdapter } = await import(\n      \"./cerebras-eval-model.js\"\n    );\n    return createRuntimeAdapter(getTrainingUseModelAdapter());\n  }\n  if (!input.runtime) {\n    throw new Error(\n      \"[prompt-compare] either `runtime` or `adapter` must be provided\",\n    );\n  }\n  return createRuntimeAdapter(input.runtime.useModel);\n}\n\ninterface PairwiseInput {\n  adapter: LlmAdapter;\n  baselinePrompt: string;\n  variantPrompt: string;\n  examples: OptimizationExample[];\n  compare: (actual: string, expected: string) => number;\n  scorerKind: ScorerKind;\n  temperature: number;\n  maxTokens: number;\n  tolerance: number;\n}\n\n/** Pairwise mode: capture baseline outputs, then compare variant\n *  outputs to those captured baselines. Both `baselineScore` and\n *  `variantScore` are reported as similarity-to-historical (same as\n *  vs_historical mode) so the two modes report a comparable axis;\n *  `delta` here additionally reflects mean pairwise self-similarity\n *  via the same compare function, which is its strength as a\n *  regression test. */\nasync function runPairwise(\n  input: PairwiseInput,\n): Promise<PromptComparisonResult> {\n  let baselineToReference = 0;\n  let variantToReference = 0;\n  let variantToBaseline = 0;\n  for (const example of input.examples) {\n    const baselineOutput = await input.adapter.complete({\n      system: input.baselinePrompt,\n      user: example.input.user,\n      temperature: input.temperature,\n      maxTokens: input.maxTokens,\n    });\n    const variantOutput = await input.adapter.complete({\n      system: input.variantPrompt,\n      user: example.input.user,\n      temperature: input.temperature,\n      maxTokens: input.maxTokens,\n    });\n    baselineToReference += input.compare(\n      baselineOutput,\n      example.expectedOutput,\n    );\n    variantToReference += input.compare(variantOutput, example.expectedOutput);\n    variantToBaseline += input.compare(variantOutput, baselineOutput);\n  }\n  const n = input.examples.length;\n  const baselineScore = baselineToReference / n;\n  const variantScore = variantToReference / n;\n  const result = finalize({\n    baselineScore,\n    variantScore,\n    examplesScored: n,\n    scorerKind: input.scorerKind,\n    mode: \"pairwise\",\n    tolerance: input.tolerance,\n  });\n  // Replace delta with the pairwise self-similarity signal; deltaPercent\n  // becomes the gap between variant→baseline similarity and 1.0.\n  const pairwise = variantToBaseline / n;\n  return {\n    ...result,\n    delta: pairwise - 1,\n    deltaPercent: (pairwise - 1) * 100,\n    passed: pairwise + input.tolerance >= 1,\n  };\n}\n\ninterface FinalizeInput {\n  baselineScore: number;\n  variantScore: number;\n  examplesScored: number;\n  scorerKind: ScorerKind;\n  mode: CompareMode;\n  tolerance: number;\n}\n\nfunction finalize(input: FinalizeInput): PromptComparisonResult {\n  const delta = input.variantScore - input.baselineScore;\n  const deltaPercent =\n    input.baselineScore === 0 ? 0 : (delta / input.baselineScore) * 100;\n  return {\n    baselineScore: input.baselineScore,\n    variantScore: input.variantScore,\n    delta,\n    deltaPercent,\n    examplesScored: input.examplesScored,\n    scorer: input.scorerKind,\n    mode: input.mode,\n    passed: delta + input.tolerance >= 0,\n    tolerance: input.tolerance,\n  };\n}\n\nfunction emptyResult(input: PromptComparisonInput): PromptComparisonResult {\n  const scorer: ScorerKind =\n    input.scorer ??\n    (input.task === \"action_planner\" ? \"planner_action\" : \"agreement\");\n  return {\n    baselineScore: 0,\n    variantScore: 0,\n    delta: 0,\n    deltaPercent: 0,\n    examplesScored: 0,\n    scorer,\n    mode: input.mode ?? \"vs_historical\",\n    passed: true,\n    tolerance: DEFAULT_REGRESSION_TOLERANCE,\n  };\n}\n\n/** Render a result as a single-line summary suitable for CLI output. */\nexport function formatComparisonSummary(\n  result: PromptComparisonResult,\n): string {\n  const sign = result.delta >= 0 ? \"+\" : \"\";\n  const verdict = result.passed ? \"PASS\" : \"FAIL\";\n  return [\n    `[prompt-compare] ${verdict} mode=${result.mode} scorer=${result.scorer}`,\n    `n=${result.examplesScored}`,\n    `baseline=${result.baselineScore.toFixed(4)}`,\n    `variant=${result.variantScore.toFixed(4)}`,\n    `delta=${sign}${result.delta.toFixed(4)} (${sign}${result.deltaPercent.toFixed(2)}%)`,\n    `tolerance=${result.tolerance}`,\n  ].join(\" \");\n}\n"],"mappings":"AA2CA,SAAS,YAAY,oBAAoB;AACzC;AAAA,EACE;AAAA,EACA;AAAA,EAGA;AAAA,EACA;AAAA,OAEK;AAiDA,MAAM,+BAA+B;AAO5C,eAAsB,eACpB,OACiC;AACjC,QAAM,WAAW,YAAY,MAAM,OAAO;AAC1C,QAAM,MACJ,OAAO,MAAM,gBAAgB,YAAY,MAAM,cAAc,IACzD,KAAK,IAAI,MAAM,aAAa,SAAS,MAAM,IAC3C,SAAS;AACf,QAAM,UAAU,SAAS,MAAM,GAAG,GAAG;AAErC,MAAI,QAAQ,WAAW,GAAG;AACxB,WAAO,YAAY,KAAK;AAAA,EAC1B;AAEA,QAAM,UAAU,MAAM,eAAe,KAAK;AAC1C,QAAM,aACJ,MAAM,WACL,MAAM,SAAS,mBAAmB,mBAAmB;AACxD,QAAM,UACJ,eAAe,mBAAmB,qBAAqB;AACzD,QAAM,OAAoB,MAAM,QAAQ;AAExC,MAAI,SAAS,YAAY;AACvB,WAAO,YAAY;AAAA,MACjB;AAAA,MACA,gBAAgB,MAAM;AAAA,MACtB,eAAe,MAAM;AAAA,MACrB,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA,aAAa,MAAM,eAAe;AAAA,MAClC,WAAW,MAAM,aAAa;AAAA,MAC9B,WAAW;AAAA,IACb,CAAC;AAAA,EACH;AAEA,QAAM,SAAS,mBAAmB,SAAS;AAAA,IACzC;AAAA,IACA,aAAa,MAAM,eAAe;AAAA,IAClC,WAAW,MAAM,aAAa;AAAA,EAChC,CAAC;AACD,QAAM,gBAAgB,MAAM,OAAO,MAAM,gBAAgB,OAAO;AAChE,QAAM,eAAe,MAAM,OAAO,MAAM,eAAe,OAAO;AAE9D,SAAO,SAAS;AAAA,IACd;AAAA,IACA;AAAA,IACA,gBAAgB,QAAQ;AAAA,IACxB;AAAA,IACA;AAAA,IACA,WAAW;AAAA,EACb,CAAC;AACH;AAEA,SAAS,YACP,SACuB;AACvB,MAAI,OAAO,YAAY,SAAU,QAAO;AACxC,MAAI,CAAC,WAAW,OAAO,GAAG;AACxB,UAAM,IAAI,MAAM,yCAAyC,OAAO,EAAE;AAAA,EACpE;AACA,QAAM,MAAM,aAAa,SAAS,OAAO;AACzC,QAAM,QAAQ,IAAI,MAAM,IAAI,EAAE,OAAO,CAAC,SAAS,KAAK,KAAK,EAAE,SAAS,CAAC;AACrE,QAAM,WAAkC,CAAC;AACzC,MAAI,QAAQ;AACZ,aAAW,QAAQ,OAAO;AACxB,UAAM,UAAU,mBAAmB,MAAM,KAAK;AAC9C,QAAI,QAAS,UAAS,KAAK,OAAO;AAClC,aAAS;AAAA,EACX;AACA,SAAO;AACT;AAiBA,SAAS,mBACP,MACA,OAC4B;AAC5B,MAAI;AACJ,MAAI;AACF,aAAS,KAAK,MAAM,IAAI;AAAA,EAC1B,QAAQ;AACN,WAAO;AAAA,EACT;AACA,MAAI;AACJ,MAAI;AACJ,MAAI;AACJ,MAAI,OAAO,OAAO,SAAS,WAAW,YAAY,OAAO,QAAQ,QAAQ;AACvE,aAAS,OAAO,QAAQ;AAAA,EAC1B;AACA,aAAW,OAAO,OAAO,SAAS,YAAY,CAAC,GAAG;AAChD,QAAI,CAAC,UAAU,IAAI,SAAS,YAAY,OAAO,IAAI,YAAY,UAAU;AACvE,eAAS,IAAI;AAAA,IACf;AACA,QAAI,IAAI,SAAS,UAAU,OAAO,IAAI,YAAY,UAAU;AAC1D,aAAO,OAAO,GAAG,IAAI;AAAA,EAAK,IAAI,OAAO,KAAK,IAAI;AAAA,IAChD;AACA,QAAI,IAAI,SAAS,eAAe,OAAO,IAAI,YAAY,UAAU;AAC/D,iBAAW,IAAI;AAAA,IACjB;AAAA,EACF;AACA,MAAI,CAAC,QAAQ,OAAO,OAAO,SAAS,WAAW,UAAU;AACvD,WAAO,OAAO,QAAQ;AAAA,EACxB;AACA,MAAI,OAAO,UAAU;AACnB,QAAI,OAAO,OAAO,SAAS,SAAS,YAAY,OAAO,SAAS,MAAM;AACpE,iBAAW,OAAO,SAAS;AAAA,IAC7B,WAAW,MAAM,QAAQ,OAAO,SAAS,SAAS,GAAG;AACnD,iBAAW,KAAK,UAAU,EAAE,WAAW,OAAO,SAAS,UAAU,CAAC;AAAA,IACpE;AAAA,EACF;AACA,MAAI,CAAC,QAAQ,CAAC,SAAU,QAAO;AAC/B,SAAO;AAAA,IACL,IAAI,OAAO,KAAK;AAAA,IAChB,OAAO,EAAE,QAAQ,KAAK;AAAA,IACtB,gBAAgB;AAAA,EAClB;AACF;AAEA,eAAe,eACb,OACqB;AACrB,MAAI,MAAM,QAAS,QAAO,MAAM;AAGhC,QAAM,gBACJ,QAAQ,IAAI,sBAAsB,KAAK,KACvC,QAAQ,IAAI,mBAAmB,KAAK;AACtC,MAAI,kBAAkB,YAAY;AAChC,UAAM,EAAE,2BAA2B,IAAI,MAAM,OAC3C,0BACF;AACA,WAAO,qBAAqB,2BAA2B,CAAC;AAAA,EAC1D;AACA,MAAI,CAAC,MAAM,SAAS;AAClB,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACA,SAAO,qBAAqB,MAAM,QAAQ,QAAQ;AACpD;AAqBA,eAAe,YACb,OACiC;AACjC,MAAI,sBAAsB;AAC1B,MAAI,qBAAqB;AACzB,MAAI,oBAAoB;AACxB,aAAW,WAAW,MAAM,UAAU;AACpC,UAAM,iBAAiB,MAAM,MAAM,QAAQ,SAAS;AAAA,MAClD,QAAQ,MAAM;AAAA,MACd,MAAM,QAAQ,MAAM;AAAA,MACpB,aAAa,MAAM;AAAA,MACnB,WAAW,MAAM;AAAA,IACnB,CAAC;AACD,UAAM,gBAAgB,MAAM,MAAM,QAAQ,SAAS;AAAA,MACjD,QAAQ,MAAM;AAAA,MACd,MAAM,QAAQ,MAAM;AAAA,MACpB,aAAa,MAAM;AAAA,MACnB,WAAW,MAAM;AAAA,IACnB,CAAC;AACD,2BAAuB,MAAM;AAAA,MAC3B;AAAA,MACA,QAAQ;AAAA,IACV;AACA,0BAAsB,MAAM,QAAQ,eAAe,QAAQ,cAAc;AACzE,yBAAqB,MAAM,QAAQ,eAAe,cAAc;AAAA,EAClE;AACA,QAAM,IAAI,MAAM,SAAS;AACzB,QAAM,gBAAgB,sBAAsB;AAC5C,QAAM,eAAe,qBAAqB;AAC1C,QAAM,SAAS,SAAS;AAAA,IACtB;AAAA,IACA;AAAA,IACA,gBAAgB;AAAA,IAChB,YAAY,MAAM;AAAA,IAClB,MAAM;AAAA,IACN,WAAW,MAAM;AAAA,EACnB,CAAC;AAGD,QAAM,WAAW,oBAAoB;AACrC,SAAO;AAAA,IACL,GAAG;AAAA,IACH,OAAO,WAAW;AAAA,IAClB,eAAe,WAAW,KAAK;AAAA,IAC/B,QAAQ,WAAW,MAAM,aAAa;AAAA,EACxC;AACF;AAWA,SAAS,SAAS,OAA8C;AAC9D,QAAM,QAAQ,MAAM,eAAe,MAAM;AACzC,QAAM,eACJ,MAAM,kBAAkB,IAAI,IAAK,QAAQ,MAAM,gBAAiB;AAClE,SAAO;AAAA,IACL,eAAe,MAAM;AAAA,IACrB,cAAc,MAAM;AAAA,IACpB;AAAA,IACA;AAAA,IACA,gBAAgB,MAAM;AAAA,IACtB,QAAQ,MAAM;AAAA,IACd,MAAM,MAAM;AAAA,IACZ,QAAQ,QAAQ,MAAM,aAAa;AAAA,IACnC,WAAW,MAAM;AAAA,EACnB;AACF;AAEA,SAAS,YAAY,OAAsD;AACzE,QAAM,SACJ,MAAM,WACL,MAAM,SAAS,mBAAmB,mBAAmB;AACxD,SAAO;AAAA,IACL,eAAe;AAAA,IACf,cAAc;AAAA,IACd,OAAO;AAAA,IACP,cAAc;AAAA,IACd,gBAAgB;AAAA,IAChB;AAAA,IACA,MAAM,MAAM,QAAQ;AAAA,IACpB,QAAQ;AAAA,IACR,WAAW;AAAA,EACb;AACF;AAGO,SAAS,wBACd,QACQ;AACR,QAAM,OAAO,OAAO,SAAS,IAAI,MAAM;AACvC,QAAM,UAAU,OAAO,SAAS,SAAS;AACzC,SAAO;AAAA,IACL,oBAAoB,OAAO,SAAS,OAAO,IAAI,WAAW,OAAO,MAAM;AAAA,IACvE,KAAK,OAAO,cAAc;AAAA,IAC1B,YAAY,OAAO,cAAc,QAAQ,CAAC,CAAC;AAAA,IAC3C,WAAW,OAAO,aAAa,QAAQ,CAAC,CAAC;AAAA,IACzC,SAAS,IAAI,GAAG,OAAO,MAAM,QAAQ,CAAC,CAAC,KAAK,IAAI,GAAG,OAAO,aAAa,QAAQ,CAAC,CAAC;AAAA,IACjF,aAAa,OAAO,SAAS;AAAA,EAC/B,EAAE,KAAK,GAAG;AACZ;","names":[]}