@elizaos/plugin-training 2.0.3-beta.5 → 2.0.3-beta.7

package/dist/optimizers/instruction-search.js

@@ -0,0 +1,108 @@
+import { subsample } from "./scoring.js";
+const REWRITE_INSTRUCTIONS = `Rewrite the SYSTEM PROMPT to make a task planner work better. The planner sees a tool list, conversation context, and a user message; it returns one JSON object with toolCalls or messageToUser.
+
+Hard constraints:
+- Preserve every input/output requirement from the original (placeholders, JSON schema, terminal-message rules).
+- Keep all literal placeholders like {{contextObject}}, {{trajectory}}, {{agentName}}, {{providers}} byte-identical and in the same locations.
+- The planner must use only the tool names that appear in the runtime-injected tool list. Do NOT enumerate any specific tool names, action names, or connector names in the rewrite \u2014 those are dynamically scoped per turn and hardcoding them in the prompt teaches the model to invent or stick to a stale list.
+- Output length must not exceed 1.3x the original prompt's character count. Aim for shorter than the original when possible.
+
+Style rules (anti-meme):
+- Imperative voice. No "You are X", no "Your job is", no role-playing framing.
+- No markdown headers, no fenced code blocks, no decorative bullets, no emojis.
+- No instructions to "still output the action with empty args" \u2014 actions need real arguments.
+- No restriction to a single tool call when the original allows a queue.
+
+Do:
+- Add concise task-specific guardrails grounded in observed failure modes.
+- Tighten phrasing of existing rules.
+- Preserve plural tool-call behaviour when the original mentions a "queue" or multiple tools.
+
+Output the rewritten prompt body only. No commentary, no preamble, no code fences.`;
+async function runInstructionSearch(input) {
+  const variants = input.options?.variants ?? 8;
+  const rounds = input.options?.rounds ?? 3;
+  const temperature = input.options?.temperature ?? 0.7;
+  const maxTokens = input.options?.maxTokens ?? 1024;
+  const rng = input.options?.rng ?? Math.random;
+  const lineage = [];
+  const heldOut = typeof input.options?.scoringSubset === "number" ? subsample(input.dataset, input.options.scoringSubset, rng) : input.dataset;
+  const baselineScore = await input.scorer(input.baselinePrompt, heldOut);
+  lineage.push({
+    round: 0,
+    variant: 0,
+    score: baselineScore,
+    notes: "baseline"
+  });
+  let bestPrompt = input.baselinePrompt;
+  let bestScore = baselineScore;
+  let currentBaseline = input.baselinePrompt;
+  for (let round = 1; round <= rounds; round += 1) {
+    let roundBestPrompt = currentBaseline;
+    let roundBestScore = bestScore;
+    for (let variant = 1; variant <= variants; variant += 1) {
+      const candidate = await input.llm.complete({
+        system: REWRITE_INSTRUCTIONS,
+        user: currentBaseline,
+        temperature,
+        maxTokens
+      });
+      const cleaned = candidate.trim();
+      if (cleaned.length === 0) {
+        lineage.push({
+          round,
+          variant,
+          score: 0,
+          notes: "empty rewrite \u2014 skipped"
+        });
+        continue;
+      }
+      const memeViolations = [
+        /^\s*you are\b/i,
+        /^\s*your job is\b/i,
+        /^\s*you're\b/i
+      ];
+      const memeHit = memeViolations.find((re) => re.test(cleaned));
+      if (memeHit) {
+        lineage.push({
+          round,
+          variant,
+          score: 0,
+          notes: `rejected: role-play opener (${memeHit})`
+        });
+        continue;
+      }
+      const lengthCap = Math.ceil(currentBaseline.length * 1.3);
+      if (cleaned.length > lengthCap) {
+        lineage.push({
+          round,
+          variant,
+          score: 0,
+          notes: `rejected: length ${cleaned.length} > cap ${lengthCap}`
+        });
+        continue;
+      }
+      const score = await input.scorer(cleaned, heldOut);
+      lineage.push({ round, variant, score });
+      if (score > roundBestScore) {
+        roundBestScore = score;
+        roundBestPrompt = cleaned;
+      }
+    }
+    if (roundBestScore > bestScore) {
+      bestScore = roundBestScore;
+      bestPrompt = roundBestPrompt;
+    }
+    currentBaseline = roundBestPrompt;
+  }
+  return {
+    optimizedPrompt: bestPrompt,
+    score: bestScore,
+    baseline: baselineScore,
+    lineage
+  };
+}
+export {
+  runInstructionSearch
+};
+//# sourceMappingURL=instruction-search.js.map

package/dist/optimizers/instruction-search.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/optimizers/instruction-search.ts"],"sourcesContent":["/**\n * MIPRO-style instruction search optimizer.\n *\n * Iterative improvement loop:\n *   1. Ask the LLM to propose N rewrites of the current baseline prompt that\n *      preserve the task contract but tighten language, add guardrails, or\n *      reorder for clarity.\n *   2. Score each candidate (plus the current baseline) on a held-out subset\n *      of the dataset.\n *   3. Keep the highest-scoring candidate as the next round's baseline.\n *   4. Repeat for `rounds` iterations.\n *\n * Returns the best prompt observed across all rounds along with full lineage\n * (every (round, variant) -> score), so the caller can render an audit trail.\n */\n\nimport { subsample } from \"./scoring.js\";\nimport type {\n  LlmAdapter,\n  OptimizationExample,\n  OptimizerLineageEntry,\n  OptimizerResult,\n  PromptScorer,\n} from \"./types.js\";\n\nexport interface InstructionSearchOptions {\n  /** Variants to propose per round. Defaults to 8. */\n  variants?: number;\n  /** Search rounds. Defaults to 3. */\n  rounds?: number;\n  /** Held-out examples scored per candidate. Defaults to all examples. */\n  scoringSubset?: number;\n  /** Sampling temperature for variant generation. Defaults to 0.7. */\n  temperature?: number;\n  /** Max tokens for the rewrite completion. Defaults to 1024. */\n  maxTokens?: number;\n  /** Deterministic RNG override (tests). Defaults to Math.random. */\n  rng?: () => number;\n}\n\nexport interface InstructionSearchInput {\n  baselinePrompt: string;\n  dataset: OptimizationExample[];\n  scorer: PromptScorer;\n  llm: LlmAdapter;\n  options?: InstructionSearchOptions;\n}\n\nconst REWRITE_INSTRUCTIONS = `Rewrite the SYSTEM PROMPT to make a task planner work better. The planner sees a tool list, conversation context, and a user message; it returns one JSON object with toolCalls or messageToUser.\n\nHard constraints:\n- Preserve every input/output requirement from the original (placeholders, JSON schema, terminal-message rules).\n- Keep all literal placeholders like {{contextObject}}, {{trajectory}}, {{agentName}}, {{providers}} byte-identical and in the same locations.\n- The planner must use only the tool names that appear in the runtime-injected tool list. Do NOT enumerate any specific tool names, action names, or connector names in the rewrite — those are dynamically scoped per turn and hardcoding them in the prompt teaches the model to invent or stick to a stale list.\n- Output length must not exceed 1.3x the original prompt's character count. Aim for shorter than the original when possible.\n\nStyle rules (anti-meme):\n- Imperative voice. No \"You are X\", no \"Your job is\", no role-playing framing.\n- No markdown headers, no fenced code blocks, no decorative bullets, no emojis.\n- No instructions to \"still output the action with empty args\" — actions need real arguments.\n- No restriction to a single tool call when the original allows a queue.\n\nDo:\n- Add concise task-specific guardrails grounded in observed failure modes.\n- Tighten phrasing of existing rules.\n- Preserve plural tool-call behaviour when the original mentions a \"queue\" or multiple tools.\n\nOutput the rewritten prompt body only. No commentary, no preamble, no code fences.`;\n\nexport async function runInstructionSearch(\n  input: InstructionSearchInput,\n): Promise<OptimizerResult> {\n  const variants = input.options?.variants ?? 8;\n  const rounds = input.options?.rounds ?? 3;\n  const temperature = input.options?.temperature ?? 0.7;\n  const maxTokens = input.options?.maxTokens ?? 1024;\n  const rng = input.options?.rng ?? Math.random;\n  const lineage: OptimizerLineageEntry[] = [];\n\n  const heldOut =\n    typeof input.options?.scoringSubset === \"number\"\n      ? subsample(input.dataset, input.options.scoringSubset, rng)\n      : input.dataset;\n\n  const baselineScore = await input.scorer(input.baselinePrompt, heldOut);\n  lineage.push({\n    round: 0,\n    variant: 0,\n    score: baselineScore,\n    notes: \"baseline\",\n  });\n\n  let bestPrompt = input.baselinePrompt;\n  let bestScore = baselineScore;\n  let currentBaseline = input.baselinePrompt;\n\n  for (let round = 1; round <= rounds; round += 1) {\n    let roundBestPrompt = currentBaseline;\n    let roundBestScore = bestScore;\n    for (let variant = 1; variant <= variants; variant += 1) {\n      const candidate = await input.llm.complete({\n        system: REWRITE_INSTRUCTIONS,\n        user: currentBaseline,\n        temperature,\n        maxTokens,\n      });\n      const cleaned = candidate.trim();\n      if (cleaned.length === 0) {\n        lineage.push({\n          round,\n          variant,\n          score: 0,\n          notes: \"empty rewrite — skipped\",\n        });\n        continue;\n      }\n      // Reject variants that violate the anti-meme rules. These patterns\n      // have been shown to reduce planner accuracy on Anthropic Haiku/Opus\n      // (role-play framing primes chat-completion behaviour over tool-use).\n      const memeViolations = [\n        /^\\s*you are\\b/i,\n        /^\\s*your job is\\b/i,\n        /^\\s*you're\\b/i,\n      ];\n      const memeHit = memeViolations.find((re) => re.test(cleaned));\n      if (memeHit) {\n        lineage.push({\n          round,\n          variant,\n          score: 0,\n          notes: `rejected: role-play opener (${memeHit})`,\n        });\n        continue;\n      }\n      const lengthCap = Math.ceil(currentBaseline.length * 1.3);\n      if (cleaned.length > lengthCap) {\n        lineage.push({\n          round,\n          variant,\n          score: 0,\n          notes: `rejected: length ${cleaned.length} > cap ${lengthCap}`,\n        });\n        continue;\n      }\n      const score = await input.scorer(cleaned, heldOut);\n      lineage.push({ round, variant, score });\n      if (score > roundBestScore) {\n        roundBestScore = score;\n        roundBestPrompt = cleaned;\n      }\n    }\n    if (roundBestScore > bestScore) {\n      bestScore = roundBestScore;\n      bestPrompt = roundBestPrompt;\n    }\n    // Carry the round winner forward as the next round's seed; this is the\n    // MIPRO greedy step. If no candidate improved, we still move forward to\n    // give the next round a chance with the same seed.\n    currentBaseline = roundBestPrompt;\n  }\n\n  return {\n    optimizedPrompt: bestPrompt,\n    score: bestScore,\n    baseline: baselineScore,\n    lineage,\n  };\n}\n"],"mappings":"AAgBA,SAAS,iBAAiB;AAgC1B,MAAM,uBAAuB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAqB7B,eAAsB,qBACpB,OAC0B;AAC1B,QAAM,WAAW,MAAM,SAAS,YAAY;AAC5C,QAAM,SAAS,MAAM,SAAS,UAAU;AACxC,QAAM,cAAc,MAAM,SAAS,eAAe;AAClD,QAAM,YAAY,MAAM,SAAS,aAAa;AAC9C,QAAM,MAAM,MAAM,SAAS,OAAO,KAAK;AACvC,QAAM,UAAmC,CAAC;AAE1C,QAAM,UACJ,OAAO,MAAM,SAAS,kBAAkB,WACpC,UAAU,MAAM,SAAS,MAAM,QAAQ,eAAe,GAAG,IACzD,MAAM;AAEZ,QAAM,gBAAgB,MAAM,MAAM,OAAO,MAAM,gBAAgB,OAAO;AACtE,UAAQ,KAAK;AAAA,IACX,OAAO;AAAA,IACP,SAAS;AAAA,IACT,OAAO;AAAA,IACP,OAAO;AAAA,EACT,CAAC;AAED,MAAI,aAAa,MAAM;AACvB,MAAI,YAAY;AAChB,MAAI,kBAAkB,MAAM;AAE5B,WAAS,QAAQ,GAAG,SAAS,QAAQ,SAAS,GAAG;AAC/C,QAAI,kBAAkB;AACtB,QAAI,iBAAiB;AACrB,aAAS,UAAU,GAAG,WAAW,UAAU,WAAW,GAAG;AACvD,YAAM,YAAY,MAAM,MAAM,IAAI,SAAS;AAAA,QACzC,QAAQ;AAAA,QACR,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF,CAAC;AACD,YAAM,UAAU,UAAU,KAAK;AAC/B,UAAI,QAAQ,WAAW,GAAG;AACxB,gBAAQ,KAAK;AAAA,UACX;AAAA,UACA;AAAA,UACA,OAAO;AAAA,UACP,OAAO;AAAA,QACT,CAAC;AACD;AAAA,MACF;AAIA,YAAM,iBAAiB;AAAA,QACrB;AAAA,QACA;AAAA,QACA;AAAA,MACF;AACA,YAAM,UAAU,eAAe,KAAK,CAAC,OAAO,GAAG,KAAK,OAAO,CAAC;AAC5D,UAAI,SAAS;AACX,gBAAQ,KAAK;AAAA,UACX;AAAA,UACA;AAAA,UACA,OAAO;AAAA,UACP,OAAO,+BAA+B,OAAO;AAAA,QAC/C,CAAC;AACD;AAAA,MACF;AACA,YAAM,YAAY,KAAK,KAAK,gBAAgB,SAAS,GAAG;AACxD,UAAI,QAAQ,SAAS,WAAW;AAC9B,gBAAQ,KAAK;AAAA,UACX;AAAA,UACA;AAAA,UACA,OAAO;AAAA,UACP,OAAO,oBAAoB,QAAQ,MAAM,UAAU,SAAS;AAAA,QAC9D,CAAC;AACD;AAAA,MACF;AACA,YAAM,QAAQ,MAAM,MAAM,OAAO,SAAS,OAAO;AACjD,cAAQ,KAAK,EAAE,OAAO,SAAS,MAAM,CAAC;AACtC,UAAI,QAAQ,gBAAgB;AAC1B,yBAAiB;AACjB,0BAAkB;AAAA,MACpB;AAAA,IACF;AACA,QAAI,iBAAiB,WAAW;AAC9B,kBAAY;AACZ,mBAAa;AAAA,IACf;AAIA,sBAAkB;AAAA,EACpB;AAEA,SAAO;AAAA,IACL,iBAAiB;AAAA,IACjB,OAAO;AAAA,IACP,UAAU;AAAA,IACV;AAAA,EACF;AACF;","names":[]}

package/dist/optimizers/prompt-evolution.d.ts

@@ -0,0 +1,39 @@
+/**
+ * GEPA-style genetic prompt evolution.
+ *
+ * Maintains a population of candidate prompts. Each generation:
+ *   1. Score every candidate on a held-out subset.
+ *   2. Select the top half as survivors.
+ *   3. For each survivor, with probability `mutationRate`, ask the LLM to
+ *      rewrite a slice (intro/rules/example) to produce a child.
+ *   4. Re-fill the population with mutated children of the survivors.
+ *
+ * The optimizer always preserves the all-time best prompt so an unlucky
+ * generation cannot regress past the baseline.
+ */
+import type { LlmAdapter, OptimizationExample, OptimizerResult, PromptScorer } from "./types.js";
+export interface PromptEvolutionOptions {
+    /** Population size. Defaults to 8. */
+    population?: number;
+    /** Generations. Defaults to 4. */
+    generations?: number;
+    /** Probability of mutating a survivor each generation. Defaults to 0.5. */
+    mutationRate?: number;
+    /** Held-out examples scored per candidate. Defaults to all examples. */
+    scoringSubset?: number;
+    /** Sampling temperature for mutation generation. Defaults to 0.8. */
+    temperature?: number;
+    /** Max tokens for the mutation completion. Defaults to 1024. */
+    maxTokens?: number;
+    /** Deterministic RNG override (tests). Defaults to Math.random. */
+    rng?: () => number;
+}
+export interface PromptEvolutionInput {
+    baselinePrompt: string;
+    dataset: OptimizationExample[];
+    scorer: PromptScorer;
+    llm: LlmAdapter;
+    options?: PromptEvolutionOptions;
+}
+export declare function runPromptEvolution(input: PromptEvolutionInput): Promise<OptimizerResult>;
+//# sourceMappingURL=prompt-evolution.d.ts.map

package/dist/optimizers/prompt-evolution.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt-evolution.d.ts","sourceRoot":"","sources":["../../src/optimizers/prompt-evolution.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,KAAK,EACV,UAAU,EACV,mBAAmB,EAEnB,eAAe,EACf,YAAY,EACb,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,sBAAsB;IACrC,sCAAsC;IACtC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,kCAAkC;IAClC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,wEAAwE;IACxE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,qEAAqE;IACrE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mEAAmE;IACnE,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,oBAAoB;IACnC,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,MAAM,EAAE,YAAY,CAAC;IACrB,GAAG,EAAE,UAAU,CAAC;IAChB,OAAO,CAAC,EAAE,sBAAsB,CAAC;CAClC;AAgBD,wBAAsB,kBAAkB,CACtC,KAAK,EAAE,oBAAoB,GAC1B,OAAO,CAAC,eAAe,CAAC,CA2F1B"}

package/dist/optimizers/prompt-evolution.js

@@ -0,0 +1,101 @@
+import { subsample } from "./scoring.js";
+const MUTATION_INSTRUCTIONS = `Mutate the SYSTEM PROMPT below to explore a different phrasing.
+
+Constraints:
+- Preserve the original task contract (inputs, outputs, format).
+- Keep all literal placeholders like {{agentName}} or {{providers}} intact.
+- Pick ONE section (intro, rules, examples, output format) and rewrite it.
+- Leave the other sections untouched, but reproduce them verbatim in the output.
+- Output ONLY the mutated prompt. No commentary, no fenced code blocks.`;
+async function runPromptEvolution(input) {
+  const population = Math.max(2, input.options?.population ?? 8);
+  const generations = input.options?.generations ?? 4;
+  const mutationRate = input.options?.mutationRate ?? 0.5;
+  const temperature = input.options?.temperature ?? 0.8;
+  const maxTokens = input.options?.maxTokens ?? 1024;
+  const rng = input.options?.rng ?? Math.random;
+  const lineage = [];
+  const heldOut = typeof input.options?.scoringSubset === "number" ? subsample(input.dataset, input.options.scoringSubset, rng) : input.dataset;
+  const baselineScore = await input.scorer(input.baselinePrompt, heldOut);
+  lineage.push({
+    round: 0,
+    variant: 0,
+    score: baselineScore,
+    notes: "baseline"
+  });
+  let pool = [
+    { prompt: input.baselinePrompt, score: baselineScore }
+  ];
+  for (let i = 1; i < population; i += 1) {
+    const seed = await mutate(input.llm, input.baselinePrompt, {
+      temperature,
+      maxTokens
+    });
+    const score = await input.scorer(seed, heldOut);
+    pool.push({ prompt: seed, score });
+    lineage.push({ round: 0, variant: i, score, notes: "seed mutation" });
+  }
+  let bestPrompt = input.baselinePrompt;
+  let bestScore = baselineScore;
+  for (const entry of pool) {
+    if (entry.score > bestScore) {
+      bestScore = entry.score;
+      bestPrompt = entry.prompt;
+    }
+  }
+  for (let gen = 1; gen <= generations; gen += 1) {
+    pool.sort((a, b) => b.score - a.score);
+    const cutoff = Math.max(1, Math.floor(pool.length / 2));
+    const survivors = pool.slice(0, cutoff);
+    const next = [...survivors];
+    let variantIdx = survivors.length;
+    while (next.length < population) {
+      const parent = survivors[next.length % survivors.length];
+      if (!parent) break;
+      const shouldMutate = rng() < mutationRate;
+      let childPrompt = parent.prompt;
+      if (shouldMutate) {
+        childPrompt = await mutate(input.llm, parent.prompt, {
+          temperature,
+          maxTokens
+        });
+      }
+      const score = await input.scorer(childPrompt, heldOut);
+      next.push({ prompt: childPrompt, score });
+      lineage.push({
+        round: gen,
+        variant: variantIdx,
+        score,
+        notes: shouldMutate ? "mutated child" : "carried forward"
+      });
+      variantIdx += 1;
+    }
+    pool = next;
+    for (const entry of pool) {
+      if (entry.score > bestScore) {
+        bestScore = entry.score;
+        bestPrompt = entry.prompt;
+      }
+    }
+  }
+  return {
+    optimizedPrompt: bestPrompt,
+    score: bestScore,
+    baseline: baselineScore,
+    lineage
+  };
+}
+async function mutate(llm, prompt, settings) {
+  const result = await llm.complete({
+    system: MUTATION_INSTRUCTIONS,
+    user: prompt,
+    temperature: settings.temperature,
+    maxTokens: settings.maxTokens
+  });
+  const cleaned = result.trim();
+  return cleaned.length > 0 ? cleaned : prompt;
+}
+export {
+  runPromptEvolution
+};
+//# sourceMappingURL=prompt-evolution.js.map

package/dist/optimizers/prompt-evolution.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/optimizers/prompt-evolution.ts"],"sourcesContent":["/**\n * GEPA-style genetic prompt evolution.\n *\n * Maintains a population of candidate prompts. Each generation:\n *   1. Score every candidate on a held-out subset.\n *   2. Select the top half as survivors.\n *   3. For each survivor, with probability `mutationRate`, ask the LLM to\n *      rewrite a slice (intro/rules/example) to produce a child.\n *   4. Re-fill the population with mutated children of the survivors.\n *\n * The optimizer always preserves the all-time best prompt so an unlucky\n * generation cannot regress past the baseline.\n */\n\nimport { subsample } from \"./scoring.js\";\nimport type {\n  LlmAdapter,\n  OptimizationExample,\n  OptimizerLineageEntry,\n  OptimizerResult,\n  PromptScorer,\n} from \"./types.js\";\n\nexport interface PromptEvolutionOptions {\n  /** Population size. Defaults to 8. */\n  population?: number;\n  /** Generations. Defaults to 4. */\n  generations?: number;\n  /** Probability of mutating a survivor each generation. Defaults to 0.5. */\n  mutationRate?: number;\n  /** Held-out examples scored per candidate. Defaults to all examples. */\n  scoringSubset?: number;\n  /** Sampling temperature for mutation generation. Defaults to 0.8. */\n  temperature?: number;\n  /** Max tokens for the mutation completion. Defaults to 1024. */\n  maxTokens?: number;\n  /** Deterministic RNG override (tests). Defaults to Math.random. */\n  rng?: () => number;\n}\n\nexport interface PromptEvolutionInput {\n  baselinePrompt: string;\n  dataset: OptimizationExample[];\n  scorer: PromptScorer;\n  llm: LlmAdapter;\n  options?: PromptEvolutionOptions;\n}\n\nconst MUTATION_INSTRUCTIONS = `Mutate the SYSTEM PROMPT below to explore a different phrasing.\n\nConstraints:\n- Preserve the original task contract (inputs, outputs, format).\n- Keep all literal placeholders like {{agentName}} or {{providers}} intact.\n- Pick ONE section (intro, rules, examples, output format) and rewrite it.\n- Leave the other sections untouched, but reproduce them verbatim in the output.\n- Output ONLY the mutated prompt. No commentary, no fenced code blocks.`;\n\ninterface ScoredPrompt {\n  prompt: string;\n  score: number;\n}\n\nexport async function runPromptEvolution(\n  input: PromptEvolutionInput,\n): Promise<OptimizerResult> {\n  const population = Math.max(2, input.options?.population ?? 8);\n  const generations = input.options?.generations ?? 4;\n  const mutationRate = input.options?.mutationRate ?? 0.5;\n  const temperature = input.options?.temperature ?? 0.8;\n  const maxTokens = input.options?.maxTokens ?? 1024;\n  const rng = input.options?.rng ?? Math.random;\n  const lineage: OptimizerLineageEntry[] = [];\n\n  const heldOut =\n    typeof input.options?.scoringSubset === \"number\"\n      ? subsample(input.dataset, input.options.scoringSubset, rng)\n      : input.dataset;\n\n  const baselineScore = await input.scorer(input.baselinePrompt, heldOut);\n  lineage.push({\n    round: 0,\n    variant: 0,\n    score: baselineScore,\n    notes: \"baseline\",\n  });\n\n  // Seed the population with mutations of the baseline so generation 0 is\n  // already diverse. The baseline itself stays at index 0 so the elite is\n  // always preserved.\n  let pool: ScoredPrompt[] = [\n    { prompt: input.baselinePrompt, score: baselineScore },\n  ];\n  for (let i = 1; i < population; i += 1) {\n    const seed = await mutate(input.llm, input.baselinePrompt, {\n      temperature,\n      maxTokens,\n    });\n    const score = await input.scorer(seed, heldOut);\n    pool.push({ prompt: seed, score });\n    lineage.push({ round: 0, variant: i, score, notes: \"seed mutation\" });\n  }\n\n  let bestPrompt = input.baselinePrompt;\n  let bestScore = baselineScore;\n  for (const entry of pool) {\n    if (entry.score > bestScore) {\n      bestScore = entry.score;\n      bestPrompt = entry.prompt;\n    }\n  }\n\n  for (let gen = 1; gen <= generations; gen += 1) {\n    pool.sort((a, b) => b.score - a.score);\n    const cutoff = Math.max(1, Math.floor(pool.length / 2));\n    const survivors = pool.slice(0, cutoff);\n    const next: ScoredPrompt[] = [...survivors];\n\n    let variantIdx = survivors.length;\n    while (next.length < population) {\n      const parent = survivors[next.length % survivors.length];\n      if (!parent) break;\n      const shouldMutate = rng() < mutationRate;\n      let childPrompt = parent.prompt;\n      if (shouldMutate) {\n        childPrompt = await mutate(input.llm, parent.prompt, {\n          temperature,\n          maxTokens,\n        });\n      }\n      const score = await input.scorer(childPrompt, heldOut);\n      next.push({ prompt: childPrompt, score });\n      lineage.push({\n        round: gen,\n        variant: variantIdx,\n        score,\n        notes: shouldMutate ? \"mutated child\" : \"carried forward\",\n      });\n      variantIdx += 1;\n    }\n\n    pool = next;\n    for (const entry of pool) {\n      if (entry.score > bestScore) {\n        bestScore = entry.score;\n        bestPrompt = entry.prompt;\n      }\n    }\n  }\n\n  return {\n    optimizedPrompt: bestPrompt,\n    score: bestScore,\n    baseline: baselineScore,\n    lineage,\n  };\n}\n\nasync function mutate(\n  llm: LlmAdapter,\n  prompt: string,\n  settings: { temperature: number; maxTokens: number },\n): Promise<string> {\n  const result = await llm.complete({\n    system: MUTATION_INSTRUCTIONS,\n    user: prompt,\n    temperature: settings.temperature,\n    maxTokens: settings.maxTokens,\n  });\n  const cleaned = result.trim();\n  return cleaned.length > 0 ? cleaned : prompt;\n}\n"],"mappings":"AAcA,SAAS,iBAAiB;AAkC1B,MAAM,wBAAwB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAc9B,eAAsB,mBACpB,OAC0B;AAC1B,QAAM,aAAa,KAAK,IAAI,GAAG,MAAM,SAAS,cAAc,CAAC;AAC7D,QAAM,cAAc,MAAM,SAAS,eAAe;AAClD,QAAM,eAAe,MAAM,SAAS,gBAAgB;AACpD,QAAM,cAAc,MAAM,SAAS,eAAe;AAClD,QAAM,YAAY,MAAM,SAAS,aAAa;AAC9C,QAAM,MAAM,MAAM,SAAS,OAAO,KAAK;AACvC,QAAM,UAAmC,CAAC;AAE1C,QAAM,UACJ,OAAO,MAAM,SAAS,kBAAkB,WACpC,UAAU,MAAM,SAAS,MAAM,QAAQ,eAAe,GAAG,IACzD,MAAM;AAEZ,QAAM,gBAAgB,MAAM,MAAM,OAAO,MAAM,gBAAgB,OAAO;AACtE,UAAQ,KAAK;AAAA,IACX,OAAO;AAAA,IACP,SAAS;AAAA,IACT,OAAO;AAAA,IACP,OAAO;AAAA,EACT,CAAC;AAKD,MAAI,OAAuB;AAAA,IACzB,EAAE,QAAQ,MAAM,gBAAgB,OAAO,cAAc;AAAA,EACvD;AACA,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK,GAAG;AACtC,UAAM,OAAO,MAAM,OAAO,MAAM,KAAK,MAAM,gBAAgB;AAAA,MACzD;AAAA,MACA;AAAA,IACF,CAAC;AACD,UAAM,QAAQ,MAAM,MAAM,OAAO,MAAM,OAAO;AAC9C,SAAK,KAAK,EAAE,QAAQ,MAAM,MAAM,CAAC;AACjC,YAAQ,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,OAAO,OAAO,gBAAgB,CAAC;AAAA,EACtE;AAEA,MAAI,aAAa,MAAM;AACvB,MAAI,YAAY;AAChB,aAAW,SAAS,MAAM;AACxB,QAAI,MAAM,QAAQ,WAAW;AAC3B,kBAAY,MAAM;AAClB,mBAAa,MAAM;AAAA,IACrB;AAAA,EACF;AAEA,WAAS,MAAM,GAAG,OAAO,aAAa,OAAO,GAAG;AAC9C,SAAK,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACrC,UAAM,SAAS,KAAK,IAAI,GAAG,KAAK,MAAM,KAAK,SAAS,CAAC,CAAC;AACtD,UAAM,YAAY,KAAK,MAAM,GAAG,MAAM;AACtC,UAAM,OAAuB,CAAC,GAAG,SAAS;AAE1C,QAAI,aAAa,UAAU;AAC3B,WAAO,KAAK,SAAS,YAAY;AAC/B,YAAM,SAAS,UAAU,KAAK,SAAS,UAAU,MAAM;AACvD,UAAI,CAAC,OAAQ;AACb,YAAM,eAAe,IAAI,IAAI;AAC7B,UAAI,cAAc,OAAO;AACzB,UAAI,cAAc;AAChB,sBAAc,MAAM,OAAO,MAAM,KAAK,OAAO,QAAQ;AAAA,UACnD;AAAA,UACA;AAAA,QACF,CAAC;AAAA,MACH;AACA,YAAM,QAAQ,MAAM,MAAM,OAAO,aAAa,OAAO;AACrD,WAAK,KAAK,EAAE,QAAQ,aAAa,MAAM,CAAC;AACxC,cAAQ,KAAK;AAAA,QACX,OAAO;AAAA,QACP,SAAS;AAAA,QACT;AAAA,QACA,OAAO,eAAe,kBAAkB;AAAA,MAC1C,CAAC;AACD,oBAAc;AAAA,IAChB;AAEA,WAAO;AACP,eAAW,SAAS,MAAM;AACxB,UAAI,MAAM,QAAQ,WAAW;AAC3B,oBAAY,MAAM;AAClB,qBAAa,MAAM;AAAA,MACrB;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AAAA,IACL,iBAAiB;AAAA,IACjB,OAAO;AAAA,IACP,UAAU;AAAA,IACV;AAAA,EACF;AACF;AAEA,eAAe,OACb,KACA,QACA,UACiB;AACjB,QAAM,SAAS,MAAM,IAAI,SAAS;AAAA,IAChC,QAAQ;AAAA,IACR,MAAM;AAAA,IACN,aAAa,SAAS;AAAA,IACtB,WAAW,SAAS;AAAA,EACtB,CAAC;AACD,QAAM,UAAU,OAAO,KAAK;AAC5B,SAAO,QAAQ,SAAS,IAAI,UAAU;AACxC;","names":[]}

package/dist/optimizers/scoring.d.ts

@@ -0,0 +1,139 @@
+/**
+ * Scoring utilities for native optimizers.
+ *
+ * The default scorer measures token-overlap agreement between the model's
+ * actual output and the expected output recorded in the trajectory dataset.
+ * It is deliberately cheap and deterministic — the optimizers run hundreds
+ * of completions per round, so we cannot afford a model-graded scorer.
+ *
+ * Token-overlap agreement (Jaccard over normalized tokens) is the same primitive
+ * that `replay-validator.ts` uses for `scoreSkill`-style success measurement,
+ * just lifted to the (output vs reference) comparison instead of (skill vs
+ * trajectory). When a richer signal becomes available, the scorer factory can
+ * be swapped without changing any optimizer code.
+ */
+import type { LlmAdapter, PromptScorer } from "./types.js";
+interface ScorerOptions {
+    /** Cap on examples scored per call. Defaults to all examples. */
+    maxExamples?: number;
+    /** Temperature passed to the adapter. Defaults to 0 for determinism. */
+    temperature?: number;
+    /** Max tokens for each completion. Defaults to 512. */
+    maxTokens?: number;
+    /**
+     * Per-example comparator. Defaults to Jaccard token overlap.
+     * Returning 1.0 means a perfect match, 0.0 means no credit.
+     */
+    compare?: (actual: string, expected: string) => number;
+}
+/**
+ * Build a `PromptScorer` backed by a real LLM adapter.
+ *
+ * For each example:
+ *   1. Run `prompt` (as system) + `example.input.user` through the adapter.
+ *   2. Compare the completion against `example.expectedOutput` via Jaccard
+ *      similarity over normalized tokens.
+ *   3. Return the mean score.
+ *
+ * Reuses the same normalization heuristic as the trajectory-task-datasets
+ * exporter (lower-case, strip punctuation, drop empty tokens).
+ */
+export declare function createPromptScorer(adapter: LlmAdapter, options?: ScorerOptions): PromptScorer;
+/**
+ * Extract the first action name from planner output. JSON is preferred; a
+ * small line-based reader keeps older key/value rows comparable.
+ *
+ * Schemas understood (in priority order):
+ *   1. v5 planner: `{toolCalls:[{name:"OWNER_TODOS","args":{...}}]}` — handled directly.
+ *   2. Legacy structured: top-level `action`/`actionName`/`name`/`type`/`actions` field.
+ *   3. Legacy line-based: `action: OWNER_TODOS` or similar key:value rows.
+ *   4. Last-resort: any uppercase identifier in the text.
+ *
+ * The regex fallback is intentionally last — it matches identifiers like
+ * `OWNER`, `OPTIONAL`, `JSON`, etc. that show up in field names, so it can
+ * mislabel non-action text. Prefer the JSON paths when the runtime emits
+ * structured output (which is the common case post-v5).
+ */
+export declare function extractPlannerAction(text: string): string | null;
+/**
+ * Extract the target view id from planner output when the chosen action is a
+ * view navigation. Understands the same shapes as {@link extractPlannerAction}:
+ *   1. tool-call: `{toolCalls:[{name:"VIEWS", args/arguments/parameters:{view}}]}`
+ *   2. bare action: `{action:"VIEWS", parameters/args:{view}}`
+ *   3. top-level alias: `{view}` / `{viewId}`
+ * Returns the lower-cased view id, or `null` when none is present.
+ */
+export declare function extractPlannerView(text: string): string | null;
+/**
+ * Action-name comparator: returns 1.0 when both outputs resolve to the same
+ * planner action name, 0.0 otherwise. This is the right primitive for
+ * optimizing the `action_planner` task because token overlap under-credits
+ * correct choices when surrounding rationale varies stochastically.
+ *
+ * View-aware refinement: when the expected output pins a specific view (a VIEWS
+ * navigation target), a matching action alone is NOT full credit — the view has
+ * to match too. Without this the optimizer can never learn correct view
+ * selection, because every `VIEWS/<anything>` would score 1.0 against a
+ * `VIEWS/calendar` reference (the exact gap that made entry-tier wrong-view
+ * outputs look perfect). Partial credit (right action, wrong/missing view =
+ * 0.5) keeps a usable gradient for the optimizer. Expected outputs without a
+ * view (every non-navigation action) are scored action-only, unchanged.
+ */
+export declare function scorePlannerAction(actual: string, expected: string): number;
+/**
+ * View-selection comparator for the contextual view evaluator (`view_context`
+ * task). Both outputs are `{viewId, reason}` (or "none"); credit is 1.0 when the
+ * chosen view id matches the reference, 0.0 otherwise. Case-insensitive, and a
+ * matching "none" (correctly declining to navigate) scores 1.0 — so it rewards
+ * both opening the right surface AND staying put on non-navigational turns.
+ */
+export declare function scoreViewSelection(actual: string, expected: string): number;
+/**
+ * Jaccard similarity over normalized token sets, in `[0, 1]`. Empty inputs
+ * collapse to 0 (no overlap to measure).
+ */
+export declare function scoreAgreement(actual: string, expected: string): number;
+/** LifeOps tasks with per-capability scorers (#8795). */
+export declare const LIFEOPS_SCORER_TASKS: readonly ["calendar_extract", "schedule_plan", "reminder_dispatch", "inbox_triage", "meeting_prep", "morning_brief", "health_checkin", "screentime_recap"];
+/** LifeOps tasks whose output is a structured JSON object (exact-field match). */
+export declare const LIFEOPS_STRUCTURED_SCORER_TASKS: readonly ["calendar_extract", "schedule_plan", "reminder_dispatch", "inbox_triage", "meeting_prep", "health_checkin", "screentime_recap"];
+/**
+ * Structured-field exact-match score in `[0, 1]`: the fraction of expected
+ * fields whose value the actual output reproduced. Both inputs are parsed as
+ * JSON or line-based fields (tolerating fences/prose). When `fields` is supplied only those keys are
+ * scored; otherwise every key in `expected` is scored. Returns 0 when expected
+ * is unparseable (nothing to credit) and 1 when both parse to empty objects.
+ */
+export declare function scoreStructuredFields(actual: string, expected: string, fields?: readonly string[]): number;
+/**
+ * Action/label set-overlap (Jaccard) in `[0, 1]`. For tasks whose target is
+ * "did the agent pick the right action/category set" rather than exact text.
+ * Two empty sets score 1.0 (both correctly produced nothing actionable).
+ */
+export declare function scoreActionSet(actual: string, expected: string): number;
+/**
+ * Authoritative per-task comparator for the LifeOps optimization tasks (the
+ * GEPA Pareto scorer dispatches through here). Extraction tasks →
+ * structured-field exact-match; the chat-shaped morning brief → token
+ * agreement (proxy for the judge rubric); anything else → token agreement.
+ */
+export declare function scoreLifeOpsTask(task: string, actual: string, expected: string): number;
+/**
+ * Random-without-replacement subsample, used by optimizer rounds to keep
+ * scoring cheap on large datasets without sacrificing comparability across
+ * rounds (deterministic when `rng` is supplied).
+ */
+export declare function subsample<T>(items: T[], count: number, rng?: () => number): T[];
+/**
+ * Wraps `IAgentRuntime.useModel` into the `LlmAdapter` shape. We accept a
+ * loose runtime type so this module stays free of `@elizaos/core` import
+ * cycles — the native backend supplies the bound `useModel` directly.
+ */
+export type UseModelHandler = (input: {
+    prompt: string;
+    temperature?: number;
+    maxTokens?: number;
+}) => Promise<string | object | undefined>;
+export declare function createRuntimeAdapter(useModel: UseModelHandler): LlmAdapter;
+export {};
+//# sourceMappingURL=scoring.d.ts.map

package/dist/optimizers/scoring.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scoring.d.ts","sourceRoot":"","sources":["../../src/optimizers/scoring.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE3D,UAAU,aAAa;IACrB,iEAAiE;IACjE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,MAAM,CAAC;CACxD;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,UAAU,EACnB,OAAO,GAAE,aAAkB,GAC1B,YAAY,CAoBd;AAiDD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAmChE;AAgBD;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAmB9D;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAW3E;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAK3E;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAYvE;AA2BD,yDAAyD;AACzD,eAAO,MAAM,oBAAoB,4JASvB,CAAC;AAEX,kFAAkF;AAClF,eAAO,MAAM,+BAA+B,2IAQlC,CAAC;AAwDX;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,GACzB,MAAM,CAgBR;AA2BD;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAWvE;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAC9B,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,GACf,MAAM,CAKR;AAED;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,CAAC,EACzB,KAAK,EAAE,CAAC,EAAE,EACV,KAAK,EAAE,MAAM,EACb,GAAG,GAAE,MAAM,MAAoB,GAC9B,CAAC,EAAE,CAYL;AAED;;;;GAIG;AACH,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAK,OAAO,CAAC,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC,CAAC;AAE3C,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,eAAe,GAAG,UAAU,CAgB1E"}