@elizaos/agent 2.0.0-alpha.173 → 2.0.0-alpha.192

package/apps/app-training/src/core/trajectory-task-datasets.js

@@ -0,0 +1,281 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+const TASK_FILE_NAMES = {
+    should_respond: "should_respond_trajectories.jsonl",
+    context_routing: "context_routing_trajectories.jsonl",
+    action_planner: "action_planner_trajectories.jsonl",
+    response: "response_trajectories.jsonl",
+    media_description: "media_description_trajectories.jsonl",
+};
+function createEmptyExampleMap() {
+    return {
+        should_respond: [],
+        context_routing: [],
+        action_planner: [],
+        response: [],
+        media_description: [],
+    };
+}
+function createEmptyCountMap() {
+    return {
+        should_respond: 0,
+        context_routing: 0,
+        action_planner: 0,
+        response: 0,
+        media_description: 0,
+    };
+}
+function createEmptyTrajectoryIdMap() {
+    return {
+        should_respond: new Set(),
+        context_routing: new Set(),
+        action_planner: new Set(),
+        response: new Set(),
+        media_description: new Set(),
+    };
+}
+function normalizeToken(value) {
+    if (typeof value !== "string") {
+        return "";
+    }
+    return value
+        .trim()
+        .replace(/([a-z0-9])([A-Z])/g, "$1_$2")
+        .toLowerCase()
+        .replace(/[^a-z0-9:_-]+/g, "_")
+        .replace(/^_+|_+$/g, "")
+        .replace(/_+/g, "_");
+}
+function collectCallHints(call) {
+    const metadata = call.metadata ?? {};
+    const tags = Array.isArray(call.tags) ? call.tags : [];
+    const values = [
+        call.purpose,
+        call.stepType,
+        call.actionType,
+        call.model,
+        metadata.modelType,
+        metadata.purpose,
+        metadata.model_type,
+        metadata.stepType,
+        ...tags,
+    ];
+    return values
+        .map(normalizeToken)
+        .filter((value, index, items) => value.length > 0 && items.indexOf(value) === index);
+}
+function hasContextRoutingFields(text) {
+    return (/(^|\n)primaryContext:/m.test(text) ||
+        /(^|\n)secondaryContexts:/m.test(text) ||
+        /(^|\n)evidenceTurnIds:/m.test(text) ||
+        /<primaryContext>/i.test(text) ||
+        /<secondaryContexts>/i.test(text) ||
+        /<evidenceTurnIds>/i.test(text));
+}
+function looksLikePlannerCall(call) {
+    const response = call.response ?? "";
+    const prompt = `${call.systemPrompt ?? ""}\n${call.userPrompt ?? ""}`;
+    return (/<actions>/i.test(response) ||
+        /(^|\n)actions:/m.test(response) ||
+        (/thought/i.test(response) && /text/i.test(response)) ||
+        /available actions/i.test(prompt) ||
+        /actionNames/i.test(prompt));
+}
+function stripContextRoutingPrompt(prompt) {
+    return prompt
+        .replace(/\ncontext_routing:[\s\S]*?\ndecision_note:/m, "\ndecision_note:")
+        .replace(/\n- primaryContext:[^\n]*/g, "")
+        .replace(/\n- secondaryContexts:[^\n]*/g, "")
+        .replace(/\n- evidenceTurnIds:[^\n]*/g, "")
+        .trim();
+}
+function stripContextRoutingResponse(response) {
+    return response
+        .replace(/\nprimaryContext:[^\n]*/g, "")
+        .replace(/\nsecondaryContexts:[^\n]*/g, "")
+        .replace(/\nevidenceTurnIds:[^\n]*/g, "")
+        .replace(/<primaryContext>[\s\S]*?<\/primaryContext>/gi, "")
+        .replace(/<secondaryContexts>[\s\S]*?<\/secondaryContexts>/gi, "")
+        .replace(/<evidenceTurnIds>[\s\S]*?<\/evidenceTurnIds>/gi, "")
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+}
+function inferTasksForCall(call) {
+    const hints = collectCallHints(call);
+    const response = call.response ?? "";
+    const tasks = new Set();
+    if (hints.includes("should_respond") ||
+        hints.includes("response_handler") ||
+        hints.includes("shouldrespond")) {
+        tasks.add("should_respond");
+    }
+    if (hasContextRoutingFields(response)) {
+        tasks.add("context_routing");
+        tasks.add("should_respond");
+    }
+    if (hints.includes("action_planner") ||
+        hints.includes("planner") ||
+        hints.includes("action") ||
+        hints.includes("runtime_use_model") ||
+        looksLikePlannerCall(call)) {
+        tasks.add("action_planner");
+    }
+    if (hints.includes("media_description") ||
+        hints.includes("image_description") ||
+        hints.includes("describe_image") ||
+        hints.includes("describe_audio") ||
+        hints.includes("describe_video")) {
+        tasks.add("media_description");
+    }
+    if (hints.includes("response") ||
+        hints.includes("reply") ||
+        hints.includes("message_response")) {
+        tasks.add("response");
+    }
+    if (tasks.size === 0 &&
+        typeof call.response === "string" &&
+        call.response.trim()) {
+        tasks.add("response");
+    }
+    return [...tasks];
+}
+function buildExampleForTask(call, task) {
+    const systemPrompt = call.systemPrompt?.trim();
+    const userPrompt = call.userPrompt?.trim();
+    const response = call.response?.trim();
+    if (!systemPrompt || !userPrompt || !response) {
+        return null;
+    }
+    if (task === "should_respond") {
+        return {
+            messages: [
+                { role: "system", content: stripContextRoutingPrompt(systemPrompt) },
+                { role: "user", content: userPrompt },
+                { role: "model", content: stripContextRoutingResponse(response) },
+            ],
+        };
+    }
+    return {
+        messages: [
+            { role: "system", content: systemPrompt },
+            { role: "user", content: userPrompt },
+            { role: "model", content: response },
+        ],
+    };
+}
+function collectTrajectoryExamplesByTask(trajectories, tasks) {
+    const requestedTasks = new Set(tasks ?? [
+        "should_respond",
+        "context_routing",
+        "action_planner",
+        "response",
+        "media_description",
+    ]);
+    const examples = createEmptyExampleMap();
+    const sourceCallCounts = createEmptyCountMap();
+    const sourceTrajectoryIds = createEmptyTrajectoryIdMap();
+    let llmCallCount = 0;
+    for (const trajectory of trajectories) {
+        const trajectoryId = trajectory.trajectoryId;
+        for (const step of trajectory.steps ?? []) {
+            for (const llmCall of step.llmCalls ?? []) {
+                llmCallCount += 1;
+                const call = llmCall;
+                const inferredTasks = inferTasksForCall(call);
+                for (const task of inferredTasks) {
+                    if (!requestedTasks.has(task)) {
+                        continue;
+                    }
+                    const example = buildExampleForTask(call, task);
+                    if (!example) {
+                        continue;
+                    }
+                    examples[task].push(example);
+                    sourceCallCounts[task] += 1;
+                    sourceTrajectoryIds[task].add(trajectoryId);
+                }
+            }
+        }
+    }
+    return {
+        examples,
+        sourceCallCounts,
+        sourceTrajectoryIds,
+        llmCallCount,
+    };
+}
+export function extractTrajectoryExamplesByTask(trajectories, tasks) {
+    return collectTrajectoryExamplesByTask(trajectories, tasks).examples;
+}
+export async function exportTrajectoryTaskDatasets(trajectories, outputDir, tasks) {
+    await mkdir(outputDir, { recursive: true });
+    const extraction = collectTrajectoryExamplesByTask(trajectories, tasks);
+    const { examples } = extraction;
+    const counts = {
+        should_respond: examples.should_respond.length,
+        context_routing: examples.context_routing.length,
+        action_planner: examples.action_planner.length,
+        response: examples.response.length,
+        media_description: examples.media_description.length,
+    };
+    const paths = {
+        shouldRespondPath: join(outputDir, TASK_FILE_NAMES.should_respond),
+        contextRoutingPath: join(outputDir, TASK_FILE_NAMES.context_routing),
+        actionPlannerPath: join(outputDir, TASK_FILE_NAMES.action_planner),
+        responsePath: join(outputDir, TASK_FILE_NAMES.response),
+        mediaDescriptionPath: join(outputDir, TASK_FILE_NAMES.media_description),
+        summaryPath: join(outputDir, "trajectory_dataset_summary.json"),
+    };
+    const summary = {
+        generatedAt: new Date().toISOString(),
+        trajectoryCount: trajectories.length,
+        llmCallCount: extraction.llmCallCount,
+        counts,
+        tasks: [
+            "should_respond",
+            "context_routing",
+            "action_planner",
+            "response",
+            "media_description",
+        ].filter((task) => tasks?.includes(task) ?? true),
+        taskMetrics: {
+            should_respond: {
+                exampleCount: counts.should_respond,
+                sourceCallCount: extraction.sourceCallCounts.should_respond,
+                sourceTrajectoryCount: extraction.sourceTrajectoryIds.should_respond.size,
+            },
+            context_routing: {
+                exampleCount: counts.context_routing,
+                sourceCallCount: extraction.sourceCallCounts.context_routing,
+                sourceTrajectoryCount: extraction.sourceTrajectoryIds.context_routing.size,
+            },
+            action_planner: {
+                exampleCount: counts.action_planner,
+                sourceCallCount: extraction.sourceCallCounts.action_planner,
+                sourceTrajectoryCount: extraction.sourceTrajectoryIds.action_planner.size,
+            },
+            response: {
+                exampleCount: counts.response,
+                sourceCallCount: extraction.sourceCallCounts.response,
+                sourceTrajectoryCount: extraction.sourceTrajectoryIds.response.size,
+            },
+            media_description: {
+                exampleCount: counts.media_description,
+                sourceCallCount: extraction.sourceCallCounts.media_description,
+                sourceTrajectoryCount: extraction.sourceTrajectoryIds.media_description.size,
+            },
+        },
+    };
+    await writeFile(paths.shouldRespondPath, `${examples.should_respond.map((example) => JSON.stringify(example)).join("\n")}${examples.should_respond.length > 0 ? "\n" : ""}`);
+    await writeFile(paths.contextRoutingPath, `${examples.context_routing.map((example) => JSON.stringify(example)).join("\n")}${examples.context_routing.length > 0 ? "\n" : ""}`);
+    await writeFile(paths.actionPlannerPath, `${examples.action_planner.map((example) => JSON.stringify(example)).join("\n")}${examples.action_planner.length > 0 ? "\n" : ""}`);
+    await writeFile(paths.responsePath, `${examples.response.map((example) => JSON.stringify(example)).join("\n")}${examples.response.length > 0 ? "\n" : ""}`);
+    await writeFile(paths.mediaDescriptionPath, `${examples.media_description.map((example) => JSON.stringify(example)).join("\n")}${examples.media_description.length > 0 ? "\n" : ""}`);
+    await writeFile(paths.summaryPath, JSON.stringify(summary, null, 2));
+    return {
+        counts,
+        paths,
+        examples,
+        summary,
+    };
+}

package/apps/app-training/src/optimizers/bootstrap-fewshot.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Bootstrap-fewshot optimizer.
+ *
+ * Picks the top-K examples from the dataset (ranked by reward, then by
+ * agreement against the baseline) and injects them as in-context
+ * demonstrations into the prompt. The output prompt is the baseline plus a
+ * `Demonstrations:` block that the runtime threads through unchanged.
+ *
+ * This is the cheapest optimizer — it does not propose new instructions, only
+ * conditions the existing prompt on a curated few-shot set. It is the fallback
+ * used when threshold-fired bootstrap runs need fast turnaround.
+ */
+import type { LlmAdapter, OptimizationExample, OptimizerResult, PromptScorer } from "./types.js";
+export interface BootstrapFewshotOptions {
+    /** Number of demonstrations to inject. Defaults to 5. */
+    k?: number;
+    /**
+     * Optional scorer override. When supplied, examples are ranked by score
+     * against the baseline prompt instead of by `example.reward`.
+     */
+    rankByScorer?: boolean;
+}
+export interface BootstrapFewshotInput {
+    baselinePrompt: string;
+    dataset: OptimizationExample[];
+    scorer: PromptScorer;
+    llm: LlmAdapter;
+    options?: BootstrapFewshotOptions;
+}
+/**
+ * Render a demonstration block exactly the way the runtime wires it back into
+ * the system prompt. Public so `OptimizedPromptService` can rebuild the
+ * combined prompt at load time.
+ */
+export declare function renderDemonstrations(examples: OptimizationExample[]): string;
+/**
+ * Combine a baseline prompt with a rendered demonstration block. Idempotent
+ * when `examples` is empty.
+ */
+export declare function withDemonstrations(baseline: string, examples: OptimizationExample[]): string;
+export declare function runBootstrapFewshot(input: BootstrapFewshotInput): Promise<OptimizerResult>;
+//# sourceMappingURL=bootstrap-fewshot.d.ts.map

package/apps/app-training/src/optimizers/bootstrap-fewshot.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bootstrap-fewshot.d.ts","sourceRoot":"","sources":["../../../../../../../apps/app-training/src/optimizers/bootstrap-fewshot.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EACX,UAAU,EACV,mBAAmB,EAEnB,eAAe,EACf,YAAY,EACZ,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,uBAAuB;IACvC,yDAAyD;IACzD,CAAC,CAAC,EAAE,MAAM,CAAC;IACX;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,qBAAqB;IACrC,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,MAAM,EAAE,YAAY,CAAC;IACrB,GAAG,EAAE,UAAU,CAAC;IAChB,OAAO,CAAC,EAAE,uBAAuB,CAAC;CAClC;AAID;;;;GAIG;AACH,wBAAgB,oBAAoB,CACnC,QAAQ,EAAE,mBAAmB,EAAE,GAC7B,MAAM,CAYR;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CACjC,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,mBAAmB,EAAE,GAC7B,MAAM,CAIR;AAED,wBAAsB,mBAAmB,CACxC,KAAK,EAAE,qBAAqB,GAC1B,OAAO,CAAC,eAAe,CAAC,CA4B1B"}

package/apps/app-training/src/optimizers/bootstrap-fewshot.js

@@ -0,0 +1,91 @@
+/**
+ * Bootstrap-fewshot optimizer.
+ *
+ * Picks the top-K examples from the dataset (ranked by reward, then by
+ * agreement against the baseline) and injects them as in-context
+ * demonstrations into the prompt. The output prompt is the baseline plus a
+ * `Demonstrations:` block that the runtime threads through unchanged.
+ *
+ * This is the cheapest optimizer — it does not propose new instructions, only
+ * conditions the existing prompt on a curated few-shot set. It is the fallback
+ * used when threshold-fired bootstrap runs need fast turnaround.
+ */
+const DEMONSTRATION_HEADER = "Demonstrations:";
+/**
+ * Render a demonstration block exactly the way the runtime wires it back into
+ * the system prompt. Public so `OptimizedPromptService` can rebuild the
+ * combined prompt at load time.
+ */
+export function renderDemonstrations(examples) {
+    if (examples.length === 0)
+        return "";
+    const lines = [DEMONSTRATION_HEADER, ""];
+    let idx = 1;
+    for (const example of examples) {
+        lines.push(`Example ${idx}:`);
+        lines.push(`Input:\n${example.input.user}`);
+        lines.push(`Expected:\n${example.expectedOutput}`);
+        lines.push("");
+        idx += 1;
+    }
+    return lines.join("\n").trimEnd();
+}
+/**
+ * Combine a baseline prompt with a rendered demonstration block. Idempotent
+ * when `examples` is empty.
+ */
+export function withDemonstrations(baseline, examples) {
+    if (examples.length === 0)
+        return baseline;
+    const demos = renderDemonstrations(examples);
+    return `${baseline.trimEnd()}\n\n${demos}\n`;
+}
+export async function runBootstrapFewshot(input) {
+    const k = Math.max(1, input.options?.k ?? 5);
+    const lineage = [];
+    const baselineScore = await input.scorer(input.baselinePrompt, input.dataset);
+    lineage.push({ round: 0, variant: 0, score: baselineScore, notes: "baseline" });
+    const ranked = await rankExamples(input, baselineScore);
+    const fewShot = ranked.slice(0, Math.min(k, ranked.length));
+    const optimizedPrompt = withDemonstrations(input.baselinePrompt, fewShot);
+    const optimizedScore = await input.scorer(optimizedPrompt, input.dataset);
+    lineage.push({
+        round: 1,
+        variant: 1,
+        score: optimizedScore,
+        notes: `injected ${fewShot.length} demonstrations`,
+    });
+    return {
+        optimizedPrompt,
+        score: optimizedScore,
+        baseline: baselineScore,
+        lineage,
+        fewShotExamples: fewShot,
+    };
+}
+async function rankExamples(input, baselineScore) {
+    if (input.options?.rankByScorer) {
+        const scored = [];
+        for (const example of input.dataset) {
+            const score = await input.scorer(input.baselinePrompt, [example]);
+            scored.push({ example, score });
+        }
+        // Pull demonstrations the baseline already gets right; they are the
+        // highest-confidence anchors for the model to imitate. Tie-break by
+        // reward when the scorer is uninformative.
+        scored.sort((a, b) => b.score - a.score ||
+            (b.example.reward ?? 0) - (a.example.reward ?? 0));
+        void baselineScore;
+        return scored.map((entry) => entry.example);
+    }
+    // Reward-first ranking. Examples without a recorded reward fall through
+    // to a stable order at the back so the dataset's natural ordering wins
+    // the tie-break.
+    const ordered = input.dataset.map((example, index) => ({
+        example,
+        index,
+        reward: example.reward ?? 0,
+    }));
+    ordered.sort((a, b) => b.reward - a.reward || a.index - b.index);
+    return ordered.map((entry) => entry.example);
+}

package/apps/app-training/src/optimizers/index.d.ts

@@ -0,0 +1,6 @@
+export type { LlmAdapter, OptimizationExample, OptimizedPromptArtifact, OptimizerLineageEntry, OptimizerName, OptimizerResult, PromptScorer, } from "./types.js";
+export { createPromptScorer, createRuntimeAdapter, scoreAgreement, subsample, type UseModelHandler, } from "./scoring.js";
+export { runInstructionSearch, type InstructionSearchInput, type InstructionSearchOptions, } from "./instruction-search.js";
+export { runPromptEvolution, type PromptEvolutionInput, type PromptEvolutionOptions, } from "./prompt-evolution.js";
+export { renderDemonstrations, runBootstrapFewshot, withDemonstrations, type BootstrapFewshotInput, type BootstrapFewshotOptions, } from "./bootstrap-fewshot.js";
+//# sourceMappingURL=index.d.ts.map

package/apps/app-training/src/optimizers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../../../apps/app-training/src/optimizers/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACX,UAAU,EACV,mBAAmB,EACnB,uBAAuB,EACvB,qBAAqB,EACrB,aAAa,EACb,eAAe,EACf,YAAY,GACZ,MAAM,YAAY,CAAC;AACpB,OAAO,EACN,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,SAAS,EACT,KAAK,eAAe,GACpB,MAAM,cAAc,CAAC;AACtB,OAAO,EACN,oBAAoB,EACpB,KAAK,sBAAsB,EAC3B,KAAK,wBAAwB,GAC7B,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACN,kBAAkB,EAClB,KAAK,oBAAoB,EACzB,KAAK,sBAAsB,GAC3B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACN,oBAAoB,EACpB,mBAAmB,EACnB,kBAAkB,EAClB,KAAK,qBAAqB,EAC1B,KAAK,uBAAuB,GAC5B,MAAM,wBAAwB,CAAC"}

package/apps/app-training/src/optimizers/index.js

@@ -0,0 +1,4 @@
+export { createPromptScorer, createRuntimeAdapter, scoreAgreement, subsample, } from "./scoring.js";
+export { runInstructionSearch, } from "./instruction-search.js";
+export { runPromptEvolution, } from "./prompt-evolution.js";
+export { renderDemonstrations, runBootstrapFewshot, withDemonstrations, } from "./bootstrap-fewshot.js";

package/apps/app-training/src/optimizers/instruction-search.d.ts

@@ -0,0 +1,39 @@
+/**
+ * MIPRO-style instruction search optimizer.
+ *
+ * Iterative improvement loop:
+ *   1. Ask the LLM to propose N rewrites of the current baseline prompt that
+ *      preserve the task contract but tighten language, add guardrails, or
+ *      reorder for clarity.
+ *   2. Score each candidate (plus the current baseline) on a held-out subset
+ *      of the dataset.
+ *   3. Keep the highest-scoring candidate as the next round's baseline.
+ *   4. Repeat for `rounds` iterations.
+ *
+ * Returns the best prompt observed across all rounds along with full lineage
+ * (every (round, variant) -> score), so the caller can render an audit trail.
+ */
+import type { LlmAdapter, OptimizationExample, OptimizerResult, PromptScorer } from "./types.js";
+export interface InstructionSearchOptions {
+    /** Variants to propose per round. Defaults to 8. */
+    variants?: number;
+    /** Search rounds. Defaults to 3. */
+    rounds?: number;
+    /** Held-out examples scored per candidate. Defaults to all examples. */
+    scoringSubset?: number;
+    /** Sampling temperature for variant generation. Defaults to 0.7. */
+    temperature?: number;
+    /** Max tokens for the rewrite completion. Defaults to 1024. */
+    maxTokens?: number;
+    /** Deterministic RNG override (tests). Defaults to Math.random. */
+    rng?: () => number;
+}
+export interface InstructionSearchInput {
+    baselinePrompt: string;
+    dataset: OptimizationExample[];
+    scorer: PromptScorer;
+    llm: LlmAdapter;
+    options?: InstructionSearchOptions;
+}
+export declare function runInstructionSearch(input: InstructionSearchInput): Promise<OptimizerResult>;
+//# sourceMappingURL=instruction-search.d.ts.map

package/apps/app-training/src/optimizers/instruction-search.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"instruction-search.d.ts","sourceRoot":"","sources":["../../../../../../../apps/app-training/src/optimizers/instruction-search.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,KAAK,EACX,UAAU,EACV,mBAAmB,EAEnB,eAAe,EACf,YAAY,EACZ,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,wBAAwB;IACxC,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oCAAoC;IACpC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wEAAwE;IACxE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mEAAmE;IACnE,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,sBAAsB;IACtC,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,MAAM,EAAE,YAAY,CAAC;IACrB,GAAG,EAAE,UAAU,CAAC;IAChB,OAAO,CAAC,EAAE,wBAAwB,CAAC;CACnC;AAWD,wBAAsB,oBAAoB,CACzC,KAAK,EAAE,sBAAsB,GAC3B,OAAO,CAAC,eAAe,CAAC,CA+D1B"}

package/apps/app-training/src/optimizers/instruction-search.js

@@ -0,0 +1,82 @@
+/**
+ * MIPRO-style instruction search optimizer.
+ *
+ * Iterative improvement loop:
+ *   1. Ask the LLM to propose N rewrites of the current baseline prompt that
+ *      preserve the task contract but tighten language, add guardrails, or
+ *      reorder for clarity.
+ *   2. Score each candidate (plus the current baseline) on a held-out subset
+ *      of the dataset.
+ *   3. Keep the highest-scoring candidate as the next round's baseline.
+ *   4. Repeat for `rounds` iterations.
+ *
+ * Returns the best prompt observed across all rounds along with full lineage
+ * (every (round, variant) -> score), so the caller can render an audit trail.
+ */
+import { subsample } from "./scoring.js";
+const REWRITE_INSTRUCTIONS = `You are a prompt engineer. Improve the SYSTEM PROMPT below for the task it describes.
+
+Constraints:
+- Preserve the original task contract (inputs, outputs, format).
+- Keep all literal placeholders like {{agentName}} or {{providers}} intact.
+- Do not invent new fields, do not remove existing ones.
+- Tighten language, add guardrails for failure modes, reorder for clarity.
+- Output ONLY the rewritten prompt. No commentary, no fenced code blocks.`;
+export async function runInstructionSearch(input) {
+    const variants = input.options?.variants ?? 8;
+    const rounds = input.options?.rounds ?? 3;
+    const temperature = input.options?.temperature ?? 0.7;
+    const maxTokens = input.options?.maxTokens ?? 1024;
+    const rng = input.options?.rng ?? Math.random;
+    const lineage = [];
+    const heldOut = typeof input.options?.scoringSubset === "number"
+        ? subsample(input.dataset, input.options.scoringSubset, rng)
+        : input.dataset;
+    const baselineScore = await input.scorer(input.baselinePrompt, heldOut);
+    lineage.push({ round: 0, variant: 0, score: baselineScore, notes: "baseline" });
+    let bestPrompt = input.baselinePrompt;
+    let bestScore = baselineScore;
+    let currentBaseline = input.baselinePrompt;
+    for (let round = 1; round <= rounds; round += 1) {
+        let roundBestPrompt = currentBaseline;
+        let roundBestScore = bestScore;
+        for (let variant = 1; variant <= variants; variant += 1) {
+            const candidate = await input.llm.complete({
+                system: REWRITE_INSTRUCTIONS,
+                user: currentBaseline,
+                temperature,
+                maxTokens,
+            });
+            const cleaned = candidate.trim();
+            if (cleaned.length === 0) {
+                lineage.push({
+                    round,
+                    variant,
+                    score: 0,
+                    notes: "empty rewrite — skipped",
+                });
+                continue;
+            }
+            const score = await input.scorer(cleaned, heldOut);
+            lineage.push({ round, variant, score });
+            if (score > roundBestScore) {
+                roundBestScore = score;
+                roundBestPrompt = cleaned;
+            }
+        }
+        if (roundBestScore > bestScore) {
+            bestScore = roundBestScore;
+            bestPrompt = roundBestPrompt;
+        }
+        // Carry the round winner forward as the next round's seed; this is the
+        // MIPRO greedy step. If no candidate improved, we still move forward to
+        // give the next round a chance with the same seed.
+        currentBaseline = roundBestPrompt;
+    }
+    return {
+        optimizedPrompt: bestPrompt,
+        score: bestScore,
+        baseline: baselineScore,
+        lineage,
+    };
+}

package/apps/app-training/src/optimizers/prompt-evolution.d.ts

@@ -0,0 +1,39 @@
+/**
+ * GEPA-style genetic prompt evolution.
+ *
+ * Maintains a population of candidate prompts. Each generation:
+ *   1. Score every candidate on a held-out subset.
+ *   2. Select the top half as survivors.
+ *   3. For each survivor, with probability `mutationRate`, ask the LLM to
+ *      rewrite a slice (intro/rules/example) to produce a child.
+ *   4. Re-fill the population with mutated children of the survivors.
+ *
+ * The optimizer always preserves the all-time best prompt so an unlucky
+ * generation cannot regress past the baseline.
+ */
+import type { LlmAdapter, OptimizationExample, OptimizerResult, PromptScorer } from "./types.js";
+export interface PromptEvolutionOptions {
+    /** Population size. Defaults to 8. */
+    population?: number;
+    /** Generations. Defaults to 4. */
+    generations?: number;
+    /** Probability of mutating a survivor each generation. Defaults to 0.5. */
+    mutationRate?: number;
+    /** Held-out examples scored per candidate. Defaults to all examples. */
+    scoringSubset?: number;
+    /** Sampling temperature for mutation generation. Defaults to 0.8. */
+    temperature?: number;
+    /** Max tokens for the mutation completion. Defaults to 1024. */
+    maxTokens?: number;
+    /** Deterministic RNG override (tests). Defaults to Math.random. */
+    rng?: () => number;
+}
+export interface PromptEvolutionInput {
+    baselinePrompt: string;
+    dataset: OptimizationExample[];
+    scorer: PromptScorer;
+    llm: LlmAdapter;
+    options?: PromptEvolutionOptions;
+}
+export declare function runPromptEvolution(input: PromptEvolutionInput): Promise<OptimizerResult>;
+//# sourceMappingURL=prompt-evolution.d.ts.map

package/apps/app-training/src/optimizers/prompt-evolution.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt-evolution.d.ts","sourceRoot":"","sources":["../../../../../../../apps/app-training/src/optimizers/prompt-evolution.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,KAAK,EACX,UAAU,EACV,mBAAmB,EAEnB,eAAe,EACf,YAAY,EACZ,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,sBAAsB;IACtC,sCAAsC;IACtC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,kCAAkC;IAClC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,wEAAwE;IACxE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,qEAAqE;IACrE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mEAAmE;IACnE,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,oBAAoB;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,MAAM,EAAE,YAAY,CAAC;IACrB,GAAG,EAAE,UAAU,CAAC;IAChB,OAAO,CAAC,EAAE,sBAAsB,CAAC;CACjC;AAgBD,wBAAsB,kBAAkB,CACvC,KAAK,EAAE,oBAAoB,GACzB,OAAO,CAAC,eAAe,CAAC,CAsF1B"}