@elizaos/agent 2.0.0-alpha.173 → 2.0.0-alpha.192

package/packages/typescript/src/services/optimized-prompt-resolver.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Helper that resolves the system prompt for one of the five core decision
+ * tasks. Each runtime call site already constructs a baseline prompt; this
+ * resolver consults `OptimizedPromptService` first and falls back to the
+ * baseline when no artifact has been loaded.
+ *
+ * Surgical-on-purpose: do NOT thread a runtime through this module. Callers
+ * pass the service directly so this helper stays a pure function.
+ */
+import type { OptimizedPromptService, OptimizedPromptTask } from "./optimized-prompt.js";
+/**
+ * Look up the optimized system prompt for `task`. Returns the baseline
+ * unchanged when no service is registered or when the service has no
+ * artifact for the task.
+ *
+ * When the artifact carries `fewShotExamples`, they are inlined into the
+ * system prompt under a `Demonstrations:` block. The structure mirrors
+ * `apps/app-training/src/optimizers/bootstrap-fewshot.ts#renderDemonstrations`
+ * so an artifact written by either backend renders identically at the call
+ * site.
+ */
+export declare function resolveOptimizedPrompt(service: OptimizedPromptService | null | undefined, task: OptimizedPromptTask, baseline: string): string;
+//# sourceMappingURL=optimized-prompt-resolver.d.ts.map

package/packages/typescript/src/services/optimized-prompt-resolver.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"optimized-prompt-resolver.d.ts","sourceRoot":"","sources":["../../../../../../typescript/src/services/optimized-prompt-resolver.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAEX,sBAAsB,EACtB,mBAAmB,EACnB,MAAM,uBAAuB,CAAC;AAE/B;;;;;;;;;;GAUG;AACH,wBAAgB,sBAAsB,CACrC,OAAO,EAAE,sBAAsB,GAAG,IAAI,GAAG,SAAS,EAClD,IAAI,EAAE,mBAAmB,EACzB,QAAQ,EAAE,MAAM,GACd,MAAM,CAWR"}

package/packages/typescript/src/services/optimized-prompt-resolver.js

@@ -0,0 +1,50 @@
+/**
+ * Helper that resolves the system prompt for one of the five core decision
+ * tasks. Each runtime call site already constructs a baseline prompt; this
+ * resolver consults `OptimizedPromptService` first and falls back to the
+ * baseline when no artifact has been loaded.
+ *
+ * Surgical-on-purpose: do NOT thread a runtime through this module. Callers
+ * pass the service directly so this helper stays a pure function.
+ */
+/**
+ * Look up the optimized system prompt for `task`. Returns the baseline
+ * unchanged when no service is registered or when the service has no
+ * artifact for the task.
+ *
+ * When the artifact carries `fewShotExamples`, they are inlined into the
+ * system prompt under a `Demonstrations:` block. The structure mirrors
+ * `apps/app-training/src/optimizers/bootstrap-fewshot.ts#renderDemonstrations`
+ * so an artifact written by either backend renders identically at the call
+ * site.
+ */
+export function resolveOptimizedPrompt(service, task, baseline) {
+    if (!service)
+        return baseline;
+    const optimized = service.getPrompt(task);
+    if (!optimized)
+        return baseline;
+    if (!optimized.fewShotExamples ||
+        optimized.fewShotExamples.length === 0) {
+        return optimized.prompt;
+    }
+    return injectDemonstrations(optimized.prompt, optimized.fewShotExamples);
+}
+function injectDemonstrations(prompt, examples) {
+    if (prompt.includes("Demonstrations:")) {
+        // The artifact already had demonstrations rendered into the prompt
+        // (this is how bootstrap-fewshot writes its artifacts). Don't
+        // double-inject.
+        return prompt;
+    }
+    const lines = [prompt.trimEnd(), "", "Demonstrations:", ""];
+    let idx = 1;
+    for (const example of examples) {
+        lines.push(`Example ${idx}:`);
+        lines.push(`Input:\n${example.input.user}`);
+        lines.push(`Expected:\n${example.expectedOutput}`);
+        lines.push("");
+        idx += 1;
+    }
+    return lines.join("\n").trimEnd();
+}

package/packages/typescript/src/services/optimized-prompt.d.ts

@@ -0,0 +1,117 @@
+/**
+ * OptimizedPromptService — runtime cache of native-optimizer artifacts.
+ *
+ * Native MIPRO/GEPA/bootstrap-fewshot optimizers (under
+ * `apps/app-training/src/optimizers/`) write a JSON artifact per task into
+ * `~/.milady/optimized-prompts/<task>/<timestamp>.json`. The runtime consults
+ * this service before constructing the system prompt for one of the five core
+ * decision tasks and substitutes the optimized prompt (plus any few-shot
+ * demonstrations) when an artifact is available.
+ *
+ * Service contract:
+ *   - `getPrompt(task)` — synchronous accessor, returns the loaded prompt or
+ *     null. Cheap to call; reads the in-memory cache. Does not refresh.
+ *   - `setPrompt(task, artifact)` — atomically writes a new artifact and
+ *     refreshes the in-memory cache for that task.
+ *   - `getMetadata(task)` — quick view of optimizer + score for diagnostics.
+ *   - `refresh()` — re-scan the disk store. Called automatically by `start()`,
+ *     also exposed for the `Settings → Auto-Training` panel.
+ *
+ * Loading rule: for each task, the artifact with the most recent
+ * `generatedAt` wins. Ties are broken by the file's `mtime` so manual
+ * intervention (touching a file) can promote an older artifact.
+ *
+ * The on-disk format intentionally mirrors `OptimizedPromptArtifact` from
+ * `apps/app-training/src/optimizers/types.ts`. We re-declare the type here
+ * (instead of importing) because `@elizaos/core` is upstream of
+ * `@elizaos/app-training` and adding the dependency would invert the layering.
+ */
+import type { IAgentRuntime } from "../types/runtime.js";
+import { Service } from "../types/service.js";
+export declare const OPTIMIZED_PROMPT_SERVICE = "optimized_prompt";
+export type OptimizedPromptTask = "should_respond" | "context_routing" | "action_planner" | "response" | "media_description";
+export declare const OPTIMIZED_PROMPT_TASKS: readonly OptimizedPromptTask[];
+export type OptimizerName = "instruction-search" | "prompt-evolution" | "bootstrap-fewshot";
+/**
+ * Mirror of `OptimizationExample` from `apps/app-training/src/optimizers/types.ts`.
+ * Kept narrow on purpose — the runtime only renders these into the prompt.
+ */
+export interface OptimizedPromptFewShotExample {
+    id?: string;
+    input: {
+        system?: string;
+        user: string;
+    };
+    expectedOutput: string;
+    reward?: number;
+    metadata?: Record<string, unknown>;
+}
+export interface OptimizedPromptLineageEntry {
+    round: number;
+    variant: number;
+    score: number;
+    notes?: string;
+}
+export interface OptimizedPromptArtifact {
+    task: OptimizedPromptTask;
+    optimizer: OptimizerName;
+    baseline: string;
+    prompt: string;
+    score: number;
+    baselineScore: number;
+    datasetId: string;
+    datasetSize: number;
+    generatedAt: string;
+    fewShotExamples?: OptimizedPromptFewShotExample[];
+    lineage: OptimizedPromptLineageEntry[];
+}
+export interface OptimizedPromptResolved {
+    prompt: string;
+    fewShotExamples?: OptimizedPromptFewShotExample[];
+    optimizerSource: OptimizerName;
+}
+export interface OptimizedPromptMetadata {
+    generatedAt: string;
+    optimizer: OptimizerName;
+    score: number;
+    baselineScore: number;
+    datasetSize: number;
+}
+/**
+ * Strict parser. We reject artifacts that are missing required fields so a
+ * corrupt file cannot silently shadow the baseline prompt with garbage.
+ */
+export declare function parseOptimizedPromptArtifact(raw: unknown): OptimizedPromptArtifact | null;
+/**
+ * Stateful service. Subclassing `Service` keeps it discoverable via
+ * `runtime.getService(OPTIMIZED_PROMPT_SERVICE)` and lets us register through
+ * the standard plugin lifecycle.
+ */
+export declare class OptimizedPromptService extends Service {
+    static serviceType: string;
+    capabilityDescription: string;
+    private storeRoot;
+    private cache;
+    constructor(runtime?: IAgentRuntime);
+    static start(runtime: IAgentRuntime): Promise<OptimizedPromptService>;
+    stop(): Promise<void>;
+    /** Override the on-disk store root. Primarily for tests. */
+    setStoreRoot(root: string): void;
+    getStoreRoot(): string;
+    /**
+     * Synchronous accessor. Returns the cached artifact for the task or null.
+     * Hot path — called per-prompt in the runtime loop.
+     */
+    getPrompt(task: OptimizedPromptTask): OptimizedPromptResolved | null;
+    getMetadata(task: OptimizedPromptTask): OptimizedPromptMetadata | null;
+    /** True iff the task has any optimized artifact loaded. */
+    hasOptimized(task: OptimizedPromptTask): boolean;
+    /**
+     * Atomic write of a new artifact. Writes to a temp file under the same
+     * directory and renames into place. Refreshes the cache for the task.
+     */
+    setPrompt(task: OptimizedPromptTask, artifact: OptimizedPromptArtifact): Promise<string>;
+    /** Re-scan the on-disk store. Safe to call repeatedly. */
+    refresh(): Promise<void>;
+}
+//# sourceMappingURL=optimized-prompt.d.ts.map

package/packages/typescript/src/services/optimized-prompt.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"optimized-prompt.d.ts","sourceRoot":"","sources":["../../../../../../typescript/src/services/optimized-prompt.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAOH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAE9C,eAAO,MAAM,wBAAwB,qBAAqB,CAAC;AAE3D,MAAM,MAAM,mBAAmB,GAC5B,gBAAgB,GAChB,iBAAiB,GACjB,gBAAgB,GAChB,UAAU,GACV,mBAAmB,CAAC;AAEvB,eAAO,MAAM,sBAAsB,EAAE,SAAS,mBAAmB,EAMvD,CAAC;AAEX,MAAM,MAAM,aAAa,GACtB,oBAAoB,GACpB,kBAAkB,GAClB,mBAAmB,CAAC;AAEvB;;;GAGG;AACH,MAAM,WAAW,6BAA6B;IAC7C,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE;QACN,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;KACb,CAAC;IACF,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,2BAA2B;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,uBAAuB;IACvC,IAAI,EAAE,mBAAmB,CAAC;IAC1B,SAAS,EAAE,aAAa,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,6BAA6B,EAAE,CAAC;IAClD,OAAO,EAAE,2BAA2B,EAAE,CAAC;CACvC;AAED,MAAM,WAAW,uBAAuB;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,CAAC,EAAE,6BAA6B,EAAE,CAAC;IAClD,eAAe,EAAE,aAAa,CAAC;CAC/B;AAED,MAAM,WAAW,uBAAuB;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,aAAa,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;CACpB;AAiCD;;;GAGG;AACH,wBAAgB,4BAA4B,CAC3C,GAAG,EAAE,OAAO,GACV,uBAAuB,GAAG,IAAI,CAiDhC;AA8BD;;;;GAIG;AACH,qBAAa,sBAAuB,SAAQ,OAAO;IAClD,OAAgB,WAAW,SAA4B;IAC9C,qBAAqB,SAC8D;IAE5F,OAAO,CAAC,SAAS,CAA8B;IAC/C,OAAO,CAAC,KAAK,CAAyD;gBAE1D,OAAO,CAAC,EAAE,aAAa;WAIb,KAAK,CAC1B,OAAO,EAAE,aAAa,GACpB,OAAO,CAAC,sBAAsB,CAAC;IAMnB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAIpC,4DAA4D;IAC5D,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAIhC,YAAY,IAAI,MAAM;IAItB;;;OAGG;IACH,SAAS,CAAC,IAAI,EAAE,mBAAmB,GAAG,uBAAuB,GAAG,IAAI;IAUpE,WAAW,CAAC,IAAI,EAAE,mBAAmB,GAAG,uBAAuB,GAAG,IAAI;IAYtE,2DAA2D;IAC3D,YAAY,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO;IAIhD;;;OAGG;IACG,SAAS,CACd,IAAI,EAAE,mBAAmB,EACzB,QAAQ,EAAE,uBAAuB,GAC/B,OAAO,CAAC,MAAM,CAAC;IA8BlB,0DAA0D;IACpD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAiC9B"}

package/packages/typescript/src/services/optimized-prompt.js

@@ -0,0 +1,262 @@
+/**
+ * OptimizedPromptService — runtime cache of native-optimizer artifacts.
+ *
+ * Native MIPRO/GEPA/bootstrap-fewshot optimizers (under
+ * `apps/app-training/src/optimizers/`) write a JSON artifact per task into
+ * `~/.milady/optimized-prompts/<task>/<timestamp>.json`. The runtime consults
+ * this service before constructing the system prompt for one of the five core
+ * decision tasks and substitutes the optimized prompt (plus any few-shot
+ * demonstrations) when an artifact is available.
+ *
+ * Service contract:
+ *   - `getPrompt(task)` — synchronous accessor, returns the loaded prompt or
+ *     null. Cheap to call; reads the in-memory cache. Does not refresh.
+ *   - `setPrompt(task, artifact)` — atomically writes a new artifact and
+ *     refreshes the in-memory cache for that task.
+ *   - `getMetadata(task)` — quick view of optimizer + score for diagnostics.
+ *   - `refresh()` — re-scan the disk store. Called automatically by `start()`,
+ *     also exposed for the `Settings → Auto-Training` panel.
+ *
+ * Loading rule: for each task, the artifact with the most recent
+ * `generatedAt` wins. Ties are broken by the file's `mtime` so manual
+ * intervention (touching a file) can promote an older artifact.
+ *
+ * The on-disk format intentionally mirrors `OptimizedPromptArtifact` from
+ * `apps/app-training/src/optimizers/types.ts`. We re-declare the type here
+ * (instead of importing) because `@elizaos/core` is upstream of
+ * `@elizaos/app-training` and adding the dependency would invert the layering.
+ */
+import { existsSync, mkdirSync, readdirSync } from "node:fs";
+import { readFile, rename, writeFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+import { logger } from "../logger.js";
+import { Service } from "../types/service.js";
+export const OPTIMIZED_PROMPT_SERVICE = "optimized_prompt";
+export const OPTIMIZED_PROMPT_TASKS = [
+    "should_respond",
+    "context_routing",
+    "action_planner",
+    "response",
+    "media_description",
+];
+function resolveStateDir() {
+    const candidate = process.env.MILADY_STATE_DIR?.trim() ||
+        process.env.ELIZA_STATE_DIR?.trim() ||
+        join(homedir(), ".milady");
+    return candidate;
+}
+function defaultStoreRoot() {
+    return join(resolveStateDir(), "optimized-prompts");
+}
+function isStringRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function isOptimizerName(value) {
+    return (value === "instruction-search" ||
+        value === "prompt-evolution" ||
+        value === "bootstrap-fewshot");
+}
+function isTask(value) {
+    return (typeof value === "string" &&
+        OPTIMIZED_PROMPT_TASKS.includes(value));
+}
+/**
+ * Strict parser. We reject artifacts that are missing required fields so a
+ * corrupt file cannot silently shadow the baseline prompt with garbage.
+ */
+export function parseOptimizedPromptArtifact(raw) {
+    if (!isStringRecord(raw))
+        return null;
+    if (!isTask(raw.task))
+        return null;
+    if (!isOptimizerName(raw.optimizer))
+        return null;
+    if (typeof raw.baseline !== "string" || typeof raw.prompt !== "string") {
+        return null;
+    }
+    if (typeof raw.score !== "number" || typeof raw.baselineScore !== "number") {
+        return null;
+    }
+    if (typeof raw.datasetId !== "string" || typeof raw.datasetSize !== "number") {
+        return null;
+    }
+    if (typeof raw.generatedAt !== "string")
+        return null;
+    if (!Array.isArray(raw.lineage))
+        return null;
+    const lineage = [];
+    for (const entry of raw.lineage) {
+        if (!isStringRecord(entry))
+            continue;
+        if (typeof entry.round === "number" &&
+            typeof entry.variant === "number" &&
+            typeof entry.score === "number") {
+            lineage.push({
+                round: entry.round,
+                variant: entry.variant,
+                score: entry.score,
+                notes: typeof entry.notes === "string" ? entry.notes : undefined,
+            });
+        }
+    }
+    const fewShot = Array.isArray(raw.fewShotExamples)
+        ? coerceFewShot(raw.fewShotExamples)
+        : undefined;
+    return {
+        task: raw.task,
+        optimizer: raw.optimizer,
+        baseline: raw.baseline,
+        prompt: raw.prompt,
+        score: raw.score,
+        baselineScore: raw.baselineScore,
+        datasetId: raw.datasetId,
+        datasetSize: raw.datasetSize,
+        generatedAt: raw.generatedAt,
+        lineage,
+        fewShotExamples: fewShot,
+    };
+}
+function coerceFewShot(value) {
+    const out = [];
+    for (const entry of value) {
+        if (!isStringRecord(entry))
+            continue;
+        const input = entry.input;
+        if (!isStringRecord(input) || typeof input.user !== "string")
+            continue;
+        if (typeof entry.expectedOutput !== "string")
+            continue;
+        out.push({
+            id: typeof entry.id === "string" ? entry.id : undefined,
+            input: {
+                user: input.user,
+                system: typeof input.system === "string" ? input.system : undefined,
+            },
+            expectedOutput: entry.expectedOutput,
+            reward: typeof entry.reward === "number" ? entry.reward : undefined,
+            metadata: isStringRecord(entry.metadata) ? entry.metadata : undefined,
+        });
+    }
+    return out.length > 0 ? out : undefined;
+}
+/**
+ * Stateful service. Subclassing `Service` keeps it discoverable via
+ * `runtime.getService(OPTIMIZED_PROMPT_SERVICE)` and lets us register through
+ * the standard plugin lifecycle.
+ */
+export class OptimizedPromptService extends Service {
+    static serviceType = OPTIMIZED_PROMPT_SERVICE;
+    capabilityDescription = "Loads and serves prompts produced by the native MIPRO/GEPA/bootstrap-fewshot optimizers.";
+    storeRoot = defaultStoreRoot();
+    cache = {};
+    constructor(runtime) {
+        super(runtime);
+    }
+    static async start(runtime) {
+        const service = new OptimizedPromptService(runtime);
+        await service.refresh();
+        return service;
+    }
+    async stop() {
+        this.cache = {};
+    }
+    /** Override the on-disk store root. Primarily for tests. */
+    setStoreRoot(root) {
+        this.storeRoot = root;
+    }
+    getStoreRoot() {
+        return this.storeRoot;
+    }
+    /**
+     * Synchronous accessor. Returns the cached artifact for the task or null.
+     * Hot path — called per-prompt in the runtime loop.
+     */
+    getPrompt(task) {
+        const entry = this.cache[task];
+        if (!entry)
+            return null;
+        return {
+            prompt: entry.artifact.prompt,
+            fewShotExamples: entry.artifact.fewShotExamples,
+            optimizerSource: entry.artifact.optimizer,
+        };
+    }
+    getMetadata(task) {
+        const entry = this.cache[task];
+        if (!entry)
+            return null;
+        return {
+            generatedAt: entry.artifact.generatedAt,
+            optimizer: entry.artifact.optimizer,
+            score: entry.artifact.score,
+            baselineScore: entry.artifact.baselineScore,
+            datasetSize: entry.artifact.datasetSize,
+        };
+    }
+    /** True iff the task has any optimized artifact loaded. */
+    hasOptimized(task) {
+        return Boolean(this.cache[task]);
+    }
+    /**
+     * Atomic write of a new artifact. Writes to a temp file under the same
+     * directory and renames into place. Refreshes the cache for the task.
+     */
+    async setPrompt(task, artifact) {
+        if (artifact.task !== task) {
+            throw new Error(`[OptimizedPromptService] artifact.task=${artifact.task} does not match target task=${task}`);
+        }
+        const dir = join(this.storeRoot, task);
+        mkdirSync(dir, { recursive: true });
+        const stamp = artifact.generatedAt.replace(/[^0-9]/g, "");
+        const finalPath = join(dir, `${stamp}.json`);
+        const tempPath = `${finalPath}.tmp-${process.pid}-${Date.now()}`;
+        const payload = `${JSON.stringify(artifact, null, 2)}\n`;
+        mkdirSync(dirname(tempPath), { recursive: true });
+        await writeFile(tempPath, payload, "utf-8");
+        await rename(tempPath, finalPath);
+        this.cache[task] = { artifact, loadedAt: Date.now() };
+        logger.info({
+            src: "service:optimized_prompt",
+            task,
+            optimizer: artifact.optimizer,
+            score: artifact.score,
+            baselineScore: artifact.baselineScore,
+            path: finalPath,
+        }, "Persisted optimized prompt artifact");
+        return finalPath;
+    }
+    /** Re-scan the on-disk store. Safe to call repeatedly. */
+    async refresh() {
+        const next = {};
+        for (const task of OPTIMIZED_PROMPT_TASKS) {
+            const dir = join(this.storeRoot, task);
+            if (!existsSync(dir))
+                continue;
+            const entries = readdirSync(dir);
+            let bestArtifact = null;
+            let bestStamp = -Infinity;
+            for (const name of entries) {
+                if (!name.endsWith(".json"))
+                    continue;
+                const path = join(dir, name);
+                const raw = await readFile(path, "utf-8");
+                const parsedJson = JSON.parse(raw);
+                const artifact = parseOptimizedPromptArtifact(parsedJson);
+                if (!artifact) {
+                    logger.warn({ src: "service:optimized_prompt", task, path }, "Optimized prompt artifact failed strict parse — skipping");
+                    continue;
+                }
+                const stamp = Date.parse(artifact.generatedAt);
+                if (Number.isFinite(stamp) && stamp > bestStamp) {
+                    bestStamp = stamp;
+                    bestArtifact = artifact;
+                }
+            }
+            if (bestArtifact) {
+                next[task] = { artifact: bestArtifact, loadedAt: Date.now() };
+            }
+        }
+        this.cache = next;
+    }
+}