@elizaos/agent 2.0.0-alpha.173 → 2.0.0-alpha.192

package/apps/app-training/src/optimizers/prompt-evolution.js

@@ -0,0 +1,111 @@
+/**
+ * GEPA-style genetic prompt evolution.
+ *
+ * Maintains a population of candidate prompts. Each generation:
+ *   1. Score every candidate on a held-out subset.
+ *   2. Select the top half as survivors.
+ *   3. For each survivor, with probability `mutationRate`, ask the LLM to
+ *      rewrite a slice (intro/rules/example) to produce a child.
+ *   4. Re-fill the population with mutated children of the survivors.
+ *
+ * The optimizer always preserves the all-time best prompt so an unlucky
+ * generation cannot regress past the baseline.
+ */
+import { subsample } from "./scoring.js";
+const MUTATION_INSTRUCTIONS = `You are a prompt engineer. Mutate the SYSTEM PROMPT below to explore a different phrasing.
+
+Constraints:
+- Preserve the original task contract (inputs, outputs, format).
+- Keep all literal placeholders like {{agentName}} or {{providers}} intact.
+- Pick ONE section (intro, rules, examples, output format) and rewrite it.
+- Leave the other sections untouched, but reproduce them verbatim in the output.
+- Output ONLY the mutated prompt. No commentary, no fenced code blocks.`;
+export async function runPromptEvolution(input) {
+    const population = Math.max(2, input.options?.population ?? 8);
+    const generations = input.options?.generations ?? 4;
+    const mutationRate = input.options?.mutationRate ?? 0.5;
+    const temperature = input.options?.temperature ?? 0.8;
+    const maxTokens = input.options?.maxTokens ?? 1024;
+    const rng = input.options?.rng ?? Math.random;
+    const lineage = [];
+    const heldOut = typeof input.options?.scoringSubset === "number"
+        ? subsample(input.dataset, input.options.scoringSubset, rng)
+        : input.dataset;
+    const baselineScore = await input.scorer(input.baselinePrompt, heldOut);
+    lineage.push({ round: 0, variant: 0, score: baselineScore, notes: "baseline" });
+    // Seed the population with mutations of the baseline so generation 0 is
+    // already diverse. The baseline itself stays at index 0 so the elite is
+    // always preserved.
+    let pool = [
+        { prompt: input.baselinePrompt, score: baselineScore },
+    ];
+    for (let i = 1; i < population; i += 1) {
+        const seed = await mutate(input.llm, input.baselinePrompt, {
+            temperature,
+            maxTokens,
+        });
+        const score = await input.scorer(seed, heldOut);
+        pool.push({ prompt: seed, score });
+        lineage.push({ round: 0, variant: i, score, notes: "seed mutation" });
+    }
+    let bestPrompt = input.baselinePrompt;
+    let bestScore = baselineScore;
+    for (const entry of pool) {
+        if (entry.score > bestScore) {
+            bestScore = entry.score;
+            bestPrompt = entry.prompt;
+        }
+    }
+    for (let gen = 1; gen <= generations; gen += 1) {
+        pool.sort((a, b) => b.score - a.score);
+        const cutoff = Math.max(1, Math.floor(pool.length / 2));
+        const survivors = pool.slice(0, cutoff);
+        const next = [...survivors];
+        let variantIdx = survivors.length;
+        while (next.length < population) {
+            const parent = survivors[next.length % survivors.length];
+            if (!parent)
+                break;
+            const shouldMutate = rng() < mutationRate;
+            let childPrompt = parent.prompt;
+            if (shouldMutate) {
+                childPrompt = await mutate(input.llm, parent.prompt, {
+                    temperature,
+                    maxTokens,
+                });
+            }
+            const score = await input.scorer(childPrompt, heldOut);
+            next.push({ prompt: childPrompt, score });
+            lineage.push({
+                round: gen,
+                variant: variantIdx,
+                score,
+                notes: shouldMutate ? "mutated child" : "carried forward",
+            });
+            variantIdx += 1;
+        }
+        pool = next;
+        for (const entry of pool) {
+            if (entry.score > bestScore) {
+                bestScore = entry.score;
+                bestPrompt = entry.prompt;
+            }
+        }
+    }
+    return {
+        optimizedPrompt: bestPrompt,
+        score: bestScore,
+        baseline: baselineScore,
+        lineage,
+    };
+}
+async function mutate(llm, prompt, settings) {
+    const result = await llm.complete({
+        system: MUTATION_INSTRUCTIONS,
+        user: prompt,
+        temperature: settings.temperature,
+        maxTokens: settings.maxTokens,
+    });
+    const cleaned = result.trim();
+    return cleaned.length > 0 ? cleaned : prompt;
+}

package/apps/app-training/src/optimizers/scoring.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Scoring utilities for native optimizers.
+ *
+ * The default scorer measures token-overlap agreement between the model's
+ * actual output and the expected output recorded in the trajectory dataset.
+ * It is deliberately cheap and deterministic — the optimizers run hundreds
+ * of completions per round, so we cannot afford a model-graded scorer.
+ *
+ * Token-overlap agreement (Jaccard over normalized tokens) is the same primitive
+ * that `replay-validator.ts` uses for `scoreSkill`-style success measurement,
+ * just lifted to the (output vs reference) comparison instead of (skill vs
+ * trajectory). When a richer signal becomes available — e.g. a structured
+ * TOON parser per task — the scorer factory can be swapped without changing
+ * any optimizer code.
+ */
+import type { LlmAdapter, PromptScorer } from "./types.js";
+interface ScorerOptions {
+    /** Cap on examples scored per call. Defaults to all examples. */
+    maxExamples?: number;
+    /** Temperature passed to the adapter. Defaults to 0 for determinism. */
+    temperature?: number;
+    /** Max tokens for each completion. Defaults to 512. */
+    maxTokens?: number;
+}
+/**
+ * Build a `PromptScorer` backed by a real LLM adapter.
+ *
+ * For each example:
+ *   1. Run `prompt` (as system) + `example.input.user` through the adapter.
+ *   2. Compare the completion against `example.expectedOutput` via Jaccard
+ *      similarity over normalized tokens.
+ *   3. Return the mean score.
+ *
+ * Reuses the same normalization heuristic as the trajectory-task-datasets
+ * exporter (lower-case, strip punctuation, drop empty tokens).
+ */
+export declare function createPromptScorer(adapter: LlmAdapter, options?: ScorerOptions): PromptScorer;
+/**
+ * Jaccard similarity over normalized token sets, in `[0, 1]`. Empty inputs
+ * collapse to 0 (no overlap to measure).
+ */
+export declare function scoreAgreement(actual: string, expected: string): number;
+/**
+ * Random-without-replacement subsample, used by optimizer rounds to keep
+ * scoring cheap on large datasets without sacrificing comparability across
+ * rounds (deterministic when `rng` is supplied).
+ */
+export declare function subsample<T>(items: T[], count: number, rng?: () => number): T[];
+/**
+ * Wraps `IAgentRuntime.useModel` into the `LlmAdapter` shape. We accept a
+ * loose runtime type so this module stays free of `@elizaos/core` import
+ * cycles — the native backend supplies the bound `useModel` directly.
+ */
+export interface UseModelHandler {
+    (input: {
+        prompt: string;
+        temperature?: number;
+        maxTokens?: number;
+    }): Promise<string | object | undefined>;
+}
+export declare function createRuntimeAdapter(useModel: UseModelHandler): LlmAdapter;
+export {};
+//# sourceMappingURL=scoring.d.ts.map

package/apps/app-training/src/optimizers/scoring.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scoring.d.ts","sourceRoot":"","sources":["../../../../../../../apps/app-training/src/optimizers/scoring.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EACX,UAAU,EAEV,YAAY,EACZ,MAAM,YAAY,CAAC;AAEpB,UAAU,aAAa;IACtB,iEAAiE;IACjE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CACjC,OAAO,EAAE,UAAU,EACnB,OAAO,GAAE,aAAkB,GACzB,YAAY,CAmBd;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAYvE;AAWD;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,CAAC,EAC1B,KAAK,EAAE,CAAC,EAAE,EACV,KAAK,EAAE,MAAM,EACb,GAAG,GAAE,MAAM,MAAoB,GAC7B,CAAC,EAAE,CAYL;AAED;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC/B,CAAC,KAAK,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;KACnB,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC,CAAC;CACzC;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,eAAe,GAAG,UAAU,CAgB1E"}

package/apps/app-training/src/optimizers/scoring.js

@@ -0,0 +1,117 @@
+/**
+ * Scoring utilities for native optimizers.
+ *
+ * The default scorer measures token-overlap agreement between the model's
+ * actual output and the expected output recorded in the trajectory dataset.
+ * It is deliberately cheap and deterministic — the optimizers run hundreds
+ * of completions per round, so we cannot afford a model-graded scorer.
+ *
+ * Token-overlap agreement (Jaccard over normalized tokens) is the same primitive
+ * that `replay-validator.ts` uses for `scoreSkill`-style success measurement,
+ * just lifted to the (output vs reference) comparison instead of (skill vs
+ * trajectory). When a richer signal becomes available — e.g. a structured
+ * TOON parser per task — the scorer factory can be swapped without changing
+ * any optimizer code.
+ */
+/**
+ * Build a `PromptScorer` backed by a real LLM adapter.
+ *
+ * For each example:
+ *   1. Run `prompt` (as system) + `example.input.user` through the adapter.
+ *   2. Compare the completion against `example.expectedOutput` via Jaccard
+ *      similarity over normalized tokens.
+ *   3. Return the mean score.
+ *
+ * Reuses the same normalization heuristic as the trajectory-task-datasets
+ * exporter (lower-case, strip punctuation, drop empty tokens).
+ */
+export function createPromptScorer(adapter, options = {}) {
+    const temperature = options.temperature ?? 0;
+    const maxTokens = options.maxTokens ?? 512;
+    return async (prompt, examples) => {
+        if (examples.length === 0)
+            return 0;
+        const cap = options.maxExamples ?? examples.length;
+        const limited = examples.slice(0, Math.max(1, cap));
+        let total = 0;
+        for (const example of limited) {
+            const completion = await adapter.complete({
+                system: prompt,
+                user: example.input.user,
+                temperature,
+                maxTokens,
+            });
+            total += scoreAgreement(completion, example.expectedOutput);
+        }
+        return total / limited.length;
+    };
+}
+/**
+ * Jaccard similarity over normalized token sets, in `[0, 1]`. Empty inputs
+ * collapse to 0 (no overlap to measure).
+ */
+export function scoreAgreement(actual, expected) {
+    const actualTokens = tokenize(actual);
+    const expectedTokens = tokenize(expected);
+    if (expectedTokens.size === 0 && actualTokens.size === 0)
+        return 1;
+    if (expectedTokens.size === 0 || actualTokens.size === 0)
+        return 0;
+    let intersection = 0;
+    for (const token of actualTokens) {
+        if (expectedTokens.has(token))
+            intersection += 1;
+    }
+    const union = actualTokens.size + expectedTokens.size - intersection;
+    if (union === 0)
+        return 0;
+    return intersection / union;
+}
+function tokenize(text) {
+    const tokens = text
+        .toLowerCase()
+        .replace(/[^a-z0-9\s_-]+/g, " ")
+        .split(/\s+/)
+        .filter((token) => token.length > 0);
+    return new Set(tokens);
+}
+/**
+ * Random-without-replacement subsample, used by optimizer rounds to keep
+ * scoring cheap on large datasets without sacrificing comparability across
+ * rounds (deterministic when `rng` is supplied).
+ */
+export function subsample(items, count, rng = Math.random) {
+    if (count >= items.length)
+        return [...items];
+    const indices = new Set();
+    const out = [];
+    while (out.length < count) {
+        const idx = Math.floor(rng() * items.length);
+        if (indices.has(idx))
+            continue;
+        indices.add(idx);
+        const item = items[idx];
+        if (item !== undefined)
+            out.push(item);
+    }
+    return out;
+}
+export function createRuntimeAdapter(useModel) {
+    return {
+        async complete(input) {
+            const composed = input.system
+                ? `${input.system}\n\n${input.user}`
+                : input.user;
+            const response = await useModel({
+                prompt: composed,
+                temperature: input.temperature,
+                maxTokens: input.maxTokens,
+            });
+            if (typeof response === "string")
+                return response;
+            if (response === undefined || response === null)
+                return "";
+            return JSON.stringify(response);
+        },
+    };
+}

package/apps/app-training/src/optimizers/types.d.ts

@@ -0,0 +1,105 @@
+/**
+ * Native prompt-optimization primitives.
+ *
+ * The optimizers under `apps/app-training/src/optimizers/` (instruction-search,
+ * prompt-evolution, bootstrap-fewshot) all operate on the same triple:
+ *
+ *   - `OptimizationExample`: a labeled (input -> expected output) row, mirroring
+ *     the JSONL shape produced by `trajectory-task-datasets.ts`.
+ *   - `PromptScorer`: pluggable evaluator that returns a score in `[0, 1]` for
+ *     a candidate prompt against a held-out subset of examples.
+ *   - `LlmAdapter`: thin wrapper over `runtime.useModel()` so optimizer code
+ *     never depends on a specific provider.
+ *
+ * Decoupling the LLM behind `LlmAdapter` makes the optimizer modules unit
+ * testable with a deterministic in-memory adapter — no HTTP, no fixtures.
+ */
+import type { TrajectoryTrainingTask } from "../core/trajectory-task-datasets.js";
+/**
+ * One row in the optimization dataset.
+ *
+ * `input.system`/`input.user` mirror the `messages` arrays produced by the
+ * trajectory dataset exporter (`should_respond`, `response`, etc.). The
+ * exporter always wires those into the `system`/`user` slots, so this is the
+ * canonical shape for all native optimizer code.
+ */
+export interface OptimizationExample {
+    /** Stable identifier for traceability. Defaults to the row index. */
+    id?: string;
+    input: {
+        system?: string;
+        user: string;
+    };
+    /** Reference output the model should produce. Compared by the scorer. */
+    expectedOutput: string;
+    /**
+     * Optional reward signal recorded with the trajectory (e.g. successful
+     * completion = 1). Bootstrap-fewshot uses this to pick top-K demonstrations.
+     */
+    reward?: number;
+    /** Optional per-row metadata (task name, source trajectory id, ...). */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * Scorer signature.
+ *
+ * Returns the mean score in `[0, 1]` across the supplied examples. A scorer
+ * MUST be deterministic given the same prompt + dataset + adapter, otherwise
+ * the optimizer cannot tell signal from noise across rounds.
+ */
+export type PromptScorer = (prompt: string, examples: OptimizationExample[]) => Promise<number>;
+/**
+ * Minimal LLM interface the optimizers depend on. Implementations route to
+ * `runtime.useModel(ModelType.TEXT_LARGE, …)` in production and to a
+ * deterministic stub in tests.
+ */
+export interface LlmAdapter {
+    /**
+     * Run a single completion. Returns plain text (no parsing).
+     *
+     * `temperature` is optional because some adapters (e.g. tests) ignore it,
+     * but optimizers should pass it explicitly when they want diverse samples.
+     */
+    complete(input: {
+        system?: string;
+        user: string;
+        temperature?: number;
+        maxTokens?: number;
+    }): Promise<string>;
+}
+/** Per-round bookkeeping returned by every optimizer. */
+export interface OptimizerLineageEntry {
+    round: number;
+    variant: number;
+    score: number;
+    notes?: string;
+}
+/** Common shape returned by all native optimizers. */
+export interface OptimizerResult {
+    optimizedPrompt: string;
+    score: number;
+    baseline: number;
+    lineage: OptimizerLineageEntry[];
+    /** Demonstrations injected into the prompt (bootstrap-fewshot only). */
+    fewShotExamples?: OptimizationExample[];
+}
+export type OptimizerName = "instruction-search" | "prompt-evolution" | "bootstrap-fewshot";
+/**
+ * Persisted artifact written by the native backend and consumed by
+ * `OptimizedPromptService`. The persisted-on-disk schema is checked field by
+ * field on read; required fields stay required (no `?? null` fallbacks).
+ */
+export interface OptimizedPromptArtifact {
+    task: TrajectoryTrainingTask;
+    optimizer: OptimizerName;
+    baseline: string;
+    prompt: string;
+    score: number;
+    baselineScore: number;
+    datasetId: string;
+    datasetSize: number;
+    generatedAt: string;
+    fewShotExamples?: OptimizationExample[];
+    lineage: OptimizerLineageEntry[];
+}
+//# sourceMappingURL=types.d.ts.map

package/apps/app-training/src/optimizers/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../../../../apps/app-training/src/optimizers/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,qCAAqC,CAAC;AAElF;;;;;;;GAOG;AACH,MAAM,WAAW,mBAAmB;IACnC,qEAAqE;IACrE,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE;QACN,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;KACb,CAAC;IACF,yEAAyE;IACzE,cAAc,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED;;;;;;GAMG;AACH,MAAM,MAAM,YAAY,GAAG,CAC1B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,mBAAmB,EAAE,KAC3B,OAAO,CAAC,MAAM,CAAC,CAAC;AAErB;;;;GAIG;AACH,MAAM,WAAW,UAAU;IAC1B;;;;;OAKG;IACH,QAAQ,CAAC,KAAK,EAAE;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;KACnB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACpB;AAED,yDAAyD;AACzD,MAAM,WAAW,qBAAqB;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;CACf;AAED,sDAAsD;AACtD,MAAM,WAAW,eAAe;IAC/B,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,wEAAwE;IACxE,eAAe,CAAC,EAAE,mBAAmB,EAAE,CAAC;CACxC;AAED,MAAM,MAAM,aAAa,GACtB,oBAAoB,GACpB,kBAAkB,GAClB,mBAAmB,CAAC;AAEvB;;;;GAIG;AACH,MAAM,WAAW,uBAAuB;IACvC,IAAI,EAAE,sBAAsB,CAAC;IAC7B,SAAS,EAAE,aAAa,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACxC,OAAO,EAAE,qBAAqB,EAAE,CAAC;CACjC"}

package/apps/app-training/src/optimizers/types.js

@@ -0,0 +1,17 @@
+/**
+ * Native prompt-optimization primitives.
+ *
+ * The optimizers under `apps/app-training/src/optimizers/` (instruction-search,
+ * prompt-evolution, bootstrap-fewshot) all operate on the same triple:
+ *
+ *   - `OptimizationExample`: a labeled (input -> expected output) row, mirroring
+ *     the JSONL shape produced by `trajectory-task-datasets.ts`.
+ *   - `PromptScorer`: pluggable evaluator that returns a score in `[0, 1]` for
+ *     a candidate prompt against a held-out subset of examples.
+ *   - `LlmAdapter`: thin wrapper over `runtime.useModel()` so optimizer code
+ *     never depends on a specific provider.
+ *
+ * Decoupling the LLM behind `LlmAdapter` makes the optimizer modules unit
+ * testable with a deterministic in-memory adapter — no HTTP, no fixtures.
+ */
+export {};

package/apps/app-training/src/services/index.d.ts

@@ -1,3 +1,4 @@
 export type { TrainingServiceLike, TrainingServiceWithRuntime } from "./training-service-like.js";
 export { detectAvailableBackends, clearBackendCache, type BackendAvailability } from "./training-backend-check.js";
+export { bootstrapOptimizationFromAccumulatedTrajectories, TrainingTriggerService, registerTrainingTriggerService, TRAINING_TRIGGER_SERVICE, type BootstrapOptimizationOptions, type RegisteredTrainingTriggerEntry, type TrainingTriggerServiceOptions, type TriggerStatusSnapshot, } from "./training-trigger.js";
 //# sourceMappingURL=index.d.ts.map

package/apps/app-training/src/services/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../../../apps/app-training/src/services/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,mBAAmB,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAClG,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,KAAK,mBAAmB,EAAE,MAAM,6BAA6B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../../../apps/app-training/src/services/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,mBAAmB,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAClG,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,KAAK,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AACnH,OAAO,EACL,gDAAgD,EAChD,sBAAsB,EACtB,8BAA8B,EAC9B,wBAAwB,EACxB,KAAK,4BAA4B,EACjC,KAAK,8BAA8B,EACnC,KAAK,6BAA6B,EAClC,KAAK,qBAAqB,GAC3B,MAAM,uBAAuB,CAAC"}

package/apps/app-training/src/services/index.js

@@ -1 +1,2 @@
 export { detectAvailableBackends, clearBackendCache } from "./training-backend-check.js";
+export { bootstrapOptimizationFromAccumulatedTrajectories, TrainingTriggerService, registerTrainingTriggerService, TRAINING_TRIGGER_SERVICE, } from "./training-trigger.js";

package/apps/app-training/src/services/training-trigger.d.ts

@@ -0,0 +1,166 @@
+/**
+ * TrainingTriggerService — counts completed trajectories per task and fires
+ * `triggerTraining` when the configured threshold is reached for a task whose
+ * cooldown has elapsed.
+ *
+ * State is held in-memory and mirrored to disk at
+ * `<state>/training/trigger-state.json` so a process restart does not lose
+ * progress mid-bucket. The persisted shape is intentionally tiny and
+ * forward-compatible: extra keys are ignored on read.
+ *
+ * Service identifier: `TRAINING_TRIGGER_SERVICE`.
+ *
+ * Trajectory storage calls `notifyTrajectoryCompleted(trajectoryId)` from
+ * `eliza/packages/agent/src/runtime/trajectory-storage.ts` whenever a
+ * trajectory transitions to `completed`. The service looks up the trajectory
+ * detail, infers the tasks it touches, and increments those counters.
+ */
+import type { AnonymizerLookup } from "../core/privacy-filter.js";
+import { type TrainingBackend, type TrainingConfig } from "../core/training-config.js";
+import { type TrainingRunRecord, triggerTraining } from "../core/training-orchestrator.js";
+import type { TrajectoryTrainingTask } from "../core/trajectory-task-datasets.js";
+export declare const TRAINING_TRIGGER_SERVICE = "TRAINING_TRIGGER_SERVICE";
+interface MinimalLogger {
+    info: (message: string) => void;
+    warn: (message: string) => void;
+    error: (message: string) => void;
+}
+interface RuntimeLike {
+    getService: (name: string) => unknown;
+    logger?: MinimalLogger;
+}
+export interface TriggerStatusSnapshot {
+    autoTrainEnabled: boolean;
+    triggerThreshold: number;
+    cooldownHours: number;
+    counters: Record<TrajectoryTrainingTask, number>;
+    lastTrain: Partial<Record<TrajectoryTrainingTask, {
+        runId: string;
+        source: string;
+        finishedAt: string;
+        status: string;
+    }>>;
+    perTaskThresholds: Record<TrajectoryTrainingTask, number>;
+    perTaskCooldownMs: Record<TrajectoryTrainingTask, number>;
+}
+export interface TrainingTriggerServiceOptions {
+    /** Defaults to <state>/training/trigger-state.json. */
+    statePath?: string;
+    /** Override config loader (tests). */
+    configLoader?: () => TrainingConfig;
+    /** Forward to the orchestrator (tests). */
+    triggerImpl?: typeof triggerTraining;
+    /** Anonymizer forwarded to the orchestrator. */
+    anonymizer?: AnonymizerLookup;
+    /** Clock injection (tests). */
+    now?: () => number;
+}
+/**
+ * Lightweight, function-based service. We follow the n8n-dispatch pattern
+ * already used in `runtime/eliza.ts` (services map insertion) rather than
+ * subclassing `Service`, because the trigger has no `start()` lifecycle —
+ * it is created once at runtime boot and lives for the runtime's lifetime.
+ */
+export declare class TrainingTriggerService {
+    readonly capabilityDescription = "Tracks completed trajectories and fires training runs when per-task counters hit the configured threshold.";
+    private readonly runtime;
+    private readonly statePath;
+    private readonly configLoader;
+    private readonly triggerImpl;
+    private readonly anonymizer?;
+    private readonly now;
+    private readonly inflight;
+    private state;
+    constructor(runtime: RuntimeLike, options?: TrainingTriggerServiceOptions);
+    /**
+     * Idempotent no-op; kept so callers can swap the service into a `Service`
+     * subclass later without changing the registration site.
+     */
+    stop(): Promise<void>;
+    getStatus(): TriggerStatusSnapshot;
+    /**
+     * Called by trajectory storage when a trajectory transitions to
+     * `completed`. Looks up the detail, increments the relevant counters, and
+     * checks each touched task's threshold + cooldown.
+     *
+     * Errors propagate by design — the caller passes through optional chaining
+     * so a missing service is a no-op, but a misconfigured service that throws
+     * during increment should not be silently lost.
+     */
+    notifyTrajectoryCompleted(trajectoryId: string): Promise<void>;
+    /** Manual fire — bypasses the autoTrain toggle (operator action). */
+    runManually(input: {
+        task?: TrajectoryTrainingTask;
+        backend?: TrainingBackend;
+        dryRun?: boolean;
+    }): Promise<TrainingRunRecord>;
+    /**
+     * Reset counters — primarily for tests and operator recovery. The persisted
+     * `lastTrainAt`/`lastTrainRecord` map is preserved so cooldowns still
+     * apply after a reset.
+     */
+    resetCounters(task?: TrajectoryTrainingTask): void;
+    private maybeFire;
+    private recordCompletion;
+}
+/**
+ * Register the service against a runtime's services map. Mirrors the
+ * n8n-dispatch pattern in `packages/app-core/src/runtime/eliza.ts`: we
+ * insert a function-shaped service entry directly rather than going through
+ * the `Service.start()` lifecycle, which expects a class.
+ *
+ * Safe to call multiple times — subsequent calls replace the entry.
+ */
+export declare function registerTrainingTriggerService(runtime: RuntimeLike & {
+    services: Map<string, unknown[]>;
+}, options?: TrainingTriggerServiceOptions): TrainingTriggerService;
+interface UserNotifier {
+    notify: (message: string) => void;
+}
+export interface BootstrapOptimizationOptions {
+    configLoader?: () => TrainingConfig;
+    notifier?: UserNotifier;
+    /**
+     * Override the service used to trigger the run. Tests pass a stub; in
+     * production the registered TrainingTriggerService is looked up from
+     * runtime.services.
+     */
+    triggerOverride?: (input: {
+        task: TrajectoryTrainingTask;
+        backend: TrainingBackend;
+    }) => Promise<TrainingRunRecord>;
+}
+/**
+ * One-shot bootstrap pass for default-on Hermes parity.
+ *
+ * Called immediately after `registerTrainingTriggerService` during runtime
+ * boot. For each high-leverage task (should_respond + action_planner):
+ *   - If `MILADY_DISABLE_AUTO_BOOTSTRAP=1`, do nothing.
+ *   - If the OptimizedPromptService already has an artifact for the task,
+ *     do nothing (the operator's previous run wins).
+ *   - If the per-task trajectory counter is below the threshold, do nothing
+ *     (we don't want to optimize against a thin dataset).
+ *   - Otherwise, fire `triggerTraining({ source: 'bootstrap', backend: 'native' })`
+ *     and notify the user that progress can be tracked in
+ *     Settings → Auto-Training.
+ *
+ * This is fire-and-forget on purpose: the runtime boot must not block on
+ * an LLM-driven optimization round. Errors propagate so the boot logger
+ * surfaces them rather than swallowing them silently.
+ */
+export declare function bootstrapOptimizationFromAccumulatedTrajectories(runtime: RuntimeLike, service: TrainingTriggerService, options?: BootstrapOptimizationOptions): Promise<TrajectoryTrainingTask[]>;
+export interface RegisteredTrainingTriggerEntry {
+    notifyTrajectoryCompleted: (trajectoryId: string) => Promise<void>;
+    getStatus: () => TriggerStatusSnapshot;
+    runManually: (input: {
+        task?: TrajectoryTrainingTask;
+        backend?: TrainingBackend;
+        dryRun?: boolean;
+    }) => Promise<TrainingRunRecord>;
+    resetCounters: (task?: TrajectoryTrainingTask) => void;
+    stop: () => Promise<void>;
+    capabilityDescription: string;
+    instance: TrainingTriggerService;
+}
+export {};
+//# sourceMappingURL=training-trigger.d.ts.map

package/apps/app-training/src/services/training-trigger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"training-trigger.d.ts","sourceRoot":"","sources":["../../../../../../../apps/app-training/src/services/training-trigger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,OAAO,KAAK,EACV,gBAAgB,EAEjB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAIL,KAAK,eAAe,EACpB,KAAK,cAAc,EAEpB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,KAAK,iBAAiB,EACtB,eAAe,EAChB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,qCAAqC,CAAC;AAElF,eAAO,MAAM,wBAAwB,6BAA6B,CAAC;AAEnE,UAAU,aAAa;IACrB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,KAAK,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CAClC;AAMD,UAAU,WAAW;IACnB,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;IACtC,MAAM,CAAC,EAAE,aAAa,CAAC;CACxB;AAcD,MAAM,WAAW,qBAAqB;IACpC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;IACjD,SAAS,EAAE,OAAO,CAChB,MAAM,CACJ,sBAAsB,EACtB;QACE,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,MAAM,CAAC;KAChB,CACF,CACF,CAAC;IACF,iBAAiB,EAAE,MAAM,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;IAC1D,iBAAiB,EAAE,MAAM,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;CAC3D;AA+ID,MAAM,WAAW,6BAA6B;IAC5C,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sCAAsC;IACtC,YAAY,CAAC,EAAE,MAAM,cAAc,CAAC;IACpC,2CAA2C;IAC3C,WAAW,CAAC,EAAE,OAAO,eAAe,CAAC;IACrC,gDAAgD;IAChD,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,+BAA+B;IAC/B,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED;;;;;GAKG;AACH,qBAAa,sBAAsB;IACjC,QAAQ,CAAC,qBAAqB,gHACiF;IAE/G,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAc;IACtC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAuB;IACpD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyB;IACrD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAmB;IAC/C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAqC;IAC9D,OAAO,CAAC,KAAK,CAAwB;gBAGnC,OAAO,EAAE,WAAW,EACpB,OAAO,GAAE,6BAAkC;IAY7C;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAE3B,SAAS,IAAI,qBAAqB;IAgClC;;;;;;;;OAQG;IACG,yBAAyB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgCpE,qEAAqE;IAC/D,WAAW,CAAC,KAAK,EAAE;QACvB,IAAI,CAAC,EAAE,sBAAsB,CAAC;QAC9B,OAAO,CAAC,EAAE,eAAe,CAAC;QAC1B,MAAM,CAAC,EAAE,OAAO,CAAC;KAClB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAY9B;;;;OAIG;IACH,aAAa,CAAC,IAAI,CAAC,EAAE,sBAAsB,GAAG,IAAI;YASpC,SAAS;IAuCvB,OAAO,CAAC,gBAAgB;CAWzB;AAED;;;;;;;GAOG;AACH,wBAAgB,8BAA8B,CAC5C,OAAO,EAAE,WAAW,GAAG;IACrB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;CAClC,EACD,OAAO,GAAE,6BAAkC,GAC1C,sBAAsB,CAaxB;AAWD,UAAU,YAAY;IACpB,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACnC;AAED,MAAM,WAAW,4BAA4B;IAC3C,YAAY,CAAC,EAAE,MAAM,cAAc,CAAC;IACpC,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB;;;;OAIG;IACH,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE;QACxB,IAAI,EAAE,sBAAsB,CAAC;QAC7B,OAAO,EAAE,eAAe,CAAC;KAC1B,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;CAClC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,gDAAgD,CACpE,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE,sBAAsB,EAC/B,OAAO,GAAE,4BAAiC,GACzC,OAAO,CAAC,sBAAsB,EAAE,CAAC,CAiCnC;AAED,MAAM,WAAW,8BAA8B;IAC7C,yBAAyB,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACnE,SAAS,EAAE,MAAM,qBAAqB,CAAC;IACvC,WAAW,EAAE,CAAC,KAAK,EAAE;QACnB,IAAI,CAAC,EAAE,sBAAsB,CAAC;QAC9B,OAAO,CAAC,EAAE,eAAe,CAAC;QAC1B,MAAM,CAAC,EAAE,OAAO,CAAC;KAClB,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACjC,aAAa,EAAE,CAAC,IAAI,CAAC,EAAE,sBAAsB,KAAK,IAAI,CAAC;IACvD,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,QAAQ,EAAE,sBAAsB,CAAC;CAClC"}