@elizaos/agent 2.0.0-alpha.173 → 2.0.0-alpha.192

package/apps/app-training/src/core/training-orchestrator.js

@@ -0,0 +1,409 @@
+/**
+ * Unified training-trigger orchestrator.
+ *
+ * Single entry point for all paths that kick off a training run:
+ *   - threshold:  TrainingTriggerService fires when the per-task counter passes
+ *                 the configured threshold.
+ *   - cron:       Scheduled job (e.g. nightly trajectory-export cron) calls
+ *                 with `source: 'cron'`.
+ *   - manual:     UI / CLI / API caller asks for an immediate run.
+ *
+ * Pipeline:
+ *   1. Fetch the most recent matching trajectories from the runtime's
+ *      trajectory service.
+ *   2. Run them through the privacy filter (REQUIRED — never bypass).
+ *   3. Bucket trajectories into per-task JSONL files via dataset-generator
+ *      (`exportTrajectoryTaskDatasets`). When `task` is supplied, only that
+ *      bucket is forwarded to the backend.
+ *   4. Dispatch the chosen task's dataset to the configured backend
+ *      (`vertex` | `atropos` | `tinker` | `native`).
+ *   5. Persist a run record at `<state>/training/runs/<runId>.json`.
+ */
+import { existsSync } from "node:fs";
+import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { applyPrivacyFilter, } from "./privacy-filter.js";
+import { ALL_TRAINING_TASKS, loadTrainingConfig, resolveTaskPolicy, trainingStateRoot, } from "./training-config.js";
+import { exportTrajectoryTaskDatasets, } from "./trajectory-task-datasets.js";
+function runsDir() {
+    return join(trainingStateRoot(), "runs");
+}
+function newRunId() {
+    return `run-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+}
+function nowIso() {
+    return new Date().toISOString();
+}
+function pathForTask(paths, task) {
+    switch (task) {
+        case "should_respond":
+            return paths.shouldRespondPath;
+        case "context_routing":
+            return paths.contextRoutingPath;
+        case "action_planner":
+            return paths.actionPlannerPath;
+        case "response":
+            return paths.responsePath;
+        case "media_description":
+            return paths.mediaDescriptionPath;
+    }
+}
+function selectTask(config, explicit, counts) {
+    if (explicit)
+        return explicit;
+    let bestTask = null;
+    let bestCount = 0;
+    for (const task of ALL_TRAINING_TASKS) {
+        const policy = resolveTaskPolicy(config, task);
+        const count = counts[task];
+        if (count > bestCount && count >= policy.threshold) {
+            bestCount = count;
+            bestTask = task;
+        }
+    }
+    return bestTask;
+}
+async function defaultDispatcher(input) {
+    switch (input.backend) {
+        case "atropos": {
+            const { runAtroposBackend } = await import("../backends/atropos.js");
+            const result = await runAtroposBackend({
+                datasetPath: input.datasetPath,
+                task: input.task,
+            });
+            return {
+                invoked: result.invoked,
+                artifactPath: result.stagedPath,
+                notes: result.invoked
+                    ? [`atropos exited with code ${result.exitCode ?? 0}`]
+                    : ["atropos CLI not configured (set ATROPOS_BIN to invoke)"],
+            };
+        }
+        case "tinker": {
+            const { runTinkerBackend } = await import("../backends/tinker.js");
+            const result = await runTinkerBackend({
+                datasetPath: input.datasetPath,
+                task: input.task,
+            });
+            return {
+                invoked: result.invoked,
+                artifactPath: result.jobId,
+                notes: result.notes,
+            };
+        }
+        case "vertex": {
+            // Vertex requires GCP project + bucket which the orchestrator does not
+            // know about by default; manual triggers must use the dedicated
+            // /api/training/vertex/orchestrate route. Threshold/cron firings cannot
+            // dispatch to vertex without explicit operator configuration, so we
+            // skip with a note rather than half-running.
+            return {
+                invoked: false,
+                notes: [
+                    "vertex backend requires explicit projectId + gcsBucket; use /api/training/vertex/orchestrate",
+                ],
+            };
+        }
+        case "native": {
+            const { runNativeBackend } = await import("../backends/native.js");
+            const useModelHandler = extractUseModel(input.runtime);
+            if (!useModelHandler) {
+                return {
+                    invoked: false,
+                    notes: [
+                        "native backend requires a runtime exposing useModel; skipped",
+                    ],
+                };
+            }
+            const baselinePrompt = await loadBaselineForTask(input.task);
+            const result = await runNativeBackend({
+                datasetPath: input.datasetPath,
+                task: input.task,
+                optimizer: "instruction-search",
+                baselinePrompt,
+                runtime: { useModel: useModelHandler },
+            });
+            const notes = [...result.notes];
+            let artifactPath;
+            if (result.invoked) {
+                const writePath = await persistOptimizedPromptArtifact(input.runtime, {
+                    task: input.task,
+                    optimizer: result.optimizer,
+                    baseline: baselinePrompt,
+                    prompt: result.result.optimizedPrompt,
+                    score: result.score,
+                    baselineScore: result.baselineScore,
+                    datasetId: input.datasetPath,
+                    datasetSize: result.datasetSize,
+                    generatedAt: new Date().toISOString(),
+                    lineage: result.result.lineage,
+                    fewShotExamples: result.result.fewShotExamples,
+                });
+                artifactPath = writePath ?? undefined;
+                if (writePath)
+                    notes.push(`artifact written to ${writePath}`);
+                else
+                    notes.push("OptimizedPromptService unavailable; artifact not persisted");
+            }
+            return {
+                invoked: result.invoked,
+                artifactPath,
+                notes,
+            };
+        }
+    }
+}
+function extractUseModel(runtime) {
+    const candidate = runtime;
+    if (typeof candidate.useModel !== "function")
+        return null;
+    return async (input) => {
+        // We always route native calls through TEXT_LARGE because the optimizers
+        // need high-quality rewrites. The hardcoding mirrors how trajectory
+        // teachers resolve the model — the operator can pin a different provider
+        // by overriding the runtime's TEXT_LARGE handler.
+        return await candidate.useModel?.("TEXT_LARGE", input);
+    };
+}
+async function persistOptimizedPromptArtifact(runtime, artifact) {
+    const service = runtime.getService("optimized_prompt");
+    if (!service || typeof service.setPrompt !== "function")
+        return null;
+    return await service.setPrompt(artifact.task, artifact);
+}
+/**
+ * Pull the live runtime template for the task. Falls back to a generic
+ * placeholder when the runtime cannot expose its template (e.g. cron tests
+ * that pass a stub runtime).
+ */
+async function loadBaselineForTask(task) {
+    const prompts = await import("@elizaos/core").catch(() => null);
+    if (!prompts) {
+        return `# baseline placeholder for ${task}`;
+    }
+    const promptModule = prompts;
+    switch (task) {
+        case "should_respond":
+        case "context_routing":
+            return typeof promptModule.shouldRespondTemplate === "string"
+                ? promptModule.shouldRespondTemplate
+                : `# baseline placeholder for ${task}`;
+        case "response":
+            return typeof promptModule.messageHandlerTemplate === "string"
+                ? promptModule.messageHandlerTemplate
+                : `# baseline placeholder for ${task}`;
+        case "action_planner":
+            return typeof promptModule.multiStepDecisionTemplate === "string"
+                ? promptModule.multiStepDecisionTemplate
+                : `# baseline placeholder for ${task}`;
+        case "media_description":
+            return typeof promptModule.imageDescriptionTemplate === "string"
+                ? promptModule.imageDescriptionTemplate
+                : `# baseline placeholder for ${task}`;
+    }
+}
+export async function recordRun(record) {
+    const dir = runsDir();
+    await mkdir(dir, { recursive: true });
+    const path = join(dir, `${record.runId}.json`);
+    await writeFile(path, `${JSON.stringify(record, null, 2)}\n`, "utf-8");
+    return path;
+}
+export async function loadRun(runId) {
+    const path = join(runsDir(), `${runId}.json`);
+    if (!existsSync(path))
+        return null;
+    const raw = await readFile(path, "utf-8");
+    return JSON.parse(raw);
+}
+export async function listRuns(limit = 20) {
+    const dir = runsDir();
+    if (!existsSync(dir))
+        return [];
+    const entries = await readdir(dir);
+    const runFiles = entries.filter((name) => name.endsWith(".json"));
+    // Filenames embed the timestamp prefix `run-<ms>-...` so a reverse
+    // lexicographic sort yields newest-first without a stat call.
+    runFiles.sort((a, b) => (a < b ? 1 : -1));
+    const sliced = runFiles.slice(0, Math.max(0, limit));
+    const records = [];
+    for (const file of sliced) {
+        const raw = await readFile(join(dir, file), "utf-8");
+        records.push(JSON.parse(raw));
+    }
+    return records;
+}
+/**
+ * Single entry point for kicking off a training run from any caller.
+ *
+ * Returns a record describing what happened, including `status: "skipped"`
+ * when the pipeline ran but the configured backend declined to invoke (no
+ * data, no backend configured, vertex without GCP creds, etc.). Errors are
+ * surfaced as `status: "failed"` with `reason`; never swallowed.
+ */
+export async function triggerTraining(runtime, options) {
+    const runId = newRunId();
+    const startedAt = nowIso();
+    const log = runtime.logger ?? {
+        info: () => { },
+        warn: () => { },
+        error: () => { },
+    };
+    const config = options.config ?? loadTrainingConfig();
+    const trajectoryService = runtime.getService("trajectories");
+    if (!trajectoryService ||
+        typeof trajectoryService.listTrajectories !== "function" ||
+        typeof trajectoryService.getTrajectoryDetail !== "function") {
+        const finishedAt = nowIso();
+        const record = {
+            runId,
+            status: "skipped",
+            reason: "trajectories service unavailable",
+            task: options.task ?? null,
+            backend: options.backend ?? null,
+            source: options.source,
+            datasetSize: 0,
+            startedAt,
+            finishedAt,
+            pulledTrajectories: 0,
+            filteredTrajectories: 0,
+            redactionCount: 0,
+            anonymizationCount: 0,
+            dryRun: options.dryRun ?? false,
+        };
+        await recordRun(record);
+        log.warn(`[TrainingOrchestrator] ${runId} skipped: trajectories service unavailable`);
+        return record;
+    }
+    const limit = options.trajectoryLimit ?? 500;
+    const list = await trajectoryService.listTrajectories({ limit });
+    const trajectories = [];
+    for (const item of list.trajectories ?? []) {
+        const detail = await trajectoryService.getTrajectoryDetail(item.id);
+        if (detail)
+            trajectories.push(detail);
+    }
+    const filtered = applyPrivacyFilter(trajectories, {
+        anonymizer: options.anonymizer,
+    });
+    const outputDir = join(trainingStateRoot(), "runs", runId, "datasets");
+    await mkdir(outputDir, { recursive: true });
+    const dataset = await exportTrajectoryTaskDatasets(filtered.trajectories, outputDir);
+    const task = selectTask(config, options.task, dataset.counts);
+    if (!task) {
+        const finishedAt = nowIso();
+        const record = {
+            runId,
+            status: "skipped",
+            reason: "no task reached its trigger threshold and none was specified explicitly",
+            task: null,
+            backend: options.backend ?? null,
+            source: options.source,
+            datasetSize: 0,
+            startedAt,
+            finishedAt,
+            pulledTrajectories: trajectories.length,
+            filteredTrajectories: filtered.trajectories.length,
+            redactionCount: filtered.redactionCount,
+            anonymizationCount: filtered.anonymizationCount,
+            datasetPaths: dataset.paths,
+            perTaskCounts: dataset.counts,
+            dryRun: options.dryRun ?? false,
+        };
+        await recordRun(record);
+        log.info(`[TrainingOrchestrator] ${runId} skipped: no task selected (counts=${JSON.stringify(dataset.counts)})`);
+        return record;
+    }
+    const policy = resolveTaskPolicy(config, task);
+    const backend = options.backend ?? policy.backend;
+    const datasetPath = pathForTask(dataset.paths, task);
+    const datasetSize = dataset.counts[task];
+    if (!backend) {
+        const finishedAt = nowIso();
+        const record = {
+            runId,
+            status: "skipped",
+            reason: "no backend configured",
+            task,
+            backend: null,
+            source: options.source,
+            datasetSize,
+            startedAt,
+            finishedAt,
+            pulledTrajectories: trajectories.length,
+            filteredTrajectories: filtered.trajectories.length,
+            redactionCount: filtered.redactionCount,
+            anonymizationCount: filtered.anonymizationCount,
+            datasetPaths: dataset.paths,
+            perTaskCounts: dataset.counts,
+            dryRun: options.dryRun ?? false,
+            notes: [
+                "Set training.backends in <state>/training/config.json to enable dispatch.",
+            ],
+        };
+        await recordRun(record);
+        log.info(`[TrainingOrchestrator] ${runId} skipped: no backend configured for task=${task}`);
+        return record;
+    }
+    if (options.dryRun) {
+        const finishedAt = nowIso();
+        const record = {
+            runId,
+            status: "succeeded",
+            reason: "dry run",
+            task,
+            backend,
+            source: options.source,
+            datasetSize,
+            startedAt,
+            finishedAt,
+            pulledTrajectories: trajectories.length,
+            filteredTrajectories: filtered.trajectories.length,
+            redactionCount: filtered.redactionCount,
+            anonymizationCount: filtered.anonymizationCount,
+            datasetPaths: dataset.paths,
+            perTaskCounts: dataset.counts,
+            dryRun: true,
+            notes: [`dry run; would dispatch ${datasetPath} to backend=${backend}`],
+        };
+        await recordRun(record);
+        log.info(`[TrainingOrchestrator] ${runId} dry-run task=${task} backend=${backend} datasetSize=${datasetSize}`);
+        return record;
+    }
+    const dispatcher = options.dispatcher ?? defaultDispatcher;
+    const dispatchResult = await dispatcher({
+        task,
+        backend,
+        datasetPath,
+        runId,
+        outputDir,
+        runtime,
+    });
+    const finishedAt = nowIso();
+    const status = dispatchResult.invoked
+        ? "succeeded"
+        : "skipped";
+    const record = {
+        runId,
+        status,
+        reason: dispatchResult.invoked ? undefined : "backend declined to invoke",
+        task,
+        backend,
+        source: options.source,
+        datasetSize,
+        startedAt,
+        finishedAt,
+        pulledTrajectories: trajectories.length,
+        filteredTrajectories: filtered.trajectories.length,
+        redactionCount: filtered.redactionCount,
+        anonymizationCount: filtered.anonymizationCount,
+        datasetPaths: dataset.paths,
+        perTaskCounts: dataset.counts,
+        artifactPath: dispatchResult.artifactPath,
+        dryRun: false,
+        notes: dispatchResult.notes,
+    };
+    await recordRun(record);
+    log.info(`[TrainingOrchestrator] ${runId} ${status} task=${task} backend=${backend} datasetSize=${datasetSize}`);
+    return record;
+}

package/apps/app-training/src/core/trajectory-task-datasets.d.ts

@@ -0,0 +1,38 @@
+import type { Trajectory } from "@elizaos/agent/types/trajectory";
+export interface GeminiTuningExample {
+    messages: Array<{
+        role: "system" | "user" | "model";
+        content: string;
+    }>;
+}
+export type TrajectoryTrainingTask = "should_respond" | "context_routing" | "action_planner" | "response" | "media_description";
+export interface TrajectoryTaskDatasetPaths {
+    shouldRespondPath: string;
+    contextRoutingPath: string;
+    actionPlannerPath: string;
+    responsePath: string;
+    mediaDescriptionPath: string;
+    summaryPath: string;
+}
+export interface TrajectoryTaskDatasetExport {
+    counts: Record<TrajectoryTrainingTask, number>;
+    paths: TrajectoryTaskDatasetPaths;
+    examples: Record<TrajectoryTrainingTask, GeminiTuningExample[]>;
+    summary: TrajectoryTaskDatasetSummary;
+}
+export interface TrajectoryTaskDatasetTaskSummary {
+    exampleCount: number;
+    sourceCallCount: number;
+    sourceTrajectoryCount: number;
+}
+export interface TrajectoryTaskDatasetSummary {
+    generatedAt: string;
+    trajectoryCount: number;
+    llmCallCount: number;
+    counts: Record<TrajectoryTrainingTask, number>;
+    tasks: TrajectoryTrainingTask[];
+    taskMetrics: Record<TrajectoryTrainingTask, TrajectoryTaskDatasetTaskSummary>;
+}
+export declare function extractTrajectoryExamplesByTask(trajectories: Trajectory[], tasks?: readonly TrajectoryTrainingTask[]): Record<TrajectoryTrainingTask, GeminiTuningExample[]>;
+export declare function exportTrajectoryTaskDatasets(trajectories: Trajectory[], outputDir: string, tasks?: readonly TrajectoryTrainingTask[]): Promise<TrajectoryTaskDatasetExport>;
+//# sourceMappingURL=trajectory-task-datasets.d.ts.map

package/apps/app-training/src/core/trajectory-task-datasets.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"trajectory-task-datasets.d.ts","sourceRoot":"","sources":["../../../../../../../apps/app-training/src/core/trajectory-task-datasets.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAqB,MAAM,iCAAiC,CAAC;AAErF,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,OAAO,CAAC;QAClC,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;CACJ;AAED,MAAM,MAAM,sBAAsB,GAC9B,gBAAgB,GAChB,iBAAiB,GACjB,gBAAgB,GAChB,UAAU,GACV,mBAAmB,CAAC;AAExB,MAAM,WAAW,0BAA0B;IACzC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,2BAA2B;IAC1C,MAAM,EAAE,MAAM,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;IAC/C,KAAK,EAAE,0BAA0B,CAAC;IAClC,QAAQ,EAAE,MAAM,CAAC,sBAAsB,EAAE,mBAAmB,EAAE,CAAC,CAAC;IAChE,OAAO,EAAE,4BAA4B,CAAC;CACvC;AAED,MAAM,WAAW,gCAAgC;IAC/C,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,4BAA4B;IAC3C,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;IAC/C,KAAK,EAAE,sBAAsB,EAAE,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC,sBAAsB,EAAE,gCAAgC,CAAC,CAAC;CAC/E;AAoRD,wBAAgB,+BAA+B,CAC7C,YAAY,EAAE,UAAU,EAAE,EAC1B,KAAK,CAAC,EAAE,SAAS,sBAAsB,EAAE,GACxC,MAAM,CAAC,sBAAsB,EAAE,mBAAmB,EAAE,CAAC,CAEvD;AAED,wBAAsB,4BAA4B,CAChD,YAAY,EAAE,UAAU,EAAE,EAC1B,SAAS,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,SAAS,sBAAsB,EAAE,GACxC,OAAO,CAAC,2BAA2B,CAAC,CAiGtC"}