@elizaos/plugin-training 2.0.3-beta.6 → 2.0.3-beta.7

package/dist/core/eval-comparison-artifact.d.ts

@@ -0,0 +1,72 @@
+export declare const EVAL_COMPARISON_ARTIFACT_SCHEMA = "eliza_eval_comparison_artifact";
+export declare const EVAL_COMPARISON_ARTIFACT_VERSION = 1;
+export interface EvalComparisonArtifactInput {
+    report: Record<string, unknown>;
+    reportPath?: string;
+    outputDir?: string;
+    source?: Record<string, unknown>;
+}
+export interface EvalComparisonRunOptions {
+    trainingRoot?: string;
+    python?: string;
+    manifestPath?: string;
+    model?: string;
+    trainedModelPath?: string;
+    backend?: "mlx" | "cuda" | "cpu";
+    promptFile?: string;
+    maxTokens?: number;
+    systemPrompt?: string;
+    outputPath?: string;
+    outputDir?: string;
+    dryRun?: boolean;
+}
+export interface EvalComparisonArtifact {
+    schema: typeof EVAL_COMPARISON_ARTIFACT_SCHEMA;
+    version: typeof EVAL_COMPARISON_ARTIFACT_VERSION;
+    generatedAt: string;
+    reportPath?: string;
+    source: Record<string, unknown>;
+    models: {
+        base: string | null;
+        trained: string | null;
+        backend: string | null;
+    };
+    metrics: {
+        baseScore: number | null;
+        trainedScore: number | null;
+        improvementAbsolute: number | null;
+        improvementPercent: number | null;
+        baseLatencyMs: number | null;
+        trainedLatencyMs: number | null;
+        latencyDeltaMs: number | null;
+        promptCount: number | null;
+        distinctResponseCount: number | null;
+    };
+    summaries: {
+        base: Record<string, unknown> | null;
+        trained: Record<string, unknown> | null;
+        comparison: Record<string, unknown> | null;
+    };
+    raw: Record<string, unknown>;
+}
+export interface EvalComparisonArtifactResult {
+    outputDir: string;
+    artifactPath: string;
+    artifact: EvalComparisonArtifact;
+}
+export interface EvalComparisonRunResult extends EvalComparisonArtifactResult {
+    trainingRoot: string;
+    command: string[];
+    reportPath: string;
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+}
+export declare function buildLocalEvalComparisonArgs(options: EvalComparisonRunOptions, resolved: {
+    trainingRoot: string;
+    reportPath: string;
+}): string[];
+export declare function buildEvalComparisonArtifactPayload(input: EvalComparisonArtifactInput): EvalComparisonArtifact;
+export declare function writeEvalComparisonArtifact(input: EvalComparisonArtifactInput): Promise<EvalComparisonArtifactResult>;
+export declare function runLocalEvalComparison(options: EvalComparisonRunOptions): Promise<EvalComparisonRunResult>;
+//# sourceMappingURL=eval-comparison-artifact.d.ts.map

package/dist/core/eval-comparison-artifact.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-comparison-artifact.d.ts","sourceRoot":"","sources":["../../src/core/eval-comparison-artifact.ts"],"names":[],"mappings":"AAKA,eAAO,MAAM,+BAA+B,mCAAmC,CAAC;AAChF,eAAO,MAAM,gCAAgC,IAAI,CAAC;AAElD,MAAM,WAAW,2BAA2B;IAC1C,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,wBAAwB;IACvC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,KAAK,CAAC;IACjC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,OAAO,+BAA+B,CAAC;IAC/C,OAAO,EAAE,OAAO,gCAAgC,CAAC;IACjD,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QACpB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QACvB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;KACxB,CAAC;IACF,OAAO,EAAE;QACP,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;QACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;QACnC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;QAClC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QAC7B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;QAChC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;QAC9B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;KACtC,CAAC;IACF,SAAS,EAAE;QACT,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;QACrC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;QACxC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;KAC5C,CAAC;IACF,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,4BAA4B;IAC3C,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,sBAAsB,CAAC;CAClC;AAED,MAAM,WAAW,uBAAwB,SAAQ,4BAA4B;IAC3E,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,4BAA4B,CAC1C,OAAO,EAAE,wBAAwB,EACjC,QAAQ,EAAE;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GACrD,MAAM,EAAE,CA8BV;AAwED,wBAAgB,kCAAkC,CAChD,KAAK,EAAE,2BAA2B,GACjC,sBAAsB,CA0FxB;AAMD,wBAAsB,2BAA2B,CAC/C,KAAK,EAAE,2BAA2B,GACjC,OAAO,CAAC,4BAA4B,CAAC,CAavC;AA6BD,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,wBAAwB,GAChC,OAAO,CAAC,uBAAuB,CAAC,CAwFlC"}

package/dist/core/eval-comparison-artifact.js

@@ -0,0 +1,281 @@
+import { spawn } from "node:child_process";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { trainingStateRoot } from "./training-config.js";
+const EVAL_COMPARISON_ARTIFACT_SCHEMA = "eliza_eval_comparison_artifact";
+const EVAL_COMPARISON_ARTIFACT_VERSION = 1;
+function buildLocalEvalComparisonArgs(options, resolved) {
+  const scriptPath = join(
+    resolved.trainingRoot,
+    "scripts",
+    "rl",
+    "compare_local_models.py"
+  );
+  const args = [scriptPath];
+  if (options.manifestPath) {
+    args.push("--manifest", options.manifestPath);
+  } else {
+    if (!options.model || !options.trainedModelPath || !options.backend) {
+      throw new Error(
+        "Provide either manifestPath or model, trainedModelPath, and backend"
+      );
+    }
+    args.push("--model", options.model);
+    args.push("--trained-model-path", options.trainedModelPath);
+    args.push("--backend", options.backend);
+  }
+  if (options.promptFile) args.push("--prompt-file", options.promptFile);
+  if (typeof options.maxTokens === "number") {
+    args.push(
+      "--max-tokens",
+      String(Math.max(1, Math.floor(options.maxTokens)))
+    );
+  }
+  if (options.systemPrompt) args.push("--system-prompt", options.systemPrompt);
+  args.push("--output", resolved.reportPath);
+  return args;
+}
+function asRecord(value) {
+  return value && typeof value === "object" && !Array.isArray(value) ? value : null;
+}
+function asString(value) {
+  return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
+}
+function asNumber(value) {
+  return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+function firstNumber(record, keys) {
+  if (!record) return null;
+  for (const key of keys) {
+    const value = asNumber(record[key]);
+    if (value !== null) return value;
+  }
+  return null;
+}
+function firstString(record, keys) {
+  if (!record) return null;
+  for (const key of keys) {
+    const value = asString(record[key]);
+    if (value) return value;
+  }
+  return null;
+}
+function nestedSummary(report, key) {
+  const variant = asRecord(report[key]);
+  return asRecord(variant?.summary);
+}
+function extractVariantModel(report, key) {
+  const variant = asRecord(report[key]);
+  return firstString(variant, [
+    "model_name",
+    "model_ref",
+    "adapter_path",
+    "served_model_id"
+  ]);
+}
+function roundMetric(value) {
+  return value === null ? null : Number(value.toFixed(4));
+}
+function improvementPercent(base, trained) {
+  if (base === null || trained === null || base === 0) return null;
+  return (trained - base) / Math.abs(base) * 100;
+}
+function buildEvalComparisonArtifactPayload(input) {
+  const report = input.report;
+  const baseSummary = nestedSummary(report, "base_model") ?? asRecord(report.base_summary);
+  const trainedSummary = nestedSummary(report, "trained_model") ?? nestedSummary(report, "adapter_model") ?? asRecord(report.trained_summary) ?? asRecord(report.adapter_summary);
+  const comparison = asRecord(report.comparison);
+  const baseScore = firstNumber(baseSummary, [
+    "avg_score",
+    "score",
+    "format_ok",
+    "content_ok",
+    "test_avg_score"
+  ]);
+  const trainedScore = firstNumber(trainedSummary, [
+    "avg_score",
+    "score",
+    "format_ok",
+    "content_ok",
+    "test_avg_score"
+  ]);
+  const baseLatencyMs = firstNumber(baseSummary, [
+    "avg_latency_ms",
+    "latency_ms"
+  ]);
+  const trainedLatencyMs = firstNumber(trainedSummary, [
+    "avg_latency_ms",
+    "latency_ms"
+  ]);
+  const promptCount = firstNumber(baseSummary, ["prompt_count", "test_sample_count"]) ?? firstNumber(trainedSummary, ["prompt_count", "test_sample_count"]);
+  const generatedAt = asString(report.timestamp) ?? asString(report.generated_at) ?? asString(report.evaluated_at) ?? (/* @__PURE__ */ new Date()).toISOString();
+  return {
+    schema: EVAL_COMPARISON_ARTIFACT_SCHEMA,
+    version: EVAL_COMPARISON_ARTIFACT_VERSION,
+    generatedAt,
+    reportPath: input.reportPath,
+    source: input.source ?? { kind: "training_eval_comparison" },
+    models: {
+      base: extractVariantModel(report, "base_model") ?? firstString(report, ["base_model", "model", "base_model_id"]),
+      trained: extractVariantModel(report, "trained_model") ?? extractVariantModel(report, "adapter_model") ?? firstString(report, [
+        "trained_model",
+        "adapter_model",
+        "trained_model_id"
+      ]),
+      backend: asString(report.backend)
+    },
+    metrics: {
+      baseScore: roundMetric(baseScore),
+      trainedScore: roundMetric(trainedScore),
+      improvementAbsolute: roundMetric(
+        baseScore !== null && trainedScore !== null ? trainedScore - baseScore : null
+      ),
+      improvementPercent: roundMetric(
+        improvementPercent(baseScore, trainedScore)
+      ),
+      baseLatencyMs: roundMetric(baseLatencyMs),
+      trainedLatencyMs: roundMetric(trainedLatencyMs),
+      latencyDeltaMs: roundMetric(
+        baseLatencyMs !== null && trainedLatencyMs !== null ? trainedLatencyMs - baseLatencyMs : null
+      ),
+      promptCount: promptCount === null ? null : Math.round(promptCount),
+      distinctResponseCount: firstNumber(comparison, ["distinct_response_count"]) ?? null
+    },
+    summaries: {
+      base: baseSummary,
+      trained: trainedSummary,
+      comparison
+    },
+    raw: report
+  };
+}
+function safeTimestamp(value) {
+  return value.replace(/[:.]/g, "-");
+}
+async function writeEvalComparisonArtifact(input) {
+  const artifact = buildEvalComparisonArtifactPayload(input);
+  const outputDir = input.outputDir ?? join(trainingStateRoot(), "evals", safeTimestamp(artifact.generatedAt));
+  await mkdir(outputDir, { recursive: true });
+  const artifactPath = join(outputDir, "eval-comparison.json");
+  await writeFile(
+    artifactPath,
+    `${JSON.stringify(artifact, null, 2)}
+`,
+    "utf-8"
+  );
+  return { outputDir, artifactPath, artifact };
+}
+function collectProcess(command, args, cwd) {
+  return new Promise((resolvePromise, reject) => {
+    const child = spawn(command, args, {
+      cwd,
+      stdio: ["ignore", "pipe", "pipe"]
+    });
+    let stdout = "";
+    let stderr = "";
+    child.stdout.setEncoding("utf-8");
+    child.stderr.setEncoding("utf-8");
+    child.stdout.on("data", (chunk) => {
+      stdout += chunk;
+    });
+    child.stderr.on("data", (chunk) => {
+      stderr += chunk;
+    });
+    child.on("error", reject);
+    child.on("close", (code) => {
+      resolvePromise({ stdout, stderr, exitCode: code ?? 1 });
+    });
+  });
+}
+async function runLocalEvalComparison(options) {
+  const trainingRoot = resolve(
+    options.trainingRoot ?? join(process.cwd(), "packages", "training")
+  );
+  const outputDir = options.outputDir ?? join(trainingStateRoot(), "evals", safeTimestamp((/* @__PURE__ */ new Date()).toISOString()));
+  await mkdir(outputDir, { recursive: true });
+  const reportPath = resolve(
+    options.outputPath ?? join(outputDir, "local_model_comparison.json")
+  );
+  const command = options.python ?? "python3";
+  const args = buildLocalEvalComparisonArgs(options, {
+    trainingRoot,
+    reportPath
+  });
+  if (options.dryRun) {
+    const artifactResult2 = await writeEvalComparisonArtifact({
+      report: {
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        backend: options.backend,
+        base_model: {
+          model_ref: options.model ?? options.manifestPath ?? null,
+          summary: {}
+        },
+        trained_model: {
+          model_ref: options.trainedModelPath ?? options.manifestPath ?? null,
+          summary: {}
+        },
+        comparison: {
+          dry_run: true
+        }
+      },
+      reportPath,
+      outputDir,
+      source: {
+        kind: "training_local_eval_comparison",
+        trainingRoot,
+        manifestPath: options.manifestPath,
+        model: options.model,
+        trainedModelPath: options.trainedModelPath,
+        backend: options.backend,
+        dryRun: true
+      }
+    });
+    return {
+      ...artifactResult2,
+      trainingRoot,
+      command: [command, ...args],
+      reportPath,
+      stdout: "[DRY RUN] Would run local eval comparison.",
+      stderr: "",
+      exitCode: 0
+    };
+  }
+  const proc = await collectProcess(command, args, trainingRoot);
+  if (proc.exitCode !== 0) {
+    throw new Error(
+      `compare_local_models.py exited with code ${proc.exitCode}: ${proc.stderr || proc.stdout}`
+    );
+  }
+  const report = JSON.parse(await readFile(reportPath, "utf-8"));
+  const artifactResult = await writeEvalComparisonArtifact({
+    report,
+    reportPath,
+    outputDir,
+    source: {
+      kind: "training_local_eval_comparison",
+      trainingRoot,
+      manifestPath: options.manifestPath,
+      model: options.model,
+      trainedModelPath: options.trainedModelPath,
+      backend: options.backend
+    }
+  });
+  return {
+    ...artifactResult,
+    trainingRoot,
+    command: [command, ...args],
+    reportPath,
+    stdout: proc.stdout,
+    stderr: proc.stderr,
+    exitCode: proc.exitCode
+  };
+}
+export {
+  EVAL_COMPARISON_ARTIFACT_SCHEMA,
+  EVAL_COMPARISON_ARTIFACT_VERSION,
+  buildEvalComparisonArtifactPayload,
+  buildLocalEvalComparisonArgs,
+  runLocalEvalComparison,
+  writeEvalComparisonArtifact
+};
+//# sourceMappingURL=eval-comparison-artifact.js.map

package/dist/core/eval-comparison-artifact.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/core/eval-comparison-artifact.ts"],"sourcesContent":["import { spawn } from \"node:child_process\";\nimport { mkdir, readFile, writeFile } from \"node:fs/promises\";\nimport { join, resolve } from \"node:path\";\nimport { trainingStateRoot } from \"./training-config.js\";\n\nexport const EVAL_COMPARISON_ARTIFACT_SCHEMA = \"eliza_eval_comparison_artifact\";\nexport const EVAL_COMPARISON_ARTIFACT_VERSION = 1;\n\nexport interface EvalComparisonArtifactInput {\n  report: Record<string, unknown>;\n  reportPath?: string;\n  outputDir?: string;\n  source?: Record<string, unknown>;\n}\n\nexport interface EvalComparisonRunOptions {\n  trainingRoot?: string;\n  python?: string;\n  manifestPath?: string;\n  model?: string;\n  trainedModelPath?: string;\n  backend?: \"mlx\" | \"cuda\" | \"cpu\";\n  promptFile?: string;\n  maxTokens?: number;\n  systemPrompt?: string;\n  outputPath?: string;\n  outputDir?: string;\n  dryRun?: boolean;\n}\n\nexport interface EvalComparisonArtifact {\n  schema: typeof EVAL_COMPARISON_ARTIFACT_SCHEMA;\n  version: typeof EVAL_COMPARISON_ARTIFACT_VERSION;\n  generatedAt: string;\n  reportPath?: string;\n  source: Record<string, unknown>;\n  models: {\n    base: string | null;\n    trained: string | null;\n    backend: string | null;\n  };\n  metrics: {\n    baseScore: number | null;\n    trainedScore: number | null;\n    improvementAbsolute: number | null;\n    improvementPercent: number | null;\n    baseLatencyMs: number | null;\n    trainedLatencyMs: number | null;\n    latencyDeltaMs: number | null;\n    promptCount: number | null;\n    distinctResponseCount: number | null;\n  };\n  summaries: {\n    base: Record<string, unknown> | null;\n    trained: Record<string, unknown> | null;\n    comparison: Record<string, unknown> | null;\n  };\n  raw: Record<string, unknown>;\n}\n\nexport interface EvalComparisonArtifactResult {\n  outputDir: string;\n  artifactPath: string;\n  artifact: EvalComparisonArtifact;\n}\n\nexport interface EvalComparisonRunResult extends EvalComparisonArtifactResult {\n  trainingRoot: string;\n  command: string[];\n  reportPath: string;\n  stdout: string;\n  stderr: string;\n  exitCode: number;\n}\n\nexport function buildLocalEvalComparisonArgs(\n  options: EvalComparisonRunOptions,\n  resolved: { trainingRoot: string; reportPath: string },\n): string[] {\n  const scriptPath = join(\n    resolved.trainingRoot,\n    \"scripts\",\n    \"rl\",\n    \"compare_local_models.py\",\n  );\n  const args = [scriptPath];\n  if (options.manifestPath) {\n    args.push(\"--manifest\", options.manifestPath);\n  } else {\n    if (!options.model || !options.trainedModelPath || !options.backend) {\n      throw new Error(\n        \"Provide either manifestPath or model, trainedModelPath, and backend\",\n      );\n    }\n    args.push(\"--model\", options.model);\n    args.push(\"--trained-model-path\", options.trainedModelPath);\n    args.push(\"--backend\", options.backend);\n  }\n  if (options.promptFile) args.push(\"--prompt-file\", options.promptFile);\n  if (typeof options.maxTokens === \"number\") {\n    args.push(\n      \"--max-tokens\",\n      String(Math.max(1, Math.floor(options.maxTokens))),\n    );\n  }\n  if (options.systemPrompt) args.push(\"--system-prompt\", options.systemPrompt);\n  args.push(\"--output\", resolved.reportPath);\n  return args;\n}\n\nfunction asRecord(value: unknown): Record<string, unknown> | null {\n  return value && typeof value === \"object\" && !Array.isArray(value)\n    ? (value as Record<string, unknown>)\n    : null;\n}\n\nfunction asString(value: unknown): string | null {\n  return typeof value === \"string\" && value.trim().length > 0\n    ? value.trim()\n    : null;\n}\n\nfunction asNumber(value: unknown): number | null {\n  return typeof value === \"number\" && Number.isFinite(value) ? value : null;\n}\n\nfunction firstNumber(\n  record: Record<string, unknown> | null,\n  keys: readonly string[],\n): number | null {\n  if (!record) return null;\n  for (const key of keys) {\n    const value = asNumber(record[key]);\n    if (value !== null) return value;\n  }\n  return null;\n}\n\nfunction firstString(\n  record: Record<string, unknown> | null,\n  keys: readonly string[],\n): string | null {\n  if (!record) return null;\n  for (const key of keys) {\n    const value = asString(record[key]);\n    if (value) return value;\n  }\n  return null;\n}\n\nfunction nestedSummary(\n  report: Record<string, unknown>,\n  key: string,\n): Record<string, unknown> | null {\n  const variant = asRecord(report[key]);\n  return asRecord(variant?.summary);\n}\n\nfunction extractVariantModel(\n  report: Record<string, unknown>,\n  key: string,\n): string | null {\n  const variant = asRecord(report[key]);\n  return firstString(variant, [\n    \"model_name\",\n    \"model_ref\",\n    \"adapter_path\",\n    \"served_model_id\",\n  ]);\n}\n\nfunction roundMetric(value: number | null): number | null {\n  return value === null ? null : Number(value.toFixed(4));\n}\n\nfunction improvementPercent(base: number | null, trained: number | null) {\n  if (base === null || trained === null || base === 0) return null;\n  return ((trained - base) / Math.abs(base)) * 100;\n}\n\nexport function buildEvalComparisonArtifactPayload(\n  input: EvalComparisonArtifactInput,\n): EvalComparisonArtifact {\n  const report = input.report;\n  const baseSummary =\n    nestedSummary(report, \"base_model\") ?? asRecord(report.base_summary);\n  const trainedSummary =\n    nestedSummary(report, \"trained_model\") ??\n    nestedSummary(report, \"adapter_model\") ??\n    asRecord(report.trained_summary) ??\n    asRecord(report.adapter_summary);\n  const comparison = asRecord(report.comparison);\n  const baseScore = firstNumber(baseSummary, [\n    \"avg_score\",\n    \"score\",\n    \"format_ok\",\n    \"content_ok\",\n    \"test_avg_score\",\n  ]);\n  const trainedScore = firstNumber(trainedSummary, [\n    \"avg_score\",\n    \"score\",\n    \"format_ok\",\n    \"content_ok\",\n    \"test_avg_score\",\n  ]);\n  const baseLatencyMs = firstNumber(baseSummary, [\n    \"avg_latency_ms\",\n    \"latency_ms\",\n  ]);\n  const trainedLatencyMs = firstNumber(trainedSummary, [\n    \"avg_latency_ms\",\n    \"latency_ms\",\n  ]);\n  const promptCount =\n    firstNumber(baseSummary, [\"prompt_count\", \"test_sample_count\"]) ??\n    firstNumber(trainedSummary, [\"prompt_count\", \"test_sample_count\"]);\n  const generatedAt =\n    asString(report.timestamp) ??\n    asString(report.generated_at) ??\n    asString(report.evaluated_at) ??\n    new Date().toISOString();\n\n  return {\n    schema: EVAL_COMPARISON_ARTIFACT_SCHEMA,\n    version: EVAL_COMPARISON_ARTIFACT_VERSION,\n    generatedAt,\n    reportPath: input.reportPath,\n    source: input.source ?? { kind: \"training_eval_comparison\" },\n    models: {\n      base:\n        extractVariantModel(report, \"base_model\") ??\n        firstString(report, [\"base_model\", \"model\", \"base_model_id\"]),\n      trained:\n        extractVariantModel(report, \"trained_model\") ??\n        extractVariantModel(report, \"adapter_model\") ??\n        firstString(report, [\n          \"trained_model\",\n          \"adapter_model\",\n          \"trained_model_id\",\n        ]),\n      backend: asString(report.backend),\n    },\n    metrics: {\n      baseScore: roundMetric(baseScore),\n      trainedScore: roundMetric(trainedScore),\n      improvementAbsolute: roundMetric(\n        baseScore !== null && trainedScore !== null\n          ? trainedScore - baseScore\n          : null,\n      ),\n      improvementPercent: roundMetric(\n        improvementPercent(baseScore, trainedScore),\n      ),\n      baseLatencyMs: roundMetric(baseLatencyMs),\n      trainedLatencyMs: roundMetric(trainedLatencyMs),\n      latencyDeltaMs: roundMetric(\n        baseLatencyMs !== null && trainedLatencyMs !== null\n          ? trainedLatencyMs - baseLatencyMs\n          : null,\n      ),\n      promptCount: promptCount === null ? null : Math.round(promptCount),\n      distinctResponseCount:\n        firstNumber(comparison, [\"distinct_response_count\"]) ?? null,\n    },\n    summaries: {\n      base: baseSummary,\n      trained: trainedSummary,\n      comparison,\n    },\n    raw: report,\n  };\n}\n\nfunction safeTimestamp(value: string): string {\n  return value.replace(/[:.]/g, \"-\");\n}\n\nexport async function writeEvalComparisonArtifact(\n  input: EvalComparisonArtifactInput,\n): Promise<EvalComparisonArtifactResult> {\n  const artifact = buildEvalComparisonArtifactPayload(input);\n  const outputDir =\n    input.outputDir ??\n    join(trainingStateRoot(), \"evals\", safeTimestamp(artifact.generatedAt));\n  await mkdir(outputDir, { recursive: true });\n  const artifactPath = join(outputDir, \"eval-comparison.json\");\n  await writeFile(\n    artifactPath,\n    `${JSON.stringify(artifact, null, 2)}\\n`,\n    \"utf-8\",\n  );\n  return { outputDir, artifactPath, artifact };\n}\n\nfunction collectProcess(\n  command: string,\n  args: string[],\n  cwd: string,\n): Promise<{ stdout: string; stderr: string; exitCode: number }> {\n  return new Promise((resolvePromise, reject) => {\n    const child = spawn(command, args, {\n      cwd,\n      stdio: [\"ignore\", \"pipe\", \"pipe\"],\n    });\n    let stdout = \"\";\n    let stderr = \"\";\n    child.stdout.setEncoding(\"utf-8\");\n    child.stderr.setEncoding(\"utf-8\");\n    child.stdout.on(\"data\", (chunk) => {\n      stdout += chunk;\n    });\n    child.stderr.on(\"data\", (chunk) => {\n      stderr += chunk;\n    });\n    child.on(\"error\", reject);\n    child.on(\"close\", (code) => {\n      resolvePromise({ stdout, stderr, exitCode: code ?? 1 });\n    });\n  });\n}\n\nexport async function runLocalEvalComparison(\n  options: EvalComparisonRunOptions,\n): Promise<EvalComparisonRunResult> {\n  const trainingRoot = resolve(\n    options.trainingRoot ?? join(process.cwd(), \"packages\", \"training\"),\n  );\n  const outputDir =\n    options.outputDir ??\n    join(trainingStateRoot(), \"evals\", safeTimestamp(new Date().toISOString()));\n  await mkdir(outputDir, { recursive: true });\n  const reportPath = resolve(\n    options.outputPath ?? join(outputDir, \"local_model_comparison.json\"),\n  );\n  const command = options.python ?? \"python3\";\n  const args = buildLocalEvalComparisonArgs(options, {\n    trainingRoot,\n    reportPath,\n  });\n  if (options.dryRun) {\n    const artifactResult = await writeEvalComparisonArtifact({\n      report: {\n        timestamp: new Date().toISOString(),\n        backend: options.backend,\n        base_model: {\n          model_ref: options.model ?? options.manifestPath ?? null,\n          summary: {},\n        },\n        trained_model: {\n          model_ref: options.trainedModelPath ?? options.manifestPath ?? null,\n          summary: {},\n        },\n        comparison: {\n          dry_run: true,\n        },\n      },\n      reportPath,\n      outputDir,\n      source: {\n        kind: \"training_local_eval_comparison\",\n        trainingRoot,\n        manifestPath: options.manifestPath,\n        model: options.model,\n        trainedModelPath: options.trainedModelPath,\n        backend: options.backend,\n        dryRun: true,\n      },\n    });\n    return {\n      ...artifactResult,\n      trainingRoot,\n      command: [command, ...args],\n      reportPath,\n      stdout: \"[DRY RUN] Would run local eval comparison.\",\n      stderr: \"\",\n      exitCode: 0,\n    };\n  }\n\n  const proc = await collectProcess(command, args, trainingRoot);\n  if (proc.exitCode !== 0) {\n    throw new Error(\n      `compare_local_models.py exited with code ${proc.exitCode}: ${proc.stderr || proc.stdout}`,\n    );\n  }\n  const report = JSON.parse(await readFile(reportPath, \"utf-8\")) as Record<\n    string,\n    unknown\n  >;\n  const artifactResult = await writeEvalComparisonArtifact({\n    report,\n    reportPath,\n    outputDir,\n    source: {\n      kind: \"training_local_eval_comparison\",\n      trainingRoot,\n      manifestPath: options.manifestPath,\n      model: options.model,\n      trainedModelPath: options.trainedModelPath,\n      backend: options.backend,\n    },\n  });\n  return {\n    ...artifactResult,\n    trainingRoot,\n    command: [command, ...args],\n    reportPath,\n    stdout: proc.stdout,\n    stderr: proc.stderr,\n    exitCode: proc.exitCode,\n  };\n}\n"],"mappings":"AAAA,SAAS,aAAa;AACtB,SAAS,OAAO,UAAU,iBAAiB;AAC3C,SAAS,MAAM,eAAe;AAC9B,SAAS,yBAAyB;AAE3B,MAAM,kCAAkC;AACxC,MAAM,mCAAmC;AAqEzC,SAAS,6BACd,SACA,UACU;AACV,QAAM,aAAa;AAAA,IACjB,SAAS;AAAA,IACT;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,QAAM,OAAO,CAAC,UAAU;AACxB,MAAI,QAAQ,cAAc;AACxB,SAAK,KAAK,cAAc,QAAQ,YAAY;AAAA,EAC9C,OAAO;AACL,QAAI,CAAC,QAAQ,SAAS,CAAC,QAAQ,oBAAoB,CAAC,QAAQ,SAAS;AACnE,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,SAAK,KAAK,WAAW,QAAQ,KAAK;AAClC,SAAK,KAAK,wBAAwB,QAAQ,gBAAgB;AAC1D,SAAK,KAAK,aAAa,QAAQ,OAAO;AAAA,EACxC;AACA,MAAI,QAAQ,WAAY,MAAK,KAAK,iBAAiB,QAAQ,UAAU;AACrE,MAAI,OAAO,QAAQ,cAAc,UAAU;AACzC,SAAK;AAAA,MACH;AAAA,MACA,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,SAAS,CAAC,CAAC;AAAA,IACnD;AAAA,EACF;AACA,MAAI,QAAQ,aAAc,MAAK,KAAK,mBAAmB,QAAQ,YAAY;AAC3E,OAAK,KAAK,YAAY,SAAS,UAAU;AACzC,SAAO;AACT;AAEA,SAAS,SAAS,OAAgD;AAChE,SAAO,SAAS,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK,IAC5D,QACD;AACN;AAEA,SAAS,SAAS,OAA+B;AAC/C,SAAO,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,IACtD,MAAM,KAAK,IACX;AACN;AAEA,SAAS,SAAS,OAA+B;AAC/C,SAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IAAI,QAAQ;AACvE;AAEA,SAAS,YACP,QACA,MACe;AACf,MAAI,CAAC,OAAQ,QAAO;AACpB,aAAW,OAAO,MAAM;AACtB,UAAM,QAAQ,SAAS,OAAO,GAAG,CAAC;AAClC,QAAI,UAAU,KAAM,QAAO;AAAA,EAC7B;AACA,SAAO;AACT;AAEA,SAAS,YACP,QACA,MACe;AACf,MAAI,CAAC,OAAQ,QAAO;AACpB,aAAW,OAAO,MAAM;AACtB,UAAM,QAAQ,SAAS,OAAO,GAAG,CAAC;AAClC,QAAI,MAAO,QAAO;AAAA,EACpB;AACA,SAAO;AACT;AAEA,SAAS,cACP,QACA,KACgC;AAChC,QAAM,UAAU,SAAS,OAAO,GAAG,CAAC;AACpC,SAAO,SAAS,SAAS,OAAO;AAClC;AAEA,SAAS,oBACP,QACA,KACe;AACf,QAAM,UAAU,SAAS,OAAO,GAAG,CAAC;AACpC,SAAO,YAAY,SAAS;AAAA,IAC1B;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AACH;AAEA,SAAS,YAAY,OAAqC;AACxD,SAAO,UAAU,OAAO,OAAO,OAAO,MAAM,QAAQ,CAAC,CAAC;AACxD;AAEA,SAAS,mBAAmB,MAAqB,SAAwB;AACvE,MAAI,SAAS,QAAQ,YAAY,QAAQ,SAAS,EAAG,QAAO;AAC5D,UAAS,UAAU,QAAQ,KAAK,IAAI,IAAI,IAAK;AAC/C;AAEO,SAAS,mCACd,OACwB;AACxB,QAAM,SAAS,MAAM;AACrB,QAAM,cACJ,cAAc,QAAQ,YAAY,KAAK,SAAS,OAAO,YAAY;AACrE,QAAM,iBACJ,cAAc,QAAQ,eAAe,KACrC,cAAc,QAAQ,eAAe,KACrC,SAAS,OAAO,eAAe,KAC/B,SAAS,OAAO,eAAe;AACjC,QAAM,aAAa,SAAS,OAAO,UAAU;AAC7C,QAAM,YAAY,YAAY,aAAa;AAAA,IACzC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AACD,QAAM,eAAe,YAAY,gBAAgB;AAAA,IAC/C;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AACD,QAAM,gBAAgB,YAAY,aAAa;AAAA,IAC7C;AAAA,IACA;AAAA,EACF,CAAC;AACD,QAAM,mBAAmB,YAAY,gBAAgB;AAAA,IACnD;AAAA,IACA;AAAA,EACF,CAAC;AACD,QAAM,cACJ,YAAY,aAAa,CAAC,gBAAgB,mBAAmB,CAAC,KAC9D,YAAY,gBAAgB,CAAC,gBAAgB,mBAAmB,CAAC;AACnE,QAAM,cACJ,SAAS,OAAO,SAAS,KACzB,SAAS,OAAO,YAAY,KAC5B,SAAS,OAAO,YAAY,MAC5B,oBAAI,KAAK,GAAE,YAAY;AAEzB,SAAO;AAAA,IACL,QAAQ;AAAA,IACR,SAAS;AAAA,IACT;AAAA,IACA,YAAY,MAAM;AAAA,IAClB,QAAQ,MAAM,UAAU,EAAE,MAAM,2BAA2B;AAAA,IAC3D,QAAQ;AAAA,MACN,MACE,oBAAoB,QAAQ,YAAY,KACxC,YAAY,QAAQ,CAAC,cAAc,SAAS,eAAe,CAAC;AAAA,MAC9D,SACE,oBAAoB,QAAQ,eAAe,KAC3C,oBAAoB,QAAQ,eAAe,KAC3C,YAAY,QAAQ;AAAA,QAClB;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAAA,MACH,SAAS,SAAS,OAAO,OAAO;AAAA,IAClC;AAAA,IACA,SAAS;AAAA,MACP,WAAW,YAAY,SAAS;AAAA,MAChC,cAAc,YAAY,YAAY;AAAA,MACtC,qBAAqB;AAAA,QACnB,cAAc,QAAQ,iBAAiB,OACnC,eAAe,YACf;AAAA,MACN;AAAA,MACA,oBAAoB;AAAA,QAClB,mBAAmB,WAAW,YAAY;AAAA,MAC5C;AAAA,MACA,eAAe,YAAY,aAAa;AAAA,MACxC,kBAAkB,YAAY,gBAAgB;AAAA,MAC9C,gBAAgB;AAAA,QACd,kBAAkB,QAAQ,qBAAqB,OAC3C,mBAAmB,gBACnB;AAAA,MACN;AAAA,MACA,aAAa,gBAAgB,OAAO,OAAO,KAAK,MAAM,WAAW;AAAA,MACjE,uBACE,YAAY,YAAY,CAAC,yBAAyB,CAAC,KAAK;AAAA,IAC5D;AAAA,IACA,WAAW;AAAA,MACT,MAAM;AAAA,MACN,SAAS;AAAA,MACT;AAAA,IACF;AAAA,IACA,KAAK;AAAA,EACP;AACF;AAEA,SAAS,cAAc,OAAuB;AAC5C,SAAO,MAAM,QAAQ,SAAS,GAAG;AACnC;AAEA,eAAsB,4BACpB,OACuC;AACvC,QAAM,WAAW,mCAAmC,KAAK;AACzD,QAAM,YACJ,MAAM,aACN,KAAK,kBAAkB,GAAG,SAAS,cAAc,SAAS,WAAW,CAAC;AACxE,QAAM,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,QAAM,eAAe,KAAK,WAAW,sBAAsB;AAC3D,QAAM;AAAA,IACJ;AAAA,IACA,GAAG,KAAK,UAAU,UAAU,MAAM,CAAC,CAAC;AAAA;AAAA,IACpC;AAAA,EACF;AACA,SAAO,EAAE,WAAW,cAAc,SAAS;AAC7C;AAEA,SAAS,eACP,SACA,MACA,KAC+D;AAC/D,SAAO,IAAI,QAAQ,CAAC,gBAAgB,WAAW;AAC7C,UAAM,QAAQ,MAAM,SAAS,MAAM;AAAA,MACjC;AAAA,MACA,OAAO,CAAC,UAAU,QAAQ,MAAM;AAAA,IAClC,CAAC;AACD,QAAI,SAAS;AACb,QAAI,SAAS;AACb,UAAM,OAAO,YAAY,OAAO;AAChC,UAAM,OAAO,YAAY,OAAO;AAChC,UAAM,OAAO,GAAG,QAAQ,CAAC,UAAU;AACjC,gBAAU;AAAA,IACZ,CAAC;AACD,UAAM,OAAO,GAAG,QAAQ,CAAC,UAAU;AACjC,gBAAU;AAAA,IACZ,CAAC;AACD,UAAM,GAAG,SAAS,MAAM;AACxB,UAAM,GAAG,SAAS,CAAC,SAAS;AAC1B,qBAAe,EAAE,QAAQ,QAAQ,UAAU,QAAQ,EAAE,CAAC;AAAA,IACxD,CAAC;AAAA,EACH,CAAC;AACH;AAEA,eAAsB,uBACpB,SACkC;AAClC,QAAM,eAAe;AAAA,IACnB,QAAQ,gBAAgB,KAAK,QAAQ,IAAI,GAAG,YAAY,UAAU;AAAA,EACpE;AACA,QAAM,YACJ,QAAQ,aACR,KAAK,kBAAkB,GAAG,SAAS,eAAc,oBAAI,KAAK,GAAE,YAAY,CAAC,CAAC;AAC5E,QAAM,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,QAAM,aAAa;AAAA,IACjB,QAAQ,cAAc,KAAK,WAAW,6BAA6B;AAAA,EACrE;AACA,QAAM,UAAU,QAAQ,UAAU;AAClC,QAAM,OAAO,6BAA6B,SAAS;AAAA,IACjD;AAAA,IACA;AAAA,EACF,CAAC;AACD,MAAI,QAAQ,QAAQ;AAClB,UAAMA,kBAAiB,MAAM,4BAA4B;AAAA,MACvD,QAAQ;AAAA,QACN,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,QAClC,SAAS,QAAQ;AAAA,QACjB,YAAY;AAAA,UACV,WAAW,QAAQ,SAAS,QAAQ,gBAAgB;AAAA,UACpD,SAAS,CAAC;AAAA,QACZ;AAAA,QACA,eAAe;AAAA,UACb,WAAW,QAAQ,oBAAoB,QAAQ,gBAAgB;AAAA,UAC/D,SAAS,CAAC;AAAA,QACZ;AAAA,QACA,YAAY;AAAA,UACV,SAAS;AAAA,QACX;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,QACN,MAAM;AAAA,QACN;AAAA,QACA,cAAc,QAAQ;AAAA,QACtB,OAAO,QAAQ;AAAA,QACf,kBAAkB,QAAQ;AAAA,QAC1B,SAAS,QAAQ;AAAA,QACjB,QAAQ;AAAA,MACV;AAAA,IACF,CAAC;AACD,WAAO;AAAA,MACL,GAAGA;AAAA,MACH;AAAA,MACA,SAAS,CAAC,SAAS,GAAG,IAAI;AAAA,MAC1B;AAAA,MACA,QAAQ;AAAA,MACR,QAAQ;AAAA,MACR,UAAU;AAAA,IACZ;AAAA,EACF;AAEA,QAAM,OAAO,MAAM,eAAe,SAAS,MAAM,YAAY;AAC7D,MAAI,KAAK,aAAa,GAAG;AACvB,UAAM,IAAI;AAAA,MACR,4CAA4C,KAAK,QAAQ,KAAK,KAAK,UAAU,KAAK,MAAM;AAAA,IAC1F;AAAA,EACF;AACA,QAAM,SAAS,KAAK,MAAM,MAAM,SAAS,YAAY,OAAO,CAAC;AAI7D,QAAM,iBAAiB,MAAM,4BAA4B;AAAA,IACvD;AAAA,IACA;AAAA,IACA;AAAA,IACA,QAAQ;AAAA,MACN,MAAM;AAAA,MACN;AAAA,MACA,cAAc,QAAQ;AAAA,MACtB,OAAO,QAAQ;AAAA,MACf,kBAAkB,QAAQ;AAAA,MAC1B,SAAS,QAAQ;AAAA,IACnB;AAAA,EACF,CAAC;AACD,SAAO;AAAA,IACL,GAAG;AAAA,IACH;AAAA,IACA,SAAS,CAAC,SAAS,GAAG,IAAI;AAAA,IAC1B;AAAA,IACA,QAAQ,KAAK;AAAA,IACb,QAAQ,KAAK;AAAA,IACb,UAAU,KAAK;AAAA,EACjB;AACF;","names":["artifactResult"]}

package/dist/core/feed-generation-runner.d.ts

@@ -0,0 +1,37 @@
+export interface FeedGenerationRunOptions {
+    workspaceRoot?: string;
+    bun?: string;
+    archetypes?: string;
+    numAgents?: number;
+    ticks?: number;
+    parallel?: number;
+    managerId?: string;
+    cleanup?: boolean;
+    dryRun?: boolean;
+    outputDir?: string;
+}
+export interface FeedGenerationRunResult {
+    workspaceRoot: string;
+    feedCliRoot: string;
+    outputDir: string;
+    artifacts: FeedGenerationArtifact[];
+    command: string[];
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+}
+export interface FeedGenerationArtifact {
+    schema: string | null;
+    manifestPath: string;
+    exportPath: string | null;
+    outputDir: string | null;
+    sourceKind: string | null;
+    trajectories: number | null;
+    archetypes: unknown;
+    generatedAt: string | null;
+}
+export declare function buildFeedGenerationArgs(options: FeedGenerationRunOptions, resolved: {
+    outputDir: string;
+}): string[];
+export declare function runFeedGeneration(options?: FeedGenerationRunOptions): Promise<FeedGenerationRunResult>;
+//# sourceMappingURL=feed-generation-runner.d.ts.map

package/dist/core/feed-generation-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"feed-generation-runner.d.ts","sourceRoot":"","sources":["../../src/core/feed-generation-runner.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,wBAAwB;IACvC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,uBAAuB;IACtC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,sBAAsB,EAAE,CAAC;IACpC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,OAAO,CAAC;IACpB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAyMD,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,wBAAwB,EACjC,QAAQ,EAAE;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,GAC9B,MAAM,EAAE,CAqBV;AAED,wBAAsB,iBAAiB,CACrC,OAAO,GAAE,wBAA6B,GACrC,OAAO,CAAC,uBAAuB,CAAC,CA8BlC"}

package/dist/core/feed-generation-runner.js

@@ -0,0 +1,232 @@
+import { spawn } from "node:child_process";
+import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { trainingStateRoot } from "./training-config.js";
+import { defaultBunCommand, resolveWorkspaceRoot } from "./workspace-runtime.js";
+function safeTimestamp(value) {
+  return value.replace(/[:.]/g, "-");
+}
+function positiveInt(value, fallback) {
+  return typeof value === "number" && Number.isFinite(value) ? Math.max(1, Math.floor(value)) : fallback;
+}
+function collectProcess(command, args, cwd) {
+  return new Promise((resolvePromise, reject) => {
+    const child = spawn(command, args, { cwd, stdio: ["ignore", "pipe", "pipe"] });
+    let stdout = "";
+    let stderr = "";
+    child.stdout.setEncoding("utf-8");
+    child.stderr.setEncoding("utf-8");
+    child.stdout.on("data", (chunk) => {
+      stdout += chunk;
+    });
+    child.stderr.on("data", (chunk) => {
+      stderr += chunk;
+    });
+    child.on("error", reject);
+    child.on("close", (code) => {
+      resolvePromise({ stdout, stderr, exitCode: code ?? 1 });
+    });
+  });
+}
+function asRecord(value) {
+  return value && typeof value === "object" && !Array.isArray(value) ? value : null;
+}
+function stringOrNull(value) {
+  return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
+}
+function numberOrNull(value) {
+  return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+function parseArchetypes(value) {
+  const archetypes = (value ?? "trader").split(",").map((item) => item.trim()).filter(Boolean);
+  return archetypes.length > 0 ? archetypes : ["trader"];
+}
+async function writeDryRunFeedGenerationArtifact(input) {
+  const generatedAt = (/* @__PURE__ */ new Date()).toISOString();
+  const archetypes = parseArchetypes(input.options.archetypes);
+  const agentsPerArchetype = positiveInt(input.options.numAgents, 1);
+  const ticks = positiveInt(input.options.ticks, 1);
+  const trajectoryRows = archetypes.flatMap(
+    (archetype) => Array.from({ length: agentsPerArchetype }, (_, index) => {
+      const ordinal = index + 1;
+      return {
+        trajectory_id: `feed-dry-run-${archetype}-${ordinal}`,
+        agent_id: `feed-dry-run-agent-${archetype}-${ordinal}`,
+        archetype,
+        scenario_id: "feed-dry-run",
+        score: null,
+        steps: [
+          {
+            action: "DRY_RUN",
+            kind: "planned_tick",
+            input: `${archetype} market observation for dry-run tick 1 of ${ticks}`,
+            output: `planned ${archetype} feed decision`,
+            tick: 1,
+            ticks
+          }
+        ],
+        reasoning: `Dry-run feed generation preview for ${archetype}.`
+      };
+    })
+  );
+  const exportPath = join(input.outputDir, "feed-dry-run-trajectories.jsonl");
+  const manifestPath = join(input.outputDir, "feed-dry-run.manifest.json");
+  const trajectoryIds = trajectoryRows.map((row) => row.trajectory_id);
+  const agentsCreated = trajectoryRows.map((row) => row.agent_id);
+  const archetypeStats = Object.fromEntries(
+    archetypes.map((archetype) => [
+      archetype,
+      {
+        agents: agentsPerArchetype,
+        trajectories: agentsPerArchetype,
+        avgTicksPerAgent: ticks
+      }
+    ])
+  );
+  await writeFile(
+    exportPath,
+    `${trajectoryRows.map((row) => JSON.stringify(row)).join("\n")}
+`,
+    "utf8"
+  );
+  await writeFile(
+    manifestPath,
+    `${JSON.stringify(
+      {
+        schema: "feed_parallel_generation",
+        schemaVersion: 1,
+        generatedAt,
+        outputDir: input.outputDir,
+        exportPath,
+        manifestPath,
+        source: {
+          kind: "feed_train_parallel_generation",
+          archetypes
+        },
+        counts: {
+          agentsCreated: agentsCreated.length,
+          trajectories: trajectoryIds.length,
+          totalTicks: agentsCreated.length * ticks,
+          errors: 0
+        },
+        durationMs: 0,
+        cleanup: input.options.cleanup === true,
+        dryRun: true,
+        agentsCreated,
+        trajectoryIds,
+        archetypeStats,
+        errors: []
+      },
+      null,
+      2
+    )}
+`,
+    "utf8"
+  );
+}
+async function discoverFeedGenerationArtifacts(outputDir) {
+  const manifestPaths = [];
+  async function walk(dir) {
+    let entries;
+    try {
+      entries = await readdir(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      const path = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        await walk(path);
+        continue;
+      }
+      if (entry.isFile() && entry.name.endsWith(".manifest.json")) {
+        manifestPaths.push(path);
+      }
+    }
+  }
+  await walk(outputDir);
+  const artifacts = [];
+  for (const manifestPath of manifestPaths) {
+    try {
+      const payload = asRecord(JSON.parse(await readFile(manifestPath, "utf8")));
+      if (!payload) continue;
+      const schema = stringOrNull(payload.schema);
+      if (schema !== "feed_parallel_generation" && schema !== "feed_training_trajectory_export") {
+        continue;
+      }
+      const counts = asRecord(payload.counts) ?? {};
+      const source = asRecord(payload.source) ?? {};
+      artifacts.push({
+        schema,
+        manifestPath,
+        exportPath: stringOrNull(payload.exportPath),
+        outputDir: stringOrNull(payload.outputDir),
+        sourceKind: stringOrNull(source.kind),
+        trajectories: numberOrNull(counts.trajectories),
+        archetypes: source.archetypes ?? source.archetype ?? null,
+        generatedAt: stringOrNull(payload.generatedAt)
+      });
+    } catch {
+    }
+  }
+  return artifacts.sort(
+    (left, right) => left.manifestPath.localeCompare(right.manifestPath)
+  );
+}
+function buildFeedGenerationArgs(options, resolved) {
+  const args = [
+    "run",
+    "src/index.ts",
+    "train",
+    "parallel",
+    "--archetypes",
+    options.archetypes?.trim() || "trader",
+    "--num-agents",
+    String(positiveInt(options.numAgents, 1)),
+    "--ticks",
+    String(positiveInt(options.ticks, 1)),
+    "--parallel",
+    String(positiveInt(options.parallel, 1)),
+    "--output-dir",
+    resolved.outputDir
+  ];
+  if (options.managerId) args.push("--manager-id", options.managerId);
+  if (options.cleanup) args.push("--cleanup");
+  if (options.dryRun) args.push("--dry-run");
+  return args;
+}
+async function runFeedGeneration(options = {}) {
+  const workspaceRoot = resolveWorkspaceRoot(options.workspaceRoot);
+  const feedCliRoot = join(workspaceRoot, "packages", "feed", "apps", "cli");
+  const stamp = safeTimestamp((/* @__PURE__ */ new Date()).toISOString());
+  const outputDir = options.outputDir ?? join(trainingStateRoot(), "feed", "parallel", stamp);
+  await mkdir(outputDir, { recursive: true });
+  const args = buildFeedGenerationArgs(options, { outputDir });
+  const command = options.bun ?? defaultBunCommand();
+  const proc = await collectProcess(command, args, feedCliRoot);
+  if (proc.exitCode !== 0) {
+    throw new Error(
+      `feed train parallel exited with code ${proc.exitCode}: ${proc.stderr || proc.stdout}`
+    );
+  }
+  let artifacts = await discoverFeedGenerationArtifacts(outputDir);
+  if (options.dryRun === true && artifacts.length === 0) {
+    await writeDryRunFeedGenerationArtifact({ outputDir, options });
+    artifacts = await discoverFeedGenerationArtifacts(outputDir);
+  }
+  return {
+    workspaceRoot,
+    feedCliRoot,
+    outputDir,
+    artifacts,
+    command: [command, ...args],
+    stdout: proc.stdout,
+    stderr: proc.stderr,
+    exitCode: proc.exitCode
+  };
+}
+export {
+  buildFeedGenerationArgs,
+  runFeedGeneration
+};
+//# sourceMappingURL=feed-generation-runner.js.map