@elizaos/plugin-training 2.0.3-beta.5 → 2.0.3-beta.7

package/dist/core/track-c-queue-task.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/core/track-c-queue-task.ts"],"sourcesContent":["/**\n * Track-C nightly jobs on the elizaOS TaskService repeat-task path (no plugin-cron).\n * Uses the same cron-boundary math as Heartbeats via `computeNextCronRunAtMs`.\n */\n\nimport { computeNextCronRunAtMs } from \"@elizaos/agent\";\nimport type { IAgentRuntime, Task } from \"@elizaos/core\";\n\nexport const TRACK_C_QUEUE_TAGS = [\"queue\", \"repeat\", \"track-c\"] as const;\n\nexport const TRACK_C_TRAJECTORY_EXPORT_TASK_NAME =\n  \"TRACK_C_TRAJECTORY_EXPORT_NIGHTLY\" as const;\nexport const TRACK_C_SKILL_SCORING_TASK_NAME =\n  \"TRACK_C_SKILL_SCORING_NIGHTLY\" as const;\n\nconst CRON_FALLBACK_MS = 60_000;\n\n/** Subset of `IAgentRuntime` required for Track-C repeat queue tasks */\nexport interface TrackCRuntimeSubset {\n  getTaskWorker: IAgentRuntime[\"getTaskWorker\"];\n  registerTaskWorker: IAgentRuntime[\"registerTaskWorker\"];\n  createTask: IAgentRuntime[\"createTask\"];\n  getTasksByName: IAgentRuntime[\"getTasksByName\"];\n  deleteTask: IAgentRuntime[\"deleteTask\"];\n}\n\ninterface MinimalLogger {\n  info: (message: string) => void;\n  warn: (message: string) => void;\n}\n\nfunction taskUpdatedMs(task: Task): number {\n  const meta = task.metadata as { updatedAt?: number } | undefined;\n  if (typeof meta?.updatedAt === \"number\") return meta.updatedAt;\n  if (typeof task.updatedAt === \"number\") return task.updatedAt;\n  if (typeof task.updatedAt === \"bigint\") return Number(task.updatedAt);\n  return 0;\n}\n\nfunction msUntilNextCron(\n  expression: string,\n  timezone: string | undefined,\n): number {\n  const now = Date.now();\n  const next = computeNextCronRunAtMs(expression, now, timezone);\n  if (next === null) return CRON_FALLBACK_MS;\n  const delta = next - now;\n  return delta > 0 ? delta : CRON_FALLBACK_MS;\n}\n\nexport async function pruneDuplicateTasksByName(\n  runtime: TrackCRuntimeSubset,\n  taskName: string,\n  log?: Pick<MinimalLogger, \"warn\">,\n  logPrefix = \"[TrackCTask]\",\n): Promise<void> {\n  const tasks = await runtime.getTasksByName(taskName);\n  if (tasks.length <= 1) return;\n\n  const sorted = [...tasks].sort((a, b) => taskUpdatedMs(b) - taskUpdatedMs(a));\n  let removed = 0;\n  for (const dup of sorted.slice(1)) {\n    if (dup.id) {\n      await runtime.deleteTask(dup.id);\n      removed += 1;\n    }\n  }\n  if (removed > 0) {\n    log?.warn(\n      `${logPrefix} removed ${removed} duplicate task(s) for \"${taskName}\"`,\n    );\n  }\n}\n\nfunction registerTrackCWorkerOnce(\n  runtime: TrackCRuntimeSubset,\n  taskName: string,\n  cronExpr: string,\n  timezone: string | undefined,\n  onExecute: (rt: IAgentRuntime) => Promise<void>,\n): void {\n  if (runtime.getTaskWorker(taskName)) return;\n\n  runtime.registerTaskWorker({\n    name: taskName,\n    execute: async (rt, _options, task) => {\n      await onExecute(rt);\n      const meta = (task.metadata ?? {}) as Record<string, unknown>;\n      const expr =\n        typeof meta.trackCCronExpr === \"string\"\n          ? meta.trackCCronExpr\n          : cronExpr;\n      const tzRaw = meta.trackCCronTz;\n      let tz: string | undefined;\n      if (tzRaw === null || tzRaw === undefined) {\n        tz = timezone;\n      } else if (typeof tzRaw === \"string\" && tzRaw.length > 0) {\n        tz = tzRaw;\n      } else {\n        tz = timezone;\n      }\n      return {\n        nextInterval: msUntilNextCron(expr, tz),\n      };\n    },\n  });\n}\n\nexport interface EnsureTrackCCronRepeatTaskParams {\n  taskName: string;\n  description: string;\n  cronExpr: string;\n  timezone?: string;\n  log?: MinimalLogger;\n  logPrefix?: string;\n  onExecute: (rt: IAgentRuntime) => Promise<void>;\n}\n\n/**\n * Registers a repeat queue task wired to TaskService ticks. Dedupes by task name,\n * invokes `onExecute` on each fire, and sets `nextInterval` from the cron boundary.\n */\nexport async function ensureTrackCCronRepeatTask(\n  runtime: TrackCRuntimeSubset,\n  params: EnsureTrackCCronRepeatTaskParams,\n): Promise<\"created\" | \"existing\"> {\n  const logPrefix = params.logPrefix ?? \"[TrackCTask]\";\n  await pruneDuplicateTasksByName(\n    runtime,\n    params.taskName,\n    params.log,\n    logPrefix,\n  );\n\n  registerTrackCWorkerOnce(\n    runtime,\n    params.taskName,\n    params.cronExpr,\n    params.timezone,\n    params.onExecute,\n  );\n\n  const existing = await runtime.getTasksByName(params.taskName);\n  if (existing.length > 0) {\n    return \"existing\";\n  }\n\n  const now = Date.now();\n  const nextInterval = msUntilNextCron(params.cronExpr, params.timezone);\n  await runtime.createTask({\n    name: params.taskName,\n    description: params.description,\n    tags: [...TRACK_C_QUEUE_TAGS],\n    metadata: {\n      updatedAt: now,\n      updateInterval: nextInterval,\n      baseInterval: nextInterval,\n      maxFailures: -1,\n      trackCCronExpr: params.cronExpr,\n      trackCCronTz: params.timezone ?? null,\n    },\n  });\n  params.log?.info(`${logPrefix} created repeat task \"${params.taskName}\"`);\n  return \"created\";\n}\n"],"mappings":"AAKA,SAAS,8BAA8B;AAGhC,MAAM,qBAAqB,CAAC,SAAS,UAAU,SAAS;AAExD,MAAM,sCACX;AACK,MAAM,kCACX;AAEF,MAAM,mBAAmB;AAgBzB,SAAS,cAAc,MAAoB;AACzC,QAAM,OAAO,KAAK;AAClB,MAAI,OAAO,MAAM,cAAc,SAAU,QAAO,KAAK;AACrD,MAAI,OAAO,KAAK,cAAc,SAAU,QAAO,KAAK;AACpD,MAAI,OAAO,KAAK,cAAc,SAAU,QAAO,OAAO,KAAK,SAAS;AACpE,SAAO;AACT;AAEA,SAAS,gBACP,YACA,UACQ;AACR,QAAM,MAAM,KAAK,IAAI;AACrB,QAAM,OAAO,uBAAuB,YAAY,KAAK,QAAQ;AAC7D,MAAI,SAAS,KAAM,QAAO;AAC1B,QAAM,QAAQ,OAAO;AACrB,SAAO,QAAQ,IAAI,QAAQ;AAC7B;AAEA,eAAsB,0BACpB,SACA,UACA,KACA,YAAY,gBACG;AACf,QAAM,QAAQ,MAAM,QAAQ,eAAe,QAAQ;AACnD,MAAI,MAAM,UAAU,EAAG;AAEvB,QAAM,SAAS,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM,cAAc,CAAC,IAAI,cAAc,CAAC,CAAC;AAC5E,MAAI,UAAU;AACd,aAAW,OAAO,OAAO,MAAM,CAAC,GAAG;AACjC,QAAI,IAAI,IAAI;AACV,YAAM,QAAQ,WAAW,IAAI,EAAE;AAC/B,iBAAW;AAAA,IACb;AAAA,EACF;AACA,MAAI,UAAU,GAAG;AACf,SAAK;AAAA,MACH,GAAG,SAAS,YAAY,OAAO,2BAA2B,QAAQ;AAAA,IACpE;AAAA,EACF;AACF;AAEA,SAAS,yBACP,SACA,UACA,UACA,UACA,WACM;AACN,MAAI,QAAQ,cAAc,QAAQ,EAAG;AAErC,UAAQ,mBAAmB;AAAA,IACzB,MAAM;AAAA,IACN,SAAS,OAAO,IAAI,UAAU,SAAS;AACrC,YAAM,UAAU,EAAE;AAClB,YAAM,OAAQ,KAAK,YAAY,CAAC;AAChC,YAAM,OACJ,OAAO,KAAK,mBAAmB,WAC3B,KAAK,iBACL;AACN,YAAM,QAAQ,KAAK;AACnB,UAAI;AACJ,UAAI,UAAU,QAAQ,UAAU,QAAW;AACzC,aAAK;AAAA,MACP,WAAW,OAAO,UAAU,YAAY,MAAM,SAAS,GAAG;AACxD,aAAK;AAAA,MACP,OAAO;AACL,aAAK;AAAA,MACP;AACA,aAAO;AAAA,QACL,cAAc,gBAAgB,MAAM,EAAE;AAAA,MACxC;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAgBA,eAAsB,2BACpB,SACA,QACiC;AACjC,QAAM,YAAY,OAAO,aAAa;AACtC,QAAM;AAAA,IACJ;AAAA,IACA,OAAO;AAAA,IACP,OAAO;AAAA,IACP;AAAA,EACF;AAEA;AAAA,IACE;AAAA,IACA,OAAO;AAAA,IACP,OAAO;AAAA,IACP,OAAO;AAAA,IACP,OAAO;AAAA,EACT;AAEA,QAAM,WAAW,MAAM,QAAQ,eAAe,OAAO,QAAQ;AAC7D,MAAI,SAAS,SAAS,GAAG;AACvB,WAAO;AAAA,EACT;AAEA,QAAM,MAAM,KAAK,IAAI;AACrB,QAAM,eAAe,gBAAgB,OAAO,UAAU,OAAO,QAAQ;AACrE,QAAM,QAAQ,WAAW;AAAA,IACvB,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,MAAM,CAAC,GAAG,kBAAkB;AAAA,IAC5B,UAAU;AAAA,MACR,WAAW;AAAA,MACX,gBAAgB;AAAA,MAChB,cAAc;AAAA,MACd,aAAa;AAAA,MACb,gBAAgB,OAAO;AAAA,MACvB,cAAc,OAAO,YAAY;AAAA,IACnC;AAAA,EACF,CAAC;AACD,SAAO,KAAK,KAAK,GAAG,SAAS,yBAAyB,OAAO,QAAQ,GAAG;AACxE,SAAO;AACT;","names":[]}

package/dist/core/training-analysis-index.d.ts

@@ -0,0 +1,104 @@
+export declare const TRAINING_ANALYSIS_INDEX_SCHEMA = "eliza_training_analysis_index";
+export declare const TRAINING_ANALYSIS_INDEX_VERSION = 1;
+export declare const TRAINING_JSONL_DATASET_SCHEMA = "eliza_training_jsonl_dataset";
+export declare const ACTION_BENCHMARK_REPORT_SCHEMA = "eliza_action_selection_benchmark_report";
+export interface TrainingAnalysisArtifact {
+    id: string;
+    kind: "trajectory_bundle" | "trajectory_dataset" | "scenario_run" | "collection_run" | "training_run" | "eval" | "benchmark_matrix" | "model";
+    title: string;
+    path: string;
+    generatedAt?: string;
+    summary: Record<string, unknown>;
+    payload: unknown;
+}
+export interface TrainingAnalysisIndexManifest {
+    schema: typeof TRAINING_ANALYSIS_INDEX_SCHEMA;
+    schemaVersion: typeof TRAINING_ANALYSIS_INDEX_VERSION;
+    generatedAt: string;
+    roots: string[];
+    outputDir: string;
+    indexHtmlPath: string;
+    manifestPath: string;
+    counts: {
+        trajectoryBundles: number;
+        trajectoryDatasets: number;
+        scenarioRuns: number;
+        collectionRuns: number;
+        trainingRuns: number;
+        evals: number;
+        benchmarkMatrices: number;
+        models: number;
+        artifacts: number;
+    };
+    coverage: TrainingAnalysisCoverageSummary;
+    artifacts: TrainingAnalysisArtifact[];
+}
+export interface TrainingAnalysisCoverageSummary {
+    dataSources: {
+        huggingFace: number;
+        feed: number;
+        natural: number;
+        scenarios: number;
+        tests: number;
+        trainingJsonl: number;
+    };
+    readableSamples: {
+        huggingFace: number;
+        feed: number;
+        natural: number;
+        scenarios: number;
+        tests: number;
+        trainingJsonl: number;
+        total: number;
+    };
+    evals: {
+        artifacts: number;
+        comparisons: number;
+        scoredComparisons: number;
+    };
+    benchmarks: {
+        matrices: number;
+        comparisons: number;
+        scoredComparisons: number;
+        caseSamples: number;
+        tiers: string[];
+        allEliza1TiersCovered: boolean;
+        tierCoverage: Array<{
+            tier: string;
+            hasBase: boolean;
+            hasTrained: boolean;
+            hasReference: boolean;
+            hasImprovement: boolean;
+            benchmarkCount: number;
+            comparisonCount: number;
+        }>;
+    };
+    models: {
+        artifacts: number;
+        stagedBundles: number;
+        inventory: Array<{
+            model: string | null;
+            tier: string | null;
+            variant: string | null;
+            baseModel: string | null;
+            outputPath: string | null;
+            baseEvalScore: number | null;
+            trainedEvalScore: number | null;
+            evalImprovementPercent: number | null;
+        }>;
+    };
+}
+export interface BuildTrainingAnalysisIndexOptions {
+    roots?: string[];
+    outputDir?: string;
+    maxDepth?: number;
+    now?: () => Date;
+}
+export interface TrainingAnalysisIndex {
+    outputDir: string;
+    indexHtmlPath: string;
+    manifestPath: string;
+    manifest: TrainingAnalysisIndexManifest;
+}
+export declare function buildTrainingAnalysisIndex(options?: BuildTrainingAnalysisIndexOptions): Promise<TrainingAnalysisIndex>;
+//# sourceMappingURL=training-analysis-index.d.ts.map

package/dist/core/training-analysis-index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"training-analysis-index.d.ts","sourceRoot":"","sources":["../../src/core/training-analysis-index.ts"],"names":[],"mappings":"AAwBA,eAAO,MAAM,8BAA8B,kCAAkC,CAAC;AAC9E,eAAO,MAAM,+BAA+B,IAAI,CAAC;AACjD,eAAO,MAAM,6BAA6B,iCAAiC,CAAC;AAC5E,eAAO,MAAM,8BAA8B,4CACA,CAAC;AAK5C,MAAM,WAAW,wBAAwB;IACvC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EACA,mBAAmB,GACnB,oBAAoB,GACpB,cAAc,GACd,gBAAgB,GAChB,cAAc,GACd,MAAM,GACN,kBAAkB,GAClB,OAAO,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,6BAA6B;IAC5C,MAAM,EAAE,OAAO,8BAA8B,CAAC;IAC9C,aAAa,EAAE,OAAO,+BAA+B,CAAC;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE;QACN,iBAAiB,EAAE,MAAM,CAAC;QAC1B,kBAAkB,EAAE,MAAM,CAAC;QAC3B,YAAY,EAAE,MAAM,CAAC;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;QACrB,KAAK,EAAE,MAAM,CAAC;QACd,iBAAiB,EAAE,MAAM,CAAC;QAC1B,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,EAAE,+BAA+B,CAAC;IAC1C,SAAS,EAAE,wBAAwB,EAAE,CAAC;CACvC;AAED,MAAM,WAAW,+BAA+B;IAC9C,WAAW,EAAE;QACX,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,eAAe,EAAE;QACf,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,aAAa,EAAE,MAAM,CAAC;QACtB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,KAAK,EAAE;QACL,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;IACF,UAAU,EAAE;QACV,QAAQ,EAAE,MAAM,CAAC;QACjB,WAAW,EAAE,MAAM,CAAC;QACpB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,WAAW,EAAE,MAAM,CAAC;QACpB,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,qBAAqB,EAAE,OAAO,CAAC;QAC/B,YAAY,EAAE,KAAK,CAAC;YAClB,IAAI,EAAE,MAAM,CAAC;YACb,OAAO,EAAE,OAAO,CAAC;YACjB,UAAU,EAAE,OAAO,CAAC;YACpB,YAAY,EAAE,OAAO,CAAC;YACtB,cAAc,EAAE,OAAO,CAAC;YACxB,cAAc,EAAE,MAAM,CAAC;YACvB,eAAe,EAAE,MAAM,CAAC;SACzB,CAAC,CAAC;KACJ,CAAC;IACF,MAAM,EAAE;QACN,SAAS,EAAE,MAAM,CAAC;QAClB,aAAa,EAAE,MAAM,CAAC;QACtB,SAAS,EAAE,KAAK,CAAC;YACf,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;YACrB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;YACpB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;YACvB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;YACzB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;YAC1B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;YAC7B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;YAChC,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;SACvC,CAAC,CAAC;KACJ,CAAC;CACH;AAED,MAAM,WAAW,iCAAiC;IAChD,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB;AAED,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,6BAA6B,CAAC;CACzC;AA8uHD,wBAAsB,0BAA0B,CAC9C,OAAO,GAAE,iCAAsC,GAC9C,OAAO,CAAC,qBAAqB,CAAC,CAkEhC"}