@elizaos/cli 1.4.4 → 1.4.5

package/dist/chunk-LG7YDBMV.js

@@ -0,0 +1,401 @@
+import {
+  askAgentViaApi
+} from "./chunk-PUZHCSGF.js";
+import {
+  bunExec
+} from "./chunk-I4L4T7QX.js";
+
+// src/commands/scenario/src/TrajectoryReconstructor.ts
+var TrajectoryReconstructor = class {
+  runtime;
+  constructor(runtime) {
+    this.runtime = runtime;
+  }
+  /**
+   * Reconstruct trajectory from memories (using same approach as TrajectoryContainsActionEvaluator)
+   */
+  async reconstructTrajectory(roomId, timeWindowMs = 3e4) {
+    const endTime = Date.now();
+    const startTime = endTime - timeWindowMs;
+    const allMemories = await this.runtime.getMemories({
+      tableName: "messages",
+      agentId: this.runtime.agentId,
+      count: 100,
+      unique: false
+    });
+    console.log(`
+\u{1F50D} [TrajectoryReconstructor] ===== MEMORY ANALYSIS START =====`);
+    console.log(
+      `\u{1F50D} [TrajectoryReconstructor] Found ${allMemories.length} total memories for agent`
+    );
+    console.log(`\u{1F50D} [TrajectoryReconstructor] All roomIds found in memories:`);
+    const uniqueRoomIds = [...new Set(allMemories.map((m) => m.roomId).filter(Boolean))];
+    uniqueRoomIds.forEach((rId, i) => {
+      const count = allMemories.filter((m) => m.roomId === rId).length;
+      console.log(`   ${i + 1}. ${rId} (${count} memories)`);
+    });
+    console.log(`\u{1F50D} [TrajectoryReconstructor] Original roomId: ${roomId}`);
+    const actualRoomId = uniqueRoomIds.length > 0 ? uniqueRoomIds[0] : roomId;
+    if (actualRoomId !== roomId) {
+      console.log(
+        `\u{1F527} [TrajectoryReconstructor] ROOMID MISMATCH DETECTED - Using actual roomId: ${actualRoomId}`
+      );
+    }
+    const memories = allMemories.filter((mem) => mem && mem.roomId === actualRoomId);
+    console.log(
+      `\u{1F50D} [TrajectoryReconstructor] Found ${memories.length} memories using actual roomId`
+    );
+    memories.forEach((mem, index) => {
+      console.log(`
+--- Memory ${index + 1}/${memories.length} ---`);
+      console.log(`ID: ${mem.id}`);
+      console.log(`CreatedAt: ${mem.createdAt} (${new Date(mem.createdAt || 0).toISOString()})`);
+      console.log(`Type: ${mem.type || "undefined"}`);
+      console.log(`Content Type: ${typeof mem.content}`);
+      if (mem.content && typeof mem.content === "object") {
+        console.log(`Content.type: ${mem.content?.type}`);
+        console.log(`Content keys:`, Object.keys(mem.content));
+        if (mem.content?.type === "action_result") {
+          console.log(
+            `\u{1F3AF} FOUND ACTION_RESULT - FULL CONTENT:`,
+            JSON.stringify(mem.content, null, 2)
+          );
+        } else if (mem.content?.type === "user" || mem.content?.type === "agent") {
+          console.log(
+            `\u{1F4AC} MESSAGE CONTENT:`,
+            JSON.stringify(
+              {
+                type: mem.content.type,
+                text: mem.content.text,
+                content: mem.content.content
+              },
+              null,
+              2
+            )
+          );
+        } else {
+          console.log(`\u{1F4CB} OTHER CONTENT:`, JSON.stringify(mem.content, null, 2));
+        }
+      } else {
+        console.log(`Raw Content:`, mem.content);
+      }
+    });
+    console.log(`\u{1F50D} [TrajectoryReconstructor] ===== MEMORY ANALYSIS END =====
+`);
+    const actionMemories = memories.filter(
+      (mem) => mem && typeof mem.content === "object" && mem.content?.type === "action_result"
+    );
+    const steps = [];
+    const runIds = /* @__PURE__ */ new Set();
+    console.log(
+      `\u{1F3AF} [TrajectoryReconstructor] Processing ${actionMemories.length} action memories...`
+    );
+    for (const memory of actionMemories) {
+      const content = memory.content;
+      console.log(`
+\u{1F504} Processing action memory ${memory.id}...`);
+      console.log(`   actionName: ${content?.actionName}`);
+      console.log(`   actionParams:`, content?.actionParams);
+      console.log(`   actionResult:`, content?.actionResult);
+      console.log(`   thought:`, content?.thought);
+      console.log(`   planThought:`, content?.planThought);
+      console.log(`   actionStatus:`, content?.actionStatus);
+      const actionName = content?.actionName || "unknown";
+      const actionParams = content?.actionParams || {};
+      const actionResult = content?.actionResult || {};
+      const thought = content?.thought || content?.planThought || "";
+      let observationContent = "";
+      if (actionResult?.text) {
+        observationContent = actionResult.text;
+      } else if (actionResult?.stdout) {
+        observationContent = actionResult.stdout;
+      } else if (actionResult?.output) {
+        observationContent = actionResult.output;
+      } else if (typeof actionResult === "string") {
+        observationContent = actionResult;
+      } else if (actionResult && typeof actionResult === "object") {
+        observationContent = JSON.stringify(actionResult);
+      }
+      console.log(
+        `   \u{1F4CB} Extracted observation (${observationContent.length} chars):`,
+        observationContent.substring(0, 200)
+      );
+      const timestamp = new Date(memory.createdAt || Date.now()).toISOString();
+      if (thought && thought.trim()) {
+        const thoughtStep = {
+          type: "thought",
+          timestamp,
+          content: thought
+        };
+        steps.push(thoughtStep);
+        console.log(`   \u{1F4AD} Created thought step:`, JSON.stringify(thoughtStep, null, 2));
+      }
+      const actionStep = {
+        type: "action",
+        timestamp,
+        content: {
+          name: actionName,
+          parameters: actionParams
+        }
+      };
+      steps.push(actionStep);
+      console.log(`   \u26A1 Created action step:`, JSON.stringify(actionStep, null, 2));
+      const observationStep = {
+        type: "observation",
+        timestamp,
+        content: observationContent
+      };
+      steps.push(observationStep);
+      console.log(`   \u{1F441}\uFE0F Created observation step:`, JSON.stringify(observationStep, null, 2));
+    }
+    steps.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
+    return {
+      steps,
+      runId: runIds.size === 1 ? Array.from(runIds)[0] : void 0,
+      startTime,
+      endTime,
+      totalSteps: steps.length
+    };
+  }
+  /**
+   * Get latest trajectory for a room (convenience method) with retry logic
+   */
+  async getLatestTrajectory(roomId) {
+    console.log(`\u{1F50D} [TrajectoryReconstructor] Starting reconstruction for room: ${roomId}`);
+    const maxRetries = 3;
+    const retryDelayMs = 2e3;
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+      console.log(`
+\u{1F504} [TrajectoryReconstructor] ===== ATTEMPT ${attempt}/${maxRetries} =====`);
+      const trajectory = await this.reconstructTrajectory(roomId, 3e4);
+      console.log(
+        `\u{1F4CA} [TrajectoryReconstructor] Found ${trajectory.steps.length} trajectory steps on attempt ${attempt}`
+      );
+      console.log(
+        `\u{1F4CA} [TrajectoryReconstructor] Time window: ${trajectory.startTime} - ${trajectory.endTime}`
+      );
+      if (trajectory.steps.length > 0) {
+        console.log(
+          `\u2705 [TrajectoryReconstructor] SUCCESS on attempt ${attempt}: Found ${trajectory.steps.length} trajectory steps`
+        );
+        console.log(
+          `\u{1F4CA} [TrajectoryReconstructor] Actions found:`,
+          trajectory.steps.filter((s) => s.type === "action").map((s) => s.content.name)
+        );
+        console.log(
+          `\u{1F4CA} [TrajectoryReconstructor] First step sample:`,
+          JSON.stringify(trajectory.steps[0], null, 2)
+        );
+        console.log(`\u{1F4CA} [TrajectoryReconstructor] ===== SUCCESS END =====
+`);
+        return trajectory.steps;
+      }
+      console.log(
+        `\u26A0\uFE0F [TrajectoryReconstructor] Attempt ${attempt} found 0 steps. ${attempt < maxRetries ? "Retrying..." : "Final attempt failed."}`
+      );
+      if (attempt < maxRetries) {
+        console.log(`\u23F3 [TrajectoryReconstructor] Waiting ${retryDelayMs}ms before retry...`);
+        await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
+      }
+    }
+    console.log(
+      `\u274C [TrajectoryReconstructor] All ${maxRetries} attempts failed - returning empty trajectory`
+    );
+    console.log(`\u{1F4CA} [TrajectoryReconstructor] ===== FINAL FAILURE =====
+`);
+    return [];
+  }
+};
+
+// src/commands/scenario/src/LocalEnvironmentProvider.ts
+import fs from "fs/promises";
+import path from "path";
+import os from "os";
+var LocalEnvironmentProvider = class {
+  tempDir = null;
+  server = null;
+  agentId = null;
+  runtime = null;
+  serverPort = null;
+  trajectoryReconstructor = null;
+  constructor(server, agentId, runtime, serverPort) {
+    this.server = server ?? null;
+    this.agentId = agentId ?? null;
+    this.runtime = runtime ?? null;
+    this.serverPort = serverPort ?? null;
+    this.trajectoryReconstructor = runtime ? new TrajectoryReconstructor(runtime) : null;
+    console.log(`\u{1F527} [DEBUG] LocalEnvironmentProvider CONSTRUCTOR:`);
+    console.log(`\u{1F527} [DEBUG]   - Server: ${server ? "present" : "null"}`);
+    console.log(`\u{1F527} [DEBUG]   - Agent ID: ${agentId}`);
+    console.log(`\u{1F527} [DEBUG]   - Runtime: ${runtime ? "present" : "null"}`);
+    console.log(`\u{1F527} [DEBUG]   - Server Port: ${serverPort}`);
+  }
+  async setup(scenario) {
+    const tempDirPrefix = path.join(os.tmpdir(), "eliza-scenario-run-");
+    this.tempDir = await fs.mkdtemp(tempDirPrefix);
+    const virtualFs = scenario.setup?.virtual_fs;
+    if (virtualFs) {
+      for (const [filePath, content] of Object.entries(virtualFs)) {
+        const fullPath = path.join(this.tempDir, filePath);
+        await fs.mkdir(path.dirname(fullPath), { recursive: true });
+        await fs.writeFile(fullPath, content);
+      }
+    }
+  }
+  async captureFileSystem() {
+    if (!this.tempDir) {
+      return {};
+    }
+    const files = {};
+    try {
+      const readDirRecursive = async (dirPath, basePath = "") => {
+        const entries = await fs.readdir(dirPath, { withFileTypes: true });
+        for (const entry of entries) {
+          const fullPath = path.join(dirPath, entry.name);
+          const relativePath = path.join(basePath, entry.name);
+          if (entry.isDirectory()) {
+            await readDirRecursive(fullPath, relativePath);
+          } else if (entry.isFile()) {
+            try {
+              const content = await fs.readFile(fullPath, "utf-8");
+              files[relativePath] = content;
+            } catch (error) {
+              files[relativePath] = "[binary or unreadable]";
+            }
+          }
+        }
+      };
+      await readDirRecursive(this.tempDir);
+      return files;
+    } catch (error) {
+      console.warn("Failed to capture file system state:", error);
+      return {};
+    }
+  }
+  async run(scenario) {
+    if (!this.tempDir) {
+      throw new Error("Setup must be called before run.");
+    }
+    const results = [];
+    for (const step of scenario.run) {
+      const startedAtMs = Date.now();
+      if (step.input) {
+        if (!this.server || !this.agentId) {
+          throw new Error(
+            "LocalEnvironmentProvider requires a pre-created server and agent for NL input"
+          );
+        }
+        const { response, roomId } = await askAgentViaApi(
+          this.server,
+          this.agentId,
+          step.input,
+          3e4,
+          // timeout
+          this.serverPort
+          // Pass the actual server port
+        );
+        await new Promise((resolve) => setTimeout(resolve, 3e3));
+        const trajectory = this.trajectoryReconstructor && roomId ? await this.trajectoryReconstructor.getLatestTrajectory(roomId) : [];
+        console.log(`\u{1F50D} [Trajectory Debug] Room ID: ${roomId}, Steps found: ${trajectory.length}`);
+        if (trajectory.length > 0) {
+          console.log(`\u{1F4CA} [Trajectory Debug] First step:`, JSON.stringify(trajectory[0], null, 2));
+        }
+        const endedAtMs = Date.now();
+        const durationMs = endedAtMs - startedAtMs;
+        results.push({
+          exitCode: 0,
+          stdout: response,
+          stderr: "",
+          files: await this.captureFileSystem(),
+          startedAtMs,
+          endedAtMs,
+          durationMs,
+          trajectory
+          // Add trajectory to execution result
+        });
+      } else if (step.code) {
+        let execCommand;
+        let execArgs;
+        switch (step.lang) {
+          case "bash":
+          case "sh":
+            execCommand = "sh";
+            execArgs = ["-c", step.code];
+            break;
+          case "node":
+          case "javascript":
+            execCommand = "node";
+            execArgs = ["-e", step.code];
+            break;
+          case "python":
+          case "python3":
+            execCommand = "python3";
+            execArgs = ["-c", step.code];
+            break;
+          default:
+            execCommand = step.lang;
+            execArgs = ["-c", step.code];
+            break;
+        }
+        try {
+          const result = await bunExec(execCommand, execArgs, { cwd: this.tempDir });
+          const { stdout, stderr } = result;
+          const files = await this.captureFileSystem();
+          const endedAtMs = Date.now();
+          const durationMs = endedAtMs - startedAtMs;
+          results.push({
+            exitCode: result.exitCode || 0,
+            stdout,
+            stderr,
+            files,
+            startedAtMs,
+            endedAtMs,
+            durationMs
+          });
+        } catch (error) {
+          const files = await this.captureFileSystem();
+          const endedAtMs = Date.now();
+          const durationMs = endedAtMs - startedAtMs;
+          let exitCode = 1;
+          let stderr = "";
+          let stdout = "";
+          if (error.exitCode !== void 0) {
+            exitCode = error.exitCode;
+          }
+          if (error.stderr) {
+            stderr = error.stderr;
+          }
+          if (error.stdout) {
+            stdout = error.stdout;
+          }
+          if (!stderr && error.message) {
+            stderr = error.message;
+          }
+          results.push({
+            exitCode,
+            stdout,
+            stderr,
+            files,
+            startedAtMs,
+            endedAtMs,
+            durationMs
+          });
+        }
+      } else {
+        throw new Error("Step must have either input or code");
+      }
+    }
+    return results;
+  }
+  async teardown() {
+    if (this.tempDir) {
+      await fs.rm(this.tempDir, { recursive: true, force: true });
+      this.tempDir = null;
+    }
+  }
+};
+
+export {
+  TrajectoryReconstructor,
+  LocalEnvironmentProvider
+};

package/dist/chunk-NHKLUXNE.js

@@ -0,0 +1,166 @@
+// src/commands/scenario/src/schema.ts
+import { z } from "zod";
+var EnhancedEvaluationResultSchema = z.object({
+  evaluator_type: z.string(),
+  success: z.boolean(),
+  summary: z.string(),
+  details: z.record(z.any())
+});
+var CapabilityCheckSchema = z.object({
+  capability: z.string(),
+  achieved: z.boolean(),
+  reasoning: z.string()
+});
+var LLMJudgeResultSchema = z.object({
+  qualitative_summary: z.string(),
+  capability_checklist: z.array(CapabilityCheckSchema)
+});
+var BaseEvaluationSchema = z.object({
+  type: z.string()
+});
+var StringContainsEvaluationSchema = BaseEvaluationSchema.extend({
+  type: z.literal("string_contains"),
+  value: z.string(),
+  case_sensitive: z.boolean().optional()
+});
+var RegexMatchEvaluationSchema = BaseEvaluationSchema.extend({
+  type: z.literal("regex_match"),
+  pattern: z.string()
+});
+var FileExistsEvaluationSchema = BaseEvaluationSchema.extend({
+  type: z.literal("file_exists"),
+  path: z.string()
+});
+var TrajectoryContainsActionSchema = BaseEvaluationSchema.extend({
+  type: z.literal("trajectory_contains_action"),
+  action: z.string()
+});
+var LLMJudgeEvaluationSchema = BaseEvaluationSchema.extend({
+  type: z.literal("llm_judge"),
+  prompt: z.string(),
+  expected: z.string(),
+  model_type: z.string().optional(),
+  temperature: z.number().min(0).max(2).optional(),
+  json_schema: z.record(z.any()).optional(),
+  // JSON schema object for response validation
+  capabilities: z.array(z.string()).min(1, "Capabilities array must not be empty").optional()
+  // Custom capabilities for evaluation
+});
+var ExecutionTimeEvaluationSchema = BaseEvaluationSchema.extend({
+  type: z.literal("execution_time"),
+  max_duration_ms: z.number(),
+  min_duration_ms: z.number().optional(),
+  target_duration_ms: z.number().optional()
+});
+var EvaluationSchema = z.discriminatedUnion("type", [
+  StringContainsEvaluationSchema,
+  RegexMatchEvaluationSchema,
+  FileExistsEvaluationSchema,
+  TrajectoryContainsActionSchema,
+  LLMJudgeEvaluationSchema,
+  ExecutionTimeEvaluationSchema
+]);
+var MockSchema = z.object({
+  service: z.string().optional(),
+  method: z.string(),
+  // Enhanced 'when' clause with multiple matching strategies
+  when: z.object({
+    // Exact argument matching (existing)
+    args: z.array(z.any()).optional(),
+    // Input parameter matching (extracted from args)
+    input: z.record(z.any()).optional(),
+    // Request context matching
+    context: z.record(z.any()).optional(),
+    // Custom JavaScript matcher function
+    matcher: z.string().optional(),
+    // Partial argument matching
+    partialArgs: z.array(z.any()).optional()
+  }).optional(),
+  // Static response (existing)
+  response: z.any(),
+  // Dynamic response generation
+  responseFn: z.string().optional(),
+  // Error simulation
+  error: z.object({
+    code: z.string(),
+    message: z.string(),
+    status: z.number().optional()
+  }).optional(),
+  // Response metadata
+  metadata: z.object({
+    delay: z.number().optional(),
+    // Simulate network delay
+    probability: z.number().min(0).max(1).optional()
+    // Random failure
+  }).optional()
+});
+var PluginConfigSchema = z.object({
+  name: z.string(),
+  version: z.string().optional(),
+  config: z.record(z.any()).optional(),
+  enabled: z.boolean().optional().default(true)
+});
+var PluginReferenceSchema = z.union([
+  z.string(),
+  // Simple string reference
+  PluginConfigSchema
+  // Full configuration object
+]);
+var SetupSchema = z.object({
+  mocks: z.array(MockSchema).optional(),
+  virtual_fs: z.record(z.string()).optional()
+});
+var RunStepSchema = z.object({
+  name: z.string().optional(),
+  lang: z.string().optional(),
+  code: z.string().optional(),
+  input: z.string().optional(),
+  // Natural language input to agent
+  evaluations: z.array(EvaluationSchema)
+});
+var JudgmentSchema = z.object({
+  strategy: z.enum(["all_pass", "any_pass"])
+});
+var ScenarioSchema = z.object({
+  name: z.string(),
+  description: z.string(),
+  plugins: z.array(PluginReferenceSchema).optional(),
+  environment: z.object({
+    type: z.enum(["e2b", "local"])
+  }),
+  setup: SetupSchema.optional(),
+  run: z.array(RunStepSchema),
+  judgment: JudgmentSchema
+});
+var ScenarioRunResultSchema = z.object({
+  run_id: z.string().min(1, "Run ID cannot be empty"),
+  matrix_combination_id: z.string().min(1, "Matrix combination ID cannot be empty"),
+  parameters: z.record(z.any()),
+  metrics: z.object({
+    execution_time_seconds: z.number().min(0),
+    llm_calls: z.number().int().min(0),
+    total_tokens: z.number().int().min(0)
+  }).catchall(z.number()),
+  // Allow additional numeric metrics
+  final_agent_response: z.string().optional(),
+  evaluations: z.array(EnhancedEvaluationResultSchema),
+  trajectory: z.array(
+    z.object({
+      type: z.enum(["thought", "action", "observation"]),
+      timestamp: z.string().refine((val) => !isNaN(Date.parse(val)), {
+        message: "Timestamp must be a valid ISO string"
+      }),
+      content: z.any()
+    })
+  ),
+  error: z.string().nullable()
+});
+
+export {
+  EnhancedEvaluationResultSchema,
+  CapabilityCheckSchema,
+  LLMJudgeResultSchema,
+  EvaluationSchema,
+  ScenarioSchema,
+  ScenarioRunResultSchema
+};