@elizaos/cli 1.4.4 → 1.4.5

package/dist/BrowserWebSocketTransport-5YQPVDV7.js

@@ -0,0 +1,7 @@
+import {
+  BrowserWebSocketTransport
+} from "./chunk-YBDC5OZO.js";
+import "./chunk-2ESYSVXG.js";
+export {
+  BrowserWebSocketTransport
+};

package/dist/EnhancedEvaluationEngine-APOQ6INN.js

@@ -0,0 +1,473 @@
+import "./chunk-2ESYSVXG.js";
+
+// src/commands/scenario/src/EnhancedEvaluationEngine.ts
+import { ModelType } from "@elizaos/core";
+import { z } from "zod";
+var EnhancedEvaluationEngine = class {
+  constructor(runtime) {
+    this.runtime = runtime;
+    this.register("string_contains", new EnhancedStringContainsEvaluator());
+    this.register("regex_match", new EnhancedRegexMatchEvaluator());
+    this.register("file_exists", new EnhancedFileExistsEvaluator());
+    this.register("trajectory_contains_action", new EnhancedTrajectoryContainsActionEvaluator());
+    this.register("llm_judge", new EnhancedLLMJudgeEvaluator());
+    this.register("execution_time", new EnhancedExecutionTimeEvaluator());
+  }
+  enhancedEvaluators = /* @__PURE__ */ new Map();
+  register(type, evaluator) {
+    this.enhancedEvaluators.set(type, evaluator);
+  }
+  /**
+   * NEW: Run evaluations with structured JSON output
+   */
+  async runEnhancedEvaluations(evaluations, runResult) {
+    const results = [];
+    for (const evaluation of evaluations) {
+      const evaluator = this.enhancedEvaluators.get(evaluation.type);
+      if (!evaluator) {
+        results.push({
+          evaluator_type: evaluation.type,
+          success: false,
+          summary: `Unknown evaluator type: '${evaluation.type}'`,
+          details: {
+            error: "evaluator_not_found",
+            requested_type: evaluation.type,
+            available_types: Array.from(this.enhancedEvaluators.keys())
+          }
+        });
+        continue;
+      }
+      try {
+        const result = await evaluator.evaluateEnhanced(evaluation, runResult, this.runtime);
+        results.push(result);
+      } catch (error) {
+        results.push({
+          evaluator_type: evaluation.type,
+          success: false,
+          summary: `Evaluator '${evaluation.type}' failed with error: ${error instanceof Error ? error.message : String(error)}`,
+          details: {
+            error: "evaluator_execution_failed",
+            error_message: error instanceof Error ? error.message : String(error),
+            evaluation_config: evaluation
+          }
+        });
+      }
+    }
+    return results;
+  }
+};
+var EnhancedStringContainsEvaluator = class {
+  async evaluateEnhanced(params, runResult) {
+    if (params.type !== "string_contains") throw new Error("Mismatched evaluator");
+    const expectedValue = params.value;
+    const actualOutput = runResult.stdout;
+    const caseSensitive = params.case_sensitive ?? false;
+    const searchIn = caseSensitive ? actualOutput : actualOutput.toLowerCase();
+    const searchFor = caseSensitive ? expectedValue : expectedValue.toLowerCase();
+    const success = searchIn.includes(searchFor);
+    return {
+      evaluator_type: "string_contains",
+      success,
+      summary: success ? `Assertion PASSED: Agent response contained the expected substring "${expectedValue}".` : `Assertion FAILED: Agent response did not contain the expected substring "${expectedValue}".`,
+      details: {
+        expected_value: expectedValue,
+        actual_output: actualOutput,
+        case_sensitive: caseSensitive,
+        search_performed: `Looking for "${searchFor}" in "${searchIn.substring(0, 100)}${searchIn.length > 100 ? "..." : ""}"`
+      }
+    };
+  }
+};
+var EnhancedRegexMatchEvaluator = class {
+  async evaluateEnhanced(params, runResult) {
+    if (params.type !== "regex_match") throw new Error("Mismatched evaluator");
+    const pattern = params.pattern;
+    const actualOutput = runResult.stdout;
+    const regex = new RegExp(pattern, "i");
+    const match = regex.exec(actualOutput);
+    const success = match !== null;
+    return {
+      evaluator_type: "regex_match",
+      success,
+      summary: success ? `Regex PASSED: Pattern "${pattern}" matched in agent output.` : `Regex FAILED: Pattern "${pattern}" did not match in agent output.`,
+      details: {
+        pattern,
+        regex_flags: "i",
+        actual_output: actualOutput,
+        match_found: match,
+        match_index: match?.index,
+        matched_text: match?.[0]
+      }
+    };
+  }
+};
+var EnhancedFileExistsEvaluator = class {
+  async evaluateEnhanced(params, runResult) {
+    if (params.type !== "file_exists") throw new Error("Mismatched evaluator");
+    const expectedPath = params.path;
+    const createdFiles = Object.keys(runResult.files);
+    const success = createdFiles.includes(expectedPath) || createdFiles.includes(`./${expectedPath}`) || createdFiles.includes(expectedPath.replace(/^\.\//, ""));
+    const matchingPath = createdFiles.find(
+      (path) => path === expectedPath || path === `./${expectedPath}` || path === expectedPath.replace(/^\.\//, "")
+    );
+    return {
+      evaluator_type: "file_exists",
+      success,
+      summary: success ? `File check PASSED: File "${expectedPath}" was created by the agent.` : `File check FAILED: File "${expectedPath}" was not created by the agent.`,
+      details: {
+        expected_path: expectedPath,
+        created_files: createdFiles,
+        matching_path: matchingPath,
+        total_files_created: createdFiles.length
+      }
+    };
+  }
+};
+var EnhancedExecutionTimeEvaluator = class {
+  async evaluateEnhanced(params, runResult) {
+    if (params.type !== "execution_time") throw new Error("Mismatched evaluator");
+    const duration = runResult.durationMs ?? (runResult.endedAtMs ?? 0) - (runResult.startedAtMs ?? 0);
+    if (duration == null || Number.isNaN(duration) || runResult.durationMs === void 0 && (runResult.startedAtMs === void 0 || runResult.endedAtMs === void 0)) {
+      return {
+        evaluator_type: "execution_time",
+        success: false,
+        summary: "Timing check FAILED: No timing information available for this step.",
+        details: {
+          error: "no_timing_data",
+          runResult_timing: {
+            durationMs: runResult.durationMs,
+            startedAtMs: runResult.startedAtMs,
+            endedAtMs: runResult.endedAtMs
+          },
+          constraints: params
+        }
+      };
+    }
+    const tooSlow = duration > params.max_duration_ms;
+    const tooFast = params.min_duration_ms != null && duration < params.min_duration_ms;
+    const success = !tooSlow && !tooFast;
+    let summary;
+    if (success) {
+      summary = `Timing check PASSED: Execution took ${duration}ms (within expected range).`;
+    } else if (tooSlow) {
+      summary = `Timing check FAILED: Execution took ${duration}ms (exceeded maximum of ${params.max_duration_ms}ms).`;
+    } else {
+      summary = `Timing check FAILED: Execution took ${duration}ms (below minimum of ${params.min_duration_ms}ms).`;
+    }
+    return {
+      evaluator_type: "execution_time",
+      success,
+      summary,
+      details: {
+        actual_duration_ms: duration,
+        max_duration_ms: params.max_duration_ms,
+        min_duration_ms: params.min_duration_ms,
+        target_duration_ms: params.target_duration_ms,
+        performance_rating: params.target_duration_ms ? Math.abs(duration - params.target_duration_ms) / params.target_duration_ms : null,
+        timing_breakdown: {
+          started_at: runResult.startedAtMs,
+          ended_at: runResult.endedAtMs,
+          calculated_duration: runResult.durationMs
+        }
+      }
+    };
+  }
+};
+var EnhancedTrajectoryContainsActionEvaluator = class {
+  async evaluateEnhanced(params, runResult, runtime) {
+    if (params.type !== "trajectory_contains_action") throw new Error("Mismatched evaluator");
+    const actionName = params.action;
+    try {
+      const actionMemories = await runtime.getMemories({
+        tableName: "messages",
+        agentId: runtime.agentId,
+        count: 50,
+        unique: false
+      });
+      const actionResults = actionMemories.filter(
+        (mem) => mem?.type === "messages" && mem.content?.type === "action_result"
+      );
+      const normalize = (name) => (typeof name === "string" ? name : "").toLowerCase().replace(/_/g, "");
+      const target = normalize(actionName);
+      const matchingAction = actionResults.find(
+        (mem) => normalize(mem.content?.actionName ?? "") === target
+      );
+      const allActionNames = actionResults.map((mem) => mem.content?.actionName || "unknown");
+      if (!matchingAction) {
+        return {
+          evaluator_type: "trajectory_contains_action",
+          success: false,
+          summary: `Action check FAILED: Action '${actionName}' was not found in the execution trajectory.`,
+          details: {
+            expected_action: actionName,
+            normalized_expected: target,
+            actions_found: allActionNames,
+            total_actions_in_trajectory: actionResults.length,
+            search_method: "case_insensitive_with_underscore_normalization"
+          }
+        };
+      }
+      const actionStatus = matchingAction.content?.actionStatus || "unknown";
+      const actionSucceeded = actionStatus === "completed";
+      return {
+        evaluator_type: "trajectory_contains_action",
+        success: true,
+        // Success means the action was found (regardless of its outcome)
+        summary: actionSucceeded ? `Action check PASSED: Action '${actionName}' was executed successfully.` : `Action check PASSED: Action '${actionName}' was found but failed execution.`,
+        details: {
+          expected_action: actionName,
+          found_action: matchingAction.content?.actionName,
+          action_status: actionStatus,
+          action_succeeded: actionSucceeded,
+          action_error: matchingAction.content?.error,
+          action_result: matchingAction.content?.result,
+          memory_id: matchingAction.id,
+          all_actions_in_trajectory: allActionNames
+        }
+      };
+    } catch (error) {
+      return {
+        evaluator_type: "trajectory_contains_action",
+        success: false,
+        summary: `Action check FAILED: Error while checking trajectory: ${error instanceof Error ? error.message : String(error)}`,
+        details: {
+          expected_action: actionName,
+          error: "trajectory_access_failed",
+          error_message: error instanceof Error ? error.message : String(error),
+          runtime_available: !!runtime,
+          agent_id: runtime?.agentId
+        }
+      };
+    }
+  }
+};
+var EnhancedLLMJudgeEvaluator = class {
+  async evaluateEnhanced(params, runResult, runtime) {
+    if (params.type !== "llm_judge") throw new Error("Mismatched evaluator");
+    const prompt = params.prompt;
+    const expected = params.expected;
+    const candidateModels = [ModelType.OBJECT_SMALL, ModelType.TEXT_LARGE, ModelType.TEXT_SMALL];
+    const temperature = params.temperature || 0.1;
+    const timeoutMs = Number(process.env.LLM_JUDGE_TIMEOUT_MS || 15e3);
+    let modelType = candidateModels.find((m) => runtime.getModel?.(m)) ?? ModelType.TEXT_LARGE;
+    const capabilities = params.capabilities;
+    if (capabilities !== void 0) {
+      try {
+        const capabilitiesSchema = z.array(z.string()).min(1, "Capabilities array must not be empty");
+        capabilitiesSchema.parse(capabilities);
+      } catch (error) {
+        throw new Error(`Invalid capabilities: ${error.message}`);
+      }
+    }
+    const structuredPrompt = this.createStructuredPrompt(runResult, prompt, expected, capabilities);
+    const jsonSchema = this.getStructuredJudgmentSchema();
+    try {
+      const modelHandler = runtime.getModel(modelType);
+      if (!modelHandler) {
+        return {
+          evaluator_type: "llm_judge",
+          success: false,
+          summary: `LLM Judge FAILED: No available model handler found.`,
+          details: {
+            error: "no_model_available",
+            attempted_models: candidateModels,
+            models_available: Object.keys(runtime.models || {}),
+            prompt,
+            expected
+          }
+        };
+      }
+      const objectParams = {
+        prompt: structuredPrompt,
+        schema: jsonSchema,
+        temperature,
+        output: "object"
+      };
+      const response = await Promise.race([
+        runtime.useModel(modelType, objectParams),
+        new Promise(
+          (_, reject) => setTimeout(() => reject(new Error(`LLM judge timeout after ${timeoutMs}ms`)), timeoutMs)
+        )
+      ]);
+      let parsedResponse;
+      try {
+        parsedResponse = this.validateStructuredResponse(response, jsonSchema);
+      } catch (parseError) {
+        return {
+          evaluator_type: "llm_judge",
+          success: false,
+          summary: `LLM Judge FAILED: Invalid LLM response - ${parseError.message}`,
+          details: {
+            error: "llm_parse_error",
+            error_type: "llm_parse_error",
+            error_message: parseError.message,
+            model_used: modelType,
+            prompt,
+            expected,
+            raw_llm_response: response,
+            custom_capabilities_provided: !!(capabilities && capabilities.length > 0),
+            capabilities_count: capabilities ? capabilities.length : 0
+          }
+        };
+      }
+      const success = this.compareWithExpected(parsedResponse, expected);
+      return {
+        evaluator_type: "llm_judge",
+        success,
+        summary: `LLM Judge ${success ? "PASSED" : "FAILED"}: ${parsedResponse.qualitative_summary.substring(0, 150)}${parsedResponse.qualitative_summary.length > 150 ? "..." : ""}`,
+        details: {
+          llm_judge_result: {
+            qualitative_summary: parsedResponse.qualitative_summary,
+            capability_checklist: parsedResponse.capability_checklist
+          },
+          custom_capabilities_provided: !!(capabilities && capabilities.length > 0),
+          capabilities_count: capabilities ? capabilities.length : 5,
+          // Default capabilities count
+          judgment_confidence: parsedResponse.confidence,
+          expected_outcome: expected,
+          model_used: modelType,
+          prompt_used: prompt,
+          raw_llm_response: response
+        }
+      };
+    } catch (error) {
+      const msg = error?.message || String(error);
+      const isTimeout = msg.toLowerCase().includes("timeout");
+      return {
+        evaluator_type: "llm_judge",
+        success: false,
+        summary: isTimeout ? `LLM Judge FAILED: Timed out after ${timeoutMs}ms.` : `LLM Judge FAILED: ${msg}`,
+        details: {
+          error: isTimeout ? "llm_timeout" : "llm_error",
+          error_type: isTimeout ? "llm_timeout" : "llm_error",
+          error_message: msg,
+          timeout_ms: timeoutMs,
+          model_attempted: modelType,
+          prompt,
+          expected,
+          custom_capabilities_provided: !!(capabilities && capabilities.length > 0),
+          capabilities_count: capabilities ? capabilities.length : 0
+        }
+      };
+    }
+  }
+  createStructuredPrompt(runResult, userPrompt, expected, capabilities) {
+    const defaultCapabilities = [
+      "Task Completion",
+      "Response Quality",
+      "User Intent Understanding",
+      "Error Handling",
+      "Appropriate Response Format"
+    ];
+    const capabilitiesToUse = capabilities && capabilities.length > 0 ? capabilities : defaultCapabilities;
+    const capabilitiesSection = capabilitiesToUse.map((capability, index) => {
+      return `${index + 1}. ${capability}`;
+    }).join("\n");
+    return `You are an expert evaluator analyzing an AI agent's performance. Provide a comprehensive, structured assessment.
+
+## Execution Context
+- Exit Code: ${runResult.exitCode}
+- Standard Output: ${runResult.stdout}
+- Standard Error: ${runResult.stderr}  
+- Files Created: ${Object.keys(runResult.files).join(", ") || "None"}
+
+## Evaluation Question
+${userPrompt}
+
+## Expected Outcome
+${expected}
+
+## Instructions
+Analyze the agent's performance and provide a detailed assessment. You must evaluate the agent against the following specific capabilities:
+
+${capabilitiesSection}
+
+For each capability listed above, you must assess whether the agent achieved it and provide detailed reasoning. Your response should include:
+
+1. **Qualitative Summary**: A comprehensive paragraph summarizing overall performance
+2. **Capability Checklist**: For each capability above, provide:
+   - capability: The exact capability name from the list
+   - achieved: Boolean indicating if the capability was demonstrated
+   - reasoning: Detailed explanation of your assessment
+
+Provide your assessment as a structured JSON response with detailed reasoning for each capability.`;
+  }
+  getStructuredJudgmentSchema() {
+    return {
+      type: "object",
+      properties: {
+        qualitative_summary: {
+          type: "string",
+          description: "A comprehensive paragraph summarizing the agent's performance, reasoning, and notable successes or failures"
+        },
+        capability_checklist: {
+          type: "array",
+          items: {
+            type: "object",
+            properties: {
+              capability: { type: "string" },
+              achieved: { type: "boolean" },
+              reasoning: { type: "string" }
+            },
+            required: ["capability", "achieved", "reasoning"]
+          }
+        },
+        confidence: {
+          type: "number",
+          minimum: 0,
+          maximum: 1,
+          description: "Confidence level in the assessment (0-1)"
+        },
+        overall_success: {
+          type: "boolean",
+          description: "Whether the agent successfully met the expected outcome"
+        }
+      },
+      required: ["qualitative_summary", "capability_checklist", "confidence", "overall_success"]
+    };
+  }
+  validateStructuredResponse(response, schema) {
+    if (typeof response === "string") {
+      response = JSON.parse(response);
+    }
+    if (!response.qualitative_summary || !response.capability_checklist) {
+      throw new Error("Invalid LLM response: missing required fields");
+    }
+    if (!Array.isArray(response.capability_checklist)) {
+      throw new Error("Invalid LLM response: capability_checklist must be an array");
+    }
+    if (response.capability_checklist.length === 0) {
+      response.capability_checklist = [
+        {
+          capability: "Task Completion",
+          achieved: response.overall_success || false,
+          reasoning: "Default capability assessment based on overall success"
+        }
+      ];
+    }
+    if (response.confidence === void 0) {
+      response.confidence = 0.8;
+    }
+    if (response.overall_success === void 0) {
+      const allAchieved = response.capability_checklist.every((cap) => cap.achieved === true);
+      response.overall_success = allAchieved;
+    }
+    return response;
+  }
+  compareWithExpected(parsedResponse, expected) {
+    const overallSuccess = parsedResponse.overall_success;
+    const confidence = parsedResponse.confidence || 0;
+    const expectedLower = expected.toLowerCase();
+    if (expectedLower === "yes" || expectedLower === "no") {
+      return expectedLower === "yes" === overallSuccess;
+    }
+    if (expectedLower.includes("+")) {
+      const threshold = parseFloat(expectedLower.replace("+", ""));
+      return confidence >= threshold;
+    }
+    return overallSuccess;
+  }
+};
+export {
+  EnhancedEvaluationEngine
+};

package/dist/EvaluationEngine-Y7ZQJBRC.js

@@ -0,0 +1,9 @@
+import {
+  EvaluationEngine,
+  TrajectoryContainsActionEvaluator
+} from "./chunk-SVHCNBHM.js";
+import "./chunk-2ESYSVXG.js";
+export {
+  EvaluationEngine,
+  TrajectoryContainsActionEvaluator
+};

package/dist/LocalEnvironmentProvider-JWFGG4IN.js

@@ -0,0 +1,15 @@
+import {
+  LocalEnvironmentProvider
+} from "./chunk-LG7YDBMV.js";
+import "./chunk-PUZHCSGF.js";
+import "./chunk-WX37MM4G.js";
+import "./chunk-I57T3WPO.js";
+import "./chunk-5IWKEMEF.js";
+import "./chunk-Q6M2K53X.js";
+import "./chunk-FQYWRHLX.js";
+import "./chunk-GXWWPFBO.js";
+import "./chunk-I4L4T7QX.js";
+import "./chunk-2ESYSVXG.js";
+export {
+  LocalEnvironmentProvider
+};

package/dist/NodeWebSocketTransport-PUO724EY.js

@@ -0,0 +1,8 @@
+import {
+  NodeWebSocketTransport
+} from "./chunk-EXUFDTUD.js";
+import "./chunk-XFJIHUT3.js";
+import "./chunk-2ESYSVXG.js";
+export {
+  NodeWebSocketTransport
+};

package/dist/ScreenRecorder-YK246DNJ.js

@@ -0,0 +1,10 @@
+import {
+  ScreenRecorder
+} from "./chunk-FWYHSCLF.js";
+import "./chunk-XPPESCCM.js";
+import "./chunk-ABGBVB74.js";
+import "./chunk-XFJIHUT3.js";
+import "./chunk-2ESYSVXG.js";
+export {
+  ScreenRecorder
+};

package/dist/agent-start-6QJQAMKA.js

@@ -0,0 +1,13 @@
+import {
+  startAgent,
+  stopAgent
+} from "./chunk-WX37MM4G.js";
+import "./chunk-Q6M2K53X.js";
+import "./chunk-FQYWRHLX.js";
+import "./chunk-GXWWPFBO.js";
+import "./chunk-I4L4T7QX.js";
+import "./chunk-2ESYSVXG.js";
+export {
+  startAgent,
+  stopAgent
+};