@elizaos/cli 1.3.1 → 1.4.2

package/templates/plugin-starter/README.md

@@ -34,89 +34,164 @@ elizaos test
 
 ## Testing
 
-ElizaOS provides a comprehensive testing structure for plugins:
+ElizaOS uses a dual testing approach that combines Bun's native test runner for component tests with a custom E2E test runner for integration testing within a live ElizaOS runtime.
 
 ### Test Structure
 
-- **Component Tests** (`__tests__/` directory):
-
-  - **Unit Tests**: Test individual functions/classes in isolation
-  - **Integration Tests**: Test how components work together
-  - Run with: `elizaos test component`
-
-- **End-to-End Tests** (`__tests__/e2e/` directory):
-
-  - Test the plugin within a full ElizaOS runtime
-  - Validate complete user scenarios with a real agent
-  - Run with: `elizaos test e2e`
+```
+src/
+  __tests__/              # All tests live inside src
+    *.test.ts            # Component tests (use Bun test runner)
+    e2e/                 # E2E tests (use ElizaOS test runner)
+      *.ts               # E2E test files
+      README.md          # E2E testing documentation
+```
 
-- **Running All Tests**:
-  - `elizaos test` runs both component and e2e tests
+### Two Types of Tests
 
-### Writing Tests
+#### 1. Component Tests (Bun Test Runner)
 
-Component tests use Vitest:
+- **Purpose**: Test individual functions/classes in isolation
+- **Location**: `src/__tests__/*.test.ts`
+- **Runner**: Bun's built-in test runner
+- **Command**: `bun test`
+- **Features**: Fast, isolated, uses mocks
 
 ```typescript
-// Unit test example (__tests__/plugin.test.ts)
+// Example: src/__tests__/plugin.test.ts
+import { describe, it, expect } from 'bun:test';
+import { starterPlugin } from '../plugin';
+
 describe('Plugin Configuration', () => {
   it('should have correct plugin metadata', () => {
     expect(starterPlugin.name).toBe('plugin-starter');
   });
 });
-
-// Integration test example (__tests__/integration.test.ts)
-describe('Integration: HelloWorld Action with StarterService', () => {
-  it('should handle HelloWorld action with StarterService', async () => {
-    // Test interactions between components
-  });
-});
 ```
 
-E2E tests run in a real ElizaOS runtime:
+#### 2. E2E Tests (ElizaOS Test Runner)
+
+- **Purpose**: Test plugin behavior within a real ElizaOS runtime
+- **Location**: `src/__tests__/e2e/*.ts`
+- **Runner**: ElizaOS custom test runner
+- **Command**: `elizaos test --type e2e`
+- **Features**: Real runtime, real database, full integration
 
 ```typescript
-// E2E test example (__tests__/e2e/starter-plugin.ts)
+// Example: src/__tests__/e2e/starter-plugin.ts
+import { type TestSuite } from '@elizaos/core';
+
 export const StarterPluginTestSuite: TestSuite = {
   name: 'plugin_starter_test_suite',
-  description: 'E2E tests for the starter plugin',
   tests: [
     {
       name: 'hello_world_action_test',
       fn: async (runtime) => {
-        // Simulate user asking agent to say hello
-        const testMessage = {
-          content: { text: 'Can you say hello?' }
-        };
-
-        // Execute action and capture response
-        const response = await helloWorldAction.handler(runtime, testMessage, ...);
-
-        // Verify agent responds with "hello world"
-        if (!response.text.includes('hello world')) {
-          throw new Error('Expected "hello world" in response');
+        // Test with real runtime - no mocks needed!
+        const action = runtime.actions.find((a) => a.name === 'HELLO_WORLD');
+        if (!action) {
+          throw new Error('Action not found');
         }
+        // Test real behavior...
       },
     },
   ],
 };
 ```
 
-#### Key E2E Testing Features:
+### Running Tests
+
+```bash
+# Run all tests (both component and E2E)
+elizaos test
+
+# Run only component tests (fast, for TDD)
+bun test
+# or
+elizaos test --type component
+
+# Run only E2E tests (slower, full integration)
+elizaos test --type e2e
+```
+
+### Key Differences
+
+| Aspect          | Component Tests      | E2E Tests               |
+| --------------- | -------------------- | ----------------------- |
+| **Runner**      | Bun test             | ElizaOS TestRunner      |
+| **Environment** | Mocked               | Real runtime            |
+| **Database**    | Mocked               | Real (PGLite)           |
+| **Speed**       | Fast (ms)            | Slower (seconds)        |
+| **Use Case**    | TDD, component logic | Integration, user flows |
+
+### E2E Test Integration
+
+E2E tests are integrated into your plugin by:
+
+1. **Creating the test suite** in `src/__tests__/e2e/`
+2. **Importing directly** in your plugin definition:
+
+```typescript
+// src/plugin.ts
+import { StarterPluginTestSuite } from './__tests__/e2e/starter-plugin';
+
+export const starterPlugin: Plugin = {
+  name: 'plugin-starter',
+  // ... other properties
+  tests: [StarterPluginTestSuite], // Direct import, no tests.ts needed
+};
+```
+
+### Writing Effective E2E Tests
+
+E2E tests receive a real `IAgentRuntime` instance, allowing you to:
+
+- Access real actions, providers, and services
+- Interact with the actual database
+- Test complete user scenarios
+- Validate plugin behavior in production-like conditions
+
+```typescript
+{
+  name: 'service_lifecycle_test',
+  fn: async (runtime) => {
+    // Get the real service
+    const service = runtime.getService('starter');
+    if (!service) {
+      throw new Error('Service not initialized');
+    }
+
+    // Test real behavior
+    await service.stop();
+    // Verify cleanup happened...
+  },
+}
+```
+
+### Best Practices
+
+1. **Use Component Tests for**:
+
+   - Algorithm logic
+   - Data transformations
+   - Input validation
+   - Error handling
 
-- **Real Runtime Environment**: Tests run with a fully initialized ElizaOS runtime
-- **Plugin Interaction**: Test how your plugin behaves with the actual agent
-- **Scenario Testing**: Validate complete user interactions, not just individual functions
-- **No Mock Required**: Access real services, actions, and providers
+2. **Use E2E Tests for**:
 
-#### Writing New E2E Tests:
+   - User scenarios
+   - Action execution flows
+   - Provider data integration
+   - Service lifecycle
+   - Plugin interactions
 
-1. Add a new test object to the `tests` array in your test suite
-2. Each test receives the runtime instance as a parameter
-3. Throw errors to indicate test failures (no assertion library needed)
-4. See the comprehensive documentation in `__tests__/e2e/starter-plugin.ts` for detailed examples
+3. **Test Organization**:
+   - Keep related tests together
+   - Use descriptive test names
+   - Include failure scenarios
+   - Document complex test setups
 
-The test utilities in `__tests__/test-utils.ts` provide mock objects and setup functions to simplify writing component tests.
+The comprehensive E2E test documentation in `src/__tests__/e2e/README.md` provides detailed examples and patterns for writing effective tests.
 
 ## Publishing & Continuous Development
 

package/templates/plugin-starter/package.json

@@ -41,7 +41,7 @@
     "tsup.config.ts"
   ],
   "dependencies": {
-    "@elizaos/core": "1.3.1",
+    "@elizaos/core": "1.4.2",
     "@tanstack/react-query": "^5.80.7",
     "clsx": "^2.1.1",
     "tailwind-merge": "^3.3.1",
@@ -50,7 +50,7 @@
     "zod": "3.24.2"
   },
   "devDependencies": {
-    "@elizaos/cli": "1.2.10",
+    "@elizaos/cli": "1.4.2",
     "@tailwindcss/vite": "^4.1.10",
     "@vitejs/plugin-react-swc": "^3.10.2",
     "dotenv": "16.4.5",
@@ -64,6 +64,7 @@
     "dev": "elizaos dev",
     "build": "tsc --noEmit && vite build && tsup",
     "lint": "prettier --write ./src",
+    "postinstall": "node -e \"const fs=require('fs');const path=require('path');const zodPath=path.join('node_modules','zod');if(fs.existsSync(zodPath)){fs.writeFileSync(path.join(zodPath,'v3.js'),'module.exports=require(\\\"./lib/index.js\\\");');fs.writeFileSync(path.join(zodPath,'v3.d.ts'),'export * from \\\"./lib/index\\\";');}\" || true",
     "test:component": "bun run test:install && bun test",
     "test:e2e": "bun run test:install && bun test",
     "test:e2e:manual": "bun run test:install && node scripts/test-e2e-manual.js",
@@ -77,7 +78,13 @@
     "access": "public"
   },
   "resolutions": {
-    "zod": "3.24.2"
+    "// Note": "langchain 0.3.30 fixes @langchain/core zod/v3 import issues",
+    "zod": "3.24.2",
+    "langchain": "0.3.30",
+    "@langchain/core": "0.3.30"
+  },
+  "overrides": {
+    "@langchain/core": "0.3.30"
   },
   "agentConfig": {
     "pluginType": "elizaos:plugin:1.0.0",

package/templates/plugin-starter/src/__tests__/e2e/README.md

@@ -2,9 +2,26 @@
 
 This directory contains end-to-end tests for the ElizaOS plugin starter template.
 
+## ElizaOS Testing Philosophy
+
+ElizaOS employs a dual testing strategy:
+
+1. **Component Tests** (`src/__tests__/*.test.ts`)
+
+   - Run with Bun's native test runner
+   - Fast, isolated tests using mocks
+   - Perfect for TDD and component logic
+   - Command: `bun test`
+
+2. **E2E Tests** (`src/__tests__/e2e/*.ts`)
+   - Run with ElizaOS custom test runner
+   - Real runtime with actual database (PGLite)
+   - Test complete user scenarios
+   - Command: `elizaos test --type e2e`
+
 ## Overview
 
-E2E tests run in a real ElizaOS runtime environment, allowing you to test your plugin's behavior as it would work in production.
+E2E tests run in a real ElizaOS runtime environment, allowing you to test your plugin's behavior as it would work in production. Unlike component tests, E2E tests provide access to a fully initialized runtime with all services, actions, and providers available.
 
 ## Test Structure
 
@@ -15,24 +32,42 @@ E2E tests run in a real ElizaOS runtime environment, allowing you to test your p
   - `hello_world_provider_test` - Tests provider functionality
   - `starter_service_test` - Tests service lifecycle
 
+## Integration with Plugin
+
+E2E tests are integrated directly into your plugin without the need for an intermediate export file:
+
+```typescript
+// src/plugin.ts
+import { StarterPluginTestSuite } from './__tests__/e2e/plugin-starter.e2e';
+
+export const myPlugin: Plugin = {
+  name: 'my-plugin',
+  // ... other properties
+  tests: [StarterPluginTestSuite], // Direct import!
+};
+```
+
 ## Running Tests
 
 ```bash
 # Run all tests (component + e2e)
-npm test
+elizaos test
 
-# Run only e2e tests
-npm run test:e2e
+# Run only e2e tests (slower, full integration)
+elizaos test --type e2e
 
-# Run only component tests
-npm run test:component
+# Run only component tests (fast, for TDD)
+bun test
+# or
+elizaos test --type component
 ```
 
 ## Implementation Details
 
-1. **Test Export**: Tests are exported through `src/tests.ts` to be included in the plugin build
+1. **Direct Import**: Tests are imported directly from the e2e test file - no intermediate export file needed
 2. **Plugin Integration**: The test suite is added to the plugin's `tests` array
-3. **Runtime Access**: Each test receives a real runtime instance with full access to:
+3. **Test Discovery**: The ElizaOS test runner automatically finds and executes tests from the plugin's `tests` array
+4. **Runtime Access**: Each test receives a real runtime instance with full access to:
    - Plugin actions, providers, and services
    - Agent character configuration
    - Database and model access
@@ -44,4 +79,4 @@ npm run test:component
 
 ## Writing New Tests
 
-See the comprehensive documentation at the top of `starter-plugin.ts` for detailed instructions on adding new tests.
+See the comprehensive documentation at the top of `plugin-starter.e2e.ts` for detailed instructions on adding new tests.

package/{dist/templates/plugin-starter/src/__tests__/e2e/starter-plugin.ts → templates/plugin-starter/src/__tests__/e2e/plugin-starter.e2e.ts}

@@ -1,4 +1,9 @@
-import { type Content, type HandlerCallback } from '@elizaos/core';
+import {
+  type Content,
+  type HandlerCallback,
+  type IAgentRuntime,
+  type TestSuite,
+} from '@elizaos/core';
 
 /**
  * E2E (End-to-End) Test Suite for ElizaOS Plugins
@@ -31,7 +36,7 @@ import { type Content, type HandlerCallback } from '@elizaos/core';
  * ```typescript
  * {
  *   name: 'my_new_test',
- *   fn: async (runtime) => {
+ *   fn: async (runtime: IAgentRuntime) => {
  *     // Your test logic here
  *     if (someCondition !== expected) {
  *       throw new Error('Test failed: reason');
@@ -60,16 +65,6 @@ import { type Content, type HandlerCallback } from '@elizaos/core';
  * For more details, see the ElizaOS documentation.
  */
 
-// Define a minimal TestSuite interface that matches what's needed
-interface TestSuite {
-  name: string;
-  description?: string;
-  tests: Array<{
-    name: string;
-    fn: (runtime: any) => Promise<any>;
-  }>;
-}
-
 // Define minimal interfaces for the types we need
 type UUID = `${string}-${string}-${string}-${string}-${string}`;
 
@@ -91,8 +86,6 @@ interface State {
 
 export const StarterPluginTestSuite: TestSuite = {
   name: 'plugin_starter_test_suite',
-  description: 'E2E tests for the starter plugin',
-
   tests: [
     /**
      * Basic Plugin Verification Test
@@ -102,7 +95,7 @@ export const StarterPluginTestSuite: TestSuite = {
      */
     {
       name: 'example_test',
-      fn: async (runtime) => {
+      fn: async (runtime: IAgentRuntime) => {
         // Test the character name
         if (runtime.character.name !== 'Eliza') {
           throw new Error(
@@ -125,7 +118,7 @@ export const StarterPluginTestSuite: TestSuite = {
      */
     {
       name: 'should_have_hello_world_action',
-      fn: async (runtime) => {
+      fn: async (runtime: IAgentRuntime) => {
         // Access actions through runtime.actions instead of getPlugin
         const actionExists = runtime.actions?.some((a) => a.name === 'HELLO_WORLD');
         if (!actionExists) {
@@ -147,7 +140,7 @@ export const StarterPluginTestSuite: TestSuite = {
      */
     {
       name: 'hello_world_action_test',
-      fn: async (runtime) => {
+      fn: async (runtime: IAgentRuntime) => {
         // Create a test message asking the agent to say hello
         const testMessage: Memory = {
           entityId: '12345678-1234-1234-1234-123456789012' as UUID,
@@ -215,7 +208,7 @@ export const StarterPluginTestSuite: TestSuite = {
      */
     {
       name: 'hello_world_provider_test',
-      fn: async (runtime) => {
+      fn: async (runtime: IAgentRuntime) => {
         // Create a test message
         const testMessage: Memory = {
           entityId: '12345678-1234-1234-1234-123456789012' as UUID,
@@ -258,7 +251,7 @@ export const StarterPluginTestSuite: TestSuite = {
      */
     {
       name: 'starter_service_test',
-      fn: async (runtime) => {
+      fn: async (runtime: IAgentRuntime) => {
         // Get the service from the runtime
         const service = runtime.getService('starter');
         if (!service) {
@@ -287,7 +280,7 @@ export const StarterPluginTestSuite: TestSuite = {
      * ```typescript
      * {
      *   name: 'your_test_name',
-     *   fn: async (runtime) => {
+     *   fn: async (runtime: IAgentRuntime) => {
      *     // Setup: Create any test data needed
      *
      *     // Action: Perform the operation you want to test

package/templates/plugin-starter/src/__tests__/integration.test.ts

@@ -1,4 +1,4 @@
-import { describe, expect, it, spyOn, beforeEach, afterAll, beforeAll } from 'bun:test';
+import { describe, expect, it, beforeEach, afterAll, beforeAll } from 'bun:test';
 import { starterPlugin, StarterService } from '../index';
 import { createMockRuntime, setupLoggerSpies, MockRuntime } from './test-utils';
 import { HandlerCallback, IAgentRuntime, Memory, State, UUID, logger } from '@elizaos/core';
@@ -33,15 +33,15 @@ describe('Integration: HelloWorld Action with StarterService', () => {
     };
 
     // Create a mock runtime with a spied getService method
-    const getServiceImpl = (serviceType) => {
+    const getServiceImpl = (serviceType: string) => {
       if (serviceType === 'starter') {
-        return mockService;
+        return mockService as any;
       }
       return null;
     };
 
     mockRuntime = createMockRuntime({
-      getService: getServiceImpl,
+      getService: getServiceImpl as any,
     });
   });
 
@@ -70,8 +70,8 @@ describe('Integration: HelloWorld Action with StarterService', () => {
     };
 
     // Create a mock callback to capture the response
-    const callbackCalls = [];
-    const callbackFn = (...args) => {
+    const callbackCalls: any[] = [];
+    const callbackFn = (...args: any[]) => {
       callbackCalls.push(args);
     };
 
@@ -88,10 +88,9 @@ describe('Integration: HelloWorld Action with StarterService', () => {
     // Verify the callback was called with expected response
     expect(callbackCalls.length).toBeGreaterThan(0);
     if (callbackCalls.length > 0) {
-      expect(callbackCalls[0][0]).toMatchObject({
-        text: 'hello world!',
-        actions: ['HELLO_WORLD'],
-      });
+      expect(callbackCalls[0][0].text).toBe('Hello world!');
+      expect(callbackCalls[0][0].actions).toEqual(['HELLO_WORLD']);
+      expect(callbackCalls[0][0].source).toBe('test');
     }
 
     // Get the service to ensure integration
@@ -107,9 +106,10 @@ describe('Integration: Plugin initialization and service registration', () => {
     const mockRuntime = createMockRuntime();
 
     // Create and install a mock registerService
-    const registerServiceCalls = [];
-    mockRuntime.registerService = (type, service) => {
-      registerServiceCalls.push({ type, service });
+    const registerServiceCalls: any[] = [];
+    mockRuntime.registerService = (service: any) => {
+      registerServiceCalls.push({ service });
+      return Promise.resolve();
     };
 
     // Run a minimal simulation of the plugin initialization process