@mastra/agent-builder 0.0.0-experimental-agent-builder-20250815195917 → 0.0.1-alpha.2

package/integration-tests/src/template-integration.test.ts

@@ -1,312 +0,0 @@
-import type { ChildProcess } from 'node:child_process';
-import { spawn, execSync } from 'node:child_process';
-import { randomUUID } from 'node:crypto';
-import { mkdtempSync, mkdirSync, rmSync, cpSync, existsSync, readFileSync } from 'node:fs';
-import { join, resolve } from 'node:path';
-import { Mastra } from '@mastra/core';
-import { describe, expect, it, beforeAll, afterAll } from 'vitest';
-import { fetchMastraTemplates } from '../../src/utils';
-import { mergeTemplateWorkflow } from '../../src/workflows';
-
-function exec(cmd: string, cwd?: string): string {
-  return execSync(cmd, { stdio: 'pipe', cwd, encoding: 'utf-8' });
-}
-
-function initGitRepo(repoDir: string) {
-  exec('git init -q', repoDir);
-  exec('git config user.email "test@example.com"', repoDir);
-  exec('git config user.name "Test User"', repoDir);
-}
-
-function commitAll(repoDir: string, message: string) {
-  exec('git add .', repoDir);
-  exec(`git commit -m "${message}" -q`, repoDir);
-}
-
-describe('Template Workflow Integration Tests', () => {
-  const integrationProjectsDir = resolve(__dirname, '../integration-projects');
-  mkdirSync(integrationProjectsDir, { recursive: true });
-  const tempRoot = mkdtempSync(join(integrationProjectsDir, 'template-workflow-test-'));
-  const fixtureProjectPath = resolve(__dirname, 'fixtures/minimal-mastra-project');
-  const targetRepo = join(tempRoot, 'test-project');
-  let mastraServer: ChildProcess;
-  let port = 4199;
-  let mastraInstance: Mastra;
-
-  beforeAll(async () => {
-    mastraInstance = new Mastra({
-      workflows: {
-        mergeTemplateWorkflow,
-      },
-    });
-
-    // Copy the fixture mastra project into temp directory
-    mkdirSync(targetRepo, { recursive: true });
-    cpSync(fixtureProjectPath, targetRepo, { recursive: true });
-
-    // Initialize git in target
-    initGitRepo(targetRepo);
-
-    // Verify .gitignore was copied
-    const gitignorePath = join(targetRepo, '.gitignore');
-    expect(existsSync(gitignorePath)).toBe(true);
-
-    commitAll(targetRepo, 'chore: initial mastra project');
-
-    // Install dependencies in the test project
-    console.log('Installing dependencies in test project...');
-    exec('pnpm install', targetRepo);
-  });
-
-  afterAll(async () => {
-    // Kill the Mastra server if it's running
-    if (mastraServer?.pid) {
-      try {
-        process.kill(-mastraServer.pid, 'SIGTERM');
-        // Wait a bit for graceful shutdown
-        await new Promise(resolve => setTimeout(resolve, 1000));
-      } catch (e) {
-        console.warn('Failed to kill Mastra server:', e);
-      }
-    }
-
-    // Cleanup temp directory
-    try {
-      rmSync(tempRoot, { recursive: true, force: true });
-    } catch {
-      // Ignore cleanup errors
-    }
-  });
-
-  it('should merge csv-to-questions template and validate functionality', async () => {
-    // Skip test if no OPENAI_API_KEY available
-    if (!process.env.OPENAI_API_KEY) {
-      console.log('Skipping test: OPENAI_API_KEY not set');
-      return;
-    }
-
-    // Get the csv-to-questions template info
-    const templates = await fetchMastraTemplates();
-    const csvTemplate = templates.find(t => t.slug === 'csv-to-questions');
-    expect(csvTemplate).toBeDefined();
-
-    console.log(`Starting template merge workflow in ${targetRepo}`);
-
-    const templateWorkflow = mastraInstance.getWorkflow(`mergeTemplateWorkflow`);
-
-    // Run the merge template workflow
-    const workflowRun = await templateWorkflow.createRunAsync();
-    const result = await workflowRun.start({
-      inputData: {
-        repo: csvTemplate!.githubUrl,
-        slug: 'csv-to-questions',
-        targetPath: targetRepo,
-      },
-    });
-
-    console.log('Workflow result:', JSON.stringify(result, null, 2));
-
-    // Verify the workflow succeeded
-    expect(result).toBeDefined();
-    expect(result.status).toBe('success');
-    expect(result.result?.success).toBe(true);
-    expect(result.result?.applied).toBe(true);
-    expect(result.result?.branchName).toBe('feat/install-template-csv-to-questions');
-
-    // Verify the template branch was created
-    const branches = exec('git branch', targetRepo);
-    expect(branches).toContain('feat/install-template-csv-to-questions');
-
-    // Verify expected template files were created
-    const expectedPaths = [
-      'src/mastra/agents/csvQuestionAgent.ts',
-      'src/mastra/tools/csvFetcherTool.ts',
-      'src/mastra/workflows/csvToQuestionsWorkflow.ts',
-    ];
-
-    for (const expectedPath of expectedPaths) {
-      const fullPath = join(targetRepo, expectedPath);
-      expect(existsSync(fullPath), `Expected ${expectedPath} to exist`).toBe(true);
-    }
-
-    // Verify package.json was updated
-    const packageJsonPath = join(targetRepo, 'package.json');
-    const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf-8'));
-    expect(packageJson.scripts).toBeDefined();
-
-    // Check for template-specific scripts or dependencies
-    const hasTemplateScript = Object.keys(packageJson.scripts || {}).some(
-      key => key.includes('csv-to-questions') || key.includes('template'),
-    );
-    expect(hasTemplateScript).toBe(true);
-
-    console.log('Template merge completed successfully');
-  }, 300000); // 5 minute timeout for full workflow
-
-  it('should start Mastra server and validate both original and new agents work', async () => {
-    // Skip test if no OPENAI_API_KEY available
-    if (!process.env.OPENAI_API_KEY) {
-      console.log('Skipping test: OPENAI_API_KEY not set');
-      return;
-    }
-
-    console.log('Starting Mastra server...');
-
-    // Start the Mastra server
-    mastraServer = spawn('pnpm', ['dev'], {
-      stdio: 'pipe',
-      cwd: targetRepo,
-      detached: true,
-    });
-
-    // Wait for server to be ready
-    await new Promise<void>((resolve, reject) => {
-      let output = '';
-      const timeout = setTimeout(() => {
-        reject(new Error('Mastra server failed to start within timeout'));
-      }, 300000);
-
-      mastraServer.stdout?.on('data', data => {
-        output += data.toString();
-        console.log('Server output:', data.toString());
-        if (output.includes('http://localhost:') || output.includes(`localhost:${port}`)) {
-          clearTimeout(timeout);
-          resolve();
-        }
-      });
-
-      mastraServer.stderr?.on('data', data => {
-        const errorStr = data.toString();
-        console.error('Mastra server error:', errorStr);
-        // Don't reject on warnings, only on actual errors
-        if (errorStr.toLowerCase().includes('error') && !errorStr.toLowerCase().includes('warning')) {
-          clearTimeout(timeout);
-          reject(new Error(`Mastra server error: ${errorStr}`));
-        }
-      });
-
-      mastraServer.on('exit', code => {
-        clearTimeout(timeout);
-        if (code !== 0) {
-          reject(new Error(`Mastra server exited with code ${code}`));
-        }
-      });
-    });
-
-    console.log(`Mastra server started on port ${port}`);
-
-    // Test the original weather agent (from fixture)
-    console.log('Testing original weather agent...');
-    const weatherResponse = await fetch(`http://localhost:${port}/api/agents/weatherAgent/generate`, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({
-        messages: [{ role: 'user', content: 'What is the weather in San Francisco?' }],
-        threadId: randomUUID(),
-        resourceId: 'test-resource',
-      }),
-    });
-
-    expect(weatherResponse.ok).toBe(true);
-    const weatherResult = await weatherResponse.json();
-    expect(weatherResult).toBeDefined();
-    expect(weatherResult.text || weatherResult.content).toContain('weather');
-
-    // Test the new CSV agent (from template)
-    console.log('Testing new CSV agent...');
-    const csvResponse = await fetch(`http://localhost:${port}/api/agents/csvQuestionAgent/generate`, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({
-        messages: [
-          {
-            role: 'user',
-            content: 'I want to analyze a CSV file with sales data. Can you help me?',
-          },
-        ],
-        threadId: randomUUID(),
-        resourceId: 'test-resource',
-      }),
-    });
-
-    expect(csvResponse.ok).toBe(true);
-    const csvResult = await csvResponse.json();
-    expect(csvResult).toBeDefined();
-    expect(csvResult.text || csvResult.content).toMatch(/csv|data|analyze/i);
-
-    // Test workflows endpoint to ensure new workflow is registered
-    console.log('Testing workflows endpoint...');
-    const workflowsResponse = await fetch(`http://localhost:${port}/api/workflows`);
-    expect(workflowsResponse.ok).toBe(true);
-    const workflows = await workflowsResponse.json();
-    expect(workflows).toBeDefined();
-
-    // Check if the CSV workflow is registered (workflows is an object, not array)
-    const hasCSVWorkflow =
-      workflows &&
-      ('csvToQuestionsWorkflow' in workflows || Object.values(workflows).some((w: any) => w.name?.includes('csv')));
-    expect(hasCSVWorkflow).toBe(true);
-
-    console.log('All agent and workflow tests passed!');
-  }, 300000); // 5 minute timeout for server startup and testing
-
-  it('should validate git history shows proper template integration', async () => {
-    // Check git log for template commits
-    const gitLog = exec('git log --oneline', targetRepo);
-    expect(gitLog).toContain('feat(template): register components from csv-to-questions@');
-    expect(gitLog).toContain('feat(template): copy 6 files from csv-to-questions@');
-    expect(gitLog).toContain('fix(template): resolve validation errors for csv-to-questions@');
-
-    // Verify we're on the template branch
-    const currentBranch = exec('git branch --show-current', targetRepo);
-    expect(currentBranch.trim()).toBe('feat/install-template-csv-to-questions');
-
-    // Verify the original default branch still exists
-    const allBranches = exec('git branch', targetRepo);
-    expect(allBranches).toMatch(/\b(main|master)\b/);
-
-    console.log('Git history validation completed');
-  });
-
-  it('should handle merge conflicts gracefully when running workflow twice', async () => {
-    // Skip test if no OPENAI_API_KEY available
-    if (!process.env.OPENAI_API_KEY) {
-      console.log('Skipping test: OPENAI_API_KEY not set');
-      return;
-    }
-
-    // Switch back to default branch
-    const defaultBranch = exec('git branch', targetRepo).includes('main') ? 'main' : 'master';
-    exec(`git checkout ${defaultBranch}`, targetRepo);
-
-    // Try to merge the same template again (should handle gracefully)
-    const templates = await fetchMastraTemplates();
-    const csvTemplate = templates.find(t => t.slug === 'csv-to-questions');
-
-    console.log('Testing duplicate template merge...');
-
-    const templateWorkflow = mastraInstance.getWorkflow(`mergeTemplateWorkflow`);
-    const workflowRun = await templateWorkflow.createRunAsync();
-    const result = await workflowRun.start({
-      inputData: {
-        repo: csvTemplate!.githubUrl,
-        slug: 'csv-to-questions',
-        targetPath: targetRepo,
-      },
-    });
-
-    // The workflow should still succeed but handle the existing files intelligently
-    expect(result.status).toBe('success');
-
-    console.log(JSON.stringify(result, null, 2));
-
-    if (result.status === 'success') {
-      expect(result.result.success).toBe(true);
-      expect(result.result.applied).toBe(true);
-      // Should create a new branch with a different name or handle existing branch
-      expect(result.result.branchName).toMatch(/feat\/install-template-csv-to-questions/);
-    }
-
-    console.log('Duplicate merge test completed');
-  }, 300000);
-});

package/integration-tests/tsconfig.json

@@ -1,13 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2020",
-    "module": "ESNext",
-    "moduleResolution": "bundler",
-    "strict": true,
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "forceConsistentCasingInFileNames": true
-  },
-  "include": ["src/**/*"],
-  "exclude": ["node_modules"]
-}

package/integration-tests/vitest.config.ts

@@ -1,17 +0,0 @@
-import { defineConfig } from 'vitest/config';
-
-export default defineConfig({
-  test: {
-    pool: 'forks',
-    globals: true,
-    environment: 'node',
-    testTimeout: 120000,
-    hookTimeout: 60000,
-    coverage: {
-      provider: 'v8',
-      reporter: ['text', 'json'],
-    },
-    reporters: 'dot',
-    bail: 1,
-  },
-});

package/src/agent-builder.test.ts

@@ -1,291 +0,0 @@
-import type { MastraLanguageModel } from '@mastra/core/agent';
-import { RuntimeContext } from '@mastra/core/runtime-context';
-import { describe, it, expect, vi } from 'vitest';
-import { AgentBuilderDefaults } from './defaults';
-import { ToolSummaryProcessor } from './processors/tool-summary';
-import { AgentBuilder } from './index';
-
-// Mock the openai model for testing
-const mockModel = {
-  modelId: 'gpt-4',
-  provider: 'openai',
-} as MastraLanguageModel;
-
-// Mock config with required properties
-const mockConfig = {
-  model: mockModel,
-  projectPath: '/test/project',
-  summaryModel: mockModel,
-};
-
-describe('AgentBuilder', () => {
-  describe('AgentBuilderDefaults', () => {
-    it('should have default instructions', () => {
-      expect(AgentBuilderDefaults.DEFAULT_INSTRUCTIONS).toContain('expert Mastra Agent Builder');
-    });
-
-    it('should have default memory config', () => {
-      expect(AgentBuilderDefaults.DEFAULT_MEMORY_CONFIG).toEqual({
-        maxMessages: 20,
-        tokenLimit: 10000,
-      });
-    });
-
-    it('should have default tools', () => {
-      expect(AgentBuilderDefaults.DEFAULT_TOOLS).toHaveProperty('manageProject');
-      expect(AgentBuilderDefaults.DEFAULT_TOOLS).toHaveProperty('searchReplace');
-      expect(AgentBuilderDefaults.DEFAULT_TOOLS).toHaveProperty('validateCode');
-    });
-  });
-
-  describe('AgentBuilder class', () => {
-    it('should create an instance with basic config', () => {
-      const builder = new AgentBuilder({
-        model: mockModel,
-        projectPath: '/test/project',
-      });
-
-      expect(builder).toBeInstanceOf(AgentBuilder);
-      expect(builder.id).toBe('agent-builder');
-      expect(builder.name).toBe('agent-builder');
-    });
-
-    it('should have access to default tools', () => {
-      const builder = new AgentBuilder({
-        model: mockModel,
-        projectPath: '/test/project',
-      });
-
-      const tools = builder.getTools({ runtimeContext: new RuntimeContext() });
-      expect(tools).toBeDefined();
-    });
-
-    it('should combine custom instructions with default instructions', () => {
-      const customInstructions = 'Custom instructions for testing';
-      const builder = new AgentBuilder({
-        model: mockModel,
-        instructions: customInstructions,
-        projectPath: '/test/project',
-      });
-
-      // Since instructions are private, we can't directly test them,
-      // but we can verify the builder was created successfully
-      expect(builder).toBeInstanceOf(AgentBuilder);
-    });
-
-    it('should merge custom tools with default tools', () => {
-      const customTool = {
-        id: 'custom-tool',
-        description: 'A custom tool for testing',
-        execute: vi.fn().mockResolvedValue({ success: true }),
-      };
-
-      const builder = new AgentBuilder({
-        model: mockModel,
-        tools: {
-          customTool,
-        },
-        projectPath: '/test/project',
-      });
-
-      expect(builder).toBeInstanceOf(AgentBuilder);
-    });
-  });
-
-  describe('AgentBuilder.defaultConfig', () => {
-    it('should return default configuration', () => {
-      const defaultConfig = AgentBuilder.defaultConfig('/test/project');
-
-      expect(defaultConfig).toHaveProperty('instructions');
-      expect(defaultConfig).toHaveProperty('memoryConfig');
-      expect(defaultConfig).toHaveProperty('tools');
-      expect(defaultConfig.memoryConfig).toEqual(AgentBuilderDefaults.DEFAULT_MEMORY_CONFIG);
-    });
-  });
-
-  describe('AgentBuilder.createWithDefaults', () => {
-    it('should create an instance with merged default settings', () => {
-      const builder = new AgentBuilder({
-        model: mockModel,
-        memoryConfig: {
-          maxMessages: 50, // Override default
-        },
-        projectPath: '/test/project',
-      });
-
-      expect(builder).toBeInstanceOf(AgentBuilder);
-    });
-  });
-
-  describe('generateAgent method', () => {
-    it('should be defined and callable', async () => {
-      const builder = new AgentBuilder(mockConfig);
-
-      expect(builder.generateAgent).toBeDefined();
-      expect(typeof builder.generateAgent).toBe('function');
-    });
-  });
-
-  describe('ToolSummaryProcessor', () => {
-    it('should cache tool call summaries', async () => {
-      const processor = new ToolSummaryProcessor({ summaryModel: mockModel });
-
-      // Check initial cache is empty
-      const initialStats = processor.getCacheStats();
-      expect(initialStats.size).toBe(0);
-
-      // Create a mock tool call
-      const mockToolCall = {
-        toolName: 'testTool',
-        args: { param1: 'value1', param2: 'value2' },
-      };
-
-      // Test cache key generation
-      const cacheKey = processor.createCacheKey(mockToolCall);
-      expect(cacheKey).toBe('testTool:{"param1":"value1","param2":"value2"}');
-
-      // Test cache clearing
-      processor.clearCache();
-      const clearedStats = processor.getCacheStats();
-      expect(clearedStats.size).toBe(0);
-    });
-
-    it('should create consistent cache keys for same arguments', () => {
-      const processor = new ToolSummaryProcessor({ summaryModel: mockModel });
-
-      const toolCall1 = {
-        toolName: 'myTool',
-        args: { a: 1, b: 2 },
-      };
-
-      const toolCall2 = {
-        toolName: 'myTool',
-        args: { b: 2, a: 1 }, // Different order
-      };
-
-      const key1 = processor.createCacheKey(toolCall1);
-      const key2 = processor.createCacheKey(toolCall2);
-
-      // Should be same key despite different argument order
-      expect(key1).toBe(key2);
-      expect(key1).toBe('myTool:{"a":1,"b":2}');
-    });
-  });
-
-  describe('Server Management Tools', () => {
-    it('should have manageServer and httpRequest tools available', async () => {
-      const builder = new AgentBuilder(mockConfig);
-      const tools = await builder.getTools({ runtimeContext: new RuntimeContext() });
-
-      expect(tools.manageServer).toBeDefined();
-      expect(tools.httpRequest).toBeDefined();
-    });
-
-    it('should validate manageServer tool schema', () => {
-      expect(AgentBuilderDefaults.checkMastraServerStatus).toBeDefined();
-      expect(AgentBuilderDefaults.startMastraServer).toBeDefined();
-      expect(AgentBuilderDefaults.stopMastraServer).toBeDefined();
-      expect(AgentBuilderDefaults.makeHttpRequest).toBeDefined();
-    });
-
-    it('should handle server stop with no running process', async () => {
-      // Mock exec to return "No process found"
-      const mockExec = vi.fn().mockResolvedValue({ stdout: 'No process found' });
-      const originalExec = require('child_process').exec;
-
-      try {
-        require('child_process').exec = mockExec;
-
-        const result = await AgentBuilderDefaults.stopMastraServer({ port: 9999 });
-
-        expect(result.success).toBe(true);
-        expect(result.status).toBe('stopped');
-        expect(result.message).toContain('No Mastra server found running on port 9999');
-      } finally {
-        require('child_process').exec = originalExec;
-      }
-    });
-  });
-
-  describe('Code Validation', () => {
-    it('should have validateCode method', () => {
-      expect(AgentBuilderDefaults.validateCode).toBeDefined();
-      expect(AgentBuilderDefaults.parseESLintErrors).toBeDefined();
-    });
-
-    it('should include full TypeScript output in validation errors', () => {
-      // Since we're now passing through raw TypeScript output,
-      // we just need to verify the validateCode method includes it
-      expect(AgentBuilderDefaults.validateCode).toBeDefined();
-
-      // The actual validation testing would require a real TypeScript project,
-      // but we can verify the method exists and handles errors properly
-      const mockTsOutput = `src/test.ts(10,5): error TS2322: Type 'string' is not assignable to type 'number'.
-src/another.ts(20,15): warning TS2345: Argument of type 'null' is not assignable to parameter of type 'string'.
-Found 2 errors in 2 files.`;
-
-      // This output would now be included directly in the error message
-      // for the agent to interpret, rather than being parsed into structured errors
-      expect(mockTsOutput).toContain('error TS2322');
-      expect(mockTsOutput).toContain('Found 2 errors');
-    });
-
-    it('should parse ESLint errors correctly', () => {
-      const eslintResults = [
-        {
-          filePath: '/path/to/file.ts',
-          messages: [
-            {
-              ruleId: 'no-unused-vars',
-              severity: 2,
-              message: "'unusedVar' is defined but never used.",
-              line: 5,
-              column: 10,
-            },
-            {
-              ruleId: 'prefer-const',
-              severity: 1,
-              message: "'data' is never reassigned. Use 'const' instead of 'let'.",
-              line: 8,
-              column: 3,
-            },
-          ],
-        },
-      ];
-
-      const errors = AgentBuilderDefaults.parseESLintErrors(eslintResults);
-
-      expect(errors).toHaveLength(2);
-      expect(errors[0]).toEqual({
-        type: 'eslint',
-        severity: 'error',
-        message: "'unusedVar' is defined but never used.",
-        file: '/path/to/file.ts',
-        line: 5,
-        column: 10,
-        code: 'no-unused-vars',
-      });
-      expect(errors[1]).toEqual({
-        type: 'eslint',
-        severity: 'warning',
-        message: "'data' is never reassigned. Use 'const' instead of 'let'.",
-        file: '/path/to/file.ts',
-        line: 8,
-        column: 3,
-        code: 'prefer-const',
-      });
-    });
-
-    it('should include validation workflow in instructions', () => {
-      const instructions = AgentBuilderDefaults.DEFAULT_INSTRUCTIONS('/test/path');
-
-      expect(instructions).toContain('VALIDATE CODE AFTER CHANGES');
-      expect(instructions).toContain('validateCode');
-      expect(instructions).toContain("['types', 'lint']");
-      expect(instructions).toContain('Re-validate until clean');
-      expect(instructions).toContain(
-        'Documentation → Web Research → Clarification → Project Exploration → Implementation → Validation',
-      );
-    });
-  });
-});