@operor/cli 0.1.0

package/src/commands/simulate.ts

@@ -0,0 +1,187 @@
+import { readConfig } from '../config.js';
+import { writeFileSync } from 'fs';
+import { formatTimestamp } from '../utils.js';
+
+export async function runSimulate(options: {
+  tests?: string[];
+  scenarios?: string[];
+  conversations?: number;
+  strategy?: 'exact' | 'similarity' | 'llm_judge';
+  real?: boolean;
+  allowWrites?: boolean;
+  dryRun?: boolean;
+  timeout?: number;
+  parallel?: boolean;
+  json?: boolean;
+  output?: string;
+}): Promise<void> {
+  const config = readConfig();
+  const clack = await import('@clack/prompts');
+
+  const { Operor } = await import('@operor/core');
+  const { MockProvider } = await import('@operor/provider-mock');
+  const {
+    MockShopifySkill,
+    SimulationRunner,
+    ECOMMERCE_SCENARIOS,
+    SkillTestHarness,
+  } = await import('@operor/testing');
+
+  // Optionally set up LLM if configured
+  let llm: any;
+  if (config.LLM_PROVIDER && config.LLM_API_KEY) {
+    const { AIProvider } = await import('@operor/llm');
+    llm = new AIProvider({
+      provider: config.LLM_PROVIDER as any,
+      apiKey: config.LLM_API_KEY,
+      model: config.LLM_MODEL,
+    });
+  }
+
+  // Determine conversation scenarios
+  let conversationScenarios: any = undefined;
+  if (options.scenarios) {
+    if (options.scenarios.includes('all')) {
+      conversationScenarios = 'builtin';
+    } else {
+      conversationScenarios = ECOMMERCE_SCENARIOS.filter((s) =>
+        options.scenarios!.some((name) =>
+          s.name.toLowerCase().includes(name.toLowerCase()) ||
+          s.id.toLowerCase().includes(name.toLowerCase())
+        )
+      );
+      if (conversationScenarios.length === 0) {
+        console.error(`No scenarios matching: ${options.scenarios.join(', ')}`);
+        console.error('Available scenarios:');
+        ECOMMERCE_SCENARIOS.forEach((s) => console.error(`  - ${s.id}: ${s.name}`));
+        process.exit(1);
+      }
+    }
+  } else {
+    // Default: use all built-in scenarios
+    conversationScenarios = 'builtin';
+  }
+
+  // Set up Operor with appropriate skills
+  const os = new Operor({ debug: false, batchWindowMs: 0 });
+  const provider = new MockProvider();
+  const shopify = new MockShopifySkill();
+
+  // Wrap skill with safety harness if --real or --dry-run
+  let skill: any = shopify;
+  if (options.real || options.dryRun) {
+    skill = new SkillTestHarness(shopify, {
+      allowWrites: options.allowWrites ?? false,
+      dryRun: options.dryRun ?? false,
+    });
+    await skill.authenticate();
+  }
+
+  await os.addProvider(provider);
+  await os.addSkill(skill);
+
+  const allTools = [shopify.tools.get_order, shopify.tools.create_discount, shopify.tools.search_products];
+
+  const agent = os.createAgent({
+    name: 'Test Agent',
+    purpose: 'Handle customer support conversations',
+    personality: 'empathetic and solution-focused',
+    triggers: ['order_tracking', 'general'],
+    tools: allTools,
+    rules: [{
+      name: 'Auto-compensation',
+      condition: async (_ctx: any, toolResults: any[]) => {
+        const order = toolResults.find((t) => t.name === 'get_order');
+        return order?.success && order.result?.isDelayed && order.result?.delayDays >= 2;
+      },
+      action: async () => {
+        const discount = await shopify.tools.create_discount.execute({ percent: 10, validDays: 30 });
+        return { type: 'discount_created', code: discount.code, percent: 10, validDays: 30 };
+      },
+    }],
+  });
+
+  // Set up Knowledge Base if enabled
+  let kbRuntime: any;
+  if (config.KB_ENABLED === 'true') {
+    try {
+      const { SQLiteKnowledgeStore, EmbeddingService, RetrievalPipeline } = await import('@operor/knowledge');
+      const embedder = new EmbeddingService({
+        provider: (config.KB_EMBEDDING_PROVIDER || config.LLM_PROVIDER) as any,
+        apiKey: config.KB_EMBEDDING_API_KEY || config.LLM_API_KEY || '',
+        model: config.KB_EMBEDDING_MODEL,
+      });
+      const kbStore = new SQLiteKnowledgeStore(config.KB_DB_PATH || './knowledge.db', embedder.dimensions);
+      await kbStore.initialize();
+      const retrieval = new RetrievalPipeline(kbStore, embedder);
+      kbRuntime = { retrieve: (q: string) => retrieval.retrieve(q) };
+      console.log('[Operor] 📚 Knowledge Base enabled for simulation');
+    } catch (kbError: any) {
+      console.warn('[Operor] ⚠️  Failed to initialize Knowledge Base:', kbError.message);
+    }
+  }
+
+  // Override agent.process with LLM-based implementation if LLM is configured
+  if (llm) {
+    const { applyLLMOverride } = await import('./llm-override.js');
+    applyLLMOverride(agent, llm, allTools, { kbRuntime });
+  }
+
+  await os.start();
+
+  const totalConversations = options.conversations ?? 10;
+
+  // Create simulation runner
+  const runner = new SimulationRunner({
+    agentOS: os,
+    config: {
+      testSuiteFiles: options.tests,
+      conversationScenarios,
+      totalConversations,
+      evaluationStrategy: options.strategy,
+      integrationMode: options.real ? 'real' : options.dryRun ? 'dry-run' : 'mock',
+      allowWrites: options.allowWrites,
+      timeout: options.timeout,
+      parallel: options.parallel,
+    },
+    llm,
+  });
+
+  if (!options.json) {
+    clack.intro('operor simulate');
+  }
+
+  const spinner = !options.json ? clack.spinner() : null;
+  spinner?.start(`Running simulation (0/${totalConversations} conversations)...`);
+
+  const report = await runner.run((completed, total, result) => {
+    const status = result.passed ? 'PASS' : 'FAIL';
+    spinner?.message(`[${formatTimestamp()}] ${completed}/${total} — ${result.scenario.name} [${status}]`);
+  });
+
+  spinner?.stop(`Simulation complete: ${report.passed}/${report.totalConversations} passed`);
+  await os.stop();
+
+  // Output results
+  if (options.json) {
+    const jsonOutput = JSON.stringify(report, null, 2);
+    if (options.output) {
+      writeFileSync(options.output, jsonOutput, 'utf-8');
+      console.log(`Report saved to ${options.output}`);
+    } else {
+      console.log(jsonOutput);
+    }
+  } else {
+    const formatted = SimulationRunner.formatReport(report);
+    console.log('\n' + formatted);
+
+    if (options.output) {
+      writeFileSync(options.output, JSON.stringify(report, null, 2), 'utf-8');
+      clack.note(`Detailed report saved to ${options.output}`);
+    }
+
+    clack.outro(report.overallPassed ? 'Simulation PASSED' : 'Simulation FAILED');
+  }
+
+  process.exit(report.overallPassed ? 0 : 1);
+}

package/src/commands/status.ts

@@ -0,0 +1,112 @@
+import fs from 'fs';
+import { readConfig } from '../config.js';
+
+export async function runStatus(): Promise<void> {
+  const config = readConfig();
+
+  console.log('\n  Operor Status\n');
+
+  // LLM
+  const llmStatus = config.LLM_PROVIDER
+    ? `${config.LLM_PROVIDER} (${config.LLM_MODEL || 'default'})`
+    : 'Not configured';
+  console.log(`  LLM:           ${config.LLM_PROVIDER ? '✓' : '✗'} ${llmStatus}`);
+
+  // Channel
+  const channelStatus = config.CHANNEL || 'Not configured';
+  console.log(`  Channel:       ${config.CHANNEL ? '✓' : '✗'} ${channelStatus}`);
+
+  // Skills (MCP)
+  if (config.SKILLS_ENABLED !== 'false') {
+    try {
+      const { loadSkillsConfig } = await import('@operor/skills');
+      const skillsConfig = loadSkillsConfig();
+      const enabledSkills = skillsConfig.skills.filter((s: any) => s.enabled !== false);
+      if (enabledSkills.length > 0) {
+        console.log(`  Skills:        ✓ ${enabledSkills.length} MCP skill(s)`);
+        for (const skill of enabledSkills) {
+          const transport = skill.transport || 'stdio';
+          const target = skill.command || skill.url || '?';
+          console.log(`    └─ ${skill.name} (${transport}: ${target})`);
+        }
+      } else {
+        console.log(`  Skills:        ─ No skills in mcp.json`);
+      }
+    } catch {
+      console.log(`  Skills:        ─ No mcp.json found`);
+    }
+  } else {
+    console.log(`  Skills:        ✗ Disabled`);
+  }
+
+  // Analytics
+  if (config.ANALYTICS_ENABLED !== 'false') {
+    const dbPath = config.ANALYTICS_DB_PATH || './analytics.db';
+    const digestStatus = config.ANALYTICS_DIGEST_ENABLED === 'true'
+      ? `${config.ANALYTICS_DIGEST_SCHEDULE || 'daily'} at ${config.ANALYTICS_DIGEST_TIME || '09:00'}`
+      : 'disabled';
+    console.log(`  Analytics:     ✓ Enabled (${dbPath})`);
+    console.log(`    └─ Digest: ${digestStatus}`);
+  } else {
+    console.log(`  Analytics:     ✗ Disabled`);
+  }
+
+  // Agents from agents/ directory
+  const agentsDir = `${process.cwd()}/agents`;
+  if (fs.existsSync(agentsDir)) {
+    try {
+      const { AgentLoader } = await import('@operor/core');
+      const loader = new AgentLoader(process.cwd());
+      const definitions = await loader.loadAll();
+
+      if (definitions.length > 0) {
+        console.log(`\n  Agents:        ✓ ${definitions.length} loaded from agents/`);
+        for (const def of definitions) {
+          const cfg = def.config;
+          const details: string[] = [];
+          if (cfg.channels?.length) details.push(`channels: ${cfg.channels.join(', ')}`);
+          if (cfg.skills?.length) details.push(`skills: ${cfg.skills.join(', ')}`);
+          if (cfg.knowledgeBase) details.push('KB');
+          if (cfg.priority) details.push(`priority: ${cfg.priority}`);
+          const suffix = details.length > 0 ? ` (${details.join(' | ')})` : '';
+          console.log(`    └─ ${cfg.name}${suffix}`);
+        }
+      } else {
+        console.log(`\n  Agents:        ✗ agents/ directory found but no valid definitions`);
+      }
+    } catch {
+      console.log(`\n  Agents:        ✗ Failed to load agents/`);
+    }
+  } else {
+    console.log(`\n  Agents:        ─ No agents/ directory (using default single-agent mode)`);
+  }
+
+  // Usage summary (last 24h)
+  if (config.ANALYTICS_ENABLED !== 'false') {
+    try {
+      const { SQLiteAnalyticsStore } = await import('@operor/analytics');
+      const dbPath = config.ANALYTICS_DB_PATH || './analytics.db';
+      if (fs.existsSync(dbPath)) {
+        const store = new SQLiteAnalyticsStore(dbPath);
+        await store.initialize();
+        const now = Date.now();
+        const range = { from: now - 24 * 60 * 60 * 1000, to: now };
+        const summary = await store.getSummary(range);
+        await store.close();
+
+        if (summary.totalMessages > 0) {
+          console.log(`  Usage (last 24h)`);
+          console.log(`    Messages:    ${summary.totalMessages}`);
+          console.log(`    Customers:   ${summary.uniqueCustomers}`);
+          console.log(`    KB answered: ${summary.kbAnsweredPct.toFixed(1)}%`);
+          console.log(`    Avg response: ${summary.avgResponseTime < 1000 ? Math.round(summary.avgResponseTime) + 'ms' : (summary.avgResponseTime / 1000).toFixed(1) + 's'}`);
+          console.log();
+        }
+      }
+    } catch {
+      // Analytics DB not available — skip silently
+    }
+  }
+
+  console.log();
+}

package/src/commands/test-suite.ts

@@ -0,0 +1,247 @@
+import { formatTimestamp } from '../utils.js';
+import { readConfig } from '../config.js';
+
+export async function runTestSuite(file: string, options: {
+  strategy?: 'exact' | 'contains' | 'similarity' | 'semantic';
+  timeout?: number;
+  parallel?: boolean;
+  verbose?: boolean;
+  json?: boolean;
+  real?: boolean;
+  allowWrites?: boolean;
+  dryRun?: boolean;
+  llm?: boolean;
+}): Promise<void> {
+  const { Operor } = await import('@operor/core');
+  const { MockProvider } = await import('@operor/provider-mock');
+  const { CSVLoader, TestSuiteRunner, SkillTestHarness } = await import('@operor/testing');
+
+  // Load test cases from file
+  let testCases;
+  try {
+    testCases = await CSVLoader.fromFile(file);
+  } catch (err) {
+    const isNotFound = err instanceof Error && 'code' in err && (err as any).code === 'ENOENT';
+    if (isNotFound) {
+      console.error(`\n  Test file not found: ${file}\n`);
+      console.error(`  To get started, create a CSV file with this format:\n`);
+      console.error(`    id,question,expected_answer,expected_tools,persona,tags`);
+      console.error(`    greeting-1,Hello,Hi! How can I help you?,,friendly,greeting`);
+      console.error(`    order-1,Where is my order?,Let me check.,get_order,helpful,order_tracking\n`);
+      console.error(`  Required columns: id, question`);
+      console.error(`  Optional columns: expected_answer, expected_tools, persona, tags\n`);
+      console.error(`  A sample file is included at tests.csv in the project root.`);
+      console.error(`  You can also use JSON format — see docs for details.\n`);
+    } else {
+      console.error(`Failed to load test cases from ${file}: ${err instanceof Error ? err.message : err}`);
+    }
+    process.exit(1);
+  }
+
+  if (testCases.length === 0) {
+    console.error('No test cases found in file.');
+    process.exit(1);
+  }
+
+  // Set up Operor
+  const os = new Operor({ debug: false, batchWindowMs: 0 });
+  const provider = new MockProvider();
+  await os.addProvider(provider);
+
+  let harness: InstanceType<typeof SkillTestHarness> | null = null;
+  let agent: any;
+  let allTools: any[] = [];
+
+  if (options.real || options.dryRun) {
+    // Real integration testing now uses MCP skills via mcp.json
+    console.error('--real/--dry-run mode has been removed. Configure MCP skills in mcp.json instead.');
+    console.error('Use the mock-based test suite (default) or Docker E2E tests for integration testing.');
+    process.exit(1);
+  } else {
+    // Default: use mocks
+    const { MockShopifySkill } = await import('@operor/testing');
+    const shopify = new MockShopifySkill();
+    await os.addSkill(shopify);
+
+    allTools = [shopify.tools.get_order, shopify.tools.create_discount, shopify.tools.search_products];
+
+    agent = os.createAgent({
+      name: 'Test Agent',
+      purpose: 'Handle customer support queries',
+      personality: 'helpful and solution-focused',
+      triggers: ['order_tracking', 'general'],
+      tools: allTools,
+      rules: [{
+        name: 'Auto-compensation',
+        condition: async (_ctx: any, toolResults: any[]) => {
+          const order = toolResults.find((t) => t.name === 'get_order');
+          return order?.success && order.result?.isDelayed && order.result?.delayDays >= 2;
+        },
+        action: async () => {
+          const discount = await shopify.tools.create_discount.execute({ percent: 10, validDays: 30 });
+          return { type: 'discount_created', code: discount.code, percent: 10, validDays: 30 };
+        },
+      }],
+    });
+  }
+
+  // Set up LLM-based agent processing if --llm flag is set
+  let llm: any;
+  if (options.llm) {
+    const config = readConfig();
+
+    if (!config.LLM_PROVIDER || !config.LLM_API_KEY) {
+      console.error('--llm requires LLM_PROVIDER and LLM_API_KEY in .env. Run "operor setup" or "operor config set" to configure.');
+      process.exit(1);
+    }
+
+    const { AIProvider } = await import('@operor/llm');
+    llm = new AIProvider({
+      provider: config.LLM_PROVIDER as any,
+      apiKey: config.LLM_API_KEY,
+      model: config.LLM_MODEL,
+    });
+
+    if (!options.json) {
+      console.log(`\n  LLM mode: ${llm.getProviderName()} (${llm.getModelName()})`);
+    }
+
+    // Override agent.process with LLM-based implementation
+    agent.process = async (context: any) => {
+      const startTime = Date.now();
+      const systemMessage = `You are a ${agent.config.personality} customer support agent. ${agent.config.purpose}.`;
+      const messages = [
+        { role: 'system' as const, content: systemMessage },
+        ...context.history.map((m: any) => ({
+          role: m.role as 'user' | 'assistant',
+          content: m.content,
+        })),
+        { role: 'user' as const, content: context.currentMessage.text },
+      ];
+
+      const toolCalls: any[] = [];
+      let finalText = '';
+      let iterations = 0;
+      const maxIterations = 5;
+
+      while (iterations < maxIterations) {
+        iterations++;
+
+        const response = await llm.complete(messages, {
+          tools: allTools.map((t) => ({
+            name: t.name,
+            description: t.description,
+            parameters: t.parameters,
+          })),
+        });
+
+        if (response.toolCalls && response.toolCalls.length > 0) {
+          const executedTools: Array<{ name: string; result: any; success: boolean; error?: string }> = [];
+
+          for (const tc of response.toolCalls) {
+            const tool = allTools.find((t) => t.name === tc.name);
+            if (!tool) continue;
+
+            try {
+              const result = await tool.execute(tc.arguments);
+              toolCalls.push({ id: tc.id, name: tc.name, params: tc.arguments, result, success: true });
+              executedTools.push({ name: tc.name, result, success: true });
+            } catch (err: any) {
+              toolCalls.push({ id: tc.id, name: tc.name, params: tc.arguments, result: null, success: false, error: err.message });
+              executedTools.push({ name: tc.name, result: null, success: false, error: err.message });
+            }
+          }
+
+          const toolResultSummary = executedTools.map(tc =>
+            `[Tool ${tc.name}]: ${JSON.stringify(tc.success ? tc.result : { error: tc.error })}`
+          ).join('\n');
+
+          messages.push(
+            { role: 'assistant', content: `I'll call ${executedTools.map(tc => tc.name).join(', ')} to help with that.` },
+            { role: 'user', content: `Tool results:\n${toolResultSummary}\n\nPlease use these results to respond to the customer.` }
+          );
+        } else {
+          finalText = response.text;
+          break;
+        }
+      }
+
+      return {
+        text: finalText,
+        toolCalls,
+        duration: Date.now() - startTime,
+      };
+    };
+  }
+
+  await os.start();
+
+  if (!options.json) {
+    console.log(`\n  Running ${testCases.length} test case(s) from ${file}...\n`);
+  }
+
+  const runner = new TestSuiteRunner({
+    agentOS: os,
+    llm,
+    timeout: options.timeout ?? 10000,
+    strategy: options.strategy,
+  });
+
+  const result = await runner.runSuite(testCases);
+  await os.stop();
+
+  // Output results
+  if (options.json) {
+    console.log(JSON.stringify(result, null, 2));
+  } else {
+    // Per-test results
+    for (const r of result.results) {
+      const status = r.evaluation.passed ? 'PASS' : 'FAIL';
+      const score = r.evaluation.score.toFixed(2);
+      const line = `  [${formatTimestamp()}] [${status}] ${r.testCase.id}: ${r.testCase.question} (score: ${score}, ${r.duration}ms)`;
+      console.log(line);
+
+      if (options.verbose && !r.evaluation.passed) {
+        console.log(`         Reason: ${r.evaluation.reasoning}`);
+        console.log(`         Response: ${r.agentResponse.slice(0, 120)}`);
+      }
+    }
+
+    // Tag breakdown
+    const tags = Object.entries(result.byTag);
+    if (tags.length > 0) {
+      console.log('\n  --- By Tag ---');
+      for (const [tag, stats] of tags) {
+        console.log(`  ${tag}: ${stats.passed}/${stats.total} passed (avg score: ${stats.avgScore.toFixed(2)})`);
+      }
+    }
+
+    // Audit log summary (when using SkillTestHarness)
+    if (harness) {
+      const log = harness.getAuditLog();
+      if (log.length > 0) {
+        console.log('\n  --- Skill Call Log ---');
+        const reads = log.filter((e) => e.classification === 'read');
+        const writes = log.filter((e) => e.classification === 'write');
+        const destructive = log.filter((e) => e.classification === 'destructive');
+        const dryRuns = log.filter((e) => (e.result as any)?.dryRun);
+
+        console.log(`  Total calls: ${log.length} (${reads.length} read, ${writes.length} write, ${destructive.length} destructive)`);
+        if (dryRuns.length > 0) {
+          console.log(`  Dry-run calls: ${dryRuns.length} (not executed)`);
+        }
+        for (const entry of log) {
+          const dryTag = (entry.result as any)?.dryRun ? ' [DRY-RUN]' : '';
+          console.log(`    ${entry.classification.toUpperCase()} ${entry.name}${dryTag} (${entry.duration}ms)`);
+        }
+      }
+    }
+
+    // Summary
+    console.log(`\n  Summary: ${result.passed}/${result.total} passed, avg score: ${result.averageScore.toFixed(2)}`);
+    console.log(`  Duration: ${(result.totalDuration / 1000).toFixed(1)}s, Cost: $${result.totalCost.toFixed(4)}`);
+    console.log(`  Result: ${result.failed === 0 ? 'PASSED' : 'FAILED'}\n`);
+  }
+
+  process.exit(result.failed === 0 ? 0 : 1);
+}