ctxpkg 0.0.1

package/src/cli/cli.agent.ts

@@ -0,0 +1,197 @@
+import type { Command } from 'commander';
+
+import { createTestRunner, type TestProgressEvent } from '../agent/agent.test-runner.ts';
+import type { ValidationMode } from '../agent/agent.test-runner.schemas.ts';
+
+import {
+  chalk,
+  formatError,
+  formatHeader,
+  formatInfo,
+  formatSuccess,
+  formatWarning,
+  withErrorHandling,
+} from './cli.utils.ts';
+
+type TestOptions = {
+  json?: boolean;
+  verbose?: boolean;
+  validationMode?: ValidationMode;
+  threshold?: number;
+  model?: string;
+};
+
+/**
+ * Format duration in human-readable form
+ */
+const formatDuration = (ms: number): string => {
+  if (ms < 1000) return `${ms}ms`;
+  if (ms < 60000) return `${(ms / 1000).toFixed(1)}s`;
+  return `${(ms / 60000).toFixed(1)}m`;
+};
+
+/**
+ * Progress callback for verbose mode
+ */
+const createProgressCallback = (verbose: boolean) => {
+  return (event: TestProgressEvent) => {
+    switch (event.type) {
+      case 'suite_start':
+        formatHeader(`Running: ${event.suiteName}`);
+        console.log(`${event.totalTests} test(s) to run\n`);
+        break;
+
+      case 'sync_start':
+        if (verbose) {
+          formatInfo('Syncing collections...');
+        }
+        break;
+
+      case 'sync_complete':
+        if (verbose) {
+          formatSuccess('Collections synced');
+          console.log();
+        }
+        break;
+
+      case 'test_start':
+        if (verbose) {
+          console.log(chalk.dim(`[${event.index + 1}] Running: ${event.testId}`));
+        }
+        break;
+
+      case 'test_complete': {
+        const { result } = event;
+        const icon = result.skipped ? chalk.yellow('○') : result.passed ? chalk.green('✓') : chalk.red('✗');
+        const status = result.skipped ? chalk.yellow('SKIP') : result.passed ? chalk.green('PASS') : chalk.red('FAIL');
+        const duration = chalk.dim(`(${formatDuration(result.durationMs)})`);
+
+        console.log(`${icon} ${result.id} ${status} ${duration}`);
+
+        if (verbose && !result.skipped) {
+          if (result.score !== undefined) {
+            console.log(chalk.dim(`  Score: ${(result.score * 100).toFixed(1)}%`));
+          }
+          if (result.reasoning) {
+            console.log(chalk.dim(`  ${result.reasoning}`));
+          }
+          if (result.error) {
+            console.log(chalk.red(`  Error: ${result.error}`));
+          }
+          if (result.keywordsFound?.length || result.keywordsMissing?.length) {
+            if (result.keywordsFound?.length) {
+              console.log(chalk.dim(`  Found: ${result.keywordsFound.join(', ')}`));
+            }
+            if (result.keywordsMissing?.length) {
+              console.log(chalk.dim(`  Missing: ${result.keywordsMissing.join(', ')}`));
+            }
+          }
+        }
+        break;
+      }
+
+      case 'suite_complete':
+        // Summary is handled separately
+        break;
+    }
+  };
+};
+
+const createAgentCli = (command: Command) => {
+  command.description('Agent testing and evaluation tools');
+
+  command
+    .command('test')
+    .description('Run agent tests from a YAML test file')
+    .argument('<test-file>', 'Path to YAML test file')
+    .option('--json', 'Output results as JSON')
+    .option('-v, --verbose', 'Show detailed progress and results')
+    .option('-m, --validation-mode <mode>', 'Override validation mode (semantic, llm, keywords)')
+    .option('-t, --threshold <number>', 'Override pass threshold (0-1)', parseFloat)
+    .option('--model <model>', 'Model to use for LLM validation (defaults to configured model)')
+    .action(
+      withErrorHandling(async (testFile: string, options: TestOptions) => {
+        const runner = createTestRunner();
+
+        try {
+          // Load test suite
+          const { suite, baseDir } = await runner.loadTestSuite(testFile);
+
+          // Validate options
+          if (options.validationMode && !['semantic', 'llm', 'keywords'].includes(options.validationMode)) {
+            formatError(`Invalid validation mode: ${options.validationMode}. Use: semantic, llm, or keywords`);
+            process.exitCode = 1;
+            return;
+          }
+
+          if (options.threshold !== undefined && (options.threshold < 0 || options.threshold > 1)) {
+            formatError('Threshold must be between 0 and 1');
+            process.exitCode = 1;
+            return;
+          }
+
+          // Run tests
+          const result = await runner.runTestSuite(suite, {
+            onProgress: options.json ? undefined : createProgressCallback(options.verbose ?? false),
+            validationMode: options.validationMode as ValidationMode | undefined,
+            passThreshold: options.threshold,
+            validationModel: options.model,
+            baseDir,
+          });
+
+          // Output results
+          if (options.json) {
+            console.log(JSON.stringify(result, null, 2));
+          } else {
+            // Summary
+            console.log();
+            formatHeader('Summary');
+            console.log(`Total:   ${result.summary.total}`);
+            console.log(`Passed:  ${chalk.green(result.summary.passed)}`);
+            console.log(
+              `Failed:  ${result.summary.failed > 0 ? chalk.red(result.summary.failed) : result.summary.failed}`,
+            );
+            console.log(
+              `Skipped: ${result.summary.skipped > 0 ? chalk.yellow(result.summary.skipped) : result.summary.skipped}`,
+            );
+            console.log(`Time:    ${formatDuration(result.durationMs)}`);
+            console.log();
+
+            // Show failed test details if not verbose (verbose already shows them)
+            if (!options.verbose && result.summary.failed > 0) {
+              formatHeader('Failed Tests');
+              for (const testResult of result.results) {
+                if (!testResult.passed && !testResult.skipped) {
+                  console.log(chalk.red(`✗ ${testResult.id}`));
+                  if (testResult.score !== undefined) {
+                    console.log(chalk.dim(`  Score: ${(testResult.score * 100).toFixed(1)}%`));
+                  }
+                  if (testResult.reasoning) {
+                    console.log(chalk.dim(`  ${testResult.reasoning}`));
+                  }
+                  if (testResult.error) {
+                    console.log(chalk.red(`  Error: ${testResult.error}`));
+                  }
+                  console.log();
+                }
+              }
+            }
+
+            // Final status
+            if (result.summary.failed === 0) {
+              formatSuccess('All tests passed!');
+            } else {
+              formatWarning(`${result.summary.failed} test(s) failed`);
+              process.exitCode = 1;
+            }
+          }
+        } finally {
+          // Clean up - TypeScript doesn't know about the destroy symbol
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          await (runner as any)[Symbol.for('destroy')]?.();
+        }
+      }),
+    );
+};
+
+export { createAgentCli };

package/src/cli/cli.chat.ts

@@ -0,0 +1,369 @@
+import { input } from '@inquirer/prompts';
+import type { Command } from 'commander';
+
+import { createDocumentAgent, getLLMConfigFromAppConfig } from '../agent/agent.ts';
+import type { AgentStep } from '../agent/agent.types.ts';
+
+import { createCliClient } from './cli.client.ts';
+import { chalk, formatError, formatHeader, formatInfo, withErrorHandling } from './cli.utils.ts';
+
+import { CollectionsService } from '#root/collections/collections.ts';
+import { Services } from '#root/utils/utils.services.ts';
+
+type ChatOptions = {
+  interactive?: boolean;
+  useCase?: string;
+  collections?: string[];
+  global: boolean;
+  model?: string;
+  verbose?: boolean;
+};
+
+/**
+ * Create a verbose step callback that logs agent reasoning to console
+ */
+const createVerboseCallback = () => {
+  return (step: AgentStep) => {
+    switch (step.type) {
+      case 'thinking':
+        console.log(chalk.dim(`  [thinking] ${step.content}`));
+        break;
+      case 'tool_call':
+        console.log(chalk.blue(`  [tool] ${step.toolName}`));
+        if (step.toolInput) {
+          const inputPreview = JSON.stringify(step.toolInput).slice(0, 100);
+          console.log(chalk.dim(`    Input: ${inputPreview}${inputPreview.length >= 100 ? '...' : ''}`));
+        }
+        break;
+      case 'tool_result':
+        console.log(chalk.green(`  [result] ${step.content}`));
+        break;
+      case 'error':
+        console.log(chalk.yellow(`  [retry] ${step.content}`));
+        break;
+    }
+  };
+};
+
+/**
+ * Resolve collection names to IDs, returning a list of resolved collection IDs
+ */
+const resolveCollectionIds = (
+  collections: string[] | undefined,
+  aliasMap: Map<string, string>,
+): string[] | undefined => {
+  if (!collections || collections.length === 0) return undefined;
+  return collections.map((c) => aliasMap.get(c) ?? c);
+};
+
+/**
+ * Run one-shot chat mode - ask a single question and get an answer.
+ */
+const runOneShotChat = async (query: string, options: ChatOptions) => {
+  // Get LLM config
+  const llmConfig = await getLLMConfigFromAppConfig();
+
+  if (!llmConfig.apiKey) {
+    formatError('LLM API key not configured. Set it with: ctxpkg config set llm.apiKey <key>');
+    formatError('Or use environment variable: CTXPKG_LLM_API_KEY=<key>');
+    process.exitCode = 1;
+    return;
+  }
+
+  // Override model if specified
+  if (options.model) {
+    llmConfig.model = options.model;
+  }
+
+  // Prompt for use case if not provided
+  let useCase = options.useCase;
+  if (!useCase) {
+    useCase = await input({
+      message: 'What is your use case? (helps find relevant information)',
+    });
+  }
+
+  if (!useCase) {
+    formatError('Use case is required to help the agent find relevant information.');
+    process.exitCode = 1;
+    return;
+  }
+
+  const client = await createCliClient();
+  const services = new Services();
+
+  try {
+    // Build alias map
+    const aliasMap = new Map<string, string>();
+    const collectionsService = services.get(CollectionsService);
+    const includeGlobal = options.global !== false;
+
+    if (includeGlobal) {
+      const allCollections = collectionsService.getAllCollections();
+      for (const [alias, { spec }] of allCollections) {
+        const collectionId = collectionsService.computeCollectionId(spec);
+        aliasMap.set(alias, collectionId);
+      }
+    } else if (collectionsService.projectConfigExists()) {
+      const projectConfig = collectionsService.readProjectConfig();
+      for (const [alias, spec] of Object.entries(projectConfig.collections)) {
+        const collectionId = collectionsService.computeCollectionId(spec);
+        aliasMap.set(alias, collectionId);
+      }
+    }
+
+    // Resolve collection filtering
+    const resolvedCollections = resolveCollectionIds(options.collections, aliasMap);
+
+    // Create verbose callback if needed
+    const onStep = options.verbose ? createVerboseCallback() : undefined;
+
+    // Create agent with collection filtering
+    const agent = createDocumentAgent({
+      client,
+      llmConfig,
+      aliasMap,
+      collections: resolvedCollections,
+      onStep,
+    });
+
+    formatInfo('Searching documentation...\n');
+
+    // Ask the question
+    const response = await agent.ask(query, useCase);
+
+    // Display answer
+    formatHeader('Answer');
+    console.log(response.answer);
+    console.log();
+
+    // Display sources
+    if (response.sources.length > 0) {
+      formatHeader('Sources');
+      for (const source of response.sources) {
+        const section = source.section ? ` → "${source.section}"` : '';
+        console.log(`${chalk.dim('•')} ${chalk.cyan(source.collection)}: ${source.document}${section}`);
+      }
+      console.log();
+    }
+
+    // Display confidence and note
+    const confidenceColor =
+      response.confidence === 'high' ? chalk.green : response.confidence === 'medium' ? chalk.yellow : chalk.red;
+    console.log(`Confidence: ${confidenceColor(response.confidence)}`);
+
+    if (response.note) {
+      console.log(`\n${chalk.dim('Note:')} ${response.note}`);
+    }
+  } finally {
+    await client.disconnect();
+    await services.destroy();
+  }
+};
+
+/**
+ * Run interactive chat mode - continuous conversation with the agent.
+ */
+const runInteractiveChat = async (options: ChatOptions) => {
+  // Get LLM config
+  const llmConfig = await getLLMConfigFromAppConfig();
+
+  if (!llmConfig.apiKey) {
+    formatError('LLM API key not configured. Set it with: ctxpkg config set llm.apiKey <key>');
+    formatError('Or use environment variable: CTXPKG_LLM_API_KEY=<key>');
+    process.exitCode = 1;
+    return;
+  }
+
+  // Override model if specified
+  if (options.model) {
+    llmConfig.model = options.model;
+  }
+
+  const client = await createCliClient();
+  const services = new Services();
+
+  try {
+    // Build alias map
+    const aliasMap = new Map<string, string>();
+    const collectionsService = services.get(CollectionsService);
+    const includeGlobal = options.global !== false;
+
+    if (includeGlobal) {
+      const allCollections = collectionsService.getAllCollections();
+      for (const [alias, { spec }] of allCollections) {
+        const collectionId = collectionsService.computeCollectionId(spec);
+        aliasMap.set(alias, collectionId);
+      }
+    } else if (collectionsService.projectConfigExists()) {
+      const projectConfig = collectionsService.readProjectConfig();
+      for (const [alias, spec] of Object.entries(projectConfig.collections)) {
+        const collectionId = collectionsService.computeCollectionId(spec);
+        aliasMap.set(alias, collectionId);
+      }
+    }
+
+    // Resolve collection filtering
+    const resolvedCollections = resolveCollectionIds(options.collections, aliasMap);
+
+    // Track verbose mode state
+    let verbose = options.verbose ?? false;
+
+    // Create agent with collection filtering
+    const agent = createDocumentAgent({
+      client,
+      llmConfig,
+      aliasMap,
+      collections: resolvedCollections,
+    });
+
+    // Display header
+    formatHeader('ctxpkg Chat');
+    console.log('Type your questions. Commands: /help, /use-case, /clear, /verbose, /quit');
+    console.log();
+
+    // Get initial use case
+    let useCase = options.useCase;
+    if (!useCase) {
+      useCase = await input({
+        message: 'What are you trying to accomplish?',
+      });
+    }
+
+    if (!useCase) {
+      formatError('Use case is required.');
+      process.exitCode = 1;
+      return;
+    }
+
+    console.log();
+    console.log(chalk.dim(`Use case: ${useCase}`));
+    if (verbose) {
+      console.log(chalk.dim('Verbose mode: on'));
+    }
+    console.log();
+
+    // Chat loop
+    while (true) {
+      let message: string;
+      try {
+        message = await input({ message: chalk.cyan('You:') });
+      } catch {
+        // Handle Ctrl+C
+        console.log('\nGoodbye!');
+        break;
+      }
+
+      if (!message.trim()) {
+        continue;
+      }
+
+      // Handle commands
+      if (message.startsWith('/')) {
+        const cmd = message.toLowerCase().trim();
+
+        if (cmd === '/quit' || cmd === '/exit' || cmd === '/q') {
+          console.log('Goodbye!');
+          break;
+        }
+
+        if (cmd === '/help' || cmd === '/h') {
+          console.log('\nCommands:');
+          console.log('  /help, /h       Show this help');
+          console.log('  /use-case, /u   Change use case');
+          console.log('  /clear, /c      Clear conversation history');
+          console.log('  /verbose, /v    Toggle verbose mode');
+          console.log('  /quit, /q       Exit chat\n');
+          continue;
+        }
+
+        if (cmd === '/use-case' || cmd === '/u') {
+          console.log(chalk.dim(`Current use case: ${useCase}`));
+          const newUseCase = await input({ message: 'New use case (or press enter to keep current):' });
+          if (newUseCase.trim()) {
+            useCase = newUseCase;
+            agent.clearHistory(); // Clear history when use case changes
+            console.log(chalk.dim(`Use case updated: ${useCase}`));
+            console.log(chalk.dim('Conversation history cleared.\n'));
+          }
+          continue;
+        }
+
+        if (cmd === '/clear' || cmd === '/c') {
+          agent.clearHistory();
+          console.log(chalk.dim('Conversation history cleared.\n'));
+          continue;
+        }
+
+        if (cmd === '/verbose' || cmd === '/v') {
+          verbose = !verbose;
+          console.log(chalk.dim(`Verbose mode: ${verbose ? 'on' : 'off'}\n`));
+          continue;
+        }
+
+        console.log(chalk.yellow(`Unknown command: ${message}. Type /help for available commands.\n`));
+        continue;
+      }
+
+      // Ask the question using chat (maintains history)
+      console.log();
+      formatInfo('Searching...');
+
+      try {
+        const onStep = verbose ? createVerboseCallback() : undefined;
+        const response = await agent.chat(message, useCase, { onStep });
+
+        console.log();
+        console.log(response.answer);
+        console.log();
+
+        if (response.sources.length > 0) {
+          console.log(chalk.dim('Sources:'));
+          for (const source of response.sources) {
+            const section = source.section ? ` → "${source.section}"` : '';
+            console.log(chalk.dim(`  • ${source.collection}: ${source.document}${section}`));
+          }
+          console.log();
+        }
+
+        // Show conversation length in verbose mode
+        if (verbose) {
+          console.log(chalk.dim(`[${agent.getHistoryLength()} messages in history]\n`));
+        }
+      } catch (error) {
+        const errMessage = error instanceof Error ? error.message : String(error);
+        formatError(`Error: ${errMessage}`);
+        console.log();
+      }
+    }
+  } finally {
+    await client.disconnect();
+    await services.destroy();
+  }
+};
+
+const createChatCli = (command: Command) => {
+  command
+    .description('Chat with your documentation using AI')
+    .argument('[query]', 'Question to ask (starts one-shot mode)')
+    .option('-i, --interactive', 'Start interactive chat session')
+    .option('-u, --use-case <text>', 'Context for why you need this information')
+    .option('-c, --collections <names...>', 'Limit to specific collections')
+    .option('--no-global', 'Exclude global collections')
+    .option('--model <model>', 'Override LLM model from config')
+    .option('--verbose', 'Show agent reasoning (not yet implemented)')
+    .action(
+      withErrorHandling(async (query: string | undefined, options: ChatOptions) => {
+        if (options.interactive) {
+          await runInteractiveChat(options);
+        } else if (query) {
+          await runOneShotChat(query, options);
+        } else {
+          // No query and not interactive - start interactive mode
+          await runInteractiveChat(options);
+        }
+      }),
+    );
+};
+
+export { createChatCli };

package/src/cli/cli.client.ts

@@ -0,0 +1,55 @@
+import { BackendClient } from '#root/client/client.ts';
+import { DaemonManager } from '#root/daemon/daemon.manager.ts';
+
+type CliClientMode = 'auto' | 'direct' | 'daemon';
+
+type CliClientOptions = {
+  mode?: CliClientMode;
+  socketPath?: string;
+  timeout?: number;
+};
+
+// Factory function for CLI commands
+const createCliClient = async (options?: CliClientOptions): Promise<BackendClient> => {
+  const mode = options?.mode ?? 'auto';
+
+  if (mode === 'auto') {
+    // Try daemon first, fall back to direct
+    const manager = new DaemonManager({ socketPath: options?.socketPath });
+    const isDaemonRunning = await manager.isRunning();
+
+    if (isDaemonRunning) {
+      const client = new BackendClient({
+        mode: 'daemon',
+        socketPath: options?.socketPath,
+        timeout: options?.timeout,
+      });
+      await client.connect();
+      return client;
+    }
+
+    // Fall back to direct mode
+    const client = new BackendClient({ mode: 'direct' });
+    await client.connect();
+    return client;
+  }
+
+  if (mode === 'daemon') {
+    const client = new BackendClient({
+      mode: 'daemon',
+      socketPath: options?.socketPath,
+      autoStartDaemon: true,
+      timeout: options?.timeout,
+    });
+    await client.connect();
+    return client;
+  }
+
+  // Direct mode
+  const client = new BackendClient({ mode: 'direct' });
+  await client.connect();
+  return client;
+};
+
+export { createCliClient };
+export type { CliClientOptions, CliClientMode };