all-hands-cli 0.1.0

package/.allhands/harness/src/__tests__/harness/assertions.ts

@@ -0,0 +1,318 @@
+/**
+ * Custom Assertion Helpers for Harness Testing
+ *
+ * Provides domain-specific assertions for CLI and hook testing.
+ */
+
+import { expect } from 'vitest';
+import type { RunResult } from './cli-runner.js';
+import type { HookResult } from './hook-runner.js';
+import type { TestFixture } from './fixture.js';
+import { readFileSync, existsSync } from 'fs';
+import { join } from 'path';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// CLI Result Assertions
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Assert that a CLI command succeeded.
+ */
+export function assertSuccess(result: RunResult, message?: string): void {
+  expect(result.success, message ?? `Expected success but got exit code ${result.exitCode}: ${result.stderr}`).toBe(true);
+}
+
+/**
+ * Assert that a CLI command failed.
+ */
+export function assertFailure(result: RunResult, message?: string): void {
+  expect(result.success, message ?? `Expected failure but command succeeded`).toBe(false);
+}
+
+/**
+ * Assert stdout contains expected text.
+ */
+export function assertStdoutContains(result: RunResult, expected: string, message?: string): void {
+  expect(
+    result.stdout.includes(expected),
+    message ?? `Expected stdout to contain "${expected}"\nActual stdout:\n${result.stdout}`
+  ).toBe(true);
+}
+
+/**
+ * Assert stderr contains expected text.
+ */
+export function assertStderrContains(result: RunResult, expected: string, message?: string): void {
+  expect(
+    result.stderr.includes(expected),
+    message ?? `Expected stderr to contain "${expected}"\nActual stderr:\n${result.stderr}`
+  ).toBe(true);
+}
+
+/**
+ * Assert stdout matches a regex.
+ */
+export function assertStdoutMatches(result: RunResult, pattern: RegExp, message?: string): void {
+  expect(
+    pattern.test(result.stdout),
+    message ?? `Expected stdout to match ${pattern}\nActual stdout:\n${result.stdout}`
+  ).toBe(true);
+}
+
+/**
+ * Assert JSON output has expected structure.
+ */
+export function assertJsonOutput<T>(
+  result: RunResult,
+  validator: (json: T) => boolean,
+  message?: string
+): void {
+  expect(result.json, 'Expected JSON output but none was parsed').toBeDefined();
+  expect(
+    validator(result.json as T),
+    message ?? `JSON output validation failed: ${JSON.stringify(result.json, null, 2)}`
+  ).toBe(true);
+}
+
+/**
+ * Assert command completed within time limit.
+ */
+export function assertTimedWithin(result: RunResult, maxMs: number, message?: string): void {
+  expect(
+    result.duration <= maxMs,
+    message ?? `Expected completion within ${maxMs}ms but took ${result.duration}ms`
+  ).toBe(true);
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Hook Result Assertions
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Assert that a hook allowed the tool.
+ */
+export function assertHookAllowed(result: HookResult, message?: string): void {
+  expect(
+    result.allowed,
+    message ?? `Expected hook to allow tool but it was ${result.denied ? 'denied' : 'blocked'}: ${result.denialReason ?? result.stderr}`
+  ).toBe(true);
+}
+
+/**
+ * Assert that a hook denied the tool.
+ */
+export function assertHookDenied(result: HookResult, message?: string): void {
+  expect(
+    result.denied,
+    message ?? `Expected hook to deny tool but it was allowed`
+  ).toBe(true);
+}
+
+/**
+ * Assert that a hook blocked the tool (PostToolUse).
+ */
+export function assertHookBlocked(result: HookResult, message?: string): void {
+  expect(
+    result.blocked,
+    message ?? `Expected hook to block tool but it was allowed`
+  ).toBe(true);
+}
+
+/**
+ * Assert that a hook injected context.
+ */
+export function assertHookInjectedContext(result: HookResult, message?: string): void {
+  expect(
+    result.systemMessage,
+    message ?? `Expected hook to inject context (systemMessage) but none was found`
+  ).toBeDefined();
+  expect(result.systemMessage!.length).toBeGreaterThan(0);
+}
+
+/**
+ * Assert that hook context contains expected text.
+ */
+export function assertHookContextContains(result: HookResult, expected: string, message?: string): void {
+  assertHookInjectedContext(result);
+  expect(
+    result.systemMessage!.includes(expected),
+    message ?? `Expected systemMessage to contain "${expected}"\nActual:\n${result.systemMessage!.substring(0, 500)}`
+  ).toBe(true);
+}
+
+/**
+ * Assert denial reason contains expected text.
+ */
+export function assertDenialReasonContains(result: HookResult, expected: string, message?: string): void {
+  assertHookDenied(result);
+  expect(
+    result.denialReason?.includes(expected),
+    message ?? `Expected denial reason to contain "${expected}"\nActual: ${result.denialReason}`
+  ).toBe(true);
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Fixture Assertions
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Assert that a file exists in the fixture.
+ */
+export function assertFileExists(fixture: TestFixture, relativePath: string, message?: string): void {
+  const fullPath = join(fixture.root, relativePath);
+  expect(
+    existsSync(fullPath),
+    message ?? `Expected file to exist: ${relativePath}`
+  ).toBe(true);
+}
+
+/**
+ * Assert that a file does not exist in the fixture.
+ */
+export function assertFileNotExists(fixture: TestFixture, relativePath: string, message?: string): void {
+  const fullPath = join(fixture.root, relativePath);
+  expect(
+    existsSync(fullPath),
+    message ?? `Expected file to not exist: ${relativePath}`
+  ).toBe(false);
+}
+
+/**
+ * Assert file content contains expected text.
+ */
+export function assertFileContains(
+  fixture: TestFixture,
+  relativePath: string,
+  expected: string,
+  message?: string
+): void {
+  assertFileExists(fixture, relativePath);
+  const content = readFileSync(join(fixture.root, relativePath), 'utf-8');
+  expect(
+    content.includes(expected),
+    message ?? `Expected ${relativePath} to contain "${expected}"\nActual:\n${content.substring(0, 500)}`
+  ).toBe(true);
+}
+
+/**
+ * Assert file content matches regex.
+ */
+export function assertFileMatches(
+  fixture: TestFixture,
+  relativePath: string,
+  pattern: RegExp,
+  message?: string
+): void {
+  assertFileExists(fixture, relativePath);
+  const content = readFileSync(join(fixture.root, relativePath), 'utf-8');
+  expect(
+    pattern.test(content),
+    message ?? `Expected ${relativePath} to match ${pattern}`
+  ).toBe(true);
+}
+
+/**
+ * Assert file has valid YAML frontmatter.
+ */
+export function assertValidFrontmatter(
+  fixture: TestFixture,
+  relativePath: string,
+  requiredFields: string[] = []
+): void {
+  assertFileExists(fixture, relativePath);
+  const content = readFileSync(join(fixture.root, relativePath), 'utf-8');
+
+  // Check frontmatter exists
+  expect(content.startsWith('---'), `Expected ${relativePath} to start with frontmatter`).toBe(true);
+
+  const endIndex = content.indexOf('---', 3);
+  expect(endIndex > 3, `Expected ${relativePath} to have closing frontmatter delimiter`).toBe(true);
+
+  const frontmatter = content.substring(3, endIndex).trim();
+
+  // Check required fields
+  for (const field of requiredFields) {
+    expect(
+      frontmatter.includes(`${field}:`),
+      `Expected frontmatter to contain "${field}" field`
+    ).toBe(true);
+  }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Git Assertions
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Assert file is tracked by git.
+ */
+export async function assertGitTracked(
+  fixture: TestFixture,
+  relativePath: string,
+  message?: string
+): Promise<void> {
+  const { execSync } = await import('child_process');
+  try {
+    execSync(`git ls-files --error-unmatch "${relativePath}"`, {
+      cwd: fixture.root,
+      stdio: 'pipe',
+    });
+  } catch {
+    expect.fail(message ?? `Expected ${relativePath} to be tracked by git`);
+  }
+}
+
+/**
+ * Assert file has uncommitted changes.
+ */
+export async function assertGitDirty(
+  fixture: TestFixture,
+  relativePath: string,
+  message?: string
+): Promise<void> {
+  const { execSync } = await import('child_process');
+  const status = execSync(`git status --porcelain "${relativePath}"`, {
+    cwd: fixture.root,
+    encoding: 'utf-8',
+  });
+  expect(
+    status.trim().length > 0,
+    message ?? `Expected ${relativePath} to have uncommitted changes`
+  ).toBe(true);
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Composite Assertions
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Assert a complete command workflow succeeded.
+ */
+export function assertWorkflowSuccess(
+  results: { name: string; result: RunResult }[],
+  message?: string
+): void {
+  const failures = results.filter((r) => !r.result.success);
+  if (failures.length > 0) {
+    const details = failures
+      .map((f) => `  ${f.name}: exit ${f.result.exitCode}, stderr: ${f.result.stderr}`)
+      .join('\n');
+    expect.fail(message ?? `Workflow had ${failures.length} failures:\n${details}`);
+  }
+}
+
+/**
+ * Assert all hook contracts passed.
+ */
+export function assertContractsPassed(
+  results: { contract: { name: string }; passed: boolean; failures: string[] }[],
+  message?: string
+): void {
+  const failed = results.filter((r) => !r.passed);
+  if (failed.length > 0) {
+    const details = failed
+      .map((f) => `  ${f.contract.name}:\n    ${f.failures.join('\n    ')}`)
+      .join('\n');
+    expect.fail(message ?? `${failed.length} contracts failed:\n${details}`);
+  }
+}

package/.allhands/harness/src/__tests__/harness/cli-runner.ts

@@ -0,0 +1,359 @@
+/**
+ * CLI Runner - Execute ah commands and capture output
+ *
+ * Provides utilities for running the CLI headlessly and asserting on results.
+ */
+
+import { spawn, SpawnOptions, ChildProcess } from 'child_process';
+import { join } from 'path';
+import type { TestFixture } from './fixture.js';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Types
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface RunOptions {
+  /** Working directory for the command */
+  cwd?: string;
+  /** Environment variables to set */
+  env?: Record<string, string>;
+  /** Timeout in milliseconds (default: 30000) */
+  timeout?: number;
+  /** Input to send to stdin */
+  stdin?: string;
+  /** Whether to expect JSON output */
+  expectJson?: boolean;
+}
+
+export interface RunResult {
+  /** Exit code (0 = success) */
+  exitCode: number;
+  /** Stdout as string */
+  stdout: string;
+  /** Stderr as string */
+  stderr: string;
+  /** Combined stdout + stderr in order received */
+  combined: string;
+  /** Whether the command succeeded (exit code 0) */
+  success: boolean;
+  /** Parsed JSON if expectJson was true and output is valid JSON */
+  json?: unknown;
+  /** Duration in milliseconds */
+  duration: number;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// CLI Paths
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Get the path to the ah CLI entry point.
+ */
+function getCliPath(): string {
+  // From src/__tests__/harness/ go up to harness root
+  return join(__dirname, '..', '..', 'cli.ts');
+}
+
+/**
+ * Get the tsx executable path.
+ */
+function getTsxPath(): string {
+  // Use npx tsx to run TypeScript directly
+  return 'npx';
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Core Runner
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Run an ah CLI command and capture output.
+ */
+export async function runCli(args: string[], options: RunOptions = {}): Promise<RunResult> {
+  const {
+    cwd = process.cwd(),
+    env = {},
+    timeout = 30000,
+    stdin,
+    expectJson = false,
+  } = options;
+
+  const startTime = Date.now();
+  const cliPath = getCliPath();
+
+  return new Promise((resolve) => {
+    const spawnEnv: Record<string, string> = {
+      ...process.env,
+      ...env,
+      // Disable color output for consistent parsing
+      NO_COLOR: '1',
+      FORCE_COLOR: '0',
+    } as Record<string, string>;
+
+    const spawnOptions: SpawnOptions = {
+      cwd,
+      env: spawnEnv,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    };
+
+    // Use tsx to run the CLI TypeScript directly
+    const child: ChildProcess = spawn('npx', ['tsx', cliPath, ...args], spawnOptions);
+
+    let stdout = '';
+    let stderr = '';
+    let combined = '';
+    let timedOut = false;
+
+    const timeoutId = setTimeout(() => {
+      timedOut = true;
+      child.kill('SIGTERM');
+    }, timeout);
+
+    child.stdout?.on('data', (data: Buffer) => {
+      const text = data.toString();
+      stdout += text;
+      combined += text;
+    });
+
+    child.stderr?.on('data', (data: Buffer) => {
+      const text = data.toString();
+      stderr += text;
+      combined += text;
+    });
+
+    if (stdin) {
+      child.stdin?.write(stdin);
+      child.stdin?.end();
+    } else {
+      child.stdin?.end();
+    }
+
+    child.on('close', (code) => {
+      clearTimeout(timeoutId);
+      const duration = Date.now() - startTime;
+
+      const result: RunResult = {
+        exitCode: timedOut ? -1 : (code ?? 1),
+        stdout: stdout.trim(),
+        stderr: stderr.trim(),
+        combined: combined.trim(),
+        success: !timedOut && code === 0,
+        duration,
+      };
+
+      // Try to parse JSON if requested
+      if (expectJson && stdout.trim()) {
+        try {
+          result.json = JSON.parse(stdout.trim());
+        } catch {
+          // Leave json undefined if parsing fails
+        }
+      }
+
+      resolve(result);
+    });
+
+    child.on('error', (err) => {
+      clearTimeout(timeoutId);
+      const duration = Date.now() - startTime;
+
+      resolve({
+        exitCode: -1,
+        stdout: '',
+        stderr: err.message,
+        combined: err.message,
+        success: false,
+        duration,
+      });
+    });
+  });
+}
+
+/**
+ * Run a command in the context of a test fixture.
+ */
+export async function runInFixture(
+  fixture: TestFixture,
+  args: string[],
+  options: Omit<RunOptions, 'cwd'> = {}
+): Promise<RunResult> {
+  return runCli(args, {
+    ...options,
+    cwd: fixture.root,
+    env: {
+      ...fixture.env,
+      ...options.env,
+    },
+  });
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Convenience Wrappers
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Run a knowledge search command.
+ */
+export async function runKnowledgeSearch(
+  query: string,
+  fixture: TestFixture,
+  options: { path?: string; k?: number } = {}
+): Promise<RunResult> {
+  const args = ['knowledge', 'search', query];
+
+  if (options.path) {
+    args.push('--path', options.path);
+  }
+  if (options.k) {
+    args.push('--k', options.k.toString());
+  }
+
+  return runInFixture(fixture, args);
+}
+
+/**
+ * Run a validation command.
+ * Uses `ah validate file <path>` syntax.
+ */
+export async function runValidate(
+  filePath: string,
+  fixture: TestFixture
+): Promise<RunResult> {
+  // If path is relative, make it absolute relative to fixture
+  const { isAbsolute } = await import('path');
+  const absPath = isAbsolute(filePath) ? filePath : join(fixture.root, filePath);
+  return runInFixture(fixture, ['validate', 'file', absPath], { expectJson: true });
+}
+
+/**
+ * Run a spawn codesearch command.
+ */
+export async function runCodeSearch(
+  query: string,
+  fixture: TestFixture,
+  options: { budget?: number } = {}
+): Promise<RunResult> {
+  const args = ['spawn', 'codesearch', query];
+
+  if (options.budget) {
+    args.push('--budget', options.budget.toString());
+  }
+
+  return runInFixture(fixture, args, { timeout: 60000 });
+}
+
+/**
+ * Run a tools list command.
+ */
+export async function runToolsList(fixture: TestFixture): Promise<RunResult> {
+  return runInFixture(fixture, ['tools', 'list'], { expectJson: true });
+}
+
+/**
+ * Run a specs list command.
+ */
+export async function runSpecsList(fixture: TestFixture): Promise<RunResult> {
+  return runInFixture(fixture, ['specs', 'list']);
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Batch Runner
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface BatchCommand {
+  name: string;
+  args: string[];
+  options?: RunOptions;
+  /** Expected outcome for assertions */
+  expect?: {
+    success?: boolean;
+    exitCode?: number;
+    stdoutContains?: string[];
+    stderrContains?: string[];
+  };
+}
+
+export interface BatchResult {
+  command: BatchCommand;
+  result: RunResult;
+  passed: boolean;
+  failures: string[];
+}
+
+/**
+ * Run multiple commands in sequence and collect results.
+ */
+export async function runBatch(
+  commands: BatchCommand[],
+  fixture: TestFixture
+): Promise<BatchResult[]> {
+  const results: BatchResult[] = [];
+
+  for (const command of commands) {
+    const result = await runInFixture(fixture, command.args, command.options);
+    const failures: string[] = [];
+
+    if (command.expect) {
+      const { expect: exp } = command;
+
+      if (exp.success !== undefined && result.success !== exp.success) {
+        failures.push(`Expected success=${exp.success}, got ${result.success}`);
+      }
+
+      if (exp.exitCode !== undefined && result.exitCode !== exp.exitCode) {
+        failures.push(`Expected exitCode=${exp.exitCode}, got ${result.exitCode}`);
+      }
+
+      if (exp.stdoutContains) {
+        for (const expected of exp.stdoutContains) {
+          if (!result.stdout.includes(expected)) {
+            failures.push(`Expected stdout to contain "${expected}"`);
+          }
+        }
+      }
+
+      if (exp.stderrContains) {
+        for (const expected of exp.stderrContains) {
+          if (!result.stderr.includes(expected)) {
+            failures.push(`Expected stderr to contain "${expected}"`);
+          }
+        }
+      }
+    }
+
+    results.push({
+      command,
+      result,
+      passed: failures.length === 0,
+      failures,
+    });
+  }
+
+  return results;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Debug Helpers
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Print a run result for debugging.
+ */
+export function debugResult(result: RunResult, label?: string): void {
+  console.log('\n' + '='.repeat(60));
+  if (label) {
+    console.log(`DEBUG: ${label}`);
+    console.log('-'.repeat(60));
+  }
+  console.log(`Exit Code: ${result.exitCode} (${result.success ? 'success' : 'failure'})`);
+  console.log(`Duration: ${result.duration}ms`);
+  console.log('\n--- STDOUT ---');
+  console.log(result.stdout || '(empty)');
+  console.log('\n--- STDERR ---');
+  console.log(result.stderr || '(empty)');
+  if (result.json) {
+    console.log('\n--- PARSED JSON ---');
+    console.log(JSON.stringify(result.json, null, 2));
+  }
+  console.log('='.repeat(60) + '\n');
+}