@specmarket/cli 0.0.5 → 0.0.6

package/src/lib/ralph-loop.test.ts

@@ -0,0 +1,404 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkdir, writeFile, readFile, rm } from 'fs/promises';
+import { join } from 'path';
+import { tmpdir } from 'os';
+import { randomUUID } from 'crypto';
+import {
+  parseTokensFromOutput,
+  ensureMetaInstructions,
+  extractTestFailures,
+  writeTestFixTasks,
+  injectSteeringMessages,
+} from './ralph-loop.js';
+import type { SteeringEntry } from './ralph-loop.js';
+import { MODEL_COST_PER_TOKEN } from '@specmarket/shared';
+import { META_INSTRUCTION_FILENAME } from './meta-instructions.js';
+
+describe('ensureMetaInstructions', () => {
+  let baseDir: string;
+  let specDir: string;
+  let runDir: string;
+
+  beforeEach(async () => {
+    baseDir = join(tmpdir(), `ralph-loop-${randomUUID()}`);
+    specDir = join(baseDir, 'spec');
+    runDir = join(baseDir, 'run');
+    await mkdir(specDir, { recursive: true });
+    await mkdir(runDir, { recursive: true });
+  });
+
+  afterEach(async () => {
+    await rm(baseDir, { recursive: true, force: true });
+  });
+
+  it('writes META_INSTRUCTION_FILENAME to runDir', async () => {
+    await writeFile(join(specDir, 'PROMPT.md'), '# Prompt');
+    await writeFile(join(specDir, 'SUCCESS_CRITERIA.md'), '# Criteria');
+    await ensureMetaInstructions(specDir, runDir);
+    const content = await readFile(join(runDir, META_INSTRUCTION_FILENAME), 'utf-8');
+    expect(content).toContain('# SpecMarket Runner Instructions');
+  });
+
+  it('detects specmarket format from spec files', async () => {
+    await writeFile(join(specDir, 'PROMPT.md'), '# Prompt');
+    await writeFile(join(specDir, 'SUCCESS_CRITERIA.md'), '# Criteria');
+    await ensureMetaInstructions(specDir, runDir);
+    const content = await readFile(join(runDir, META_INSTRUCTION_FILENAME), 'utf-8');
+    expect(content).toContain('SpecMarket (Native)');
+  });
+
+  it('detects speckit format from spec.md', async () => {
+    await writeFile(join(specDir, 'spec.md'), '# Spec');
+    await writeFile(join(specDir, 'tasks.md'), '- [ ] Task 1');
+    await ensureMetaInstructions(specDir, runDir);
+    const content = await readFile(join(runDir, META_INSTRUCTION_FILENAME), 'utf-8');
+    expect(content).toContain('Spec Kit');
+  });
+
+  it('respects formatOverride and skips detection', async () => {
+    // No spec files present — would default to custom without override
+    await ensureMetaInstructions(specDir, runDir, 'bmad');
+    const content = await readFile(join(runDir, META_INSTRUCTION_FILENAME), 'utf-8');
+    expect(content).toContain('BMAD Method');
+  });
+
+  it('overwrites existing meta-instructions file', async () => {
+    await writeFile(join(runDir, META_INSTRUCTION_FILENAME), 'old content');
+    await ensureMetaInstructions(specDir, runDir, 'ralph');
+    const content = await readFile(join(runDir, META_INSTRUCTION_FILENAME), 'utf-8');
+    expect(content).not.toBe('old content');
+    expect(content).toContain('Ralph');
+  });
+});
+
+describe('parseTokensFromOutput', () => {
+  // ---- Strategy 1: JSON parsing ----
+
+  it('returns 0 for empty output', () => {
+    expect(parseTokensFromOutput('')).toBe(0);
+    expect(parseTokensFromOutput('   ')).toBe(0);
+  });
+
+  it('extracts total_tokens from top-level JSON field', () => {
+    const json = JSON.stringify({ total_tokens: 12345 });
+    expect(parseTokensFromOutput(json)).toBe(12345);
+  });
+
+  it('extracts totalTokens (camelCase) from top-level JSON field', () => {
+    const json = JSON.stringify({ totalTokens: 9999 });
+    expect(parseTokensFromOutput(json)).toBe(9999);
+  });
+
+  it('extracts usage.total_tokens from nested JSON', () => {
+    const json = JSON.stringify({ usage: { total_tokens: 5000 } });
+    expect(parseTokensFromOutput(json)).toBe(5000);
+  });
+
+  it('sums usage.input_tokens + usage.output_tokens', () => {
+    const json = JSON.stringify({ usage: { input_tokens: 3000, output_tokens: 1000 } });
+    expect(parseTokensFromOutput(json)).toBe(4000);
+  });
+
+  it('sums usage.prompt_tokens + usage.completion_tokens', () => {
+    const json = JSON.stringify({ usage: { prompt_tokens: 2500, completion_tokens: 500 } });
+    expect(parseTokensFromOutput(json)).toBe(3000);
+  });
+
+  it('extracts result.usage.total_tokens from nested result object', () => {
+    const json = JSON.stringify({ result: { usage: { total_tokens: 7777 } } });
+    expect(parseTokensFromOutput(json)).toBe(7777);
+  });
+
+  it('uses cost_usd with default (Sonnet) pricing when model is omitted', () => {
+    const costUsd = 0.006; // 0.006 / 0.000006 = 1000 tokens
+    const json = JSON.stringify({ cost_usd: costUsd });
+    expect(parseTokensFromOutput(json)).toBe(1000);
+  });
+
+  it('uses cost_usd with default (Sonnet) pricing for unrecognised model', () => {
+    const costUsd = 0.006;
+    const json = JSON.stringify({ cost_usd: costUsd });
+    expect(parseTokensFromOutput(json, 'claude-unknown-model')).toBe(1000);
+  });
+
+  it('uses Haiku pricing when model contains "haiku" (case-insensitive)', () => {
+    // costPerToken for haiku = 0.0000005; 0.0005 / 0.0000005 = 1000 tokens
+    const costUsd = 0.0000005 * 1000;
+    const json = JSON.stringify({ cost_usd: costUsd });
+    expect(parseTokensFromOutput(json, 'claude-haiku-4-5-20251001')).toBe(1000);
+  });
+
+  it('uses Opus pricing when model contains "opus" (case-insensitive)', () => {
+    // costPerToken for opus = 0.00003; 0.03 / 0.00003 = 1000 tokens
+    const costUsd = 0.00003 * 1000;
+    const json = JSON.stringify({ cost_usd: costUsd });
+    expect(parseTokensFromOutput(json, 'claude-opus-4-6')).toBe(1000);
+  });
+
+  it('uses Sonnet pricing for model containing "sonnet"', () => {
+    // costPerToken for sonnet/default = 0.000006; 0.006 / 0.000006 = 1000 tokens
+    const costUsd = MODEL_COST_PER_TOKEN.default * 1000;
+    const json = JSON.stringify({ cost_usd: costUsd });
+    expect(parseTokensFromOutput(json, 'claude-sonnet-4-6')).toBe(1000);
+  });
+
+  it('skips cost_usd when cost is 0', () => {
+    // cost_usd=0 means no billable work — fall through to heuristic
+    const json = JSON.stringify({ cost_usd: 0 });
+    // Should NOT return 0 via cost path; falls to character heuristic
+    const result = parseTokensFromOutput(json);
+    // The JSON string is ~17 chars → heuristic gives Math.ceil(17/4) = 5
+    expect(result).toBeGreaterThan(0);
+  });
+
+  it('handles newline-delimited JSON and picks first valid structure', () => {
+    const lines = [
+      'not json',
+      JSON.stringify({ total_tokens: 42 }),
+      JSON.stringify({ total_tokens: 99 }),
+    ].join('\n');
+    expect(parseTokensFromOutput(lines)).toBe(42);
+  });
+
+  // ---- Strategy 2: Regex text patterns ----
+
+  it('extracts from "total tokens: N" text pattern', () => {
+    const output = 'Run complete. Total tokens: 3,500 used.';
+    expect(parseTokensFromOutput(output)).toBe(3500);
+  });
+
+  it('extracts from "tokens used: N" text pattern', () => {
+    const output = 'Tokens used: 1200';
+    expect(parseTokensFromOutput(output)).toBe(1200);
+  });
+
+  it('extracts from quoted "total_tokens": N pattern', () => {
+    const output = 'Stats: "total_tokens": 8000 and other info';
+    expect(parseTokensFromOutput(output)).toBe(8000);
+  });
+
+  it('extracts from input_tokens + output_tokens text pattern', () => {
+    const output = 'input_tokens: 4000, output_tokens: 1000';
+    expect(parseTokensFromOutput(output)).toBe(5000);
+  });
+
+  // ---- Strategy 3: Character heuristic fallback ----
+
+  it('falls back to character heuristic for unrecognised output', () => {
+    const output = 'x'.repeat(400); // 400 chars → ceil(400/4) = 100 tokens
+    expect(parseTokensFromOutput(output)).toBe(100);
+  });
+});
+
+describe('extractTestFailures', () => {
+  it('returns empty array for empty output', () => {
+    expect(extractTestFailures('')).toEqual([]);
+  });
+
+  it('parses Vitest/Jest file-level FAIL lines', () => {
+    const output = [
+      'FAIL src/commands/login.test.ts',
+      'FAIL src/lib/auth.test.ts',
+      ' ✓ passing test',
+    ].join('\n');
+    const failures = extractTestFailures(output);
+    expect(failures).toContain('src/commands/login.test.ts');
+    expect(failures).toContain('src/lib/auth.test.ts');
+    expect(failures).not.toContain('passing test');
+  });
+
+  it('parses Vitest/Jest individual test × failures', () => {
+    const output = [
+      '  × login should reject bad token',
+      '  × auth should expire session',
+      '  ✓ passing test',
+    ].join('\n');
+    const failures = extractTestFailures(output);
+    expect(failures).toContain('login should reject bad token');
+    expect(failures).toContain('auth should expire session');
+    expect(failures).not.toContain('passing test');
+  });
+
+  it('parses pytest FAILED lines', () => {
+    const output = [
+      'FAILED tests/test_auth.py::test_login_bad_token',
+      'FAILED tests/test_api.py::test_rate_limit',
+      'passed 5 tests',
+    ].join('\n');
+    const failures = extractTestFailures(output);
+    expect(failures).toContain('tests/test_auth.py::test_login_bad_token');
+    expect(failures).toContain('tests/test_api.py::test_rate_limit');
+  });
+
+  it('falls back to generic summary when specific names cannot be parsed', () => {
+    const output = 'Test run complete: 3 failed, 10 passed';
+    const failures = extractTestFailures(output);
+    expect(failures).toHaveLength(1);
+    expect(failures[0]).toContain('3 test(s) failed');
+    expect(failures[0]).toContain('TEST_FAILURES.md');
+  });
+
+  it('deduplicates repeated failure names', () => {
+    const output = [
+      'FAIL src/foo.test.ts',
+      'FAIL src/foo.test.ts',
+    ].join('\n');
+    const failures = extractTestFailures(output);
+    expect(failures.filter((f) => f === 'src/foo.test.ts')).toHaveLength(1);
+  });
+
+  it('caps results at 10 entries', () => {
+    const lines = Array.from({ length: 15 }, (_, i) => `FAIL src/test${i}.test.ts`);
+    const failures = extractTestFailures(lines.join('\n'));
+    expect(failures.length).toBeLessThanOrEqual(10);
+  });
+});
+
+describe('writeTestFixTasks', () => {
+  let dir: string;
+
+  beforeEach(async () => {
+    dir = join(tmpdir(), `write-test-fix-${randomUUID()}`);
+    await mkdir(dir, { recursive: true });
+  });
+
+  afterEach(async () => {
+    await rm(dir, { recursive: true, force: true });
+  });
+
+  it('writes TEST_FAILURES.md with the test output', async () => {
+    const output = 'FAIL src/foo.test.ts\n3 failed, 2 passed';
+    await writeTestFixTasks(dir, output);
+    const content = await readFile(join(dir, 'TEST_FAILURES.md'), 'utf-8');
+    expect(content).toContain('# Test Failures');
+    expect(content).toContain('FAIL src/foo.test.ts');
+  });
+
+  it('appends fix tasks to an existing TASKS.md', async () => {
+    await writeFile(join(dir, 'TASKS.md'), '# Tasks\n\n- [x] Completed task\n', 'utf-8');
+    const output = 'FAIL src/bar.test.ts\n1 failed';
+    await writeTestFixTasks(dir, output);
+    const content = await readFile(join(dir, 'TASKS.md'), 'utf-8');
+    expect(content).toContain('- [x] Completed task');
+    expect(content).toContain('- [ ] Fix: src/bar.test.ts');
+    expect(content).toContain('## Test Failures (Auto-Generated)');
+  });
+
+  it('creates TASKS.md if it does not exist', async () => {
+    const output = 'FAIL src/new.test.ts\n1 failed';
+    await writeTestFixTasks(dir, output);
+    const content = await readFile(join(dir, 'TASKS.md'), 'utf-8');
+    expect(content).toContain('- [ ] Fix: src/new.test.ts');
+  });
+
+  it('replaces previous auto-generated section on second call', async () => {
+    const first = 'FAIL src/first.test.ts\n1 failed';
+    const second = 'FAIL src/second.test.ts\n1 failed';
+    await writeTestFixTasks(dir, first);
+    await writeTestFixTasks(dir, second);
+    const content = await readFile(join(dir, 'TASKS.md'), 'utf-8');
+    // Old section replaced — only second failure should appear
+    expect(content).not.toContain('src/first.test.ts');
+    expect(content).toContain('src/second.test.ts');
+    // Section header appears exactly once
+    const sectionCount = (content.match(/## Test Failures \(Auto-Generated\)/g) ?? []).length;
+    expect(sectionCount).toBe(1);
+  });
+
+  it('does not write fix tasks when no failures can be extracted', async () => {
+    await writeFile(join(dir, 'TASKS.md'), '# Tasks\n', 'utf-8');
+    // Output has no recognisable failure patterns
+    await writeTestFixTasks(dir, 'All tests passed. 5 passed.');
+    const content = await readFile(join(dir, 'TASKS.md'), 'utf-8');
+    expect(content).not.toContain('## Test Failures (Auto-Generated)');
+    // TEST_FAILURES.md is still written (for agent reference)
+    const failuresFile = await readFile(join(dir, 'TEST_FAILURES.md'), 'utf-8');
+    expect(failuresFile).toContain('# Test Failures');
+  });
+
+  it('truncates very long test output in TEST_FAILURES.md', async () => {
+    const longOutput = 'x'.repeat(10000);
+    await writeTestFixTasks(dir, longOutput);
+    const content = await readFile(join(dir, 'TEST_FAILURES.md'), 'utf-8');
+    // Should be truncated to 8KB of the output portion
+    expect(content.length).toBeLessThan(10000 + 200); // 200 bytes for the header
+  });
+});
+
+describe('injectSteeringMessages', () => {
+  let dir: string;
+
+  beforeEach(async () => {
+    dir = join(tmpdir(), `inject-steering-${randomUUID()}`);
+    await mkdir(dir, { recursive: true });
+  });
+
+  afterEach(async () => {
+    await rm(dir, { recursive: true, force: true });
+  });
+
+  it('appends a steering section to an existing meta-instructions file', async () => {
+    await writeFile(join(dir, META_INSTRUCTION_FILENAME), '# SpecMarket Runner Instructions\n\nOriginal content.');
+    const log: SteeringEntry[] = [];
+    await injectSteeringMessages(dir, ['Focus on the auth module'], log);
+
+    const content = await readFile(join(dir, META_INSTRUCTION_FILENAME), 'utf-8');
+    expect(content).toContain('# SpecMarket Runner Instructions');
+    expect(content).toContain('Original content.');
+    expect(content).toContain('## Steering Input');
+    expect(content).toContain('> Focus on the auth module');
+  });
+
+  it('creates the meta-instructions file when it does not exist', async () => {
+    const log: SteeringEntry[] = [];
+    await injectSteeringMessages(dir, ['Fix the failing tests'], log);
+
+    const content = await readFile(join(dir, META_INSTRUCTION_FILENAME), 'utf-8');
+    expect(content).toContain('## Steering Input');
+    expect(content).toContain('> Fix the failing tests');
+  });
+
+  it('pushes SteeringEntry objects into the log', async () => {
+    const log: SteeringEntry[] = [];
+    await injectSteeringMessages(dir, ['First message', 'Second message'], log);
+
+    expect(log).toHaveLength(2);
+    expect(log[0].content).toBe('First message');
+    expect(log[1].content).toBe('Second message');
+    expect(log[0].timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T/); // ISO-8601 prefix
+    expect(log[0].timestamp).toBe(log[1].timestamp); // Same batch → same timestamp
+  });
+
+  it('accumulates entries from multiple calls', async () => {
+    const log: SteeringEntry[] = [];
+    await injectSteeringMessages(dir, ['First steering'], log);
+    await injectSteeringMessages(dir, ['Second steering'], log);
+
+    expect(log).toHaveLength(2);
+    const content = await readFile(join(dir, META_INSTRUCTION_FILENAME), 'utf-8');
+    // Both sections should be present
+    const sectionCount = (content.match(/## Steering Input/g) ?? []).length;
+    expect(sectionCount).toBe(2);
+    expect(content).toContain('> First steering');
+    expect(content).toContain('> Second steering');
+  });
+
+  it('does nothing when messages array is empty', async () => {
+    await writeFile(join(dir, META_INSTRUCTION_FILENAME), '# Original');
+    const log: SteeringEntry[] = [];
+    await injectSteeringMessages(dir, [], log);
+
+    expect(log).toHaveLength(0);
+    const content = await readFile(join(dir, META_INSTRUCTION_FILENAME), 'utf-8');
+    expect(content).toBe('# Original'); // File unchanged
+  });
+
+  it('includes a timestamp header in the injected section', async () => {
+    const log: SteeringEntry[] = [];
+    await injectSteeringMessages(dir, ['Check the database layer'], log);
+
+    const content = await readFile(join(dir, META_INSTRUCTION_FILENAME), 'utf-8');
+    // Section header includes the ISO timestamp
+    expect(content).toMatch(/## Steering Input \(injected at \d{4}-\d{2}-\d{2}T/);
+  });
+});