cognitive-core 0.1.2 → 0.2.1

package/tests/search/verification-runner.test.ts

@@ -0,0 +1,357 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+import {
+  VerificationRunner,
+  createVerificationRunner,
+  TestRunners,
+  type CommandVerificationConfig,
+} from '../../src/search/verification-runner.js';
+import { createTrajectory } from '../../src/types/trajectory.js';
+import { createTask } from '../../src/types/task.js';
+import { createStep } from '../../src/types/step.js';
+import { successOutcome } from '../../src/types/outcome.js';
+import type { Trajectory } from '../../src/types/trajectory.js';
+import type { Task } from '../../src/types/task.js';
+
+function makeTask(): Task {
+  return createTask({
+    domain: 'test-domain',
+    description: 'test task',
+  });
+}
+
+function makeTrajectory(): Trajectory {
+  return createTrajectory({
+    task: makeTask(),
+    steps: [createStep({ action: 'echo hello', observation: 'hello' })],
+    outcome: successOutcome('hello world'),
+    agentId: 'agent',
+  });
+}
+
+describe('VerificationRunner', () => {
+  let runner: VerificationRunner;
+
+  beforeEach(() => {
+    runner = createVerificationRunner({ cwd: '/tmp', timeout: 5000 });
+  });
+
+  describe('runCommand', () => {
+    it('should execute a simple command and capture stdout', async () => {
+      const result = await runner.runCommand('echo hello', {});
+      expect(result.stdout.trim()).toBe('hello');
+      expect(result.exitCode).toBe(0);
+      expect(result.timedOut).toBe(false);
+    });
+
+    it('should capture stderr', async () => {
+      const result = await runner.runCommand('echo error >&2', {});
+      expect(result.stderr.trim()).toBe('error');
+    });
+
+    it('should return non-zero exit code for failing commands', async () => {
+      const result = await runner.runCommand('exit 42', {});
+      expect(result.exitCode).toBe(42);
+    });
+
+    it('should handle command timeout', { timeout: 20000 }, async () => {
+      // Use a bash loop that responds to SIGTERM better than plain sleep
+      const result = await runner.runCommand('while true; do sleep 0.1; done', { timeout: 500 });
+      expect(result.timedOut).toBe(true);
+    });
+
+    it('should handle command not found', async () => {
+      const result = await runner.runCommand('nonexistent_command_xyz_12345', {});
+      expect(result.exitCode).not.toBe(0);
+    });
+
+    it('should pass environment variables', async () => {
+      const result = await runner.runCommand('echo $MY_TEST_VAR', {
+        env: { MY_TEST_VAR: 'hello123' },
+      });
+      expect(result.stdout.trim()).toBe('hello123');
+    });
+
+    it('should use specified working directory', async () => {
+      const result = await runner.runCommand('pwd', { cwd: '/tmp' });
+      expect(result.stdout.trim()).toMatch(/\/tmp/);
+    });
+  });
+
+  describe('runCommandSync', () => {
+    it('should execute command synchronously', () => {
+      const result = runner.runCommandSync('echo sync-test');
+      expect(result.stdout.trim()).toBe('sync-test');
+      expect(result.exitCode).toBe(0);
+    });
+
+    it('should handle sync command failure', () => {
+      const result = runner.runCommandSync('exit 1');
+      expect(result.exitCode).toBe(1);
+    });
+
+    it('should detect sync command failure on timeout', () => {
+      const result = runner.runCommandSync('sleep 30', { timeout: 500 });
+      // execSync throws on timeout; killed + SIGTERM detection may vary by platform
+      // The important thing is that the command does not succeed
+      expect(result.exitCode).not.toBe(0);
+    });
+  });
+
+  describe('createVerifier', () => {
+    it('should create verifier that passes on exit code 0', async () => {
+      const config: CommandVerificationConfig = {
+        command: 'echo "all tests pass"',
+        timeout: 5000,
+      };
+
+      const verifier = runner.createVerifier(config);
+      const result = await verifier(makeTrajectory(), makeTask());
+
+      expect(result.passed).toBe(true);
+      expect(result.confidence).toBe(0.7); // No pattern/extractor → 0.7
+    });
+
+    it('should create verifier that fails on non-zero exit', async () => {
+      const config: CommandVerificationConfig = {
+        command: 'exit 1',
+        timeout: 5000,
+      };
+
+      const verifier = runner.createVerifier(config);
+      const result = await verifier(makeTrajectory(), makeTask());
+
+      expect(result.passed).toBe(false);
+      expect(result.issues!.length).toBeGreaterThan(0);
+    });
+
+    it('should check success pattern when provided', async () => {
+      const config: CommandVerificationConfig = {
+        command: 'echo "Tests: 5 passed, 0 failed"',
+        successPattern: /\d+ passed, 0 failed/,
+        timeout: 5000,
+      };
+
+      const verifier = runner.createVerifier(config);
+      const result = await verifier(makeTrajectory(), makeTask());
+
+      expect(result.passed).toBe(true);
+      expect(result.confidence).toBe(0.9); // Has successPattern → 0.9
+    });
+
+    it('should check failure pattern when provided', async () => {
+      const config: CommandVerificationConfig = {
+        command: 'echo "FAIL test_login"',
+        failurePattern: /FAIL/,
+        successOnZeroExit: false,
+        timeout: 5000,
+      };
+
+      const verifier = runner.createVerifier(config);
+      const result = await verifier(makeTrajectory(), makeTask());
+
+      expect(result.passed).toBe(false);
+    });
+
+    it('should use custom issue extractor', async () => {
+      const config: CommandVerificationConfig = {
+        command: 'echo "ERROR: missing return"',
+        timeout: 5000,
+        successOnZeroExit: false,
+        issueExtractor: (output) => {
+          const match = output.match(/ERROR:\s*(.+)/);
+          return match
+            ? [{ type: 'error' as const, description: match[1], severity: 'critical' as const }]
+            : [];
+        },
+      };
+
+      const verifier = runner.createVerifier(config);
+      const result = await verifier(makeTrajectory(), makeTask());
+
+      expect(result.confidence).toBe(0.9); // Has issueExtractor
+    });
+
+    it('should handle timeout in verifier', { timeout: 20000 }, async () => {
+      const config: CommandVerificationConfig = {
+        command: 'while true; do sleep 0.1; done',
+        timeout: 500,
+      };
+
+      const verifier = runner.createVerifier(config);
+      const result = await verifier(makeTrajectory(), makeTask());
+
+      expect(result.passed).toBe(false);
+      expect(result.confidence).toBe(0.5); // Timeout → 0.5
+      expect(result.details).toContain('timed out');
+    });
+
+    it('should interpolate {{taskId}} and {{trajectoryId}} in command', async () => {
+      const config: CommandVerificationConfig = {
+        command: 'echo "task={{taskId}} traj={{trajectoryId}}"',
+        timeout: 5000,
+      };
+
+      const trajectory = makeTrajectory();
+      const task = makeTask();
+      const verifier = runner.createVerifier(config);
+      const result = await verifier(trajectory, task);
+
+      expect(result.details).toContain(`task=${task.id}`);
+      expect(result.details).toContain(`traj=${trajectory.id}`);
+    });
+  });
+
+  describe('createCompositeVerifier', () => {
+    it('should pass when all verifiers pass', async () => {
+      const composite = runner.createCompositeVerifier([
+        { command: 'echo pass1', timeout: 5000 },
+        { command: 'echo pass2', timeout: 5000 },
+      ]);
+
+      const result = await composite(makeTrajectory(), makeTask());
+      expect(result.passed).toBe(true);
+    });
+
+    it('should fail when any verifier fails', async () => {
+      const composite = runner.createCompositeVerifier([
+        { command: 'echo pass', timeout: 5000 },
+        { command: 'exit 1', timeout: 5000 },
+      ]);
+
+      const result = await composite(makeTrajectory(), makeTask());
+      expect(result.passed).toBe(false);
+    });
+
+    it('should aggregate issues from all verifiers', async () => {
+      const composite = runner.createCompositeVerifier([
+        { command: 'exit 1', timeout: 5000 },
+        { command: 'exit 2', timeout: 5000 },
+      ]);
+
+      const result = await composite(makeTrajectory(), makeTask());
+      expect(result.issues!.length).toBeGreaterThanOrEqual(1);
+    });
+
+    it('should fail fast on critical issues', async () => {
+      let secondRan = false;
+      const configs: CommandVerificationConfig[] = [
+        {
+          command: 'exit 1',
+          timeout: 5000,
+          issueExtractor: () => [
+            { type: 'error', description: 'critical failure', severity: 'critical' },
+          ],
+        },
+        {
+          command: 'echo should-not-run',
+          timeout: 5000,
+          issueExtractor: () => {
+            secondRan = true;
+            return [];
+          },
+        },
+      ];
+
+      const composite = runner.createCompositeVerifier(configs);
+      await composite(makeTrajectory(), makeTask());
+
+      // Second verifier should not have run due to fail-fast
+      expect(secondRan).toBe(false);
+    });
+
+    it('should average confidence across verifiers', async () => {
+      const composite = runner.createCompositeVerifier([
+        { command: 'echo pass', timeout: 5000 }, // confidence 0.7
+        { command: 'echo pass', timeout: 5000 }, // confidence 0.7
+      ]);
+
+      const result = await composite(makeTrajectory(), makeTask());
+      expect(result.confidence).toBeCloseTo(0.7, 1);
+    });
+  });
+
+  describe('TestRunners presets', () => {
+    it('should create vitest configuration', () => {
+      const config = TestRunners.vitest('tests/foo.test.ts');
+      expect(config.command).toContain('vitest');
+      expect(config.command).toContain('tests/foo.test.ts');
+      expect(config.timeout).toBe(60000);
+      expect(config.issueExtractor).toBeDefined();
+    });
+
+    it('should create jest configuration', () => {
+      const config = TestRunners.jest();
+      expect(config.command).toContain('jest');
+      expect(config.issueExtractor).toBeDefined();
+    });
+
+    it('should create typescript configuration', () => {
+      const config = TestRunners.typescript();
+      expect(config.command).toContain('tsc --noEmit');
+      expect(config.issueExtractor).toBeDefined();
+    });
+
+    it('should create eslint configuration', () => {
+      const config = TestRunners.eslint('src/');
+      expect(config.command).toContain('eslint');
+      expect(config.command).toContain('src/');
+    });
+
+    it('should create pytest configuration', () => {
+      const config = TestRunners.pytest();
+      expect(config.command).toContain('pytest');
+      expect(config.failurePattern).toBeDefined();
+    });
+
+    it('should create go test configuration', () => {
+      const config = TestRunners.goTest();
+      expect(config.command).toContain('go test');
+    });
+
+    it('should create cargo test configuration', () => {
+      const config = TestRunners.cargoTest();
+      expect(config.command).toContain('cargo test');
+      expect(config.timeout).toBe(120000);
+    });
+  });
+
+  describe('default issue extraction', () => {
+    it('should extract error messages from output', async () => {
+      const config: CommandVerificationConfig = {
+        command: 'echo "Error: cannot read file" && exit 1',
+        timeout: 5000,
+      };
+
+      const verifier = runner.createVerifier(config);
+      const result = await verifier(makeTrajectory(), makeTask());
+
+      expect(result.issues!.length).toBeGreaterThan(0);
+      expect(result.issues![0].description).toContain('cannot read file');
+    });
+
+    it('should extract fail messages from output', async () => {
+      const config: CommandVerificationConfig = {
+        command: 'echo "Fail: test_login" && exit 1',
+        timeout: 5000,
+      };
+
+      const verifier = runner.createVerifier(config);
+      const result = await verifier(makeTrajectory(), makeTask());
+
+      expect(result.issues!.length).toBeGreaterThan(0);
+    });
+
+    it('should generate generic issue when no pattern matches', async () => {
+      const config: CommandVerificationConfig = {
+        command: 'exit 99',
+        timeout: 5000,
+      };
+
+      const verifier = runner.createVerifier(config);
+      const result = await verifier(makeTrajectory(), makeTask());
+
+      expect(result.issues!.length).toBeGreaterThan(0);
+      expect(result.issues![0].description).toContain('exit code 99');
+    });
+  });
+});

package/tests/session-bank/fixtures/sessionlog-root-metadata.json

@@ -0,0 +1,16 @@
+{
+  "cliVersion": "opentasks-sessionlog",
+  "checkpointID": "c4d5e6f7a8b9",
+  "strategy": "manual-commit",
+  "branch": "main",
+  "checkpointsCount": 3,
+  "filesTouched": ["src/api.ts", "tests/api.test.ts"],
+  "sessions": [
+    {
+      "metadata": "1/metadata.json",
+      "transcript": "1/full.jsonl",
+      "context": "1/context.md",
+      "prompt": "1/prompt.txt"
+    }
+  ]
+}

package/tests/session-bank/fixtures/sessionlog-session/full.jsonl

@@ -0,0 +1,6 @@
+{"type":"user","uuid":"msg-sl-001","message":{"content":"Add input validation to the API endpoint using Zod"}}
+{"type":"assistant","uuid":"msg-sl-002","message":{"content":[{"type":"text","text":"I'll examine the API endpoint first."},{"type":"tool_use","id":"tool-sl-001","name":"Read","input":{"file_path":"/src/api.ts"}}]}}
+{"type":"user","uuid":"msg-sl-003","message":{"content":[{"type":"tool_result","tool_use_id":"tool-sl-001","content":"export function handler(req, res) {\n  const body = req.body;\n  // TODO: validate\n  res.json({ ok: true });\n}"}]}}
+{"type":"assistant","uuid":"msg-sl-004","message":{"content":[{"type":"text","text":"I'll add Zod validation."},{"type":"tool_use","id":"tool-sl-002","name":"Edit","input":{"file_path":"/src/api.ts","old_string":"const body = req.body;","new_string":"const body = schema.parse(req.body);"}}]}}
+{"type":"user","uuid":"msg-sl-005","message":{"content":[{"type":"tool_result","tool_use_id":"tool-sl-002","content":"File edited successfully"}]}}
+{"type":"assistant","uuid":"msg-sl-006","message":{"content":[{"type":"text","text":"Validation added. The endpoint now validates input using the Zod schema before processing."}]}}

package/tests/session-bank/fixtures/sessionlog-session/metadata.json

@@ -0,0 +1,55 @@
+{
+  "cliVersion": "opentasks-sessionlog",
+  "checkpointID": "c4d5e6f7a8b9",
+  "sessionID": "2026-02-15-aabbccdd-1122-3344-5566-778899aabbcc",
+  "strategy": "manual-commit",
+  "createdAt": "2026-02-15T09:00:00Z",
+  "branch": "main",
+  "filesTouched": ["src/api.ts", "tests/api.test.ts"],
+  "agent": "Claude Code",
+  "turnID": "turn-001",
+  "isTask": false,
+  "checkpointTranscriptStart": 0,
+  "tokenUsage": {
+    "inputTokens": 20000,
+    "cacheCreationTokens": 3000,
+    "cacheReadTokens": 6000,
+    "outputTokens": 4000,
+    "apiCallCount": 10,
+    "subagentTokens": {
+      "inputTokens": 2000,
+      "cacheCreationTokens": 0,
+      "cacheReadTokens": 0,
+      "outputTokens": 800,
+      "apiCallCount": 3
+    }
+  },
+  "summary": {
+    "intent": "Add validation to the API endpoint",
+    "outcome": "Added input validation with Zod schema",
+    "learnings": {
+      "repo": ["API routes use Zod for validation"],
+      "code": [
+        {
+          "path": "src/api.ts",
+          "line": 15,
+          "finding": "Zod schema must be defined before route handler"
+        }
+      ],
+      "workflow": ["Run API tests with --reporter=verbose flag"]
+    },
+    "friction": ["Had to look up Zod coercion syntax"],
+    "openItems": ["Add validation for the PUT endpoint too"]
+  },
+  "initialAttribution": {
+    "calculatedAt": "2026-02-15T09:30:00Z",
+    "agentLines": 30,
+    "humanAdded": 3,
+    "humanModified": 1,
+    "humanRemoved": 0,
+    "totalCommitted": 34,
+    "agentPercentage": 0.88
+  },
+  "tasks": {},
+  "skillsUsed": []
+}

package/tests/session-bank/git-reader.test.ts

@@ -3,7 +3,7 @@ import { mkdtemp, rm, readFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { execFileSync } from 'node:child_process';
-import { EntireGitReader } from '../../src/session-bank/git-reader.js';
+import { CheckpointGitReader } from '../../src/session-bank/git-reader.js';
 
 const FIXTURES = join(import.meta.dirname, 'fixtures');
 
@@ -126,14 +126,14 @@ async function createFixtureRepo(): Promise<string> {
   const orphanCommit = commitTree(tmpDir, rootTree, 'checkpoint data');
   git(tmpDir, [
     'update-ref',
-    'refs/heads/entire/checkpoints/v1',
+    'refs/heads/sessionlog/checkpoints/v1',
     orphanCommit,
   ]);
 
   return tmpDir;
 }
 
-describe('EntireGitReader', () => {
+describe('CheckpointGitReader', () => {
   let tmpDir: string;
 
   beforeEach(async () => {
@@ -145,12 +145,12 @@ describe('EntireGitReader', () => {
   });
 
   it('should detect that the orphan branch exists', async () => {
-    const reader = new EntireGitReader({ repoPath: tmpDir });
+    const reader = new CheckpointGitReader({ repoPath: tmpDir });
     expect(await reader.branchExists()).toBe(true);
   });
 
   it('should return false for non-existent branch', async () => {
-    const reader = new EntireGitReader({
+    const reader = new CheckpointGitReader({
       repoPath: tmpDir,
       branch: 'nonexistent/branch',
     });
@@ -158,7 +158,7 @@ describe('EntireGitReader', () => {
   });
 
   it('should list checkpoint paths', async () => {
-    const reader = new EntireGitReader({ repoPath: tmpDir });
+    const reader = new CheckpointGitReader({ repoPath: tmpDir });
     const checkpoints = await reader.listCheckpoints();
 
     expect(checkpoints).toHaveLength(1);
@@ -166,7 +166,7 @@ describe('EntireGitReader', () => {
   });
 
   it('should list session indices', async () => {
-    const reader = new EntireGitReader({ repoPath: tmpDir });
+    const reader = new CheckpointGitReader({ repoPath: tmpDir });
     const sessions = await reader.listSessions('a3/b2c4d5e6f7');
 
     expect(sessions).toHaveLength(1);
@@ -174,7 +174,7 @@ describe('EntireGitReader', () => {
   });
 
   it('should read a file from the orphan branch', async () => {
-    const reader = new EntireGitReader({ repoPath: tmpDir });
+    const reader = new CheckpointGitReader({ repoPath: tmpDir });
     const content = await reader.readFile(
       'a3/b2c4d5e6f7/0/metadata.json',
     );
@@ -185,7 +185,7 @@ describe('EntireGitReader', () => {
   });
 
   it('should batch-read multiple files', async () => {
-    const reader = new EntireGitReader({ repoPath: tmpDir });
+    const reader = new CheckpointGitReader({ repoPath: tmpDir });
     const files = await reader.readFiles([
       'a3/b2c4d5e6f7/0/metadata.json',
       'a3/b2c4d5e6f7/0/full.jsonl',
@@ -199,13 +199,13 @@ describe('EntireGitReader', () => {
   });
 
   it('should return null for missing files', async () => {
-    const reader = new EntireGitReader({ repoPath: tmpDir });
+    const reader = new CheckpointGitReader({ repoPath: tmpDir });
     const content = await reader.readFileOrNull('does/not/exist');
     expect(content).toBeNull();
   });
 
   it('should get the orphan branch HEAD SHA', async () => {
-    const reader = new EntireGitReader({ repoPath: tmpDir });
+    const reader = new CheckpointGitReader({ repoPath: tmpDir });
     const sha = await reader.getBranchHead();
 
     expect(sha).toBeDefined();
@@ -213,7 +213,7 @@ describe('EntireGitReader', () => {
   });
 
   it('should return null for HEAD of non-existent branch', async () => {
-    const reader = new EntireGitReader({
+    const reader = new CheckpointGitReader({
       repoPath: tmpDir,
       branch: 'nope',
     });
@@ -222,7 +222,7 @@ describe('EntireGitReader', () => {
   });
 
   it('should return empty checkpoints for non-existent branch', async () => {
-    const reader = new EntireGitReader({
+    const reader = new CheckpointGitReader({
       repoPath: tmpDir,
       branch: 'nope',
     });