cognitive-core 0.2.4 → 0.3.1

package/tests/integration/dataclaw-e2e.test.ts

@@ -1,559 +0,0 @@
-/**
- * End-to-end integration test: DataclawTrajectorySource
- *
- * Writes realistic dataclaw JSONL to temp files, loads through the source,
- * and verifies the trajectories are valid and compatible with the learning
- * pipeline (analyzer, schema validation, knowledge extraction).
- */
-import { describe, it, expect, beforeAll, afterAll } from 'vitest';
-import { mkdtemp, rm, writeFile, mkdir } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { DataclawTrajectorySource } from '../../src/learning/trajectory-sources/dataclaw.js';
-import { TrajectorySchema } from '../../src/types/trajectory.js';
-import { TrajectoryAnalyzer, createAnalyzer } from '../../src/learning/analyzer.js';
-import type { DataclawSession } from '../../src/types/dataclaw.js';
-
-// ── Realistic dataclaw sessions simulating different sources ──
-
-/** A Claude Code session fixing a TypeScript bug */
-function claudeSession(): DataclawSession {
-  return {
-    session_id: 'claude-session-001',
-    project: 'my-web-app',
-    model: 'claude-opus-4-6',
-    source: 'claude',
-    git_branch: 'fix/auth-bug',
-    start_time: '2026-02-01T09:00:00+00:00',
-    end_time: '2026-02-01T09:25:00+00:00',
-    messages: [
-      {
-        role: 'user',
-        content: 'The login endpoint returns 500 when the user has no email. Can you fix it?',
-        timestamp: '2026-02-01T09:00:05+00:00',
-      },
-      {
-        role: 'assistant',
-        content: "Let me look at the login handler to understand what's happening.",
-        thinking: 'User reports a 500 error on the login endpoint when user has no email. This is likely a null reference issue in the user validation code.',
-        tool_uses: [
-          {
-            tool: 'Read',
-            input: { file_path: '/src/routes/auth.ts' },
-            output: { text: 'export async function login(req: Request) {\n  const user = await findUser(req.body.id);\n  const email = user.email.toLowerCase();\n  // ...\n}' },
-            status: 'success',
-          },
-        ],
-        timestamp: '2026-02-01T09:00:15+00:00',
-      },
-      {
-        role: 'assistant',
-        content: "Found it — `user.email.toLowerCase()` throws when email is null. Let me add a null check.",
-        tool_uses: [
-          {
-            tool: 'Edit',
-            input: {
-              file_path: '/src/routes/auth.ts',
-              old_string: 'const email = user.email.toLowerCase();',
-              new_string: 'const email = user.email?.toLowerCase() ?? \'\';',
-            },
-            output: { text: 'File edited successfully' },
-            status: 'success',
-          },
-        ],
-        timestamp: '2026-02-01T09:01:00+00:00',
-      },
-      {
-        role: 'assistant',
-        content: "Now let me run the tests to make sure this fix doesn't break anything.",
-        tool_uses: [
-          {
-            tool: 'bash',
-            input: { command: 'npm test -- --grep "login"' },
-            output: { text: 'PASS  tests/auth.test.ts\n  login endpoint\n    ✓ returns 200 for valid user (45ms)\n    ✓ returns 200 for user without email (12ms)\n    ✓ returns 401 for invalid credentials (8ms)\n\nTest Suites: 1 passed, 1 total\nTests:       3 passed, 3 total' },
-            status: 'success',
-          },
-        ],
-        timestamp: '2026-02-01T09:02:00+00:00',
-      },
-      {
-        role: 'assistant',
-        content: 'All tests pass. The fix adds optional chaining to handle users without an email address, preventing the 500 error.',
-        timestamp: '2026-02-01T09:02:30+00:00',
-      },
-    ],
-    stats: {
-      user_messages: 1,
-      assistant_messages: 4,
-      tool_uses: 3,
-      input_tokens: 12000,
-      output_tokens: 2500,
-    },
-  };
-}
-
-/** A Codex session working with Python */
-function codexSession(): DataclawSession {
-  return {
-    session_id: 'codex-session-042',
-    project: 'ml-pipeline',
-    model: 'gpt-5.3-codex',
-    source: 'codex',
-    git_branch: 'feature/data-loader',
-    start_time: '2026-02-10T14:00:00+00:00',
-    end_time: '2026-02-10T14:45:00+00:00',
-    messages: [
-      {
-        role: 'user',
-        content: 'Add a CSV data loader to the pipeline module that supports chunked reading for large files',
-        timestamp: '2026-02-10T14:00:00+00:00',
-      },
-      {
-        role: 'assistant',
-        content: "I'll create a chunked CSV loader. Let me first check the existing pipeline structure.",
-        tool_uses: [
-          {
-            tool: 'exec_command',
-            input: { command: 'ls src/pipeline/' },
-            output: { exit_code: 0, wall_time: '0.1 seconds', output: '__init__.py\nbase.py\njson_loader.py\nparquet_loader.py' },
-            status: 'success',
-          },
-        ],
-        timestamp: '2026-02-10T14:00:30+00:00',
-      },
-      {
-        role: 'assistant',
-        content: 'Creating the CSV loader following the existing pattern.',
-        tool_uses: [
-          {
-            tool: 'write_file',
-            input: { file_path: 'src/pipeline/csv_loader.py', content: 'import pandas as pd\nfrom .base import BaseLoader\n\nclass CSVLoader(BaseLoader):\n    def load(self, path: str, chunk_size: int = 10000):\n        for chunk in pd.read_csv(path, chunksize=chunk_size):\n            yield self.process(chunk)\n' },
-            output: {},
-            status: 'success',
-          },
-        ],
-        timestamp: '2026-02-10T14:05:00+00:00',
-      },
-      {
-        role: 'assistant',
-        content: "Let me add the import to __init__.py and run the tests.",
-        tool_uses: [
-          {
-            tool: 'exec_command',
-            input: { command: 'python -m pytest tests/test_csv_loader.py -v' },
-            output: { exit_code: 0, wall_time: '3 seconds', output: 'tests/test_csv_loader.py::test_load_small_csv PASSED\ntests/test_csv_loader.py::test_chunked_reading PASSED\n\n2 passed in 2.1s' },
-            status: 'success',
-          },
-        ],
-        timestamp: '2026-02-10T14:10:00+00:00',
-      },
-      {
-        role: 'assistant',
-        content: 'The CSV data loader is implemented and tested. It supports chunked reading for memory efficiency with large files.',
-        timestamp: '2026-02-10T14:11:00+00:00',
-      },
-    ],
-    stats: {
-      user_messages: 1,
-      assistant_messages: 4,
-      tool_uses: 3,
-      input_tokens: 8000,
-      output_tokens: 2000,
-    },
-  };
-}
-
-/** A session that fails — error-heavy with failure signals */
-function failedSession(): DataclawSession {
-  return {
-    session_id: 'failed-session-007',
-    project: 'my-web-app',
-    model: 'claude-opus-4-6',
-    source: 'claude',
-    git_branch: 'main',
-    start_time: '2026-02-05T16:00:00+00:00',
-    end_time: '2026-02-05T16:10:00+00:00',
-    messages: [
-      {
-        role: 'user',
-        content: 'Deploy the app to production',
-        timestamp: '2026-02-05T16:00:00+00:00',
-      },
-      {
-        role: 'assistant',
-        content: 'Attempting deployment.',
-        tool_uses: [
-          {
-            tool: 'bash',
-            input: { command: 'npm run deploy' },
-            output: { text: 'Error: Missing DEPLOY_TOKEN environment variable' },
-            status: 'error',
-          },
-        ],
-        timestamp: '2026-02-05T16:01:00+00:00',
-      },
-      {
-        role: 'assistant',
-        content: 'Retrying with alternate method.',
-        tool_uses: [
-          {
-            tool: 'bash',
-            input: { command: 'npx deploy --force' },
-            output: { text: 'Fatal: permission denied' },
-            status: 'error',
-          },
-        ],
-        timestamp: '2026-02-05T16:02:00+00:00',
-      },
-      {
-        role: 'assistant',
-        content: 'I was unable to deploy. The deployment requires a DEPLOY_TOKEN that is not available in this environment.',
-        timestamp: '2026-02-05T16:03:00+00:00',
-      },
-    ],
-    stats: {
-      user_messages: 1,
-      assistant_messages: 3,
-      tool_uses: 2,
-      input_tokens: 5000,
-      output_tokens: 1000,
-    },
-  };
-}
-
-/** A minimal no-tools Q&A session */
-function conversationOnlySession(): DataclawSession {
-  return {
-    session_id: 'qa-session-099',
-    project: 'my-web-app',
-    model: 'claude-opus-4-6',
-    source: 'claude',
-    git_branch: null,
-    start_time: '2026-02-15T11:00:00+00:00',
-    end_time: '2026-02-15T11:05:00+00:00',
-    messages: [
-      {
-        role: 'user',
-        content: 'Explain how the middleware chain works in our Express app',
-        timestamp: '2026-02-15T11:00:00+00:00',
-      },
-      {
-        role: 'assistant',
-        content: 'Your Express middleware chain processes requests in order: first auth validation, then rate limiting, then the route handler. Each middleware calls next() to pass control.',
-        timestamp: '2026-02-15T11:00:30+00:00',
-      },
-    ],
-    stats: {
-      user_messages: 1,
-      assistant_messages: 1,
-      tool_uses: 0,
-      input_tokens: 3000,
-      output_tokens: 500,
-    },
-  };
-}
-
-// ============================================================================
-// E2E Tests
-// ============================================================================
-
-describe('DataClaw E2E Integration', () => {
-  let tmpDir: string;
-  let dataDir: string;
-
-  beforeAll(async () => {
-    tmpDir = await mkdtemp(join(tmpdir(), 'dataclaw-e2e-'));
-    dataDir = join(tmpDir, 'dataclaw-export');
-    await mkdir(dataDir);
-
-    // Write a multi-file export — resembling a real dataclaw dataset
-    const allSessions = [
-      claudeSession(),
-      codexSession(),
-      failedSession(),
-      conversationOnlySession(),
-    ];
-
-    // Split across two files like a real export might
-    await writeFile(
-      join(dataDir, 'claude-sessions.jsonl'),
-      [claudeSession(), failedSession(), conversationOnlySession()]
-        .map((s) => JSON.stringify(s))
-        .join('\n'),
-      'utf-8',
-    );
-    await writeFile(
-      join(dataDir, 'codex-sessions.jsonl'),
-      JSON.stringify(codexSession()),
-      'utf-8',
-    );
-
-    // Companion metadata file (should be ignored by the source)
-    await writeFile(
-      join(dataDir, 'metadata.json'),
-      JSON.stringify({
-        sessions: 4,
-        skipped: 0,
-        redactions: 2,
-        models: { 'claude-opus-4-6': 3, 'gpt-5.3-codex': 1 },
-        projects: ['my-web-app', 'ml-pipeline'],
-        exported_at: '2026-02-20T00:00:00Z',
-      }),
-      'utf-8',
-    );
-  });
-
-  afterAll(async () => {
-    await rm(tmpDir, { recursive: true, force: true });
-  });
-
-  describe('load all sessions from directory', () => {
-    it('should load all 4 sessions from the dataset', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      const items = await source.load();
-      expect(items).toHaveLength(4);
-    });
-
-    it('should produce valid Trajectory objects for every session', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      const items = await source.load();
-
-      for (const { trajectory } of items) {
-        const parsed = TrajectorySchema.safeParse(trajectory);
-        expect(parsed.success, `Invalid trajectory: ${trajectory.id}`).toBe(true);
-      }
-    });
-
-    it('should correctly identify the source provenance in metadata', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      const items = await source.load();
-
-      for (const { trajectory } of items) {
-        expect(trajectory.metadata.source).toBe('dataclaw');
-        expect(trajectory.metadata.dataclawSource).toBeTruthy();
-        expect(trajectory.metadata.sessionId).toBeTruthy();
-      }
-    });
-  });
-
-  describe('trajectory quality — Claude session', () => {
-    it('should extract meaningful task description', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      const items = await source.load();
-      const claude = items.find((i) => i.trajectory.id.includes('claude-session-001'));
-      expect(claude).toBeDefined();
-      expect(claude!.trajectory.task.description).toContain('login endpoint');
-      expect(claude!.trajectory.task.description).toContain('500');
-    });
-
-    it('should extract tool-use steps with correct thought/action/observation pattern', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      const items = await source.load();
-      const claude = items.find((i) => i.trajectory.id.includes('claude-session-001'));
-      const steps = claude!.trajectory.steps;
-
-      // Should have tool steps + final text step
-      expect(steps.length).toBeGreaterThanOrEqual(3);
-
-      // Read step
-      const readStep = steps.find((s) => s.metadata?.toolName === 'Read');
-      expect(readStep).toBeDefined();
-      expect(readStep!.action).toContain('Read');
-      expect(readStep!.observation).toContain('user.email.toLowerCase');
-      expect(readStep!.thought).toBeTruthy(); // should have thinking
-
-      // Edit step
-      const editStep = steps.find((s) => s.metadata?.toolName === 'Edit');
-      expect(editStep).toBeDefined();
-      expect(editStep!.observation).toContain('File edited');
-
-      // Bash step (test run)
-      const bashStep = steps.find((s) => s.metadata?.toolName === 'bash');
-      expect(bashStep).toBeDefined();
-      expect(bashStep!.observation).toContain('3 passed');
-    });
-
-    it('should infer TypeScript domain from file paths', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      const items = await source.load();
-      const claude = items.find((i) => i.trajectory.id.includes('claude-session-001'));
-      expect(claude!.trajectory.task.domain).toBe('typescript');
-    });
-
-    it('should infer success for a clean session', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      const items = await source.load();
-      const claude = items.find((i) => i.trajectory.id.includes('claude-session-001'));
-      expect(claude!.trajectory.outcome.success).toBe(true);
-    });
-
-    it('should compute correct wall time and token totals', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      const items = await source.load();
-      const claude = items.find((i) => i.trajectory.id.includes('claude-session-001'));
-      expect(claude!.trajectory.wallTimeSeconds).toBe(25 * 60); // 25 minutes
-      expect(claude!.trajectory.totalTokens).toBe(14500); // 12000 + 2500
-    });
-  });
-
-  describe('trajectory quality — Codex session', () => {
-    it('should correctly map Codex tool format (exec_command)', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      const items = await source.load();
-      const codex = items.find((i) => i.trajectory.id.includes('codex-session'));
-      expect(codex).toBeDefined();
-
-      // exec_command output uses { output: "..." } shape
-      const lsStep = codex!.trajectory.steps.find((s) =>
-        s.metadata?.toolName === 'exec_command',
-      );
-      expect(lsStep).toBeDefined();
-      expect(lsStep!.observation).toContain('json_loader.py');
-    });
-
-    it('should infer Python domain from Python file paths', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      const items = await source.load();
-      const codex = items.find((i) => i.trajectory.id.includes('codex-session'));
-      expect(codex!.trajectory.task.domain).toBe('python');
-    });
-
-    it('should set agentId from model field', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      const items = await source.load();
-      const codex = items.find((i) => i.trajectory.id.includes('codex-session'));
-      expect(codex!.trajectory.agentId).toBe('gpt-5.3-codex');
-    });
-  });
-
-  describe('trajectory quality — failed session', () => {
-    it('should detect failure from high error rate and failure signals', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      const items = await source.load();
-      const failed = items.find((i) => i.trajectory.id.includes('failed-session'));
-      expect(failed).toBeDefined();
-      expect(failed!.trajectory.outcome.success).toBe(false);
-    });
-
-    it('should report a partial score reflecting the error rate', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      const items = await source.load();
-      const failed = items.find((i) => i.trajectory.id.includes('failed-session'));
-      expect(failed!.trajectory.outcome.partialScore).toBeLessThan(0.5);
-    });
-  });
-
-  describe('trajectory quality — conversation-only session', () => {
-    it('should handle pure Q&A with no tool usage', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      const items = await source.load();
-      const qa = items.find((i) => i.trajectory.id.includes('qa-session'));
-      expect(qa).toBeDefined();
-      expect(qa!.trajectory.steps).toHaveLength(1);
-      expect(qa!.trajectory.steps[0].thought).toContain('middleware');
-    });
-  });
-
-  describe('filtering', () => {
-    it('should filter by source (claude only)', async () => {
-      const source = new DataclawTrajectorySource(dataDir, {
-        sources: ['claude'],
-      });
-      const items = await source.load();
-      expect(items).toHaveLength(3); // claude + failed + qa
-      for (const { trajectory } of items) {
-        expect(trajectory.metadata.dataclawSource).toBe('claude');
-      }
-    });
-
-    it('should filter by model', async () => {
-      const source = new DataclawTrajectorySource(dataDir, {
-        models: ['gpt-5.3-codex'],
-      });
-      const items = await source.load();
-      expect(items).toHaveLength(1);
-      expect(items[0].trajectory.metadata.model).toBe('gpt-5.3-codex');
-    });
-
-    it('should filter successOnly', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      const items = await source.load({ successOnly: true });
-      for (const { trajectory } of items) {
-        expect(trajectory.outcome.success).toBe(true);
-      }
-      // Failed session should be excluded
-      expect(
-        items.find((i) => i.trajectory.id.includes('failed-session')),
-      ).toBeUndefined();
-    });
-
-    it('should filter by domain', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      const items = await source.load({ domain: 'python' });
-      expect(items).toHaveLength(1);
-      expect(items[0].trajectory.task.domain).toBe('python');
-    });
-
-    it('should filter by since date', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      const items = await source.load({
-        since: new Date('2026-02-10T00:00:00Z'),
-      });
-      // Only codex (Feb 10) and qa (Feb 15) sessions
-      expect(items).toHaveLength(2);
-    });
-
-    it('should combine source + successOnly filters', async () => {
-      const source = new DataclawTrajectorySource(dataDir, {
-        sources: ['claude'],
-      });
-      const items = await source.load({ successOnly: true });
-      // Claude source has 3 sessions, but 1 failed
-      expect(items).toHaveLength(2);
-    });
-  });
-
-  describe('analyzer compatibility', () => {
-    it('should produce trajectories analyzable by TrajectoryAnalyzer', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      const items = await source.load({ successOnly: true, limit: 2 });
-      const analyzer = createAnalyzer('simple');
-
-      for (const { trajectory } of items) {
-        const analysis = await analyzer.analyze(trajectory);
-
-        expect(analysis).toBeDefined();
-        expect(typeof analysis.success).toBe('boolean');
-        expect(analysis.stepAttribution).toHaveLength(trajectory.steps.length);
-        expect(analysis.keySteps).toBeDefined();
-        expect(typeof analysis.abstractable).toBe('boolean');
-      }
-    });
-
-    it('should have meaningful key steps identified by the analyzer', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      const items = await source.load();
-      const claude = items.find((i) => i.trajectory.id.includes('claude-session-001'));
-      const analyzer = createAnalyzer('simple');
-      const analysis = await analyzer.analyze(claude!.trajectory);
-
-      // A 4-step trajectory with tool usage should have identifiable key steps
-      expect(analysis.stepAttribution.length).toBeGreaterThan(0);
-      // At least one step should have non-zero attribution
-      expect(analysis.stepAttribution.some((s) => s > 0)).toBe(true);
-    });
-  });
-
-  describe('count', () => {
-    it('should return correct count for all sessions', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      expect(await source.count()).toBe(4);
-    });
-
-    it('should return correct count with filters', async () => {
-      const source = new DataclawTrajectorySource(dataDir);
-      expect(await source.count({ successOnly: true })).toBeLessThan(4);
-      expect(await source.count({ domain: 'python' })).toBe(1);
-    });
-  });
-});