cognitive-core 0.1.0 → 0.2.0

package/tests/integration/dataclaw-e2e.test.ts

@@ -0,0 +1,559 @@
+/**
+ * End-to-end integration test: DataclawTrajectorySource
+ *
+ * Writes realistic dataclaw JSONL to temp files, loads through the source,
+ * and verifies the trajectories are valid and compatible with the learning
+ * pipeline (analyzer, schema validation, knowledge extraction).
+ */
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { mkdtemp, rm, writeFile, mkdir } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { DataclawTrajectorySource } from '../../src/learning/trajectory-sources/dataclaw.js';
+import { TrajectorySchema } from '../../src/types/trajectory.js';
+import { TrajectoryAnalyzer, createAnalyzer } from '../../src/learning/analyzer.js';
+import type { DataclawSession } from '../../src/types/dataclaw.js';
+
+// ── Realistic dataclaw sessions simulating different sources ──
+
+/** A Claude Code session fixing a TypeScript bug */
+function claudeSession(): DataclawSession {
+  return {
+    session_id: 'claude-session-001',
+    project: 'my-web-app',
+    model: 'claude-opus-4-6',
+    source: 'claude',
+    git_branch: 'fix/auth-bug',
+    start_time: '2026-02-01T09:00:00+00:00',
+    end_time: '2026-02-01T09:25:00+00:00',
+    messages: [
+      {
+        role: 'user',
+        content: 'The login endpoint returns 500 when the user has no email. Can you fix it?',
+        timestamp: '2026-02-01T09:00:05+00:00',
+      },
+      {
+        role: 'assistant',
+        content: "Let me look at the login handler to understand what's happening.",
+        thinking: 'User reports a 500 error on the login endpoint when user has no email. This is likely a null reference issue in the user validation code.',
+        tool_uses: [
+          {
+            tool: 'Read',
+            input: { file_path: '/src/routes/auth.ts' },
+            output: { text: 'export async function login(req: Request) {\n  const user = await findUser(req.body.id);\n  const email = user.email.toLowerCase();\n  // ...\n}' },
+            status: 'success',
+          },
+        ],
+        timestamp: '2026-02-01T09:00:15+00:00',
+      },
+      {
+        role: 'assistant',
+        content: "Found it — `user.email.toLowerCase()` throws when email is null. Let me add a null check.",
+        tool_uses: [
+          {
+            tool: 'Edit',
+            input: {
+              file_path: '/src/routes/auth.ts',
+              old_string: 'const email = user.email.toLowerCase();',
+              new_string: 'const email = user.email?.toLowerCase() ?? \'\';',
+            },
+            output: { text: 'File edited successfully' },
+            status: 'success',
+          },
+        ],
+        timestamp: '2026-02-01T09:01:00+00:00',
+      },
+      {
+        role: 'assistant',
+        content: "Now let me run the tests to make sure this fix doesn't break anything.",
+        tool_uses: [
+          {
+            tool: 'bash',
+            input: { command: 'npm test -- --grep "login"' },
+            output: { text: 'PASS  tests/auth.test.ts\n  login endpoint\n    ✓ returns 200 for valid user (45ms)\n    ✓ returns 200 for user without email (12ms)\n    ✓ returns 401 for invalid credentials (8ms)\n\nTest Suites: 1 passed, 1 total\nTests:       3 passed, 3 total' },
+            status: 'success',
+          },
+        ],
+        timestamp: '2026-02-01T09:02:00+00:00',
+      },
+      {
+        role: 'assistant',
+        content: 'All tests pass. The fix adds optional chaining to handle users without an email address, preventing the 500 error.',
+        timestamp: '2026-02-01T09:02:30+00:00',
+      },
+    ],
+    stats: {
+      user_messages: 1,
+      assistant_messages: 4,
+      tool_uses: 3,
+      input_tokens: 12000,
+      output_tokens: 2500,
+    },
+  };
+}
+
+/** A Codex session working with Python */
+function codexSession(): DataclawSession {
+  return {
+    session_id: 'codex-session-042',
+    project: 'ml-pipeline',
+    model: 'gpt-5.3-codex',
+    source: 'codex',
+    git_branch: 'feature/data-loader',
+    start_time: '2026-02-10T14:00:00+00:00',
+    end_time: '2026-02-10T14:45:00+00:00',
+    messages: [
+      {
+        role: 'user',
+        content: 'Add a CSV data loader to the pipeline module that supports chunked reading for large files',
+        timestamp: '2026-02-10T14:00:00+00:00',
+      },
+      {
+        role: 'assistant',
+        content: "I'll create a chunked CSV loader. Let me first check the existing pipeline structure.",
+        tool_uses: [
+          {
+            tool: 'exec_command',
+            input: { command: 'ls src/pipeline/' },
+            output: { exit_code: 0, wall_time: '0.1 seconds', output: '__init__.py\nbase.py\njson_loader.py\nparquet_loader.py' },
+            status: 'success',
+          },
+        ],
+        timestamp: '2026-02-10T14:00:30+00:00',
+      },
+      {
+        role: 'assistant',
+        content: 'Creating the CSV loader following the existing pattern.',
+        tool_uses: [
+          {
+            tool: 'write_file',
+            input: { file_path: 'src/pipeline/csv_loader.py', content: 'import pandas as pd\nfrom .base import BaseLoader\n\nclass CSVLoader(BaseLoader):\n    def load(self, path: str, chunk_size: int = 10000):\n        for chunk in pd.read_csv(path, chunksize=chunk_size):\n            yield self.process(chunk)\n' },
+            output: {},
+            status: 'success',
+          },
+        ],
+        timestamp: '2026-02-10T14:05:00+00:00',
+      },
+      {
+        role: 'assistant',
+        content: "Let me add the import to __init__.py and run the tests.",
+        tool_uses: [
+          {
+            tool: 'exec_command',
+            input: { command: 'python -m pytest tests/test_csv_loader.py -v' },
+            output: { exit_code: 0, wall_time: '3 seconds', output: 'tests/test_csv_loader.py::test_load_small_csv PASSED\ntests/test_csv_loader.py::test_chunked_reading PASSED\n\n2 passed in 2.1s' },
+            status: 'success',
+          },
+        ],
+        timestamp: '2026-02-10T14:10:00+00:00',
+      },
+      {
+        role: 'assistant',
+        content: 'The CSV data loader is implemented and tested. It supports chunked reading for memory efficiency with large files.',
+        timestamp: '2026-02-10T14:11:00+00:00',
+      },
+    ],
+    stats: {
+      user_messages: 1,
+      assistant_messages: 4,
+      tool_uses: 3,
+      input_tokens: 8000,
+      output_tokens: 2000,
+    },
+  };
+}
+
+/** A session that fails — error-heavy with failure signals */
+function failedSession(): DataclawSession {
+  return {
+    session_id: 'failed-session-007',
+    project: 'my-web-app',
+    model: 'claude-opus-4-6',
+    source: 'claude',
+    git_branch: 'main',
+    start_time: '2026-02-05T16:00:00+00:00',
+    end_time: '2026-02-05T16:10:00+00:00',
+    messages: [
+      {
+        role: 'user',
+        content: 'Deploy the app to production',
+        timestamp: '2026-02-05T16:00:00+00:00',
+      },
+      {
+        role: 'assistant',
+        content: 'Attempting deployment.',
+        tool_uses: [
+          {
+            tool: 'bash',
+            input: { command: 'npm run deploy' },
+            output: { text: 'Error: Missing DEPLOY_TOKEN environment variable' },
+            status: 'error',
+          },
+        ],
+        timestamp: '2026-02-05T16:01:00+00:00',
+      },
+      {
+        role: 'assistant',
+        content: 'Retrying with alternate method.',
+        tool_uses: [
+          {
+            tool: 'bash',
+            input: { command: 'npx deploy --force' },
+            output: { text: 'Fatal: permission denied' },
+            status: 'error',
+          },
+        ],
+        timestamp: '2026-02-05T16:02:00+00:00',
+      },
+      {
+        role: 'assistant',
+        content: 'I was unable to deploy. The deployment requires a DEPLOY_TOKEN that is not available in this environment.',
+        timestamp: '2026-02-05T16:03:00+00:00',
+      },
+    ],
+    stats: {
+      user_messages: 1,
+      assistant_messages: 3,
+      tool_uses: 2,
+      input_tokens: 5000,
+      output_tokens: 1000,
+    },
+  };
+}
+
+/** A minimal no-tools Q&A session */
+function conversationOnlySession(): DataclawSession {
+  return {
+    session_id: 'qa-session-099',
+    project: 'my-web-app',
+    model: 'claude-opus-4-6',
+    source: 'claude',
+    git_branch: null,
+    start_time: '2026-02-15T11:00:00+00:00',
+    end_time: '2026-02-15T11:05:00+00:00',
+    messages: [
+      {
+        role: 'user',
+        content: 'Explain how the middleware chain works in our Express app',
+        timestamp: '2026-02-15T11:00:00+00:00',
+      },
+      {
+        role: 'assistant',
+        content: 'Your Express middleware chain processes requests in order: first auth validation, then rate limiting, then the route handler. Each middleware calls next() to pass control.',
+        timestamp: '2026-02-15T11:00:30+00:00',
+      },
+    ],
+    stats: {
+      user_messages: 1,
+      assistant_messages: 1,
+      tool_uses: 0,
+      input_tokens: 3000,
+      output_tokens: 500,
+    },
+  };
+}
+
+// ============================================================================
+// E2E Tests
+// ============================================================================
+
+describe('DataClaw E2E Integration', () => {
+  let tmpDir: string;
+  let dataDir: string;
+
+  beforeAll(async () => {
+    tmpDir = await mkdtemp(join(tmpdir(), 'dataclaw-e2e-'));
+    dataDir = join(tmpDir, 'dataclaw-export');
+    await mkdir(dataDir);
+
+    // Write a multi-file export — resembling a real dataclaw dataset
+    const allSessions = [
+      claudeSession(),
+      codexSession(),
+      failedSession(),
+      conversationOnlySession(),
+    ];
+
+    // Split across two files like a real export might
+    await writeFile(
+      join(dataDir, 'claude-sessions.jsonl'),
+      [claudeSession(), failedSession(), conversationOnlySession()]
+        .map((s) => JSON.stringify(s))
+        .join('\n'),
+      'utf-8',
+    );
+    await writeFile(
+      join(dataDir, 'codex-sessions.jsonl'),
+      JSON.stringify(codexSession()),
+      'utf-8',
+    );
+
+    // Companion metadata file (should be ignored by the source)
+    await writeFile(
+      join(dataDir, 'metadata.json'),
+      JSON.stringify({
+        sessions: 4,
+        skipped: 0,
+        redactions: 2,
+        models: { 'claude-opus-4-6': 3, 'gpt-5.3-codex': 1 },
+        projects: ['my-web-app', 'ml-pipeline'],
+        exported_at: '2026-02-20T00:00:00Z',
+      }),
+      'utf-8',
+    );
+  });
+
+  afterAll(async () => {
+    await rm(tmpDir, { recursive: true, force: true });
+  });
+
+  describe('load all sessions from directory', () => {
+    it('should load all 4 sessions from the dataset', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      const items = await source.load();
+      expect(items).toHaveLength(4);
+    });
+
+    it('should produce valid Trajectory objects for every session', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      const items = await source.load();
+
+      for (const { trajectory } of items) {
+        const parsed = TrajectorySchema.safeParse(trajectory);
+        expect(parsed.success, `Invalid trajectory: ${trajectory.id}`).toBe(true);
+      }
+    });
+
+    it('should correctly identify the source provenance in metadata', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      const items = await source.load();
+
+      for (const { trajectory } of items) {
+        expect(trajectory.metadata.source).toBe('dataclaw');
+        expect(trajectory.metadata.dataclawSource).toBeTruthy();
+        expect(trajectory.metadata.sessionId).toBeTruthy();
+      }
+    });
+  });
+
+  describe('trajectory quality — Claude session', () => {
+    it('should extract meaningful task description', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      const items = await source.load();
+      const claude = items.find((i) => i.trajectory.id.includes('claude-session-001'));
+      expect(claude).toBeDefined();
+      expect(claude!.trajectory.task.description).toContain('login endpoint');
+      expect(claude!.trajectory.task.description).toContain('500');
+    });
+
+    it('should extract tool-use steps with correct thought/action/observation pattern', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      const items = await source.load();
+      const claude = items.find((i) => i.trajectory.id.includes('claude-session-001'));
+      const steps = claude!.trajectory.steps;
+
+      // Should have tool steps + final text step
+      expect(steps.length).toBeGreaterThanOrEqual(3);
+
+      // Read step
+      const readStep = steps.find((s) => s.metadata?.toolName === 'Read');
+      expect(readStep).toBeDefined();
+      expect(readStep!.action).toContain('Read');
+      expect(readStep!.observation).toContain('user.email.toLowerCase');
+      expect(readStep!.thought).toBeTruthy(); // should have thinking
+
+      // Edit step
+      const editStep = steps.find((s) => s.metadata?.toolName === 'Edit');
+      expect(editStep).toBeDefined();
+      expect(editStep!.observation).toContain('File edited');
+
+      // Bash step (test run)
+      const bashStep = steps.find((s) => s.metadata?.toolName === 'bash');
+      expect(bashStep).toBeDefined();
+      expect(bashStep!.observation).toContain('3 passed');
+    });
+
+    it('should infer TypeScript domain from file paths', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      const items = await source.load();
+      const claude = items.find((i) => i.trajectory.id.includes('claude-session-001'));
+      expect(claude!.trajectory.task.domain).toBe('typescript');
+    });
+
+    it('should infer success for a clean session', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      const items = await source.load();
+      const claude = items.find((i) => i.trajectory.id.includes('claude-session-001'));
+      expect(claude!.trajectory.outcome.success).toBe(true);
+    });
+
+    it('should compute correct wall time and token totals', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      const items = await source.load();
+      const claude = items.find((i) => i.trajectory.id.includes('claude-session-001'));
+      expect(claude!.trajectory.wallTimeSeconds).toBe(25 * 60); // 25 minutes
+      expect(claude!.trajectory.totalTokens).toBe(14500); // 12000 + 2500
+    });
+  });
+
+  describe('trajectory quality — Codex session', () => {
+    it('should correctly map Codex tool format (exec_command)', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      const items = await source.load();
+      const codex = items.find((i) => i.trajectory.id.includes('codex-session'));
+      expect(codex).toBeDefined();
+
+      // exec_command output uses { output: "..." } shape
+      const lsStep = codex!.trajectory.steps.find((s) =>
+        s.metadata?.toolName === 'exec_command',
+      );
+      expect(lsStep).toBeDefined();
+      expect(lsStep!.observation).toContain('json_loader.py');
+    });
+
+    it('should infer Python domain from Python file paths', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      const items = await source.load();
+      const codex = items.find((i) => i.trajectory.id.includes('codex-session'));
+      expect(codex!.trajectory.task.domain).toBe('python');
+    });
+
+    it('should set agentId from model field', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      const items = await source.load();
+      const codex = items.find((i) => i.trajectory.id.includes('codex-session'));
+      expect(codex!.trajectory.agentId).toBe('gpt-5.3-codex');
+    });
+  });
+
+  describe('trajectory quality — failed session', () => {
+    it('should detect failure from high error rate and failure signals', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      const items = await source.load();
+      const failed = items.find((i) => i.trajectory.id.includes('failed-session'));
+      expect(failed).toBeDefined();
+      expect(failed!.trajectory.outcome.success).toBe(false);
+    });
+
+    it('should report a partial score reflecting the error rate', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      const items = await source.load();
+      const failed = items.find((i) => i.trajectory.id.includes('failed-session'));
+      expect(failed!.trajectory.outcome.partialScore).toBeLessThan(0.5);
+    });
+  });
+
+  describe('trajectory quality — conversation-only session', () => {
+    it('should handle pure Q&A with no tool usage', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      const items = await source.load();
+      const qa = items.find((i) => i.trajectory.id.includes('qa-session'));
+      expect(qa).toBeDefined();
+      expect(qa!.trajectory.steps).toHaveLength(1);
+      expect(qa!.trajectory.steps[0].thought).toContain('middleware');
+    });
+  });
+
+  describe('filtering', () => {
+    it('should filter by source (claude only)', async () => {
+      const source = new DataclawTrajectorySource(dataDir, {
+        sources: ['claude'],
+      });
+      const items = await source.load();
+      expect(items).toHaveLength(3); // claude + failed + qa
+      for (const { trajectory } of items) {
+        expect(trajectory.metadata.dataclawSource).toBe('claude');
+      }
+    });
+
+    it('should filter by model', async () => {
+      const source = new DataclawTrajectorySource(dataDir, {
+        models: ['gpt-5.3-codex'],
+      });
+      const items = await source.load();
+      expect(items).toHaveLength(1);
+      expect(items[0].trajectory.metadata.model).toBe('gpt-5.3-codex');
+    });
+
+    it('should filter successOnly', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      const items = await source.load({ successOnly: true });
+      for (const { trajectory } of items) {
+        expect(trajectory.outcome.success).toBe(true);
+      }
+      // Failed session should be excluded
+      expect(
+        items.find((i) => i.trajectory.id.includes('failed-session')),
+      ).toBeUndefined();
+    });
+
+    it('should filter by domain', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      const items = await source.load({ domain: 'python' });
+      expect(items).toHaveLength(1);
+      expect(items[0].trajectory.task.domain).toBe('python');
+    });
+
+    it('should filter by since date', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      const items = await source.load({
+        since: new Date('2026-02-10T00:00:00Z'),
+      });
+      // Only codex (Feb 10) and qa (Feb 15) sessions
+      expect(items).toHaveLength(2);
+    });
+
+    it('should combine source + successOnly filters', async () => {
+      const source = new DataclawTrajectorySource(dataDir, {
+        sources: ['claude'],
+      });
+      const items = await source.load({ successOnly: true });
+      // Claude source has 3 sessions, but 1 failed
+      expect(items).toHaveLength(2);
+    });
+  });
+
+  describe('analyzer compatibility', () => {
+    it('should produce trajectories analyzable by TrajectoryAnalyzer', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      const items = await source.load({ successOnly: true, limit: 2 });
+      const analyzer = createAnalyzer('simple');
+
+      for (const { trajectory } of items) {
+        const analysis = await analyzer.analyze(trajectory);
+
+        expect(analysis).toBeDefined();
+        expect(typeof analysis.success).toBe('boolean');
+        expect(analysis.stepAttribution).toHaveLength(trajectory.steps.length);
+        expect(analysis.keySteps).toBeDefined();
+        expect(typeof analysis.abstractable).toBe('boolean');
+      }
+    });
+
+    it('should have meaningful key steps identified by the analyzer', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      const items = await source.load();
+      const claude = items.find((i) => i.trajectory.id.includes('claude-session-001'));
+      const analyzer = createAnalyzer('simple');
+      const analysis = await analyzer.analyze(claude!.trajectory);
+
+      // A 4-step trajectory with tool usage should have identifiable key steps
+      expect(analysis.stepAttribution.length).toBeGreaterThan(0);
+      // At least one step should have non-zero attribution
+      expect(analysis.stepAttribution.some((s) => s > 0)).toBe(true);
+    });
+  });
+
+  describe('count', () => {
+    it('should return correct count for all sessions', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      expect(await source.count()).toBe(4);
+    });
+
+    it('should return correct count with filters', async () => {
+      const source = new DataclawTrajectorySource(dataDir);
+      expect(await source.count({ successOnly: true })).toBeLessThan(4);
+      expect(await source.count({ domain: 'python' })).toBe(1);
+    });
+  });
+});