cognitive-core 0.0.2 → 0.1.2

package/tests/learning/dataclaw-trajectory-source.test.ts

@@ -0,0 +1,642 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { DataclawTrajectorySource } from '../../src/learning/trajectory-sources/dataclaw.js';
+import { TrajectorySchema } from '../../src/types/trajectory.js';
+import type { DataclawSession } from '../../src/types/dataclaw.js';
+
+function makeSession(overrides?: Partial<DataclawSession>): DataclawSession {
+  return {
+    session_id: 'sess-abc-123',
+    project: 'my-project',
+    model: 'claude-opus-4-6',
+    source: 'claude',
+    git_branch: 'main',
+    start_time: '2026-01-15T10:00:00+00:00',
+    end_time: '2026-01-15T10:30:00+00:00',
+    messages: [
+      {
+        role: 'user',
+        content: 'Fix the login bug in auth.ts',
+        timestamp: '2026-01-15T10:00:10+00:00',
+      },
+      {
+        role: 'assistant',
+        content: "I'll read the file to understand the issue.",
+        thinking: 'The user wants me to fix a bug in auth.ts. Let me read it first.',
+        tool_uses: [
+          {
+            tool: 'Read',
+            input: { file_path: '/src/auth.ts' },
+            output: { text: 'export function login() { /* broken */ }' },
+            status: 'success',
+          },
+        ],
+        timestamp: '2026-01-15T10:00:20+00:00',
+      },
+      {
+        role: 'user',
+        content: 'looks right, go ahead',
+        timestamp: '2026-01-15T10:01:00+00:00',
+      },
+      {
+        role: 'assistant',
+        content: 'Found the issue — missing null check. Let me fix it.',
+        tool_uses: [
+          {
+            tool: 'Edit',
+            input: {
+              file_path: '/src/auth.ts',
+              old_string: 'user.token',
+              new_string: 'user?.token',
+            },
+            output: { text: 'File edited successfully' },
+            status: 'success',
+          },
+        ],
+        timestamp: '2026-01-15T10:01:30+00:00',
+      },
+      {
+        role: 'assistant',
+        content: 'The fix has been applied. The login function now handles null users correctly.',
+        timestamp: '2026-01-15T10:02:00+00:00',
+      },
+    ],
+    stats: {
+      user_messages: 2,
+      assistant_messages: 3,
+      tool_uses: 2,
+      input_tokens: 15000,
+      output_tokens: 3000,
+    },
+    ...overrides,
+  };
+}
+
+async function writeJsonl(dir: string, filename: string, sessions: DataclawSession[]): Promise<string> {
+  const filePath = join(dir, filename);
+  const content = sessions.map((s) => JSON.stringify(s)).join('\n');
+  await writeFile(filePath, content, 'utf-8');
+  return filePath;
+}
+
+// ============================================================================
+// DataclawTrajectorySource — synthesis tests
+// ============================================================================
+
+describe('DataclawTrajectorySource', () => {
+  describe('synthesize', () => {
+    it('should produce a valid Trajectory', () => {
+      const session = makeSession();
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      const parsed = TrajectorySchema.safeParse(trajectory);
+      expect(parsed.success).toBe(true);
+    });
+
+    it('should set correct trajectory ID', () => {
+      const session = makeSession();
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      expect(trajectory.id).toBe('dataclaw-claude-sess-abc-123');
+    });
+
+    it('should use first user message as task description', () => {
+      const session = makeSession();
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      expect(trajectory.task.description).toBe('Fix the login bug in auth.ts');
+    });
+
+    it('should fall back to "Unknown task" when no messages', () => {
+      const session = makeSession({ messages: [] });
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      expect(trajectory.task.description).toBe('Unknown task');
+    });
+
+    it('should truncate very long first user messages', () => {
+      const longMessage = 'a'.repeat(600);
+      const session = makeSession({
+        messages: [
+          { role: 'user', content: longMessage },
+        ],
+      });
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      expect(trajectory.task.description.length).toBeLessThanOrEqual(503); // 500 + '...'
+      expect(trajectory.task.description).toMatch(/\.\.\.$/);
+    });
+
+    it('should infer typescript domain from tool inputs', () => {
+      const session = makeSession();
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      expect(trajectory.task.domain).toBe('typescript');
+    });
+
+    it('should infer testing domain from test file extensions', () => {
+      const session = makeSession({
+        messages: [
+          { role: 'user', content: 'Run the tests' },
+          {
+            role: 'assistant',
+            content: 'Running tests',
+            tool_uses: [
+              {
+                tool: 'bash',
+                input: { command: 'vitest src/auth.test.ts' },
+                output: { text: 'All tests passed' },
+                status: 'success',
+              },
+            ],
+          },
+        ],
+      });
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      expect(trajectory.task.domain).toBe('testing');
+    });
+
+    it('should infer python domain', () => {
+      const session = makeSession({
+        messages: [
+          { role: 'user', content: 'Fix the Python script' },
+          {
+            role: 'assistant',
+            content: 'Reading',
+            tool_uses: [
+              {
+                tool: 'Read',
+                input: { file_path: '/src/main.py' },
+                output: { text: 'import os' },
+                status: 'success',
+              },
+            ],
+          },
+        ],
+      });
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      expect(trajectory.task.domain).toBe('python');
+    });
+
+    it('should use configured domain override', () => {
+      const session = makeSession();
+      const source = new DataclawTrajectorySource('/dev/null', {
+        domain: 'custom-domain',
+      });
+      const trajectory = source.synthesize(session);
+
+      expect(trajectory.task.domain).toBe('custom-domain');
+    });
+
+    it('should extract steps from tool_use messages', () => {
+      const session = makeSession();
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      // 2 tool-use steps + 1 text-only step
+      expect(trajectory.steps.length).toBe(3);
+
+      // First step: Read tool
+      expect(trajectory.steps[0].action).toContain('Read');
+      expect(trajectory.steps[0].observation).toContain('login');
+      expect(trajectory.steps[0].metadata?.toolName).toBe('Read');
+      expect(trajectory.steps[0].metadata?.toolStatus).toBe('success');
+
+      // Second step: Edit tool
+      expect(trajectory.steps[1].action).toContain('Edit');
+      expect(trajectory.steps[1].observation).toContain('File edited');
+      expect(trajectory.steps[1].metadata?.toolName).toBe('Edit');
+
+      // Third step: pure text response (content used as both thought and action)
+      expect(trajectory.steps[2].thought).toContain('fix has been applied');
+      expect(trajectory.steps[2].action).toContain('fix has been applied');
+    });
+
+    it('should preserve thinking as thought on tool steps', () => {
+      const session = makeSession();
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      // First tool step has explicit thinking
+      expect(trajectory.steps[0].thought).toContain('user wants me to fix');
+    });
+
+    it('should infer success heuristically for normal sessions', () => {
+      const session = makeSession();
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      expect(trajectory.outcome.success).toBe(true);
+      expect(trajectory.outcome.partialScore).toBeGreaterThan(0);
+    });
+
+    it('should infer failure when high tool error rate', () => {
+      const session = makeSession({
+        messages: [
+          { role: 'user', content: 'Fix it' },
+          {
+            role: 'assistant',
+            content: 'Trying',
+            tool_uses: [
+              { tool: 'bash', input: { command: 'npm build' }, output: { text: 'error' }, status: 'error' },
+              { tool: 'bash', input: { command: 'npm build' }, output: { text: 'error' }, status: 'error' },
+              { tool: 'bash', input: { command: 'npm build' }, output: { text: 'ok' }, status: 'success' },
+            ],
+          },
+        ],
+      });
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      expect(trajectory.outcome.success).toBe(false);
+    });
+
+    it('should use optimistic strategy when configured', () => {
+      const session = makeSession({
+        messages: [
+          { role: 'user', content: 'do something' },
+          { role: 'assistant', content: 'I was unable to complete the task.' },
+        ],
+      });
+      const source = new DataclawTrajectorySource('/dev/null', {
+        outcomeStrategy: 'optimistic',
+      });
+      const trajectory = source.synthesize(session);
+
+      expect(trajectory.outcome.success).toBe(true);
+      expect(trajectory.outcome.verificationDetails).toEqual({
+        strategy: 'optimistic',
+      });
+    });
+
+    it('should compute total tokens from stats', () => {
+      const session = makeSession();
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      expect(trajectory.totalTokens).toBe(18000); // 15000 + 3000
+    });
+
+    it('should compute wall time from start/end', () => {
+      const session = makeSession();
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      expect(trajectory.wallTimeSeconds).toBe(1800); // 30 minutes
+    });
+
+    it('should handle missing start/end times', () => {
+      const session = makeSession({
+        start_time: null,
+        end_time: null,
+      });
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      expect(trajectory.wallTimeSeconds).toBe(0);
+    });
+
+    it('should set agentId from model', () => {
+      const session = makeSession();
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      expect(trajectory.agentId).toBe('claude-opus-4-6');
+    });
+
+    it('should preserve dataclaw-specific metadata', () => {
+      const session = makeSession();
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      expect(trajectory.metadata.source).toBe('dataclaw');
+      expect(trajectory.metadata.dataclawSource).toBe('claude');
+      expect(trajectory.metadata.project).toBe('my-project');
+      expect(trajectory.metadata.model).toBe('claude-opus-4-6');
+      expect(trajectory.metadata.gitBranch).toBe('main');
+      expect(trajectory.metadata.sessionId).toBe('sess-abc-123');
+    });
+
+    it('should estimate LLM calls from assistant message count', () => {
+      const session = makeSession();
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      expect(trajectory.llmCalls).toBe(3); // 3 assistant messages
+    });
+
+    it('should handle codex-style tool output format', () => {
+      const session = makeSession({
+        messages: [
+          { role: 'user', content: 'Run ls' },
+          {
+            role: 'assistant',
+            content: 'Running',
+            tool_uses: [
+              {
+                tool: 'exec_command',
+                input: { command: 'ls' },
+                output: { exit_code: 0, wall_time: '1 seconds', output: 'file1.ts\nfile2.ts' },
+                status: 'success',
+              },
+            ],
+          },
+        ],
+      });
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      expect(trajectory.steps[0].observation).toBe('file1.ts\nfile2.ts');
+    });
+
+    it('should handle empty tool output', () => {
+      const session = makeSession({
+        messages: [
+          { role: 'user', content: 'Do it' },
+          {
+            role: 'assistant',
+            content: 'Done',
+            tool_uses: [
+              {
+                tool: 'Write',
+                input: { file_path: '/foo.ts', content: 'hello' },
+                output: {},
+                status: 'success',
+              },
+            ],
+          },
+        ],
+      });
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      expect(trajectory.steps[0].observation).toBe('');
+    });
+
+    it('should handle session with no tool uses', () => {
+      const session = makeSession({
+        messages: [
+          { role: 'user', content: 'What is TypeScript?' },
+          {
+            role: 'assistant',
+            content: 'TypeScript is a typed superset of JavaScript.',
+          },
+        ],
+      });
+      const source = new DataclawTrajectorySource('/dev/null');
+      const trajectory = source.synthesize(session);
+
+      expect(trajectory.steps.length).toBe(1);
+      expect(trajectory.steps[0].thought).toBe(
+        'TypeScript is a typed superset of JavaScript.',
+      );
+    });
+  });
+
+  // ============================================================================
+  // File loading tests
+  // ============================================================================
+
+  describe('load (from files)', () => {
+    let tmpDir: string;
+
+    beforeEach(async () => {
+      tmpDir = await mkdtemp(join(tmpdir(), 'dataclaw-source-test-'));
+    });
+
+    afterEach(async () => {
+      await rm(tmpDir, { recursive: true, force: true });
+    });
+
+    it('should have name "dataclaw"', () => {
+      const source = new DataclawTrajectorySource(tmpDir);
+      expect(source.name).toBe('dataclaw');
+    });
+
+    it('should load sessions from a JSONL file', async () => {
+      const sessions = [
+        makeSession({ session_id: 'sess-1' }),
+        makeSession({ session_id: 'sess-2' }),
+      ];
+      const filePath = await writeJsonl(tmpDir, 'conversations.jsonl', sessions);
+
+      const source = new DataclawTrajectorySource(filePath);
+      const items = await source.load();
+      expect(items).toHaveLength(2);
+      expect(items[0].trajectory.id).toBe('dataclaw-claude-sess-1');
+      expect(items[1].trajectory.id).toBe('dataclaw-claude-sess-2');
+    });
+
+    it('should load all .jsonl files from a directory', async () => {
+      await writeJsonl(tmpDir, 'batch1.jsonl', [
+        makeSession({ session_id: 'sess-1' }),
+      ]);
+      await writeJsonl(tmpDir, 'batch2.jsonl', [
+        makeSession({ session_id: 'sess-2' }),
+        makeSession({ session_id: 'sess-3' }),
+      ]);
+
+      const source = new DataclawTrajectorySource(tmpDir);
+      const items = await source.load();
+      expect(items).toHaveLength(3);
+    });
+
+    it('should skip non-jsonl files in directory', async () => {
+      await writeJsonl(tmpDir, 'conversations.jsonl', [
+        makeSession({ session_id: 'sess-1' }),
+      ]);
+      await writeFile(
+        join(tmpDir, 'metadata.json'),
+        JSON.stringify({ sessions: 1 }),
+        'utf-8',
+      );
+      await writeFile(join(tmpDir, 'README.md'), '# Dataset', 'utf-8');
+
+      const source = new DataclawTrajectorySource(tmpDir);
+      const items = await source.load();
+      expect(items).toHaveLength(1);
+    });
+
+    it('should skip malformed JSONL lines', async () => {
+      const content = [
+        JSON.stringify(makeSession({ session_id: 'good-1' })),
+        'this is not json',
+        JSON.stringify(makeSession({ session_id: 'good-2' })),
+        '{"incomplete": true',
+      ].join('\n');
+      await writeFile(join(tmpDir, 'data.jsonl'), content, 'utf-8');
+
+      const source = new DataclawTrajectorySource(tmpDir);
+      const items = await source.load();
+      expect(items).toHaveLength(2);
+    });
+
+    it('should skip empty lines in JSONL', async () => {
+      const content = [
+        JSON.stringify(makeSession({ session_id: 'sess-1' })),
+        '',
+        '  ',
+        JSON.stringify(makeSession({ session_id: 'sess-2' })),
+      ].join('\n');
+      await writeFile(join(tmpDir, 'data.jsonl'), content, 'utf-8');
+
+      const source = new DataclawTrajectorySource(tmpDir);
+      const items = await source.load();
+      expect(items).toHaveLength(2);
+    });
+
+    it('should handle non-existent path gracefully', async () => {
+      const source = new DataclawTrajectorySource('/tmp/does-not-exist-99999');
+      const items = await source.load();
+      expect(items).toHaveLength(0);
+    });
+
+    it('should handle empty directory', async () => {
+      const source = new DataclawTrajectorySource(tmpDir);
+      const items = await source.load();
+      expect(items).toHaveLength(0);
+    });
+
+    it('should filter by domain', async () => {
+      await writeJsonl(tmpDir, 'data.jsonl', [
+        makeSession({
+          session_id: 'sess-ts',
+          messages: [
+            { role: 'user', content: 'Fix' },
+            {
+              role: 'assistant',
+              content: 'Ok',
+              tool_uses: [
+                { tool: 'Read', input: { file_path: '/src/main.ts' }, output: { text: '' }, status: 'success' },
+              ],
+            },
+          ],
+        }),
+        makeSession({
+          session_id: 'sess-py',
+          messages: [
+            { role: 'user', content: 'Fix' },
+            {
+              role: 'assistant',
+              content: 'Ok',
+              tool_uses: [
+                { tool: 'Read', input: { file_path: '/src/main.py' }, output: { text: '' }, status: 'success' },
+              ],
+            },
+          ],
+        }),
+      ]);
+
+      const source = new DataclawTrajectorySource(tmpDir);
+      const items = await source.load({ domain: 'python' });
+      expect(items).toHaveLength(1);
+      expect(items[0].trajectory.task.domain).toBe('python');
+    });
+
+    it('should filter by successOnly', async () => {
+      await writeJsonl(tmpDir, 'data.jsonl', [
+        makeSession({ session_id: 'good-session' }),
+        makeSession({
+          session_id: 'bad-session',
+          messages: [
+            { role: 'user', content: 'Fix' },
+            {
+              role: 'assistant',
+              content: 'Trying',
+              tool_uses: [
+                { tool: 'bash', input: { command: 'x' }, output: {}, status: 'error' },
+                { tool: 'bash', input: { command: 'x' }, output: {}, status: 'error' },
+              ],
+            },
+          ],
+        }),
+      ]);
+
+      const source = new DataclawTrajectorySource(tmpDir);
+      const items = await source.load({ successOnly: true });
+      expect(items).toHaveLength(1);
+    });
+
+    it('should respect limit', async () => {
+      await writeJsonl(tmpDir, 'data.jsonl', [
+        makeSession({ session_id: 'sess-1' }),
+        makeSession({ session_id: 'sess-2' }),
+        makeSession({ session_id: 'sess-3' }),
+      ]);
+
+      const source = new DataclawTrajectorySource(tmpDir);
+      const items = await source.load({ limit: 2 });
+      expect(items).toHaveLength(2);
+    });
+
+    it('should filter by since', async () => {
+      await writeJsonl(tmpDir, 'data.jsonl', [
+        makeSession({
+          session_id: 'old',
+          start_time: '2025-01-01T00:00:00+00:00',
+        }),
+        makeSession({
+          session_id: 'new',
+          start_time: '2026-06-01T00:00:00+00:00',
+        }),
+      ]);
+
+      const source = new DataclawTrajectorySource(tmpDir);
+      const items = await source.load({
+        since: new Date('2026-01-01'),
+      });
+      expect(items).toHaveLength(1);
+      expect(items[0].trajectory.metadata.sessionId).toBe('new');
+    });
+
+    it('should count items', async () => {
+      await writeJsonl(tmpDir, 'data.jsonl', [
+        makeSession({ session_id: 'sess-1' }),
+        makeSession({ session_id: 'sess-2' }),
+      ]);
+
+      const source = new DataclawTrajectorySource(tmpDir);
+      expect(await source.count()).toBe(2);
+    });
+
+    it('should filter by dataclaw source option', async () => {
+      await writeJsonl(tmpDir, 'data.jsonl', [
+        makeSession({ session_id: 'sess-claude', source: 'claude' }),
+        makeSession({ session_id: 'sess-codex', source: 'codex' }),
+        makeSession({ session_id: 'sess-gemini', source: 'gemini' }),
+      ]);
+
+      const source = new DataclawTrajectorySource(tmpDir, {
+        sources: ['claude', 'codex'],
+      });
+      const items = await source.load();
+      expect(items).toHaveLength(2);
+    });
+
+    it('should filter by model option', async () => {
+      await writeJsonl(tmpDir, 'data.jsonl', [
+        makeSession({ session_id: 'sess-1', model: 'claude-opus-4-6' }),
+        makeSession({ session_id: 'sess-2', model: 'gpt-5.3-codex' }),
+      ]);
+
+      const source = new DataclawTrajectorySource(tmpDir, {
+        models: ['claude-opus-4-6'],
+      });
+      const items = await source.load();
+      expect(items).toHaveLength(1);
+      expect(items[0].trajectory.metadata.model).toBe('claude-opus-4-6');
+    });
+  });
+});