cognitive-core 0.1.0 → 0.1.2

package/tests/integration/huggingface-e2e.test.ts

@@ -0,0 +1,627 @@
+/**
+ * End-to-end integration test: HuggingFaceTrajectorySource → full pipeline
+ *
+ * Loads dataclaw-format sessions through a mock HF datasets-server,
+ * then runs the full pipeline: load → analyze → extract knowledge.
+ *
+ * Uses realistic dataclaw session data matching the real HF dataset schema
+ * (peteromallet/dataclaw-peteromallet format).
+ */
+import { describe, it, expect, vi } from 'vitest';
+import { HuggingFaceTrajectorySource } from '../../src/learning/trajectory-sources/huggingface.js';
+import { TrajectorySchema } from '../../src/types/trajectory.js';
+import { createAnalyzer } from '../../src/learning/analyzer.js';
+import { createKnowledgeExtractor } from '../../src/learning/knowledge-extractor.js';
+import type { DataclawSession } from '../../src/types/dataclaw.js';
+
+// ── Realistic dataclaw sessions matching real HF exports ──────────────
+
+/** Claude Code session: multi-step bug fix with Read/Edit/bash */
+function claudeBugfixSession(): DataclawSession {
+  return {
+    session_id: 'a1b2c3d4-e5f6-7890-abcd-ef1234567890',
+    project: 'my-web-app',
+    model: 'claude-opus-4-6',
+    source: 'claude',
+    git_branch: 'fix/null-email',
+    start_time: '2026-02-01T09:00:00+00:00',
+    end_time: '2026-02-01T09:25:00+00:00',
+    messages: [
+      {
+        role: 'user',
+        content: 'The login endpoint returns 500 when the user has no email. Can you fix it?',
+        timestamp: '2026-02-01T09:00:05+00:00',
+      },
+      {
+        role: 'assistant',
+        content: "Let me look at the login handler to understand what's happening.",
+        thinking: 'User reports a 500 error on login when user has no email. Likely a null reference in validation code.',
+        tool_uses: [
+          {
+            tool: 'Read',
+            input: { file_path: '/src/routes/auth.ts' },
+            output: { text: 'export async function login(req: Request) {\n  const user = await findUser(req.body.id);\n  const email = user.email.toLowerCase();\n  // ...\n}' },
+            status: 'success',
+          },
+        ],
+        timestamp: '2026-02-01T09:00:15+00:00',
+      },
+      {
+        role: 'assistant',
+        content: "Found it — `user.email.toLowerCase()` throws when email is null. Adding optional chaining.",
+        tool_uses: [
+          {
+            tool: 'Edit',
+            input: {
+              file_path: '/src/routes/auth.ts',
+              old_string: 'const email = user.email.toLowerCase();',
+              new_string: "const email = user.email?.toLowerCase() ?? '';",
+            },
+            output: { text: 'File edited successfully' },
+            status: 'success',
+          },
+        ],
+        timestamp: '2026-02-01T09:01:00+00:00',
+      },
+      {
+        role: 'assistant',
+        content: "Running tests to verify the fix.",
+        tool_uses: [
+          {
+            tool: 'bash',
+            input: { command: 'npm test -- --grep "login"' },
+            output: { text: 'PASS  tests/auth.test.ts\n  login endpoint\n    ✓ returns 200 for valid user (45ms)\n    ✓ returns 200 for user without email (12ms)\n    ✓ returns 401 for invalid credentials (8ms)\n\nTest Suites: 1 passed, 1 total\nTests:       3 passed, 3 total' },
+            status: 'success',
+          },
+        ],
+        timestamp: '2026-02-01T09:02:00+00:00',
+      },
+      {
+        role: 'assistant',
+        content: 'All tests pass. The fix adds optional chaining to handle users without an email, preventing the 500 error.',
+        timestamp: '2026-02-01T09:02:30+00:00',
+      },
+    ],
+    stats: {
+      user_messages: 1,
+      assistant_messages: 4,
+      tool_uses: 3,
+      input_tokens: 12000,
+      output_tokens: 2500,
+    },
+  };
+}
+
+/** Codex session: Python feature with exec_command (different tool format) */
+function codexFeatureSession(): DataclawSession {
+  return {
+    session_id: 'f9e8d7c6-b5a4-3210-fedc-ba0987654321',
+    project: 'ml-pipeline',
+    model: 'gpt-5.3-codex',
+    source: 'codex',
+    git_branch: 'feature/csv-loader',
+    start_time: '2026-02-10T14:00:00+00:00',
+    end_time: '2026-02-10T14:45:00+00:00',
+    messages: [
+      {
+        role: 'user',
+        content: 'Add a CSV data loader to the pipeline module that supports chunked reading for large files',
+        timestamp: '2026-02-10T14:00:00+00:00',
+      },
+      {
+        role: 'assistant',
+        content: "Let me check the existing pipeline structure.",
+        tool_uses: [
+          {
+            tool: 'exec_command',
+            input: { command: 'ls src/pipeline/' },
+            output: { exit_code: 0, wall_time: '0.1 seconds', output: '__init__.py\nbase.py\njson_loader.py\nparquet_loader.py' },
+            status: 'success',
+          },
+        ],
+        timestamp: '2026-02-10T14:00:30+00:00',
+      },
+      {
+        role: 'assistant',
+        content: 'Creating the CSV loader following the existing pattern.',
+        tool_uses: [
+          {
+            tool: 'write_file',
+            input: { file_path: 'src/pipeline/csv_loader.py', content: 'import pandas as pd\nfrom .base import BaseLoader\n\nclass CSVLoader(BaseLoader):\n    def load(self, path: str, chunk_size: int = 10000):\n        for chunk in pd.read_csv(path, chunksize=chunk_size):\n            yield self.process(chunk)\n' },
+            output: {},
+            status: 'success',
+          },
+        ],
+        timestamp: '2026-02-10T14:05:00+00:00',
+      },
+      {
+        role: 'assistant',
+        content: 'Running tests.',
+        tool_uses: [
+          {
+            tool: 'exec_command',
+            input: { command: 'python -m pytest tests/test_csv_loader.py -v' },
+            output: { exit_code: 0, wall_time: '3 seconds', output: 'tests/test_csv_loader.py::test_load_small_csv PASSED\ntests/test_csv_loader.py::test_chunked_reading PASSED\n\n2 passed in 2.1s' },
+            status: 'success',
+          },
+        ],
+        timestamp: '2026-02-10T14:10:00+00:00',
+      },
+      {
+        role: 'assistant',
+        content: 'The CSV data loader is implemented and tested. It supports chunked reading for memory efficiency.',
+        timestamp: '2026-02-10T14:11:00+00:00',
+      },
+    ],
+    stats: {
+      user_messages: 1,
+      assistant_messages: 4,
+      tool_uses: 3,
+      input_tokens: 8000,
+      output_tokens: 2000,
+    },
+  };
+}
+
+/** A failed session with error tools and failure signals in final message */
+function failedDeploySession(): DataclawSession {
+  return {
+    session_id: 'dead-beef-0000-1111-222233334444',
+    project: 'my-web-app',
+    model: 'claude-opus-4-6',
+    source: 'claude',
+    git_branch: 'main',
+    start_time: '2026-02-05T16:00:00+00:00',
+    end_time: '2026-02-05T16:10:00+00:00',
+    messages: [
+      {
+        role: 'user',
+        content: 'Deploy the app to production',
+        timestamp: '2026-02-05T16:00:00+00:00',
+      },
+      {
+        role: 'assistant',
+        content: 'Attempting deployment.',
+        tool_uses: [
+          {
+            tool: 'bash',
+            input: { command: 'npm run deploy' },
+            output: { text: 'Error: Missing DEPLOY_TOKEN environment variable' },
+            status: 'error',
+          },
+        ],
+        timestamp: '2026-02-05T16:01:00+00:00',
+      },
+      {
+        role: 'assistant',
+        content: 'Retrying with alternate method.',
+        tool_uses: [
+          {
+            tool: 'bash',
+            input: { command: 'npx deploy --force' },
+            output: { text: 'Fatal: permission denied' },
+            status: 'error',
+          },
+        ],
+        timestamp: '2026-02-05T16:02:00+00:00',
+      },
+      {
+        role: 'assistant',
+        content: 'I was unable to deploy. The deployment requires a DEPLOY_TOKEN that is not available in this environment.',
+        timestamp: '2026-02-05T16:03:00+00:00',
+      },
+    ],
+    stats: {
+      user_messages: 1,
+      assistant_messages: 3,
+      tool_uses: 2,
+      input_tokens: 5000,
+      output_tokens: 1000,
+    },
+  };
+}
+
+/** Minimal conversation-only session — no tools, null git_branch */
+function conversationOnlySession(): DataclawSession {
+  return {
+    session_id: 'aaaa-bbbb-cccc-dddd-eeee11112222',
+    project: 'my-web-app',
+    model: 'claude-opus-4-6',
+    source: 'claude',
+    git_branch: null,
+    start_time: '2026-02-15T11:00:00+00:00',
+    end_time: '2026-02-15T11:05:00+00:00',
+    messages: [
+      {
+        role: 'user',
+        content: 'Explain how the middleware chain works in our Express app',
+        timestamp: '2026-02-15T11:00:00+00:00',
+      },
+      {
+        role: 'assistant',
+        content: 'Your Express middleware chain processes requests in order: first auth validation, then rate limiting, then the route handler. Each middleware calls next() to pass control to the next function.',
+        timestamp: '2026-02-15T11:00:30+00:00',
+      },
+    ],
+    stats: {
+      user_messages: 1,
+      assistant_messages: 1,
+      tool_uses: 0,
+      input_tokens: 3000,
+      output_tokens: 500,
+    },
+  };
+}
+
+// ── Mock HF datasets-server ──────────────────────────────────────────
+
+type FetchFn = typeof globalThis.fetch;
+
+const ALL_SESSIONS = [
+  claudeBugfixSession(),
+  codexFeatureSession(),
+  failedDeploySession(),
+  conversationOnlySession(),
+];
+
+function createMockHfServer(
+  sessions: DataclawSession[] = ALL_SESSIONS,
+): FetchFn {
+  return vi.fn(async (input: RequestInfo | URL, _init?: RequestInit) => {
+    const url = typeof input === 'string' ? input : input.toString();
+
+    if (url.includes('/info')) {
+      return new Response(
+        JSON.stringify({
+          dataset_info: {
+            default: { num_rows: sessions.length, dataset_name: 'test' },
+          },
+        }),
+        { status: 200, headers: { 'Content-Type': 'application/json' } },
+      );
+    }
+
+    if (url.includes('/rows')) {
+      const parsed = new URL(url);
+      const offset = parseInt(parsed.searchParams.get('offset') ?? '0', 10);
+      const length = parseInt(parsed.searchParams.get('length') ?? '100', 10);
+      const page = sessions.slice(offset, offset + length);
+
+      return new Response(
+        JSON.stringify({
+          features: [],
+          rows: page.map((s, i) => ({ row_idx: offset + i, row: s })),
+          num_rows_total: sessions.length,
+          num_rows_per_page: 100,
+          partial: false,
+        }),
+        { status: 200, headers: { 'Content-Type': 'application/json' } },
+      );
+    }
+
+    return new Response('Not Found', { status: 404 });
+  }) as unknown as FetchFn;
+}
+
+// ============================================================================
+// E2E Tests
+// ============================================================================
+
+describe('HuggingFace E2E: load → analyze → extract', () => {
+  // ── Stage 1: Load from HF ─────────────────────────────────────────
+
+  describe('Stage 1: HF dataset loading', () => {
+    it('should load all sessions from mock HF dataset', async () => {
+      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
+        fetchFn: createMockHfServer(),
+      });
+      const items = await source.load();
+      expect(items).toHaveLength(4);
+    });
+
+    it('should produce valid Trajectory objects for every session', async () => {
+      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
+        fetchFn: createMockHfServer(),
+      });
+      const items = await source.load();
+
+      for (const { trajectory } of items) {
+        const parsed = TrajectorySchema.safeParse(trajectory);
+        expect(parsed.success, `Invalid trajectory: ${trajectory.id}`).toBe(true);
+      }
+    });
+
+    it('should tag all trajectories with dataclaw source metadata', async () => {
+      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
+        fetchFn: createMockHfServer(),
+      });
+      const items = await source.load();
+
+      for (const { trajectory } of items) {
+        expect(trajectory.metadata.source).toBe('dataclaw');
+        expect(trajectory.metadata.dataclawSource).toBeTruthy();
+        expect(trajectory.metadata.sessionId).toBeTruthy();
+      }
+    });
+
+    it('should infer correct domains from tool file paths', async () => {
+      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
+        fetchFn: createMockHfServer(),
+      });
+      const items = await source.load();
+
+      const claude = items.find((i) => i.trajectory.id.includes('a1b2c3d4'));
+      const codex = items.find((i) => i.trajectory.id.includes('f9e8d7c6'));
+
+      expect(claude!.trajectory.task.domain).toBe('typescript');
+      expect(codex!.trajectory.task.domain).toBe('python');
+    });
+
+    it('should detect success/failure correctly', async () => {
+      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
+        fetchFn: createMockHfServer(),
+      });
+      const items = await source.load();
+
+      const claude = items.find((i) => i.trajectory.id.includes('a1b2c3d4'));
+      const failed = items.find((i) => i.trajectory.id.includes('dead-beef'));
+
+      expect(claude!.trajectory.outcome.success).toBe(true);
+      expect(failed!.trajectory.outcome.success).toBe(false);
+    });
+
+    it('should paginate across multiple pages', async () => {
+      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
+        fetchFn: createMockHfServer(),
+        pageSize: 2,  // Force 2 pages for 4 sessions
+      });
+      const items = await source.load();
+
+      expect(items).toHaveLength(4);
+    });
+  });
+
+  // ── Stage 2: Analyzer ─────────────────────────────────────────────
+
+  describe('Stage 2: TrajectoryAnalyzer on HF-loaded trajectories', () => {
+    it('should analyze all successful trajectories without error', async () => {
+      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
+        fetchFn: createMockHfServer(),
+      });
+      const items = await source.load({ successOnly: true });
+      const analyzer = createAnalyzer('simple');
+
+      for (const { trajectory } of items) {
+        const analysis = await analyzer.analyze(trajectory);
+
+        expect(analysis).toBeDefined();
+        expect(typeof analysis.success).toBe('boolean');
+        expect(analysis.stepAttribution).toHaveLength(trajectory.steps.length);
+        expect(analysis.keySteps).toBeDefined();
+        expect(typeof analysis.abstractable).toBe('boolean');
+      }
+    });
+
+    it('should analyze failed trajectories and detect error patterns', async () => {
+      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
+        fetchFn: createMockHfServer(),
+      });
+      const items = await source.load();
+      const failed = items.find((i) => i.trajectory.id.includes('dead-beef'));
+      const analyzer = createAnalyzer('simple');
+
+      const analysis = await analyzer.analyze(failed!.trajectory);
+
+      expect(analysis.success).toBe(false);
+      expect(analysis.stepAttribution.length).toBeGreaterThan(0);
+    });
+
+    it('should identify key steps in the Claude bugfix session', async () => {
+      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
+        fetchFn: createMockHfServer(),
+      });
+      const items = await source.load();
+      const claude = items.find((i) => i.trajectory.id.includes('a1b2c3d4'));
+      const analyzer = createAnalyzer('simple');
+
+      const analysis = await analyzer.analyze(claude!.trajectory);
+
+      // Multi-step trajectory should have identifiable key steps
+      expect(analysis.keySteps.length).toBeGreaterThan(0);
+      // At least one step should have non-zero attribution
+      expect(analysis.stepAttribution.some((s) => s > 0)).toBe(true);
+    });
+
+    it('should mark multi-step successful trajectories as abstractable', async () => {
+      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
+        fetchFn: createMockHfServer(),
+      });
+      const items = await source.load();
+      const claude = items.find((i) => i.trajectory.id.includes('a1b2c3d4'));
+      const analyzer = createAnalyzer('simple');
+
+      const analysis = await analyzer.analyze(claude!.trajectory);
+
+      // A 4-step successful session with tool usage should be abstractable
+      expect(analysis.abstractable).toBe(true);
+    });
+
+    it('should extract training examples from successful trajectories', async () => {
+      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
+        fetchFn: createMockHfServer(),
+      });
+      const items = await source.load();
+      const claude = items.find((i) => i.trajectory.id.includes('a1b2c3d4'));
+      const analyzer = createAnalyzer('simple');
+
+      const analysis = await analyzer.analyze(claude!.trajectory);
+
+      expect(analysis.trainingExamples.length).toBeGreaterThan(0);
+      for (const ex of analysis.trainingExamples) {
+        expect(ex.input).toBeTruthy();
+        expect(typeof ex.stepIndex).toBe('number');
+      }
+    });
+  });
+
+  // ── Stage 3: Knowledge Extraction ─────────────────────────────────
+
+  describe('Stage 3: KnowledgeExtractor on HF-loaded & analyzed trajectories', () => {
+    it('should extract knowledge from successful Claude session', async () => {
+      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
+        fetchFn: createMockHfServer(),
+      });
+      const items = await source.load();
+      const claude = items.find((i) => i.trajectory.id.includes('a1b2c3d4'));
+      const analyzer = createAnalyzer('simple');
+      const extractor = createKnowledgeExtractor();
+
+      const analysis = await analyzer.analyze(claude!.trajectory);
+      const knowledge = extractor.extract(claude!.trajectory, analysis);
+
+      expect(knowledge).toBeDefined();
+      expect(knowledge.observations).toBeDefined();
+      expect(knowledge.entityUpdates).toBeDefined();
+      expect(knowledge.links).toBeDefined();
+      expect(knowledge.playbookLinks).toBeDefined();
+      expect(knowledge.contradictions).toBeDefined();
+    });
+
+    it('should extract causal chains from failed session', async () => {
+      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
+        fetchFn: createMockHfServer(),
+      });
+      const items = await source.load();
+      const failed = items.find((i) => i.trajectory.id.includes('dead-beef'));
+      const analyzer = createAnalyzer('simple');
+      const extractor = createKnowledgeExtractor();
+
+      const analysis = await analyzer.analyze(failed!.trajectory);
+      const knowledge = extractor.extract(failed!.trajectory, analysis);
+
+      // The failed session has error→failure pattern that should produce causal chains
+      // or at minimum, the extraction should complete without throwing
+      expect(knowledge).toBeDefined();
+      expect(Array.isArray(knowledge.observations)).toBe(true);
+      expect(Array.isArray(knowledge.links)).toBe(true);
+    });
+
+    it('should run full pipeline on all sessions without errors', async () => {
+      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
+        fetchFn: createMockHfServer(),
+      });
+      const items = await source.load();
+      const analyzer = createAnalyzer('simple');
+      const extractor = createKnowledgeExtractor();
+
+      for (const { trajectory } of items) {
+        const analysis = await analyzer.analyze(trajectory);
+        const knowledge = extractor.extract(trajectory, analysis);
+
+        // Every trajectory should produce valid analysis and knowledge
+        expect(analysis.stepAttribution).toHaveLength(trajectory.steps.length);
+        expect(knowledge).toBeDefined();
+        expect(Array.isArray(knowledge.observations)).toBe(true);
+      }
+    });
+
+    it('should extract observations from the Codex Python session', async () => {
+      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
+        fetchFn: createMockHfServer(),
+      });
+      const items = await source.load();
+      const codex = items.find((i) => i.trajectory.id.includes('f9e8d7c6'));
+      const analyzer = createAnalyzer('simple');
+      const extractor = createKnowledgeExtractor();
+
+      const analysis = await analyzer.analyze(codex!.trajectory);
+      const knowledge = extractor.extract(codex!.trajectory, analysis);
+
+      expect(knowledge).toBeDefined();
+      // Codex session writes Python files — extraction should succeed
+      expect(Array.isArray(knowledge.observations)).toBe(true);
+    });
+
+    it('should handle conversation-only session through full pipeline', async () => {
+      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
+        fetchFn: createMockHfServer(),
+      });
+      const items = await source.load();
+      const qa = items.find((i) => i.trajectory.id.includes('aaaa-bbbb'));
+      const analyzer = createAnalyzer('simple');
+      const extractor = createKnowledgeExtractor();
+
+      const analysis = await analyzer.analyze(qa!.trajectory);
+      const knowledge = extractor.extract(qa!.trajectory, analysis);
+
+      // Single-step Q&A should still work through the full pipeline
+      expect(analysis.stepAttribution).toHaveLength(1);
+      expect(knowledge).toBeDefined();
+    });
+  });
+
+  // ── Stage 4: Full pipeline with filtering ─────────────────────────
+
+  describe('Stage 4: filtered pipeline runs', () => {
+    it('should pipeline only Claude sessions', async () => {
+      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
+        fetchFn: createMockHfServer(),
+        sources: ['claude'],
+      });
+      const items = await source.load({ successOnly: true });
+      const analyzer = createAnalyzer('simple');
+      const extractor = createKnowledgeExtractor();
+
+      // Should have 2 Claude success sessions (bugfix + conversation)
+      expect(items).toHaveLength(2);
+
+      let totalObservations = 0;
+      let totalTrainingExamples = 0;
+
+      for (const { trajectory } of items) {
+        expect(trajectory.metadata.dataclawSource).toBe('claude');
+        expect(trajectory.outcome.success).toBe(true);
+
+        const analysis = await analyzer.analyze(trajectory);
+        const knowledge = extractor.extract(trajectory, analysis);
+
+        totalObservations += knowledge.observations.length;
+        totalTrainingExamples += analysis.trainingExamples.length;
+      }
+
+      // The pipeline should produce some learning signal
+      expect(totalObservations + totalTrainingExamples).toBeGreaterThan(0);
+    });
+
+    it('should pipeline TypeScript-domain only', async () => {
+      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
+        fetchFn: createMockHfServer(),
+      });
+      const items = await source.load({ domain: 'typescript' });
+      const analyzer = createAnalyzer('simple');
+
+      expect(items).toHaveLength(1);
+      expect(items[0].trajectory.task.domain).toBe('typescript');
+
+      const analysis = await analyzer.analyze(items[0].trajectory);
+      expect(analysis.abstractable).toBe(true);
+    });
+
+    it('should pipeline with limit', async () => {
+      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
+        fetchFn: createMockHfServer(),
+      });
+      const items = await source.load({ limit: 2 });
+      const analyzer = createAnalyzer('simple');
+      const extractor = createKnowledgeExtractor();
+
+      expect(items).toHaveLength(2);
+
+      for (const { trajectory } of items) {
+        const analysis = await analyzer.analyze(trajectory);
+        const knowledge = extractor.extract(trajectory, analysis);
+        expect(knowledge).toBeDefined();
+      }
+    });
+  });
+});