cognitive-core 0.2.4 → 0.3.1

package/tests/integration/huggingface-e2e.test.ts

@@ -1,627 +0,0 @@
-/**
- * End-to-end integration test: HuggingFaceTrajectorySource → full pipeline
- *
- * Loads dataclaw-format sessions through a mock HF datasets-server,
- * then runs the full pipeline: load → analyze → extract knowledge.
- *
- * Uses realistic dataclaw session data matching the real HF dataset schema
- * (peteromallet/dataclaw-peteromallet format).
- */
-import { describe, it, expect, vi } from 'vitest';
-import { HuggingFaceTrajectorySource } from '../../src/learning/trajectory-sources/huggingface.js';
-import { TrajectorySchema } from '../../src/types/trajectory.js';
-import { createAnalyzer } from '../../src/learning/analyzer.js';
-import { createKnowledgeExtractor } from '../../src/learning/knowledge-extractor.js';
-import type { DataclawSession } from '../../src/types/dataclaw.js';
-
-// ── Realistic dataclaw sessions matching real HF exports ──────────────
-
-/** Claude Code session: multi-step bug fix with Read/Edit/bash */
-function claudeBugfixSession(): DataclawSession {
-  return {
-    session_id: 'a1b2c3d4-e5f6-7890-abcd-ef1234567890',
-    project: 'my-web-app',
-    model: 'claude-opus-4-6',
-    source: 'claude',
-    git_branch: 'fix/null-email',
-    start_time: '2026-02-01T09:00:00+00:00',
-    end_time: '2026-02-01T09:25:00+00:00',
-    messages: [
-      {
-        role: 'user',
-        content: 'The login endpoint returns 500 when the user has no email. Can you fix it?',
-        timestamp: '2026-02-01T09:00:05+00:00',
-      },
-      {
-        role: 'assistant',
-        content: "Let me look at the login handler to understand what's happening.",
-        thinking: 'User reports a 500 error on login when user has no email. Likely a null reference in validation code.',
-        tool_uses: [
-          {
-            tool: 'Read',
-            input: { file_path: '/src/routes/auth.ts' },
-            output: { text: 'export async function login(req: Request) {\n  const user = await findUser(req.body.id);\n  const email = user.email.toLowerCase();\n  // ...\n}' },
-            status: 'success',
-          },
-        ],
-        timestamp: '2026-02-01T09:00:15+00:00',
-      },
-      {
-        role: 'assistant',
-        content: "Found it — `user.email.toLowerCase()` throws when email is null. Adding optional chaining.",
-        tool_uses: [
-          {
-            tool: 'Edit',
-            input: {
-              file_path: '/src/routes/auth.ts',
-              old_string: 'const email = user.email.toLowerCase();',
-              new_string: "const email = user.email?.toLowerCase() ?? '';",
-            },
-            output: { text: 'File edited successfully' },
-            status: 'success',
-          },
-        ],
-        timestamp: '2026-02-01T09:01:00+00:00',
-      },
-      {
-        role: 'assistant',
-        content: "Running tests to verify the fix.",
-        tool_uses: [
-          {
-            tool: 'bash',
-            input: { command: 'npm test -- --grep "login"' },
-            output: { text: 'PASS  tests/auth.test.ts\n  login endpoint\n    ✓ returns 200 for valid user (45ms)\n    ✓ returns 200 for user without email (12ms)\n    ✓ returns 401 for invalid credentials (8ms)\n\nTest Suites: 1 passed, 1 total\nTests:       3 passed, 3 total' },
-            status: 'success',
-          },
-        ],
-        timestamp: '2026-02-01T09:02:00+00:00',
-      },
-      {
-        role: 'assistant',
-        content: 'All tests pass. The fix adds optional chaining to handle users without an email, preventing the 500 error.',
-        timestamp: '2026-02-01T09:02:30+00:00',
-      },
-    ],
-    stats: {
-      user_messages: 1,
-      assistant_messages: 4,
-      tool_uses: 3,
-      input_tokens: 12000,
-      output_tokens: 2500,
-    },
-  };
-}
-
-/** Codex session: Python feature with exec_command (different tool format) */
-function codexFeatureSession(): DataclawSession {
-  return {
-    session_id: 'f9e8d7c6-b5a4-3210-fedc-ba0987654321',
-    project: 'ml-pipeline',
-    model: 'gpt-5.3-codex',
-    source: 'codex',
-    git_branch: 'feature/csv-loader',
-    start_time: '2026-02-10T14:00:00+00:00',
-    end_time: '2026-02-10T14:45:00+00:00',
-    messages: [
-      {
-        role: 'user',
-        content: 'Add a CSV data loader to the pipeline module that supports chunked reading for large files',
-        timestamp: '2026-02-10T14:00:00+00:00',
-      },
-      {
-        role: 'assistant',
-        content: "Let me check the existing pipeline structure.",
-        tool_uses: [
-          {
-            tool: 'exec_command',
-            input: { command: 'ls src/pipeline/' },
-            output: { exit_code: 0, wall_time: '0.1 seconds', output: '__init__.py\nbase.py\njson_loader.py\nparquet_loader.py' },
-            status: 'success',
-          },
-        ],
-        timestamp: '2026-02-10T14:00:30+00:00',
-      },
-      {
-        role: 'assistant',
-        content: 'Creating the CSV loader following the existing pattern.',
-        tool_uses: [
-          {
-            tool: 'write_file',
-            input: { file_path: 'src/pipeline/csv_loader.py', content: 'import pandas as pd\nfrom .base import BaseLoader\n\nclass CSVLoader(BaseLoader):\n    def load(self, path: str, chunk_size: int = 10000):\n        for chunk in pd.read_csv(path, chunksize=chunk_size):\n            yield self.process(chunk)\n' },
-            output: {},
-            status: 'success',
-          },
-        ],
-        timestamp: '2026-02-10T14:05:00+00:00',
-      },
-      {
-        role: 'assistant',
-        content: 'Running tests.',
-        tool_uses: [
-          {
-            tool: 'exec_command',
-            input: { command: 'python -m pytest tests/test_csv_loader.py -v' },
-            output: { exit_code: 0, wall_time: '3 seconds', output: 'tests/test_csv_loader.py::test_load_small_csv PASSED\ntests/test_csv_loader.py::test_chunked_reading PASSED\n\n2 passed in 2.1s' },
-            status: 'success',
-          },
-        ],
-        timestamp: '2026-02-10T14:10:00+00:00',
-      },
-      {
-        role: 'assistant',
-        content: 'The CSV data loader is implemented and tested. It supports chunked reading for memory efficiency.',
-        timestamp: '2026-02-10T14:11:00+00:00',
-      },
-    ],
-    stats: {
-      user_messages: 1,
-      assistant_messages: 4,
-      tool_uses: 3,
-      input_tokens: 8000,
-      output_tokens: 2000,
-    },
-  };
-}
-
-/** A failed session with error tools and failure signals in final message */
-function failedDeploySession(): DataclawSession {
-  return {
-    session_id: 'dead-beef-0000-1111-222233334444',
-    project: 'my-web-app',
-    model: 'claude-opus-4-6',
-    source: 'claude',
-    git_branch: 'main',
-    start_time: '2026-02-05T16:00:00+00:00',
-    end_time: '2026-02-05T16:10:00+00:00',
-    messages: [
-      {
-        role: 'user',
-        content: 'Deploy the app to production',
-        timestamp: '2026-02-05T16:00:00+00:00',
-      },
-      {
-        role: 'assistant',
-        content: 'Attempting deployment.',
-        tool_uses: [
-          {
-            tool: 'bash',
-            input: { command: 'npm run deploy' },
-            output: { text: 'Error: Missing DEPLOY_TOKEN environment variable' },
-            status: 'error',
-          },
-        ],
-        timestamp: '2026-02-05T16:01:00+00:00',
-      },
-      {
-        role: 'assistant',
-        content: 'Retrying with alternate method.',
-        tool_uses: [
-          {
-            tool: 'bash',
-            input: { command: 'npx deploy --force' },
-            output: { text: 'Fatal: permission denied' },
-            status: 'error',
-          },
-        ],
-        timestamp: '2026-02-05T16:02:00+00:00',
-      },
-      {
-        role: 'assistant',
-        content: 'I was unable to deploy. The deployment requires a DEPLOY_TOKEN that is not available in this environment.',
-        timestamp: '2026-02-05T16:03:00+00:00',
-      },
-    ],
-    stats: {
-      user_messages: 1,
-      assistant_messages: 3,
-      tool_uses: 2,
-      input_tokens: 5000,
-      output_tokens: 1000,
-    },
-  };
-}
-
-/** Minimal conversation-only session — no tools, null git_branch */
-function conversationOnlySession(): DataclawSession {
-  return {
-    session_id: 'aaaa-bbbb-cccc-dddd-eeee11112222',
-    project: 'my-web-app',
-    model: 'claude-opus-4-6',
-    source: 'claude',
-    git_branch: null,
-    start_time: '2026-02-15T11:00:00+00:00',
-    end_time: '2026-02-15T11:05:00+00:00',
-    messages: [
-      {
-        role: 'user',
-        content: 'Explain how the middleware chain works in our Express app',
-        timestamp: '2026-02-15T11:00:00+00:00',
-      },
-      {
-        role: 'assistant',
-        content: 'Your Express middleware chain processes requests in order: first auth validation, then rate limiting, then the route handler. Each middleware calls next() to pass control to the next function.',
-        timestamp: '2026-02-15T11:00:30+00:00',
-      },
-    ],
-    stats: {
-      user_messages: 1,
-      assistant_messages: 1,
-      tool_uses: 0,
-      input_tokens: 3000,
-      output_tokens: 500,
-    },
-  };
-}
-
-// ── Mock HF datasets-server ──────────────────────────────────────────
-
-type FetchFn = typeof globalThis.fetch;
-
-const ALL_SESSIONS = [
-  claudeBugfixSession(),
-  codexFeatureSession(),
-  failedDeploySession(),
-  conversationOnlySession(),
-];
-
-function createMockHfServer(
-  sessions: DataclawSession[] = ALL_SESSIONS,
-): FetchFn {
-  return vi.fn(async (input: RequestInfo | URL, _init?: RequestInit) => {
-    const url = typeof input === 'string' ? input : input.toString();
-
-    if (url.includes('/info')) {
-      return new Response(
-        JSON.stringify({
-          dataset_info: {
-            default: { num_rows: sessions.length, dataset_name: 'test' },
-          },
-        }),
-        { status: 200, headers: { 'Content-Type': 'application/json' } },
-      );
-    }
-
-    if (url.includes('/rows')) {
-      const parsed = new URL(url);
-      const offset = parseInt(parsed.searchParams.get('offset') ?? '0', 10);
-      const length = parseInt(parsed.searchParams.get('length') ?? '100', 10);
-      const page = sessions.slice(offset, offset + length);
-
-      return new Response(
-        JSON.stringify({
-          features: [],
-          rows: page.map((s, i) => ({ row_idx: offset + i, row: s })),
-          num_rows_total: sessions.length,
-          num_rows_per_page: 100,
-          partial: false,
-        }),
-        { status: 200, headers: { 'Content-Type': 'application/json' } },
-      );
-    }
-
-    return new Response('Not Found', { status: 404 });
-  }) as unknown as FetchFn;
-}
-
-// ============================================================================
-// E2E Tests
-// ============================================================================
-
-describe('HuggingFace E2E: load → analyze → extract', () => {
-  // ── Stage 1: Load from HF ─────────────────────────────────────────
-
-  describe('Stage 1: HF dataset loading', () => {
-    it('should load all sessions from mock HF dataset', async () => {
-      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
-        fetchFn: createMockHfServer(),
-      });
-      const items = await source.load();
-      expect(items).toHaveLength(4);
-    });
-
-    it('should produce valid Trajectory objects for every session', async () => {
-      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
-        fetchFn: createMockHfServer(),
-      });
-      const items = await source.load();
-
-      for (const { trajectory } of items) {
-        const parsed = TrajectorySchema.safeParse(trajectory);
-        expect(parsed.success, `Invalid trajectory: ${trajectory.id}`).toBe(true);
-      }
-    });
-
-    it('should tag all trajectories with dataclaw source metadata', async () => {
-      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
-        fetchFn: createMockHfServer(),
-      });
-      const items = await source.load();
-
-      for (const { trajectory } of items) {
-        expect(trajectory.metadata.source).toBe('dataclaw');
-        expect(trajectory.metadata.dataclawSource).toBeTruthy();
-        expect(trajectory.metadata.sessionId).toBeTruthy();
-      }
-    });
-
-    it('should infer correct domains from tool file paths', async () => {
-      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
-        fetchFn: createMockHfServer(),
-      });
-      const items = await source.load();
-
-      const claude = items.find((i) => i.trajectory.id.includes('a1b2c3d4'));
-      const codex = items.find((i) => i.trajectory.id.includes('f9e8d7c6'));
-
-      expect(claude!.trajectory.task.domain).toBe('typescript');
-      expect(codex!.trajectory.task.domain).toBe('python');
-    });
-
-    it('should detect success/failure correctly', async () => {
-      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
-        fetchFn: createMockHfServer(),
-      });
-      const items = await source.load();
-
-      const claude = items.find((i) => i.trajectory.id.includes('a1b2c3d4'));
-      const failed = items.find((i) => i.trajectory.id.includes('dead-beef'));
-
-      expect(claude!.trajectory.outcome.success).toBe(true);
-      expect(failed!.trajectory.outcome.success).toBe(false);
-    });
-
-    it('should paginate across multiple pages', async () => {
-      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
-        fetchFn: createMockHfServer(),
-        pageSize: 2,  // Force 2 pages for 4 sessions
-      });
-      const items = await source.load();
-
-      expect(items).toHaveLength(4);
-    });
-  });
-
-  // ── Stage 2: Analyzer ─────────────────────────────────────────────
-
-  describe('Stage 2: TrajectoryAnalyzer on HF-loaded trajectories', () => {
-    it('should analyze all successful trajectories without error', async () => {
-      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
-        fetchFn: createMockHfServer(),
-      });
-      const items = await source.load({ successOnly: true });
-      const analyzer = createAnalyzer('simple');
-
-      for (const { trajectory } of items) {
-        const analysis = await analyzer.analyze(trajectory);
-
-        expect(analysis).toBeDefined();
-        expect(typeof analysis.success).toBe('boolean');
-        expect(analysis.stepAttribution).toHaveLength(trajectory.steps.length);
-        expect(analysis.keySteps).toBeDefined();
-        expect(typeof analysis.abstractable).toBe('boolean');
-      }
-    });
-
-    it('should analyze failed trajectories and detect error patterns', async () => {
-      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
-        fetchFn: createMockHfServer(),
-      });
-      const items = await source.load();
-      const failed = items.find((i) => i.trajectory.id.includes('dead-beef'));
-      const analyzer = createAnalyzer('simple');
-
-      const analysis = await analyzer.analyze(failed!.trajectory);
-
-      expect(analysis.success).toBe(false);
-      expect(analysis.stepAttribution.length).toBeGreaterThan(0);
-    });
-
-    it('should identify key steps in the Claude bugfix session', async () => {
-      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
-        fetchFn: createMockHfServer(),
-      });
-      const items = await source.load();
-      const claude = items.find((i) => i.trajectory.id.includes('a1b2c3d4'));
-      const analyzer = createAnalyzer('simple');
-
-      const analysis = await analyzer.analyze(claude!.trajectory);
-
-      // Multi-step trajectory should have identifiable key steps
-      expect(analysis.keySteps.length).toBeGreaterThan(0);
-      // At least one step should have non-zero attribution
-      expect(analysis.stepAttribution.some((s) => s > 0)).toBe(true);
-    });
-
-    it('should mark multi-step successful trajectories as abstractable', async () => {
-      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
-        fetchFn: createMockHfServer(),
-      });
-      const items = await source.load();
-      const claude = items.find((i) => i.trajectory.id.includes('a1b2c3d4'));
-      const analyzer = createAnalyzer('simple');
-
-      const analysis = await analyzer.analyze(claude!.trajectory);
-
-      // A 4-step successful session with tool usage should be abstractable
-      expect(analysis.abstractable).toBe(true);
-    });
-
-    it('should extract training examples from successful trajectories', async () => {
-      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
-        fetchFn: createMockHfServer(),
-      });
-      const items = await source.load();
-      const claude = items.find((i) => i.trajectory.id.includes('a1b2c3d4'));
-      const analyzer = createAnalyzer('simple');
-
-      const analysis = await analyzer.analyze(claude!.trajectory);
-
-      expect(analysis.trainingExamples.length).toBeGreaterThan(0);
-      for (const ex of analysis.trainingExamples) {
-        expect(ex.input).toBeTruthy();
-        expect(typeof ex.stepIndex).toBe('number');
-      }
-    });
-  });
-
-  // ── Stage 3: Knowledge Extraction ─────────────────────────────────
-
-  describe('Stage 3: KnowledgeExtractor on HF-loaded & analyzed trajectories', () => {
-    it('should extract knowledge from successful Claude session', async () => {
-      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
-        fetchFn: createMockHfServer(),
-      });
-      const items = await source.load();
-      const claude = items.find((i) => i.trajectory.id.includes('a1b2c3d4'));
-      const analyzer = createAnalyzer('simple');
-      const extractor = createKnowledgeExtractor();
-
-      const analysis = await analyzer.analyze(claude!.trajectory);
-      const knowledge = extractor.extract(claude!.trajectory, analysis);
-
-      expect(knowledge).toBeDefined();
-      expect(knowledge.observations).toBeDefined();
-      expect(knowledge.entityUpdates).toBeDefined();
-      expect(knowledge.links).toBeDefined();
-      expect(knowledge.playbookLinks).toBeDefined();
-      expect(knowledge.contradictions).toBeDefined();
-    });
-
-    it('should extract causal chains from failed session', async () => {
-      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
-        fetchFn: createMockHfServer(),
-      });
-      const items = await source.load();
-      const failed = items.find((i) => i.trajectory.id.includes('dead-beef'));
-      const analyzer = createAnalyzer('simple');
-      const extractor = createKnowledgeExtractor();
-
-      const analysis = await analyzer.analyze(failed!.trajectory);
-      const knowledge = extractor.extract(failed!.trajectory, analysis);
-
-      // The failed session has error→failure pattern that should produce causal chains
-      // or at minimum, the extraction should complete without throwing
-      expect(knowledge).toBeDefined();
-      expect(Array.isArray(knowledge.observations)).toBe(true);
-      expect(Array.isArray(knowledge.links)).toBe(true);
-    });
-
-    it('should run full pipeline on all sessions without errors', async () => {
-      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
-        fetchFn: createMockHfServer(),
-      });
-      const items = await source.load();
-      const analyzer = createAnalyzer('simple');
-      const extractor = createKnowledgeExtractor();
-
-      for (const { trajectory } of items) {
-        const analysis = await analyzer.analyze(trajectory);
-        const knowledge = extractor.extract(trajectory, analysis);
-
-        // Every trajectory should produce valid analysis and knowledge
-        expect(analysis.stepAttribution).toHaveLength(trajectory.steps.length);
-        expect(knowledge).toBeDefined();
-        expect(Array.isArray(knowledge.observations)).toBe(true);
-      }
-    });
-
-    it('should extract observations from the Codex Python session', async () => {
-      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
-        fetchFn: createMockHfServer(),
-      });
-      const items = await source.load();
-      const codex = items.find((i) => i.trajectory.id.includes('f9e8d7c6'));
-      const analyzer = createAnalyzer('simple');
-      const extractor = createKnowledgeExtractor();
-
-      const analysis = await analyzer.analyze(codex!.trajectory);
-      const knowledge = extractor.extract(codex!.trajectory, analysis);
-
-      expect(knowledge).toBeDefined();
-      // Codex session writes Python files — extraction should succeed
-      expect(Array.isArray(knowledge.observations)).toBe(true);
-    });
-
-    it('should handle conversation-only session through full pipeline', async () => {
-      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
-        fetchFn: createMockHfServer(),
-      });
-      const items = await source.load();
-      const qa = items.find((i) => i.trajectory.id.includes('aaaa-bbbb'));
-      const analyzer = createAnalyzer('simple');
-      const extractor = createKnowledgeExtractor();
-
-      const analysis = await analyzer.analyze(qa!.trajectory);
-      const knowledge = extractor.extract(qa!.trajectory, analysis);
-
-      // Single-step Q&A should still work through the full pipeline
-      expect(analysis.stepAttribution).toHaveLength(1);
-      expect(knowledge).toBeDefined();
-    });
-  });
-
-  // ── Stage 4: Full pipeline with filtering ─────────────────────────
-
-  describe('Stage 4: filtered pipeline runs', () => {
-    it('should pipeline only Claude sessions', async () => {
-      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
-        fetchFn: createMockHfServer(),
-        sources: ['claude'],
-      });
-      const items = await source.load({ successOnly: true });
-      const analyzer = createAnalyzer('simple');
-      const extractor = createKnowledgeExtractor();
-
-      // Should have 2 Claude success sessions (bugfix + conversation)
-      expect(items).toHaveLength(2);
-
-      let totalObservations = 0;
-      let totalTrainingExamples = 0;
-
-      for (const { trajectory } of items) {
-        expect(trajectory.metadata.dataclawSource).toBe('claude');
-        expect(trajectory.outcome.success).toBe(true);
-
-        const analysis = await analyzer.analyze(trajectory);
-        const knowledge = extractor.extract(trajectory, analysis);
-
-        totalObservations += knowledge.observations.length;
-        totalTrainingExamples += analysis.trainingExamples.length;
-      }
-
-      // The pipeline should produce some learning signal
-      expect(totalObservations + totalTrainingExamples).toBeGreaterThan(0);
-    });
-
-    it('should pipeline TypeScript-domain only', async () => {
-      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
-        fetchFn: createMockHfServer(),
-      });
-      const items = await source.load({ domain: 'typescript' });
-      const analyzer = createAnalyzer('simple');
-
-      expect(items).toHaveLength(1);
-      expect(items[0].trajectory.task.domain).toBe('typescript');
-
-      const analysis = await analyzer.analyze(items[0].trajectory);
-      expect(analysis.abstractable).toBe(true);
-    });
-
-    it('should pipeline with limit', async () => {
-      const source = new HuggingFaceTrajectorySource('peteromallet/dataclaw-peteromallet', {
-        fetchFn: createMockHfServer(),
-      });
-      const items = await source.load({ limit: 2 });
-      const analyzer = createAnalyzer('simple');
-      const extractor = createKnowledgeExtractor();
-
-      expect(items).toHaveLength(2);
-
-      for (const { trajectory } of items) {
-        const analysis = await analyzer.analyze(trajectory);
-        const knowledge = extractor.extract(trajectory, analysis);
-        expect(knowledge).toBeDefined();
-      }
-    });
-  });
-});