@plaited/acp-harness 0.3.1 → 0.3.2

package/src/tests/summarize-helpers.spec.ts

@@ -0,0 +1,339 @@
+import { describe, expect, test } from 'bun:test'
+import type { CaptureResult } from '../schemas.ts'
+import { formatMarkdown, formatSummary } from '../summarize.ts'
+
+// ============================================================================
+// Test Fixtures
+// ============================================================================
+
+const createBasicResult = (overrides?: Partial<CaptureResult>): CaptureResult => ({
+  id: 'test-001',
+  input: 'What is 2+2?',
+  output: 'The answer is 4.',
+  trajectory: [
+    { type: 'thought', content: 'Let me think about this...', timestamp: 0 },
+    { type: 'message', content: 'The answer is 4.', timestamp: 100 },
+  ],
+  metadata: { category: 'math', agent: 'test-agent' },
+  timing: { start: 1000, end: 2000 },
+  toolErrors: false,
+  ...overrides,
+})
+
+const createResultWithToolCalls = (): CaptureResult => ({
+  id: 'test-002',
+  input: 'Read and summarize file.txt',
+  output: 'File contains important data.',
+  trajectory: [
+    { type: 'thought', content: 'I will read the file first.', timestamp: 0 },
+    {
+      type: 'tool_call',
+      name: 'Read',
+      status: 'completed',
+      input: { file_path: '/path/to/file.txt' },
+      output: 'file contents here',
+      duration: 50,
+      timestamp: 100,
+    },
+    {
+      type: 'tool_call',
+      name: 'Write',
+      status: 'completed',
+      input: { file_path: '/output.md', content: 'Summary here' },
+      duration: 30,
+      timestamp: 200,
+    },
+    { type: 'message', content: 'File contains important data.', timestamp: 300 },
+  ],
+  metadata: { agent: 'test-agent' },
+  timing: { start: 1000, end: 1500 },
+  toolErrors: false,
+})
+
+// ============================================================================
+// formatSummary
+// ============================================================================
+
+describe('formatSummary', () => {
+  test('extracts id, input, and output', () => {
+    const result = createBasicResult()
+    const summary = formatSummary(result)
+
+    expect(summary.id).toBe('test-001')
+    expect(summary.input).toBe('What is 2+2?')
+    expect(summary.output).toBe('The answer is 4.')
+  })
+
+  test('extracts tool call names', () => {
+    const result = createResultWithToolCalls()
+    const summary = formatSummary(result)
+
+    expect(summary.toolCalls).toEqual(['Read', 'Write'])
+  })
+
+  test('calculates duration from timing', () => {
+    const result = createBasicResult()
+    const summary = formatSummary(result)
+
+    expect(summary.duration).toBe(1000) // 2000 - 1000
+  })
+
+  test('handles empty trajectory', () => {
+    const result = createBasicResult({ trajectory: [] })
+    const summary = formatSummary(result)
+
+    expect(summary.toolCalls).toEqual([])
+  })
+
+  test('filters only tool_call steps for toolCalls list', () => {
+    const result = createBasicResult()
+    const summary = formatSummary(result)
+
+    // trajectory has thought and message, but no tool_call
+    expect(summary.toolCalls).toEqual([])
+  })
+
+  test('handles trajectory with only messages', () => {
+    const result = createBasicResult({
+      trajectory: [
+        { type: 'message', content: 'First message', timestamp: 0 },
+        { type: 'message', content: 'Second message', timestamp: 100 },
+      ],
+    })
+    const summary = formatSummary(result)
+
+    expect(summary.toolCalls).toEqual([])
+  })
+
+  test('preserves original input/output exactly', () => {
+    const result = createBasicResult({
+      input: 'Input with\nnewlines and "quotes"',
+      output: 'Output with\ttabs',
+    })
+    const summary = formatSummary(result)
+
+    expect(summary.input).toBe('Input with\nnewlines and "quotes"')
+    expect(summary.output).toBe('Output with\ttabs')
+  })
+})
+
+// ============================================================================
+// formatMarkdown
+// ============================================================================
+
+describe('formatMarkdown', () => {
+  test('includes evaluation record header with id', () => {
+    const result = createBasicResult()
+    const markdown = formatMarkdown(result)
+
+    expect(markdown).toContain('## Evaluation Record: test-001')
+  })
+
+  test('includes input field', () => {
+    const result = createBasicResult()
+    const markdown = formatMarkdown(result)
+
+    expect(markdown).toContain('**Input:** What is 2+2?')
+  })
+
+  test('includes trajectory section', () => {
+    const result = createBasicResult()
+    const markdown = formatMarkdown(result)
+
+    expect(markdown).toContain('**Trajectory:**')
+  })
+
+  test('formats thought steps with truncation', () => {
+    const result = createBasicResult({
+      trajectory: [
+        { type: 'thought', content: 'Short thought', timestamp: 0 },
+        { type: 'thought', content: 'A'.repeat(150), timestamp: 100 },
+      ],
+    })
+    const markdown = formatMarkdown(result)
+
+    expect(markdown).toContain('[THOUGHT] Short thought')
+    expect(markdown).toContain(`[THOUGHT] ${'A'.repeat(100)}...`)
+  })
+
+  test('formats tool calls with status and duration', () => {
+    const result = createResultWithToolCalls()
+    const markdown = formatMarkdown(result)
+
+    expect(markdown).toContain('[TOOL:Read] -> completed (50ms)')
+    expect(markdown).toContain('[TOOL:Write] -> completed (30ms)')
+  })
+
+  test('includes file path for tool calls', () => {
+    const result = createResultWithToolCalls()
+    const markdown = formatMarkdown(result)
+
+    expect(markdown).toContain('File: /path/to/file.txt')
+    expect(markdown).toContain('File: /output.md')
+  })
+
+  test('includes step IDs for reference', () => {
+    const result = createBasicResult()
+    const markdown = formatMarkdown(result)
+
+    expect(markdown).toContain('[→test-001-step-1]')
+    expect(markdown).toContain('[→test-001-step-2]')
+  })
+
+  test('formats plan steps', () => {
+    const result = createBasicResult({
+      trajectory: [
+        {
+          type: 'plan',
+          entries: [
+            { content: 'Step 1', status: 'completed' },
+            { content: 'Step 2', status: 'in_progress' },
+          ],
+          timestamp: 0,
+        },
+      ],
+    })
+    const markdown = formatMarkdown(result)
+
+    expect(markdown).toContain('[PLAN]')
+    expect(markdown).toContain('Step 1: completed')
+    expect(markdown).toContain('Step 2: in_progress')
+  })
+
+  test('truncates long plan summaries', () => {
+    const result = createBasicResult({
+      trajectory: [
+        {
+          type: 'plan',
+          entries: [
+            { content: 'A very long step description that goes on and on', status: 'completed' },
+            { content: 'Another very long step description', status: 'pending' },
+            { content: 'Yet another step', status: 'pending' },
+          ],
+          timestamp: 0,
+        },
+      ],
+    })
+    const markdown = formatMarkdown(result)
+
+    expect(markdown).toContain('...')
+  })
+
+  test('formats message steps', () => {
+    const result = createBasicResult({
+      trajectory: [{ type: 'message', content: 'Here is my response to your question.', timestamp: 0 }],
+    })
+    const markdown = formatMarkdown(result)
+
+    expect(markdown).toContain('[MESSAGE] Here is my response')
+  })
+
+  test('includes output preview', () => {
+    const result = createBasicResult()
+    const markdown = formatMarkdown(result)
+
+    expect(markdown).toContain('**Output:** The answer is 4.')
+  })
+
+  test('truncates long output', () => {
+    const result = createBasicResult({
+      output: 'X'.repeat(300),
+    })
+    const markdown = formatMarkdown(result)
+
+    expect(markdown).toContain(`${'X'.repeat(200)}...`)
+  })
+
+  test('includes metadata', () => {
+    const result = createBasicResult()
+    const markdown = formatMarkdown(result)
+
+    expect(markdown).toContain('**Metadata:**')
+    expect(markdown).toContain('category=math')
+    expect(markdown).toContain('agent=test-agent')
+  })
+
+  test('includes tool errors status', () => {
+    const result = createBasicResult({ toolErrors: true })
+    const markdown = formatMarkdown(result)
+
+    expect(markdown).toContain('**Tool Errors:** true')
+  })
+
+  test('includes duration', () => {
+    const result = createBasicResult()
+    const markdown = formatMarkdown(result)
+
+    expect(markdown).toContain('**Duration:** 1000ms')
+  })
+
+  test('includes score when present', () => {
+    const result = createBasicResult({
+      score: {
+        pass: true,
+        score: 0.95,
+        reasoning: 'Correct answer provided',
+      },
+    })
+    const markdown = formatMarkdown(result)
+
+    expect(markdown).toContain('**Score:** PASS (0.95)')
+    expect(markdown).toContain('**Reasoning:** Correct answer provided')
+  })
+
+  test('handles failed score', () => {
+    const result = createBasicResult({
+      score: {
+        pass: false,
+        score: 0.2,
+        reasoning: 'Incorrect answer',
+      },
+    })
+    const markdown = formatMarkdown(result)
+
+    expect(markdown).toContain('**Score:** FAIL (0.2)')
+  })
+
+  test('includes content preview with syntax highlighting', () => {
+    const result: CaptureResult = {
+      id: 'test-003',
+      input: 'Write a function',
+      output: 'Done',
+      trajectory: [
+        {
+          type: 'tool_call',
+          name: 'Write',
+          status: 'completed',
+          input: {
+            file_path: '/src/utils.ts',
+            content: 'export const add = (a: number, b: number) => a + b;',
+          },
+          duration: 20,
+          timestamp: 0,
+        },
+      ],
+      metadata: { agent: 'test' },
+      timing: { start: 0, end: 100 },
+      toolErrors: false,
+    }
+    const markdown = formatMarkdown(result)
+
+    expect(markdown).toContain('```ts')
+    expect(markdown).toContain('export const add')
+  })
+
+  test('ends with horizontal rule separator', () => {
+    const result = createBasicResult()
+    const markdown = formatMarkdown(result)
+
+    expect(markdown).toContain('---')
+  })
+
+  test('handles empty trajectory', () => {
+    const result = createBasicResult({ trajectory: [] })
+    const markdown = formatMarkdown(result)
+
+    expect(markdown).toContain('**Trajectory:**')
+    expect(markdown).toContain('**Output:**')
+  })
+})

package/src/tests/trials-calculations.spec.ts

@@ -0,0 +1,209 @@
+import { describe, expect, test } from 'bun:test'
+import { calculatePassAtK, calculatePassExpK } from '../trials.ts'
+
+// ============================================================================
+// calculatePassAtK
+// ============================================================================
+
+describe('calculatePassAtK', () => {
+  test('returns 1 when all trials pass', () => {
+    expect(calculatePassAtK(5, 5)).toBe(1)
+    expect(calculatePassAtK(10, 10)).toBe(1)
+    expect(calculatePassAtK(1, 1)).toBe(1)
+  })
+
+  test('returns 0 when no trials pass', () => {
+    expect(calculatePassAtK(0, 5)).toBe(0)
+    expect(calculatePassAtK(0, 10)).toBe(0)
+    expect(calculatePassAtK(0, 1)).toBe(0)
+  })
+
+  test('calculates probability correctly for partial passes', () => {
+    // pass@k = 1 - (1 - passRate)^k
+    // For 3 passes out of 5: passRate = 0.6
+    // pass@5 = 1 - (0.4)^5 = 1 - 0.01024 = 0.98976
+    const result = calculatePassAtK(3, 5)
+    expect(result).toBeCloseTo(0.98976, 5)
+  })
+
+  test('k=1 equals the pass rate', () => {
+    // For k=1, pass@1 = 1 - (1 - p)^1 = p
+    expect(calculatePassAtK(1, 1)).toBe(1)
+
+    // More interesting: 0 passes, 1 trial
+    expect(calculatePassAtK(0, 1)).toBe(0)
+  })
+
+  test('higher pass rate yields higher pass@k', () => {
+    const lowPassRate = calculatePassAtK(1, 5) // 20% pass rate
+    const highPassRate = calculatePassAtK(4, 5) // 80% pass rate
+
+    expect(highPassRate).toBeGreaterThan(lowPassRate)
+  })
+
+  test('larger k amplifies probability of at least one pass', () => {
+    // With 50% pass rate, larger k means higher chance of at least one pass
+    // k=2: 1 - (0.5)^2 = 0.75
+    // k=4: 1 - (0.5)^4 = 0.9375
+
+    const k2 = calculatePassAtK(1, 2) // 50% pass rate
+    const k4 = calculatePassAtK(2, 4) // Also 50% pass rate
+
+    expect(k4).toBeGreaterThan(k2)
+  })
+
+  test('handles edge case where passes equals k', () => {
+    expect(calculatePassAtK(3, 3)).toBe(1)
+  })
+
+  test('handles passes greater than k (returns 1)', () => {
+    // This shouldn't happen in practice, but the function handles it
+    expect(calculatePassAtK(10, 5)).toBe(1)
+  })
+
+  test('mathematical verification with known values', () => {
+    // 1 out of 3 passes: passRate = 1/3
+    // pass@3 = 1 - (2/3)^3 = 1 - 8/27 = 19/27 ≈ 0.7037
+    const result = calculatePassAtK(1, 3)
+    expect(result).toBeCloseTo(19 / 27, 5)
+
+    // 2 out of 4 passes: passRate = 0.5
+    // pass@4 = 1 - (0.5)^4 = 1 - 0.0625 = 0.9375
+    const result2 = calculatePassAtK(2, 4)
+    expect(result2).toBeCloseTo(0.9375, 5)
+  })
+})
+
+// ============================================================================
+// calculatePassExpK
+// ============================================================================
+
+describe('calculatePassExpK', () => {
+  test('returns 1 when all trials pass', () => {
+    expect(calculatePassExpK(5, 5)).toBe(1)
+    expect(calculatePassExpK(10, 10)).toBe(1)
+    expect(calculatePassExpK(1, 1)).toBe(1)
+  })
+
+  test('returns 0 when no trials pass', () => {
+    expect(calculatePassExpK(0, 5)).toBe(0)
+    expect(calculatePassExpK(0, 10)).toBe(0)
+    expect(calculatePassExpK(0, 1)).toBe(0)
+  })
+
+  test('calculates probability correctly', () => {
+    // pass^k = passRate^k
+    // For 3 passes out of 5: passRate = 0.6
+    // pass^5 = (0.6)^5 = 0.07776
+    const result = calculatePassExpK(3, 5)
+    expect(result).toBeCloseTo(0.07776, 5)
+  })
+
+  test('k=1 equals the pass rate', () => {
+    // For k=1, pass^1 = p^1 = p
+    expect(calculatePassExpK(1, 1)).toBe(1)
+  })
+
+  test('higher pass rate yields higher pass^k', () => {
+    const lowPassRate = calculatePassExpK(1, 5) // 20% pass rate
+    const highPassRate = calculatePassExpK(4, 5) // 80% pass rate
+
+    expect(highPassRate).toBeGreaterThan(lowPassRate)
+  })
+
+  test('larger k reduces probability of all passing (for non-100% rates)', () => {
+    // With 80% pass rate:
+    // k=2: (0.8)^2 = 0.64
+    // k=5: (0.8)^5 = 0.32768
+
+    // Mathematical verification using known formulas
+    const k2_fair = 0.8 ** 2 // = 0.64
+    const k5_fair = 0.8 ** 5 // = 0.32768
+
+    expect(k5_fair).toBeLessThan(k2_fair)
+
+    // Also verify our function produces consistent results
+    // 4 out of 5 gives 80% pass rate
+    const result = calculatePassExpK(4, 5)
+    expect(result).toBeCloseTo(k5_fair, 5)
+  })
+
+  test('handles edge case where passes equals k', () => {
+    expect(calculatePassExpK(3, 3)).toBe(1)
+  })
+
+  test('mathematical verification with known values', () => {
+    // 1 out of 3 passes: passRate = 1/3
+    // pass^3 = (1/3)^3 = 1/27 ≈ 0.037
+    const result = calculatePassExpK(1, 3)
+    expect(result).toBeCloseTo(1 / 27, 5)
+
+    // 2 out of 4 passes: passRate = 0.5
+    // pass^4 = (0.5)^4 = 0.0625
+    const result2 = calculatePassExpK(2, 4)
+    expect(result2).toBeCloseTo(0.0625, 5)
+
+    // 3 out of 4 passes: passRate = 0.75
+    // pass^4 = (0.75)^4 = 0.31640625
+    const result3 = calculatePassExpK(3, 4)
+    expect(result3).toBeCloseTo(0.31640625, 5)
+  })
+
+  test('pass^k is always less than or equal to pass@k', () => {
+    // For any pass rate < 100%, pass^k <= pass@k
+    // This is because "all pass" is a subset of "at least one passes"
+
+    const testCases = [
+      { passes: 1, k: 5 },
+      { passes: 2, k: 5 },
+      { passes: 3, k: 5 },
+      { passes: 4, k: 5 },
+      { passes: 1, k: 3 },
+      { passes: 2, k: 4 },
+    ]
+
+    for (const { passes, k } of testCases) {
+      const passExpK = calculatePassExpK(passes, k)
+      const passAtK = calculatePassAtK(passes, k)
+      expect(passExpK).toBeLessThanOrEqual(passAtK)
+    }
+  })
+})
+
+// ============================================================================
+// Combined behavior tests
+// ============================================================================
+
+describe('pass@k and pass^k relationship', () => {
+  test('100% pass rate: both metrics equal 1', () => {
+    expect(calculatePassAtK(5, 5)).toBe(1)
+    expect(calculatePassExpK(5, 5)).toBe(1)
+  })
+
+  test('0% pass rate: both metrics equal 0', () => {
+    expect(calculatePassAtK(0, 5)).toBe(0)
+    expect(calculatePassExpK(0, 5)).toBe(0)
+  })
+
+  test('gap between metrics varies with pass rate', () => {
+    // At 50% pass rate, the gap is maximized
+    // At extreme pass rates (0% or 100%), the gap is 0
+
+    // 50% pass rate with k=4
+    const midAtK = calculatePassAtK(2, 4) // 0.9375
+    const midExpK = calculatePassExpK(2, 4) // 0.0625
+    const midGap = midAtK - midExpK // 0.875
+
+    // 80% pass rate with k=5
+    const highAtK = calculatePassAtK(4, 5)
+    const highExpK = calculatePassExpK(4, 5)
+    const highGap = highAtK - highExpK
+
+    // Both gaps should be positive (pass@k > pass^k for partial pass rates)
+    expect(midGap).toBeGreaterThan(0)
+    expect(highGap).toBeGreaterThan(0)
+
+    // Mid-range pass rate has larger gap than high pass rate
+    expect(midGap).toBeGreaterThan(highGap)
+  })
+})

package/src/trials.ts

@@ -34,8 +34,14 @@ import { McpServerSchema } from './schemas.ts'
  *
  * For our case where n = k (we run exactly k trials per prompt):
  * pass@k = 1 - (1 - passRate)^k (simplified)
+ *
+ * @param passes - Number of passing trials
+ * @param k - Total number of trials
+ * @returns Probability of at least one pass
+ *
+ * @public
  */
-const calculatePassAtK = (passes: number, k: number): number => {
+export const calculatePassAtK = (passes: number, k: number): number => {
   if (passes >= k) return 1
   if (passes === 0) return 0
 
@@ -49,8 +55,14 @@ const calculatePassAtK = (passes: number, k: number): number => {
  *
  * @remarks
  * This is simply passRate^k
+ *
+ * @param passes - Number of passing trials
+ * @param k - Total number of trials
+ * @returns Probability of all k samples passing
+ *
+ * @public
  */
-const calculatePassExpK = (passes: number, k: number): number => {
+export const calculatePassExpK = (passes: number, k: number): number => {
   if (passes === k) return 1
   if (passes === 0) return 0