opencastle 0.26.1 → 0.27.1

package/src/cli/convoy/events.test.ts

@@ -1,25 +1,21 @@
-import { mkdtempSync, readFileSync, rmSync, existsSync } from 'node:fs'
+import { mkdtempSync, readFileSync, writeFileSync, rmSync, existsSync, mkdirSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
-import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
+import { realpathSync } from 'node:fs'
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
 import { createConvoyStore } from './store.js'
 import { createEventEmitter } from './events.js'
+import { recoverNdjson } from './engine.js'
 import type { ConvoyStore } from './store.js'
 
-vi.mock('../log.js', () => ({
-  appendEvent: vi.fn().mockResolvedValue(undefined),
-}))
-
-import { appendEvent } from '../log.js'
-const mockAppend = vi.mocked(appendEvent)
-
 let tmpDir: string
 let store: ConvoyStore
+let ndjsonPath: string
 
 beforeEach(() => {
-  tmpDir = mkdtempSync(join(tmpdir(), 'emitter-test-'))
+  tmpDir = realpathSync(mkdtempSync(join(tmpdir(), 'emitter-test-')))
   store = createConvoyStore(join(tmpDir, 'test.db'))
-  vi.clearAllMocks()
+  ndjsonPath = join(tmpDir, 'events.ndjson')
 
   store.insertConvoy({
     id: 'c1',
@@ -41,6 +37,7 @@ describe('createEventEmitter', () => {
   it('inserts the event into SQLite', () => {
     const emitter = createEventEmitter(store)
     emitter.emit('task_started', { msg: 'started' }, { convoy_id: 'c1' })
+    emitter.close()
     const events = store.getEvents('c1')
     expect(events).toHaveLength(1)
     expect(events[0].type).toBe('task_started')
@@ -50,6 +47,7 @@ describe('createEventEmitter', () => {
   it('serializes event data to JSON in SQLite', () => {
     const emitter = createEventEmitter(store)
     emitter.emit('task_done', { exitCode: 0, output: 'ok' }, { convoy_id: 'c1' })
+    emitter.close()
     const events = store.getEvents('c1')
     const parsed = JSON.parse(events[0].data!)
     expect(parsed.exitCode).toBe(0)
@@ -59,60 +57,203 @@ describe('createEventEmitter', () => {
   it('stores null data when no data object is provided', () => {
     const emitter = createEventEmitter(store)
     emitter.emit('heartbeat', undefined, { convoy_id: 'c1' })
+    emitter.close()
     const events = store.getEvents('c1')
     expect(events[0].data).toBeNull()
   })
 
-  it('calls appendEvent for NDJSON dual-write', () => {
-    const emitter = createEventEmitter(store)
+  it('writes NDJSON when ndjsonPath is provided', () => {
+    const emitter = createEventEmitter(store, { ndjsonPath })
     emitter.emit('convoy_started', { name: 'test' }, { convoy_id: 'c1' })
-    expect(mockAppend).toHaveBeenCalledOnce()
+    emitter.close()
+    expect(existsSync(ndjsonPath)).toBe(true)
+    const content = readFileSync(ndjsonPath, 'utf8')
+    expect(content.trim()).not.toBe('')
+    const line = JSON.parse(content.trim())
+    expect(line.type).toBe('convoy_started')
+    expect(line.convoy_id).toBe('c1')
   })
 
-  it('passes logs dir to appendEvent', () => {
-    const emitter = createEventEmitter(store, '/some/logs')
+  it('writes _event_id to NDJSON matching SQLite rowid', () => {
+    const emitter = createEventEmitter(store, { ndjsonPath })
     emitter.emit('convoy_started', {}, { convoy_id: 'c1' })
-    expect(mockAppend).toHaveBeenCalledWith(
-      expect.objectContaining({ type: 'convoy_started', convoy_id: 'c1' }),
-      '/some/logs',
-    )
+    emitter.close()
+    const sqliteEvents = store.getEvents('c1')
+    const ndjsonLine = JSON.parse(readFileSync(ndjsonPath, 'utf8').trim())
+    expect(ndjsonLine._event_id).toBe(sqliteEvents[0].id)
   })
 
   it('defaults all ids to null when ids are not provided', () => {
-    const emitter = createEventEmitter(store)
+    const emitter = createEventEmitter(store, { ndjsonPath })
     emitter.emit('generic_event')
-    const db = require('node:sqlite').DatabaseSync
-    // Verify via NDJSON mock payload
-    expect(mockAppend).toHaveBeenCalledWith(
-      expect.objectContaining({
-        convoy_id: null,
-        task_id: null,
-        worker_id: null,
-      }),
-      null,
-    )
+    emitter.close()
+    const events = store.getEvents('c1')
+    expect(events).toHaveLength(0)
+    // No convoy_id so not retrievable via getEvents('c1'), but event was inserted
+    const content = readFileSync(ndjsonPath, 'utf8')
+    const line = JSON.parse(content.trim())
+    expect(line.convoy_id).toBeNull()
+    expect(line.task_id).toBeNull()
+    expect(line.worker_id).toBeNull()
   })
 
   it('includes all provided ids in the NDJSON record', () => {
-    const emitter = createEventEmitter(store, tmpDir)
+    const emitter = createEventEmitter(store, { ndjsonPath })
     emitter.emit('worker_spawned', {}, { convoy_id: 'c1', task_id: 't1', worker_id: 'w1' })
-    expect(mockAppend).toHaveBeenCalledWith(
-      expect.objectContaining({ convoy_id: 'c1', task_id: 't1', worker_id: 'w1' }),
-      tmpDir,
-    )
+    emitter.close()
+    const line = JSON.parse(readFileSync(ndjsonPath, 'utf8').trim())
+    expect(line.convoy_id).toBe('c1')
+    expect(line.task_id).toBe('t1')
+    expect(line.worker_id).toBe('w1')
   })
 
   it('SQLite event stores correct ids', () => {
     const emitter = createEventEmitter(store)
     emitter.emit('worker_done', {}, { convoy_id: 'c1', task_id: 'task-x', worker_id: 'wkr-y' })
+    emitter.close()
     const events = store.getEvents('c1')
     expect(events[0].task_id).toBe('task-x')
     expect(events[0].worker_id).toBe('wkr-y')
   })
 
-  it('does not throw if NDJSON write fails', () => {
-    mockAppend.mockRejectedValueOnce(new Error('disk full'))
+  it('does not throw if NDJSON path is not provided', () => {
     const emitter = createEventEmitter(store)
     expect(() => emitter.emit('test', {}, { convoy_id: 'c1' })).not.toThrow()
+    emitter.close()
+  })
+
+  it('close() is idempotent', () => {
+    const emitter = createEventEmitter(store, { ndjsonPath })
+    emitter.close()
+    expect(() => emitter.close()).not.toThrow()
+  })
+})
+
+describe('crash resilience', () => {
+  it('1. mid-write crash: SQLite has events, recovery writes NDJSON', () => {
+    // Emit events using emitter WITHOUT ndjsonPath — simulates crash after SQLite commit
+    const emitter = createEventEmitter(store)
+    emitter.emit('task_started', { step: 1 }, { convoy_id: 'c1', task_id: 't1' })
+    emitter.emit('task_done', { step: 2 }, { convoy_id: 'c1', task_id: 't1' })
+    emitter.close()
+
+    // SQLite has both events
+    const sqliteEvents = store.getEvents('c1')
+    expect(sqliteEvents).toHaveLength(2)
+
+    // NDJSON file does not exist
+    expect(existsSync(ndjsonPath)).toBe(false)
+
+    // Recovery writes the missing events to NDJSON
+    recoverNdjson(store, 'c1', ndjsonPath)
+
+    expect(existsSync(ndjsonPath)).toBe(true)
+    const lines = readFileSync(ndjsonPath, 'utf8').split('\n').filter(l => l.trim())
+    expect(lines).toHaveLength(2)
+    const types = lines.map(l => JSON.parse(l).type)
+    expect(types).toContain('task_started')
+    expect(types).toContain('task_done')
+  })
+
+  it('2. recovery consistency: missing events replayed after partial crash', () => {
+    // Write some events to both SQLite + NDJSON via emitter
+    const emitter = createEventEmitter(store, { ndjsonPath })
+    emitter.emit('convoy_started', {}, { convoy_id: 'c1' })
+    emitter.close()
+
+    // Simulate crash: two more events go only to SQLite (bypass emitter)
+    store.insertEvent({
+      convoy_id: 'c1', task_id: 't1', worker_id: null,
+      type: 'task_started', data: null, created_at: new Date().toISOString(),
+    })
+    store.insertEvent({
+      convoy_id: 'c1', task_id: 't1', worker_id: null,
+      type: 'task_done', data: null, created_at: new Date().toISOString(),
+    })
+
+    // Before recovery: NDJSON has 1 line, SQLite has 3
+    const beforeLines = readFileSync(ndjsonPath, 'utf8').split('\n').filter(l => l.trim())
+    expect(beforeLines).toHaveLength(1)
+    expect(store.getEvents('c1')).toHaveLength(3)
+
+    // Recovery replays the 2 missing events
+    recoverNdjson(store, 'c1', ndjsonPath)
+
+    const afterLines = readFileSync(ndjsonPath, 'utf8').split('\n').filter(l => l.trim())
+    expect(afterLines).toHaveLength(3)
+  })
+
+  it('3. no duplication: idempotent recovery when all synced', () => {
+    // Write 5 events — all go to both SQLite and NDJSON
+    const emitter = createEventEmitter(store, { ndjsonPath })
+    for (let i = 0; i < 5; i++) {
+      emitter.emit('task_done', { i }, { convoy_id: 'c1', task_id: `t${i}` })
+    }
+    emitter.close()
+
+    const linesBefore = readFileSync(ndjsonPath, 'utf8').split('\n').filter(l => l.trim())
+    expect(linesBefore).toHaveLength(5)
+
+    // Run recovery — nothing should be added since all events already in NDJSON
+    recoverNdjson(store, 'c1', ndjsonPath)
+
+    const linesAfter = readFileSync(ndjsonPath, 'utf8').split('\n').filter(l => l.trim())
+    expect(linesAfter).toHaveLength(5)
+  })
+
+  it('4. partial line recovery: incomplete write truncated and replayed', () => {
+    // Write one complete event then append a partial JSON line (no \\n terminator)
+    const emitter = createEventEmitter(store, { ndjsonPath })
+    emitter.emit('convoy_started', {}, { convoy_id: 'c1' })
+    emitter.close()
+
+    // Append a partial line directly (simulating a crash mid-write)
+    const partialLine = '{"_event_id":999,"type":"partial_crash","convoy_id":"c1"'  // no closing } or \n
+    const existingContent = readFileSync(ndjsonPath, 'utf8')
+    writeFileSync(ndjsonPath, existingContent + partialLine)
+
+    // Recovery should truncate the partial line and replay anything missing
+    recoverNdjson(store, 'c1', ndjsonPath)
+
+    const recovered = readFileSync(ndjsonPath, 'utf8')
+    // Every line should be valid JSON
+    const lines = recovered.split('\n').filter(l => l.trim())
+    for (const line of lines) {
+      expect(() => JSON.parse(line)).not.toThrow()
+    }
+    // The original complete event should be present
+    const types = lines.map(l => JSON.parse(l).type)
+    expect(types).toContain('convoy_started')
+    // The partial line should not appear
+    expect(types).not.toContain('partial_crash')
+  })
+
+  it('5. large file: 1000 events all readable after emit and recovery', () => {
+    const count = 1000
+    const emitter = createEventEmitter(store, { ndjsonPath })
+    for (let i = 0; i < count; i++) {
+      emitter.emit('bench_event', { index: i }, { convoy_id: 'c1', task_id: `t${i}` })
+    }
+    emitter.close()
+
+    // All events in SQLite
+    expect(store.getEvents('c1')).toHaveLength(count)
+
+    // All events in NDJSON
+    const lines = readFileSync(ndjsonPath, 'utf8').split('\n').filter(l => l.trim())
+    expect(lines).toHaveLength(count)
+
+    // Each line is valid JSON with the right type
+    for (const line of lines) {
+      const parsed = JSON.parse(line)
+      expect(parsed.type).toBe('bench_event')
+      expect(parsed.convoy_id).toBe('c1')
+    }
+
+    // Recovery is a no-op (everything is synced)
+    recoverNdjson(store, 'c1', ndjsonPath)
+    const linesAfter = readFileSync(ndjsonPath, 'utf8').split('\n').filter(l => l.trim())
+    expect(linesAfter).toHaveLength(count)
   })
 })
+

package/src/cli/convoy/events.ts

@@ -1,5 +1,6 @@
-import { appendEvent as appendNdjson } from '../log.js'
+import { appendFileSync, closeSync, fsyncSync, openSync } from 'node:fs'
 import type { ConvoyStore } from './store.js'
+import { scanForSecrets } from './gates.js'
 
 export interface ConvoyEventEmitter {
   emit(
@@ -7,35 +8,106 @@ export interface ConvoyEventEmitter {
     data?: Record<string, unknown>,
     ids?: { convoy_id?: string; task_id?: string; worker_id?: string },
   ): void
+  close(): void
 }
 
-export function createEventEmitter(store: ConvoyStore, logsDir?: string): ConvoyEventEmitter {
-  return {
-    emit(type, data, ids) {
-      const now = new Date().toISOString()
+export function createEventEmitter(
+  store: ConvoyStore,
+  options?: { ndjsonPath?: string },
+): ConvoyEventEmitter {
+  let fd: number | null = null
+  if (options?.ndjsonPath) {
+    fd = openSync(options.ndjsonPath, 'a')
+  }
+
+  // NDJSON writes are supplementary — SQLite is the primary store. Use async
+  // retries to avoid blocking the Node.js event loop.
+  async function writeNdjson(
+    type: string,
+    data: Record<string, unknown> | undefined,
+    ids: { convoy_id?: string; task_id?: string; worker_id?: string } | undefined,
+    now: string,
+    eventId: number,
+    currentFd: number,
+  ): Promise<void> {
+    const record = {
+      _event_id: eventId,
+      timestamp: now,
+      type,
+      convoy_id: ids?.convoy_id ?? null,
+      task_id: ids?.task_id ?? null,
+      worker_id: ids?.worker_id ?? null,
+      ...(data ?? {}),
+    }
+    const jsonLine = JSON.stringify(record) + '\n'
 
+    const scanResult = scanForSecrets(jsonLine, 'ndjson')
+    if (!scanResult.clean) {
+      // Block the NDJSON write — record the blocked event in SQLite only
       store.insertEvent({
         convoy_id: ids?.convoy_id ?? null,
         task_id: ids?.task_id ?? null,
         worker_id: ids?.worker_id ?? null,
-        type,
-        data: data !== undefined ? JSON.stringify(data) : null,
+        type: 'secret_leak_prevented',
+        data: JSON.stringify({ original_type: type, patterns: scanResult.findings.map(f => f.pattern) }),
         created_at: now,
       })
+      return
+    }
 
-      appendNdjson(
-        {
-          timestamp: now,
-          type,
+    try {
+      appendFileSync(currentFd, jsonLine)
+      fsyncSync(currentFd)
+    } catch {
+      // Retry once after 100ms (non-blocking)
+      await new Promise<void>(resolve => setTimeout(resolve, 100))
+      try {
+        appendFileSync(currentFd, jsonLine)
+        fsyncSync(currentFd)
+      } catch {
+        // Emit failure meta-event to SQLite only (do NOT recurse into NDJSON write)
+        store.insertEvent({
           convoy_id: ids?.convoy_id ?? null,
           task_id: ids?.task_id ?? null,
           worker_id: ids?.worker_id ?? null,
-          ...(data ?? {}),
-        },
-        logsDir ?? null,
-      ).catch(() => {
-        // fire-and-forget: NDJSON write failure must not crash the convoy engine
+          type: 'ndjson_write_failed',
+          data: JSON.stringify({ original_type: type }),
+          created_at: new Date().toISOString(),
+        })
+      }
+    }
+  }
+
+  return {
+    emit(type, data, ids) {
+      // Event data is NOT secret-scanned here because all user-generated content
+      // (task output, DLQ entries) is scanned at its source before reaching the
+      // event emitter. Re-scanning would be redundant. See MF-4 in panel report.
+      const now = new Date().toISOString()
+
+      const eventId = store.insertEvent({
+        convoy_id: ids?.convoy_id ?? null,
+        task_id: ids?.task_id ?? null,
+        worker_id: ids?.worker_id ?? null,
+        type,
+        data: data !== undefined ? JSON.stringify(data) : null,
+        created_at: now,
       })
+
+      // Fire-and-forget: SQLite record (above) is the source of truth.
+      // NDJSON is supplementary — no need to await or block on it.
+      if (fd !== null) {
+        writeNdjson(type, data, ids, now, eventId, fd).catch(() => {
+          // Swallow unhandled rejection — failure already recorded in SQLite via writeNdjson
+        })
+      }
+    },
+
+    close() {
+      if (fd !== null) {
+        closeSync(fd)
+        fd = null
+      }
     },
   }
 }

package/src/cli/convoy/expertise.test.ts

@@ -0,0 +1,128 @@
+import { mkdtempSync, rmSync, realpathSync, writeFileSync, mkdirSync, readFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
+import { readExpertise, updateExpertise, feedCircuitBreaker } from './expertise.js'
+
+vi.mock('./gates.js', () => ({
+  scanForSecrets: vi.fn(() => ({ clean: true, findings: [] })),
+}))
+
+const EXPERTISE_REL = '.opencastle/AGENT-EXPERTISE.md'
+
+function makeBase(): string {
+  const dir = realpathSync(mkdtempSync(join(tmpdir(), 'expertise-test-')))
+  mkdirSync(join(dir, '.opencastle'), { recursive: true })
+  return dir
+}
+
+let tmpDir: string
+
+beforeEach(() => {
+  tmpDir = makeBase()
+  vi.clearAllMocks()
+})
+
+afterEach(() => {
+  rmSync(tmpDir, { recursive: true, force: true })
+})
+
+describe('readExpertise', () => {
+  it('returns empty expertise for missing file', () => {
+    const result = readExpertise('developer', tmpDir)
+    expect(result).toEqual({ strong: [], weak: [], files: [] })
+  })
+
+  it('returns empty expertise when agent section not present', () => {
+    writeFileSync(join(tmpDir, EXPERTISE_REL), '# Agent Expertise\n\n## other-agent\n### Strong Areas\n- Knows stuff\n')
+    const result = readExpertise('developer', tmpDir)
+    expect(result).toEqual({ strong: [], weak: [], files: [] })
+  })
+
+  it('parses strong areas', () => {
+    writeFileSync(
+      join(tmpDir, EXPERTISE_REL),
+      '# Agent Expertise\n\n## developer\n### Strong Areas\n- TypeScript typing\n- React hooks\n### Weak Areas\n### File Familiarity\n',
+    )
+    const result = readExpertise('developer', tmpDir)
+    expect(result.strong).toContain('TypeScript typing')
+    expect(result.strong).toContain('React hooks')
+  })
+
+  it('parses weak areas', () => {
+    writeFileSync(
+      join(tmpDir, EXPERTISE_REL),
+      '# Agent Expertise\n\n## developer\n### Strong Areas\n### Weak Areas\n- CSS animations\n### File Familiarity\n',
+    )
+    const result = readExpertise('developer', tmpDir)
+    expect(result.weak).toContain('CSS animations')
+  })
+
+  it('parses file familiarity', () => {
+    writeFileSync(
+      join(tmpDir, EXPERTISE_REL),
+      '# Agent Expertise\n\n## developer\n### Strong Areas\n### Weak Areas\n### File Familiarity\n- src/cli/engine.ts\n',
+    )
+    const result = readExpertise('developer', tmpDir)
+    expect(result.files).toContain('src/cli/engine.ts')
+  })
+})
+
+describe('updateExpertise', () => {
+  it('creates file if missing when updating success with no retries', () => {
+    updateExpertise('developer', { taskId: 'task-1', success: true, retries: 0, files: ['src/app.ts'] }, tmpDir)
+    const content = readFileSync(join(tmpDir, EXPERTISE_REL), 'utf8')
+    expect(content).toContain('## developer')
+    expect(content).toContain('src/app.ts')
+  })
+
+  it('appends success to Strong Areas', () => {
+    updateExpertise('developer', { taskId: 'task-1', success: true, retries: 0, files: ['src/engine.ts'] }, tmpDir)
+    const result = readExpertise('developer', tmpDir)
+    expect(result.strong.some(s => s.includes('task-1'))).toBe(true)
+  })
+
+  it('appends failure to Weak Areas', () => {
+    updateExpertise('developer', { taskId: 'task-fail', success: false, retries: 3, files: ['src/hard.ts'] }, tmpDir)
+    const result = readExpertise('developer', tmpDir)
+    expect(result.weak.some(w => w.includes('task-fail'))).toBe(true)
+  })
+
+  it('appends to Weak Areas when success with retries > 0', () => {
+    updateExpertise('developer', { taskId: 'task-retry', success: true, retries: 2, files: ['src/tricky.ts'] }, tmpDir)
+    const result = readExpertise('developer', tmpDir)
+    expect(result.weak.some(w => w.includes('task-retry'))).toBe(true)
+  })
+
+  it('rejects secrets in expertise content', async () => {
+    const { scanForSecrets } = await import('./gates.js')
+    vi.mocked(scanForSecrets).mockReturnValueOnce({
+      clean: false,
+      findings: [{ pattern: 'Token', file: '', line: 1, snippet: 'x' }],
+    })
+    // Should not throw; silently skips the write
+    expect(() => {
+      updateExpertise('developer', { taskId: 'leak', success: true, retries: 0, files: [] }, tmpDir)
+    }).not.toThrow()
+  })
+})
+
+describe('feedCircuitBreaker', () => {
+  it('returns weak areas for the agent', () => {
+    writeFileSync(
+      join(tmpDir, EXPERTISE_REL),
+      '# Agent Expertise\n\n## developer\n### Strong Areas\n### Weak Areas\n- database migrations\n### File Familiarity\n',
+    )
+    const result = feedCircuitBreaker('developer', tmpDir)
+    expect(result).toContain('database migrations')
+  })
+
+  it('returns empty array when the agent has no weak areas', () => {
+    writeFileSync(
+      join(tmpDir, EXPERTISE_REL),
+      '# Agent Expertise\n\n## developer\n### Strong Areas\n- everything\n### Weak Areas\n### File Familiarity\n',
+    )
+    const result = feedCircuitBreaker('developer', tmpDir)
+    expect(result).toEqual([])
+  })
+})