opencastle 0.27.0 → 0.27.2

package/src/cli/convoy/events.ts

@@ -1,5 +1,20 @@
-import { appendEvent as appendNdjson } from '../log.js'
+import { appendFileSync, closeSync, fsyncSync, mkdirSync, openSync, readFileSync, writeFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
 import type { ConvoyStore } from './store.js'
+import { KNOWN_EVENT_TYPES } from './types.js'
+import { validateEventData } from './event-schemas.js'
+
+const RESERVED_KEYS = new Set(['_event_id', 'convoy_id', 'task_id', 'worker_id', 'timestamp', 'type'])
+import { scanForSecrets } from './gates.js'
+
+export function validateEventType(type: string): boolean {
+  return KNOWN_EVENT_TYPES.has(type)
+}
+
+export function ndjsonPathForConvoy(convoyId: string, basePath?: string): string {
+  const base = basePath ?? process.cwd()
+  return join(base, '.opencastle', 'logs', 'convoys', `${convoyId}.ndjson`)
+}
 
 export interface ConvoyEventEmitter {
   emit(
@@ -7,35 +22,206 @@ export interface ConvoyEventEmitter {
     data?: Record<string, unknown>,
     ids?: { convoy_id?: string; task_id?: string; worker_id?: string },
   ): void
+  close(): void
 }
 
-export function createEventEmitter(store: ConvoyStore, logsDir?: string): ConvoyEventEmitter {
-  return {
-    emit(type, data, ids) {
-      const now = new Date().toISOString()
+export function createEventEmitter(
+  store: ConvoyStore,
+  options?: { ndjsonPath?: string },
+): ConvoyEventEmitter {
+  if (typeof options === 'string') {
+    throw new TypeError('createEventEmitter options must be an object, not a string')
+  }
+
+  let fd: number | null = null
+  if (options?.ndjsonPath) {
+    mkdirSync(dirname(options.ndjsonPath), { recursive: true })
+    fd = openSync(options.ndjsonPath, 'a')
+  }
 
+  // NDJSON writes are supplementary — SQLite is the primary store. Use async
+  // retries to avoid blocking the Node.js event loop.
+  async function writeNdjson(
+    type: string,
+    data: Record<string, unknown> | undefined,
+    ids: { convoy_id?: string; task_id?: string; worker_id?: string } | undefined,
+    now: string,
+    eventId: number,
+    currentFd: number,
+  ): Promise<void> {
+    const safeData: Record<string, unknown> = {}
+    if (data) {
+      for (const [k, v] of Object.entries(data)) {
+        if (!RESERVED_KEYS.has(k)) safeData[k] = v
+      }
+    }
+    const record = {
+      _event_id: eventId,
+      timestamp: now,
+      type,
+      convoy_id: ids?.convoy_id ?? null,
+      task_id: ids?.task_id ?? null,
+      worker_id: ids?.worker_id ?? null,
+      ...safeData,
+    }
+    const jsonLine = JSON.stringify(record) + '\n'
+
+    const scanResult = scanForSecrets(jsonLine, 'ndjson')
+    if (!scanResult.clean) {
+      // Block the NDJSON write — record the blocked event in SQLite only
       store.insertEvent({
         convoy_id: ids?.convoy_id ?? null,
         task_id: ids?.task_id ?? null,
         worker_id: ids?.worker_id ?? null,
-        type,
-        data: data !== undefined ? JSON.stringify(data) : null,
+        type: 'secret_leak_prevented',
+        data: JSON.stringify({ original_type: type, patterns: scanResult.findings.map(f => f.pattern) }),
         created_at: now,
       })
+      return
+    }
 
-      appendNdjson(
-        {
-          timestamp: now,
-          type,
+    try {
+      appendFileSync(currentFd, jsonLine)
+      fsyncSync(currentFd)
+    } catch {
+      // Retry once after 100ms (non-blocking)
+      await new Promise<void>(resolve => setTimeout(resolve, 100))
+      try {
+        appendFileSync(currentFd, jsonLine)
+        fsyncSync(currentFd)
+      } catch {
+        // Emit failure meta-event to SQLite only (do NOT recurse into NDJSON write)
+        store.insertEvent({
           convoy_id: ids?.convoy_id ?? null,
           task_id: ids?.task_id ?? null,
           worker_id: ids?.worker_id ?? null,
-          ...(data ?? {}),
-        },
-        logsDir ?? null,
-      ).catch(() => {
-        // fire-and-forget: NDJSON write failure must not crash the convoy engine
+          type: 'ndjson_write_failed',
+          data: JSON.stringify({ original_type: type }),
+          created_at: new Date().toISOString(),
+        })
+      }
+    }
+  }
+
+  return {
+    emit(type, data, ids) {
+      // SQLite insert is not scanned; NDJSON write is scanned via writeNdjson().
+      // User-generated content (task output, DLQ entries) is scanned at its source
+      // before reaching the event emitter. See MF-4 in panel report.
+      if (!validateEventType(type)) {
+        console.warn(`[convoy] Unknown event type: "${type}"`)
+      }
+      const dataValidation = validateEventData(type, data)
+      if (!dataValidation.valid) {
+        console.warn(`[convoy] Invalid data for event type "${type}": ${dataValidation.issues?.join(', ')}`)
+      }
+      const now = new Date().toISOString()
+
+      const eventId = store.insertEvent({
+        convoy_id: ids?.convoy_id ?? null,
+        task_id: ids?.task_id ?? null,
+        worker_id: ids?.worker_id ?? null,
+        type,
+        data: data !== undefined ? JSON.stringify(data) : null,
+        created_at: now,
       })
+
+      // Fire-and-forget: SQLite record (above) is the source of truth.
+      // NDJSON is supplementary — no need to await or block on it.
+      if (fd !== null) {
+        writeNdjson(type, data, ids, now, eventId, fd).catch(() => {
+          // Swallow unhandled rejection — failure already recorded in SQLite via writeNdjson
+        })
+      }
     },
+
+    close() {
+      if (fd !== null) {
+        closeSync(fd)
+        fd = null
+      }
+    },
+  }
+}
+
+function safeJsonParse(raw: string): Record<string, unknown> {
+  try {
+    return JSON.parse(raw) as Record<string, unknown>
+  } catch {
+    return {}
+  }
+}
+
+/**
+ * Truncate any trailing partial line in the NDJSON file, then replay any SQLite
+ * events for the given convoy that are missing from the file.
+ * Exported for unit testing.
+ */
+export function recoverNdjson(store: ConvoyStore, convoyId: string, ndjsonPath: string): void {
+  // 1. Read the NDJSON file (if it exists)
+  let fileContent: string
+  try {
+    fileContent = readFileSync(ndjsonPath, 'utf8')
+  } catch {
+    fileContent = ''
+  }
+
+  // 2. Truncate any partial trailing line (no \n terminator)
+  if (fileContent.length > 0 && !fileContent.endsWith('\n')) {
+    const lastNewline = fileContent.lastIndexOf('\n')
+    if (lastNewline === -1) {
+      writeFileSync(ndjsonPath, '')
+      fileContent = ''
+    } else {
+      writeFileSync(ndjsonPath, fileContent.slice(0, lastNewline + 1))
+      fileContent = fileContent.slice(0, lastNewline + 1)
+    }
+  }
+
+  // 3. Count valid NDJSON event IDs for this convoy
+  const ndjsonIds = new Set<number>()
+  for (const line of fileContent.split('\n')) {
+    if (!line.trim()) continue
+    try {
+      const parsed = JSON.parse(line) as Record<string, unknown>
+      if (parsed.convoy_id === convoyId && parsed._event_id != null) {
+        ndjsonIds.add(parsed._event_id as number)
+      }
+    } catch {
+      // Skip unparseable lines
+    }
+  }
+
+  // 4. Get all SQLite events for this convoy
+  const sqliteEvents = store.getEvents(convoyId)
+
+  // 5. Replay missing events (those in SQLite but not in NDJSON)
+  const missing = sqliteEvents.filter(e => e.id != null && !ndjsonIds.has(e.id!))
+  if (missing.length > 0) {
+    const fd = openSync(ndjsonPath, 'a')
+    try {
+      for (const event of missing) {
+        const parsedData = event.data ? safeJsonParse(event.data) : {}
+        // Strip reserved keys from event.data to prevent attacker-controlled
+        // values from overriding canonical fields from the DB row.
+        const safeData: Record<string, unknown> = {}
+        for (const [key, value] of Object.entries(parsedData)) {
+          if (!RESERVED_KEYS.has(key)) safeData[key] = value
+        }
+        const record = {
+          ...safeData,
+          _event_id: event.id,
+          timestamp: event.created_at,
+          type: event.type,
+          convoy_id: event.convoy_id,
+          task_id: event.task_id,
+          worker_id: event.worker_id,
+        }
+        appendFileSync(fd, JSON.stringify(record) + '\n')
+      }
+      fsyncSync(fd)
+    } finally {
+      closeSync(fd)
+    }
   }
 }

package/src/cli/convoy/expertise.test.ts

@@ -0,0 +1,128 @@
+import { mkdtempSync, rmSync, realpathSync, writeFileSync, mkdirSync, readFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
+import { readExpertise, updateExpertise, feedCircuitBreaker } from './expertise.js'
+
+vi.mock('./gates.js', () => ({
+  scanForSecrets: vi.fn(() => ({ clean: true, findings: [] })),
+}))
+
+const EXPERTISE_REL = '.opencastle/AGENT-EXPERTISE.md'
+
+function makeBase(): string {
+  const dir = realpathSync(mkdtempSync(join(tmpdir(), 'expertise-test-')))
+  mkdirSync(join(dir, '.opencastle'), { recursive: true })
+  return dir
+}
+
+let tmpDir: string
+
+beforeEach(() => {
+  tmpDir = makeBase()
+  vi.clearAllMocks()
+})
+
+afterEach(() => {
+  rmSync(tmpDir, { recursive: true, force: true })
+})
+
+describe('readExpertise', () => {
+  it('returns empty expertise for missing file', () => {
+    const result = readExpertise('developer', tmpDir)
+    expect(result).toEqual({ strong: [], weak: [], files: [] })
+  })
+
+  it('returns empty expertise when agent section not present', () => {
+    writeFileSync(join(tmpDir, EXPERTISE_REL), '# Agent Expertise\n\n## other-agent\n### Strong Areas\n- Knows stuff\n')
+    const result = readExpertise('developer', tmpDir)
+    expect(result).toEqual({ strong: [], weak: [], files: [] })
+  })
+
+  it('parses strong areas', () => {
+    writeFileSync(
+      join(tmpDir, EXPERTISE_REL),
+      '# Agent Expertise\n\n## developer\n### Strong Areas\n- TypeScript typing\n- React hooks\n### Weak Areas\n### File Familiarity\n',
+    )
+    const result = readExpertise('developer', tmpDir)
+    expect(result.strong).toContain('TypeScript typing')
+    expect(result.strong).toContain('React hooks')
+  })
+
+  it('parses weak areas', () => {
+    writeFileSync(
+      join(tmpDir, EXPERTISE_REL),
+      '# Agent Expertise\n\n## developer\n### Strong Areas\n### Weak Areas\n- CSS animations\n### File Familiarity\n',
+    )
+    const result = readExpertise('developer', tmpDir)
+    expect(result.weak).toContain('CSS animations')
+  })
+
+  it('parses file familiarity', () => {
+    writeFileSync(
+      join(tmpDir, EXPERTISE_REL),
+      '# Agent Expertise\n\n## developer\n### Strong Areas\n### Weak Areas\n### File Familiarity\n- src/cli/engine.ts\n',
+    )
+    const result = readExpertise('developer', tmpDir)
+    expect(result.files).toContain('src/cli/engine.ts')
+  })
+})
+
+describe('updateExpertise', () => {
+  it('creates file if missing when updating success with no retries', () => {
+    updateExpertise('developer', { taskId: 'task-1', success: true, retries: 0, files: ['src/app.ts'] }, tmpDir)
+    const content = readFileSync(join(tmpDir, EXPERTISE_REL), 'utf8')
+    expect(content).toContain('## developer')
+    expect(content).toContain('src/app.ts')
+  })
+
+  it('appends success to Strong Areas', () => {
+    updateExpertise('developer', { taskId: 'task-1', success: true, retries: 0, files: ['src/engine.ts'] }, tmpDir)
+    const result = readExpertise('developer', tmpDir)
+    expect(result.strong.some(s => s.includes('task-1'))).toBe(true)
+  })
+
+  it('appends failure to Weak Areas', () => {
+    updateExpertise('developer', { taskId: 'task-fail', success: false, retries: 3, files: ['src/hard.ts'] }, tmpDir)
+    const result = readExpertise('developer', tmpDir)
+    expect(result.weak.some(w => w.includes('task-fail'))).toBe(true)
+  })
+
+  it('appends to Weak Areas when success with retries > 0', () => {
+    updateExpertise('developer', { taskId: 'task-retry', success: true, retries: 2, files: ['src/tricky.ts'] }, tmpDir)
+    const result = readExpertise('developer', tmpDir)
+    expect(result.weak.some(w => w.includes('task-retry'))).toBe(true)
+  })
+
+  it('rejects secrets in expertise content', async () => {
+    const { scanForSecrets } = await import('./gates.js')
+    vi.mocked(scanForSecrets).mockReturnValueOnce({
+      clean: false,
+      findings: [{ pattern: 'Token', file: '', line: 1, snippet: 'x' }],
+    })
+    // Should not throw; silently skips the write
+    expect(() => {
+      updateExpertise('developer', { taskId: 'leak', success: true, retries: 0, files: [] }, tmpDir)
+    }).not.toThrow()
+  })
+})
+
+describe('feedCircuitBreaker', () => {
+  it('returns weak areas for the agent', () => {
+    writeFileSync(
+      join(tmpDir, EXPERTISE_REL),
+      '# Agent Expertise\n\n## developer\n### Strong Areas\n### Weak Areas\n- database migrations\n### File Familiarity\n',
+    )
+    const result = feedCircuitBreaker('developer', tmpDir)
+    expect(result).toContain('database migrations')
+  })
+
+  it('returns empty array when the agent has no weak areas', () => {
+    writeFileSync(
+      join(tmpDir, EXPERTISE_REL),
+      '# Agent Expertise\n\n## developer\n### Strong Areas\n- everything\n### Weak Areas\n### File Familiarity\n',
+    )
+    const result = feedCircuitBreaker('developer', tmpDir)
+    expect(result).toEqual([])
+  })
+})

package/src/cli/convoy/expertise.ts

@@ -0,0 +1,163 @@
+import { existsSync, readFileSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { scanForSecrets } from './gates.js'
+
+const EXPERTISE_PATH = '.opencastle/AGENT-EXPERTISE.md'
+
+export function readExpertise(
+  agentName: string,
+  basePath?: string,
+): { strong: string[]; weak: string[]; files: string[] } {
+  const base = basePath ?? process.cwd()
+  const filePath = join(base, EXPERTISE_PATH)
+  const empty = { strong: [] as string[], weak: [] as string[], files: [] as string[] }
+  if (!existsSync(filePath)) return empty
+
+  const content = readFileSync(filePath, 'utf8')
+  const lines = content.split('\n')
+  const agentHeaderRegex = new RegExp(
+    '^## ' + agentName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '\\s*$',
+    'i',
+  )
+
+  let inAgentSection = false
+  let currentSubsection = ''
+  const result = { strong: [] as string[], weak: [] as string[], files: [] as string[] }
+
+  for (const line of lines) {
+    if (agentHeaderRegex.test(line)) {
+      inAgentSection = true
+      continue
+    }
+    if (inAgentSection) {
+      if (line.startsWith('## ')) break
+      if (line.startsWith('### ')) {
+        currentSubsection = line.replace(/^###\s*/, '').trim()
+        continue
+      }
+      if (line.startsWith('- ')) {
+        const item = line.replace(/^-\s*/, '').trim()
+        if (currentSubsection === 'Strong Areas') result.strong.push(item)
+        else if (currentSubsection === 'Weak Areas') result.weak.push(item)
+        else if (currentSubsection === 'File Familiarity') result.files.push(item)
+      }
+    }
+  }
+
+  return result
+}
+
+function appendBulletToSubsection(
+  lines: string[],
+  agentName: string,
+  subsection: string,
+  item: string,
+): void {
+  const agentHeaderRegex = new RegExp(
+    '^## ' + agentName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '\\s*$',
+    'i',
+  )
+  const subsectionHeader = '### ' + subsection
+
+  let inAgentSection = false
+  let inSubsection = false
+  let insertAfterLine = -1
+
+  for (let i = 0; i < lines.length; i++) {
+    if (agentHeaderRegex.test(lines[i])) {
+      inAgentSection = true
+      continue
+    }
+    if (inAgentSection) {
+      if (lines[i].startsWith('## ')) break
+      if (lines[i] === subsectionHeader) {
+        inSubsection = true
+        insertAfterLine = i
+        continue
+      }
+      if (inSubsection) {
+        if (lines[i].startsWith('### ') || lines[i].startsWith('## ')) break
+        if (lines[i].startsWith('- ')) {
+          insertAfterLine = i
+        }
+      }
+    }
+  }
+
+  if (insertAfterLine !== -1) {
+    lines.splice(insertAfterLine + 1, 0, '- ' + item)
+  }
+}
+
+function ensureAgentSection(lines: string[], agentName: string): void {
+  const agentHeaderRegex = new RegExp(
+    '^## ' + agentName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '\\s*$',
+    'i',
+  )
+  const exists = lines.some(l => agentHeaderRegex.test(l))
+  if (!exists) {
+    lines.push(
+      '',
+      '## ' + agentName,
+      '',
+      '### Strong Areas',
+      '',
+      '### Weak Areas',
+      '',
+      '### File Familiarity',
+      '',
+    )
+  }
+}
+
+export function updateExpertise(
+  agentName: string,
+  taskResult: { taskId: string; success: boolean; retries: number; files: string[] },
+  basePath?: string,
+): { updated: boolean; reason?: string } {
+  const base = basePath ?? process.cwd()
+  const filePath = join(base, EXPERTISE_PATH)
+
+  const date = new Date().toISOString().slice(0, 10)
+  const outcome = taskResult.success ? 'success' : 'failed'
+  const entryText =
+    '[' + date + '] ' + taskResult.taskId + ': ' + outcome +
+    '/retries=' + taskResult.retries + ', files: [' + taskResult.files.join(', ') + ']'
+
+  const scanResult = scanForSecrets(entryText, 'expertise')
+  if (!scanResult.clean) {
+    return { updated: false, reason: 'secrets_detected' }
+  }
+
+  const initialContent = existsSync(filePath)
+    ? readFileSync(filePath, 'utf8')
+    : '# Agent Expertise\n\nTracking agent performance across tasks.\n'
+
+  const lines = initialContent.split('\n')
+  ensureAgentSection(lines, agentName)
+
+  if (taskResult.success && taskResult.retries === 0) {
+    appendBulletToSubsection(lines, agentName, 'Strong Areas', entryText)
+  } else if (taskResult.success && taskResult.retries > 0) {
+    appendBulletToSubsection(lines, agentName, 'Strong Areas', entryText)
+    appendBulletToSubsection(lines, agentName, 'Weak Areas', entryText)
+  } else {
+    appendBulletToSubsection(lines, agentName, 'Weak Areas', entryText)
+  }
+
+  const existing = readExpertise(agentName, base)
+  const currentFiles = new Set(existing.files)
+  for (const f of taskResult.files) {
+    if (!currentFiles.has(f)) {
+      appendBulletToSubsection(lines, agentName, 'File Familiarity', f)
+      currentFiles.add(f)
+    }
+  }
+
+  writeFileSync(filePath, lines.join('\n'), 'utf8')
+  return { updated: true }
+}
+
+export function feedCircuitBreaker(agentName: string, basePath?: string): string[] {
+  return readExpertise(agentName, basePath).weak
+}

package/src/cli/convoy/export.test.ts

@@ -49,6 +49,7 @@ function insertTask(
     max_retries: 1,
     files: null,
     depends_on: null,
+    gates: null,
   })
   if (status !== 'pending') {
     store.updateTaskStatus(taskId, convoyId, status, {