opencastle 0.27.1 → 0.27.2

package/src/cli/convoy/event-schemas.ts

@@ -0,0 +1,195 @@
+import * as v from 'valibot'
+
+type AnySchema = v.BaseSchema<unknown, unknown, v.BaseIssue<unknown>>
+
+export const EVENT_DATA_SCHEMAS: Record<string, AnySchema> = {
+  convoy_started: v.looseObject({ name: v.optional(v.string()) }),
+  convoy_finished: v.looseObject({ status: v.string() }),
+  convoy_failed: v.looseObject({ status: v.string(), reason: v.optional(v.string()) }),
+  convoy_guard: v.looseObject({ checks: v.optional(v.array(v.string())) }),
+
+  task_started: v.looseObject({ worker_id: v.optional(v.string()) }),
+  task_done: v.looseObject({
+    status: v.optional(v.string()),
+    retries: v.optional(v.number()),
+    worker_id: v.optional(v.string()),
+  }),
+  task_failed: v.looseObject({
+    reason: v.string(),
+    worker_id: v.optional(v.string()),
+    gate: v.optional(v.string()),
+    hook: v.optional(v.string()),
+  }),
+  task_skipped: v.looseObject({ reason: v.string() }),
+  task_retried: v.looseObject({ previous_status: v.string() }),
+  task_waiting_input: v.looseObject({
+    task_id: v.optional(v.string()),
+    reason: v.optional(v.string()),
+  }),
+
+  review_started: v.looseObject({
+    level: v.string(),
+    task_id: v.optional(v.string()),
+    model: v.optional(v.string()),
+  }),
+  review_verdict: v.looseObject({
+    level: v.string(),
+    verdict: v.string(),
+    tokens: v.number(),
+    model: v.optional(v.string()),
+    feedback_length: v.optional(v.number()),
+    budget_exceeded: v.optional(v.boolean()),
+    budget_downgrade: v.optional(v.boolean()),
+    budget_skip: v.optional(v.boolean()),
+    passes: v.optional(v.number()),
+    blocks: v.optional(v.number()),
+  }),
+  dispute_opened: v.looseObject({
+    dispute_id: v.string(),
+    task_id: v.string(),
+    agent: v.optional(v.string()),
+    reason: v.optional(v.string()),
+  }),
+  dlq_entry_created: v.looseObject({
+    dlq_id: v.string(),
+    task_id: v.string(),
+    agent: v.optional(v.string()),
+    attempts: v.optional(v.number()),
+  }),
+
+  drift_check_result: v.looseObject({
+    score: v.optional(v.number()),
+    threshold: v.optional(v.number()),
+    passed: v.optional(v.boolean()),
+  }),
+  drift_detected: v.looseObject({
+    score: v.optional(v.number()),
+    files: v.optional(v.array(v.string())),
+  }),
+
+  circuit_breaker_tripped: v.looseObject({
+    agent: v.optional(v.string()),
+    failure_count: v.optional(v.number()),
+    threshold: v.optional(v.number()),
+  }),
+  circuit_breaker_fallback: v.looseObject({
+    original_agent: v.optional(v.string()),
+    fallback_agent: v.optional(v.string()),
+    task_id: v.optional(v.string()),
+  }),
+  circuit_breaker_blocked: v.looseObject({
+    agent: v.optional(v.string()),
+    task_id: v.optional(v.string()),
+  }),
+
+  merge_conflict_detected: v.looseObject({
+    task_id: v.optional(v.string()),
+    files: v.optional(v.array(v.string())),
+  }),
+  merge_conflict_failed: v.looseObject({
+    task_id: v.optional(v.string()),
+    error: v.optional(v.string()),
+  }),
+
+  file_injection_received: v.looseObject({
+    task_id: v.optional(v.string()),
+    from_task: v.optional(v.string()),
+    name: v.optional(v.string()),
+  }),
+  artifact_limit_reached: v.looseObject({
+    task_id: v.optional(v.string()),
+    limit: v.optional(v.number()),
+    current: v.optional(v.number()),
+  }),
+
+  agent_identity_captured: v.looseObject({
+    agent: v.optional(v.string()),
+    task_id: v.optional(v.string()),
+  }),
+  agent_identity_rejected: v.looseObject({
+    agent: v.optional(v.string()),
+    task_id: v.optional(v.string()),
+    reason: v.optional(v.string()),
+  }),
+
+  weak_area_skipped: v.looseObject({
+    agent: v.optional(v.string()),
+    weak_areas: v.optional(v.array(v.string())),
+    task_files: v.optional(v.array(v.string())),
+  }),
+  swarm_concurrency_update: v.looseObject({
+    new_concurrency: v.optional(v.number()),
+    reason: v.optional(v.string()),
+  }),
+  post_convoy_hook_failed: v.looseObject({
+    hook: v.optional(v.string()),
+    error: v.optional(v.string()),
+  }),
+  session: v.looseObject({
+    agent: v.optional(v.string()),
+    model: v.optional(v.string()),
+    task: v.optional(v.string()),
+    outcome: v.optional(v.string()),
+    duration_min: v.optional(v.number()),
+  }),
+  delegation: v.looseObject({
+    agent: v.optional(v.string()),
+    model: v.optional(v.string()),
+    tier: v.optional(v.string()),
+    mechanism: v.optional(v.string()),
+    outcome: v.optional(v.string()),
+  }),
+  secret_leak_prevented: v.looseObject({
+    original_type: v.optional(v.string()),
+    patterns: v.optional(v.array(v.string())),
+    task_id: v.optional(v.string()),
+    findings_count: v.optional(v.number()),
+    context: v.optional(v.string()),
+  }),
+  ndjson_write_failed: v.looseObject({ original_type: v.optional(v.string()) }),
+  built_in_gate_result: v.looseObject({
+    gate: v.string(),
+    passed: v.boolean(),
+    output: v.optional(v.string()),
+    level: v.optional(v.string()),
+  }),
+  watch_started: v.looseObject({
+    trigger_type: v.optional(v.string()),
+    pid: v.optional(v.number()),
+  }),
+  watch_cycle_start: v.looseObject({
+    cycle_number: v.optional(v.number()),
+    triggered_by: v.optional(v.string()),
+  }),
+  watch_cycle_end: v.looseObject({
+    cycle_number: v.optional(v.number()),
+    status: v.optional(v.string()),
+  }),
+  watch_stopped: v.looseObject({ reason: v.optional(v.string()) }),
+  worker_killed: v.looseObject({
+    reason: v.optional(v.string()),
+    worker_id: v.optional(v.string()),
+    task_id: v.optional(v.string()),
+  }),
+  discovered_issue: v.looseObject({
+    task_id: v.optional(v.string()),
+    title: v.optional(v.string()),
+    file: v.optional(v.string()),
+    description: v.optional(v.string()),
+    severity: v.optional(v.string()),
+  }),
+}
+export function validateEventData(
+  type: string,
+  data: unknown,
+): { valid: boolean; issues?: string[] } {
+  const schema = EVENT_DATA_SCHEMAS[type]
+  if (schema === undefined) return { valid: true }
+  if (data === undefined || data === null) return { valid: true }
+  const result = v.safeParse(schema, data)
+  if (result.success) return { valid: true }
+  return {
+    valid: false,
+    issues: result.issues.map((i) => i.message),
+  }
+}

package/src/cli/convoy/events.test.ts

@@ -2,10 +2,10 @@ import { mkdtempSync, readFileSync, writeFileSync, rmSync, existsSync, mkdirSync
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { realpathSync } from 'node:fs'
-import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
 import { createConvoyStore } from './store.js'
-import { createEventEmitter } from './events.js'
-import { recoverNdjson } from './engine.js'
+import { createEventEmitter, ndjsonPathForConvoy, recoverNdjson, validateEventType } from './events.js'
+import { KNOWN_EVENT_TYPES } from './types.js'
 import type { ConvoyStore } from './store.js'
 
 let tmpDir: string
@@ -34,6 +34,13 @@ afterEach(() => {
 })
 
 describe('createEventEmitter', () => {
+  it('throws TypeError when options is a string', () => {
+    expect(() => createEventEmitter(store, 'bad' as unknown as any)).toThrow(TypeError)
+    expect(() => createEventEmitter(store, 'bad' as unknown as any)).toThrow(
+      'createEventEmitter options must be an object, not a string',
+    )
+  })
+
   it('inserts the event into SQLite', () => {
     const emitter = createEventEmitter(store)
     emitter.emit('task_started', { msg: 'started' }, { convoy_id: 'c1' })
@@ -127,6 +134,33 @@ describe('createEventEmitter', () => {
     emitter.close()
     expect(() => emitter.close()).not.toThrow()
   })
+
+  it('sanitizes reserved keys from caller data before NDJSON write', () => {
+    const emitter = createEventEmitter(store, { ndjsonPath })
+    emitter.emit(
+      'task_started',
+      {
+        convoy_id: 'attacker',
+        timestamp: 'fake',
+        _event_id: 999,
+        type: 'evil',
+        task_id: 'injected',
+        worker_id: 'hacker',
+        custom_field: 'ok',
+      },
+      { convoy_id: 'c1', task_id: 't1', worker_id: 'w1' },
+    )
+    emitter.close()
+    const content = readFileSync(ndjsonPath, 'utf8')
+    const line = JSON.parse(content.trim())
+    expect(line.convoy_id).toBe('c1')
+    expect(line.task_id).toBe('t1')
+    expect(line.worker_id).toBe('w1')
+    expect(line.type).toBe('task_started')
+    expect(line._event_id).not.toBe(999)
+    expect(line.timestamp).not.toBe('fake')
+    expect(line.custom_field).toBe('ok')
+  })
 })
 
 describe('crash resilience', () => {
@@ -255,5 +289,175 @@ describe('crash resilience', () => {
     const linesAfter = readFileSync(ndjsonPath, 'utf8').split('\n').filter(l => l.trim())
     expect(linesAfter).toHaveLength(count)
   })
+
+  it('6. canonical-field protection: event.data cannot override DB row fields', () => {
+    // Insert an event whose data contains attacker-controlled reserved keys
+    store.insertEvent({
+      convoy_id: 'c1',
+      task_id: 'legit-task',
+      worker_id: 'legit-worker',
+      type: 'task_done',
+      data: JSON.stringify({
+        convoy_id: 'attacker',
+        timestamp: 'fake',
+        _event_id: 9999,
+        type: 'evil',
+        task_id: 'injected',
+        worker_id: 'hacker',
+        safe_field: 'this-is-fine',
+      }),
+      created_at: new Date().toISOString(),
+    })
+
+    recoverNdjson(store, 'c1', ndjsonPath)
+
+    const lines = readFileSync(ndjsonPath, 'utf8').split('\n').filter(l => l.trim())
+    expect(lines).toHaveLength(1)
+    const record = JSON.parse(lines[0]) as Record<string, unknown>
+
+    // Canonical fields must come from the DB row, not from data
+    expect(record.convoy_id).toBe('c1')
+    expect(record.task_id).toBe('legit-task')
+    expect(record.worker_id).toBe('legit-worker')
+    expect(record.type).toBe('task_done')
+    expect(record._event_id).not.toBe(9999)
+    expect(record.timestamp).not.toBe('fake')
+
+    // Attacker values must not appear
+    expect(record.convoy_id).not.toBe('attacker')
+    expect(record.type).not.toBe('evil')
+    expect(record.task_id).not.toBe('injected')
+    expect(record.worker_id).not.toBe('hacker')
+
+    // Safe non-reserved fields are preserved
+    expect(record.safe_field).toBe('this-is-fine')
+  })
+})
+
+describe('KNOWN_EVENT_TYPES', () => {
+  it('contains all canonical event types', () => {
+    const canonical = [
+      'convoy_started', 'convoy_finished', 'convoy_failed', 'convoy_guard',
+      'task_started', 'task_done', 'task_failed', 'task_skipped', 'task_retried', 'task_waiting_input',
+      'review_started', 'review_verdict', 'dispute_opened', 'dlq_entry_created',
+      'drift_check_result', 'drift_detected',
+      'circuit_breaker_tripped', 'circuit_breaker_fallback', 'circuit_breaker_blocked',
+      'merge_conflict_detected', 'merge_conflict_failed',
+      'file_injection_received', 'artifact_limit_reached',
+      'agent_identity_captured', 'agent_identity_rejected',
+      'weak_area_skipped', 'swarm_concurrency_update', 'post_convoy_hook_failed',
+      'session', 'delegation',
+      'secret_leak_prevented', 'ndjson_write_failed', 'built_in_gate_result',
+      'watch_started', 'watch_cycle_start', 'watch_cycle_end', 'watch_stopped',
+      'worker_killed', 'discovered_issue',
+    ]
+    for (const type of canonical) {
+      expect(KNOWN_EVENT_TYPES.has(type)).toBe(true)
+    }
+  })
+
+  it('has no duplicates (Set size matches array)', () => {
+    expect(KNOWN_EVENT_TYPES.size).toBeGreaterThanOrEqual(37)
+  })
+})
+
+describe('ndjsonPathForConvoy', () => {
+  it('returns correct per-convoy path with default basePath', () => {
+    const result = ndjsonPathForConvoy('abc-123')
+    expect(result).toBe(join(process.cwd(), '.opencastle', 'logs', 'convoys', 'abc-123.ndjson'))
+  })
+
+  it('returns correct per-convoy path with custom basePath', () => {
+    const result = ndjsonPathForConvoy('xyz-456', '/custom/base')
+    expect(result).toBe(join('/custom/base', '.opencastle', 'logs', 'convoys', 'xyz-456.ndjson'))
+  })
+})
+
+describe('createEventEmitter directory creation', () => {
+  it('creates parent directory when ndjsonPath is in a non-existent directory', () => {
+    const nestedPath = join(tmpDir, 'nested', 'dir', 'events.ndjson')
+    const emitter = createEventEmitter(store, { ndjsonPath: nestedPath })
+    emitter.emit('convoy_started', { name: 'test' }, { convoy_id: 'c1' })
+    emitter.close()
+    expect(existsSync(nestedPath)).toBe(true)
+    const content = readFileSync(nestedPath, 'utf8')
+    const line = JSON.parse(content.trim())
+    expect(line.type).toBe('convoy_started')
+  })
+})
+
+describe('validateEventType', () => {
+  it('returns true for known event types', () => {
+    expect(validateEventType('convoy_started')).toBe(true)
+    expect(validateEventType('task_done')).toBe(true)
+    expect(validateEventType('watch_stopped')).toBe(true)
+  })
+
+  it('returns false for unknown event types', () => {
+    expect(validateEventType('unknown_event')).toBe(false)
+    expect(validateEventType('')).toBe(false)
+    expect(validateEventType('convoy_start')).toBe(false)
+  })
+})
+
+describe('emit-time data validation', () => {
+  let warnSpy: ReturnType<typeof vi.spyOn>
+
+  beforeEach(() => {
+    warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+  })
+
+  afterEach(() => {
+    warnSpy.mockRestore()
+  })
+
+  it('valid data passes without warning', () => {
+    const emitter = createEventEmitter(store)
+    emitter.emit('convoy_finished', { status: 'done' }, { convoy_id: 'c1' })
+    emitter.close()
+    expect(warnSpy).not.toHaveBeenCalled()
+  })
+
+  it('invalid data shape warns with correct message', () => {
+    const emitter = createEventEmitter(store)
+    emitter.emit('convoy_finished', { status: 123 } as unknown as Record<string, unknown>, { convoy_id: 'c1' })
+    emitter.close()
+    expect(warnSpy).toHaveBeenCalledWith(
+      expect.stringContaining('Invalid data for event type "convoy_finished"'),
+    )
+  })
+
+  it('missing required field warns', () => {
+    const emitter = createEventEmitter(store)
+    // task_failed requires reason: string — passing {} should fail
+    emitter.emit('task_failed', {} as Record<string, unknown>, { convoy_id: 'c1' })
+    emitter.close()
+    expect(warnSpy).toHaveBeenCalledWith(
+      expect.stringContaining('Invalid data for event type "task_failed"'),
+    )
+  })
+
+  it('extra fields are allowed (no warning)', () => {
+    const emitter = createEventEmitter(store)
+    emitter.emit('convoy_started', { name: 'test', extra: true } as Record<string, unknown>, { convoy_id: 'c1' })
+    emitter.close()
+    expect(warnSpy).not.toHaveBeenCalled()
+  })
+
+  it('undefined data is valid', () => {
+    const emitter = createEventEmitter(store)
+    emitter.emit('convoy_started', undefined, { convoy_id: 'c1' })
+    emitter.close()
+    expect(warnSpy).not.toHaveBeenCalled()
+  })
+
+  it('unknown event type bypasses data validation (only one warning)', () => {
+    const emitter = createEventEmitter(store)
+    emitter.emit('unknown_type_xyz', { any: 'data' }, { convoy_id: 'c1' })
+    emitter.close()
+    expect(warnSpy).toHaveBeenCalledTimes(1)
+    expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('Unknown event type: "unknown_type_xyz"'))
+    expect(warnSpy).not.toHaveBeenCalledWith(expect.stringContaining('Invalid data'))
+  })
 })
 

package/src/cli/convoy/events.ts

@@ -1,7 +1,21 @@
-import { appendFileSync, closeSync, fsyncSync, openSync } from 'node:fs'
+import { appendFileSync, closeSync, fsyncSync, mkdirSync, openSync, readFileSync, writeFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
 import type { ConvoyStore } from './store.js'
+import { KNOWN_EVENT_TYPES } from './types.js'
+import { validateEventData } from './event-schemas.js'
+
+const RESERVED_KEYS = new Set(['_event_id', 'convoy_id', 'task_id', 'worker_id', 'timestamp', 'type'])
 import { scanForSecrets } from './gates.js'
 
+export function validateEventType(type: string): boolean {
+  return KNOWN_EVENT_TYPES.has(type)
+}
+
+export function ndjsonPathForConvoy(convoyId: string, basePath?: string): string {
+  const base = basePath ?? process.cwd()
+  return join(base, '.opencastle', 'logs', 'convoys', `${convoyId}.ndjson`)
+}
+
 export interface ConvoyEventEmitter {
   emit(
     type: string,
@@ -15,8 +29,13 @@ export function createEventEmitter(
   store: ConvoyStore,
   options?: { ndjsonPath?: string },
 ): ConvoyEventEmitter {
+  if (typeof options === 'string') {
+    throw new TypeError('createEventEmitter options must be an object, not a string')
+  }
+
   let fd: number | null = null
   if (options?.ndjsonPath) {
+    mkdirSync(dirname(options.ndjsonPath), { recursive: true })
     fd = openSync(options.ndjsonPath, 'a')
   }
 
@@ -30,6 +49,12 @@ export function createEventEmitter(
     eventId: number,
     currentFd: number,
   ): Promise<void> {
+    const safeData: Record<string, unknown> = {}
+    if (data) {
+      for (const [k, v] of Object.entries(data)) {
+        if (!RESERVED_KEYS.has(k)) safeData[k] = v
+      }
+    }
     const record = {
       _event_id: eventId,
       timestamp: now,
@@ -37,7 +62,7 @@ export function createEventEmitter(
       convoy_id: ids?.convoy_id ?? null,
       task_id: ids?.task_id ?? null,
       worker_id: ids?.worker_id ?? null,
-      ...(data ?? {}),
+      ...safeData,
     }
     const jsonLine = JSON.stringify(record) + '\n'
 
@@ -80,9 +105,16 @@ export function createEventEmitter(
 
   return {
     emit(type, data, ids) {
-      // Event data is NOT secret-scanned here because all user-generated content
-      // (task output, DLQ entries) is scanned at its source before reaching the
-      // event emitter. Re-scanning would be redundant. See MF-4 in panel report.
+      // SQLite insert is not scanned; NDJSON write is scanned via writeNdjson().
+      // User-generated content (task output, DLQ entries) is scanned at its source
+      // before reaching the event emitter. See MF-4 in panel report.
+      if (!validateEventType(type)) {
+        console.warn(`[convoy] Unknown event type: "${type}"`)
+      }
+      const dataValidation = validateEventData(type, data)
+      if (!dataValidation.valid) {
+        console.warn(`[convoy] Invalid data for event type "${type}": ${dataValidation.issues?.join(', ')}`)
+      }
       const now = new Date().toISOString()
 
       const eventId = store.insertEvent({
@@ -111,3 +143,85 @@ export function createEventEmitter(
     },
   }
 }
+
+function safeJsonParse(raw: string): Record<string, unknown> {
+  try {
+    return JSON.parse(raw) as Record<string, unknown>
+  } catch {
+    return {}
+  }
+}
+
+/**
+ * Truncate any trailing partial line in the NDJSON file, then replay any SQLite
+ * events for the given convoy that are missing from the file.
+ * Exported for unit testing.
+ */
+export function recoverNdjson(store: ConvoyStore, convoyId: string, ndjsonPath: string): void {
+  // 1. Read the NDJSON file (if it exists)
+  let fileContent: string
+  try {
+    fileContent = readFileSync(ndjsonPath, 'utf8')
+  } catch {
+    fileContent = ''
+  }
+
+  // 2. Truncate any partial trailing line (no \n terminator)
+  if (fileContent.length > 0 && !fileContent.endsWith('\n')) {
+    const lastNewline = fileContent.lastIndexOf('\n')
+    if (lastNewline === -1) {
+      writeFileSync(ndjsonPath, '')
+      fileContent = ''
+    } else {
+      writeFileSync(ndjsonPath, fileContent.slice(0, lastNewline + 1))
+      fileContent = fileContent.slice(0, lastNewline + 1)
+    }
+  }
+
+  // 3. Count valid NDJSON event IDs for this convoy
+  const ndjsonIds = new Set<number>()
+  for (const line of fileContent.split('\n')) {
+    if (!line.trim()) continue
+    try {
+      const parsed = JSON.parse(line) as Record<string, unknown>
+      if (parsed.convoy_id === convoyId && parsed._event_id != null) {
+        ndjsonIds.add(parsed._event_id as number)
+      }
+    } catch {
+      // Skip unparseable lines
+    }
+  }
+
+  // 4. Get all SQLite events for this convoy
+  const sqliteEvents = store.getEvents(convoyId)
+
+  // 5. Replay missing events (those in SQLite but not in NDJSON)
+  const missing = sqliteEvents.filter(e => e.id != null && !ndjsonIds.has(e.id!))
+  if (missing.length > 0) {
+    const fd = openSync(ndjsonPath, 'a')
+    try {
+      for (const event of missing) {
+        const parsedData = event.data ? safeJsonParse(event.data) : {}
+        // Strip reserved keys from event.data to prevent attacker-controlled
+        // values from overriding canonical fields from the DB row.
+        const safeData: Record<string, unknown> = {}
+        for (const [key, value] of Object.entries(parsedData)) {
+          if (!RESERVED_KEYS.has(key)) safeData[key] = value
+        }
+        const record = {
+          ...safeData,
+          _event_id: event.id,
+          timestamp: event.created_at,
+          type: event.type,
+          convoy_id: event.convoy_id,
+          task_id: event.task_id,
+          worker_id: event.worker_id,
+        }
+        appendFileSync(fd, JSON.stringify(record) + '\n')
+      }
+      fsyncSync(fd)
+    } finally {
+      closeSync(fd)
+    }
+  }
+}