opencastle 0.27.1 → 0.27.3

package/src/cli/convoy/events.ts

@@ -1,7 +1,21 @@
-import { appendFileSync, closeSync, fsyncSync, openSync } from 'node:fs'
+import { appendFileSync, closeSync, fsyncSync, mkdirSync, openSync, readFileSync, writeFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
 import type { ConvoyStore } from './store.js'
+import { KNOWN_EVENT_TYPES } from './types.js'
+import { validateEventData } from './event-schemas.js'
+
+const RESERVED_KEYS = new Set(['_event_id', 'convoy_id', 'task_id', 'worker_id', 'timestamp', 'type'])
 import { scanForSecrets } from './gates.js'
 
+export function validateEventType(type: string): boolean {
+  return KNOWN_EVENT_TYPES.has(type)
+}
+
+export function ndjsonPathForConvoy(convoyId: string, basePath?: string): string {
+  const base = basePath ?? process.cwd()
+  return join(base, '.opencastle', 'logs', 'convoys', `${convoyId}.ndjson`)
+}
+
 export interface ConvoyEventEmitter {
   emit(
     type: string,
@@ -15,8 +29,13 @@ export function createEventEmitter(
   store: ConvoyStore,
   options?: { ndjsonPath?: string },
 ): ConvoyEventEmitter {
+  if (typeof options === 'string') {
+    throw new TypeError('createEventEmitter options must be an object, not a string')
+  }
+
   let fd: number | null = null
   if (options?.ndjsonPath) {
+    mkdirSync(dirname(options.ndjsonPath), { recursive: true })
     fd = openSync(options.ndjsonPath, 'a')
   }
 
@@ -30,6 +49,12 @@ export function createEventEmitter(
     eventId: number,
     currentFd: number,
   ): Promise<void> {
+    const safeData: Record<string, unknown> = {}
+    if (data) {
+      for (const [k, v] of Object.entries(data)) {
+        if (!RESERVED_KEYS.has(k)) safeData[k] = v
+      }
+    }
     const record = {
       _event_id: eventId,
       timestamp: now,
@@ -37,7 +62,7 @@ export function createEventEmitter(
       convoy_id: ids?.convoy_id ?? null,
       task_id: ids?.task_id ?? null,
       worker_id: ids?.worker_id ?? null,
-      ...(data ?? {}),
+      ...safeData,
     }
     const jsonLine = JSON.stringify(record) + '\n'
 
@@ -80,9 +105,16 @@ export function createEventEmitter(
 
   return {
     emit(type, data, ids) {
-      // Event data is NOT secret-scanned here because all user-generated content
-      // (task output, DLQ entries) is scanned at its source before reaching the
-      // event emitter. Re-scanning would be redundant. See MF-4 in panel report.
+      // SQLite insert is not scanned; NDJSON write is scanned via writeNdjson().
+      // User-generated content (task output, DLQ entries) is scanned at its source
+      // before reaching the event emitter. See MF-4 in panel report.
+      if (!validateEventType(type)) {
+        console.warn(`[convoy] Unknown event type: "${type}"`)
+      }
+      const dataValidation = validateEventData(type, data)
+      if (!dataValidation.valid) {
+        console.warn(`[convoy] Invalid data for event type "${type}": ${dataValidation.issues?.join(', ')}`)
+      }
       const now = new Date().toISOString()
 
       const eventId = store.insertEvent({
@@ -111,3 +143,85 @@ export function createEventEmitter(
     },
   }
 }
+
+function safeJsonParse(raw: string): Record<string, unknown> {
+  try {
+    return JSON.parse(raw) as Record<string, unknown>
+  } catch {
+    return {}
+  }
+}
+
+/**
+ * Truncate any trailing partial line in the NDJSON file, then replay any SQLite
+ * events for the given convoy that are missing from the file.
+ * Exported for unit testing.
+ */
+export function recoverNdjson(store: ConvoyStore, convoyId: string, ndjsonPath: string): void {
+  // 1. Read the NDJSON file (if it exists)
+  let fileContent: string
+  try {
+    fileContent = readFileSync(ndjsonPath, 'utf8')
+  } catch {
+    fileContent = ''
+  }
+
+  // 2. Truncate any partial trailing line (no \n terminator)
+  if (fileContent.length > 0 && !fileContent.endsWith('\n')) {
+    const lastNewline = fileContent.lastIndexOf('\n')
+    if (lastNewline === -1) {
+      writeFileSync(ndjsonPath, '')
+      fileContent = ''
+    } else {
+      writeFileSync(ndjsonPath, fileContent.slice(0, lastNewline + 1))
+      fileContent = fileContent.slice(0, lastNewline + 1)
+    }
+  }
+
+  // 3. Count valid NDJSON event IDs for this convoy
+  const ndjsonIds = new Set<number>()
+  for (const line of fileContent.split('\n')) {
+    if (!line.trim()) continue
+    try {
+      const parsed = JSON.parse(line) as Record<string, unknown>
+      if (parsed.convoy_id === convoyId && parsed._event_id != null) {
+        ndjsonIds.add(parsed._event_id as number)
+      }
+    } catch {
+      // Skip unparseable lines
+    }
+  }
+
+  // 4. Get all SQLite events for this convoy
+  const sqliteEvents = store.getEvents(convoyId)
+
+  // 5. Replay missing events (those in SQLite but not in NDJSON)
+  const missing = sqliteEvents.filter(e => e.id != null && !ndjsonIds.has(e.id!))
+  if (missing.length > 0) {
+    const fd = openSync(ndjsonPath, 'a')
+    try {
+      for (const event of missing) {
+        const parsedData = event.data ? safeJsonParse(event.data) : {}
+        // Strip reserved keys from event.data to prevent attacker-controlled
+        // values from overriding canonical fields from the DB row.
+        const safeData: Record<string, unknown> = {}
+        for (const [key, value] of Object.entries(parsedData)) {
+          if (!RESERVED_KEYS.has(key)) safeData[key] = value
+        }
+        const record = {
+          ...safeData,
+          _event_id: event.id,
+          timestamp: event.created_at,
+          type: event.type,
+          convoy_id: event.convoy_id,
+          task_id: event.task_id,
+          worker_id: event.worker_id,
+        }
+        appendFileSync(fd, JSON.stringify(record) + '\n')
+      }
+      fsyncSync(fd)
+    } finally {
+      closeSync(fd)
+    }
+  }
+}

package/src/cli/convoy/log-merge.test.ts

@@ -0,0 +1,179 @@
+import { mkdtempSync, rmSync, realpathSync, mkdirSync, writeFileSync, readFileSync, existsSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { mergeConvoyLogs } from '../log.js'
+
+const CONVOYS_REL = '.opencastle/logs/convoys'
+const OUTPUT_REL = '.opencastle/logs/convoy-events.ndjson'
+
+function makeBase(): string {
+  const dir = realpathSync(mkdtempSync(join(tmpdir(), 'log-merge-test-')))
+  mkdirSync(join(dir, CONVOYS_REL), { recursive: true })
+  return dir
+}
+
+function writeConvoyFile(base: string, convoyId: string, records: object[]): void {
+  const path = join(base, CONVOYS_REL, `${convoyId}.ndjson`)
+  writeFileSync(path, records.map(r => JSON.stringify(r)).join('\n') + '\n', 'utf8')
+}
+
+let tmpDir: string
+
+beforeEach(() => {
+  tmpDir = makeBase()
+})
+
+afterEach(() => {
+  rmSync(tmpDir, { recursive: true, force: true })
+})
+
+describe('mergeConvoyLogs', () => {
+  it('returns zeros when convoys directory is missing', async () => {
+    rmSync(join(tmpDir, '.opencastle'), { recursive: true, force: true })
+    const result = await mergeConvoyLogs({ basePath: tmpDir })
+    expect(result).toEqual({ merged: 0, deduplicated: 0, written: 0 })
+  })
+
+  it('returns zeros when convoys directory is empty', async () => {
+    const result = await mergeConvoyLogs({ basePath: tmpDir })
+    expect(result).toEqual({ merged: 0, deduplicated: 0, written: 0 })
+  })
+
+  it('merges records from 3 convoy files', async () => {
+    writeConvoyFile(tmpDir, 'convoy-a', [
+      { _event_id: 1, timestamp: '2026-01-01T10:00:00.000Z', type: 'task_started' },
+    ])
+    writeConvoyFile(tmpDir, 'convoy-b', [
+      { _event_id: 2, timestamp: '2026-01-02T10:00:00.000Z', type: 'task_done' },
+    ])
+    writeConvoyFile(tmpDir, 'convoy-c', [
+      { _event_id: 3, timestamp: '2026-01-03T10:00:00.000Z', type: 'session' },
+    ])
+
+    const result = await mergeConvoyLogs({ basePath: tmpDir })
+    expect(result.merged).toBe(3)
+    expect(result.written).toBe(3)
+  })
+
+  it('output is sorted by timestamp ascending', async () => {
+    writeConvoyFile(tmpDir, 'convoy-z', [
+      { _event_id: 10, timestamp: '2026-03-01T00:00:00.000Z', type: 'task_done' },
+      { _event_id: 11, timestamp: '2026-01-01T00:00:00.000Z', type: 'task_started' },
+    ])
+    writeConvoyFile(tmpDir, 'convoy-a', [
+      { _event_id: 12, timestamp: '2026-02-01T00:00:00.000Z', type: 'session' },
+    ])
+
+    const outputPath = join(tmpDir, 'merged.ndjson')
+    await mergeConvoyLogs({ basePath: tmpDir, output: outputPath })
+
+    const lines = readFileSync(outputPath, 'utf8').split('\n').filter(l => l.trim())
+    const timestamps = lines.map(l => (JSON.parse(l) as { timestamp: string }).timestamp)
+    expect(timestamps).toEqual([
+      '2026-01-01T00:00:00.000Z',
+      '2026-02-01T00:00:00.000Z',
+      '2026-03-01T00:00:00.000Z',
+    ])
+  })
+
+  it('deduplicates records by _event_id (keeps first occurrence)', async () => {
+    writeConvoyFile(tmpDir, 'convoy-a', [
+      { _event_id: 5, timestamp: '2026-01-01T00:00:00.000Z', type: 'task_started', note: 'first' },
+    ])
+    writeConvoyFile(tmpDir, 'convoy-b', [
+      { _event_id: 5, timestamp: '2026-01-01T00:00:00.000Z', type: 'task_started', note: 'duplicate' },
+      { _event_id: 6, timestamp: '2026-01-02T00:00:00.000Z', type: 'task_done' },
+    ])
+
+    const outputPath = join(tmpDir, 'merged.ndjson')
+    const result = await mergeConvoyLogs({ basePath: tmpDir, output: outputPath })
+
+    expect(result.merged).toBe(3)
+    expect(result.deduplicated).toBe(1)
+    expect(result.written).toBe(2)
+
+    const lines = readFileSync(outputPath, 'utf8').split('\n').filter(l => l.trim())
+    expect(lines).toHaveLength(2)
+    const first = JSON.parse(lines[0]) as { note: string }
+    expect(first.note).toBe('first')
+  })
+
+  it('filters by --since (inclusive)', async () => {
+    writeConvoyFile(tmpDir, 'convoy-a', [
+      { _event_id: 1, timestamp: '2026-01-01T00:00:00.000Z', type: 'session' },
+      { _event_id: 2, timestamp: '2026-02-01T00:00:00.000Z', type: 'session' },
+      { _event_id: 3, timestamp: '2026-03-01T00:00:00.000Z', type: 'session' },
+    ])
+
+    const outputPath = join(tmpDir, 'merged.ndjson')
+    const result = await mergeConvoyLogs({ basePath: tmpDir, since: '2026-02-01T00:00:00.000Z', output: outputPath })
+
+    expect(result.written).toBe(2)
+    const lines = readFileSync(outputPath, 'utf8').split('\n').filter(l => l.trim())
+    expect(lines).toHaveLength(2)
+  })
+
+  it('filters by --until (inclusive)', async () => {
+    writeConvoyFile(tmpDir, 'convoy-a', [
+      { _event_id: 1, timestamp: '2026-01-01T00:00:00.000Z', type: 'session' },
+      { _event_id: 2, timestamp: '2026-02-01T00:00:00.000Z', type: 'session' },
+      { _event_id: 3, timestamp: '2026-03-01T00:00:00.000Z', type: 'session' },
+    ])
+
+    const outputPath = join(tmpDir, 'merged.ndjson')
+    const result = await mergeConvoyLogs({ basePath: tmpDir, until: '2026-02-01T00:00:00.000Z', output: outputPath })
+
+    expect(result.written).toBe(2)
+  })
+
+  it('filters by --since and --until together', async () => {
+    writeConvoyFile(tmpDir, 'convoy-a', [
+      { _event_id: 1, timestamp: '2026-01-01T00:00:00.000Z', type: 'session' },
+      { _event_id: 2, timestamp: '2026-02-15T00:00:00.000Z', type: 'session' },
+      { _event_id: 3, timestamp: '2026-03-01T00:00:00.000Z', type: 'session' },
+    ])
+
+    const outputPath = join(tmpDir, 'merged.ndjson')
+    const result = await mergeConvoyLogs({
+      basePath: tmpDir,
+      since: '2026-02-01T00:00:00.000Z',
+      until: '2026-02-28T23:59:59.999Z',
+      output: outputPath,
+    })
+
+    expect(result.written).toBe(1)
+    const lines = readFileSync(outputPath, 'utf8').split('\n').filter(l => l.trim())
+    const record = JSON.parse(lines[0]) as { timestamp: string }
+    expect(record.timestamp).toBe('2026-02-15T00:00:00.000Z')
+  })
+
+  it('writes to default output path when --output not specified', async () => {
+    writeConvoyFile(tmpDir, 'convoy-a', [
+      { _event_id: 1, timestamp: '2026-01-01T00:00:00.000Z', type: 'session' },
+    ])
+
+    await mergeConvoyLogs({ basePath: tmpDir })
+
+    const defaultPath = join(tmpDir, '.opencastle', 'logs', 'convoy-events.ndjson')
+    expect(existsSync(defaultPath)).toBe(true)
+  })
+
+  it('returns written: 0 when all records filtered out', async () => {
+    writeConvoyFile(tmpDir, 'convoy-a', [
+      { _event_id: 1, timestamp: '2026-01-01T00:00:00.000Z', type: 'session' },
+    ])
+
+    const result = await mergeConvoyLogs({ basePath: tmpDir, since: '2027-01-01T00:00:00.000Z' })
+    expect(result.written).toBe(0)
+    expect(result.merged).toBe(1)
+  })
+
+  it('skips malformed JSON lines gracefully', async () => {
+    const path = join(tmpDir, CONVOYS_REL, 'convoy-bad.ndjson')
+    writeFileSync(path, '{"_event_id":1,"timestamp":"2026-01-01T00:00:00.000Z","type":"session"}\nnot-valid-json\n{"_event_id":2,"timestamp":"2026-01-02T00:00:00.000Z","type":"task_done"}\n', 'utf8')
+
+    const result = await mergeConvoyLogs({ basePath: tmpDir })
+    expect(result.written).toBe(2)
+  })
+})