opencastle 0.27.0 → 0.27.2

package/src/cli/convoy/types.ts

@@ -1,4 +1,4 @@
-export type ConvoyStatus = 'pending' | 'running' | 'done' | 'failed' | 'gate-failed'
+export type ConvoyStatus = 'pending' | 'running' | 'done' | 'failed' | 'gate-failed' | 'hook-failed'
 
 export type ConvoyTaskStatus =
   | 'pending'
@@ -6,8 +6,13 @@ export type ConvoyTaskStatus =
   | 'running'
   | 'done'
   | 'failed'
+  | 'gate-failed'
+  | 'review-blocked'
   | 'timed-out'
   | 'skipped'
+  | 'hook-failed'
+  | 'disputed'
+  | 'wait-for-input'
 
 export type WorkerStatus = 'spawned' | 'running' | 'done' | 'failed' | 'killed'
 
@@ -24,8 +29,11 @@ export interface ConvoyRecord {
   finished_at: string | null
   spec_yaml: string
   total_tokens: number | null
-  total_cost_usd: string | null
+  total_cost_usd: number | null
   pipeline_id: string | null
+  circuit_state: string | null
+  review_tokens_total: number | null
+  review_budget: number | null
 }
 
 export interface TaskRecord {
@@ -51,7 +59,25 @@ export interface TaskRecord {
   prompt_tokens: number | null
   completion_tokens: number | null
   total_tokens: number | null
-  cost_usd: string | null
+  cost_usd: number | null
+  gates: string | null
+  on_exhausted: 'dlq' | 'skip' | 'stop'
+  injected: number
+  provenance: string | null
+  idempotency_key: string | null
+  current_step: number | null
+  total_steps: number | null
+  review_level: string | null
+  review_verdict: string | null
+  review_tokens: number | null
+  review_model: string | null
+  panel_attempts: number
+  dispute_id: string | null
+  drift_score: number | null
+  drift_retried: number
+  outputs?: string | null          // JSON array of TaskOutput
+  inputs?: string | null           // JSON array of TaskInput
+  discovered_issues?: string | null // JSON array
 }
 
 export interface WorkerRecord {
@@ -88,5 +114,253 @@ export interface PipelineRecord {
   started_at: string | null
   finished_at: string | null
   total_tokens: number | null
-  total_cost_usd: string | null
+  total_cost_usd: number | null
 }
+
+export interface BuiltInGatesConfig {
+  secret_scan?: boolean
+  blast_radius?: boolean
+  dependency_audit?: 'auto' | boolean
+  regression_test?: 'auto' | boolean
+  browser_test?: 'auto' | boolean
+  gate_timeout?: number
+}
+
+
+export interface BrowserTestConfig {
+  urls: string[]
+  check_console_errors?: boolean
+  visual_diff_threshold?: number
+  a11y?: boolean
+  severity_threshold?: 'critical' | 'serious' | 'moderate' | 'minor'
+  baselines_dir?: string
+}
+export interface GuardConfig {
+  enabled?: boolean      // default: true
+  agent?: string         // optional agent name (e.g. 'session-guard')
+  checks?: string[]      // e.g. ['observability', 'cleanup', 'cost-report']
+}
+
+export interface DlqRecord {
+  id: string
+  convoy_id: string
+  task_id: string
+  agent: string
+  failure_type: string
+  error_output: string | null
+  attempts: number
+  tokens_spent: number | null
+  escalation_task_id: string | null
+  resolved: number
+  resolution: string | null
+  created_at: string
+  resolved_at: string | null
+}
+
+export interface CircuitBreakerConfig {
+  threshold?: number      // failures before Open (default: 3)
+  cooldown_ms?: number    // ms in Open before Half-Open (default: 300000 = 5min)
+  fallback_agent?: string // reassign pending tasks when circuit opens
+}
+
+export interface TaskOutput {
+  name: string
+  type: 'file' | 'summary' | 'json'
+  description?: string
+}
+
+export interface TaskInput {
+  from: string
+  name: string
+  as?: string
+}
+
+export interface ArtifactRecord {
+  id: string
+  convoy_id: string
+  task_id: string
+  name: string
+  type: 'file' | 'summary' | 'json'
+  content: string
+  created_at: string
+}
+
+export interface AgentIdentityRecord {
+  id: string
+  agent: string
+  convoy_id: string
+  task_id: string
+  summary: string
+  created_at: string
+  retention_days: number
+}
+
+export interface StepCondition {
+  step: string         // reference previous step by id
+  exitCode?: { eq?: number; ne?: number; gt?: number; lt?: number }
+  fileExists?: { path: string }
+}
+
+export interface TaskStep {
+  id?: string
+  prompt: string
+  gates?: string[]
+  max_retries?: number // inherits from task if omitted
+  if?: StepCondition
+}
+
+export interface Hook {
+  type: 'review' | 'guard' | 'agent' | 'command' | 'validate'
+  name?: string
+  prompt?: string  // for agent hooks
+  command?: string // for command hooks
+  on?: 'pre_task' | 'post_task' | 'post_convoy'
+}
+
+export interface TaskStepRecord {
+  id: number
+  task_id: string
+  step_index: number
+  prompt: string
+  gates: string | null
+  status: string
+  exit_code: number | null
+  output: string | null
+  started_at: string | null
+  finished_at: string | null
+}
+
+export interface WatchTrigger {
+  type: 'file-change' | 'cron' | 'git-push'
+  glob?: string        // for file-change: glob pattern to watch
+  schedule?: string    // for cron: 5-field cron expression
+  branch?: string      // for git-push: branch name pattern
+  debounce_ms?: number // file-change debounce (default: 500ms)
+}
+
+export interface WatchConfig {
+  triggers: WatchTrigger[]
+  clear_scratchpad?: boolean // clear scratchpad on watch start
+  scratchpad_retention_days?: number // auto-clear scratchpad entries older than N days
+}
+
+export interface ScratchpadRecord {
+  key: string
+  value: string
+  updated_at: string
+}
+
+export interface MCPServerConfig {
+  name: string
+  type: string
+  local?: boolean
+  command?: string
+  args?: string[]
+  url?: string
+  config?: Record<string, unknown>
+}
+
+// ---------------------------------------------------------------------------
+// Discriminated union covering every canonical convoy event type.
+// Each variant constrains the `data` shape that callers may pass to emit().
+// ---------------------------------------------------------------------------
+export type ConvoyEventType =
+  | { type: 'convoy_started'; data?: { name?: string } }
+  | { type: 'convoy_finished'; data?: { status: string } }
+  | { type: 'convoy_failed'; data?: { status: string; reason?: string } }
+  | { type: 'convoy_guard'; data?: { checks?: string[]; [key: string]: unknown } }
+  | { type: 'task_started'; data?: { worker_id?: string } }
+  | { type: 'task_done'; data?: { status?: string; retries?: number; worker_id?: string } }
+  | { type: 'task_failed'; data?: { reason: string; worker_id?: string; gate?: string; hook?: string } }
+  | { type: 'task_skipped'; data?: { reason: string } }
+  | { type: 'task_retried'; data?: { previous_status: string } }
+  | { type: 'task_waiting_input'; data?: { task_id?: string; reason?: string } }
+  | { type: 'review_started'; data?: { level: string; task_id?: string; model?: string } }
+  | {
+      type: 'review_verdict'
+      data?: {
+        level: string
+        verdict: string
+        tokens: number
+        model?: string
+        feedback_length?: number
+        budget_exceeded?: boolean
+        budget_downgrade?: boolean
+        budget_skip?: boolean
+        passes?: number
+        blocks?: number
+      }
+    }
+  | { type: 'dispute_opened'; data?: { dispute_id: string; task_id: string; agent?: string; reason?: string } }
+  | { type: 'dlq_entry_created'; data?: { dlq_id: string; task_id: string; agent?: string; attempts?: number } }
+  | { type: 'drift_check_result'; data?: { score?: number; threshold?: number; passed?: boolean } }
+  | { type: 'drift_detected'; data?: { score?: number; files?: string[] } }
+  | { type: 'circuit_breaker_tripped'; data?: { agent?: string; failure_count?: number; threshold?: number } }
+  | { type: 'circuit_breaker_fallback'; data?: { original_agent?: string; fallback_agent?: string; task_id?: string } }
+  | { type: 'circuit_breaker_blocked'; data?: { agent?: string; task_id?: string } }
+  | { type: 'merge_conflict_detected'; data?: { task_id?: string; files?: string[] } }
+  | { type: 'merge_conflict_failed'; data?: { task_id?: string; error?: string } }
+  | { type: 'file_injection_received'; data?: { task_id?: string; from_task?: string; name?: string } }
+  | { type: 'artifact_limit_reached'; data?: { task_id?: string; limit?: number; current?: number } }
+  | { type: 'agent_identity_captured'; data?: { agent?: string; task_id?: string } }
+  | { type: 'agent_identity_rejected'; data?: { agent?: string; task_id?: string; reason?: string } }
+  | { type: 'weak_area_skipped'; data?: { agent?: string; weak_areas?: string[]; task_files?: string[] } }
+  | { type: 'swarm_concurrency_update'; data?: { new_concurrency?: number; reason?: string } }
+  | { type: 'post_convoy_hook_failed'; data?: { hook?: string; error?: string } }
+  | { type: 'session'; data?: { agent?: string; model?: string; task?: string; outcome?: string; duration_min?: number } }
+  | { type: 'delegation'; data?: { agent?: string; model?: string; tier?: string; mechanism?: string; outcome?: string } }
+  | {
+      type: 'secret_leak_prevented'
+      data?: { original_type?: string; patterns?: string[]; task_id?: string; findings_count?: number; context?: string }
+    }
+  | { type: 'ndjson_write_failed'; data?: { original_type?: string } }
+  | { type: 'built_in_gate_result'; data?: { gate: string; passed: boolean; output?: string; level?: string } }
+  | { type: 'watch_started'; data?: { trigger_type?: string; pid?: number } }
+  | { type: 'watch_cycle_start'; data?: { cycle_number?: number; triggered_by?: string } }
+  | { type: 'watch_cycle_end'; data?: { cycle_number?: number; status?: string } }
+  | { type: 'watch_stopped'; data?: { reason?: string } }
+  | { type: 'worker_killed'; data?: { reason?: string; worker_id?: string; task_id?: string } }
+  | { type: 'discovered_issue'; data?: { task_id?: string; title?: string; file?: string; description?: string; severity?: string } }
+
+/** All canonical convoy event type strings. Used for runtime validation. */
+export const KNOWN_EVENT_TYPES: Set<string> = new Set<ConvoyEventType['type']>([
+  'convoy_started',
+  'convoy_finished',
+  'convoy_failed',
+  'convoy_guard',
+  'task_started',
+  'task_done',
+  'task_failed',
+  'task_skipped',
+  'task_retried',
+  'task_waiting_input',
+  'review_started',
+  'review_verdict',
+  'dispute_opened',
+  'dlq_entry_created',
+  'drift_check_result',
+  'drift_detected',
+  'circuit_breaker_tripped',
+  'circuit_breaker_fallback',
+  'circuit_breaker_blocked',
+  'merge_conflict_detected',
+  'merge_conflict_failed',
+  'file_injection_received',
+  'artifact_limit_reached',
+  'agent_identity_captured',
+  'agent_identity_rejected',
+  'weak_area_skipped',
+  'swarm_concurrency_update',
+  'post_convoy_hook_failed',
+  'session',
+  'delegation',
+  'secret_leak_prevented',
+  'ndjson_write_failed',
+  'built_in_gate_result',
+  'watch_started',
+  'watch_cycle_start',
+  'watch_cycle_end',
+  'watch_stopped',
+  'worker_killed',
+  'discovered_issue',
+])

package/src/cli/log.ts

@@ -1,18 +1,29 @@
 import { mkdir, appendFile, stat } from 'node:fs/promises'
+import { readdirSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs'
 import { join, dirname } from 'node:path'
 import type { CliContext } from './types.js'
 
 const HELP = `
   opencastle log [options]
+  opencastle log merge [--since <ISO-date>] [--until <ISO-date>] [--output <path>]
 
-  Append a structured event to the observability log (events.ndjson).
+  Append a structured event to the observability log (events.ndjson),
+  or merge per-convoy NDJSON files into a single file.
 
-  Options:
+  Subcommands:
+    merge         Merge all .opencastle/logs/convoys/*.ndjson into convoy-events.ndjson
+
+  Options (log append):
     --type <type>          Event type (required): session|delegation|review|panel|dispute
     --<field> <value>      Any field from the event schema (see documentation)
     --logs-dir <path>      Override the logs directory path
     --help, -h             Show this help
 
+  Options (merge):
+    --since <ISO-date>     Only include records at or after this date
+    --until <ISO-date>     Only include records at or before this date
+    --output <path>        Output path (default: .opencastle/logs/convoy-events.ndjson)
+
   Array fields (comma-separated): file_partition, lessons_added, discoveries, reviewing_agents
   Boolean fields: escalated, weighted
   Numeric fields: auto-detected from value
@@ -21,6 +32,8 @@ const HELP = `
     opencastle log --type session --agent Developer --model claude-sonnet-4-6 --task "Fix bug" --outcome success
     opencastle log --type delegation --session_id feat/prj-1 --agent Developer --tier fast --mechanism sub-agent --outcome success
     opencastle log --type panel --panel_key auth-review --verdict pass --pass_count 3 --block_count 0
+    opencastle log merge --since 2026-01-01 --output /tmp/merged.ndjson
+    opencastle log merge
 `
 
 const VALID_TYPES = ['session', 'delegation', 'review', 'panel', 'dispute']
@@ -59,6 +72,94 @@ export async function resolveLogsDir(override?: string | null): Promise<string>
   return join(process.cwd(), '.opencastle', 'logs')
 }
 
+/** Merge per-convoy NDJSON files into a single deduplicated, sorted file. */
+export async function mergeConvoyLogs(options: {
+  since?: string
+  until?: string
+  output?: string
+  basePath?: string
+}): Promise<{ merged: number; deduplicated: number; written: number }> {
+  const base = options.basePath ?? process.cwd()
+  const convoysDir = join(base, '.opencastle', 'logs', 'convoys')
+
+  let files: string[] = []
+  try {
+    files = readdirSync(convoysDir)
+      .filter(f => f.endsWith('.ndjson'))
+      .map(f => join(convoysDir, f))
+  } catch {
+    return { merged: 0, deduplicated: 0, written: 0 }
+  }
+
+  if (files.length === 0) {
+    return { merged: 0, deduplicated: 0, written: 0 }
+  }
+
+  const allRecords: Array<Record<string, unknown>> = []
+  let totalRead = 0
+
+  for (const file of files) {
+    const content = readFileSync(file, 'utf8')
+    const lines = content.split('\n').filter(l => l.trim())
+    for (const line of lines) {
+      try {
+        allRecords.push(JSON.parse(line) as Record<string, unknown>)
+        totalRead++
+      } catch {
+        // skip malformed lines
+      }
+    }
+  }
+
+  // Deduplicate by _event_id — keep first occurrence
+  const seen = new Set<unknown>()
+  const unique: Array<Record<string, unknown>> = []
+  for (const record of allRecords) {
+    const id = record['_event_id']
+    if (id !== undefined) {
+      if (seen.has(id)) continue
+      seen.add(id)
+    }
+    unique.push(record)
+  }
+
+  const deduplicatedCount = totalRead - unique.length
+
+  // Filter by since/until
+  let filtered = unique
+  if (options.since) {
+    const since = options.since
+    filtered = filtered.filter(r => {
+      const ts = r['timestamp'] as string | undefined
+      return ts !== undefined && ts >= since
+    })
+  }
+  if (options.until) {
+    const until = options.until
+    filtered = filtered.filter(r => {
+      const ts = r['timestamp'] as string | undefined
+      return ts !== undefined && ts <= until
+    })
+  }
+
+  // Sort by timestamp ascending
+  filtered.sort((a, b) => {
+    const ta = (a['timestamp'] as string) ?? ''
+    const tb = (b['timestamp'] as string) ?? ''
+    return ta < tb ? -1 : ta > tb ? 1 : 0
+  })
+
+  if (filtered.length === 0) {
+    return { merged: totalRead, deduplicated: deduplicatedCount, written: 0 }
+  }
+
+  const outputPath = options.output ?? join(base, '.opencastle', 'logs', 'convoy-events.ndjson')
+  mkdirSync(dirname(outputPath), { recursive: true })
+  writeFileSync(outputPath, filtered.map(r => JSON.stringify(r)).join('\n') + '\n', 'utf8')
+
+  return { merged: totalRead, deduplicated: deduplicatedCount, written: filtered.length }
+}
+
 /** Append a structured event record to events.ndjson. */
 export async function appendEvent(
   record: Record<string, unknown>,
@@ -77,6 +178,23 @@ export default async function log({ args }: CliContext): Promise<void> {
     return
   }
 
+  // merge subcommand
+  if (args[0] === 'merge') {
+    const mergeArgs = args.slice(1)
+    let since: string | undefined
+    let until: string | undefined
+    let output: string | undefined
+    for (let i = 0; i < mergeArgs.length; i++) {
+      const a = mergeArgs[i]
+      if (a === '--since' && i + 1 < mergeArgs.length) { since = mergeArgs[++i]; continue }
+      if (a === '--until' && i + 1 < mergeArgs.length) { until = mergeArgs[++i]; continue }
+      if (a === '--output' && i + 1 < mergeArgs.length) { output = mergeArgs[++i]; continue }
+    }
+    const result = await mergeConvoyLogs({ since, until, output })
+    console.log(`  Merged: ${result.merged} records, Deduplicated: ${result.deduplicated}, Written: ${result.written}`)
+    return
+  }
+
   let type: string | null = null
   let logsDir: string | null = null
   const fields: Record<string, unknown> = {}