opencastle 0.26.1 → 0.27.1

package/src/cli/convoy/store.ts

@@ -1,3 +1,4 @@
+import { copyFileSync } from 'node:fs'
 import { DatabaseSync } from 'node:sqlite'
 import type {
   ConvoyRecord,
@@ -9,12 +10,61 @@ import type {
   EventRecord,
   PipelineRecord,
   PipelineStatus,
+  DlqRecord,
+  ArtifactRecord,
+  AgentIdentityRecord,
+  TaskStepRecord,
 } from './types.js'
 
-const SCHEMA_VERSION = 4
+const SCHEMA_VERSION = 9
+
+// ── Size limits (bytes) ────────────────────────────────────────────────────────
+const LIMIT_SPEC_YAML = 256 * 1024      // 256 KB
+const LIMIT_OUTPUT = 1024 * 1024         // 1 MB (head 512KB + tail 512KB)
+const LIMIT_OUTPUT_HALF = 512 * 1024     // 512 KB per half
+const LIMIT_EVENT_DATA = 64 * 1024       // 64 KB
+const LIMIT_SUMMARY = 4096              // 4 KB
+
+export class FieldSizeLimitError extends Error {
+  constructor(field: string, actual: number, limit: number) {
+    super(`Field "${field}" exceeds size limit: ${actual} bytes > ${limit} bytes`)
+    this.name = 'FieldSizeLimitError'
+  }
+}
+
+function enforceLimit(value: string | null | undefined, field: string, limit: number): void {
+  if (value == null) return
+  const size = Buffer.byteLength(value, 'utf8')
+  if (size > limit) {
+    throw new FieldSizeLimitError(field, size, limit)
+  }
+}
+
+function truncateOutput(value: string | null | undefined): string | null {
+  if (value == null) return null
+  const size = Buffer.byteLength(value, 'utf8')
+  if (size <= LIMIT_OUTPUT) return value
+  // Head + tail truncation with marker
+  const head = value.slice(0, LIMIT_OUTPUT_HALF)
+  const tail = value.slice(-LIMIT_OUTPUT_HALF)
+  return head + '\n\n... [truncated: ' + size + ' bytes total, showing first/last 512KB] ...\n\n' + tail
+}
+
+export class ConvoyArtifactLimitError extends Error {
+  constructor(convoyId: string) {
+    super(`Convoy ${convoyId} has reached the maximum of 50 artifacts`)
+    this.name = 'ConvoyArtifactLimitError'
+  }
+}
 
 export interface ConvoyStore {
-  insertConvoy(record: Omit<ConvoyRecord, 'started_at' | 'finished_at' | 'total_tokens' | 'total_cost_usd' | 'pipeline_id'> & { pipeline_id?: string | null }): void
+  insertConvoy(
+    record: Omit<
+      ConvoyRecord,
+      | 'started_at' | 'finished_at' | 'total_tokens' | 'total_cost_usd'
+      | 'pipeline_id' | 'circuit_state' | 'review_tokens_total' | 'review_budget'
+    > & { pipeline_id?: string | null },
+  ): void
   getConvoy(id: string): ConvoyRecord | undefined
   getLatestConvoy(): ConvoyRecord | undefined
   updateConvoyStatus(
@@ -22,23 +72,54 @@ export interface ConvoyStore {
     status: ConvoyStatus,
     extra?: { started_at?: string; finished_at?: string; total_tokens?: number | null; total_cost_usd?: string | null },
   ): void
+  updateConvoyReviewTokens(convoyId: string, tokens: number): void
+  updateConvoyCircuitState(convoyId: string, state: string | null): void
   insertTask(
     record: Omit<
       TaskRecord,
-      'worker_id' | 'worktree' | 'output' | 'exit_code' | 'started_at' | 'finished_at' | 'prompt_tokens' | 'completion_tokens' | 'total_tokens' | 'cost_usd'
-    >,
+      | 'worker_id' | 'worktree' | 'output' | 'exit_code' | 'started_at' | 'finished_at'
+      | 'prompt_tokens' | 'completion_tokens' | 'total_tokens' | 'cost_usd'
+      | 'on_exhausted' | 'injected' | 'provenance' | 'idempotency_key'
+      | 'current_step' | 'total_steps' | 'review_level' | 'review_verdict'
+      | 'review_tokens' | 'review_model' | 'panel_attempts' | 'dispute_id'
+      | 'drift_score' | 'drift_retried' | 'discovered_issues'
+    > & { outputs?: string | null; inputs?: string | null },
   ): void
+  insertInjectedTask(record: TaskRecord): void
   getTask(id: string, convoyId: string): TaskRecord | undefined
   getTasksByConvoy(convoyId: string): TaskRecord[]
+  getTaskByIdempotencyKey(convoyId: string, key: string): TaskRecord | undefined
+  getTaskByDisputeId(disputeId: string): TaskRecord | undefined
+  getDisputedTasks(convoyId?: string): TaskRecord[]
   updateTaskStatus(
     id: string,
     convoyId: string,
     status: ConvoyTaskStatus,
     extra?: Partial<
-      Pick<TaskRecord, 'worker_id' | 'worktree' | 'output' | 'exit_code' | 'started_at' | 'finished_at' | 'retries' | 'prompt_tokens' | 'completion_tokens' | 'total_tokens' | 'cost_usd'>
+      Pick<
+        TaskRecord,
+        | 'worker_id' | 'worktree' | 'output' | 'exit_code' | 'started_at' | 'finished_at'
+        | 'retries' | 'prompt_tokens' | 'completion_tokens' | 'total_tokens' | 'cost_usd' | 'prompt'
+      >
     >,
   ): void
+  updateTaskReview(
+    taskId: string,
+    convoyId: string,
+    fields: Partial<Pick<TaskRecord, 'review_level' | 'review_verdict' | 'review_tokens' | 'review_model' | 'panel_attempts' | 'dispute_id'>>,
+  ): void
+  updateTaskDrift(
+    taskId: string,
+    convoyId: string,
+    fields: Partial<Pick<TaskRecord, 'drift_score' | 'drift_retried'>>,
+  ): void
+  updateTaskDisputeStatus(taskId: string, convoyId: string, status: ConvoyTaskStatus, disputeId: string): void
   getReadyTasks(convoyId: string): TaskRecord[]
+  insertTaskStep(record: Omit<TaskStepRecord, 'id'>): number
+  updateTaskStep(
+    id: number,
+    fields: Partial<Pick<TaskStepRecord, 'status' | 'exit_code' | 'output' | 'started_at' | 'finished_at'>>,
+  ): void
   insertWorker(record: Omit<WorkerRecord, 'finished_at' | 'last_heartbeat'>): void
   getWorker(id: string): WorkerRecord | undefined
   updateWorkerStatus(
@@ -46,8 +127,24 @@ export interface ConvoyStore {
     status: WorkerStatus,
     extra?: Partial<Pick<WorkerRecord, 'finished_at' | 'last_heartbeat' | 'pid'>>,
   ): void
-  insertEvent(record: Omit<EventRecord, 'id'>): void
+  insertEvent(record: Omit<EventRecord, 'id'>): number
   getEvents(convoyId: string): EventRecord[]
+  insertDlqEntry(record: DlqRecord): void
+  listDlqEntries(convoyIdFilter?: string): DlqRecord[]
+  resolveDlqEntry(id: string, resolution: string): void
+  insertArtifact(record: ArtifactRecord): void
+  getArtifact(convoyId: string, name: string): ArtifactRecord | undefined
+  getArtifactsByTask(taskId: string): ArtifactRecord[]
+  deleteArtifactsOlderThan(days: number): number
+  insertAgentIdentity(record: AgentIdentityRecord): void
+  getAgentIdentities(agent: string, limit: number): AgentIdentityRecord[]
+  listAgentIdentitySummary(): Array<{ agent: string; task_count: number; latest_date: string }>
+  purgeAgentIdentities(agent: string): number
+  deleteAgentIdentitiesOlderThan(days: number): number
+  getScratchpadValue(key: string): string | null
+  setScratchpadValue(key: string, value: string): void
+  clearScratchpad(): void
+  clearScratchpadOlderThan(days: number): void
   insertPipeline(record: Omit<PipelineRecord, 'started_at' | 'finished_at' | 'total_tokens' | 'total_cost_usd'>): void
   getPipeline(id: string): PipelineRecord | undefined
   getLatestPipeline(): PipelineRecord | undefined
@@ -63,8 +160,10 @@ export interface ConvoyStore {
 
 class ConvoyStoreImpl implements ConvoyStore {
   private db: DatabaseSync
+  private dbPath: string
 
   constructor(dbPath: string) {
+    this.dbPath = dbPath
     this.db = new DatabaseSync(dbPath)
     this.db.exec('PRAGMA journal_mode = WAL')
     this.db.exec('PRAGMA synchronous = NORMAL')
@@ -76,18 +175,21 @@ class ConvoyStoreImpl implements ConvoyStore {
     if (version === 0) {
       this.db.exec(`
         CREATE TABLE IF NOT EXISTS convoy (
-          id          TEXT PRIMARY KEY,
-          name        TEXT NOT NULL,
-          spec_hash   TEXT NOT NULL,
-          status      TEXT NOT NULL DEFAULT 'pending',
-          branch      TEXT,
-          created_at  TEXT NOT NULL,
-          started_at  TEXT,
-          finished_at    TEXT,
-          spec_yaml      TEXT NOT NULL,
-          total_tokens   INTEGER,
-          total_cost_usd TEXT,
-          pipeline_id    TEXT
+          id                   TEXT PRIMARY KEY,
+          name                 TEXT NOT NULL,
+          spec_hash            TEXT NOT NULL,
+          status               TEXT NOT NULL DEFAULT 'pending',
+          branch               TEXT,
+          created_at           TEXT NOT NULL,
+          started_at           TEXT,
+          finished_at          TEXT,
+          spec_yaml            TEXT NOT NULL,
+          total_tokens         INTEGER,
+          total_cost_usd       TEXT,
+          pipeline_id          TEXT,
+          circuit_state        TEXT,
+          review_tokens_total  INTEGER,
+          review_budget        INTEGER
         );
 
         CREATE TABLE IF NOT EXISTS pipeline (
@@ -127,7 +229,41 @@ class ConvoyStoreImpl implements ConvoyStore {
           prompt_tokens     INTEGER,
           completion_tokens INTEGER,
           total_tokens      INTEGER,
-          cost_usd          TEXT
+          cost_usd          TEXT,
+          gates             TEXT,
+          on_exhausted      TEXT NOT NULL DEFAULT 'dlq',
+          injected          INTEGER NOT NULL DEFAULT 0,
+          provenance        TEXT,
+          idempotency_key   TEXT,
+          current_step      INTEGER,
+          total_steps       INTEGER,
+          review_level      TEXT,
+          review_verdict    TEXT,
+          review_tokens     INTEGER,
+          review_model      TEXT,
+          panel_attempts    INTEGER NOT NULL DEFAULT 0,
+          dispute_id        TEXT,
+          drift_score       REAL,
+          drift_retried     INTEGER NOT NULL DEFAULT 0,
+          outputs           TEXT,
+          inputs            TEXT,
+          discovered_issues TEXT
+        );
+
+        CREATE UNIQUE INDEX IF NOT EXISTS idx_task_idempotency ON task(convoy_id, idempotency_key)
+          WHERE idempotency_key IS NOT NULL;
+
+        CREATE TABLE IF NOT EXISTS task_step (
+          id          INTEGER PRIMARY KEY AUTOINCREMENT,
+          task_id     TEXT NOT NULL REFERENCES task(id),
+          step_index  INTEGER NOT NULL,
+          prompt      TEXT NOT NULL,
+          gates       TEXT,
+          status      TEXT NOT NULL DEFAULT 'pending',
+          exit_code   INTEGER,
+          output      TEXT,
+          started_at  TEXT,
+          finished_at TEXT
         );
 
         CREATE TABLE IF NOT EXISTS worker (
@@ -152,6 +288,57 @@ class ConvoyStoreImpl implements ConvoyStore {
           data       TEXT,
           created_at TEXT NOT NULL
         );
+
+        CREATE TABLE IF NOT EXISTS dlq (
+          id                TEXT PRIMARY KEY,
+          convoy_id         TEXT NOT NULL REFERENCES convoy(id),
+          task_id           TEXT NOT NULL REFERENCES task(id),
+          agent             TEXT NOT NULL,
+          failure_type      TEXT NOT NULL,
+          error_output      TEXT,
+          attempts          INTEGER NOT NULL,
+          tokens_spent      INTEGER,
+          escalation_task_id TEXT,
+          resolved          INTEGER NOT NULL DEFAULT 0,
+          resolution        TEXT,
+          created_at        TEXT NOT NULL,
+          resolved_at       TEXT
+        );
+
+        CREATE TABLE IF NOT EXISTS artifact (
+          id          TEXT PRIMARY KEY,
+          convoy_id   TEXT NOT NULL REFERENCES convoy(id),
+          task_id     TEXT NOT NULL REFERENCES task(id),
+          name        TEXT NOT NULL,
+          type        TEXT NOT NULL,
+          content     TEXT NOT NULL CHECK (length(content) <= 1048576),
+          created_at  TEXT NOT NULL,
+          UNIQUE(convoy_id, name)
+        );
+
+        CREATE TABLE IF NOT EXISTS agent_identity (
+          id             TEXT PRIMARY KEY,
+          agent          TEXT NOT NULL,
+          convoy_id      TEXT NOT NULL,
+          task_id        TEXT NOT NULL,
+          summary        TEXT NOT NULL,
+          created_at     TEXT NOT NULL,
+          retention_days INTEGER NOT NULL DEFAULT 90
+        );
+
+        CREATE TABLE IF NOT EXISTS scratchpad (
+          key        TEXT PRIMARY KEY,
+          value      TEXT NOT NULL,
+          updated_at TEXT NOT NULL
+        );
+
+        CREATE TABLE IF NOT EXISTS engine_lock (
+          id             INTEGER PRIMARY KEY,
+          pid            INTEGER NOT NULL,
+          hostname       TEXT NOT NULL,
+          started_at     TEXT NOT NULL,
+          last_heartbeat TEXT NOT NULL
+        );
       `)
       this.db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`)
       version = SCHEMA_VERSION
@@ -191,13 +378,44 @@ class ConvoyStoreImpl implements ConvoyStore {
       this.db.exec('PRAGMA user_version = 4')
       version = 4
     }
+    if (version === 4) {
+      migrateSchema(this.db, this.dbPath, 4, 5)
+      version = 5
+    }
+    if (version === 5) {
+      migrateSchema(this.db, this.dbPath, 5, 6)
+      version = 6
+    }
+    if (version === 6) {
+      migrateSchema(this.db, this.dbPath, 6, 7)
+      version = 7
+    }
+    if (version === 7) {
+      migrateSchema(this.db, this.dbPath, 7, 8)
+      version = 8
+    }
+    if (version === 8) {
+      migrateSchema(this.db, this.dbPath, 8, 9)
+      version = 9
+    }
   }
 
-  insertConvoy(record: Omit<ConvoyRecord, 'started_at' | 'finished_at' | 'total_tokens' | 'total_cost_usd' | 'pipeline_id'> & { pipeline_id?: string | null }): void {
+  insertConvoy(
+    record: Omit<
+      ConvoyRecord,
+      | 'started_at' | 'finished_at' | 'total_tokens' | 'total_cost_usd'
+      | 'pipeline_id' | 'circuit_state' | 'review_tokens_total' | 'review_budget'
+    > & { pipeline_id?: string | null },
+  ): void {
+    enforceLimit(record.spec_yaml, 'spec_yaml', LIMIT_SPEC_YAML)
     this.db
       .prepare(
-        `INSERT INTO convoy (id, name, spec_hash, status, branch, created_at, started_at, finished_at, spec_yaml, pipeline_id)
-         VALUES (:id, :name, :spec_hash, :status, :branch, :created_at, NULL, NULL, :spec_yaml, :pipeline_id)`,
+        `INSERT INTO convoy
+           (id, name, spec_hash, status, branch, created_at, started_at, finished_at,
+            spec_yaml, pipeline_id)
+         VALUES
+           (:id, :name, :spec_hash, :status, :branch, :created_at, NULL, NULL,
+            :spec_yaml, :pipeline_id)`,
       )
       .run({ ...record, pipeline_id: record.pipeline_id ?? null })
   }
@@ -242,24 +460,72 @@ class ConvoyStoreImpl implements ConvoyStore {
     this.db.prepare(`UPDATE convoy SET ${sets.join(', ')} WHERE id = :id`).run(params)
   }
 
+  updateConvoyReviewTokens(convoyId: string, tokens: number): void {
+    this.db
+      .prepare(
+        `UPDATE convoy
+         SET review_tokens_total = :tokens
+         WHERE id = :id`,
+      )
+      .run({ id: convoyId, tokens })
+  }
+
+  updateConvoyCircuitState(convoyId: string, state: string | null): void {
+    this.db
+      .prepare('UPDATE convoy SET circuit_state = :state WHERE id = :id')
+      .run({ id: convoyId, state: state ?? null })
+  }
+
   insertTask(
     record: Omit<
       TaskRecord,
-      'worker_id' | 'worktree' | 'output' | 'exit_code' | 'started_at' | 'finished_at' | 'prompt_tokens' | 'completion_tokens' | 'total_tokens' | 'cost_usd'
-    >,
+      | 'worker_id' | 'worktree' | 'output' | 'exit_code' | 'started_at' | 'finished_at'
+      | 'prompt_tokens' | 'completion_tokens' | 'total_tokens' | 'cost_usd'
+      | 'on_exhausted' | 'injected' | 'provenance' | 'idempotency_key'
+      | 'current_step' | 'total_steps' | 'review_level' | 'review_verdict'
+      | 'review_tokens' | 'review_model' | 'panel_attempts' | 'dispute_id'
+      | 'drift_score' | 'drift_retried' | 'discovered_issues'
+    > & { outputs?: string | null; inputs?: string | null },
   ): void {
     this.db
       .prepare(
         `INSERT INTO task
            (id, convoy_id, phase, prompt, agent, adapter, model, timeout_ms, status,
             worker_id, worktree, output, exit_code, started_at, finished_at,
-            retries, max_retries, files, depends_on)
+            retries, max_retries, files, depends_on, gates,
+            on_exhausted, injected, provenance, idempotency_key,
+            outputs, inputs)
          VALUES
            (:id, :convoy_id, :phase, :prompt, :agent, :adapter, :model, :timeout_ms, :status,
             NULL, NULL, NULL, NULL, NULL, NULL,
-            :retries, :max_retries, :files, :depends_on)`,
+            :retries, :max_retries, :files, :depends_on, :gates,
+            'dlq', 0, NULL, NULL,
+            :outputs, :inputs)`,
       )
-      .run(record)
+      .run({ ...record, outputs: record.outputs ?? null, inputs: record.inputs ?? null })
+  }
+
+  insertInjectedTask(record: TaskRecord): void {
+    this.db
+      .prepare(
+        `INSERT INTO task
+           (id, convoy_id, phase, prompt, agent, adapter, model, timeout_ms, status,
+            worker_id, worktree, output, exit_code, started_at, finished_at,
+            retries, max_retries, files, depends_on, gates,
+            on_exhausted, injected, provenance, idempotency_key,
+            current_step, total_steps, review_level, review_verdict,
+            review_tokens, review_model, panel_attempts, dispute_id,
+            drift_score, drift_retried, outputs, inputs, discovered_issues)
+         VALUES
+           (:id, :convoy_id, :phase, :prompt, :agent, :adapter, :model, :timeout_ms, :status,
+            :worker_id, :worktree, :output, :exit_code, :started_at, :finished_at,
+            :retries, :max_retries, :files, :depends_on, :gates,
+            :on_exhausted, :injected, :provenance, :idempotency_key,
+            :current_step, :total_steps, :review_level, :review_verdict,
+            :review_tokens, :review_model, :panel_attempts, :dispute_id,
+            :drift_score, :drift_retried, :outputs, :inputs, :discovered_issues)`,
+      )
+      .run(record as unknown as Record<string, string | number | null>)
   }
 
   getTask(id: string, convoyId: string): TaskRecord | undefined {
@@ -274,17 +540,50 @@ class ConvoyStoreImpl implements ConvoyStore {
       .all({ convoy_id: convoyId }) as unknown as TaskRecord[]
   }
 
+  getTaskByIdempotencyKey(convoyId: string, key: string): TaskRecord | undefined {
+    return this.db
+      .prepare('SELECT * FROM task WHERE convoy_id = :convoy_id AND idempotency_key = :key')
+      .get({ convoy_id: convoyId, key }) as TaskRecord | undefined
+  }
+
+  getTaskByDisputeId(disputeId: string): TaskRecord | undefined {
+    return this.db
+      .prepare('SELECT * FROM task WHERE dispute_id = :dispute_id LIMIT 1')
+      .get({ dispute_id: disputeId }) as TaskRecord | undefined
+  }
+
+  getDisputedTasks(convoyId?: string): TaskRecord[] {
+    if (convoyId) {
+      return this.db
+        .prepare("SELECT * FROM task WHERE status = 'disputed' AND convoy_id = :convoy_id ORDER BY phase, id")
+        .all({ convoy_id: convoyId }) as unknown as TaskRecord[]
+    }
+    return this.db
+      .prepare("SELECT * FROM task WHERE status = 'disputed' ORDER BY convoy_id, phase, id")
+      .all({}) as unknown as TaskRecord[]
+  }
+
   updateTaskStatus(
     id: string,
     convoyId: string,
     status: ConvoyTaskStatus,
     extra?: Partial<
-      Pick<TaskRecord, 'worker_id' | 'worktree' | 'output' | 'exit_code' | 'started_at' | 'finished_at' | 'retries' | 'prompt_tokens' | 'completion_tokens' | 'total_tokens' | 'cost_usd'>
+      Pick<
+        TaskRecord,
+        | 'worker_id' | 'worktree' | 'output' | 'exit_code' | 'started_at' | 'finished_at'
+        | 'retries' | 'prompt_tokens' | 'completion_tokens' | 'total_tokens' | 'cost_usd' | 'prompt'
+      >
     >,
   ): void {
+    if (extra?.output !== undefined) {
+      extra = { ...extra, output: truncateOutput(extra.output) }
+    }
     const sets = ['status = :status']
     const params: Record<string, string | number | null> = { id, convoy_id: convoyId, status }
-    const extraFields = ['worker_id', 'worktree', 'output', 'exit_code', 'started_at', 'finished_at', 'retries', 'prompt_tokens', 'completion_tokens', 'total_tokens', 'cost_usd'] as const
+    const extraFields = [
+      'worker_id', 'worktree', 'output', 'exit_code', 'started_at', 'finished_at',
+      'retries', 'prompt_tokens', 'completion_tokens', 'total_tokens', 'cost_usd', 'prompt',
+    ] as const
 
     if (extra) {
       for (const field of extraFields) {
@@ -312,6 +611,88 @@ class ConvoyStoreImpl implements ConvoyStore {
     })
   }
 
+  insertTaskStep(record: Omit<TaskStepRecord, 'id'>): number {
+    this.db
+      .prepare(
+        `INSERT INTO task_step
+           (task_id, step_index, prompt, gates, status, exit_code, output, started_at, finished_at)
+         VALUES
+           (:task_id, :step_index, :prompt, :gates, :status, :exit_code, :output, :started_at, :finished_at)`,
+      )
+      .run(record)
+    const row = this.db.prepare('SELECT last_insert_rowid() AS id').get() as { id: number }
+    return row.id
+  }
+
+  updateTaskStep(
+    id: number,
+    fields: Partial<Pick<TaskStepRecord, 'status' | 'exit_code' | 'output' | 'started_at' | 'finished_at'>>,
+  ): void {
+    const sets: string[] = []
+    const params: Record<string, string | number | null> = { id }
+    const stepFields = ['status', 'exit_code', 'output', 'started_at', 'finished_at'] as const
+
+    for (const field of stepFields) {
+      if (field in fields && fields[field] !== undefined) {
+        sets.push(`${field} = :${field}`)
+        params[field] = fields[field] as string | number | null
+      }
+    }
+
+    if (sets.length === 0) return
+    this.db.prepare(`UPDATE task_step SET ${sets.join(', ')} WHERE id = :id`).run(params)
+  }
+
+  updateTaskReview(
+    taskId: string,
+    convoyId: string,
+    fields: Partial<Pick<TaskRecord, 'review_level' | 'review_verdict' | 'review_tokens' | 'review_model' | 'panel_attempts' | 'dispute_id'>>,
+  ): void {
+    const sets: string[] = []
+    const params: Record<string, string | number | null> = { id: taskId, convoy_id: convoyId }
+    const reviewFields = ['review_level', 'review_verdict', 'review_tokens', 'review_model', 'panel_attempts', 'dispute_id'] as const
+
+    for (const field of reviewFields) {
+      if (field in fields && fields[field] !== undefined) {
+        sets.push(`${field} = :${field}`)
+        params[field] = fields[field] as string | number | null
+      }
+    }
+
+    if (sets.length === 0) return
+    this.db.prepare(`UPDATE task SET ${sets.join(', ')} WHERE id = :id AND convoy_id = :convoy_id`).run(params)
+  }
+
+  updateTaskDrift(
+    taskId: string,
+    convoyId: string,
+    fields: Partial<Pick<TaskRecord, 'drift_score' | 'drift_retried'>>,
+  ): void {
+    const sets: string[] = []
+    const params: Record<string, string | number | null> = { id: taskId, convoy_id: convoyId }
+
+    if (fields.drift_score !== undefined) {
+      sets.push('drift_score = :drift_score')
+      params.drift_score = fields.drift_score
+    }
+    if (fields.drift_retried !== undefined) {
+      sets.push('drift_retried = :drift_retried')
+      params.drift_retried = fields.drift_retried
+    }
+
+    if (sets.length === 0) return
+    this.db.prepare(`UPDATE task SET ${sets.join(', ')} WHERE id = :id AND convoy_id = :convoy_id`).run(params)
+  }
+
+  updateTaskDisputeStatus(taskId: string, convoyId: string, status: ConvoyTaskStatus, disputeId: string): void {
+    this.db
+      .prepare(
+        `UPDATE task SET status = :status, dispute_id = :dispute_id
+         WHERE id = :id AND convoy_id = :convoy_id`,
+      )
+      .run({ id: taskId, convoy_id: convoyId, status, dispute_id: disputeId })
+  }
+
   insertWorker(record: Omit<WorkerRecord, 'finished_at' | 'last_heartbeat'>): void {
     this.db
       .prepare(
@@ -355,13 +736,16 @@ class ConvoyStoreImpl implements ConvoyStore {
     this.db.prepare(`UPDATE worker SET ${sets.join(', ')} WHERE id = :id`).run(params)
   }
 
-  insertEvent(record: Omit<EventRecord, 'id'>): void {
+  insertEvent(record: Omit<EventRecord, 'id'>): number {
+    enforceLimit(record.data, 'event.data', LIMIT_EVENT_DATA)
     this.db
       .prepare(
         `INSERT INTO event (convoy_id, task_id, worker_id, type, data, created_at)
          VALUES (:convoy_id, :task_id, :worker_id, :type, :data, :created_at)`,
       )
       .run(record)
+    const row = this.db.prepare('SELECT last_insert_rowid() AS id').get() as { id: number }
+    return row.id
   }
 
   getEvents(convoyId: string): EventRecord[] {
@@ -370,7 +754,160 @@ class ConvoyStoreImpl implements ConvoyStore {
       .all({ convoy_id: convoyId }) as unknown as EventRecord[]
   }
 
+  insertDlqEntry(record: DlqRecord): void {
+    this.db
+      .prepare(
+        `INSERT INTO dlq
+           (id, convoy_id, task_id, agent, failure_type, error_output, attempts,
+            tokens_spent, escalation_task_id, resolved, resolution, created_at, resolved_at)
+         VALUES
+           (:id, :convoy_id, :task_id, :agent, :failure_type, :error_output, :attempts,
+            :tokens_spent, :escalation_task_id, :resolved, :resolution, :created_at, :resolved_at)`,
+      )
+      .run(record as unknown as Record<string, string | number | null>)
+  }
+
+  listDlqEntries(convoyIdFilter?: string): DlqRecord[] {
+    if (convoyIdFilter) {
+      return this.db
+        .prepare('SELECT * FROM dlq WHERE convoy_id = :convoy_id ORDER BY created_at DESC')
+        .all({ convoy_id: convoyIdFilter }) as unknown as DlqRecord[]
+    }
+    return this.db
+      .prepare('SELECT * FROM dlq ORDER BY created_at DESC')
+      .all() as unknown as DlqRecord[]
+  }
+
+  resolveDlqEntry(id: string, resolution: string): void {
+    this.db
+      .prepare(
+        `UPDATE dlq SET resolved = 1, resolution = :resolution, resolved_at = :resolved_at
+         WHERE id = :id`,
+      )
+      .run({ id, resolution, resolved_at: new Date().toISOString() })
+  }
+
+  insertArtifact(record: ArtifactRecord): void {
+    const count = (
+      this.db
+        .prepare('SELECT COUNT(*) AS cnt FROM artifact WHERE convoy_id = :convoy_id')
+        .get({ convoy_id: record.convoy_id }) as { cnt: number }
+    ).cnt
+    if (count >= 50) {
+      throw new ConvoyArtifactLimitError(record.convoy_id)
+    }
+    this.db
+      .prepare(
+        `INSERT INTO artifact (id, convoy_id, task_id, name, type, content, created_at)
+         VALUES (:id, :convoy_id, :task_id, :name, :type, :content, :created_at)`,
+      )
+      .run(record as unknown as Record<string, string | number | null>)
+  }
+
+  getArtifact(convoyId: string, name: string): ArtifactRecord | undefined {
+    return this.db
+      .prepare('SELECT * FROM artifact WHERE convoy_id = :convoy_id AND name = :name')
+      .get({ convoy_id: convoyId, name }) as ArtifactRecord | undefined
+  }
+
+  getArtifactsByTask(taskId: string): ArtifactRecord[] {
+    return this.db
+      .prepare('SELECT * FROM artifact WHERE task_id = :task_id ORDER BY created_at')
+      .all({ task_id: taskId }) as unknown as ArtifactRecord[]
+  }
+
+  deleteArtifactsOlderThan(days: number): number {
+    const cutoff = new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString()
+    const result = this.db
+      .prepare(
+        `DELETE FROM artifact WHERE convoy_id IN (
+           SELECT id FROM convoy WHERE finished_at IS NOT NULL AND finished_at < :cutoff
+         )`,
+      )
+      .run({ cutoff })
+    return (result as unknown as { changes: number }).changes
+  }
+
+  insertAgentIdentity(record: AgentIdentityRecord): void {
+    const summarySize = Buffer.byteLength(record.summary, 'utf8')
+    const truncatedSummary = summarySize > LIMIT_SUMMARY
+      ? record.summary.slice(0, LIMIT_SUMMARY)
+      : record.summary
+    this.db
+      .prepare(
+        `INSERT INTO agent_identity
+           (id, agent, convoy_id, task_id, summary, created_at, retention_days)
+         VALUES
+           (:id, :agent, :convoy_id, :task_id, :summary, :created_at, :retention_days)`,
+      )
+      .run({ ...record, summary: truncatedSummary } as unknown as Record<string, string | number | null>)
+  }
+
+  getAgentIdentities(agent: string, limit: number): AgentIdentityRecord[] {
+    return this.db
+      .prepare(
+        'SELECT * FROM agent_identity WHERE agent = :agent ORDER BY created_at DESC LIMIT :limit',
+      )
+      .all({ agent, limit }) as unknown as AgentIdentityRecord[]
+  }
+
+  listAgentIdentitySummary(): Array<{ agent: string; task_count: number; latest_date: string }> {
+    return this.db
+      .prepare(
+        `SELECT agent, COUNT(*) AS task_count, MAX(created_at) AS latest_date
+         FROM agent_identity GROUP BY agent ORDER BY agent`,
+      )
+      .all() as unknown as Array<{ agent: string; task_count: number; latest_date: string }>
+  }
+
+  purgeAgentIdentities(agent: string): number {
+    const result = this.db
+      .prepare('DELETE FROM agent_identity WHERE agent = :agent')
+      .run({ agent })
+    return (result as unknown as { changes: number }).changes
+  }
+
+  deleteAgentIdentitiesOlderThan(days: number): number {
+    const cutoff = new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString()
+    const result = this.db
+      .prepare(
+        `DELETE FROM agent_identity
+         WHERE created_at < :cutoff
+            OR (retention_days IS NOT NULL
+                AND created_at < datetime('now', '-' || retention_days || ' days'))`,
+      )
+      .run({ cutoff })
+    return (result as unknown as { changes: number }).changes
+  }
+
+  getScratchpadValue(key: string): string | null {
+    const row = this.db
+      .prepare('SELECT value FROM scratchpad WHERE key = :key')
+      .get({ key }) as { value: string } | undefined
+    return row?.value ?? null
+  }
+
+  setScratchpadValue(key: string, value: string): void {
+    this.db
+      .prepare(
+        `INSERT INTO scratchpad (key, value, updated_at)
+         VALUES (:key, :value, :updated_at)
+         ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at`,
+      )
+      .run({ key, value, updated_at: new Date().toISOString() })
+  }
+
+  clearScratchpad(): void {
+    this.db.exec('DELETE FROM scratchpad')
+  }
+
+  clearScratchpadOlderThan(days: number): void {
+    const cutoff = new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString()
+    this.db.prepare('DELETE FROM scratchpad WHERE updated_at < :cutoff').run({ cutoff })
+  }
+
   insertPipeline(record: Omit<PipelineRecord, 'started_at' | 'finished_at' | 'total_tokens' | 'total_cost_usd'>): void {
+    enforceLimit(record.spec_yaml, 'pipeline.spec_yaml', LIMIT_SPEC_YAML)
     this.db
       .prepare(
         `INSERT INTO pipeline (id, name, status, branch, spec_yaml, convoy_specs, created_at,
@@ -444,6 +981,115 @@ class ConvoyStoreImpl implements ConvoyStore {
   }
 }
 
+export function migrateSchema(db: DatabaseSync, dbPath: string, fromVersion: number, toVersion: number): void {
+  for (let v = fromVersion; v < toVersion; v++) {
+    const backupPath = `${dbPath}.v${v}.bak`
+    copyFileSync(dbPath, backupPath)
+    db.exec('BEGIN')
+    try {
+      if (v === 4) {
+        db.exec(`
+          ALTER TABLE task ADD COLUMN gates TEXT;
+          ALTER TABLE task ADD COLUMN on_exhausted TEXT NOT NULL DEFAULT 'dlq';
+          ALTER TABLE task ADD COLUMN injected INTEGER NOT NULL DEFAULT 0;
+          ALTER TABLE task ADD COLUMN provenance TEXT;
+          ALTER TABLE task ADD COLUMN idempotency_key TEXT;
+          CREATE UNIQUE INDEX idx_task_idempotency ON task(convoy_id, idempotency_key)
+            WHERE idempotency_key IS NOT NULL;
+          ALTER TABLE convoy ADD COLUMN circuit_state TEXT;
+          CREATE TABLE dlq (
+            id                TEXT PRIMARY KEY,
+            convoy_id         TEXT NOT NULL REFERENCES convoy(id),
+            task_id           TEXT NOT NULL REFERENCES task(id),
+            agent             TEXT NOT NULL,
+            failure_type      TEXT NOT NULL,
+            error_output      TEXT,
+            attempts          INTEGER NOT NULL,
+            tokens_spent      INTEGER,
+            escalation_task_id TEXT,
+            resolved          INTEGER NOT NULL DEFAULT 0,
+            resolution        TEXT,
+            created_at        TEXT NOT NULL,
+            resolved_at       TEXT
+          );
+        `)
+      }
+      if (v === 5) {
+        db.exec(`
+          ALTER TABLE task ADD COLUMN current_step INTEGER;
+          ALTER TABLE task ADD COLUMN total_steps INTEGER;
+          ALTER TABLE task ADD COLUMN review_level TEXT;
+          ALTER TABLE task ADD COLUMN review_verdict TEXT;
+          ALTER TABLE task ADD COLUMN review_tokens INTEGER;
+          ALTER TABLE task ADD COLUMN review_model TEXT;
+          ALTER TABLE task ADD COLUMN panel_attempts INTEGER NOT NULL DEFAULT 0;
+          ALTER TABLE task ADD COLUMN dispute_id TEXT;
+          ALTER TABLE convoy ADD COLUMN review_tokens_total INTEGER;
+          ALTER TABLE convoy ADD COLUMN review_budget INTEGER;
+          CREATE TABLE task_step (
+            id          INTEGER PRIMARY KEY AUTOINCREMENT,
+            task_id     TEXT NOT NULL REFERENCES task(id),
+            step_index  INTEGER NOT NULL,
+            prompt      TEXT NOT NULL,
+            gates       TEXT,
+            status      TEXT NOT NULL DEFAULT 'pending',
+            exit_code   INTEGER,
+            output      TEXT,
+            started_at  TEXT,
+            finished_at TEXT
+          );
+        `)
+      }
+      if (v === 6) {
+        db.exec(`
+          ALTER TABLE task ADD COLUMN drift_score REAL;
+          ALTER TABLE task ADD COLUMN drift_retried INTEGER NOT NULL DEFAULT 0;
+        `)
+      }
+      if (v === 7) {
+        db.exec(`
+          ALTER TABLE task ADD COLUMN outputs TEXT;
+          ALTER TABLE task ADD COLUMN inputs TEXT;
+          ALTER TABLE task ADD COLUMN discovered_issues TEXT;
+          CREATE TABLE artifact (
+            id          TEXT PRIMARY KEY,
+            convoy_id   TEXT NOT NULL REFERENCES convoy(id),
+            task_id     TEXT NOT NULL REFERENCES task(id),
+            name        TEXT NOT NULL,
+            type        TEXT NOT NULL,
+            content     TEXT NOT NULL CHECK (length(content) <= 1048576),
+            created_at  TEXT NOT NULL,
+            UNIQUE(convoy_id, name)
+          );
+          CREATE TABLE agent_identity (
+            id             TEXT PRIMARY KEY,
+            agent          TEXT NOT NULL,
+            convoy_id      TEXT NOT NULL,
+            task_id        TEXT NOT NULL,
+            summary        TEXT NOT NULL,
+            created_at     TEXT NOT NULL,
+            retention_days INTEGER NOT NULL DEFAULT 90
+          );
+        `)
+      }
+      if (v === 8) {
+        db.exec(`
+          CREATE TABLE scratchpad (
+            key        TEXT PRIMARY KEY,
+            value      TEXT NOT NULL,
+            updated_at TEXT NOT NULL
+          );
+        `)
+      }
+      db.exec('COMMIT')
+    } catch (err) {
+      try { db.exec('ROLLBACK') } catch { /* ignore */ }
+      throw new Error(`Migration v${v}→v${v + 1} failed. Backup at ${backupPath}. Original error: ${(err as Error).message}`)
+    }
+    db.exec(`PRAGMA user_version = ${v + 1}`)
+  }
+}
+
 export function createConvoyStore(dbPath: string): ConvoyStore {
   return new ConvoyStoreImpl(dbPath)
 }