agent-relay-server 0.33.0 → 0.34.0

package/src/db/migrations.ts

@@ -0,0 +1,431 @@
+import { Database } from "bun:sqlite";
+import { randomUUID } from "node:crypto";
+import { isRecord, stringValue, isMechanicalMessageKind } from "agent-relay-sdk";
+import { ORCHESTRATOR_PROTOCOL_VERSION, VERSION } from "../config.ts";
+import { parseJson } from "../utils";
+import { isLiveIsolatedWorkspace } from "../workspace-phase";
+import {
+  CONTRACT_REQUIREMENTS,
+  contractCompatibility,
+  parseRuntimeCapabilities,
+  parseRuntimeContracts,
+  parseRuntimePackage,
+  type RuntimeContracts,
+} from "../contracts";
+import { STALE_TTL_MS, DAY_MS, CLAIM_LEASE_MS, POOL_CLAIM_LEASE_MS, WORKSPACE_MERGE_LEASE_MS } from "../config";
+import { matchAgents } from "../agent-ref";
+import { getDb } from "./connection.ts";
+import { normalizeReactionEmoji } from "./mappers.ts";
+import type {
+  AgentCard,
+  ActivityEvent,
+  ActivityEventInput,
+  AgentKind,
+  AgentSessionGuard,
+  Artifact,
+  ArtifactBlob,
+  ArtifactKind,
+  ArtifactLink,
+  ArtifactSensitivity,
+  ArtifactVisibility,
+  AttachmentRef,
+  ChannelBinding,
+  ChannelBindingMode,
+  ChannelRouteTarget,
+  ChatHistoryImport,
+  ChatHistoryImportEntry,
+  ChannelSummary,
+  ChannelTargetHealth,
+  CreatePairInput,
+  HealthCheck,
+  HealthReport,
+  ManagedAgent,
+  ManagedSessionExitDiagnostics,
+  Message,
+  MessageDeliveryAttempt,
+  MessageDeliveryStatus,
+  Orchestrator,
+  OrchestratorHealth,
+  OrchestratorRuntimeInput,
+  OrchestratorStatus,
+  OrchestratorUpgradeState,
+  PairActionInput,
+  PairMessageInput,
+  PairSession,
+  PairStatus,
+  RegisterAgentInput,
+  ReplyObligation,
+  RegisterOrchestratorInput,
+  SendMessageInput,
+  PollQuery,
+  SpawnApprovalMode,
+  SpawnProvider,
+  Task,
+  TaskEvent,
+  TaskSeverity,
+  TaskStatus,
+  IntegrationEventInput,
+  IntegrationSummary,
+  IntegrationTaskStats,
+  InboxDraft,
+  InboxState,
+  InboxThreadState,
+  ContextSnapshot,
+  ContextState,
+  ProviderCapabilities,
+  TaskStatusInput,
+  WorkspaceMetadata,
+  WorkspaceRecord,
+  WorkspaceStatus,
+} from "../types";
+
+// One-time normalization of legacy reaction rows, run on every open (idempotent).
+function normalizeExistingMessageReactions(): void {
+  const rows = getDb().query("SELECT message_id, actor_id, emoji, created_at, updated_at FROM message_reactions").all() as Array<{
+    message_id: number;
+    actor_id: string;
+    emoji: string;
+    created_at: number;
+    updated_at: number;
+  }>;
+  const migrate = getDb().transaction((items: typeof rows) => {
+    const upsert = getDb().query(`
+      INSERT INTO message_reactions (message_id, actor_id, emoji, created_at, updated_at)
+      VALUES (?, ?, ?, ?, ?)
+      ON CONFLICT(message_id, actor_id, emoji) DO UPDATE SET
+        created_at = min(message_reactions.created_at, excluded.created_at),
+        updated_at = max(message_reactions.updated_at, excluded.updated_at)
+    `);
+    const remove = getDb().query("DELETE FROM message_reactions WHERE message_id = ? AND actor_id = ? AND emoji = ?");
+    for (const row of items) {
+      const normalized = normalizeReactionEmoji(row.emoji);
+      if (normalized === row.emoji) continue;
+      upsert.run(row.message_id, row.actor_id, normalized, row.created_at, row.updated_at);
+      remove.run(row.message_id, row.actor_id, row.emoji);
+    }
+  });
+  migrate(rows);
+}
+
+
+// Incremental schema migrations + one-shot backfills, applied after the base DDL
+// in initDb(). Ordering matters: each block guards on the current column/table
+// shape, so they must run in historical order. Built-in agent seeding and the
+// read_by backfill live here too — both run unconditionally and idempotently.
+export function applyMigrations(): void {
+  normalizeExistingMessageReactions();
+
+  // Migrations
+  const cols = getDb().query("PRAGMA table_info(messages)").all() as any[];
+  const colNames = cols.map((c: any) => c.name);
+  if (!colNames.includes("thread_id")) {
+    getDb().run("ALTER TABLE messages ADD COLUMN thread_id INTEGER");
+    getDb().run("ALTER TABLE messages ADD COLUMN reply_to INTEGER REFERENCES messages(id)");
+  }
+  if (!colNames.includes("reply_expected")) {
+    getDb().run("ALTER TABLE messages ADD COLUMN reply_expected INTEGER NOT NULL DEFAULT 1");
+  }
+  if (!colNames.includes("claimable")) {
+    getDb().run("ALTER TABLE messages ADD COLUMN claimable INTEGER NOT NULL DEFAULT 0");
+    getDb().run("ALTER TABLE messages ADD COLUMN claimed_by TEXT");
+    getDb().run("ALTER TABLE messages ADD COLUMN claimed_at INTEGER");
+  }
+  if (!colNames.includes("claim_expires_at")) {
+    getDb().run("ALTER TABLE messages ADD COLUMN claim_expires_at INTEGER");
+  }
+  if (!colNames.includes("idempotency_key")) {
+    getDb().run("ALTER TABLE messages ADD COLUMN idempotency_key TEXT");
+  }
+  if (!colNames.includes("delivery_status")) {
+    getDb().run("ALTER TABLE messages ADD COLUMN delivery_status TEXT NOT NULL DEFAULT 'pending'");
+  }
+  if (!colNames.includes("delivery_attempts")) {
+    getDb().run("ALTER TABLE messages ADD COLUMN delivery_attempts INTEGER NOT NULL DEFAULT 0");
+  }
+  if (!colNames.includes("delivery_last_error")) {
+    getDb().run("ALTER TABLE messages ADD COLUMN delivery_last_error TEXT");
+  }
+  if (!colNames.includes("delivery_next_retry_at")) {
+    getDb().run("ALTER TABLE messages ADD COLUMN delivery_next_retry_at INTEGER");
+  }
+  if (!colNames.includes("delivery_poison_reason")) {
+    getDb().run("ALTER TABLE messages ADD COLUMN delivery_poison_reason TEXT");
+  }
+  if (!colNames.includes("delivery_updated_at")) {
+    getDb().run("ALTER TABLE messages ADD COLUMN delivery_updated_at INTEGER");
+  }
+  if (!colNames.includes("queued_at")) {
+    getDb().run("ALTER TABLE messages ADD COLUMN queued_at INTEGER");
+  }
+  if (!colNames.includes("max_age_seconds")) {
+    getDb().run("ALTER TABLE messages ADD COLUMN max_age_seconds INTEGER");
+  }
+  if (!colNames.includes("resolved_to_agent")) {
+    getDb().run("ALTER TABLE messages ADD COLUMN resolved_to_agent TEXT");
+  }
+  // Event time (#196): when a Runner queues a message in its durable outbox during an
+  // outage, occurred_at preserves when it really happened vs. the later receive time.
+  if (!colNames.includes("occurred_at")) {
+    getDb().run("ALTER TABLE messages ADD COLUMN occurred_at INTEGER");
+  }
+  getDb().query(
+    "UPDATE messages SET claim_expires_at = coalesce(claimed_at, ?) + ? WHERE claimed_by IS NOT NULL AND claim_expires_at IS NULL",
+  ).run(Date.now(), CLAIM_LEASE_MS);
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_msg_thread ON messages(thread_id)");
+  getDb().run("CREATE UNIQUE INDEX IF NOT EXISTS idx_msg_idempotency ON messages(from_agent, idempotency_key) WHERE idempotency_key IS NOT NULL");
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_msg_delivery_status ON messages(delivery_status)");
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_msg_resolved_to_agent ON messages(resolved_to_agent)");
+
+  const tokenCols = getDb().query("PRAGMA table_info(tokens)").all() as any[];
+  const tokenColNames = tokenCols.map((c: any) => c.name);
+  if (!tokenColNames.includes("constraints")) {
+    getDb().run("ALTER TABLE tokens ADD COLUMN constraints TEXT");
+  }
+  if (!tokenColNames.includes("profile_id")) {
+    getDb().run("ALTER TABLE tokens ADD COLUMN profile_id TEXT");
+  }
+
+  getDb().run(`
+    CREATE TABLE IF NOT EXISTS message_delivery_attempts (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      message_id INTEGER NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
+      agent_id TEXT,
+      action TEXT NOT NULL DEFAULT 'attempt',
+      status TEXT NOT NULL,
+      error TEXT,
+      next_retry_at INTEGER,
+      poison_reason TEXT,
+      created_at INTEGER NOT NULL
+    )
+  `);
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_mda_message ON message_delivery_attempts(message_id, created_at DESC)");
+
+  const channelBindingsSql = getDb().query("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'channel_bindings'").get() as { sql?: string } | undefined;
+  if (channelBindingsSql?.sql?.includes("UNIQUE(channel_id, conversation_key)")) {
+    getDb().transaction(() => {
+      getDb().run("ALTER TABLE channel_bindings RENAME TO channel_bindings_old");
+      getDb().run(`
+        CREATE TABLE channel_bindings (
+          id TEXT PRIMARY KEY,
+          channel_id TEXT NOT NULL REFERENCES channels(id) ON DELETE CASCADE,
+          conversation_key TEXT NOT NULL DEFAULT '',
+          conversation_id TEXT,
+          target_type TEXT NOT NULL,
+          target_id TEXT NOT NULL,
+          mode TEXT NOT NULL DEFAULT 'exclusive',
+          priority INTEGER NOT NULL DEFAULT 0,
+          created_at INTEGER NOT NULL,
+          updated_at INTEGER NOT NULL
+        )
+      `);
+      getDb().run(`
+        INSERT INTO channel_bindings (id, channel_id, conversation_key, conversation_id, target_type, target_id, mode, priority, created_at, updated_at)
+        SELECT id, channel_id, conversation_key, conversation_id, target_type, target_id, mode, priority, created_at, updated_at
+        FROM channel_bindings_old
+      `);
+      getDb().run("DROP TABLE channel_bindings_old");
+    })();
+  }
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_channel_bindings_channel ON channel_bindings(channel_id, priority)");
+  getDb().run("CREATE UNIQUE INDEX IF NOT EXISTS idx_channel_bindings_target ON channel_bindings(channel_id, conversation_key, target_type, target_id)");
+
+  const bindingColNames = (getDb().query("PRAGMA table_info(channel_bindings)").all() as any[]).map((c: any) => c.name);
+  if (!bindingColNames.includes("pool_selector")) {
+    getDb().run("ALTER TABLE channel_bindings ADD COLUMN pool_selector TEXT");
+    getDb().run("ALTER TABLE channel_bindings ADD COLUMN pool_agent_id TEXT");
+    getDb().run("ALTER TABLE channel_bindings ADD COLUMN pool_agent_epoch INTEGER");
+    getDb().run("ALTER TABLE channel_bindings ADD COLUMN pool_claim_expires_at INTEGER");
+  }
+
+  if (!colNames.includes("kind")) {
+    getDb().run("ALTER TABLE messages ADD COLUMN kind TEXT NOT NULL DEFAULT 'chat'");
+  }
+  if (!colNames.includes("payload")) {
+    getDb().run("ALTER TABLE messages ADD COLUMN payload TEXT NOT NULL DEFAULT '{}'");
+  }
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_msg_kind ON messages(kind)");
+
+  // Backfill thread_id for pre-migration rows (self-threaded).
+  getDb().run("UPDATE messages SET thread_id = id WHERE thread_id IS NULL");
+
+  const taskCols = getDb().query("PRAGMA table_info(tasks)").all() as any[];
+  const taskColNames = taskCols.map((c: any) => c.name);
+  if (!taskColNames.includes("claim_expires_at")) {
+    getDb().run("ALTER TABLE tasks ADD COLUMN claim_expires_at INTEGER");
+  }
+  getDb().query(
+    "UPDATE tasks SET claim_expires_at = coalesce(claimed_at, ?) + ? WHERE claimed_by IS NOT NULL AND claim_expires_at IS NULL",
+  ).run(Date.now(), CLAIM_LEASE_MS);
+
+  // Migration: orchestrators.api_url
+  const orchCols = getDb().query("PRAGMA table_info(orchestrators)").all() as any[];
+  const orchColNames = orchCols.map((c: any) => c.name);
+  if (!orchColNames.includes("api_url")) {
+    getDb().run("ALTER TABLE orchestrators ADD COLUMN api_url TEXT");
+  }
+
+  const managedStateCols = getDb().query("PRAGMA table_info(managed_agent_state)").all() as any[];
+  const managedStateColNames = managedStateCols.map((c: any) => c.name);
+  if (!managedStateColNames.includes("workspace_id")) {
+    getDb().run("ALTER TABLE managed_agent_state ADD COLUMN workspace_id TEXT");
+    getDb().run("ALTER TABLE managed_agent_state ADD COLUMN workspace_path TEXT");
+    getDb().run("ALTER TABLE managed_agent_state ADD COLUMN workspace_branch TEXT");
+  }
+
+  // message_reads: relational replacement for the read_by JSON array.
+  getDb().run(`
+    CREATE TABLE IF NOT EXISTS message_reads (
+      message_id INTEGER NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
+      agent_id TEXT NOT NULL,
+      read_at INTEGER NOT NULL,
+      PRIMARY KEY (message_id, agent_id)
+    );
+    CREATE INDEX IF NOT EXISTS idx_mr_agent ON message_reads(agent_id);
+  `);
+
+  // Migration: agents.label
+  const agentCols = getDb().query("PRAGMA table_info(agents)").all() as any[];
+  const agentColNames = agentCols.map((c: any) => c.name);
+  if (!agentColNames.includes("label")) {
+    getDb().run("ALTER TABLE agents ADD COLUMN label TEXT");
+  }
+  if (!agentColNames.includes("ready")) {
+    getDb().run("ALTER TABLE agents ADD COLUMN ready INTEGER NOT NULL DEFAULT 0");
+  }
+  if (!agentColNames.includes("instance_id")) {
+    getDb().run("ALTER TABLE agents ADD COLUMN instance_id TEXT");
+  }
+  if (!agentColNames.includes("epoch")) {
+    getDb().run("ALTER TABLE agents ADD COLUMN epoch INTEGER NOT NULL DEFAULT 0");
+  }
+  if (!agentColNames.includes("kind")) {
+    getDb().run("ALTER TABLE agents ADD COLUMN kind TEXT NOT NULL DEFAULT 'provider'");
+    getDb().run(`
+      UPDATE agents
+      SET kind = CASE
+        WHEN id = 'user' THEN 'user'
+        WHEN id = 'system' THEN 'system'
+        WHEN json_extract(meta, '$.kind') = 'channel' THEN 'channel'
+        WHEN EXISTS (SELECT 1 FROM json_each(tags) WHERE value = 'channel') THEN 'channel'
+        ELSE 'provider'
+      END
+    `);
+  }
+  if (!agentColNames.includes("provider_capabilities")) {
+    getDb().run("ALTER TABLE agents ADD COLUMN provider_capabilities TEXT");
+  }
+  if (!agentColNames.includes("context_state")) {
+    getDb().run("ALTER TABLE agents ADD COLUMN context_state TEXT");
+  }
+  if (!agentColNames.includes("spawned_by")) {
+    getDb().run("ALTER TABLE agents ADD COLUMN spawned_by TEXT");
+  }
+  getDb().run(`
+    CREATE TABLE IF NOT EXISTS context_snapshots (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      agent_id TEXT NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
+      utilization REAL NOT NULL,
+      lifecycle_state TEXT NOT NULL,
+      tokens_used INTEGER,
+      tokens_max INTEGER,
+      source TEXT NOT NULL,
+      confidence TEXT NOT NULL,
+      context_state TEXT NOT NULL,
+      captured_at INTEGER NOT NULL
+    )
+  `);
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_context_snapshots_agent_time ON context_snapshots(agent_id, captured_at DESC)");
+
+  getDb().run(`
+    CREATE TABLE IF NOT EXISTS memories (
+      id TEXT PRIMARY KEY,
+      type TEXT NOT NULL,
+      scope TEXT NOT NULL,
+      title TEXT NOT NULL,
+      content TEXT NOT NULL,
+      tags TEXT NOT NULL DEFAULT '[]',
+      visibility TEXT NOT NULL DEFAULT 'project',
+      sensitivity TEXT NOT NULL DEFAULT 'normal',
+      confidence TEXT NOT NULL DEFAULT 'reported',
+      redaction_state TEXT NOT NULL DEFAULT 'raw',
+      relevance_score REAL DEFAULT 1.0,
+      source_agent TEXT,
+      source_task INTEGER REFERENCES tasks(id) ON DELETE SET NULL,
+      created_by TEXT,
+      content_hash TEXT,
+      metadata TEXT NOT NULL DEFAULT '{}',
+      access_count INTEGER DEFAULT 0,
+      last_accessed_at INTEGER,
+      created_at INTEGER NOT NULL,
+      updated_at INTEGER NOT NULL,
+      expires_at INTEGER
+    )
+  `);
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_memories_type ON memories(type)");
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_memories_scope ON memories(scope)");
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_memories_tags ON memories(tags)");
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_memories_relevance ON memories(relevance_score)");
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_memories_updated ON memories(updated_at)");
+  getDb().run(`
+    CREATE TABLE IF NOT EXISTS agent_active_memories (
+      agent_id TEXT NOT NULL,
+      memory_id TEXT NOT NULL REFERENCES memories(id) ON DELETE CASCADE,
+      loaded_at INTEGER NOT NULL,
+      PRIMARY KEY (agent_id, memory_id)
+    )
+  `);
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_agent_active_memories_agent ON agent_active_memories(agent_id)");
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_agents_label ON agents(label)");
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_agents_kind ON agents(kind)");
+
+  getDb().run(`
+    CREATE TABLE IF NOT EXISTS integration_registry (
+      name TEXT PRIMARY KEY,
+      display_name TEXT,
+      description TEXT,
+      enabled INTEGER NOT NULL DEFAULT 1,
+      scopes TEXT NOT NULL DEFAULT '[]',
+      targets TEXT NOT NULL DEFAULT '[]',
+      channels TEXT NOT NULL DEFAULT '[]',
+      type TEXT,
+      icon TEXT,
+      accent_color TEXT,
+      tags TEXT NOT NULL DEFAULT '[]',
+      homepage_url TEXT,
+      repository_url TEXT,
+      docs_url TEXT,
+      manifest TEXT NOT NULL DEFAULT '{}',
+      source TEXT NOT NULL DEFAULT 'api',
+      created_at INTEGER NOT NULL,
+      updated_at INTEGER NOT NULL,
+      last_event_at INTEGER,
+      last_task_at INTEGER,
+      last_auth_success_at INTEGER,
+      last_auth_failure_at INTEGER
+    )
+  `);
+
+  // Built-in agents — registered unconditionally so sends from these ids
+  // pass the sendMessage validation. The reaper exempts these by checking
+  // meta.builtin (or by id for "user").
+  const now = Date.now();
+  const builtinStmt = getDb().query(`
+    INSERT INTO agents (id, name, kind, tags, machine, rig, capabilities, ready, status, meta, last_seen, created_at)
+    VALUES (?, ?, ?, ?, NULL, NULL, '[]', 1, 'online', '{"builtin":true}', ?, ?)
+    ON CONFLICT(id) DO UPDATE SET status = 'online', ready = 1, last_seen = excluded.last_seen
+  `);
+  builtinStmt.run("user", "User", "user", '["human"]', now, now);
+  builtinStmt.run("system", "System", "system", '["system"]', now, now);
+
+  // One-shot migration: backfill message_reads from legacy read_by JSON
+  // if that column still carries data. Safe to run repeatedly (INSERT OR IGNORE).
+  if (colNames.includes("read_by")) {
+    getDb().run(`
+      INSERT OR IGNORE INTO message_reads (message_id, agent_id, read_at)
+      SELECT m.id, je.value, m.created_at
+      FROM messages m, json_each(m.read_by) je
+      WHERE json_valid(m.read_by)
+    `);
+  }
+
+}