@desplega.ai/agent-swarm 1.20.0 → 1.49.0

package/src/be/migrations/006_vcs_provider.sql

@@ -0,0 +1,94 @@
+-- Add VCS provider abstraction: rename github* columns to vcs*, add vcsProvider,
+-- and update source CHECK constraint to include 'gitlab'.
+-- Uses the 12-step table recreation approach since SQLite doesn't support ALTER CHECK.
+
+PRAGMA defer_foreign_keys = ON;
+
+-- ═══════════════════════════════════════════════════════════════════
+-- agent_tasks: recreate with vcs* columns + updated CHECK constraint
+-- ═══════════════════════════════════════════════════════════════════
+
+CREATE TABLE agent_tasks_new (
+    id TEXT PRIMARY KEY,
+    agentId TEXT,
+    creatorAgentId TEXT,
+    task TEXT NOT NULL,
+    status TEXT NOT NULL DEFAULT 'pending',
+    source TEXT NOT NULL DEFAULT 'mcp' CHECK(source IN ('mcp', 'slack', 'api', 'github', 'gitlab', 'agentmail', 'system', 'schedule', 'workflow')),
+    taskType TEXT,
+    tags TEXT DEFAULT '[]',
+    priority INTEGER DEFAULT 50,
+    dependsOn TEXT DEFAULT '[]',
+    offeredTo TEXT,
+    offeredAt TEXT,
+    acceptedAt TEXT,
+    rejectionReason TEXT,
+    slackChannelId TEXT,
+    slackThreadTs TEXT,
+    slackUserId TEXT,
+    mentionMessageId TEXT,
+    mentionChannelId TEXT,
+    vcsProvider TEXT,
+    vcsRepo TEXT,
+    vcsEventType TEXT,
+    vcsNumber INTEGER,
+    vcsCommentId INTEGER,
+    vcsAuthor TEXT,
+    vcsUrl TEXT,
+    epicId TEXT REFERENCES epics(id) ON DELETE SET NULL,
+    parentTaskId TEXT,
+    claudeSessionId TEXT,
+    agentmailInboxId TEXT,
+    agentmailMessageId TEXT,
+    agentmailThreadId TEXT,
+    model TEXT,
+    scheduleId TEXT,
+    workflowRunId TEXT REFERENCES workflow_runs(id),
+    workflowRunStepId TEXT REFERENCES workflow_run_steps(id),
+    createdAt TEXT NOT NULL,
+    lastUpdatedAt TEXT NOT NULL,
+    finishedAt TEXT,
+    failureReason TEXT,
+    output TEXT,
+    progress TEXT,
+    notifiedAt TEXT
+);
+
+-- Copy data, renaming github* → vcs* and adding vcsProvider backfill
+INSERT INTO agent_tasks_new SELECT
+    id, agentId, creatorAgentId, task, status, source,
+    taskType, tags, priority, dependsOn, offeredTo, offeredAt,
+    acceptedAt, rejectionReason, slackChannelId, slackThreadTs, slackUserId,
+    mentionMessageId, mentionChannelId,
+    CASE WHEN githubRepo IS NOT NULL THEN 'github' ELSE NULL END,
+    githubRepo, githubEventType, githubNumber, githubCommentId, githubAuthor, githubUrl,
+    epicId, parentTaskId, claudeSessionId,
+    agentmailInboxId, agentmailMessageId, agentmailThreadId,
+    model, scheduleId, workflowRunId, workflowRunStepId,
+    createdAt, lastUpdatedAt, finishedAt, failureReason, output, progress, notifiedAt
+FROM agent_tasks;
+
+DROP TABLE agent_tasks;
+ALTER TABLE agent_tasks_new RENAME TO agent_tasks;
+
+-- Recreate indexes
+CREATE INDEX IF NOT EXISTS idx_agent_tasks_agentId ON agent_tasks(agentId);
+CREATE INDEX IF NOT EXISTS idx_agent_tasks_status ON agent_tasks(status);
+CREATE INDEX IF NOT EXISTS idx_agent_tasks_offeredTo ON agent_tasks(offeredTo);
+CREATE INDEX IF NOT EXISTS idx_agent_tasks_taskType ON agent_tasks(taskType);
+CREATE INDEX IF NOT EXISTS idx_agent_tasks_epicId ON agent_tasks(epicId);
+CREATE INDEX IF NOT EXISTS idx_agent_tasks_agentmailThreadId ON agent_tasks(agentmailThreadId);
+CREATE INDEX IF NOT EXISTS idx_agent_tasks_schedule_id ON agent_tasks(scheduleId);
+CREATE INDEX IF NOT EXISTS idx_agent_tasks_workflow_run ON agent_tasks(workflowRunId);
+
+-- ═══════════════════════════════════════════════════════════════════
+-- epics: rename github* → vcs* and add vcsProvider
+-- (No CHECK constraint to update, so ALTER TABLE is fine)
+-- ═══════════════════════════════════════════════════════════════════
+
+ALTER TABLE epics RENAME COLUMN githubRepo TO vcsRepo;
+ALTER TABLE epics RENAME COLUMN githubMilestone TO vcsMilestone;
+ALTER TABLE epics ADD COLUMN vcsProvider TEXT DEFAULT NULL;
+
+-- Backfill vcsProvider for epics
+UPDATE epics SET vcsProvider = 'github' WHERE vcsRepo IS NOT NULL;

package/src/be/migrations/007_task_dir.sql

@@ -0,0 +1,2 @@
+-- Add optional working directory field to tasks
+ALTER TABLE agent_tasks ADD COLUMN dir TEXT;

package/src/be/migrations/008_workflow_redesign.sql

@@ -0,0 +1,85 @@
+-- Workflow Engine Redesign: drop and recreate workflow tables with new schema.
+-- This is a clean-slate migration — no existing workflow data is preserved.
+
+-- Disable FK enforcement so we can drop referenced tables
+PRAGMA foreign_keys = OFF;
+
+-- NULL out FK columns on agent_tasks that reference workflow tables
+UPDATE agent_tasks SET workflowRunId = NULL, workflowRunStepId = NULL
+  WHERE workflowRunId IS NOT NULL OR workflowRunStepId IS NOT NULL;
+
+-- Drop old tables in FK-safe order
+DROP TABLE IF EXISTS workflow_run_steps;
+DROP TABLE IF EXISTS workflow_runs;
+DROP TABLE IF EXISTS workflows;
+
+-- Recreate workflows with new columns (triggers, cooldown, input; no webhookSecret)
+CREATE TABLE IF NOT EXISTS workflows (
+  id TEXT PRIMARY KEY,
+  name TEXT NOT NULL UNIQUE,
+  description TEXT,
+  enabled INTEGER NOT NULL DEFAULT 1,
+  definition TEXT NOT NULL,
+  triggers TEXT NOT NULL DEFAULT '[]',
+  cooldown TEXT,
+  input TEXT,
+  createdByAgentId TEXT,
+  createdAt TEXT NOT NULL DEFAULT (datetime('now')),
+  lastUpdatedAt TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+-- Recreate workflow_runs with 'skipped' status
+CREATE TABLE IF NOT EXISTS workflow_runs (
+  id TEXT PRIMARY KEY,
+  workflowId TEXT NOT NULL REFERENCES workflows(id),
+  status TEXT NOT NULL DEFAULT 'running'
+    CHECK(status IN ('running', 'waiting', 'completed', 'failed', 'skipped')),
+  triggerData TEXT,
+  context TEXT,
+  error TEXT,
+  startedAt TEXT NOT NULL DEFAULT (datetime('now')),
+  lastUpdatedAt TEXT NOT NULL DEFAULT (datetime('now')),
+  finishedAt TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_workflow_runs_workflowId ON workflow_runs(workflowId);
+CREATE INDEX IF NOT EXISTS idx_workflow_runs_status ON workflow_runs(status);
+
+-- Recreate workflow_run_steps with retry columns
+CREATE TABLE IF NOT EXISTS workflow_run_steps (
+  id TEXT PRIMARY KEY,
+  runId TEXT NOT NULL REFERENCES workflow_runs(id),
+  nodeId TEXT NOT NULL,
+  nodeType TEXT NOT NULL,
+  status TEXT NOT NULL DEFAULT 'pending'
+    CHECK(status IN ('pending', 'running', 'waiting', 'completed', 'failed', 'skipped')),
+  input TEXT,
+  output TEXT,
+  error TEXT,
+  startedAt TEXT NOT NULL DEFAULT (datetime('now')),
+  finishedAt TEXT,
+  retryCount INTEGER NOT NULL DEFAULT 0,
+  maxRetries INTEGER NOT NULL DEFAULT 3,
+  nextRetryAt TEXT,
+  idempotencyKey TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_workflow_run_steps_runId ON workflow_run_steps(runId);
+CREATE INDEX IF NOT EXISTS idx_wrs_retry
+  ON workflow_run_steps(status, nextRetryAt)
+  WHERE status = 'failed' AND nextRetryAt IS NOT NULL;
+CREATE INDEX IF NOT EXISTS idx_wrs_idempotency ON workflow_run_steps(idempotencyKey);
+
+-- New: workflow version history
+CREATE TABLE IF NOT EXISTS workflow_versions (
+  id TEXT PRIMARY KEY,
+  workflowId TEXT NOT NULL REFERENCES workflows(id) ON DELETE CASCADE,
+  version INTEGER NOT NULL,
+  snapshot TEXT NOT NULL,
+  changedByAgentId TEXT,
+  createdAt TEXT NOT NULL DEFAULT (datetime('now')),
+  UNIQUE(workflowId, version)
+);
+
+-- Re-enable FK enforcement
+PRAGMA foreign_keys = ON;

package/src/be/migrations/009_tracker_integration.sql

@@ -0,0 +1,144 @@
+-- Tracker integration: generic OAuth, entity sync mapping, agent mapping tables.
+-- Also adds 'linear' to the agent_tasks source CHECK constraint.
+
+PRAGMA defer_foreign_keys = ON;
+
+-- ═══════════════════════════════════════════════════════════════════
+-- Generic OAuth tables (reusable across any OAuth provider in the swarm)
+-- ═══════════════════════════════════════════════════════════════════
+
+-- Generic OAuth application configuration (one row per provider)
+CREATE TABLE IF NOT EXISTS oauth_apps (
+    id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))),
+    provider TEXT NOT NULL UNIQUE,
+    clientId TEXT NOT NULL,
+    clientSecret TEXT NOT NULL,
+    authorizeUrl TEXT NOT NULL,
+    tokenUrl TEXT NOT NULL,
+    redirectUri TEXT NOT NULL,
+    scopes TEXT NOT NULL,
+    metadata TEXT DEFAULT '{}',
+    createdAt TEXT NOT NULL DEFAULT (datetime('now')),
+    updatedAt TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+-- Generic OAuth token storage (one active token set per provider)
+CREATE TABLE IF NOT EXISTS oauth_tokens (
+    id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))),
+    provider TEXT NOT NULL UNIQUE,
+    accessToken TEXT NOT NULL,
+    refreshToken TEXT,
+    expiresAt TEXT NOT NULL,
+    scope TEXT,
+    createdAt TEXT NOT NULL DEFAULT (datetime('now')),
+    updatedAt TEXT NOT NULL DEFAULT (datetime('now')),
+    FOREIGN KEY (provider) REFERENCES oauth_apps(provider)
+);
+
+-- ═══════════════════════════════════════════════════════════════════
+-- Generic tracker tables (reusable across any ticket tracker)
+-- ═══════════════════════════════════════════════════════════════════
+
+-- Generic tracker entity mapping
+CREATE TABLE IF NOT EXISTS tracker_sync (
+    id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))),
+    provider TEXT NOT NULL,
+    entityType TEXT NOT NULL CHECK (entityType IN ('task', 'epic')),
+    providerEntityType TEXT,
+    swarmId TEXT NOT NULL,
+    externalId TEXT NOT NULL,
+    externalIdentifier TEXT,
+    externalUrl TEXT,
+    lastSyncedAt TEXT NOT NULL DEFAULT (datetime('now')),
+    lastSyncOrigin TEXT CHECK (lastSyncOrigin IN ('swarm', 'external')),
+    lastDeliveryId TEXT,
+    syncDirection TEXT NOT NULL DEFAULT 'inbound' CHECK (syncDirection IN ('inbound', 'outbound', 'bidirectional')),
+    createdAt TEXT NOT NULL DEFAULT (datetime('now')),
+    UNIQUE(provider, entityType, swarmId),
+    UNIQUE(provider, entityType, externalId)
+);
+
+-- Generic agent-to-external-user mapping
+CREATE TABLE IF NOT EXISTS tracker_agent_mapping (
+    id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))),
+    provider TEXT NOT NULL,
+    agentId TEXT NOT NULL,
+    externalUserId TEXT NOT NULL,
+    agentName TEXT NOT NULL,
+    createdAt TEXT NOT NULL DEFAULT (datetime('now')),
+    UNIQUE(provider, agentId),
+    UNIQUE(provider, externalUserId)
+);
+
+-- Indexes
+CREATE INDEX IF NOT EXISTS idx_oauth_tokens_provider ON oauth_tokens(provider);
+CREATE INDEX IF NOT EXISTS idx_tracker_sync_swarm ON tracker_sync(provider, entityType, swarmId);
+CREATE INDEX IF NOT EXISTS idx_tracker_sync_external ON tracker_sync(provider, entityType, externalId);
+CREATE INDEX IF NOT EXISTS idx_tracker_agent_agentId ON tracker_agent_mapping(provider, agentId);
+
+-- ═══════════════════════════════════════════════════════════════════
+-- agent_tasks: recreate with 'linear' added to source CHECK constraint
+-- (copied from previous 008_linear_integration.sql lines 54-114)
+-- ═══════════════════════════════════════════════════════════════════
+
+CREATE TABLE agent_tasks_new (
+    id TEXT PRIMARY KEY,
+    agentId TEXT,
+    creatorAgentId TEXT,
+    task TEXT NOT NULL,
+    status TEXT NOT NULL DEFAULT 'pending',
+    source TEXT NOT NULL DEFAULT 'mcp' CHECK(source IN ('mcp', 'slack', 'api', 'github', 'gitlab', 'agentmail', 'system', 'schedule', 'workflow', 'linear')),
+    taskType TEXT,
+    tags TEXT DEFAULT '[]',
+    priority INTEGER DEFAULT 50,
+    dependsOn TEXT DEFAULT '[]',
+    offeredTo TEXT,
+    offeredAt TEXT,
+    acceptedAt TEXT,
+    rejectionReason TEXT,
+    slackChannelId TEXT,
+    slackThreadTs TEXT,
+    slackUserId TEXT,
+    mentionMessageId TEXT,
+    mentionChannelId TEXT,
+    vcsProvider TEXT,
+    vcsRepo TEXT,
+    vcsEventType TEXT,
+    vcsNumber INTEGER,
+    vcsCommentId INTEGER,
+    vcsAuthor TEXT,
+    vcsUrl TEXT,
+    epicId TEXT REFERENCES epics(id) ON DELETE SET NULL,
+    parentTaskId TEXT,
+    claudeSessionId TEXT,
+    agentmailInboxId TEXT,
+    agentmailMessageId TEXT,
+    agentmailThreadId TEXT,
+    model TEXT,
+    scheduleId TEXT,
+    workflowRunId TEXT REFERENCES workflow_runs(id),
+    workflowRunStepId TEXT REFERENCES workflow_run_steps(id),
+    createdAt TEXT NOT NULL,
+    lastUpdatedAt TEXT NOT NULL,
+    finishedAt TEXT,
+    failureReason TEXT,
+    output TEXT,
+    progress TEXT,
+    notifiedAt TEXT,
+    dir TEXT
+);
+
+INSERT INTO agent_tasks_new SELECT * FROM agent_tasks;
+
+DROP TABLE agent_tasks;
+ALTER TABLE agent_tasks_new RENAME TO agent_tasks;
+
+-- Recreate indexes
+CREATE INDEX IF NOT EXISTS idx_agent_tasks_agentId ON agent_tasks(agentId);
+CREATE INDEX IF NOT EXISTS idx_agent_tasks_status ON agent_tasks(status);
+CREATE INDEX IF NOT EXISTS idx_agent_tasks_offeredTo ON agent_tasks(offeredTo);
+CREATE INDEX IF NOT EXISTS idx_agent_tasks_taskType ON agent_tasks(taskType);
+CREATE INDEX IF NOT EXISTS idx_agent_tasks_epicId ON agent_tasks(epicId);
+CREATE INDEX IF NOT EXISTS idx_agent_tasks_agentmailThreadId ON agent_tasks(agentmailThreadId);
+CREATE INDEX IF NOT EXISTS idx_agent_tasks_schedule_id ON agent_tasks(scheduleId);
+CREATE INDEX IF NOT EXISTS idx_agent_tasks_workflow_run ON agent_tasks(workflowRunId);

package/src/be/migrations/010_step_diagnostics.sql

@@ -0,0 +1 @@
+ALTER TABLE workflow_run_steps ADD COLUMN diagnostics TEXT;

package/src/be/migrations/011_step_next_port.sql

@@ -0,0 +1 @@
+ALTER TABLE workflow_run_steps ADD COLUMN nextPort TEXT;

package/src/be/migrations/012_trigger_schema.sql

@@ -0,0 +1 @@
+ALTER TABLE workflows ADD COLUMN triggerSchema TEXT;

package/src/be/migrations/013_task_output_schema.sql

@@ -0,0 +1,2 @@
+-- Add outputSchema column to agent_tasks for structured output validation
+ALTER TABLE agent_tasks ADD COLUMN outputSchema TEXT;

package/src/be/migrations/runner.ts

@@ -0,0 +1,188 @@
+import type { Database } from "bun:sqlite";
+import { createHash } from "node:crypto";
+import { readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+
+interface Migration {
+  version: number;
+  name: string;
+  sql: string;
+  checksum: string;
+}
+
+interface AppliedMigration {
+  version: number;
+  name: string;
+  checksum: string;
+}
+
+const BASELINE_TABLES = [
+  "agents",
+  "channels",
+  "epics",
+  "agent_tasks",
+  "agent_log",
+  "channel_messages",
+  "channel_read_state",
+  "services",
+  "session_logs",
+  "session_costs",
+  "inbox_messages",
+  "scheduled_tasks",
+  "swarm_config",
+  "swarm_repos",
+  "agent_memory",
+  "active_sessions",
+  "agentmail_inbox_mappings",
+  "context_versions",
+];
+
+function shouldBootstrapInitialMigration(db: Database): boolean {
+  const rows = db
+    .prepare<{ name: string }, []>(
+      "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE '\\_migrations' ESCAPE '\\'",
+    )
+    .all();
+
+  if (rows.length === 0) {
+    return false;
+  }
+
+  const existingTables = new Set(rows.map((row) => row.name));
+  return BASELINE_TABLES.every((table) => existingTables.has(table));
+}
+
+/**
+ * Runs all pending database migrations.
+ *
+ * - Creates the `_migrations` tracking table if it doesn't exist
+ * - Reads `.sql` files from the migrations directory (sorted by numeric prefix)
+ * - For existing databases (pre-migration-system), bootstraps by marking 001_initial as applied
+ * - Applies pending migrations in order, each within its own transaction
+ * - Verifies checksums of previously-applied migrations (warns on mismatch)
+ */
+export function runMigrations(db: Database): void {
+  // 1. Ensure tracking table exists
+  db.run(`
+    CREATE TABLE IF NOT EXISTS _migrations (
+      version INTEGER PRIMARY KEY,
+      name TEXT NOT NULL,
+      applied_at TEXT NOT NULL,
+      checksum TEXT NOT NULL
+    )
+  `);
+
+  // 2. Load migration files
+  // import.meta.dir resolves to /$bunfs/root in compiled binaries, which isn't scannable.
+  // Fall back to MIGRATIONS_DIR env var (set in Dockerfile) for compiled binary support.
+  const migrationsDir = (() => {
+    const dir = import.meta.dir;
+    try {
+      readdirSync(dir);
+      return dir;
+    } catch {
+      const fallback = process.env.MIGRATIONS_DIR;
+      if (fallback) return fallback;
+      console.warn(
+        "[migrations] Cannot read migrations directory and MIGRATIONS_DIR not set — skipping",
+      );
+      return null;
+    }
+  })();
+
+  if (!migrationsDir) return;
+
+  const files = readdirSync(migrationsDir)
+    .filter((f) => f.endsWith(".sql"))
+    .sort();
+
+  const migrations: Migration[] = files.map((file) => {
+    const version = parseInt(file.split("_")[0] ?? "0", 10);
+    const name = file.replace(".sql", "");
+    const sql = readFileSync(join(migrationsDir, file), "utf-8");
+    const checksum = createHash("sha256").update(sql).digest("hex");
+    return { version, name, sql, checksum };
+  });
+
+  if (migrations.length === 0) {
+    return;
+  }
+
+  // 3. Get applied migrations
+  const applied = new Map<number, AppliedMigration>();
+  const rows = db
+    .prepare("SELECT version, name, checksum FROM _migrations")
+    .all() as AppliedMigration[];
+  for (const row of rows) {
+    applied.set(row.version, {
+      version: row.version,
+      name: row.name,
+      checksum: row.checksum,
+    });
+  }
+
+  // 4. Bootstrap existing databases
+  // If no migrations have been applied yet and database tables already exist,
+  // this is a pre-migration-system database. Mark 001_initial as applied without
+  // executing it (the schema already exists).
+  if (applied.size === 0) {
+    const initialMigration = migrations.find((m) => m.version === 1);
+    if (initialMigration) {
+      if (shouldBootstrapInitialMigration(db)) {
+        console.log("[migrations] Existing database detected — bootstrapping migration tracking");
+        db.run(
+          "INSERT INTO _migrations (version, name, applied_at, checksum) VALUES (?, ?, ?, ?)",
+          [
+            initialMigration.version,
+            initialMigration.name,
+            new Date().toISOString(),
+            initialMigration.checksum,
+          ],
+        );
+        applied.set(initialMigration.version, {
+          version: initialMigration.version,
+          name: initialMigration.name,
+          checksum: initialMigration.checksum,
+        });
+      } else {
+        console.log(
+          "[migrations] Existing database appears incomplete — applying 001_initial migration",
+        );
+      }
+    }
+  }
+
+  // 5. Run pending migrations
+  for (const migration of migrations) {
+    const existing = applied.get(migration.version);
+
+    if (existing) {
+      // Verify checksum hasn't changed
+      if (existing.checksum !== migration.checksum) {
+        console.warn(
+          `[migrations] WARNING: Migration ${migration.name} checksum mismatch. ` +
+            `Applied: ${existing.checksum.slice(0, 12)}..., Current: ${migration.checksum.slice(0, 12)}... ` +
+            `Do not modify applied migrations — create a new one instead.`,
+        );
+      }
+      continue;
+    }
+
+    // Apply migration in a transaction
+    console.log(`[migrations] Applying: ${migration.name}`);
+    const start = performance.now();
+
+    db.transaction(() => {
+      db.exec(migration.sql);
+      db.run("INSERT INTO _migrations (version, name, applied_at, checksum) VALUES (?, ?, ?, ?)", [
+        migration.version,
+        migration.name,
+        new Date().toISOString(),
+        migration.checksum,
+      ]);
+    })();
+
+    const elapsed = (performance.now() - start).toFixed(1);
+    console.log(`[migrations] Applied: ${migration.name} (${elapsed}ms)`);
+  }
+}