@desplega.ai/agent-swarm 1.20.0 → 1.51.2

package/src/http/mcp.ts

@@ -0,0 +1,77 @@
+import { randomUUID } from "node:crypto";
+import type { IncomingMessage, ServerResponse } from "node:http";
+import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
+import { isInitializeRequest } from "@modelcontextprotocol/sdk/types.js";
+import { createServer } from "@/server";
+
+export async function handleMcp(
+  req: IncomingMessage,
+  res: ServerResponse,
+  transports: Record<string, StreamableHTTPServerTransport>,
+): Promise<boolean> {
+  const sessionId = req.headers["mcp-session-id"] as string | undefined;
+
+  if (req.url !== "/mcp") {
+    return false;
+  }
+
+  if (req.method === "POST") {
+    const chunks: Buffer[] = [];
+    for await (const chunk of req) {
+      chunks.push(chunk);
+    }
+    const body = JSON.parse(Buffer.concat(chunks).toString());
+
+    let transport: StreamableHTTPServerTransport;
+
+    if (sessionId && transports[sessionId]) {
+      transport = transports[sessionId];
+    } else if (!sessionId && isInitializeRequest(body)) {
+      transport = new StreamableHTTPServerTransport({
+        sessionIdGenerator: () => randomUUID(),
+        onsessioninitialized: (id) => {
+          transports[id] = transport;
+        },
+        onsessionclosed: (id) => {
+          delete transports[id];
+        },
+      });
+
+      transport.onclose = () => {
+        if (transport.sessionId) {
+          delete transports[transport.sessionId];
+        }
+      };
+
+      const server = createServer();
+      await server.connect(transport);
+    } else {
+      res.writeHead(400, { "Content-Type": "application/json" });
+      res.end(
+        JSON.stringify({
+          jsonrpc: "2.0",
+          error: { code: -32000, message: "Invalid session" },
+          id: null,
+        }),
+      );
+      return true;
+    }
+
+    await transport.handleRequest(req, res, body);
+    return true;
+  }
+
+  if (req.method === "GET" || req.method === "DELETE") {
+    if (sessionId && transports[sessionId]) {
+      await transports[sessionId].handleRequest(req, res);
+      return true;
+    }
+    res.writeHead(400);
+    res.end("Invalid session");
+    return true;
+  }
+
+  res.writeHead(405);
+  res.end("Method not allowed");
+  return true;
+}

package/src/http/memory.ts

@@ -0,0 +1,168 @@
+import type { IncomingMessage, ServerResponse } from "node:http";
+import { z } from "zod";
+import { chunkContent } from "../be/chunking";
+import {
+  createMemory,
+  deleteMemoriesBySourcePath,
+  getDb,
+  searchMemoriesByVector,
+  updateMemoryEmbedding,
+} from "../be/db";
+import { getEmbedding, serializeEmbedding } from "../be/embedding";
+import { AgentMemoryScopeSchema, AgentMemorySourceSchema } from "../types";
+import { route } from "./route-def";
+import { json, jsonError } from "./utils";
+
+// ─── Route Definitions ───────────────────────────────────────────────────────
+
+const indexMemory = route({
+  method: "post",
+  path: "/api/memory/index",
+  pattern: ["api", "memory", "index"],
+  summary: "Ingest content into memory system (async embedding)",
+  tags: ["Memory"],
+  body: z.object({
+    agentId: z.string().uuid().optional(),
+    content: z.string().min(1),
+    name: z.string().min(1),
+    scope: AgentMemoryScopeSchema,
+    source: AgentMemorySourceSchema,
+    sourceTaskId: z.string().uuid().optional(),
+    sourcePath: z.string().optional(),
+    tags: z.array(z.string()).optional(),
+  }),
+  responses: {
+    202: { description: "Content queued for embedding" },
+    400: { description: "Validation error" },
+  },
+});
+
+const searchMemory = route({
+  method: "post",
+  path: "/api/memory/search",
+  pattern: ["api", "memory", "search"],
+  summary: "Search memories by natural language query",
+  tags: ["Memory"],
+  auth: { apiKey: true, agentId: true },
+  body: z.object({
+    query: z.string().min(1),
+    limit: z.number().int().min(1).max(20).default(5),
+  }),
+  responses: {
+    200: { description: "Search results" },
+    400: { description: "Missing query or agent ID" },
+  },
+});
+
+// ─── Handler ─────────────────────────────────────────────────────────────────
+
+export async function handleMemory(
+  req: IncomingMessage,
+  res: ServerResponse,
+  pathSegments: string[],
+  myAgentId: string | undefined,
+): Promise<boolean> {
+  if (indexMemory.match(req.method, pathSegments)) {
+    const parsed = await indexMemory.parse(req, res, pathSegments, new URLSearchParams());
+    if (!parsed) return true;
+
+    const { agentId, content, name, scope, source, sourceTaskId, sourcePath, tags } = parsed.body;
+
+    // Chunk content and create memories in a transaction (with dedup)
+    const contentChunks = chunkContent(content);
+    if (contentChunks.length === 0) {
+      // Content too small to chunk — create a single memory
+      contentChunks.push({
+        content: content.trim(),
+        chunkIndex: 0,
+        totalChunks: 1,
+        headings: [],
+      });
+    }
+
+    const memoryIds = getDb().transaction(() => {
+      // Delete old chunks if re-indexing same file
+      if (sourcePath && agentId) {
+        deleteMemoriesBySourcePath(sourcePath, agentId);
+      }
+
+      const ids: string[] = [];
+      for (const chunk of contentChunks) {
+        const memory = createMemory({
+          agentId: agentId || null,
+          content: chunk.content,
+          name,
+          scope,
+          source,
+          sourcePath: sourcePath || null,
+          sourceTaskId: sourceTaskId || null,
+          chunkIndex: chunk.chunkIndex,
+          totalChunks: chunk.totalChunks,
+          tags: tags || [],
+        });
+        ids.push(memory.id);
+      }
+      return ids;
+    })();
+
+    // Async embedding — fire and forget
+    (async () => {
+      for (let i = 0; i < contentChunks.length; i++) {
+        try {
+          const embedding = await getEmbedding(contentChunks[i]!.content);
+          if (embedding) {
+            updateMemoryEmbedding(memoryIds[i]!, serializeEmbedding(embedding));
+          }
+        } catch (err) {
+          console.error(`[memory] Failed to embed chunk ${memoryIds[i]}:`, (err as Error).message);
+        }
+      }
+    })();
+
+    json(res, { queued: true, memoryIds }, 202);
+    return true;
+  }
+
+  if (searchMemory.match(req.method, pathSegments)) {
+    if (!myAgentId) {
+      jsonError(res, "Missing required fields: query, X-Agent-ID header", 400);
+      return true;
+    }
+
+    const parsed = await searchMemory.parse(req, res, pathSegments, new URLSearchParams());
+    if (!parsed) return true;
+
+    const { query, limit } = parsed.body;
+
+    try {
+      const queryEmbedding = await getEmbedding(query);
+      if (!queryEmbedding) {
+        json(res, { results: [] });
+        return true;
+      }
+
+      const results = searchMemoriesByVector(queryEmbedding, myAgentId, {
+        scope: "all",
+        limit: Math.min(limit, 20),
+        isLead: false,
+      });
+
+      json(res, {
+        results: results.map((r) => ({
+          id: r.id,
+          name: r.name,
+          content: r.content,
+          similarity: r.similarity,
+          source: r.source,
+          scope: r.scope,
+        })),
+      });
+    } catch (err) {
+      console.error("[memory-search] Error:", (err as Error).message);
+      json(res, { results: [] });
+    }
+    return true;
+  }
+
+  return false;
+}

package/src/http/openapi.ts

@@ -0,0 +1,109 @@
+import {
+  extendZodWithOpenApi,
+  OpenAPIRegistry,
+  OpenApiGeneratorV31,
+} from "@asteasolutions/zod-to-openapi";
+import { z } from "zod";
+import { routeRegistry } from "./route-def";
+
+extendZodWithOpenApi(z);
+
+let cachedSpec: string | null = null;
+
+interface OpenApiOptions {
+  version: string;
+  serverUrl?: string;
+}
+
+export function generateOpenApiSpec(opts: OpenApiOptions): string {
+  if (cachedSpec) return cachedSpec;
+
+  const registry = new OpenAPIRegistry();
+
+  // Register Bearer auth
+  registry.registerComponent("securitySchemes", "bearerAuth", {
+    type: "http",
+    scheme: "bearer",
+    description: "API key via Authorization: Bearer <API_KEY>",
+  });
+
+  // Register X-Agent-ID header
+  registry.registerComponent("securitySchemes", "agentId", {
+    type: "apiKey",
+    in: "header",
+    name: "X-Agent-ID",
+    description: "Agent UUID for agent-scoped operations",
+  });
+
+  // Convert route definitions to OpenAPI paths
+  for (const routeDef of routeRegistry) {
+    const responses: Record<
+      string,
+      { description: string; content?: Record<string, { schema: z.ZodType }> }
+    > = {};
+    for (const [code, resDef] of Object.entries(routeDef.responses)) {
+      responses[code] = {
+        description: resDef.description,
+        ...(resDef.schema && {
+          content: {
+            "application/json": { schema: resDef.schema },
+          },
+        }),
+      };
+    }
+
+    const request: Record<string, unknown> = {};
+    if (routeDef.params) request.params = routeDef.params;
+    if (routeDef.query) request.query = routeDef.query;
+    if (routeDef.body) {
+      request.body = {
+        content: { "application/json": { schema: routeDef.body } },
+      };
+    }
+
+    registry.registerPath({
+      method: routeDef.method,
+      path: routeDef.path,
+      summary: routeDef.summary,
+      description: routeDef.description,
+      tags: routeDef.tags,
+      request,
+      responses,
+      security: routeDef.auth?.apiKey !== false ? [{ bearerAuth: [] }] : undefined,
+    });
+  }
+
+  const serverUrl =
+    opts.serverUrl || process.env.MCP_BASE_URL || `http://localhost:${process.env.PORT || "3013"}`;
+
+  const generator = new OpenApiGeneratorV31(registry.definitions);
+  const doc = generator.generateDocument({
+    openapi: "3.1.0",
+    info: {
+      title: "Agent Swarm API",
+      version: opts.version,
+      description:
+        "Multi-agent orchestration API for Claude Code, Codex, and Gemini CLI. " +
+        "Enables task distribution, agent communication, and service discovery.\n\n" +
+        "MCP tools are documented separately in [MCP.md](./MCP.md).",
+    },
+    servers: [
+      {
+        url: serverUrl,
+        description: serverUrl.includes("localhost") ? "Local development" : "Production",
+      },
+    ],
+  });
+
+  cachedSpec = JSON.stringify(doc, null, 2);
+  return cachedSpec;
+}
+
+export const SCALAR_HTML = `<!DOCTYPE html>
+<html>
+<head><title>Agent Swarm API</title><meta charset="utf-8" /></head>
+<body>
+  <script id="api-reference" data-url="/openapi.json"></script>
+  <script src="https://cdn.jsdelivr.net/npm/@scalar/api-reference"></script>
+</body>
+</html>`;

package/src/http/poll.ts

@@ -0,0 +1,299 @@
+import type { IncomingMessage, ServerResponse } from "node:http";
+import { z } from "zod";
+import {
+  claimMentions,
+  claimOfferedTask,
+  claimTask,
+  getAgentById,
+  getAllChannelActivityCursors,
+  getDb,
+  getEpicsWithProgressUpdates,
+  getInboxSummary,
+  getOfferedTasksForAgent,
+  getPendingTaskForAgent,
+  getUnassignedTaskIds,
+  hasCapacity,
+  markEpicsProgressNotified,
+  startTask,
+  upsertChannelActivityCursor,
+} from "../be/db";
+import { fetchChannelActivity } from "../slack/channel-activity";
+import { route } from "./route-def";
+import { json, jsonError } from "./utils";
+
+// ─── Route Definitions ───────────────────────────────────────────────────────
+
+const pollTriggers = route({
+  method: "get",
+  path: "/api/poll",
+  pattern: ["api", "poll"],
+  summary: "Poll for triggers (tasks, mentions, epic updates)",
+  tags: ["Poll"],
+  auth: { apiKey: true, agentId: true },
+  responses: {
+    200: { description: "Trigger data or null" },
+    400: { description: "Missing X-Agent-ID" },
+    404: { description: "Agent not found" },
+  },
+});
+
+// ─── Channel Activity Throttle ──────────────────────────────────────────────
+
+const CHANNEL_ACTIVITY_INTERVAL_MS = 60_000; // Check at most once per 60s
+let lastChannelActivityCheckAt = 0;
+
+// ─── Cursor Commit Endpoint ─────────────────────────────────────────────────
+
+const commitCursorsRoute = route({
+  method: "post",
+  path: "/api/channel-activity/commit-cursors",
+  pattern: ["api", "channel-activity", "commit-cursors"],
+  summary: "Commit channel activity cursors after successful processing",
+  tags: ["Poll"],
+  auth: { apiKey: true },
+  body: z.object({
+    cursorUpdates: z.array(
+      z.object({
+        channelId: z.string(),
+        ts: z.string(),
+      }),
+    ),
+  }),
+  responses: {
+    200: { description: "Cursors committed" },
+    400: { description: "Invalid request" },
+  },
+});
+
+// ─── Handler ─────────────────────────────────────────────────────────────────
+
+export async function handlePoll(
+  req: IncomingMessage,
+  res: ServerResponse,
+  pathSegments: string[],
+  queryParams: URLSearchParams,
+  myAgentId: string | undefined,
+): Promise<boolean> {
+  // Handle cursor commit endpoint
+  if (commitCursorsRoute.match(req.method, pathSegments)) {
+    const parsed = await commitCursorsRoute.parse(req, res, pathSegments, queryParams);
+    if (!parsed) return true;
+    for (const { channelId, ts } of parsed.body.cursorUpdates) {
+      if (channelId && ts) {
+        upsertChannelActivityCursor(channelId, ts);
+      }
+    }
+    json(res, { success: true, committed: parsed.body.cursorUpdates.length });
+    return true;
+  }
+
+  if (pollTriggers.match(req.method, pathSegments)) {
+    if (!myAgentId) {
+      jsonError(res, "Missing X-Agent-ID header", 400);
+      return true;
+    }
+
+    // Use transaction for consistent reads across all trigger checks
+    let result:
+      | { error: string; status: number }
+      | { trigger: { type: string; [key: string]: unknown } | null };
+    try {
+      result = getDb().transaction(() => {
+        const agent = getAgentById(myAgentId);
+        if (!agent) {
+          return { error: "Agent not found", status: 404 };
+        }
+
+        // Check for offered tasks first (highest priority for both workers and leads)
+        // Atomically claim the task for review to prevent duplicate processing
+        const offeredTasks = getOfferedTasksForAgent(myAgentId);
+        const firstOfferedTask = offeredTasks[0];
+        if (firstOfferedTask) {
+          const claimedTask = claimOfferedTask(firstOfferedTask.id, myAgentId);
+          if (claimedTask) {
+            return {
+              trigger: {
+                type: "task_offered",
+                taskId: claimedTask.id,
+                task: claimedTask,
+              },
+            };
+          }
+        }
+
+        // Check for pending tasks (assigned directly to this agent)
+        // Only return a task if agent has capacity (server-side enforcement)
+        if (hasCapacity(myAgentId)) {
+          const pendingTask = getPendingTaskForAgent(myAgentId);
+          if (pendingTask) {
+            // Mark task as in_progress immediately to prevent duplicate polling
+            startTask(pendingTask.id);
+            return {
+              trigger: {
+                type: "task_assigned",
+                taskId: pendingTask.id,
+                task: { ...pendingTask, status: "in_progress" },
+              },
+            };
+          }
+        }
+
+        // Check for unread mentions (internal chat) - all agents can be woken by @mentions
+        // Uses atomic claiming via processing_since to prevent duplicate processing.
+        // Only idle agents poll, so busy workers won't be interrupted.
+        const claimedChannels = claimMentions(myAgentId);
+        if (claimedChannels.length > 0) {
+          // Recalculate inbox summary now that we've claimed
+          const inbox = getInboxSummary(myAgentId);
+          return {
+            trigger: {
+              type: "unread_mentions",
+              mentionsCount: inbox.mentionsCount,
+              claimedChannels: claimedChannels.map((c) => c.channelId), // Include for tracking
+            },
+          };
+        }
+
+        if (agent.isLead) {
+          // === LEAD-SPECIFIC TRIGGERS ===
+
+          // NOTE: tasks_finished trigger has been replaced by follow-up task creation
+          // in store-progress. When a worker completes/fails a task, a follow-up task
+          // is created and assigned to the lead, which is picked up via the normal
+          // task_assigned trigger above. This is more reliable and visible than the
+          // old poll-based notification approach.
+
+          // Check for epic progress updates (tasks completed/failed for active epics)
+          // This trigger helps lead plan next steps for epics - similar to ralph loop
+          const epicsWithUpdates = getEpicsWithProgressUpdates();
+          if (epicsWithUpdates.length > 0) {
+            // Atomically mark as notified within this transaction
+            const epicIds = epicsWithUpdates.map((e) => e.epic.id);
+            markEpicsProgressNotified(epicIds);
+
+            return {
+              trigger: {
+                type: "epic_progress_changed",
+                count: epicsWithUpdates.length,
+                epics: epicsWithUpdates,
+              },
+            };
+          }
+        } else {
+          // === WORKER-SPECIFIC TRIGGERS ===
+
+          // Auto-claim: atomically claim an unassigned task for this worker.
+          // claimTask() uses an atomic UPDATE WHERE status='unassigned', so only
+          // one worker wins if multiple poll simultaneously.
+          // This ensures session logs are correctly associated with the real task ID
+          // from the start (no reassociation needed).
+          if (hasCapacity(myAgentId)) {
+            const unassignedIds = getUnassignedTaskIds(5);
+            for (const candidateId of unassignedIds) {
+              const claimed = claimTask(candidateId, myAgentId);
+              if (claimed) {
+                return {
+                  trigger: {
+                    type: "task_assigned",
+                    taskId: claimed.id,
+                    task: claimed,
+                  },
+                };
+              }
+              // Claim failed (another worker got it) — try next
+            }
+          }
+        }
+
+        // No trigger found
+        return { trigger: null };
+      })();
+    } catch (error) {
+      console.error("[/api/poll] Database error:", error);
+      jsonError(
+        res,
+        `Database error occurred while polling for triggers: ${error instanceof Error ? error.message : String(error)}`,
+        500,
+      );
+      return true;
+    }
+
+    // Handle error case
+    if ("error" in result) {
+      jsonError(res, result.error, result.status ?? 500);
+      return true;
+    }
+
+    // If no trigger found and agent is lead, check for Slack channel activity.
+    // This is the lowest-priority trigger, checked AFTER all others.
+    // Runs outside the transaction because it requires async Slack API calls.
+    // Throttled to avoid Slack API rate limits (~50 calls/min).
+    if (
+      result.trigger === null &&
+      process.env.LEAD_MONITOR_CHANNELS === "true" &&
+      Date.now() - lastChannelActivityCheckAt >= CHANNEL_ACTIVITY_INTERVAL_MS
+    ) {
+      const agent = getAgentById(myAgentId);
+      if (agent?.isLead) {
+        lastChannelActivityCheckAt = Date.now();
+        try {
+          const cursors = getAllChannelActivityCursors();
+          const cursorMap = new Map(cursors.map((c) => [c.channelId, c.lastSeenTs]));
+
+          // Parse optional channel allowlist from env
+          const allowedIds = process.env.LEAD_MONITOR_CHANNEL_IDS
+            ? process.env.LEAD_MONITOR_CHANNEL_IDS.split(",")
+                .map((s) => s.trim())
+                .filter(Boolean)
+            : undefined;
+
+          const { messages, seedCursors } = await fetchChannelActivity(cursorMap, allowedIds);
+
+          // Commit seed cursors immediately (cold-start initialization, no trigger)
+          for (const [channelId, ts] of seedCursors) {
+            upsertChannelActivityCursor(channelId, ts);
+          }
+
+          if (messages.length > 0) {
+            // Compute cursor updates but DON'T commit them yet.
+            // They're included in the trigger payload so the runner can commit
+            // them after the lead successfully processes the messages.
+            const latestPerChannel = new Map<string, string>();
+            for (const msg of messages) {
+              const existing = latestPerChannel.get(msg.channelId);
+              if (!existing || Number.parseFloat(msg.ts) > Number.parseFloat(existing)) {
+                latestPerChannel.set(msg.channelId, msg.ts);
+              }
+            }
+
+            result = {
+              trigger: {
+                type: "channel_activity",
+                count: messages.length,
+                messages: messages.map((m) => ({
+                  channelId: m.channelId,
+                  channelName: m.channelName,
+                  ts: m.ts,
+                  user: m.user,
+                  text: m.text.slice(0, 500),
+                })),
+                cursorUpdates: Array.from(latestPerChannel.entries()).map(([channelId, ts]) => ({
+                  channelId,
+                  ts,
+                })),
+              },
+            };
+          }
+        } catch (err) {
+          console.warn("[/api/poll] Channel activity check failed:", err);
+          // Don't fail the poll — just skip this trigger
+        }
+      }
+    }
+
+    json(res, result);
+    return true;
+  }
+
+  return false;
+}