@desplega.ai/agent-swarm 1.149.1 → 1.151.0

package/src/extensions/lifecycle.ts

@@ -0,0 +1,281 @@
+import {
+  activateExtensionVersionSnapshot,
+  getExtensionById,
+  getExtensionFiles,
+  getExtensionVersion,
+  listExtensions,
+  pruneExtensionRuns,
+  setExtensionState,
+} from "../be/extensions/db";
+import type { Extension } from "../types";
+import { ExtensionManifestSchema } from "../types";
+import { scrubSecrets } from "../utils/secret-scrubber";
+import { listRegistered, registerLoaded, unregister } from "./dispatcher";
+import { deactivateExtensionAgent, ensureExtensionAgent } from "./identity";
+import {
+  cleanExtensionTmpRoot,
+  type LoadableExtension,
+  loadExtension,
+  removeExtensionTmpRoot,
+} from "./loader";
+import { initExtensionPostBridge, teardownExtensionPostBridge } from "./post-bridge";
+
+const EXTENSION_RELOAD_INTERVAL_MS = 30_000;
+
+let reloadTimer: ReturnType<typeof setTimeout> | undefined;
+let pollInFlight: Promise<void> | undefined;
+let polling = false;
+let loopbackBaseUrl: string | undefined;
+let lastObservedUpdate = "";
+const attemptedFingerprints = new Map<string, string>();
+
+export class ExtensionLifecycleError extends Error {
+  constructor(
+    message: string,
+    readonly status: 400 | 404 = 400,
+  ) {
+    super(message);
+    this.name = "ExtensionLifecycleError";
+  }
+}
+
+export function setExtensionLoopbackBaseUrl(url: string): void {
+  loopbackBaseUrl = url;
+}
+
+export function getExtensionLoopbackBaseUrl(): string {
+  if (!loopbackBaseUrl) throw new ExtensionLifecycleError("extension loopback not ready");
+  return loopbackBaseUrl;
+}
+
+function extensionFingerprint(extension: Extension): string {
+  return JSON.stringify({
+    activeVersion: extension.activeVersion,
+    configJson: extension.configJson,
+    enabled: extension.enabled,
+    priority: extension.priority,
+  });
+}
+
+function maxUpdatedAt(extensions: Extension[]): string {
+  return extensions.reduce(
+    (latest, extension) => (extension.updatedAt > latest ? extension.updatedAt : latest),
+    "",
+  );
+}
+
+async function loadableExtension(record: Extension): Promise<LoadableExtension> {
+  if (record.activeVersion !== record.version) {
+    const version = await getExtensionVersion(record.id, record.activeVersion);
+    if (!version) {
+      throw new ExtensionLifecycleError(
+        `Extension version ${record.activeVersion} was not found`,
+        404,
+      );
+    }
+    return {
+      record: {
+        ...record,
+        manifestJson: version.manifestJson,
+        contentHash: version.contentHash,
+      },
+      manifest: ExtensionManifestSchema.parse(JSON.parse(version.manifestJson)),
+      files: JSON.parse(version.filesJson) as Record<string, string>,
+    };
+  }
+  return {
+    record,
+    manifest: ExtensionManifestSchema.parse(JSON.parse(record.manifestJson)),
+    files: await getExtensionFiles(record.id),
+  };
+}
+
+async function removeRegistered(id: string): Promise<void> {
+  const previous = unregister(id);
+  if (previous) await previous.dispose();
+}
+
+export async function enableExtension(
+  id: string,
+  opts: { by?: string | null } = {},
+): Promise<Extension> {
+  initExtensionPostBridge();
+  let record = await getExtensionById(id);
+  if (!record) throw new ExtensionLifecycleError("Extension not found", 404);
+
+  const agentId = await ensureExtensionAgent(record.name);
+  const identified = await setExtensionState(id, { agentId, updatedBy: opts.by });
+  if (!identified) throw new ExtensionLifecycleError("Extension not found", 404);
+  record = identified;
+
+  await removeRegistered(id);
+  attemptedFingerprints.set(id, extensionFingerprint(record));
+  const loaded = await loadExtension(await loadableExtension(record));
+  registerLoaded(loaded);
+  try {
+    const enabled = await setExtensionState(id, {
+      enabled: true,
+      status: "enabled",
+      consecutiveFailures: 0,
+      lastError: null,
+      agentId,
+      updatedBy: opts.by,
+    });
+    if (!enabled) throw new ExtensionLifecycleError("Extension not found", 404);
+    loaded.record = enabled;
+    attemptedFingerprints.set(id, extensionFingerprint(enabled));
+    return enabled;
+  } catch (error) {
+    await removeRegistered(id);
+    throw error;
+  }
+}
+
+export async function disableExtension(
+  id: string,
+  opts: { by?: string | null } = {},
+): Promise<Extension> {
+  const record = await getExtensionById(id);
+  if (!record) throw new ExtensionLifecycleError("Extension not found", 404);
+  await removeRegistered(id);
+  attemptedFingerprints.delete(id);
+  await deactivateExtensionAgent(record.name);
+  const disabled = await setExtensionState(id, {
+    enabled: false,
+    status: "disabled",
+    updatedBy: opts.by,
+  });
+  if (!disabled) throw new ExtensionLifecycleError("Extension not found", 404);
+  return disabled;
+}
+
+export async function activateVersion(
+  id: string,
+  version: number,
+  opts: { by?: string | null } = {},
+): Promise<Extension> {
+  const current = await getExtensionById(id);
+  if (!current) throw new ExtensionLifecycleError("Extension not found", 404);
+  if (!(await getExtensionVersion(id, version))) {
+    throw new ExtensionLifecycleError(`Extension version ${version} was not found`, 404);
+  }
+
+  const wasEnabled = current.enabled;
+  if (wasEnabled) await disableExtension(id, opts);
+  const activated = await activateExtensionVersionSnapshot(id, version, opts.by);
+  if (!activated)
+    throw new ExtensionLifecycleError(`Extension version ${version} was not found`, 404);
+  return wasEnabled ? await enableExtension(id, opts) : activated;
+}
+
+export async function reloadExtension(
+  id: string,
+  opts: { by?: string | null } = {},
+): Promise<Extension> {
+  const record = await getExtensionById(id);
+  if (!record) throw new ExtensionLifecycleError("Extension not found", 404);
+  if (!record.enabled) return record;
+  return await enableExtension(id, opts);
+}
+
+async function syncEnabledExtensions(extensions: Extension[]): Promise<void> {
+  const enabledIds = new Set(
+    extensions.filter((extension) => extension.enabled).map((extension) => extension.id),
+  );
+  for (const loaded of listRegistered()) {
+    if (!enabledIds.has(loaded.record.id)) {
+      await removeRegistered(loaded.record.id);
+      attemptedFingerprints.delete(loaded.record.id);
+    }
+  }
+
+  for (const extension of extensions) {
+    if (!extension.enabled) continue;
+    const fingerprint = extensionFingerprint(extension);
+    const registered = listRegistered().find((loaded) => loaded.record.id === extension.id);
+    if (registered && extensionFingerprint(registered.record) === fingerprint) continue;
+    if (!registered && attemptedFingerprints.get(extension.id) === fingerprint) continue;
+    attemptedFingerprints.set(extension.id, fingerprint);
+    try {
+      await enableExtension(extension.id);
+    } catch (error) {
+      console.error(
+        `[extensions] Reload failed for ${extension.name}:`,
+        scrubSecrets(error instanceof Error ? error.message : String(error)),
+      );
+    }
+  }
+}
+
+async function pollForExtensionChanges(): Promise<void> {
+  const extensions = await listExtensions();
+  const observedUpdate = maxUpdatedAt(extensions);
+  if (observedUpdate !== lastObservedUpdate) {
+    lastObservedUpdate = observedUpdate;
+    await syncEnabledExtensions(extensions);
+  }
+  for (const extension of extensions) await pruneExtensionRuns(extension.id);
+}
+
+function startReloadPoll(): void {
+  if (polling) return;
+  polling = true;
+  scheduleReloadPoll();
+}
+
+function scheduleReloadPoll(): void {
+  if (!polling) return;
+  reloadTimer = setTimeout(() => {
+    reloadTimer = undefined;
+    pollInFlight = pollForExtensionChanges()
+      .catch((error) => {
+        console.error(
+          "[extensions] Reload poll failed:",
+          scrubSecrets(error instanceof Error ? error.message : String(error)),
+        );
+      })
+      .finally(() => {
+        pollInFlight = undefined;
+        scheduleReloadPoll();
+      });
+  }, EXTENSION_RELOAD_INTERVAL_MS);
+  reloadTimer.unref?.();
+}
+
+export async function loadEnabledExtensions(): Promise<void> {
+  initExtensionPostBridge();
+  for (const loaded of listRegistered()) await removeRegistered(loaded.record.id);
+  await cleanExtensionTmpRoot();
+  attemptedFingerprints.clear();
+
+  const extensions = await listExtensions();
+  for (const extension of extensions) {
+    await pruneExtensionRuns(extension.id);
+    if (!extension.enabled) continue;
+    attemptedFingerprints.set(extension.id, extensionFingerprint(extension));
+    try {
+      await enableExtension(extension.id);
+    } catch (error) {
+      console.error(
+        `[extensions] Load failed for ${extension.name}:`,
+        scrubSecrets(error instanceof Error ? error.message : String(error)),
+      );
+    }
+  }
+  lastObservedUpdate = maxUpdatedAt(await listExtensions());
+  startReloadPoll();
+}
+
+export async function stopExtensionRuntime(): Promise<void> {
+  polling = false;
+  if (reloadTimer) {
+    clearTimeout(reloadTimer);
+    reloadTimer = undefined;
+  }
+  await pollInFlight;
+  loopbackBaseUrl = undefined;
+  teardownExtensionPostBridge();
+  for (const loaded of listRegistered()) await removeRegistered(loaded.record.id);
+  attemptedFingerprints.clear();
+  await removeExtensionTmpRoot();
+}

package/src/extensions/loader.ts

@@ -0,0 +1,207 @@
+import { tmpdir } from "node:os";
+import { resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import type { z } from "zod";
+import { insertExtensionRun, setExtensionState } from "../be/extensions/db";
+import { writeBareImportShim } from "../scripts-runtime/executors/native";
+import type { Extension, ExtensionManifest } from "../types";
+import { ExtensionManifestSchema } from "../types";
+import { scrubSecrets } from "../utils/secret-scrubber";
+import { isSafeBundlePath } from "./bundle-path";
+import type { ApiCtx, ExtensionApi, HooksModule, SwarmEventMap } from "./contract";
+
+const EXTENSIONS_TMP_PARENT = `${tmpdir()}/swarm-extensions`;
+export const EXTENSIONS_TMP_ROOT = `${EXTENSIONS_TMP_PARENT}/${process.pid}`;
+
+export type ExtensionHandler = {
+  event: keyof SwarmEventMap;
+  handler: (event: never, ctx: ApiCtx) => unknown | Promise<unknown>;
+  priority: number;
+};
+
+export type LoadableExtension = {
+  record: Extension;
+  manifest: ExtensionManifest;
+  files: Record<string, string>;
+};
+
+export type LoadedExtension = LoadableExtension & {
+  handlers: ExtensionHandler[];
+  configSchema?: z.ZodTypeAny;
+  config: Record<string, unknown>;
+  sourcePath: string;
+  dispose: () => Promise<void>;
+};
+
+export class ExtensionConfigError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ExtensionConfigError";
+  }
+}
+
+const successfulDirectoryByName = new Map<string, string>();
+
+function isZodSchema(value: unknown): value is z.ZodTypeAny {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    "safeParseAsync" in value &&
+    typeof value.safeParseAsync === "function"
+  );
+}
+
+async function writeRuntimeShims(directory: string): Promise<void> {
+  const runtimeDir = process.env.SCRIPT_RUNTIME_DIR;
+  if (runtimeDir) {
+    await Promise.all([
+      writeBareImportShim(directory, "stdlib", pathToFileURL(`${runtimeDir}/stdlib.bundle.js`)),
+      writeBareImportShim(directory, "zod", pathToFileURL(`${runtimeDir}/zod.bundle.js`)),
+      writeBareImportShim(
+        directory,
+        "swarm-extension",
+        pathToFileURL(`${runtimeDir}/extensions-contract.bundle.js`),
+      ),
+    ]);
+    return;
+  }
+
+  const zodEntry = Bun.resolveSync("zod", import.meta.dir);
+  await Promise.all([
+    writeBareImportShim(
+      directory,
+      "stdlib",
+      new URL("../scripts-runtime/stdlib/index.ts", import.meta.url),
+    ),
+    writeBareImportShim(directory, "zod", pathToFileURL(zodEntry)),
+    writeBareImportShim(
+      directory,
+      "swarm-extension",
+      new URL("./contract-runtime.ts", import.meta.url),
+    ),
+  ]);
+}
+
+async function persistLoadError(record: Extension, error: unknown): Promise<void> {
+  const message = scrubSecrets(error instanceof Error ? error.message : String(error));
+  try {
+    await insertExtensionRun({
+      extensionId: record.id,
+      version: record.activeVersion,
+      event: "load",
+      action: "load-error",
+      message,
+    });
+    await setExtensionState(record.id, { status: "error", lastError: message });
+  } catch (persistError) {
+    console.error(
+      "[extensions] Failed to persist load error:",
+      scrubSecrets(persistError instanceof Error ? persistError.message : String(persistError)),
+    );
+  }
+}
+
+export async function cleanExtensionTmpRoot(): Promise<void> {
+  await removeExtensionTmpRoot();
+  const siblings = new Bun.Glob("*");
+  await Bun.$`mkdir -p ${EXTENSIONS_TMP_PARENT}`.quiet();
+  for await (const name of siblings.scan({ cwd: EXTENSIONS_TMP_PARENT, onlyFiles: false })) {
+    if (!/^[1-9]\d*$/.test(name)) continue;
+    try {
+      process.kill(Number(name), 0);
+    } catch {
+      await Bun.$`rm -rf ${EXTENSIONS_TMP_PARENT}/${name}`.quiet();
+    }
+  }
+}
+
+export async function removeExtensionTmpRoot(): Promise<void> {
+  await Bun.$`rm -rf ${EXTENSIONS_TMP_ROOT}`.quiet();
+  successfulDirectoryByName.clear();
+}
+
+export async function loadExtension(input: LoadableExtension): Promise<LoadedExtension> {
+  const { record } = input;
+  const directory = `${EXTENSIONS_TMP_ROOT}/${record.name}/${record.contentHash}`;
+
+  try {
+    const bundlePath = (path: string): string => {
+      const filePath = resolve(directory, path);
+      if (!isSafeBundlePath(path) || !filePath.startsWith(`${directory}/`)) {
+        throw new Error(`Unsafe bundle file path: ${JSON.stringify(path)}`);
+      }
+      return filePath;
+    };
+    const sourcePath = bundlePath(input.manifest.assets.hooks);
+    const files = Object.entries(input.files).map(
+      ([path, content]) => [bundlePath(path), content] as const,
+    );
+    await Bun.$`mkdir -p ${directory}`.quiet();
+    for (const [filePath, content] of files) {
+      const parent = filePath.slice(0, filePath.lastIndexOf("/"));
+      await Bun.$`mkdir -p ${parent}`.quiet();
+      await Bun.write(filePath, content);
+    }
+    await writeRuntimeShims(directory);
+
+    const mod = (await import(sourcePath)) as Partial<HooksModule>;
+    if (typeof mod.default !== "function") {
+      throw new Error("Extension hooks must export a default function");
+    }
+    if (mod.config !== undefined && !isZodSchema(mod.config)) {
+      throw new ExtensionConfigError("Extension config export must be a Zod schema");
+    }
+
+    let config: Record<string, unknown>;
+    try {
+      config = JSON.parse(record.configJson) as Record<string, unknown>;
+    } catch {
+      throw new ExtensionConfigError("Extension configJson must contain a JSON object");
+    }
+    if (mod.config) {
+      const parsed = await mod.config.safeParseAsync(config);
+      if (!parsed.success) {
+        throw new ExtensionConfigError(`Extension config is invalid: ${parsed.error.message}`);
+      }
+      config = parsed.data as Record<string, unknown>;
+    }
+
+    const handlers: ExtensionHandler[] = [];
+    const api: ExtensionApi = {
+      on(event, handler, opts) {
+        handlers.push({
+          event,
+          handler: handler as ExtensionHandler["handler"],
+          priority: opts?.priority ?? record.priority,
+        });
+      },
+    };
+    mod.default(api);
+
+    const previousDirectory = successfulDirectoryByName.get(record.name);
+    if (previousDirectory && previousDirectory !== directory) {
+      await Bun.$`rm -rf ${previousDirectory}`.quiet();
+    }
+    successfulDirectoryByName.set(record.name, directory);
+
+    return {
+      ...input,
+      handlers,
+      configSchema: mod.config,
+      config,
+      sourcePath,
+      // Bun retains the imported module in its registry and cannot unload it.
+      // Disposal removes only the source directory from disk.
+      dispose: async () => {
+        await Bun.$`rm -rf ${directory}`.quiet();
+      },
+    };
+  } catch (error) {
+    await persistLoadError(record, error);
+    throw error;
+  }
+}
+
+export function parseStoredManifest(record: Extension): ExtensionManifest {
+  return ExtensionManifestSchema.parse(JSON.parse(record.manifestJson));
+}

package/src/extensions/post-bridge.ts

@@ -0,0 +1,161 @@
+import { getTaskById } from "../be/db";
+import { scrubSecrets } from "../utils/secret-scrubber";
+import { workflowEventBus } from "../workflows/event-bus";
+import { dispatchPost, extensionIdForAgent } from "./dispatcher";
+
+let subscribed = false;
+
+function observed(
+  eventName: string,
+  handler: (data: unknown) => Promise<void>,
+): (data: unknown) => void {
+  return (data): void => {
+    handler(data).catch((error) => {
+      console.error(
+        `[extensions] ${eventName} bridge failed:`,
+        scrubSecrets(error instanceof Error ? error.message : String(error)),
+      );
+    });
+  };
+}
+
+async function taskFrom(data: unknown) {
+  const taskId = (data as { taskId?: unknown } | null)?.taskId;
+  if (typeof taskId !== "string") return null;
+  return await getTaskById(taskId);
+}
+
+const onTaskCreated = observed("task.created", async (data) => {
+  const task = await taskFrom(data);
+  if (task)
+    await dispatchPost(
+      "post.task.created",
+      { task },
+      { skipExtensionId: extensionIdForAgent(task.creatorAgentId) },
+    );
+});
+
+const onTaskCompleted = observed("task.completed", async (data) => {
+  const task = await taskFrom(data);
+  if (!task) return;
+  const output = (data as { output?: unknown }).output;
+  await dispatchPost(
+    "post.task.completed",
+    {
+      task,
+      output: typeof output === "string" ? output : (task.output ?? ""),
+    },
+    { skipExtensionId: extensionIdForAgent(task.creatorAgentId) },
+  );
+});
+
+const onTaskFailed = observed("task.failed", async (data) => {
+  const task = await taskFrom(data);
+  if (!task) return;
+  const failureReason = (data as { failureReason?: unknown }).failureReason;
+  await dispatchPost(
+    "post.task.failed",
+    {
+      task,
+      failureReason: typeof failureReason === "string" ? failureReason : (task.failureReason ?? ""),
+    },
+    { skipExtensionId: extensionIdForAgent(task.creatorAgentId) },
+  );
+});
+
+const onTaskCancelled = observed("task.cancelled", async (data) => {
+  const task = await taskFrom(data);
+  if (task)
+    await dispatchPost(
+      "post.task.cancelled",
+      { task },
+      { skipExtensionId: extensionIdForAgent(task.creatorAgentId) },
+    );
+});
+
+const onTaskSuperseded = observed("task.superseded", async (data) => {
+  const task = await taskFrom(data);
+  if (!task) return;
+  const source = data as { resumeTaskId?: unknown; supersededBy?: unknown };
+  const supersededBy =
+    typeof source.supersededBy === "string"
+      ? source.supersededBy
+      : typeof source.resumeTaskId === "string"
+        ? source.resumeTaskId
+        : "";
+  await dispatchPost(
+    "post.task.superseded",
+    { task, supersededBy },
+    { skipExtensionId: extensionIdForAgent(task.creatorAgentId) },
+  );
+});
+
+const onTaskProgress = observed("task.progress", async (data) => {
+  const task = await taskFrom(data);
+  if (!task) return;
+  const progress = (data as { progress?: unknown }).progress;
+  await dispatchPost(
+    "post.task.progress",
+    {
+      task,
+      progress: typeof progress === "string" ? progress : (task.progress ?? ""),
+    },
+    { skipExtensionId: extensionIdForAgent(task.creatorAgentId) },
+  );
+});
+
+const onSlackMessage = observed("slack.message", async (data) => {
+  const source = data as {
+    channel?: unknown;
+    channelId?: unknown;
+    user?: unknown;
+    userId?: unknown;
+    text?: unknown;
+    threadTs?: unknown;
+    taskId?: unknown;
+  };
+  const channelId =
+    typeof source.channelId === "string"
+      ? source.channelId
+      : typeof source.channel === "string"
+        ? source.channel
+        : undefined;
+  const userId =
+    typeof source.userId === "string"
+      ? source.userId
+      : typeof source.user === "string"
+        ? source.user
+        : undefined;
+  if (!channelId || !userId || typeof source.text !== "string") return;
+  await dispatchPost("post.slack.message", {
+    channelId,
+    userId,
+    text: source.text,
+    ...(typeof source.threadTs === "string" ? { threadTs: source.threadTs } : {}),
+    ...(typeof source.taskId === "string" ? { taskId: source.taskId } : {}),
+  });
+});
+
+export function initExtensionPostBridge(): void {
+  if (subscribed) return;
+  subscribed = true;
+  workflowEventBus.on("task.created", onTaskCreated);
+  workflowEventBus.on("task.completed", onTaskCompleted);
+  workflowEventBus.on("task.failed", onTaskFailed);
+  workflowEventBus.on("task.cancelled", onTaskCancelled);
+  workflowEventBus.on("task.superseded", onTaskSuperseded);
+  workflowEventBus.on("task.progress", onTaskProgress);
+  workflowEventBus.on("slack.message", onSlackMessage);
+}
+
+export function teardownExtensionPostBridge(): void {
+  if (!subscribed) return;
+  subscribed = false;
+  workflowEventBus.off("task.created", onTaskCreated);
+  workflowEventBus.off("task.completed", onTaskCompleted);
+  workflowEventBus.off("task.failed", onTaskFailed);
+  workflowEventBus.off("task.cancelled", onTaskCancelled);
+  workflowEventBus.off("task.superseded", onTaskSuperseded);
+  workflowEventBus.off("task.progress", onTaskProgress);
+  workflowEventBus.off("slack.message", onSlackMessage);
+}