@checkstack/backend 0.8.1 → 0.9.0

package/src/plugin-deregistration.test.ts

@@ -0,0 +1,137 @@
+import { describe, it, expect, mock, beforeEach } from "bun:test";
+import { PluginManager } from "./plugin-manager";
+
+/**
+ * Integration test for the originator/in-process split introduced in the
+ * runtime plugin system.
+ *
+ * Key contract: `deregisterPluginInProcess` must NOT touch any persistent
+ * state. It runs on every instance via the broadcast hook; the destructive
+ * cleanup (drop schema, delete plugin_configs, delete plugin_artifacts,
+ * delete plugins rows) only runs on the originator via `deletePluginData`.
+ *
+ * If a future refactor accidentally pushes destructive ops back into the
+ * in-process path, these tests fire.
+ */
+
+describe("deregisterPluginInProcess", () => {
+  let pluginManager: PluginManager;
+
+  beforeEach(() => {
+    pluginManager = new PluginManager();
+  });
+
+  it("clears in-memory cleanup handlers and runs them in LIFO order", async () => {
+    const order: string[] = [];
+    const handlerA = mock(async () => {
+      order.push("a");
+    });
+    const handlerB = mock(async () => {
+      order.push("b");
+    });
+
+    // Inject cleanup handlers using the same mechanism plugins use during
+    // registration. We poke the private map because the public path (full
+    // backendPlugin.register) needs a real plugin module.
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const handlers = (pluginManager as any).cleanupHandlers as Map<
+      string,
+      Array<() => Promise<void>>
+    >;
+    handlers.set("test-plugin", [handlerA, handlerB]);
+
+    await pluginManager.deregisterPluginInProcess("test-plugin");
+
+    // LIFO: handlerB ran before handlerA
+    expect(order).toEqual(["b", "a"]);
+    expect(handlers.has("test-plugin")).toBe(false);
+  });
+
+  it("removes router, contract, metadata and access-rule entries", async () => {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const pm = pluginManager as any;
+    pm.pluginRpcRouters.set("foo", { router: true });
+    pm.pluginContractRegistry.set("foo", { contract: true });
+    pm.pluginMetadataRegistry.set("foo", { pluginId: "foo" });
+    pm.registeredAccessRules.push({
+      id: "foo.something",
+      pluginId: "foo",
+      action: "read",
+      resourceType: "thing",
+      description: "test",
+    });
+
+    await pluginManager.deregisterPluginInProcess("foo");
+
+    expect(pm.pluginRpcRouters.has("foo")).toBe(false);
+    expect(pm.pluginContractRegistry.has("foo")).toBe(false);
+    expect(pm.pluginMetadataRegistry.has("foo")).toBe(false);
+    expect(
+      pm.registeredAccessRules.some(
+        (r: { pluginId: string }) => r.pluginId === "foo",
+      ),
+    ).toBe(false);
+  });
+
+  it("does not invoke any artifact-store or destructive ops", async () => {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const pm = pluginManager as any;
+
+    const artifactStoreCalls: string[] = [];
+    const fakeArtifactStore = {
+      maxArtifactSize: 1,
+      store: () => {
+        artifactStoreCalls.push("store");
+        return Promise.resolve({ artifactId: "x", contentHash: "x" });
+      },
+      fetch: () => {
+        artifactStoreCalls.push("fetch");
+        return Promise.resolve(undefined);
+      },
+      fetchById: () => {
+        artifactStoreCalls.push("fetchById");
+        return Promise.resolve(undefined);
+      },
+      delete: () => {
+        artifactStoreCalls.push("delete");
+        return Promise.resolve();
+      },
+      deleteByBundle: () => {
+        artifactStoreCalls.push("deleteByBundle");
+        return Promise.resolve();
+      },
+    };
+    // The artifact store is registered via coreServices.pluginArtifactStore.
+    // Even when present, deregisterPluginInProcess should not consult it.
+    pm.registry.register(
+      { id: "core.pluginArtifactStore" },
+      fakeArtifactStore,
+    );
+
+    pm.cleanupHandlers.set("foo", []);
+    await pluginManager.deregisterPluginInProcess("foo");
+
+    expect(artifactStoreCalls).toEqual([]);
+  });
+
+  it("handler errors do not block subsequent cleanup or remove-from-registry", async () => {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const pm = pluginManager as any;
+
+    const goodHandler = mock(async () => {});
+    const badHandler = mock(async () => {
+      throw new Error("intentional");
+    });
+
+    pm.cleanupHandlers.set("plugin-a", [goodHandler, badHandler]);
+    pm.pluginRpcRouters.set("plugin-a", {});
+
+    await pluginManager.deregisterPluginInProcess("plugin-a");
+
+    expect(badHandler).toHaveBeenCalled();
+    expect(goodHandler).toHaveBeenCalled();
+    // Even though one handler threw, the in-memory state was cleared:
+    expect(pm.pluginRpcRouters.has("plugin-a")).toBe(false);
+    expect(pm.cleanupHandlers.has("plugin-a")).toBe(false);
+  });
+});

package/src/plugin-manager/api-router.ts

@@ -20,6 +20,8 @@ import type {
 import type { ServiceRegistry } from "../services/service-registry";
 import type { EventBus } from "@checkstack/backend-api";
 import type { PluginMetadata } from "@checkstack/common";
+import { rootLogger } from "../logger";
+import { extractErrorMessage } from "@checkstack/common";
 
 /**
  * Creates the API route handler for Hono.
@@ -71,17 +73,18 @@ export function createApiRouteHandler({
             try {
               return await next(rest);
             } catch (error) {
-              if (logger) {
-                logger.error(`RPC procedure error: ${String(error)}`);
-                const stack =
-                  error !== null &&
-                  typeof error === "object" &&
-                  "stack" in error
-                    ? (error as { stack: string }).stack
-                    : undefined;
-                if (stack) {
-                  logger.error(`Stack trace: ${stack}`);
-                }
+              const target = (logger ?? rootLogger) as Logger;
+              target.error(
+                `RPC ${pathname} failed: ${extractErrorMessage(error)}`,
+              );
+              const stack =
+                error !== null &&
+                typeof error === "object" &&
+                "stack" in error
+                  ? (error as { stack: string }).stack
+                  : undefined;
+              if (stack) {
+                target.error(`Stack trace: ${stack}`);
               }
               throw error;
             }
@@ -121,6 +124,22 @@ export function createApiRouteHandler({
       !cacheManager ||
       !eventBus
     ) {
+      const missing = [
+        !auth && "auth",
+        !logger && "logger",
+        !db && "db",
+        !fetch && "fetch",
+        !healthCheckRegistry && "healthCheckRegistry",
+        !collectorRegistry && "collectorRegistry",
+        !queuePluginRegistry && "queuePluginRegistry",
+        !queueManager && "queueManager",
+        !cachePluginRegistry && "cachePluginRegistry",
+        !cacheManager && "cacheManager",
+        !eventBus && "eventBus",
+      ].filter(Boolean).join(", ");
+      (logger ?? rootLogger).error(
+        `${pathname}: core services not initialized — missing: ${missing}`,
+      );
       return c.json({ error: "Core services not initialized" }, 500);
     }
 
@@ -136,6 +155,11 @@ export function createApiRouteHandler({
       pluginMetadataRegistry.get(pluginId);
 
     if (!pluginMetadata) {
+      (logger as Logger).error(
+        `${pathname}: no plugin metadata registered for pluginId='${pluginId}'. ` +
+          `Regular plugins populate this during register(); core routers must call ` +
+          `pluginManager.registerCorePluginMetadata().`,
+      );
       return c.json({ error: "Plugin metadata not found in registry" }, 500);
     }
 

package/src/plugin-manager/core-services.ts

@@ -30,6 +30,10 @@ import {
   WebSocketRouteStoreImpl,
   createScopedWsRegistry,
 } from "../services/ws-route-registry";
+import {
+  CoreReadinessRegistry,
+  createScopedReadinessRegistry,
+} from "../services/readiness-registry";
 
 /**
  * Check if a PostgreSQL schema exists.
@@ -59,7 +63,11 @@ export function registerCoreServices({
   pluginRpcRouters: Map<string, unknown>;
   pluginHttpHandlers: Map<string, (req: Request) => Promise<Response>>;
   pluginContractRegistry: Map<string, unknown>;
-}): { collectorRegistry: CoreCollectorRegistry; wsStore: WebSocketRouteStoreImpl } {
+}): {
+  collectorRegistry: CoreCollectorRegistry;
+  wsStore: WebSocketRouteStoreImpl;
+  readinessRegistry: CoreReadinessRegistry;
+} {
   // 1. Database Factory (Scoped)
   registry.registerFactory(coreServices.database, async (metadata) => {
     const { pluginId, previousPluginIds } = metadata;
@@ -356,6 +364,17 @@ export function registerCoreServices({
     createScopedWsRegistry(globalWsStore, metadata.pluginId),
   );
 
+  // 11. Readiness Registry (Scoped Factory)
+  // Plugins contribute probes that are aggregated by the /ready endpoint.
+  const globalReadinessRegistry = new CoreReadinessRegistry();
+  registry.registerFactory(coreServices.readinessRegistry, () =>
+    createScopedReadinessRegistry(globalReadinessRegistry),
+  );
+
   // Return global registries for lifecycle cleanup
-  return { collectorRegistry: globalCollectorRegistry, wsStore: globalWsStore };
+  return {
+    collectorRegistry: globalCollectorRegistry,
+    wsStore: globalWsStore,
+    readinessRegistry: globalReadinessRegistry,
+  };
 }

package/src/plugin-manager/plugin-loader.ts

@@ -56,6 +56,14 @@ export interface PluginLoaderDeps {
    * Map of pluginId -> contract for OpenAPI generation.
    */
   pluginContractRegistry: Map<string, AnyContractRouter>;
+  /**
+   * Called once `/api/:pluginId/*` is added to the root router and Phase 2
+   * (per-plugin init) is about to start. From this point on, plugin RPC
+   * routers come online incrementally as each plugin initializes — so
+   * self-referencing HTTP calls (e.g. RPC made from `afterPluginsReady`)
+   * can be allowed through the boot-time request gate without deadlocking.
+   */
+  onApiRouteRegistered?: () => void;
 }
 
 /**
@@ -216,6 +224,14 @@ export async function loadPlugins({
     // 2. Sync local plugins to database
     await syncPluginsToDatabase({ localPlugins, db: deps.db });
 
+    // 2.5 Bootstrap runtime-installed plugins missing from node_modules.
+    // For every `is_uninstallable=true` row in `plugins`, ensure the
+    // package is installed in the runtime dir. If not, fetch its tarball
+    // from `plugin_artifacts` and run `bun install` so the import below
+    // resolves. This is what lets a freshly spun replica recover the
+    // full plugin set from Postgres alone.
+    await bootstrapRuntimePlugins({ db: deps.db, registry: deps.registry });
+
     // 3. Load all enabled BACKEND plugins from database
     allPlugins = await deps.db
       .select()
@@ -295,6 +311,19 @@ export async function loadPlugins({
   });
   registerApiRoute(rootRouter, apiHandler);
 
+  // Routes are now registered on the root router. Signal readiness so the
+  // server can stop blocking incoming requests in `waitForInit()`. We open
+  // the gate here (BEFORE Phase 2 / Phase 3) so that:
+  //   - the static module-load endpoints (/api/plugins, /api/about, …) stop
+  //     hanging behind the boot gate;
+  //   - cross-plugin RPC calls made from `afterPluginsReady` can self-loop
+  //     through the HTTP server without deadlocking on init completion.
+  // Plugin RPC routers come online incrementally as each plugin's Phase 2
+  // init runs; requests targeting a not-yet-initialized plugin fall through
+  // to the api-router's "Plugin metadata not found" 500, which is the
+  // pre-existing behavior and is preferable to a multi-second hang.
+  deps.onApiRouteRegistered?.();
+
   for (const id of sortedIds) {
     const p = pendingInits.find((x) => x.metadata.pluginId === id)!;
     rootLogger.info(`🚀 Initializing ${p.metadata.pluginId}...`);
@@ -620,3 +649,68 @@ function validateContractAccessRules({
     }
   }
 }
+
+/**
+ * Fresh-instance bootstrap.
+ *
+ * Walks every `is_uninstallable=true` row in the `plugins` table and ensures
+ * the corresponding package is installed under the runtime dir. Missing
+ * packages are recovered from `plugin_artifacts` (bytea) and installed with
+ * `bun install --no-save --ignore-scripts`. Once this returns, the normal
+ * `pluginModule = await import(...)` in Phase 1 below resolves.
+ */
+async function bootstrapRuntimePlugins({
+  db,
+  registry,
+}: {
+  db: SafeDatabase<Record<string, unknown>>;
+  registry: ServiceRegistry;
+}): Promise<void> {
+  const installerRegistry = await registry
+    .get(coreServices.pluginInstallerRegistry, { pluginId: "core" })
+    .catch(() => {});
+  const artifactStore = await registry
+    .get(coreServices.pluginArtifactStore, { pluginId: "core" })
+    .catch(() => {});
+  if (!installerRegistry || !artifactStore) {
+    rootLogger.debug(
+      "   -> Skipping runtime plugin bootstrap (services not registered)",
+    );
+    return;
+  }
+
+  const remoteRows = await db
+    .select()
+    .from(plugins)
+    .where(eq(plugins.isUninstallable, true));
+
+  for (const row of remoteRows) {
+    if (!row.path) continue;
+    const pkgJsonPath = path.join(row.path, "package.json");
+    if (fs.existsSync(pkgJsonPath)) {
+      rootLogger.debug(`   -> ${row.name} already present, skipping bootstrap`);
+      continue;
+    }
+    rootLogger.info(
+      `🔌 Bootstrapping runtime plugin from artifact store: ${row.name}@${row.version}`,
+    );
+    const artifact = await artifactStore.fetch({
+      pluginName: row.name,
+      version: row.version,
+    });
+    if (!artifact) {
+      rootLogger.warn(
+        `   -> No artifact for ${row.name}@${row.version} — skipping. Re-install from the original source.`,
+      );
+      continue;
+    }
+    const allowInstallScripts =
+      (row.metadata as { checkstack?: { allowInstallScripts?: boolean } })
+        ?.checkstack?.allowInstallScripts === true;
+    await installerRegistry.forSource("npm").installFromArtifact({
+      tarball: artifact.tarball,
+      pluginName: row.name,
+      allowInstallScripts,
+    });
+  }
+}