@decocms/sandbox-controller 0.6.0 → 0.7.0

package/dist/index.js

@@ -842,6 +842,16 @@ function refSlugSource(projectRef) {
 }
 
 // ../sandbox/server/provider/types.ts
+var SANDBOX_PLACEMENT_REASONS = [
+  "existing",
+  "requested",
+  "split",
+  "capacity",
+  "fallback",
+  "image",
+  "warm-pool",
+  "freestyle-unavailable"
+];
 function sandboxIdKey(id2) {
   return `${id2.userId}:${id2.projectRef}`;
 }
@@ -6749,7 +6759,7 @@ var Freestyle = class {
 // ../sandbox/package.json
 var package_default = {
   name: "@decocms/sandbox",
-  version: "1.79.0",
+  version: "1.80.0",
   private: true,
   type: "module",
   description: "Sandbox runtime for isolated Studio tool execution",
@@ -6827,6 +6837,17 @@ var DAEMON_PORT = 9e3;
 var WORKDIR = "/app";
 var DEFAULT_DEV_PORT2 = 3e3;
 var CONTAINER = "sandbox";
+var SIDECAR = "orgfs";
+var DEFAULT_SIDECAR_IMAGE = "ghcr.io/decocms/studio/orgfs-sidecar:0.1.0";
+var HOST_ORG_DIR = "/srv/sandbox/org";
+var HOST_CTL_DIR = "/srv/sandbox/orgfs-ctl";
+var CTL_DIR = "/run/orgfs";
+var HOST_SETUP = [
+  `mkdir -p ${HOST_ORG_DIR} ${HOST_CTL_DIR}`,
+  `grep -q ' ${HOST_ORG_DIR} ' /etc/fstab || printf '%s\\n' '${HOST_ORG_DIR} ${HOST_ORG_DIR} none bind,rshared 0 0' 'tmpfs ${HOST_CTL_DIR} tmpfs uid=1000,gid=1000,mode=0755,size=1m 0 0' >> /etc/fstab`,
+  `mountpoint -q ${HOST_ORG_DIR} || mount ${HOST_ORG_DIR}`,
+  `mountpoint -q ${HOST_CTL_DIR} || mount ${HOST_CTL_DIR}`
+].join(" && ");
 var TOKEN_KEY = "studio-daemon-token";
 var ORG_KEY = "studio-org-id";
 var USER_KEY = "studio-user-id";
@@ -6838,10 +6859,14 @@ var LOOKUP_TTL_MS = 5 * 6e4;
 var MISS_TTL_MS = 1e4;
 var LOOKUP_MAX = 5e3;
 var WATCH_TIMEOUT_MS = 5 * 6e4;
+var LOOKUP_TIMEOUT_MS = 3e3;
 var FreestyleSandboxProvider = class {
   freestyle;
+  lookupClient;
   image;
+  sidecarImage;
   idleTimeoutSeconds;
+  autoDeleteSeconds;
   domainSuffix;
   timeDaemonRequest;
   inflight = new Inflight();
@@ -6849,9 +6874,24 @@ var FreestyleSandboxProvider = class {
   lookups = /* @__PURE__ */ new Map();
   releaseTimers = /* @__PURE__ */ new Map();
   constructor(opts) {
-    this.freestyle = new Freestyle({ apiKey: opts.apiKey });
+    this.freestyle = opts.client ?? new Freestyle({ apiKey: opts.apiKey });
+    this.lookupClient = opts.client ?? new Freestyle({
+      apiKey: opts.apiKey,
+      fetch: Object.assign(
+        (input, init) => fetch(input, {
+          ...init,
+          signal: init?.signal ? AbortSignal.any([
+            init.signal,
+            AbortSignal.timeout(LOOKUP_TIMEOUT_MS)
+          ]) : AbortSignal.timeout(LOOKUP_TIMEOUT_MS)
+        }),
+        { preconnect: fetch.preconnect }
+      )
+    });
     this.image = opts.image ?? `ghcr.io/decocms/studio/studio-sandbox-go:${package_default.version}`;
+    this.sidecarImage = opts.orgFsSidecarImage ?? DEFAULT_SIDECAR_IMAGE;
     this.idleTimeoutSeconds = opts.idleTimeoutSeconds ?? 15 * 60;
+    this.autoDeleteSeconds = opts.autoDeleteSeconds ?? 3 * 24 * 60 * 60;
     this.domainSuffix = opts.domainSuffix ?? "style.dev";
     this.timeDaemonRequest = daemonProxyTimer(opts.meter);
   }
@@ -6870,7 +6910,7 @@ var FreestyleSandboxProvider = class {
     }
     let found = null;
     try {
-      const data = await this.freestyle.vms.get(handle);
+      const data = await this.lookupClient.vms.get(handle);
       const token = data.metadata[TOKEN_KEY];
       if (token) {
         found = {
@@ -6894,12 +6934,13 @@ var FreestyleSandboxProvider = class {
   }
   // ---- Base snapshot --------------------------------------------------------
   /**
-   * A snapshot of a VM with the image already pulled, so a sandbox boots in
-   * seconds instead of pulling ~2GB. One per image; built on first use.
+   * A snapshot of a VM with both images already pulled, so a sandbox boots in
+   * seconds instead of pulling ~2GB. One per image pair; built on first use.
    */
   baseSnapshot() {
-    const image = this.image;
-    const slug = `studio-sandbox-${createHash3("sha256").update(image).digest("hex").slice(0, 16)}`;
+    const { image, sidecarImage } = this;
+    const slug = `studio-sandbox-${createHash3("sha256").update(`${image}
+${sidecarImage}`).digest("hex").slice(0, 16)}`;
     return this.snapshotInflight.run(slug, async () => {
       if (await this.snapshotExists(slug)) return slug;
       console.log(`[${LOG_LABEL2}] building base snapshot ${slug} for ${image}`);
@@ -6912,9 +6953,11 @@ var FreestyleSandboxProvider = class {
         }
       });
       try {
-        await this.run(vm, `docker pull -q ${shellQuote(image)}`, {
-          timeoutMs: IMAGE_PULL_TIMEOUT_MS
-        });
+        await this.run(
+          vm,
+          `docker pull -q ${shellQuote(image)} && docker pull -q ${shellQuote(sidecarImage)}`,
+          { timeoutMs: IMAGE_PULL_TIMEOUT_MS }
+        );
         await vm.snapshot({ slug });
       } catch (err) {
         if (!await this.snapshotExists(slug)) throw err;
@@ -6972,6 +7015,13 @@ var FreestyleSandboxProvider = class {
     const { state } = await found.vm.data();
     if (state !== "running") await found.vm.start();
     await this.waitForDaemon(found.daemon.url);
+    if (opts.orgFsConfigJson) {
+      await postOrgFsConfig(
+        found.daemon.url,
+        found.daemon.token,
+        opts.orgFsConfigJson
+      ).catch((err) => console.warn(`[${LOG_LABEL2}] org-fs relay failed`, err));
+    }
     const patch = gitCredentialRefreshPatch(opts);
     if (patch) {
       await postConfig(found.daemon.url, found.daemon.token, patch).catch(
@@ -6984,7 +7034,7 @@ var FreestyleSandboxProvider = class {
   async create(handle, opts) {
     const snapshotId = await this.baseSnapshot();
     const token = randomBytes2(24).toString("hex");
-    const { vm } = await this.freestyle.vms.create({
+    const { vm } = await this.createVm({
       snapshotId,
       slug: handle,
       idleTimeoutSeconds: this.idleTimeoutSeconds,
@@ -7013,12 +7063,19 @@ var FreestyleSandboxProvider = class {
         DAEMON_TOKEN: token,
         DAEMON_BOOT_ID: crypto.randomUUID(),
         APP_ROOT: WORKDIR,
-        PROXY_PORT: String(DAEMON_PORT)
+        PROXY_PORT: String(DAEMON_PORT),
+        ORGFS_SIDECAR_CONFIG_PATH: `${CTL_DIR}/config.json`,
+        ORGFS_SIDECAR_STATUS_PATH: `${CTL_DIR}/status.json`
       };
       const envFlags = Object.keys(env).map((k) => `-e ${shellQuote(k)}`).join(" ");
       await this.run(
         vm,
-        `docker run -d --name ${CONTAINER} --restart=always -p ${DAEMON_PORT}:${DAEMON_PORT} ${envFlags} ${shellQuote(this.image)}`,
+        `sudo sh -c ${shellQuote(HOST_SETUP)} && docker run -d --name ${SIDECAR} --restart=always --privileged --device /dev/fuse -e APP_ROOT=${WORKDIR} -v ${HOST_ORG_DIR}:${WORKDIR}/org:rshared -v ${HOST_CTL_DIR}:${CTL_DIR} ${shellQuote(this.sidecarImage)}`,
+        { timeoutMs: IMAGE_PULL_TIMEOUT_MS }
+      );
+      await this.run(
+        vm,
+        `docker run -d --name ${CONTAINER} --restart=always -p ${DAEMON_PORT}:${DAEMON_PORT} -v ${HOST_ORG_DIR}:${WORKDIR}/org:rslave -v ${HOST_CTL_DIR}:${CTL_DIR} ${envFlags} ${shellQuote(this.image)}`,
         { env, timeoutMs: IMAGE_PULL_TIMEOUT_MS }
       );
       await this.waitForDaemon(daemon.url);
@@ -7036,6 +7093,21 @@ var FreestyleSandboxProvider = class {
     }
     this.remember(handle, { vm, daemon });
   }
+  /** A plan that caps autoDelete lower answers 400; omitting it gets the cap. */
+  async createVm(opts) {
+    try {
+      return await this.freestyle.vms.create({
+        ...opts,
+        autoDeleteSeconds: this.autoDeleteSeconds
+      });
+    } catch (err) {
+      if (!(err instanceof FreestyleApiError && err.status === 400)) throw err;
+      console.warn(
+        `[${LOG_LABEL2}] autoDeleteSeconds=${this.autoDeleteSeconds} refused (${err.message}); creating with the plan's cap`
+      );
+      return this.freestyle.vms.create(opts);
+    }
+  }
   configPayload(opts) {
     return buildConfigPayload({
       runtime: opts.workload?.runtime ?? "node",
@@ -7065,7 +7137,10 @@ var FreestyleSandboxProvider = class {
     const found = await this.find(handle, { refresh: true });
     this.lookups.delete(handle);
     if (!found) return;
-    await found.vm.exec({ command: `docker stop -t 30 ${CONTAINER}`, timeoutMs: 45e3 }).catch(() => {
+    await found.vm.exec({
+      command: `docker stop -t 30 ${CONTAINER}; docker stop -t 10 ${SIDECAR}`,
+      timeoutMs: 6e4
+    }).catch(() => {
     });
     await found.vm.delete();
   }
@@ -7228,6 +7303,10 @@ function handleShare(handle) {
   return Number.parseInt(hex, 16) / 2 ** 32;
 }
 var PICKED_MAX = 5e3;
+var other = (kind) => kind === "kubernetes" ? "freestyle" : "kubernetes";
+function repeatsElsewhere(err) {
+  return err instanceof ConfigRequestError;
+}
 var SandboxProviderRouter = class {
   constructor(providers, opts = {}) {
     this.providers = providers;
@@ -7238,6 +7317,14 @@ var SandboxProviderRouter = class {
   }
   providers;
   picked = /* @__PURE__ */ new Map();
+  /**
+   * Handles whose sandbox on this provider outlived a fallback because its
+   * delete failed. Retried when `place` next sees the handle.
+   *
+   * ponytail: in-process, lost on restart; Freestyle's autoDeleteSeconds
+   * reclaims a stray paused VM anyway, only later.
+   */
+  strays = /* @__PURE__ */ new Map();
   freestyleShare;
   get(kind) {
     const provider = this.providers[kind];
@@ -7260,12 +7347,13 @@ var SandboxProviderRouter = class {
     if (!kubernetes) return "freestyle";
     const picked = this.picked.get(handle);
     if (picked) return picked;
-    return await freestyle.owns(handle) ? "freestyle" : "kubernetes";
+    const owned = await freestyle.owns(handle).catch(() => false);
+    return owned ? "freestyle" : "kubernetes";
   }
   async owner(handle) {
     return this.get(await this.providerOf(handle));
   }
-  /** Where `ensure` puts this sandbox. */
+  /** Where `ensure` puts this sandbox, and why. */
   async place(handle, opts) {
     const { kubernetes, freestyle } = this.providers;
     const image = opts.sandboxImage ?? "default";
@@ -7273,30 +7361,98 @@ var SandboxProviderRouter = class {
       if (!kubernetes) {
         throw new Error(`sandbox image ${image} needs the kubernetes provider`);
       }
-      return "kubernetes";
+      return { kind: "kubernetes", reason: freestyle ? "image" : void 0 };
+    }
+    if (!kubernetes) return { kind: "freestyle" };
+    if (!freestyle) return { kind: "kubernetes" };
+    const stray = this.strays.get(handle);
+    if (stray) this.retire(stray, handle);
+    if (stray !== "kubernetes" && await kubernetes.alive(handle).catch(() => false)) {
+      return { kind: "kubernetes", reason: "existing" };
+    }
+    const owned = stray === "freestyle" ? false : await freestyle.owns(handle).catch(() => null);
+    if (owned) return { kind: "freestyle", reason: "existing" };
+    if (owned === null) {
+      return { kind: "kubernetes", reason: "freestyle-unavailable" };
+    }
+    if (opts.provider) return { kind: opts.provider, reason: "requested" };
+    if (this.hasWarmPool(opts, image)) {
+      return { kind: "kubernetes", reason: "warm-pool" };
+    }
+    if (handleShare(handle) < this.freestyleShare) {
+      return { kind: "freestyle", reason: "split" };
     }
-    if (!kubernetes) return "freestyle";
-    if (!freestyle) return "kubernetes";
-    if (await freestyle.owns(handle)) return "freestyle";
-    if (await kubernetes.alive(handle).catch(() => false)) return "kubernetes";
-    if (opts.provider) return opts.provider;
-    if (handleShare(handle) < this.freestyleShare) return "freestyle";
     const roomy = await kubernetes.hasSchedulableCapacity().catch(() => false);
-    return roomy ? "kubernetes" : "freestyle";
+    return roomy ? { kind: "kubernetes", reason: "split" } : { kind: "freestyle", reason: "capacity" };
+  }
+  /** The org has pods already cloned and running for this image. */
+  hasWarmPool(opts, image) {
+    const tenant = claimTenantKey(opts);
+    if (!tenant) return false;
+    const pools = this.providers.kubernetes?.listTenantPools?.() ?? [];
+    return pools.some((p) => p.tenant === tenant && p.image === image);
   }
   async ensure(id2, opts = {}) {
     const { provider: _requested, ...rest } = opts;
     const handle = computeHandle(id2);
-    const kind = await this.place(handle, opts);
-    this.remember(handle, kind);
+    const placed = await this.place(handle, opts);
+    this.remember(handle, placed.kind);
+    if (this.strays.get(handle) === placed.kind) this.strays.delete(handle);
+    let failure;
+    try {
+      const sandbox2 = await this.get(placed.kind).ensure(id2, rest);
+      return placed.reason ? { ...sandbox2, placement: { reason: placed.reason } } : sandbox2;
+    } catch (err) {
+      failure = tagSandboxProvider(err, placed.kind);
+    }
+    const fallback = other(placed.kind);
+    if (!placed.reason || placed.reason === "image" || repeatsElsewhere(failure)) {
+      throw failure;
+    }
+    console.warn(
+      `[SandboxProviderRouter] ensure ${handle} failed on ${placed.kind}, trying ${fallback}:`,
+      failure instanceof Error ? failure.message : String(failure)
+    );
+    this.remember(handle, fallback);
+    let sandbox;
     try {
-      return await this.get(kind).ensure(id2, rest);
+      sandbox = await this.get(fallback).ensure(id2, rest);
     } catch (err) {
-      throw tagSandboxProvider(err, kind);
+      this.picked.delete(handle);
+      throw tagSandboxProvider(err, fallback);
     }
+    if (placed.reason === "existing") this.retire(placed.kind, handle);
+    return {
+      ...sandbox,
+      placement: { reason: "fallback", fallbackFrom: placed.kind }
+    };
+  }
+  /** Delete the sandbox a fallback left behind on `kind`, in the background. */
+  retire(kind, handle) {
+    this.strays.set(handle, kind);
+    void retry(
+      async () => {
+        if (this.strays.get(handle) === kind)
+          await this.get(kind).delete(handle);
+      },
+      { maxAttempts: 3 }
+    ).then(() => {
+      if (this.strays.get(handle) === kind) this.strays.delete(handle);
+    }).catch(
+      (err) => console.warn(
+        `[SandboxProviderRouter] delete of ${kind} sandbox ${handle} after fallback failed:`,
+        err instanceof Error ? err.message : String(err)
+      )
+    );
   }
   async delete(handle) {
-    await (await this.owner(handle)).delete(handle);
+    const stray = this.strays.get(handle);
+    this.strays.delete(handle);
+    await Promise.all([
+      (await this.owner(handle)).delete(handle),
+      stray && this.get(stray).delete(handle).catch(() => {
+      })
+    ]);
     this.picked.delete(handle);
   }
   async lastTermination(handle) {
@@ -7356,6 +7512,7 @@ var SandboxProviderRouter = class {
     this.providers.kubernetes?.close();
     this.providers.freestyle?.close();
     this.picked.clear();
+    this.strays.clear();
   }
 };
 
@@ -7440,6 +7597,11 @@ var ensureOutputSchema = z4.object({
   previewUrl: z4.string().nullable(),
   warmPoolAdopted: z4.boolean(),
   provider: providerKindSchema.default("kubernetes"),
+  // A reason this build doesn't know drops the label, not the ensure.
+  placement: z4.object({
+    reason: z4.enum(SANDBOX_PLACEMENT_REASONS),
+    fallbackFrom: providerKindSchema.optional()
+  }).optional().catch(void 0),
   daemon: daemonSchema
 });
 var handleInputSchema = z4.object({ handle: z4.string().min(1) });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@decocms/sandbox-controller",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "Runs Studio agent sandboxes on Kubernetes, and serves them to a remote Studio.",
   "repository": {
     "type": "git",