@decocms/sandbox-controller 0.8.0 → 0.8.2

package/dist/index.d.ts

@@ -3717,6 +3717,13 @@ interface FreestyleSandboxProviderOptions {
     orgFsSidecarImage?: string;
     /** Pause a VM after this long without network activity. Default 5 min. */
     idleTimeoutSeconds?: number;
+    /**
+     * Also pause a running VM once its daemon has gone this long unused. The
+     * idle timeout counts any traffic, so a dev server holding an outbound
+     * connection open keeps the VM awake without it. Checked every minute;
+     * 0 turns it off. Default 10 min.
+     */
+    idlePauseMs?: number;
     /**
      * Delete a VM after this long without running. Default 1 day. A plan
      * that caps it lower gets its cap.
@@ -3730,18 +3737,24 @@ interface FreestyleSandboxProviderOptions {
     client?: Freestyle;
 }
 declare class FreestyleSandboxProvider implements SandboxProvider {
+    private readonly apiKey;
     private readonly freestyle;
     private readonly lookupClient;
     private readonly image;
     private readonly sidecarImage;
     private readonly idleTimeoutSeconds;
+    private readonly idlePauseMs;
     private readonly autoDeleteSeconds;
     private readonly domainSuffix;
     private readonly timeDaemonRequest;
     private readonly inflight;
     private readonly snapshotInflight;
+    /** Snapshots are immutable once built, so one positive lookup is enough. */
+    private readonly knownSnapshots;
     private readonly lookups;
     private readonly releaseTimers;
+    private readonly idleSweep;
+    private sweeping;
     constructor(opts: FreestyleSandboxProviderOptions);
     /** Whether this handle is a Freestyle sandbox. Cached; see `MISS_TTL_MS`. */
     owns(handle: string): Promise<boolean>;
@@ -3749,8 +3762,9 @@ declare class FreestyleSandboxProvider implements SandboxProvider {
     private find;
     private remember;
     /**
-     * A snapshot of a VM with both images already pulled, so a sandbox boots in
-     * seconds instead of pulling ~2GB. One per image pair; built on first use.
+     * A snapshot of a VM with both containers RUNNING and the daemon healthy but
+     * unclaimed, so a fork resumes a live daemon (Freestyle snapshots memory) and
+     * is configured with one request instead of booted. One per image pair.
      */
     /**
      * Build this image's base snapshot now, so the first sandbox after an image
@@ -3758,11 +3772,24 @@ declare class FreestyleSandboxProvider implements SandboxProvider {
      */
     warm(): Promise<void>;
     private baseSnapshot;
+    /**
+     * The bearer the snapshot's daemon boots with. Every fork shares it until its
+     * first config rotates it, so it is derived (any replica can compute it) and
+     * never stored.
+     */
+    private sentinelToken;
+    private startContainers;
     private snapshotExists;
     private run;
     ensure(id: SandboxId, opts?: EnsureOptions): Promise<Sandbox>;
     private resume;
     private create;
+    /**
+     * The fork's first config: swap the snapshot's sentinel bearer for this
+     * sandbox's own, retried until the new domain routes. A 401 means an earlier
+     * attempt already rotated (its response was lost), so finish with the new one.
+     */
+    private claimDaemon;
     /** A plan that caps autoDelete lower answers 400; omitting it gets the cap. */
     private createVm;
     private configPayload;
@@ -3792,6 +3819,12 @@ declare class FreestyleSandboxProvider implements SandboxProvider {
     releaseAfter(handle: string, graceMs: number): Promise<void>;
     /** The VM's idle timeout already counts every request as activity. */
     renewTtl(handle: string): Promise<void>;
+    /**
+     * Pause every running VM of ours whose daemon reports `idlePauseMs` without
+     * use. The daemon's clock counts requests and streaming runs, not the dev
+     * server's own traffic. Returns the handles paused; never throws.
+     */
+    pauseIdle(): Promise<string[]>;
     private cancelRelease;
     close(): void;
 }

package/dist/index.js

@@ -4475,7 +4475,7 @@ function tenantPoolReader(store, cluster, { refreshMs = DEFAULT_TENANT_POOL_REFR
 }
 
 // ../sandbox/server/provider/freestyle.ts
-import { createHash as createHash3, randomBytes as randomBytes2 } from "crypto";
+import { createHash as createHash3, createHmac, randomBytes as randomBytes2 } from "crypto";
 
 // ../../node_modules/.bun/freestyle@0.2.16/node_modules/freestyle/dist/errors.js
 var FreestyleApiError = class _FreestyleApiError extends Error {
@@ -6759,7 +6759,7 @@ var Freestyle = class {
 // ../sandbox/package.json
 var package_default = {
   name: "@decocms/sandbox",
-  version: "1.83.0",
+  version: "1.85.2",
   private: true,
   type: "module",
   description: "Sandbox runtime for isolated Studio tool execution",
@@ -6853,6 +6853,7 @@ var ORG_KEY = "studio-org-id";
 var USER_KEY = "studio-user-id";
 var DAEMON_BOOT_TIMEOUT_MS = 12e4;
 var HEALTH_POLL_MS = 500;
+var CLAIM_RETRY_MS = 100;
 var HEALTH_PROBE_TIMEOUT_MS2 = 5e3;
 var IMAGE_PULL_TIMEOUT_MS = 3e5;
 var LOOKUP_TTL_MS = 5 * 6e4;
@@ -6860,20 +6861,29 @@ var MISS_TTL_MS = 1e4;
 var LOOKUP_MAX = 5e3;
 var WATCH_TIMEOUT_MS = 5 * 6e4;
 var LOOKUP_TIMEOUT_MS = 3e3;
+var IDLE_SWEEP_MS = 6e4;
+var IDLE_SWEEP_PAGE = 100;
 var FreestyleSandboxProvider = class {
+  apiKey;
   freestyle;
   lookupClient;
   image;
   sidecarImage;
   idleTimeoutSeconds;
+  idlePauseMs;
   autoDeleteSeconds;
   domainSuffix;
   timeDaemonRequest;
   inflight = new Inflight();
   snapshotInflight = new Inflight();
+  /** Snapshots are immutable once built, so one positive lookup is enough. */
+  knownSnapshots = /* @__PURE__ */ new Set();
   lookups = /* @__PURE__ */ new Map();
   releaseTimers = /* @__PURE__ */ new Map();
+  idleSweep = null;
+  sweeping = false;
   constructor(opts) {
+    this.apiKey = opts.apiKey;
     this.freestyle = opts.client ?? new Freestyle({ apiKey: opts.apiKey });
     this.lookupClient = opts.client ?? new Freestyle({
       apiKey: opts.apiKey,
@@ -6891,9 +6901,14 @@ var FreestyleSandboxProvider = class {
     this.image = opts.image ?? `ghcr.io/decocms/studio/studio-sandbox-go:${package_default.version}`;
     this.sidecarImage = opts.orgFsSidecarImage ?? DEFAULT_SIDECAR_IMAGE;
     this.idleTimeoutSeconds = opts.idleTimeoutSeconds ?? 5 * 60;
+    this.idlePauseMs = opts.idlePauseMs ?? 10 * 6e4;
     this.autoDeleteSeconds = opts.autoDeleteSeconds ?? 24 * 60 * 60;
     this.domainSuffix = opts.domainSuffix ?? "style.dev";
     this.timeDaemonRequest = daemonProxyTimer(opts.meter);
+    if (this.idlePauseMs > 0) {
+      this.idleSweep = setInterval(() => void this.pauseIdle(), IDLE_SWEEP_MS);
+      this.idleSweep.unref?.();
+    }
   }
   /** Whether this handle is a Freestyle sandbox. Cached; see `MISS_TTL_MS`. */
   async owns(handle) {
@@ -6934,8 +6949,9 @@ var FreestyleSandboxProvider = class {
   }
   // ---- Base snapshot --------------------------------------------------------
   /**
-   * A snapshot of a VM with both images already pulled, so a sandbox boots in
-   * seconds instead of pulling ~2GB. One per image pair; built on first use.
+   * A snapshot of a VM with both containers RUNNING and the daemon healthy but
+   * unclaimed, so a fork resumes a live daemon (Freestyle snapshots memory) and
+   * is configured with one request instead of booted. One per image pair.
    */
   /**
    * Build this image's base snapshot now, so the first sandbox after an image
@@ -6949,9 +6965,14 @@ var FreestyleSandboxProvider = class {
   baseSnapshot() {
     const { image, sidecarImage } = this;
     const slug = `studio-sandbox-${createHash3("sha256").update(`${image}
-${sidecarImage}`).digest("hex").slice(0, 16)}`;
+${sidecarImage}
+running`).digest("hex").slice(0, 16)}`;
+    if (this.knownSnapshots.has(slug)) return Promise.resolve(slug);
     return this.snapshotInflight.run(slug, async () => {
-      if (await this.snapshotExists(slug)) return slug;
+      if (await this.snapshotExists(slug)) {
+        this.knownSnapshots.add(slug);
+        return slug;
+      }
       console.log(`[${LOG_LABEL2}] building base snapshot ${slug} for ${image}`);
       const { vm } = await this.freestyle.vms.create({
         ttlSeconds: 1800,
@@ -6967,7 +6988,14 @@ ${sidecarImage}`).digest("hex").slice(0, 16)}`;
           `docker pull -q ${shellQuote(image)} && docker pull -q ${shellQuote(sidecarImage)}`,
           { timeoutMs: IMAGE_PULL_TIMEOUT_MS }
         );
+        await this.startContainers(vm, this.sentinelToken(slug));
+        await this.run(
+          vm,
+          `for i in $(seq 1 240); do curl -sf -o /dev/null http://127.0.0.1:${DAEMON_PORT}/health && exit 0; sleep 0.5; done; exit 1`,
+          { timeoutMs: DAEMON_BOOT_TIMEOUT_MS + 1e4 }
+        );
         await vm.snapshot({ slug });
+        this.knownSnapshots.add(slug);
       } catch (err) {
         if (!await this.snapshotExists(slug)) throw err;
       } finally {
@@ -6977,6 +7005,34 @@ ${sidecarImage}`).digest("hex").slice(0, 16)}`;
       return slug;
     });
   }
+  /**
+   * The bearer the snapshot's daemon boots with. Every fork shares it until its
+   * first config rotates it, so it is derived (any replica can compute it) and
+   * never stored.
+   */
+  sentinelToken(slug) {
+    return createHmac("sha256", this.apiKey).update(`daemon-sentinel:${slug}`).digest("hex").slice(0, 48);
+  }
+  async startContainers(vm, token) {
+    const env = {
+      DAEMON_TOKEN: token,
+      APP_ROOT: WORKDIR,
+      PROXY_PORT: String(DAEMON_PORT),
+      ORGFS_SIDECAR_CONFIG_PATH: `${CTL_DIR}/config.json`,
+      ORGFS_SIDECAR_STATUS_PATH: `${CTL_DIR}/status.json`
+    };
+    const envFlags = Object.keys(env).map((k) => `-e ${shellQuote(k)}`).join(" ");
+    await this.run(
+      vm,
+      `sudo sh -c ${shellQuote(HOST_SETUP)} && docker run -d --name ${SIDECAR} --restart=always --privileged --device /dev/fuse -e APP_ROOT=${WORKDIR} -e ORGFS_FEED_IDLE_MS=60000 -v ${HOST_ORG_DIR}:${WORKDIR}/org:rshared -v ${HOST_CTL_DIR}:${CTL_DIR} ${shellQuote(this.sidecarImage)}`,
+      { timeoutMs: IMAGE_PULL_TIMEOUT_MS }
+    );
+    await this.run(
+      vm,
+      `docker run -d --name ${CONTAINER} --restart=always -p ${DAEMON_PORT}:${DAEMON_PORT} -v ${HOST_ORG_DIR}:${WORKDIR}/org:rslave -v ${HOST_CTL_DIR}:${CTL_DIR} ${envFlags} ${shellQuote(this.image)}`,
+      { env, timeoutMs: IMAGE_PULL_TIMEOUT_MS }
+    );
+  }
   async snapshotExists(slug) {
     try {
       await this.freestyle.vms.snapshots.get(slug);
@@ -7041,7 +7097,19 @@ ${sidecarImage}`).digest("hex").slice(0, 16)}`;
     }
   }
   async create(handle, opts) {
+    if (opts.env && Object.keys(opts.env).length > 0) {
+      throw new Error(
+        `${LOG_LABEL2}: per-sandbox container env is not supported`
+      );
+    }
+    const ms = {};
+    let last = Date.now();
+    const step = (name) => {
+      ms[name] = Date.now() - last;
+      last = Date.now();
+    };
     const snapshotId = await this.baseSnapshot();
+    step("snapshot");
     const token = randomBytes2(24).toString("hex");
     const { vm } = await this.createVm({
       snapshotId,
@@ -7065,35 +7133,23 @@ ${sidecarImage}`).digest("hex").slice(0, 16)}`;
         ]
       }
     });
+    step("vm-fork");
     const daemon = { url: this.daemonUrl(handle), token };
     try {
-      const env = {
-        ...opts.env,
-        DAEMON_TOKEN: token,
-        DAEMON_BOOT_ID: crypto.randomUUID(),
-        APP_ROOT: WORKDIR,
-        PROXY_PORT: String(DAEMON_PORT),
-        ORGFS_SIDECAR_CONFIG_PATH: `${CTL_DIR}/config.json`,
-        ORGFS_SIDECAR_STATUS_PATH: `${CTL_DIR}/status.json`
-      };
-      const envFlags = Object.keys(env).map((k) => `-e ${shellQuote(k)}`).join(" ");
-      await this.run(
-        vm,
-        `sudo sh -c ${shellQuote(HOST_SETUP)} && docker run -d --name ${SIDECAR} --restart=always --privileged --device /dev/fuse -e APP_ROOT=${WORKDIR} -e ORGFS_FEED_IDLE_MS=60000 -v ${HOST_ORG_DIR}:${WORKDIR}/org:rshared -v ${HOST_CTL_DIR}:${CTL_DIR} ${shellQuote(this.sidecarImage)}`,
-        { timeoutMs: IMAGE_PULL_TIMEOUT_MS }
-      );
-      await this.run(
-        vm,
-        `docker run -d --name ${CONTAINER} --restart=always -p ${DAEMON_PORT}:${DAEMON_PORT} -v ${HOST_ORG_DIR}:${WORKDIR}/org:rslave -v ${HOST_CTL_DIR}:${CTL_DIR} ${envFlags} ${shellQuote(this.image)}`,
-        { env, timeoutMs: IMAGE_PULL_TIMEOUT_MS }
+      await this.claimDaemon(
+        daemon.url,
+        this.sentinelToken(snapshotId),
+        token,
+        this.configPayload(opts) ?? {}
       );
-      await this.waitForDaemon(daemon.url);
-      await postConfig(daemon.url, token, this.configPayload(opts) ?? {});
+      step("claim");
       if (opts.orgFsConfigJson) {
         await postOrgFsConfig(daemon.url, token, opts.orgFsConfigJson).catch(
           (err) => console.warn(`[${LOG_LABEL2}] org-fs relay failed`, err)
         );
       }
+      step("orgfs-config");
+      console.log(`[${LOG_LABEL2}] create ${handle} ms=${JSON.stringify(ms)}`);
     } catch (err) {
       await vm.delete().catch(() => {
       });
@@ -7102,6 +7158,30 @@ ${sidecarImage}`).digest("hex").slice(0, 16)}`;
     }
     this.remember(handle, { vm, daemon });
   }
+  /**
+   * The fork's first config: swap the snapshot's sentinel bearer for this
+   * sandbox's own, retried until the new domain routes. A 401 means an earlier
+   * attempt already rotated (its response was lost), so finish with the new one.
+   */
+  async claimDaemon(url, sentinel, token, payload) {
+    const deadline = Date.now() + DAEMON_BOOT_TIMEOUT_MS;
+    for (; ; ) {
+      try {
+        await postConfig(url, sentinel, payload, { rotateToken: token });
+        return;
+      } catch (err) {
+        if (err instanceof ConfigRequestError && err.status === 401) {
+          await postConfig(url, token, payload);
+          return;
+        }
+        if (err instanceof ConfigRequestError && (err.status === 400 || err.status === 409)) {
+          throw err;
+        }
+        if (Date.now() > deadline) throw err;
+        await delay(CLAIM_RETRY_MS);
+      }
+    }
+  }
   /** A plan that caps autoDelete lower answers 400; omitting it gets the cap. */
   async createVm(opts) {
     try {
@@ -7259,12 +7339,64 @@ ${sidecarImage}`).digest("hex").slice(0, 16)}`;
   async renewTtl(handle) {
     this.cancelRelease(handle);
   }
+  /**
+   * Pause every running VM of ours whose daemon reports `idlePauseMs` without
+   * use. The daemon's clock counts requests and streaming runs, not the dev
+   * server's own traffic. Returns the handles paused; never throws.
+   */
+  async pauseIdle() {
+    if (this.sweeping) return [];
+    this.sweeping = true;
+    try {
+      const running = [];
+      for (let offset = 0; ; offset += IDLE_SWEEP_PAGE) {
+        const { vms } = await this.freestyle.vms.list({
+          state: "running",
+          limit: IDLE_SWEEP_PAGE,
+          offset
+        });
+        running.push(...vms);
+        if (vms.length < IDLE_SWEEP_PAGE) break;
+      }
+      const paused = await Promise.all(
+        running.map(async (data) => {
+          const handle = data.slug;
+          if (!handle || !data.metadata[TOKEN_KEY]) return null;
+          if (Date.now() - Date.parse(data.createdAt) < this.idlePauseMs) {
+            return null;
+          }
+          const idleMs = await daemonIdleMs(this.daemonUrl(handle));
+          if (idleMs === null || idleMs < this.idlePauseMs) return null;
+          try {
+            await this.freestyle.vms.ref(data.id).pause();
+          } catch (err) {
+            console.warn(
+              `[${LOG_LABEL2}] pause ${handle} failed: ${errMsg2(err)}`
+            );
+            return null;
+          }
+          this.cancelRelease(handle);
+          console.log(
+            `[${LOG_LABEL2}] paused ${handle}, daemon idle ${Math.round(idleMs / 6e4)}m`
+          );
+          return handle;
+        })
+      );
+      return paused.filter((h) => h !== null);
+    } catch (err) {
+      console.warn(`[${LOG_LABEL2}] idle sweep failed: ${errMsg2(err)}`);
+      return [];
+    } finally {
+      this.sweeping = false;
+    }
+  }
   cancelRelease(handle) {
     const timer = this.releaseTimers.get(handle);
     if (timer) clearTimeout(timer);
     this.releaseTimers.delete(handle);
   }
   close() {
+    if (this.idleSweep) clearInterval(this.idleSweep);
     for (const timer of this.releaseTimers.values()) clearTimeout(timer);
     this.releaseTimers.clear();
     this.lookups.clear();
@@ -7281,6 +7413,21 @@ async function daemonAnswers(url) {
     return false;
   }
 }
+async function daemonIdleMs(url) {
+  try {
+    const res = await fetch(`${url}/_sandbox/idle`, {
+      signal: AbortSignal.timeout(HEALTH_PROBE_TIMEOUT_MS2)
+    });
+    if (!res.ok) {
+      await res.body?.cancel();
+      return null;
+    }
+    const body = await res.json();
+    return typeof body === "object" && body !== null && "idleMs" in body && typeof body.idleMs === "number" ? body.idleMs : null;
+  } catch {
+    return null;
+  }
+}
 function shellQuote(s) {
   return `'${s.replaceAll("'", `'\\''`)}'`;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@decocms/sandbox-controller",
-  "version": "0.8.0",
+  "version": "0.8.2",
   "description": "Runs Studio agent sandboxes on Kubernetes, and serves them to a remote Studio.",
   "repository": {
     "type": "git",