@decocms/sandbox-controller 0.15.4 → 0.16.0

package/dist/index.d.ts

@@ -3750,6 +3750,8 @@ interface FreestyleSandboxProviderOptions {
     meter?: Meter;
     /** Replaces the API client, for tests. */
     client?: Freestyle;
+    /** How long a started VM's daemon gets to answer /health, for tests. */
+    daemonBootTimeoutMs?: number;
 }
 declare class FreestyleSandboxProvider implements SandboxProvider {
     private readonly apiKey;
@@ -3761,6 +3763,7 @@ declare class FreestyleSandboxProvider implements SandboxProvider {
     private readonly idleTimeoutSeconds;
     private readonly idlePauseMs;
     private readonly autoDeleteSeconds;
+    private readonly daemonBootTimeoutMs;
     private readonly domainSuffix;
     private readonly timeDaemonRequest;
     private readonly inflight;
@@ -3867,7 +3870,19 @@ declare class FreestyleSandboxProvider implements SandboxProvider {
     private createVm;
     private configPayload;
     private waitForDaemon;
+    /**
+     * Deletes the VM only once its shutdown sync is confirmed. Otherwise the
+     * user's unpushed work may still be on its disk, so it is detached instead:
+     * the handle is free for a new sandbox, and the VM stays recoverable by id
+     * until Freestyle's auto-delete.
+     */
     delete(handle: string): Promise<void>;
+    /**
+     * Stops the daemon, which publishes unsaved work to git on SIGTERM, then the
+     * sidecar, which flushes its mounts. The daemon exits 0 even when that
+     * publish fails, so its log decides.
+     */
+    private syncedOnShutdown;
     lastTermination(_handle: string): Promise<PodTermination | null>;
     /** A paused VM is alive: the next request resumes it. */
     alive(handle: string): Promise<boolean>;
@@ -3954,6 +3969,13 @@ declare class SandboxProviderRouter implements SandboxProvider {
      * reclaims a stray paused VM anyway, only later.
      */
     private readonly strays;
+    /**
+     * The latest ensure per handle. Placement reads what exists, so two at once
+     * each found nothing and put one sandbox on each provider.
+     *
+     * ponytail: in-process; replicas would need the state store's lock.
+     */
+    private readonly ensuring;
     private readonly freestyleShare;
     constructor(providers: SandboxProviders, opts?: SandboxProviderRouterOptions);
     private get;
@@ -3964,6 +3986,7 @@ declare class SandboxProviderRouter implements SandboxProvider {
     /** Where `ensure` puts this sandbox, and why. */
     place(handle: string, opts: EnsureOptions): Promise<Placement>;
     ensure(id: SandboxId, opts?: EnsureOptions): Promise<Sandbox>;
+    private ensureOne;
     /** Delete the sandbox a fallback left behind on `kind`, in the background. */
     private retire;
     delete(handle: string): Promise<void>;

package/dist/index.js

@@ -6804,16 +6804,17 @@ var Freestyle = class {
 // ../sandbox/package.json
 var package_default = {
   name: "@decocms/sandbox",
-  version: "1.95.4",
+  version: "1.96.0",
   private: true,
   type: "module",
   description: "Sandbox runtime for isolated Studio tool execution",
   scripts: {
     check: "tsc --noEmit",
     test: "bun test",
-    build: "bun run build:typegen && bun run build:harness-runner",
+    build: "bun run build:typegen && bun run build:harness-runner && bun run build:studio-cli",
     "build:typegen": "mkdir -p dist && bun run --cwd=../typegen build && bun --cwd=../typegen pm pack --filename $PWD/dist/typegen.tgz --quiet",
-    "build:harness-runner": "mkdir -p dist && bun --cwd=../harness-runner pm pack --filename $PWD/dist/harness-runner.tgz --quiet"
+    "build:harness-runner": "mkdir -p dist && bun --cwd=../harness-runner pm pack --filename $PWD/dist/harness-runner.tgz --quiet",
+    "build:studio-cli": "bun scripts/build-studio-cli.ts dist/decocms"
   },
   exports: {
     "./shared": "./shared.ts",
@@ -6926,6 +6927,7 @@ var FreestyleSandboxProvider = class {
   idleTimeoutSeconds;
   idlePauseMs;
   autoDeleteSeconds;
+  daemonBootTimeoutMs;
   domainSuffix;
   timeDaemonRequest;
   inflight = new Inflight();
@@ -6966,6 +6968,7 @@ var FreestyleSandboxProvider = class {
     this.idleTimeoutSeconds = opts.idleTimeoutSeconds ?? 5 * 60;
     this.idlePauseMs = opts.idlePauseMs ?? 10 * 6e4;
     this.autoDeleteSeconds = opts.autoDeleteSeconds ?? 24 * 60 * 60;
+    this.daemonBootTimeoutMs = opts.daemonBootTimeoutMs ?? DAEMON_BOOT_TIMEOUT_MS;
     this.domainSuffix = opts.domainSuffix ?? "style.dev";
     this.timeDaemonRequest = daemonProxyTimer(opts.meter);
     if (this.idlePauseMs > 0) {
@@ -7300,7 +7303,12 @@ ${head}`).digest("hex").slice(0, 16)}`;
   async resume(handle, found, opts) {
     const { state, sourceSnapshotSlugAtCreate, metadata } = await found.vm.data();
     if (state !== "running") await found.vm.start();
-    await this.waitForDaemon(found.daemon.url);
+    await this.waitForDaemon(found.daemon.url).catch(async (err) => {
+      console.warn(`[${LOG_LABEL2}] ${handle}: ${errMsg2(err)}; rebooting`);
+      await found.vm.exec({ command: "reboot -f", timeoutMs: 1e4 }).catch(() => {
+      });
+      await this.waitForDaemon(found.daemon.url);
+    });
     const sentinelSlug = metadata[BASE_KEY] ?? sourceSnapshotSlugAtCreate;
     if (sentinelSlug) {
       await this.reclaim(found, sentinelSlug, opts);
@@ -7539,26 +7547,63 @@ ${head}`).digest("hex").slice(0, 16)}`;
     });
   }
   async waitForDaemon(url) {
-    const deadline = Date.now() + DAEMON_BOOT_TIMEOUT_MS;
+    const deadline = Date.now() + this.daemonBootTimeoutMs;
     while (Date.now() < deadline) {
       if (await daemonAnswers(url)) return;
       await delay(HEALTH_POLL_MS);
     }
     throw new Error(
-      `sandbox daemon at ${url} did not answer /health within ${DAEMON_BOOT_TIMEOUT_MS / 1e3}s`
+      `sandbox daemon at ${url} did not answer /health within ${this.daemonBootTimeoutMs / 1e3}s`
     );
   }
+  /**
+   * Deletes the VM only once its shutdown sync is confirmed. Otherwise the
+   * user's unpushed work may still be on its disk, so it is detached instead:
+   * the handle is free for a new sandbox, and the VM stays recoverable by id
+   * until Freestyle's auto-delete.
+   */
   async delete(handle) {
     this.cancelRelease(handle);
     const found = await this.find(handle, { refresh: true });
     this.lookups.delete(handle);
     if (!found) return;
-    await found.vm.exec({
-      command: `docker stop -t 30 ${CONTAINER}; docker stop -t 10 ${SIDECAR}`,
-      timeoutMs: 6e4
-    }).catch(() => {
+    if (await this.syncedOnShutdown(found.vm)) {
+      await found.vm.delete();
+      return;
+    }
+    const { rules } = await this.freestyle.tls.rules.list({
+      vmId: found.vm.id
     });
-    await found.vm.delete();
+    for (const rule of rules) await this.freestyle.tls.rules.delete(rule.id);
+    await found.vm.update({ slug: "" });
+    this.appIngress.delete(handle);
+    console.warn(
+      `[${LOG_LABEL2}] ${handle}: shutdown sync not confirmed; detached VM ${found.vm.id} instead of deleting it`
+    );
+  }
+  /**
+   * Stops the daemon, which publishes unsaved work to git on SIGTERM, then the
+   * sidecar, which flushes its mounts. The daemon exits 0 even when that
+   * publish fails, so its log decides.
+   */
+  async syncedOnShutdown(vm) {
+    try {
+      const { state } = await vm.data();
+      if (state !== "running") await vm.start();
+      const res = await vm.exec({
+        command: [
+          "since=$(date +%s)",
+          `docker stop -t 30 ${CONTAINER} >/dev/null`,
+          `code=$(docker inspect -f '{{.State.ExitCode}}' ${CONTAINER})`,
+          `docker stop -t 10 ${SIDECAR} >/dev/null`,
+          `[ "$code" = 0 ] && ! docker logs --since "$since" ${CONTAINER} 2>&1 | grep -q "shutdown publish failed"`
+        ].join("; "),
+        timeoutMs: 6e4
+      });
+      return res.statusCode === 0;
+    } catch {
+      return false;
+    }
   }
   async lastTermination(_handle) {
     return null;
@@ -7861,6 +7906,13 @@ var SandboxProviderRouter = class {
    * reclaims a stray paused VM anyway, only later.
    */
   strays = /* @__PURE__ */ new Map();
+  /**
+   * The latest ensure per handle. Placement reads what exists, so two at once
+   * each found nothing and put one sandbox on each provider.
+   *
+   * ponytail: in-process; replicas would need the state store's lock.
+   */
+  ensuring = /* @__PURE__ */ new Map();
   freestyleShare;
   get(kind) {
     const provider = this.providers[kind];
@@ -7919,8 +7971,18 @@ var SandboxProviderRouter = class {
     return roomy ? { kind: "kubernetes", reason: "split" } : { kind: "freestyle", reason: "capacity" };
   }
   async ensure(id2, opts = {}) {
-    const { provider: _requested, ...rest } = opts;
     const handle = computeHandle(id2);
+    const run = (this.ensuring.get(handle) ?? Promise.resolve()).catch(() => {
+    }).then(() => this.ensureOne(handle, id2, opts));
+    this.ensuring.set(handle, run);
+    try {
+      return await run;
+    } finally {
+      if (this.ensuring.get(handle) === run) this.ensuring.delete(handle);
+    }
+  }
+  async ensureOne(handle, id2, opts) {
+    const { provider: _requested, ...rest } = opts;
     const placed = await this.place(handle, opts);
     this.remember(handle, placed.kind);
     if (this.strays.get(handle) === placed.kind) this.strays.delete(handle);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@decocms/sandbox-controller",
-  "version": "0.15.4",
+  "version": "0.16.0",
   "description": "Runs Studio agent sandboxes on Kubernetes, and serves them to a remote Studio.",
   "repository": {
     "type": "git",