@knpkv/codecommit-core 0.15.0 → 0.16.0

package/dist/SandboxService/SandboxService.js

@@ -6,7 +6,7 @@
  *
  * @module
  */
-import { Clock, Config, Context, Crypto, Duration, Effect, Layer, Option, Predicate, Random, Ref, Result, Schedule, Stream } from "effect";
+import { Clock, Config, Context, Crypto, Duration, Effect, Fiber, Layer, Option, Predicate, Random, Ref, Result, Schedule, Semaphore, Stream } from "effect";
 import * as FileSystem from "effect/FileSystem";
 import { ChildProcess } from "effect/unstable/process";
 import { SandboxRepo } from "../CacheService/repos/SandboxRepo.js";
@@ -14,7 +14,7 @@ import * as ChildEnv from "../ChildEnv.js";
 import { ConfigService, defaultSandboxConfig, validateSandboxConfig } from "../ConfigService/index.js";
 import { PullRequestId, RepositoryName, SandboxId } from "../Domain.js";
 import { SandboxError } from "../Errors.js";
-import { DockerService, isMissingContainerError } from "./DockerService.js";
+import { DockerService, isAlreadyStoppedContainerError, isMissingContainerError } from "./DockerService.js";
 import { PluginService } from "./PluginService.js";
 import { SandboxWorkerScope } from "./SandboxWorkerScope.js";
 /** Keep the container non-root while matching the bind-mounted workspace owner when available. */
@@ -28,6 +28,12 @@ export const sandboxRuntimeHome = "/tmp";
 export const sandboxRuntimeXdgDataHome = `${sandboxRuntimeHome}/.local/share`;
 const isRegionlessSandbox = (row) => row.region === undefined || row.region === null || row.region === "";
 const isPreContainerSandboxStatus = (status) => status === "creating" || status === "cloning" || status === "starting";
+const isTerminalSandboxStatus = (status) => status === "stopped" || status === "error";
+const isConfirmedStoppedContainer = (state) => state === "exited" || state === "dead";
+const isCompletedLegacyRetirement = (row) => row.accessPassword !== null && row.status === "stopped" && row.legacyRetiredAt !== null;
+const isPendingLegacyRetirement = (row) => row.accessPassword !== null && row.status !== "stopped" && row.legacyRetiredAt !== null;
+const isOrdinaryStoppingSandbox = (row) => row.status === "stopping" && row.legacyRetiredAt === null;
+const isDiscoveredAwsAccountId = (value) => /^\d{12}$/u.test(value);
 const homeDir = Config.string("HOME").pipe(Config.orElse(() => Config.string("USERPROFILE")));
 const sandboxesDir = homeDir.pipe(Config.map((h) => `${h}/.codecommit/sandboxes`));
 const expandHome = (p, home) => p.startsWith("~/") ? `${home}${p.slice(1)}` : p;
@@ -66,14 +72,101 @@ const makeSandboxService = Effect.gen(function* () {
     const cryptoService = yield* Crypto.Crypto;
     const homePath = yield* homeDir.pipe(Effect.orDie);
     const basePath = yield* sandboxesDir.pipe(Effect.orDie);
-    const activeWorkerIds = yield* Ref.make(new Set());
+    const activeWorkerIds = yield* Ref.make(new Map());
+    const lifecycleAdmission = yield* Semaphore.make(1);
+    const containerAdmission = yield* Semaphore.make(1);
+    const createAdmission = yield* Semaphore.make(1);
+    const stopRequestedIds = yield* Ref.make(new Set());
+    const stopOwnerIds = yield* Ref.make(new Set());
+    const retiredLegacyIds = yield* Ref.make(new Set());
+    const markWorkerActive = (id) => Ref.update(activeWorkerIds, (workers) => {
+        const next = new Map(workers);
+        next.set(id, (next.get(id) ?? 0) + 1);
+        return next;
+    });
+    const releaseWorker = (id) => Ref.modify(activeWorkerIds, (workers) => {
+        const next = new Map(workers);
+        const count = next.get(id) ?? 0;
+        if (count <= 1)
+            next.delete(id);
+        else
+            next.set(id, count - 1);
+        return [count <= 1, next];
+    });
+    const hasActiveWorker = (id) => Ref.get(activeWorkerIds).pipe(Effect.map((workers) => (workers.get(id) ?? 0) > 0));
+    const isStopRequested = (id) => Ref.get(stopRequestedIds).pipe(Effect.map((ids) => ids.has(id)));
+    const requestWorkerStops = (ids) => Ref.update(stopRequestedIds, (requested) => new Set([...requested, ...ids]));
+    const claimStopOwners = (ids) => Ref.update(stopOwnerIds, (owners) => new Set([...owners, ...ids]));
+    const isStopOwned = (id) => Ref.get(stopOwnerIds).pipe(Effect.map((ids) => ids.has(id)));
+    const clearWorkerStopRequest = (id) => Ref.update(stopRequestedIds, (ids) => {
+        if (!ids.has(id))
+            return ids;
+        const next = new Set(ids);
+        next.delete(id);
+        return next;
+    });
+    const findFallbackCandidates = (params) => !isDiscoveredAwsAccountId(params.awsAccountId)
+        ? Effect.gen(function* () {
+            const candidates = (yield* repo.findAll()).filter((row) => isDiscoveredAwsAccountId(row.awsAccountId) &&
+                row.awsAccountId !== params.profile &&
+                row.pullRequestId === params.pullRequestId &&
+                row.repositoryName === params.repositoryName &&
+                (row.region === params.region || row.region === null || row.region === undefined || row.region === ""));
+            const terminalCandidates = [];
+            for (const candidate of candidates) {
+                if (!isTerminalSandboxStatus(candidate.status)) {
+                    return [{ row: candidate, blocksCreation: true }];
+                }
+                const containers = yield* docker.listContainersByLabel("codecommit.sandbox.id", candidate.id);
+                if (containers.some((container) => !isConfirmedStoppedContainer(container.State))) {
+                    return [{ row: candidate, blocksCreation: true }];
+                }
+                terminalCandidates.push(candidate);
+            }
+            return terminalCandidates.map((row) => ({ row, blocksCreation: false }));
+        })
+        : Effect.succeed([]);
     const loadSandboxConfig = configService.load.pipe(Effect.map((config) => config.sandbox), Effect.catch(() => Effect.succeed(defaultSandboxConfig)));
     const updateStatus = (id, status, extra) => repo.updateStatus(id, status, extra);
-    const recordCreationFailure = (id, error) => Effect.gen(function* () {
+    const updateWorkerStatus = (id, status, extra) => lifecycleAdmission.withPermits(1)(Effect.gen(function* () {
+        if (yield* isStopRequested(String(id)))
+            return;
+        yield* updateStatus(id, status, extra);
+    }));
+    const releaseWorkerReservation = (id) => lifecycleAdmission.withPermits(1)(Effect.gen(function* () {
+        const lastWorker = yield* releaseWorker(id);
+        if (lastWorker && !(yield* isStopOwned(id))) {
+            if (yield* isStopRequested(id)) {
+                yield* updateStatus(SandboxId.make(id), "stopped").pipe(Effect.catch((error) => Effect.logError(`Failed to finalize stopped sandbox ${id}`, error)));
+            }
+            yield* clearWorkerStopRequest(id);
+        }
+    }));
+    const releaseStopOwner = (id, finalizeUnowned = false) => lifecycleAdmission.withPermits(1)(Effect.gen(function* () {
+        const hadStopRequest = yield* isStopRequested(id);
+        yield* Ref.update(stopOwnerIds, (ids) => {
+            if (!ids.has(id))
+                return ids;
+            const next = new Set(ids);
+            next.delete(id);
+            return next;
+        });
+        if (!(yield* hasActiveWorker(id))) {
+            if (finalizeUnowned && hadStopRequest) {
+                yield* repo.findById(SandboxId.make(id)).pipe(Effect.flatMap((row) => (row.containerId === null || row.containerId.length === 0) && isPreContainerSandboxStatus(row.status)
+                    ? updateStatus(SandboxId.make(id), "stopped")
+                    : Effect.void), Effect.catch((error) => Effect.logError(`Failed to finalize stopped sandbox ${id}`, error)));
+            }
+            yield* clearWorkerStopRequest(id);
+        }
+    }));
+    const recordCreationFailure = (id, error) => lifecycleAdmission.withPermits(1)(Effect.gen(function* () {
+        if (yield* isStopRequested(String(id)))
+            return;
         yield* Effect.logError(`Sandbox ${id} creation failed`, error);
         const errorDetail = Result.try(() => String(Predicate.isError(error) ? error.message : error)).pipe(Result.getOrElse(() => "Unknown error"));
         yield* updateStatus(id, "error", { error: errorDetail.slice(0, 500) }).pipe(Effect.catch((statusError) => Effect.logError("Failed to update sandbox error status", statusError)), Effect.catchDefect((statusDefect) => Effect.logError("Defect while updating sandbox error status", statusDefect)));
-    });
+    }));
     const progress = (id, detail) => Clock.currentTimeMillis.pipe(Effect.flatMap((ms) => {
         const ts = new Date(ms).toISOString();
         return repo.updateDetail(id, detail).pipe(Effect.tap(() => repo.appendLog(id, `[${ts}] ${detail}`)), Effect.tap(() => Effect.logInfo(`Sandbox ${id}: ${detail}`)));
@@ -90,15 +183,90 @@ const makeSandboxService = Effect.gen(function* () {
             sourceBranch: row.sourceBranch
         }
     });
+    const retireLegacySandbox = (legacy) => lifecycleAdmission.withPermits(1)(Effect.gen(function* () {
+        const activeWorker = yield* hasActiveWorker(legacy.id);
+        if (activeWorker) {
+            return yield* new SandboxError({
+                sandboxId: SandboxId.make(legacy.id),
+                message: "Legacy sandbox is still active; retry after its worker exits"
+            });
+        }
+        const discovered = yield* docker.listContainersByLabel("codecommit.sandbox.id", legacy.id);
+        const containerIds = new Set([
+            ...(legacy.containerId === null || legacy.containerId.length === 0 ? [] : [legacy.containerId]),
+            ...discovered.map((container) => container.Id)
+        ]);
+        if (legacy.accessPassword === null) {
+            yield* Effect.forEach(containerIds, (containerId) => docker.stopContainer(containerId).pipe(Effect.catchIf((error) => isMissingContainerError(error) || isAlreadyStoppedContainerError(error), () => Effect.void)), { discard: true });
+            yield* updateStatus(SandboxId.make(legacy.id), "error", {
+                error: "Legacy unauthenticated sandbox stopped; delete and recreate it"
+            });
+            return;
+        }
+        yield* markLegacyRetirementStarted(SandboxId.make(legacy.id));
+        yield* Effect.forEach(containerIds, (containerId) => docker.inspectContainer(containerId).pipe(Effect.flatMap((info) => info.State.Running ? docker.stopContainer(containerId) : Effect.void), Effect.catchIf((error) => isMissingContainerError(error) || isAlreadyStoppedContainerError(error), () => Effect.void)), { discard: true });
+        yield* Clock.currentTimeMillis.pipe(Effect.flatMap((ms) => updateStatus(SandboxId.make(legacy.id), "stopped", { legacyRetiredAt: new Date(ms).toISOString() })));
+        yield* Ref.update(retiredLegacyIds, (ids) => new Set(ids).add(legacy.id));
+    }));
+    const markLegacyRetired = (id) => Clock.currentTimeMillis.pipe(Effect.flatMap((ms) => updateStatus(id, "stopped", { legacyRetiredAt: new Date(ms).toISOString() })));
+    const markLegacyRetirementStarted = (id) => Clock.currentTimeMillis.pipe(Effect.flatMap((ms) => updateStatus(id, "stopping", { legacyRetiredAt: new Date(ms).toISOString() })));
     const service = {
         create: (params) => Effect.gen(function* () {
+            yield* Effect.uninterruptible(createAdmission.take(1));
             // Singleton check — one active sandbox per PR
             const existing = yield* repo.findByPr(params.awsAccountId, params.pullRequestId, params.repositoryName, params.region);
-            const exactExisting = Option.isSome(existing) && existing.value.region === params.region
+            const exactCandidate = Option.isSome(existing) && existing.value.region === params.region
                 ? existing.value
                 : undefined;
-            const regionless = yield* repo.findRegionlessByPrAll(params.awsAccountId, params.pullRequestId, params.repositoryName);
-            const legacyResults = yield* Effect.forEach(regionless, (legacy) => Effect.gen(function* () {
+            const exactExisting = exactCandidate !== undefined &&
+                !isCompletedLegacyRetirement(exactCandidate) &&
+                !isPendingLegacyRetirement(exactCandidate)
+                ? exactCandidate
+                : undefined;
+            const pendingExact = exactCandidate !== undefined && isPendingLegacyRetirement(exactCandidate)
+                ? [exactCandidate]
+                : [];
+            const emptyAccountRows = params.awsAccountId.length === 0
+                ? []
+                : (yield* repo.findAll()).filter((row) => row.awsAccountId === "" &&
+                    row.pullRequestId === params.pullRequestId &&
+                    row.repositoryName === params.repositoryName &&
+                    row.region === params.region);
+            const profileRows = isDiscoveredAwsAccountId(params.awsAccountId) &&
+                !isDiscoveredAwsAccountId(params.profile) &&
+                params.profile !== params.awsAccountId
+                ? (yield* repo.findAll()).filter((row) => row.awsAccountId === params.profile &&
+                    row.pullRequestId === params.pullRequestId &&
+                    row.repositoryName === params.repositoryName &&
+                    row.region === params.region)
+                : [];
+            const effectiveExisting = yield* Effect.forEach([...emptyAccountRows, ...profileRows, ...pendingExact]
+                .filter((row) => !isCompletedLegacyRetirement(row)), retireLegacySandbox, { discard: true }).pipe(Effect.as(exactExisting));
+            const regionlessKeys = Array.from(new Set([
+                params.awsAccountId,
+                isDiscoveredAwsAccountId(params.awsAccountId)
+                    ? params.profile
+                    : undefined
+            ])).filter((key) => key !== undefined && key.length > 0);
+            const regionlessCandidates = (yield* Effect.forEach(regionlessKeys, (key) => repo.findRegionlessByPrAll(key, params.pullRequestId, params.repositoryName), { concurrency: 1 })).flat()
+                .filter((row) => regionlessKeys.includes(row.awsAccountId) && !isCompletedLegacyRetirement(row));
+            const regionless = regionlessCandidates.filter((row, index, candidates) => candidates.findIndex((candidate) => candidate.id === row.id) === index);
+            const ambiguousNumericProfile = isDiscoveredAwsAccountId(params.profile)
+                ? regionless.find((row) => row.awsAccountId === params.profile)
+                : undefined;
+            if (ambiguousNumericProfile !== undefined) {
+                return yield* new SandboxError({
+                    sandboxId: SandboxId.make(ambiguousNumericProfile.id),
+                    message: "Configured numeric profile matches an existing regionless sandbox; account identity is ambiguous"
+                });
+            }
+            const legacyResults = yield* Effect.forEach(regionless, (legacy) => lifecycleAdmission.withPermits(1)(Effect.gen(function* () {
+                if (legacy.accessPassword !== null && (yield* hasActiveWorker(legacy.id))) {
+                    return yield* new SandboxError({
+                        sandboxId: SandboxId.make(legacy.id),
+                        message: "Regionless legacy sandbox is still active; retry after its worker exits"
+                    });
+                }
                 const discovered = yield* docker.listContainersByLabel("codecommit.sandbox.id", legacy.id);
                 const containerIds = new Set([
                     ...(legacy.containerId === null || legacy.containerId.length === 0 ? [] : [legacy.containerId]),
@@ -110,17 +278,20 @@ const makeSandboxService = Effect.gen(function* () {
                         message: "Regionless legacy sandbox is still starting; retry after its worker reports a container"
                     });
                 }
+                if (legacy.accessPassword !== null) {
+                    yield* markLegacyRetirementStarted(SandboxId.make(legacy.id));
+                }
                 if (legacy.accessPassword === null) {
-                    yield* Effect.forEach(containerIds, (containerId) => docker.stopContainer(containerId).pipe(Effect.catchIf(isMissingContainerError, () => Effect.void)), { discard: true });
+                    yield* Effect.forEach(containerIds, (containerId) => docker.stopContainer(containerId).pipe(Effect.catchIf((error) => isMissingContainerError(error) || isAlreadyStoppedContainerError(error), () => Effect.void)), { discard: true });
                     yield* updateStatus(SandboxId.make(legacy.id), "error", {
                         error: "Legacy unauthenticated sandbox stopped; delete and recreate it"
                     });
                     return Option.some(legacy);
                 }
-                yield* Effect.forEach(containerIds, (containerId) => docker.inspectContainer(containerId).pipe(Effect.flatMap((info) => info.State.Running ? docker.stopContainer(containerId) : Effect.void), Effect.catchIf(isMissingContainerError, () => Effect.void)), { discard: true });
-                yield* updateStatus(SandboxId.make(legacy.id), "stopped");
+                yield* Effect.forEach(containerIds, (containerId) => docker.inspectContainer(containerId).pipe(Effect.flatMap((info) => info.State.Running ? docker.stopContainer(containerId) : Effect.void), Effect.catchIf((error) => isMissingContainerError(error) || isAlreadyStoppedContainerError(error), () => Effect.void)), { discard: true });
+                yield* markLegacyRetired(SandboxId.make(legacy.id));
                 return Option.none();
-            }), { concurrency: 1 });
+            })), { concurrency: 1 });
             const unauthenticated = legacyResults.find(Option.isSome);
             if (unauthenticated !== undefined && Option.isSome(unauthenticated)) {
                 return yield* new SandboxError({
@@ -128,17 +299,24 @@ const makeSandboxService = Effect.gen(function* () {
                     message: "Regionless legacy sandbox requires explicit cleanup before recreation"
                 });
             }
-            if (exactExisting !== undefined) {
-                if (exactExisting.containerId === null && isPreContainerSandboxStatus(exactExisting.status)) {
-                    const activeWorker = yield* Ref.get(activeWorkerIds).pipe(Effect.map((ids) => ids.has(exactExisting.id)));
-                    if (activeWorker)
-                        return exactExisting;
-                    yield* updateStatus(SandboxId.make(exactExisting.id), "error", {
+            if (exactCandidate !== undefined && isOrdinaryStoppingSandbox(exactCandidate)) {
+                return yield* new SandboxError({
+                    sandboxId: SandboxId.make(exactCandidate.id),
+                    message: "Sandbox is still stopping; retry after it has stopped"
+                });
+            }
+            if (effectiveExisting !== undefined) {
+                if (effectiveExisting.containerId === null && isPreContainerSandboxStatus(effectiveExisting.status)) {
+                    const activeWorker = yield* hasActiveWorker(effectiveExisting.id);
+                    if (activeWorker) {
+                        return effectiveExisting;
+                    }
+                    yield* updateStatus(SandboxId.make(effectiveExisting.id), "error", {
                         error: "Orphaned (no container)"
                     });
                 }
                 else {
-                    return exactExisting;
+                    return effectiveExisting;
                 }
             }
             const sandboxCfg = yield* loadSandboxConfig;
@@ -150,28 +328,17 @@ const makeSandboxService = Effect.gen(function* () {
             const accessPassword = yield* cryptoService.randomUUIDv4;
             const workspacePath = `${basePath}/${id}`;
             const now = new Date(nowMs).toISOString();
-            yield* repo.insert({
-                id,
-                pullRequestId: params.pullRequestId,
-                awsAccountId: params.awsAccountId,
-                repositoryName: params.repositoryName,
-                region: params.region,
-                sourceBranch: params.sourceBranch,
-                accessPassword,
-                workspacePath,
-                status: "creating",
-                createdAt: now,
-                lastActivityAt: now
-            });
-            // Fork daemon for async lifecycle
-            yield* Ref.update(activeWorkerIds, (ids) => new Set(ids).add(String(id)));
-            yield* ownerScope.fork(Effect.gen(function* () {
+            const workerTransferred = yield* Ref.make(false);
+            const releaseWorkerReservationForCreate = () => releaseWorkerReservation(String(id));
+            const worker = Effect.gen(function* () {
                 const fs = yield* FileSystem.FileSystem;
                 const host = yield* ChildEnv.HostEnvironment;
                 const log = (detail) => progress(id, detail);
+                if (yield* isStopRequested(String(id)))
+                    return;
                 yield* log("Sandbox config validated");
                 // Clone via HTTPS + AWS credential helper
-                yield* updateStatus(id, "cloning");
+                yield* updateWorkerStatus(id, "cloning");
                 yield* fs.makeDirectory(workspacePath, { recursive: true });
                 const cloneUrl = `https://git-codecommit.${params.region}.amazonaws.com/v1/repos/${params.repositoryName}`;
                 const branch = params.sourceBranch.replace(/^refs\/heads\//, "");
@@ -212,12 +379,14 @@ const makeSandboxService = Effect.gen(function* () {
                 if (cloneResult.exitCode !== 0) {
                     const stderrText = cloneResult.stderrText;
                     yield* log(`Clone failed: ${stderrText}`);
-                    yield* updateStatus(id, "error", {
+                    yield* updateWorkerStatus(id, "error", {
                         error: stderrText || `git clone failed (exit ${cloneResult.exitCode})`
                     });
                     return;
                 }
                 yield* log("Clone complete");
+                if (yield* isStopRequested(String(id)))
+                    return;
                 const workspaceInfo = yield* fs.stat(workspacePath);
                 const containerIdentity = sandboxContainerIdentityForWorkspaceOwner(Option.getOrUndefined(workspaceInfo.uid), Option.getOrUndefined(workspaceInfo.gid));
                 if (containerIdentity.repairRootOwnedWorkspace) {
@@ -231,22 +400,40 @@ const makeSandboxService = Effect.gen(function* () {
                     }
                 }
                 // Pull image
-                yield* updateStatus(id, "starting");
+                yield* updateWorkerStatus(id, "starting");
                 yield* log(`Pulling image ${sandboxCfg.image}`);
                 yield* docker.pullImage(sandboxCfg.image).pipe(Effect.tap(() => log("Image ready")), Effect.catchIf(() => true, () => log("Image pull skipped (using cached)")));
-                // Create + start container
-                yield* log("Creating container");
-                yield* validateSandboxConfig(sandboxCfg, homePath);
-                const containerConfig = makeContainerConfig(workspacePath, port, id, params.pullRequestId, sandboxCfg, homePath, containerIdentity.user, accessPassword);
-                const containerId = yield* docker.createContainer(containerConfig);
-                const cid = containerId.trim();
-                yield* log(`Container ${cid.slice(0, 12)} created, starting`);
-                yield* docker.startContainer(cid);
-                yield* updateStatus(id, "starting", { containerId: cid, port });
-                yield* log(`Container started on port ${port}`);
+                // Create + start container. Stop and provisioning share this gate so a
+                // stop request cannot pass while a worker is between create and start.
+                const cid = yield* lifecycleAdmission.withPermits(1)(Effect.gen(function* () {
+                    if (yield* isStopRequested(String(id)))
+                        return undefined;
+                    yield* log("Creating container");
+                    yield* validateSandboxConfig(sandboxCfg, homePath);
+                    const containerConfig = makeContainerConfig(workspacePath, port, id, params.pullRequestId, sandboxCfg, homePath, containerIdentity.user, accessPassword);
+                    const containerId = yield* docker.createContainer(containerConfig);
+                    const nextCid = containerId.trim();
+                    if (yield* isStopRequested(String(id))) {
+                        yield* updateStatus(id, "stopping", { containerId: nextCid, port });
+                        return undefined;
+                    }
+                    yield* log(`Container ${nextCid.slice(0, 12)} created, starting`);
+                    yield* docker.startContainer(nextCid);
+                    yield* updateStatus(id, "starting", { containerId: nextCid, port });
+                    yield* log(`Container started on port ${port}`);
+                    if (yield* isStopRequested(String(id)))
+                        return undefined;
+                    return nextCid;
+                }));
+                if (cid === undefined)
+                    return;
+                if (yield* isStopRequested(String(id)))
+                    return;
                 // Wait for code-server to be ready (poll health)
                 yield* log("Waiting for code-server health check");
                 yield* docker.exec(cid, ["curl", "-sf", "http://localhost:8080/healthz"]).pipe(Effect.retry(Schedule.max([Schedule.recurs(30), Schedule.spaced(Duration.seconds(1))])), Effect.tap(() => log("code-server ready")), Effect.tapError((e) => log(`Health check failed: ${String(e)}`)), Effect.catchIf(() => true, () => Effect.void));
+                if (yield* isStopRequested(String(id)))
+                    return;
                 // Run plugin hooks
                 const row = yield* repo.findById(id);
                 const ctx = makeSandboxContext(row);
@@ -263,36 +450,98 @@ const makeSandboxService = Effect.gen(function* () {
                     yield* log(`Running ${sandboxCfg.setupCommands.length} setup command(s)`);
                     yield* Effect.forEach(sandboxCfg.setupCommands, (cmd, i) => log(`[${i + 1}/${sandboxCfg.setupCommands.length}] ${cmd}`).pipe(Effect.andThen(docker.exec(cid, ["sh", "-c", cmd])), Effect.tap((output) => log(`Command done: ${cmd.slice(0, 60)}${output.length > 0 ? `\n${output.trim()}` : ""}`)), Effect.tapError((e) => log(`Command failed: ${cmd.slice(0, 60)} — ${String(e)}`)), Effect.catchIf(() => true, () => Effect.void)), { discard: true });
                 }
-                yield* updateStatus(id, "running");
+                yield* updateWorkerStatus(id, "running");
                 yield* log("Sandbox ready");
             }).pipe(Effect.catch((error) => recordCreationFailure(id, error)), 
             // Observe and persist unexpected defects without recovering them.
             // `tapDefect` leaves the original Cause / Exit unchanged.
-            Effect.tapDefect((defect) => recordCreationFailure(id, defect)), Effect.ensuring(Ref.update(activeWorkerIds, (ids) => {
-                const next = new Set(ids);
-                next.delete(String(id));
-                return next;
-            }))));
+            Effect.tapDefect((defect) => recordCreationFailure(id, defect)));
+            yield* Effect.acquireUseRelease(lifecycleAdmission.withPermits(1)(Effect.gen(function* () {
+                const fallbackCandidates = yield* findFallbackCandidates(params);
+                const blockingCandidate = fallbackCandidates.find((candidate) => candidate.blocksCreation);
+                if (blockingCandidate !== undefined) {
+                    return yield* new SandboxError({
+                        sandboxId: SandboxId.make(blockingCandidate.row.id),
+                        message: "AWS account identity is unavailable; retry after account discovery"
+                    });
+                }
+                const fallbackStopGuards = fallbackCandidates.map((candidate) => candidate.row.id);
+                yield* claimStopOwners(fallbackStopGuards);
+                yield* requestWorkerStops(fallbackStopGuards);
+                return fallbackStopGuards;
+            })), () => Effect.gen(function* () {
+                yield* markWorkerActive(String(id));
+                yield* repo.insert({
+                    id,
+                    pullRequestId: params.pullRequestId,
+                    awsAccountId: params.awsAccountId,
+                    repositoryName: params.repositoryName,
+                    region: params.region,
+                    sourceBranch: params.sourceBranch,
+                    accessPassword,
+                    workspacePath,
+                    status: "creating",
+                    createdAt: now,
+                    lastActivityAt: now
+                }).pipe(Effect.andThen(Effect.uninterruptible(ownerScope.fork(worker, releaseWorkerReservationForCreate()).pipe(Effect.flatMap(({ fiber, started }) => Effect.race(started.pipe(Effect.as(true)), Fiber.await(fiber).pipe(Effect.as(false)))), Effect.flatMap((started) => started ? Ref.set(workerTransferred, true) : Effect.void)))), Effect.ensuring(Ref.get(workerTransferred).pipe(Effect.flatMap((transferred) => transferred ? Effect.void : releaseWorkerReservationForCreate()))));
+            }), (fallbackStopGuards) => Effect.forEach(fallbackStopGuards, (guardId) => releaseStopOwner(guardId), { discard: true }));
             return yield* repo.findById(id);
-        }).pipe(Effect.mapError((cause) => Predicate.isTagged(cause, "SandboxError")
+        }).pipe(Effect.ensuring(createAdmission.release(1)), Effect.mapError((cause) => Predicate.isTagged(cause, "SandboxError")
             ? cause
             : new SandboxError({ message: "Failed to create sandbox", cause }))),
         get: (id) => repo.findById(id).pipe(Effect.mapError((cause) => new SandboxError({ sandboxId: id, message: "Sandbox not found", cause }))),
         list: () => repo.findActive(),
         listAll: () => repo.findAll(),
-        stop: (id) => Effect.gen(function* () {
-            const row = yield* repo.findById(id);
-            yield* updateStatus(id, "stopping");
-            if (row.containerId !== null) {
-                const ctx = makeSandboxContext(row);
-                yield* plugins.executeHook("onSandboxDestroy", ctx);
-                yield* docker.stopContainer(row.containerId).pipe(Effect.catchIf(() => true, () => Effect.void));
-            }
-            yield* updateStatus(id, "stopped");
-            yield* Effect.logInfo(`Sandbox ${id} stopped`);
+        stop: (id) => Effect.uninterruptibleMask((restore) => {
+            const stopAdmissionAcquired = Effect.gen(function* () {
+                const acquired = yield* Ref.make(false);
+                return acquired;
+            });
+            const stopSandbox = lifecycleAdmission.withPermits(1)(Effect.gen(function* () {
+                const row = yield* repo.findById(id);
+                yield* updateStatus(id, "stopping");
+                if (row.containerId !== null) {
+                    const containerId = row.containerId;
+                    const ctx = makeSandboxContext(row);
+                    yield* plugins.executeHook("onSandboxDestroy", ctx);
+                    yield* containerAdmission.withPermits(1)(Effect.gen(function* () {
+                        const stop = row.legacyRetiredAt === null
+                            ? docker.stopContainer(containerId).pipe(Effect.catchIf(() => true, () => Effect.void))
+                            : docker.stopContainer(containerId).pipe(Effect.catchIf((error) => isMissingContainerError(error) || isAlreadyStoppedContainerError(error), () => Effect.void));
+                        yield* stop;
+                    }));
+                }
+                yield* updateStatus(id, "stopped");
+                yield* Effect.logInfo(`Sandbox ${id} stopped`);
+            }));
+            return Effect.flatMap(stopAdmissionAcquired, (admissionAcquired) => Effect.ensuring(Effect.gen(function* () {
+                // Publish the stop intent before waiting for create admission.
+                // A provisioning worker may hold that permit while it inserts its row.
+                yield* claimStopOwners([String(id)]);
+                yield* requestWorkerStops([String(id)]);
+                yield* restore(createAdmission.withPermits(1)(Effect.uninterruptible(Effect.gen(function* () {
+                    yield* Ref.set(admissionAcquired, true);
+                    yield* stopSandbox;
+                }))));
+            }), Ref.get(admissionAcquired).pipe(Effect.flatMap((acquired) => releaseStopOwner(String(id), !acquired)))));
         }).pipe(Effect.mapError((cause) => new SandboxError({ sandboxId: id, message: "Failed to stop sandbox", cause }))),
-        restart: (id) => Effect.gen(function* () {
+        restart: (id) => Effect.uninterruptibleMask((restore) => Effect.acquireUseRelease(Effect.gen(function* () {
+            yield* lifecycleAdmission.take(1);
+            return {
+                permitReleased: yield* Ref.make(false),
+                workerMarked: yield* Ref.make(false)
+            };
+        }), ({ permitReleased, workerMarked }) => Effect.gen(function* () {
             const row = yield* repo.findById(id);
+            if (yield* isStopRequested(String(id)))
+                return;
+            const wasRetired = yield* Ref.get(retiredLegacyIds).pipe(Effect.map((ids) => ids.has(String(id))));
+            if (wasRetired || row.legacyRetiredAt !== null) {
+                return yield* new SandboxError({
+                    sandboxId: id,
+                    message: "Legacy sandbox was retired; create a replacement"
+                });
+            }
             if (row.accessPassword === null) {
                 return yield* new SandboxError({
                     sandboxId: id,
@@ -302,18 +551,47 @@ const makeSandboxService = Effect.gen(function* () {
             if (row.containerId === null) {
                 return yield* new SandboxError({ sandboxId: id, message: "No container to restart" });
             }
-            yield* updateStatus(id, "starting");
-            yield* progress(id, "Restarting container");
-            yield* docker.startContainer(row.containerId);
-            yield* updateStatus(id, "starting", row.port !== null ? { port: row.port } : {});
-            yield* progress(id, "Waiting for code-server health check");
-            yield* docker.exec(row.containerId, ["curl", "-sf", "http://localhost:8080/healthz"]).pipe(Effect.retry(Schedule.max([Schedule.recurs(30), Schedule.spaced(Duration.seconds(1))])), Effect.tap(() => progress(id, "code-server ready")), Effect.catchIf(() => true, () => Effect.void));
-            const ctx = makeSandboxContext(yield* repo.findById(id));
-            yield* plugins.executeHook("onSandboxReady", ctx);
-            yield* updateStatus(id, "running");
-            yield* progress(id, "Sandbox restarted");
-            yield* Effect.logInfo(`Sandbox ${id} restarted`);
-        }).pipe(Effect.mapError((cause) => new SandboxError({ sandboxId: id, message: "Failed to restart sandbox", cause }))),
+            const containerId = row.containerId;
+            yield* markWorkerActive(String(id));
+            yield* Ref.set(workerMarked, true);
+            yield* lifecycleAdmission.release(1);
+            yield* Ref.set(permitReleased, true);
+            yield* restore(Effect.gen(function* () {
+                if (yield* isStopRequested(String(id)))
+                    return;
+                yield* updateWorkerStatus(id, "starting");
+                if (yield* isStopRequested(String(id)))
+                    return;
+                yield* progress(id, "Restarting container");
+                const started = yield* containerAdmission.withPermits(1)(Effect.gen(function* () {
+                    if (yield* isStopRequested(String(id)))
+                        return false;
+                    yield* docker.startContainer(containerId);
+                    yield* updateStatus(id, "starting", row.port !== null ? { port: row.port } : {});
+                    return true;
+                }));
+                if (!started || (yield* isStopRequested(String(id))))
+                    return;
+                yield* progress(id, "Waiting for code-server health check");
+                yield* docker.exec(containerId, ["curl", "-sf", "http://localhost:8080/healthz"]).pipe(Effect.retry(Schedule.max([Schedule.recurs(30), Schedule.spaced(Duration.seconds(1))])), Effect.tap(() => progress(id, "code-server ready")), Effect.catchIf(() => true, () => Effect.void));
+                if (yield* isStopRequested(String(id)))
+                    return;
+                const ctx = makeSandboxContext(yield* repo.findById(id));
+                yield* plugins.executeHook("onSandboxReady", ctx);
+                if (yield* isStopRequested(String(id)))
+                    return;
+                yield* updateWorkerStatus(id, "running");
+                if (yield* isStopRequested(String(id)))
+                    return;
+                yield* progress(id, "Sandbox restarted");
+                yield* Effect.logInfo(`Sandbox ${id} restarted`);
+            }));
+        }), ({ permitReleased, workerMarked }) => Effect.gen(function* () {
+            if (!(yield* Ref.getAndSet(permitReleased, true)))
+                yield* lifecycleAdmission.release(1);
+            if (yield* Ref.getAndSet(workerMarked, false))
+                yield* releaseWorkerReservation(String(id));
+        }))).pipe(Effect.mapError((cause) => new SandboxError({ sandboxId: id, message: "Failed to restart sandbox", cause }))),
         cleanup: (id) => Effect.gen(function* () {
             const row = yield* repo.findById(id);
             const fs = yield* FileSystem.FileSystem;
@@ -325,10 +603,13 @@ const makeSandboxService = Effect.gen(function* () {
             yield* Effect.logInfo(`Sandbox ${id} cleaned up`);
         }).pipe(Effect.mapError((cause) => new SandboxError({ sandboxId: id, message: "Failed to cleanup sandbox", cause }))),
         reconcile: () => Effect.gen(function* () {
+            yield* Effect.uninterruptible(lifecycleAdmission.take(1));
             const active = yield* repo.findActive();
             const all = yield* repo.findAll();
             const activeIds = new Set(active.map((row) => row.id));
-            const rows = all.filter((row) => row.accessPassword === null || activeIds.has(row.id) || isRegionlessSandbox(row));
+            const rows = all.filter((row) => !isCompletedLegacyRetirement(row) &&
+                (row.accessPassword === null || activeIds.has(row.id) || isRegionlessSandbox(row) ||
+                    (row.legacyRetiredAt !== null && row.status !== "stopped")));
             yield* Effect.forEach(rows, (row) => Effect.gen(function* () {
                 if (isRegionlessSandbox(row)) {
                     const discovered = yield* docker.listContainersByLabel("codecommit.sandbox.id", row.id);
@@ -339,24 +620,38 @@ const makeSandboxService = Effect.gen(function* () {
                     if (row.accessPassword !== null &&
                         containerIds.size === 0 &&
                         isPreContainerSandboxStatus(row.status)) {
-                        const activeWorker = yield* Ref.get(activeWorkerIds).pipe(Effect.map((ids) => ids.has(row.id)));
+                        const activeWorker = yield* hasActiveWorker(row.id);
                         if (activeWorker)
                             return;
                         yield* updateStatus(SandboxId.make(row.id), "error", { error: "Orphaned (no container)" });
                         return;
                     }
                     if (row.accessPassword === null) {
-                        yield* Effect.forEach(containerIds, (containerId) => docker.stopContainer(containerId).pipe(Effect.catchIf(isMissingContainerError, () => Effect.void)), { discard: true });
+                        yield* Effect.forEach(containerIds, (containerId) => docker.stopContainer(containerId).pipe(Effect.catchIf((error) => isMissingContainerError(error) || isAlreadyStoppedContainerError(error), () => Effect.void)), { discard: true });
                         yield* updateStatus(SandboxId.make(row.id), "error", {
                             error: "Legacy unauthenticated sandbox stopped; delete and recreate it"
                         });
                         return;
                     }
-                    yield* Effect.forEach(containerIds, (containerId) => docker.inspectContainer(containerId).pipe(Effect.flatMap((info) => info.State.Running ? docker.stopContainer(containerId) : Effect.void), Effect.catchIf(isMissingContainerError, () => Effect.void)), { discard: true });
-                    yield* updateStatus(SandboxId.make(row.id), "stopped");
+                    if (yield* hasActiveWorker(row.id)) {
+                        return;
+                    }
+                    yield* markLegacyRetirementStarted(SandboxId.make(row.id));
+                    yield* Effect.forEach(containerIds, (containerId) => docker.inspectContainer(containerId).pipe(Effect.flatMap((info) => info.State.Running ? docker.stopContainer(containerId) : Effect.void), Effect.catchIf((error) => isMissingContainerError(error) || isAlreadyStoppedContainerError(error), () => Effect.void)), { discard: true });
+                    yield* markLegacyRetired(SandboxId.make(row.id));
                     yield* Effect.logInfo(`Reconciled regionless sandbox ${row.id}`);
                     return;
                 }
+                if (row.accessPassword !== null && row.legacyRetiredAt !== null) {
+                    const discovered = yield* docker.listContainersByLabel("codecommit.sandbox.id", row.id);
+                    const containerIds = new Set([
+                        ...(row.containerId === null || row.containerId.length === 0 ? [] : [row.containerId]),
+                        ...discovered.map((container) => container.Id)
+                    ]);
+                    yield* Effect.forEach(containerIds, (containerId) => docker.inspectContainer(containerId).pipe(Effect.flatMap((info) => info.State.Running ? docker.stopContainer(containerId) : Effect.void), Effect.catchIf((error) => isMissingContainerError(error) || isAlreadyStoppedContainerError(error), () => Effect.void)), { discard: true });
+                    yield* updateStatus(SandboxId.make(row.id), "stopped", { legacyRetiredAt: row.legacyRetiredAt });
+                    return;
+                }
                 if (row.accessPassword === null) {
                     const discovered = yield* docker.listContainersByLabel("codecommit.sandbox.id", row.id);
                     const containerIds = new Set([
@@ -365,7 +660,7 @@ const makeSandboxService = Effect.gen(function* () {
                     ]);
                     // Do not consider any legacy row reconciled until every persisted
                     // or labeled passwordless container has confirmed shutdown.
-                    yield* Effect.forEach(containerIds, (containerId) => docker.stopContainer(containerId).pipe(Effect.catchIf(isMissingContainerError, () => Effect.void)), { discard: true });
+                    yield* Effect.forEach(containerIds, (containerId) => docker.stopContainer(containerId).pipe(Effect.catchIf((error) => isMissingContainerError(error) || isAlreadyStoppedContainerError(error), () => Effect.void)), { discard: true });
                     yield* updateStatus(SandboxId.make(row.id), "error", {
                         error: "Legacy unauthenticated sandbox stopped; delete and recreate it"
                     });
@@ -373,7 +668,7 @@ const makeSandboxService = Effect.gen(function* () {
                 }
                 if (row.containerId === null) {
                     if (isPreContainerSandboxStatus(row.status)) {
-                        const activeWorker = yield* Ref.get(activeWorkerIds).pipe(Effect.map((ids) => ids.has(row.id)));
+                        const activeWorker = yield* hasActiveWorker(row.id);
                         if (activeWorker)
                             return;
                     }
@@ -386,8 +681,8 @@ const makeSandboxService = Effect.gen(function* () {
                     yield* Effect.logInfo(`Reconciled orphaned sandbox ${row.id}`);
                 }
             }), { discard: true });
-        }).pipe(Effect.as(true), Effect.catch((cause) => Effect.logWarning("Sandbox reconcile failed", cause).pipe(Effect.as(false)))),
-        hasLegacyUnauthenticated: () => repo.findAll().pipe(Effect.map((rows) => rows.some((row) => row.accessPassword === null || isRegionlessSandbox(row)))),
+        }).pipe(Effect.as(true), Effect.catch((cause) => Effect.logWarning("Sandbox reconcile failed", cause).pipe(Effect.as(false))), Effect.ensuring(lifecycleAdmission.release(1))),
+        hasLegacyUnauthenticated: () => repo.findAll().pipe(Effect.map((rows) => rows.some((row) => row.accessPassword === null || (isRegionlessSandbox(row) && !isCompletedLegacyRetirement(row))))),
         gcIdle: (idleTimeout = Duration.minutes(30), cleanupDelay = Duration.hours(24)) => Effect.gen(function* () {
             const all = yield* repo.findAll();
             const now = yield* Clock.currentTimeMillis;