@baochunli/flakes 0.0.4 → 0.0.6

package/dist/node_modules/@flakes/store/dist/managed-sandbox-instances-store.js

@@ -4,7 +4,17 @@ import { HOST_PROVISIONING_QUOTAS, HostProvisioningQuotaError, jsonByteLength, }
 import { addMs, fromJson, toJson } from "./json.js";
 import { mapHost, mapHostCredential, mapHostPool, mapManagedSandboxInstance, mapProvisioningLease, mapSandboxCredential, mapSandboxIdentity, } from "./mappers.js";
 const nonTerminalManagedInstanceStatuses = ["pending_launch", "launching", "running", "draining", "stopping"];
-const cleanupManagedInstanceStatuses = ["stopped", "failed", "expired", "cancelled"];
+const cleanupManagedInstanceStatuses = ["stopped", "failed", "expired", "cancelled", "deleting"];
+const noProviderDeleteManagedInstanceStatuses = [
+    "stopped",
+    "failed",
+    "expired",
+    "cancelled",
+    "running",
+    "draining",
+    "stopping",
+];
+const providerDeleteManagedInstanceStatuses = [...noProviderDeleteManagedInstanceStatuses, "deleting"];
 const nonTerminalProvisioningLeaseStates = [
     "pending",
     "claimed",
@@ -38,6 +48,7 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
                 owner_key: input.ownerKey,
                 name,
                 status: input.status ?? "created",
+                project_workspace_root: null,
                 last_seen_at: null,
                 connection_id: null,
                 stream_generation: 0,
@@ -261,6 +272,7 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
             "host.owner_key as auth_host_owner_key",
             "host.name as auth_host_name",
             "host.status as auth_host_status",
+            "host.project_workspace_root as auth_host_project_workspace_root",
             "host.last_seen_at as auth_host_last_seen_at",
             "host.connection_id as auth_host_connection_id",
             "host.stream_generation as auth_host_stream_generation",
@@ -404,6 +416,7 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
             owner_key: input.ownerKey,
             name: input.hostName,
             status: "created",
+            project_workspace_root: null,
             last_seen_at: null,
             connection_id: null,
             stream_generation: 0,
@@ -930,6 +943,7 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
                 owner_kind: input.ownerKind,
                 owner_id: input.ownerId,
                 owner_key: input.ownerKey,
+                project_id: input.projectId ?? null,
                 host_id: input.hostId,
                 host_pool_id: input.hostPoolId,
                 pool_id: input.hostPoolId,
@@ -1057,6 +1071,261 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
             clearedAssignments: cleared.cleared,
         };
     }
+    /** Deletes or queues cleanup for an owner-scoped stale managed sandbox row. */
+    async deleteManagedSandboxInstance(input) {
+        return this.withTransaction(async (trx) => {
+            const liveness = await this.lockManagedSandboxLiveness(trx, input.sandboxId);
+            const instance = await trx
+                .selectFrom("managed_sandbox_instances as instance")
+                .innerJoin("sandbox_identities as identity", "identity.sandbox_id", "instance.sandbox_id")
+                .select([
+                "instance.instance_id as instance_id",
+                "instance.owner_kind as owner_kind",
+                "instance.owner_id as owner_id",
+                "instance.owner_key as owner_key",
+                "instance.project_id as project_id",
+                "instance.host_id as host_id",
+                "instance.host_pool_id as host_pool_id",
+                "instance.sandbox_id as sandbox_id",
+                "instance.status as status",
+                "instance.provider_resource_id as provider_resource_id",
+                "identity.sandbox_identity_id as sandbox_identity_id",
+                "identity.disabled_at as identity_disabled_at",
+            ])
+                .where("instance.owner_key", "=", input.ownerKey)
+                .where("instance.sandbox_id", "=", input.sandboxId)
+                .where("identity.owner_key", "=", input.ownerKey)
+                .forUpdate()
+                .executeTakeFirst();
+            if (!instance) {
+                return { status: "not_found" };
+            }
+            const blockers = [];
+            const providerResourceId = instance.provider_resource_id;
+            const eligibleStatuses = providerResourceId === null
+                ? noProviderDeleteManagedInstanceStatuses
+                : providerDeleteManagedInstanceStatuses;
+            if (!eligibleStatuses.includes(instance.status)) {
+                blockers.push("unsafe_status");
+            }
+            if (liveness?.status === "active") {
+                blockers.push("live_sandbox");
+            }
+            if (blockers.length > 0) {
+                return {
+                    status: "blocked",
+                    sandboxId: input.sandboxId,
+                    blockers,
+                };
+            }
+            const latestLiveness = liveness ??
+                (await this.lockManagedSandboxLiveness(trx, input.sandboxId));
+            if (latestLiveness?.status === "active") {
+                return {
+                    status: "blocked",
+                    sandboxId: input.sandboxId,
+                    blockers: ["live_sandbox"],
+                };
+            }
+            const hostPoolId = instance.host_pool_id;
+            if (providerResourceId !== null) {
+                if (hostPoolId === null) {
+                    return {
+                        status: "blocked",
+                        sandboxId: input.sandboxId,
+                        blockers: ["provider_cleanup_required"],
+                    };
+                }
+                if (await this.managedSandboxHasAcceptedRunningAssignments(trx, input.sandboxId)) {
+                    return {
+                        status: "blocked",
+                        sandboxId: input.sandboxId,
+                        blockers: ["accepted_running_assignments"],
+                    };
+                }
+                const clearedAssignments = await this.clearReservedSandboxAssignmentsForDelete(trx, input.sandboxId);
+                const now = this.nowIso();
+                const reason = redactReason(input.reason) ?? "managed sandbox deleted";
+                await this.cancelSupersededProviderStopLeasesForDelete(trx, {
+                    ownerKey: instance.owner_key,
+                    hostId: instance.host_id,
+                    instanceId: instance.instance_id,
+                }, now, reason);
+                const cleanupLease = await this.queueManagedSandboxDeleteCleanup(trx, {
+                    instanceId: instance.instance_id,
+                    ownerKind: instance.owner_kind,
+                    ownerId: instance.owner_id,
+                    ownerKey: instance.owner_key,
+                    projectId: instance.project_id,
+                    hostId: instance.host_id,
+                    hostPoolId,
+                    sandboxId: instance.sandbox_id,
+                    providerResourceId,
+                }, now);
+                await trx
+                    .updateTable("managed_sandbox_instances")
+                    .set({
+                    status: "deleting",
+                    last_error: reason,
+                    updated_at: now,
+                })
+                    .where("instance_id", "=", instance.instance_id)
+                    .where("owner_key", "=", input.ownerKey)
+                    .where("sandbox_id", "=", input.sandboxId)
+                    .where("provider_resource_id", "=", providerResourceId)
+                    .where("status", "in", [...providerDeleteManagedInstanceStatuses])
+                    .executeTakeFirst();
+                await this.invalidateManagedSandboxRuntimeAuthForDelete(trx, {
+                    sandboxIdentityId: instance.sandbox_identity_id,
+                    identityDisabledAt: instance.identity_disabled_at,
+                    reason,
+                    now,
+                });
+                return {
+                    status: "cleanup_queued",
+                    sandboxId: input.sandboxId,
+                    instanceId: instance.instance_id,
+                    leaseId: cleanupLease.leaseId,
+                    providerResourceId,
+                    clearedAssignments,
+                };
+            }
+            if (await this.managedSandboxHasAcceptedRunningAssignments(trx, input.sandboxId)) {
+                return {
+                    status: "blocked",
+                    sandboxId: input.sandboxId,
+                    blockers: ["accepted_running_assignments"],
+                };
+            }
+            const clearedAssignments = await this.clearReservedSandboxAssignmentsForDelete(trx, input.sandboxId);
+            const now = this.nowIso();
+            const reason = redactReason(input.reason) ?? "managed sandbox deleted";
+            await this.invalidateManagedSandboxRuntimeAuthForDelete(trx, {
+                sandboxIdentityId: instance.sandbox_identity_id,
+                identityDisabledAt: instance.identity_disabled_at,
+                reason,
+                now,
+            });
+            await trx
+                .deleteFrom("managed_sandbox_instances")
+                .where("instance_id", "=", instance.instance_id)
+                .where("owner_key", "=", input.ownerKey)
+                .where("sandbox_id", "=", input.sandboxId)
+                .executeTakeFirst();
+            return {
+                status: "deleted",
+                sandboxId: input.sandboxId,
+                instanceId: instance.instance_id,
+                clearedAssignments,
+            };
+        });
+    }
+    async invalidateManagedSandboxRuntimeAuthForDelete(executor, input) {
+        await executor
+            .updateTable("sandbox_credentials")
+            .set({
+            revoked_at: input.now,
+            revoked_reason: input.reason,
+            updated_at: input.now,
+        })
+            .where("sandbox_identity_id", "=", input.sandboxIdentityId)
+            .where("revoked_at", "is", null)
+            .execute();
+        await executor
+            .updateTable("sandbox_identities")
+            .set({
+            status: "disabled",
+            disabled_at: input.identityDisabledAt ?? input.now,
+            updated_at: input.now,
+        })
+            .where("sandbox_identity_id", "=", input.sandboxIdentityId)
+            .execute();
+    }
+    async queueManagedSandboxDeleteCleanup(executor, instance, now) {
+        const existing = await executor
+            .selectFrom("provisioning_leases")
+            .selectAll()
+            .where("owner_key", "=", instance.ownerKey)
+            .where("host_id", "=", instance.hostId)
+            .where("instance_id", "=", instance.instanceId)
+            .where("action", "=", "provider_cleanup")
+            .where("state", "in", [...nonTerminalProvisioningLeaseStates])
+            .forUpdate()
+            .executeTakeFirst();
+        if (existing) {
+            if (existing.state === "pending" && existing.lease_deadline_at <= now) {
+                const refreshed = await executor
+                    .updateTable("provisioning_leases")
+                    .set({
+                    lease_deadline_at: addMs(this.clock.now(), providerCleanupLeaseTtlMs),
+                    last_error: null,
+                    updated_at: now,
+                })
+                    .where("lease_id", "=", existing.lease_id)
+                    .where("owner_key", "=", instance.ownerKey)
+                    .where("host_id", "=", instance.hostId)
+                    .where("action", "=", "provider_cleanup")
+                    .where("state", "=", "pending")
+                    .returningAll()
+                    .executeTakeFirst();
+                return mapProvisioningLease(refreshed ?? existing);
+            }
+            return mapProvisioningLease(existing);
+        }
+        await this.assertProvisioningLeaseQuota(executor, instance.ownerKey);
+        const row = await executor
+            .insertInto("provisioning_leases")
+            .values({
+            lease_id: this.idGenerator.createId("provisioning_lease"),
+            owner_kind: instance.ownerKind,
+            owner_id: instance.ownerId,
+            owner_key: instance.ownerKey,
+            project_id: instance.projectId,
+            host_project_binding_id: null,
+            host_id: instance.hostId,
+            host_pool_id: instance.hostPoolId,
+            instance_id: instance.instanceId,
+            sandbox_id: instance.sandboxId,
+            demand_task_id: null,
+            action: "provider_cleanup",
+            state: "pending",
+            claim_id: null,
+            attempt: 0,
+            max_attempts: providerCleanupLeaseMaxAttempts,
+            lease_deadline_at: addMs(this.clock.now(), providerCleanupLeaseTtlMs),
+            claim_deadline_at: null,
+            hello_deadline_at: null,
+            accepted_at: null,
+            running_at: null,
+            completed_at: null,
+            credential_ref: null,
+            provider_resource_id: instance.providerResourceId,
+            provider_result_json: null,
+            last_error: null,
+            release_audit_json: null,
+            created_at: now,
+            updated_at: now,
+        })
+            .returningAll()
+            .executeTakeFirstOrThrow();
+        return mapProvisioningLease(row);
+    }
+    async cancelSupersededProviderStopLeasesForDelete(executor, input, now, reason) {
+        await executor
+            .updateTable("provisioning_leases")
+            .set({
+            state: "cancelled",
+            completed_at: now,
+            last_error: reason,
+            updated_at: now,
+        })
+            .where("owner_key", "=", input.ownerKey)
+            .where("host_id", "=", input.hostId)
+            .where("instance_id", "=", input.instanceId)
+            .where("action", "=", "provider_stop")
+            .where("state", "in", [...nonTerminalProvisioningLeaseStates])
+            .execute();
+    }
     /** Returns one owner/host-scoped provisioning lease. */
     async getProvisioningLease(input) {
         const row = await this.db
@@ -1111,6 +1380,25 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
             if (input.maxAttempts <= 0) {
                 throw new Error("lease maxAttempts must be positive");
             }
+            const projectId = input.projectId ?? instance.project_id;
+            if (projectId !== instance.project_id) {
+                throw new Error("launch lease project mismatch");
+            }
+            if (projectId !== null && input.hostProjectBindingId === undefined) {
+                throw new Error("host project binding is required for project launch");
+            }
+            if (projectId === null && input.hostProjectBindingId !== undefined) {
+                throw new Error("host project binding requires project launch");
+            }
+            if (projectId !== null && input.hostProjectBindingId !== undefined) {
+                await this.requireActiveHostProjectBindingForPool(trx, {
+                    ownerKey: input.ownerKey,
+                    hostId: input.hostId,
+                    projectId,
+                    hostProjectBindingId: input.hostProjectBindingId,
+                    hostPoolId: input.hostPoolId,
+                });
+            }
             await this.assertProvisioningLeaseQuota(trx, input.ownerKey);
             const now = this.nowIso();
             const leaseId = input.leaseId ?? this.idGenerator.createId("provisioning_lease");
@@ -1122,6 +1410,8 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
                     owner_kind: host.owner_kind,
                     owner_id: host.owner_id,
                     owner_key: host.owner_key,
+                    project_id: projectId,
+                    host_project_binding_id: input.hostProjectBindingId ?? null,
                     host_id: input.hostId,
                     host_pool_id: input.hostPoolId,
                     instance_id: input.instanceId,
@@ -1229,6 +1519,8 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
                     owner_kind: host.owner_kind,
                     owner_id: host.owner_id,
                     owner_key: host.owner_key,
+                    project_id: instance.project_id,
+                    host_project_binding_id: null,
                     host_id: input.hostId,
                     host_pool_id: instance.host_pool_id,
                     instance_id: input.instanceId,
@@ -1335,6 +1627,8 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
                     owner_kind: host.owner_kind,
                     owner_id: host.owner_id,
                     owner_key: host.owner_key,
+                    project_id: instance.project_id,
+                    host_project_binding_id: null,
                     host_id: input.hostId,
                     host_pool_id: instance.host_pool_id,
                     instance_id: input.instanceId,
@@ -1462,6 +1756,43 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
                     !hasAllCapabilities(poolCapabilities, requiredCapabilities)) {
                     return { status: "skipped", reason: "pool_mismatch" };
                 }
+                const taskProjectId = task.project_id;
+                if (taskProjectId === null) {
+                    return { status: "skipped", reason: "pool_mismatch" };
+                }
+                if (input.projectId !== undefined &&
+                    input.projectId !== taskProjectId) {
+                    return { status: "skipped", reason: "pool_mismatch" };
+                }
+                if (taskProjectId !== null) {
+                    if (input.hostProjectBindingId === undefined) {
+                        return { status: "skipped", reason: "pool_mismatch" };
+                    }
+                    const binding = await trx
+                        .selectFrom("host_project_bindings as binding")
+                        .innerJoin("projects as project", (join) => join
+                        .onRef("project.project_id", "=", "binding.project_id")
+                        .onRef("project.owner_key", "=", "binding.owner_key"))
+                        .select([
+                        "binding.allowed_pool_keys_json",
+                        "binding.availability",
+                        "project.status as project_status",
+                    ])
+                        .where("binding.binding_id", "=", input.hostProjectBindingId)
+                        .where("binding.owner_key", "=", input.ownerKey)
+                        .where("binding.host_id", "=", input.hostId)
+                        .where("binding.project_id", "=", taskProjectId)
+                        .forUpdate()
+                        .executeTakeFirst();
+                    const allowedPoolKeys = binding
+                        ? (fromJson(binding.allowed_pool_keys_json) ?? [])
+                        : [];
+                    if (binding?.project_status !== "active" ||
+                        binding.availability !== "available" ||
+                        !allowedPoolKeys.includes(pool.pool_key)) {
+                        return { status: "skipped", reason: "pool_mismatch" };
+                    }
+                }
                 const activeInstances = await trx
                     .selectFrom("managed_sandbox_instances")
                     .select(sql `count(*)::text`.as("count"))
@@ -1500,6 +1831,7 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
                     owner_kind: input.ownerKind,
                     owner_id: input.ownerId,
                     owner_key: input.ownerKey,
+                    project_id: task.project_id,
                     host_id: input.hostId,
                     host_pool_id: input.hostPoolId,
                     pool_id: input.hostPoolId,
@@ -1523,6 +1855,8 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
                     owner_kind: host.owner_kind,
                     owner_id: host.owner_id,
                     owner_key: host.owner_key,
+                    project_id: task.project_id,
+                    host_project_binding_id: input.hostProjectBindingId ?? null,
                     host_id: input.hostId,
                     host_pool_id: input.hostPoolId,
                     instance_id: instanceRow.instance_id,
@@ -1606,15 +1940,30 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
                 "available") {
                 return [];
             }
-            const now = this.nowIso();
+            const nowDate = this.clock.now();
+            const now = nowDate.toISOString();
             const candidates = await trx
                 .selectFrom("provisioning_leases as lease")
                 .innerJoin("host_pools as pool", "pool.host_pool_id", "lease.host_pool_id")
                 .innerJoin("managed_sandbox_instances as instance", "instance.instance_id", "lease.instance_id")
                 .innerJoin("sandbox_identities as identity", "identity.sandbox_id", "lease.sandbox_id")
-                .select(["lease.lease_id"])
+                .innerJoin("projects as project", (join) => join
+                .onRef("project.project_id", "=", "lease.project_id")
+                .onRef("project.owner_key", "=", "lease.owner_key"))
+                .innerJoin("host_project_bindings as binding", (join) => join
+                .onRef("binding.binding_id", "=", "lease.host_project_binding_id")
+                .onRef("binding.project_id", "=", "lease.project_id")
+                .onRef("binding.owner_key", "=", "lease.owner_key")
+                .onRef("binding.host_id", "=", "lease.host_id"))
+                .select([
+                "lease.lease_id",
+                "binding.binding_id as host_project_binding_id",
+                "project.project_key",
+            ])
                 .where("lease.owner_key", "=", input.ownerKey)
                 .where("lease.host_id", "=", input.hostId)
+                .where("lease.project_id", "is not", null)
+                .whereRef("instance.project_id", "=", "lease.project_id")
                 .where("lease.action", "=", "launch")
                 .where("lease.state", "=", "pending")
                 .where("lease.lease_deadline_at", ">", now)
@@ -1622,6 +1971,13 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
                 .where("pool.owner_key", "=", input.ownerKey)
                 .where("pool.host_id", "=", input.hostId)
                 .where("pool.status", "=", "available")
+                .where("project.status", "=", "active")
+                .where("binding.availability", "=", "available")
+                .where(sql `exists (
+            select 1
+              from jsonb_array_elements_text(binding.allowed_pool_keys_json::jsonb) as allowed(pool_key)
+             where allowed.pool_key = pool.pool_key
+          )`)
                 .where("instance.owner_key", "=", input.ownerKey)
                 .whereRef("instance.host_id", "=", "lease.host_id")
                 .whereRef("instance.host_pool_id", "=", "lease.host_pool_id")
@@ -1652,7 +2008,12 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
                     hostStreamFence: input.hostStreamFence,
                 });
                 if (result?.credential) {
-                    claimed.push(result);
+                    claimed.push({
+                        lease: result.lease,
+                        credential: result.credential,
+                        hostProjectBindingId: candidate.host_project_binding_id,
+                        projectKey: candidate.project_key,
+                    });
                 }
             }
             return claimed;
@@ -1680,7 +2041,8 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
                 action: "provider_stop",
                 limit,
             });
-            const now = this.nowIso();
+            const nowDate = this.clock.now();
+            const now = nowDate.toISOString();
             const candidates = await trx
                 .selectFrom("provisioning_leases as lease")
                 .innerJoin("host_pools as pool", "pool.host_pool_id", "lease.host_pool_id")
@@ -1746,17 +2108,26 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
                 action: "provider_cleanup",
                 limit,
             });
-            const now = this.nowIso();
+            const nowDate = this.clock.now();
+            const now = nowDate.toISOString();
             const candidates = await trx
                 .selectFrom("provisioning_leases as lease")
                 .innerJoin("host_pools as pool", "pool.host_pool_id", "lease.host_pool_id")
                 .innerJoin("managed_sandbox_instances as instance", "instance.instance_id", "lease.instance_id")
-                .select(["lease.lease_id", "pool.provider"])
+                .select([
+                "lease.lease_id",
+                "lease.lease_deadline_at",
+                "pool.provider",
+                "instance.status as instance_status",
+            ])
                 .where("lease.owner_key", "=", input.ownerKey)
                 .where("lease.host_id", "=", input.hostId)
                 .where("lease.action", "=", "provider_cleanup")
                 .where("lease.state", "=", "pending")
-                .where("lease.lease_deadline_at", ">", now)
+                .where((eb) => eb.or([
+                eb("lease.lease_deadline_at", ">", now),
+                eb("instance.status", "=", "deleting"),
+            ]))
                 .where("lease.provider_resource_id", "is not", null)
                 .where("pool.owner_key", "=", input.ownerKey)
                 .where("pool.host_id", "=", input.hostId)
@@ -1773,6 +2144,25 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
                 .execute();
             const claimed = [];
             for (const candidate of candidates) {
+                if (candidate.instance_status === "deleting" &&
+                    candidate.lease_deadline_at <= now) {
+                    const refreshed = await trx
+                        .updateTable("provisioning_leases")
+                        .set({
+                        lease_deadline_at: addMs(nowDate, providerCleanupLeaseTtlMs),
+                        last_error: null,
+                        updated_at: now,
+                    })
+                        .where("lease_id", "=", candidate.lease_id)
+                        .where("owner_key", "=", input.ownerKey)
+                        .where("host_id", "=", input.hostId)
+                        .where("action", "=", "provider_cleanup")
+                        .where("state", "=", "pending")
+                        .executeTakeFirst();
+                    if (refreshed.numUpdatedRows === 0n) {
+                        continue;
+                    }
+                }
                 const result = await this.claimLeaseInTransaction(trx, {
                     ownerKey: input.ownerKey,
                     hostId: input.hostId,
@@ -2147,6 +2537,8 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
                         owner_kind: current.owner_kind,
                         owner_id: current.owner_id,
                         owner_key: current.owner_key,
+                        project_id: current.project_id,
+                        host_project_binding_id: current.host_project_binding_id,
                         host_id: current.host_id,
                         host_pool_id: current.host_pool_id,
                         instance_id: current.instance_id,
@@ -2263,6 +2655,20 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
             if (!instanceRow) {
                 return { status: "rejected" };
             }
+            const deletingIdentity = input.status === "succeeded" && instanceRow.status === "deleting"
+                ? await trx
+                    .selectFrom("sandbox_identities")
+                    .select(["sandbox_identity_id", "disabled_at"])
+                    .where("owner_key", "=", current.owner_key)
+                    .where("sandbox_id", "=", current.sandbox_id ?? "")
+                    .forUpdate()
+                    .executeTakeFirst()
+                : undefined;
+            if (input.status === "succeeded" && instanceRow.status === "deleting") {
+                if (!deletingIdentity) {
+                    return { status: "rejected" };
+                }
+            }
             const row = await trx
                 .updateTable("provisioning_leases")
                 .set({
@@ -2288,6 +2694,22 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
             if (!row) {
                 throw new Error("failed to record provider cleanup lease result");
             }
+            if (deletingIdentity) {
+                await this.invalidateManagedSandboxRuntimeAuthForDelete(trx, {
+                    sandboxIdentityId: deletingIdentity.sandbox_identity_id,
+                    identityDisabledAt: deletingIdentity.disabled_at,
+                    reason: redactReason(input.reason) ?? "managed sandbox deleted",
+                    now,
+                });
+                await trx
+                    .deleteFrom("managed_sandbox_instances")
+                    .where("instance_id", "=", current.instance_id)
+                    .where("owner_key", "=", current.owner_key)
+                    .where("host_id", "=", current.host_id)
+                    .where("status", "=", "deleting")
+                    .where("provider_resource_id", "is", null)
+                    .executeTakeFirst();
+            }
             return {
                 status: "recorded",
                 lease: mapProvisioningLease(row),
@@ -2477,7 +2899,7 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
         if (current.action === "launch" && current.host_pool_id !== null) {
             const pool = await trx
                 .selectFrom("host_pools")
-                .select(["status"])
+                .select(["pool_key", "status"])
                 .where("host_pool_id", "=", current.host_pool_id)
                 .where("owner_key", "=", current.owner_key)
                 .where("host_id", "=", current.host_id)
@@ -2486,6 +2908,35 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
             if (pool?.status !== "available") {
                 return undefined;
             }
+            if (current.project_id !== null) {
+                if (current.host_project_binding_id === null) {
+                    return undefined;
+                }
+                const binding = await trx
+                    .selectFrom("host_project_bindings as binding")
+                    .innerJoin("projects as project", (join) => join
+                    .onRef("project.project_id", "=", "binding.project_id")
+                    .onRef("project.owner_key", "=", "binding.owner_key"))
+                    .select([
+                    "binding.allowed_pool_keys_json",
+                    "binding.availability",
+                    "project.status as project_status",
+                ])
+                    .where("binding.binding_id", "=", current.host_project_binding_id)
+                    .where("binding.owner_key", "=", current.owner_key)
+                    .where("binding.host_id", "=", current.host_id)
+                    .where("binding.project_id", "=", current.project_id)
+                    .forUpdate()
+                    .executeTakeFirst();
+                const allowedPoolKeys = binding
+                    ? (fromJson(binding.allowed_pool_keys_json) ?? [])
+                    : [];
+                if (binding?.project_status !== "active" ||
+                    binding.availability !== "available" ||
+                    !allowedPoolKeys.includes(pool.pool_key)) {
+                    return undefined;
+                }
+            }
         }
         const nowDate = this.clock.now();
         const now = nowDate.toISOString();
@@ -2538,6 +2989,7 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
                 .where("pool.status", "=", "available")
                 .where("instance.owner_key", "=", current.owner_key)
                 .where("instance.host_id", "=", current.host_id)
+                .where(sql `instance.project_id is not distinct from lease.project_id`)
                 .where("instance.host_pool_id", "=", current.host_pool_id)
                 .where("instance.sandbox_id", "=", current.sandbox_id)
                 .where("instance.launch_lease_id", "=", current.lease_id)
@@ -2709,8 +3161,84 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
         if (nonterminalManagedInstances.count > 0) {
             blockers.push("nonterminal_managed_instances");
         }
+        const retryableDeletingProviderInstances = await executor
+            .selectFrom("managed_sandbox_instances")
+            .select(sql `count(*)::int`.as("count"))
+            .where("owner_key", "=", ownerKey)
+            .where("host_id", "=", hostId)
+            .where("status", "=", "deleting")
+            .where("provider_resource_id", "is not", null)
+            .executeTakeFirstOrThrow();
+        if (retryableDeletingProviderInstances.count > 0 &&
+            !blockers.includes("nonterminal_managed_instances")) {
+            blockers.push("nonterminal_managed_instances");
+        }
         return blockers;
     }
+    async lockManagedSandboxLiveness(executor, sandboxId) {
+        return executor
+            .selectFrom("sandboxes")
+            .select(["sandbox_id", "status"])
+            .where("sandbox_id", "=", sandboxId)
+            .forUpdate()
+            .executeTakeFirst();
+    }
+    async managedSandboxHasAcceptedRunningAssignments(executor, sandboxId) {
+        const row = await executor
+            .selectFrom("tasks")
+            .select("task_id")
+            .where("status", "=", "RUNNING")
+            .where("assigned_sandbox_id", "=", sandboxId)
+            .where("assignment_accepted_at", "is not", null)
+            .where("current_assignment_id", "is not", null)
+            .forUpdate()
+            .executeTakeFirst();
+        return row !== undefined;
+    }
+    async clearReservedSandboxAssignmentsForDelete(executor, sandboxId) {
+        const tasks = await executor
+            .selectFrom("tasks")
+            .select(["task_id", "run_id"])
+            .where("status", "=", "RUNNING")
+            .where("assigned_sandbox_id", "=", sandboxId)
+            .where("assignment_accepted_at", "is", null)
+            .where("current_assignment_id", "is not", null)
+            .forUpdate()
+            .execute();
+        if (tasks.length === 0) {
+            return 0;
+        }
+        const now = this.nowIso();
+        const taskIds = tasks.map((task) => task.task_id);
+        const cleared = await executor
+            .updateTable("tasks")
+            .set({
+            status: "QUEUED",
+            current_attempt_number: null,
+            current_assignment_id: null,
+            assigned_sandbox_id: null,
+            assignment_created_at: null,
+            assignment_sent_at: null,
+            assignment_delivery_deadline_at: null,
+            assignment_accepted_at: null,
+            assignment_expires_at: null,
+            assignment_heartbeat_at: null,
+            assignment_delivery_attempt_count: 0,
+            assignment_last_delivery_error: "managed_sandbox_deleted",
+            not_before_at: null,
+            updated_at: now,
+            queued_at: now,
+        })
+            .where("task_id", "in", taskIds)
+            .where("status", "=", "RUNNING")
+            .where("assigned_sandbox_id", "=", sandboxId)
+            .where("assignment_accepted_at", "is", null)
+            .executeTakeFirst();
+        for (const runId of new Set(tasks.map((task) => task.run_id))) {
+            await this.updateRunState(executor, runId);
+        }
+        return Number(cleared.numUpdatedRows);
+    }
     async requireHostPoolOwner(executor, hostPoolId, hostId, ownerKey) {
         const pool = await executor
             .selectFrom("host_pools")
@@ -2725,6 +3253,35 @@ export class ManagedSandboxInstancesStore extends ArtifactsEventsStore {
         }
         return pool;
     }
+    async requireActiveHostProjectBindingForPool(executor, input) {
+        const binding = await executor
+            .selectFrom("host_project_bindings as binding")
+            .innerJoin("projects as project", (join) => join
+            .onRef("project.project_id", "=", "binding.project_id")
+            .onRef("project.owner_key", "=", "binding.owner_key"))
+            .innerJoin("host_pools as pool", (join) => join
+            .onRef("pool.owner_key", "=", "binding.owner_key")
+            .onRef("pool.host_id", "=", "binding.host_id"))
+            .select([
+            "binding.allowed_pool_keys_json",
+            "project.status as project_status",
+            "pool.pool_key",
+        ])
+            .where("binding.binding_id", "=", input.hostProjectBindingId)
+            .where("binding.owner_key", "=", input.ownerKey)
+            .where("binding.host_id", "=", input.hostId)
+            .where("binding.project_id", "=", input.projectId)
+            .where("pool.host_pool_id", "=", input.hostPoolId)
+            .executeTakeFirst();
+        if (!binding) {
+            throw new Error("host project binding not found");
+        }
+        const allowedPoolKeys = fromJson(binding.allowed_pool_keys_json) ?? [];
+        if (binding.project_status !== "active" ||
+            !allowedPoolKeys.includes(binding.pool_key)) {
+            throw new Error("host project binding is not valid for project launch");
+        }
+    }
     async requireSandboxIdentityOwner(executor, sandboxId, ownerKey) {
         const sandbox = await executor
             .selectFrom("sandbox_identities")
@@ -3105,6 +3662,7 @@ function mapHostCredentialAuth(row) {
             owner_key: row.auth_host_owner_key,
             name: row.auth_host_name,
             status: row.auth_host_status,
+            project_workspace_root: row.auth_host_project_workspace_root,
             last_seen_at: row.auth_host_last_seen_at,
             connection_id: row.auth_host_connection_id,
             stream_generation: row.auth_host_stream_generation,