@substrat-run/adapter-cloudflare 0.148.0 → 0.149.0

package/dist/host.js

@@ -1,9 +1,10 @@
 import { env as ambientEnv } from 'cloudflare:workers';
 import { isRewindRefusal, REWIND_REFUSED } from './rewind-refusal.js';
 import { isSystemDoorMoved, SYSTEM_DOOR_MOVED, SYSTEM_DOOR_REGATES } from './system-door.js';
-import { delegatedReadParams, operationSeriesCount, fromWireFailure, isSubstratError, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, assertNoBindingShapeBreak, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, findingEntry, findingRuleEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, mintedCapability, boundedBecomeMint, boundedBecomeRevoke, boundedUnbind, entityRef, becomeLinkState, principalBecomeCapabilityInput, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, scopeStorageSample, subjectRef, createConnectionInput, connectLinkFilter, connectLinkKey, consumeConnectLinkInput, mintConnectLinkInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, memberChangeAudit, peerBindingAudit, connectionBindingAudit, copyMarkAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, SCOPE_GATE_REASONS, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, peerInstanceResolution, peerBinding as peerBindingSchema, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
+import { delegatedReadParams, operationSeriesCount, fromWireFailure, isSubstratError, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, assertNoBindingShapeBreak, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, findingEntry, findingRuleEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, mintedCapability, boundedBecomeMint, boundedBecomeRevoke, boundedUnbind, entityRef, becomeLinkState, principalBecomeCapabilityInput, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, scopeStorageSample, subjectRef, createConnectionInput, connectLinkFilter, connectLinkKey, consumeConnectLinkInput, mintConnectLinkInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, memberChangeAudit, peerBindingAudit, connectionBindingAudit, copyMarkAudit, orphanSweepAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, SCOPE_GATE_REASONS, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, peerInstanceResolution, peerBinding as peerBindingSchema, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
 import { normalizeHostname, toRouteTarget } from './route-resolver.js';
-import { attachmentBlobKey, UPLOAD_LINK_WRONG_TARGET, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, assertBecomeLinkStateIds, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, executorOutcomeOf, isDeliveryRefusal, refusalJournalText, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, assertLeaseMs, runDueJobRuns, SYSTEM_DOOR_WAIT, startJobRun, attachmentTextJob, assertAttachmentExtractors, resolveAttachmentTextBounds, assertJobRegistrable, ATTACHMENT_TEXT_BACKFILL_JOB, ATTACHMENT_TEXT_JOB, kernelJobFor, attachmentTextBackfillJob, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, recordedOffFromWire, peerSubjectRef, SWITCH_KINDS, reassertActionOf, reassertEntry, reassertOffRow, reassertOnRow, switchActionOf, switchAuditSubject, recordAuditOutcome, auditedCapabilityRevoke, auditWarningOf, switchNotFoundMessage, recordWriteSuperseded, switchSubjectOf, switchSupersededMessage, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, scopeMoveOf, runningVersionOf, strandJudgmentsOf, boundShapesOfColumn, installReconciled, collectPeers, peerCallerAt, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, emptyImportResult, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, assertNoCallerPurge, purgeReportOf, purgeStillDue, assertBoundOperations, registerTrashTargets, memberAddedAudit, shapeTopUpBatch, } from '@substrat-run/kernel';
+import { attachmentBlobKey, normalizeUploadMediaType, redeemOrReplayUploadLink, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, assertBecomeLinkStateIds, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, executorOutcomeOf, isDeliveryRefusal, refusalJournalText, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, assertLeaseMs, runDueJobRuns, SYSTEM_DOOR_WAIT, startJobRun, attachmentTextJob, assertAttachmentExtractors, resolveAttachmentTextBounds, assertJobRegistrable, ATTACHMENT_TEXT_BACKFILL_JOB, ATTACHMENT_TEXT_JOB, kernelJobFor, attachmentTextBackfillJob, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, recordedOffFromWire, peerSubjectRef, SWITCH_KINDS, reassertActionOf, reassertEntry, reassertOffRow, reassertOnRow, switchActionOf, switchAuditSubject, recordAuditOutcome, auditedCapabilityRevoke, auditWarningOf, switchNotFoundMessage, recordWriteSuperseded, switchSubjectOf, switchSupersededMessage, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, assertPromoteReviewCurrent, promoteRaceRefusal, rebindRaceRefusal, bindExportBreaksOf, bindExportBreakRefusal, scopeMoveOf, runningVersionOf, strandJudgmentsOf, boundShapesOfColumn, installReconciled, collectPeers, peerCallerAt, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, publishedMigrationDigest, migrationChangeRefusal, unknownRoleError, assertRowLimit, assertRowOffset, emptyImportResult, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, assertNoCallerPurge, purgeReportOf, purgeStillDue, assertBoundOperations, registerTrashTargets, memberAddedAudit, shapeTopUpBatch, } from '@substrat-run/kernel';
+import { fenceAttachmentSweep, missingAttachmentStore, sweepOrphanAttachments, unsettledCopyRefusal } from '@substrat-run/kernel';
 import { COPY_RESTORE_FENCE_LAPSED, scopeScriptCopyOf } from '@substrat-run/kernel';
 import { attributedView, isModuleErasureCounts, moduleErasurePlan, moduleRowsErased, callReservedStoreOperation, callScopeStore, observeStoreCall, replayStoreErasures, scopeStoreLifecycleOf, scopeStoreRunnersOf } from '@substrat-run/kernel';
 import { PLATFORM_BINDING_PREFIX, scopeStoreBindingName, scopeStoreObjectName, STORE_CALL_URL, StoreError } from '@substrat-run/contracts';
@@ -320,7 +321,7 @@ export class CloudflareScopeHost {
     scopeStoreRunners;
     scopeStoreVertical;
     scopeStoreLifecycle;
-    scopeStoreKinds;
+    scopeStoreDrivers;
     scopeStoreTransport;
     cp;
     servicePrincipals;
@@ -360,6 +361,7 @@ export class CloudflareScopeHost {
     blobStores;
     /** Worker-side attachment-bucket resolver (#473); the ambient per-tenant binding unless overridden (#1995). */
     attachmentBuckets;
+    orphanSweepClock;
     /** The parsers attachment text is extracted with (K-43); the host's own, never imported here. */
     attachmentExtractors;
     attachmentTextBounds;
@@ -429,6 +431,7 @@ export class CloudflareScopeHost {
         this.tenantStores = options.tenantStores;
         this.blobStores = options.blobStores;
         this.attachmentBuckets = options.attachmentBuckets ?? ambientAttachmentBucket;
+        this.orphanSweepClock = options.orphanSweepClock ?? (() => new Date().toISOString());
         assertAttachmentExtractors(options.attachmentExtractors ?? []);
         this.attachmentExtractors = options.attachmentExtractors ?? [];
         this.attachmentTextBounds = resolveAttachmentTextBounds(options.attachmentTextBounds);
@@ -437,7 +440,7 @@ export class CloudflareScopeHost {
         this.scopeLocalPermissions = options.scopeLocalPermissions ?? false;
         this.scopeNs = options.scope;
         this.scopeStoreVertical = options.scopeStores?.vertical;
-        this.scopeStoreKinds = options.scopeStores?.kinds ?? {};
+        this.scopeStoreDrivers = options.scopeStores?.drivers ?? {};
         this.scopeStoreTransport = scopeStoreTransportOf(options);
         this.scopeStoreRunners = scopeStoreRunnersOf(options.scopeStores);
         this.scopeStoreLifecycle = scopeStoreLifecycleOf(this.scopeStoreRunners);
@@ -1054,6 +1057,8 @@ export class CloudflareScopeHost {
     opts) {
         if (opts?.markCopy)
             assertCopyLineage(opts.markCopy);
+        // #2314: a load replaces the scope's rows, so the orphan sweep is fenced first, as for a rewind.
+        await this.fenceOrphanSweep(scopeId, opts?.tenantId);
         const carry = opts ? recordedOffFromWire(opts) : undefined;
         const load = {
             switchOff: carry ? { at: new Date().toISOString(), ...carry } : undefined,
@@ -1366,7 +1371,119 @@ export class CloudflareScopeHost {
      * run nothing until the switch is back in the scope. See `rewindHolding`.
      */
     async rewindScopeLocal(scopeId, bookmark, opts) {
-        return this.rewindHolding(scopeId, () => this.scopeStub(scopeId).rewindToBookmark(bookmark, opts));
+        // #2314: the orphan sweep's fence first. The rewind can take rows away that a later restore
+        // brings back, so their bytes must outlive the PITR window; no fence, no rewind.
+        try {
+            await this.fenceOrphanSweep(scopeId, opts?.tenantId);
+        }
+        catch (err) {
+            throw new Error(`${REWIND_REFUSED}${err instanceof Error ? err.message : String(err)}`);
+        }
+        return this.rewindHolding(scopeId, () => this.scopeStub(scopeId).rewindToBookmark(bookmark, { force: opts?.force }));
+    }
+    /**
+     * #2314: write the orphan sweep's restore fence for a scope whose state is about to be
+     * replaced, into the tenant's attachment bucket, so no pass deletes its bytes while a row the
+     * replacement takes away can still be restored. The tenant is the one named, else the scope's
+     * own provision receipt. A scope whose tenant is neither holds no row the replacement could
+     * take from it unless it holds an attachment row (a fresh scope, a clean-room preview), and
+     * only then is it refused, because its bucket cannot be found. A tenant this script holds no
+     * bucket for has no bytes here, so nothing is fenced. Throws when the fence cannot be written:
+     * the caller refuses the replacement.
+     */
+    async fenceOrphanSweep(scopeId, named) {
+        let tenantId = named ?? null;
+        if (!tenantId) {
+            const facts = await this.scopeStub(scopeId).sweepFenceFacts();
+            if (!facts.tenant && !facts.holdsAttachments)
+                return;
+            tenantId = facts.tenant;
+        }
+        if (!tenantId) {
+            throw new Error(`the tenant of scope ${scopeId} is unknown (no provision receipt, and none was named), so the ` +
+                `orphan-attachment sweep's restore fence (#2314) cannot be written. Reconcile the scope, or ` +
+                `name its tenant, then retry`);
+        }
+        const bucket = await this.attachmentBuckets(tenantId);
+        if (!bucket)
+            return;
+        try {
+            await fenceAttachmentSweep(r2TenantBlobStore(bucket), scopeId, this.orphanSweepClock());
+        }
+        catch (err) {
+            throw new Error(`the orphan-attachment sweep's restore fence (#2314) could not be written for scope ${scopeId}: ` +
+                `${err instanceof Error ? err.message : String(err)}. Nothing was replaced; retry once the attachment bucket answers`);
+        }
+    }
+    /**
+     * #2314: one orphan-sweep pass over a scope's attachment bytes, judged by THIS deployment's
+     * copy of the scope (`sweepOrphanAttachments` in the kernel has the guards). The caller is the
+     * platform, and only for the deployment that serves the scope: the copy refuses to judge if it
+     * is not the scope's provisioned primary, but a stale copy in an older version's script cannot
+     * tell that it is stale.
+     */
+    async sweepOrphanAttachments(tenantId, scopeId, request = {}) {
+        // With a directory, it is the word on the scope, and it is read BEFORE the object is addressed,
+        // since addressing a reaped or never-held object would wake an empty one: this tenant's, active,
+        // a primary, and with no other script's copy that may hold rows this one lacks. A CP-less
+        // deployment has none: its scope's own storage is the word (receipt, lifecycle, copy markers,
+        // in the DO), and the copy ledger is the caller's to have read.
+        let vouched = false;
+        if (!this.cpLess) {
+            const record = await this.cp.getScopeRecord(tenantId, scopeId);
+            if (!record)
+                throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
+            if (record.status !== 'active') {
+                throw substratError('conflict', `refused: scope ${scopeId} is ${record.status}, and only an active scope's bytes are swept`);
+            }
+            if (!isPrimaryScopeRow(record)) {
+                throw substratError('conflict', `refused: scope ${scopeId} is a copy, and a copy may not judge orphan attachments`);
+            }
+            // Every ledger entry of the scope, page by page, so no page can hide one: a pass needs all of
+            // them settled (`SETTLED_COPY_STATES` is an allowlist), and stops at the first that is not.
+            let after;
+            for (;;) {
+                const page = (await this.cp.listScopeScriptCopies({ tenantId, scopeId, limit: 100, ...(after ? { after } : {}) })).map(scopeScriptCopyOf);
+                const unsettled = unsettledCopyRefusal(scopeId, page);
+                if (unsettled)
+                    throw unsettled;
+                const last = page[page.length - 1];
+                if (page.length < 100 || !last)
+                    break;
+                after = { scriptRef: last.scriptRef, moveId: last.moveId };
+            }
+            // The tenant's status too, as every door into a scope asks.
+            await this.assertLive(tenantId, scopeId);
+            vouched = true;
+        }
+        else if (request.copiesSettled !== true) {
+            throw substratError('conflict', `refused: this deployment cannot read the copy ledger, so a pass over scope ${scopeId} needs its caller ` +
+                'to have found no unsettled copy of it (copiesSettled)');
+        }
+        const stub = this.scopeStub(scopeId);
+        const now = this.orphanSweepClock();
+        const expected = request.expectedVersionId;
+        const judge = (condemn) => async (ids, at) => unwrapReply(await stub.orphanAttachmentsReply([...ids], at, tenantId, condemn, vouched, expected));
+        // The scope's copy gives its verdict first, with no ids: every refusal it makes, and whether this
+        // script is the one the platform routed the pass to, in the pass's own mode. So a destructive
+        // pass routed to the wrong script is refused even when there is nothing to condemn.
+        await judge(request.dryRun !== true)([], now);
+        // A script with no attachment store for the tenant holds none of its bytes: a vertical whose
+        // modules keep no files. Answered as that, marked, rather than the throw `resolveAttachmentStore`
+        // makes for a write, so the platform's pass can tell it from a failure and from a store that held
+        // nothing. A scope that holds an attachment row does keep files, so its missing store is a
+        // misconfiguration, and it fails.
+        const bucket = await this.attachmentBuckets(tenantId);
+        if (!bucket) {
+            if ((await stub.sweepFenceFacts()).holdsAttachments)
+                throw missingAttachmentStore(tenantId, scopeId);
+            return { listed: 0, candidates: 0, orphans: 0, deleted: 0, keys: [], dryRun: request.dryRun === true, bucket: 'none' };
+        }
+        const store = r2TenantBlobStore(bucket);
+        return sweepOrphanAttachments(store, scopeId, {
+            rowless: (ids) => judge(false)(ids, now),
+            condemn: judge(true),
+        }, { now, ...(request.dryRun ? { dryRun: true } : {}), ...(request.limit !== undefined ? { limit: request.limit } : {}) });
     }
     registerModule(registration) {
         const manifest = moduleManifest.parse(registration.manifest);
@@ -1646,41 +1763,45 @@ export class CloudflareScopeHost {
         const store = await this.resolveAttachmentStore(tenantId);
         return this.buildAttachmentSurface({ principal }, tenantId, scopeId, store);
     }
-    async inspectUploadLink(tenantId, scopeId, secret) {
+    async inspectUploadLink(tenantId, scopeId, secret, target, idempotencyKey) {
         await this.assertLive(tenantId, scopeId);
         await this.migrateAndRecord(scopeId);
-        return unwrapReply(await this.scopeStub(scopeId).uploadLinkInspectReply(secret, tenantId, scopeId)).view;
+        return unwrapReply(await this.scopeStub(scopeId).uploadLinkInspectReply(secret, tenantId, scopeId, target, idempotencyKey)).view;
     }
     async redeemUploadLink(tenantId, scopeId, secret, input) {
         await this.assertLive(tenantId, scopeId);
         await this.migrateAndRecord(scopeId);
         const stub = this.scopeStub(scopeId);
-        const { view, mintedBy } = unwrapReply(await stub.uploadLinkInspectReply(secret, tenantId, scopeId));
-        if (view.entity.entityType !== input.entity.entityType || view.entity.entityId !== input.entity.entityId) {
-            throw substratError('not_found', 'This upload link is not valid for that target.', {
-                reason: UPLOAD_LINK_WRONG_TARGET,
+        // The target first (the kernel judges it before the state), so a wrong path never reads as
+        // gone; a replay (#2315) is answered from the inspection and never reaches the write.
+        const inspect = async () => unwrapReply(await stub.uploadLinkInspectReply(secret, tenantId, scopeId, input.entity, input.idempotencyKey));
+        return redeemOrReplayUploadLink(inspect, input, async ({ view, mintedBy }) => {
+            const store = await this.resolveAttachmentStore(tenantId);
+            // A fresh id per request, so its key is its own: compensating a refusal can never remove
+            // the bytes of the redemption that won the link.
+            const id = ulid();
+            const key = attachmentBlobKey(scopeId, id);
+            const filename = view.filename ?? input.filename;
+            // Recorded as it is judged: the essence, never a parameter or a stray space.
+            const contentType = normalizeUploadMediaType(input.contentType);
+            const record = attachmentRecord.parse({
+                id,
+                entity: view.entity,
+                filename,
+                contentType,
+                size: input.body.byteLength,
+                sha256: await attachmentSha256(input.body),
+                visibility: view.visibility,
+                createdBy: mintedBy,
+                createdAt: new Date().toISOString(),
             });
-        }
-        const store = await this.resolveAttachmentStore(tenantId);
-        const id = ulid();
-        const key = attachmentBlobKey(scopeId, id);
-        const filename = view.filename ?? input.filename;
-        const record = attachmentRecord.parse({
-            id,
-            entity: view.entity,
-            filename,
-            contentType: input.contentType,
-            size: input.body.byteLength,
-            sha256: await attachmentSha256(input.body),
-            visibility: view.visibility,
-            createdBy: mintedBy,
-            createdAt: new Date().toISOString(),
+            // Bytes first, row second, exactly as an upload: a refusal (size, type, a lost key, a link
+            // someone else spent first) compensates the object away, and the link is unspent with it.
+            // A lost reply keeps the bytes (`compensatingRefusal`), and a retry carrying the same
+            // `Idempotency-Key` is answered as a replay (#2315).
+            await store.put(key, input.body, { contentType });
+            return unwrapReply(await compensatingRefusal(store, key, () => stub.attachmentRedeemUploadLinkReply(record, secret, tenantId, scopeId, input.idempotencyKey)));
         });
-        // Bytes first, row second, exactly as an upload: a refusal (size, type, a lost key, a link
-        // someone else spent first) compensates the object away, and the link is unspent with it.
-        // A lost reply keeps the bytes (`compensatingRefusal`); retrying the link is #2315.
-        await store.put(key, input.body, { contentType: input.contentType });
-        return unwrapReply(await compensatingRefusal(store, key, () => stub.attachmentRedeemUploadLinkReply(record, secret, tenantId, scopeId)));
     }
     async getSystemAttachments(moduleId, tenantId, scopeId) {
         // Reuse the system invoke door's module, tenant and lifecycle gate, and its rewind gate
@@ -1888,6 +2009,8 @@ export class CloudflareScopeHost {
         if (!existing)
             throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
         this.refuseUnusableScopeStores(dump.stores, existing.vertical);
+        // #2314: the load replaces the scope's rows, so the orphan sweep is fenced first.
+        await this.fenceOrphanSweep(scopeId, tenantId);
         // #1742: a dump from before a switch was pulled brings the module's grants back live, so
         // the directory's recorded-off modules go back off in the replay's own event. Only where
         // this is the scope's real store, as for the provision's seat above; a hosted scope is
@@ -2188,23 +2311,23 @@ export class CloudflareScopeHost {
         }
         return erasures;
     }
-    /** A runner's kind, or the kind declared for an ambient binding. Never invented. */
-    scopeStoreKind(binding) {
-        const kind = this.scopeStoreRunners.get(binding)?.kind ?? this.scopeStoreKinds[binding];
-        if (!kind) {
-            throw substratError('unavailable', `scope store '${binding}' has no declared kind`);
+    /** A runner's driver, or the driver declared for an ambient binding. Never invented. */
+    scopeStoreDriver(binding) {
+        const driver = this.scopeStoreRunners.get(binding)?.driver ?? this.scopeStoreDrivers[binding];
+        if (!driver) {
+            throw substratError('unavailable', `scope store '${binding}' has no declared driver`);
         }
-        return kind;
+        return driver;
     }
     /**
      * A dump's stores this host cannot load are refused before any write (#2211, L8): a
-     * binding that is not bound, a kind other than the bound one, or any store at all for
+     * binding that is not bound, a driver other than the bound one, or any store at all for
      * a destination with no vertical (no store object can be named for it, so loading would
      * drop them), or a destination on a vertical other than the one this host is pinned to.
      * `vertical` undefined is a caller that cannot read the destination here;
      * `importScopeStores` then refuses a scope with no vertical itself, still before a store
-     * write. A store with no `kind` is refused: the dump contract requires one, and an
-     * unparsed dump would otherwise skip the kind comparison.
+     * write. A store with no `driver` is refused: the dump contract requires one, and an
+     * unparsed dump would otherwise skip the driver comparison.
      */
     refuseUnusableScopeStores(stores, vertical) {
         const listed = stores ?? [];
@@ -2221,12 +2344,12 @@ export class CloudflareScopeHost {
             if (!bound.has(store.binding)) {
                 throw substratError('not_found', `dump names scope store '${store.binding}', which is not bound`);
             }
-            if (!store.kind) {
-                throw substratError('validation_failed', `dump's scope store '${store.binding}' names no kind`);
+            if (!store.driver) {
+                throw substratError('validation_failed', `dump's scope store '${store.binding}' names no driver`);
             }
-            const kind = this.scopeStoreRunners.get(store.binding)?.kind ?? this.scopeStoreKinds[store.binding];
-            if (store.kind !== kind) {
-                throw substratError('conflict', `dump's scope store '${store.binding}' is kind '${store.kind}', and this host binds kind '${kind ?? 'undeclared'}'`);
+            const driver = this.scopeStoreRunners.get(store.binding)?.driver ?? this.scopeStoreDrivers[store.binding];
+            if (store.driver !== driver) {
+                throw substratError('conflict', `dump's scope store '${store.binding}' is driver '${store.driver}', and this host binds driver '${driver ?? 'undeclared'}'`);
             }
         }
     }
@@ -2244,7 +2367,7 @@ export class CloudflareScopeHost {
             const exported = await this.invokeReservedStore({ tenantId, vertical, scopeId, binding }, 'substrat.export');
             stores.push({
                 binding,
-                kind: this.scopeStoreKind(binding),
+                driver: this.scopeStoreDriver(binding),
                 tables: exported.tables,
             });
         }
@@ -3018,9 +3141,10 @@ export class CloudflareScopeHost {
                     // #119: a purge horizon's schedule runs its operation once per due entity, each its own
                     // call and transaction — and the SCOPE decides which are due and runs them, so purge
                     // authority is never an argument any stub can pass (`runPurgeSweep`).
-                    const pass = await door.through((instance) => stub.runPurgeSweep(schedule.operation, tenantId, scopeId, instance));
+                    // #2192: under the run's own id, so the purges and their deliveries join this run's line.
+                    const pass = await door.through((instance) => stub.runPurgeSweep(schedule.operation, tenantId, scopeId, instance, invocationId));
                     // The purges' events, delivered the way each invoke's own tail delivers them.
-                    await this.drainExecutors(tenantId, scopeId, null);
+                    await this.drainExecutors(tenantId, scopeId, invocationId);
                     stillDue = purgeStillDue(pass);
                     const outcome = purgeReportOf(schedule.operation, schedule.purge.entityType, pass);
                     report.errors.push(...outcome.errors);
@@ -4408,8 +4532,13 @@ export class CloudflareScopeHost {
                 // The SQL migrations are stored apart from the manifest (#1764), so no read of the
                 // version but the promote review's carries them.
                 const split = splitManifestMigrations(manifestJson ?? null, 'push');
+                // #2339: the digest a promote compares is the one derived from what is stored, never the pushed one.
+                const migrationDigest = await publishedMigrationDigest({
+                    id: audited.id, migrationDigest: audited.migrationDigest, manifestJson: manifestJson ?? null, migrations: split.migrations,
+                });
                 await this.directory('insertVersion', {
                     ...audited,
+                    migrationDigest,
                     manifestJson: split.manifestJson,
                     migrations: split.migrations,
                     originJson: origin ? JSON.stringify(origin) : null,
@@ -4419,6 +4548,8 @@ export class CloudflareScopeHost {
                 });
                 await this.recordAdmin(actor, 'publishVersion', { tenantId: null }, null, {
                     ...audited,
+                    // The digest the registry holds, which differs from the pushed one only for an unheld version.
+                    migrationDigest,
                     ...(origin ? { origin } : {}),
                     admission: selfAdmits ? 'admitted' : 'pending',
                 });
@@ -4585,6 +4716,20 @@ export class CloudflareScopeHost {
                 ], incoming.id, incoming, served || owned.length > 0
                     ? await this.cp.provisionedVersionsOf(verticalSlug, served, owned.length > 0 ? owning.owner_tenant : null)
                     : []);
+                // #2333: the acknowledgements answer the review they were given against, or none at all.
+                const reviewed = async (v) => ({
+                    id: v.id,
+                    permissionDigest: v.permission_digest,
+                    migrationDigest: v.migration_digest,
+                    migrations: (await this.cp.readVersionMigrations(v.id))?.migrations ?? null,
+                });
+                await assertPromoteReviewCurrent({
+                    ack,
+                    channel,
+                    baseId: outgoing?.id ?? null,
+                    migrationsChange: outgoing !== undefined && outgoing.migration_digest !== incoming.migration_digest,
+                    read: async () => ({ base: outgoing ? await reviewed(outgoing) : null, incoming: await reviewed(incoming) }),
+                });
                 // §4's checkpoints, at the moment of exposure. A first promotion has
                 // nothing to diff against — the gate is about change, not existence.
                 if (outgoing) {
@@ -4593,8 +4738,7 @@ export class CloudflareScopeHost {
                             `${incoming.permission_digest}) — acknowledge it explicitly to promote`);
                     }
                     if (outgoing.migration_digest !== incoming.migration_digest && !ack.migrationChange) {
-                        throw substratError('conflict', `promotion changes migrations (${outgoing.migration_digest} → ` +
-                            `${incoming.migration_digest}) — acknowledge it explicitly to promote`);
+                        throw substratError('conflict', migrationChangeRefusal(outgoing.migration_digest, incoming.migration_digest));
                     }
                     // #1705 PR 3: an export an installed consumer imports, dropped or re-versioned.
                     if (!ack.exportBreak) {
@@ -4604,10 +4748,13 @@ export class CloudflareScopeHost {
                     }
                 }
                 const promotedAt = new Date().toISOString();
-                await this.cp.setChannel(verticalSlug, channel, versionId, promotedAt);
-                // The timeline row: what makes rollback a choice among recorded moments, and
-                // `at` the PITR anchor a data rollback would rewind to.
-                await this.cp.insertChannelHistory({
+                // #2333: a compare-and-set. Every gate above — the review binding, the digest checkpoints,
+                // the export breaks — was judged against `outgoing`, across awaits another promote can run
+                // in. The control plane moves the pointer, and writes the timeline row (what makes rollback a
+                // choice among recorded moments, and `at` the PITR anchor a data rollback would rewind to),
+                // only if the pointer still names `outgoing`; the promote that lost the race is refused as
+                // stale and writes nothing.
+                const moved = await this.cp.moveChannel(outgoing?.id ?? null, {
                     id: ulid(),
                     vertical_slug: verticalSlug,
                     channel,
@@ -4616,6 +4763,8 @@ export class CloudflareScopeHost {
                     actor,
                     at: promotedAt,
                 });
+                if (!moved)
+                    throw promoteRaceRefusal(channel);
                 // For a PRIVATE vertical, prod IS what the owner's apps run: re-point the
                 // owning tenant's live scopes in the same act, so merge-to-main (push +
                 // promote) is a complete deploy and a rollback promote reaches the running
@@ -4688,19 +4837,27 @@ export class CloudflareScopeHost {
             },
             bindScopeVersion: async (actor, tenantId, scopeId, versionId, opts) => {
                 const { v, scope } = await bindTarget(tenantId, scopeId, versionId);
-                if (opts?.expectedVersionId !== undefined && scope.vertical_version_id !== opts.expectedVersionId) {
-                    throw substratError('precondition_failed', 'scope binding changed; reload the scope and retry');
-                }
+                // #2347: a rebind judged against both lineages loses its race as stale, so its client reads
+                // the review again; any other conditional bind keeps the refusal it always had.
+                const lost = () => opts?.expectedServing
+                    ? rebindRaceRefusal(opts.expectedServing.vertical)
+                    : substratError('precondition_failed', 'scope binding changed; reload the scope and retry');
+                if (opts?.expectedVersionId !== undefined && scope.vertical_version_id !== opts.expectedVersionId)
+                    throw lost();
+                if (opts?.expectedServing && !await this.cp.servesNow(opts.expectedServing))
+                    throw lost();
                 if (opts?.expectedErasureEpoch !== undefined && await this.cp.scopeErasureEpoch(tenantId, scopeId) !== opts.expectedErasureEpoch) {
                     throw substratError('precondition_failed', 'scope erasure changed; reload the scope and retry');
                 }
                 const ack = bindAcknowledgement.parse(opts?.acknowledge ?? {});
+                // #2347: a route moving in the same write is judged with it, as `bindingImpact` is.
+                const servingRef = opts?.servingRef;
                 // #2178 first: no acknowledgement passes it, so none should be asked for ahead of it.
-                await this.assertBindingShapesKept(mapScope(scope), v);
+                await this.assertBindingShapesKept(mapScope(scope), v, servingRef);
                 // #1756: an export an app in this tenant imports, dropped or re-versioned by what this
                 // scope would run. Before the snapshot, so a refused bind leaves nothing behind.
                 if (!ack.exportBreak) {
-                    const breaks = await this.bindBreaks(actor, mapScope(scope), v);
+                    const breaks = await this.bindBreaks(actor, mapScope(scope), v, servingRef);
                     if (breaks.length > 0)
                         throw substratError('precondition_failed', bindExportBreakRefusal(breaks));
                 }
@@ -4712,12 +4869,19 @@ export class CloudflareScopeHost {
                         await this.snapshotScope(actor, tenantId, scopeId);
                     }
                 }
-                if (!await this.cp.bindScopeVersion(scopeId, versionId, v.vertical_slug, opts?.expectedVersionId, opts?.expectedErasureEpoch, opts?.confirmMove)) {
-                    throw substratError('precondition_failed', 'scope binding changed; reload the scope and retry');
+                if (!await this.cp.bindScopeVersion(scopeId, versionId, v.vertical_slug, opts?.expectedVersionId, opts?.expectedErasureEpoch, opts?.confirmMove, servingRef, opts?.expectedServing)) {
+                    // The erasure and copy-move conditions keep their own refusal: only a moved binding or a
+                    // moved serving version is the race a rebind's review answers.
+                    const now = await this.cp.getScopeRecord(tenantId, scopeId);
+                    const raced = (opts?.expectedVersionId !== undefined && (now?.vertical_version_id ?? null) !== opts.expectedVersionId)
+                        || (opts?.expectedServing !== undefined && !await this.cp.servesNow(opts.expectedServing));
+                    throw raced ? lost() : substratError('precondition_failed', 'scope binding changed; reload the scope and retry');
                 }
-                await this.recordAdmin(actor, 'bindScopeVersion', { tenantId, scopeId }, null, {
+                await this.recordAdmin(actor, 'bindScopeVersion', { tenantId, scopeId }, servingRef === undefined ? null : { servingRef: scope.serving_ref ?? null }, {
                     versionId, vertical: v.vertical_slug, version: v.version,
+                    ...(servingRef !== undefined ? { servingRef } : {}),
                     ...(opts?.expectedVersionId !== undefined ? { expectedVersionId: opts.expectedVersionId } : {}),
+                    ...(opts?.expectedServing ? { expectedServing: opts.expectedServing } : {}),
                     ...(ack.exportBreak ? { acknowledged: ack } : {}),
                 });
             },
@@ -4898,7 +5062,7 @@ export class CloudflareScopeHost {
                     return { rewindingTo: bookmark };
                 }
                 // #1819: the same rewind a dispatched vertical runs, holding what the scope switched off.
-                return this.rewindScopeLocal(scopeId, bookmark, { force: opts?.force });
+                return this.rewindScopeLocal(scopeId, bookmark, { force: opts?.force, tenantId });
             },
             createOrg: async (actor, input) => {
                 const parsed = createOrgInput.parse(input);
@@ -5050,43 +5214,63 @@ export class CloudflareScopeHost {
                     createdAt: r.created_at,
                 }));
             },
-            // -- store-kind releases, rollouts and holds (D-63, #2300) -------------------
-            recordStoreKindRelease: async (actor, input) => {
+            // -- store-driver releases, rollouts and holds (D-63, #2300) -------------------
+            recordStoreDriverRelease: async (actor, input) => {
                 const at = new Date().toISOString();
-                const { record, created } = await this.directory('recordStoreKindRelease', { ...input, actor, at });
+                const { record, created } = await this.directory('recordStoreDriverRelease', { ...input, actor, at });
                 if (created)
-                    await this.recordAdmin(actor, 'uploadStoreKindRelease', { tenantId: null }, null, record);
+                    await this.recordAdmin(actor, 'uploadStoreDriverRelease', { tenantId: null }, null, record);
                 return record;
             },
-            getStoreKindRelease: async (actor, kind, sha256) => {
-                const record = await this.cp.getStoreKindRelease(kind, sha256);
-                await this.recordAccess(actor, 'getStoreKindRelease', { tenantId: null }, { kind, sha256 }, record ? 1 : 0);
+            getStoreDriverRelease: async (actor, driver, sha256) => {
+                const record = await this.cp.getStoreDriverRelease(driver, sha256);
+                await this.recordAccess(actor, 'getStoreDriverRelease', { tenantId: null }, { driver, sha256 }, record ? 1 : 0);
                 return record;
             },
-            storeKindState: async (actor, kind) => {
-                const state = await this.cp.storeKindState(kind);
-                await this.recordAccess(actor, 'storeKindState', { tenantId: null }, { kind }, state.rollouts.length);
+            storeDriverState: async (actor, driver) => {
+                const state = await this.cp.storeDriverState(driver);
+                await this.recordAccess(actor, 'storeDriverState', { tenantId: null }, { driver }, state.rollouts.length);
                 return state;
             },
-            storeKindDeclarations: async (actor, kind) => {
-                const declared = await this.directory('storeKindDeclarations', kind);
-                await this.recordAccess(actor, 'storeKindDeclarations', { tenantId: null }, { kind }, declared.verticals.length);
+            storeDriverDeclarations: async (actor, driver) => {
+                const declared = await this.directory('storeDriverDeclarations', driver);
+                await this.recordAccess(actor, 'storeDriverDeclarations', { tenantId: null }, { driver }, declared.verticals.length);
                 return declared;
             },
-            holdStoreKind: async (_actor, input) => this.directory('holdStoreKind', input),
-            releaseStoreKindHold: async (_actor, input) => {
-                await this.cp.releaseStoreKindHold(input);
+            storeDriverRolloutPending: async (actor, input) => {
+                const page = await this.directory('storeDriverRolloutPending', input);
+                await this.recordAccess(actor, 'storeDriverRolloutPending', { tenantId: null }, { driver: input.driver, sha256: input.sha256, action: input.action }, page.verticals.length);
+                return page;
             },
-            beginStoreKindRollout: async (actor, input) => {
-                const row = await this.directory('beginStoreKindRollout', { ...input, actor, at: new Date().toISOString() });
-                await this.recordAdmin(actor, input.action === 'rollback' ? 'rollBackStoreKind' : 'rollOutStoreKind', { tenantId: null }, null, row);
+            storeDriverRolloutWalked: async (actor, input) => {
+                const walked = await this.directory('storeDriverRolloutWalked', input);
+                await this.recordAccess(actor, 'storeDriverRolloutWalked', { tenantId: null }, { driver: input.driver, seq: input.seq }, walked.verticals.length);
+                return walked;
+            },
+            holdStoreDriver: async (_actor, input) => this.directory('holdStoreDriver', input),
+            releaseStoreDriverHold: async (_actor, input) => {
+                await this.cp.releaseStoreDriverHold(input);
+            },
+            beginStoreDriverRollout: async (actor, input) => {
+                const row = await this.directory('beginStoreDriverRollout', { ...input, actor, at: new Date().toISOString() });
+                await this.recordAdmin(actor, input.action === 'rollback' ? 'rollBackStoreDriver' : 'rollOutStoreDriver', { tenantId: null }, null, row);
                 return row;
             },
-            finishStoreKindRollout: async (actor, input) => {
-                const row = await this.directory('finishStoreKindRollout', { ...input, at: new Date().toISOString() });
-                await this.recordAdmin(actor, row.action === 'rollback' ? 'rollBackStoreKind' : 'rollOutStoreKind', { tenantId: null }, null, row);
+            finishStoreDriverRollout: async (actor, input) => {
+                const row = await this.directory('finishStoreDriverRollout', { ...input, at: new Date().toISOString() });
+                await this.recordAdmin(actor, row.action === 'rollback' ? 'rollBackStoreDriver' : 'rollOutStoreDriver', { tenantId: null }, null, row);
                 return row;
             },
+            storeDriverRebuild: async (actor, driver) => {
+                const rebuild = await this.cp.storeDriverRebuild(driver);
+                await this.recordAccess(actor, 'storeDriverRebuild', { tenantId: null }, { driver }, rebuild ? 1 : 0);
+                return rebuild;
+            },
+            claimStoreDriverRebuild: async (_actor, input) => this.directory('claimStoreDriverRebuild', { ...input, at: new Date().toISOString() }),
+            advanceStoreDriverRebuild: async (_actor, input) => this.directory('advanceStoreDriverRebuild', { ...input, at: new Date().toISOString() }),
+            recordStoreDriverRebuildRun: async (actor, entry) => {
+                await this.recordAdmin(actor, 'rebuildStoreDriver', { tenantId: null }, null, entry);
+            },
             listScopeTables: async (actor, tenantId, scopeId) => {
                 // K-3 cross-check on the shared directory BEFORE reaching the scope DO: a pair
                 // that does not resolve is unreachable, never another tenant's database.
@@ -6030,6 +6214,11 @@ export class CloudflareScopeHost {
                 const { tenantId, scopeId, action, ...after } = copyMarkAudit.parse(entry);
                 await this.recordAdmin(actor, action === 'mark' ? 'markScopeCopy' : 'clearScopeCopyMark', { tenantId, scopeId }, null, after);
             },
+            /** #2314: one orphan-attachment pass over a scope, written after the vertical's own pass. */
+            recordOrphanSweep: async (actor, entry) => {
+                const { tenantId, scopeId, ...after } = orphanSweepAudit.parse(entry);
+                await this.recordAdmin(actor, 'sweepOrphanAttachments', { tenantId, scopeId }, null, after);
+            },
             accessLog: async (actor, filter) => {
                 const rows = await this.cp.accessLog({
                     actor: filter?.actor,