@substrat-run/adapter-cloudflare 0.147.0 → 0.149.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/control-plane-do.d.ts +131 -10
- package/dist/control-plane-do.d.ts.map +1 -1
- package/dist/control-plane-do.js +165 -35
- package/dist/control-plane-do.js.map +1 -1
- package/dist/host.d.ts +75 -16
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +395 -61
- package/dist/host.js.map +1 -1
- package/dist/route-resolver.js +2 -2
- package/dist/route-resolver.js.map +1 -1
- package/dist/scope-do.d.ts +7 -0
- package/dist/scope-do.d.ts.map +1 -1
- package/dist/scope-do.js +180 -11
- package/dist/scope-do.js.map +1 -1
- package/package.json +9 -9
package/dist/host.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { env as ambientEnv } from 'cloudflare:workers';
|
|
2
2
|
import { isRewindRefusal, REWIND_REFUSED } from './rewind-refusal.js';
|
|
3
3
|
import { isSystemDoorMoved, SYSTEM_DOOR_MOVED, SYSTEM_DOOR_REGATES } from './system-door.js';
|
|
4
|
-
import { delegatedReadParams, operationSeriesCount, fromWireFailure, isSubstratError, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, assertNoBindingShapeBreak, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, findingEntry, findingRuleEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, mintedCapability, boundedBecomeMint, boundedBecomeRevoke, boundedUnbind, entityRef, becomeLinkState, principalBecomeCapabilityInput, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, scopeStorageSample, subjectRef, createConnectionInput, connectLinkFilter, connectLinkKey, consumeConnectLinkInput, mintConnectLinkInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, memberChangeAudit, peerBindingAudit, copyMarkAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, SCOPE_GATE_REASONS, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, peerInstanceResolution, peerBinding as peerBindingSchema, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
|
|
4
|
+
import { delegatedReadParams, operationSeriesCount, fromWireFailure, isSubstratError, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, assertNoBindingShapeBreak, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, findingEntry, findingRuleEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, mintedCapability, boundedBecomeMint, boundedBecomeRevoke, boundedUnbind, entityRef, becomeLinkState, principalBecomeCapabilityInput, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, scopeStorageSample, subjectRef, createConnectionInput, connectLinkFilter, connectLinkKey, consumeConnectLinkInput, mintConnectLinkInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, memberChangeAudit, peerBindingAudit, connectionBindingAudit, copyMarkAudit, orphanSweepAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, SCOPE_GATE_REASONS, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, peerInstanceResolution, peerBinding as peerBindingSchema, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
|
|
5
5
|
import { normalizeHostname, toRouteTarget } from './route-resolver.js';
|
|
6
|
-
import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, assertBecomeLinkStateIds, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, executorOutcomeOf, isDeliveryRefusal, refusalJournalText, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, assertLeaseMs, runDueJobRuns, SYSTEM_DOOR_WAIT, startJobRun, attachmentTextJob, assertAttachmentExtractors, resolveAttachmentTextBounds, assertJobRegistrable, ATTACHMENT_TEXT_BACKFILL_JOB, ATTACHMENT_TEXT_JOB, kernelJobFor, attachmentTextBackfillJob, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, recordedOffFromWire, peerSubjectRef, SWITCH_KINDS, reassertActionOf, reassertEntry, reassertOffRow, reassertOnRow, switchActionOf, switchAuditSubject, recordAuditOutcome, auditedCapabilityRevoke, auditWarningOf, switchNotFoundMessage, recordWriteSuperseded, switchSubjectOf, switchSupersededMessage, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, scopeMoveOf, runningVersionOf, strandJudgmentsOf, boundShapesOfColumn, installReconciled, collectPeers, peerCallerAt, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, emptyImportResult, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, assertNoCallerPurge, purgeReportOf, purgeStillDue, assertBoundOperations, registerTrashTargets, memberAddedAudit, shapeTopUpBatch, } from '@substrat-run/kernel';
|
|
6
|
+
import { attachmentBlobKey, normalizeUploadMediaType, redeemOrReplayUploadLink, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, assertBecomeLinkStateIds, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, executorOutcomeOf, isDeliveryRefusal, refusalJournalText, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, assertLeaseMs, runDueJobRuns, SYSTEM_DOOR_WAIT, startJobRun, attachmentTextJob, assertAttachmentExtractors, resolveAttachmentTextBounds, assertJobRegistrable, ATTACHMENT_TEXT_BACKFILL_JOB, ATTACHMENT_TEXT_JOB, kernelJobFor, attachmentTextBackfillJob, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, recordedOffFromWire, peerSubjectRef, SWITCH_KINDS, reassertActionOf, reassertEntry, reassertOffRow, reassertOnRow, switchActionOf, switchAuditSubject, recordAuditOutcome, auditedCapabilityRevoke, auditWarningOf, switchNotFoundMessage, recordWriteSuperseded, switchSubjectOf, switchSupersededMessage, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, assertPromoteReviewCurrent, promoteRaceRefusal, rebindRaceRefusal, bindExportBreaksOf, bindExportBreakRefusal, scopeMoveOf, runningVersionOf, strandJudgmentsOf, boundShapesOfColumn, installReconciled, collectPeers, peerCallerAt, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, publishedMigrationDigest, migrationChangeRefusal, unknownRoleError, assertRowLimit, assertRowOffset, emptyImportResult, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, assertNoCallerPurge, purgeReportOf, purgeStillDue, assertBoundOperations, registerTrashTargets, memberAddedAudit, shapeTopUpBatch, } from '@substrat-run/kernel';
|
|
7
|
+
import { fenceAttachmentSweep, missingAttachmentStore, sweepOrphanAttachments, unsettledCopyRefusal } from '@substrat-run/kernel';
|
|
7
8
|
import { COPY_RESTORE_FENCE_LAPSED, scopeScriptCopyOf } from '@substrat-run/kernel';
|
|
8
9
|
import { attributedView, isModuleErasureCounts, moduleErasurePlan, moduleRowsErased, callReservedStoreOperation, callScopeStore, observeStoreCall, replayStoreErasures, scopeStoreLifecycleOf, scopeStoreRunnersOf } from '@substrat-run/kernel';
|
|
9
10
|
import { PLATFORM_BINDING_PREFIX, scopeStoreBindingName, scopeStoreObjectName, STORE_CALL_URL, StoreError } from '@substrat-run/contracts';
|
|
@@ -179,6 +180,21 @@ class UnattestedSwitch extends Error {
|
|
|
179
180
|
const unfencedMessage = (scopeId) => `the deployment serving scope ${scopeId} predates the switch fence (#2045): it cannot refuse an older ` +
|
|
180
181
|
`switch call's OFF landing after this ON, so no switch is turned on there. Switching off still works. ` +
|
|
181
182
|
`Redeploy the vertical, then retry.`;
|
|
183
|
+
/**
|
|
184
|
+
* The attachment write's compensation, after its bytes are in the store (#2311). The object
|
|
185
|
+
* is deleted only when the ScopeDO ANSWERED a refusal: `replyOf` turns every DO-side throw
|
|
186
|
+
* into a `failure` value, and the DO does not let a post-commit drain fault become one, so an
|
|
187
|
+
* answered failure means the row did not commit. A rejected call is a transport failure with
|
|
188
|
+
* the commit unknown. A delete there could leave a committed row without its bytes, so the
|
|
189
|
+
* bytes are kept and the error rethrown. If nothing committed, the object is an orphan that
|
|
190
|
+
* no row points at, which the sweep of #2314 is for.
|
|
191
|
+
*/
|
|
192
|
+
async function compensatingRefusal(store, key, call) {
|
|
193
|
+
const reply = await call();
|
|
194
|
+
if (reply.failure)
|
|
195
|
+
await store.delete(key).catch(() => { });
|
|
196
|
+
return reply;
|
|
197
|
+
}
|
|
182
198
|
/** A tenant-held module's OFF answered without `deniesTenantGrants` (#1823). */
|
|
183
199
|
const undeniedMessage = (scopeId, moduleId) => `the deployment serving scope ${scopeId} predates the kill switch's tenant-grant denial (#1823): ` +
|
|
184
200
|
`module '${moduleId}' holds a tenant-level grant that its OFF would leave authorizing there. ` +
|
|
@@ -223,6 +239,23 @@ function isScopeStoreNamespace(value) {
|
|
|
223
239
|
typeof Reflect.get(value, 'get') === 'function');
|
|
224
240
|
}
|
|
225
241
|
const SCOPE_STORE_BINDING_PREFIX = `${PLATFORM_BINDING_PREFIX}STORE_`;
|
|
242
|
+
/**
|
|
243
|
+
* `platform` unless this host was handed another way to reach a store (#2288): a
|
|
244
|
+
* non-empty `runners` list (in-process), or `transport: 'binding'`, which is the only
|
|
245
|
+
* thing that reads an ambient `SUBSTRAT_STORE_*` namespace. A control plane does not
|
|
246
|
+
* change the answer: whoever wants the direct namespace says so. An explicit
|
|
247
|
+
* `'platform'` beside runners is refused, since the runners would never be used.
|
|
248
|
+
*/
|
|
249
|
+
function scopeStoreTransportOf(options) {
|
|
250
|
+
const runners = options.scopeStores?.runners?.length ?? 0;
|
|
251
|
+
const transport = options.scopeStores?.transport;
|
|
252
|
+
if (transport === 'platform' && runners > 0) {
|
|
253
|
+
throw substratError('validation_failed', "scopeStores: transport 'platform' takes no runners; they would never be used");
|
|
254
|
+
}
|
|
255
|
+
if (transport)
|
|
256
|
+
return transport;
|
|
257
|
+
return runners > 0 ? 'binding' : 'platform';
|
|
258
|
+
}
|
|
226
259
|
/** Bindings this script declares by having a `SUBSTRAT_STORE_*` namespace. */
|
|
227
260
|
function ambientScopeStoreBindings() {
|
|
228
261
|
const names = [];
|
|
@@ -288,7 +321,7 @@ export class CloudflareScopeHost {
|
|
|
288
321
|
scopeStoreRunners;
|
|
289
322
|
scopeStoreVertical;
|
|
290
323
|
scopeStoreLifecycle;
|
|
291
|
-
|
|
324
|
+
scopeStoreDrivers;
|
|
292
325
|
scopeStoreTransport;
|
|
293
326
|
cp;
|
|
294
327
|
servicePrincipals;
|
|
@@ -328,6 +361,7 @@ export class CloudflareScopeHost {
|
|
|
328
361
|
blobStores;
|
|
329
362
|
/** Worker-side attachment-bucket resolver (#473); the ambient per-tenant binding unless overridden (#1995). */
|
|
330
363
|
attachmentBuckets;
|
|
364
|
+
orphanSweepClock;
|
|
331
365
|
/** The parsers attachment text is extracted with (K-43); the host's own, never imported here. */
|
|
332
366
|
attachmentExtractors;
|
|
333
367
|
attachmentTextBounds;
|
|
@@ -397,6 +431,7 @@ export class CloudflareScopeHost {
|
|
|
397
431
|
this.tenantStores = options.tenantStores;
|
|
398
432
|
this.blobStores = options.blobStores;
|
|
399
433
|
this.attachmentBuckets = options.attachmentBuckets ?? ambientAttachmentBucket;
|
|
434
|
+
this.orphanSweepClock = options.orphanSweepClock ?? (() => new Date().toISOString());
|
|
400
435
|
assertAttachmentExtractors(options.attachmentExtractors ?? []);
|
|
401
436
|
this.attachmentExtractors = options.attachmentExtractors ?? [];
|
|
402
437
|
this.attachmentTextBounds = resolveAttachmentTextBounds(options.attachmentTextBounds);
|
|
@@ -405,8 +440,8 @@ export class CloudflareScopeHost {
|
|
|
405
440
|
this.scopeLocalPermissions = options.scopeLocalPermissions ?? false;
|
|
406
441
|
this.scopeNs = options.scope;
|
|
407
442
|
this.scopeStoreVertical = options.scopeStores?.vertical;
|
|
408
|
-
this.
|
|
409
|
-
this.scopeStoreTransport = options
|
|
443
|
+
this.scopeStoreDrivers = options.scopeStores?.drivers ?? {};
|
|
444
|
+
this.scopeStoreTransport = scopeStoreTransportOf(options);
|
|
410
445
|
this.scopeStoreRunners = scopeStoreRunnersOf(options.scopeStores);
|
|
411
446
|
this.scopeStoreLifecycle = scopeStoreLifecycleOf(this.scopeStoreRunners);
|
|
412
447
|
this.cpLess = !options.controlPlane;
|
|
@@ -1022,6 +1057,8 @@ export class CloudflareScopeHost {
|
|
|
1022
1057
|
opts) {
|
|
1023
1058
|
if (opts?.markCopy)
|
|
1024
1059
|
assertCopyLineage(opts.markCopy);
|
|
1060
|
+
// #2314: a load replaces the scope's rows, so the orphan sweep is fenced first, as for a rewind.
|
|
1061
|
+
await this.fenceOrphanSweep(scopeId, opts?.tenantId);
|
|
1025
1062
|
const carry = opts ? recordedOffFromWire(opts) : undefined;
|
|
1026
1063
|
const load = {
|
|
1027
1064
|
switchOff: carry ? { at: new Date().toISOString(), ...carry } : undefined,
|
|
@@ -1254,6 +1291,7 @@ export class CloudflareScopeHost {
|
|
|
1254
1291
|
* audited the ask; this end does not write the access log again.
|
|
1255
1292
|
*/
|
|
1256
1293
|
async scopeStoreSizesLocal(tenantId, scopeId) {
|
|
1294
|
+
this.refuseLocalStoresOnPlatform('size');
|
|
1257
1295
|
return this.readScopeStoreSizes(tenantId, scopeId);
|
|
1258
1296
|
}
|
|
1259
1297
|
/**
|
|
@@ -1261,6 +1299,7 @@ export class CloudflareScopeHost {
|
|
|
1261
1299
|
* plane calls this once on the serving script, then destroys the key itself.
|
|
1262
1300
|
*/
|
|
1263
1301
|
async scopeStoreExportLocal(tenantId, scopeId) {
|
|
1302
|
+
this.refuseLocalStoresOnPlatform('export');
|
|
1264
1303
|
return this.exportScopeStores(tenantId, scopeId);
|
|
1265
1304
|
}
|
|
1266
1305
|
async scopeStoreImportLocal(tenantId, scopeId, stores) {
|
|
@@ -1268,8 +1307,19 @@ export class CloudflareScopeHost {
|
|
|
1268
1307
|
await this.importScopeStores(tenantId, scopeId, stores);
|
|
1269
1308
|
}
|
|
1270
1309
|
async scopeStoreEraseLocal(tenantId, scopeId, subjectId) {
|
|
1310
|
+
this.refuseLocalStoresOnPlatform('erase');
|
|
1271
1311
|
return this.eraseScopeStores(tenantId, scopeId, subjectId);
|
|
1272
1312
|
}
|
|
1313
|
+
/**
|
|
1314
|
+
* On the platform hop this host holds no store object, so it cannot say what its stores
|
|
1315
|
+
* hold: an empty answer would read as "no stores" (#2242). It refuses instead, and the
|
|
1316
|
+
* control plane, which reaches the store script, is the one to ask.
|
|
1317
|
+
*/
|
|
1318
|
+
refuseLocalStoresOnPlatform(what) {
|
|
1319
|
+
if (this.scopeStoreTransport === 'platform') {
|
|
1320
|
+
throw substratError('unavailable', `this host reaches its scope stores through the platform and cannot ${what} them here; ask the control plane`);
|
|
1321
|
+
}
|
|
1322
|
+
}
|
|
1273
1323
|
/**
|
|
1274
1324
|
* The oldest not-yet-drained events of this host's own scope (#1334) — the far end of
|
|
1275
1325
|
* the control plane's `EventDrainDelegation`. Bounded the same way the audited verb is,
|
|
@@ -1321,7 +1371,119 @@ export class CloudflareScopeHost {
|
|
|
1321
1371
|
* run nothing until the switch is back in the scope. See `rewindHolding`.
|
|
1322
1372
|
*/
|
|
1323
1373
|
async rewindScopeLocal(scopeId, bookmark, opts) {
|
|
1324
|
-
|
|
1374
|
+
// #2314: the orphan sweep's fence first. The rewind can take rows away that a later restore
|
|
1375
|
+
// brings back, so their bytes must outlive the PITR window; no fence, no rewind.
|
|
1376
|
+
try {
|
|
1377
|
+
await this.fenceOrphanSweep(scopeId, opts?.tenantId);
|
|
1378
|
+
}
|
|
1379
|
+
catch (err) {
|
|
1380
|
+
throw new Error(`${REWIND_REFUSED}${err instanceof Error ? err.message : String(err)}`);
|
|
1381
|
+
}
|
|
1382
|
+
return this.rewindHolding(scopeId, () => this.scopeStub(scopeId).rewindToBookmark(bookmark, { force: opts?.force }));
|
|
1383
|
+
}
|
|
1384
|
+
/**
|
|
1385
|
+
* #2314: write the orphan sweep's restore fence for a scope whose state is about to be
|
|
1386
|
+
* replaced, into the tenant's attachment bucket, so no pass deletes its bytes while a row the
|
|
1387
|
+
* replacement takes away can still be restored. The tenant is the one named, else the scope's
|
|
1388
|
+
* own provision receipt. A scope whose tenant is neither holds no row the replacement could
|
|
1389
|
+
* take from it unless it holds an attachment row (a fresh scope, a clean-room preview), and
|
|
1390
|
+
* only then is it refused, because its bucket cannot be found. A tenant this script holds no
|
|
1391
|
+
* bucket for has no bytes here, so nothing is fenced. Throws when the fence cannot be written:
|
|
1392
|
+
* the caller refuses the replacement.
|
|
1393
|
+
*/
|
|
1394
|
+
async fenceOrphanSweep(scopeId, named) {
|
|
1395
|
+
let tenantId = named ?? null;
|
|
1396
|
+
if (!tenantId) {
|
|
1397
|
+
const facts = await this.scopeStub(scopeId).sweepFenceFacts();
|
|
1398
|
+
if (!facts.tenant && !facts.holdsAttachments)
|
|
1399
|
+
return;
|
|
1400
|
+
tenantId = facts.tenant;
|
|
1401
|
+
}
|
|
1402
|
+
if (!tenantId) {
|
|
1403
|
+
throw new Error(`the tenant of scope ${scopeId} is unknown (no provision receipt, and none was named), so the ` +
|
|
1404
|
+
`orphan-attachment sweep's restore fence (#2314) cannot be written. Reconcile the scope, or ` +
|
|
1405
|
+
`name its tenant, then retry`);
|
|
1406
|
+
}
|
|
1407
|
+
const bucket = await this.attachmentBuckets(tenantId);
|
|
1408
|
+
if (!bucket)
|
|
1409
|
+
return;
|
|
1410
|
+
try {
|
|
1411
|
+
await fenceAttachmentSweep(r2TenantBlobStore(bucket), scopeId, this.orphanSweepClock());
|
|
1412
|
+
}
|
|
1413
|
+
catch (err) {
|
|
1414
|
+
throw new Error(`the orphan-attachment sweep's restore fence (#2314) could not be written for scope ${scopeId}: ` +
|
|
1415
|
+
`${err instanceof Error ? err.message : String(err)}. Nothing was replaced; retry once the attachment bucket answers`);
|
|
1416
|
+
}
|
|
1417
|
+
}
|
|
1418
|
+
/**
|
|
1419
|
+
* #2314: one orphan-sweep pass over a scope's attachment bytes, judged by THIS deployment's
|
|
1420
|
+
* copy of the scope (`sweepOrphanAttachments` in the kernel has the guards). The caller is the
|
|
1421
|
+
* platform, and only for the deployment that serves the scope: the copy refuses to judge if it
|
|
1422
|
+
* is not the scope's provisioned primary, but a stale copy in an older version's script cannot
|
|
1423
|
+
* tell that it is stale.
|
|
1424
|
+
*/
|
|
1425
|
+
async sweepOrphanAttachments(tenantId, scopeId, request = {}) {
|
|
1426
|
+
// With a directory, it is the word on the scope, and it is read BEFORE the object is addressed,
|
|
1427
|
+
// since addressing a reaped or never-held object would wake an empty one: this tenant's, active,
|
|
1428
|
+
// a primary, and with no other script's copy that may hold rows this one lacks. A CP-less
|
|
1429
|
+
// deployment has none: its scope's own storage is the word (receipt, lifecycle, copy markers,
|
|
1430
|
+
// in the DO), and the copy ledger is the caller's to have read.
|
|
1431
|
+
let vouched = false;
|
|
1432
|
+
if (!this.cpLess) {
|
|
1433
|
+
const record = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
1434
|
+
if (!record)
|
|
1435
|
+
throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
|
|
1436
|
+
if (record.status !== 'active') {
|
|
1437
|
+
throw substratError('conflict', `refused: scope ${scopeId} is ${record.status}, and only an active scope's bytes are swept`);
|
|
1438
|
+
}
|
|
1439
|
+
if (!isPrimaryScopeRow(record)) {
|
|
1440
|
+
throw substratError('conflict', `refused: scope ${scopeId} is a copy, and a copy may not judge orphan attachments`);
|
|
1441
|
+
}
|
|
1442
|
+
// Every ledger entry of the scope, page by page, so no page can hide one: a pass needs all of
|
|
1443
|
+
// them settled (`SETTLED_COPY_STATES` is an allowlist), and stops at the first that is not.
|
|
1444
|
+
let after;
|
|
1445
|
+
for (;;) {
|
|
1446
|
+
const page = (await this.cp.listScopeScriptCopies({ tenantId, scopeId, limit: 100, ...(after ? { after } : {}) })).map(scopeScriptCopyOf);
|
|
1447
|
+
const unsettled = unsettledCopyRefusal(scopeId, page);
|
|
1448
|
+
if (unsettled)
|
|
1449
|
+
throw unsettled;
|
|
1450
|
+
const last = page[page.length - 1];
|
|
1451
|
+
if (page.length < 100 || !last)
|
|
1452
|
+
break;
|
|
1453
|
+
after = { scriptRef: last.scriptRef, moveId: last.moveId };
|
|
1454
|
+
}
|
|
1455
|
+
// The tenant's status too, as every door into a scope asks.
|
|
1456
|
+
await this.assertLive(tenantId, scopeId);
|
|
1457
|
+
vouched = true;
|
|
1458
|
+
}
|
|
1459
|
+
else if (request.copiesSettled !== true) {
|
|
1460
|
+
throw substratError('conflict', `refused: this deployment cannot read the copy ledger, so a pass over scope ${scopeId} needs its caller ` +
|
|
1461
|
+
'to have found no unsettled copy of it (copiesSettled)');
|
|
1462
|
+
}
|
|
1463
|
+
const stub = this.scopeStub(scopeId);
|
|
1464
|
+
const now = this.orphanSweepClock();
|
|
1465
|
+
const expected = request.expectedVersionId;
|
|
1466
|
+
const judge = (condemn) => async (ids, at) => unwrapReply(await stub.orphanAttachmentsReply([...ids], at, tenantId, condemn, vouched, expected));
|
|
1467
|
+
// The scope's copy gives its verdict first, with no ids: every refusal it makes, and whether this
|
|
1468
|
+
// script is the one the platform routed the pass to, in the pass's own mode. So a destructive
|
|
1469
|
+
// pass routed to the wrong script is refused even when there is nothing to condemn.
|
|
1470
|
+
await judge(request.dryRun !== true)([], now);
|
|
1471
|
+
// A script with no attachment store for the tenant holds none of its bytes: a vertical whose
|
|
1472
|
+
// modules keep no files. Answered as that, marked, rather than the throw `resolveAttachmentStore`
|
|
1473
|
+
// makes for a write, so the platform's pass can tell it from a failure and from a store that held
|
|
1474
|
+
// nothing. A scope that holds an attachment row does keep files, so its missing store is a
|
|
1475
|
+
// misconfiguration, and it fails.
|
|
1476
|
+
const bucket = await this.attachmentBuckets(tenantId);
|
|
1477
|
+
if (!bucket) {
|
|
1478
|
+
if ((await stub.sweepFenceFacts()).holdsAttachments)
|
|
1479
|
+
throw missingAttachmentStore(tenantId, scopeId);
|
|
1480
|
+
return { listed: 0, candidates: 0, orphans: 0, deleted: 0, keys: [], dryRun: request.dryRun === true, bucket: 'none' };
|
|
1481
|
+
}
|
|
1482
|
+
const store = r2TenantBlobStore(bucket);
|
|
1483
|
+
return sweepOrphanAttachments(store, scopeId, {
|
|
1484
|
+
rowless: (ids) => judge(false)(ids, now),
|
|
1485
|
+
condemn: judge(true),
|
|
1486
|
+
}, { now, ...(request.dryRun ? { dryRun: true } : {}), ...(request.limit !== undefined ? { limit: request.limit } : {}) });
|
|
1325
1487
|
}
|
|
1326
1488
|
registerModule(registration) {
|
|
1327
1489
|
const manifest = moduleManifest.parse(registration.manifest);
|
|
@@ -1601,6 +1763,46 @@ export class CloudflareScopeHost {
|
|
|
1601
1763
|
const store = await this.resolveAttachmentStore(tenantId);
|
|
1602
1764
|
return this.buildAttachmentSurface({ principal }, tenantId, scopeId, store);
|
|
1603
1765
|
}
|
|
1766
|
+
async inspectUploadLink(tenantId, scopeId, secret, target, idempotencyKey) {
|
|
1767
|
+
await this.assertLive(tenantId, scopeId);
|
|
1768
|
+
await this.migrateAndRecord(scopeId);
|
|
1769
|
+
return unwrapReply(await this.scopeStub(scopeId).uploadLinkInspectReply(secret, tenantId, scopeId, target, idempotencyKey)).view;
|
|
1770
|
+
}
|
|
1771
|
+
async redeemUploadLink(tenantId, scopeId, secret, input) {
|
|
1772
|
+
await this.assertLive(tenantId, scopeId);
|
|
1773
|
+
await this.migrateAndRecord(scopeId);
|
|
1774
|
+
const stub = this.scopeStub(scopeId);
|
|
1775
|
+
// The target first (the kernel judges it before the state), so a wrong path never reads as
|
|
1776
|
+
// gone; a replay (#2315) is answered from the inspection and never reaches the write.
|
|
1777
|
+
const inspect = async () => unwrapReply(await stub.uploadLinkInspectReply(secret, tenantId, scopeId, input.entity, input.idempotencyKey));
|
|
1778
|
+
return redeemOrReplayUploadLink(inspect, input, async ({ view, mintedBy }) => {
|
|
1779
|
+
const store = await this.resolveAttachmentStore(tenantId);
|
|
1780
|
+
// A fresh id per request, so its key is its own: compensating a refusal can never remove
|
|
1781
|
+
// the bytes of the redemption that won the link.
|
|
1782
|
+
const id = ulid();
|
|
1783
|
+
const key = attachmentBlobKey(scopeId, id);
|
|
1784
|
+
const filename = view.filename ?? input.filename;
|
|
1785
|
+
// Recorded as it is judged: the essence, never a parameter or a stray space.
|
|
1786
|
+
const contentType = normalizeUploadMediaType(input.contentType);
|
|
1787
|
+
const record = attachmentRecord.parse({
|
|
1788
|
+
id,
|
|
1789
|
+
entity: view.entity,
|
|
1790
|
+
filename,
|
|
1791
|
+
contentType,
|
|
1792
|
+
size: input.body.byteLength,
|
|
1793
|
+
sha256: await attachmentSha256(input.body),
|
|
1794
|
+
visibility: view.visibility,
|
|
1795
|
+
createdBy: mintedBy,
|
|
1796
|
+
createdAt: new Date().toISOString(),
|
|
1797
|
+
});
|
|
1798
|
+
// Bytes first, row second, exactly as an upload: a refusal (size, type, a lost key, a link
|
|
1799
|
+
// someone else spent first) compensates the object away, and the link is unspent with it.
|
|
1800
|
+
// A lost reply keeps the bytes (`compensatingRefusal`), and a retry carrying the same
|
|
1801
|
+
// `Idempotency-Key` is answered as a replay (#2315).
|
|
1802
|
+
await store.put(key, input.body, { contentType });
|
|
1803
|
+
return unwrapReply(await compensatingRefusal(store, key, () => stub.attachmentRedeemUploadLinkReply(record, secret, tenantId, scopeId, input.idempotencyKey)));
|
|
1804
|
+
});
|
|
1805
|
+
}
|
|
1604
1806
|
async getSystemAttachments(moduleId, tenantId, scopeId) {
|
|
1605
1807
|
// Reuse the system invoke door's module, tenant and lifecycle gate, and its rewind gate
|
|
1606
1808
|
// (#1834). The DO authorizes each open against the attachment target's read permission as
|
|
@@ -1739,16 +1941,10 @@ export class CloudflareScopeHost {
|
|
|
1739
1941
|
createdBy,
|
|
1740
1942
|
createdAt: new Date().toISOString(),
|
|
1741
1943
|
});
|
|
1742
|
-
// Bytes first, row second — a crash between the two leaves an orphaned object
|
|
1743
|
-
//
|
|
1944
|
+
// Bytes first, row second — a crash between the two leaves an orphaned object (#2314);
|
|
1945
|
+
// a refused gate compensates, and only an answered refusal does (`compensatingRefusal`).
|
|
1744
1946
|
await store.put(key, input.body, { contentType: input.contentType });
|
|
1745
|
-
|
|
1746
|
-
return unwrapReply(await stub.attachmentAddReply(record, asPrincipalId, tenantId, scopeId, connectionId));
|
|
1747
|
-
}
|
|
1748
|
-
catch (err) {
|
|
1749
|
-
await store.delete(key).catch(() => { });
|
|
1750
|
-
throw err;
|
|
1751
|
-
}
|
|
1947
|
+
return unwrapReply(await compensatingRefusal(store, key, () => stub.attachmentAddReply(record, asPrincipalId, tenantId, scopeId, connectionId)));
|
|
1752
1948
|
},
|
|
1753
1949
|
list: async (entity) => unwrapReply(await stub.attachmentListReply(entity, asPrincipalId, tenantId, scopeId, connectionId)),
|
|
1754
1950
|
open: async (attachmentId) => {
|
|
@@ -1813,6 +2009,8 @@ export class CloudflareScopeHost {
|
|
|
1813
2009
|
if (!existing)
|
|
1814
2010
|
throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
|
|
1815
2011
|
this.refuseUnusableScopeStores(dump.stores, existing.vertical);
|
|
2012
|
+
// #2314: the load replaces the scope's rows, so the orphan sweep is fenced first.
|
|
2013
|
+
await this.fenceOrphanSweep(scopeId, tenantId);
|
|
1816
2014
|
// #1742: a dump from before a switch was pulled brings the module's grants back live, so
|
|
1817
2015
|
// the directory's recorded-off modules go back off in the replay's own event. Only where
|
|
1818
2016
|
// this is the scope's real store, as for the provision's seat above; a hosted scope is
|
|
@@ -2031,8 +2229,15 @@ export class CloudflareScopeHost {
|
|
|
2031
2229
|
}
|
|
2032
2230
|
return vertical;
|
|
2033
2231
|
}
|
|
2034
|
-
/**
|
|
2232
|
+
/**
|
|
2233
|
+
* Configured runners, or the ambient namespaces when this host was told to use a
|
|
2234
|
+
* direct binding. The platform hop holds no object in this isolate, so an ambient
|
|
2235
|
+
* namespace is not one of its stores, and the admin's store paths answer for none: the
|
|
2236
|
+
* control plane reaches the platform's stores itself (see `eraseScopeStores`).
|
|
2237
|
+
*/
|
|
2035
2238
|
scopeStoreBindingNames() {
|
|
2239
|
+
if (this.scopeStoreTransport === 'platform')
|
|
2240
|
+
return [];
|
|
2036
2241
|
if (this.scopeStoreRunners.size > 0)
|
|
2037
2242
|
return [...this.scopeStoreRunners.keys()];
|
|
2038
2243
|
return ambientScopeStoreBindings();
|
|
@@ -2084,8 +2289,13 @@ export class CloudflareScopeHost {
|
|
|
2084
2289
|
}
|
|
2085
2290
|
/**
|
|
2086
2291
|
* `substrat.erase` on each declared binding (#2068). A store error throws so the
|
|
2087
|
-
* caller can leave the subject key untouched.
|
|
2088
|
-
*
|
|
2292
|
+
* caller can leave the subject key untouched. An empty answer means this host holds no
|
|
2293
|
+
* store object for the scope, never that it could not tell. On the platform hop that is
|
|
2294
|
+
* true by construction (`scopeStoreBindingNames` is empty): the stores live behind the
|
|
2295
|
+
* store script, and control-plane-api erases, exports, sizes and wipes them itself around
|
|
2296
|
+
* its calls to this host's admin (`shredSubject`, `exportScope`, `scopeStoreSizes`, the
|
|
2297
|
+
* wipe and the replay); an erasure reaches them before the host's shred destroys the key. Only the vertical-host `*Local` reads, which no control plane fronts,
|
|
2298
|
+
* refuse on the hop instead (`refuseLocalStoresOnPlatform`).
|
|
2089
2299
|
*/
|
|
2090
2300
|
async eraseScopeStores(tenantId, scopeId, subjectId) {
|
|
2091
2301
|
const bindings = this.scopeStoreBindingNames();
|
|
@@ -2101,23 +2311,23 @@ export class CloudflareScopeHost {
|
|
|
2101
2311
|
}
|
|
2102
2312
|
return erasures;
|
|
2103
2313
|
}
|
|
2104
|
-
/** A runner's
|
|
2105
|
-
|
|
2106
|
-
const
|
|
2107
|
-
if (!
|
|
2108
|
-
throw substratError('unavailable', `scope store '${binding}' has no declared
|
|
2314
|
+
/** A runner's driver, or the driver declared for an ambient binding. Never invented. */
|
|
2315
|
+
scopeStoreDriver(binding) {
|
|
2316
|
+
const driver = this.scopeStoreRunners.get(binding)?.driver ?? this.scopeStoreDrivers[binding];
|
|
2317
|
+
if (!driver) {
|
|
2318
|
+
throw substratError('unavailable', `scope store '${binding}' has no declared driver`);
|
|
2109
2319
|
}
|
|
2110
|
-
return
|
|
2320
|
+
return driver;
|
|
2111
2321
|
}
|
|
2112
2322
|
/**
|
|
2113
2323
|
* A dump's stores this host cannot load are refused before any write (#2211, L8): a
|
|
2114
|
-
* binding that is not bound, a
|
|
2324
|
+
* binding that is not bound, a driver other than the bound one, or any store at all for
|
|
2115
2325
|
* a destination with no vertical (no store object can be named for it, so loading would
|
|
2116
2326
|
* drop them), or a destination on a vertical other than the one this host is pinned to.
|
|
2117
2327
|
* `vertical` undefined is a caller that cannot read the destination here;
|
|
2118
2328
|
* `importScopeStores` then refuses a scope with no vertical itself, still before a store
|
|
2119
|
-
* write. A store with no `
|
|
2120
|
-
* unparsed dump would otherwise skip the
|
|
2329
|
+
* write. A store with no `driver` is refused: the dump contract requires one, and an
|
|
2330
|
+
* unparsed dump would otherwise skip the driver comparison.
|
|
2121
2331
|
*/
|
|
2122
2332
|
refuseUnusableScopeStores(stores, vertical) {
|
|
2123
2333
|
const listed = stores ?? [];
|
|
@@ -2134,12 +2344,12 @@ export class CloudflareScopeHost {
|
|
|
2134
2344
|
if (!bound.has(store.binding)) {
|
|
2135
2345
|
throw substratError('not_found', `dump names scope store '${store.binding}', which is not bound`);
|
|
2136
2346
|
}
|
|
2137
|
-
if (!store.
|
|
2138
|
-
throw substratError('validation_failed', `dump's scope store '${store.binding}' names no
|
|
2347
|
+
if (!store.driver) {
|
|
2348
|
+
throw substratError('validation_failed', `dump's scope store '${store.binding}' names no driver`);
|
|
2139
2349
|
}
|
|
2140
|
-
const
|
|
2141
|
-
if (store.
|
|
2142
|
-
throw substratError('conflict', `dump's scope store '${store.binding}' is
|
|
2350
|
+
const driver = this.scopeStoreRunners.get(store.binding)?.driver ?? this.scopeStoreDrivers[store.binding];
|
|
2351
|
+
if (store.driver !== driver) {
|
|
2352
|
+
throw substratError('conflict', `dump's scope store '${store.binding}' is driver '${store.driver}', and this host binds driver '${driver ?? 'undeclared'}'`);
|
|
2143
2353
|
}
|
|
2144
2354
|
}
|
|
2145
2355
|
}
|
|
@@ -2157,7 +2367,7 @@ export class CloudflareScopeHost {
|
|
|
2157
2367
|
const exported = await this.invokeReservedStore({ tenantId, vertical, scopeId, binding }, 'substrat.export');
|
|
2158
2368
|
stores.push({
|
|
2159
2369
|
binding,
|
|
2160
|
-
|
|
2370
|
+
driver: this.scopeStoreDriver(binding),
|
|
2161
2371
|
tables: exported.tables,
|
|
2162
2372
|
});
|
|
2163
2373
|
}
|
|
@@ -2931,9 +3141,10 @@ export class CloudflareScopeHost {
|
|
|
2931
3141
|
// #119: a purge horizon's schedule runs its operation once per due entity, each its own
|
|
2932
3142
|
// call and transaction — and the SCOPE decides which are due and runs them, so purge
|
|
2933
3143
|
// authority is never an argument any stub can pass (`runPurgeSweep`).
|
|
2934
|
-
|
|
3144
|
+
// #2192: under the run's own id, so the purges and their deliveries join this run's line.
|
|
3145
|
+
const pass = await door.through((instance) => stub.runPurgeSweep(schedule.operation, tenantId, scopeId, instance, invocationId));
|
|
2935
3146
|
// The purges' events, delivered the way each invoke's own tail delivers them.
|
|
2936
|
-
await this.drainExecutors(tenantId, scopeId,
|
|
3147
|
+
await this.drainExecutors(tenantId, scopeId, invocationId);
|
|
2937
3148
|
stillDue = purgeStillDue(pass);
|
|
2938
3149
|
const outcome = purgeReportOf(schedule.operation, schedule.purge.entityType, pass);
|
|
2939
3150
|
report.errors.push(...outcome.errors);
|
|
@@ -4205,12 +4416,12 @@ export class CloudflareScopeHost {
|
|
|
4205
4416
|
});
|
|
4206
4417
|
await this.recordAdmin(actor, 'setHostnameIssuance', { tenantId: row.tenant_id, scopeId: row.scope_id }, { status: row.status }, { status: fields.status, note: fields.note ?? null });
|
|
4207
4418
|
},
|
|
4208
|
-
unbindHostname: async (actor, raw) => {
|
|
4419
|
+
unbindHostname: async (actor, raw, owner) => {
|
|
4209
4420
|
const hostname = raw.toLowerCase(); // DNS is case-insensitive; the map is normalized
|
|
4210
|
-
|
|
4421
|
+
// The row the delete removed, so an owner is judged by the delete itself (#2285).
|
|
4422
|
+
const row = await this.cp.deleteHostname(hostname, owner);
|
|
4211
4423
|
if (!row)
|
|
4212
4424
|
return; // idempotent, and a no-op is not audited
|
|
4213
|
-
await this.cp.deleteHostname(hostname);
|
|
4214
4425
|
await this.recordAdmin(actor, 'unbindHostname', { tenantId: row.tenant_id, scopeId: row.scope_id }, { hostname, status: row.status }, null);
|
|
4215
4426
|
},
|
|
4216
4427
|
listHostnames: async (actor, filter) => {
|
|
@@ -4321,8 +4532,13 @@ export class CloudflareScopeHost {
|
|
|
4321
4532
|
// The SQL migrations are stored apart from the manifest (#1764), so no read of the
|
|
4322
4533
|
// version but the promote review's carries them.
|
|
4323
4534
|
const split = splitManifestMigrations(manifestJson ?? null, 'push');
|
|
4324
|
-
|
|
4535
|
+
// #2339: the digest a promote compares is the one derived from what is stored, never the pushed one.
|
|
4536
|
+
const migrationDigest = await publishedMigrationDigest({
|
|
4537
|
+
id: audited.id, migrationDigest: audited.migrationDigest, manifestJson: manifestJson ?? null, migrations: split.migrations,
|
|
4538
|
+
});
|
|
4539
|
+
await this.directory('insertVersion', {
|
|
4325
4540
|
...audited,
|
|
4541
|
+
migrationDigest,
|
|
4326
4542
|
manifestJson: split.manifestJson,
|
|
4327
4543
|
migrations: split.migrations,
|
|
4328
4544
|
originJson: origin ? JSON.stringify(origin) : null,
|
|
@@ -4332,10 +4548,17 @@ export class CloudflareScopeHost {
|
|
|
4332
4548
|
});
|
|
4333
4549
|
await this.recordAdmin(actor, 'publishVersion', { tenantId: null }, null, {
|
|
4334
4550
|
...audited,
|
|
4551
|
+
// The digest the registry holds, which differs from the pushed one only for an unheld version.
|
|
4552
|
+
migrationDigest,
|
|
4335
4553
|
...(origin ? { origin } : {}),
|
|
4336
4554
|
admission: selfAdmits ? 'admitted' : 'pending',
|
|
4337
4555
|
});
|
|
4338
4556
|
},
|
|
4557
|
+
verticalScopeStores: async (actor, verticalSlug) => {
|
|
4558
|
+
const stores = await this.directory('verticalScopeStores', verticalSlug);
|
|
4559
|
+
await this.recordAccess(actor, 'verticalScopeStores', {}, { verticalSlug }, stores.length);
|
|
4560
|
+
return stores;
|
|
4561
|
+
},
|
|
4339
4562
|
listVersions: async (actor, verticalSlug, page) => {
|
|
4340
4563
|
const rows = await this.cp.listVersions(verticalSlug, page);
|
|
4341
4564
|
await this.recordAccess(actor, 'listVersions', {}, { verticalSlug }, rows.length);
|
|
@@ -4493,6 +4716,20 @@ export class CloudflareScopeHost {
|
|
|
4493
4716
|
], incoming.id, incoming, served || owned.length > 0
|
|
4494
4717
|
? await this.cp.provisionedVersionsOf(verticalSlug, served, owned.length > 0 ? owning.owner_tenant : null)
|
|
4495
4718
|
: []);
|
|
4719
|
+
// #2333: the acknowledgements answer the review they were given against, or none at all.
|
|
4720
|
+
const reviewed = async (v) => ({
|
|
4721
|
+
id: v.id,
|
|
4722
|
+
permissionDigest: v.permission_digest,
|
|
4723
|
+
migrationDigest: v.migration_digest,
|
|
4724
|
+
migrations: (await this.cp.readVersionMigrations(v.id))?.migrations ?? null,
|
|
4725
|
+
});
|
|
4726
|
+
await assertPromoteReviewCurrent({
|
|
4727
|
+
ack,
|
|
4728
|
+
channel,
|
|
4729
|
+
baseId: outgoing?.id ?? null,
|
|
4730
|
+
migrationsChange: outgoing !== undefined && outgoing.migration_digest !== incoming.migration_digest,
|
|
4731
|
+
read: async () => ({ base: outgoing ? await reviewed(outgoing) : null, incoming: await reviewed(incoming) }),
|
|
4732
|
+
});
|
|
4496
4733
|
// §4's checkpoints, at the moment of exposure. A first promotion has
|
|
4497
4734
|
// nothing to diff against — the gate is about change, not existence.
|
|
4498
4735
|
if (outgoing) {
|
|
@@ -4501,8 +4738,7 @@ export class CloudflareScopeHost {
|
|
|
4501
4738
|
`${incoming.permission_digest}) — acknowledge it explicitly to promote`);
|
|
4502
4739
|
}
|
|
4503
4740
|
if (outgoing.migration_digest !== incoming.migration_digest && !ack.migrationChange) {
|
|
4504
|
-
throw substratError('conflict',
|
|
4505
|
-
`${incoming.migration_digest}) — acknowledge it explicitly to promote`);
|
|
4741
|
+
throw substratError('conflict', migrationChangeRefusal(outgoing.migration_digest, incoming.migration_digest));
|
|
4506
4742
|
}
|
|
4507
4743
|
// #1705 PR 3: an export an installed consumer imports, dropped or re-versioned.
|
|
4508
4744
|
if (!ack.exportBreak) {
|
|
@@ -4512,10 +4748,13 @@ export class CloudflareScopeHost {
|
|
|
4512
4748
|
}
|
|
4513
4749
|
}
|
|
4514
4750
|
const promotedAt = new Date().toISOString();
|
|
4515
|
-
|
|
4516
|
-
//
|
|
4517
|
-
//
|
|
4518
|
-
|
|
4751
|
+
// #2333: a compare-and-set. Every gate above — the review binding, the digest checkpoints,
|
|
4752
|
+
// the export breaks — was judged against `outgoing`, across awaits another promote can run
|
|
4753
|
+
// in. The control plane moves the pointer, and writes the timeline row (what makes rollback a
|
|
4754
|
+
// choice among recorded moments, and `at` the PITR anchor a data rollback would rewind to),
|
|
4755
|
+
// only if the pointer still names `outgoing`; the promote that lost the race is refused as
|
|
4756
|
+
// stale and writes nothing.
|
|
4757
|
+
const moved = await this.cp.moveChannel(outgoing?.id ?? null, {
|
|
4519
4758
|
id: ulid(),
|
|
4520
4759
|
vertical_slug: verticalSlug,
|
|
4521
4760
|
channel,
|
|
@@ -4524,6 +4763,8 @@ export class CloudflareScopeHost {
|
|
|
4524
4763
|
actor,
|
|
4525
4764
|
at: promotedAt,
|
|
4526
4765
|
});
|
|
4766
|
+
if (!moved)
|
|
4767
|
+
throw promoteRaceRefusal(channel);
|
|
4527
4768
|
// For a PRIVATE vertical, prod IS what the owner's apps run: re-point the
|
|
4528
4769
|
// owning tenant's live scopes in the same act, so merge-to-main (push +
|
|
4529
4770
|
// promote) is a complete deploy and a rollback promote reaches the running
|
|
@@ -4596,19 +4837,27 @@ export class CloudflareScopeHost {
|
|
|
4596
4837
|
},
|
|
4597
4838
|
bindScopeVersion: async (actor, tenantId, scopeId, versionId, opts) => {
|
|
4598
4839
|
const { v, scope } = await bindTarget(tenantId, scopeId, versionId);
|
|
4599
|
-
|
|
4600
|
-
|
|
4601
|
-
|
|
4840
|
+
// #2347: a rebind judged against both lineages loses its race as stale, so its client reads
|
|
4841
|
+
// the review again; any other conditional bind keeps the refusal it always had.
|
|
4842
|
+
const lost = () => opts?.expectedServing
|
|
4843
|
+
? rebindRaceRefusal(opts.expectedServing.vertical)
|
|
4844
|
+
: substratError('precondition_failed', 'scope binding changed; reload the scope and retry');
|
|
4845
|
+
if (opts?.expectedVersionId !== undefined && scope.vertical_version_id !== opts.expectedVersionId)
|
|
4846
|
+
throw lost();
|
|
4847
|
+
if (opts?.expectedServing && !await this.cp.servesNow(opts.expectedServing))
|
|
4848
|
+
throw lost();
|
|
4602
4849
|
if (opts?.expectedErasureEpoch !== undefined && await this.cp.scopeErasureEpoch(tenantId, scopeId) !== opts.expectedErasureEpoch) {
|
|
4603
4850
|
throw substratError('precondition_failed', 'scope erasure changed; reload the scope and retry');
|
|
4604
4851
|
}
|
|
4605
4852
|
const ack = bindAcknowledgement.parse(opts?.acknowledge ?? {});
|
|
4853
|
+
// #2347: a route moving in the same write is judged with it, as `bindingImpact` is.
|
|
4854
|
+
const servingRef = opts?.servingRef;
|
|
4606
4855
|
// #2178 first: no acknowledgement passes it, so none should be asked for ahead of it.
|
|
4607
|
-
await this.assertBindingShapesKept(mapScope(scope), v);
|
|
4856
|
+
await this.assertBindingShapesKept(mapScope(scope), v, servingRef);
|
|
4608
4857
|
// #1756: an export an app in this tenant imports, dropped or re-versioned by what this
|
|
4609
4858
|
// scope would run. Before the snapshot, so a refused bind leaves nothing behind.
|
|
4610
4859
|
if (!ack.exportBreak) {
|
|
4611
|
-
const breaks = await this.bindBreaks(actor, mapScope(scope), v);
|
|
4860
|
+
const breaks = await this.bindBreaks(actor, mapScope(scope), v, servingRef);
|
|
4612
4861
|
if (breaks.length > 0)
|
|
4613
4862
|
throw substratError('precondition_failed', bindExportBreakRefusal(breaks));
|
|
4614
4863
|
}
|
|
@@ -4620,12 +4869,19 @@ export class CloudflareScopeHost {
|
|
|
4620
4869
|
await this.snapshotScope(actor, tenantId, scopeId);
|
|
4621
4870
|
}
|
|
4622
4871
|
}
|
|
4623
|
-
if (!await this.cp.bindScopeVersion(scopeId, versionId, v.vertical_slug, opts?.expectedVersionId, opts?.expectedErasureEpoch, opts?.confirmMove)) {
|
|
4624
|
-
|
|
4872
|
+
if (!await this.cp.bindScopeVersion(scopeId, versionId, v.vertical_slug, opts?.expectedVersionId, opts?.expectedErasureEpoch, opts?.confirmMove, servingRef, opts?.expectedServing)) {
|
|
4873
|
+
// The erasure and copy-move conditions keep their own refusal: only a moved binding or a
|
|
4874
|
+
// moved serving version is the race a rebind's review answers.
|
|
4875
|
+
const now = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
4876
|
+
const raced = (opts?.expectedVersionId !== undefined && (now?.vertical_version_id ?? null) !== opts.expectedVersionId)
|
|
4877
|
+
|| (opts?.expectedServing !== undefined && !await this.cp.servesNow(opts.expectedServing));
|
|
4878
|
+
throw raced ? lost() : substratError('precondition_failed', 'scope binding changed; reload the scope and retry');
|
|
4625
4879
|
}
|
|
4626
|
-
await this.recordAdmin(actor, 'bindScopeVersion', { tenantId, scopeId }, null, {
|
|
4880
|
+
await this.recordAdmin(actor, 'bindScopeVersion', { tenantId, scopeId }, servingRef === undefined ? null : { servingRef: scope.serving_ref ?? null }, {
|
|
4627
4881
|
versionId, vertical: v.vertical_slug, version: v.version,
|
|
4882
|
+
...(servingRef !== undefined ? { servingRef } : {}),
|
|
4628
4883
|
...(opts?.expectedVersionId !== undefined ? { expectedVersionId: opts.expectedVersionId } : {}),
|
|
4884
|
+
...(opts?.expectedServing ? { expectedServing: opts.expectedServing } : {}),
|
|
4629
4885
|
...(ack.exportBreak ? { acknowledged: ack } : {}),
|
|
4630
4886
|
});
|
|
4631
4887
|
},
|
|
@@ -4806,7 +5062,7 @@ export class CloudflareScopeHost {
|
|
|
4806
5062
|
return { rewindingTo: bookmark };
|
|
4807
5063
|
}
|
|
4808
5064
|
// #1819: the same rewind a dispatched vertical runs, holding what the scope switched off.
|
|
4809
|
-
return this.rewindScopeLocal(scopeId, bookmark, { force: opts?.force });
|
|
5065
|
+
return this.rewindScopeLocal(scopeId, bookmark, { force: opts?.force, tenantId });
|
|
4810
5066
|
},
|
|
4811
5067
|
createOrg: async (actor, input) => {
|
|
4812
5068
|
const parsed = createOrgInput.parse(input);
|
|
@@ -4958,6 +5214,63 @@ export class CloudflareScopeHost {
|
|
|
4958
5214
|
createdAt: r.created_at,
|
|
4959
5215
|
}));
|
|
4960
5216
|
},
|
|
5217
|
+
// -- store-driver releases, rollouts and holds (D-63, #2300) -------------------
|
|
5218
|
+
recordStoreDriverRelease: async (actor, input) => {
|
|
5219
|
+
const at = new Date().toISOString();
|
|
5220
|
+
const { record, created } = await this.directory('recordStoreDriverRelease', { ...input, actor, at });
|
|
5221
|
+
if (created)
|
|
5222
|
+
await this.recordAdmin(actor, 'uploadStoreDriverRelease', { tenantId: null }, null, record);
|
|
5223
|
+
return record;
|
|
5224
|
+
},
|
|
5225
|
+
getStoreDriverRelease: async (actor, driver, sha256) => {
|
|
5226
|
+
const record = await this.cp.getStoreDriverRelease(driver, sha256);
|
|
5227
|
+
await this.recordAccess(actor, 'getStoreDriverRelease', { tenantId: null }, { driver, sha256 }, record ? 1 : 0);
|
|
5228
|
+
return record;
|
|
5229
|
+
},
|
|
5230
|
+
storeDriverState: async (actor, driver) => {
|
|
5231
|
+
const state = await this.cp.storeDriverState(driver);
|
|
5232
|
+
await this.recordAccess(actor, 'storeDriverState', { tenantId: null }, { driver }, state.rollouts.length);
|
|
5233
|
+
return state;
|
|
5234
|
+
},
|
|
5235
|
+
storeDriverDeclarations: async (actor, driver) => {
|
|
5236
|
+
const declared = await this.directory('storeDriverDeclarations', driver);
|
|
5237
|
+
await this.recordAccess(actor, 'storeDriverDeclarations', { tenantId: null }, { driver }, declared.verticals.length);
|
|
5238
|
+
return declared;
|
|
5239
|
+
},
|
|
5240
|
+
storeDriverRolloutPending: async (actor, input) => {
|
|
5241
|
+
const page = await this.directory('storeDriverRolloutPending', input);
|
|
5242
|
+
await this.recordAccess(actor, 'storeDriverRolloutPending', { tenantId: null }, { driver: input.driver, sha256: input.sha256, action: input.action }, page.verticals.length);
|
|
5243
|
+
return page;
|
|
5244
|
+
},
|
|
5245
|
+
storeDriverRolloutWalked: async (actor, input) => {
|
|
5246
|
+
const walked = await this.directory('storeDriverRolloutWalked', input);
|
|
5247
|
+
await this.recordAccess(actor, 'storeDriverRolloutWalked', { tenantId: null }, { driver: input.driver, seq: input.seq }, walked.verticals.length);
|
|
5248
|
+
return walked;
|
|
5249
|
+
},
|
|
5250
|
+
holdStoreDriver: async (_actor, input) => this.directory('holdStoreDriver', input),
|
|
5251
|
+
releaseStoreDriverHold: async (_actor, input) => {
|
|
5252
|
+
await this.cp.releaseStoreDriverHold(input);
|
|
5253
|
+
},
|
|
5254
|
+
beginStoreDriverRollout: async (actor, input) => {
|
|
5255
|
+
const row = await this.directory('beginStoreDriverRollout', { ...input, actor, at: new Date().toISOString() });
|
|
5256
|
+
await this.recordAdmin(actor, input.action === 'rollback' ? 'rollBackStoreDriver' : 'rollOutStoreDriver', { tenantId: null }, null, row);
|
|
5257
|
+
return row;
|
|
5258
|
+
},
|
|
5259
|
+
finishStoreDriverRollout: async (actor, input) => {
|
|
5260
|
+
const row = await this.directory('finishStoreDriverRollout', { ...input, at: new Date().toISOString() });
|
|
5261
|
+
await this.recordAdmin(actor, row.action === 'rollback' ? 'rollBackStoreDriver' : 'rollOutStoreDriver', { tenantId: null }, null, row);
|
|
5262
|
+
return row;
|
|
5263
|
+
},
|
|
5264
|
+
storeDriverRebuild: async (actor, driver) => {
|
|
5265
|
+
const rebuild = await this.cp.storeDriverRebuild(driver);
|
|
5266
|
+
await this.recordAccess(actor, 'storeDriverRebuild', { tenantId: null }, { driver }, rebuild ? 1 : 0);
|
|
5267
|
+
return rebuild;
|
|
5268
|
+
},
|
|
5269
|
+
claimStoreDriverRebuild: async (_actor, input) => this.directory('claimStoreDriverRebuild', { ...input, at: new Date().toISOString() }),
|
|
5270
|
+
advanceStoreDriverRebuild: async (_actor, input) => this.directory('advanceStoreDriverRebuild', { ...input, at: new Date().toISOString() }),
|
|
5271
|
+
recordStoreDriverRebuildRun: async (actor, entry) => {
|
|
5272
|
+
await this.recordAdmin(actor, 'rebuildStoreDriver', { tenantId: null }, null, entry);
|
|
5273
|
+
},
|
|
4961
5274
|
listScopeTables: async (actor, tenantId, scopeId) => {
|
|
4962
5275
|
// K-3 cross-check on the shared directory BEFORE reaching the scope DO: a pair
|
|
4963
5276
|
// that does not resolve is unreachable, never another tenant's database.
|
|
@@ -5347,11 +5660,12 @@ export class CloudflareScopeHost {
|
|
|
5347
5660
|
await this.recordAccess(actor, 'listShreddedSubjects', { tenantId, scopeId }, null, subjects.length);
|
|
5348
5661
|
return { subjects, nextCursor };
|
|
5349
5662
|
},
|
|
5350
|
-
shredSubject: async (actor, tenantId, scopeId, subjectId) => {
|
|
5663
|
+
shredSubject: async (actor, tenantId, scopeId, subjectId, fence) => {
|
|
5351
5664
|
await this.assertScope(tenantId, scopeId);
|
|
5352
|
-
// #1722: a direct shred
|
|
5353
|
-
//
|
|
5354
|
-
|
|
5665
|
+
// #1722: a direct shred records the epoch it ran under, as an orchestrated one records the
|
|
5666
|
+
// epoch its claim compared, so the backfill can order every erasure. With a fence (#2285) it
|
|
5667
|
+
// claims what it read just before the key goes, and records the epoch that claim compared.
|
|
5668
|
+
const stamp = { erasureEpoch: fence?.epoch ?? await this.cp.scopeErasureEpoch(tenantId, scopeId), path: 'direct' };
|
|
5355
5669
|
// Redact the live spine FIRST, destroy the key LAST. Both halves are idempotent and
|
|
5356
5670
|
// a crash between them converges on retry, so the order is decided by which
|
|
5357
5671
|
// half-done state harms the person: dying after the redaction leaves ciphertext in
|
|
@@ -5413,6 +5727,8 @@ export class CloudflareScopeHost {
|
|
|
5413
5727
|
await this.cp.redactSubjectText({ tenantId, scopeId, subjectId, intentIds });
|
|
5414
5728
|
// Scope stores after the scope-side redaction and before the key (#2068).
|
|
5415
5729
|
const storeErasures = await this.eraseScopeStores(tenantId, scopeId, subjectId);
|
|
5730
|
+
if (fence)
|
|
5731
|
+
await this.claimSubjectErasure(tenantId, scopeId, fence);
|
|
5416
5732
|
const at = new Date().toISOString();
|
|
5417
5733
|
const { existed } = await this.subjectKeysFor(tenantId, scopeId).destroy(subjectId, at);
|
|
5418
5734
|
const receipt = subjectShredReceipt.parse({
|
|
@@ -5439,9 +5755,7 @@ export class CloudflareScopeHost {
|
|
|
5439
5755
|
await this.assertScope(tenantId, scopeId);
|
|
5440
5756
|
if (redactions.length === 0)
|
|
5441
5757
|
throw substratError('conflict', 'subject erasure has no confirmed scope redaction');
|
|
5442
|
-
|
|
5443
|
-
throw substratError('precondition_failed', 'scope route or copy inventory changed during subject erasure; retry after it settles');
|
|
5444
|
-
}
|
|
5758
|
+
await this.claimSubjectErasure(tenantId, scopeId, expected);
|
|
5445
5759
|
const intentIds = [...new Set(redactions.flatMap((r) => r.intentIds))];
|
|
5446
5760
|
await this.cp.redactSubjectText({ tenantId, scopeId, subjectId, intentIds });
|
|
5447
5761
|
const { existed } = await this.subjectKeysFor(tenantId, scopeId).destroy(subjectId, new Date().toISOString());
|
|
@@ -5877,6 +6191,10 @@ export class CloudflareScopeHost {
|
|
|
5877
6191
|
const { tenantId, scopeId, ...after } = memberChangeAudit.parse(entry);
|
|
5878
6192
|
await this.recordAdmin(actor, 'manageScopeMember', { tenantId, scopeId }, null, after);
|
|
5879
6193
|
},
|
|
6194
|
+
recordConnectionBindingChange: async (actor, entry) => {
|
|
6195
|
+
const { tenantId, scopeId, ...after } = connectionBindingAudit.parse(entry);
|
|
6196
|
+
await this.recordAdmin(actor, 'bindConnectionScope', { tenantId, scopeId }, null, after);
|
|
6197
|
+
},
|
|
5880
6198
|
recordPeerBindingChange: async (actor, entry) => {
|
|
5881
6199
|
const { tenantId, scopeId, ...after } = peerBindingAudit.parse(entry);
|
|
5882
6200
|
await this.recordAdmin(actor, 'setPeerBinding', { tenantId, scopeId }, null, after);
|
|
@@ -5896,6 +6214,11 @@ export class CloudflareScopeHost {
|
|
|
5896
6214
|
const { tenantId, scopeId, action, ...after } = copyMarkAudit.parse(entry);
|
|
5897
6215
|
await this.recordAdmin(actor, action === 'mark' ? 'markScopeCopy' : 'clearScopeCopyMark', { tenantId, scopeId }, null, after);
|
|
5898
6216
|
},
|
|
6217
|
+
/** #2314: one orphan-attachment pass over a scope, written after the vertical's own pass. */
|
|
6218
|
+
recordOrphanSweep: async (actor, entry) => {
|
|
6219
|
+
const { tenantId, scopeId, ...after } = orphanSweepAudit.parse(entry);
|
|
6220
|
+
await this.recordAdmin(actor, 'sweepOrphanAttachments', { tenantId, scopeId }, null, after);
|
|
6221
|
+
},
|
|
5899
6222
|
accessLog: async (actor, filter) => {
|
|
5900
6223
|
const rows = await this.cp.accessLog({
|
|
5901
6224
|
actor: filter?.actor,
|
|
@@ -6381,6 +6704,12 @@ export class CloudflareScopeHost {
|
|
|
6381
6704
|
* subject-key operation touches anything. Without it a caller could reach another
|
|
6382
6705
|
* tenant's keys by naming their scope id.
|
|
6383
6706
|
*/
|
|
6707
|
+
/** The erasure claim (`SUBJECT_ERASURE_CLAIM_SQL`); a scope that moved refuses with the key kept. */
|
|
6708
|
+
async claimSubjectErasure(tenantId, scopeId, fence) {
|
|
6709
|
+
if (!await this.cp.claimSubjectErasure(tenantId, scopeId, fence)) {
|
|
6710
|
+
throw substratError('precondition_failed', 'scope route or copy inventory changed during subject erasure; retry after it settles');
|
|
6711
|
+
}
|
|
6712
|
+
}
|
|
6384
6713
|
async assertScope(tenantId, scopeId) {
|
|
6385
6714
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
6386
6715
|
if (!rec)
|
|
@@ -6465,6 +6794,11 @@ export class CloudflareScopeHost {
|
|
|
6465
6794
|
*/
|
|
6466
6795
|
async connectionGrantsInScope(tenantId, scopeId) {
|
|
6467
6796
|
await this.validateScopeAccess(tenantId, scopeId);
|
|
6797
|
+
if (this.connectorDelegation) {
|
|
6798
|
+
if (!this.connectorDelegation.grants)
|
|
6799
|
+
throw new Error('connector grant inspection is not configured for the serving deployment');
|
|
6800
|
+
return this.connectorDelegation.grants({ tenantId, scopeId });
|
|
6801
|
+
}
|
|
6468
6802
|
await this.migrateAndRecord(scopeId);
|
|
6469
6803
|
const now = new Date().toISOString();
|
|
6470
6804
|
// Both stores again, but the CF split is not the pure adapter's. The DO holds the
|