@substrat-run/adapter-cloudflare 0.136.0 → 0.138.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/checker.d.ts +7 -1
- package/dist/checker.d.ts.map +1 -1
- package/dist/checker.js +5 -3
- package/dist/checker.js.map +1 -1
- package/dist/control-plane-do.d.ts +123 -52
- package/dist/control-plane-do.d.ts.map +1 -1
- package/dist/control-plane-do.js +321 -73
- package/dist/control-plane-do.js.map +1 -1
- package/dist/do-reply.d.ts +25 -0
- package/dist/do-reply.d.ts.map +1 -0
- package/dist/do-reply.js +17 -0
- package/dist/do-reply.js.map +1 -0
- package/dist/host.d.ts +221 -16
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +1073 -401
- package/dist/host.js.map +1 -1
- package/dist/live-reads.d.ts +39 -1
- package/dist/live-reads.d.ts.map +1 -1
- package/dist/live-reads.js +79 -0
- package/dist/live-reads.js.map +1 -1
- package/dist/scope-do.d.ts +4 -7
- package/dist/scope-do.d.ts.map +1 -1
- package/dist/scope-do.js +595 -134
- package/dist/scope-do.js.map +1 -1
- package/dist/system-door.d.ts +21 -0
- package/dist/system-door.d.ts.map +1 -0
- package/dist/system-door.js +13 -0
- package/dist/system-door.js.map +1 -0
- package/package.json +11 -11
package/dist/scope-do.js
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { REWIND_REFUSED } from './rewind-refusal.js';
|
|
2
|
+
import { SYSTEM_DOOR_MOVED } from './system-door.js';
|
|
2
3
|
import { DurableObject } from 'cloudflare:workers';
|
|
3
4
|
import { ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, domainEvent, domainEventInput, eventId, instant, objectRef, entityObjectRef, assertModuleEmittableType, assertKernelAuthoredType, toWireFailure, grantRefFromProof, principalId, scopeId as scopeIdOf, tenantId as tenantIdOf, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, MAX_PENDING_SWEEP_RUNS, SWEEP_RUNS_KIND, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, listLimitOf, requestFingerprint, substratError, errorCodeOf, assertReplayableDump, REDRAIN_BATCH, } from '@substrat-run/contracts';
|
|
4
|
-
import { ulid, DO_SQL_LIMITS, unknownRoleError, createUlid, assertAllowed, ConnectionSealingKeyUnavailableError, noSealingKeyMessage, sealTo, assertReadOnlyQuery, entitlementDenial, platformRequestHistoryQuery, platformRequestOf, PLATFORM_REQUEST_COLUMNS, PLATFORM_REQUEST_REDACTION_SQL, platformRequestRedactionParams, platformRequestRedactionQuery, intentPayloadCarriesSubject, redactSubjectJobRuns, redactSubjectScopeText, assertRowLimit, assertRowOffset, JOB_RUN_PATCH_SQL, JOB_STEP_RECORD_SQL, DELIVERY_ERROR_REDACTION_SQL, REDACTED_DELIVERY_NOTE, seatScopeTuple, effectiveRoleGrantQuery, switchRecordedOff, switchSystemSchedules, peerGrantsStatus, systemGrantsStatus, systemScheduleState, systemSwitchedOff, denialListQuery, refusalListQuery, mapRefusalRow, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, PermissionDenied, assertImpersonationWrites, assertModuleEnqueueableKind, impersonationStampOf, createAtomic, NotSearchable, isSearchIndexTable, searchIndexDdl, searchIndexPlans, NotListable, listIndexPlans, moduleMigrations, listQuery, cursorOf, searchLimit, searchMatchExpression, searchQuery, ATTACHMENT_TEXT_DDL, attachmentRecordOfRow, enqueueAttachmentText, reconcileAttachmentText, recordAttachmentText, searchAttachments, IDEMPOTENCY_DDL,
|
|
5
|
-
import { isUpgradeRequest, readSubscription, LIVE_FANOUT_LIMIT, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
|
|
5
|
+
import { ulid, DO_SQL_LIMITS, unknownRoleError, createUlid, assertAllowed, ConnectionSealingKeyUnavailableError, noSealingKeyMessage, sealTo, assertReadOnlyQuery, entitlementDenial, platformRequestHistoryQuery, platformRequestOf, PLATFORM_REQUEST_COLUMNS, PLATFORM_REQUEST_REDACTION_SQL, platformRequestRedactionParams, platformRequestRedactionQuery, intentPayloadCarriesSubject, redactSubjectJobRuns, redactSubjectScopeText, assertRowLimit, assertRowOffset, JOB_RUN_PATCH_SQL, JOB_RUN_CLAIM_SQL, JOB_RUN_RENEW_SQL, JOB_RUN_BEGIN_SQL, JOB_RUN_MISS_SQL, JOB_RUN_MISS_SETTLE_SQL, admissionMissOutcome, JOB_LEASE_EXPIRED_NOTE, JOB_STEP_RECORD_SQL, DELIVERY_ERROR_REDACTION_SQL, REDACTED_DELIVERY_NOTE, seatScopeTuple, effectiveRoleGrantQuery, switchRecordedOff, switchSystemSchedules, peerGrantsStatus, systemGrantsStatus, systemScheduleState, systemSwitchedOff, denialListQuery, refusalListQuery, mapRefusalRow, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, PermissionDenied, assertImpersonationWrites, assertModuleEnqueueableKind, impersonationStampOf, createAtomic, NotSearchable, isSearchIndexTable, searchIndexDdl, searchIndexPlans, NotListable, listIndexPlans, moduleMigrations, listQuery, cursorOf, searchLimit, searchMatchExpression, searchQuery, ATTACHMENT_TEXT_DDL, attachmentRecordOfRow, enqueueAttachmentText, reconcileAttachmentText, recordAttachmentText, searchAttachments, IDEMPOTENCY_DDL, REFUSALS_TABLE_DDL, REFUSALS_INDEX, refusalInsert, refusalOf, markGuardRefusal, REFUSALS_REBUILD, refusalsAdmitGuards, assertIdempotencyKey, assertPermissionKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, SCHEDULE_STATE_DDL, SCHEDULE_STATE_REBUILD, scheduleStateHasKind, VERTICAL_EVENTS_DDL, EXPORT_HOPS_SQL, emptyImportResult, IMPORT_CURSORS_SQL, IMPORT_CURSOR_OF_SQL, OUTBOX_MARK_SQL, exportedSinceQuery, emittedSinceQuery, emittedReportOf, EMITTED_REPORT_CAP, IMPORT_CURSOR_ADVANCE_SQL, IMPORT_RECORD_SQL, moveImportCursor, CrossVerticalRegistry, exportReadPlan, exportReadQuery, planExportBatch, withheldNote, JOB_RUN_DDL, JOB_RUN_DUE_AT, jobRunListLimit, SYSTEM_DOOR_WAIT, ancestorsWithin, } from '@substrat-run/kernel';
|
|
6
|
+
import { isUpgradeRequest, readSubscription, LIVE_FANOUT_LIMIT, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, LIVE_WITHIN_HEADER, decodeLiveWithin, } from './live-reads.js';
|
|
7
|
+
import { replyOf } from './do-reply.js';
|
|
6
8
|
import { OperationQueue } from './serialization.js';
|
|
7
9
|
import { doScopedSql, doBuiltColumnsOf, doRedactionSql, doSpineSql } from './sql.js';
|
|
8
|
-
import { actorOf, admitPeer, collectPeers, switchPeer, assertNoSecret, CAPABILITY_DDL, COPY_ORIGIN_DDL, CAPABILITY_EXCHANGE_OPERATION, capabilityAttachmentWriteRefused, createCapabilityVerbs, createEntityEdgeVerbs, exchangeCapability, guardSecrets, mintBecomeCapability, redactSecrets, redactSecretText, moduleLog, asyncInvocationId, asyncLinePass, resolveCapabilitySession, revokeCapabilityAsPlatform, readCapabilityPage, domainEventOf, facetEvents, readDeadLetters, readLifecycleFlow, readHistory, readInvocation, readUndrainedOutbox, walkEventCause, walkEventEffects, } from '@substrat-run/kernel';
|
|
9
|
-
import { createDoTupleChecker, createLocalControlPlaneReader } from './checker.js';
|
|
10
|
+
import { actorOf, admitPeer, collectPeers, switchPeer, peerSubjectRef, subjectGrantState, assertNoSecret, CAPABILITY_DDL, COPY_ORIGIN_DDL, SWITCH_FENCES_DDL, CAPABILITY_EXCHANGE_OPERATION, capabilityAttachmentWriteRefused, createCapabilityVerbs, createEntityEdgeVerbs, exchangeCapability, guardSecrets, mintBecomeCapability, redactSecrets, redactSecretText, moduleLog, asyncInvocationId, asyncLinePass, resolveCapabilitySession, revokeCapabilityAsPlatform, readCapabilityPage, domainEventOf, facetEvents, readDeadLetters, readLifecycleFlow, readOperationSeries, readHistory, readInvocation, readUndrainedOutbox, walkEventCause, walkEventEffects, } from '@substrat-run/kernel';
|
|
11
|
+
import { createDoTupleChecker, createLocalControlPlaneReader, scopeTupleReader } from './checker.js';
|
|
10
12
|
import { CARRIED_AWAY_KEY, COPY_MARK_CLEARED_KEY, KEPT_COPY_REFUSAL, KEPT_DIVERGENT_KEY, LOAD_STAMP_KEY, STORE_LOCAL_META_KEYS, WRITE_REVISION_KEY, carriedAwayDump, isCopyMarkInsert, isWriteStatement, assertNoSpineReference, assertSpineTablesBuilt, capabilitiesForLoad, clearCopyMarker, dumpRowsInsert, isSpineTable, markCopyOrigin, repointScopeGrants, settleCopiedWork, emittedHere, IS_COPY_SQL, isCopyLoad, isLifecycleWrite, readLifecycle, settleLifecycleAfterLoad, writeLifecycle, spineColumnAdditions } from '@substrat-run/kernel';
|
|
11
13
|
/**
|
|
12
14
|
* The key marking a scope DO whose storage was destroyed (`destroyStorage`).
|
|
@@ -15,6 +17,21 @@ import { CARRIED_AWAY_KEY, COPY_MARK_CLEARED_KEY, KEPT_COPY_REFUSAL, KEPT_DIVERG
|
|
|
15
17
|
* without asking the directory.
|
|
16
18
|
*/
|
|
17
19
|
const REAPED_MARKER = '_substrat_reaped';
|
|
20
|
+
/**
|
|
21
|
+
* The `_substrat_meta` key of the tenant a scope was provisioned for (#1738): written by the first
|
|
22
|
+
* projection, never re-pointed, and what every door's pair check holds a request to (#2016).
|
|
23
|
+
*/
|
|
24
|
+
const PROVISIONED_FOR_KEY = 'provisioned_for';
|
|
25
|
+
/** #2016: the refusal of a write that would re-point a scope's tenant, in one wording. */
|
|
26
|
+
const tenantReceiptRefusal = (held, asked) => `refused: this scope was provisioned for tenant ${held}, not ${asked}`;
|
|
27
|
+
/** #2016: the spine tables whose rows say a scope holds state (`ScopeDO.holdsData`). */
|
|
28
|
+
const HOLDS_DATA_TABLES = ['_substrat_migrations', '_substrat_tuples', '_substrat_outbox', '_substrat_schedule_state'];
|
|
29
|
+
/** #2016: why `tenantId` is foreign to a scope (`held`, the receipt `tenantVerdict` read), in one wording. */
|
|
30
|
+
const foreignTenant = (held, tenantId) => held !== null
|
|
31
|
+
? tenantReceiptRefusal(held, tenantId)
|
|
32
|
+
: `refused: this scope holds role rows for another tenant and none for ${tenantId}`;
|
|
33
|
+
/** #2016: the load refusals `tenantReceiptRefusal` raised, told apart by identity, not by text. */
|
|
34
|
+
const tenantRefusals = new WeakSet();
|
|
18
35
|
/**
|
|
19
36
|
* The scope spine, as this adapter builds it — one of two hand-written copies (#969).
|
|
20
37
|
*
|
|
@@ -174,6 +191,7 @@ const KERNEL_DDL = `
|
|
|
174
191
|
taken_at TEXT NOT NULL,
|
|
175
192
|
pending TEXT NOT NULL
|
|
176
193
|
);
|
|
194
|
+
${SWITCH_FENCES_DDL}
|
|
177
195
|
CREATE TABLE IF NOT EXISTS _substrat_tuples (
|
|
178
196
|
subject TEXT NOT NULL,
|
|
179
197
|
relation TEXT NOT NULL,
|
|
@@ -322,7 +340,8 @@ const KERNEL_DDL = `
|
|
|
322
340
|
-- #116: the request-dedupe table, kernel-owned so no vertical migrates for it.
|
|
323
341
|
${IDEMPOTENCY_DDL}
|
|
324
342
|
-- #1745: refused transitions, recorded after the rollback like a denial. Kernel-owned.
|
|
325
|
-
${
|
|
343
|
+
${REFUSALS_TABLE_DDL}
|
|
344
|
+
${REFUSALS_INDEX}
|
|
326
345
|
-- #1672: capabilities — authority carried by a secret (a link share), and the sessions
|
|
327
346
|
-- an exchange trades that secret for. Shared with the pure adapter from
|
|
328
347
|
-- @substrat-run/kernel so the two cannot part company; the column comments are there.
|
|
@@ -644,6 +663,15 @@ export function splitSqlStatements(sql) {
|
|
|
644
663
|
out.push(cur.trim());
|
|
645
664
|
return out;
|
|
646
665
|
}
|
|
666
|
+
/**
|
|
667
|
+
* #1834: the pin missed. Thrown only by `assertSystemDoor`, and module-private, so no operation can
|
|
668
|
+
* throw one: the two RPCs a door call reaches turn it into the `SystemDoorMoved` ANSWER, by class
|
|
669
|
+
* identity, and an operation's own error, whatever its text, stays a failure.
|
|
670
|
+
*/
|
|
671
|
+
class SystemDoorMovedError extends Error {
|
|
672
|
+
}
|
|
673
|
+
/** #1834: the brand only `assertSystemDoor` sets, so nothing else can make a `SystemDoorPass`. */
|
|
674
|
+
const SYSTEM_DOOR_PASSED = Symbol('system door passed');
|
|
647
675
|
export function defineScopeDO(modules, bareOps) {
|
|
648
676
|
return class ScopeDO extends DurableObject {
|
|
649
677
|
sql;
|
|
@@ -1041,7 +1069,16 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1041
1069
|
* On the queue, so no operation is mid-transaction while the outbox is read. An event
|
|
1042
1070
|
* must not reach another vertical before the transaction that wrote it has committed.
|
|
1043
1071
|
*/
|
|
1044
|
-
async exportedEventsRead(input, tenantId, scopeId
|
|
1072
|
+
async exportedEventsRead(input, tenantId, scopeId,
|
|
1073
|
+
/**
|
|
1074
|
+
* #2029: the consumer is a peer, so the read is gated by its door. `instance` pins it to the
|
|
1075
|
+
* gate's read, as every door call is; `held` says the rewind hold keeps the consumer off, and
|
|
1076
|
+
* then it holds nothing here: every key is missing, the answer a switched-off consumer gets.
|
|
1077
|
+
*/
|
|
1078
|
+
door) {
|
|
1079
|
+
const held = door !== undefined && 'held' in door;
|
|
1080
|
+
if (!held && this.peerDoorMoved(input.consumer, door?.instance))
|
|
1081
|
+
return SYSTEM_DOOR_MOVED;
|
|
1045
1082
|
await this.ensureMigrations();
|
|
1046
1083
|
const plan = exportReadPlan(this.crossVertical.exports(), input.wants);
|
|
1047
1084
|
const quiet = {
|
|
@@ -1054,9 +1091,11 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1054
1091
|
};
|
|
1055
1092
|
if (plan.types.length === 0)
|
|
1056
1093
|
return quiet;
|
|
1057
|
-
const missing =
|
|
1058
|
-
|
|
1059
|
-
.
|
|
1094
|
+
const missing = held
|
|
1095
|
+
? plan.keys
|
|
1096
|
+
: (await this.peerCoverage(tenantId, scopeId, input.consumer, plan.keys))
|
|
1097
|
+
.filter((c) => !c.held)
|
|
1098
|
+
.map((c) => c.permission);
|
|
1060
1099
|
if (missing.length > 0)
|
|
1061
1100
|
return { ...quiet, paused: { missing } };
|
|
1062
1101
|
return await this.queue.enqueue(() => {
|
|
@@ -1106,25 +1145,76 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1106
1145
|
* pair check), and a stray role row can no longer make a scope serve a tenant it was never
|
|
1107
1146
|
* provisioned for.
|
|
1108
1147
|
*
|
|
1109
|
-
* A scope with NO receipt was provisioned before it existed
|
|
1110
|
-
*
|
|
1111
|
-
*
|
|
1112
|
-
*
|
|
1113
|
-
* (reconcile, or the restore's repair) writes the receipt, after which
|
|
1114
|
-
* never consulted again.
|
|
1148
|
+
* A scope with NO receipt was provisioned before it existed (a load never brings the dump's:
|
|
1149
|
+
* it describes the scope the dump came from, and the store keeps its own, #2016). Only then
|
|
1150
|
+
* is the old inference used: `provisionScopeLocal` projected role definitions under the
|
|
1151
|
+
* tenant, so a `_substrat_roles` row for it says the same thing. The next projection
|
|
1152
|
+
* (reconcile, or the restore's repair) or lifecycle delivery writes the receipt, after which
|
|
1153
|
+
* the inference is never consulted again.
|
|
1115
1154
|
*
|
|
1116
1155
|
* Read without migrating, so asking about a foreign scope leaves its DO as empty as it found it.
|
|
1117
1156
|
*/
|
|
1118
1157
|
async servesTenant(tenantId) {
|
|
1119
|
-
const
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1158
|
+
const { verdict } = this.tenantVerdict(tenantId);
|
|
1159
|
+
return verdict === 'recorded' || verdict === 'inferred';
|
|
1160
|
+
}
|
|
1161
|
+
/**
|
|
1162
|
+
* #2016: what one door into this scope needs from its storage on a CP-less host, in ONE call —
|
|
1163
|
+
* how `tenantId` reads against the scope (`tenantVerdict`), and the lifecycle the platform
|
|
1164
|
+
* delivered (#1713). The coordinator refuses a foreign pair before any guard, handler or store
|
|
1165
|
+
* lookup runs, and reports an `unknown` one it lets through. Read without migrating, like
|
|
1166
|
+
* `servesTenant`.
|
|
1167
|
+
*/
|
|
1168
|
+
admission(tenantId) {
|
|
1169
|
+
return { verdict: this.tenantVerdict(tenantId).verdict, lifecycle: readLifecycle(this.switchSql()) };
|
|
1170
|
+
}
|
|
1171
|
+
/**
|
|
1172
|
+
* #1738 / #2016: this scope's tenant against `tenantId` — the one reading behind the served-here
|
|
1173
|
+
* gate, every door's pair check and the lifecycle delivery's back-fill.
|
|
1174
|
+
*
|
|
1175
|
+
* - `recorded`: the `provisioned_for` receipt names `tenantId`.
|
|
1176
|
+
* - `foreign`: the receipt names another tenant, or — on a scope provisioned before receipts
|
|
1177
|
+
* existed — the scope holds role rows and none of them is `tenantId`'s.
|
|
1178
|
+
* - `inferred`: no receipt, and a role row for `tenantId` (what `provisionScopeLocal` projected
|
|
1179
|
+
* before the receipt existed).
|
|
1180
|
+
* - `unknown`: no receipt and no role rows — never provisioned here, or loaded from a world
|
|
1181
|
+
* that keeps its roles elsewhere and not yet repaired. Nothing to hold the pair against.
|
|
1182
|
+
*
|
|
1183
|
+
* `held` is the receipt read, so a refusal can name it. Read without migrating, so asking about
|
|
1184
|
+
* a foreign scope leaves its DO as empty as it found it.
|
|
1185
|
+
*/
|
|
1186
|
+
tenantVerdict(tenantId) {
|
|
1187
|
+
const held = this.provisionedFor();
|
|
1188
|
+
if (held !== null)
|
|
1189
|
+
return { verdict: held === tenantId ? 'recorded' : 'foreign', held };
|
|
1190
|
+
if (!this.hasTable('_substrat_roles'))
|
|
1191
|
+
return { verdict: 'unknown', held };
|
|
1192
|
+
if (this.sql.exec('SELECT 1 FROM _substrat_roles WHERE tenant_id = ? LIMIT 1', tenantId).toArray().length > 0) {
|
|
1193
|
+
return { verdict: 'inferred', held };
|
|
1124
1194
|
}
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1195
|
+
const anyRole = this.sql.exec('SELECT 1 FROM _substrat_roles LIMIT 1').toArray().length > 0;
|
|
1196
|
+
return { verdict: anyRole ? 'foreign' : 'unknown', held };
|
|
1197
|
+
}
|
|
1198
|
+
/**
|
|
1199
|
+
* #2016: whether a scope with no receipt and no role rows holds state — rather than being a DO
|
|
1200
|
+
* nothing ever provisioned or loaded here. Only such a scope takes its tenant from a lifecycle
|
|
1201
|
+
* delivery: recording one in an empty DO would make `servesTenant` answer for a scope this
|
|
1202
|
+
* deployment does not hold.
|
|
1203
|
+
*
|
|
1204
|
+
* Every provision since #1738 writes the receipt itself (that IS the durable provision marker),
|
|
1205
|
+
* so this only ever judges a scope from before it, or a load from an older platform. A module
|
|
1206
|
+
* with no SQL migrations still leaves spine state, so any of it counts, not migrations alone:
|
|
1207
|
+
* applied migrations, scope tuples (its grants, `system:` ones included), events, schedule state.
|
|
1208
|
+
*/
|
|
1209
|
+
holdsData() {
|
|
1210
|
+
return HOLDS_DATA_TABLES.some((table) => this.hasTable(table) && this.sql.exec(`SELECT 1 FROM ${table} LIMIT 1`).toArray().length > 0);
|
|
1211
|
+
}
|
|
1212
|
+
/** #1738: the tenant this scope's `provisioned_for` receipt names, or null; read without migrating. */
|
|
1213
|
+
provisionedFor() {
|
|
1214
|
+
return this.hasTable('_substrat_meta') ? this.metaValue(PROVISIONED_FOR_KEY) : null;
|
|
1215
|
+
}
|
|
1216
|
+
hasTable(name) {
|
|
1217
|
+
return this.sql.exec(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?`, name).toArray().length > 0;
|
|
1128
1218
|
}
|
|
1129
1219
|
/** #1705 PR 2: the outbox's insertion mark (`OUTBOX_MARK_SQL`). */
|
|
1130
1220
|
outboxMark() {
|
|
@@ -1144,20 +1234,16 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1144
1234
|
* The coordinator parses the batch before it gets here and runs the executors after.
|
|
1145
1235
|
* Executors live on the coordinator here, as they do after an invoke.
|
|
1146
1236
|
*/
|
|
1147
|
-
async importApply(batch, tenantId, scopeId
|
|
1148
|
-
|
|
1237
|
+
async importApply(batch, tenantId, scopeId,
|
|
1238
|
+
/** #2029: the instance the peer door's gate read for the producer; on another, nothing applies. */
|
|
1239
|
+
doorInstance) {
|
|
1149
1240
|
const source = batch.source;
|
|
1241
|
+
if (this.peerDoorMoved(source.vertical, doorInstance))
|
|
1242
|
+
return SYSTEM_DOOR_MOVED;
|
|
1243
|
+
await this.ensureMigrations();
|
|
1150
1244
|
return await this.queue.enqueue(async () => {
|
|
1151
1245
|
const liveSince = this.liveHighWaterMark();
|
|
1152
|
-
const result =
|
|
1153
|
-
delivered: 0,
|
|
1154
|
-
deadLettered: 0,
|
|
1155
|
-
duplicates: 0,
|
|
1156
|
-
withheld: 0,
|
|
1157
|
-
cursor: batch.after,
|
|
1158
|
-
stale: false,
|
|
1159
|
-
paused: null,
|
|
1160
|
-
};
|
|
1246
|
+
const result = emptyImportResult(batch);
|
|
1161
1247
|
// #1706's door, for a delivery (`operation: null`), inside the queued body: the producer
|
|
1162
1248
|
// is a declared peer with its switch on, or nothing runs and the edge pauses.
|
|
1163
1249
|
let peerSubject;
|
|
@@ -1383,6 +1469,10 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1383
1469
|
lifecycleFlow(input) {
|
|
1384
1470
|
return readLifecycleFlow({ sql: doScopedSql(this.sql) }, input);
|
|
1385
1471
|
}
|
|
1472
|
+
/** #1750: business volumes per bucket, counted over this scope's outbox. */
|
|
1473
|
+
operationSeries(input) {
|
|
1474
|
+
return readOperationSeries({ sql: doScopedSql(this.sql) }, input);
|
|
1475
|
+
}
|
|
1386
1476
|
migrationBookmarks(limit = 20) {
|
|
1387
1477
|
return this.sql
|
|
1388
1478
|
.exec(`SELECT bookmark, taken_at, pending FROM _substrat_migration_bookmarks
|
|
@@ -1625,21 +1715,33 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1625
1715
|
* `vertical.honoured`: an old DO that ignored this argument would run the call as that
|
|
1626
1716
|
* placeholder, and the coordinator refuses a success without the acknowledgement.
|
|
1627
1717
|
*/
|
|
1628
|
-
verticalCaller
|
|
1718
|
+
verticalCaller,
|
|
1719
|
+
/**
|
|
1720
|
+
* #1834: set on a system-door call — the instance whose state read the door's gate
|
|
1721
|
+
* consulted the rewind hold against. On any other instance nothing runs and the answer is
|
|
1722
|
+
* `systemDoorMoved` instead, because a PITR restore always restarts this object, so another
|
|
1723
|
+
* instance may be rewound storage the gate never saw. The reply carries `systemDoor.honoured`.
|
|
1724
|
+
*/
|
|
1725
|
+
systemDoorInstance) {
|
|
1629
1726
|
if (!failureEnvelope) {
|
|
1630
1727
|
// Legacy path, byte-for-byte what it was: rewrapped so a non-plain error (a
|
|
1631
1728
|
// ZodError, whose `message` is a getter) still arrives with its message.
|
|
1632
1729
|
try {
|
|
1633
|
-
return await this.invokeOrThrow(operation, input, principal, tenantId, scopeId, connectionId, requiredEntitlement, systemModuleId, invokeOptions, impersonation, capabilitySession, verticalCaller);
|
|
1730
|
+
return await this.invokeOrThrow(operation, input, principal, tenantId, scopeId, connectionId, requiredEntitlement, systemModuleId, invokeOptions, impersonation, capabilitySession, verticalCaller, systemDoorInstance);
|
|
1634
1731
|
}
|
|
1635
1732
|
catch (err) {
|
|
1733
|
+
if (err instanceof SystemDoorMovedError)
|
|
1734
|
+
return { result: undefined, platformRequests: 0, ...SYSTEM_DOOR_MOVED };
|
|
1636
1735
|
throw toRpcError(err);
|
|
1637
1736
|
}
|
|
1638
1737
|
}
|
|
1639
1738
|
try {
|
|
1640
|
-
return await this.invokeOrThrow(operation, input, principal, tenantId, scopeId, connectionId, requiredEntitlement, systemModuleId, invokeOptions, impersonation, capabilitySession, verticalCaller);
|
|
1739
|
+
return await this.invokeOrThrow(operation, input, principal, tenantId, scopeId, connectionId, requiredEntitlement, systemModuleId, invokeOptions, impersonation, capabilitySession, verticalCaller, systemDoorInstance);
|
|
1641
1740
|
}
|
|
1642
1741
|
catch (err) {
|
|
1742
|
+
// #1834: the pin missed — an answer, never a failure an operation could have produced.
|
|
1743
|
+
if (err instanceof SystemDoorMovedError)
|
|
1744
|
+
return { result: undefined, platformRequests: 0, ...SYSTEM_DOOR_MOVED };
|
|
1643
1745
|
// The ONE place the error keeps its structure: flattened here, rebuilt by the
|
|
1644
1746
|
// coordinator. `toRpcError` is not applied — that exists to make a throw
|
|
1645
1747
|
// survivable, and this is not a throw.
|
|
@@ -1653,7 +1755,9 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1653
1755
|
/** #1672: a capability session's hash. See `invoke` above. */
|
|
1654
1756
|
capabilitySession,
|
|
1655
1757
|
/** #1706: the calling vertical, as the platform named it. See `invoke` above. */
|
|
1656
|
-
verticalCaller
|
|
1758
|
+
verticalCaller,
|
|
1759
|
+
/** #1834: the instance the system door's gate read. See `invoke` above. */
|
|
1760
|
+
systemDoorInstance) {
|
|
1657
1761
|
await this.ensureMigrations();
|
|
1658
1762
|
const handler = this.operations.get(operation);
|
|
1659
1763
|
// `not_found`, not a bare throw (#113): every vertical hand-matched this message
|
|
@@ -1736,6 +1840,9 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1736
1840
|
// one call. Same placement as the SQLite adapter's actor task, for this reason.
|
|
1737
1841
|
this.invocationId = invokeOptions?.invocationId ?? null;
|
|
1738
1842
|
try {
|
|
1843
|
+
// #1834: before anything opens. This instance is the storage the call meets, for the
|
|
1844
|
+
// whole call: a restore restarts the object, so it cannot move under a running body.
|
|
1845
|
+
const systemDoor = this.assertSystemDoor(systemModuleId, systemDoorInstance);
|
|
1739
1846
|
// #1672: the capability session, resolved on EVERY call and here — inside the queued
|
|
1740
1847
|
// body, the one region where this call holds the DO to itself — so nothing can
|
|
1741
1848
|
// revoke between this read and the transaction. Refuses a stale session, a revoked
|
|
@@ -1751,6 +1858,8 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1751
1858
|
// operation off its allowlist are refused `forbidden`; none is a K-35 denial.
|
|
1752
1859
|
let peerSubject;
|
|
1753
1860
|
if (verticalCaller !== undefined) {
|
|
1861
|
+
// #2029: pinned to the instance the peer door's gate read against the rewind hold.
|
|
1862
|
+
this.assertPeerDoor(verticalCaller.vertical, systemDoorInstance);
|
|
1754
1863
|
peerSubject = admitPeer(this.switchSql(), this.peers, verticalCaller, operation);
|
|
1755
1864
|
idempotencySubjectRef = peerSubject;
|
|
1756
1865
|
}
|
|
@@ -1810,7 +1919,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1810
1919
|
// emitted events back as one — verified across `await` in workerd.
|
|
1811
1920
|
try {
|
|
1812
1921
|
await this.revision.transaction(async () => {
|
|
1813
|
-
const ctx = this.operationContext(principal, tenantId, scopeId, undefined, connectionId,
|
|
1922
|
+
const ctx = this.operationContext(principal, tenantId, scopeId, undefined, connectionId, systemDoor, signals, impersonation, operation, capabilityId, minted, undefined, peerSubject);
|
|
1814
1923
|
// #116: a retry is answered from the recording, and nothing else runs
|
|
1815
1924
|
// — not the guards, not the handler, not the permission check inside
|
|
1816
1925
|
// it. Keyed by SUBJECT, so a caller only ever reaches its own
|
|
@@ -1878,6 +1987,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1878
1987
|
impersonation: { honoured: true },
|
|
1879
1988
|
...(capabilitySession !== undefined ? { capability: { honoured: true } } : {}),
|
|
1880
1989
|
...(verticalCaller !== undefined ? { vertical: { honoured: true } } : {}),
|
|
1990
|
+
...(systemDoorInstance !== undefined ? { systemDoor: { honoured: true } } : {}),
|
|
1881
1991
|
...(idempotencyKey !== undefined
|
|
1882
1992
|
? { idempotency: { keyHonoured: true, replayed } }
|
|
1883
1993
|
: {}),
|
|
@@ -1942,6 +2052,8 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1942
2052
|
...(capabilitySession !== undefined ? { capability: { honoured: true } } : {}),
|
|
1943
2053
|
// #1706: the same, for the peer door.
|
|
1944
2054
|
...(verticalCaller !== undefined ? { vertical: { honoured: true } } : {}),
|
|
2055
|
+
// #1834: the same, for the system door's instance pin.
|
|
2056
|
+
...(systemDoorInstance !== undefined ? { systemDoor: { honoured: true } } : {}),
|
|
1945
2057
|
// The acknowledgement the coordinator's skew check reads (#116), on the
|
|
1946
2058
|
// same reasoning as `ifMatchChecked` below and with a sharper failure: a
|
|
1947
2059
|
// DO too old to know about keys would EXECUTE THE OPERATION AGAIN and
|
|
@@ -2022,6 +2134,14 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2022
2134
|
status: 500,
|
|
2023
2135
|
});
|
|
2024
2136
|
}
|
|
2137
|
+
// A narrowing the coordinator sent but this end cannot read is refused, never
|
|
2138
|
+
// dropped: dropping it would open the whole scope's feed — to a vouched
|
|
2139
|
+
// subscriber, one the principal's check was never going to filter (#1853).
|
|
2140
|
+
const withinHeader = request.headers.get(LIVE_WITHIN_HEADER);
|
|
2141
|
+
const within = withinHeader === null ? undefined : decodeLiveWithin(withinHeader);
|
|
2142
|
+
if (withinHeader !== null && !within) {
|
|
2143
|
+
return new Response('live reads: unreadable within narrowing', { status: 500 });
|
|
2144
|
+
}
|
|
2025
2145
|
// A subscriber arriving before the scope's migrations have run would be told
|
|
2026
2146
|
// about events against a schema it cannot read back through. Same gate every
|
|
2027
2147
|
// other entry point takes, for the same reason.
|
|
@@ -2046,6 +2166,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2046
2166
|
tenantId: tenantIdOf.parse(tenantId),
|
|
2047
2167
|
scopeId: scopeIdOf.parse(scopeId),
|
|
2048
2168
|
since: new Date().toISOString(),
|
|
2169
|
+
...(within ? { within } : {}),
|
|
2049
2170
|
});
|
|
2050
2171
|
return new Response(null, { status: 101, webSocket: client });
|
|
2051
2172
|
}
|
|
@@ -2169,6 +2290,11 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2169
2290
|
* to nobody. Knowing that a row exists and changed at 14:02 is information about
|
|
2170
2291
|
* that row, so the empty payload is not what makes this safe; this is.
|
|
2171
2292
|
*
|
|
2293
|
+
* *Narrowed, when the subscription asked to be* (#1853). A socket opened `within` an
|
|
2294
|
+
* entity hears only rows that reach it through live parent edges — each row's
|
|
2295
|
+
* ancestors are walked once per pass, however many sockets ask. A vouched root
|
|
2296
|
+
* replaces the check above, and its frames are `LiveNudge`s.
|
|
2297
|
+
*
|
|
2172
2298
|
* *Never able to fail the operation.* The write has committed and the caller has
|
|
2173
2299
|
* its answer. A socket that has gone away mid-fan-out, or a check that cannot be
|
|
2174
2300
|
* evaluated, costs a subscriber its live update — which it survives, because the
|
|
@@ -2188,6 +2314,25 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2188
2314
|
const announceable = rows.filter((r) => this.liveTargets.has(r.entity_type));
|
|
2189
2315
|
if (announceable.length === 0)
|
|
2190
2316
|
return;
|
|
2317
|
+
/**
|
|
2318
|
+
* Is a row's entity beneath a `within` root (#1853)? Each row's ancestors are walked
|
|
2319
|
+
* once per pass, on first need, so a check per socket is a set lookup — every widget
|
|
2320
|
+
* visitor's root is a different session, and a walk per (row, socket) would read the
|
|
2321
|
+
* same parent edges once per visitor. Post-commit state: a `ctx.relink` in the same
|
|
2322
|
+
* operation has already moved the edge the walk follows.
|
|
2323
|
+
*/
|
|
2324
|
+
const parents = scopeTupleReader(this.sql);
|
|
2325
|
+
const now = new Date().toISOString();
|
|
2326
|
+
const ancestors = new Map();
|
|
2327
|
+
const reaches = async (row, root) => {
|
|
2328
|
+
let up = ancestors.get(row.id);
|
|
2329
|
+
if (!up) {
|
|
2330
|
+
// A walk that cannot answer is a frame not sent — the same fail-closed rule as the check.
|
|
2331
|
+
up = ancestorsWithin(parents, { entityType: row.entity_type, entityId: row.entity_id }, now).catch(() => new Set());
|
|
2332
|
+
ancestors.set(row.id, up);
|
|
2333
|
+
}
|
|
2334
|
+
return (await up).has(`${root.entityType}:${root.entityId}`);
|
|
2335
|
+
};
|
|
2191
2336
|
for (const ws of sockets) {
|
|
2192
2337
|
let subscription = null;
|
|
2193
2338
|
try {
|
|
@@ -2206,8 +2351,10 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2206
2351
|
// assumption about how the subscription was created.
|
|
2207
2352
|
if (subscription.tenantId !== tenantId || subscription.scopeId !== scopeId)
|
|
2208
2353
|
continue;
|
|
2354
|
+
const { within } = subscription;
|
|
2209
2355
|
// One context per subscriber, not per event: `ctx.check` is the expensive part
|
|
2210
|
-
// and the context is only the subject it is evaluated for.
|
|
2356
|
+
// and the context is only the subject it is evaluated for. Built on first use, so
|
|
2357
|
+
// a vouched subscriber — whose frames the walk alone decides — never builds one.
|
|
2211
2358
|
//
|
|
2212
2359
|
// A frame this subscriber does not pass is NOT recorded as a denial (K-35),
|
|
2213
2360
|
// and that is deliberate: `recordDenial` is called on a refused REQUEST, where
|
|
@@ -2218,35 +2365,50 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2218
2365
|
//
|
|
2219
2366
|
// The operation name is carried anyway, for the events a fan-out cannot emit
|
|
2220
2367
|
// but a future reader of this context might.
|
|
2221
|
-
|
|
2368
|
+
let ctx;
|
|
2369
|
+
const { principal } = subscription;
|
|
2370
|
+
const context = () => (ctx ??= this.operationContext(principal, tenantId, scopeId, undefined, undefined, undefined, undefined, undefined, 'live.subscribe'));
|
|
2222
2371
|
for (const row of announceable) {
|
|
2223
|
-
//
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2372
|
+
// Narrowing first: it is memoised across sockets, and a row outside the root
|
|
2373
|
+
// is out whatever the principal holds.
|
|
2374
|
+
if (within && !(await reaches(row, within)))
|
|
2375
|
+
continue;
|
|
2376
|
+
let frame;
|
|
2377
|
+
if (within?.vouched !== undefined) {
|
|
2378
|
+
// The vertical vouched for the root, so the walk above was the whole filter
|
|
2379
|
+
// — and the subscriber holds no read on this row, so it is told only that
|
|
2380
|
+
// something beneath its root changed. Never which row, never how.
|
|
2381
|
+
frame = { kind: 'nudge', id: row.id, at: row.occurred_at };
|
|
2382
|
+
}
|
|
2383
|
+
else {
|
|
2384
|
+
// Non-null: `announceable` is exactly the rows whose type is in the map.
|
|
2385
|
+
const permission = this.liveTargets.get(row.entity_type);
|
|
2386
|
+
let allowed = false;
|
|
2387
|
+
try {
|
|
2388
|
+
const decision = await context().check(permission, {
|
|
2389
|
+
entityType: row.entity_type,
|
|
2390
|
+
entityId: row.entity_id,
|
|
2391
|
+
});
|
|
2392
|
+
allowed = decision.allowed;
|
|
2393
|
+
}
|
|
2394
|
+
catch {
|
|
2395
|
+
// A check that cannot answer is a check that refuses. The alternative —
|
|
2396
|
+
// treating an evaluator failure as an allow — turns an outage in the
|
|
2397
|
+
// permission path into a disclosure, which is the one failure mode this
|
|
2398
|
+
// surface must not have.
|
|
2399
|
+
allowed = false;
|
|
2400
|
+
}
|
|
2401
|
+
if (!allowed)
|
|
2402
|
+
continue;
|
|
2403
|
+
frame = {
|
|
2404
|
+
kind: 'change',
|
|
2405
|
+
id: row.id,
|
|
2406
|
+
type: row.type,
|
|
2228
2407
|
entityType: row.entity_type,
|
|
2229
2408
|
entityId: row.entity_id,
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
}
|
|
2233
|
-
catch {
|
|
2234
|
-
// A check that cannot answer is a check that refuses. The alternative —
|
|
2235
|
-
// treating an evaluator failure as an allow — turns an outage in the
|
|
2236
|
-
// permission path into a disclosure, which is the one failure mode this
|
|
2237
|
-
// surface must not have.
|
|
2238
|
-
allowed = false;
|
|
2409
|
+
at: row.occurred_at,
|
|
2410
|
+
};
|
|
2239
2411
|
}
|
|
2240
|
-
if (!allowed)
|
|
2241
|
-
continue;
|
|
2242
|
-
const frame = {
|
|
2243
|
-
kind: 'change',
|
|
2244
|
-
id: row.id,
|
|
2245
|
-
type: row.type,
|
|
2246
|
-
entityType: row.entity_type,
|
|
2247
|
-
entityId: row.entity_id,
|
|
2248
|
-
at: row.occurred_at,
|
|
2249
|
-
};
|
|
2250
2412
|
try {
|
|
2251
2413
|
ws.send(JSON.stringify(frame));
|
|
2252
2414
|
}
|
|
@@ -2383,8 +2545,19 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2383
2545
|
return record;
|
|
2384
2546
|
}
|
|
2385
2547
|
/** Authorize a job's read as its registered module, never as a person. */
|
|
2386
|
-
async systemAttachmentAuthorize(attachmentId, moduleId, tenantId, scopeId
|
|
2548
|
+
async systemAttachmentAuthorize(attachmentId, moduleId, tenantId, scopeId,
|
|
2549
|
+
/** #1834: the instance the system door's gate read; see `invoke`. */
|
|
2550
|
+
systemDoorInstance) {
|
|
2387
2551
|
await this.ensureMigrations();
|
|
2552
|
+
let systemDoor;
|
|
2553
|
+
try {
|
|
2554
|
+
systemDoor = this.assertSystemDoor(moduleId, systemDoorInstance);
|
|
2555
|
+
}
|
|
2556
|
+
catch (err) {
|
|
2557
|
+
if (err instanceof SystemDoorMovedError)
|
|
2558
|
+
return SYSTEM_DOOR_MOVED;
|
|
2559
|
+
throw toRpcError(err);
|
|
2560
|
+
}
|
|
2388
2561
|
if (!this.modules.has(moduleId)) {
|
|
2389
2562
|
throw toRpcError(substratError('not_found', `module not registered in this scope: ${moduleId}`));
|
|
2390
2563
|
}
|
|
@@ -2393,7 +2566,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2393
2566
|
return null;
|
|
2394
2567
|
const gate = this.attachmentGate(record.entity.entityType);
|
|
2395
2568
|
try {
|
|
2396
|
-
const ctx = this.operationContext(this.systemPrincipal, tenantId, scopeId, undefined, undefined,
|
|
2569
|
+
const ctx = this.operationContext(this.systemPrincipal, tenantId, scopeId, undefined, undefined, systemDoor, undefined, undefined, 'attachments.open');
|
|
2397
2570
|
assertAllowed(await ctx.check(gate.read, record.entity));
|
|
2398
2571
|
}
|
|
2399
2572
|
catch (err) {
|
|
@@ -2457,13 +2630,10 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2457
2630
|
*/
|
|
2458
2631
|
async attachmentSearch(term, limit, principal, tenantId, scopeId, connectionId) {
|
|
2459
2632
|
await this.ensureMigrations();
|
|
2460
|
-
|
|
2633
|
+
return replyOf(() => {
|
|
2461
2634
|
const ctx = this.operationContext(principal, tenantId, scopeId, undefined, connectionId, undefined, undefined, undefined, 'attachments.search');
|
|
2462
|
-
return
|
|
2463
|
-
}
|
|
2464
|
-
catch (err) {
|
|
2465
|
-
return { failure: toWireFailure(err) };
|
|
2466
|
-
}
|
|
2635
|
+
return this.searchAttachmentsAs(ctx, term, limit);
|
|
2636
|
+
});
|
|
2467
2637
|
}
|
|
2468
2638
|
/**
|
|
2469
2639
|
* #1575: the extraction job's read of one record. No gate, deliberately — extraction
|
|
@@ -2585,6 +2755,70 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2585
2755
|
async systemScheduleState(moduleId) {
|
|
2586
2756
|
return systemScheduleState(this.switchSql(), moduleId, new Date().toISOString());
|
|
2587
2757
|
}
|
|
2758
|
+
/**
|
|
2759
|
+
* #1834: the system door's state read — `systemScheduleState`, and the instance that
|
|
2760
|
+
* answered it. The door consults the rewind hold after this read and pins every call it
|
|
2761
|
+
* then makes to this instance (`assertSystemDoor`).
|
|
2762
|
+
*/
|
|
2763
|
+
async systemDoorState(moduleId) {
|
|
2764
|
+
return { state: await this.systemScheduleState(moduleId), instance: this.instanceId };
|
|
2765
|
+
}
|
|
2766
|
+
/**
|
|
2767
|
+
* #1834: the system door's check on this side, and the ONLY maker of the `SystemDoorPass`
|
|
2768
|
+
* the operation context needs to act as `system:<moduleId>`. A call acting as a module must
|
|
2769
|
+
* carry the instance the host's door gate read (refused without one: no door gated it), and
|
|
2770
|
+
* is refused on any other instance. A PITR restore always restarts this object, so a call
|
|
2771
|
+
* landing on a new instance may meet rewound storage whose module the gate never checked
|
|
2772
|
+
* against the rewind hold. Both are refused before anything opens; the moved one is the
|
|
2773
|
+
* module-private `SystemDoorMovedError`, which the RPC answers as `SystemDoorMoved`, so the door
|
|
2774
|
+
* gates again and retries. No module, no pass.
|
|
2775
|
+
*/
|
|
2776
|
+
assertSystemDoor(moduleId, expected) {
|
|
2777
|
+
if (moduleId === undefined)
|
|
2778
|
+
return undefined;
|
|
2779
|
+
this.assertDoorPin(`module '${moduleId}'`, 'system door', expected);
|
|
2780
|
+
return { moduleId, [SYSTEM_DOOR_PASSED]: true };
|
|
2781
|
+
}
|
|
2782
|
+
/** #2029: the peer door's pin, for a call acting as `vertical:<slug>` — `assertDoorPin`. */
|
|
2783
|
+
assertPeerDoor(vertical, expected) {
|
|
2784
|
+
this.assertDoorPin(`vertical '${vertical}'`, 'peer door', expected);
|
|
2785
|
+
}
|
|
2786
|
+
/**
|
|
2787
|
+
* #2029: the peer door's pin for an RPC that answers a missed pin as `SystemDoorMoved` rather
|
|
2788
|
+
* than through `invoke`'s envelope: true when it missed. A call with no pin still throws.
|
|
2789
|
+
*/
|
|
2790
|
+
peerDoorMoved(vertical, expected) {
|
|
2791
|
+
try {
|
|
2792
|
+
this.assertPeerDoor(vertical, expected);
|
|
2793
|
+
return false;
|
|
2794
|
+
}
|
|
2795
|
+
catch (err) {
|
|
2796
|
+
if (err instanceof SystemDoorMovedError)
|
|
2797
|
+
return true;
|
|
2798
|
+
throw err;
|
|
2799
|
+
}
|
|
2800
|
+
}
|
|
2801
|
+
/**
|
|
2802
|
+
* #1834's pin, for any door (#2029: the peer door's too). A call acting as a switched subject
|
|
2803
|
+
* must carry the instance the host's door gate read, and is refused on any other instance —
|
|
2804
|
+
* the moved one as the module-private `SystemDoorMovedError`, so the door re-gates.
|
|
2805
|
+
*/
|
|
2806
|
+
assertDoorPin(what, door, expected) {
|
|
2807
|
+
// Strict, and deliberately so: no pin means no door gated this call, so it is refused rather
|
|
2808
|
+
// than run unchecked. The one innocent caller is a worker a deploy behind, still running host
|
|
2809
|
+
// code from before the pin, during a rolling deploy. Its refusal is transient: the host and this
|
|
2810
|
+
// object ship in the same bundle, so the window lasts as long as the rollout. Nothing ran, and
|
|
2811
|
+
// the reason says "not now" (`SYSTEM_DOOR_WAIT`) to whoever reads it: a schedule fires on the
|
|
2812
|
+
// next pass, a job run retries on its next drive, and a peer caller retries.
|
|
2813
|
+
if (expected === undefined) {
|
|
2814
|
+
throw substratError('forbidden', `a call acting as ${what} reached this scope without passing the ${door}`, {
|
|
2815
|
+
reason: SYSTEM_DOOR_WAIT,
|
|
2816
|
+
});
|
|
2817
|
+
}
|
|
2818
|
+
if (expected !== this.instanceId) {
|
|
2819
|
+
throw new SystemDoorMovedError(`the scope restarted after the ${door}'s gate read it; gate it again`);
|
|
2820
|
+
}
|
|
2821
|
+
}
|
|
2588
2822
|
/**
|
|
2589
2823
|
* The pre-#1666 read, kept for ONE reason: a coordinator a deploy behind this DO still
|
|
2590
2824
|
* calls it. It answers the new question, not the old one — the old predicate counted any
|
|
@@ -2614,8 +2848,10 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2614
2848
|
* `switchSystemSchedules`, serialized on the queue with every other tuple write and run
|
|
2615
2849
|
* as one `transactionSync`, so its reads and writes are one unit.
|
|
2616
2850
|
*/
|
|
2617
|
-
async switchSystemSchedules(moduleId, scopeId, to, at
|
|
2618
|
-
|
|
2851
|
+
async switchSystemSchedules(moduleId, scopeId, to, at, tenantHeld = false,
|
|
2852
|
+
/** #2045: the switch call's fence — the scope refuses a move older than the one it applied. */
|
|
2853
|
+
fence) {
|
|
2854
|
+
const outcome = await this.queue.enqueue(() => this.revision.transactionSync(() => switchSystemSchedules(this.switchSql(), { moduleId, scopeId, to, at, tenantHeld, fence })));
|
|
2619
2855
|
// #1819: the instance that applied it, which the rewind hold's release rule reads.
|
|
2620
2856
|
return { ...outcome, instance: this.instanceId };
|
|
2621
2857
|
}
|
|
@@ -2636,8 +2872,25 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2636
2872
|
* Move one PEER's kill switch on this scope (#1706) — the kernel's `switchPeer`, which is
|
|
2637
2873
|
* the schedule switch's statement with a `vertical:` subject. Queued, one transaction.
|
|
2638
2874
|
*/
|
|
2639
|
-
async switchPeer(vertical, scopeId, to, at
|
|
2640
|
-
|
|
2875
|
+
async switchPeer(vertical, scopeId, to, at,
|
|
2876
|
+
/** #2030: the directory holds a live tenant-level `vertical:` grant for the peer. */
|
|
2877
|
+
tenantHeld = false,
|
|
2878
|
+
/** #2045: the switch call's fence — the scope refuses a move older than the one it applied. */
|
|
2879
|
+
fence) {
|
|
2880
|
+
const outcome = await this.queue.enqueue(() => this.revision.transactionSync(() => switchPeer(this.switchSql(), { vertical, scopeId, to, at, tenantHeld, fence })));
|
|
2881
|
+
// #2029: the instance that applied it, which the rewind hold's release rule reads.
|
|
2882
|
+
return { ...outcome, instance: this.instanceId };
|
|
2883
|
+
}
|
|
2884
|
+
/**
|
|
2885
|
+
* #2029: the peer door's state read — where one peer stands here (`subjectGrantState` over its
|
|
2886
|
+
* subject, the predicate `admitPeer` refuses on), and the instance that answered it. The door
|
|
2887
|
+
* consults the rewind hold after this read and pins every call it then makes to this instance.
|
|
2888
|
+
*/
|
|
2889
|
+
async peerDoorState(vertical) {
|
|
2890
|
+
return {
|
|
2891
|
+
state: subjectGrantState(this.switchSql(), peerSubjectRef(vertical), new Date().toISOString()),
|
|
2892
|
+
instance: this.instanceId,
|
|
2893
|
+
};
|
|
2641
2894
|
}
|
|
2642
2895
|
/**
|
|
2643
2896
|
* Does peer `vertical` hold each key at this scope's node now (#1706) — the checker's own
|
|
@@ -2645,7 +2898,15 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2645
2898
|
* is the target scope only to satisfy the type: `covers` records nothing, and the tuple ref
|
|
2646
2899
|
* it walks is the slug alone.
|
|
2647
2900
|
*/
|
|
2648
|
-
async peerCovers(tenantId, scopeId, vertical, permissions
|
|
2901
|
+
async peerCovers(tenantId, scopeId, vertical, permissions,
|
|
2902
|
+
/** #2029: the instance the peer door's gate read; on another, nothing is read. */
|
|
2903
|
+
doorInstance) {
|
|
2904
|
+
if (this.peerDoorMoved(vertical, doorInstance))
|
|
2905
|
+
return SYSTEM_DOOR_MOVED;
|
|
2906
|
+
return this.peerCoverage(tenantId, scopeId, vertical, permissions);
|
|
2907
|
+
}
|
|
2908
|
+
/** `peerCovers`' read, for this object's own callers, which gated the peer themselves. */
|
|
2909
|
+
async peerCoverage(tenantId, scopeId, vertical, permissions) {
|
|
2649
2910
|
await this.ensureMigrations();
|
|
2650
2911
|
return this.queue.enqueue(async () => {
|
|
2651
2912
|
const coverage = await this.checker.covers({ kind: 'vertical', id: vertical, scope: scopeId }, permissions, { tenantId, scopeId });
|
|
@@ -2658,9 +2919,11 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2658
2919
|
* so no invoke's open transaction is read half-done. `null` when the tenant defines no such
|
|
2659
2920
|
* role: a typed error thrown here would reach the host flattened, so the host types it.
|
|
2660
2921
|
*/
|
|
2661
|
-
async canAssignFor(tenantId, scopeId, principal, roleKey
|
|
2922
|
+
async canAssignFor(tenantId, scopeId, principal, roleKey,
|
|
2923
|
+
/** #1184: bound at the TENANT node instead — a membership executor assigning tenant-wide. */
|
|
2924
|
+
atTenant = false) {
|
|
2662
2925
|
await this.ensureMigrations();
|
|
2663
|
-
return this.queue.enqueue(() => this.assignmentBound({ kind: 'principal', id: principal }, tenantId, scopeId, roleKey));
|
|
2926
|
+
return this.queue.enqueue(() => this.assignmentBound({ kind: 'principal', id: principal }, tenantId, atTenant ? null : scopeId, roleKey));
|
|
2664
2927
|
}
|
|
2665
2928
|
/** Check and grant in one serialized scope task; a refusal never writes a tuple. */
|
|
2666
2929
|
async assignScopeRoleBoundedFor(tenantId, scopeId, caller, assignee, roleKey) {
|
|
@@ -2701,7 +2964,9 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2701
2964
|
// the host can tell storage the rewind will discard from storage it restored. On ONE object
|
|
2702
2965
|
// per deployment, the one named `SWITCH_HOLDS_NAME` and never a scope's own: the claims
|
|
2703
2966
|
// themselves, because a rewind replaces the rewound scope's whole storage. The claims table
|
|
2704
|
-
// is created on first use, so no scope ever carries it.
|
|
2967
|
+
// is created on first use, so no scope ever carries it. #2029: `module_id` holds a hold KEY —
|
|
2968
|
+
// a module id, bare as #1819 stored it, or a peer's `vertical:<slug>`, which no module id can
|
|
2969
|
+
// spell (a module id has no `:`).
|
|
2705
2970
|
/** This instance, and nothing before or after it: a restart is a new id. */
|
|
2706
2971
|
instanceId = crypto.randomUUID();
|
|
2707
2972
|
/** Whether this instance armed a rewind: then its writes are discarded at the restart. */
|
|
@@ -3061,9 +3326,16 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
3061
3326
|
});
|
|
3062
3327
|
return row;
|
|
3063
3328
|
}
|
|
3064
|
-
/**
|
|
3065
|
-
|
|
3066
|
-
|
|
3329
|
+
/**
|
|
3330
|
+
* #2034 (#2042 review r1): the FENCE for a coordinator from before leases. Its only caller is
|
|
3331
|
+
* that coordinator's drive, which re-reads a run here and then runs the pass WITHOUT claiming
|
|
3332
|
+
* it, so a row handed back could run beside a claimed pass. It answers "no such run" to it,
|
|
3333
|
+
* always: the old drive skips, and only a coordinator that claims ever drives. Nothing else
|
|
3334
|
+
* reads a run through this method (the operator read is `jobRunList`). The coordinator ships in
|
|
3335
|
+
* the same script as this class, so an old one exists only for a deploy's overlap.
|
|
3336
|
+
*/
|
|
3337
|
+
async jobRunById(_id) {
|
|
3338
|
+
return null;
|
|
3067
3339
|
}
|
|
3068
3340
|
/** Insert a fresh run. The coordinator has already refused a non-queue-safe payload. */
|
|
3069
3341
|
async jobRunInsert(row) {
|
|
@@ -3073,24 +3345,25 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
3073
3345
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, row.id, row.module_id, row.job, row.instance, row.payload, row.status, row.cursor, row.counters, row.attempts, row.last_error, row.started_at, row.updated_at, row.next_attempt_at, row.ended_at, row.subject_id ?? null);
|
|
3074
3346
|
}
|
|
3075
3347
|
/**
|
|
3076
|
-
* `running` runs whose backoff has elapsed, oldest first, after `afterId
|
|
3077
|
-
*
|
|
3078
|
-
* The cursor is what lets the coordinator page past runs it cannot drive: it
|
|
3079
|
-
* skips any whose job this deployment does not register, and without a cursor
|
|
3080
|
-
* those rows head every batch forever (`runDueJobRuns`). `afterId` is LAST, as
|
|
3081
|
-
* every argument added to an RPC on this interface must be.
|
|
3348
|
+
* Was: `running` runs whose backoff has elapsed, oldest first, after `afterId` — the due read of
|
|
3349
|
+
* a coordinator from before #1834, which drove every row it returned. Now always empty (below).
|
|
3082
3350
|
*/
|
|
3083
|
-
async jobRunsDue(
|
|
3351
|
+
async jobRunsDue(_now, _limit, _afterId) {
|
|
3352
|
+
// #2034 (#2042 review r1): fenced, as `jobRunById` is. Its only caller is a coordinator from
|
|
3353
|
+
// before #1834, which runs every row this returns without claiming it.
|
|
3354
|
+
return [];
|
|
3355
|
+
}
|
|
3356
|
+
/**
|
|
3357
|
+
* #1834: the drive's ONE snapshot of due runs — keys only, in the order they became due
|
|
3358
|
+
* (`JOB_RUN_DUE_AT`, then id). Spelled exactly as the pure adapter spells it. A coordinator
|
|
3359
|
+
* from before #2034 reads this too, but drives nothing from it: its re-read (`jobRunById`)
|
|
3360
|
+
* is fenced.
|
|
3361
|
+
*/
|
|
3362
|
+
async jobRunsDueKeys(now, max) {
|
|
3084
3363
|
return this.sql
|
|
3085
|
-
.exec(
|
|
3086
|
-
// `afterId` bound TWICE rather than as `?2`: mixing anonymous and numbered
|
|
3087
|
-
// parameters makes the anonymous ones resume from the highest index used,
|
|
3088
|
-
// which is a footgun for the next person to add a clause. Spelled exactly
|
|
3089
|
-
// as the pure adapter spells it.
|
|
3090
|
-
`SELECT * FROM _substrat_job_runs
|
|
3364
|
+
.exec(`SELECT id, module_id, job FROM _substrat_job_runs
|
|
3091
3365
|
WHERE status = 'running' AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
|
|
3092
|
-
|
|
3093
|
-
ORDER BY id LIMIT ?`, now, afterId ?? null, afterId ?? null, limit)
|
|
3366
|
+
ORDER BY ${JOB_RUN_DUE_AT}, id LIMIT ?`, now, max)
|
|
3094
3367
|
.toArray();
|
|
3095
3368
|
}
|
|
3096
3369
|
/** The operator read, newest first. */
|
|
@@ -3123,20 +3396,99 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
3123
3396
|
* partial update that wrote three of the four would leave a run carrying the
|
|
3124
3397
|
* last failure's error beside the new cursor, which reads as broken forever.
|
|
3125
3398
|
*/
|
|
3126
|
-
async jobRunPatch(id, patch) {
|
|
3127
|
-
// A compare-and-set on `running` (#1632) — see
|
|
3128
|
-
|
|
3399
|
+
async jobRunPatch(id, patch, owner) {
|
|
3400
|
+
// A compare-and-set on `running` (#1632) and on the pass's lease (#2034) — see
|
|
3401
|
+
// `JOB_RUN_PATCH_SQL`. `owner` is LAST and optional: a coordinator from before leases
|
|
3402
|
+
// sends none, and patches the unleased rows it drives.
|
|
3403
|
+
return this.sql.exec(JOB_RUN_PATCH_SQL, ...this.jobPatchArgs(id, patch, owner)).rowsWritten > 0;
|
|
3404
|
+
}
|
|
3405
|
+
jobPatchArgs(id, patch, owner) {
|
|
3406
|
+
return [
|
|
3407
|
+
patch.status, patch.cursor, patch.counters, patch.attempts, patch.lastError,
|
|
3408
|
+
patch.updatedAt, patch.nextAttemptAt, patch.endedAt, id, owner ?? null,
|
|
3409
|
+
];
|
|
3129
3410
|
}
|
|
3130
|
-
/**
|
|
3411
|
+
/**
|
|
3412
|
+
* #2034: the claim — `JOB_RUN_CLAIM_SQL`, and whether it took over a lease, read in ONE
|
|
3413
|
+
* `transactionSync`: the single round trip is what makes it indivisible, as in
|
|
3414
|
+
* `jobRunStartOrJoin`. Null = the run is no longer running and due.
|
|
3415
|
+
*/
|
|
3416
|
+
async jobRunClaim(id, owner, leaseMs) {
|
|
3417
|
+
return this.revision.transactionSync(() => {
|
|
3418
|
+
// #2042 r2, r4: a takeover is charged only when the lease it takes had BEGUN its pass.
|
|
3419
|
+
const before = this.sql.exec('SELECT lease_began_at FROM _substrat_job_runs WHERE id = ?', id).toArray()[0];
|
|
3420
|
+
// #2042 r4: the due test and the expiry are THIS object's clock, never the coordinator's.
|
|
3421
|
+
const now = Date.now();
|
|
3422
|
+
const at = new Date(now).toISOString();
|
|
3423
|
+
const claimed = this.sql
|
|
3424
|
+
.exec(JOB_RUN_CLAIM_SQL, owner, new Date(now + leaseMs).toISOString(), at, JOB_LEASE_EXPIRED_NOTE, id, at)
|
|
3425
|
+
.toArray()[0];
|
|
3426
|
+
return claimed ? { run: claimed, takeover: (before?.lease_began_at ?? null) !== null } : null;
|
|
3427
|
+
});
|
|
3428
|
+
}
|
|
3429
|
+
/**
|
|
3430
|
+
* #2042 r3, r4: an admission miss — `JOB_RUN_MISS_SQL` (the relative count) and
|
|
3431
|
+
* `JOB_RUN_MISS_SETTLE_SQL` (the backoff or failure that count calls for) in ONE transaction,
|
|
3432
|
+
* by this object's clock. Null = the claim no longer held the run, or had begun.
|
|
3433
|
+
*/
|
|
3434
|
+
async jobRunMiss(id, owner, note) {
|
|
3435
|
+
return this.revision.transactionSync(() => {
|
|
3436
|
+
const at = new Date(Date.now()).toISOString();
|
|
3437
|
+
const counted = this.sql.exec(JOB_RUN_MISS_SQL, at, id, owner).toArray()[0];
|
|
3438
|
+
if (!counted)
|
|
3439
|
+
return null;
|
|
3440
|
+
const o = admissionMissOutcome(counted.admission_misses, at, note);
|
|
3441
|
+
this.sql.exec(JOB_RUN_MISS_SETTLE_SQL, o.status, o.nextAttemptAt, o.endedAt, o.lastError, id, counted.admission_misses);
|
|
3442
|
+
return { misses: counted.admission_misses, failed: o.status === 'failed' };
|
|
3443
|
+
});
|
|
3444
|
+
}
|
|
3445
|
+
/**
|
|
3446
|
+
* #2034 (#2042 r4): BEGIN a claimed pass — `JOB_RUN_BEGIN_SQL`, the commitment point: one
|
|
3447
|
+
* compare-and-set judged by this object's clock as it runs, never by a time the coordinator
|
|
3448
|
+
* computed. The coordinator invokes the handler if and only if this wrote.
|
|
3449
|
+
*/
|
|
3450
|
+
async jobRunBegin(id, owner, marginMs) {
|
|
3451
|
+
const now = Date.now();
|
|
3452
|
+
return (this.sql.exec(JOB_RUN_BEGIN_SQL, new Date(now).toISOString(), id, owner, new Date(now + marginMs).toISOString())
|
|
3453
|
+
.rowsWritten > 0);
|
|
3454
|
+
}
|
|
3455
|
+
/**
|
|
3456
|
+
* #2034: a step boundary — renew the pass's lease, then read the step's ledger row, in one
|
|
3457
|
+
* round trip. A pass that lost its lease reads nothing and stops.
|
|
3458
|
+
*/
|
|
3459
|
+
async jobStepBegin(runId, step, owner, leaseMs) {
|
|
3460
|
+
// #2042 r4: renewed to this object's now plus the lease.
|
|
3461
|
+
const until = new Date(Date.now() + leaseMs).toISOString();
|
|
3462
|
+
if (this.sql.exec(JOB_RUN_RENEW_SQL, until, runId, owner).rowsWritten === 0)
|
|
3463
|
+
return { held: false, row: null };
|
|
3464
|
+
return { held: true, row: this.stepRow(runId, step) };
|
|
3465
|
+
}
|
|
3466
|
+
/**
|
|
3467
|
+
* One step's ledger row — a non-null `result` is what means completed. Not called by a
|
|
3468
|
+
* coordinator since #2034 (`jobStepBegin` reads it while renewing); kept for one a deploy behind.
|
|
3469
|
+
*/
|
|
3131
3470
|
async jobStepRow(runId, step) {
|
|
3471
|
+
return this.stepRow(runId, step);
|
|
3472
|
+
}
|
|
3473
|
+
stepRow(runId, step) {
|
|
3132
3474
|
return (this.sql
|
|
3133
3475
|
.exec('SELECT step, result, attempts, last_error FROM _substrat_job_steps WHERE run_id = ? AND step = ?', runId, step)
|
|
3134
3476
|
.toArray()[0] ?? null);
|
|
3135
3477
|
}
|
|
3136
3478
|
/** Record one step attempt. `result` non-null = it completed and must not re-run. */
|
|
3137
|
-
async jobStepRecord(runId, step, result, attempts, lastError, at) {
|
|
3138
|
-
// Only while the run is still `running` (#1632)
|
|
3139
|
-
this
|
|
3479
|
+
async jobStepRecord(runId, step, result, attempts, lastError, at, owner, leaseMs) {
|
|
3480
|
+
// Only while the run is still `running` (#1632) and the pass holds its lease (#2034),
|
|
3481
|
+
// renewing the lease — to this object's now plus `leaseMs` (#2042 r4) — in the same
|
|
3482
|
+
// transaction; see `JOB_STEP_RECORD_SQL`. A coordinator from before leases sends neither,
|
|
3483
|
+
// holds no lease, and renews nothing.
|
|
3484
|
+
return this.revision.transactionSync(() => {
|
|
3485
|
+
const until = leaseMs === undefined ? null : new Date(Date.now() + leaseMs).toISOString();
|
|
3486
|
+
if (until !== null && this.sql.exec(JOB_RUN_RENEW_SQL, until, runId, owner ?? null).rowsWritten === 0) {
|
|
3487
|
+
return false;
|
|
3488
|
+
}
|
|
3489
|
+
this.sql.exec(JOB_STEP_RECORD_SQL, runId, step, result, attempts, lastError, at, runId, owner ?? null);
|
|
3490
|
+
return true;
|
|
3491
|
+
});
|
|
3140
3492
|
}
|
|
3141
3493
|
/**
|
|
3142
3494
|
* A COMMITTED pass: the run's new state and the dropping of its step ledger, in
|
|
@@ -3149,13 +3501,17 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
3149
3501
|
* results, not to the cursor) then skips work it never did. The kernel's
|
|
3150
3502
|
* `runJobPass` carries the full argument.
|
|
3151
3503
|
*/
|
|
3152
|
-
async jobCommitPass(id, patch) {
|
|
3504
|
+
async jobCommitPass(id, patch, owner) {
|
|
3153
3505
|
// `transactionSync` with both statements inline — NOT `await
|
|
3154
3506
|
// this.jobRunPatch(...)` then the delete. The await is an output-gate
|
|
3155
3507
|
// boundary, which is precisely the gap this method exists to close.
|
|
3156
|
-
this.revision.transactionSync(() => {
|
|
3157
|
-
|
|
3508
|
+
return this.revision.transactionSync(() => {
|
|
3509
|
+
// #2034: the ledger goes only with a patch that applied — a stale holder's commit must
|
|
3510
|
+
// not empty the ledger of the pass that took the run over.
|
|
3511
|
+
if (this.sql.exec(JOB_RUN_PATCH_SQL, ...this.jobPatchArgs(id, patch, owner)).rowsWritten === 0)
|
|
3512
|
+
return false;
|
|
3158
3513
|
this.sql.exec('DELETE FROM _substrat_job_steps WHERE run_id = ?', id);
|
|
3514
|
+
return true;
|
|
3159
3515
|
});
|
|
3160
3516
|
}
|
|
3161
3517
|
// -- guards (K-17) --------------------------------------------------------
|
|
@@ -3174,7 +3530,15 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
3174
3530
|
throw new Error(`unknown guard predicate: '${guard.predicate}' — declared by ${guard.declaredBy} ` +
|
|
3175
3531
|
`before '${operation}'; no registered module contributes it (operation blocked)`);
|
|
3176
3532
|
}
|
|
3177
|
-
|
|
3533
|
+
try {
|
|
3534
|
+
await predicate.handler(ctx, guard.config, input);
|
|
3535
|
+
}
|
|
3536
|
+
catch (err) {
|
|
3537
|
+
// #1745: a guard refusing is recorded after the rollback, as a transition is —
|
|
3538
|
+
// marked here, where which guard threw is still known. Always rethrown.
|
|
3539
|
+
markGuardRefusal(err, guard.predicate, operation);
|
|
3540
|
+
throw err;
|
|
3541
|
+
}
|
|
3178
3542
|
}
|
|
3179
3543
|
}
|
|
3180
3544
|
// -- migrations (port of applyPendingMigrations) --------------------------
|
|
@@ -3324,10 +3688,28 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
3324
3688
|
* older than the one held. Bookkeeping, like the copy marker: it changes what runs here, not
|
|
3325
3689
|
* the scope's data, so it does not advance the write revision a carry fences on.
|
|
3326
3690
|
*/
|
|
3327
|
-
setLifecycle(next
|
|
3691
|
+
setLifecycle(next,
|
|
3692
|
+
/** #2016: the tenant the directory delivered this lifecycle for. A scope it is foreign to
|
|
3693
|
+
* (`tenantVerdict`) refuses the delivery untouched; one provisioned before the receipt
|
|
3694
|
+
* existed, whose role rows name this tenant, records it — the back-fill. */
|
|
3695
|
+
tenantId) {
|
|
3696
|
+
const { verdict, held } = tenantId === undefined ? { verdict: null, held: null } : this.tenantVerdict(tenantId);
|
|
3697
|
+
if (verdict === 'foreign') {
|
|
3698
|
+
return { refused: 'tenant', message: `lifecycle delivery ${foreignTenant(held, tenantId)}` };
|
|
3699
|
+
}
|
|
3700
|
+
// The back-fill: a scope provisioned before the receipt (its role rows agree), or one whose
|
|
3701
|
+
// tenant nothing recorded although it holds data (a load from a world that keeps its roles
|
|
3702
|
+
// elsewhere). The directory's word records it; a DO never provisioned here records nothing.
|
|
3703
|
+
const backfill = verdict === 'inferred' || (verdict === 'unknown' && this.holdsData());
|
|
3328
3704
|
let out;
|
|
3329
3705
|
this.revision.transactionSync(() => {
|
|
3330
|
-
|
|
3706
|
+
// A write the carry fences on, not bookkeeping: it happens once per legacy scope, and
|
|
3707
|
+
// over-counting only refuses a restore that could have landed. A carried-away copy takes
|
|
3708
|
+
// no write at all, so it is left to the next projection.
|
|
3709
|
+
if (backfill && !this.carriedAwayCopy) {
|
|
3710
|
+
this.sql.exec(`INSERT INTO _substrat_meta (key, value) VALUES (?, ?)`, PROVISIONED_FOR_KEY, tenantId);
|
|
3711
|
+
}
|
|
3712
|
+
out = { ...this.revision.bookkeeping(() => writeLifecycle(this.switchSql(), next)), tenantRecorded: this.provisionedFor() !== null };
|
|
3331
3713
|
});
|
|
3332
3714
|
return out;
|
|
3333
3715
|
}
|
|
@@ -3582,13 +3964,26 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
3582
3964
|
system: isSystemTable(name),
|
|
3583
3965
|
}));
|
|
3584
3966
|
}
|
|
3585
|
-
/** A bounded page of one table. Unknown table names throw — never queried blind.
|
|
3967
|
+
/** A bounded page of one table. Unknown table names throw — never queried blind. Kept for a
|
|
3968
|
+
* coordinator from before #113; the current one calls `introspectTableReply`. */
|
|
3586
3969
|
introspectTable(table, limit, offset) {
|
|
3970
|
+
try {
|
|
3971
|
+
return this.tablePage(table, limit, offset);
|
|
3972
|
+
}
|
|
3973
|
+
catch (e) {
|
|
3974
|
+
throw toRpcError(e);
|
|
3975
|
+
}
|
|
3976
|
+
}
|
|
3977
|
+
/** `introspectTable`, its refusal answered as DATA so its code survives the hop (#113). */
|
|
3978
|
+
introspectTableReply(table, limit, offset) {
|
|
3979
|
+
return replyOf(() => this.tablePage(table, limit, offset));
|
|
3980
|
+
}
|
|
3981
|
+
tablePage(table, limit, offset) {
|
|
3587
3982
|
const known = new Set(this.sql
|
|
3588
3983
|
.exec(`SELECT name FROM sqlite_master WHERE type = 'table'`)
|
|
3589
3984
|
.toArray().map((r) => r.name));
|
|
3590
3985
|
if (!known.has(table))
|
|
3591
|
-
throw
|
|
3986
|
+
throw substratError('not_found', `unknown table '${table}'`);
|
|
3592
3987
|
// The ceiling clamps; a bound SQLite would misread (NaN, non-finite, fractional,
|
|
3593
3988
|
// negative) is refused instead of reaching LIMIT / OFFSET (#1632).
|
|
3594
3989
|
const l = Math.min(assertRowLimit('limit', limit), SCOPE_TABLE_PAGE_MAX);
|
|
@@ -3607,8 +4002,21 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
3607
4002
|
* (no sqlite3_stmt_readonly analogue), a transaction that ALWAYS rolls back — a
|
|
3608
4003
|
* statement the gate misclassified still cannot persist a write. Rows are capped
|
|
3609
4004
|
* at SCOPE_QUERY_ROW_MAX with `truncated` set, never an error.
|
|
4005
|
+
* Kept for a coordinator from before #113; the current one calls `introspectQueryReply`.
|
|
3610
4006
|
*/
|
|
3611
4007
|
async introspectQuery(sql) {
|
|
4008
|
+
try {
|
|
4009
|
+
return await this.readOnlyQuery(sql);
|
|
4010
|
+
}
|
|
4011
|
+
catch (e) {
|
|
4012
|
+
throw toRpcError(e);
|
|
4013
|
+
}
|
|
4014
|
+
}
|
|
4015
|
+
/** `introspectQuery`, its refusal answered as DATA so its code survives the hop (#113). */
|
|
4016
|
+
introspectQueryReply(sql) {
|
|
4017
|
+
return replyOf(() => this.readOnlyQuery(sql));
|
|
4018
|
+
}
|
|
4019
|
+
async readOnlyQuery(sql) {
|
|
3612
4020
|
const stmt = assertReadOnlyQuery(sql);
|
|
3613
4021
|
let result;
|
|
3614
4022
|
const rollback = new Error('read-only console rollback');
|
|
@@ -3632,7 +4040,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
3632
4040
|
}
|
|
3633
4041
|
catch (e) {
|
|
3634
4042
|
if (e !== rollback)
|
|
3635
|
-
throw
|
|
4043
|
+
throw e;
|
|
3636
4044
|
}
|
|
3637
4045
|
return result;
|
|
3638
4046
|
}
|
|
@@ -3753,6 +4161,12 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
3753
4161
|
'ALTER TABLE _substrat_tuples ADD COLUMN revoked_at TEXT',
|
|
3754
4162
|
// #1632: legacy runs retain an unknown subject; no content-based backfill.
|
|
3755
4163
|
'ALTER TABLE _substrat_job_runs ADD COLUMN subject_id TEXT',
|
|
4164
|
+
// #2034: the lease. NULL = nobody holds the run, which is right for every row already there.
|
|
4165
|
+
'ALTER TABLE _substrat_job_runs ADD COLUMN lease_owner TEXT',
|
|
4166
|
+
// #2042 r2, r4: whether the holder BEGAN its pass. NULL for any lease already there: free to take over.
|
|
4167
|
+
'ALTER TABLE _substrat_job_runs ADD COLUMN lease_began_at TEXT',
|
|
4168
|
+
// #2042 r3: consecutive admission misses. NULL = none, right for every run already there.
|
|
4169
|
+
'ALTER TABLE _substrat_job_runs ADD COLUMN admission_misses INTEGER',
|
|
3756
4170
|
// Executor retry state (#100). The defaults read as "terminal", which is
|
|
3757
4171
|
// right for every row already there: each is a completed delivery or a
|
|
3758
4172
|
// consumer dead-letter.
|
|
@@ -3826,6 +4240,14 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
3826
4240
|
// both adapters by the query-plan test rather than by that gate.
|
|
3827
4241
|
this.sql.exec('CREATE INDEX IF NOT EXISTS _substrat_outbox_invocation ON _substrat_outbox (invocation_id, id)');
|
|
3828
4242
|
this.ensureScheduleStateKind();
|
|
4243
|
+
this.ensureRefusalsAdmitGuards();
|
|
4244
|
+
}
|
|
4245
|
+
/**
|
|
4246
|
+
* #1745: `_substrat_refusals`, rebuilt to admit a guard row (a nullable `from_state`, plus
|
|
4247
|
+
* `guard` and `reason`) on a scope DO whose table predates it.
|
|
4248
|
+
*/
|
|
4249
|
+
ensureRefusalsAdmitGuards() {
|
|
4250
|
+
this.rebuildIfStale('_substrat_refusals', refusalsAdmitGuards, REFUSALS_REBUILD);
|
|
3829
4251
|
}
|
|
3830
4252
|
/**
|
|
3831
4253
|
* #1288: `_substrat_schedule_state`, rebuilt with `kind` in its key on a scope DO
|
|
@@ -3837,23 +4259,31 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
3837
4259
|
* the stored DDL is the one probe both adapters can make.
|
|
3838
4260
|
*/
|
|
3839
4261
|
ensureScheduleStateKind() {
|
|
4262
|
+
this.rebuildIfStale('_substrat_schedule_state', scheduleStateHasKind, SCHEDULE_STATE_REBUILD);
|
|
4263
|
+
}
|
|
4264
|
+
/**
|
|
4265
|
+
* Run a kernel create-copy-drop-rename `script` over `table` when the stored DDL fails
|
|
4266
|
+
* `isCurrent` — the one shape `ensureScheduleStateKind` and `ensureRefusalsAdmitGuards`
|
|
4267
|
+
* share, so the pure adapter's twin rebuilds byte-identically.
|
|
4268
|
+
*/
|
|
4269
|
+
rebuildIfStale(table, isCurrent, script) {
|
|
3840
4270
|
const row = this.sql
|
|
3841
|
-
.exec(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`,
|
|
4271
|
+
.exec(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`, table)
|
|
3842
4272
|
.toArray()[0];
|
|
3843
|
-
if (!row ||
|
|
4273
|
+
if (!row || isCurrent(row.sql))
|
|
3844
4274
|
return;
|
|
3845
|
-
// In a transaction, for the reason the kernel
|
|
4275
|
+
// In a transaction, for the reason the kernel constants spell out: create-copy-
|
|
3846
4276
|
// drop-rename has two intermediate states and both are unrecoverable on the next
|
|
3847
4277
|
// wake. `transactionSync`, not the async one every operation uses — the DO
|
|
3848
4278
|
// runtime forbids a manual BEGIN through `sql.exec`, and this body is wholly
|
|
3849
4279
|
// synchronous, which is the one case the sync API is for (it commits at the
|
|
3850
4280
|
// first await, and there is none). It also has to be sync because the caller is.
|
|
3851
4281
|
this.revision.transactionSync(() => {
|
|
3852
|
-
for (const stmt of splitSqlStatements(
|
|
4282
|
+
for (const stmt of splitSqlStatements(script))
|
|
3853
4283
|
this.sql.exec(stmt);
|
|
3854
4284
|
});
|
|
3855
4285
|
}
|
|
3856
|
-
async importDump(tables, destScopeId, { switchOff, sourceScopeId, exact, loadStamp, expect, resolveKept, markCopy, } = {}) {
|
|
4286
|
+
async importDump(tables, destScopeId, { switchOff, sourceScopeId, exact, loadStamp, expect, resolveKept, markCopy, provisionedFor, } = {}) {
|
|
3857
4287
|
// The WHOLE drop-then-replay runs under deferred foreign keys, in one transaction.
|
|
3858
4288
|
//
|
|
3859
4289
|
// Two distinct FK hazards, and both need the deferral — the second is why it cannot
|
|
@@ -3892,6 +4322,15 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
3892
4322
|
const before = this.loadMarker();
|
|
3893
4323
|
// #1713: the lifecycle the platform delivered here, read before the drops take it.
|
|
3894
4324
|
const lifecycleBefore = readLifecycle(this.switchSql());
|
|
4325
|
+
// #2016: this store's own tenant receipt, read before the drops take it. A load never
|
|
4326
|
+
// changes which tenant the scope belongs to: the platform's word for it (`provisionedFor`)
|
|
4327
|
+
// must agree with the receipt held, and is refused before the first drop if it does not.
|
|
4328
|
+
const receiptBefore = this.provisionedFor();
|
|
4329
|
+
if (provisionedFor !== undefined && receiptBefore !== null && receiptBefore !== provisionedFor) {
|
|
4330
|
+
const refused = substratError('conflict', tenantReceiptRefusal(receiptBefore, provisionedFor));
|
|
4331
|
+
tenantRefusals.add(refused);
|
|
4332
|
+
throw refused;
|
|
4333
|
+
}
|
|
3895
4334
|
// #1722 (Codex #2008 r7): a kept copy holds writes nothing else has. No load replaces it
|
|
3896
4335
|
// except its own resolution, and that only at the revision the operator acted on.
|
|
3897
4336
|
const kept = this.metaValue(KEPT_DIVERGENT_KEY);
|
|
@@ -3962,9 +4401,15 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
3962
4401
|
this.sql.exec(insert, ...row);
|
|
3963
4402
|
}
|
|
3964
4403
|
// #1738: a dump's `provisioned_for` names the scope (and tenant) it was captured from,
|
|
3965
|
-
// not this one
|
|
3966
|
-
//
|
|
3967
|
-
|
|
4404
|
+
// not this one, so it is never carried over. #2016: what stays is THIS store's own —
|
|
4405
|
+
// the platform's word when the load carries it (a copy into a fresh scope records its
|
|
4406
|
+
// own tenant here), else the receipt held before the drops. A store that held none
|
|
4407
|
+
// keeps none, and the repair projection that follows writes it.
|
|
4408
|
+
this.sql.exec(`DELETE FROM _substrat_meta WHERE key = ?`, PROVISIONED_FOR_KEY);
|
|
4409
|
+
const receipt = provisionedFor ?? receiptBefore;
|
|
4410
|
+
if (receipt !== null) {
|
|
4411
|
+
this.sql.exec(`INSERT INTO _substrat_meta (key, value) VALUES (?, ?)`, PROVISIONED_FOR_KEY, receipt);
|
|
4412
|
+
}
|
|
3968
4413
|
// #1722: whatever stamp an export read here no longer describes this store, so every load
|
|
3969
4414
|
// replaces it: with the carry's own, or with none. And the write revision moves on from
|
|
3970
4415
|
// where it stood before the drops, so it never goes back.
|
|
@@ -4077,6 +4522,8 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
4077
4522
|
const code = errorCodeOf(e);
|
|
4078
4523
|
if (code === 'precondition_failed')
|
|
4079
4524
|
return { refused: 'changed' };
|
|
4525
|
+
if (e instanceof Error && tenantRefusals.has(e))
|
|
4526
|
+
return { refused: 'tenant', message: e.message };
|
|
4080
4527
|
if (code === 'conflict')
|
|
4081
4528
|
return { refused: 'kept' };
|
|
4082
4529
|
throw toRpcError(e);
|
|
@@ -4485,9 +4932,9 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
4485
4932
|
* survives — the whole point, since the denial is the write the operation could not make.
|
|
4486
4933
|
*/
|
|
4487
4934
|
/** K-42: the session a refused call ran under travels with the denial row. */
|
|
4488
|
-
/** #1745: a refused transition the operation failed with, written after its rollback. */
|
|
4935
|
+
/** #1745: a refused transition or refusing guard the operation failed with, written after its rollback. */
|
|
4489
4936
|
recordRefusal(subject, tenantId, scopeId, operation, err, invocationId, impersonation) {
|
|
4490
|
-
const refused =
|
|
4937
|
+
const refused = refusalOf(err);
|
|
4491
4938
|
if (!refused)
|
|
4492
4939
|
return;
|
|
4493
4940
|
const q = refusalInsert({
|
|
@@ -4538,7 +4985,12 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
4538
4985
|
invocationId, new Date().toISOString());
|
|
4539
4986
|
}
|
|
4540
4987
|
// -- operation context (port of operationContext) -------------------------
|
|
4541
|
-
operationContext(principal, tenantId, scopeId, systemActor, connectionId,
|
|
4988
|
+
operationContext(principal, tenantId, scopeId, systemActor, connectionId,
|
|
4989
|
+
/**
|
|
4990
|
+
* #1834: the system subject, only as the door check hands it out (`assertSystemDoor`) — so
|
|
4991
|
+
* no path can act as `system:<moduleId>` without passing that check.
|
|
4992
|
+
*/
|
|
4993
|
+
systemDoor,
|
|
4542
4994
|
/** #458: per-invoke tally of `ctx.requestPlatform` calls; absent for consumer dispatch. */
|
|
4543
4995
|
signals,
|
|
4544
4996
|
/**
|
|
@@ -4603,8 +5055,8 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
4603
5055
|
? peerSubject
|
|
4604
5056
|
: capabilityId
|
|
4605
5057
|
? { kind: 'capability', id: capabilityId }
|
|
4606
|
-
:
|
|
4607
|
-
? { kind: 'system', id:
|
|
5058
|
+
: systemDoor
|
|
5059
|
+
? { kind: 'system', id: systemDoor.moduleId }
|
|
4608
5060
|
: connectionId
|
|
4609
5061
|
? { kind: 'connection', id: connectionId }
|
|
4610
5062
|
: { kind: 'principal', id: principal };
|
|
@@ -4990,6 +5442,10 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
4990
5442
|
this.sql.exec(`INSERT OR REPLACE INTO _substrat_meta (key, value) VALUES ('permission_source', ?)`, source);
|
|
4991
5443
|
});
|
|
4992
5444
|
}
|
|
5445
|
+
/** `applyProjection`, its refusal answered as DATA so its code survives the hop (#113). */
|
|
5446
|
+
applyProjectionReply(...args) {
|
|
5447
|
+
return replyOf(() => this.applyProjection(...args));
|
|
5448
|
+
}
|
|
4993
5449
|
/**
|
|
4994
5450
|
* Replace this scope's projected view of a tenant's roles + tenant-level tuples
|
|
4995
5451
|
* with a fresh snapshot, and make it authoritative (source = 'local'). A full
|
|
@@ -5048,14 +5504,19 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
5048
5504
|
// #1738: the receipt `servesTenant` reads. First writer wins: absent or equal is written,
|
|
5049
5505
|
// a receipt for ANOTHER tenant refuses the whole projection (K-3), before a single row
|
|
5050
5506
|
// moves, so a misdirected projection can never re-point a scope or leave its roles behind.
|
|
5051
|
-
// A restore does not trip this: `importDump`
|
|
5052
|
-
// projection that follows finds
|
|
5053
|
-
|
|
5054
|
-
|
|
5055
|
-
|
|
5507
|
+
// A restore does not trip this: `importDump` keeps the destination's own receipt and
|
|
5508
|
+
// never the dump's (#2016), so the repair projection that follows finds its own tenant.
|
|
5509
|
+
// #2016: a scope with no receipt yet takes its first one only where its role rows agree
|
|
5510
|
+
// (`tenantVerdict`, the same reading the lifecycle back-fill uses), so a projection for
|
|
5511
|
+
// another tenant cannot pin a legacy scope to it.
|
|
5512
|
+
const { verdict, held } = this.tenantVerdict(tenantId);
|
|
5513
|
+
if (verdict === 'foreign') {
|
|
5514
|
+
throw substratError('conflict', held !== null
|
|
5515
|
+
? `applyProjection refused: this scope was provisioned for tenant ${held}, and a projection for tenant ${tenantId} would re-point it`
|
|
5516
|
+
: `applyProjection ${foreignTenant(held, tenantId)}`);
|
|
5056
5517
|
}
|
|
5057
5518
|
// Written before any guard below can return early, so every projection leaves it.
|
|
5058
|
-
this.sql.exec(`INSERT OR REPLACE INTO _substrat_meta (key, value) VALUES (
|
|
5519
|
+
this.sql.exec(`INSERT OR REPLACE INTO _substrat_meta (key, value) VALUES (?, ?)`, PROVISIONED_FOR_KEY, tenantId);
|
|
5059
5520
|
this.sql.exec(`DELETE FROM _substrat_roles WHERE tenant_id = ?`, tenantId);
|
|
5060
5521
|
for (const r of roles) {
|
|
5061
5522
|
this.sql.exec(`INSERT OR REPLACE INTO _substrat_roles (tenant_id, role_key, permissions, source, revoked_at)
|