@substrat-run/adapter-cloudflare 0.114.0 → 0.117.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/checker.d.ts.map +1 -1
- package/dist/checker.js +7 -1
- package/dist/checker.js.map +1 -1
- package/dist/control-plane-do.d.ts +57 -0
- package/dist/control-plane-do.d.ts.map +1 -1
- package/dist/control-plane-do.js +102 -11
- package/dist/control-plane-do.js.map +1 -1
- package/dist/host.d.ts +153 -16
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +563 -75
- package/dist/host.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/live-reads.d.ts +116 -0
- package/dist/live-reads.d.ts.map +1 -0
- package/dist/live-reads.js +106 -0
- package/dist/live-reads.js.map +1 -0
- package/dist/scope-do.d.ts +0 -16
- package/dist/scope-do.d.ts.map +1 -1
- package/dist/scope-do.js +1106 -202
- package/dist/scope-do.js.map +1 -1
- package/dist/sql.d.ts +6 -0
- package/dist/sql.d.ts.map +1 -1
- package/dist/sql.js +14 -0
- package/dist/sql.js.map +1 -1
- package/package.json +4 -4
package/dist/scope-do.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { DurableObject } from 'cloudflare:workers';
|
|
2
|
-
import { ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, domainEvent, domainEventInput, eventId, instant, objectRef, toWireFailure, grantRefFromProof, principalId, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, MAX_PENDING_SWEEP_RUNS, SWEEP_RUNS_KIND,
|
|
3
|
-
import { ulid, createUlid, assertAllowed, ConnectionSealingKeyUnavailableError, noSealingKeyMessage, sealTo, assertReadOnlyQuery, entitlementDenial, platformRequestHistoryQuery, PLATFORM_REQUEST_COLUMNS, denialListQuery, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, PermissionDenied, assertImpersonationWrites, assertModuleEnqueueableKind, impersonationStampOf, createAtomic, NotSearchable, isSearchIndexTable, searchIndexDdl, searchIndexMigrations, searchIndexPlans, NotListable, listIndexMigrations, listIndexPlans, listQuery, cursorOf, searchLimit, searchMatchExpression, searchQuery, IDEMPOTENCY_DDL, assertIdempotencyKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, } from '@substrat-run/kernel';
|
|
2
|
+
import { ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, domainEvent, domainEventInput, eventId, instant, objectRef, toWireFailure, grantRefFromProof, principalId, scopeId as scopeIdOf, tenantId as tenantIdOf, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, MAX_PENDING_SWEEP_RUNS, SWEEP_RUNS_KIND, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, listLimitOf, requestFingerprint, substratError, assertReplayableDump, REDRAIN_BATCH, } from '@substrat-run/contracts';
|
|
3
|
+
import { ulid, createUlid, assertAllowed, ConnectionSealingKeyUnavailableError, noSealingKeyMessage, sealTo, assertReadOnlyQuery, entitlementDenial, platformRequestHistoryQuery, platformRequestOf, PLATFORM_REQUEST_COLUMNS, PLATFORM_REQUEST_REDACTION_SQL, platformRequestRedactionParams, platformRequestRedactionQuery, intentPayloadCarriesSubject, seatScopeTuple, effectiveRoleGrantQuery, switchSystemSchedules, systemScheduleState, systemSwitchedOff, denialListQuery, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, PermissionDenied, assertImpersonationWrites, assertModuleEnqueueableKind, impersonationStampOf, createAtomic, NotSearchable, isSearchIndexTable, searchIndexDdl, searchIndexMigrations, searchIndexPlans, NotListable, listIndexMigrations, listIndexPlans, listQuery, cursorOf, searchLimit, searchMatchExpression, searchQuery, IDEMPOTENCY_DDL, assertIdempotencyKey, assertPermissionKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, SCHEDULE_STATE_DDL, SCHEDULE_STATE_REBUILD, scheduleStateHasKind, JOB_RUN_DDL, jobRunListLimit, } from '@substrat-run/kernel';
|
|
4
|
+
import { isUpgradeRequest, readSubscription, LIVE_FANOUT_LIMIT, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
|
|
4
5
|
import { OperationQueue } from './serialization.js';
|
|
5
|
-
import { doScopedSql } from './sql.js';
|
|
6
|
-
import { facetEvents, readDeadLetters, readHistory, readInvocation, walkEventCause, walkEventEffects } from '@substrat-run/kernel';
|
|
6
|
+
import { doScopedSql, doSpineSql } from './sql.js';
|
|
7
|
+
import { actorOf, assertNoSecret, CAPABILITY_DDL, CAPABILITY_EXCHANGE_OPERATION, createCapabilityVerbs, exchangeCapability, guardSecrets, mintBecomeCapability, redactSecrets, resolveCapabilitySession, revokeCapabilityAsPlatform, domainEventOf, facetEvents, readDeadLetters, readHistory, readInvocation, readUndrainedOutbox, walkEventCause, walkEventEffects, } from '@substrat-run/kernel';
|
|
7
8
|
import { createDoTupleChecker, createLocalControlPlaneReader } from './checker.js';
|
|
8
9
|
/**
|
|
9
10
|
* The key marking a scope DO whose storage was destroyed (`destroyStorage`).
|
|
@@ -127,31 +128,28 @@ const KERNEL_DDL = `
|
|
|
127
128
|
operation TEXT,
|
|
128
129
|
-- K-42: WHICH of the two actors was refused is exactly what this log is for.
|
|
129
130
|
impersonation TEXT,
|
|
131
|
+
-- #1525: the INVOCATION this refusal happened during, the same id #1237 stamps on
|
|
132
|
+
-- every event of a call. A denial joins to nothing but actor and time otherwise --
|
|
133
|
+
-- the operation column names what was attempted, never WHICH attempt -- so "what
|
|
134
|
+
-- else did this request do" cannot reach a refusal, which is the one thing an
|
|
135
|
+
-- incident asks about first. NULL = the transport carried no id (a seed, a test,
|
|
136
|
+
-- an internal call, an attachment RPC), or the row predates the column.
|
|
137
|
+
invocation_id TEXT,
|
|
130
138
|
at TEXT NOT NULL,
|
|
131
139
|
drained_at TEXT
|
|
132
140
|
);
|
|
133
|
-
-- #383: the platform sweep's per-scope gating state
|
|
134
|
-
--
|
|
135
|
-
--
|
|
136
|
-
--
|
|
137
|
-
--
|
|
138
|
-
|
|
139
|
-
--
|
|
140
|
-
--
|
|
141
|
-
--
|
|
142
|
-
--
|
|
143
|
-
--
|
|
144
|
-
|
|
145
|
-
-- that declared a schedule operation literally named "freshness:orders.placed"
|
|
146
|
-
-- would share a row with the evaluator and nothing today would reject it.
|
|
147
|
-
-- #1288 tracks giving the table a column (or a name) that says this outright,
|
|
148
|
-
-- which is also what would let the collision be refused rather than avoided.
|
|
149
|
-
-- Spine (kernel-written), never a module migration.
|
|
150
|
-
CREATE TABLE IF NOT EXISTS _substrat_schedule_state (
|
|
151
|
-
schedule_op TEXT PRIMARY KEY,
|
|
152
|
-
last_run_at TEXT,
|
|
153
|
-
last_status TEXT
|
|
154
|
-
);
|
|
141
|
+
-- #383 / #1232 / #1288: the platform sweep's per-scope gating state, holding two
|
|
142
|
+
-- families of row that the kind COLUMN -- not the spelling of a key -- tells
|
|
143
|
+
-- apart. Spine (kernel-written), never a module migration. Shared with the pure
|
|
144
|
+
-- adapter from @substrat-run/kernel, so the shape a rebuild produces and the shape
|
|
145
|
+
-- a fresh store gets cannot part company; the column comments are in there.
|
|
146
|
+
${SCHEDULE_STATE_DDL}
|
|
147
|
+
-- #1577: the resumable-run driver's record and the step ledger of the pass it
|
|
148
|
+
-- currently has in flight. Spine (kernel-written), never a module migration.
|
|
149
|
+
-- Shared with the pure adapter from @substrat-run/kernel so the shape production
|
|
150
|
+
-- builds and the shape a self-host builds cannot part company; the column
|
|
151
|
+
-- comments, and the reason coalescing is NOT a unique index, are in there.
|
|
152
|
+
${JOB_RUN_DDL}
|
|
155
153
|
CREATE TABLE IF NOT EXISTS _substrat_migrations (
|
|
156
154
|
module_id TEXT NOT NULL,
|
|
157
155
|
version TEXT NOT NULL,
|
|
@@ -191,6 +189,17 @@ const KERNEL_DDL = `
|
|
|
191
189
|
-- next_attempt_at IS NULL -> terminal: error IS NULL delivered, else dead
|
|
192
190
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
193
191
|
next_attempt_at TEXT,
|
|
192
|
+
-- #1525: the invocation THIS ATTEMPT ran in, the same id #1237 stamps on every
|
|
193
|
+
-- event of a call. Not the emitting event's -- that one is already on the outbox
|
|
194
|
+
-- and a delivery joins to it through event_id. What that join cannot say is
|
|
195
|
+
-- which call attempted the DELIVERY, and for an executor the two differ by
|
|
196
|
+
-- design: attempt one runs in the emitting call's post-commit tail, every retry
|
|
197
|
+
-- afterwards in a drain that is a different call or none. Moves with the row on
|
|
198
|
+
-- an upsert, like delivered_at, error and attempts -- it describes the
|
|
199
|
+
-- latest attempt, not the first. NULL = that attempt carried no call (a
|
|
200
|
+
-- scheduled drain, an alarm, a seed, an attachment RPC), or the row predates
|
|
201
|
+
-- the column.
|
|
202
|
+
invocation_id TEXT,
|
|
194
203
|
PRIMARY KEY (event_id, consumer_module)
|
|
195
204
|
);
|
|
196
205
|
-- Scope-local permissions (docs/architecture/scope-local-permissions.md): the
|
|
@@ -301,6 +310,10 @@ const KERNEL_DDL = `
|
|
|
301
310
|
${OUTBOX_ENTITY_INDEX}
|
|
302
311
|
-- #116: the request-dedupe table, kernel-owned so no vertical migrates for it.
|
|
303
312
|
${IDEMPOTENCY_DDL}
|
|
313
|
+
-- #1672: capabilities — authority carried by a secret (a link share), and the sessions
|
|
314
|
+
-- an exchange trades that secret for. Shared with the pure adapter from
|
|
315
|
+
-- @substrat-run/kernel so the two cannot part company; the column comments are there.
|
|
316
|
+
${CAPABILITY_DDL}
|
|
304
317
|
`;
|
|
305
318
|
/**
|
|
306
319
|
* Workers RPC carries a plain `Error`'s MESSAGE faithfully and nothing else. A custom
|
|
@@ -359,27 +372,6 @@ function attachSubject(principal, connectionId) {
|
|
|
359
372
|
function isSystemTable(name) {
|
|
360
373
|
return name.startsWith('_substrat') || name.startsWith('sqlite_');
|
|
361
374
|
}
|
|
362
|
-
/**
|
|
363
|
-
* A stored row → the `PlatformRequest` contract shape (JSON columns parsed). The coordinator
|
|
364
|
-
* maps the RPC's raw rows the same way in `host.ts`; this copy exists because `ctx.platformRequests`
|
|
365
|
-
* (#618) answers INSIDE the DO, where the row never crosses an RPC boundary at all.
|
|
366
|
-
*/
|
|
367
|
-
function rowToPlatformRequest(r) {
|
|
368
|
-
return platformRequest.parse({
|
|
369
|
-
id: r.id,
|
|
370
|
-
kind: r.kind,
|
|
371
|
-
payload: JSON.parse(r.payload),
|
|
372
|
-
requestedBy: JSON.parse(r.requested_by),
|
|
373
|
-
impersonation: r.impersonation == null ? null : JSON.parse(r.impersonation),
|
|
374
|
-
status: r.status,
|
|
375
|
-
attempts: r.attempts,
|
|
376
|
-
lastError: r.last_error,
|
|
377
|
-
failure: r.last_failure == null ? null : JSON.parse(r.last_failure),
|
|
378
|
-
result: r.result === null ? null : JSON.parse(r.result),
|
|
379
|
-
requestedAt: r.requested_at,
|
|
380
|
-
settledAt: r.settled_at,
|
|
381
|
-
});
|
|
382
|
-
}
|
|
383
375
|
/** SQLite cell → a JSON-safe value: bigints stringify, blobs (ArrayBuffer) read as null. */
|
|
384
376
|
function cellToJson(v) {
|
|
385
377
|
if (v == null)
|
|
@@ -505,6 +497,12 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
505
497
|
listPlans = new Map();
|
|
506
498
|
/** entityType → the declared attachment gate (#473): read key + write key (default: read). */
|
|
507
499
|
attachmentTargets = new Map();
|
|
500
|
+
/**
|
|
501
|
+
* entityType → the declared live-read gate (#938): the key a subscriber must hold
|
|
502
|
+
* ON THAT ENTITY before a change to it is announced. Absent = announced to nobody;
|
|
503
|
+
* `fanOutLive` treats a miss as silence, never as "unguarded".
|
|
504
|
+
*/
|
|
505
|
+
liveTargets = new Map();
|
|
508
506
|
checker;
|
|
509
507
|
systemPrincipal = principalId.parse(ulid());
|
|
510
508
|
/**
|
|
@@ -622,6 +620,20 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
622
620
|
}
|
|
623
621
|
this.attachmentTargets.set(target.entityType, gate);
|
|
624
622
|
}
|
|
623
|
+
// Live-read targets (#938): entityType → the read key a subscriber must pass ON
|
|
624
|
+
// THAT ENTITY before a change to it is announced. Validated exactly like the
|
|
625
|
+
// attachment gate above, and for a sharper reason: two modules disagreeing about
|
|
626
|
+
// which key guards an entity type would make "who may watch this" depend on
|
|
627
|
+
// registration order, and the losing declaration would be the one that was meant
|
|
628
|
+
// to be stricter. An entity type nobody declares is announced to nobody.
|
|
629
|
+
for (const target of manifest.liveTargets ?? []) {
|
|
630
|
+
const existing = this.liveTargets.get(target.entityType);
|
|
631
|
+
if (existing && existing !== target.readPermission) {
|
|
632
|
+
throw new Error(`conflicting liveTargets for '${target.entityType}': ` +
|
|
633
|
+
`(${existing}) vs (${target.readPermission})`);
|
|
634
|
+
}
|
|
635
|
+
this.liveTargets.set(target.entityType, target.readPermission);
|
|
636
|
+
}
|
|
625
637
|
for (const name of manifest.withdraws ?? []) {
|
|
626
638
|
this.withdrawn.set(name, manifest.id);
|
|
627
639
|
this.operations.delete(name);
|
|
@@ -680,9 +692,11 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
680
692
|
* and skips the write when nothing changed.
|
|
681
693
|
*
|
|
682
694
|
* `ensureMigrations` memoises its promise, so every later call on a warm DO
|
|
683
|
-
* resolves to the SAME `true` without applying anything
|
|
684
|
-
*
|
|
685
|
-
*
|
|
695
|
+
* resolves to the SAME `true` without applying anything — until something
|
|
696
|
+
* clears the memo (`retryMigrations`, `importDump`), which is a fresh pass and
|
|
697
|
+
* may apply. Reporting on each of the cached ones would bill a control-plane
|
|
698
|
+
* RPC per stub mint to store a number that has not moved — hence the
|
|
699
|
+
* once-per-instance latch.
|
|
686
700
|
*/
|
|
687
701
|
async migrate() {
|
|
688
702
|
const applied = await this.ensureMigrations();
|
|
@@ -738,6 +752,14 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
738
752
|
migrationFailure() {
|
|
739
753
|
return this.lastFailure ? { ...this.lastFailure, applied: this.applied.size } : null;
|
|
740
754
|
}
|
|
755
|
+
/**
|
|
756
|
+
* This scope's database size in bytes (#1524): `SqlStorage.databaseSize`, which Cloudflare
|
|
757
|
+
* bills on. Its one caller is an on-demand storage reading, never a sweep, because reaching
|
|
758
|
+
* it wakes this DO.
|
|
759
|
+
*/
|
|
760
|
+
databaseSize() {
|
|
761
|
+
return this.sql.databaseSize;
|
|
762
|
+
}
|
|
741
763
|
/**
|
|
742
764
|
* The PITR bookmarks this scope recorded before migration passes (#286),
|
|
743
765
|
* newest first — what a backout UI offers as rewind points. Rows taken after
|
|
@@ -762,26 +784,26 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
762
784
|
}));
|
|
763
785
|
}
|
|
764
786
|
/**
|
|
765
|
-
* The events not yet shipped to Tier 2 (#1334), oldest first
|
|
766
|
-
* is chronological (ULID) and stable, so a drain
|
|
787
|
+
* The events not yet shipped to Tier 2 (#1334), oldest first — and what the read
|
|
788
|
+
* stepped over (#1636). `ORDER BY id` is chronological (ULID) and stable, so a drain
|
|
789
|
+
* resumes where it stopped.
|
|
790
|
+
*
|
|
791
|
+
* The kernel's read, shared with the pure adapter: a row that will not decode is
|
|
792
|
+
* neither returned nor stamped, and the rows behind it still come back. An object
|
|
793
|
+
* rather than an array because it crosses the RPC — a property on an array would not.
|
|
794
|
+
*/
|
|
795
|
+
undrainedEventsRead(limit) {
|
|
796
|
+
return readUndrainedOutbox((offset, count) => this.sql
|
|
797
|
+
.exec(`SELECT * FROM _substrat_outbox WHERE drained_at IS NULL ORDER BY id LIMIT ? OFFSET ?`, count, offset)
|
|
798
|
+
.toArray(), limit);
|
|
799
|
+
}
|
|
800
|
+
/**
|
|
801
|
+
* The same read as a bare array, for a coordinator deployed before
|
|
802
|
+
* `undrainedEventsRead` (#1636) — kept so that pairing still drains, and still steps
|
|
803
|
+
* over a bad row rather than stalling on it. It just cannot say that it did.
|
|
767
804
|
*/
|
|
768
805
|
undrainedEvents(limit) {
|
|
769
|
-
|
|
770
|
-
.exec(`SELECT * FROM _substrat_outbox WHERE drained_at IS NULL ORDER BY id LIMIT ?`, limit)
|
|
771
|
-
.toArray();
|
|
772
|
-
return rows.map((r) => ({
|
|
773
|
-
...this.parseOutboxRow(r),
|
|
774
|
-
operation: r.operation ?? null,
|
|
775
|
-
version: r.version ?? null,
|
|
776
|
-
// #1237 — lifted like the two above, and for the same reason: the column
|
|
777
|
-
// exists on the outbox but not on the envelope `parseOutboxRow` returns,
|
|
778
|
-
// whose `domainEvent.parse` strips anything it does not declare.
|
|
779
|
-
causedBy: r.caused_by ?? null,
|
|
780
|
-
// …and the invocation, for the same reason again: a lake that kept cause and
|
|
781
|
-
// dropped the call could say what set an event off and never which request did
|
|
782
|
-
// it, which is the grouping a trace is built on.
|
|
783
|
-
invocationId: r.invocation_id ?? null,
|
|
784
|
-
}));
|
|
806
|
+
return this.undrainedEventsRead(limit).events;
|
|
785
807
|
}
|
|
786
808
|
/**
|
|
787
809
|
* Stamp `drained_at` on shipped events (#1334). Idempotent — a re-mark is a no-op,
|
|
@@ -843,6 +865,24 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
843
865
|
return ids.length;
|
|
844
866
|
});
|
|
845
867
|
}
|
|
868
|
+
/**
|
|
869
|
+
* How many rows `redrainEvents` WOULD reopen for the same instant (#1545), reopening
|
|
870
|
+
* none of them. A separate verb rather than a flag on the one above, deliberately: the
|
|
871
|
+
* two answers are indistinguishable once they are numbers, so the caller that asks for
|
|
872
|
+
* a count must not be able to reach the reopen by losing an argument on the way.
|
|
873
|
+
*
|
|
874
|
+
* UNBOUNDED where the reopen is batched. The batch exists because an UPDATE over the
|
|
875
|
+
* whole window rewrites every row and its index entries inside ONE Durable Object
|
|
876
|
+
* request, against a fixed budget. An aggregate materialises no rows, so that budget is
|
|
877
|
+
* not the binding constraint — and a partial count would be worse than useless: the
|
|
878
|
+
* number is the whole point, and "5000, or possibly more" answers nothing.
|
|
879
|
+
*/
|
|
880
|
+
async redrainCount(drainedBefore) {
|
|
881
|
+
return this.sql
|
|
882
|
+
.exec(`SELECT COUNT(*) AS c FROM _substrat_outbox
|
|
883
|
+
WHERE drained_at IS NOT NULL AND drained_at < ?`, drainedBefore)
|
|
884
|
+
.toArray()[0].c;
|
|
885
|
+
}
|
|
846
886
|
/**
|
|
847
887
|
* Facet this scope's own outbox (#1239) — `facetEvents`, which is the
|
|
848
888
|
* sanctioned read: an erased payload yields the same NULL a missing field
|
|
@@ -960,13 +1000,30 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
960
1000
|
return rows.sort((a, b) => a.subject.localeCompare(b.subject) || a.relation.localeCompare(b.relation));
|
|
961
1001
|
});
|
|
962
1002
|
}
|
|
963
|
-
/**
|
|
1003
|
+
/**
|
|
1004
|
+
* Admin scope-tuple write (role assignment / grant scoped to this scope) — the
|
|
1005
|
+
* EXPLICIT grant. `INSERT OR REPLACE`, so it clears a tombstone: a re-grant grants.
|
|
1006
|
+
* Provisioning does not come through here; it seats with `seatTuple` (#1659).
|
|
1007
|
+
*/
|
|
964
1008
|
async writeTuple(subject, relation, object, expiresAt) {
|
|
965
1009
|
await this.queue.enqueue(() => {
|
|
966
1010
|
this.sql.exec(`INSERT OR REPLACE INTO _substrat_tuples (subject, relation, object, expires_at)
|
|
967
1011
|
VALUES (?, ?, ?, ?)`, subject, relation, object, expiresAt);
|
|
968
1012
|
});
|
|
969
1013
|
}
|
|
1014
|
+
/**
|
|
1015
|
+
* Provisioning's scope-tuple write (#1659): create the row if it is missing, follow the
|
|
1016
|
+
* platform's expiry if it is live, and leave it alone if it was revoked — so a re-run
|
|
1017
|
+
* provision cannot undo an operator's revoke — and seat nothing at all for a module
|
|
1018
|
+
* whose schedule kill switch is off (#1666). `seatScopeTuple` is the statement, shared
|
|
1019
|
+
* with `applyProjection`'s `scopeTuples` and with the pure adapter.
|
|
1020
|
+
*/
|
|
1021
|
+
async seatTuple(subject, relation, object, expiresAt) {
|
|
1022
|
+
await this.queue.enqueue(() => {
|
|
1023
|
+
const seat = seatScopeTuple(subject, relation, object, expiresAt);
|
|
1024
|
+
this.sql.exec(seat.sql, ...seat.params);
|
|
1025
|
+
});
|
|
1026
|
+
}
|
|
970
1027
|
/**
|
|
971
1028
|
* The dispatch capability's admission (#726 remedy B): this delivery may read the
|
|
972
1029
|
* attachments of the entity its own spine row names, and no others.
|
|
@@ -1056,19 +1113,28 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1056
1113
|
* this scope; the DO uses it for the two-actor stamp and for the read-only
|
|
1057
1114
|
* bound, and nothing here can be reached without it having been minted.
|
|
1058
1115
|
*/
|
|
1059
|
-
impersonation
|
|
1116
|
+
impersonation,
|
|
1117
|
+
/**
|
|
1118
|
+
* #1672: the HASH of a capability session token — the coordinator hashed the token
|
|
1119
|
+
* and the plaintext never crosses. Resolved to its capability INSIDE the queued body,
|
|
1120
|
+
* on every call, so a revoke between two calls refuses the second. `principal` is
|
|
1121
|
+
* then a random placeholder that holds nothing, and the reply carries
|
|
1122
|
+
* `capability.honoured`: an old DO that ignored this argument would run the call as
|
|
1123
|
+
* that placeholder, and the coordinator refuses a success without the acknowledgement.
|
|
1124
|
+
*/
|
|
1125
|
+
capabilitySession) {
|
|
1060
1126
|
if (!failureEnvelope) {
|
|
1061
1127
|
// Legacy path, byte-for-byte what it was: rewrapped so a non-plain error (a
|
|
1062
1128
|
// ZodError, whose `message` is a getter) still arrives with its message.
|
|
1063
1129
|
try {
|
|
1064
|
-
return await this.invokeOrThrow(operation, input, principal, tenantId, scopeId, connectionId, requiredEntitlement, systemModuleId, invokeOptions, impersonation);
|
|
1130
|
+
return await this.invokeOrThrow(operation, input, principal, tenantId, scopeId, connectionId, requiredEntitlement, systemModuleId, invokeOptions, impersonation, capabilitySession);
|
|
1065
1131
|
}
|
|
1066
1132
|
catch (err) {
|
|
1067
1133
|
throw toRpcError(err);
|
|
1068
1134
|
}
|
|
1069
1135
|
}
|
|
1070
1136
|
try {
|
|
1071
|
-
return await this.invokeOrThrow(operation, input, principal, tenantId, scopeId, connectionId, requiredEntitlement, systemModuleId, invokeOptions, impersonation);
|
|
1137
|
+
return await this.invokeOrThrow(operation, input, principal, tenantId, scopeId, connectionId, requiredEntitlement, systemModuleId, invokeOptions, impersonation, capabilitySession);
|
|
1072
1138
|
}
|
|
1073
1139
|
catch (err) {
|
|
1074
1140
|
// The ONE place the error keeps its structure: flattened here, rebuilt by the
|
|
@@ -1080,7 +1146,9 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1080
1146
|
/** The operation path itself. Throws; `invoke` decides how that reaches the caller. */
|
|
1081
1147
|
async invokeOrThrow(operation, input, principal, tenantId, scopeId, connectionId, requiredEntitlement, systemModuleId, invokeOptions,
|
|
1082
1148
|
/** K-42: the session, resolved coordinator-side. See `invoke` above. */
|
|
1083
|
-
impersonation
|
|
1149
|
+
impersonation,
|
|
1150
|
+
/** #1672: a capability session's hash. See `invoke` above. */
|
|
1151
|
+
capabilitySession) {
|
|
1084
1152
|
await this.ensureMigrations();
|
|
1085
1153
|
const handler = this.operations.get(operation);
|
|
1086
1154
|
// `not_found`, not a bare throw (#113): every vertical hand-matched this message
|
|
@@ -1139,7 +1207,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1139
1207
|
// The subject a key is scoped to — the same three-way read `recordDenial`
|
|
1140
1208
|
// makes below, hoisted because both need it. A key belongs to whoever sent
|
|
1141
1209
|
// it: two principals choosing `1` must not reach each other's response.
|
|
1142
|
-
|
|
1210
|
+
let idempotencySubjectRef = systemModuleId
|
|
1143
1211
|
? { kind: 'system', id: systemModuleId }
|
|
1144
1212
|
: connectionId
|
|
1145
1213
|
? { kind: 'connection', id: connectionId }
|
|
@@ -1163,6 +1231,34 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1163
1231
|
// one call. Same placement as the SQLite adapter's actor task, for this reason.
|
|
1164
1232
|
this.invocationId = invokeOptions?.invocationId ?? null;
|
|
1165
1233
|
try {
|
|
1234
|
+
// #1672: the capability session, resolved on EVERY call and here — inside the queued
|
|
1235
|
+
// body, the one region where this call holds the DO to itself — so nothing can
|
|
1236
|
+
// revoke between this read and the transaction. Refuses a stale session, a revoked
|
|
1237
|
+
// or expired capability and an operation off its allowlist before anything opens;
|
|
1238
|
+
// none of those is a K-35 denial (no key was checked).
|
|
1239
|
+
let capabilityId;
|
|
1240
|
+
if (capabilitySession !== undefined) {
|
|
1241
|
+
capabilityId = resolveCapabilitySession(doSpineSql(this.sql), capabilitySession, instant.parse(new Date().toISOString()), operation);
|
|
1242
|
+
idempotencySubjectRef = { kind: 'capability', id: capabilityId };
|
|
1243
|
+
}
|
|
1244
|
+
// #1672: the secrets this call mints — withheld from its idempotency recording, and
|
|
1245
|
+
// what the tripwire on its writes looks for.
|
|
1246
|
+
const minted = [];
|
|
1247
|
+
/**
|
|
1248
|
+
* #938: the outbox's high-water mark BEFORE this call wrote anything, so the
|
|
1249
|
+
* post-commit fan-out can name exactly the events this call (and the consumers
|
|
1250
|
+
* it set off) added. Read here — inside the queued body, before the
|
|
1251
|
+
* transaction — because that is the region where this call holds the DO to
|
|
1252
|
+
* itself, the same reason `invocationId` is set here.
|
|
1253
|
+
*
|
|
1254
|
+
* A socket that connects between here and the fan-out is served whatever this
|
|
1255
|
+
* call committed, and one that connects while `null` was decided hears nothing
|
|
1256
|
+
* about it. Both are harmless and neither is worth a lock: a frame is an
|
|
1257
|
+
* invalidation, so hearing about a change from just before you subscribed costs
|
|
1258
|
+
* one redundant re-read, and missing one costs a wait for the client's poll —
|
|
1259
|
+
* which is the floor this whole surface sits on.
|
|
1260
|
+
*/
|
|
1261
|
+
const liveSince = this.liveHighWaterMark();
|
|
1166
1262
|
let result;
|
|
1167
1263
|
let committedVersion = null;
|
|
1168
1264
|
// #116: set when this invocation was answered from a recording rather
|
|
@@ -1190,7 +1286,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1190
1286
|
// emitted events back as one — verified across `await` in workerd.
|
|
1191
1287
|
try {
|
|
1192
1288
|
await this.ctx.storage.transaction(async () => {
|
|
1193
|
-
const ctx = this.operationContext(principal, tenantId, scopeId, undefined, connectionId, systemModuleId, signals, impersonation, operation);
|
|
1289
|
+
const ctx = this.operationContext(principal, tenantId, scopeId, undefined, connectionId, systemModuleId, signals, impersonation, operation, capabilityId, minted);
|
|
1194
1290
|
// #116: a retry is answered from the recording, and nothing else runs
|
|
1195
1291
|
// — not the guards, not the handler, not the permission check inside
|
|
1196
1292
|
// it. Keyed by SUBJECT, so a caller only ever reaches its own
|
|
@@ -1231,7 +1327,9 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1231
1327
|
// it describes. The prune rides along, on the only path that adds a row.
|
|
1232
1328
|
if (idempotencyKey !== undefined && fingerprint !== undefined) {
|
|
1233
1329
|
const at = new Date().toISOString();
|
|
1234
|
-
const record = idempotencyRecordStatement(idempotencySubjectRef, idempotencyKey, operation, fingerprint,
|
|
1330
|
+
const record = idempotencyRecordStatement(idempotencySubjectRef, idempotencyKey, operation, fingerprint,
|
|
1331
|
+
// #1672: a replay of a mint returns the placeholder, never the secret.
|
|
1332
|
+
redactSecrets(result, minted), committedVersion, at);
|
|
1235
1333
|
this.sql.exec(record.sql, ...record.params);
|
|
1236
1334
|
const prune = idempotencyPruneStatement(at);
|
|
1237
1335
|
this.sql.exec(prune.sql, ...prune.params);
|
|
@@ -1254,6 +1352,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1254
1352
|
result,
|
|
1255
1353
|
platformRequests: 0,
|
|
1256
1354
|
impersonation: { honoured: true },
|
|
1355
|
+
...(capabilitySession !== undefined ? { capability: { honoured: true } } : {}),
|
|
1257
1356
|
...(idempotencyKey !== undefined
|
|
1258
1357
|
? { idempotency: { keyHonoured: true, replayed } }
|
|
1259
1358
|
: {}),
|
|
@@ -1270,11 +1369,12 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1270
1369
|
// K-35: the transaction has rolled back; record a refused check now, as its own
|
|
1271
1370
|
// write (outside that transaction), so the denial survives the rollback.
|
|
1272
1371
|
if (err instanceof PermissionDenied) {
|
|
1273
|
-
this.recordDenial(
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1372
|
+
this.recordDenial(
|
|
1373
|
+
// The subject the call acted as — the capability's, once resolved above.
|
|
1374
|
+
idempotencySubjectRef, tenantId, operation, err,
|
|
1375
|
+
// Inside the queued body, which is the one region where this call holds
|
|
1376
|
+
// the DO to itself — so the field is this call's own (#1237).
|
|
1377
|
+
this.invocationId, impersonation);
|
|
1278
1378
|
}
|
|
1279
1379
|
// The ORIGINAL error, deliberately: `invoke` flattens it for the envelope
|
|
1280
1380
|
// (which keeps its code and extensions) or rewraps it for the legacy throw
|
|
@@ -1285,12 +1385,20 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1285
1385
|
// Skipped on a replay: nothing was written, so there is nothing this
|
|
1286
1386
|
// invocation added to drain. Anything the ORIGINAL left undrained is the
|
|
1287
1387
|
// outbox's own retry backstop, which is what that backstop is for.
|
|
1388
|
+
// Drain to consumers, then announce what landed (#938). Skipped whole on a
|
|
1389
|
+
// replay: nothing was written, so there is nothing this invocation added to
|
|
1390
|
+
// drain or to announce. Anything the ORIGINAL left undrained is the outbox's
|
|
1391
|
+
// own retry backstop, which is what that backstop is for.
|
|
1392
|
+
// #1525: still inside the queued body that set it, so these deliveries are this
|
|
1393
|
+
// call's own work — the same tail its consumers' emits are stamped in.
|
|
1288
1394
|
if (!replayed)
|
|
1289
|
-
await this.
|
|
1395
|
+
await this.settleCommitted(tenantId, scopeId, liveSince, this.invocationId);
|
|
1290
1396
|
return {
|
|
1291
1397
|
result,
|
|
1292
1398
|
platformRequests: signals.platformRequests,
|
|
1293
1399
|
...(impersonation ? { impersonation: { honoured: true } } : {}),
|
|
1400
|
+
// #1672: the acknowledgement the coordinator's skew check reads — see `invoke`.
|
|
1401
|
+
...(capabilitySession !== undefined ? { capability: { honoured: true } } : {}),
|
|
1294
1402
|
// The acknowledgement the coordinator's skew check reads (#116), on the
|
|
1295
1403
|
// same reasoning as `ifMatchChecked` below and with a sharper failure: a
|
|
1296
1404
|
// DO too old to know about keys would EXECUTE THE OPERATION AGAIN and
|
|
@@ -1319,6 +1427,288 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1319
1427
|
}
|
|
1320
1428
|
});
|
|
1321
1429
|
}
|
|
1430
|
+
// -- live reads (#938): the subscription half of the change feed ------------
|
|
1431
|
+
// The ONLY part of this DO addressed as a fetch target rather than over RPC, and
|
|
1432
|
+
// only because a WebSocket cannot cross RPC — a socket is not serializable, so the
|
|
1433
|
+
// one way to hand one back is a `Response` carrying a `webSocket`. Everything the
|
|
1434
|
+
// coordinator asserts on the way in is named in `live-reads.ts`, shared with
|
|
1435
|
+
// `host.ts` so the two ends cannot drift.
|
|
1436
|
+
/**
|
|
1437
|
+
* Accept a subscription to this scope's changes.
|
|
1438
|
+
*
|
|
1439
|
+
* The coordinator has already decided that this connection can carry a push at all
|
|
1440
|
+
* (the O2O check) and WHO is asking. What is decided here is nothing about
|
|
1441
|
+
* authority: a subscription is not an authorization, and accepting one grants the
|
|
1442
|
+
* subscriber no read it did not already have. Every frame is checked on its way
|
|
1443
|
+
* out, individually, against the tuple state at that moment — so a grant revoked
|
|
1444
|
+
* while the socket is open stops the frames it used to allow, which a
|
|
1445
|
+
* subscription-time check would not.
|
|
1446
|
+
*
|
|
1447
|
+
* **This adds no authority to a holder of the stub, and the question is worth
|
|
1448
|
+
* answering rather than leaving to be asked.** Being a public method, anything with
|
|
1449
|
+
* the `SCOPE` binding can call it and assert whatever principal it likes in the
|
|
1450
|
+
* headers. That is already true of `invoke`, which takes the principal as an
|
|
1451
|
+
* argument and acts on it: a stub is the key to the scope, which is precisely why
|
|
1452
|
+
* the router is not given one. The trust boundary is who holds the binding, not
|
|
1453
|
+
* what this method checks — and what it does NOT do is let a stub-holder read
|
|
1454
|
+
* anything the asserted principal could not, because the filter downstream re-checks
|
|
1455
|
+
* that principal against every frame.
|
|
1456
|
+
*/
|
|
1457
|
+
async fetch(request) {
|
|
1458
|
+
const url = new URL(request.url);
|
|
1459
|
+
if (url.pathname !== LIVE_SUBSCRIBE_PATH) {
|
|
1460
|
+
// The DO has exactly one fetch surface. Anything else reaching here is a
|
|
1461
|
+
// coordinator bug, and a 404 says so without guessing at an intent.
|
|
1462
|
+
return new Response('this scope has no such surface', { status: 404 });
|
|
1463
|
+
}
|
|
1464
|
+
if (!isUpgradeRequest(request)) {
|
|
1465
|
+
return new Response('live reads are a WebSocket surface', {
|
|
1466
|
+
status: 426,
|
|
1467
|
+
headers: { [LIVE_MODE_HEADER]: 'not-an-upgrade' },
|
|
1468
|
+
});
|
|
1469
|
+
}
|
|
1470
|
+
const principal = request.headers.get(LIVE_PRINCIPAL_HEADER);
|
|
1471
|
+
const tenantId = request.headers.get(LIVE_TENANT_HEADER);
|
|
1472
|
+
const scopeId = request.headers.get(LIVE_SCOPE_HEADER);
|
|
1473
|
+
if (!principal || !tenantId || !scopeId) {
|
|
1474
|
+
// Fail closed and loudly. An unnamed subscriber is one whose permissions
|
|
1475
|
+
// cannot be evaluated, and the only safe thing to do with a channel we cannot
|
|
1476
|
+
// filter is to refuse to open it. 500, not 400: the caller is the coordinator
|
|
1477
|
+
// in this same package, so a missing assertion is OUR bug, not the client's.
|
|
1478
|
+
return new Response('live reads require an asserted principal, tenant and scope', {
|
|
1479
|
+
status: 500,
|
|
1480
|
+
});
|
|
1481
|
+
}
|
|
1482
|
+
// A subscriber arriving before the scope's migrations have run would be told
|
|
1483
|
+
// about events against a schema it cannot read back through. Same gate every
|
|
1484
|
+
// other entry point takes, for the same reason.
|
|
1485
|
+
await this.ensureMigrations();
|
|
1486
|
+
const pair = new WebSocketPair();
|
|
1487
|
+
const [client, server] = Object.values(pair);
|
|
1488
|
+
// HIBERNATABLE, not `server.accept()`. A scope with a watcher open would
|
|
1489
|
+
// otherwise be pinned in memory for as long as somebody has a tab open, which is
|
|
1490
|
+
// the cost model inverted: a support desk being WATCHED is the normal state.
|
|
1491
|
+
// Hibernation also fixes the worse half — an in-memory roster does not survive
|
|
1492
|
+
// eviction, so the socket would stay open and silently stop receiving, which is
|
|
1493
|
+
// indistinguishable from a quiet scope.
|
|
1494
|
+
this.ctx.acceptWebSocket(server);
|
|
1495
|
+
// `.parse`, not a cast: these three arrived as header strings, and the branded
|
|
1496
|
+
// ids are what every check downstream is keyed on. A malformed one would
|
|
1497
|
+
// otherwise be carried all the way to a `ctx.check` that quietly matches nothing —
|
|
1498
|
+
// which reads as "this subscriber may see nothing" and is indistinguishable from
|
|
1499
|
+
// a correct denial. Refused here instead, where it is still one subscriber's
|
|
1500
|
+
// problem. Throwing is right: the coordinator built this request.
|
|
1501
|
+
server.serializeAttachment({
|
|
1502
|
+
principal: principalId.parse(principal),
|
|
1503
|
+
tenantId: tenantIdOf.parse(tenantId),
|
|
1504
|
+
scopeId: scopeIdOf.parse(scopeId),
|
|
1505
|
+
since: new Date().toISOString(),
|
|
1506
|
+
});
|
|
1507
|
+
return new Response(null, { status: 101, webSocket: client });
|
|
1508
|
+
}
|
|
1509
|
+
/**
|
|
1510
|
+
* A subscriber said something.
|
|
1511
|
+
*
|
|
1512
|
+
* The channel is one-way by design — the server announces, the client re-reads
|
|
1513
|
+
* through the ordinary operation — so there is no client message that can cause a
|
|
1514
|
+
* read, a write, or a change of subscription. `ping`/`pong` is the whole protocol,
|
|
1515
|
+
* and it exists so an idle connection can be kept alive by either end.
|
|
1516
|
+
*
|
|
1517
|
+
* Deliberately NOT a place to let a client narrow or widen what it receives: a
|
|
1518
|
+
* filter the client chooses is a filter the client can choose wrongly, and the
|
|
1519
|
+
* only filter that matters here is the one it does not control.
|
|
1520
|
+
*/
|
|
1521
|
+
webSocketMessage(ws, message) {
|
|
1522
|
+
if (message === 'ping')
|
|
1523
|
+
ws.send('pong');
|
|
1524
|
+
}
|
|
1525
|
+
/**
|
|
1526
|
+
* The client hung up. Close our end so the runtime stops holding the subscription.
|
|
1527
|
+
*
|
|
1528
|
+
* The runtime passes `reason` and `wasClean` too; neither is read, so neither is
|
|
1529
|
+
* named — there is nothing to do differently for an unclean close, because the
|
|
1530
|
+
* socket is going away either way and the client's poll is what covers the gap.
|
|
1531
|
+
*/
|
|
1532
|
+
webSocketClose(ws, code) {
|
|
1533
|
+
// 1006 is reserved: it is what the runtime REPORTS for an abnormal closure and
|
|
1534
|
+
// is not a code anything may SEND, so echoing it back throws — on precisely the
|
|
1535
|
+
// path where the connection is already in trouble.
|
|
1536
|
+
try {
|
|
1537
|
+
ws.close(code === 1006 ? 1000 : code, 'scope closing the subscription');
|
|
1538
|
+
}
|
|
1539
|
+
catch {
|
|
1540
|
+
// Already gone. Nothing to do, and nothing worth reporting.
|
|
1541
|
+
}
|
|
1542
|
+
}
|
|
1543
|
+
/**
|
|
1544
|
+
* A socket failed. Logged and not rethrown: there is no caller to fail, and the
|
|
1545
|
+
* runtime delivers `webSocketClose` after this, which is what does the cleanup.
|
|
1546
|
+
*/
|
|
1547
|
+
webSocketError(_ws, error) {
|
|
1548
|
+
console.error('substrat: live-read socket error', error);
|
|
1549
|
+
}
|
|
1550
|
+
/**
|
|
1551
|
+
* The outbox's high-water mark before a committing path runs (#938), or `null`
|
|
1552
|
+
* when nobody is listening.
|
|
1553
|
+
*
|
|
1554
|
+
* `null` and `''` are different answers: `null` means no subscriber, so nothing is
|
|
1555
|
+
* read at all and a scope pays nothing for a feature it is not using; `''` is the
|
|
1556
|
+
* honest empty-outbox answer, and every ULID sorts above it.
|
|
1557
|
+
*
|
|
1558
|
+
* Read BEFORE the transaction by every caller, which is what makes the pair below
|
|
1559
|
+
* able to name exactly the events that path added.
|
|
1560
|
+
*/
|
|
1561
|
+
liveHighWaterMark() {
|
|
1562
|
+
if (this.ctx.getWebSockets().length === 0)
|
|
1563
|
+
return null;
|
|
1564
|
+
return (this.sql.exec('SELECT MAX(id) AS id FROM _substrat_outbox').toArray()[0]?.id ?? '');
|
|
1565
|
+
}
|
|
1566
|
+
/**
|
|
1567
|
+
* What every committing path does after its transaction closes: drain the outbox
|
|
1568
|
+
* to consumers, then announce what landed to whoever may see it.
|
|
1569
|
+
*
|
|
1570
|
+
* **One step, because there is one rule.** The fan-out was wired into `invoke`
|
|
1571
|
+
* alone at first, and `attachmentAdd`/`attachmentRemove` commit and emit too —
|
|
1572
|
+
* `attachment.added` and `attachment.removed`, about a real entity. A watcher of
|
|
1573
|
+
* that entity would have missed them: no error, no gap it could see, just a screen
|
|
1574
|
+
* that did not update for one kind of change. That is the exact failure the
|
|
1575
|
+
* hibernation design exists to prevent, arriving through a different door, so the
|
|
1576
|
+
* answer is a step both doors take rather than a second call both must remember.
|
|
1577
|
+
*
|
|
1578
|
+
* Order is load-bearing: drain FIRST, announce after. A consumer's own emits are
|
|
1579
|
+
* changes too, and they land in the outbox above `liveSince` — so announcing first
|
|
1580
|
+
* would tell a subscriber about the cause and not the effect, and it would re-read
|
|
1581
|
+
* too early.
|
|
1582
|
+
*/
|
|
1583
|
+
async settleCommitted(tenantId, scopeId, liveSince,
|
|
1584
|
+
/**
|
|
1585
|
+
* #1525: the call whose commit this is settling, or null — PASSED rather than
|
|
1586
|
+
* read off `this.invocationId` inside `dispatch`, so each of the three doors
|
|
1587
|
+
* says what it actually knows: `invoke` its own id, the two attachment verbs
|
|
1588
|
+
* null. Same discipline `recordDenial` adopted, and for the same reason.
|
|
1589
|
+
*/
|
|
1590
|
+
invocationId) {
|
|
1591
|
+
await this.dispatch(tenantId, scopeId, invocationId);
|
|
1592
|
+
if (liveSince === null)
|
|
1593
|
+
return;
|
|
1594
|
+
try {
|
|
1595
|
+
await this.fanOutLive(liveSince, tenantId, scopeId);
|
|
1596
|
+
}
|
|
1597
|
+
catch (err) {
|
|
1598
|
+
// The write has COMMITTED and the caller is owed its answer. A failure to
|
|
1599
|
+
// announce is a failure of a hint, and the client's poll is the floor
|
|
1600
|
+
// underneath it — so this is logged and never rethrown. Rethrowing would turn
|
|
1601
|
+
// a delivered write into a 500 the caller would reasonably retry.
|
|
1602
|
+
console.error('substrat: live-read fan-out failed after commit', err);
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
/**
|
|
1606
|
+
* Announce what just committed, to whoever may see it (#938).
|
|
1607
|
+
*
|
|
1608
|
+
* **Three properties, and each is load-bearing.**
|
|
1609
|
+
*
|
|
1610
|
+
* *Post-commit.* Called after the operation's transaction has closed and after the
|
|
1611
|
+
* consumer drain, so nothing is announced that a rollback could take back and
|
|
1612
|
+
* nothing a consumer emitted is missed. A subscriber told about a row that then
|
|
1613
|
+
* vanished would re-read, find nothing, and have no way to tell that from a
|
|
1614
|
+
* deletion.
|
|
1615
|
+
*
|
|
1616
|
+
* *Filtered per subscriber, per event, after the check.* The declared
|
|
1617
|
+
* `liveTargets` key is checked ON THE EVENT'S OWN ENTITY, through `ctx.check` —
|
|
1618
|
+
* the same evaluator, the same entity-narrowed grants, the same parent walk as the
|
|
1619
|
+
* read the client is about to make. An entity type no module declared is announced
|
|
1620
|
+
* to nobody. Knowing that a row exists and changed at 14:02 is information about
|
|
1621
|
+
* that row, so the empty payload is not what makes this safe; this is.
|
|
1622
|
+
*
|
|
1623
|
+
* *Never able to fail the operation.* The write has committed and the caller has
|
|
1624
|
+
* its answer. A socket that has gone away mid-fan-out, or a check that cannot be
|
|
1625
|
+
* evaluated, costs a subscriber its live update — which it survives, because the
|
|
1626
|
+
* client's contract is that a push is a hint and the poll is the floor.
|
|
1627
|
+
*/
|
|
1628
|
+
async fanOutLive(sinceEventId, tenantId, scopeId) {
|
|
1629
|
+
const sockets = this.ctx.getWebSockets();
|
|
1630
|
+
if (sockets.length === 0)
|
|
1631
|
+
return;
|
|
1632
|
+
const rows = this.sql
|
|
1633
|
+
.exec(`SELECT id, type, entity_type, entity_id, occurred_at FROM _substrat_outbox
|
|
1634
|
+
WHERE id > ? ORDER BY id LIMIT ?`, sinceEventId, LIVE_FANOUT_LIMIT)
|
|
1635
|
+
.toArray();
|
|
1636
|
+
// Drop the undeclared entity types BEFORE any per-subscriber work. Not an
|
|
1637
|
+
// optimisation: it is the fail-closed rule stated once, in the one place that
|
|
1638
|
+
// decides, rather than relied upon inside the loop below.
|
|
1639
|
+
const announceable = rows.filter((r) => this.liveTargets.has(r.entity_type));
|
|
1640
|
+
if (announceable.length === 0)
|
|
1641
|
+
return;
|
|
1642
|
+
for (const ws of sockets) {
|
|
1643
|
+
let subscription = null;
|
|
1644
|
+
try {
|
|
1645
|
+
subscription = readSubscription(ws.deserializeAttachment());
|
|
1646
|
+
}
|
|
1647
|
+
catch {
|
|
1648
|
+
subscription = null;
|
|
1649
|
+
}
|
|
1650
|
+
// A socket we cannot name is a socket we cannot filter for. Skipped, never
|
|
1651
|
+
// sent to — see `readSubscription` for why every unusable shape fails closed.
|
|
1652
|
+
if (!subscription)
|
|
1653
|
+
continue;
|
|
1654
|
+
// A socket that outlived a scope rebind, or was somehow accepted for another
|
|
1655
|
+
// node, must not be fed this scope's events. Cheap, and it makes the identity
|
|
1656
|
+
// the frames are filtered against an explicit precondition rather than an
|
|
1657
|
+
// assumption about how the subscription was created.
|
|
1658
|
+
if (subscription.tenantId !== tenantId || subscription.scopeId !== scopeId)
|
|
1659
|
+
continue;
|
|
1660
|
+
// One context per subscriber, not per event: `ctx.check` is the expensive part
|
|
1661
|
+
// and the context is only the subject it is evaluated for.
|
|
1662
|
+
//
|
|
1663
|
+
// A frame this subscriber does not pass is NOT recorded as a denial (K-35),
|
|
1664
|
+
// and that is deliberate: `recordDenial` is called on a refused REQUEST, where
|
|
1665
|
+
// somebody asked for something and was told no. Nobody asked for these. Logging
|
|
1666
|
+
// one row per unentitled subscriber per event would bury the denials that mean
|
|
1667
|
+
// something — a broken screen, or somebody walking the surface — under the
|
|
1668
|
+
// ordinary, correct working of a filter.
|
|
1669
|
+
//
|
|
1670
|
+
// The operation name is carried anyway, for the events a fan-out cannot emit
|
|
1671
|
+
// but a future reader of this context might.
|
|
1672
|
+
const ctx = this.operationContext(subscription.principal, tenantId, scopeId, undefined, undefined, undefined, undefined, undefined, 'live.subscribe');
|
|
1673
|
+
for (const row of announceable) {
|
|
1674
|
+
// Non-null: `announceable` is exactly the rows whose type is in the map.
|
|
1675
|
+
const permission = this.liveTargets.get(row.entity_type);
|
|
1676
|
+
let allowed = false;
|
|
1677
|
+
try {
|
|
1678
|
+
const decision = await ctx.check(permission, {
|
|
1679
|
+
entityType: row.entity_type,
|
|
1680
|
+
entityId: row.entity_id,
|
|
1681
|
+
});
|
|
1682
|
+
allowed = decision.allowed;
|
|
1683
|
+
}
|
|
1684
|
+
catch {
|
|
1685
|
+
// A check that cannot answer is a check that refuses. The alternative —
|
|
1686
|
+
// treating an evaluator failure as an allow — turns an outage in the
|
|
1687
|
+
// permission path into a disclosure, which is the one failure mode this
|
|
1688
|
+
// surface must not have.
|
|
1689
|
+
allowed = false;
|
|
1690
|
+
}
|
|
1691
|
+
if (!allowed)
|
|
1692
|
+
continue;
|
|
1693
|
+
const frame = {
|
|
1694
|
+
kind: 'change',
|
|
1695
|
+
id: row.id,
|
|
1696
|
+
type: row.type,
|
|
1697
|
+
entityType: row.entity_type,
|
|
1698
|
+
entityId: row.entity_id,
|
|
1699
|
+
at: row.occurred_at,
|
|
1700
|
+
};
|
|
1701
|
+
try {
|
|
1702
|
+
ws.send(JSON.stringify(frame));
|
|
1703
|
+
}
|
|
1704
|
+
catch {
|
|
1705
|
+
// The socket went away between `getWebSockets()` and here. Stop writing to
|
|
1706
|
+
// this one and move on; the runtime will deliver `webSocketClose`.
|
|
1707
|
+
break;
|
|
1708
|
+
}
|
|
1709
|
+
}
|
|
1710
|
+
}
|
|
1711
|
+
}
|
|
1322
1712
|
// -- attachments (#473): the metadata half of the attachment surface --------
|
|
1323
1713
|
// The coordinator (worker) holds the bytes and the per-tenant R2 binding; this
|
|
1324
1714
|
// DO holds the permission gate and the metadata fact, under the same per-scope
|
|
@@ -1359,6 +1749,10 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1359
1749
|
const parsed = attachmentRecord.parse(record);
|
|
1360
1750
|
const gate = this.attachmentGate(parsed.entity.entityType);
|
|
1361
1751
|
return this.queue.enqueue(async () => {
|
|
1752
|
+
// #938: this path commits and emits too, so it takes the same mark-then-settle
|
|
1753
|
+
// pair `invoke` takes. Inside the queued body and before the transaction, for
|
|
1754
|
+
// the reason it is read there: that is where this call holds the DO to itself.
|
|
1755
|
+
const liveSince = this.liveHighWaterMark();
|
|
1362
1756
|
try {
|
|
1363
1757
|
await this.ctx.storage.transaction(async () => {
|
|
1364
1758
|
const ctx = this.operationContext(principal, tenantId, scopeId, undefined, connectionId, undefined, undefined, undefined, 'attachments.upload');
|
|
@@ -1378,11 +1772,13 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1378
1772
|
}
|
|
1379
1773
|
catch (err) {
|
|
1380
1774
|
if (err instanceof PermissionDenied) {
|
|
1381
|
-
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.upload', err);
|
|
1775
|
+
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.upload', err, null);
|
|
1382
1776
|
}
|
|
1383
1777
|
throw toRpcError(err);
|
|
1384
1778
|
}
|
|
1385
|
-
|
|
1779
|
+
// #1525: null. An attachment RPC carries no invocation, exactly as its denial
|
|
1780
|
+
// records none — so the deliveries it drains name no call rather than a wrong one.
|
|
1781
|
+
await this.settleCommitted(tenantId, scopeId, liveSince, null);
|
|
1386
1782
|
return parsed;
|
|
1387
1783
|
});
|
|
1388
1784
|
}
|
|
@@ -1396,7 +1792,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1396
1792
|
}
|
|
1397
1793
|
catch (err) {
|
|
1398
1794
|
if (err instanceof PermissionDenied) {
|
|
1399
|
-
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.list', err);
|
|
1795
|
+
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.list', err, null);
|
|
1400
1796
|
}
|
|
1401
1797
|
throw toRpcError(err);
|
|
1402
1798
|
}
|
|
@@ -1427,7 +1823,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1427
1823
|
}
|
|
1428
1824
|
catch (err) {
|
|
1429
1825
|
if (err instanceof PermissionDenied) {
|
|
1430
|
-
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.open', err);
|
|
1826
|
+
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.open', err, null);
|
|
1431
1827
|
}
|
|
1432
1828
|
throw toRpcError(err);
|
|
1433
1829
|
}
|
|
@@ -1440,7 +1836,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1440
1836
|
}
|
|
1441
1837
|
catch (err) {
|
|
1442
1838
|
if (err instanceof PermissionDenied) {
|
|
1443
|
-
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.open', err);
|
|
1839
|
+
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.open', err, null);
|
|
1444
1840
|
}
|
|
1445
1841
|
throw toRpcError(err);
|
|
1446
1842
|
}
|
|
@@ -1454,6 +1850,8 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1454
1850
|
if (!record)
|
|
1455
1851
|
return null;
|
|
1456
1852
|
const gate = this.attachmentGate(record.entity.entityType);
|
|
1853
|
+
// #938: same mark-then-settle pair as the upload path above.
|
|
1854
|
+
const liveSince = this.liveHighWaterMark();
|
|
1457
1855
|
try {
|
|
1458
1856
|
await this.ctx.storage.transaction(async () => {
|
|
1459
1857
|
const ctx = this.operationContext(principal, tenantId, scopeId, undefined, connectionId, undefined, undefined, undefined, 'attachments.remove');
|
|
@@ -1470,32 +1868,128 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1470
1868
|
}
|
|
1471
1869
|
catch (err) {
|
|
1472
1870
|
if (err instanceof PermissionDenied) {
|
|
1473
|
-
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.remove', err);
|
|
1871
|
+
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.remove', err, null);
|
|
1474
1872
|
}
|
|
1475
1873
|
throw toRpcError(err);
|
|
1476
1874
|
}
|
|
1477
|
-
|
|
1875
|
+
// #1525: null. An attachment RPC carries no invocation, exactly as its denial
|
|
1876
|
+
// records none — so the deliveries it drains name no call rather than a wrong one.
|
|
1877
|
+
await this.settleCommitted(tenantId, scopeId, liveSince, null);
|
|
1478
1878
|
return record;
|
|
1479
1879
|
});
|
|
1480
1880
|
}
|
|
1881
|
+
/** The kernel's schedule-switch SQL (#1666), over this DO's storage. */
|
|
1882
|
+
switchSql() {
|
|
1883
|
+
return {
|
|
1884
|
+
all: (sql, ...params) => this.sql.exec(sql, ...params).toArray(),
|
|
1885
|
+
run: (sql, ...params) => {
|
|
1886
|
+
this.sql.exec(sql, ...params);
|
|
1887
|
+
},
|
|
1888
|
+
};
|
|
1889
|
+
}
|
|
1890
|
+
/**
|
|
1891
|
+
* Where a module's schedules stand on this scope (#383, #1666) — the kernel's
|
|
1892
|
+
* `systemScheduleState`, the predicate the pure adapter runs too: `on` with a live
|
|
1893
|
+
* `system:<moduleId>` grant, `off` while the kill switch's marker is live, whatever
|
|
1894
|
+
* else is, and `ungranted` on a scope that never ran the module (a foreign vertical's,
|
|
1895
|
+
* which the sweep skips quietly).
|
|
1896
|
+
*/
|
|
1897
|
+
async systemScheduleState(moduleId) {
|
|
1898
|
+
return systemScheduleState(this.switchSql(), moduleId, new Date().toISOString());
|
|
1899
|
+
}
|
|
1481
1900
|
/**
|
|
1482
|
-
*
|
|
1483
|
-
*
|
|
1484
|
-
*
|
|
1901
|
+
* The pre-#1666 read, kept for ONE reason: a coordinator a deploy behind this DO still
|
|
1902
|
+
* calls it. It answers the new question, not the old one — the old predicate counted any
|
|
1903
|
+
* live `system:` tuple, and the kill switch's OFF marker is one, so an old coordinator
|
|
1904
|
+
* asking the old question would run a switched-off scope's schedules.
|
|
1485
1905
|
*/
|
|
1486
1906
|
async hasSystemGrant(moduleId) {
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1907
|
+
return (await this.systemScheduleState(moduleId)) === 'on';
|
|
1908
|
+
}
|
|
1909
|
+
/**
|
|
1910
|
+
* `grantToSystem`'s scope-level write (#1666): the explicit grant — `INSERT OR REPLACE`,
|
|
1911
|
+
* as `writeTuple` — EXCEPT while the module's schedule kill switch is off, when it writes
|
|
1912
|
+
* nothing and answers `false`. The check and the write are one queued unit, so no switch
|
|
1913
|
+
* can move between them. Restore is the lever; a grant is not.
|
|
1914
|
+
*/
|
|
1915
|
+
async writeSystemGrant(moduleId, relation, object, expiresAt) {
|
|
1916
|
+
return this.queue.enqueue(() => {
|
|
1917
|
+
if (systemSwitchedOff(this.switchSql(), moduleId))
|
|
1918
|
+
return false;
|
|
1919
|
+
this.sql.exec(`INSERT OR REPLACE INTO _substrat_tuples (subject, relation, object, expires_at)
|
|
1920
|
+
VALUES (?, ?, ?, ?)`, `system:${moduleId}`, relation, object, expiresAt);
|
|
1921
|
+
return true;
|
|
1922
|
+
});
|
|
1494
1923
|
}
|
|
1495
|
-
/**
|
|
1924
|
+
/**
|
|
1925
|
+
* Move a module's schedule switch on this scope (#1666) — the kernel's
|
|
1926
|
+
* `switchSystemSchedules`, serialized on the queue with every other tuple write and run
|
|
1927
|
+
* as one `transactionSync`, so its reads and writes are one unit.
|
|
1928
|
+
*/
|
|
1929
|
+
async switchSystemSchedules(moduleId, scopeId, to, at) {
|
|
1930
|
+
return this.queue.enqueue(() => this.ctx.storage.transactionSync(() => switchSystemSchedules(this.switchSql(), { moduleId, scopeId, to, at })));
|
|
1931
|
+
}
|
|
1932
|
+
/**
|
|
1933
|
+
* The exchange (#1672) — a capability's secret traded for a session, or for the
|
|
1934
|
+
* principal a `become` capability yields. Runs the kernel's `exchangeCapability`, the
|
|
1935
|
+
* function the pure adapter runs, in this DO: one queued body and one storage
|
|
1936
|
+
* transaction, so the use it takes and the `capability.exercised` event recording it
|
|
1937
|
+
* commit together or not at all; the event's consumers then settle as an invoke's do.
|
|
1938
|
+
*
|
|
1939
|
+
* The secret itself reaches this DO — it must, to be hashed and looked up — and goes
|
|
1940
|
+
* no further: nothing here stores or returns it.
|
|
1941
|
+
*/
|
|
1942
|
+
async exchangeCapability(secret, tenantId, scopeId, mode) {
|
|
1943
|
+
await this.ensureMigrations();
|
|
1944
|
+
return await this.queue.enqueue(async () => {
|
|
1945
|
+
const liveSince = this.liveHighWaterMark();
|
|
1946
|
+
// ONE instant for the whole exchange: the row's `last_used_at`, the session's times
|
|
1947
|
+
// and the event's `occurredAt` are one fact, and two clock reads could disagree.
|
|
1948
|
+
const now = instant.parse(new Date().toISOString());
|
|
1949
|
+
let outcome = null;
|
|
1950
|
+
await this.ctx.storage.transaction(async () => {
|
|
1951
|
+
outcome = await exchangeCapability({
|
|
1952
|
+
sql: doSpineSql(this.sql),
|
|
1953
|
+
now,
|
|
1954
|
+
// Stamped `{ capability }` by the ordinary emit path, under the exchange's own
|
|
1955
|
+
// pseudo-operation name — the actor is the capability, as on every event it
|
|
1956
|
+
// goes on to cause.
|
|
1957
|
+
emit: (capability, event) => this.operationContext(principalId.parse(ulid()), tenantId, scopeId, undefined, undefined, undefined, undefined, undefined, CAPABILITY_EXCHANGE_OPERATION, capability, [], now).emit(event),
|
|
1958
|
+
}, secret, mode);
|
|
1959
|
+
});
|
|
1960
|
+
if (outcome)
|
|
1961
|
+
await this.settleCommitted(tenantId, scopeId, liveSince, null);
|
|
1962
|
+
return outcome;
|
|
1963
|
+
});
|
|
1964
|
+
}
|
|
1965
|
+
/**
|
|
1966
|
+
* The platform's mint (#1672, `HostAdmin.mintCapability`) — a `become` capability,
|
|
1967
|
+
* serialized on the queue with every other spine write. One INSERT, so it needs no
|
|
1968
|
+
* transaction of its own; the hash is computed before it.
|
|
1969
|
+
*/
|
|
1970
|
+
async mintBecomeCapability(input, actor) {
|
|
1971
|
+
await this.ensureMigrations();
|
|
1972
|
+
return await this.queue.enqueue(() => mintBecomeCapability(doSpineSql(this.sql), input, actor, instant.parse(new Date().toISOString())));
|
|
1973
|
+
}
|
|
1974
|
+
/**
|
|
1975
|
+
* The platform's revoke (#1672, `HostAdmin.revokeCapability`) — of any capability in
|
|
1976
|
+
* this scope. Returns the record as it stood before, for the admin log, or `null`.
|
|
1977
|
+
*/
|
|
1978
|
+
async revokeCapabilityAsPlatform(id, actor) {
|
|
1979
|
+
await this.ensureMigrations();
|
|
1980
|
+
return await this.queue.enqueue(() => this.ctx.storage.transactionSync(() => revokeCapabilityAsPlatform(doSpineSql(this.sql), id, actor, instant.parse(new Date().toISOString()))) ?? null);
|
|
1981
|
+
}
|
|
1982
|
+
/**
|
|
1983
|
+
* The last time a schedule's operation ran on this scope (#383), or null.
|
|
1984
|
+
*
|
|
1985
|
+
* `kind = 'schedule'` is not decoration (#1288): an operation may legally be
|
|
1986
|
+
* named `freshness:<something>`, and before the column this read answered with
|
|
1987
|
+
* the EVALUATOR's last recorded time for that event type — a cadence gate
|
|
1988
|
+
* driven by a verdict nothing ran.
|
|
1989
|
+
*/
|
|
1496
1990
|
async scheduleLastRun(operation) {
|
|
1497
1991
|
const row = this.sql
|
|
1498
|
-
.exec(`SELECT last_run_at FROM _substrat_schedule_state WHERE schedule_op = ?`, operation)
|
|
1992
|
+
.exec(`SELECT last_run_at FROM _substrat_schedule_state WHERE kind = 'schedule' AND schedule_op = ?`, operation)
|
|
1499
1993
|
.toArray()[0];
|
|
1500
1994
|
return row?.last_run_at ?? null;
|
|
1501
1995
|
}
|
|
@@ -1503,8 +1997,9 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1503
1997
|
* #1232: everything the freshness evaluator needs about this scope, one round
|
|
1504
1998
|
* trip regardless of how many types are declared — the newest matching event
|
|
1505
1999
|
* per type, plus each type's recorded evaluator state (last recorded at +
|
|
1506
|
-
* outcome, kept in `_substrat_schedule_state` under
|
|
1507
|
-
*
|
|
2000
|
+
* outcome, kept in `_substrat_schedule_state` under `kind = 'freshness'`, which
|
|
2001
|
+
* since #1288 is what separates these rows from the schedule rows beside them —
|
|
2002
|
+
* the `freshness:` prefix on the key is retained but no longer load-bearing).
|
|
1508
2003
|
*/
|
|
1509
2004
|
async freshnessProbe(types) {
|
|
1510
2005
|
const out = {};
|
|
@@ -1521,7 +2016,8 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1521
2016
|
}
|
|
1522
2017
|
const keys = types.map((t) => `freshness:${t}`);
|
|
1523
2018
|
for (const row of this.sql
|
|
1524
|
-
.exec(`SELECT schedule_op, last_run_at, last_status FROM _substrat_schedule_state
|
|
2019
|
+
.exec(`SELECT schedule_op, last_run_at, last_status FROM _substrat_schedule_state
|
|
2020
|
+
WHERE kind = 'freshness' AND schedule_op IN (${marks})`, ...keys)
|
|
1525
2021
|
.toArray()) {
|
|
1526
2022
|
const t = row.schedule_op.slice('freshness:'.length);
|
|
1527
2023
|
if (out[t]) {
|
|
@@ -1533,16 +2029,173 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1533
2029
|
}
|
|
1534
2030
|
/**
|
|
1535
2031
|
* Write one `_substrat_schedule_state` row (#383). Spine, kernel-written.
|
|
1536
|
-
*
|
|
2032
|
+
*
|
|
2033
|
+
* `kind` is PASSED, never derived from the shape of `unit` (#1288) — deriving it
|
|
2034
|
+
* is precisely the convention this column replaced, and it is wrong for the one
|
|
2035
|
+
* input that matters: a schedule operation named `freshness:orders.placed` is a
|
|
2036
|
+
* schedule row, whatever its key looks like. `unit` is that row's key: a schedule
|
|
1537
2037
|
* operation (`module/verb`, and then `at`/`status` are when it ran and how it
|
|
1538
2038
|
* ended) or a freshness key (`freshness:<eventType>`, and then they are when the
|
|
1539
2039
|
* verdict was recorded and what it was — nothing ran). See the bootstrap DDL.
|
|
2040
|
+
*
|
|
2041
|
+
* `kind` is LAST because this is a positional RPC — see the interface's own note
|
|
2042
|
+
* in `host.ts`. An old DO drops a trailing argument; a leading one it binds, and
|
|
2043
|
+
* every value after it lands one column to the left, silently.
|
|
1540
2044
|
*/
|
|
1541
|
-
async recordScheduleRun(unit, at, status) {
|
|
1542
|
-
this.sql.exec(`INSERT INTO _substrat_schedule_state (schedule_op, last_run_at, last_status)
|
|
1543
|
-
VALUES (?, ?, ?)
|
|
1544
|
-
ON CONFLICT(schedule_op) DO UPDATE SET last_run_at = excluded.last_run_at,
|
|
1545
|
-
|
|
2045
|
+
async recordScheduleRun(unit, at, status, kind) {
|
|
2046
|
+
this.sql.exec(`INSERT INTO _substrat_schedule_state (kind, schedule_op, last_run_at, last_status)
|
|
2047
|
+
VALUES (?, ?, ?, ?)
|
|
2048
|
+
ON CONFLICT(kind, schedule_op) DO UPDATE SET last_run_at = excluded.last_run_at,
|
|
2049
|
+
last_status = excluded.last_status`, kind, unit, at, status);
|
|
2050
|
+
}
|
|
2051
|
+
// -- the resumable-run driver's store (#1577) -----------------------------
|
|
2052
|
+
//
|
|
2053
|
+
// D-14's DURABLE driver, and the only half of it that lives in the DO: the run
|
|
2054
|
+
// record and the step ledger. Every decision — what coalescing means, when a
|
|
2055
|
+
// step is skipped, when a run fails — is in the kernel's `JobRunStore` callers,
|
|
2056
|
+
// which the coordinator drives; these are reads and writes and nothing else, so
|
|
2057
|
+
// the two drivers cannot disagree about any of it.
|
|
2058
|
+
//
|
|
2059
|
+
// Each step commits on its OWN round trip rather than the pass batching them at
|
|
2060
|
+
// the end. That is what a mid-pass eviction keeps, and a DO is evicted and
|
|
2061
|
+
// revived constantly — batching would lose exactly the work resume exists for.
|
|
2062
|
+
/**
|
|
2063
|
+
* Coalescing, as ONE round trip: the live (`running`) run for this key, or this
|
|
2064
|
+
* row inserted and returned.
|
|
2065
|
+
*
|
|
2066
|
+
* **The single RPC is the atomicity**, and that is the whole reason it is shaped
|
|
2067
|
+
* this way rather than as a `jobRunLive` the coordinator follows with a
|
|
2068
|
+
* `jobRunInsert`. A Durable Object serializes its RPCs, so the lookup and the
|
|
2069
|
+
* insert cannot be interleaved by another caller; split across two calls they
|
|
2070
|
+
* can, and two concurrent starts then both find nothing and both insert. The
|
|
2071
|
+
* schema carries no unique constraint to catch that, deliberately — a crashed
|
|
2072
|
+
* run must stay restartable — so the indivisibility has to come from here.
|
|
2073
|
+
*/
|
|
2074
|
+
async jobRunStartOrJoin(moduleId, job, instance, row) {
|
|
2075
|
+
const live = this.sql
|
|
2076
|
+
.exec(`SELECT * FROM _substrat_job_runs
|
|
2077
|
+
WHERE module_id = ? AND job = ? AND instance = ? AND status = 'running'
|
|
2078
|
+
ORDER BY id DESC LIMIT 1`, moduleId, job, instance)
|
|
2079
|
+
.toArray()[0] ?? null;
|
|
2080
|
+
if (live)
|
|
2081
|
+
return live;
|
|
2082
|
+
// `transactionSync`, and the two writes inlined rather than reached through
|
|
2083
|
+
// `jobRunInsert`: an `await` between them is an output-gate boundary, and the
|
|
2084
|
+
// point of doing this in one RPC is that there is no boundary to be evicted
|
|
2085
|
+
// at. Same reason `SCHEDULE_STATE_REBUILD` insists on it.
|
|
2086
|
+
this.ctx.storage.transactionSync(() => {
|
|
2087
|
+
this.sql.exec(`INSERT INTO _substrat_job_runs
|
|
2088
|
+
(id, module_id, job, instance, payload, status, cursor, counters, attempts,
|
|
2089
|
+
last_error, started_at, updated_at, next_attempt_at, ended_at)
|
|
2090
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, row.id, row.module_id, row.job, row.instance, row.payload, row.status, row.cursor, row.counters, row.attempts, row.last_error, row.started_at, row.updated_at, row.next_attempt_at, row.ended_at);
|
|
2091
|
+
});
|
|
2092
|
+
return row;
|
|
2093
|
+
}
|
|
2094
|
+
/** One run by id, whatever its status. */
|
|
2095
|
+
async jobRunById(id) {
|
|
2096
|
+
return (this.sql.exec('SELECT * FROM _substrat_job_runs WHERE id = ?', id).toArray()[0] ?? null);
|
|
2097
|
+
}
|
|
2098
|
+
/** Insert a fresh run. The coordinator has already refused a non-queue-safe payload. */
|
|
2099
|
+
async jobRunInsert(row) {
|
|
2100
|
+
this.sql.exec(`INSERT INTO _substrat_job_runs
|
|
2101
|
+
(id, module_id, job, instance, payload, status, cursor, counters, attempts,
|
|
2102
|
+
last_error, started_at, updated_at, next_attempt_at, ended_at)
|
|
2103
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, row.id, row.module_id, row.job, row.instance, row.payload, row.status, row.cursor, row.counters, row.attempts, row.last_error, row.started_at, row.updated_at, row.next_attempt_at, row.ended_at);
|
|
2104
|
+
}
|
|
2105
|
+
/**
|
|
2106
|
+
* `running` runs whose backoff has elapsed, oldest first, after `afterId`.
|
|
2107
|
+
*
|
|
2108
|
+
* The cursor is what lets the coordinator page past runs it cannot drive: it
|
|
2109
|
+
* skips any whose job this deployment does not register, and without a cursor
|
|
2110
|
+
* those rows head every batch forever (`runDueJobRuns`). `afterId` is LAST, as
|
|
2111
|
+
* every argument added to an RPC on this interface must be.
|
|
2112
|
+
*/
|
|
2113
|
+
async jobRunsDue(now, limit, afterId) {
|
|
2114
|
+
return this.sql
|
|
2115
|
+
.exec(
|
|
2116
|
+
// `afterId` bound TWICE rather than as `?2`: mixing anonymous and numbered
|
|
2117
|
+
// parameters makes the anonymous ones resume from the highest index used,
|
|
2118
|
+
// which is a footgun for the next person to add a clause. Spelled exactly
|
|
2119
|
+
// as the pure adapter spells it.
|
|
2120
|
+
`SELECT * FROM _substrat_job_runs
|
|
2121
|
+
WHERE status = 'running' AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
|
|
2122
|
+
AND (? IS NULL OR id > ?)
|
|
2123
|
+
ORDER BY id LIMIT ?`, now, afterId ?? null, afterId ?? null, limit)
|
|
2124
|
+
.toArray();
|
|
2125
|
+
}
|
|
2126
|
+
/** The operator read, newest first. */
|
|
2127
|
+
async jobRunList(filter) {
|
|
2128
|
+
const where = [];
|
|
2129
|
+
const params = [];
|
|
2130
|
+
for (const [column, value] of [
|
|
2131
|
+
['module_id', filter.moduleId],
|
|
2132
|
+
['job', filter.job],
|
|
2133
|
+
['instance', filter.instance],
|
|
2134
|
+
['status', filter.status],
|
|
2135
|
+
]) {
|
|
2136
|
+
if (value !== undefined) {
|
|
2137
|
+
where.push(`${column} = ?`);
|
|
2138
|
+
params.push(value);
|
|
2139
|
+
}
|
|
2140
|
+
}
|
|
2141
|
+
params.push(jobRunListLimit(filter.limit));
|
|
2142
|
+
return this.sql
|
|
2143
|
+
.exec(`SELECT * FROM _substrat_job_runs
|
|
2144
|
+
${where.length ? `WHERE ${where.join(' AND ')}` : ''}
|
|
2145
|
+
ORDER BY id DESC LIMIT ?`, ...params)
|
|
2146
|
+
.toArray();
|
|
2147
|
+
}
|
|
2148
|
+
/**
|
|
2149
|
+
* Write a pass outcome onto the run row — every column at once.
|
|
2150
|
+
*
|
|
2151
|
+
* Whole rather than partial, and that is the shape not an accident: a commit
|
|
2152
|
+
* writes cursor AND counters AND a cleared error AND a cleared backoff, and a
|
|
2153
|
+
* partial update that wrote three of the four would leave a run carrying the
|
|
2154
|
+
* last failure's error beside the new cursor, which reads as broken forever.
|
|
2155
|
+
*/
|
|
2156
|
+
async jobRunPatch(id, patch) {
|
|
2157
|
+
this.sql.exec(`UPDATE _substrat_job_runs
|
|
2158
|
+
SET status = ?, cursor = ?, counters = ?, attempts = ?, last_error = ?,
|
|
2159
|
+
updated_at = ?, next_attempt_at = ?, ended_at = ?
|
|
2160
|
+
WHERE id = ?`, patch.status, patch.cursor, patch.counters, patch.attempts, patch.lastError, patch.updatedAt, patch.nextAttemptAt, patch.endedAt, id);
|
|
2161
|
+
}
|
|
2162
|
+
/** One step's ledger row — a non-null `result` is what means completed. */
|
|
2163
|
+
async jobStepRow(runId, step) {
|
|
2164
|
+
return (this.sql
|
|
2165
|
+
.exec('SELECT step, result, attempts, last_error FROM _substrat_job_steps WHERE run_id = ? AND step = ?', runId, step)
|
|
2166
|
+
.toArray()[0] ?? null);
|
|
2167
|
+
}
|
|
2168
|
+
/** Record one step attempt. `result` non-null = it completed and must not re-run. */
|
|
2169
|
+
async jobStepRecord(runId, step, result, attempts, lastError, at) {
|
|
2170
|
+
this.sql.exec(`INSERT INTO _substrat_job_steps (run_id, step, result, attempts, last_error, recorded_at)
|
|
2171
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
2172
|
+
ON CONFLICT (run_id, step) DO UPDATE SET result = excluded.result,
|
|
2173
|
+
attempts = excluded.attempts,
|
|
2174
|
+
last_error = excluded.last_error,
|
|
2175
|
+
recorded_at = excluded.recorded_at`, runId, step, result, attempts, lastError, at);
|
|
2176
|
+
}
|
|
2177
|
+
/**
|
|
2178
|
+
* A COMMITTED pass: the run's new state and the dropping of its step ledger, in
|
|
2179
|
+
* ONE RPC so they cannot come apart.
|
|
2180
|
+
*
|
|
2181
|
+
* Same reasoning as `jobRunStartOrJoin` — the single round trip is the
|
|
2182
|
+
* atomicity. As two calls, a stop in between leaves the advanced cursor beside
|
|
2183
|
+
* the finished pass's memo rows, and a handler that reuses a step name across
|
|
2184
|
+
* passes (legal: the determinism rule binds names to the payload and prior
|
|
2185
|
+
* results, not to the cursor) then skips work it never did. The kernel's
|
|
2186
|
+
* `runJobPass` carries the full argument.
|
|
2187
|
+
*/
|
|
2188
|
+
async jobCommitPass(id, patch) {
|
|
2189
|
+
// `transactionSync` with both statements inline — NOT `await
|
|
2190
|
+
// this.jobRunPatch(...)` then the delete. The await is an output-gate
|
|
2191
|
+
// boundary, which is precisely the gap this method exists to close.
|
|
2192
|
+
this.ctx.storage.transactionSync(() => {
|
|
2193
|
+
this.sql.exec(`UPDATE _substrat_job_runs
|
|
2194
|
+
SET status = ?, cursor = ?, counters = ?, attempts = ?, last_error = ?,
|
|
2195
|
+
updated_at = ?, next_attempt_at = ?, ended_at = ?
|
|
2196
|
+
WHERE id = ?`, patch.status, patch.cursor, patch.counters, patch.attempts, patch.lastError, patch.updatedAt, patch.nextAttemptAt, patch.endedAt, id);
|
|
2197
|
+
this.sql.exec('DELETE FROM _substrat_job_steps WHERE run_id = ?', id);
|
|
2198
|
+
});
|
|
1546
2199
|
}
|
|
1547
2200
|
// -- guards (K-17) --------------------------------------------------------
|
|
1548
2201
|
/** This scope's spine, read under whatever transaction the caller already opened. */
|
|
@@ -1644,8 +2297,15 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1644
2297
|
*
|
|
1645
2298
|
* "Not yet consumed" means never attempted, or retrying and now due (#100).
|
|
1646
2299
|
* Terminal rows — delivered or dead-lettered — are excluded by the join.
|
|
2300
|
+
*
|
|
2301
|
+
* Decoded per row (#1636). A row that will not decode comes back in `undecodable`,
|
|
2302
|
+
* never as an event: the coordinator journals it as a dead letter, and its handler is
|
|
2303
|
+
* never handed an event built from stand-ins. The whole list used to be one
|
|
2304
|
+
* `rows.map(decode)`, so one bad row threw the executor's pending list on every pass.
|
|
2305
|
+
* The journal write stays on the coordinator, beside every other attempt it records —
|
|
2306
|
+
* this stays a read.
|
|
1647
2307
|
*/
|
|
1648
|
-
|
|
2308
|
+
pendingExecutorDeliveries(deliveryId, eventType) {
|
|
1649
2309
|
const rows = this.sql
|
|
1650
2310
|
.exec(`SELECT o.* FROM _substrat_outbox o
|
|
1651
2311
|
LEFT JOIN _substrat_deliveries d
|
|
@@ -1655,7 +2315,26 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1655
2315
|
OR (d.next_attempt_at IS NOT NULL AND d.next_attempt_at <= ?))
|
|
1656
2316
|
ORDER BY o.id`, deliveryId, eventType, new Date().toISOString())
|
|
1657
2317
|
.toArray();
|
|
1658
|
-
|
|
2318
|
+
const events = [];
|
|
2319
|
+
const undecodable = [];
|
|
2320
|
+
for (const r of rows) {
|
|
2321
|
+
try {
|
|
2322
|
+
events.push(domainEventOf(r));
|
|
2323
|
+
}
|
|
2324
|
+
catch (err) {
|
|
2325
|
+
undecodable.push({ eventId: r.id, error: String(err) });
|
|
2326
|
+
}
|
|
2327
|
+
}
|
|
2328
|
+
return { events, undecodable };
|
|
2329
|
+
}
|
|
2330
|
+
/**
|
|
2331
|
+
* The same read as a bare list of events, for a coordinator deployed before
|
|
2332
|
+
* `pendingExecutorDeliveries` (#1636). An undecodable row is left out rather than
|
|
2333
|
+
* thrown, so that pairing still delivers the rows behind it; the next coordinator
|
|
2334
|
+
* dead-letters it.
|
|
2335
|
+
*/
|
|
2336
|
+
pendingExecutorEvents(deliveryId, eventType) {
|
|
2337
|
+
return this.pendingExecutorDeliveries(deliveryId, eventType).events;
|
|
1659
2338
|
}
|
|
1660
2339
|
/**
|
|
1661
2340
|
* Journal one executor attempt (#100). `error` null means delivered;
|
|
@@ -1665,20 +2344,44 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1665
2344
|
* Written AFTER the effect, so a crash mid-effect retries rather than silently
|
|
1666
2345
|
* marking success. The coordinator computes the backoff because it owns the
|
|
1667
2346
|
* per-executor policy; the DO owns the state.
|
|
2347
|
+
*
|
|
2348
|
+
* `invocationId` is LAST because this is a positional RPC — the same rule
|
|
2349
|
+
* `recordScheduleRun` states. An old DO drops a trailing argument; a leading one
|
|
2350
|
+
* it binds, and every value after it lands one column to the left, silently.
|
|
1668
2351
|
*/
|
|
1669
|
-
recordExecutorAttempt(eventId, deliveryId, error, nextAttemptAt
|
|
2352
|
+
recordExecutorAttempt(eventId, deliveryId, error, nextAttemptAt,
|
|
2353
|
+
/**
|
|
2354
|
+
* #1525: the call THIS attempt ran in, or null. Passed rather than read off
|
|
2355
|
+
* `this.invocationId`, and here there is no choice about it: executors run on
|
|
2356
|
+
* the COORDINATOR, so by the time this RPC arrives the queued body that held
|
|
2357
|
+
* the id has long returned and the field reads null. The coordinator is the
|
|
2358
|
+
* only side that knows, which is why the SQLite twin passes it too — a recorded
|
|
2359
|
+
* fact whose correctness argument differs per adapter is the kind that drifts.
|
|
2360
|
+
*
|
|
2361
|
+
* Defaulted, for the other half of the skew: a coordinator too old to pass it
|
|
2362
|
+
* calls this with four arguments, and `undefined` is not a value SQLite binds.
|
|
2363
|
+
* Null is the honest answer there anyway — that coordinator recorded no call.
|
|
2364
|
+
*/
|
|
2365
|
+
invocationId = null) {
|
|
1670
2366
|
const prior = this.sql
|
|
1671
2367
|
.exec('SELECT attempts FROM _substrat_deliveries WHERE event_id = ? AND consumer_module = ?', eventId, deliveryId)
|
|
1672
2368
|
.toArray()[0];
|
|
1673
2369
|
const attempts = (prior?.attempts ?? 0) + 1;
|
|
1674
2370
|
this.sql.exec(`INSERT INTO _substrat_deliveries
|
|
1675
|
-
(event_id, consumer_module, delivered_at, error, attempts, next_attempt_at
|
|
1676
|
-
|
|
2371
|
+
(event_id, consumer_module, delivered_at, error, attempts, next_attempt_at,
|
|
2372
|
+
invocation_id)
|
|
2373
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
1677
2374
|
ON CONFLICT (event_id, consumer_module) DO UPDATE SET
|
|
1678
2375
|
delivered_at = excluded.delivered_at,
|
|
1679
2376
|
error = excluded.error,
|
|
1680
2377
|
attempts = excluded.attempts,
|
|
1681
|
-
next_attempt_at = excluded.next_attempt_at
|
|
2378
|
+
next_attempt_at = excluded.next_attempt_at,
|
|
2379
|
+
-- #1525: overwritten, like the four above. The row describes the LATEST
|
|
2380
|
+
-- attempt, so a retry drained by a different call (or by none) must not
|
|
2381
|
+
-- keep claiming the call that made the first one.
|
|
2382
|
+
invocation_id = excluded.invocation_id`, eventId, deliveryId, new Date().toISOString(), error, attempts, nextAttemptAt,
|
|
2383
|
+
// #1525: the call this attempt ran in, as the coordinator named it.
|
|
2384
|
+
invocationId);
|
|
1682
2385
|
return attempts;
|
|
1683
2386
|
}
|
|
1684
2387
|
/** How many attempts this delivery has already had — the backoff input. */
|
|
@@ -1720,12 +2423,21 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1720
2423
|
* `result` is COALESCE'd so a value written on an earlier pass (e.g. a minted sibling scope id,
|
|
1721
2424
|
* for two-phase idempotency) survives a null on retry. `attempts` bumps each settle; `settled_at`
|
|
1722
2425
|
* is set only on a terminal outcome.
|
|
2426
|
+
*
|
|
2427
|
+
* **Compare-and-set on `pending` (#1600 review).** The drain reads pending rows, runs a
|
|
2428
|
+
* handler, then settles — and between the read and the settle a subject erasure can redact
|
|
2429
|
+
* the row. Settling by `id` alone let that stale pass overwrite the redaction and write a
|
|
2430
|
+
* provider's reply, which can quote the person, back into `last_error`. Nothing legitimate
|
|
2431
|
+
* is refused: `pendingPlatformRequests` returns only pending rows, so every settle targets
|
|
2432
|
+
* one that was pending when it was read. A settle that finds the row already terminal does
|
|
2433
|
+
* nothing, deliberately silently — throwing would make the drain's blanket catch retry a
|
|
2434
|
+
* row that is correctly over.
|
|
1723
2435
|
*/
|
|
1724
2436
|
settlePlatformRequest(id, status, result, lastError, lastFailure = null) {
|
|
1725
2437
|
this.sql.exec(`UPDATE _substrat_platform_requests
|
|
1726
2438
|
SET status = ?, result = COALESCE(?, result), last_error = ?, last_failure = ?,
|
|
1727
2439
|
attempts = attempts + 1, settled_at = ?
|
|
1728
|
-
WHERE id =
|
|
2440
|
+
WHERE id = ? AND status = 'pending'`, status, result, lastError, lastFailure, status === 'pending' ? null : new Date().toISOString(), id);
|
|
1729
2441
|
}
|
|
1730
2442
|
/**
|
|
1731
2443
|
* Turn one connector delivery into a `connector:<provider>` platform intent (#574
|
|
@@ -1736,8 +2448,18 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1736
2448
|
* the event on the next drain). Backpressure throws BEFORE any write — the caller
|
|
1737
2449
|
* records a failed attempt and the delivery retries on its own backoff, exactly as
|
|
1738
2450
|
* a throwing handler would.
|
|
2451
|
+
*
|
|
2452
|
+
* `invocationId` is LAST, and defaulted, for the RPC-skew reason `recordExecutorAttempt`
|
|
2453
|
+
* states at length.
|
|
1739
2454
|
*/
|
|
1740
|
-
routeExecutorEventToPlatform(eventId, deliveryId, kind, payload, requestedBy
|
|
2455
|
+
routeExecutorEventToPlatform(eventId, deliveryId, kind, payload, requestedBy,
|
|
2456
|
+
/**
|
|
2457
|
+
* #1525: the call this routing ran in, or null. On the hosted path this is the
|
|
2458
|
+
* dominant executor journal — every vertical is CP-less, so a connector delivery
|
|
2459
|
+
* becomes an intent here rather than running in `recordExecutorAttempt`'s caller —
|
|
2460
|
+
* so leaving it null would make the column read "no call" for the common case.
|
|
2461
|
+
*/
|
|
2462
|
+
invocationId = null) {
|
|
1741
2463
|
const pending = Number(this.sql
|
|
1742
2464
|
.exec(`SELECT COUNT(*) AS c FROM _substrat_platform_requests WHERE status = 'pending'`)
|
|
1743
2465
|
.toArray()[0].c);
|
|
@@ -1757,7 +2479,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1757
2479
|
this.sql.exec(`INSERT INTO _substrat_platform_requests
|
|
1758
2480
|
(id, kind, payload, requested_by, impersonation, status, attempts, requested_at)
|
|
1759
2481
|
VALUES (?, ?, ?, ?, ?, 'pending', 0, ?)`, id, kind, payload, requestedBy, source?.impersonation ?? null, instant.parse(new Date().toISOString()));
|
|
1760
|
-
this.recordExecutorAttempt(eventId, deliveryId, null, null);
|
|
2482
|
+
this.recordExecutorAttempt(eventId, deliveryId, null, null, invocationId);
|
|
1761
2483
|
return id;
|
|
1762
2484
|
}
|
|
1763
2485
|
/**
|
|
@@ -1933,6 +2655,9 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1933
2655
|
* `destScopeId` is the scope being written INTO. A dump carries scope-level tuples
|
|
1934
2656
|
* naming the scope it was captured from, so restoring one anywhere else needs them
|
|
1935
2657
|
* re-pointed — see `rewriteScopeTuples`.
|
|
2658
|
+
*
|
|
2659
|
+
* What the dump did NOT carry is rebuilt by the next migration pass, which is why
|
|
2660
|
+
* this ends by forgetting the memoised one (#1589) — see the tail of the method.
|
|
1936
2661
|
*/
|
|
1937
2662
|
/**
|
|
1938
2663
|
* The additive spine-column migrations. KERNEL_DDL is all IF NOT EXISTS, so a
|
|
@@ -1951,6 +2676,11 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1951
2676
|
// consumer dead-letter.
|
|
1952
2677
|
'ALTER TABLE _substrat_deliveries ADD COLUMN attempts INTEGER NOT NULL DEFAULT 0',
|
|
1953
2678
|
'ALTER TABLE _substrat_deliveries ADD COLUMN next_attempt_at TEXT',
|
|
2679
|
+
// #1525: the invocation an attempt ran in, on a scope DO created before the
|
|
2680
|
+
// column. Nullable, and the null is honestly "no call was carried" — which
|
|
2681
|
+
// attempt produced a delivery already journalled cannot be decided afterwards,
|
|
2682
|
+
// exactly as #1237's outbox column argued.
|
|
2683
|
+
'ALTER TABLE _substrat_deliveries ADD COLUMN invocation_id TEXT',
|
|
1954
2684
|
// K-34: the authorization column on a scope DO created before it existed. Nullable,
|
|
1955
2685
|
// so legacy outbox rows read as "unrecorded". (_substrat_denials is a new table,
|
|
1956
2686
|
// covered by KERNEL_DDL's IF NOT EXISTS with no ALTER.)
|
|
@@ -1966,6 +2696,10 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1966
2696
|
'ALTER TABLE _substrat_outbox ADD COLUMN impersonation TEXT',
|
|
1967
2697
|
'ALTER TABLE _substrat_platform_requests ADD COLUMN impersonation TEXT',
|
|
1968
2698
|
'ALTER TABLE _substrat_denials ADD COLUMN impersonation TEXT',
|
|
2699
|
+
// #1525: the invocation a refusal happened during, on a scope DO created before
|
|
2700
|
+
// the column. Nullable, and the null is honestly "no id was carried" — a past
|
|
2701
|
+
// denial's call cannot be decided afterwards, as #1237's outbox column argued.
|
|
2702
|
+
'ALTER TABLE _substrat_denials ADD COLUMN invocation_id TEXT',
|
|
1969
2703
|
// #1231: the emitting operation, on a scope DO created before the column.
|
|
1970
2704
|
// Nullable so every legacy row reads as unrecorded rather than named.
|
|
1971
2705
|
'ALTER TABLE _substrat_outbox ADD COLUMN operation TEXT',
|
|
@@ -2001,6 +2735,33 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2001
2735
|
// boot. `lint:spine-ddl` compares KERNEL_DDL's indexes only, so this one is held to
|
|
2002
2736
|
// both adapters by the query-plan test rather than by that gate.
|
|
2003
2737
|
this.sql.exec('CREATE INDEX IF NOT EXISTS _substrat_outbox_invocation ON _substrat_outbox (invocation_id, id)');
|
|
2738
|
+
this.ensureScheduleStateKind();
|
|
2739
|
+
}
|
|
2740
|
+
/**
|
|
2741
|
+
* #1288: `_substrat_schedule_state`, rebuilt with `kind` in its key on a scope DO
|
|
2742
|
+
* whose table predates the column. Not in the ALTER list above, because `kind`
|
|
2743
|
+
* joins the PRIMARY KEY and no ALTER can widen a key — the statements are the
|
|
2744
|
+
* kernel's, so the pure adapter rebuilds byte-identically.
|
|
2745
|
+
*
|
|
2746
|
+
* Detected from `sqlite_master.sql`: DO SQLite restricts `PRAGMA`, and reading
|
|
2747
|
+
* the stored DDL is the one probe both adapters can make.
|
|
2748
|
+
*/
|
|
2749
|
+
ensureScheduleStateKind() {
|
|
2750
|
+
const row = this.sql
|
|
2751
|
+
.exec(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`, '_substrat_schedule_state')
|
|
2752
|
+
.toArray()[0];
|
|
2753
|
+
if (!row || scheduleStateHasKind(row.sql))
|
|
2754
|
+
return;
|
|
2755
|
+
// In a transaction, for the reason the kernel constant spells out: create-copy-
|
|
2756
|
+
// drop-rename has two intermediate states and both are unrecoverable on the next
|
|
2757
|
+
// wake. `transactionSync`, not the async one every operation uses — the DO
|
|
2758
|
+
// runtime forbids a manual BEGIN through `sql.exec`, and this body is wholly
|
|
2759
|
+
// synchronous, which is the one case the sync API is for (it commits at the
|
|
2760
|
+
// first await, and there is none). It also has to be sync because the caller is.
|
|
2761
|
+
this.ctx.storage.transactionSync(() => {
|
|
2762
|
+
for (const stmt of splitSqlStatements(SCHEDULE_STATE_REBUILD))
|
|
2763
|
+
this.sql.exec(stmt);
|
|
2764
|
+
});
|
|
2004
2765
|
}
|
|
2005
2766
|
async importDump(tables, destScopeId) {
|
|
2006
2767
|
// The WHOLE drop-then-replay runs under deferred foreign keys, in one transaction.
|
|
@@ -2100,6 +2861,29 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2100
2861
|
.toArray()) {
|
|
2101
2862
|
this.applied.add(`${row.module_id}@${row.version}`);
|
|
2102
2863
|
}
|
|
2864
|
+
// …and forget that this INSTANCE ever ran a migration pass (#1589). Refreshing
|
|
2865
|
+
// the set above is not enough on its own: `ensureMigrations` memoises its
|
|
2866
|
+
// promise, so a warm DO answers "already migrated" from the cache and never
|
|
2867
|
+
// reads the set again. A restore replays only what the dump carries, so a dump
|
|
2868
|
+
// that omits a module's tables — a world that keeps part of the spine
|
|
2869
|
+
// elsewhere, a targeted repair supplying only the tables being fixed — leaves
|
|
2870
|
+
// them dropped and never rebuilt, and the next operation touching one fails
|
|
2871
|
+
// with a bare `no such table` until an eviction or `retryMigrations` resets the
|
|
2872
|
+
// latch. The pure host has no such memo (it re-reads `appliedMigrations` on
|
|
2873
|
+
// every pass), so this is the line that makes the two adapters agree; without
|
|
2874
|
+
// it the divergence is invisible to dev, CI and self-host, which is the #969
|
|
2875
|
+
// class exactly. Same two fields `retryMigrations` clears, for the same reason:
|
|
2876
|
+
// the next pass has to be a fresh one. Already-journaled versions are skipped
|
|
2877
|
+
// by the `applied` set and the in-transaction re-check, so a dump that DID
|
|
2878
|
+
// carry its tables re-applies nothing.
|
|
2879
|
+
//
|
|
2880
|
+
// `schemaVersionReported` is deliberately left set, as `retryMigrations` leaves
|
|
2881
|
+
// it. `migrate()` reports only when a pass APPLIED something, and a pass that
|
|
2882
|
+
// applies after this reset ends with every code-defined migration journaled —
|
|
2883
|
+
// the same `applied.size` the first pass already reported. A dump whose frontier
|
|
2884
|
+
// is complete applies nothing, so there is nothing to report either way.
|
|
2885
|
+
this.migrationPromise = undefined;
|
|
2886
|
+
this.lastFailure = null;
|
|
2103
2887
|
}
|
|
2104
2888
|
/**
|
|
2105
2889
|
* Re-point scope-level tuples at the scope they now live in.
|
|
@@ -2161,12 +2945,23 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2161
2945
|
* and transaction facts remain". A timeline still shows that something happened, to
|
|
2162
2946
|
* what, and when; it no longer shows who, or what was said about them.
|
|
2163
2947
|
*
|
|
2948
|
+
* **Two tables, one verb (#1600).** The outbox is not the only place the spine holds
|
|
2949
|
+
* an event's payload: this host is the CP-less one, so every connector delivery it
|
|
2950
|
+
* cannot run becomes a `connector:<provider>` intent carrying the whole event, and
|
|
2951
|
+
* nothing ever deletes those rows. Redacting one table and not the other left the
|
|
2952
|
+
* name in the live database and in every copy taken from it afterwards. One RPC
|
|
2953
|
+
* rather than two for `routeExecutorEventToPlatform`'s reason — a crash cannot land
|
|
2954
|
+
* half an erasure — and both halves are idempotent anyway, so a retry converges.
|
|
2955
|
+
*
|
|
2164
2956
|
* This is the one sanctioned write that mutates the outbox. It is kernel code, not
|
|
2165
2957
|
* module code, and an erasure request is precisely the case the append-only rule has
|
|
2166
2958
|
* to yield to — the alternative is telling a data subject that the spine's convenience
|
|
2167
2959
|
* outranks their Article 17 right.
|
|
2168
2960
|
*/
|
|
2169
2961
|
async redactSubject(subjectId) {
|
|
2962
|
+
// One instant for the whole erasure — the intent tombstones must not disagree with
|
|
2963
|
+
// each other about when a person was erased.
|
|
2964
|
+
const at = new Date().toISOString();
|
|
2170
2965
|
const doomed = this.sql
|
|
2171
2966
|
.exec(`SELECT id FROM _substrat_outbox
|
|
2172
2967
|
WHERE subject_id = ? AND pii_class != 'none' AND payload IS NOT NULL`, subjectId)
|
|
@@ -2174,7 +2969,30 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2174
2969
|
for (const id of doomed) {
|
|
2175
2970
|
this.sql.exec('UPDATE _substrat_outbox SET payload = NULL WHERE id = ?', id);
|
|
2176
2971
|
}
|
|
2177
|
-
return doomed.length;
|
|
2972
|
+
return { events: doomed.length, intents: this.redactSubjectIntents(subjectId, at) };
|
|
2973
|
+
}
|
|
2974
|
+
/**
|
|
2975
|
+
* The intent-journal half of `redactSubject` (#1600) — the SQLite adapter's twin.
|
|
2976
|
+
*
|
|
2977
|
+
* Row-by-row rather than one `UPDATE … WHERE`, because the decision is structural and
|
|
2978
|
+
* SQL cannot make it: an intent payload is opaque JSON with no `pii_class` column to
|
|
2979
|
+
* test, so the SQL narrows to rows that could possibly match and the kernel predicate
|
|
2980
|
+
* decides. Query, predicate and statement all come from the kernel, so this adapter
|
|
2981
|
+
* and the pure one cannot drift about what a redacted intent is.
|
|
2982
|
+
*/
|
|
2983
|
+
redactSubjectIntents(subjectId, at) {
|
|
2984
|
+
const q = platformRequestRedactionQuery(subjectId);
|
|
2985
|
+
const candidates = this.sql
|
|
2986
|
+
.exec(q.sql, ...q.params)
|
|
2987
|
+
.toArray();
|
|
2988
|
+
let redacted = 0;
|
|
2989
|
+
for (const row of candidates) {
|
|
2990
|
+
if (!intentPayloadCarriesSubject(row.payload, subjectId))
|
|
2991
|
+
continue;
|
|
2992
|
+
this.sql.exec(PLATFORM_REQUEST_REDACTION_SQL, ...platformRequestRedactionParams(row.id, subjectId, at));
|
|
2993
|
+
redacted += 1;
|
|
2994
|
+
}
|
|
2995
|
+
return redacted;
|
|
2178
2996
|
}
|
|
2179
2997
|
// -- event dispatch (port of dispatch) ------------------------------------
|
|
2180
2998
|
/**
|
|
@@ -2200,7 +3018,13 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2200
3018
|
* later alarm-driven drain with a call it had nothing to do with.
|
|
2201
3019
|
*/
|
|
2202
3020
|
invocationId = null;
|
|
2203
|
-
async dispatch(tenantId, scopeId
|
|
3021
|
+
async dispatch(tenantId, scopeId,
|
|
3022
|
+
/**
|
|
3023
|
+
* #1525: the call this drain is running in, or null. Every round of the loop
|
|
3024
|
+
* belongs to it: a consumer's own emit is delivered in the same tail, so the
|
|
3025
|
+
* whole cascade is one call's work.
|
|
3026
|
+
*/
|
|
3027
|
+
invocationId) {
|
|
2204
3028
|
for (let round = 0; round < 50; round++) {
|
|
2205
3029
|
let deliveredAny = false;
|
|
2206
3030
|
for (const mod of this.modules.values()) {
|
|
@@ -2215,7 +3039,21 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2215
3039
|
ORDER BY o.id`, consumer.eventType, mod.id)
|
|
2216
3040
|
.toArray();
|
|
2217
3041
|
for (const row of rows) {
|
|
2218
|
-
|
|
3042
|
+
let event;
|
|
3043
|
+
try {
|
|
3044
|
+
event = domainEventOf(row);
|
|
3045
|
+
}
|
|
3046
|
+
catch (err) {
|
|
3047
|
+
// #1636: dead-letter an event that does not decode, exactly as a failed
|
|
3048
|
+
// handler is — the decode sat ABOVE the `try` below, so one bad row halted
|
|
3049
|
+
// every event of this type behind it, on every pass. The consumer is never
|
|
3050
|
+
// handed it: an event built from stand-ins is not one it may act on. Only
|
|
3051
|
+
// the decode is caught; the journal write is not.
|
|
3052
|
+
this.sql.exec(`INSERT INTO _substrat_deliveries
|
|
3053
|
+
(event_id, consumer_module, delivered_at, error, invocation_id)
|
|
3054
|
+
VALUES (?, ?, ?, ?, ?)`, row.id, mod.id, new Date().toISOString(), String(err), invocationId);
|
|
3055
|
+
continue;
|
|
3056
|
+
}
|
|
2219
3057
|
// #1237: anything this consumer emits was emitted BECAUSE of this event
|
|
2220
3058
|
// — the step a backwards walk used to stop dead at, since a consumer
|
|
2221
3059
|
// emit records no operation either.
|
|
@@ -2226,16 +3064,18 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2226
3064
|
system: mod.id,
|
|
2227
3065
|
});
|
|
2228
3066
|
await consumer.handler(ctx, event);
|
|
2229
|
-
this.sql.exec(`INSERT INTO _substrat_deliveries
|
|
2230
|
-
|
|
3067
|
+
this.sql.exec(`INSERT INTO _substrat_deliveries
|
|
3068
|
+
(event_id, consumer_module, delivered_at, invocation_id)
|
|
3069
|
+
VALUES (?, ?, ?, ?)`, event.id, mod.id, new Date().toISOString(), invocationId);
|
|
2231
3070
|
});
|
|
2232
3071
|
deliveredAny = true;
|
|
2233
3072
|
}
|
|
2234
3073
|
catch (err) {
|
|
2235
3074
|
// Dead-letter (v0): journal the failure so one poison event
|
|
2236
3075
|
// can't wedge the loop. Written outside the rolled-back txn.
|
|
2237
|
-
this.sql.exec(`INSERT INTO _substrat_deliveries
|
|
2238
|
-
|
|
3076
|
+
this.sql.exec(`INSERT INTO _substrat_deliveries
|
|
3077
|
+
(event_id, consumer_module, delivered_at, error, invocation_id)
|
|
3078
|
+
VALUES (?, ?, ?, ?, ?)`, event.id, mod.id, new Date().toISOString(), String(err), invocationId);
|
|
2239
3079
|
}
|
|
2240
3080
|
finally {
|
|
2241
3081
|
// Cleared on BOTH paths. Left set, the id leaks onto every later emit
|
|
@@ -2257,43 +3097,40 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2257
3097
|
* survives — the whole point, since the denial is the write the operation could not make.
|
|
2258
3098
|
*/
|
|
2259
3099
|
/** K-42: the session a refused call ran under travels with the denial row. */
|
|
2260
|
-
recordDenial(subject, tenantId, operation, err,
|
|
3100
|
+
recordDenial(subject, tenantId, operation, err,
|
|
3101
|
+
/**
|
|
3102
|
+
* #1525: the invocation this refusal belongs to, or null — PASSED, never read off
|
|
3103
|
+
* `this.invocationId` here.
|
|
3104
|
+
*
|
|
3105
|
+
* Reading the ambient field is only self-evidently right where one call holds the
|
|
3106
|
+
* DO to itself, and two denial paths do not: `attachmentList` and
|
|
3107
|
+
* `attachmentAuthorize` run OUTSIDE `this.queue`, and both await before they
|
|
3108
|
+
* record (`ensureMigrations`, `ctx.check`). Whether the input gate can actually
|
|
3109
|
+
* reopen far enough for one of them to observe an in-flight call's id is NOT
|
|
3110
|
+
* settled here — a probe that raced twelve attachment refusals against invokes
|
|
3111
|
+
* holding an id (including one awaiting the control plane) recorded null every
|
|
3112
|
+
* time, so the gate evidently holds more than the shape of the code promises.
|
|
3113
|
+
*
|
|
3114
|
+
* Passed anyway, because the argument for the ambient read is an argument about
|
|
3115
|
+
* workerd's gate semantics, and the argument for a parameter is local: the invoke
|
|
3116
|
+
* path passes its own id, every attachment path passes null, and each says what it
|
|
3117
|
+
* actually knows. That is the property worth having on a recorded fact, and it
|
|
3118
|
+
* costs one argument. No test accompanies it — the condition could not be
|
|
3119
|
+
* reproduced, and a test that passes either way would be worse than none.
|
|
3120
|
+
*/
|
|
3121
|
+
invocationId, impersonation) {
|
|
2261
3122
|
// Only an ENFORCED denial (assertAllowed, which attaches the checked permission +
|
|
2262
3123
|
// node) is recorded. A module's own hand-thrown `new PermissionDenied('…')` carries
|
|
2263
3124
|
// no permission key and is left to the module.
|
|
2264
3125
|
if (!err.permission || !err.node)
|
|
2265
3126
|
return;
|
|
2266
|
-
const actor = subject
|
|
2267
|
-
? { system: subject.id }
|
|
2268
|
-
: subject.kind === 'connection'
|
|
2269
|
-
? { connection: subject.id }
|
|
2270
|
-
: subject.id;
|
|
3127
|
+
const actor = actorOf(subject);
|
|
2271
3128
|
this.sql.exec(`INSERT INTO _substrat_denials
|
|
2272
|
-
(id, actor, permission, tenant_id, scope_id, operation, impersonation,
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
id: row.id,
|
|
2278
|
-
type: row.type,
|
|
2279
|
-
schemaVersion: row.schema_version,
|
|
2280
|
-
occurredAt: row.occurred_at,
|
|
2281
|
-
tenantId: row.tenant_id,
|
|
2282
|
-
scopeId: row.scope_id,
|
|
2283
|
-
actor: JSON.parse(row.actor),
|
|
2284
|
-
entity: { entityType: row.entity_type, entityId: row.entity_id },
|
|
2285
|
-
piiClass: row.pii_class,
|
|
2286
|
-
...(row.subject_id ? { subjectId: row.subject_id } : {}),
|
|
2287
|
-
...(row.authorization ? { authorization: JSON.parse(row.authorization) } : {}),
|
|
2288
|
-
// K-42: the stamp survives the read, so a consumer's event and an executor's
|
|
2289
|
-
// are the same fact the stored row is. Absent rather than null when nobody
|
|
2290
|
-
// was impersonating, because `DomainEvent.impersonation` is optional — the
|
|
2291
|
-
// shape module code never sees is also the shape it cannot branch on.
|
|
2292
|
-
...(row.impersonation ? { impersonation: JSON.parse(row.impersonation) } : {}),
|
|
2293
|
-
// #1231: absent rather than null, the same shape rule as the stamp above.
|
|
2294
|
-
...(row.operation ? { operation: row.operation } : {}),
|
|
2295
|
-
payload: row.payload === null ? undefined : JSON.parse(row.payload),
|
|
2296
|
-
});
|
|
3129
|
+
(id, actor, permission, tenant_id, scope_id, operation, impersonation,
|
|
3130
|
+
invocation_id, at)
|
|
3131
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, ulid(), JSON.stringify(actor), err.permission, err.node.tenantId, err.node.scopeId ?? null, operation, impersonation ? JSON.stringify(impersonationStampOf(impersonation)) : null,
|
|
3132
|
+
// #1525: the call this refusal belongs to, as the caller named it.
|
|
3133
|
+
invocationId, new Date().toISOString());
|
|
2297
3134
|
}
|
|
2298
3135
|
// -- operation context (port of operationContext) -------------------------
|
|
2299
3136
|
operationContext(principal, tenantId, scopeId, systemActor, connectionId, systemModuleId,
|
|
@@ -2312,7 +3149,28 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2312
3149
|
* onto every event it emits. Absent for consumer dispatch: a consumer runs on
|
|
2313
3150
|
* behalf of no operation, and the emitted row's NULL says so.
|
|
2314
3151
|
*/
|
|
2315
|
-
operation
|
|
3152
|
+
operation,
|
|
3153
|
+
/**
|
|
3154
|
+
* #1672: set when the caller holds a CAPABILITY session — already resolved from its
|
|
3155
|
+
* hash inside the queued body, before the transaction. Mutually exclusive with
|
|
3156
|
+
* `connectionId` and `systemModuleId`; `principal` is then a placeholder that the
|
|
3157
|
+
* subject below never reads.
|
|
3158
|
+
*/
|
|
3159
|
+
capabilityId,
|
|
3160
|
+
/**
|
|
3161
|
+
* #1672: the secrets `ctx.capabilities.mint` hands out during this invocation. The
|
|
3162
|
+
* caller owns the array (it withholds them from the idempotency recording); this
|
|
3163
|
+
* context appends to it and holds `ctx.sql`, `ctx.emit` and `ctx.requestPlatform` to
|
|
3164
|
+
* it — the tripwire that catches a module persisting one by accident (not a boundary
|
|
3165
|
+
* against one that means to; see `assertNoSecret`).
|
|
3166
|
+
*/
|
|
3167
|
+
minted = [],
|
|
3168
|
+
/**
|
|
3169
|
+
* #1672: the instant to stamp, when the caller already read one. The exchange passes
|
|
3170
|
+
* the instant its row write used, so the `capability.exercised` event's `occurredAt`
|
|
3171
|
+
* and the row's `last_used_at` are the same value — a second read could disagree.
|
|
3172
|
+
*/
|
|
3173
|
+
instantOverride) {
|
|
2316
3174
|
const checker = this.checker;
|
|
2317
3175
|
const relations = this.relations;
|
|
2318
3176
|
const searchPlans = this.searchPlans;
|
|
@@ -2326,20 +3184,18 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2326
3184
|
* on, and it holds identically here: `ctx.now()`, every `occurredAt` and
|
|
2327
3185
|
* every `requested_at` in one operation are the same value.
|
|
2328
3186
|
*/
|
|
2329
|
-
const at = instant.parse(new Date().toISOString());
|
|
3187
|
+
const at = instantOverride ?? instant.parse(new Date().toISOString());
|
|
2330
3188
|
// The permission subject and the derived event actor for a NON-override caller
|
|
2331
3189
|
// (#383/#97): a scheduled module, a connection, or a person. `systemActor` (the
|
|
2332
3190
|
// override, used only by consumer dispatch) stays a separate bypass path below.
|
|
2333
|
-
const subject =
|
|
2334
|
-
? { kind: '
|
|
2335
|
-
:
|
|
2336
|
-
? { kind: '
|
|
2337
|
-
:
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
? { connection: subject.id }
|
|
2342
|
-
: principal;
|
|
3191
|
+
const subject = capabilityId
|
|
3192
|
+
? { kind: 'capability', id: capabilityId }
|
|
3193
|
+
: systemModuleId
|
|
3194
|
+
? { kind: 'system', id: systemModuleId }
|
|
3195
|
+
: connectionId
|
|
3196
|
+
? { kind: 'connection', id: connectionId }
|
|
3197
|
+
: { kind: 'principal', id: principal };
|
|
3198
|
+
const derivedActor = actorOf(subject);
|
|
2343
3199
|
// #304: entitlement reads pick the same local-vs-RPC reader the permission checker
|
|
2344
3200
|
// uses (projected scope → local table; console-managed → CP over RPC), resolved per
|
|
2345
3201
|
// call so a scope that flips to 'local' is picked up without rebuilding the context.
|
|
@@ -2368,7 +3224,10 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2368
3224
|
// Lifted so `grant` reuses the SAME check the operation itself passes —
|
|
2369
3225
|
// a delegation check that could differ from the operation's would be a
|
|
2370
3226
|
// second opinion about what the caller holds.
|
|
2371
|
-
const runCheck = async (
|
|
3227
|
+
const runCheck = async (unparsed, entity) => {
|
|
3228
|
+
// #1642: parsed before the system actor's early return, which never reaches
|
|
3229
|
+
// the checker — a cast key would otherwise become that path's proof relation.
|
|
3230
|
+
const permission = assertPermissionKey(unparsed);
|
|
2372
3231
|
if (systemActor) {
|
|
2373
3232
|
return {
|
|
2374
3233
|
allowed: true,
|
|
@@ -2403,15 +3262,19 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2403
3262
|
}
|
|
2404
3263
|
return checker.covers(subject, role.permissions, { tenantId, scopeId });
|
|
2405
3264
|
};
|
|
2406
|
-
|
|
3265
|
+
const ctxRef = {
|
|
2407
3266
|
tenantId,
|
|
2408
3267
|
scopeId,
|
|
2409
|
-
|
|
2410
|
-
|
|
3268
|
+
// #1672: a capability's own id stands in so the type holds — it is not a person, and
|
|
3269
|
+
// the event actor says what it is instead. Every other door passes its own value.
|
|
3270
|
+
principal: capabilityId ? capabilityId : principal,
|
|
3271
|
+
sql: guardSecrets(doScopedSql(sql), minted),
|
|
2411
3272
|
now: () => at,
|
|
2412
3273
|
emit: (event) => {
|
|
2413
3274
|
assertImpersonationWrites(impersonation, 'ctx.emit');
|
|
2414
3275
|
const parsed = domainEventInput.parse(event);
|
|
3276
|
+
// #1672: the COMPLETE parsed event — entity id, type and subject as well as payload.
|
|
3277
|
+
assertNoSecret('ctx.emit', parsed, minted);
|
|
2415
3278
|
const full = domainEvent.parse({
|
|
2416
3279
|
...parsed,
|
|
2417
3280
|
// #956: from the operation's instant, not a second reading of the clock.
|
|
@@ -2448,6 +3311,8 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2448
3311
|
requestPlatform: (request) => {
|
|
2449
3312
|
assertImpersonationWrites(impersonation, 'ctx.requestPlatform');
|
|
2450
3313
|
const input = platformRequestInput.parse(request);
|
|
3314
|
+
// #1672: the COMPLETE parsed request — its `kind` is persisted as surely as its payload.
|
|
3315
|
+
assertNoSecret('ctx.requestPlatform', input, minted);
|
|
2451
3316
|
// #1474: a platform-authored kind (`sweep-runs`) never comes from module code —
|
|
2452
3317
|
// the sweeper enqueues it through `enqueueSweepRuns`, which does not pass here.
|
|
2453
3318
|
assertModuleEnqueueableKind(input.kind);
|
|
@@ -2472,7 +3337,9 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2472
3337
|
// predicate is needed or possible: the DO IS the scope.
|
|
2473
3338
|
platformRequests: (filter) => {
|
|
2474
3339
|
const q = platformRequestHistoryQuery(filter);
|
|
2475
|
-
|
|
3340
|
+
// The kernel's decoder, the one the coordinator maps the RPC's rows with (#1588):
|
|
3341
|
+
// tolerant, so one undecodable row cannot hide this scope's other intents from it.
|
|
3342
|
+
return sql.exec(q.sql, ...q.params).toArray().map(platformRequestOf);
|
|
2476
3343
|
},
|
|
2477
3344
|
// #901. Mirror of the pure adapter, and the reason the contract suite
|
|
2478
3345
|
// runs on both: the query is ordinary SQL, but it is only a seek rather
|
|
@@ -2551,6 +3418,22 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2551
3418
|
sql.exec(`DELETE FROM _substrat_tuples WHERE subject = ? AND relation = ? AND object = ?`, `principal:${principal}`, `granted:${permission}`, `${entity.entityType}:${entity.entityId}`);
|
|
2552
3419
|
},
|
|
2553
3420
|
atomic: createAtomic(runSub, { passed, signals }),
|
|
3421
|
+
// #1672: mint / revoke / list, written once in the kernel — the pure adapter hands
|
|
3422
|
+
// the same function the same four things. The raw spine seam (the kernel's own write
|
|
3423
|
+
// to `_substrat_capabilities`), the operation's OWN check, and `ctx.emit`. A
|
|
3424
|
+
// consumer's override actor is passed as the system actor it is, so a consumer
|
|
3425
|
+
// cannot mint: its checks allow unconditionally, which would make "the minter holds
|
|
3426
|
+
// it" vacuous.
|
|
3427
|
+
capabilities: createCapabilityVerbs({
|
|
3428
|
+
sql: doSpineSql(sql),
|
|
3429
|
+
subject: systemActor ? { kind: 'system', id: systemActor.system } : subject,
|
|
3430
|
+
now: at,
|
|
3431
|
+
check: runCheck,
|
|
3432
|
+
emit: (event) => ctxRef.emit(event),
|
|
3433
|
+
isOperation: (name) => this.operations.has(name),
|
|
3434
|
+
assertWrites: (verb) => assertImpersonationWrites(impersonation, verb),
|
|
3435
|
+
minted,
|
|
3436
|
+
}),
|
|
2554
3437
|
link: (child, parent) => {
|
|
2555
3438
|
assertImpersonationWrites(impersonation, 'ctx.link');
|
|
2556
3439
|
const allowed = relations.get(child.entityType);
|
|
@@ -2584,6 +3467,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2584
3467
|
return sealTo({ keyId: row.key_id, publicKey: row.public_key }, plaintext);
|
|
2585
3468
|
},
|
|
2586
3469
|
};
|
|
3470
|
+
return ctxRef;
|
|
2587
3471
|
}
|
|
2588
3472
|
/** True once this scope has had entitlements projected at least once (#304) — the switch
|
|
2589
3473
|
* from trust-upstream to strict fail-closed entitlement enforcement on the local path. */
|
|
@@ -2686,11 +3570,16 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2686
3570
|
* while passing a list — even `[]` — full-replaces them. This keeps pre-#304 callers
|
|
2687
3571
|
* from silently wiping a scope's entitlements. */
|
|
2688
3572
|
entitlements,
|
|
2689
|
-
/** Scope-level tuples (e.g. the owner's role grant at provision)
|
|
3573
|
+
/** Scope-level tuples (e.g. the owner's role grant at provision) seated into
|
|
2690
3574
|
* `_substrat_tuples` in this SAME transaction, additively (#332). Preserve-on-undefined:
|
|
2691
3575
|
* omitting it leaves existing scope tuples untouched, so a role-only re-projection keeps
|
|
2692
3576
|
* the owner grant. Passing them here (rather than a follow-up `writeTuple`) is what makes
|
|
2693
|
-
* provision atomic — the grant and the enforcement flip land together or not at all.
|
|
3577
|
+
* provision atomic — the grant and the enforcement flip land together or not at all.
|
|
3578
|
+
*
|
|
3579
|
+
* Seated, not replaced (#1659): a missing tuple is created, a revoked one stays revoked.
|
|
3580
|
+
* `lockout_reseat` marks the one exception — the owner-of-record's seat, which is
|
|
3581
|
+
* re-seated even over a revoke when the scope would otherwise hold no effective role
|
|
3582
|
+
* grant (`hasEffectiveRoleGrant`). */
|
|
2694
3583
|
scopeTuples,
|
|
2695
3584
|
/** The tenant's identity links (#406) — projected alongside the rest so a CP-less
|
|
2696
3585
|
* vertical's auth adapter resolves logins locally. Same preserve-on-undefined
|
|
@@ -2765,18 +3654,39 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2765
3654
|
// — NOT a full replace — so existing scope tuples are preserved. This is what keeps a
|
|
2766
3655
|
// scope from ever being left "roles projected, permission_source=local, zero tuples" by
|
|
2767
3656
|
// a write that lands the projection but drops before a follow-up owner grant.
|
|
3657
|
+
//
|
|
3658
|
+
// #1659: SEATED, so a reconcile creates what is missing and leaves a revoke alone. It
|
|
3659
|
+
// used to be `INSERT OR REPLACE … revoked_at = NULL`, which undid an operator's revoke
|
|
3660
|
+
// of the owner seat or of a `system:` schedule grant on the next reconcile. And a
|
|
3661
|
+
// module switched off (#1666) gets no `system:` grant seated, new or old.
|
|
2768
3662
|
for (const st of scopeTuples ?? []) {
|
|
2769
|
-
|
|
2770
|
-
|
|
3663
|
+
const seat = seatScopeTuple(st.subject, st.relation, st.object, st.expires_at);
|
|
3664
|
+
this.sql.exec(seat.sql, ...seat.params);
|
|
3665
|
+
}
|
|
3666
|
+
// #1659's one exception: the owner-of-record's seat comes back over a revoke when
|
|
3667
|
+
// NOTHING else would let anyone act here — roles projected, no effective role grant.
|
|
3668
|
+
// That is the #332 lockout this path exists to repair, and it is decided by the same
|
|
3669
|
+
// predicate as the flip guard below, so "locked out" means one thing in this unit.
|
|
3670
|
+
// With any other effective holder, the revoke stands: a hand-over that seats a
|
|
3671
|
+
// successor before unseating the owner is not undone by the next promote. A holder of
|
|
3672
|
+
// a role the vertical no longer defines is NOT one — it passes no check, so it must
|
|
3673
|
+
// not stand in for the holder this repair exists to restore.
|
|
3674
|
+
if (roles.length > 0 && !this.hasEffectiveRoleGrant(tenantId)) {
|
|
3675
|
+
for (const st of scopeTuples ?? []) {
|
|
3676
|
+
if (!st.lockout_reseat)
|
|
3677
|
+
continue;
|
|
3678
|
+
this.sql.exec(`INSERT OR REPLACE INTO _substrat_tuples (subject, relation, object, expires_at, revoked_at)
|
|
3679
|
+
VALUES (?, ?, ?, ?, NULL)`, st.subject, st.relation, st.object, st.expires_at);
|
|
3680
|
+
}
|
|
2771
3681
|
}
|
|
2772
3682
|
// #332: only switch on strict local enforcement when SOMEONE actually holds a role.
|
|
2773
|
-
// A projection that leaves role definitions but no
|
|
2774
|
-
// every check fail closed — a scope serving nothing but denials, unfixable from
|
|
2775
|
-
// Leave `permission_source` as-is instead; a reconcile that restores the owner
|
|
2776
|
-
// re-runs this and flips safely. (A CP-less vertical uses the local reader
|
|
2777
|
-
// this flag, so the owner grant is written above in the same unit — this
|
|
2778
|
-
// belt to that suspenders, and it protects the CP-backed flip outright.)
|
|
2779
|
-
if (roles.length > 0 && !this.
|
|
3683
|
+
// A projection that leaves role definitions but no effective principal→role grant would
|
|
3684
|
+
// make every check fail closed — a scope serving nothing but denials, unfixable from
|
|
3685
|
+
// inside. Leave `permission_source` as-is instead; a reconcile that restores the owner
|
|
3686
|
+
// grant re-runs this and flips safely. (A CP-less vertical uses the local reader
|
|
3687
|
+
// regardless of this flag, so the owner grant is written above in the same unit — this
|
|
3688
|
+
// guard is the belt to that suspenders, and it protects the CP-backed flip outright.)
|
|
3689
|
+
if (roles.length > 0 && !this.hasEffectiveRoleGrant(tenantId))
|
|
2780
3690
|
return;
|
|
2781
3691
|
this.sql.exec(`INSERT OR REPLACE INTO _substrat_meta (key, value) VALUES ('permission_source', 'local')`);
|
|
2782
3692
|
});
|
|
@@ -2797,23 +3707,17 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2797
3707
|
return undefined;
|
|
2798
3708
|
return { principal: row.principal_id, scopeId: row.scope_id };
|
|
2799
3709
|
}
|
|
2800
|
-
/** True if
|
|
2801
|
-
* at scope OR tenant level
|
|
2802
|
-
*
|
|
2803
|
-
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
|
|
2808
|
-
|
|
2809
|
-
|
|
2810
|
-
|
|
2811
|
-
return true;
|
|
2812
|
-
return (this.sql
|
|
2813
|
-
.exec(`SELECT 1 FROM _substrat_tenant_tuples
|
|
2814
|
-
WHERE tenant_id = ? AND relation LIKE 'role:%' AND revoked_at IS NULL AND (expires_at IS NULL OR expires_at > ?)
|
|
2815
|
-
LIMIT 1`, tenantId, now)
|
|
2816
|
-
.toArray().length > 0);
|
|
3710
|
+
/** True if some principal holds a role this scope can actually EXPAND — a live
|
|
3711
|
+
* (non-revoked, unexpired) `role:<key>` tuple, at scope OR tenant level, whose key names a
|
|
3712
|
+
* current, non-revoked role definition for this tenant. The one predicate behind both the
|
|
3713
|
+
* #332 flip guard and #1659's owner re-seat in `applyProjection`; the query is the
|
|
3714
|
+
* kernel's `effectiveRoleGrantQuery`, where it is tested against a real SQLite. A tuple
|
|
3715
|
+
* for a role the vertical no longer defines counts for nothing, exactly as in the local
|
|
3716
|
+
* checker, which expands a role only through its definition. */
|
|
3717
|
+
hasEffectiveRoleGrant(tenantId) {
|
|
3718
|
+
const q = effectiveRoleGrantQuery(tenantId, new Date().toISOString());
|
|
3719
|
+
const row = this.sql.exec(q.sql, ...q.params).toArray()[0];
|
|
3720
|
+
return row?.effective === 1;
|
|
2817
3721
|
}
|
|
2818
3722
|
};
|
|
2819
3723
|
}
|