@substrat-run/adapter-cloudflare 0.113.0 → 0.116.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/control-plane-do.d.ts +26 -0
- package/dist/control-plane-do.d.ts.map +1 -1
- package/dist/control-plane-do.js +48 -0
- package/dist/control-plane-do.js.map +1 -1
- package/dist/host.d.ts +63 -2
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +274 -29
- package/dist/host.js.map +1 -1
- package/dist/live-reads.d.ts +116 -0
- package/dist/live-reads.d.ts.map +1 -0
- package/dist/live-reads.js +106 -0
- package/dist/live-reads.js.map +1 -0
- package/dist/scope-do.d.ts.map +1 -1
- package/dist/scope-do.js +694 -59
- package/dist/scope-do.js.map +1 -1
- package/package.json +4 -4
package/dist/scope-do.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { DurableObject } from 'cloudflare:workers';
|
|
2
|
-
import { ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, domainEvent, domainEventInput, eventId, instant, objectRef, toWireFailure, grantRefFromProof, principalId, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, MAX_PENDING_SWEEP_RUNS, SWEEP_RUNS_KIND, platformRequest, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, listLimitOf, requestFingerprint, substratError, assertReplayableDump, REDRAIN_BATCH, } from '@substrat-run/contracts';
|
|
3
|
-
import { ulid, createUlid, assertAllowed, ConnectionSealingKeyUnavailableError, noSealingKeyMessage, sealTo, assertReadOnlyQuery, entitlementDenial, platformRequestHistoryQuery, PLATFORM_REQUEST_COLUMNS, denialListQuery, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, PermissionDenied, assertImpersonationWrites, assertModuleEnqueueableKind, impersonationStampOf, createAtomic, NotSearchable, isSearchIndexTable, searchIndexDdl, searchIndexMigrations, searchIndexPlans, NotListable, listIndexMigrations, listIndexPlans, listQuery, cursorOf, searchLimit, searchMatchExpression, searchQuery, IDEMPOTENCY_DDL, assertIdempotencyKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, } from '@substrat-run/kernel';
|
|
2
|
+
import { ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, domainEvent, domainEventInput, eventId, instant, objectRef, toWireFailure, grantRefFromProof, principalId, scopeId as scopeIdOf, tenantId as tenantIdOf, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, MAX_PENDING_SWEEP_RUNS, SWEEP_RUNS_KIND, platformRequest, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, listLimitOf, requestFingerprint, substratError, assertReplayableDump, REDRAIN_BATCH, } from '@substrat-run/contracts';
|
|
3
|
+
import { ulid, createUlid, assertAllowed, ConnectionSealingKeyUnavailableError, noSealingKeyMessage, sealTo, assertReadOnlyQuery, entitlementDenial, platformRequestHistoryQuery, PLATFORM_REQUEST_COLUMNS, denialListQuery, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, PermissionDenied, assertImpersonationWrites, assertModuleEnqueueableKind, impersonationStampOf, createAtomic, NotSearchable, isSearchIndexTable, searchIndexDdl, searchIndexMigrations, searchIndexPlans, NotListable, listIndexMigrations, listIndexPlans, listQuery, cursorOf, searchLimit, searchMatchExpression, searchQuery, IDEMPOTENCY_DDL, assertIdempotencyKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, SCHEDULE_STATE_DDL, SCHEDULE_STATE_REBUILD, scheduleStateHasKind, JOB_RUN_DDL, jobRunListLimit, } from '@substrat-run/kernel';
|
|
4
|
+
import { isUpgradeRequest, readSubscription, LIVE_FANOUT_LIMIT, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
|
|
4
5
|
import { OperationQueue } from './serialization.js';
|
|
5
6
|
import { doScopedSql } from './sql.js';
|
|
6
|
-
import { facetEvents, readHistory, readInvocation, walkEventCause, walkEventEffects } from '@substrat-run/kernel';
|
|
7
|
+
import { facetEvents, readDeadLetters, readHistory, readInvocation, walkEventCause, walkEventEffects } from '@substrat-run/kernel';
|
|
7
8
|
import { createDoTupleChecker, createLocalControlPlaneReader } from './checker.js';
|
|
8
9
|
/**
|
|
9
10
|
* The key marking a scope DO whose storage was destroyed (`destroyStorage`).
|
|
@@ -127,31 +128,28 @@ const KERNEL_DDL = `
|
|
|
127
128
|
operation TEXT,
|
|
128
129
|
-- K-42: WHICH of the two actors was refused is exactly what this log is for.
|
|
129
130
|
impersonation TEXT,
|
|
131
|
+
-- #1525: the INVOCATION this refusal happened during, the same id #1237 stamps on
|
|
132
|
+
-- every event of a call. A denial joins to nothing but actor and time otherwise --
|
|
133
|
+
-- the operation column names what was attempted, never WHICH attempt -- so "what
|
|
134
|
+
-- else did this request do" cannot reach a refusal, which is the one thing an
|
|
135
|
+
-- incident asks about first. NULL = the transport carried no id (a seed, a test,
|
|
136
|
+
-- an internal call, an attachment RPC), or the row predates the column.
|
|
137
|
+
invocation_id TEXT,
|
|
130
138
|
at TEXT NOT NULL,
|
|
131
139
|
drained_at TEXT
|
|
132
140
|
);
|
|
133
|
-
-- #383: the platform sweep's per-scope gating state
|
|
134
|
-
--
|
|
135
|
-
--
|
|
136
|
-
--
|
|
137
|
-
--
|
|
138
|
-
|
|
139
|
-
--
|
|
140
|
-
--
|
|
141
|
-
--
|
|
142
|
-
--
|
|
143
|
-
--
|
|
144
|
-
|
|
145
|
-
-- that declared a schedule operation literally named "freshness:orders.placed"
|
|
146
|
-
-- would share a row with the evaluator and nothing today would reject it.
|
|
147
|
-
-- #1288 tracks giving the table a column (or a name) that says this outright,
|
|
148
|
-
-- which is also what would let the collision be refused rather than avoided.
|
|
149
|
-
-- Spine (kernel-written), never a module migration.
|
|
150
|
-
CREATE TABLE IF NOT EXISTS _substrat_schedule_state (
|
|
151
|
-
schedule_op TEXT PRIMARY KEY,
|
|
152
|
-
last_run_at TEXT,
|
|
153
|
-
last_status TEXT
|
|
154
|
-
);
|
|
141
|
+
-- #383 / #1232 / #1288: the platform sweep's per-scope gating state, holding two
|
|
142
|
+
-- families of row that the kind COLUMN -- not the spelling of a key -- tells
|
|
143
|
+
-- apart. Spine (kernel-written), never a module migration. Shared with the pure
|
|
144
|
+
-- adapter from @substrat-run/kernel, so the shape a rebuild produces and the shape
|
|
145
|
+
-- a fresh store gets cannot part company; the column comments are in there.
|
|
146
|
+
${SCHEDULE_STATE_DDL}
|
|
147
|
+
-- #1577: the resumable-run driver's record and the step ledger of the pass it
|
|
148
|
+
-- currently has in flight. Spine (kernel-written), never a module migration.
|
|
149
|
+
-- Shared with the pure adapter from @substrat-run/kernel so the shape production
|
|
150
|
+
-- builds and the shape a self-host builds cannot part company; the column
|
|
151
|
+
-- comments, and the reason coalescing is NOT a unique index, are in there.
|
|
152
|
+
${JOB_RUN_DDL}
|
|
155
153
|
CREATE TABLE IF NOT EXISTS _substrat_migrations (
|
|
156
154
|
module_id TEXT NOT NULL,
|
|
157
155
|
version TEXT NOT NULL,
|
|
@@ -191,6 +189,17 @@ const KERNEL_DDL = `
|
|
|
191
189
|
-- next_attempt_at IS NULL -> terminal: error IS NULL delivered, else dead
|
|
192
190
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
193
191
|
next_attempt_at TEXT,
|
|
192
|
+
-- #1525: the invocation THIS ATTEMPT ran in, the same id #1237 stamps on every
|
|
193
|
+
-- event of a call. Not the emitting event's -- that one is already on the outbox
|
|
194
|
+
-- and a delivery joins to it through event_id. What that join cannot say is
|
|
195
|
+
-- which call attempted the DELIVERY, and for an executor the two differ by
|
|
196
|
+
-- design: attempt one runs in the emitting call's post-commit tail, every retry
|
|
197
|
+
-- afterwards in a drain that is a different call or none. Moves with the row on
|
|
198
|
+
-- an upsert, like delivered_at, error and attempts -- it describes the
|
|
199
|
+
-- latest attempt, not the first. NULL = that attempt carried no call (a
|
|
200
|
+
-- scheduled drain, an alarm, a seed, an attachment RPC), or the row predates
|
|
201
|
+
-- the column.
|
|
202
|
+
invocation_id TEXT,
|
|
194
203
|
PRIMARY KEY (event_id, consumer_module)
|
|
195
204
|
);
|
|
196
205
|
-- Scope-local permissions (docs/architecture/scope-local-permissions.md): the
|
|
@@ -505,6 +514,12 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
505
514
|
listPlans = new Map();
|
|
506
515
|
/** entityType → the declared attachment gate (#473): read key + write key (default: read). */
|
|
507
516
|
attachmentTargets = new Map();
|
|
517
|
+
/**
|
|
518
|
+
* entityType → the declared live-read gate (#938): the key a subscriber must hold
|
|
519
|
+
* ON THAT ENTITY before a change to it is announced. Absent = announced to nobody;
|
|
520
|
+
* `fanOutLive` treats a miss as silence, never as "unguarded".
|
|
521
|
+
*/
|
|
522
|
+
liveTargets = new Map();
|
|
508
523
|
checker;
|
|
509
524
|
systemPrincipal = principalId.parse(ulid());
|
|
510
525
|
/**
|
|
@@ -622,6 +637,20 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
622
637
|
}
|
|
623
638
|
this.attachmentTargets.set(target.entityType, gate);
|
|
624
639
|
}
|
|
640
|
+
// Live-read targets (#938): entityType → the read key a subscriber must pass ON
|
|
641
|
+
// THAT ENTITY before a change to it is announced. Validated exactly like the
|
|
642
|
+
// attachment gate above, and for a sharper reason: two modules disagreeing about
|
|
643
|
+
// which key guards an entity type would make "who may watch this" depend on
|
|
644
|
+
// registration order, and the losing declaration would be the one that was meant
|
|
645
|
+
// to be stricter. An entity type nobody declares is announced to nobody.
|
|
646
|
+
for (const target of manifest.liveTargets ?? []) {
|
|
647
|
+
const existing = this.liveTargets.get(target.entityType);
|
|
648
|
+
if (existing && existing !== target.readPermission) {
|
|
649
|
+
throw new Error(`conflicting liveTargets for '${target.entityType}': ` +
|
|
650
|
+
`(${existing}) vs (${target.readPermission})`);
|
|
651
|
+
}
|
|
652
|
+
this.liveTargets.set(target.entityType, target.readPermission);
|
|
653
|
+
}
|
|
625
654
|
for (const name of manifest.withdraws ?? []) {
|
|
626
655
|
this.withdrawn.set(name, manifest.id);
|
|
627
656
|
this.operations.delete(name);
|
|
@@ -843,6 +872,24 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
843
872
|
return ids.length;
|
|
844
873
|
});
|
|
845
874
|
}
|
|
875
|
+
/**
|
|
876
|
+
* How many rows `redrainEvents` WOULD reopen for the same instant (#1545), reopening
|
|
877
|
+
* none of them. A separate verb rather than a flag on the one above, deliberately: the
|
|
878
|
+
* two answers are indistinguishable once they are numbers, so the caller that asks for
|
|
879
|
+
* a count must not be able to reach the reopen by losing an argument on the way.
|
|
880
|
+
*
|
|
881
|
+
* UNBOUNDED where the reopen is batched. The batch exists because an UPDATE over the
|
|
882
|
+
* whole window rewrites every row and its index entries inside ONE Durable Object
|
|
883
|
+
* request, against a fixed budget. An aggregate materialises no rows, so that budget is
|
|
884
|
+
* not the binding constraint — and a partial count would be worse than useless: the
|
|
885
|
+
* number is the whole point, and "5000, or possibly more" answers nothing.
|
|
886
|
+
*/
|
|
887
|
+
async redrainCount(drainedBefore) {
|
|
888
|
+
return this.sql
|
|
889
|
+
.exec(`SELECT COUNT(*) AS c FROM _substrat_outbox
|
|
890
|
+
WHERE drained_at IS NOT NULL AND drained_at < ?`, drainedBefore)
|
|
891
|
+
.toArray()[0].c;
|
|
892
|
+
}
|
|
846
893
|
/**
|
|
847
894
|
* Facet this scope's own outbox (#1239) — `facetEvents`, which is the
|
|
848
895
|
* sanctioned read: an erased payload yields the same NULL a missing field
|
|
@@ -879,6 +926,10 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
879
926
|
invocationEvents(input) {
|
|
880
927
|
return readInvocation({ sql: doScopedSql(this.sql) }, input.invocationId, input.limit);
|
|
881
928
|
}
|
|
929
|
+
/** #1525: every delivery in this scope that gave up, inside the DO where both tables live. */
|
|
930
|
+
deadLetters(input) {
|
|
931
|
+
return readDeadLetters({ sql: doScopedSql(this.sql) }, { limit: input.limit, cursor: input.cursor });
|
|
932
|
+
}
|
|
882
933
|
migrationBookmarks(limit = 20) {
|
|
883
934
|
return this.sql
|
|
884
935
|
.exec(`SELECT bookmark, taken_at, pending FROM _substrat_migration_bookmarks
|
|
@@ -1159,6 +1210,21 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1159
1210
|
// one call. Same placement as the SQLite adapter's actor task, for this reason.
|
|
1160
1211
|
this.invocationId = invokeOptions?.invocationId ?? null;
|
|
1161
1212
|
try {
|
|
1213
|
+
/**
|
|
1214
|
+
* #938: the outbox's high-water mark BEFORE this call wrote anything, so the
|
|
1215
|
+
* post-commit fan-out can name exactly the events this call (and the consumers
|
|
1216
|
+
* it set off) added. Read here — inside the queued body, before the
|
|
1217
|
+
* transaction — because that is the region where this call holds the DO to
|
|
1218
|
+
* itself, the same reason `invocationId` is set here.
|
|
1219
|
+
*
|
|
1220
|
+
* A socket that connects between here and the fan-out is served whatever this
|
|
1221
|
+
* call committed, and one that connects while `null` was decided hears nothing
|
|
1222
|
+
* about it. Both are harmless and neither is worth a lock: a frame is an
|
|
1223
|
+
* invalidation, so hearing about a change from just before you subscribed costs
|
|
1224
|
+
* one redundant re-read, and missing one costs a wait for the client's poll —
|
|
1225
|
+
* which is the floor this whole surface sits on.
|
|
1226
|
+
*/
|
|
1227
|
+
const liveSince = this.liveHighWaterMark();
|
|
1162
1228
|
let result;
|
|
1163
1229
|
let committedVersion = null;
|
|
1164
1230
|
// #116: set when this invocation was answered from a recording rather
|
|
@@ -1270,7 +1336,10 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1270
1336
|
? { kind: 'system', id: systemModuleId }
|
|
1271
1337
|
: connectionId
|
|
1272
1338
|
? { kind: 'connection', id: connectionId }
|
|
1273
|
-
: { kind: 'principal', id: principal }, tenantId, operation, err,
|
|
1339
|
+
: { kind: 'principal', id: principal }, tenantId, operation, err,
|
|
1340
|
+
// Inside the queued body, which is the one region where this call holds
|
|
1341
|
+
// the DO to itself — so the field is this call's own (#1237).
|
|
1342
|
+
this.invocationId, impersonation);
|
|
1274
1343
|
}
|
|
1275
1344
|
// The ORIGINAL error, deliberately: `invoke` flattens it for the envelope
|
|
1276
1345
|
// (which keeps its code and extensions) or rewraps it for the legacy throw
|
|
@@ -1281,8 +1350,14 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1281
1350
|
// Skipped on a replay: nothing was written, so there is nothing this
|
|
1282
1351
|
// invocation added to drain. Anything the ORIGINAL left undrained is the
|
|
1283
1352
|
// outbox's own retry backstop, which is what that backstop is for.
|
|
1353
|
+
// Drain to consumers, then announce what landed (#938). Skipped whole on a
|
|
1354
|
+
// replay: nothing was written, so there is nothing this invocation added to
|
|
1355
|
+
// drain or to announce. Anything the ORIGINAL left undrained is the outbox's
|
|
1356
|
+
// own retry backstop, which is what that backstop is for.
|
|
1357
|
+
// #1525: still inside the queued body that set it, so these deliveries are this
|
|
1358
|
+
// call's own work — the same tail its consumers' emits are stamped in.
|
|
1284
1359
|
if (!replayed)
|
|
1285
|
-
await this.
|
|
1360
|
+
await this.settleCommitted(tenantId, scopeId, liveSince, this.invocationId);
|
|
1286
1361
|
return {
|
|
1287
1362
|
result,
|
|
1288
1363
|
platformRequests: signals.platformRequests,
|
|
@@ -1315,6 +1390,288 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1315
1390
|
}
|
|
1316
1391
|
});
|
|
1317
1392
|
}
|
|
1393
|
+
// -- live reads (#938): the subscription half of the change feed ------------
|
|
1394
|
+
// The ONLY part of this DO addressed as a fetch target rather than over RPC, and
|
|
1395
|
+
// only because a WebSocket cannot cross RPC — a socket is not serializable, so the
|
|
1396
|
+
// one way to hand one back is a `Response` carrying a `webSocket`. Everything the
|
|
1397
|
+
// coordinator asserts on the way in is named in `live-reads.ts`, shared with
|
|
1398
|
+
// `host.ts` so the two ends cannot drift.
|
|
1399
|
+
/**
|
|
1400
|
+
* Accept a subscription to this scope's changes.
|
|
1401
|
+
*
|
|
1402
|
+
* The coordinator has already decided that this connection can carry a push at all
|
|
1403
|
+
* (the O2O check) and WHO is asking. What is decided here is nothing about
|
|
1404
|
+
* authority: a subscription is not an authorization, and accepting one grants the
|
|
1405
|
+
* subscriber no read it did not already have. Every frame is checked on its way
|
|
1406
|
+
* out, individually, against the tuple state at that moment — so a grant revoked
|
|
1407
|
+
* while the socket is open stops the frames it used to allow, which a
|
|
1408
|
+
* subscription-time check would not.
|
|
1409
|
+
*
|
|
1410
|
+
* **This adds no authority to a holder of the stub, and the question is worth
|
|
1411
|
+
* answering rather than leaving to be asked.** Being a public method, anything with
|
|
1412
|
+
* the `SCOPE` binding can call it and assert whatever principal it likes in the
|
|
1413
|
+
* headers. That is already true of `invoke`, which takes the principal as an
|
|
1414
|
+
* argument and acts on it: a stub is the key to the scope, which is precisely why
|
|
1415
|
+
* the router is not given one. The trust boundary is who holds the binding, not
|
|
1416
|
+
* what this method checks — and what it does NOT do is let a stub-holder read
|
|
1417
|
+
* anything the asserted principal could not, because the filter downstream re-checks
|
|
1418
|
+
* that principal against every frame.
|
|
1419
|
+
*/
|
|
1420
|
+
async fetch(request) {
|
|
1421
|
+
const url = new URL(request.url);
|
|
1422
|
+
if (url.pathname !== LIVE_SUBSCRIBE_PATH) {
|
|
1423
|
+
// The DO has exactly one fetch surface. Anything else reaching here is a
|
|
1424
|
+
// coordinator bug, and a 404 says so without guessing at an intent.
|
|
1425
|
+
return new Response('this scope has no such surface', { status: 404 });
|
|
1426
|
+
}
|
|
1427
|
+
if (!isUpgradeRequest(request)) {
|
|
1428
|
+
return new Response('live reads are a WebSocket surface', {
|
|
1429
|
+
status: 426,
|
|
1430
|
+
headers: { [LIVE_MODE_HEADER]: 'not-an-upgrade' },
|
|
1431
|
+
});
|
|
1432
|
+
}
|
|
1433
|
+
const principal = request.headers.get(LIVE_PRINCIPAL_HEADER);
|
|
1434
|
+
const tenantId = request.headers.get(LIVE_TENANT_HEADER);
|
|
1435
|
+
const scopeId = request.headers.get(LIVE_SCOPE_HEADER);
|
|
1436
|
+
if (!principal || !tenantId || !scopeId) {
|
|
1437
|
+
// Fail closed and loudly. An unnamed subscriber is one whose permissions
|
|
1438
|
+
// cannot be evaluated, and the only safe thing to do with a channel we cannot
|
|
1439
|
+
// filter is to refuse to open it. 500, not 400: the caller is the coordinator
|
|
1440
|
+
// in this same package, so a missing assertion is OUR bug, not the client's.
|
|
1441
|
+
return new Response('live reads require an asserted principal, tenant and scope', {
|
|
1442
|
+
status: 500,
|
|
1443
|
+
});
|
|
1444
|
+
}
|
|
1445
|
+
// A subscriber arriving before the scope's migrations have run would be told
|
|
1446
|
+
// about events against a schema it cannot read back through. Same gate every
|
|
1447
|
+
// other entry point takes, for the same reason.
|
|
1448
|
+
await this.ensureMigrations();
|
|
1449
|
+
const pair = new WebSocketPair();
|
|
1450
|
+
const [client, server] = Object.values(pair);
|
|
1451
|
+
// HIBERNATABLE, not `server.accept()`. A scope with a watcher open would
|
|
1452
|
+
// otherwise be pinned in memory for as long as somebody has a tab open, which is
|
|
1453
|
+
// the cost model inverted: a support desk being WATCHED is the normal state.
|
|
1454
|
+
// Hibernation also fixes the worse half — an in-memory roster does not survive
|
|
1455
|
+
// eviction, so the socket would stay open and silently stop receiving, which is
|
|
1456
|
+
// indistinguishable from a quiet scope.
|
|
1457
|
+
this.ctx.acceptWebSocket(server);
|
|
1458
|
+
// `.parse`, not a cast: these three arrived as header strings, and the branded
|
|
1459
|
+
// ids are what every check downstream is keyed on. A malformed one would
|
|
1460
|
+
// otherwise be carried all the way to a `ctx.check` that quietly matches nothing —
|
|
1461
|
+
// which reads as "this subscriber may see nothing" and is indistinguishable from
|
|
1462
|
+
// a correct denial. Refused here instead, where it is still one subscriber's
|
|
1463
|
+
// problem. Throwing is right: the coordinator built this request.
|
|
1464
|
+
server.serializeAttachment({
|
|
1465
|
+
principal: principalId.parse(principal),
|
|
1466
|
+
tenantId: tenantIdOf.parse(tenantId),
|
|
1467
|
+
scopeId: scopeIdOf.parse(scopeId),
|
|
1468
|
+
since: new Date().toISOString(),
|
|
1469
|
+
});
|
|
1470
|
+
return new Response(null, { status: 101, webSocket: client });
|
|
1471
|
+
}
|
|
1472
|
+
/**
|
|
1473
|
+
* A subscriber said something.
|
|
1474
|
+
*
|
|
1475
|
+
* The channel is one-way by design — the server announces, the client re-reads
|
|
1476
|
+
* through the ordinary operation — so there is no client message that can cause a
|
|
1477
|
+
* read, a write, or a change of subscription. `ping`/`pong` is the whole protocol,
|
|
1478
|
+
* and it exists so an idle connection can be kept alive by either end.
|
|
1479
|
+
*
|
|
1480
|
+
* Deliberately NOT a place to let a client narrow or widen what it receives: a
|
|
1481
|
+
* filter the client chooses is a filter the client can choose wrongly, and the
|
|
1482
|
+
* only filter that matters here is the one it does not control.
|
|
1483
|
+
*/
|
|
1484
|
+
webSocketMessage(ws, message) {
|
|
1485
|
+
if (message === 'ping')
|
|
1486
|
+
ws.send('pong');
|
|
1487
|
+
}
|
|
1488
|
+
/**
|
|
1489
|
+
* The client hung up. Close our end so the runtime stops holding the subscription.
|
|
1490
|
+
*
|
|
1491
|
+
* The runtime passes `reason` and `wasClean` too; neither is read, so neither is
|
|
1492
|
+
* named — there is nothing to do differently for an unclean close, because the
|
|
1493
|
+
* socket is going away either way and the client's poll is what covers the gap.
|
|
1494
|
+
*/
|
|
1495
|
+
webSocketClose(ws, code) {
|
|
1496
|
+
// 1006 is reserved: it is what the runtime REPORTS for an abnormal closure and
|
|
1497
|
+
// is not a code anything may SEND, so echoing it back throws — on precisely the
|
|
1498
|
+
// path where the connection is already in trouble.
|
|
1499
|
+
try {
|
|
1500
|
+
ws.close(code === 1006 ? 1000 : code, 'scope closing the subscription');
|
|
1501
|
+
}
|
|
1502
|
+
catch {
|
|
1503
|
+
// Already gone. Nothing to do, and nothing worth reporting.
|
|
1504
|
+
}
|
|
1505
|
+
}
|
|
1506
|
+
/**
|
|
1507
|
+
* A socket failed. Logged and not rethrown: there is no caller to fail, and the
|
|
1508
|
+
* runtime delivers `webSocketClose` after this, which is what does the cleanup.
|
|
1509
|
+
*/
|
|
1510
|
+
webSocketError(_ws, error) {
|
|
1511
|
+
console.error('substrat: live-read socket error', error);
|
|
1512
|
+
}
|
|
1513
|
+
/**
|
|
1514
|
+
* The outbox's high-water mark before a committing path runs (#938), or `null`
|
|
1515
|
+
* when nobody is listening.
|
|
1516
|
+
*
|
|
1517
|
+
* `null` and `''` are different answers: `null` means no subscriber, so nothing is
|
|
1518
|
+
* read at all and a scope pays nothing for a feature it is not using; `''` is the
|
|
1519
|
+
* honest empty-outbox answer, and every ULID sorts above it.
|
|
1520
|
+
*
|
|
1521
|
+
* Read BEFORE the transaction by every caller, which is what makes the pair below
|
|
1522
|
+
* able to name exactly the events that path added.
|
|
1523
|
+
*/
|
|
1524
|
+
liveHighWaterMark() {
|
|
1525
|
+
if (this.ctx.getWebSockets().length === 0)
|
|
1526
|
+
return null;
|
|
1527
|
+
return (this.sql.exec('SELECT MAX(id) AS id FROM _substrat_outbox').toArray()[0]?.id ?? '');
|
|
1528
|
+
}
|
|
1529
|
+
/**
|
|
1530
|
+
* What every committing path does after its transaction closes: drain the outbox
|
|
1531
|
+
* to consumers, then announce what landed to whoever may see it.
|
|
1532
|
+
*
|
|
1533
|
+
* **One step, because there is one rule.** The fan-out was wired into `invoke`
|
|
1534
|
+
* alone at first, and `attachmentAdd`/`attachmentRemove` commit and emit too —
|
|
1535
|
+
* `attachment.added` and `attachment.removed`, about a real entity. A watcher of
|
|
1536
|
+
* that entity would have missed them: no error, no gap it could see, just a screen
|
|
1537
|
+
* that did not update for one kind of change. That is the exact failure the
|
|
1538
|
+
* hibernation design exists to prevent, arriving through a different door, so the
|
|
1539
|
+
* answer is a step both doors take rather than a second call both must remember.
|
|
1540
|
+
*
|
|
1541
|
+
* Order is load-bearing: drain FIRST, announce after. A consumer's own emits are
|
|
1542
|
+
* changes too, and they land in the outbox above `liveSince` — so announcing first
|
|
1543
|
+
* would tell a subscriber about the cause and not the effect, and it would re-read
|
|
1544
|
+
* too early.
|
|
1545
|
+
*/
|
|
1546
|
+
async settleCommitted(tenantId, scopeId, liveSince,
|
|
1547
|
+
/**
|
|
1548
|
+
* #1525: the call whose commit this is settling, or null — PASSED rather than
|
|
1549
|
+
* read off `this.invocationId` inside `dispatch`, so each of the three doors
|
|
1550
|
+
* says what it actually knows: `invoke` its own id, the two attachment verbs
|
|
1551
|
+
* null. Same discipline `recordDenial` adopted, and for the same reason.
|
|
1552
|
+
*/
|
|
1553
|
+
invocationId) {
|
|
1554
|
+
await this.dispatch(tenantId, scopeId, invocationId);
|
|
1555
|
+
if (liveSince === null)
|
|
1556
|
+
return;
|
|
1557
|
+
try {
|
|
1558
|
+
await this.fanOutLive(liveSince, tenantId, scopeId);
|
|
1559
|
+
}
|
|
1560
|
+
catch (err) {
|
|
1561
|
+
// The write has COMMITTED and the caller is owed its answer. A failure to
|
|
1562
|
+
// announce is a failure of a hint, and the client's poll is the floor
|
|
1563
|
+
// underneath it — so this is logged and never rethrown. Rethrowing would turn
|
|
1564
|
+
// a delivered write into a 500 the caller would reasonably retry.
|
|
1565
|
+
console.error('substrat: live-read fan-out failed after commit', err);
|
|
1566
|
+
}
|
|
1567
|
+
}
|
|
1568
|
+
/**
|
|
1569
|
+
* Announce what just committed, to whoever may see it (#938).
|
|
1570
|
+
*
|
|
1571
|
+
* **Three properties, and each is load-bearing.**
|
|
1572
|
+
*
|
|
1573
|
+
* *Post-commit.* Called after the operation's transaction has closed and after the
|
|
1574
|
+
* consumer drain, so nothing is announced that a rollback could take back and
|
|
1575
|
+
* nothing a consumer emitted is missed. A subscriber told about a row that then
|
|
1576
|
+
* vanished would re-read, find nothing, and have no way to tell that from a
|
|
1577
|
+
* deletion.
|
|
1578
|
+
*
|
|
1579
|
+
* *Filtered per subscriber, per event, after the check.* The declared
|
|
1580
|
+
* `liveTargets` key is checked ON THE EVENT'S OWN ENTITY, through `ctx.check` —
|
|
1581
|
+
* the same evaluator, the same entity-narrowed grants, the same parent walk as the
|
|
1582
|
+
* read the client is about to make. An entity type no module declared is announced
|
|
1583
|
+
* to nobody. Knowing that a row exists and changed at 14:02 is information about
|
|
1584
|
+
* that row, so the empty payload is not what makes this safe; this is.
|
|
1585
|
+
*
|
|
1586
|
+
* *Never able to fail the operation.* The write has committed and the caller has
|
|
1587
|
+
* its answer. A socket that has gone away mid-fan-out, or a check that cannot be
|
|
1588
|
+
* evaluated, costs a subscriber its live update — which it survives, because the
|
|
1589
|
+
* client's contract is that a push is a hint and the poll is the floor.
|
|
1590
|
+
*/
|
|
1591
|
+
async fanOutLive(sinceEventId, tenantId, scopeId) {
|
|
1592
|
+
const sockets = this.ctx.getWebSockets();
|
|
1593
|
+
if (sockets.length === 0)
|
|
1594
|
+
return;
|
|
1595
|
+
const rows = this.sql
|
|
1596
|
+
.exec(`SELECT id, type, entity_type, entity_id, occurred_at FROM _substrat_outbox
|
|
1597
|
+
WHERE id > ? ORDER BY id LIMIT ?`, sinceEventId, LIVE_FANOUT_LIMIT)
|
|
1598
|
+
.toArray();
|
|
1599
|
+
// Drop the undeclared entity types BEFORE any per-subscriber work. Not an
|
|
1600
|
+
// optimisation: it is the fail-closed rule stated once, in the one place that
|
|
1601
|
+
// decides, rather than relied upon inside the loop below.
|
|
1602
|
+
const announceable = rows.filter((r) => this.liveTargets.has(r.entity_type));
|
|
1603
|
+
if (announceable.length === 0)
|
|
1604
|
+
return;
|
|
1605
|
+
for (const ws of sockets) {
|
|
1606
|
+
let subscription = null;
|
|
1607
|
+
try {
|
|
1608
|
+
subscription = readSubscription(ws.deserializeAttachment());
|
|
1609
|
+
}
|
|
1610
|
+
catch {
|
|
1611
|
+
subscription = null;
|
|
1612
|
+
}
|
|
1613
|
+
// A socket we cannot name is a socket we cannot filter for. Skipped, never
|
|
1614
|
+
// sent to — see `readSubscription` for why every unusable shape fails closed.
|
|
1615
|
+
if (!subscription)
|
|
1616
|
+
continue;
|
|
1617
|
+
// A socket that outlived a scope rebind, or was somehow accepted for another
|
|
1618
|
+
// node, must not be fed this scope's events. Cheap, and it makes the identity
|
|
1619
|
+
// the frames are filtered against an explicit precondition rather than an
|
|
1620
|
+
// assumption about how the subscription was created.
|
|
1621
|
+
if (subscription.tenantId !== tenantId || subscription.scopeId !== scopeId)
|
|
1622
|
+
continue;
|
|
1623
|
+
// One context per subscriber, not per event: `ctx.check` is the expensive part
|
|
1624
|
+
// and the context is only the subject it is evaluated for.
|
|
1625
|
+
//
|
|
1626
|
+
// A frame this subscriber does not pass is NOT recorded as a denial (K-35),
|
|
1627
|
+
// and that is deliberate: `recordDenial` is called on a refused REQUEST, where
|
|
1628
|
+
// somebody asked for something and was told no. Nobody asked for these. Logging
|
|
1629
|
+
// one row per unentitled subscriber per event would bury the denials that mean
|
|
1630
|
+
// something — a broken screen, or somebody walking the surface — under the
|
|
1631
|
+
// ordinary, correct working of a filter.
|
|
1632
|
+
//
|
|
1633
|
+
// The operation name is carried anyway, for the events a fan-out cannot emit
|
|
1634
|
+
// but a future reader of this context might.
|
|
1635
|
+
const ctx = this.operationContext(subscription.principal, tenantId, scopeId, undefined, undefined, undefined, undefined, undefined, 'live.subscribe');
|
|
1636
|
+
for (const row of announceable) {
|
|
1637
|
+
// Non-null: `announceable` is exactly the rows whose type is in the map.
|
|
1638
|
+
const permission = this.liveTargets.get(row.entity_type);
|
|
1639
|
+
let allowed = false;
|
|
1640
|
+
try {
|
|
1641
|
+
const decision = await ctx.check(permission, {
|
|
1642
|
+
entityType: row.entity_type,
|
|
1643
|
+
entityId: row.entity_id,
|
|
1644
|
+
});
|
|
1645
|
+
allowed = decision.allowed;
|
|
1646
|
+
}
|
|
1647
|
+
catch {
|
|
1648
|
+
// A check that cannot answer is a check that refuses. The alternative —
|
|
1649
|
+
// treating an evaluator failure as an allow — turns an outage in the
|
|
1650
|
+
// permission path into a disclosure, which is the one failure mode this
|
|
1651
|
+
// surface must not have.
|
|
1652
|
+
allowed = false;
|
|
1653
|
+
}
|
|
1654
|
+
if (!allowed)
|
|
1655
|
+
continue;
|
|
1656
|
+
const frame = {
|
|
1657
|
+
kind: 'change',
|
|
1658
|
+
id: row.id,
|
|
1659
|
+
type: row.type,
|
|
1660
|
+
entityType: row.entity_type,
|
|
1661
|
+
entityId: row.entity_id,
|
|
1662
|
+
at: row.occurred_at,
|
|
1663
|
+
};
|
|
1664
|
+
try {
|
|
1665
|
+
ws.send(JSON.stringify(frame));
|
|
1666
|
+
}
|
|
1667
|
+
catch {
|
|
1668
|
+
// The socket went away between `getWebSockets()` and here. Stop writing to
|
|
1669
|
+
// this one and move on; the runtime will deliver `webSocketClose`.
|
|
1670
|
+
break;
|
|
1671
|
+
}
|
|
1672
|
+
}
|
|
1673
|
+
}
|
|
1674
|
+
}
|
|
1318
1675
|
// -- attachments (#473): the metadata half of the attachment surface --------
|
|
1319
1676
|
// The coordinator (worker) holds the bytes and the per-tenant R2 binding; this
|
|
1320
1677
|
// DO holds the permission gate and the metadata fact, under the same per-scope
|
|
@@ -1355,6 +1712,10 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1355
1712
|
const parsed = attachmentRecord.parse(record);
|
|
1356
1713
|
const gate = this.attachmentGate(parsed.entity.entityType);
|
|
1357
1714
|
return this.queue.enqueue(async () => {
|
|
1715
|
+
// #938: this path commits and emits too, so it takes the same mark-then-settle
|
|
1716
|
+
// pair `invoke` takes. Inside the queued body and before the transaction, for
|
|
1717
|
+
// the reason it is read there: that is where this call holds the DO to itself.
|
|
1718
|
+
const liveSince = this.liveHighWaterMark();
|
|
1358
1719
|
try {
|
|
1359
1720
|
await this.ctx.storage.transaction(async () => {
|
|
1360
1721
|
const ctx = this.operationContext(principal, tenantId, scopeId, undefined, connectionId, undefined, undefined, undefined, 'attachments.upload');
|
|
@@ -1374,11 +1735,13 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1374
1735
|
}
|
|
1375
1736
|
catch (err) {
|
|
1376
1737
|
if (err instanceof PermissionDenied) {
|
|
1377
|
-
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.upload', err);
|
|
1738
|
+
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.upload', err, null);
|
|
1378
1739
|
}
|
|
1379
1740
|
throw toRpcError(err);
|
|
1380
1741
|
}
|
|
1381
|
-
|
|
1742
|
+
// #1525: null. An attachment RPC carries no invocation, exactly as its denial
|
|
1743
|
+
// records none — so the deliveries it drains name no call rather than a wrong one.
|
|
1744
|
+
await this.settleCommitted(tenantId, scopeId, liveSince, null);
|
|
1382
1745
|
return parsed;
|
|
1383
1746
|
});
|
|
1384
1747
|
}
|
|
@@ -1392,7 +1755,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1392
1755
|
}
|
|
1393
1756
|
catch (err) {
|
|
1394
1757
|
if (err instanceof PermissionDenied) {
|
|
1395
|
-
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.list', err);
|
|
1758
|
+
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.list', err, null);
|
|
1396
1759
|
}
|
|
1397
1760
|
throw toRpcError(err);
|
|
1398
1761
|
}
|
|
@@ -1423,7 +1786,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1423
1786
|
}
|
|
1424
1787
|
catch (err) {
|
|
1425
1788
|
if (err instanceof PermissionDenied) {
|
|
1426
|
-
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.open', err);
|
|
1789
|
+
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.open', err, null);
|
|
1427
1790
|
}
|
|
1428
1791
|
throw toRpcError(err);
|
|
1429
1792
|
}
|
|
@@ -1436,7 +1799,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1436
1799
|
}
|
|
1437
1800
|
catch (err) {
|
|
1438
1801
|
if (err instanceof PermissionDenied) {
|
|
1439
|
-
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.open', err);
|
|
1802
|
+
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.open', err, null);
|
|
1440
1803
|
}
|
|
1441
1804
|
throw toRpcError(err);
|
|
1442
1805
|
}
|
|
@@ -1450,6 +1813,8 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1450
1813
|
if (!record)
|
|
1451
1814
|
return null;
|
|
1452
1815
|
const gate = this.attachmentGate(record.entity.entityType);
|
|
1816
|
+
// #938: same mark-then-settle pair as the upload path above.
|
|
1817
|
+
const liveSince = this.liveHighWaterMark();
|
|
1453
1818
|
try {
|
|
1454
1819
|
await this.ctx.storage.transaction(async () => {
|
|
1455
1820
|
const ctx = this.operationContext(principal, tenantId, scopeId, undefined, connectionId, undefined, undefined, undefined, 'attachments.remove');
|
|
@@ -1466,11 +1831,13 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1466
1831
|
}
|
|
1467
1832
|
catch (err) {
|
|
1468
1833
|
if (err instanceof PermissionDenied) {
|
|
1469
|
-
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.remove', err);
|
|
1834
|
+
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.remove', err, null);
|
|
1470
1835
|
}
|
|
1471
1836
|
throw toRpcError(err);
|
|
1472
1837
|
}
|
|
1473
|
-
|
|
1838
|
+
// #1525: null. An attachment RPC carries no invocation, exactly as its denial
|
|
1839
|
+
// records none — so the deliveries it drains name no call rather than a wrong one.
|
|
1840
|
+
await this.settleCommitted(tenantId, scopeId, liveSince, null);
|
|
1474
1841
|
return record;
|
|
1475
1842
|
});
|
|
1476
1843
|
}
|
|
@@ -1488,10 +1855,17 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1488
1855
|
.toArray()[0];
|
|
1489
1856
|
return row !== undefined;
|
|
1490
1857
|
}
|
|
1491
|
-
/**
|
|
1858
|
+
/**
|
|
1859
|
+
* The last time a schedule's operation ran on this scope (#383), or null.
|
|
1860
|
+
*
|
|
1861
|
+
* `kind = 'schedule'` is not decoration (#1288): an operation may legally be
|
|
1862
|
+
* named `freshness:<something>`, and before the column this read answered with
|
|
1863
|
+
* the EVALUATOR's last recorded time for that event type — a cadence gate
|
|
1864
|
+
* driven by a verdict nothing ran.
|
|
1865
|
+
*/
|
|
1492
1866
|
async scheduleLastRun(operation) {
|
|
1493
1867
|
const row = this.sql
|
|
1494
|
-
.exec(`SELECT last_run_at FROM _substrat_schedule_state WHERE schedule_op = ?`, operation)
|
|
1868
|
+
.exec(`SELECT last_run_at FROM _substrat_schedule_state WHERE kind = 'schedule' AND schedule_op = ?`, operation)
|
|
1495
1869
|
.toArray()[0];
|
|
1496
1870
|
return row?.last_run_at ?? null;
|
|
1497
1871
|
}
|
|
@@ -1499,8 +1873,9 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1499
1873
|
* #1232: everything the freshness evaluator needs about this scope, one round
|
|
1500
1874
|
* trip regardless of how many types are declared — the newest matching event
|
|
1501
1875
|
* per type, plus each type's recorded evaluator state (last recorded at +
|
|
1502
|
-
* outcome, kept in `_substrat_schedule_state` under
|
|
1503
|
-
*
|
|
1876
|
+
* outcome, kept in `_substrat_schedule_state` under `kind = 'freshness'`, which
|
|
1877
|
+
* since #1288 is what separates these rows from the schedule rows beside them —
|
|
1878
|
+
* the `freshness:` prefix on the key is retained but no longer load-bearing).
|
|
1504
1879
|
*/
|
|
1505
1880
|
async freshnessProbe(types) {
|
|
1506
1881
|
const out = {};
|
|
@@ -1517,7 +1892,8 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1517
1892
|
}
|
|
1518
1893
|
const keys = types.map((t) => `freshness:${t}`);
|
|
1519
1894
|
for (const row of this.sql
|
|
1520
|
-
.exec(`SELECT schedule_op, last_run_at, last_status FROM _substrat_schedule_state
|
|
1895
|
+
.exec(`SELECT schedule_op, last_run_at, last_status FROM _substrat_schedule_state
|
|
1896
|
+
WHERE kind = 'freshness' AND schedule_op IN (${marks})`, ...keys)
|
|
1521
1897
|
.toArray()) {
|
|
1522
1898
|
const t = row.schedule_op.slice('freshness:'.length);
|
|
1523
1899
|
if (out[t]) {
|
|
@@ -1529,16 +1905,173 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1529
1905
|
}
|
|
1530
1906
|
/**
|
|
1531
1907
|
* Write one `_substrat_schedule_state` row (#383). Spine, kernel-written.
|
|
1532
|
-
*
|
|
1908
|
+
*
|
|
1909
|
+
* `kind` is PASSED, never derived from the shape of `unit` (#1288) — deriving it
|
|
1910
|
+
* is precisely the convention this column replaced, and it is wrong for the one
|
|
1911
|
+
* input that matters: a schedule operation named `freshness:orders.placed` is a
|
|
1912
|
+
* schedule row, whatever its key looks like. `unit` is that row's key: a schedule
|
|
1533
1913
|
* operation (`module/verb`, and then `at`/`status` are when it ran and how it
|
|
1534
1914
|
* ended) or a freshness key (`freshness:<eventType>`, and then they are when the
|
|
1535
1915
|
* verdict was recorded and what it was — nothing ran). See the bootstrap DDL.
|
|
1916
|
+
*
|
|
1917
|
+
* `kind` is LAST because this is a positional RPC — see the interface's own note
|
|
1918
|
+
* in `host.ts`. An old DO drops a trailing argument; a leading one it binds, and
|
|
1919
|
+
* every value after it lands one column to the left, silently.
|
|
1920
|
+
*/
|
|
1921
|
+
async recordScheduleRun(unit, at, status, kind) {
|
|
1922
|
+
this.sql.exec(`INSERT INTO _substrat_schedule_state (kind, schedule_op, last_run_at, last_status)
|
|
1923
|
+
VALUES (?, ?, ?, ?)
|
|
1924
|
+
ON CONFLICT(kind, schedule_op) DO UPDATE SET last_run_at = excluded.last_run_at,
|
|
1925
|
+
last_status = excluded.last_status`, kind, unit, at, status);
|
|
1926
|
+
}
|
|
1927
|
+
// -- the resumable-run driver's store (#1577) -----------------------------
|
|
1928
|
+
//
|
|
1929
|
+
// D-14's DURABLE driver, and the only half of it that lives in the DO: the run
|
|
1930
|
+
// record and the step ledger. Every decision — what coalescing means, when a
|
|
1931
|
+
// step is skipped, when a run fails — is in the kernel's `JobRunStore` callers,
|
|
1932
|
+
// which the coordinator drives; these are reads and writes and nothing else, so
|
|
1933
|
+
// the two drivers cannot disagree about any of it.
|
|
1934
|
+
//
|
|
1935
|
+
// Each step commits on its OWN round trip rather than the pass batching them at
|
|
1936
|
+
// the end. That is what a mid-pass eviction keeps, and a DO is evicted and
|
|
1937
|
+
// revived constantly — batching would lose exactly the work resume exists for.
|
|
1938
|
+
/**
|
|
1939
|
+
* Coalescing, as ONE round trip: the live (`running`) run for this key, or this
|
|
1940
|
+
* row inserted and returned.
|
|
1941
|
+
*
|
|
1942
|
+
* **The single RPC is the atomicity**, and that is the whole reason it is shaped
|
|
1943
|
+
* this way rather than as a `jobRunLive` the coordinator follows with a
|
|
1944
|
+
* `jobRunInsert`. A Durable Object serializes its RPCs, so the lookup and the
|
|
1945
|
+
* insert cannot be interleaved by another caller; split across two calls they
|
|
1946
|
+
* can, and two concurrent starts then both find nothing and both insert. The
|
|
1947
|
+
* schema carries no unique constraint to catch that, deliberately — a crashed
|
|
1948
|
+
* run must stay restartable — so the indivisibility has to come from here.
|
|
1949
|
+
*/
|
|
1950
|
+
async jobRunStartOrJoin(moduleId, job, instance, row) {
|
|
1951
|
+
const live = this.sql
|
|
1952
|
+
.exec(`SELECT * FROM _substrat_job_runs
|
|
1953
|
+
WHERE module_id = ? AND job = ? AND instance = ? AND status = 'running'
|
|
1954
|
+
ORDER BY id DESC LIMIT 1`, moduleId, job, instance)
|
|
1955
|
+
.toArray()[0] ?? null;
|
|
1956
|
+
if (live)
|
|
1957
|
+
return live;
|
|
1958
|
+
// `transactionSync`, and the two writes inlined rather than reached through
|
|
1959
|
+
// `jobRunInsert`: an `await` between them is an output-gate boundary, and the
|
|
1960
|
+
// point of doing this in one RPC is that there is no boundary to be evicted
|
|
1961
|
+
// at. Same reason `SCHEDULE_STATE_REBUILD` insists on it.
|
|
1962
|
+
this.ctx.storage.transactionSync(() => {
|
|
1963
|
+
this.sql.exec(`INSERT INTO _substrat_job_runs
|
|
1964
|
+
(id, module_id, job, instance, payload, status, cursor, counters, attempts,
|
|
1965
|
+
last_error, started_at, updated_at, next_attempt_at, ended_at)
|
|
1966
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, row.id, row.module_id, row.job, row.instance, row.payload, row.status, row.cursor, row.counters, row.attempts, row.last_error, row.started_at, row.updated_at, row.next_attempt_at, row.ended_at);
|
|
1967
|
+
});
|
|
1968
|
+
return row;
|
|
1969
|
+
}
|
|
1970
|
+
/** One run by id, whatever its status. */
|
|
1971
|
+
async jobRunById(id) {
|
|
1972
|
+
return (this.sql.exec('SELECT * FROM _substrat_job_runs WHERE id = ?', id).toArray()[0] ?? null);
|
|
1973
|
+
}
|
|
1974
|
+
/** Insert a fresh run. The coordinator has already refused a non-queue-safe payload. */
|
|
1975
|
+
async jobRunInsert(row) {
|
|
1976
|
+
this.sql.exec(`INSERT INTO _substrat_job_runs
|
|
1977
|
+
(id, module_id, job, instance, payload, status, cursor, counters, attempts,
|
|
1978
|
+
last_error, started_at, updated_at, next_attempt_at, ended_at)
|
|
1979
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, row.id, row.module_id, row.job, row.instance, row.payload, row.status, row.cursor, row.counters, row.attempts, row.last_error, row.started_at, row.updated_at, row.next_attempt_at, row.ended_at);
|
|
1980
|
+
}
|
|
1981
|
+
/**
|
|
1982
|
+
* `running` runs whose backoff has elapsed, oldest first, after `afterId`.
|
|
1983
|
+
*
|
|
1984
|
+
* The cursor is what lets the coordinator page past runs it cannot drive: it
|
|
1985
|
+
* skips any whose job this deployment does not register, and without a cursor
|
|
1986
|
+
* those rows head every batch forever (`runDueJobRuns`). `afterId` is LAST, as
|
|
1987
|
+
* every argument added to an RPC on this interface must be.
|
|
1536
1988
|
*/
|
|
1537
|
-
async
|
|
1538
|
-
this.sql
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1989
|
+
async jobRunsDue(now, limit, afterId) {
|
|
1990
|
+
return this.sql
|
|
1991
|
+
.exec(
|
|
1992
|
+
// `afterId` bound TWICE rather than as `?2`: mixing anonymous and numbered
|
|
1993
|
+
// parameters makes the anonymous ones resume from the highest index used,
|
|
1994
|
+
// which is a footgun for the next person to add a clause. Spelled exactly
|
|
1995
|
+
// as the pure adapter spells it.
|
|
1996
|
+
`SELECT * FROM _substrat_job_runs
|
|
1997
|
+
WHERE status = 'running' AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
|
|
1998
|
+
AND (? IS NULL OR id > ?)
|
|
1999
|
+
ORDER BY id LIMIT ?`, now, afterId ?? null, afterId ?? null, limit)
|
|
2000
|
+
.toArray();
|
|
2001
|
+
}
|
|
2002
|
+
/** The operator read, newest first. */
|
|
2003
|
+
async jobRunList(filter) {
|
|
2004
|
+
const where = [];
|
|
2005
|
+
const params = [];
|
|
2006
|
+
for (const [column, value] of [
|
|
2007
|
+
['module_id', filter.moduleId],
|
|
2008
|
+
['job', filter.job],
|
|
2009
|
+
['instance', filter.instance],
|
|
2010
|
+
['status', filter.status],
|
|
2011
|
+
]) {
|
|
2012
|
+
if (value !== undefined) {
|
|
2013
|
+
where.push(`${column} = ?`);
|
|
2014
|
+
params.push(value);
|
|
2015
|
+
}
|
|
2016
|
+
}
|
|
2017
|
+
params.push(jobRunListLimit(filter.limit));
|
|
2018
|
+
return this.sql
|
|
2019
|
+
.exec(`SELECT * FROM _substrat_job_runs
|
|
2020
|
+
${where.length ? `WHERE ${where.join(' AND ')}` : ''}
|
|
2021
|
+
ORDER BY id DESC LIMIT ?`, ...params)
|
|
2022
|
+
.toArray();
|
|
2023
|
+
}
|
|
2024
|
+
/**
|
|
2025
|
+
* Write a pass outcome onto the run row — every column at once.
|
|
2026
|
+
*
|
|
2027
|
+
* Whole rather than partial, and that is the shape not an accident: a commit
|
|
2028
|
+
* writes cursor AND counters AND a cleared error AND a cleared backoff, and a
|
|
2029
|
+
* partial update that wrote three of the four would leave a run carrying the
|
|
2030
|
+
* last failure's error beside the new cursor, which reads as broken forever.
|
|
2031
|
+
*/
|
|
2032
|
+
async jobRunPatch(id, patch) {
|
|
2033
|
+
this.sql.exec(`UPDATE _substrat_job_runs
|
|
2034
|
+
SET status = ?, cursor = ?, counters = ?, attempts = ?, last_error = ?,
|
|
2035
|
+
updated_at = ?, next_attempt_at = ?, ended_at = ?
|
|
2036
|
+
WHERE id = ?`, patch.status, patch.cursor, patch.counters, patch.attempts, patch.lastError, patch.updatedAt, patch.nextAttemptAt, patch.endedAt, id);
|
|
2037
|
+
}
|
|
2038
|
+
/** One step's ledger row — a non-null `result` is what means completed. */
|
|
2039
|
+
async jobStepRow(runId, step) {
|
|
2040
|
+
return (this.sql
|
|
2041
|
+
.exec('SELECT step, result, attempts, last_error FROM _substrat_job_steps WHERE run_id = ? AND step = ?', runId, step)
|
|
2042
|
+
.toArray()[0] ?? null);
|
|
2043
|
+
}
|
|
2044
|
+
/** Record one step attempt. `result` non-null = it completed and must not re-run. */
|
|
2045
|
+
async jobStepRecord(runId, step, result, attempts, lastError, at) {
|
|
2046
|
+
this.sql.exec(`INSERT INTO _substrat_job_steps (run_id, step, result, attempts, last_error, recorded_at)
|
|
2047
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
2048
|
+
ON CONFLICT (run_id, step) DO UPDATE SET result = excluded.result,
|
|
2049
|
+
attempts = excluded.attempts,
|
|
2050
|
+
last_error = excluded.last_error,
|
|
2051
|
+
recorded_at = excluded.recorded_at`, runId, step, result, attempts, lastError, at);
|
|
2052
|
+
}
|
|
2053
|
+
/**
|
|
2054
|
+
* A COMMITTED pass: the run's new state and the dropping of its step ledger, in
|
|
2055
|
+
* ONE RPC so they cannot come apart.
|
|
2056
|
+
*
|
|
2057
|
+
* Same reasoning as `jobRunStartOrJoin` — the single round trip is the
|
|
2058
|
+
* atomicity. As two calls, a stop in between leaves the advanced cursor beside
|
|
2059
|
+
* the finished pass's memo rows, and a handler that reuses a step name across
|
|
2060
|
+
* passes (legal: the determinism rule binds names to the payload and prior
|
|
2061
|
+
* results, not to the cursor) then skips work it never did. The kernel's
|
|
2062
|
+
* `runJobPass` carries the full argument.
|
|
2063
|
+
*/
|
|
2064
|
+
async jobCommitPass(id, patch) {
|
|
2065
|
+
// `transactionSync` with both statements inline — NOT `await
|
|
2066
|
+
// this.jobRunPatch(...)` then the delete. The await is an output-gate
|
|
2067
|
+
// boundary, which is precisely the gap this method exists to close.
|
|
2068
|
+
this.ctx.storage.transactionSync(() => {
|
|
2069
|
+
this.sql.exec(`UPDATE _substrat_job_runs
|
|
2070
|
+
SET status = ?, cursor = ?, counters = ?, attempts = ?, last_error = ?,
|
|
2071
|
+
updated_at = ?, next_attempt_at = ?, ended_at = ?
|
|
2072
|
+
WHERE id = ?`, patch.status, patch.cursor, patch.counters, patch.attempts, patch.lastError, patch.updatedAt, patch.nextAttemptAt, patch.endedAt, id);
|
|
2073
|
+
this.sql.exec('DELETE FROM _substrat_job_steps WHERE run_id = ?', id);
|
|
2074
|
+
});
|
|
1542
2075
|
}
|
|
1543
2076
|
// -- guards (K-17) --------------------------------------------------------
|
|
1544
2077
|
/** This scope's spine, read under whatever transaction the caller already opened. */
|
|
@@ -1661,20 +2194,44 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1661
2194
|
* Written AFTER the effect, so a crash mid-effect retries rather than silently
|
|
1662
2195
|
* marking success. The coordinator computes the backoff because it owns the
|
|
1663
2196
|
* per-executor policy; the DO owns the state.
|
|
2197
|
+
*
|
|
2198
|
+
* `invocationId` is LAST because this is a positional RPC — the same rule
|
|
2199
|
+
* `recordScheduleRun` states. An old DO drops a trailing argument; a leading one
|
|
2200
|
+
* it binds, and every value after it lands one column to the left, silently.
|
|
2201
|
+
*/
|
|
2202
|
+
recordExecutorAttempt(eventId, deliveryId, error, nextAttemptAt,
|
|
2203
|
+
/**
|
|
2204
|
+
* #1525: the call THIS attempt ran in, or null. Passed rather than read off
|
|
2205
|
+
* `this.invocationId`, and here there is no choice about it: executors run on
|
|
2206
|
+
* the COORDINATOR, so by the time this RPC arrives the queued body that held
|
|
2207
|
+
* the id has long returned and the field reads null. The coordinator is the
|
|
2208
|
+
* only side that knows, which is why the SQLite twin passes it too — a recorded
|
|
2209
|
+
* fact whose correctness argument differs per adapter is the kind that drifts.
|
|
2210
|
+
*
|
|
2211
|
+
* Defaulted, for the other half of the skew: a coordinator too old to pass it
|
|
2212
|
+
* calls this with four arguments, and `undefined` is not a value SQLite binds.
|
|
2213
|
+
* Null is the honest answer there anyway — that coordinator recorded no call.
|
|
1664
2214
|
*/
|
|
1665
|
-
|
|
2215
|
+
invocationId = null) {
|
|
1666
2216
|
const prior = this.sql
|
|
1667
2217
|
.exec('SELECT attempts FROM _substrat_deliveries WHERE event_id = ? AND consumer_module = ?', eventId, deliveryId)
|
|
1668
2218
|
.toArray()[0];
|
|
1669
2219
|
const attempts = (prior?.attempts ?? 0) + 1;
|
|
1670
2220
|
this.sql.exec(`INSERT INTO _substrat_deliveries
|
|
1671
|
-
(event_id, consumer_module, delivered_at, error, attempts, next_attempt_at
|
|
1672
|
-
|
|
2221
|
+
(event_id, consumer_module, delivered_at, error, attempts, next_attempt_at,
|
|
2222
|
+
invocation_id)
|
|
2223
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
1673
2224
|
ON CONFLICT (event_id, consumer_module) DO UPDATE SET
|
|
1674
2225
|
delivered_at = excluded.delivered_at,
|
|
1675
2226
|
error = excluded.error,
|
|
1676
2227
|
attempts = excluded.attempts,
|
|
1677
|
-
next_attempt_at = excluded.next_attempt_at
|
|
2228
|
+
next_attempt_at = excluded.next_attempt_at,
|
|
2229
|
+
-- #1525: overwritten, like the four above. The row describes the LATEST
|
|
2230
|
+
-- attempt, so a retry drained by a different call (or by none) must not
|
|
2231
|
+
-- keep claiming the call that made the first one.
|
|
2232
|
+
invocation_id = excluded.invocation_id`, eventId, deliveryId, new Date().toISOString(), error, attempts, nextAttemptAt,
|
|
2233
|
+
// #1525: the call this attempt ran in, as the coordinator named it.
|
|
2234
|
+
invocationId);
|
|
1678
2235
|
return attempts;
|
|
1679
2236
|
}
|
|
1680
2237
|
/** How many attempts this delivery has already had — the backoff input. */
|
|
@@ -1732,8 +2289,18 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1732
2289
|
* the event on the next drain). Backpressure throws BEFORE any write — the caller
|
|
1733
2290
|
* records a failed attempt and the delivery retries on its own backoff, exactly as
|
|
1734
2291
|
* a throwing handler would.
|
|
2292
|
+
*
|
|
2293
|
+
* `invocationId` is LAST, and defaulted, for the RPC-skew reason `recordExecutorAttempt`
|
|
2294
|
+
* states at length.
|
|
2295
|
+
*/
|
|
2296
|
+
routeExecutorEventToPlatform(eventId, deliveryId, kind, payload, requestedBy,
|
|
2297
|
+
/**
|
|
2298
|
+
* #1525: the call this routing ran in, or null. On the hosted path this is the
|
|
2299
|
+
* dominant executor journal — every vertical is CP-less, so a connector delivery
|
|
2300
|
+
* becomes an intent here rather than running in `recordExecutorAttempt`'s caller —
|
|
2301
|
+
* so leaving it null would make the column read "no call" for the common case.
|
|
1735
2302
|
*/
|
|
1736
|
-
|
|
2303
|
+
invocationId = null) {
|
|
1737
2304
|
const pending = Number(this.sql
|
|
1738
2305
|
.exec(`SELECT COUNT(*) AS c FROM _substrat_platform_requests WHERE status = 'pending'`)
|
|
1739
2306
|
.toArray()[0].c);
|
|
@@ -1753,7 +2320,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1753
2320
|
this.sql.exec(`INSERT INTO _substrat_platform_requests
|
|
1754
2321
|
(id, kind, payload, requested_by, impersonation, status, attempts, requested_at)
|
|
1755
2322
|
VALUES (?, ?, ?, ?, ?, 'pending', 0, ?)`, id, kind, payload, requestedBy, source?.impersonation ?? null, instant.parse(new Date().toISOString()));
|
|
1756
|
-
this.recordExecutorAttempt(eventId, deliveryId, null, null);
|
|
2323
|
+
this.recordExecutorAttempt(eventId, deliveryId, null, null, invocationId);
|
|
1757
2324
|
return id;
|
|
1758
2325
|
}
|
|
1759
2326
|
/**
|
|
@@ -1947,6 +2514,11 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1947
2514
|
// consumer dead-letter.
|
|
1948
2515
|
'ALTER TABLE _substrat_deliveries ADD COLUMN attempts INTEGER NOT NULL DEFAULT 0',
|
|
1949
2516
|
'ALTER TABLE _substrat_deliveries ADD COLUMN next_attempt_at TEXT',
|
|
2517
|
+
// #1525: the invocation an attempt ran in, on a scope DO created before the
|
|
2518
|
+
// column. Nullable, and the null is honestly "no call was carried" — which
|
|
2519
|
+
// attempt produced a delivery already journalled cannot be decided afterwards,
|
|
2520
|
+
// exactly as #1237's outbox column argued.
|
|
2521
|
+
'ALTER TABLE _substrat_deliveries ADD COLUMN invocation_id TEXT',
|
|
1950
2522
|
// K-34: the authorization column on a scope DO created before it existed. Nullable,
|
|
1951
2523
|
// so legacy outbox rows read as "unrecorded". (_substrat_denials is a new table,
|
|
1952
2524
|
// covered by KERNEL_DDL's IF NOT EXISTS with no ALTER.)
|
|
@@ -1962,6 +2534,10 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1962
2534
|
'ALTER TABLE _substrat_outbox ADD COLUMN impersonation TEXT',
|
|
1963
2535
|
'ALTER TABLE _substrat_platform_requests ADD COLUMN impersonation TEXT',
|
|
1964
2536
|
'ALTER TABLE _substrat_denials ADD COLUMN impersonation TEXT',
|
|
2537
|
+
// #1525: the invocation a refusal happened during, on a scope DO created before
|
|
2538
|
+
// the column. Nullable, and the null is honestly "no id was carried" — a past
|
|
2539
|
+
// denial's call cannot be decided afterwards, as #1237's outbox column argued.
|
|
2540
|
+
'ALTER TABLE _substrat_denials ADD COLUMN invocation_id TEXT',
|
|
1965
2541
|
// #1231: the emitting operation, on a scope DO created before the column.
|
|
1966
2542
|
// Nullable so every legacy row reads as unrecorded rather than named.
|
|
1967
2543
|
'ALTER TABLE _substrat_outbox ADD COLUMN operation TEXT',
|
|
@@ -1997,6 +2573,33 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1997
2573
|
// boot. `lint:spine-ddl` compares KERNEL_DDL's indexes only, so this one is held to
|
|
1998
2574
|
// both adapters by the query-plan test rather than by that gate.
|
|
1999
2575
|
this.sql.exec('CREATE INDEX IF NOT EXISTS _substrat_outbox_invocation ON _substrat_outbox (invocation_id, id)');
|
|
2576
|
+
this.ensureScheduleStateKind();
|
|
2577
|
+
}
|
|
2578
|
+
/**
|
|
2579
|
+
* #1288: `_substrat_schedule_state`, rebuilt with `kind` in its key on a scope DO
|
|
2580
|
+
* whose table predates the column. Not in the ALTER list above, because `kind`
|
|
2581
|
+
* joins the PRIMARY KEY and no ALTER can widen a key — the statements are the
|
|
2582
|
+
* kernel's, so the pure adapter rebuilds byte-identically.
|
|
2583
|
+
*
|
|
2584
|
+
* Detected from `sqlite_master.sql`: DO SQLite restricts `PRAGMA`, and reading
|
|
2585
|
+
* the stored DDL is the one probe both adapters can make.
|
|
2586
|
+
*/
|
|
2587
|
+
ensureScheduleStateKind() {
|
|
2588
|
+
const row = this.sql
|
|
2589
|
+
.exec(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`, '_substrat_schedule_state')
|
|
2590
|
+
.toArray()[0];
|
|
2591
|
+
if (!row || scheduleStateHasKind(row.sql))
|
|
2592
|
+
return;
|
|
2593
|
+
// In a transaction, for the reason the kernel constant spells out: create-copy-
|
|
2594
|
+
// drop-rename has two intermediate states and both are unrecoverable on the next
|
|
2595
|
+
// wake. `transactionSync`, not the async one every operation uses — the DO
|
|
2596
|
+
// runtime forbids a manual BEGIN through `sql.exec`, and this body is wholly
|
|
2597
|
+
// synchronous, which is the one case the sync API is for (it commits at the
|
|
2598
|
+
// first await, and there is none). It also has to be sync because the caller is.
|
|
2599
|
+
this.ctx.storage.transactionSync(() => {
|
|
2600
|
+
for (const stmt of splitSqlStatements(SCHEDULE_STATE_REBUILD))
|
|
2601
|
+
this.sql.exec(stmt);
|
|
2602
|
+
});
|
|
2000
2603
|
}
|
|
2001
2604
|
async importDump(tables, destScopeId) {
|
|
2002
2605
|
// The WHOLE drop-then-replay runs under deferred foreign keys, in one transaction.
|
|
@@ -2196,7 +2799,13 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2196
2799
|
* later alarm-driven drain with a call it had nothing to do with.
|
|
2197
2800
|
*/
|
|
2198
2801
|
invocationId = null;
|
|
2199
|
-
async dispatch(tenantId, scopeId
|
|
2802
|
+
async dispatch(tenantId, scopeId,
|
|
2803
|
+
/**
|
|
2804
|
+
* #1525: the call this drain is running in, or null. Every round of the loop
|
|
2805
|
+
* belongs to it: a consumer's own emit is delivered in the same tail, so the
|
|
2806
|
+
* whole cascade is one call's work.
|
|
2807
|
+
*/
|
|
2808
|
+
invocationId) {
|
|
2200
2809
|
for (let round = 0; round < 50; round++) {
|
|
2201
2810
|
let deliveredAny = false;
|
|
2202
2811
|
for (const mod of this.modules.values()) {
|
|
@@ -2222,16 +2831,18 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2222
2831
|
system: mod.id,
|
|
2223
2832
|
});
|
|
2224
2833
|
await consumer.handler(ctx, event);
|
|
2225
|
-
this.sql.exec(`INSERT INTO _substrat_deliveries
|
|
2226
|
-
|
|
2834
|
+
this.sql.exec(`INSERT INTO _substrat_deliveries
|
|
2835
|
+
(event_id, consumer_module, delivered_at, invocation_id)
|
|
2836
|
+
VALUES (?, ?, ?, ?)`, event.id, mod.id, new Date().toISOString(), invocationId);
|
|
2227
2837
|
});
|
|
2228
2838
|
deliveredAny = true;
|
|
2229
2839
|
}
|
|
2230
2840
|
catch (err) {
|
|
2231
2841
|
// Dead-letter (v0): journal the failure so one poison event
|
|
2232
2842
|
// can't wedge the loop. Written outside the rolled-back txn.
|
|
2233
|
-
this.sql.exec(`INSERT INTO _substrat_deliveries
|
|
2234
|
-
|
|
2843
|
+
this.sql.exec(`INSERT INTO _substrat_deliveries
|
|
2844
|
+
(event_id, consumer_module, delivered_at, error, invocation_id)
|
|
2845
|
+
VALUES (?, ?, ?, ?, ?)`, event.id, mod.id, new Date().toISOString(), String(err), invocationId);
|
|
2235
2846
|
}
|
|
2236
2847
|
finally {
|
|
2237
2848
|
// Cleared on BOTH paths. Left set, the id leaks onto every later emit
|
|
@@ -2253,7 +2864,28 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2253
2864
|
* survives — the whole point, since the denial is the write the operation could not make.
|
|
2254
2865
|
*/
|
|
2255
2866
|
/** K-42: the session a refused call ran under travels with the denial row. */
|
|
2256
|
-
recordDenial(subject, tenantId, operation, err,
|
|
2867
|
+
recordDenial(subject, tenantId, operation, err,
|
|
2868
|
+
/**
|
|
2869
|
+
* #1525: the invocation this refusal belongs to, or null — PASSED, never read off
|
|
2870
|
+
* `this.invocationId` here.
|
|
2871
|
+
*
|
|
2872
|
+
* Reading the ambient field is only self-evidently right where one call holds the
|
|
2873
|
+
* DO to itself, and two denial paths do not: `attachmentList` and
|
|
2874
|
+
* `attachmentAuthorize` run OUTSIDE `this.queue`, and both await before they
|
|
2875
|
+
* record (`ensureMigrations`, `ctx.check`). Whether the input gate can actually
|
|
2876
|
+
* reopen far enough for one of them to observe an in-flight call's id is NOT
|
|
2877
|
+
* settled here — a probe that raced twelve attachment refusals against invokes
|
|
2878
|
+
* holding an id (including one awaiting the control plane) recorded null every
|
|
2879
|
+
* time, so the gate evidently holds more than the shape of the code promises.
|
|
2880
|
+
*
|
|
2881
|
+
* Passed anyway, because the argument for the ambient read is an argument about
|
|
2882
|
+
* workerd's gate semantics, and the argument for a parameter is local: the invoke
|
|
2883
|
+
* path passes its own id, every attachment path passes null, and each says what it
|
|
2884
|
+
* actually knows. That is the property worth having on a recorded fact, and it
|
|
2885
|
+
* costs one argument. No test accompanies it — the condition could not be
|
|
2886
|
+
* reproduced, and a test that passes either way would be worse than none.
|
|
2887
|
+
*/
|
|
2888
|
+
invocationId, impersonation) {
|
|
2257
2889
|
// Only an ENFORCED denial (assertAllowed, which attaches the checked permission +
|
|
2258
2890
|
// node) is recorded. A module's own hand-thrown `new PermissionDenied('…')` carries
|
|
2259
2891
|
// no permission key and is left to the module.
|
|
@@ -2265,8 +2897,11 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2265
2897
|
? { connection: subject.id }
|
|
2266
2898
|
: subject.id;
|
|
2267
2899
|
this.sql.exec(`INSERT INTO _substrat_denials
|
|
2268
|
-
(id, actor, permission, tenant_id, scope_id, operation, impersonation,
|
|
2269
|
-
|
|
2900
|
+
(id, actor, permission, tenant_id, scope_id, operation, impersonation,
|
|
2901
|
+
invocation_id, at)
|
|
2902
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, ulid(), JSON.stringify(actor), err.permission, err.node.tenantId, err.node.scopeId ?? null, operation, impersonation ? JSON.stringify(impersonationStampOf(impersonation)) : null,
|
|
2903
|
+
// #1525: the call this refusal belongs to, as the caller named it.
|
|
2904
|
+
invocationId, new Date().toISOString());
|
|
2270
2905
|
}
|
|
2271
2906
|
parseOutboxRow(row) {
|
|
2272
2907
|
return domainEvent.parse({
|