@substrat-run/adapter-cloudflare 0.114.0 → 0.116.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/control-plane-do.d.ts +26 -0
- package/dist/control-plane-do.d.ts.map +1 -1
- package/dist/control-plane-do.js +48 -0
- package/dist/control-plane-do.js.map +1 -1
- package/dist/host.d.ts +60 -1
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +263 -29
- package/dist/host.js.map +1 -1
- package/dist/live-reads.d.ts +116 -0
- package/dist/live-reads.d.ts.map +1 -0
- package/dist/live-reads.js +106 -0
- package/dist/live-reads.js.map +1 -0
- package/dist/scope-do.d.ts.map +1 -1
- package/dist/scope-do.js +689 -58
- package/dist/scope-do.js.map +1 -1
- package/package.json +4 -4
package/dist/scope-do.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { DurableObject } from 'cloudflare:workers';
|
|
2
|
-
import { ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, domainEvent, domainEventInput, eventId, instant, objectRef, toWireFailure, grantRefFromProof, principalId, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, MAX_PENDING_SWEEP_RUNS, SWEEP_RUNS_KIND, platformRequest, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, listLimitOf, requestFingerprint, substratError, assertReplayableDump, REDRAIN_BATCH, } from '@substrat-run/contracts';
|
|
3
|
-
import { ulid, createUlid, assertAllowed, ConnectionSealingKeyUnavailableError, noSealingKeyMessage, sealTo, assertReadOnlyQuery, entitlementDenial, platformRequestHistoryQuery, PLATFORM_REQUEST_COLUMNS, denialListQuery, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, PermissionDenied, assertImpersonationWrites, assertModuleEnqueueableKind, impersonationStampOf, createAtomic, NotSearchable, isSearchIndexTable, searchIndexDdl, searchIndexMigrations, searchIndexPlans, NotListable, listIndexMigrations, listIndexPlans, listQuery, cursorOf, searchLimit, searchMatchExpression, searchQuery, IDEMPOTENCY_DDL, assertIdempotencyKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, } from '@substrat-run/kernel';
|
|
2
|
+
import { ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, domainEvent, domainEventInput, eventId, instant, objectRef, toWireFailure, grantRefFromProof, principalId, scopeId as scopeIdOf, tenantId as tenantIdOf, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, MAX_PENDING_SWEEP_RUNS, SWEEP_RUNS_KIND, platformRequest, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, listLimitOf, requestFingerprint, substratError, assertReplayableDump, REDRAIN_BATCH, } from '@substrat-run/contracts';
|
|
3
|
+
import { ulid, createUlid, assertAllowed, ConnectionSealingKeyUnavailableError, noSealingKeyMessage, sealTo, assertReadOnlyQuery, entitlementDenial, platformRequestHistoryQuery, PLATFORM_REQUEST_COLUMNS, denialListQuery, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, PermissionDenied, assertImpersonationWrites, assertModuleEnqueueableKind, impersonationStampOf, createAtomic, NotSearchable, isSearchIndexTable, searchIndexDdl, searchIndexMigrations, searchIndexPlans, NotListable, listIndexMigrations, listIndexPlans, listQuery, cursorOf, searchLimit, searchMatchExpression, searchQuery, IDEMPOTENCY_DDL, assertIdempotencyKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, SCHEDULE_STATE_DDL, SCHEDULE_STATE_REBUILD, scheduleStateHasKind, JOB_RUN_DDL, jobRunListLimit, } from '@substrat-run/kernel';
|
|
4
|
+
import { isUpgradeRequest, readSubscription, LIVE_FANOUT_LIMIT, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
|
|
4
5
|
import { OperationQueue } from './serialization.js';
|
|
5
6
|
import { doScopedSql } from './sql.js';
|
|
6
7
|
import { facetEvents, readDeadLetters, readHistory, readInvocation, walkEventCause, walkEventEffects } from '@substrat-run/kernel';
|
|
@@ -127,31 +128,28 @@ const KERNEL_DDL = `
|
|
|
127
128
|
operation TEXT,
|
|
128
129
|
-- K-42: WHICH of the two actors was refused is exactly what this log is for.
|
|
129
130
|
impersonation TEXT,
|
|
131
|
+
-- #1525: the INVOCATION this refusal happened during, the same id #1237 stamps on
|
|
132
|
+
-- every event of a call. A denial joins to nothing but actor and time otherwise --
|
|
133
|
+
-- the operation column names what was attempted, never WHICH attempt -- so "what
|
|
134
|
+
-- else did this request do" cannot reach a refusal, which is the one thing an
|
|
135
|
+
-- incident asks about first. NULL = the transport carried no id (a seed, a test,
|
|
136
|
+
-- an internal call, an attachment RPC), or the row predates the column.
|
|
137
|
+
invocation_id TEXT,
|
|
130
138
|
at TEXT NOT NULL,
|
|
131
139
|
drained_at TEXT
|
|
132
140
|
);
|
|
133
|
-
-- #383: the platform sweep's per-scope gating state
|
|
134
|
-
--
|
|
135
|
-
--
|
|
136
|
-
--
|
|
137
|
-
--
|
|
138
|
-
|
|
139
|
-
--
|
|
140
|
-
--
|
|
141
|
-
--
|
|
142
|
-
--
|
|
143
|
-
--
|
|
144
|
-
|
|
145
|
-
-- that declared a schedule operation literally named "freshness:orders.placed"
|
|
146
|
-
-- would share a row with the evaluator and nothing today would reject it.
|
|
147
|
-
-- #1288 tracks giving the table a column (or a name) that says this outright,
|
|
148
|
-
-- which is also what would let the collision be refused rather than avoided.
|
|
149
|
-
-- Spine (kernel-written), never a module migration.
|
|
150
|
-
CREATE TABLE IF NOT EXISTS _substrat_schedule_state (
|
|
151
|
-
schedule_op TEXT PRIMARY KEY,
|
|
152
|
-
last_run_at TEXT,
|
|
153
|
-
last_status TEXT
|
|
154
|
-
);
|
|
141
|
+
-- #383 / #1232 / #1288: the platform sweep's per-scope gating state, holding two
|
|
142
|
+
-- families of row that the kind COLUMN -- not the spelling of a key -- tells
|
|
143
|
+
-- apart. Spine (kernel-written), never a module migration. Shared with the pure
|
|
144
|
+
-- adapter from @substrat-run/kernel, so the shape a rebuild produces and the shape
|
|
145
|
+
-- a fresh store gets cannot part company; the column comments are in there.
|
|
146
|
+
${SCHEDULE_STATE_DDL}
|
|
147
|
+
-- #1577: the resumable-run driver's record and the step ledger of the pass it
|
|
148
|
+
-- currently has in flight. Spine (kernel-written), never a module migration.
|
|
149
|
+
-- Shared with the pure adapter from @substrat-run/kernel so the shape production
|
|
150
|
+
-- builds and the shape a self-host builds cannot part company; the column
|
|
151
|
+
-- comments, and the reason coalescing is NOT a unique index, are in there.
|
|
152
|
+
${JOB_RUN_DDL}
|
|
155
153
|
CREATE TABLE IF NOT EXISTS _substrat_migrations (
|
|
156
154
|
module_id TEXT NOT NULL,
|
|
157
155
|
version TEXT NOT NULL,
|
|
@@ -191,6 +189,17 @@ const KERNEL_DDL = `
|
|
|
191
189
|
-- next_attempt_at IS NULL -> terminal: error IS NULL delivered, else dead
|
|
192
190
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
193
191
|
next_attempt_at TEXT,
|
|
192
|
+
-- #1525: the invocation THIS ATTEMPT ran in, the same id #1237 stamps on every
|
|
193
|
+
-- event of a call. Not the emitting event's -- that one is already on the outbox
|
|
194
|
+
-- and a delivery joins to it through event_id. What that join cannot say is
|
|
195
|
+
-- which call attempted the DELIVERY, and for an executor the two differ by
|
|
196
|
+
-- design: attempt one runs in the emitting call's post-commit tail, every retry
|
|
197
|
+
-- afterwards in a drain that is a different call or none. Moves with the row on
|
|
198
|
+
-- an upsert, like delivered_at, error and attempts -- it describes the
|
|
199
|
+
-- latest attempt, not the first. NULL = that attempt carried no call (a
|
|
200
|
+
-- scheduled drain, an alarm, a seed, an attachment RPC), or the row predates
|
|
201
|
+
-- the column.
|
|
202
|
+
invocation_id TEXT,
|
|
194
203
|
PRIMARY KEY (event_id, consumer_module)
|
|
195
204
|
);
|
|
196
205
|
-- Scope-local permissions (docs/architecture/scope-local-permissions.md): the
|
|
@@ -505,6 +514,12 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
505
514
|
listPlans = new Map();
|
|
506
515
|
/** entityType → the declared attachment gate (#473): read key + write key (default: read). */
|
|
507
516
|
attachmentTargets = new Map();
|
|
517
|
+
/**
|
|
518
|
+
* entityType → the declared live-read gate (#938): the key a subscriber must hold
|
|
519
|
+
* ON THAT ENTITY before a change to it is announced. Absent = announced to nobody;
|
|
520
|
+
* `fanOutLive` treats a miss as silence, never as "unguarded".
|
|
521
|
+
*/
|
|
522
|
+
liveTargets = new Map();
|
|
508
523
|
checker;
|
|
509
524
|
systemPrincipal = principalId.parse(ulid());
|
|
510
525
|
/**
|
|
@@ -622,6 +637,20 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
622
637
|
}
|
|
623
638
|
this.attachmentTargets.set(target.entityType, gate);
|
|
624
639
|
}
|
|
640
|
+
// Live-read targets (#938): entityType → the read key a subscriber must pass ON
|
|
641
|
+
// THAT ENTITY before a change to it is announced. Validated exactly like the
|
|
642
|
+
// attachment gate above, and for a sharper reason: two modules disagreeing about
|
|
643
|
+
// which key guards an entity type would make "who may watch this" depend on
|
|
644
|
+
// registration order, and the losing declaration would be the one that was meant
|
|
645
|
+
// to be stricter. An entity type nobody declares is announced to nobody.
|
|
646
|
+
for (const target of manifest.liveTargets ?? []) {
|
|
647
|
+
const existing = this.liveTargets.get(target.entityType);
|
|
648
|
+
if (existing && existing !== target.readPermission) {
|
|
649
|
+
throw new Error(`conflicting liveTargets for '${target.entityType}': ` +
|
|
650
|
+
`(${existing}) vs (${target.readPermission})`);
|
|
651
|
+
}
|
|
652
|
+
this.liveTargets.set(target.entityType, target.readPermission);
|
|
653
|
+
}
|
|
625
654
|
for (const name of manifest.withdraws ?? []) {
|
|
626
655
|
this.withdrawn.set(name, manifest.id);
|
|
627
656
|
this.operations.delete(name);
|
|
@@ -843,6 +872,24 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
843
872
|
return ids.length;
|
|
844
873
|
});
|
|
845
874
|
}
|
|
875
|
+
/**
|
|
876
|
+
* How many rows `redrainEvents` WOULD reopen for the same instant (#1545), reopening
|
|
877
|
+
* none of them. A separate verb rather than a flag on the one above, deliberately: the
|
|
878
|
+
* two answers are indistinguishable once they are numbers, so the caller that asks for
|
|
879
|
+
* a count must not be able to reach the reopen by losing an argument on the way.
|
|
880
|
+
*
|
|
881
|
+
* UNBOUNDED where the reopen is batched. The batch exists because an UPDATE over the
|
|
882
|
+
* whole window rewrites every row and its index entries inside ONE Durable Object
|
|
883
|
+
* request, against a fixed budget. An aggregate materialises no rows, so that budget is
|
|
884
|
+
* not the binding constraint — and a partial count would be worse than useless: the
|
|
885
|
+
* number is the whole point, and "5000, or possibly more" answers nothing.
|
|
886
|
+
*/
|
|
887
|
+
async redrainCount(drainedBefore) {
|
|
888
|
+
return this.sql
|
|
889
|
+
.exec(`SELECT COUNT(*) AS c FROM _substrat_outbox
|
|
890
|
+
WHERE drained_at IS NOT NULL AND drained_at < ?`, drainedBefore)
|
|
891
|
+
.toArray()[0].c;
|
|
892
|
+
}
|
|
846
893
|
/**
|
|
847
894
|
* Facet this scope's own outbox (#1239) — `facetEvents`, which is the
|
|
848
895
|
* sanctioned read: an erased payload yields the same NULL a missing field
|
|
@@ -1163,6 +1210,21 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1163
1210
|
// one call. Same placement as the SQLite adapter's actor task, for this reason.
|
|
1164
1211
|
this.invocationId = invokeOptions?.invocationId ?? null;
|
|
1165
1212
|
try {
|
|
1213
|
+
/**
|
|
1214
|
+
* #938: the outbox's high-water mark BEFORE this call wrote anything, so the
|
|
1215
|
+
* post-commit fan-out can name exactly the events this call (and the consumers
|
|
1216
|
+
* it set off) added. Read here — inside the queued body, before the
|
|
1217
|
+
* transaction — because that is the region where this call holds the DO to
|
|
1218
|
+
* itself, the same reason `invocationId` is set here.
|
|
1219
|
+
*
|
|
1220
|
+
* A socket that connects between here and the fan-out is served whatever this
|
|
1221
|
+
* call committed, and one that connects while `null` was decided hears nothing
|
|
1222
|
+
* about it. Both are harmless and neither is worth a lock: a frame is an
|
|
1223
|
+
* invalidation, so hearing about a change from just before you subscribed costs
|
|
1224
|
+
* one redundant re-read, and missing one costs a wait for the client's poll —
|
|
1225
|
+
* which is the floor this whole surface sits on.
|
|
1226
|
+
*/
|
|
1227
|
+
const liveSince = this.liveHighWaterMark();
|
|
1166
1228
|
let result;
|
|
1167
1229
|
let committedVersion = null;
|
|
1168
1230
|
// #116: set when this invocation was answered from a recording rather
|
|
@@ -1274,7 +1336,10 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1274
1336
|
? { kind: 'system', id: systemModuleId }
|
|
1275
1337
|
: connectionId
|
|
1276
1338
|
? { kind: 'connection', id: connectionId }
|
|
1277
|
-
: { kind: 'principal', id: principal }, tenantId, operation, err,
|
|
1339
|
+
: { kind: 'principal', id: principal }, tenantId, operation, err,
|
|
1340
|
+
// Inside the queued body, which is the one region where this call holds
|
|
1341
|
+
// the DO to itself — so the field is this call's own (#1237).
|
|
1342
|
+
this.invocationId, impersonation);
|
|
1278
1343
|
}
|
|
1279
1344
|
// The ORIGINAL error, deliberately: `invoke` flattens it for the envelope
|
|
1280
1345
|
// (which keeps its code and extensions) or rewraps it for the legacy throw
|
|
@@ -1285,8 +1350,14 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1285
1350
|
// Skipped on a replay: nothing was written, so there is nothing this
|
|
1286
1351
|
// invocation added to drain. Anything the ORIGINAL left undrained is the
|
|
1287
1352
|
// outbox's own retry backstop, which is what that backstop is for.
|
|
1353
|
+
// Drain to consumers, then announce what landed (#938). Skipped whole on a
|
|
1354
|
+
// replay: nothing was written, so there is nothing this invocation added to
|
|
1355
|
+
// drain or to announce. Anything the ORIGINAL left undrained is the outbox's
|
|
1356
|
+
// own retry backstop, which is what that backstop is for.
|
|
1357
|
+
// #1525: still inside the queued body that set it, so these deliveries are this
|
|
1358
|
+
// call's own work — the same tail its consumers' emits are stamped in.
|
|
1288
1359
|
if (!replayed)
|
|
1289
|
-
await this.
|
|
1360
|
+
await this.settleCommitted(tenantId, scopeId, liveSince, this.invocationId);
|
|
1290
1361
|
return {
|
|
1291
1362
|
result,
|
|
1292
1363
|
platformRequests: signals.platformRequests,
|
|
@@ -1319,6 +1390,288 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1319
1390
|
}
|
|
1320
1391
|
});
|
|
1321
1392
|
}
|
|
1393
|
+
// -- live reads (#938): the subscription half of the change feed ------------
|
|
1394
|
+
// The ONLY part of this DO addressed as a fetch target rather than over RPC, and
|
|
1395
|
+
// only because a WebSocket cannot cross RPC — a socket is not serializable, so the
|
|
1396
|
+
// one way to hand one back is a `Response` carrying a `webSocket`. Everything the
|
|
1397
|
+
// coordinator asserts on the way in is named in `live-reads.ts`, shared with
|
|
1398
|
+
// `host.ts` so the two ends cannot drift.
|
|
1399
|
+
/**
|
|
1400
|
+
* Accept a subscription to this scope's changes.
|
|
1401
|
+
*
|
|
1402
|
+
* The coordinator has already decided that this connection can carry a push at all
|
|
1403
|
+
* (the O2O check) and WHO is asking. What is decided here is nothing about
|
|
1404
|
+
* authority: a subscription is not an authorization, and accepting one grants the
|
|
1405
|
+
* subscriber no read it did not already have. Every frame is checked on its way
|
|
1406
|
+
* out, individually, against the tuple state at that moment — so a grant revoked
|
|
1407
|
+
* while the socket is open stops the frames it used to allow, which a
|
|
1408
|
+
* subscription-time check would not.
|
|
1409
|
+
*
|
|
1410
|
+
* **This adds no authority to a holder of the stub, and the question is worth
|
|
1411
|
+
* answering rather than leaving to be asked.** Being a public method, anything with
|
|
1412
|
+
* the `SCOPE` binding can call it and assert whatever principal it likes in the
|
|
1413
|
+
* headers. That is already true of `invoke`, which takes the principal as an
|
|
1414
|
+
* argument and acts on it: a stub is the key to the scope, which is precisely why
|
|
1415
|
+
* the router is not given one. The trust boundary is who holds the binding, not
|
|
1416
|
+
* what this method checks — and what it does NOT do is let a stub-holder read
|
|
1417
|
+
* anything the asserted principal could not, because the filter downstream re-checks
|
|
1418
|
+
* that principal against every frame.
|
|
1419
|
+
*/
|
|
1420
|
+
async fetch(request) {
|
|
1421
|
+
const url = new URL(request.url);
|
|
1422
|
+
if (url.pathname !== LIVE_SUBSCRIBE_PATH) {
|
|
1423
|
+
// The DO has exactly one fetch surface. Anything else reaching here is a
|
|
1424
|
+
// coordinator bug, and a 404 says so without guessing at an intent.
|
|
1425
|
+
return new Response('this scope has no such surface', { status: 404 });
|
|
1426
|
+
}
|
|
1427
|
+
if (!isUpgradeRequest(request)) {
|
|
1428
|
+
return new Response('live reads are a WebSocket surface', {
|
|
1429
|
+
status: 426,
|
|
1430
|
+
headers: { [LIVE_MODE_HEADER]: 'not-an-upgrade' },
|
|
1431
|
+
});
|
|
1432
|
+
}
|
|
1433
|
+
const principal = request.headers.get(LIVE_PRINCIPAL_HEADER);
|
|
1434
|
+
const tenantId = request.headers.get(LIVE_TENANT_HEADER);
|
|
1435
|
+
const scopeId = request.headers.get(LIVE_SCOPE_HEADER);
|
|
1436
|
+
if (!principal || !tenantId || !scopeId) {
|
|
1437
|
+
// Fail closed and loudly. An unnamed subscriber is one whose permissions
|
|
1438
|
+
// cannot be evaluated, and the only safe thing to do with a channel we cannot
|
|
1439
|
+
// filter is to refuse to open it. 500, not 400: the caller is the coordinator
|
|
1440
|
+
// in this same package, so a missing assertion is OUR bug, not the client's.
|
|
1441
|
+
return new Response('live reads require an asserted principal, tenant and scope', {
|
|
1442
|
+
status: 500,
|
|
1443
|
+
});
|
|
1444
|
+
}
|
|
1445
|
+
// A subscriber arriving before the scope's migrations have run would be told
|
|
1446
|
+
// about events against a schema it cannot read back through. Same gate every
|
|
1447
|
+
// other entry point takes, for the same reason.
|
|
1448
|
+
await this.ensureMigrations();
|
|
1449
|
+
const pair = new WebSocketPair();
|
|
1450
|
+
const [client, server] = Object.values(pair);
|
|
1451
|
+
// HIBERNATABLE, not `server.accept()`. A scope with a watcher open would
|
|
1452
|
+
// otherwise be pinned in memory for as long as somebody has a tab open, which is
|
|
1453
|
+
// the cost model inverted: a support desk being WATCHED is the normal state.
|
|
1454
|
+
// Hibernation also fixes the worse half — an in-memory roster does not survive
|
|
1455
|
+
// eviction, so the socket would stay open and silently stop receiving, which is
|
|
1456
|
+
// indistinguishable from a quiet scope.
|
|
1457
|
+
this.ctx.acceptWebSocket(server);
|
|
1458
|
+
// `.parse`, not a cast: these three arrived as header strings, and the branded
|
|
1459
|
+
// ids are what every check downstream is keyed on. A malformed one would
|
|
1460
|
+
// otherwise be carried all the way to a `ctx.check` that quietly matches nothing —
|
|
1461
|
+
// which reads as "this subscriber may see nothing" and is indistinguishable from
|
|
1462
|
+
// a correct denial. Refused here instead, where it is still one subscriber's
|
|
1463
|
+
// problem. Throwing is right: the coordinator built this request.
|
|
1464
|
+
server.serializeAttachment({
|
|
1465
|
+
principal: principalId.parse(principal),
|
|
1466
|
+
tenantId: tenantIdOf.parse(tenantId),
|
|
1467
|
+
scopeId: scopeIdOf.parse(scopeId),
|
|
1468
|
+
since: new Date().toISOString(),
|
|
1469
|
+
});
|
|
1470
|
+
return new Response(null, { status: 101, webSocket: client });
|
|
1471
|
+
}
|
|
1472
|
+
/**
|
|
1473
|
+
* A subscriber said something.
|
|
1474
|
+
*
|
|
1475
|
+
* The channel is one-way by design — the server announces, the client re-reads
|
|
1476
|
+
* through the ordinary operation — so there is no client message that can cause a
|
|
1477
|
+
* read, a write, or a change of subscription. `ping`/`pong` is the whole protocol,
|
|
1478
|
+
* and it exists so an idle connection can be kept alive by either end.
|
|
1479
|
+
*
|
|
1480
|
+
* Deliberately NOT a place to let a client narrow or widen what it receives: a
|
|
1481
|
+
* filter the client chooses is a filter the client can choose wrongly, and the
|
|
1482
|
+
* only filter that matters here is the one it does not control.
|
|
1483
|
+
*/
|
|
1484
|
+
webSocketMessage(ws, message) {
|
|
1485
|
+
if (message === 'ping')
|
|
1486
|
+
ws.send('pong');
|
|
1487
|
+
}
|
|
1488
|
+
/**
|
|
1489
|
+
* The client hung up. Close our end so the runtime stops holding the subscription.
|
|
1490
|
+
*
|
|
1491
|
+
* The runtime passes `reason` and `wasClean` too; neither is read, so neither is
|
|
1492
|
+
* named — there is nothing to do differently for an unclean close, because the
|
|
1493
|
+
* socket is going away either way and the client's poll is what covers the gap.
|
|
1494
|
+
*/
|
|
1495
|
+
webSocketClose(ws, code) {
|
|
1496
|
+
// 1006 is reserved: it is what the runtime REPORTS for an abnormal closure and
|
|
1497
|
+
// is not a code anything may SEND, so echoing it back throws — on precisely the
|
|
1498
|
+
// path where the connection is already in trouble.
|
|
1499
|
+
try {
|
|
1500
|
+
ws.close(code === 1006 ? 1000 : code, 'scope closing the subscription');
|
|
1501
|
+
}
|
|
1502
|
+
catch {
|
|
1503
|
+
// Already gone. Nothing to do, and nothing worth reporting.
|
|
1504
|
+
}
|
|
1505
|
+
}
|
|
1506
|
+
/**
|
|
1507
|
+
* A socket failed. Logged and not rethrown: there is no caller to fail, and the
|
|
1508
|
+
* runtime delivers `webSocketClose` after this, which is what does the cleanup.
|
|
1509
|
+
*/
|
|
1510
|
+
webSocketError(_ws, error) {
|
|
1511
|
+
console.error('substrat: live-read socket error', error);
|
|
1512
|
+
}
|
|
1513
|
+
/**
|
|
1514
|
+
* The outbox's high-water mark before a committing path runs (#938), or `null`
|
|
1515
|
+
* when nobody is listening.
|
|
1516
|
+
*
|
|
1517
|
+
* `null` and `''` are different answers: `null` means no subscriber, so nothing is
|
|
1518
|
+
* read at all and a scope pays nothing for a feature it is not using; `''` is the
|
|
1519
|
+
* honest empty-outbox answer, and every ULID sorts above it.
|
|
1520
|
+
*
|
|
1521
|
+
* Read BEFORE the transaction by every caller, which is what makes the pair below
|
|
1522
|
+
* able to name exactly the events that path added.
|
|
1523
|
+
*/
|
|
1524
|
+
liveHighWaterMark() {
|
|
1525
|
+
if (this.ctx.getWebSockets().length === 0)
|
|
1526
|
+
return null;
|
|
1527
|
+
return (this.sql.exec('SELECT MAX(id) AS id FROM _substrat_outbox').toArray()[0]?.id ?? '');
|
|
1528
|
+
}
|
|
1529
|
+
/**
|
|
1530
|
+
* What every committing path does after its transaction closes: drain the outbox
|
|
1531
|
+
* to consumers, then announce what landed to whoever may see it.
|
|
1532
|
+
*
|
|
1533
|
+
* **One step, because there is one rule.** The fan-out was wired into `invoke`
|
|
1534
|
+
* alone at first, and `attachmentAdd`/`attachmentRemove` commit and emit too —
|
|
1535
|
+
* `attachment.added` and `attachment.removed`, about a real entity. A watcher of
|
|
1536
|
+
* that entity would have missed them: no error, no gap it could see, just a screen
|
|
1537
|
+
* that did not update for one kind of change. That is the exact failure the
|
|
1538
|
+
* hibernation design exists to prevent, arriving through a different door, so the
|
|
1539
|
+
* answer is a step both doors take rather than a second call both must remember.
|
|
1540
|
+
*
|
|
1541
|
+
* Order is load-bearing: drain FIRST, announce after. A consumer's own emits are
|
|
1542
|
+
* changes too, and they land in the outbox above `liveSince` — so announcing first
|
|
1543
|
+
* would tell a subscriber about the cause and not the effect, and it would re-read
|
|
1544
|
+
* too early.
|
|
1545
|
+
*/
|
|
1546
|
+
async settleCommitted(tenantId, scopeId, liveSince,
|
|
1547
|
+
/**
|
|
1548
|
+
* #1525: the call whose commit this is settling, or null — PASSED rather than
|
|
1549
|
+
* read off `this.invocationId` inside `dispatch`, so each of the three doors
|
|
1550
|
+
* says what it actually knows: `invoke` its own id, the two attachment verbs
|
|
1551
|
+
* null. Same discipline `recordDenial` adopted, and for the same reason.
|
|
1552
|
+
*/
|
|
1553
|
+
invocationId) {
|
|
1554
|
+
await this.dispatch(tenantId, scopeId, invocationId);
|
|
1555
|
+
if (liveSince === null)
|
|
1556
|
+
return;
|
|
1557
|
+
try {
|
|
1558
|
+
await this.fanOutLive(liveSince, tenantId, scopeId);
|
|
1559
|
+
}
|
|
1560
|
+
catch (err) {
|
|
1561
|
+
// The write has COMMITTED and the caller is owed its answer. A failure to
|
|
1562
|
+
// announce is a failure of a hint, and the client's poll is the floor
|
|
1563
|
+
// underneath it — so this is logged and never rethrown. Rethrowing would turn
|
|
1564
|
+
// a delivered write into a 500 the caller would reasonably retry.
|
|
1565
|
+
console.error('substrat: live-read fan-out failed after commit', err);
|
|
1566
|
+
}
|
|
1567
|
+
}
|
|
1568
|
+
/**
|
|
1569
|
+
* Announce what just committed, to whoever may see it (#938).
|
|
1570
|
+
*
|
|
1571
|
+
* **Three properties, and each is load-bearing.**
|
|
1572
|
+
*
|
|
1573
|
+
* *Post-commit.* Called after the operation's transaction has closed and after the
|
|
1574
|
+
* consumer drain, so nothing is announced that a rollback could take back and
|
|
1575
|
+
* nothing a consumer emitted is missed. A subscriber told about a row that then
|
|
1576
|
+
* vanished would re-read, find nothing, and have no way to tell that from a
|
|
1577
|
+
* deletion.
|
|
1578
|
+
*
|
|
1579
|
+
* *Filtered per subscriber, per event, after the check.* The declared
|
|
1580
|
+
* `liveTargets` key is checked ON THE EVENT'S OWN ENTITY, through `ctx.check` —
|
|
1581
|
+
* the same evaluator, the same entity-narrowed grants, the same parent walk as the
|
|
1582
|
+
* read the client is about to make. An entity type no module declared is announced
|
|
1583
|
+
* to nobody. Knowing that a row exists and changed at 14:02 is information about
|
|
1584
|
+
* that row, so the empty payload is not what makes this safe; this is.
|
|
1585
|
+
*
|
|
1586
|
+
* *Never able to fail the operation.* The write has committed and the caller has
|
|
1587
|
+
* its answer. A socket that has gone away mid-fan-out, or a check that cannot be
|
|
1588
|
+
* evaluated, costs a subscriber its live update — which it survives, because the
|
|
1589
|
+
* client's contract is that a push is a hint and the poll is the floor.
|
|
1590
|
+
*/
|
|
1591
|
+
async fanOutLive(sinceEventId, tenantId, scopeId) {
|
|
1592
|
+
const sockets = this.ctx.getWebSockets();
|
|
1593
|
+
if (sockets.length === 0)
|
|
1594
|
+
return;
|
|
1595
|
+
const rows = this.sql
|
|
1596
|
+
.exec(`SELECT id, type, entity_type, entity_id, occurred_at FROM _substrat_outbox
|
|
1597
|
+
WHERE id > ? ORDER BY id LIMIT ?`, sinceEventId, LIVE_FANOUT_LIMIT)
|
|
1598
|
+
.toArray();
|
|
1599
|
+
// Drop the undeclared entity types BEFORE any per-subscriber work. Not an
|
|
1600
|
+
// optimisation: it is the fail-closed rule stated once, in the one place that
|
|
1601
|
+
// decides, rather than relied upon inside the loop below.
|
|
1602
|
+
const announceable = rows.filter((r) => this.liveTargets.has(r.entity_type));
|
|
1603
|
+
if (announceable.length === 0)
|
|
1604
|
+
return;
|
|
1605
|
+
for (const ws of sockets) {
|
|
1606
|
+
let subscription = null;
|
|
1607
|
+
try {
|
|
1608
|
+
subscription = readSubscription(ws.deserializeAttachment());
|
|
1609
|
+
}
|
|
1610
|
+
catch {
|
|
1611
|
+
subscription = null;
|
|
1612
|
+
}
|
|
1613
|
+
// A socket we cannot name is a socket we cannot filter for. Skipped, never
|
|
1614
|
+
// sent to — see `readSubscription` for why every unusable shape fails closed.
|
|
1615
|
+
if (!subscription)
|
|
1616
|
+
continue;
|
|
1617
|
+
// A socket that outlived a scope rebind, or was somehow accepted for another
|
|
1618
|
+
// node, must not be fed this scope's events. Cheap, and it makes the identity
|
|
1619
|
+
// the frames are filtered against an explicit precondition rather than an
|
|
1620
|
+
// assumption about how the subscription was created.
|
|
1621
|
+
if (subscription.tenantId !== tenantId || subscription.scopeId !== scopeId)
|
|
1622
|
+
continue;
|
|
1623
|
+
// One context per subscriber, not per event: `ctx.check` is the expensive part
|
|
1624
|
+
// and the context is only the subject it is evaluated for.
|
|
1625
|
+
//
|
|
1626
|
+
// A frame this subscriber does not pass is NOT recorded as a denial (K-35),
|
|
1627
|
+
// and that is deliberate: `recordDenial` is called on a refused REQUEST, where
|
|
1628
|
+
// somebody asked for something and was told no. Nobody asked for these. Logging
|
|
1629
|
+
// one row per unentitled subscriber per event would bury the denials that mean
|
|
1630
|
+
// something — a broken screen, or somebody walking the surface — under the
|
|
1631
|
+
// ordinary, correct working of a filter.
|
|
1632
|
+
//
|
|
1633
|
+
// The operation name is carried anyway, for the events a fan-out cannot emit
|
|
1634
|
+
// but a future reader of this context might.
|
|
1635
|
+
const ctx = this.operationContext(subscription.principal, tenantId, scopeId, undefined, undefined, undefined, undefined, undefined, 'live.subscribe');
|
|
1636
|
+
for (const row of announceable) {
|
|
1637
|
+
// Non-null: `announceable` is exactly the rows whose type is in the map.
|
|
1638
|
+
const permission = this.liveTargets.get(row.entity_type);
|
|
1639
|
+
let allowed = false;
|
|
1640
|
+
try {
|
|
1641
|
+
const decision = await ctx.check(permission, {
|
|
1642
|
+
entityType: row.entity_type,
|
|
1643
|
+
entityId: row.entity_id,
|
|
1644
|
+
});
|
|
1645
|
+
allowed = decision.allowed;
|
|
1646
|
+
}
|
|
1647
|
+
catch {
|
|
1648
|
+
// A check that cannot answer is a check that refuses. The alternative —
|
|
1649
|
+
// treating an evaluator failure as an allow — turns an outage in the
|
|
1650
|
+
// permission path into a disclosure, which is the one failure mode this
|
|
1651
|
+
// surface must not have.
|
|
1652
|
+
allowed = false;
|
|
1653
|
+
}
|
|
1654
|
+
if (!allowed)
|
|
1655
|
+
continue;
|
|
1656
|
+
const frame = {
|
|
1657
|
+
kind: 'change',
|
|
1658
|
+
id: row.id,
|
|
1659
|
+
type: row.type,
|
|
1660
|
+
entityType: row.entity_type,
|
|
1661
|
+
entityId: row.entity_id,
|
|
1662
|
+
at: row.occurred_at,
|
|
1663
|
+
};
|
|
1664
|
+
try {
|
|
1665
|
+
ws.send(JSON.stringify(frame));
|
|
1666
|
+
}
|
|
1667
|
+
catch {
|
|
1668
|
+
// The socket went away between `getWebSockets()` and here. Stop writing to
|
|
1669
|
+
// this one and move on; the runtime will deliver `webSocketClose`.
|
|
1670
|
+
break;
|
|
1671
|
+
}
|
|
1672
|
+
}
|
|
1673
|
+
}
|
|
1674
|
+
}
|
|
1322
1675
|
// -- attachments (#473): the metadata half of the attachment surface --------
|
|
1323
1676
|
// The coordinator (worker) holds the bytes and the per-tenant R2 binding; this
|
|
1324
1677
|
// DO holds the permission gate and the metadata fact, under the same per-scope
|
|
@@ -1359,6 +1712,10 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1359
1712
|
const parsed = attachmentRecord.parse(record);
|
|
1360
1713
|
const gate = this.attachmentGate(parsed.entity.entityType);
|
|
1361
1714
|
return this.queue.enqueue(async () => {
|
|
1715
|
+
// #938: this path commits and emits too, so it takes the same mark-then-settle
|
|
1716
|
+
// pair `invoke` takes. Inside the queued body and before the transaction, for
|
|
1717
|
+
// the reason it is read there: that is where this call holds the DO to itself.
|
|
1718
|
+
const liveSince = this.liveHighWaterMark();
|
|
1362
1719
|
try {
|
|
1363
1720
|
await this.ctx.storage.transaction(async () => {
|
|
1364
1721
|
const ctx = this.operationContext(principal, tenantId, scopeId, undefined, connectionId, undefined, undefined, undefined, 'attachments.upload');
|
|
@@ -1378,11 +1735,13 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1378
1735
|
}
|
|
1379
1736
|
catch (err) {
|
|
1380
1737
|
if (err instanceof PermissionDenied) {
|
|
1381
|
-
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.upload', err);
|
|
1738
|
+
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.upload', err, null);
|
|
1382
1739
|
}
|
|
1383
1740
|
throw toRpcError(err);
|
|
1384
1741
|
}
|
|
1385
|
-
|
|
1742
|
+
// #1525: null. An attachment RPC carries no invocation, exactly as its denial
|
|
1743
|
+
// records none — so the deliveries it drains name no call rather than a wrong one.
|
|
1744
|
+
await this.settleCommitted(tenantId, scopeId, liveSince, null);
|
|
1386
1745
|
return parsed;
|
|
1387
1746
|
});
|
|
1388
1747
|
}
|
|
@@ -1396,7 +1755,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1396
1755
|
}
|
|
1397
1756
|
catch (err) {
|
|
1398
1757
|
if (err instanceof PermissionDenied) {
|
|
1399
|
-
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.list', err);
|
|
1758
|
+
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.list', err, null);
|
|
1400
1759
|
}
|
|
1401
1760
|
throw toRpcError(err);
|
|
1402
1761
|
}
|
|
@@ -1427,7 +1786,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1427
1786
|
}
|
|
1428
1787
|
catch (err) {
|
|
1429
1788
|
if (err instanceof PermissionDenied) {
|
|
1430
|
-
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.open', err);
|
|
1789
|
+
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.open', err, null);
|
|
1431
1790
|
}
|
|
1432
1791
|
throw toRpcError(err);
|
|
1433
1792
|
}
|
|
@@ -1440,7 +1799,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1440
1799
|
}
|
|
1441
1800
|
catch (err) {
|
|
1442
1801
|
if (err instanceof PermissionDenied) {
|
|
1443
|
-
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.open', err);
|
|
1802
|
+
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.open', err, null);
|
|
1444
1803
|
}
|
|
1445
1804
|
throw toRpcError(err);
|
|
1446
1805
|
}
|
|
@@ -1454,6 +1813,8 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1454
1813
|
if (!record)
|
|
1455
1814
|
return null;
|
|
1456
1815
|
const gate = this.attachmentGate(record.entity.entityType);
|
|
1816
|
+
// #938: same mark-then-settle pair as the upload path above.
|
|
1817
|
+
const liveSince = this.liveHighWaterMark();
|
|
1457
1818
|
try {
|
|
1458
1819
|
await this.ctx.storage.transaction(async () => {
|
|
1459
1820
|
const ctx = this.operationContext(principal, tenantId, scopeId, undefined, connectionId, undefined, undefined, undefined, 'attachments.remove');
|
|
@@ -1470,11 +1831,13 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1470
1831
|
}
|
|
1471
1832
|
catch (err) {
|
|
1472
1833
|
if (err instanceof PermissionDenied) {
|
|
1473
|
-
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.remove', err);
|
|
1834
|
+
this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.remove', err, null);
|
|
1474
1835
|
}
|
|
1475
1836
|
throw toRpcError(err);
|
|
1476
1837
|
}
|
|
1477
|
-
|
|
1838
|
+
// #1525: null. An attachment RPC carries no invocation, exactly as its denial
|
|
1839
|
+
// records none — so the deliveries it drains name no call rather than a wrong one.
|
|
1840
|
+
await this.settleCommitted(tenantId, scopeId, liveSince, null);
|
|
1478
1841
|
return record;
|
|
1479
1842
|
});
|
|
1480
1843
|
}
|
|
@@ -1492,10 +1855,17 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1492
1855
|
.toArray()[0];
|
|
1493
1856
|
return row !== undefined;
|
|
1494
1857
|
}
|
|
1495
|
-
/**
|
|
1858
|
+
/**
|
|
1859
|
+
* The last time a schedule's operation ran on this scope (#383), or null.
|
|
1860
|
+
*
|
|
1861
|
+
* `kind = 'schedule'` is not decoration (#1288): an operation may legally be
|
|
1862
|
+
* named `freshness:<something>`, and before the column this read answered with
|
|
1863
|
+
* the EVALUATOR's last recorded time for that event type — a cadence gate
|
|
1864
|
+
* driven by a verdict nothing ran.
|
|
1865
|
+
*/
|
|
1496
1866
|
async scheduleLastRun(operation) {
|
|
1497
1867
|
const row = this.sql
|
|
1498
|
-
.exec(`SELECT last_run_at FROM _substrat_schedule_state WHERE schedule_op = ?`, operation)
|
|
1868
|
+
.exec(`SELECT last_run_at FROM _substrat_schedule_state WHERE kind = 'schedule' AND schedule_op = ?`, operation)
|
|
1499
1869
|
.toArray()[0];
|
|
1500
1870
|
return row?.last_run_at ?? null;
|
|
1501
1871
|
}
|
|
@@ -1503,8 +1873,9 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1503
1873
|
* #1232: everything the freshness evaluator needs about this scope, one round
|
|
1504
1874
|
* trip regardless of how many types are declared — the newest matching event
|
|
1505
1875
|
* per type, plus each type's recorded evaluator state (last recorded at +
|
|
1506
|
-
* outcome, kept in `_substrat_schedule_state` under
|
|
1507
|
-
*
|
|
1876
|
+
* outcome, kept in `_substrat_schedule_state` under `kind = 'freshness'`, which
|
|
1877
|
+
* since #1288 is what separates these rows from the schedule rows beside them —
|
|
1878
|
+
* the `freshness:` prefix on the key is retained but no longer load-bearing).
|
|
1508
1879
|
*/
|
|
1509
1880
|
async freshnessProbe(types) {
|
|
1510
1881
|
const out = {};
|
|
@@ -1521,7 +1892,8 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1521
1892
|
}
|
|
1522
1893
|
const keys = types.map((t) => `freshness:${t}`);
|
|
1523
1894
|
for (const row of this.sql
|
|
1524
|
-
.exec(`SELECT schedule_op, last_run_at, last_status FROM _substrat_schedule_state
|
|
1895
|
+
.exec(`SELECT schedule_op, last_run_at, last_status FROM _substrat_schedule_state
|
|
1896
|
+
WHERE kind = 'freshness' AND schedule_op IN (${marks})`, ...keys)
|
|
1525
1897
|
.toArray()) {
|
|
1526
1898
|
const t = row.schedule_op.slice('freshness:'.length);
|
|
1527
1899
|
if (out[t]) {
|
|
@@ -1533,16 +1905,173 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1533
1905
|
}
|
|
1534
1906
|
/**
|
|
1535
1907
|
* Write one `_substrat_schedule_state` row (#383). Spine, kernel-written.
|
|
1536
|
-
*
|
|
1908
|
+
*
|
|
1909
|
+
* `kind` is PASSED, never derived from the shape of `unit` (#1288) — deriving it
|
|
1910
|
+
* is precisely the convention this column replaced, and it is wrong for the one
|
|
1911
|
+
* input that matters: a schedule operation named `freshness:orders.placed` is a
|
|
1912
|
+
* schedule row, whatever its key looks like. `unit` is that row's key: a schedule
|
|
1537
1913
|
* operation (`module/verb`, and then `at`/`status` are when it ran and how it
|
|
1538
1914
|
* ended) or a freshness key (`freshness:<eventType>`, and then they are when the
|
|
1539
1915
|
* verdict was recorded and what it was — nothing ran). See the bootstrap DDL.
|
|
1916
|
+
*
|
|
1917
|
+
* `kind` is LAST because this is a positional RPC — see the interface's own note
|
|
1918
|
+
* in `host.ts`. An old DO drops a trailing argument; a leading one it binds, and
|
|
1919
|
+
* every value after it lands one column to the left, silently.
|
|
1920
|
+
*/
|
|
1921
|
+
async recordScheduleRun(unit, at, status, kind) {
|
|
1922
|
+
this.sql.exec(`INSERT INTO _substrat_schedule_state (kind, schedule_op, last_run_at, last_status)
|
|
1923
|
+
VALUES (?, ?, ?, ?)
|
|
1924
|
+
ON CONFLICT(kind, schedule_op) DO UPDATE SET last_run_at = excluded.last_run_at,
|
|
1925
|
+
last_status = excluded.last_status`, kind, unit, at, status);
|
|
1926
|
+
}
|
|
1927
|
+
// -- the resumable-run driver's store (#1577) -----------------------------
|
|
1928
|
+
//
|
|
1929
|
+
// D-14's DURABLE driver, and the only half of it that lives in the DO: the run
|
|
1930
|
+
// record and the step ledger. Every decision — what coalescing means, when a
|
|
1931
|
+
// step is skipped, when a run fails — is in the kernel's `JobRunStore` callers,
|
|
1932
|
+
// which the coordinator drives; these are reads and writes and nothing else, so
|
|
1933
|
+
// the two drivers cannot disagree about any of it.
|
|
1934
|
+
//
|
|
1935
|
+
// Each step commits on its OWN round trip rather than the pass batching them at
|
|
1936
|
+
// the end. That is what a mid-pass eviction keeps, and a DO is evicted and
|
|
1937
|
+
// revived constantly — batching would lose exactly the work resume exists for.
|
|
1938
|
+
/**
|
|
1939
|
+
* Coalescing, as ONE round trip: the live (`running`) run for this key, or this
|
|
1940
|
+
* row inserted and returned.
|
|
1941
|
+
*
|
|
1942
|
+
* **The single RPC is the atomicity**, and that is the whole reason it is shaped
|
|
1943
|
+
* this way rather than as a `jobRunLive` the coordinator follows with a
|
|
1944
|
+
* `jobRunInsert`. A Durable Object serializes its RPCs, so the lookup and the
|
|
1945
|
+
* insert cannot be interleaved by another caller; split across two calls they
|
|
1946
|
+
* can, and two concurrent starts then both find nothing and both insert. The
|
|
1947
|
+
* schema carries no unique constraint to catch that, deliberately — a crashed
|
|
1948
|
+
* run must stay restartable — so the indivisibility has to come from here.
|
|
1949
|
+
*/
|
|
1950
|
+
async jobRunStartOrJoin(moduleId, job, instance, row) {
|
|
1951
|
+
const live = this.sql
|
|
1952
|
+
.exec(`SELECT * FROM _substrat_job_runs
|
|
1953
|
+
WHERE module_id = ? AND job = ? AND instance = ? AND status = 'running'
|
|
1954
|
+
ORDER BY id DESC LIMIT 1`, moduleId, job, instance)
|
|
1955
|
+
.toArray()[0] ?? null;
|
|
1956
|
+
if (live)
|
|
1957
|
+
return live;
|
|
1958
|
+
// `transactionSync`, and the two writes inlined rather than reached through
|
|
1959
|
+
// `jobRunInsert`: an `await` between them is an output-gate boundary, and the
|
|
1960
|
+
// point of doing this in one RPC is that there is no boundary to be evicted
|
|
1961
|
+
// at. Same reason `SCHEDULE_STATE_REBUILD` insists on it.
|
|
1962
|
+
this.ctx.storage.transactionSync(() => {
|
|
1963
|
+
this.sql.exec(`INSERT INTO _substrat_job_runs
|
|
1964
|
+
(id, module_id, job, instance, payload, status, cursor, counters, attempts,
|
|
1965
|
+
last_error, started_at, updated_at, next_attempt_at, ended_at)
|
|
1966
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, row.id, row.module_id, row.job, row.instance, row.payload, row.status, row.cursor, row.counters, row.attempts, row.last_error, row.started_at, row.updated_at, row.next_attempt_at, row.ended_at);
|
|
1967
|
+
});
|
|
1968
|
+
return row;
|
|
1969
|
+
}
|
|
1970
|
+
/** One run by id, whatever its status. */
|
|
1971
|
+
async jobRunById(id) {
|
|
1972
|
+
return (this.sql.exec('SELECT * FROM _substrat_job_runs WHERE id = ?', id).toArray()[0] ?? null);
|
|
1973
|
+
}
|
|
1974
|
+
/** Insert a fresh run. The coordinator has already refused a non-queue-safe payload. */
|
|
1975
|
+
async jobRunInsert(row) {
|
|
1976
|
+
this.sql.exec(`INSERT INTO _substrat_job_runs
|
|
1977
|
+
(id, module_id, job, instance, payload, status, cursor, counters, attempts,
|
|
1978
|
+
last_error, started_at, updated_at, next_attempt_at, ended_at)
|
|
1979
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, row.id, row.module_id, row.job, row.instance, row.payload, row.status, row.cursor, row.counters, row.attempts, row.last_error, row.started_at, row.updated_at, row.next_attempt_at, row.ended_at);
|
|
1980
|
+
}
|
|
1981
|
+
/**
|
|
1982
|
+
* `running` runs whose backoff has elapsed, oldest first, after `afterId`.
|
|
1983
|
+
*
|
|
1984
|
+
* The cursor is what lets the coordinator page past runs it cannot drive: it
|
|
1985
|
+
* skips any whose job this deployment does not register, and without a cursor
|
|
1986
|
+
* those rows head every batch forever (`runDueJobRuns`). `afterId` is LAST, as
|
|
1987
|
+
* every argument added to an RPC on this interface must be.
|
|
1540
1988
|
*/
|
|
1541
|
-
async
|
|
1542
|
-
this.sql
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1989
|
+
async jobRunsDue(now, limit, afterId) {
|
|
1990
|
+
return this.sql
|
|
1991
|
+
.exec(
|
|
1992
|
+
// `afterId` bound TWICE rather than as `?2`: mixing anonymous and numbered
|
|
1993
|
+
// parameters makes the anonymous ones resume from the highest index used,
|
|
1994
|
+
// which is a footgun for the next person to add a clause. Spelled exactly
|
|
1995
|
+
// as the pure adapter spells it.
|
|
1996
|
+
`SELECT * FROM _substrat_job_runs
|
|
1997
|
+
WHERE status = 'running' AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
|
|
1998
|
+
AND (? IS NULL OR id > ?)
|
|
1999
|
+
ORDER BY id LIMIT ?`, now, afterId ?? null, afterId ?? null, limit)
|
|
2000
|
+
.toArray();
|
|
2001
|
+
}
|
|
2002
|
+
/** The operator read, newest first. */
|
|
2003
|
+
async jobRunList(filter) {
|
|
2004
|
+
const where = [];
|
|
2005
|
+
const params = [];
|
|
2006
|
+
for (const [column, value] of [
|
|
2007
|
+
['module_id', filter.moduleId],
|
|
2008
|
+
['job', filter.job],
|
|
2009
|
+
['instance', filter.instance],
|
|
2010
|
+
['status', filter.status],
|
|
2011
|
+
]) {
|
|
2012
|
+
if (value !== undefined) {
|
|
2013
|
+
where.push(`${column} = ?`);
|
|
2014
|
+
params.push(value);
|
|
2015
|
+
}
|
|
2016
|
+
}
|
|
2017
|
+
params.push(jobRunListLimit(filter.limit));
|
|
2018
|
+
return this.sql
|
|
2019
|
+
.exec(`SELECT * FROM _substrat_job_runs
|
|
2020
|
+
${where.length ? `WHERE ${where.join(' AND ')}` : ''}
|
|
2021
|
+
ORDER BY id DESC LIMIT ?`, ...params)
|
|
2022
|
+
.toArray();
|
|
2023
|
+
}
|
|
2024
|
+
/**
|
|
2025
|
+
* Write a pass outcome onto the run row — every column at once.
|
|
2026
|
+
*
|
|
2027
|
+
* Whole rather than partial, and that is the shape not an accident: a commit
|
|
2028
|
+
* writes cursor AND counters AND a cleared error AND a cleared backoff, and a
|
|
2029
|
+
* partial update that wrote three of the four would leave a run carrying the
|
|
2030
|
+
* last failure's error beside the new cursor, which reads as broken forever.
|
|
2031
|
+
*/
|
|
2032
|
+
async jobRunPatch(id, patch) {
|
|
2033
|
+
this.sql.exec(`UPDATE _substrat_job_runs
|
|
2034
|
+
SET status = ?, cursor = ?, counters = ?, attempts = ?, last_error = ?,
|
|
2035
|
+
updated_at = ?, next_attempt_at = ?, ended_at = ?
|
|
2036
|
+
WHERE id = ?`, patch.status, patch.cursor, patch.counters, patch.attempts, patch.lastError, patch.updatedAt, patch.nextAttemptAt, patch.endedAt, id);
|
|
2037
|
+
}
|
|
2038
|
+
/** One step's ledger row — a non-null `result` is what means completed. */
|
|
2039
|
+
async jobStepRow(runId, step) {
|
|
2040
|
+
return (this.sql
|
|
2041
|
+
.exec('SELECT step, result, attempts, last_error FROM _substrat_job_steps WHERE run_id = ? AND step = ?', runId, step)
|
|
2042
|
+
.toArray()[0] ?? null);
|
|
2043
|
+
}
|
|
2044
|
+
/** Record one step attempt. `result` non-null = it completed and must not re-run. */
|
|
2045
|
+
async jobStepRecord(runId, step, result, attempts, lastError, at) {
|
|
2046
|
+
this.sql.exec(`INSERT INTO _substrat_job_steps (run_id, step, result, attempts, last_error, recorded_at)
|
|
2047
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
2048
|
+
ON CONFLICT (run_id, step) DO UPDATE SET result = excluded.result,
|
|
2049
|
+
attempts = excluded.attempts,
|
|
2050
|
+
last_error = excluded.last_error,
|
|
2051
|
+
recorded_at = excluded.recorded_at`, runId, step, result, attempts, lastError, at);
|
|
2052
|
+
}
|
|
2053
|
+
/**
|
|
2054
|
+
* A COMMITTED pass: the run's new state and the dropping of its step ledger, in
|
|
2055
|
+
* ONE RPC so they cannot come apart.
|
|
2056
|
+
*
|
|
2057
|
+
* Same reasoning as `jobRunStartOrJoin` — the single round trip is the
|
|
2058
|
+
* atomicity. As two calls, a stop in between leaves the advanced cursor beside
|
|
2059
|
+
* the finished pass's memo rows, and a handler that reuses a step name across
|
|
2060
|
+
* passes (legal: the determinism rule binds names to the payload and prior
|
|
2061
|
+
* results, not to the cursor) then skips work it never did. The kernel's
|
|
2062
|
+
* `runJobPass` carries the full argument.
|
|
2063
|
+
*/
|
|
2064
|
+
async jobCommitPass(id, patch) {
|
|
2065
|
+
// `transactionSync` with both statements inline — NOT `await
|
|
2066
|
+
// this.jobRunPatch(...)` then the delete. The await is an output-gate
|
|
2067
|
+
// boundary, which is precisely the gap this method exists to close.
|
|
2068
|
+
this.ctx.storage.transactionSync(() => {
|
|
2069
|
+
this.sql.exec(`UPDATE _substrat_job_runs
|
|
2070
|
+
SET status = ?, cursor = ?, counters = ?, attempts = ?, last_error = ?,
|
|
2071
|
+
updated_at = ?, next_attempt_at = ?, ended_at = ?
|
|
2072
|
+
WHERE id = ?`, patch.status, patch.cursor, patch.counters, patch.attempts, patch.lastError, patch.updatedAt, patch.nextAttemptAt, patch.endedAt, id);
|
|
2073
|
+
this.sql.exec('DELETE FROM _substrat_job_steps WHERE run_id = ?', id);
|
|
2074
|
+
});
|
|
1546
2075
|
}
|
|
1547
2076
|
// -- guards (K-17) --------------------------------------------------------
|
|
1548
2077
|
/** This scope's spine, read under whatever transaction the caller already opened. */
|
|
@@ -1665,20 +2194,44 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1665
2194
|
* Written AFTER the effect, so a crash mid-effect retries rather than silently
|
|
1666
2195
|
* marking success. The coordinator computes the backoff because it owns the
|
|
1667
2196
|
* per-executor policy; the DO owns the state.
|
|
2197
|
+
*
|
|
2198
|
+
* `invocationId` is LAST because this is a positional RPC — the same rule
|
|
2199
|
+
* `recordScheduleRun` states. An old DO drops a trailing argument; a leading one
|
|
2200
|
+
* it binds, and every value after it lands one column to the left, silently.
|
|
2201
|
+
*/
|
|
2202
|
+
recordExecutorAttempt(eventId, deliveryId, error, nextAttemptAt,
|
|
2203
|
+
/**
|
|
2204
|
+
* #1525: the call THIS attempt ran in, or null. Passed rather than read off
|
|
2205
|
+
* `this.invocationId`, and here there is no choice about it: executors run on
|
|
2206
|
+
* the COORDINATOR, so by the time this RPC arrives the queued body that held
|
|
2207
|
+
* the id has long returned and the field reads null. The coordinator is the
|
|
2208
|
+
* only side that knows, which is why the SQLite twin passes it too — a recorded
|
|
2209
|
+
* fact whose correctness argument differs per adapter is the kind that drifts.
|
|
2210
|
+
*
|
|
2211
|
+
* Defaulted, for the other half of the skew: a coordinator too old to pass it
|
|
2212
|
+
* calls this with four arguments, and `undefined` is not a value SQLite binds.
|
|
2213
|
+
* Null is the honest answer there anyway — that coordinator recorded no call.
|
|
1668
2214
|
*/
|
|
1669
|
-
|
|
2215
|
+
invocationId = null) {
|
|
1670
2216
|
const prior = this.sql
|
|
1671
2217
|
.exec('SELECT attempts FROM _substrat_deliveries WHERE event_id = ? AND consumer_module = ?', eventId, deliveryId)
|
|
1672
2218
|
.toArray()[0];
|
|
1673
2219
|
const attempts = (prior?.attempts ?? 0) + 1;
|
|
1674
2220
|
this.sql.exec(`INSERT INTO _substrat_deliveries
|
|
1675
|
-
(event_id, consumer_module, delivered_at, error, attempts, next_attempt_at
|
|
1676
|
-
|
|
2221
|
+
(event_id, consumer_module, delivered_at, error, attempts, next_attempt_at,
|
|
2222
|
+
invocation_id)
|
|
2223
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
1677
2224
|
ON CONFLICT (event_id, consumer_module) DO UPDATE SET
|
|
1678
2225
|
delivered_at = excluded.delivered_at,
|
|
1679
2226
|
error = excluded.error,
|
|
1680
2227
|
attempts = excluded.attempts,
|
|
1681
|
-
next_attempt_at = excluded.next_attempt_at
|
|
2228
|
+
next_attempt_at = excluded.next_attempt_at,
|
|
2229
|
+
-- #1525: overwritten, like the four above. The row describes the LATEST
|
|
2230
|
+
-- attempt, so a retry drained by a different call (or by none) must not
|
|
2231
|
+
-- keep claiming the call that made the first one.
|
|
2232
|
+
invocation_id = excluded.invocation_id`, eventId, deliveryId, new Date().toISOString(), error, attempts, nextAttemptAt,
|
|
2233
|
+
// #1525: the call this attempt ran in, as the coordinator named it.
|
|
2234
|
+
invocationId);
|
|
1682
2235
|
return attempts;
|
|
1683
2236
|
}
|
|
1684
2237
|
/** How many attempts this delivery has already had — the backoff input. */
|
|
@@ -1736,8 +2289,18 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1736
2289
|
* the event on the next drain). Backpressure throws BEFORE any write — the caller
|
|
1737
2290
|
* records a failed attempt and the delivery retries on its own backoff, exactly as
|
|
1738
2291
|
* a throwing handler would.
|
|
2292
|
+
*
|
|
2293
|
+
* `invocationId` is LAST, and defaulted, for the RPC-skew reason `recordExecutorAttempt`
|
|
2294
|
+
* states at length.
|
|
2295
|
+
*/
|
|
2296
|
+
routeExecutorEventToPlatform(eventId, deliveryId, kind, payload, requestedBy,
|
|
2297
|
+
/**
|
|
2298
|
+
* #1525: the call this routing ran in, or null. On the hosted path this is the
|
|
2299
|
+
* dominant executor journal — every vertical is CP-less, so a connector delivery
|
|
2300
|
+
* becomes an intent here rather than running in `recordExecutorAttempt`'s caller —
|
|
2301
|
+
* so leaving it null would make the column read "no call" for the common case.
|
|
1739
2302
|
*/
|
|
1740
|
-
|
|
2303
|
+
invocationId = null) {
|
|
1741
2304
|
const pending = Number(this.sql
|
|
1742
2305
|
.exec(`SELECT COUNT(*) AS c FROM _substrat_platform_requests WHERE status = 'pending'`)
|
|
1743
2306
|
.toArray()[0].c);
|
|
@@ -1757,7 +2320,7 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1757
2320
|
this.sql.exec(`INSERT INTO _substrat_platform_requests
|
|
1758
2321
|
(id, kind, payload, requested_by, impersonation, status, attempts, requested_at)
|
|
1759
2322
|
VALUES (?, ?, ?, ?, ?, 'pending', 0, ?)`, id, kind, payload, requestedBy, source?.impersonation ?? null, instant.parse(new Date().toISOString()));
|
|
1760
|
-
this.recordExecutorAttempt(eventId, deliveryId, null, null);
|
|
2323
|
+
this.recordExecutorAttempt(eventId, deliveryId, null, null, invocationId);
|
|
1761
2324
|
return id;
|
|
1762
2325
|
}
|
|
1763
2326
|
/**
|
|
@@ -1951,6 +2514,11 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1951
2514
|
// consumer dead-letter.
|
|
1952
2515
|
'ALTER TABLE _substrat_deliveries ADD COLUMN attempts INTEGER NOT NULL DEFAULT 0',
|
|
1953
2516
|
'ALTER TABLE _substrat_deliveries ADD COLUMN next_attempt_at TEXT',
|
|
2517
|
+
// #1525: the invocation an attempt ran in, on a scope DO created before the
|
|
2518
|
+
// column. Nullable, and the null is honestly "no call was carried" — which
|
|
2519
|
+
// attempt produced a delivery already journalled cannot be decided afterwards,
|
|
2520
|
+
// exactly as #1237's outbox column argued.
|
|
2521
|
+
'ALTER TABLE _substrat_deliveries ADD COLUMN invocation_id TEXT',
|
|
1954
2522
|
// K-34: the authorization column on a scope DO created before it existed. Nullable,
|
|
1955
2523
|
// so legacy outbox rows read as "unrecorded". (_substrat_denials is a new table,
|
|
1956
2524
|
// covered by KERNEL_DDL's IF NOT EXISTS with no ALTER.)
|
|
@@ -1966,6 +2534,10 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
1966
2534
|
'ALTER TABLE _substrat_outbox ADD COLUMN impersonation TEXT',
|
|
1967
2535
|
'ALTER TABLE _substrat_platform_requests ADD COLUMN impersonation TEXT',
|
|
1968
2536
|
'ALTER TABLE _substrat_denials ADD COLUMN impersonation TEXT',
|
|
2537
|
+
// #1525: the invocation a refusal happened during, on a scope DO created before
|
|
2538
|
+
// the column. Nullable, and the null is honestly "no id was carried" — a past
|
|
2539
|
+
// denial's call cannot be decided afterwards, as #1237's outbox column argued.
|
|
2540
|
+
'ALTER TABLE _substrat_denials ADD COLUMN invocation_id TEXT',
|
|
1969
2541
|
// #1231: the emitting operation, on a scope DO created before the column.
|
|
1970
2542
|
// Nullable so every legacy row reads as unrecorded rather than named.
|
|
1971
2543
|
'ALTER TABLE _substrat_outbox ADD COLUMN operation TEXT',
|
|
@@ -2001,6 +2573,33 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2001
2573
|
// boot. `lint:spine-ddl` compares KERNEL_DDL's indexes only, so this one is held to
|
|
2002
2574
|
// both adapters by the query-plan test rather than by that gate.
|
|
2003
2575
|
this.sql.exec('CREATE INDEX IF NOT EXISTS _substrat_outbox_invocation ON _substrat_outbox (invocation_id, id)');
|
|
2576
|
+
this.ensureScheduleStateKind();
|
|
2577
|
+
}
|
|
2578
|
+
/**
|
|
2579
|
+
* #1288: `_substrat_schedule_state`, rebuilt with `kind` in its key on a scope DO
|
|
2580
|
+
* whose table predates the column. Not in the ALTER list above, because `kind`
|
|
2581
|
+
* joins the PRIMARY KEY and no ALTER can widen a key — the statements are the
|
|
2582
|
+
* kernel's, so the pure adapter rebuilds byte-identically.
|
|
2583
|
+
*
|
|
2584
|
+
* Detected from `sqlite_master.sql`: DO SQLite restricts `PRAGMA`, and reading
|
|
2585
|
+
* the stored DDL is the one probe both adapters can make.
|
|
2586
|
+
*/
|
|
2587
|
+
ensureScheduleStateKind() {
|
|
2588
|
+
const row = this.sql
|
|
2589
|
+
.exec(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`, '_substrat_schedule_state')
|
|
2590
|
+
.toArray()[0];
|
|
2591
|
+
if (!row || scheduleStateHasKind(row.sql))
|
|
2592
|
+
return;
|
|
2593
|
+
// In a transaction, for the reason the kernel constant spells out: create-copy-
|
|
2594
|
+
// drop-rename has two intermediate states and both are unrecoverable on the next
|
|
2595
|
+
// wake. `transactionSync`, not the async one every operation uses — the DO
|
|
2596
|
+
// runtime forbids a manual BEGIN through `sql.exec`, and this body is wholly
|
|
2597
|
+
// synchronous, which is the one case the sync API is for (it commits at the
|
|
2598
|
+
// first await, and there is none). It also has to be sync because the caller is.
|
|
2599
|
+
this.ctx.storage.transactionSync(() => {
|
|
2600
|
+
for (const stmt of splitSqlStatements(SCHEDULE_STATE_REBUILD))
|
|
2601
|
+
this.sql.exec(stmt);
|
|
2602
|
+
});
|
|
2004
2603
|
}
|
|
2005
2604
|
async importDump(tables, destScopeId) {
|
|
2006
2605
|
// The WHOLE drop-then-replay runs under deferred foreign keys, in one transaction.
|
|
@@ -2200,7 +2799,13 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2200
2799
|
* later alarm-driven drain with a call it had nothing to do with.
|
|
2201
2800
|
*/
|
|
2202
2801
|
invocationId = null;
|
|
2203
|
-
async dispatch(tenantId, scopeId
|
|
2802
|
+
async dispatch(tenantId, scopeId,
|
|
2803
|
+
/**
|
|
2804
|
+
* #1525: the call this drain is running in, or null. Every round of the loop
|
|
2805
|
+
* belongs to it: a consumer's own emit is delivered in the same tail, so the
|
|
2806
|
+
* whole cascade is one call's work.
|
|
2807
|
+
*/
|
|
2808
|
+
invocationId) {
|
|
2204
2809
|
for (let round = 0; round < 50; round++) {
|
|
2205
2810
|
let deliveredAny = false;
|
|
2206
2811
|
for (const mod of this.modules.values()) {
|
|
@@ -2226,16 +2831,18 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2226
2831
|
system: mod.id,
|
|
2227
2832
|
});
|
|
2228
2833
|
await consumer.handler(ctx, event);
|
|
2229
|
-
this.sql.exec(`INSERT INTO _substrat_deliveries
|
|
2230
|
-
|
|
2834
|
+
this.sql.exec(`INSERT INTO _substrat_deliveries
|
|
2835
|
+
(event_id, consumer_module, delivered_at, invocation_id)
|
|
2836
|
+
VALUES (?, ?, ?, ?)`, event.id, mod.id, new Date().toISOString(), invocationId);
|
|
2231
2837
|
});
|
|
2232
2838
|
deliveredAny = true;
|
|
2233
2839
|
}
|
|
2234
2840
|
catch (err) {
|
|
2235
2841
|
// Dead-letter (v0): journal the failure so one poison event
|
|
2236
2842
|
// can't wedge the loop. Written outside the rolled-back txn.
|
|
2237
|
-
this.sql.exec(`INSERT INTO _substrat_deliveries
|
|
2238
|
-
|
|
2843
|
+
this.sql.exec(`INSERT INTO _substrat_deliveries
|
|
2844
|
+
(event_id, consumer_module, delivered_at, error, invocation_id)
|
|
2845
|
+
VALUES (?, ?, ?, ?, ?)`, event.id, mod.id, new Date().toISOString(), String(err), invocationId);
|
|
2239
2846
|
}
|
|
2240
2847
|
finally {
|
|
2241
2848
|
// Cleared on BOTH paths. Left set, the id leaks onto every later emit
|
|
@@ -2257,7 +2864,28 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2257
2864
|
* survives — the whole point, since the denial is the write the operation could not make.
|
|
2258
2865
|
*/
|
|
2259
2866
|
/** K-42: the session a refused call ran under travels with the denial row. */
|
|
2260
|
-
recordDenial(subject, tenantId, operation, err,
|
|
2867
|
+
recordDenial(subject, tenantId, operation, err,
|
|
2868
|
+
/**
|
|
2869
|
+
* #1525: the invocation this refusal belongs to, or null — PASSED, never read off
|
|
2870
|
+
* `this.invocationId` here.
|
|
2871
|
+
*
|
|
2872
|
+
* Reading the ambient field is only self-evidently right where one call holds the
|
|
2873
|
+
* DO to itself, and two denial paths do not: `attachmentList` and
|
|
2874
|
+
* `attachmentAuthorize` run OUTSIDE `this.queue`, and both await before they
|
|
2875
|
+
* record (`ensureMigrations`, `ctx.check`). Whether the input gate can actually
|
|
2876
|
+
* reopen far enough for one of them to observe an in-flight call's id is NOT
|
|
2877
|
+
* settled here — a probe that raced twelve attachment refusals against invokes
|
|
2878
|
+
* holding an id (including one awaiting the control plane) recorded null every
|
|
2879
|
+
* time, so the gate evidently holds more than the shape of the code promises.
|
|
2880
|
+
*
|
|
2881
|
+
* Passed anyway, because the argument for the ambient read is an argument about
|
|
2882
|
+
* workerd's gate semantics, and the argument for a parameter is local: the invoke
|
|
2883
|
+
* path passes its own id, every attachment path passes null, and each says what it
|
|
2884
|
+
* actually knows. That is the property worth having on a recorded fact, and it
|
|
2885
|
+
* costs one argument. No test accompanies it — the condition could not be
|
|
2886
|
+
* reproduced, and a test that passes either way would be worse than none.
|
|
2887
|
+
*/
|
|
2888
|
+
invocationId, impersonation) {
|
|
2261
2889
|
// Only an ENFORCED denial (assertAllowed, which attaches the checked permission +
|
|
2262
2890
|
// node) is recorded. A module's own hand-thrown `new PermissionDenied('…')` carries
|
|
2263
2891
|
// no permission key and is left to the module.
|
|
@@ -2269,8 +2897,11 @@ export function defineScopeDO(modules, bareOps) {
|
|
|
2269
2897
|
? { connection: subject.id }
|
|
2270
2898
|
: subject.id;
|
|
2271
2899
|
this.sql.exec(`INSERT INTO _substrat_denials
|
|
2272
|
-
(id, actor, permission, tenant_id, scope_id, operation, impersonation,
|
|
2273
|
-
|
|
2900
|
+
(id, actor, permission, tenant_id, scope_id, operation, impersonation,
|
|
2901
|
+
invocation_id, at)
|
|
2902
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, ulid(), JSON.stringify(actor), err.permission, err.node.tenantId, err.node.scopeId ?? null, operation, impersonation ? JSON.stringify(impersonationStampOf(impersonation)) : null,
|
|
2903
|
+
// #1525: the call this refusal belongs to, as the caller named it.
|
|
2904
|
+
invocationId, new Date().toISOString());
|
|
2274
2905
|
}
|
|
2275
2906
|
parseOutboxRow(row) {
|
|
2276
2907
|
return domainEvent.parse({
|