@substrat-run/kernel 0.114.0 → 0.117.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/capability.d.ts +220 -0
- package/dist/capability.d.ts.map +1 -0
- package/dist/capability.js +537 -0
- package/dist/capability.js.map +1 -0
- package/dist/check-key.d.ts +30 -0
- package/dist/check-key.d.ts.map +1 -0
- package/dist/check-key.js +37 -0
- package/dist/check-key.js.map +1 -0
- package/dist/denial-query.d.ts +35 -2
- package/dist/denial-query.d.ts.map +1 -1
- package/dist/denial-query.js +70 -28
- package/dist/denial-query.js.map +1 -1
- package/dist/index.d.ts +23 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +16 -4
- package/dist/index.js.map +1 -1
- package/dist/job-run.d.ts +493 -0
- package/dist/job-run.d.ts.map +1 -0
- package/dist/job-run.js +655 -0
- package/dist/job-run.js.map +1 -0
- package/dist/outbox-event.d.ts +131 -0
- package/dist/outbox-event.d.ts.map +1 -0
- package/dist/outbox-event.js +185 -0
- package/dist/outbox-event.js.map +1 -0
- package/dist/permission-checker.d.ts +8 -1
- package/dist/permission-checker.d.ts.map +1 -1
- package/dist/permission-checker.js +18 -0
- package/dist/permission-checker.js.map +1 -1
- package/dist/permission-eval.d.ts +8 -0
- package/dist/permission-eval.d.ts.map +1 -1
- package/dist/permission-eval.js +164 -91
- package/dist/permission-eval.js.map +1 -1
- package/dist/platform-request-query.d.ts +58 -1
- package/dist/platform-request-query.d.ts.map +1 -1
- package/dist/platform-request-query.js +61 -1
- package/dist/platform-request-query.js.map +1 -1
- package/dist/platform-sweep.d.ts +180 -8
- package/dist/platform-sweep.d.ts.map +1 -1
- package/dist/platform-sweep.js +243 -34
- package/dist/platform-sweep.js.map +1 -1
- package/dist/row-decode.d.ts +107 -0
- package/dist/row-decode.d.ts.map +1 -0
- package/dist/row-decode.js +93 -0
- package/dist/row-decode.js.map +1 -0
- package/dist/scope-host.d.ts +444 -9
- package/dist/scope-host.d.ts.map +1 -1
- package/dist/scope-host.js +33 -0
- package/dist/scope-host.js.map +1 -1
- package/dist/scope-tuple-seat.d.ts +72 -0
- package/dist/scope-tuple-seat.d.ts.map +1 -0
- package/dist/scope-tuple-seat.js +93 -0
- package/dist/scope-tuple-seat.js.map +1 -0
- package/dist/subject-redaction.d.ts +160 -0
- package/dist/subject-redaction.d.ts.map +1 -0
- package/dist/subject-redaction.js +210 -0
- package/dist/subject-redaction.js.map +1 -0
- package/dist/system-switch.d.ts +108 -0
- package/dist/system-switch.d.ts.map +1 -0
- package/dist/system-switch.js +145 -0
- package/dist/system-switch.js.map +1 -0
- package/dist/timeline.d.ts.map +1 -1
- package/dist/timeline.js +115 -52
- package/dist/timeline.js.map +1 -1
- package/package.json +2 -2
package/dist/platform-sweep.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { AccessLogEntry, ConnectionId, MigrationProgress, MigrationStraggler, PlatformActorId, ScopeId, TenantId, DrainedEvent } from '@substrat-run/contracts';
|
|
1
|
+
import type { AccessLogEntry, ConnectionId, MigrationProgress, MigrationStraggler, PlatformActorId, Scope, ScopeId, TenantId, DrainedEvent } from '@substrat-run/contracts';
|
|
2
2
|
import type { ExecutorDrainReport, FetchLike, ScopeHost, SweepRunInput } from './scope-host.js';
|
|
3
3
|
/**
|
|
4
4
|
* A connector's reconcile sweep — the unit `runPlatformSweep` calls per live
|
|
@@ -118,11 +118,45 @@ export interface PlatformSweepOptions {
|
|
|
118
118
|
* What makes the phase necessary: a vertical's `onProvision` runs ONCE per scope, at
|
|
119
119
|
* install. Anything the vertical mints for itself there — a service principal, a site
|
|
120
120
|
* registration — therefore never reaches an install that predates it. The new code
|
|
121
|
-
* deploys, and the thing it depends on was never created. Comparing the
|
|
122
|
-
*
|
|
123
|
-
* and this fn is how it fixes it.
|
|
121
|
+
* deploys, and the thing it depends on was never created. Comparing the version whose
|
|
122
|
+
* code RUNS on the scope (`runningVersionOf`, #1653) against the one its provision last
|
|
123
|
+
* ran against is how the platform sees that, and this fn is how it fixes it.
|
|
124
|
+
*
|
|
125
|
+
* `expected` is the version the phase will record on success — the one running on the
|
|
126
|
+
* scope. A fn whose deployment for the scope runs a DIFFERENT version (its resolver fell
|
|
127
|
+
* back from the serving script to the bound version's, say) must throw rather than
|
|
128
|
+
* resolve: the receipt would otherwise name a hook that never ran, and the scope would
|
|
129
|
+
* read as repaired while the code it runs was never provisioned for. A throw leaves it
|
|
130
|
+
* unmarked and counted `failed`, which is where it belongs.
|
|
131
|
+
*
|
|
132
|
+
* Resolves `'unsupported'` when the vertical answered that it implements no reconcile
|
|
133
|
+
* (a 501 from `/internal/reconcile`). That is neither a success nor a failure: no
|
|
134
|
+
* receipt is written, since nothing ran, and it is counted apart from `failed` so a
|
|
135
|
+
* vertical that simply lacks the route does not drown the refusals somebody must read.
|
|
136
|
+
*/
|
|
137
|
+
reconcileScopeFn?: (tenantId: TenantId, scopeId: ScopeId, expected: string) => Promise<void | 'unsupported'>;
|
|
138
|
+
/**
|
|
139
|
+
* The most scopes the #1172 phase reconciles in ONE pass (#1653). Default
|
|
140
|
+
* {@link PROVISION_RECONCILE_BATCH}.
|
|
141
|
+
*
|
|
142
|
+
* A promote of a LISTED vertical puts every install of it behind at once — each one is
|
|
143
|
+
* another tenant's scope, and a popular vertical has thousands. Reconciling them all in
|
|
144
|
+
* one tick is how a cron becomes an incident, so the phase takes at most this many and
|
|
145
|
+
* the rest wait for the next pass (`deferred`). `0` reconciles nothing: the phase still
|
|
146
|
+
* counts who is behind, which makes it the pause switch.
|
|
124
147
|
*/
|
|
125
|
-
|
|
148
|
+
provisionReconcileBatch?: number;
|
|
149
|
+
/**
|
|
150
|
+
* Where the #1172 window starts, as a number in [0, 1) — `Math.random` unless given.
|
|
151
|
+
*
|
|
152
|
+
* The behind scopes are ordered by id and the pass takes a contiguous window of
|
|
153
|
+
* `provisionReconcileBatch` from a random starting point, wrapping. A FIXED start would
|
|
154
|
+
* let installs that fail on every pass hold the same slots forever and starve every
|
|
155
|
+
* healthy one behind them; a random start gives each behind scope a chance of
|
|
156
|
+
* batch/behind per pass however many of the others keep failing. Injectable so that
|
|
157
|
+
* property is tested rather than hoped for.
|
|
158
|
+
*/
|
|
159
|
+
provisionReconcileRng?: () => number;
|
|
126
160
|
/**
|
|
127
161
|
* Also reap expired snapshots (preview-and-snapshots.md §3/§9): any FORK
|
|
128
162
|
* (`forkedFrom` set) whose `expiresAt` has passed is hard-deleted via
|
|
@@ -257,6 +291,63 @@ export interface MigrationSweepReport extends MigrationProgress {
|
|
|
257
291
|
/** Attempts where this host had nothing pending — the scope's modules run in another deployment. */
|
|
258
292
|
noops: number;
|
|
259
293
|
}
|
|
294
|
+
/**
|
|
295
|
+
* Which family a `_substrat_schedule_state` row belongs to (#1288) — the two of
|
|
296
|
+
* `sweepRunKind`'s three names that a SCOPE can hold gating state for. (`connector`
|
|
297
|
+
* is the third and is deliberately absent: a connection is swept host-wide and its
|
|
298
|
+
* state lives nowhere in a scope.)
|
|
299
|
+
*/
|
|
300
|
+
export type ScheduleStateKind = 'schedule' | 'freshness';
|
|
301
|
+
/**
|
|
302
|
+
* The platform sweep's per-scope gating state (#383), as both adapters build it.
|
|
303
|
+
*
|
|
304
|
+
* Shared rather than spelled twice because `lint:spine-ddl` can compare the copies
|
|
305
|
+
* only where they are DDL a `KERNEL_DDL` executes — and this table is also rebuilt
|
|
306
|
+
* by `SCHEDULE_STATE_REBUILD` below, on a store that predates the key. One
|
|
307
|
+
* definition is what keeps the rebuilt shape and the created shape the same shape;
|
|
308
|
+
* the gate then holds each adapter to including it.
|
|
309
|
+
*/
|
|
310
|
+
export declare const SCHEDULE_STATE_DDL = "\n CREATE TABLE IF NOT EXISTS _substrat_schedule_state (\n -- #1288: WHICH of the two families this row belongs to, named with the same two\n -- words _substrat_sweep_runs already records its entries under:\n -- * 'schedule' -- keyed by the operation (module/verb). last_run_at and\n -- last_status are when that operation RAN here and how it ended.\n -- * 'freshness' -- keyed freshness:<eventType> (#1232). last_run_at and\n -- last_status are when the evaluator last RECORDED a verdict for that event\n -- type and what it was. Nothing ran; the row gates what the sweep records.\n kind TEXT NOT NULL,\n schedule_op TEXT NOT NULL,\n last_run_at TEXT,\n last_status TEXT,\n -- The key LEADS with kind, and that is the point of #1288 rather than a tidy-up.\n -- Until it did, the two families were told apart by the spelling of one column:\n -- a freshness key could never LOOK like an operation (an event type has passed\n -- contracts' eventType regex -- lowercase ns.verb, no colon, no slash), but the\n -- other direction was convention only, because scheduleSpec.operation is\n -- z.string().min(1). A module declaring a schedule literally named\n -- \"freshness:orders.placed\" shared the evaluator's row and nothing refused it:\n -- each write clobbered the other's verdict, and the sweep read back whichever\n -- ran last. With kind in the key those are two rows that cannot meet.\n PRIMARY KEY (kind, schedule_op)\n );\n";
|
|
311
|
+
/**
|
|
312
|
+
* `_substrat_schedule_state`, rebuilt with its #1288 key on a store created before
|
|
313
|
+
* it. Create-copy-drop-rename, because `kind` joins the PRIMARY KEY and SQLite
|
|
314
|
+
* cannot widen a key in place — the same shape the directory's `ensureIdentityKey`
|
|
315
|
+
* uses, and detected the same way (from `sqlite_master.sql`, which DO SQLite serves
|
|
316
|
+
* and `PRAGMA` does not, so both adapters migrate by one strategy).
|
|
317
|
+
*
|
|
318
|
+
* The new table is `SCHEDULE_STATE_DDL` under a temporary name, so the rebuilt shape
|
|
319
|
+
* cannot drift from the created one.
|
|
320
|
+
*
|
|
321
|
+
* The backfill derives `kind` from the `freshness:` prefix because that prefix IS how
|
|
322
|
+
* the two families were told apart until now: every row the evaluator ever wrote
|
|
323
|
+
* carries it, and no operation name in existence does. `substr(...) = 'freshness:'`
|
|
324
|
+
* rather than `LIKE`, which is case-insensitive over ASCII in SQLite and would file a
|
|
325
|
+
* schedule named `FRESHNESS:x` under the evaluator.
|
|
326
|
+
*
|
|
327
|
+
* Keys are copied VERBATIM, prefix included. What changes is which rows can coexist,
|
|
328
|
+
* not what any row says — so a deployment rolled back to code that looks a freshness
|
|
329
|
+
* key up under its old name still finds it.
|
|
330
|
+
*
|
|
331
|
+
* **Run these inside the adapter's transaction API**, which is what both callers do —
|
|
332
|
+
* `db.transaction` on the pure side, `ctx.storage.transactionSync` on the DO side.
|
|
333
|
+
* Un-wrapped, a stop between the CREATE and the DROP leaves the scratch table behind
|
|
334
|
+
* and the NEXT wake dies on `table _substrat_schedule_state_new already exists`, which
|
|
335
|
+
* is a scope that cannot open; a stop between the DROP and the RENAME is worse and
|
|
336
|
+
* quieter, because the next wake's `CREATE TABLE IF NOT EXISTS` puts an EMPTY table
|
|
337
|
+
* of the new shape in place, the detection below then reads it as already migrated,
|
|
338
|
+
* and every copied row stays orphaned in the scratch table. Atomically, neither state
|
|
339
|
+
* is reachable — which is why there is no recovery path here to go with them.
|
|
340
|
+
*
|
|
341
|
+
* The leading `DROP TABLE IF EXISTS` is belt to that braces, not the fix: it costs one
|
|
342
|
+
* statement and makes the rebuild idempotent against a scratch table left by anything
|
|
343
|
+
* below the transaction (a torn copy of the file, a restore that carried one).
|
|
344
|
+
*/
|
|
345
|
+
export declare const SCHEDULE_STATE_REBUILD: string;
|
|
346
|
+
/**
|
|
347
|
+
* Whether a store's `_substrat_schedule_state` already carries the #1288 key, read
|
|
348
|
+
* off the `sql` column of `sqlite_master`. `false` means the rebuild is due.
|
|
349
|
+
*/
|
|
350
|
+
export declare function scheduleStateHasKind(tableSql: string): boolean;
|
|
260
351
|
/**
|
|
261
352
|
* What the recurring-schedule phase did in one pass (#383), summed across every
|
|
262
353
|
* module's live scopes. `null` on `PlatformSweepReport` means the phase was
|
|
@@ -275,13 +366,68 @@ export interface ScheduleSweepReport {
|
|
|
275
366
|
}
|
|
276
367
|
/** What the #1172 phase did — see `reconcileScopeFn`. */
|
|
277
368
|
export interface ProvisionReconcileReport {
|
|
278
|
-
/**
|
|
369
|
+
/**
|
|
370
|
+
* Scopes whose running version differed from the one their provision last ran against
|
|
371
|
+
* this pass. Always `reconciled + failed + unsupported + deferred`.
|
|
372
|
+
*/
|
|
279
373
|
behind: number;
|
|
280
374
|
/** Of those, the ones whose reconcile succeeded and were marked. */
|
|
281
375
|
reconciled: number;
|
|
282
376
|
/** Of those, the ones whose reconcile threw. They stay behind and retry next pass. */
|
|
283
377
|
failed: number;
|
|
378
|
+
/**
|
|
379
|
+
* Of those, the ones whose vertical implements no reconcile (a 501). Unmarked like a
|
|
380
|
+
* failure, so they are asked again next pass; counted apart so they cannot pass for one.
|
|
381
|
+
* `scopeIds` is capped at {@link PROVISION_RECONCILE_REPORTED_IDS}; `count` is exact.
|
|
382
|
+
*/
|
|
383
|
+
unsupported: {
|
|
384
|
+
count: number;
|
|
385
|
+
scopeIds: string[];
|
|
386
|
+
};
|
|
387
|
+
/** Of those, the ones past this pass's `provisionReconcileBatch`. Next pass's work. */
|
|
388
|
+
deferred: number;
|
|
389
|
+
}
|
|
390
|
+
/** The #1172 phase's per-pass bound when `provisionReconcileBatch` is unset (#1653). */
|
|
391
|
+
export declare const PROVISION_RECONCILE_BATCH = 50;
|
|
392
|
+
/** How many `unsupported` scope ids one report carries — the count stays exact. */
|
|
393
|
+
export declare const PROVISION_RECONCILE_REPORTED_IDS = 50;
|
|
394
|
+
/**
|
|
395
|
+
* A scope the platform may treat as the REAL install, rather than a copy of one (#1172,
|
|
396
|
+
* #1653). The directory is the only oracle for this, and it takes BOTH tests, not one.
|
|
397
|
+
*
|
|
398
|
+
* A fork is an archive or a preview of somebody else's data, and re-provisioning one
|
|
399
|
+
* runs the vertical's install-side hook against a copy, minting a second set of whatever
|
|
400
|
+
* it mints — or, since #1656, putting a copy of production on a sweeper that reaps and
|
|
401
|
+
* assigns. But a CLEAN-ROOM preview (#509) is an empty scope with no source to copy, so
|
|
402
|
+
* it carries `kind: 'preview'` and NO `forkedFrom` — the reap sweep keys on `kind` for
|
|
403
|
+
* exactly that reason. Filtering on lineage alone would let those through, which is the
|
|
404
|
+
* one shape of scope where the hook's effects are least wanted.
|
|
405
|
+
*/
|
|
406
|
+
export declare function isPrimaryScope(scope: Pick<Scope, 'forkedFrom' | 'kind'>): boolean;
|
|
407
|
+
/** What a vertical's stable serving script runs (#286) — `Vertical.servingRef`/`servingVersionId`. */
|
|
408
|
+
export interface ServingPointer {
|
|
409
|
+
ref: string;
|
|
410
|
+
versionId: string;
|
|
284
411
|
}
|
|
412
|
+
/**
|
|
413
|
+
* The version whose code actually executes for `scope` (#1653) — which is not always the
|
|
414
|
+
* one it is bound to.
|
|
415
|
+
*
|
|
416
|
+
* A scope on its vertical's stable serving script (`servingRef`, #286) runs whatever that
|
|
417
|
+
* script serves, and a promote re-uploads the script in place: the code changes under
|
|
418
|
+
* the scope at that moment, whatever its version pointer says. For a PRIVATE vertical
|
|
419
|
+
* the promote moves the pointer too, so the two agree. For a LISTED one it does not —
|
|
420
|
+
* the pointer is the tenant's, and moves only on their Update — so an install can run
|
|
421
|
+
* version N+1 with its provision state still at N. Measuring against the running version
|
|
422
|
+
* is what lets the reconcile follow the code instead of the pointer.
|
|
423
|
+
*
|
|
424
|
+
* Falls back to the bound version whenever the serving script cannot be named: a scope
|
|
425
|
+
* with no `servingRef` (legacy per-version dispatch, where the bound version's own
|
|
426
|
+
* script IS what runs), a vertical with nothing served in place yet, or a `servingRef`
|
|
427
|
+
* that is not the vertical's current one. The fallback is exactly the pre-#1653 answer,
|
|
428
|
+
* so an unknown never manufactures a reconcile.
|
|
429
|
+
*/
|
|
430
|
+
export declare function runningVersionOf(scope: Pick<Scope, 'verticalVersionId' | 'servingRef'>, serving: ServingPointer | null | undefined): string | null;
|
|
285
431
|
export interface PlatformSweepReport {
|
|
286
432
|
/** Active scopes `drainDue` ran on. */
|
|
287
433
|
scopesDrained: number;
|
|
@@ -300,8 +446,9 @@ export interface PlatformSweepReport {
|
|
|
300
446
|
/** Platform-intent drain outcomes summed across scopes (platform-intents.md). */
|
|
301
447
|
platformRequestTotals: PlatformRequestDrainTotals;
|
|
302
448
|
/**
|
|
303
|
-
* Scopes whose provision was re-run because
|
|
304
|
-
* it last ran against (#1172), or null when no `reconcileScopeFn` was
|
|
449
|
+
* Scopes whose provision was re-run because the version running on them had moved past
|
|
450
|
+
* the one it last ran against (#1172, #1653), or null when no `reconcileScopeFn` was
|
|
451
|
+
* supplied.
|
|
305
452
|
*
|
|
306
453
|
* Null and `{ reconciled: 0 }` are different facts, as everywhere else in this report:
|
|
307
454
|
* the first is "nobody looked".
|
|
@@ -349,6 +496,31 @@ export interface EventDrainReport {
|
|
|
349
496
|
* it. Reported rather than looped, so one busy scope cannot starve the pass.
|
|
350
497
|
*/
|
|
351
498
|
incomplete: number;
|
|
499
|
+
/**
|
|
500
|
+
* Undrained rows a scope's read stepped over because they would not decode (#1636), plus
|
|
501
|
+
* any event this sweep refused itself because the published `drainedEvent` schema did not
|
|
502
|
+
* accept it (#1641 — what an older vertical, which does not validate, can still send),
|
|
503
|
+
* one entry per scope that had any — ABSENT when no scope did.
|
|
504
|
+
*
|
|
505
|
+
* These rows are never shipped and never stamped: the lake is append-only, so a row
|
|
506
|
+
* built from stand-ins could not be taken back, and the stamp is the only record of
|
|
507
|
+
* what left. So they are missing from the lake for as long as they stay undecodable,
|
|
508
|
+
* and every pass reads them again and reports them again. The event is still in
|
|
509
|
+
* Tier 1, where `readHistory` returns it with a `decodeError`.
|
|
510
|
+
*
|
|
511
|
+
* Not an `errors` entry: nothing about this pass failed, and the failure digest mails
|
|
512
|
+
* every error — a standing condition would be re-sent on every tick.
|
|
513
|
+
*/
|
|
514
|
+
skipped?: EventDrainSkipped[];
|
|
515
|
+
}
|
|
516
|
+
/** One scope's share of `EventDrainReport.skipped`. */
|
|
517
|
+
export interface EventDrainSkipped {
|
|
518
|
+
tenantId: TenantId;
|
|
519
|
+
scopeId: ScopeId;
|
|
520
|
+
/** Exact. */
|
|
521
|
+
count: number;
|
|
522
|
+
/** The first of them, oldest first; capped (`UNDRAINED_SKIPPED_IDS`), so `count` may be larger. */
|
|
523
|
+
eventIds: string[];
|
|
352
524
|
}
|
|
353
525
|
export interface AccessLogSweepReport {
|
|
354
526
|
/** Rows handed to the sink and confirmed durable. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"platform-sweep.d.ts","sourceRoot":"","sources":["../src/platform-sweep.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,cAAc,EACd,YAAY,EACZ,iBAAiB,EACjB,kBAAkB,EAClB,eAAe,
|
|
1
|
+
{"version":3,"file":"platform-sweep.d.ts","sourceRoot":"","sources":["../src/platform-sweep.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,cAAc,EACd,YAAY,EACZ,iBAAiB,EACjB,kBAAkB,EAClB,eAAe,EACf,KAAK,EACL,OAAO,EACP,QAAQ,EACR,YAAY,EACb,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAE,mBAAmB,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAWhG;;;;;;;GAOG;AACH,MAAM,WAAW,gBAAgB;IAC/B,CAAC,IAAI,EAAE,SAAS,EAAE,YAAY,EAAE,YAAY,EAAE,IAAI,EAAE;QAAE,KAAK,EAAE,SAAS,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC7F;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,aAAa;IAC5B;;;;OAIG;IACH,IAAI,CAAC,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC3D;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,WAAW,SAAS;IACxB;;;;OAIG;IACH,IAAI,CAAC,KAAK,EAAE;QAAE,QAAQ,EAAE,QAAQ,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACzG;AAQD,MAAM,WAAW,oBAAoB;IACnC;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;IAChD,wFAAwF;IACxF,KAAK,EAAE,eAAe,CAAC;IACvB,0DAA0D;IAC1D,KAAK,EAAE,SAAS,CAAC;IACjB,+FAA+F;IAC/F,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC3C,yEAAyE;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,uBAAuB,CAAC,EAAE,CACxB,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,OAAO,KACb,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjF;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,GAAG,aAAa,CAAC,CAAC;IAC7G;;;;;;;;;OASG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC;;;;;;;;;OASG;IACH,qBAAqB,CAAC,EAAE,MAAM,MAAM,CAAC;IACrC;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3E;;;;;;;;;OASG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtE;;;;;;;;;;OAUG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B;;;;OAIG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,sDAAsD;IACtD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;OAQG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;;;;;;OAQG;IACH,YAAY,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACrD;;;;;;OAMG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;;;;OASG;IACH,gBAAgB,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjE,uEAAuE;IACvE,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC;;;;;;;OAOG;IACH,mBAAmB,CAAC,EAAE,CAAC,OAAO,EAAE,kBAAkB,EAAE,KAAK,IAAI,CAAC;CAC/D;AAED;;;;GAIG;AACH,MAAM,WAAW,oBAAqB,SAAQ,iBAAiB;IAC7D,sEAAsE;IACtE,SAAS,EAAE,MAAM,CAAC;IAClB,oDAAoD;IACpD,QAAQ,EAAE,MAAM,CAAC;IACjB,8EAA8E;IAC9E,QAAQ,EAAE,MAAM,CAAC;IACjB,oGAAoG;IACpG,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;GAKG;AACH,MAAM,MAAM,iBAAiB,GAAG,UAAU,GAAG,WAAW,CAAC;AAEzD;;;;;;;;GAQG;AACH,eAAO,MAAM,kBAAkB,4+CAwB9B,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,eAAO,MAAM,sBAAsB,QAYlC,CAAC;AAEF;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAE9D;AAED;;;;;GAKG;AACH,MAAM,WAAW,mBAAmB;IAClC,kEAAkE;IAClE,MAAM,EAAE,MAAM,CAAC;IACf,sDAAsD;IACtD,KAAK,EAAE,MAAM,CAAC;IACd,uEAAuE;IACvE,OAAO,EAAE,MAAM,CAAC;IAChB,qEAAqE;IACrE,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,yDAAyD;AACzD,MAAM,WAAW,wBAAwB;IACvC;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAC;IACf,oEAAoE;IACpE,UAAU,EAAE,MAAM,CAAC;IACnB,sFAAsF;IACtF,MAAM,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,WAAW,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IACnD,uFAAuF;IACvF,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wFAAwF;AACxF,eAAO,MAAM,yBAAyB,KAAK,CAAC;AAE5C,mFAAmF;AACnF,eAAO,MAAM,gCAAgC,KAAK,CAAC;AAEnD;;;;;;;;;;;GAWG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,YAAY,GAAG,MAAM,CAAC,GAAG,OAAO,CAEjF;AAED,sGAAsG;AACtG,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,mBAAmB,GAAG,YAAY,CAAC,EACtD,OAAO,EAAE,cAAc,GAAG,IAAI,GAAG,SAAS,GACzC,MAAM,GAAG,IAAI,CAGf;AAeD,MAAM,WAAW,mBAAmB;IAClC,uCAAuC;IACvC,aAAa,EAAE,MAAM,CAAC;IACtB,2CAA2C;IAC3C,WAAW,EAAE,mBAAmB,CAAC;IACjC,qCAAqC;IACrC,gBAAgB,EAAE,MAAM,CAAC;IACzB,kFAAkF;IAClF,kBAAkB,EAAE,MAAM,CAAC;IAC3B,0DAA0D;IAC1D,eAAe,EAAE,MAAM,CAAC;IACxB,2EAA2E;IAC3E,oBAAoB,EAAE,MAAM,CAAC;IAC7B,+DAA+D;IAC/D,aAAa,EAAE,MAAM,CAAC;IACtB,iFAAiF;IACjF,qBAAqB,EAAE,0BAA0B,CAAC;IAClD;;;;;;;OAOG;IACH,kBAAkB,EAAE,wBAAwB,GAAG,IAAI,CAAC;IACpD;;;;;;OAMG;IACH,UAAU,EAAE,oBAAoB,GAAG,IAAI,CAAC;IACxC;;;;;OAKG;IACH,SAAS,EAAE,mBAAmB,GAAG,IAAI,CAAC;IACtC;;;;OAIG;IACH,SAAS,EAAE,oBAAoB,GAAG,IAAI,CAAC;IACvC,kFAAkF;IAClF,UAAU,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACpC,oFAAoF;IACpF,MAAM,EAAE;QACN,IAAI,EACA,OAAO,GACP,OAAO,GACP,IAAI,GACJ,MAAM,GACN,aAAa,GACb,SAAS,GACT,kBAAkB,GAClB,qBAAqB,GACrB,UAAU,GACV,WAAW,GACX,YAAY,GAEZ,aAAa,CAAC;QAClB,EAAE,EAAE,MAAM,CAAC;QACX,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED,sEAAsE;AACtE,4DAA4D;AAC5D,MAAM,WAAW,gBAAgB;IAC/B,qEAAqE;IACrE,MAAM,EAAE,MAAM,CAAC;IACf,uDAAuD;IACvD,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,EAAE,iBAAiB,EAAE,CAAC;CAC/B;AAED,uDAAuD;AACvD,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,aAAa;IACb,KAAK,EAAE,MAAM,CAAC;IACd,mGAAmG;IACnG,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,oBAAoB;IACnC,qDAAqD;IACrD,OAAO,EAAE,MAAM,CAAC;IAChB,8EAA8E;IAC9E,MAAM,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,MAAM,EAAE,MAAM,CAAC;IACf,kFAAkF;IAClF,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;CACpB;AAED,sEAAsE;AACtE,MAAM,WAAW,0BAA0B;IACzC,0DAA0D;IAC1D,MAAM,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,OAAO,EAAE,MAAM,CAAC;IAChB,6BAA6B;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,mEAAmE;IACnE,MAAM,EAAE,MAAM,CAAC;IACf,0DAA0D;IAC1D,OAAO,EAAE,MAAM,CAAC;CACjB;AAsBD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,SAAS,EACf,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,mBAAmB,CAAC,CAmiB9B;AA8HD,wFAAwF;AACxF,MAAM,WAAW,qBAAqB;IACpC,IAAI,IAAI,IAAI,CAAC;CACd;AAED,MAAM,WAAW,2BAA4B,SAAQ,oBAAoB;IACvE;;;;OAIG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB,sFAAsF;IACtF,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,mBAAmB,GAAG;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IACpE,0DAA0D;IAC1D,QAAQ,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,IAAI,EAAE,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC;IACnD,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,IAAI,CAAC;CACxC;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,SAAS,EACf,OAAO,EAAE,2BAA2B,GACnC,qBAAqB,CAuBvB"}
|
package/dist/platform-sweep.js
CHANGED
|
@@ -1,10 +1,152 @@
|
|
|
1
|
-
import { instant } from '@substrat-run/contracts';
|
|
1
|
+
import { drainedEvent, instant } from '@substrat-run/contracts';
|
|
2
2
|
import { backoffAt } from './scope-host.js';
|
|
3
3
|
import { MIGRATION_FLAG_THRESHOLD, migrationFleet, migrationProgress, scopeMigrationState } from './migration-progress.js';
|
|
4
|
+
import { UNDRAINED_SKIPPED_IDS } from './outbox-event.js';
|
|
4
5
|
/** How many access rows one pass ships, and one pass prunes, by default. */
|
|
5
6
|
const ACCESS_LOG_BATCH = 500;
|
|
6
7
|
/** How many events one pass drains PER SCOPE by default. */
|
|
7
8
|
const EVENT_DRAIN_BATCH = 200;
|
|
9
|
+
/**
|
|
10
|
+
* The platform sweep's per-scope gating state (#383), as both adapters build it.
|
|
11
|
+
*
|
|
12
|
+
* Shared rather than spelled twice because `lint:spine-ddl` can compare the copies
|
|
13
|
+
* only where they are DDL a `KERNEL_DDL` executes — and this table is also rebuilt
|
|
14
|
+
* by `SCHEDULE_STATE_REBUILD` below, on a store that predates the key. One
|
|
15
|
+
* definition is what keeps the rebuilt shape and the created shape the same shape;
|
|
16
|
+
* the gate then holds each adapter to including it.
|
|
17
|
+
*/
|
|
18
|
+
export const SCHEDULE_STATE_DDL = `
|
|
19
|
+
CREATE TABLE IF NOT EXISTS _substrat_schedule_state (
|
|
20
|
+
-- #1288: WHICH of the two families this row belongs to, named with the same two
|
|
21
|
+
-- words _substrat_sweep_runs already records its entries under:
|
|
22
|
+
-- * 'schedule' -- keyed by the operation (module/verb). last_run_at and
|
|
23
|
+
-- last_status are when that operation RAN here and how it ended.
|
|
24
|
+
-- * 'freshness' -- keyed freshness:<eventType> (#1232). last_run_at and
|
|
25
|
+
-- last_status are when the evaluator last RECORDED a verdict for that event
|
|
26
|
+
-- type and what it was. Nothing ran; the row gates what the sweep records.
|
|
27
|
+
kind TEXT NOT NULL,
|
|
28
|
+
schedule_op TEXT NOT NULL,
|
|
29
|
+
last_run_at TEXT,
|
|
30
|
+
last_status TEXT,
|
|
31
|
+
-- The key LEADS with kind, and that is the point of #1288 rather than a tidy-up.
|
|
32
|
+
-- Until it did, the two families were told apart by the spelling of one column:
|
|
33
|
+
-- a freshness key could never LOOK like an operation (an event type has passed
|
|
34
|
+
-- contracts' eventType regex -- lowercase ns.verb, no colon, no slash), but the
|
|
35
|
+
-- other direction was convention only, because scheduleSpec.operation is
|
|
36
|
+
-- z.string().min(1). A module declaring a schedule literally named
|
|
37
|
+
-- "freshness:orders.placed" shared the evaluator's row and nothing refused it:
|
|
38
|
+
-- each write clobbered the other's verdict, and the sweep read back whichever
|
|
39
|
+
-- ran last. With kind in the key those are two rows that cannot meet.
|
|
40
|
+
PRIMARY KEY (kind, schedule_op)
|
|
41
|
+
);
|
|
42
|
+
`;
|
|
43
|
+
/**
|
|
44
|
+
* `_substrat_schedule_state`, rebuilt with its #1288 key on a store created before
|
|
45
|
+
* it. Create-copy-drop-rename, because `kind` joins the PRIMARY KEY and SQLite
|
|
46
|
+
* cannot widen a key in place — the same shape the directory's `ensureIdentityKey`
|
|
47
|
+
* uses, and detected the same way (from `sqlite_master.sql`, which DO SQLite serves
|
|
48
|
+
* and `PRAGMA` does not, so both adapters migrate by one strategy).
|
|
49
|
+
*
|
|
50
|
+
* The new table is `SCHEDULE_STATE_DDL` under a temporary name, so the rebuilt shape
|
|
51
|
+
* cannot drift from the created one.
|
|
52
|
+
*
|
|
53
|
+
* The backfill derives `kind` from the `freshness:` prefix because that prefix IS how
|
|
54
|
+
* the two families were told apart until now: every row the evaluator ever wrote
|
|
55
|
+
* carries it, and no operation name in existence does. `substr(...) = 'freshness:'`
|
|
56
|
+
* rather than `LIKE`, which is case-insensitive over ASCII in SQLite and would file a
|
|
57
|
+
* schedule named `FRESHNESS:x` under the evaluator.
|
|
58
|
+
*
|
|
59
|
+
* Keys are copied VERBATIM, prefix included. What changes is which rows can coexist,
|
|
60
|
+
* not what any row says — so a deployment rolled back to code that looks a freshness
|
|
61
|
+
* key up under its old name still finds it.
|
|
62
|
+
*
|
|
63
|
+
* **Run these inside the adapter's transaction API**, which is what both callers do —
|
|
64
|
+
* `db.transaction` on the pure side, `ctx.storage.transactionSync` on the DO side.
|
|
65
|
+
* Un-wrapped, a stop between the CREATE and the DROP leaves the scratch table behind
|
|
66
|
+
* and the NEXT wake dies on `table _substrat_schedule_state_new already exists`, which
|
|
67
|
+
* is a scope that cannot open; a stop between the DROP and the RENAME is worse and
|
|
68
|
+
* quieter, because the next wake's `CREATE TABLE IF NOT EXISTS` puts an EMPTY table
|
|
69
|
+
* of the new shape in place, the detection below then reads it as already migrated,
|
|
70
|
+
* and every copied row stays orphaned in the scratch table. Atomically, neither state
|
|
71
|
+
* is reachable — which is why there is no recovery path here to go with them.
|
|
72
|
+
*
|
|
73
|
+
* The leading `DROP TABLE IF EXISTS` is belt to that braces, not the fix: it costs one
|
|
74
|
+
* statement and makes the rebuild idempotent against a scratch table left by anything
|
|
75
|
+
* below the transaction (a torn copy of the file, a restore that carried one).
|
|
76
|
+
*/
|
|
77
|
+
export const SCHEDULE_STATE_REBUILD = `
|
|
78
|
+
DROP TABLE IF EXISTS _substrat_schedule_state_new;
|
|
79
|
+
${SCHEDULE_STATE_DDL.replace('CREATE TABLE IF NOT EXISTS _substrat_schedule_state', 'CREATE TABLE _substrat_schedule_state_new')}
|
|
80
|
+
INSERT INTO _substrat_schedule_state_new (kind, schedule_op, last_run_at, last_status)
|
|
81
|
+
SELECT CASE WHEN substr(schedule_op, 1, 10) = 'freshness:' THEN 'freshness' ELSE 'schedule' END,
|
|
82
|
+
schedule_op, last_run_at, last_status
|
|
83
|
+
FROM _substrat_schedule_state;
|
|
84
|
+
DROP TABLE _substrat_schedule_state;
|
|
85
|
+
ALTER TABLE _substrat_schedule_state_new RENAME TO _substrat_schedule_state;
|
|
86
|
+
`;
|
|
87
|
+
/**
|
|
88
|
+
* Whether a store's `_substrat_schedule_state` already carries the #1288 key, read
|
|
89
|
+
* off the `sql` column of `sqlite_master`. `false` means the rebuild is due.
|
|
90
|
+
*/
|
|
91
|
+
export function scheduleStateHasKind(tableSql) {
|
|
92
|
+
return tableSql.includes('PRIMARY KEY (kind, schedule_op)');
|
|
93
|
+
}
|
|
94
|
+
/** The #1172 phase's per-pass bound when `provisionReconcileBatch` is unset (#1653). */
|
|
95
|
+
export const PROVISION_RECONCILE_BATCH = 50;
|
|
96
|
+
/** How many `unsupported` scope ids one report carries — the count stays exact. */
|
|
97
|
+
export const PROVISION_RECONCILE_REPORTED_IDS = 50;
|
|
98
|
+
/**
|
|
99
|
+
* A scope the platform may treat as the REAL install, rather than a copy of one (#1172,
|
|
100
|
+
* #1653). The directory is the only oracle for this, and it takes BOTH tests, not one.
|
|
101
|
+
*
|
|
102
|
+
* A fork is an archive or a preview of somebody else's data, and re-provisioning one
|
|
103
|
+
* runs the vertical's install-side hook against a copy, minting a second set of whatever
|
|
104
|
+
* it mints — or, since #1656, putting a copy of production on a sweeper that reaps and
|
|
105
|
+
* assigns. But a CLEAN-ROOM preview (#509) is an empty scope with no source to copy, so
|
|
106
|
+
* it carries `kind: 'preview'` and NO `forkedFrom` — the reap sweep keys on `kind` for
|
|
107
|
+
* exactly that reason. Filtering on lineage alone would let those through, which is the
|
|
108
|
+
* one shape of scope where the hook's effects are least wanted.
|
|
109
|
+
*/
|
|
110
|
+
export function isPrimaryScope(scope) {
|
|
111
|
+
return !scope.forkedFrom && scope.kind !== 'preview';
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* The version whose code actually executes for `scope` (#1653) — which is not always the
|
|
115
|
+
* one it is bound to.
|
|
116
|
+
*
|
|
117
|
+
* A scope on its vertical's stable serving script (`servingRef`, #286) runs whatever that
|
|
118
|
+
* script serves, and a promote re-uploads the script in place: the code changes under
|
|
119
|
+
* the scope at that moment, whatever its version pointer says. For a PRIVATE vertical
|
|
120
|
+
* the promote moves the pointer too, so the two agree. For a LISTED one it does not —
|
|
121
|
+
* the pointer is the tenant's, and moves only on their Update — so an install can run
|
|
122
|
+
* version N+1 with its provision state still at N. Measuring against the running version
|
|
123
|
+
* is what lets the reconcile follow the code instead of the pointer.
|
|
124
|
+
*
|
|
125
|
+
* Falls back to the bound version whenever the serving script cannot be named: a scope
|
|
126
|
+
* with no `servingRef` (legacy per-version dispatch, where the bound version's own
|
|
127
|
+
* script IS what runs), a vertical with nothing served in place yet, or a `servingRef`
|
|
128
|
+
* that is not the vertical's current one. The fallback is exactly the pre-#1653 answer,
|
|
129
|
+
* so an unknown never manufactures a reconcile.
|
|
130
|
+
*/
|
|
131
|
+
export function runningVersionOf(scope, serving) {
|
|
132
|
+
if (scope.servingRef && serving && scope.servingRef === serving.ref)
|
|
133
|
+
return serving.versionId;
|
|
134
|
+
return scope.verticalVersionId;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* This pass's share of the behind scopes: at most `batch`, as a contiguous window over
|
|
138
|
+
* the id order, starting at `rng()` of the way round and wrapping. Deterministic for a
|
|
139
|
+
* given `rng`, which is what makes the fairness claim testable.
|
|
140
|
+
*/
|
|
141
|
+
function reconcileWindow(behind, batch, rng) {
|
|
142
|
+
if (behind.length <= batch)
|
|
143
|
+
return [...behind];
|
|
144
|
+
if (batch <= 0)
|
|
145
|
+
return [];
|
|
146
|
+
const ordered = [...behind].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
|
147
|
+
const start = Math.min(ordered.length - 1, Math.max(0, Math.floor(rng() * ordered.length)));
|
|
148
|
+
return Array.from({ length: batch }, (_, i) => ordered[(start + i) % ordered.length]);
|
|
149
|
+
}
|
|
8
150
|
const message = (err) => (err instanceof Error ? err.message : String(err));
|
|
9
151
|
/**
|
|
10
152
|
* Run `fn` over `items` with at most `limit` in flight. `fn` owns its errors —
|
|
@@ -198,53 +340,81 @@ export async function runPlatformSweep(host, options) {
|
|
|
198
340
|
}
|
|
199
341
|
});
|
|
200
342
|
}
|
|
201
|
-
// -- provision reconcile (#1172)
|
|
343
|
+
// -- provision reconcile (#1172, #1653) -------------------------------------
|
|
202
344
|
// A vertical's `onProvision` runs once per scope, at install. So a scope serving code
|
|
203
345
|
// whose provision hook never ran against it is missing whatever that hook creates —
|
|
204
346
|
// and no other path will ever deliver it. This phase is what makes a push repair its
|
|
205
|
-
// own installs:
|
|
347
|
+
// own installs: running version != provisioned version ⇒ reconcile once, then record it.
|
|
348
|
+
//
|
|
349
|
+
// RUNNING, not bound (#1653): a listed vertical's promote re-serves every install in
|
|
350
|
+
// place and moves none of their pointers, so comparing against the pointer never saw
|
|
351
|
+
// those installs at all. See `runningVersionOf`.
|
|
206
352
|
//
|
|
207
353
|
// AFTER the migration phase and skipping what failed there, like every phase that
|
|
208
354
|
// touches a vertical: a fail-closed scope would only re-throw its migration error.
|
|
209
355
|
if (options.reconcileScopeFn) {
|
|
210
356
|
const reconcile = options.reconcileScopeFn;
|
|
211
|
-
const provisionReconcile = {
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
357
|
+
const provisionReconcile = {
|
|
358
|
+
behind: 0,
|
|
359
|
+
reconciled: 0,
|
|
360
|
+
failed: 0,
|
|
361
|
+
unsupported: { count: 0, scopeIds: [] },
|
|
362
|
+
deferred: 0,
|
|
363
|
+
};
|
|
364
|
+
const scopes = (await host.admin.listScopes(options.actor, { status: 'active' })).filter((s) => isPrimaryScope(s) && !failedThisPass.has(s.id));
|
|
365
|
+
// One directory read for every vertical's serving pointer — and only when some scope
|
|
366
|
+
// is actually on a serving script, since nothing else can differ from its binding.
|
|
367
|
+
const serving = new Map();
|
|
368
|
+
if (scopes.some((s) => s.servingRef)) {
|
|
369
|
+
for (const v of await host.admin.listVerticals(options.actor)) {
|
|
370
|
+
if (v.servingRef && v.servingVersionId) {
|
|
371
|
+
serving.set(v.slug, { ref: v.servingRef, versionId: v.servingVersionId });
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
const behind = [];
|
|
376
|
+
for (const s of scopes) {
|
|
377
|
+
const running = runningVersionOf(s, s.vertical ? serving.get(s.vertical) : null);
|
|
378
|
+
// No running version ⇒ nothing to compare, and nothing a reconcile could target.
|
|
379
|
+
if (!running)
|
|
380
|
+
continue;
|
|
229
381
|
// A null receipt is "unknown", NOT "up to date": a scope provisioned before the
|
|
230
382
|
// platform recorded this has no evidence either way, and guessing the optimistic
|
|
231
383
|
// answer leaves exactly the broken installs this phase exists to heal. It costs
|
|
232
384
|
// one reconcile per pre-existing scope, once, and then the receipt is there.
|
|
233
|
-
if (s.provisionedVersionId ===
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
385
|
+
if (s.provisionedVersionId === running)
|
|
386
|
+
continue;
|
|
387
|
+
behind.push({ id: s.id, tenantId: s.tenantId, target: running });
|
|
388
|
+
}
|
|
389
|
+
provisionReconcile.behind = behind.length;
|
|
390
|
+
const batch = options.provisionReconcileBatch ?? PROVISION_RECONCILE_BATCH;
|
|
391
|
+
const window = reconcileWindow(behind, batch, options.provisionReconcileRng ?? Math.random);
|
|
392
|
+
provisionReconcile.deferred = behind.length - window.length;
|
|
393
|
+
await mapBounded(window, concurrency, async (s) => {
|
|
237
394
|
try {
|
|
238
|
-
await reconcile(s.tenantId, s.id);
|
|
395
|
+
const outcome = await reconcile(s.tenantId, s.id, s.target);
|
|
396
|
+
if (outcome === 'unsupported') {
|
|
397
|
+
// Nothing ran, so there is nothing true to record: left unmarked, like a
|
|
398
|
+
// failure, and asked again next pass. Not an error — the vertical answered
|
|
399
|
+
// exactly, and it is not the kind of refusal a person has to act on per pass.
|
|
400
|
+
provisionReconcile.unsupported.count += 1;
|
|
401
|
+
if (provisionReconcile.unsupported.scopeIds.length < PROVISION_RECONCILE_REPORTED_IDS) {
|
|
402
|
+
provisionReconcile.unsupported.scopeIds.push(s.id);
|
|
403
|
+
}
|
|
404
|
+
return;
|
|
405
|
+
}
|
|
239
406
|
// Marked with the version we RECONCILED against, read before the call — not
|
|
240
|
-
// whatever the scope
|
|
241
|
-
//
|
|
242
|
-
|
|
407
|
+
// whatever the scope runs now. A promote that lands mid-pass must not have its
|
|
408
|
+
// new version marked by a reconcile that ran against the old one. The serving
|
|
409
|
+
// pointer is written only after its upload succeeds, so the version read here is
|
|
410
|
+
// one the script was already running when the pass read it.
|
|
411
|
+
await host.admin.markScopeProvisioned(options.actor, s.tenantId, s.id, s.target);
|
|
243
412
|
provisionReconcile.reconciled += 1;
|
|
244
413
|
}
|
|
245
414
|
catch (err) {
|
|
246
415
|
// Left unmarked on purpose: it stays behind and is retried next pass, which is
|
|
247
|
-
// the whole difference between a backstop and a one-shot.
|
|
416
|
+
// the whole difference between a backstop and a one-shot. One scope's failure
|
|
417
|
+
// is its own: the rest of the window carries on.
|
|
248
418
|
provisionReconcile.failed += 1;
|
|
249
419
|
report.errors.push({ kind: 'provision-reconcile', id: s.id, error: message(err) });
|
|
250
420
|
}
|
|
@@ -500,11 +670,14 @@ export async function runPlatformSweep(host, options) {
|
|
|
500
670
|
const drainable = await host.admin.listScopes(options.actor, { status: 'active' });
|
|
501
671
|
await mapBounded(drainable, concurrency, async (s) => {
|
|
502
672
|
try {
|
|
503
|
-
const shipped = await drainScopeEvents(host, options, sink, {
|
|
673
|
+
const { shipped, skipped } = await drainScopeEvents(host, options, sink, {
|
|
504
674
|
tenantId: s.tenantId,
|
|
505
675
|
scopeId: s.id,
|
|
506
676
|
budget,
|
|
507
677
|
});
|
|
678
|
+
if (skipped) {
|
|
679
|
+
(report.eventDrain.skipped ??= []).push({ tenantId: s.tenantId, scopeId: s.id, ...skipped });
|
|
680
|
+
}
|
|
508
681
|
if (shipped === 0)
|
|
509
682
|
return;
|
|
510
683
|
report.eventDrain.scopes += 1;
|
|
@@ -558,14 +731,50 @@ export async function runPlatformSweep(host, options) {
|
|
|
558
731
|
*
|
|
559
732
|
* Nothing is pruned. The outbox is still read by consumers, replay and
|
|
560
733
|
* `readHistory`; what the stamp buys today is knowing what has left.
|
|
734
|
+
*
|
|
735
|
+
* What the read stepped over (#1636) comes back beside the count, for the report —
|
|
736
|
+
* and ONLY for the report: those rows were never in `events`, so nothing here can
|
|
737
|
+
* ship or stamp them. A host too old to say leaves it undefined.
|
|
738
|
+
*
|
|
739
|
+
* **And every event is parsed here, by the published `drainedEvent`, before the sink sees
|
|
740
|
+
* it** (#1641). This is the last point before an append-only lake, and it is the control
|
|
741
|
+
* plane's: a hosted scope's events arrive over HTTP from the vertical's own deployment,
|
|
742
|
+
* decoded by whatever adapter version that vertical was pushed with. A vertical older
|
|
743
|
+
* than #1636 decodes the envelope and copies its lifted columns unvalidated, so without
|
|
744
|
+
* this parse a corrupt one would still ship — the invariant would hold only for verticals
|
|
745
|
+
* new enough to hold it themselves. An event that fails is treated exactly as the read's
|
|
746
|
+
* own skips are: not shipped, not stamped, and folded into `skipped`.
|
|
561
747
|
*/
|
|
562
748
|
async function drainScopeEvents(host, options, sink, input) {
|
|
563
|
-
const
|
|
749
|
+
const read = await host.admin.readUndrainedEvents(options.actor, input.tenantId, input.scopeId, input.budget);
|
|
750
|
+
// A plain array to the sink, of the published schema's own output: nothing typed as a
|
|
751
|
+
// DrainedEvent reaches the lake unless the schema accepted it here.
|
|
752
|
+
const events = [];
|
|
753
|
+
const refused = [];
|
|
754
|
+
for (const event of read) {
|
|
755
|
+
const parsed = drainedEvent.safeParse(event);
|
|
756
|
+
if (parsed.success)
|
|
757
|
+
events.push(parsed.data);
|
|
758
|
+
else
|
|
759
|
+
refused.push(String(event?.id));
|
|
760
|
+
}
|
|
761
|
+
const skipped = skippedOf(read.skipped, refused);
|
|
564
762
|
if (events.length === 0)
|
|
565
|
-
return 0;
|
|
763
|
+
return { shipped: 0, skipped };
|
|
566
764
|
await sink.ship({ tenantId: input.tenantId, scopeId: input.scopeId }, events);
|
|
567
765
|
await host.admin.markEventsDrained(options.actor, input.tenantId, input.scopeId, events.map((e) => e.id));
|
|
568
|
-
return events.length;
|
|
766
|
+
return { shipped: events.length, skipped };
|
|
767
|
+
}
|
|
768
|
+
/**
|
|
769
|
+
* The read's own skip, plus the events this side refused (#1641): one count, exact, and the
|
|
770
|
+
* ids capped as the read caps them. Undefined when there was neither — a clean pass reports
|
|
771
|
+
* nothing rather than a zero.
|
|
772
|
+
*/
|
|
773
|
+
function skippedOf(read, refused) {
|
|
774
|
+
const count = (read?.count ?? 0) + refused.length;
|
|
775
|
+
if (count === 0)
|
|
776
|
+
return undefined;
|
|
777
|
+
return { count, eventIds: [...(read?.eventIds ?? []), ...refused].slice(0, UNDRAINED_SKIPPED_IDS) };
|
|
569
778
|
}
|
|
570
779
|
/**
|
|
571
780
|
* One ship→stamp→prune cycle over the access log. Split out because the ORDER is the
|