@substrat-run/contract-tests 0.114.0 → 0.117.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,35 @@
1
1
  import { describe, it, expect, beforeAll, afterAll } from 'vitest';
2
2
  import { connectorCalls, connectorTestFetch, resetConnectorCalls } from './connector-fixture.js';
3
- import { connectionId, dataSubjectId, eventId, instant, moduleManifest, orgId, AUTO_ADMISSION_NOTE, permissionKey, platformActorId, principalId, scopeId, tenantId, SCOPE_QUERY_ROW_MAX, } from '@substrat-run/contracts';
4
- import { isSearchIndexTable, runPlatformSweep, ulid, } from '@substrat-run/kernel';
3
+ import { connectionId, dataSubjectId, domainEvent, errorCodeOf, eventId, instant, moduleManifest, orgId, AUTO_ADMISSION_NOTE, permissionKey, platformActorId, platformRequestId, principalId, scopeId, tenantId, SCOPE_QUERY_ROW_MAX, } from '@substrat-run/contracts';
4
+ import { isSearchIndexTable, REDACTED_INTENT_MARKER, runPlatformSweep, ulid, } from '@substrat-run/kernel';
5
5
  import { billedMod, contractTestBareOps, contractTestInitialModules, gateModManifest, lateMod, testModManifest, victimModManifest, } from './modules.js';
6
+ /**
7
+ * Assert a refusal by the CODE it declared, not by the sentence it happens to carry
8
+ * (#113 phase 5).
9
+ *
10
+ * `errorCodeOf` is the same reading the control plane's `mapError` does, so what this
11
+ * pins and the problem document a transport renders agree by construction. A `/message/`
12
+ * match agreed with it only by coincidence — and that coincidence was load-bearing:
13
+ * `CODE_PATTERNS` in `packages/control-plane-api/src/errors.ts` guessed the code by
14
+ * regex, so rewording a throw silently changed a 404 into a 500, and the suite's own
15
+ * message assertion was the only thing standing in the way. Moving the assertion here
16
+ * makes the message free to change and the TYPE the thing that cannot.
17
+ *
18
+ * A promise that RESOLVES fails this too: `errorCodeOf(undefined)` is `undefined`, which
19
+ * is never a code — so it cannot pass by not throwing at all.
20
+ *
21
+ * `message` is for the minority of refusals where the sentence carries something the code
22
+ * cannot: WHICH branch fired, when two branches share one code. `deleteVertical` refuses a
23
+ * live scope with "delete or rebind" and an archived one with "reap or restore" — both
24
+ * `conflict`, and an adapter that answered the wrong one would send an operator to a button
25
+ * that is not there. Pass it only for that; a code is the contract everywhere else.
26
+ */
27
+ const expectRefusal = async (p, code, message) => {
28
+ const err = await p.then(() => undefined, (e) => e);
29
+ expect(errorCodeOf(err)).toBe(code);
30
+ if (message !== undefined)
31
+ expect(err?.message).toMatch(message);
32
+ };
6
33
  /**
7
34
  * The scope-host contract suite (design doc §11). Every adapter — pure SQLite,
8
35
  * Cloudflare, and any future one — must pass this unchanged (D-14). If an
@@ -349,6 +376,574 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
349
376
  expect(settled.lastError).toMatch(/personal number field/);
350
377
  expect(settled.payload).toEqual({ contract: 'c1' });
351
378
  });
379
+ /**
380
+ * #1588. Every read of the intent journal returns a LIST, and the row decode behind all
381
+ * three was strict — so ONE row whose JSON would not parse threw out of the map and took
382
+ * the scope's every other intent with it. The drain could not read its own queue, and the
383
+ * history read, which exists so a failed intent explains itself (#618), was switched off
384
+ * by exactly the row that failed.
385
+ *
386
+ * Nothing module code can do produces such a row (`ctx.sql` refuses `_substrat_*` writes,
387
+ * #954). A restore does: `importDump` replays rows verbatim, so a dump from another world
388
+ * or edited by hand is the whole reproduction — the lever #1587's run-driver test uses.
389
+ * The dump is the scope's OWN export with only the journal's rows replaced, so the rest of
390
+ * the spine comes back as it was and the in-scope read can still be invoked.
391
+ */
392
+ describe('one undecodable intent row (#1588)', () => {
393
+ /** Sorts before every ULID, so the pending read (ORDER BY id) meets it FIRST. */
394
+ const brokenPendingId = platformRequestId.parse('00000000000000000000000000');
395
+ const brokenSettledId = platformRequestId.parse('00000000000000000000000001');
396
+ const at = '2026-09-01T00:00:00.000Z';
397
+ const healthyPending = {
398
+ id: ulid(),
399
+ kind: 'connector:test',
400
+ payload: JSON.stringify({ doc: 1 }),
401
+ requested_by: JSON.stringify({ system: 'connector-dispatch' }),
402
+ status: 'pending',
403
+ attempts: 0,
404
+ requested_at: at,
405
+ };
406
+ const healthySettled = {
407
+ id: ulid(),
408
+ kind: 'connector:test',
409
+ payload: JSON.stringify({ doc: 2 }),
410
+ requested_by: JSON.stringify(alice),
411
+ status: 'failed',
412
+ attempts: 1,
413
+ last_error: 'HTTP 409 requires valid personal number field',
414
+ last_failure: JSON.stringify({ origin: 'provider', code: null, permission: null }),
415
+ result: JSON.stringify({ eventId: 'E1' }),
416
+ requested_at: at,
417
+ settled_at: at,
418
+ };
419
+ /** The queue's bad row: a payload nothing could act on. */
420
+ const brokenPending = { ...healthyPending, id: brokenPendingId, payload: 'not json at all' };
421
+ /** The journal's bad row: the failure is on record, beside two columns that are not. */
422
+ const brokenSettled = {
423
+ ...healthySettled,
424
+ id: brokenSettledId,
425
+ last_error: 'HTTP 422 the evidence an operator came here to read',
426
+ last_failure: 'nope',
427
+ result: '{"eventId":',
428
+ };
429
+ /** A fresh scope whose journal holds exactly `rows`, planted by a restore. */
430
+ const restoredWith = async (rows) => {
431
+ const s = scopeId.parse(ulid());
432
+ await host.provisionScope(staff, {
433
+ tenantId: t1,
434
+ scopeId: s,
435
+ jurisdiction: 'eu',
436
+ vertical: 'connector-vertical',
437
+ });
438
+ await host.admin.activateScope(staff, t1, s);
439
+ const backup = await host.admin.exportScope(staff, t1, s);
440
+ const journal = backup.tables.find((t) => t.name === '_substrat_platform_requests');
441
+ // Without the table in the export this test would plant nothing and pass vacuously.
442
+ expect(journal).toBeDefined();
443
+ await host.restoreScope(staff, t1, s, {
444
+ ...backup,
445
+ tables: backup.tables.map((t) => t === journal ? { ...t, rows: rows.map((r) => t.columns.map((c) => r[c] ?? null)) } : t),
446
+ });
447
+ return s;
448
+ };
449
+ /** All three reads, so each of the three decode sites is on the hook. */
450
+ const readAll = async (s) => ({
451
+ pending: await host.listPlatformRequests(t1, s),
452
+ history: await host.listPlatformRequestHistory(t1, s),
453
+ inScope: await (await host.getScope(alice, t1, s)).invoke('platform/intents'),
454
+ });
455
+ const byId = (rows, id) => rows.find((r) => r.id === id);
456
+ it('a clean journal reads exactly as before — no decodeError anywhere (the positive twin)', async () => {
457
+ const s = await restoredWith([healthyPending, healthySettled]);
458
+ const { pending, history, inScope } = await readAll(s);
459
+ expect(pending.map((r) => r.id)).toEqual([healthyPending.id]);
460
+ expect(history.map((r) => r.id).sort()).toEqual([healthyPending.id, healthySettled.id].sort());
461
+ expect(inScope.map((r) => r.id).sort()).toEqual([healthyPending.id, healthySettled.id].sort());
462
+ for (const row of [...pending, ...history, ...inScope]) {
463
+ // ABSENT rather than null, so a healthy list is the shape it always was.
464
+ expect(row).not.toHaveProperty('decodeError');
465
+ }
466
+ expect(byId(history, healthySettled.id)).toMatchObject({
467
+ payload: { doc: 2 },
468
+ requestedBy: alice,
469
+ failure: { origin: 'provider', code: null, permission: null },
470
+ result: { eventId: 'E1' },
471
+ });
472
+ });
473
+ it('returns every other row beside the bad one, and the bad one says why', async () => {
474
+ const clean = await readAll(await restoredWith([healthyPending, healthySettled]));
475
+ const s = await restoredWith([brokenPending, healthyPending, brokenSettled, healthySettled]);
476
+ const { pending, history, inScope } = await readAll(s);
477
+ // The healthy rows are all there, and read EXACTLY as they do with no bad neighbour.
478
+ expect(byId(pending, healthyPending.id)).toEqual(byId(clean.pending, healthyPending.id));
479
+ for (const [read, twin] of [
480
+ [history, clean.history],
481
+ [inScope, clean.inScope],
482
+ ]) {
483
+ expect(read).toHaveLength(4);
484
+ expect(byId(read, healthyPending.id)).toEqual(byId(twin, healthyPending.id));
485
+ expect(byId(read, healthySettled.id)).toEqual(byId(twin, healthySettled.id));
486
+ }
487
+ // The drain's queue: the bad row is listed — it was listed first — and flagged, with
488
+ // an EMPTY payload rather than the raw text, which would read as a string payload.
489
+ expect(pending.map((r) => r.id)).toEqual([brokenPendingId, healthyPending.id]);
490
+ const queued = byId(pending, brokenPendingId);
491
+ expect(queued.decodeError).toMatch(/^payload: .*JSON/);
492
+ expect(queued.payload).toBeNull();
493
+ expect(queued.kind).toBe('connector:test');
494
+ // The journal: the failure the row records is READABLE, which is the whole point of
495
+ // the read — and every column that did not decode is named, not only the first.
496
+ for (const read of [history, inScope]) {
497
+ const settled = byId(read, brokenSettledId);
498
+ expect(settled.lastError).toBe('HTTP 422 the evidence an operator came here to read');
499
+ expect(settled.status).toBe('failed');
500
+ expect(settled.decodeError).toMatch(/^last_failure: .*; result: /);
501
+ // `null` beside a reason, never a silent `null` that reads as "unclassified".
502
+ expect(settled.failure).toBeNull();
503
+ expect(settled.result).toBeNull();
504
+ }
505
+ });
506
+ it('lets the drain settle the bad row failed — out of the queue, the evidence kept', async () => {
507
+ // The drain lives in control-plane-api and refuses any row carrying `decodeError`
508
+ // without running a handler (tested there, against this adapter's pure twin). What
509
+ // BOTH adapters owe it is that the refusal lands: a bad row can be settled like any
510
+ // other, leaves the queue, and still reads back with its reason.
511
+ const s = await restoredWith([brokenPending, healthyPending]);
512
+ const refusal = 'not executed: the intent row could not be decoded (payload: …)';
513
+ await host.settlePlatformRequest(t1, s, brokenPendingId, {
514
+ status: 'failed',
515
+ lastError: refusal,
516
+ failure: { origin: 'platform', code: 'validation_failed', permission: null },
517
+ });
518
+ expect((await host.listPlatformRequests(t1, s)).map((r) => r.id)).toEqual([healthyPending.id]);
519
+ const after = byId(await host.listPlatformRequestHistory(t1, s), brokenPendingId);
520
+ expect(after.status).toBe('failed');
521
+ expect(after.lastError).toBe(refusal);
522
+ expect(after.failure).toEqual({ origin: 'platform', code: 'validation_failed', permission: null });
523
+ // The stored payload is untouched by the settle, so the row still says it is not whole.
524
+ expect(after.decodeError).toMatch(/^payload: /);
525
+ });
526
+ });
527
+ /**
528
+ * #1636 — #1588's shape everywhere else it lived. One outbox or denial row whose JSON
529
+ * would not parse still took a WHOLE list with it: an entity's history and its walks,
530
+ * the denial log and its summary — and, worse, the outbox's executed paths, where the
531
+ * decode sat above each loop's per-event containment, so one bad event halted every
532
+ * event behind it on every pass.
533
+ *
534
+ * Planted the #1588 way: a fresh scope's OWN export, restored with one table's rows
535
+ * rewritten, because `importDump` replays rows verbatim and nothing else can write a
536
+ * spine row that module code did not (#954). The bad row's id sorts before every ULID,
537
+ * so each read and each loop meets it FIRST — the position that used to halt it.
538
+ */
539
+ describe('one undecodable spine row (#1636)', () => {
540
+ /** Sorts before every ULID; `n` keeps several of them distinct and in order. */
541
+ const badId = (n = 0) => `0000000000000000000000${String(n).padStart(4, '0')}`;
542
+ const UNDECODED = { system: 'undecodable' };
543
+ /** A fresh scope, seeded through real operations, and its export. */
544
+ const seeded = async (vertical, seed) => {
545
+ const s = scopeId.parse(ulid());
546
+ await host.provisionScope(staff, { tenantId: t1, scopeId: s, jurisdiction: 'eu', vertical });
547
+ await host.admin.activateScope(staff, t1, s);
548
+ await seed(await host.getScope(alice, t1, s));
549
+ return { s, dump: await host.admin.exportScope(staff, t1, s) };
550
+ };
551
+ const rowsOf = (dump, table) => {
552
+ const t = dump.tables.find((x) => x.name === table);
553
+ // Without the table in the export these tests would plant nothing and pass vacuously.
554
+ expect(t).toBeDefined();
555
+ return t.rows.map((r) => Object.fromEntries(t.columns.map((c, i) => [c, r[i]])));
556
+ };
557
+ /** Restore `s` from `dump` with each named table's rows replaced by `replace[table]`. */
558
+ const restore = async (s, dump, replace) => {
559
+ for (const name of Object.keys(replace))
560
+ expect(dump.tables.some((t) => t.name === name)).toBe(true);
561
+ await host.restoreScope(staff, t1, s, {
562
+ ...dump,
563
+ tables: dump.tables.map((t) => t.name in replace ? { ...t, rows: replace[t.name].map((r) => t.columns.map((c) => r[c] ?? null)) } : t),
564
+ });
565
+ };
566
+ describe('pure reads: the bad row comes back beside its neighbours, saying why', () => {
567
+ it('history, timeline, both walks and an invocation read (the outbox)', async () => {
568
+ const call = ulid();
569
+ const { s, dump } = await seeded('connector-vertical', async (stub) => {
570
+ for (let i = 0; i < 3; i++)
571
+ await stub.invoke('test/emit-event');
572
+ });
573
+ // One call's worth, so `invocationEvents` has something to read.
574
+ const outbox = rowsOf(dump, '_substrat_outbox').map((r) => r.type === 'test.happened' ? { ...r, invocation_id: call } : r);
575
+ const target = outbox.filter((r) => r.type === 'test.happened')[1];
576
+ const targetId = eventId.parse(target.id);
577
+ const readAll = async () => ({
578
+ history: (await host.admin.entityHistory(staff, t1, s, { entityType: 'test-thing', entityId: 'x1' })).entries,
579
+ timeline: (await (await host.getScope(alice, t1, s)).invoke('test/timeline', {
580
+ entityType: 'test-thing',
581
+ entityId: 'x1',
582
+ })).entries,
583
+ cause: await host.admin.eventCause(staff, t1, s, { eventId: targetId }),
584
+ effects: await host.admin.eventEffects(staff, t1, s, { eventId: targetId }),
585
+ call: (await host.admin.invocationEvents(staff, t1, s, { invocationId: call })).events,
586
+ });
587
+ // The positive twin: the scope's own rows, untouched — nothing says it is not whole.
588
+ await restore(s, dump, { _substrat_outbox: outbox });
589
+ const clean = await readAll();
590
+ expect(clean.history).toHaveLength(3);
591
+ for (const entry of [...clean.history, ...clean.timeline, ...clean.cause.chain, ...clean.call]) {
592
+ expect(entry).not.toHaveProperty('decodeError');
593
+ }
594
+ expect(clean.effects.root.event).not.toHaveProperty('decodeError');
595
+ // The same scope with ONE row's actor and payload unreadable.
596
+ await restore(s, dump, {
597
+ _substrat_outbox: outbox.map((r) => (r.id === target.id ? { ...r, actor: '{', payload: 'not json at all' } : r)),
598
+ });
599
+ const planted = await readAll();
600
+ const reason = 'actor: not valid JSON; payload: not valid JSON';
601
+ // Every row comes back, and every healthy one reads EXACTLY as with no bad neighbour.
602
+ expect(planted.history.map((e) => e.id)).toEqual(clean.history.map((e) => e.id));
603
+ planted.history.forEach((entry, i) => {
604
+ if (entry.id === target.id)
605
+ return;
606
+ expect(entry).toEqual(clean.history[i]);
607
+ });
608
+ const bad = planted.history.find((e) => e.id === target.id);
609
+ expect(bad.decodeError).toBe(reason);
610
+ expect(bad.actor).toEqual(UNDECODED);
611
+ // Null beside a reason — the one thing that keeps it from reading as an erasure.
612
+ expect(bad.payload).toBeNull();
613
+ expect(bad.type).toBe('test.happened');
614
+ expect(planted.timeline).toHaveLength(3);
615
+ expect(planted.timeline.find((e) => e.id === target.id)).toMatchObject({
616
+ actor: UNDECODED,
617
+ decodeError: 'actor: not valid JSON',
618
+ });
619
+ // The walks read it rather than ending in an exception, and end where they did.
620
+ expect(planted.cause.chain[0]).toMatchObject({ id: target.id, decodeError: reason });
621
+ expect(planted.cause.terminal).toBe(clean.cause.terminal);
622
+ expect(planted.effects.root.event.decodeError).toBe(reason);
623
+ expect(planted.call.map((e) => e.id)).toEqual(clean.call.map((e) => e.id));
624
+ expect(planted.call.find((e) => e.id === target.id).decodeError).toBe(reason);
625
+ });
626
+ it('the denial log and its summary', async () => {
627
+ const at = '2026-09-01T00:00:00.000Z';
628
+ const { s, dump } = await seeded('connector-vertical', async () => undefined);
629
+ const denial = (id, over = {}) => ({
630
+ id,
631
+ actor: JSON.stringify(alice),
632
+ permission: 'test:read',
633
+ tenant_id: t1,
634
+ scope_id: s,
635
+ operation: 'test/read',
636
+ impersonation: null,
637
+ invocation_id: null,
638
+ at,
639
+ drained_at: null,
640
+ ...over,
641
+ });
642
+ const healthy = [
643
+ denial(ulid()),
644
+ denial(ulid(), { actor: JSON.stringify({ system: '@test/flow' }), permission: 'test:write' }),
645
+ ];
646
+ const readAll = async () => ({
647
+ rows: await host.admin.listDenials(staff, t1, s),
648
+ summary: await host.admin.summarizeDenials(staff, t1, s),
649
+ });
650
+ await restore(s, dump, { _substrat_denials: healthy });
651
+ const clean = await readAll();
652
+ expect(clean.rows).toHaveLength(2);
653
+ for (const row of clean.rows)
654
+ expect(row).not.toHaveProperty('decodeError');
655
+ for (const bucket of clean.summary.buckets)
656
+ expect(bucket).not.toHaveProperty('decodeError');
657
+ const bad = denial(badId(), { actor: '{', impersonation: 'nope' });
658
+ // A key a module CAST rather than declared — reachable from live code, since nothing
659
+ // validates a checked key at runtime. The key is the evidence, so it is kept.
660
+ const badKey = denial(ulid(), { permission: 'Workorder:Read' });
661
+ await restore(s, dump, { _substrat_denials: [bad, badKey, ...healthy] });
662
+ const planted = await readAll();
663
+ expect(planted.rows).toHaveLength(4);
664
+ for (const row of clean.rows)
665
+ expect(planted.rows.find((r) => r.id === row.id)).toEqual(row);
666
+ const refused = planted.rows.find((r) => r.id === bad.id);
667
+ expect(refused.decodeError).toBe('actor: not valid JSON; impersonation: not valid JSON');
668
+ expect(refused.actor).toEqual(UNDECODED);
669
+ // Null beside a reason, never a silent null that reads as "nobody was impersonating".
670
+ expect(refused.impersonation).toBeNull();
671
+ expect(refused.permission).toBe('test:read');
672
+ // The malformed key is listed under the marker, the stored key quoted verbatim.
673
+ const castKey = planted.rows.find((r) => r.id === badKey.id);
674
+ expect(castKey.permission).toBe('undecodable:permission');
675
+ expect(castKey.decodeError).toMatch(/^permission: .* \(stored "Workorder:Read"\)$/);
676
+ expect(castKey.actor).toBe(alice);
677
+ // The summary — "the read a console opens first" — keeps both buckets, counted.
678
+ expect(planted.summary.total).toBe(4);
679
+ expect(planted.summary.groupBy).toBe('actor-permission');
680
+ const buckets = planted.summary.groupBy === 'actor-permission' ? planted.summary.buckets : [];
681
+ expect(buckets).toHaveLength(4);
682
+ expect(buckets.find((b) => b.decodeError === 'actor: not valid JSON')).toMatchObject({
683
+ actor: UNDECODED,
684
+ permission: 'test:read',
685
+ count: 1,
686
+ });
687
+ expect(buckets.find((b) => b.permission === 'undecodable:permission')).toMatchObject({
688
+ actor: alice,
689
+ count: 1,
690
+ decodeError: expect.stringContaining('(stored "Workorder:Read")'),
691
+ });
692
+ for (const bucket of clean.summary.buckets)
693
+ expect(planted.summary.buckets).toContainEqual(bucket);
694
+ });
695
+ });
696
+ describe('the three mappers #1641 left casting (#1643): decoded, or thrown naming the column', () => {
697
+ const at = '2026-09-01T00:00:00.000Z';
698
+ it('dead letters and deliveries: a healthy row reads whole; a broken `attempts` is refused, never typed as valid', async () => {
699
+ const { s, dump } = await seeded('connector-vertical', async (stub) => {
700
+ for (let i = 0; i < 2; i++)
701
+ await stub.invoke('test/emit-event');
702
+ });
703
+ const [first, second] = rowsOf(dump, '_substrat_outbox').filter((r) => r.type === 'test.happened');
704
+ const delivery = (event, over = {}) => ({
705
+ event_id: event.id,
706
+ consumer_module: '@test/doomed',
707
+ delivered_at: at,
708
+ error: 'gave up',
709
+ attempts: 3,
710
+ next_attempt_at: null,
711
+ invocation_id: null,
712
+ ...over,
713
+ });
714
+ const readAll = async (event) => ({
715
+ letters: (await host.admin.deadLetters(staff, t1, s, {})).entries,
716
+ effects: (await host.admin.eventEffects(staff, t1, s, { eventId: eventId.parse(event.id) })).root.deliveries,
717
+ });
718
+ // The positive twin: two dead letters, one of them an EXECUTOR's — whose id is not a
719
+ // module id, and which a decode against a bare `moduleId` would have refused.
720
+ await restore(s, dump, {
721
+ _substrat_deliveries: [delivery(first), delivery(second, { consumer_module: 'executor:mailer' })],
722
+ });
723
+ const clean = await readAll(second);
724
+ expect(clean.letters).toHaveLength(2);
725
+ expect(clean.letters.map((l) => l.consumer).sort()).toEqual(['@test/doomed', 'executor:mailer']);
726
+ for (const l of clean.letters)
727
+ expect(l).not.toHaveProperty('decodeError');
728
+ expect(clean.effects).toMatchObject([{ consumer: 'executor:mailer', state: 'dead', attempts: 3 }]);
729
+ expect(clean.effects[0]).not.toHaveProperty('decodeError');
730
+ // The same rows with one `attempts` that cannot be decoded as a non-negative integer (a
731
+ // negative number, or text). It has no honest empty value, so the read names the column —
732
+ // on both reads, on both adapters.
733
+ for (const attempts of [-1, 'many']) {
734
+ await restore(s, dump, { _substrat_deliveries: [delivery(first), delivery(second, { attempts })] });
735
+ await expect(host.admin.deadLetters(staff, t1, s, {})).rejects.toThrow(/DeadLetter — attempts: /);
736
+ await expect(host.admin.eventEffects(staff, t1, s, { eventId: eventId.parse(second.id) })).rejects.toThrow(/valid EventDelivery — attempts: /);
737
+ }
738
+ // …and the row that is fine is still fine once the bad one is repaired.
739
+ await restore(s, dump, { _substrat_deliveries: [delivery(first), delivery(second, { attempts: 2 })] });
740
+ expect((await readAll(second)).letters.map((l) => l.attempts).sort()).toEqual([2, 3]);
741
+ });
742
+ it('an executor delivery is read whatever id registration let through — the reader matches the writer', async () => {
743
+ // `registerExecutor` accepts any string and both adapters persist `executor:${id}`, so the
744
+ // kernel can itself write an empty id or one with a newline. Refusing those would throw
745
+ // the whole page on a delivery the kernel wrote.
746
+ const { s, dump } = await seeded('connector-vertical', (stub) => stub.invoke('test/emit-event'));
747
+ const [event] = rowsOf(dump, '_substrat_outbox').filter((r) => r.type === 'test.happened');
748
+ const consumers = ['executor:mailer', 'executor:', 'executor:line\nbreak', 'executor:a b/ç'];
749
+ await restore(s, dump, {
750
+ _substrat_deliveries: consumers.map((consumer_module) => ({
751
+ event_id: event.id,
752
+ consumer_module,
753
+ delivered_at: at,
754
+ error: 'gave up',
755
+ attempts: 3,
756
+ next_attempt_at: null,
757
+ invocation_id: null,
758
+ })),
759
+ });
760
+ const letters = (await host.admin.deadLetters(staff, t1, s, {})).entries;
761
+ expect(letters.map((l) => l.consumer).sort()).toEqual([...consumers].sort());
762
+ const effects = (await host.admin.eventEffects(staff, t1, s, { eventId: eventId.parse(event.id) })).root.deliveries;
763
+ expect(effects.map((d) => d.consumer).sort()).toEqual([...consumers].sort());
764
+ for (const row of [...letters, ...effects])
765
+ expect(row).not.toHaveProperty('decodeError');
766
+ });
767
+ it('the operation summary: healthy buckets read whole; a broken time is refused naming its column', async () => {
768
+ const { s, dump } = await seeded('connector-vertical', async () => undefined);
769
+ const denial = (over = {}) => ({
770
+ id: ulid(),
771
+ actor: JSON.stringify(alice),
772
+ permission: 'test:read',
773
+ tenant_id: t1,
774
+ scope_id: s,
775
+ operation: 'test/read',
776
+ impersonation: null,
777
+ invocation_id: null,
778
+ at,
779
+ drained_at: null,
780
+ ...over,
781
+ });
782
+ const summarize = () => host.admin.summarizeDenials(staff, t1, s, { groupBy: 'operation' });
783
+ await restore(s, dump, {
784
+ _substrat_denials: [denial(), denial(), denial({ operation: 'test/write' }), denial({ operation: null })],
785
+ });
786
+ const clean = await summarize();
787
+ expect(clean.groupBy).toBe('operation');
788
+ expect(clean.buckets).toHaveLength(3);
789
+ const byOp = (op) => clean.buckets.find((b) => b.operation === op);
790
+ expect(byOp('test/read')).toMatchObject({ count: 2, firstAt: at, lastAt: at });
791
+ expect(byOp('test/write')).toMatchObject({ count: 1 });
792
+ // The refusal outside any operation is a bucket of its own — and that null is a fact.
793
+ expect(byOp(null)).toMatchObject({ count: 1 });
794
+ for (const b of clean.buckets)
795
+ expect(b).not.toHaveProperty('decodeError');
796
+ // One denial with an empty time: the bucket it lands in cannot state when it began.
797
+ await restore(s, dump, {
798
+ _substrat_denials: [denial({ operation: 'test/write', at: '' }), denial()],
799
+ });
800
+ await expect(summarize()).rejects.toThrow(/DenialOperationBucket — first_at: /);
801
+ });
802
+ });
803
+ describe('executed rows: the bad event is dead-lettered, and the one behind it runs', () => {
804
+ it('consumer delivery', async () => {
805
+ const { s, dump } = await seeded('flow-vertical', (stub) => stub.invoke('flow/produce'));
806
+ const step1 = rowsOf(dump, '_substrat_outbox').find((r) => r.type === 'flow.step1');
807
+ const healthy = { ...step1, id: ulid() };
808
+ const deliveredIds = async () => {
809
+ const stub = await host.getScope(alice, t1, s);
810
+ // Any invoke drains the scope's consumers in its tail.
811
+ await stub.invoke('flow/produce');
812
+ return {
813
+ log: (await stub.invoke('flow/log')).map((r) => r.event_id),
814
+ deliveries: await stub.invoke('flow/deliveries'),
815
+ };
816
+ };
817
+ // Undelivered, the both of them: no deliveries and no consumer-side rows.
818
+ const fresh = { _substrat_deliveries: [], flow_log: [] };
819
+ await restore(s, dump, { ...fresh, _substrat_outbox: [healthy] });
820
+ const clean = await deliveredIds();
821
+ expect(clean.log).toContain(healthy.id);
822
+ expect(clean.deliveries.every((d) => d.error === null)).toBe(true);
823
+ await restore(s, dump, { ...fresh, _substrat_outbox: [{ ...step1, id: badId(), actor: '{' }, healthy] });
824
+ const planted = await deliveredIds();
825
+ // The event BEHIND the bad one is delivered — the handler ran on it — and the bad
826
+ // one never reached the handler at all.
827
+ expect(planted.log).toContain(healthy.id);
828
+ expect(planted.log).not.toContain(badId());
829
+ const dead = planted.deliveries.find((d) => d.event_id === badId());
830
+ expect(dead.error).toMatch(/cannot be decoded — actor: not valid JSON/);
831
+ expect(planted.deliveries.find((d) => d.event_id === healthy.id).error).toBeNull();
832
+ // …and it is a dead letter like any other, readable where operators look.
833
+ const letters = await host.admin.deadLetters(staff, t1, s, {});
834
+ expect(letters.entries.find((d) => d.eventId === badId())).toMatchObject({ consumer: '@test/flow' });
835
+ });
836
+ it('executor dispatch', async () => {
837
+ const tag = `after-bad-${ulid()}`;
838
+ const { s, dump } = await seeded('connector-vertical', (stub) => stub.invoke('connector/request-effect', { tag: `seed-${ulid()}` }));
839
+ const template = rowsOf(dump, '_substrat_outbox').find((r) => r.type === 'effect.requested');
840
+ const healthy = { ...template, id: ulid(), payload: JSON.stringify({ tag }) };
841
+ const fresh = { _substrat_deliveries: [] };
842
+ // The positive twin: one healthy event, delivered, nothing dead.
843
+ await restore(s, dump, { ...fresh, _substrat_outbox: [healthy] });
844
+ const cleanReport = await host.drainDue(t1, s);
845
+ expect(cleanReport).toMatchObject({ attempted: 1, delivered: 1, deadLettered: 0 });
846
+ expect(effected).toContain(tag);
847
+ const next = `after-bad-${ulid()}`;
848
+ const behind = { ...template, id: ulid(), payload: JSON.stringify({ tag: next }) };
849
+ await restore(s, dump, {
850
+ ...fresh,
851
+ _substrat_outbox: [{ ...template, id: badId(), payload: 'not json at all' }, behind],
852
+ });
853
+ const report = await host.drainDue(t1, s);
854
+ // Both attempted; the bad one dead at once — `flaky-effector` allows five attempts,
855
+ // and a decode that failed once can never succeed.
856
+ expect(report).toMatchObject({ attempted: 2, delivered: 1, deadLettered: 1, retrying: 0 });
857
+ expect(effected).toContain(next);
858
+ const dead = (await host.admin.deadLetters(staff, t1, s, {})).entries.find((d) => d.eventId === badId());
859
+ expect(dead).toMatchObject({ consumer: 'executor:flaky-effector', attempts: 1 });
860
+ expect(dead.error).toMatch(/cannot be decoded — payload: not valid JSON/);
861
+ // Terminal: the next pass does not attempt it again.
862
+ expect(await host.drainDue(t1, s)).toMatchObject({ attempted: 0 });
863
+ });
864
+ it('the Tier-2 read steps over it, ships nothing built from stand-ins, and says so', async () => {
865
+ const { s, dump } = await seeded('connector-vertical', async (stub) => {
866
+ await stub.invoke('test/emit-event');
867
+ await stub.invoke('test/emit-event');
868
+ });
869
+ const rows = rowsOf(dump, '_substrat_outbox');
870
+ const events = rows.filter((r) => r.type === 'test.happened');
871
+ // The positive twin: a clean read carries no `skipped` at all.
872
+ await restore(s, dump, { _substrat_outbox: rows });
873
+ const clean = await host.admin.readUndrainedEvents(staff, t1, s, 200);
874
+ expect(clean.map((e) => e.id)).toEqual(expect.arrayContaining(events.map((e) => e.id)));
875
+ expect(clean.skipped).toBeUndefined();
876
+ const bad = { ...events[0], id: badId(), actor: '{' };
877
+ await restore(s, dump, { _substrat_outbox: [bad, ...rows] });
878
+ const read = await host.admin.readUndrainedEvents(staff, t1, s, 200);
879
+ // Every healthy row, exactly as the clean read had it; the bad one nowhere in it.
880
+ expect(read.map((e) => e.id)).toEqual(clean.map((e) => e.id));
881
+ expect([...read]).toEqual([...clean]);
882
+ expect(read.skipped).toEqual({ count: 1, eventIds: [bad.id] });
883
+ // The cost of the skip, pinned: it is never stamped, so it is still undrained — and
884
+ // still reported — once everything behind it has shipped. The lake never has it.
885
+ await host.admin.markEventsDrained(staff, t1, s, read.map((e) => e.id));
886
+ const after = await host.admin.readUndrainedEvents(staff, t1, s, 200);
887
+ expect(after).toHaveLength(0);
888
+ expect(after.skipped).toEqual({ count: 1, eventIds: [bad.id] });
889
+ });
890
+ // #1641 review. Two shapes the first cut let through: an optional column stored as
891
+ // `''` was tested for truthiness and read as ABSENT, skipping validation; and the
892
+ // drain's lifted columns were copied onto a validated envelope unparsed. Both are
893
+ // values the published schema refuses, and both must be contained like any other.
894
+ it('an empty optional column, or a corrupt lifted one, is contained too', async () => {
895
+ const tag = `behind-${ulid()}`;
896
+ const { s, dump } = await seeded('connector-vertical', (stub) => stub.invoke('connector/request-effect', { tag: `seed-${ulid()}` }));
897
+ const template = rowsOf(dump, '_substrat_outbox').find((r) => r.type === 'effect.requested');
898
+ const behind = { ...template, id: ulid(), payload: JSON.stringify({ tag }) };
899
+ const planted = [
900
+ // Envelope columns — every path: delivery, dispatch and the drain.
901
+ { ...template, id: badId(0), impersonation: '' },
902
+ { ...template, id: badId(1), authorization: '' },
903
+ // Lifted columns — the drain's alone.
904
+ { ...template, id: badId(2), version: '' },
905
+ { ...template, id: badId(3), caused_by: 'not-an-event-id' },
906
+ ];
907
+ await restore(s, dump, { _substrat_deliveries: [], _substrat_outbox: [...planted, behind] });
908
+ // The drain: none of the four ships; the event behind them does; all four counted.
909
+ const read = await host.admin.readUndrainedEvents(staff, t1, s, 200);
910
+ expect(read.map((e) => e.id)).toEqual([behind.id]);
911
+ expect(read.skipped).toEqual({ count: 4, eventIds: planted.map((r) => r.id) });
912
+ // Dispatch: the two envelope-broken rows are dead at once and never handled; the
913
+ // lifted columns are not part of an executor's event, so those two deliver as before.
914
+ const report = await host.drainDue(t1, s);
915
+ expect(report).toMatchObject({ attempted: 5, delivered: 3, deadLettered: 2 });
916
+ expect(effected).toContain(tag);
917
+ const dead = (await host.admin.deadLetters(staff, t1, s, {})).entries.map((d) => d.eventId).sort();
918
+ expect(dead).toEqual([badId(0), badId(1)]);
919
+ });
920
+ it('the Tier-2 read is bounded: past limit × 10 bad rows in a row, a pass ships nothing', async () => {
921
+ const { s, dump } = await seeded('connector-vertical', (stub) => stub.invoke('test/emit-event'));
922
+ const rows = rowsOf(dump, '_substrat_outbox');
923
+ const event = rows.find((r) => r.type === 'test.happened');
924
+ const others = rows.filter((r) => r !== event);
925
+ const bad = (n) => Array.from({ length: n }, (_, i) => ({ ...event, id: badId(i), payload: '{' }));
926
+ // Already-drained, so the only undrained rows are the planted ones and `event`.
927
+ const drainedOthers = others.map((r) => ({ ...r, drained_at: '2026-09-01T00:00:00.000Z' }));
928
+ // One under the bound (limit 1 → ten rows looked at): the healthy row still ships.
929
+ await restore(s, dump, { _substrat_outbox: [...bad(9), event, ...drainedOthers] });
930
+ const under = await host.admin.readUndrainedEvents(staff, t1, s, 1);
931
+ expect(under.map((e) => e.id)).toEqual([event.id]);
932
+ expect(under.skipped?.count).toBe(9);
933
+ // At the bound: nothing ships, this pass or the next — the known limit. A spine
934
+ // this broken is a restore to repair; the count is what makes it visible.
935
+ await restore(s, dump, { _substrat_outbox: [...bad(10), event, ...drainedOthers] });
936
+ for (let pass = 0; pass < 2; pass++) {
937
+ const stalled = await host.admin.readUndrainedEvents(staff, t1, s, 1);
938
+ expect(stalled).toHaveLength(0);
939
+ expect(stalled.skipped?.count).toBe(10);
940
+ }
941
+ // A larger budget reaches past it.
942
+ const wider = await host.admin.readUndrainedEvents(staff, t1, s, 2);
943
+ expect(wider.map((e) => e.id)).toEqual([event.id]);
944
+ });
945
+ });
946
+ });
352
947
  it('isolates scope storage: a write in one scope is invisible in another', async () => {
353
948
  const stub1 = await host.getScope(alice, t1, s1);
354
949
  const stub2 = await host.getScope(alice, t2, s2);
@@ -451,6 +1046,44 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
451
1046
  // rebuild instant went to the new table, and reopening it would write it there twice.
452
1047
  await expect(host.admin.redrainEvents(staff, t1, sDrain, { drainedBefore: stampedAt })).resolves.toBe(0);
453
1048
  const justAfter = new Date(Date.parse(stampedAt) + 1).toISOString();
1049
+ // COUNT ONLY (#1545) — the same window asked read-only, which is what a dry run needs
1050
+ // and what nothing could answer before: the verb only ever reopened, so a caller could
1051
+ // learn the number by changing the rows or not at all.
1052
+ const redrainRows = async () => (await host.admin.auditLog(staff, { tenantId: t1 })).filter((r) => r.action === 'redrainEvents' && r.scopeId === sDrain).length;
1053
+ const receiptsBefore = await redrainRows();
1054
+ const undrainedBefore = await host.admin.readUndrainedEvents(staff, t1, sDrain, 200);
1055
+ // Strictly before, on the same boundary the reopen holds: a count that included the
1056
+ // instant would rehearse a different window from the one the real run touches.
1057
+ await expect(host.admin.redrainEvents(staff, t1, sDrain, { drainedBefore: stampedAt, countOnly: true })).resolves.toBe(0);
1058
+ // Asked TWICE on purpose: a "count" that reopened its own window would answer
1059
+ // differently the second time, which is the one failure a rehearsal must not have.
1060
+ await expect(host.admin.redrainEvents(staff, t1, sDrain, { drainedBefore: justAfter, countOnly: true })).resolves.toBe(first.length);
1061
+ await expect(host.admin.redrainEvents(staff, t1, sDrain, { drainedBefore: justAfter, countOnly: true })).resolves.toBe(first.length);
1062
+ // …and it moved nothing: the stamped rows are still stamped, so the drain has exactly
1063
+ // the same work in front of it as before the count.
1064
+ expect((await host.admin.readUndrainedEvents(staff, t1, sDrain, 200)).map((e) => e.id)).toEqual(undrainedBefore.map((e) => e.id));
1065
+ // …and wrote no receipt. The intent row exists because a reopen that crashed before
1066
+ // its outcome row would leave a second egress with no trace; a count egresses nothing,
1067
+ // and a row claiming a redrain on a scope that was only counted is a false statement
1068
+ // in the one log that is evidence.
1069
+ expect(await redrainRows()).toBe(receiptsBefore);
1070
+ // It is still a READ, and K-24 takes all reads rather than a chosen subset — so the
1071
+ // access log carries a row per count, naming the window and the number it found.
1072
+ // "Nobody can tell who counted every tenant's outbox" is the hole that would be.
1073
+ const counts = (await host.admin.accessLog(staff, { tenantId: t1, method: 'redrainEvents' })).filter((r) => r.scopeId === sDrain);
1074
+ expect(counts).toHaveLength(3);
1075
+ expect(counts.map((r) => r.resultCount)).toEqual(expect.arrayContaining([0, first.length]));
1076
+ // The future refusal is the verb's, not the reopen's alone — the window rule is
1077
+ // checked before the two branches part.
1078
+ await expect(host.admin.redrainEvents(staff, t1, sDrain, {
1079
+ drainedBefore: new Date(Date.now() + 60_000).toISOString(),
1080
+ countOnly: true,
1081
+ })).rejects.toThrow(/future/);
1082
+ // K-3 holds for the count as for the reopen: a foreign pair is refused, never answered
1083
+ // 0 — "nothing to reopen" would read as a finished rehearsal of a scope never touched.
1084
+ await expect(host.admin.redrainEvents(staff, t2, sDrain, { drainedBefore: justAfter, countOnly: true })).rejects.toThrow(/unknown scope/);
1085
+ // The number the count gave IS the number the reopen changes — the property that makes
1086
+ // a dry run a rehearsal rather than a second, differently-shaped question.
454
1087
  await expect(host.admin.redrainEvents(staff, t1, sDrain, { drainedBefore: justAfter })).resolves.toBe(first.length);
455
1088
  // The reopened rows come back through the ordinary read, unchanged — nothing about the
456
1089
  // event moves except that it may leave again.
@@ -820,6 +1453,29 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
820
1453
  it('rejects an unknown table', async () => {
821
1454
  await expect(host.admin.readScopeTable(staff, t1, s1, { table: 'no_such_table', limit: 50, offset: 0 })).rejects.toThrow(/unknown table/);
822
1455
  });
1456
+ // #1524: the size behind the on-demand storage reading. The growth assertion is what
1457
+ // separates a real size from a constant. A host that answered a fixed number, or
1458
+ // the size of some other database, passes "positive integer" and fails this.
1459
+ it('answers the scope database size in bytes, and it grows with what is written (#1524)', async () => {
1460
+ const stub = await host.getScope(alice, t1, s1);
1461
+ const before = await host.admin.scopeDatabaseSize(staff, t1, s1);
1462
+ expect(Number.isInteger(before)).toBe(true);
1463
+ expect(before).toBeGreaterThan(0);
1464
+ // ~1 MiB of marker rows: far past any page-rounding or free-page slack.
1465
+ const chunk = 'x'.repeat(64 * 1024);
1466
+ for (let i = 0; i < 16; i += 1)
1467
+ await stub.invoke('test/write-marker', { v: `${i}-${chunk}` });
1468
+ const after = await host.admin.scopeDatabaseSize(staff, t1, s1);
1469
+ expect(after - before).toBeGreaterThanOrEqual(512 * 1024);
1470
+ // A size of ANOTHER scope must not answer for this one.
1471
+ const other = await host.admin.scopeDatabaseSize(staff, t2, s2);
1472
+ expect(other).toBeLessThan(after);
1473
+ const reads = await host.admin.accessLog(staff, { tenantId: t1, method: 'scopeDatabaseSize' });
1474
+ expect(reads.some((r) => r.scopeId === s1)).toBe(true);
1475
+ });
1476
+ it('refuses a database size for a mismatched (tenantId, scopeId) pair (K-3, #1524)', async () => {
1477
+ await expect(host.admin.scopeDatabaseSize(staff, t2, s1)).rejects.toThrow();
1478
+ });
823
1479
  it('fails closed on a mismatched (tenantId, scopeId) pair (K-3)', async () => {
824
1480
  await expect(host.admin.listScopeTables(staff, t2, s1)).rejects.toThrow();
825
1481
  await expect(host.admin.readScopeTable(staff, t2, s1, { table: 'marker', limit: 50, offset: 0 })).rejects.toThrow();
@@ -1063,6 +1719,217 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
1063
1719
  await expect(host.admin.shredSubject(staff, t2, s1, subject)).rejects.toThrow();
1064
1720
  await expect(host.admin.sealSubjectPayloads(staff, t2, s1, [{ subjectId: subject, plaintext: 'x' }])).rejects.toThrow();
1065
1721
  });
1722
+ // -- the spine's OTHER copy of an event (#1600) -------------------------
1723
+ //
1724
+ // `_substrat_platform_requests` holds whole `DomainEvent`s. A CP-less host cannot
1725
+ // run a connector, so each delivery becomes a `connector:<provider>` intent whose
1726
+ // payload is the entire envelope, payload included — deliberately fat, because the
1727
+ // platform's handler needs what the in-process handler would have been handed. And
1728
+ // nothing ever deletes those rows: `listPlatformRequestHistory` exists precisely so
1729
+ // a settled one stays readable. For a year the redaction reached the outbox and not
1730
+ // this, so a shredded subject's name sat in the live scope database, and in every
1731
+ // export, backup and PITR window taken from it afterwards.
1732
+ //
1733
+ // These drive the intent through `ctx.requestPlatform` with the exact payload the
1734
+ // CP-less router writes (`{ executorId, event }`, `connectorDispatchPayload`), which
1735
+ // is the one shape both adapters can be held to — the routing itself only exists on
1736
+ // the Durable-Object host, and `connector-route.test.ts` asserts that what it really
1737
+ // writes is a payload this redaction selects.
1738
+ describe('the intent journal (#1600)', () => {
1739
+ /** The whole spine envelope, as an intent embeds it. Parsed, so the fixture IS one. */
1740
+ const envelopeFor = (subject, said) => domainEvent.parse({
1741
+ id: eventId.parse(ulid()),
1742
+ type: 'protocol.signatures-requested',
1743
+ schemaVersion: 1,
1744
+ occurredAt: instant.parse(new Date().toISOString()),
1745
+ tenantId: t1,
1746
+ scopeId: s1,
1747
+ actor: { system: '@substrat-run/engine-protocol' },
1748
+ entity: { entityType: 'protocol', entityId: ulid() },
1749
+ piiClass: 'direct',
1750
+ subjectId: subject,
1751
+ payload: { senderParty: { label: said }, parties: [{ label: said }] },
1752
+ });
1753
+ /** Enqueue one routed-connector-shaped intent and hand back its id. */
1754
+ const routeIntent = async (kind, subject, said) => {
1755
+ const scope = await host.getScope(alice, t1, s1);
1756
+ const id = await scope.invoke('platform/request', {
1757
+ kind,
1758
+ payload: { executorId: 'signer', event: envelopeFor(subject, said) },
1759
+ });
1760
+ return id;
1761
+ };
1762
+ const journal = async (kind) => host.listPlatformRequestHistory(t1, s1, { kind });
1763
+ it('redacts the routed copy of the event, and leaves the neighbour intact', async () => {
1764
+ // A distinct kind per test: this scope's journal is shared with every other
1765
+ // suite that enqueues, and `connector:<provider>` IS a family of kinds.
1766
+ const kind = `connector:erasure-${ulid()}`;
1767
+ const erased = dataSubjectId.parse(ulid());
1768
+ const spared = dataSubjectId.parse(ulid());
1769
+ const theirs = await routeIntent(kind, erased, 'Anna Ek');
1770
+ const neighbour = await routeIntent(kind, spared, 'Bo Lund');
1771
+ const before = await journal(kind);
1772
+ expect(JSON.stringify(before.find((r) => r.id === theirs))).toContain('Anna Ek');
1773
+ const receipt = await host.admin.shredSubject(staff, t1, s1, erased);
1774
+ const after = await journal(kind);
1775
+ const mine = after.find((r) => r.id === theirs);
1776
+ // THE property, asserted before the receipt that claims it: no trace of the
1777
+ // name, anywhere in the row.
1778
+ expect(JSON.stringify(mine)).not.toContain('Anna Ek');
1779
+ // And the positive twin, without which a redact-everything bug passes the
1780
+ // line above: the intent belonging to somebody else still says what it said,
1781
+ // and is still waiting to be drained.
1782
+ const other = after.find((r) => r.id === neighbour);
1783
+ expect(JSON.stringify(other)).toContain('Bo Lund');
1784
+ expect(other.status).toBe('pending');
1785
+ expect(receipt.intentsRedacted).toBe(1);
1786
+ // Settled afterwards so this test does not spend one of the scope's 32 pending
1787
+ // slots for the rest of the run.
1788
+ await host.settlePlatformRequest(t1, s1, neighbour, { status: 'done' });
1789
+ });
1790
+ it('keeps the envelope — the row is still there, still listed, still dated', async () => {
1791
+ const kind = `connector:erasure-${ulid()}`;
1792
+ const erased = dataSubjectId.parse(ulid());
1793
+ const id = await routeIntent(kind, erased, 'Anna Ek');
1794
+ const before = (await journal(kind)).find((r) => r.id === id);
1795
+ await host.admin.shredSubject(staff, t1, s1, erased);
1796
+ const after = (await journal(kind)).find((r) => r.id === id);
1797
+ // The retention is intended, so the redaction must not become a delete: what
1798
+ // this row proves — that something was asked of the platform, by whom, and
1799
+ // when — is the audit trail, and it survives the erasure of what was said.
1800
+ expect(after).toBeDefined();
1801
+ expect(after.kind).toBe(kind);
1802
+ expect(after.requestedAt).toBe(before.requestedAt);
1803
+ expect(after.requestedBy).toEqual(before.requestedBy);
1804
+ // Obviously redacted rather than plausible data — a reader must not need the
1805
+ // kind's schema to see that the content is gone.
1806
+ expect(after.payload).toEqual({
1807
+ [REDACTED_INTENT_MARKER]: expect.objectContaining({
1808
+ reason: 'subject-erasure',
1809
+ subjectId: erased,
1810
+ }),
1811
+ });
1812
+ });
1813
+ it('never leaves a redacted intent drainable — a pending one settles failed', async () => {
1814
+ const kind = `connector:erasure-${ulid()}`;
1815
+ const erased = dataSubjectId.parse(ulid());
1816
+ const id = await routeIntent(kind, erased, 'Anna Ek');
1817
+ expect((await host.listPlatformRequests(t1, s1)).some((r) => r.id === id)).toBe(true);
1818
+ await host.admin.shredSubject(staff, t1, s1, erased);
1819
+ // Redacting and leaving it pending would hand the drain a tombstone to execute;
1820
+ // leaving it pending and intact would keep the name. `failed` is the truthful
1821
+ // terminal state — the delivery did not happen and now cannot.
1822
+ const settled = (await journal(kind)).find((r) => r.id === id);
1823
+ expect(settled.status).toBe('failed');
1824
+ expect(settled.settledAt).not.toBeNull();
1825
+ expect(settled.lastError).toContain('subject erasure');
1826
+ const stillPending = await host.listPlatformRequests(t1, s1);
1827
+ expect(stillPending.some((r) => r.id === id)).toBe(false);
1828
+ });
1829
+ it('redacts what the provider said back, not only what we sent', async () => {
1830
+ const kind = `connector:erasure-${ulid()}`;
1831
+ const erased = dataSubjectId.parse(ulid());
1832
+ const id = await routeIntent(kind, erased, 'Anna Ek');
1833
+ // #618 made a provider's own sentence readable on the journal, which is exactly
1834
+ // why it can quote the person back at us. An erasure that emptied `payload` and
1835
+ // left a name two columns over is the same defect, one column to the right.
1836
+ await host.settlePlatformRequest(t1, s1, id, {
1837
+ status: 'failed',
1838
+ lastError: 'HTTP 409: Anna Ek requires a valid personal number field',
1839
+ });
1840
+ await host.admin.shredSubject(staff, t1, s1, erased);
1841
+ const after = (await journal(kind)).find((r) => r.id === id);
1842
+ expect(after.lastError).not.toContain('Anna Ek');
1843
+ // Already settled, so the redaction does not re-date it or reopen it.
1844
+ expect(after.status).toBe('failed');
1845
+ });
1846
+ it('keeps `result`, which is envelope — what the intent DID, not what it said', async () => {
1847
+ // The redaction spares `result` on purpose, and until now nothing held it to
1848
+ // that: for a routed dispatch it is `{ eventId }`, which names the event the
1849
+ // outbox already keeps a redacted row for, so it is the same class of fact as
1850
+ // `kind` and `requestedAt`. Asserted beside the two columns that DO go, so the
1851
+ // line between them is a test rather than a sentence in the PR.
1852
+ const kind = `connector:erasure-${ulid()}`;
1853
+ const erased = dataSubjectId.parse(ulid());
1854
+ const id = await routeIntent(kind, erased, 'Anna Ek');
1855
+ await host.settlePlatformRequest(t1, s1, id, {
1856
+ status: 'failed',
1857
+ result: { eventId: 'evt-01', deliveredAt: '2026-09-20T00:00:00.000Z' },
1858
+ lastError: 'HTTP 409: Anna Ek requires a valid personal number field',
1859
+ });
1860
+ await host.admin.shredSubject(staff, t1, s1, erased);
1861
+ const after = (await journal(kind)).find((r) => r.id === id);
1862
+ // What was said goes — both columns that could carry it.
1863
+ expect(JSON.stringify(after.payload)).not.toContain('Anna Ek');
1864
+ expect(after.lastError).not.toContain('Anna Ek');
1865
+ expect(after.lastError).toContain('subject erasure');
1866
+ // What happened stays, unchanged and whole.
1867
+ expect(after.result).toEqual({
1868
+ eventId: 'evt-01',
1869
+ deliveredAt: '2026-09-20T00:00:00.000Z',
1870
+ });
1871
+ });
1872
+ it('refuses a stale drain settling a row the erasure already redacted', async () => {
1873
+ // The drain reads pending rows, runs the handler, then settles — so a settle can
1874
+ // arrive after the erasure landed, carrying a provider's reply that quotes the
1875
+ // person. Settling by id alone wrote that name back into `last_error` on a row
1876
+ // whose payload had just been emptied. The settle is a compare-and-set on
1877
+ // `pending`, so the stale pass is ignored rather than undoing the erasure.
1878
+ //
1879
+ // What this does NOT claim to stop is the DELIVERY: a handler that already read
1880
+ // the payload has it. Only the writeback onto the redacted row is refused.
1881
+ const kind = `connector:erasure-${ulid()}`;
1882
+ const erased = dataSubjectId.parse(ulid());
1883
+ const id = await routeIntent(kind, erased, 'Anna Ek');
1884
+ await host.admin.shredSubject(staff, t1, s1, erased);
1885
+ await host.settlePlatformRequest(t1, s1, id, {
1886
+ status: 'done',
1887
+ result: { eventId: 'delivered-before-the-shred' },
1888
+ lastError: 'HTTP 409: Anna Ek requires a valid personal number field',
1889
+ });
1890
+ const after = (await journal(kind)).find((r) => r.id === id);
1891
+ expect(JSON.stringify(after)).not.toContain('Anna Ek');
1892
+ // The erasure's own settlement stands; the stale pass changed nothing at all.
1893
+ expect(after.status).toBe('failed');
1894
+ expect(after.result).toBeNull();
1895
+ expect(after.attempts).toBe(0);
1896
+ });
1897
+ it('spares an intent whose embedded event is classified `none`', async () => {
1898
+ // The outbox spares such an event even when it names the subject, so a COPY of
1899
+ // it judged more harshly than its original would be incoherent, not stricter.
1900
+ const kind = `connector:erasure-${ulid()}`;
1901
+ const erased = dataSubjectId.parse(ulid());
1902
+ const scope = await host.getScope(alice, t1, s1);
1903
+ const id = (await scope.invoke('platform/request', {
1904
+ kind,
1905
+ payload: {
1906
+ executorId: 'signer',
1907
+ event: {
1908
+ ...envelopeFor(erased, 'Anna Ek'),
1909
+ piiClass: 'none',
1910
+ payload: { reference: 'nothing about anybody' },
1911
+ },
1912
+ },
1913
+ }));
1914
+ const receipt = await host.admin.shredSubject(staff, t1, s1, erased);
1915
+ expect(receipt.intentsRedacted).toBe(0);
1916
+ const after = (await journal(kind)).find((r) => r.id === id);
1917
+ expect(JSON.stringify(after.payload)).toContain('nothing about anybody');
1918
+ await host.settlePlatformRequest(t1, s1, id, { status: 'done' });
1919
+ });
1920
+ it('is idempotent — a second shred finds the tombstone and changes nothing', async () => {
1921
+ const kind = `connector:erasure-${ulid()}`;
1922
+ const erased = dataSubjectId.parse(ulid());
1923
+ const id = await routeIntent(kind, erased, 'Anna Ek');
1924
+ expect((await host.admin.shredSubject(staff, t1, s1, erased)).intentsRedacted).toBe(1);
1925
+ const once = (await journal(kind)).find((r) => r.id === id);
1926
+ // The tombstone names the subject, so it survives the candidate read and is
1927
+ // declined by the predicate — which is the property, not a lucky shape.
1928
+ expect((await host.admin.shredSubject(staff, t1, s1, erased)).intentsRedacted).toBe(0);
1929
+ const twice = (await journal(kind)).find((r) => r.id === id);
1930
+ expect(twice).toEqual(once);
1931
+ });
1932
+ });
1066
1933
  it('records the erasure in BOTH logs — mutation and evidence-destruction', async () => {
1067
1934
  const subject = dataSubjectId.parse(ulid());
1068
1935
  await host.admin.shredSubject(staff, t1, s1, subject);
@@ -1395,6 +2262,103 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
1395
2262
  const tables = await host.admin.listScopeTables(staff, t1, spineScope);
1396
2263
  expect(tables.some((t) => t.name === '_substrat_migrations')).toBe(true);
1397
2264
  });
2265
+ it('rebuilds a module table the dump did not carry — on a host that has ALREADY migrated (#1589)', async () => {
2266
+ // A restore replays only what the dump carries and then re-asserts the spine,
2267
+ // so a MODULE's own tables are dropped and not rebuilt by the restore itself.
2268
+ // Both adapters lean on the next migration pass to recreate them, and this is
2269
+ // the assertion that holds them to the same answer.
2270
+ //
2271
+ // ALREADY MIGRATED is the whole point, not scene-setting. The pure host reads
2272
+ // its applied-migration set on every pass, so it re-applies by construction —
2273
+ // but the Cloudflare ScopeDO memoises the pass itself in `migrationPromise`,
2274
+ // and a WARM instance kept answering "migrations are done" over a scope whose
2275
+ // tables the dump had just dropped. The next operation failed with a bare
2276
+ // `no such table`, and kept failing until an eviction or `migrateScope` — so
2277
+ // the restore reported success and the scope was broken (#1589). A cold host
2278
+ // re-derives everything and would pass this vacuously; the operation below is
2279
+ // what makes the host warm before the dump lands.
2280
+ const memoScope = scopeId.parse(ulid());
2281
+ await host.provisionScope(staff, {
2282
+ tenantId: t1,
2283
+ scopeId: memoScope,
2284
+ jurisdiction: 'eu',
2285
+ vertical: 'connector-vertical',
2286
+ });
2287
+ await host.admin.activateScope(staff, t1, memoScope);
2288
+ const warm = await host.getScope(alice, t1, memoScope);
2289
+ await warm.invoke('testmod/add', { id: 'before-restore', box: 'b1' });
2290
+ // The dump of a targeted repair: the spine and every other module's tables,
2291
+ // but neither `@test/mod`'s tables nor the journal rows claiming they exist.
2292
+ // Stripping the journal row is what makes the dump self-consistent — a dump
2293
+ // that dropped the table and KEPT the row would be asking for a schema no
2294
+ // migration pass is allowed to rebuild.
2295
+ const backup = await host.admin.exportScope(staff, t1, memoScope);
2296
+ const journal = backup.tables.find((t) => t.name === '_substrat_migrations');
2297
+ const moduleCol = journal.columns.indexOf('module_id');
2298
+ expect(backup.tables.some((t) => t.name === 'testmod_items')).toBe(true);
2299
+ expect(journal.rows.some((r) => r[moduleCol] === '@test/mod')).toBe(true);
2300
+ const stripped = {
2301
+ ...backup,
2302
+ tables: backup.tables
2303
+ // BOTH of the module's tables: its one migration creates the pair in a
2304
+ // single statement list, so leaving `testmod_notes` behind would stop the
2305
+ // re-apply on "table testmod_notes already exists" — a different failure
2306
+ // wearing this one's clothes.
2307
+ .filter((t) => t.name !== 'testmod_items' && t.name !== 'testmod_notes')
2308
+ .map((t) => t.name === '_substrat_migrations'
2309
+ ? { ...t, rows: t.rows.filter((r) => r[moduleCol] !== '@test/mod') }
2310
+ : t),
2311
+ };
2312
+ await host.restoreScope(staff, t1, memoScope, stripped);
2313
+ // Both tables are back, and the module writes on them — this is the line that
2314
+ // threw `no such table: testmod_items` on the hosted adapter.
2315
+ const after = await host.getScope(alice, t1, memoScope);
2316
+ await after.invoke('testmod/add', { id: 'after-restore', box: 'b1' });
2317
+ expect(await after.invoke('testmod/read-items')).toEqual([
2318
+ { id: 'after-restore' },
2319
+ ]);
2320
+ expect(await after.invoke('testmod/read-notes')).toEqual([]);
2321
+ // The journal records the re-applied version exactly once, so the pass after
2322
+ // this one has nothing to do.
2323
+ const frontier = await after.invoke('testmod/read-journal');
2324
+ expect(frontier.filter((r) => r.module_id === '@test/mod')).toEqual([
2325
+ { module_id: '@test/mod', version: '0001-init' },
2326
+ ]);
2327
+ });
2328
+ it('leaves a module table the dump DID carry exactly as the dump left it (#1589)', async () => {
2329
+ // The twin of the test above, and what stops "rebuild what the dump dropped"
2330
+ // from becoming "rebuild on every restore". A dump carrying the table and its
2331
+ // journal row must come back as the dump left it: re-running the migration
2332
+ // here would stop on "table testmod_items already exists" and fail the scope
2333
+ // closed — the same outage, reached from the other side — and a restore that
2334
+ // rebuilt the table empty would lose the rows it exists to bring back.
2335
+ const keptScope = scopeId.parse(ulid());
2336
+ await host.provisionScope(staff, {
2337
+ tenantId: t1,
2338
+ scopeId: keptScope,
2339
+ jurisdiction: 'eu',
2340
+ vertical: 'connector-vertical',
2341
+ });
2342
+ await host.admin.activateScope(staff, t1, keptScope);
2343
+ const warm = await host.getScope(alice, t1, keptScope);
2344
+ await warm.invoke('testmod/add', { id: 'kept', box: 'b1' });
2345
+ const backup = await host.admin.exportScope(staff, t1, keptScope);
2346
+ expect(backup.tables.some((t) => t.name === 'testmod_items')).toBe(true);
2347
+ await warm.invoke('testmod/add', { id: 'zz-diverged', box: 'b1' });
2348
+ await host.restoreScope(staff, t1, keptScope, backup);
2349
+ // The dump's row, and only it — the divergence is gone and nothing was wiped.
2350
+ const after = await host.getScope(alice, t1, keptScope);
2351
+ expect(await after.invoke('testmod/read-items')).toEqual([
2352
+ { id: 'kept' },
2353
+ ]);
2354
+ // …and the scope still writes: a re-applied migration would have failed the
2355
+ // scope closed here rather than accepting this row.
2356
+ await after.invoke('testmod/add', { id: 'post-restore', box: 'b1' });
2357
+ expect(await after.invoke('testmod/read-items')).toEqual([
2358
+ { id: 'kept' },
2359
+ { id: 'post-restore' },
2360
+ ]);
2361
+ });
1398
2362
  it('loads a dump whose child table sorts before its parent — FK order is not alphabetical', async () => {
1399
2363
  // A dump is ordered by table NAME. A vertical whose child sorts first (a CRM's
1400
2364
  // `crm_bank_accounts` before `crm_vendors`) used to fail its first insert with a
@@ -1762,6 +2726,11 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
1762
2726
  // -- the integrations hub: connections (#101) -----------------------------
1763
2727
  // -- model usage (#1054): meter 3's ledger, idempotent on the intent id ----------------
1764
2728
  describe('model usage (#1054)', () => {
2729
+ // Window edges are derived from the real clock, never literals: the host prunes model
2730
+ // usage older than MODEL_USAGE_RETENTION_DAYS against Date.now(), so a fixed date is a
2731
+ // fixture that expires. `day(n)` is midnight UTC, `n` days after a base 40 days back.
2732
+ const BASE = new Date(Date.now() - 40 * 86_400_000).setUTCHours(0, 0, 0, 0);
2733
+ const day = (n, hour = 0) => new Date(BASE + n * 86_400_000 + hour * 3_600_000).toISOString();
1765
2734
  const line = (at, over = {}) => ({
1766
2735
  attribution: {
1767
2736
  tenant: t1,
@@ -1786,23 +2755,23 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
1786
2755
  it('records a line once per intent id, lists it, and folds the window exactly', async () => {
1787
2756
  const first = await host.admin.recordModelUsage({
1788
2757
  requestId: 'req-model-usage-1',
1789
- line: line('2026-08-01T10:00:00.000Z', { listUsd: '0.75' }),
2758
+ line: line(day(0, 10), { listUsd: '0.75' }),
1790
2759
  });
1791
2760
  expect(first.recorded).toBe(true);
1792
2761
  // The drain replays a settled-then-retried intent: nothing is billed twice.
1793
2762
  const again = await host.admin.recordModelUsage({
1794
2763
  requestId: 'req-model-usage-1',
1795
- line: line('2026-08-01T10:00:00.000Z', { listUsd: '0.75' }),
2764
+ line: line(day(0, 10), { listUsd: '0.75' }),
1796
2765
  });
1797
2766
  expect(again.recorded).toBe(false);
1798
2767
  await host.admin.recordModelUsage({
1799
2768
  requestId: 'req-model-usage-2',
1800
- line: line('2026-08-02T10:00:00.000Z', { listUsd: '0.5', inputTokens: 50_000, outputTokens: 5_000 }),
2769
+ line: line(day(1, 10), { listUsd: '0.5', inputTokens: 50_000, outputTokens: 5_000 }),
1801
2770
  });
1802
2771
  // An unpriced call: counted, never folded in as $0.
1803
2772
  await host.admin.recordModelUsage({
1804
2773
  requestId: 'req-model-usage-3',
1805
- line: line('2026-08-03T10:00:00.000Z', {
2774
+ line: line(day(2, 10), {
1806
2775
  listUsd: null,
1807
2776
  model: 'cloudflare:@cf/meta/llama-3.1-8b-instruct-fast',
1808
2777
  provider: 'cloudflare',
@@ -1814,13 +2783,13 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
1814
2783
  // Outside the window below.
1815
2784
  await host.admin.recordModelUsage({
1816
2785
  requestId: 'req-model-usage-4',
1817
- line: line('2026-09-01T00:00:00.000Z', { listUsd: '9' }),
2786
+ line: line(day(30), { listUsd: '9' }),
1818
2787
  });
1819
- const listed = await host.admin.listModelUsage(staff, { tenantId: t1, since: '2026-08-01T00:00:00.000Z' });
2788
+ const listed = await host.admin.listModelUsage(staff, { tenantId: t1, since: day(0) });
1820
2789
  expect(listed.length).toBe(4);
1821
2790
  expect(listed[0].requestId).toBe('req-model-usage-4'); // newest first
1822
2791
  expect(listed.every((e) => e.attribution.tenant === t1)).toBe(true);
1823
- const summary = await host.admin.summarizeModelUsage(staff, { tenantId: t1, since: '2026-08-01T00:00:00.000Z', until: '2026-09-01T00:00:00.000Z' }, 20);
2792
+ const summary = await host.admin.summarizeModelUsage(staff, { tenantId: t1, since: day(0), until: day(30) }, 20);
1824
2793
  expect(summary.marginPercent).toBe(20);
1825
2794
  expect(summary.rows.length).toBe(2);
1826
2795
  const opus = summary.rows.find((r) => r.model === 'anthropic:claude-opus-5');
@@ -2087,14 +3056,18 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
2087
3056
  expect(row.error).toBeNull();
2088
3057
  expect(row.elapsedMs).toBeNull();
2089
3058
  });
2090
- it('dedupes on (requestId, unit) — a replayed drained batch writes nothing twice (#1232)', async () => {
3059
+ it('dedupes on (requestId, kind, unit) — a replayed drained batch writes nothing twice (#1232)', async () => {
2091
3060
  const unit = `${ulid()}:sched/tick`;
3061
+ // Relative to the real clock, never a literal: the host prunes a sweep run older
3062
+ // than SWEEP_RUN_RETENTION_DAYS against Date.now(), so a fixed date is a fixture
3063
+ // that expires and takes the suite with it.
3064
+ const passAt = new Date(Date.now() - 3_600_000).toISOString();
2092
3065
  const write = () => host.admin.recordSweepRun({
2093
3066
  kind: 'schedule',
2094
3067
  unit,
2095
3068
  outcome: 'ok',
2096
3069
  tenantId: t1,
2097
- at: '2026-09-07T10:00:00.000Z',
3070
+ at: passAt,
2098
3071
  requestId: '01JDEDUPEINTENTAAAAAAAAAAA',
2099
3072
  });
2100
3073
  await write();
@@ -2102,7 +3075,7 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
2102
3075
  const rows = await host.admin.listSweepRuns(staff, { unit });
2103
3076
  expect(rows).toHaveLength(1);
2104
3077
  // The carried pass time survives, never overwritten by write time.
2105
- expect(rows[0].at).toBe('2026-09-07T10:00:00.000Z');
3078
+ expect(rows[0].at).toBe(passAt);
2106
3079
  // The DIRECT path (no requestId) never dedupes: NULLs are distinct, and two
2107
3080
  // real passes over one unit are two facts.
2108
3081
  const direct = `${ulid()}:direct`;
@@ -2110,6 +3083,28 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
2110
3083
  await host.admin.recordSweepRun({ kind: 'schedule', unit: direct, outcome: 'ok' });
2111
3084
  expect(await host.admin.listSweepRuns(staff, { unit: direct })).toHaveLength(2);
2112
3085
  });
3086
+ it('keeps a schedule row and a freshness row of one batch that derive the same unit (#1572)', async () => {
3087
+ // The drain derives a schedule's unit from its operation and a freshness unit
3088
+ // from its event type, and nothing keeps those apart: `orders.placed` is an
3089
+ // ordinary spelling for both. One batch shares one requestId, so under the old
3090
+ // (requestId, unit) key the second write was IGNOREd, silently.
3091
+ const unit = `${ulid()}:orders.placed`;
3092
+ const requestId = '01JKINDINTENTAAAAAAAAAAAAA';
3093
+ const schedule = () => host.admin.recordSweepRun({ kind: 'schedule', unit, outcome: 'skipped', operation: 'orders.placed', requestId });
3094
+ const freshness = () => host.admin.recordSweepRun({ kind: 'freshness', unit, outcome: 'failed', eventType: 'orders.placed', requestId });
3095
+ await schedule();
3096
+ await freshness();
3097
+ const both = await host.admin.listSweepRuns(staff, { unit });
3098
+ expect(both.map((r) => [r.kind, r.outcome]).sort()).toEqual([
3099
+ ['freshness', 'failed'],
3100
+ ['schedule', 'skipped'],
3101
+ ]);
3102
+ // The twin: kind widened the key, it did not drop it. A replay of either entry
3103
+ // is still the same row and still writes nothing.
3104
+ await schedule();
3105
+ await freshness();
3106
+ expect(await host.admin.listSweepRuns(staff, { unit })).toHaveLength(2);
3107
+ });
2113
3108
  it('bounds the recorded error — one runaway provider body never becomes a runaway row', async () => {
2114
3109
  const unit = ulid();
2115
3110
  await host.admin.recordSweepRun({
@@ -2950,8 +3945,8 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
2950
3945
  expect((await at('listtest'))?.publishRequestedAt).toBeTruthy();
2951
3946
  await host.admin.setVerticalListed(staff, 'listtest', true);
2952
3947
  expect((await at('listtest'))?.publishRequestedAt).toBeUndefined(); // cleared on review
2953
- await expect(host.admin.setVerticalListed(staff, 'no-such-vertical', true)).rejects.toThrow(/unknown vertical/);
2954
- await expect(host.admin.requestPublish(staff, 'no-such-vertical')).rejects.toThrow(/unknown vertical/);
3948
+ await expectRefusal(host.admin.setVerticalListed(staff, 'no-such-vertical', true), 'not_found');
3949
+ await expectRefusal(host.admin.requestPublish(staff, 'no-such-vertical'), 'not_found');
2955
3950
  });
2956
3951
  it('blocks new installs (setVerticalInstallsBlocked) — a provisioning gate, not a delete', async () => {
2957
3952
  const at = (slug) => host.admin.listVerticals(staff).then((vs) => vs.find((v) => v.slug === slug));
@@ -2964,7 +3959,7 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
2964
3959
  expect((await at('blocktest'))?.listed).toBe(false);
2965
3960
  await host.admin.setVerticalInstallsBlocked(staff, 'blocktest', false);
2966
3961
  expect((await at('blocktest'))?.installsBlocked).toBe(false);
2967
- await expect(host.admin.setVerticalInstallsBlocked(staff, 'no-such-vertical', true)).rejects.toThrow(/unknown vertical/);
3962
+ await expectRefusal(host.admin.setVerticalInstallsBlocked(staff, 'no-such-vertical', true), 'not_found');
2968
3963
  });
2969
3964
  it('grants the tenant-provisioner capability (setVerticalTenantProvisioner) — a staff grant a re-push cannot touch', async () => {
2970
3965
  const at = (slug) => host.admin.listVerticals(staff).then((vs) => vs.find((v) => v.slug === slug));
@@ -2979,7 +3974,7 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
2979
3974
  expect((await at('managertest'))?.tenantProvisioner).toBe(true);
2980
3975
  await host.admin.setVerticalTenantProvisioner(staff, 'managertest', false);
2981
3976
  expect((await at('managertest'))?.tenantProvisioner).toBe(false);
2982
- await expect(host.admin.setVerticalTenantProvisioner(staff, 'no-such-vertical', true)).rejects.toThrow(/unknown vertical/);
3977
+ await expectRefusal(host.admin.setVerticalTenantProvisioner(staff, 'no-such-vertical', true), 'not_found');
2983
3978
  });
2984
3979
  it('carries the declared provisioner intent (#455) — a refreshable request, orthogonal to the grant', async () => {
2985
3980
  const at = (slug) => host.admin.listVerticals(staff).then((vs) => vs.find((v) => v.slug === slug));
@@ -3030,12 +4025,13 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
3030
4025
  expect((await at('mailer'))?.emailSender).toBe(true);
3031
4026
  await host.admin.setVerticalEmailSender(staff, 'mailer', false);
3032
4027
  expect((await at('mailer'))?.emailSender).toBe(false);
3033
- await expect(host.admin.setVerticalEmailSender(staff, 'no-such-vertical', true)).rejects.toThrow(/unknown vertical/);
4028
+ await expectRefusal(host.admin.setVerticalEmailSender(staff, 'no-such-vertical', true), 'not_found');
3034
4029
  });
3035
4030
  it('deletes a vertical — refused while a scope is bound, total once nothing is', async () => {
3036
4031
  // 'callout' still backs s1 (bound above): the refusal that stops a delete from
3037
- // stranding a live scope's version pin and routing.
3038
- await expect(host.admin.deleteVertical(staff, 'callout')).rejects.toThrow(/still backs/);
4032
+ // stranding a live scope's version pin and routing. Pinned by the CODE (#113
4033
+ // phase 5) — the sentence is free to change, the 409 is not.
4034
+ await expectRefusal(host.admin.deleteVertical(staff, 'callout'), 'conflict');
3039
4035
  expect((await host.admin.listVerticals(staff)).some((v) => v.slug === 'callout')).toBe(true);
3040
4036
  // A vertical nothing is bound to deletes totally: row, versions, channels.
3041
4037
  const versionId = ulid();
@@ -3059,7 +4055,55 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
3059
4055
  await host.admin.registerVertical(staff, { slug: 'deletable', name: 'Deletable', source: 'cli', ownerTenant: t2 });
3060
4056
  expect(await host.admin.listVersions(staff, 'deletable')).toEqual([]); // no resurrected versions
3061
4057
  await host.admin.deleteVertical(staff, 'deletable');
3062
- await expect(host.admin.deleteVertical(staff, 'no-such-vertical')).rejects.toThrow(/unknown vertical/);
4058
+ await expectRefusal(host.admin.deleteVertical(staff, 'no-such-vertical'), 'not_found');
4059
+ });
4060
+ it('refuses the serving-state read and write for a vertical that does not exist', async () => {
4061
+ await expectRefusal(host.admin.verticalServing(staff, 'no-such-vertical'), 'not_found');
4062
+ await expectRefusal(host.admin.setVerticalServing(staff, 'no-such-vertical', {
4063
+ ref: 'serving-script', versionId: ulid(), doClasses: [], migrationTag: 'g1',
4064
+ }), 'not_found');
4065
+ });
4066
+ it('refuses every registry verb that addresses a version which does not exist (#113 phase 5)', async () => {
4067
+ // The five `unknown version …` throw sites, one per verb, on both adapters. Nothing
4068
+ // asserted them before this case: each was a bare `Error` whose 404 came from a
4069
+ // `/unknown version /` regex in the control plane, so a reword — or a regression to
4070
+ // `new Error` — changed the status with no test to notice. The code is the contract
4071
+ // now, and it is pinned here rather than in the transport.
4072
+ //
4073
+ // The id is well-formed and simply names nothing, which is what separates this from
4074
+ // a parse failure: `not_found`, never `validation_failed`.
4075
+ const ghost = ulid();
4076
+ await expectRefusal(host.admin.admitVersion(staff, ghost), 'not_found');
4077
+ await expectRefusal(host.admin.rejectVersion(staff, ghost, 'no such version'), 'not_found');
4078
+ await expectRefusal(host.admin.promoteVersion(staff, 'callout', 'prod', ghost), 'not_found');
4079
+ await expectRefusal(host.admin.bindScopeVersion(staff, t1, s1, ghost), 'not_found');
4080
+ await expectRefusal(host.admin.versionManifest(staff, 'callout', ghost), 'not_found');
4081
+ // versionManifest refuses on the PAIR — `!v || v.vertical_slug !== verticalSlug` —
4082
+ // so the absent id above only exercises HALF its guard. A version that really
4083
+ // exists, under a DIFFERENT vertical, must read as absent too: asking for it under
4084
+ // someone else's slug is not a way in. Without this second call an implementation
4085
+ // that dropped the ownership check would still pass the case, which is the same
4086
+ // shape of hole this whole change exists to close.
4087
+ await host.admin.registerVertical(staff, {
4088
+ slug: 'otherowner',
4089
+ name: 'OtherOwner',
4090
+ source: 'cli',
4091
+ ownerTenant: t1,
4092
+ });
4093
+ const elsewhere = ulid();
4094
+ await host.admin.publishVersion(staff, {
4095
+ id: elsewhere,
4096
+ verticalSlug: 'otherowner',
4097
+ version: '1.0.0',
4098
+ manifestDigest: 'm-other',
4099
+ permissionDigest: 'p-other',
4100
+ migrationDigest: 'g-other',
4101
+ deploymentRef: null,
4102
+ });
4103
+ // It resolves under its OWN vertical — so the refusal below is the pair check
4104
+ // firing, not the version failing to exist.
4105
+ await host.admin.versionManifest(staff, 'otherowner', elsewhere);
4106
+ await expectRefusal(host.admin.versionManifest(staff, 'callout', elsewhere), 'not_found');
3063
4107
  });
3064
4108
  it('an archived scope blocks the delete naming the reap step; a reaped tombstone never blocks', async () => {
3065
4109
  // A deleted app leaves an `archived` row (restorable via unarchive), then a
@@ -3070,9 +4114,14 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
3070
4114
  const sRet = scopeId.parse(ulid());
3071
4115
  await host.admin.registerVertical(staff, { slug: 'retirable', name: 'Retirable', source: 'cli', ownerTenant: t2 });
3072
4116
  await host.provisionScope(staff, { tenantId: t2, scopeId: sRet, jurisdiction: 'eu', vertical: 'retirable' });
3073
- await expect(host.admin.deleteVertical(staff, 'retirable')).rejects.toThrow(/still backs 1 scope\(s\) — delete or rebind/);
4117
+ // Both halves are `conflict` and the sentence is what tells them apart, so both are
4118
+ // asserted (#113 phase 5). The CODE is what the transport renders the 409 from —
4119
+ // the archived half reached the control plane as a generic 500 until these throws
4120
+ // were typed, because the pattern row that carried the live one read
4121
+ // `/still backs \d+ scope\(s\)/` and the word `archived` sits in the middle of it.
4122
+ await expectRefusal(host.admin.deleteVertical(staff, 'retirable'), 'conflict', /still backs 1 scope\(s\) — delete or rebind/);
3074
4123
  await host.admin.archiveScope(staff, t2, sRet);
3075
- await expect(host.admin.deleteVertical(staff, 'retirable')).rejects.toThrow(/1 archived scope\(s\) — reap or restore/);
4124
+ await expectRefusal(host.admin.deleteVertical(staff, 'retirable'), 'conflict', /1 archived scope\(s\) — reap or restore/);
3076
4125
  await host.admin.reapScope(staff, t2, sRet);
3077
4126
  await host.admin.deleteVertical(staff, 'retirable');
3078
4127
  expect((await host.admin.listVerticals(staff)).some((v) => v.slug === 'retirable')).toBe(false);
@@ -3104,7 +4153,7 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
3104
4153
  await host.admin.rejectVersion(staff, versionId, 'permission diff widened a role');
3105
4154
  await expect(host.admin.bindScopeVersion(staff, t1, s1, versionId)).rejects.toThrow(/rejected, not admitted/);
3106
4155
  // Terminal: a rejected version is not resurrected, a new one is published.
3107
- await expect(host.admin.admitVersion(staff, versionId)).rejects.toThrow(/was rejected/);
4156
+ await expectRefusal(host.admin.admitVersion(staff, versionId), 'conflict');
3108
4157
  const rejected = (await host.admin.listVersions(staff, 'callout')).find((v) => v.id === versionId);
3109
4158
  expect(rejected?.admissionNote).toContain('widened a role');
3110
4159
  });
@@ -3120,7 +4169,7 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
3120
4169
  }
3121
4170
  });
3122
4171
  it('refuses a version for a vertical nobody registered', async () => {
3123
- await expect(host.admin.publishVersion(staff, {
4172
+ await expectRefusal(host.admin.publishVersion(staff, {
3124
4173
  id: ulid(),
3125
4174
  verticalSlug: 'ghost',
3126
4175
  version: '1.0.0',
@@ -3128,7 +4177,7 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
3128
4177
  permissionDigest: 'p',
3129
4178
  migrationDigest: 'g',
3130
4179
  deploymentRef: null,
3131
- })).rejects.toThrow(/unknown vertical/);
4180
+ }), 'not_found');
3132
4181
  });
3133
4182
  // -- channels and promotion-time checkpoints (#31 step 2) ----------------
3134
4183
  // Promotion is the moment a change reaches anyone, so it is where §4's two
@@ -3871,6 +4920,124 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
3871
4920
  expect(reactions).toHaveLength(2);
3872
4921
  expect(new Set(reactions.map((r) => r.caused_by)).size).toBe(2);
3873
4922
  });
4923
+ // -- the invocation a DELIVERY belongs to (#1525) -------------------------
4924
+ //
4925
+ // #1237 put `invocation_id` on the outbox, which answers "which call emitted this
4926
+ // event". A delivery joins to that through `event_id`, so the tempting reading is
4927
+ // that deliveries needed nothing. What the event's id cannot say is which call
4928
+ // attempted the DELIVERY — and for an executor those come apart by design: attempt
4929
+ // one runs in the emitting call's post-commit tail, every retry after it in a drain
4930
+ // that is a different call or none at all.
4931
+ //
4932
+ // Each test takes its own scope: they assert over the whole delivery log they read.
4933
+ describe('the invocation a delivery belongs to (#1525)', () => {
4934
+ const flowScope = async () => {
4935
+ const s = scopeId.parse(ulid());
4936
+ await host.provisionScope(staff, { tenantId: t1, scopeId: s, vertical: 'flow-vertical' });
4937
+ await host.admin.activateScope(staff, t1, s);
4938
+ return s;
4939
+ };
4940
+ it('stamps a consumer delivery with the invocation the call carried', async () => {
4941
+ const s = await flowScope();
4942
+ const stub = await host.getScope(alice, t1, s);
4943
+ const call = ulid();
4944
+ await stub.invoke('flow/produce', undefined, { invocationId: call });
4945
+ // The VALUE, not merely the column: a delivery that recorded null here would
4946
+ // satisfy every "the column exists" assertion and join to nothing.
4947
+ const rows = (await stub.invoke('flow/deliveries'));
4948
+ expect(rows).toHaveLength(2);
4949
+ expect(rows.map((r) => r.invocation_id)).toEqual([call, call]);
4950
+ // A SECOND call is a second id. The field is cleared after each invocation
4951
+ // (#1237), so a leak would file this call's deliveries under the previous one
4952
+ // — which a single-call assertion cannot see.
4953
+ const next = ulid();
4954
+ await stub.invoke('flow/produce', undefined, { invocationId: next });
4955
+ const after = (await stub.invoke('flow/deliveries'));
4956
+ expect(after).toHaveLength(4);
4957
+ expect(after.filter((r) => r.invocation_id === call)).toHaveLength(2);
4958
+ expect(after.filter((r) => r.invocation_id === next)).toHaveLength(2);
4959
+ });
4960
+ it('records null when the caller carried no invocation, and still delivers', async () => {
4961
+ // A seed, a test, an internal call. Null is the honest answer — and the
4962
+ // delivery itself must still happen and still be journalled, since a drain that
4963
+ // failed because no id was carried is a far worse outcome than an unjoined row.
4964
+ // The positive twin above shares this mechanism, so breaking the write reddens
4965
+ // that one rather than leaving both green on a column nobody fills.
4966
+ const s = await flowScope();
4967
+ const stub = await host.getScope(alice, t1, s);
4968
+ await stub.invoke('flow/produce');
4969
+ const rows = (await stub.invoke('flow/deliveries'));
4970
+ expect(rows).toHaveLength(2);
4971
+ expect(rows.every((r) => r.invocation_id === null)).toBe(true);
4972
+ expect(rows.every((r) => r.error === null)).toBe(true);
4973
+ });
4974
+ it('carries the delivery\'s invocation out through the effects walk', async () => {
4975
+ // The column is written; this is what makes it reachable. `walkEventEffects`
4976
+ // is the read that renders a delivery, so a column added to the store and
4977
+ // missed there would leave the join invisible to every surface that shows one.
4978
+ const s = await flowScope();
4979
+ const stub = await host.getScope(alice, t1, s);
4980
+ const call = ulid();
4981
+ await stub.invoke('flow/produce', undefined, { invocationId: call });
4982
+ const rows = (await stub.invoke('flow/causes'));
4983
+ const step1 = rows.find((r) => r.type === 'flow.step1');
4984
+ const tree = await host.admin.eventEffects(staff, t1, s, {
4985
+ eventId: eventId.parse(step1.id),
4986
+ });
4987
+ expect(tree.root.deliveries).not.toHaveLength(0);
4988
+ expect(tree.root.deliveries.every((d) => d.invocationId === call)).toBe(true);
4989
+ });
4990
+ it('stamps an executor delivery with the call whose tail ran it', async () => {
4991
+ // Executors are the half the event's own id could never have covered. On the
4992
+ // hosted adapter this journal is written from the COORDINATOR over an RPC, so
4993
+ // there is no ambient field to read and the id has to survive the hop.
4994
+ const s = scopeId.parse(ulid());
4995
+ await host.provisionScope(staff, {
4996
+ tenantId: t1,
4997
+ scopeId: s,
4998
+ vertical: 'connector-vertical',
4999
+ });
5000
+ await host.admin.activateScope(staff, t1, s);
5001
+ const stub = await host.getScope(alice, t1, s);
5002
+ const call = ulid();
5003
+ await stub.invoke('connector/request-effect', { tag: 'inv-ok' }, { invocationId: call });
5004
+ const emitted = await host.admin.invocationEvents(staff, t1, s, { invocationId: call });
5005
+ const requested = emitted.events.find((e) => e.type === 'effect.requested');
5006
+ const tree = await host.admin.eventEffects(staff, t1, s, { eventId: requested.id });
5007
+ const delivery = tree.root.deliveries.find((d) => d.consumer === 'executor:flaky-effector');
5008
+ expect(delivery).toBeDefined();
5009
+ expect(delivery.state).toBe('delivered');
5010
+ expect(delivery.invocationId).toBe(call);
5011
+ });
5012
+ it('files a RETRY under the call that re-attempted it, not the one that emitted', async () => {
5013
+ // The test that proves this column is not a copy of the outbox's. A doomed
5014
+ // executor is attempted once in the emitting call's tail and once more on a
5015
+ // drain — and the drain is not a call. So the dead row's event still names the
5016
+ // request that produced it while the attempt that gave up names none, which is
5017
+ // exactly the pair `DeadLetter` could not previously tell apart.
5018
+ const s = scopeId.parse(ulid());
5019
+ await host.provisionScope(staff, {
5020
+ tenantId: t1,
5021
+ scopeId: s,
5022
+ vertical: 'connector-vertical',
5023
+ });
5024
+ await host.admin.activateScope(staff, t1, s);
5025
+ const stub = await host.getScope(alice, t1, s);
5026
+ const call = ulid();
5027
+ await stub.invoke('connector/request-doomed', { tag: 'inv-doomed' }, { invocationId: call });
5028
+ // maxAttempts 2: attempt one ran inline under `call`, attempt two runs here.
5029
+ expect((await host.drainDue(t1, s)).deadLettered).toBe(1);
5030
+ const page = await host.admin.deadLetters(staff, t1, s, {});
5031
+ const dead = page.entries.find((d) => d.consumer === 'executor:doomed-effector');
5032
+ expect(dead).toBeDefined();
5033
+ // The EVENT still joins to the request that emitted it…
5034
+ expect(dead.invocationId).toBe(call);
5035
+ // …and the ATTEMPT that gave up names no call, because a drain is not one. An
5036
+ // implementation that copied the event's id here would read `call` and claim a
5037
+ // request had done something it did not do.
5038
+ expect(dead.attemptInvocationId).toBeNull();
5039
+ });
5040
+ });
3874
5041
  it('creates a tenant record, idempotently; only real creates are audited', async () => {
3875
5042
  await host.admin.createTenant(staff, { id: t3, slug: 'acme-co', name: 'Acme Co' });
3876
5043
  await host.admin.createTenant(staff, { id: t3, slug: 'acme-co', name: 'Acme Co' }); // no-op
@@ -3986,6 +5153,9 @@ export function scopeHostContractSuite(adapterName, makeFixture, opts = {}) {
3986
5153
  await expect(host.admin.facetEvents(staff, t3, s, { groupBy: { kind: 'type' } })).rejects.toThrow(/reaped/);
3987
5154
  await expect(host.admin.entityHistory(staff, t3, s, { entityType: 'test-thing', entityId: 'x1' })).rejects.toThrow(/reaped/);
3988
5155
  await expect(host.admin.listScopeTables(staff, t3, s)).rejects.toThrow(/reaped/);
5156
+ // #1524: a size read is refused BEFORE it reaches the storage. Addressing a reaped
5157
+ // DO to ask its size would recreate an empty database and report that as the scope.
5158
+ await expect(host.admin.scopeDatabaseSize(staff, t3, s)).rejects.toThrow(/reaped/);
3989
5159
  // Audited as reapScope against the right scope + actor.
3990
5160
  const reapEntry = (await host.admin.auditLog(staff, { tenantId: t3 })).find((r) => r.action === 'reapScope' && r.scopeId === s);
3991
5161
  expect(reapEntry?.actor).toBe(staff);