@substrat-run/kernel 0.142.0 → 0.144.0

package/dist/entity-grant-shape.js

@@ -1,7 +1,7 @@
 import { ENTITY_GRANTS_RETIRED, ENTITY_GRANTS_TOPPED_UP, ENTITY_SHAPE_MARKER_RELATION, domainEvent, entityGrantsRetiredPayload, entityGrantsToppedUpPayload, entityObjectRef, eventId, } from '@substrat-run/contracts';
 import { assertKernelAuthoredType, substratError } from '@substrat-run/contracts';
 import { KERNEL_ACTOR, kernelOutboxInsertSql } from './kernel-outbox.js';
-import { GRANTEE_KEY_RELATION, explicitTupleSql } from './entity-grant.js';
+import { SHAPE_PROTECTED_KEY_RELATION, explicitTupleSql } from './entity-grant.js';
 import { liveTupleSql } from './permission-eval.js';
 import { assertSqlIdentifier } from './sql-identifier.js';
 /**
@@ -40,13 +40,25 @@ import { assertSqlIdentifier } from './sql-identifier.js';
  *   the principal; a table column naming the principal), and a person is marked only on their
  *   own record, and only when they hold a row there (live or tombstoned) for some key of the
  *   shape — evidence they were given it. Someone `ctx.grant`ed even the whole shape on another
- *   person's record is never marked. A shape with no `holder` gets no backfill. Each batch marks
- *   at most what the pass's budget allows; the markers are the progress, since a marked pair is
- *   no longer a candidate, and a record tuple (`shape:<type>`, `shape-backfilled`, `scope:<id>`)
- *   ends it once a batch comes back short.
+ *   person's record is never marked. A shape with no `holder` gets no backfill. It is read in
+ *   windows like the walks (#2154): each pass reads at most its read budget of rows, in an index
+ *   order, marks at most its write budget, and hands back the last row it read as the cursor's
+ *   `row`, so the next pass seeks past it rather than re-reading from the start. A record tuple
+ *   (`shape:<type>`, `shape-backfilled`, `scope:<id>`) ends it once a read reaches the end of the
+ *   range, and never before. A table holder's read is keyed on `idColumn` and then the row's own
+ *   identity (its `rowid`, or a `WITHOUT ROWID` table's primary key), because nothing makes
+ *   `idColumn` unique or the principal non-NULL, and a key that ties lets a window's edge skip a row.
+ *
+ *   The cut-off is the cursor. Evidence that lands behind it while the backfill runs (a direct
+ *   grant of a shape key to the person on their own record, for `'self'` or a table holder) is not
+ *   marked, and once the record is written nothing is: the same accepted gap as a grant landing
+ *   after the record. There is deliberately no confirming walk, as a retirement has: it would
+ *   double every first backfill's reads to catch a grant that a shape grant should have made. For
+ *   `'grantee'` nothing can land at all, since the first pass records the grantee keys and the
+ *   tuple writer refuses them to every other grant from then on.
  * - `holder: 'grantee'` (#2083), for a record that names no principal (a portal customer, a
  *   contact): whoever holds a live current or retired key of the shape on such an entity was given it.
- *   Each pass records current and retired keys (`shape-grantee-key`), and the explicit tuple
+ *   Each reconcile records current and retired keys (`shape-grantee-key`), and the explicit tuple
  *   writer makes non-shape grants refuse them on that entity type. A tuple from before the
  *   first such pass cannot be told apart. A marker already written by a shape grant persists
  *   through retirement, so its holder receives the new keys even if their old key was K alone.
@@ -55,8 +67,36 @@ import { assertSqlIdentifier } from './sql-identifier.js';
  * record. A sharing shape, which people reach through `ctx.grant` (todo's `list`), is skipped
  * whole, backfill included, so a sharee is never marked a holder.
  */
+/**
+ * The index every marker walk reads (#2083): markers only, in `(object, subject)` order, so one
+ * shape's markers are one range (`<type>:` up to `<type>;`) and a pass's cursor is a seek into
+ * it. Without it a pass scanned the whole `_substrat_tuples` table, even when it had nothing to
+ * do. Partial, so it costs a row per marker and nothing for any other tuple. SQLite uses a
+ * partial index only where it can prove the index's WHERE from the query's, so the walks write the
+ * relation as this same literal: proved from the text alone, with no dependence on SQLite
+ * re-planning once a parameter is bound. In KERNEL_DDL on both adapters, which every wake re-runs,
+ * so an existing scope builds it on its next wake.
+ */
+export const SHAPE_MARKER_INDEX_DDL = `CREATE INDEX IF NOT EXISTS _substrat_tuples_shape_marker ON _substrat_tuples (object, subject) WHERE relation = '${ENTITY_SHAPE_MARKER_RELATION}'`;
 /** Holders topped up per pass — one transaction each, so a large scope never holds one long. */
 export const SHAPE_TOP_UP_BATCH = 500;
+/**
+ * Rows a pass may READ, per row of work it may write: markers on a walk, and the tuples or table
+ * rows the backfill reads for candidates (#2154). The budget bounds writes, but finding
+ * the holders who need one means reading past the ones who do not, and on a scope where nobody
+ * does (the common reconcile: nothing changed) that is every marker. Ten per row keeps a pass at
+ * 5000 index rows by default, short on any scope, while a rollout still spends most of each pass
+ * writing. A pass that reads its window without filling its budget hands back a cursor, and the
+ * next pass reads on from there.
+ */
+export const SHAPE_MARKER_READS_PER_ROW = 10;
+/**
+ * Confirming walks that may take keys, per shape per reconcile, before its retirement is left open
+ * for the next reconcile. Each one that takes keys starts another, so a steady stream of grants of
+ * a retired key landing ahead of the walk would otherwise keep a single reconcile walking forever.
+ * Left open, the retirement keeps no record, so the next reconcile simply runs it again.
+ */
+export const SHAPE_RETIRE_CONFIRMS_MAX = 3;
 /** The largest batch a caller may ask for: past this a pass is the long transaction it exists to avoid. */
 export const SHAPE_TOP_UP_BATCH_MAX = 5000;
 /** `batch`, or `validation_failed`: a pass size is a positive integer no larger than the max. */
@@ -75,6 +115,11 @@ const recordRefs = (entityType, scopeId) => [`shape:${entityType}`, `scope:${sco
 /** The writer of the top-up events: the kernel, as no module or person acted. */
 const PRINCIPAL = 'principal:';
 const keysOf = (permissions) => [...new Set(permissions)].sort();
+/**
+ * Every object of one entity type, as an index range: `prefix` is `<type>:`, and `;` is the byte
+ * after `:`. A range, not `substr(object, …) = ?`, so an index leading with `object` can seek it.
+ */
+const typeRange = (prefix) => [prefix, `${prefix.slice(0, -1)};`];
 /**
  * The shape's grant to one person on one entity: the marker and every key, each an explicit
  * write, so a re-grant brings back what a revoke tombstoned — as `ctx.grant` does.
@@ -92,21 +137,31 @@ export function grantEntityShapeIn(db, principal, entity, permissions) {
     }
 }
 /**
- * One pass of the reconcile over one scope, at most `limit` rows of work: backfill marks for any
- * shape whose backfill is not done here, then holders whose retired keys are taken back, each
- * with its `entity.grants-retired` event, then holders topped up, each with its
- * `entity.grants-topped-up` event. `done` is false when the budget ran out, and the caller runs
- * another pass. Run it inside ONE transaction, so the keys and their events commit together.
- * Re-running a finished scope writes nothing.
+ * One pass of the reconcile over one scope, at most `limit` rows of work and
+ * `limit × SHAPE_MARKER_READS_PER_ROW` rows read (markers, and the backfill's tuples or table
+ * rows): backfill marks for any shape whose backfill
+ * is not done here, then holders whose retired keys are taken back, each with its
+ * `entity.grants-retired` event, then holders topped up, each with its `entity.grants-topped-up`
+ * event. `next` is `null` once the scope is done; otherwise the caller runs another pass with it
+ * as `after`. `retirementsLeftOpen` counts the shapes whose retirement this pass gave up on at
+ * `SHAPE_RETIRE_CONFIRMS_MAX`, for the next reconcile to run again. Run it inside ONE
+ * transaction, so the keys and their events commit together. Re-running a finished scope writes
+ * nothing.
  */
 export function topUpEntityGrantShapes(db, pass) {
-    // Here as well as at each entry point: a pass with no budget never reports done, so a caller
-    // looping until it does would never stop.
-    let budget = shapeTopUpBatch(pass.limit);
-    recordGranteeKeys(db, pass.shapes);
+    // Here as well as at each entry point: a pass with no budget never hands back `next: null`, so
+    // a caller looping until it does would never stop.
+    const limit = shapeTopUpBatch(pass.limit);
+    const room = { writes: limit, reads: limit * SHAPE_MARKER_READS_PER_ROW };
+    // The registry is the same for every pass of one reconcile, so its facts are recorded by the first.
+    if (!pass.after)
+        recordShapeFacts(db, pass.shapes);
     let toppedUp = 0;
     let retired = 0;
-    for (const shape of pass.shapes) {
+    let retirementsLeftOpen = 0;
+    const from = pass.after ?? null;
+    for (let i = from?.shape ?? 0; i < pass.shapes.length; i++) {
+        const shape = pass.shapes[i];
         // A sharing shape is never reconciled — not topped up, not backfilled, nothing retired.
         if (!shape.bootstrap)
             continue;
@@ -115,80 +170,155 @@ export function topUpEntityGrantShapes(db, pass) {
         const gone = keysOf(shape.retired ?? []).filter((k) => !keys.includes(k));
         if (keys.length === 0 && gone.length === 0)
             continue;
+        // Where this shape starts: the cursor's step if it points here, else the beginning.
+        const resume = from?.shape === i ? from : null;
+        const at = resume?.step ?? 'backfill';
+        const marker = (step) => (resume?.step === step ? resume.marker : null);
+        const stop = (step, last, more) => ({
+            toppedUp,
+            retired,
+            retirementsLeftOpen,
+            next: { shape: i, step, marker: last, ...more },
+        });
         const prefix = `${shape.entityType}:`;
-        const json = JSON.stringify(keys);
-        // A live retired key still identifies a legacy holder so this pass can retire it.
-        // The grantee query excludes tombstoned tuples.
-        budget -= backfill(db, pass, shape, prefix, JSON.stringify([...keys, ...gone]), budget);
-        if (budget === 0)
-            return { toppedUp, retired, done: false };
-        reopenRetirements(db, pass, shape.entityType, keys);
-        const took = retire(db, pass, shape.entityType, prefix, gone, budget);
-        budget -= took;
-        retired += took;
-        if (budget === 0)
-            return { toppedUp, retired, done: false };
+        if (at === 'backfill') {
+            // A live retired key still identifies a legacy holder so this pass can retire it.
+            // The grantee query excludes tombstoned tuples.
+            const read = backfill(db, pass, shape, prefix, JSON.stringify([...keys, ...gone]), room, resume?.row ?? null);
+            if (!read.done)
+                return stop('backfill', null, read.at ? { row: read.at } : undefined);
+        }
+        if (at !== 'topUp') {
+            reopenRetirements(db, pass, shape.entityType, keys);
+            let start = marker('retire');
+            // The first walk is never a confirming one; nor is a resumed walk that does not say it is.
+            let confirm = resume?.step === 'retire' ? resume.confirm : undefined;
+            for (;;) {
+                const took = retire(db, pass, shape.entityType, prefix, gone, start, room, confirm?.clean === true);
+                retired += took.found.length;
+                if (confirm)
+                    confirm = { ...confirm, clean: confirm.clean && took.found.length === 0 };
+                if (!took.done)
+                    return stop('retire', took.at, confirm ? { confirm } : undefined);
+                if (confirm?.clean)
+                    break; // `retire` recorded the retirement finished
+                // A confirming walk from the first marker, for what landed behind this one; a walk that
+                // ended here having taken keys counts against the cap if it was itself confirming.
+                const count = confirm ? confirm.count + 1 : 0;
+                if (count >= SHAPE_RETIRE_CONFIRMS_MAX) {
+                    retirementsLeftOpen++;
+                    break;
+                }
+                [start, confirm] = [null, { clean: true, count }];
+            }
+        }
         if (keys.length === 0)
             continue;
-        const holders = db.all(`SELECT m.subject, m.object FROM _substrat_tuples m
-        WHERE m.relation = ? AND ${liveTupleSql('m')}
-          AND substr(m.subject, 1, ${PRINCIPAL.length}) = '${PRINCIPAL}'
-          AND substr(m.object, 1, ?) = ?
-          AND EXISTS (SELECT 1 FROM json_each(?) k
-                       WHERE NOT EXISTS (SELECT 1 FROM _substrat_tuples t
-                                          WHERE t.subject = m.subject AND t.object = m.object
-                                            AND t.relation = 'granted:' || k.value))
-        ORDER BY m.subject, m.object
-        LIMIT ?`, ENTITY_SHAPE_MARKER_RELATION, pass.now, prefix.length, prefix, json, budget);
-        for (const h of holders) {
-            const added = db.all(`SELECT k.value AS key FROM json_each(?) k
-            WHERE NOT EXISTS (SELECT 1 FROM _substrat_tuples t
-                               WHERE t.subject = ? AND t.object = ? AND t.relation = 'granted:' || k.value)
-            ORDER BY k.value`, json, h.subject, h.object).map((m) => m.key);
-            for (const key of added) {
-                db.run(`INSERT OR IGNORE INTO _substrat_tuples (subject, relation, object) VALUES (?, ?, ?)`, h.subject, `granted:${key}`, h.object);
-            }
-            const st = kernelOutboxInsertSql(shapeEvent(pass, ENTITY_GRANTS_TOPPED_UP, entityGrantsToppedUpPayload, {
-                principal: h.subject.slice(PRINCIPAL.length),
-                entity: { entityType: shape.entityType, entityId: h.object.slice(prefix.length) },
-                added,
-            }), pass.version);
-            db.run(st.sql, ...st.params);
+        const gave = topUp(db, pass, shape.entityType, prefix, keys, marker('topUp'), room);
+        toppedUp += gave.found.length;
+        if (!gave.done)
+            return stop('topUp', gave.at);
+    }
+    return { toppedUp, retired, retirementsLeftOpen, next: null };
+}
+/** A step with nothing to walk: done, having read nothing. */
+const NOTHING_TO_WALK = { found: [], done: true, at: null };
+/**
+ * One window of a step's read, fetched in sort order with each row's `hit` computed in the SELECT
+ * rather than filtered in the WHERE, so `LIMIT window` bounds the rows read and not only the rows
+ * wanted. Takes the wanted pairs, each once, until the write budget is spent, and charges every
+ * row FETCHED to the read budget: the whole window, not only up to where the writes ran out,
+ * because SQLite evaluated `hit` on every row it returned. `done` once every row fetched was taken
+ * and the fetch came back short of the window; `last` is the last row taken, where the next
+ * window starts after.
+ */
+function takeWindow(rows, window, room) {
+    const found = [];
+    // A pair can come back on several rows of one window (a tuple per key it holds); it is one write.
+    const seen = new Set();
+    let read = 0;
+    while (read < rows.length && found.length < room.writes) {
+        const r = rows[read++];
+        const pair = `${r.subject}\n${r.object}`;
+        if (!r.hit || seen.has(pair))
+            continue;
+        seen.add(pair);
+        found.push(r);
+    }
+    room.reads -= rows.length;
+    room.writes -= found.length;
+    return { found, done: read === rows.length && rows.length < window, last: rows[read - 1] };
+}
+/**
+ * The live markers of one shape after `after`, in index order, that are `wanted` (a SQL predicate
+ * on the marker `m`, binding one parameter: `json`), as many as `room` allows, read as one
+ * {@link takeWindow}.
+ */
+function walkMarkers(db, now, prefix, after, room, wanted, json) {
+    if (room.writes === 0 || room.reads === 0)
+        return { found: [], done: false, at: after };
+    const [, end] = typeRange(prefix);
+    const from = after ?? { object: prefix, subject: '' };
+    const window = room.reads;
+    const rows = db.all(`SELECT m.object, m.subject,
+            (substr(m.subject, 1, ${PRINCIPAL.length}) = '${PRINCIPAL}' AND ${liveTupleSql('m')} AND ${wanted}) AS hit
+       FROM _substrat_tuples m
+      WHERE m.relation = '${ENTITY_SHAPE_MARKER_RELATION}' AND (m.object, m.subject) > (?, ?) AND m.object < ?
+      ORDER BY m.object, m.subject
+      LIMIT ?`, now, json, 
+    // A row-value seek: a resumed walk starts after the cursor and reads none of what it passed.
+    from.object, from.subject, end, window);
+    const { found, done, last } = takeWindow(rows, window, room);
+    return { found, done, at: last ? { object: last.object, subject: last.subject } : after };
+}
+/**
+ * One bounded walk of a shape's top-up: each live marker whose entity lacks a key of the current
+ * shape (no row at all — a tombstone is a revoke, kept) gets that key, with one event.
+ */
+function topUp(db, pass, entityType, prefix, keys, after, room) {
+    const json = JSON.stringify(keys);
+    const walk = walkMarkers(db, pass.now, prefix, after, room, `EXISTS (SELECT 1 FROM json_each(?) k
+              WHERE NOT EXISTS (SELECT 1 FROM _substrat_tuples t
+                                 WHERE t.subject = m.subject AND t.object = m.object
+                                   AND t.relation = 'granted:' || k.value))`, json);
+    for (const h of walk.found) {
+        const added = db.all(`SELECT k.value AS key FROM json_each(?) k
+          WHERE NOT EXISTS (SELECT 1 FROM _substrat_tuples t
+                             WHERE t.subject = ? AND t.object = ? AND t.relation = 'granted:' || k.value)
+          ORDER BY k.value`, json, h.subject, h.object).map((m) => m.key);
+        for (const key of added) {
+            db.run(`INSERT OR IGNORE INTO _substrat_tuples (subject, relation, object) VALUES (?, ?, ?)`, h.subject, `granted:${key}`, h.object);
         }
-        budget -= holders.length;
-        toppedUp += holders.length;
-        if (budget === 0)
-            return { toppedUp, retired, done: false };
+        const st = kernelOutboxInsertSql(shapeEvent(pass, ENTITY_GRANTS_TOPPED_UP, entityGrantsToppedUpPayload, {
+            principal: h.subject.slice(PRINCIPAL.length),
+            entity: { entityType, entityId: h.object.slice(prefix.length) },
+            added,
+        }), pass.version);
+        db.run(st.sql, ...st.params);
     }
-    return { toppedUp, retired, done: true };
+    return walk;
 }
 /**
- * One bounded batch of a shape's retirement: at most `budget` (person, entity) pairs, each a
- * live marker still holding a retired key live there, whose retired keys are tombstoned with one
- * event. Keys whose retirement already finished on this scope are skipped, and a short batch
- * records the rest finished. Returns how many pairs it took keys from.
+ * One bounded walk of a shape's retirement: each live marker still holding a retired key live
+ * there has its retired keys tombstoned, with one event. Keys whose retirement already finished
+ * on this scope are skipped. The rest are recorded finished only by a `clean` (confirming) walk
+ * that reaches the shape's last marker having taken nothing.
  */
-function retire(db, pass, entityType, prefix, gone, budget) {
+function retire(db, pass, entityType, prefix, gone, after, room, clean) {
     if (gone.length === 0)
-        return 0;
+        return NOTHING_TO_WALK;
     const [shapeRef, scopeRef] = recordRefs(entityType, pass.scopeId);
     const finished = new Set(db.all(`SELECT substr(relation, ${RETIRED_RELATION.length + 1}) AS key FROM _substrat_tuples
           WHERE subject = ? AND object = ? AND revoked_at IS NULL
             AND relation IN (SELECT '${RETIRED_RELATION}' || value FROM json_each(?))`, shapeRef, scopeRef, JSON.stringify(gone)).map((r) => r.key));
     const open = gone.filter((k) => !finished.has(k));
     if (open.length === 0)
-        return 0;
+        return NOTHING_TO_WALK;
     const json = JSON.stringify(open);
-    const holders = db.all(`SELECT m.subject, m.object FROM _substrat_tuples m
-      WHERE m.relation = ? AND ${liveTupleSql('m')}
-        AND substr(m.subject, 1, ${PRINCIPAL.length}) = '${PRINCIPAL}'
-        AND substr(m.object, 1, ?) = ?
-        AND EXISTS (SELECT 1 FROM _substrat_tuples t
-                     WHERE t.subject = m.subject AND t.object = m.object AND t.revoked_at IS NULL
-                       AND t.relation IN (SELECT 'granted:' || value FROM json_each(?)))
-      ORDER BY m.subject, m.object
-      LIMIT ?`, ENTITY_SHAPE_MARKER_RELATION, pass.now, prefix.length, prefix, json, budget);
-    for (const h of holders) {
+    const walk = walkMarkers(db, pass.now, prefix, after, room, `EXISTS (SELECT 1 FROM _substrat_tuples t
+              WHERE t.subject = m.subject AND t.object = m.object AND t.revoked_at IS NULL
+                AND t.relation IN (SELECT 'granted:' || value FROM json_each(?)))`, json);
+    for (const h of walk.found) {
         // Tombstoned and read back in one statement: the keys this holder lost are what it touched.
         const removed = keysOf(db.all(`UPDATE _substrat_tuples SET revoked_at = ?
             WHERE subject = ? AND object = ? AND revoked_at IS NULL
@@ -201,12 +331,12 @@ function retire(db, pass, entityType, prefix, gone, budget) {
         }), pass.version);
         db.run(st.sql, ...st.params);
     }
-    if (holders.length < budget) {
+    if (walk.done && clean && walk.found.length === 0) {
         for (const k of open) {
             db.run('INSERT OR REPLACE INTO _substrat_tuples (subject, relation, object) VALUES (?, ?, ?)', shapeRef, `${RETIRED_RELATION}${k}`, scopeRef);
         }
     }
-    return holders.length;
+    return walk;
 }
 /**
  * A key back in the shape ends its retirement's record (tombstoned, like any tuple), so a later
@@ -220,93 +350,225 @@ function reopenRetirements(db, pass, entityType, keys) {
         AND relation IN (SELECT '${RETIRED_RELATION}' || value FROM json_each(?))`, pass.now, ...recordRefs(entityType, pass.scopeId), JSON.stringify(keys));
 }
 /**
- * One bounded batch of the backfill for one shape — at most `budget` marks — and the run-once
- * record when the batch comes back short. Returns how many it marked. See the module comment:
- * provenance only, from the shape's declared `holder`.
+ * One window of the backfill for one shape after `after` (#2154), as one {@link takeWindow}: at
+ * most `room.reads` rows read, in an index order a cursor can seek, and at most `room.writes`
+ * marks; `done` once it read its whole range, else `at` is the last row it read. The run-once record is written only by the window that reads to the end of the range, so
+ * a backfill cut short by either budget hands back where it stopped and keeps no record. See the
+ * module comment: provenance only, from the shape's declared `holder`.
  */
-function backfill(db, pass, shape, prefix, json, budget) {
-    if (!shape.holder || budget === 0)
-        return 0;
+function backfill(db, pass, shape, prefix, json, room, after) {
+    if (!shape.holder)
+        return { done: true, at: null };
     const [shapeRef, scopeRef] = recordRefs(shape.entityType, pass.scopeId);
     const record = [shapeRef, BACKFILLED_RELATION, scopeRef];
     if (db.all('SELECT 1 FROM _substrat_tuples WHERE subject = ? AND relation = ? AND object = ?', ...record).length > 0) {
-        return 0;
+        return { done: true, at: null };
     }
+    if (room.writes === 0 || room.reads === 0)
+        return { done: false, at: after };
+    const window = room.reads;
     // Not yet a holder in any state: a tombstoned marker is a revoke of the holding, kept.
     const unmarked = (subject, object) => `NOT EXISTS (SELECT 1 FROM _substrat_tuples m WHERE m.subject = ${subject} AND m.relation = '${ENTITY_SHAPE_MARKER_RELATION}' AND m.object = ${object})`;
     const heldThere = (subject, object) => `EXISTS (SELECT 1 FROM _substrat_tuples g WHERE g.subject = ${subject} AND g.object = ${object}
                AND g.relation IN (SELECT 'granted:' || value FROM json_each(?)))`;
-    let candidates;
-    if (shape.holder === 'self') {
-        // The entity id IS the principal: `owner:<p>` belongs to `principal:<p>` and nobody else.
-        candidates = db.all(`SELECT DISTINCT t.subject, t.object FROM _substrat_tuples t
-        WHERE substr(t.object, 1, ?) = ?
-          AND t.subject = '${PRINCIPAL}' || substr(t.object, ?)
-          AND t.relation IN (SELECT 'granted:' || value FROM json_each(?))
-          AND ${unmarked('t.subject', 't.object')}
-        ORDER BY t.subject, t.object
-        LIMIT ?`, prefix.length, prefix, prefix.length + 1, json, budget);
-    }
-    else if (shape.holder === 'grantee') {
-        // Whoever holds a LIVE key of the shape on an entity of this type. Live, unlike the other two
-        // holders: with no record naming the person, a key taken back from them is no evidence they
-        // still belong there, and marking them would hand a revoked portal the next key it gains.
-        candidates = db.all(`SELECT DISTINCT t.subject, t.object FROM _substrat_tuples t
-        WHERE substr(t.object, 1, ?) = ?
-          AND substr(t.subject, 1, ${PRINCIPAL.length}) = '${PRINCIPAL}'
-          AND t.relation IN (SELECT 'granted:' || value FROM json_each(?))
-          AND ${liveTupleSql('t')}
-          AND ${unmarked('t.subject', 't.object')}
-        ORDER BY t.subject, t.object
-        LIMIT ?`, prefix.length, prefix, json, pass.now, budget);
+    const shapeKey = `t.relation IN (SELECT 'granted:' || value FROM json_each(?))`;
+    // The rows of the window, and the sort key the cursor carries for a row.
+    let rows;
+    let keyOf;
+    if (shape.holder === 'self' || shape.holder === 'grantee') {
+        // The type's tuples on `_substrat_tuples_object_relation_subject`, in its order, so the cursor
+        // is a row-value seek into the range. Every tuple of the type is read, a key of another shape
+        // or a marker included; which of them make a candidate is `hit`.
+        const [, end] = typeRange(prefix);
+        const from = after ?? [prefix, '', ''];
+        const [holds, holdsParams] = shape.holder === 'self'
+            ? // The entity id IS the principal: `owner:<p>` belongs to `principal:<p>` and nobody else.
+                [`t.subject = '${PRINCIPAL}' || substr(t.object, ${prefix.length + 1})`, []]
+            : // Whoever holds a LIVE key of the shape on an entity of this type. Live, unlike the other
+                // two holders: with no record naming the person, a key taken back from them is no evidence
+                // they still belong there, and marking them would hand a revoked portal the next key it gains.
+                [`substr(t.subject, 1, ${PRINCIPAL.length}) = '${PRINCIPAL}' AND ${liveTupleSql('t')}`, [pass.now]];
+        rows = db.all(`SELECT t.object, t.relation, t.subject, (${shapeKey} AND ${holds} AND ${unmarked('t.subject', 't.object')}) AS hit
+         FROM _substrat_tuples t
+        WHERE (t.object, t.relation, t.subject) > (?, ?, ?) AND t.object < ?
+        ORDER BY t.object, t.relation, t.subject
+        LIMIT ?`, json, ...holdsParams, ...from, end, window);
+        keyOf = (r) => [r.object, r.relation, r.subject];
     }
     else {
-        const { table, idColumn, principalColumn } = shape.holder;
-        const where = `entityGrants holder of '${shape.entityType}'`;
-        for (const [kind, name] of [['a table', table], ['a column', idColumn], ['a column', principalColumn]]) {
-            assertSqlIdentifier('reconcileEntityGrantShapes', kind, name, where);
-        }
+        const { table, idColumn, principalColumn } = holderNames(shape.holder, 'reconcileEntityGrantShapes', `entityGrants holder of '${shape.entityType}'`);
         // The vertical's table may not exist yet on this scope (a module not migrated): nothing to
         // mark, and the backfill stays open for the next reconcile rather than being recorded done.
-        if (db.all(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?`, table).length === 0)
-            return 0;
+        // Asked of SQLite, not read off the DDL's text, which keeps table options in any order and
+        // comments after them (#2159 review).
+        const listed = db.all(`SELECT wr FROM pragma_table_list WHERE schema = 'main' AND type = 'table' AND name = ?`, table)[0];
+        if (!listed)
+            return { done: true, at: null };
         const subject = `'${PRINCIPAL}' || e."${principalColumn}"`;
         const object = `? || e."${idColumn}"`;
-        candidates = db.all(`SELECT ${subject} AS subject, ${object} AS object FROM "${table}" e
-        WHERE e."${principalColumn}" IS NOT NULL
-          AND ${heldThere(subject, object)}
-          AND ${unmarked(subject, object)}
-        ORDER BY 1, 2
-        LIMIT ?`, prefix, prefix, json, prefix, budget);
+        // The table in `idColumn` order, the cursor a seek on it: the raw values, so they compare as the
+        // columns do. Nothing makes `idColumn` unique, and the principal may be NULL, so the key goes on
+        // to the row's own identity (#2159 review): rows sharing an id are never skipped at a window's
+        // edge. Bounded only where `idColumn` leads an index (an entity id is the primary key);
+        // otherwise every window sorts the whole table, which `warnUnboundedRead` reports.
+        const key = [`e."${idColumn}"`, ...rowIdentity(db, table, idColumn, Number(listed['wr']) === 1)];
+        const sql = `SELECT ${key.map((k, i) => `${k} AS k${i}`).join(', ')}, ${subject} AS subject, ${object} AS object,
+                        (e."${principalColumn}" IS NOT NULL AND ${heldThere(subject, object)} AND ${unmarked(subject, object)}) AS hit
+                   FROM "${table}" e
+                  WHERE ${after ? `(${key.join(', ')}) > (${key.map(() => '?').join(', ')})` : `e."${idColumn}" IS NOT NULL`}
+                  ORDER BY ${key.join(', ')}
+                  LIMIT ?`;
+        const params = [prefix, prefix, json, prefix, ...(after ?? []), window];
+        if (!after)
+            warnUnboundedRead(db, pass, shape.entityType, { table, idColumn }, sql, params);
+        rows = db.all(sql, ...params);
+        keyOf = (r) => key.map((_, i) => r[`k${i}`]);
     }
-    for (const c of candidates) {
+    const { found, done, last } = takeWindow(rows, window, room);
+    for (const c of found) {
         db.run('INSERT OR IGNORE INTO _substrat_tuples (subject, relation, object) VALUES (?, ?, ?)', c.subject, ENTITY_SHAPE_MARKER_RELATION, c.object);
     }
-    if (candidates.length < budget) {
+    if (done)
         db.run('INSERT OR IGNORE INTO _substrat_tuples (subject, relation, object) VALUES (?, ?, ?)', ...record);
+    return { done, at: last ? keyOf(last) : after };
+}
+/**
+ * What tells apart two rows of a holder table that share an `idColumn`, after it in the read's key:
+ * the `rowid`, unique, never NULL, and already the last column of every index on a rowid table, so
+ * an index leading with `idColumn` serves the whole key with no sort. Named by the first of its three
+ * names that no column of the table has taken: a column called `rowid` is that column, not the row's
+ * identity. A table that takes all three is refused (`validation_failed`) rather than read with a key
+ * that can tie: a rowid table's primary key may hold NULLs, so it cannot stand in. A `WITHOUT ROWID` table has
+ * none; its primary key columns are NOT NULL, unique, and end each of its indexes the same way, so
+ * they stand in, read in key order (those already `idColumn`'s left out: a key of `idColumn`
+ * alone is unique, and needs nothing after it).
+ */
+function rowIdentity(db, table, idColumn, withoutRowid) {
+    const columns = db.all('SELECT name, pk FROM pragma_table_info(?) ORDER BY pk', table).map((r) => ({ name: String(r['name']), pk: Number(r['pk']) }));
+    if (withoutRowid) {
+        return columns.filter((c) => c.pk > 0 && c.name !== idColumn).map((c) => `e."${c.name.replaceAll('"', '""')}"`);
+    }
+    const taken = new Set(columns.map((c) => c.name.toLowerCase()));
+    const alias = ROWID_NAMES.find((n) => !taken.has(n));
+    if (!alias) {
+        throw substratError('validation_failed', `reconcileEntityGrantShapes: holder table "${table}" has columns named rowid, _rowid_ and oid, so its rows cannot be told apart; rename one`, { errors: [{ path: 'holder.table', message: 'a table that leaves one of rowid, _rowid_ or oid to the row' }] });
     }
-    return candidates.length;
+    return [`e.${alias}`];
 }
+/** The names SQLite answers a rowid table's row identity to, unless a column has taken one. */
+const ROWID_NAMES = ['rowid', '_rowid_', 'oid'];
 /**
- * The current and retired keys of every `holder: 'grantee'` bootstrap shape, recorded so the
- * explicit tuple writer can refuse them to every non-shape grant. A reconcile carries the
- * reached version's whole reviewed registry,
- * so the records are made to match it exactly: a declaration a later version drops stops
- * refusing. Bounded by the declared keys, so it runs in every pass without a budget.
+ * A table holder's backfill is bounded only where SQLite reads the table in `idColumn` order off
+ * an index; where it must sort instead (`idColumn` leads no index), every window reads the whole
+ * table. The schema is the vertical's, known only on the scope, so the backfill's first window
+ * asks SQLite for the plan of the read it is about to send and writes one structured warn line
+ * when the plan sorts the whole table (`USE TEMP B-TREE FOR ORDER BY`), its keys a contract as the
+ * log lines' are. The read still runs: the backfill is correct, only unbounded. A sort of only the
+ * key's LAST TERM is not reported: an index on `(idColumn, principalColumn)` serves `idColumn` but
+ * not the row identity after it, so SQLite sorts the rows of one id at a time, which is bounded by
+ * the rows sharing an id, not by the table.
  */
-function recordGranteeKeys(db, shapes) {
-    const rows = shapes
-        .filter((s) => s.bootstrap && s.holder === 'grantee')
-        .flatMap((s) => keysOf([...s.permissions, ...(s.retired ?? [])]).map((k) => [`shape:${s.entityType}`, `granted:${k}`]));
+function warnUnboundedRead(db, pass, entityType, holder, sql, params) {
+    const plan = db.all(`EXPLAIN QUERY PLAN ${sql}`, ...params).map((r) => String(r['detail']));
+    if (!plan.some((d) => d.startsWith('USE TEMP B-TREE FOR ORDER BY')))
+        return;
+    console.warn(JSON.stringify({
+        substrat: 'shape-backfill-unbounded',
+        level: 'warn',
+        tenantId: pass.tenantId,
+        scopeId: pass.scopeId,
+        entityType,
+        ...holder,
+        message: `the entity-grant backfill of '${entityType}' sorts all of "${holder.table}" on every pass: index "${holder.idColumn}" to bound it`,
+    }));
+}
+/**
+ * A BINDING shape's record (#1686), on the scope's tuple spine beside the grantee keys: which
+ * table column names the record's person (`<table>.<idColumn>.<principalColumn>`), the key that
+ * lets someone bind or unbind them, and the shape's current keys — what a binding invite's claim
+ * gives. The claim and an unbind read these, never the vertical's code: the shape is the reviewed
+ * registry the platform last reconciled this scope with. Each identifier is a SQL identifier, so
+ * `.` cannot occur inside one.
+ */
+export const BIND_HOLDER_RELATION = 'shape-bind-holder';
+export const BIND_PERMISSION_RELATION = 'shape-bind-permission';
+export const BIND_KEY_RELATION = 'shape-bind-key';
+/**
+ * `(shape:<type>, shape-bound, bound)`: this scope has declared a binding for `<type>` (#1686).
+ * Written by the first reconcile that records the binding (and by every claim, to the same effect)
+ * and STICKY — no reconcile removes it, whatever later
+ * versions declare — so an erasure keeps releasing that type's holders after a version drops the
+ * shape, drops its `bind` or moves its holder column. A spine tuple like the others here, so
+ * module code can never write one (`ctx.sql` refuses every `_substrat_*` write).
+ */
+export const BOUND_EVER_RELATION = 'shape-bound';
+/** A shape the reconcile records as binding: bootstrap, a table holder, and a `bind`. */
+const binds = (s) => !!s.bootstrap && !!s.bind && typeof s.holder === 'object';
+/** A table holder's three names, each checked as a SQL identifier before anything interpolates it. */
+function holderNames(holder, area, where) {
+    for (const [kind, name] of [['a table', holder.table], ['a column', holder.idColumn], ['a column', holder.principalColumn]]) {
+        assertSqlIdentifier(area, kind, name, where);
+    }
+    return holder;
+}
+/**
+ * The registry facts a reconcile records for the explicit tuple writer and a binding claim, made to
+ * match the reached version's whole reviewed registry exactly, so a declaration a later version
+ * drops stops counting:
+ * - every current and retired key of each `holder: 'grantee'` or BOUND bootstrap shape
+ *   (`shape-grantee-key`), which every non-shape grant then refuses on that entity type;
+ * - each bound shape's holder column and current keys, and — only while it declares `bind` — its
+ *   bind key (#1686).
+ *
+ * A type that has ever bound (`shape-bound`) keeps its protected keys whatever later versions
+ * declare, so the keys stay refused to direct grants while its holders keep the shape; a key a
+ * later version gives the shape joins them.
+ *
+ * A shape is BOUND while it declares `bind`, and stays bound after a version drops `bind` for as
+ * long as it is still a bootstrap shape with the same table holder: records may already name their
+ * people in that column, holding the shape through it, and an erasure releases them by these
+ * facts. Without the bind key nobody can mint, claim or unbind any more; what was bound stays as a
+ * shape holder does when a key is dropped without being retired. A version that drops the shape,
+ * or changes its holder, drops the facts with it.
+ * Three statements, bounded by the declared shapes, so it runs without a budget.
+ */
+function recordShapeFacts(db, shapes) {
+    const ref = (s) => `shape:${s.entityType}`;
+    const holderOf = (s) => {
+        const { table, idColumn, principalColumn } = holderNames(s.holder, 'reconcileEntityGrantShapes', `entityGrants bind of '${s.entityType}'`);
+        return `${table}.${idColumn}.${principalColumn}`;
+    };
+    const recorded = new Map(db.all(`SELECT subject, object FROM _substrat_tuples WHERE subject >= 'shape:' AND subject < 'shape;' AND relation = ?`, BIND_HOLDER_RELATION).map((r) => [r.subject, r.object]));
+    const tableHeld = (s) => !!s.bootstrap && typeof s.holder === 'object';
+    const bound = shapes.filter(tableHeld).filter((s) => binds(s) || recorded.get(ref(s)) === holderOf(s));
+    const boundTypes = new Set(bound.map((s) => s.entityType));
+    const facts = [
+        ...shapes
+            .filter((s) => (s.bootstrap && s.holder === 'grantee') || boundTypes.has(s.entityType))
+            .flatMap((s) => keysOf([...s.permissions, ...(s.retired ?? [])]).map((k) => [ref(s), SHAPE_PROTECTED_KEY_RELATION, `granted:${k}`])),
+        ...bound.map((s) => [ref(s), BIND_HOLDER_RELATION, holderOf(s)]),
+        ...shapes.filter(binds).map((s) => [ref(s), BIND_PERMISSION_RELATION, s.bind.permission]),
+        // A declared binding is sticky from the first reconcile that records it, not only from its first
+        // claim: its keys stay refused and its holders (backfilled ones included) stay reachable by an
+        // erasure, whatever later versions declare. Never deleted: the DELETE below names other relations.
+        ...shapes.filter(binds).map((s) => [ref(s), BOUND_EVER_RELATION, 'bound']),
+        ...bound.flatMap((s) => keysOf(s.permissions).map((k) => [ref(s), BIND_KEY_RELATION, `granted:${k}`])),
+    ];
+    const json = JSON.stringify(facts);
     db.run(
     // The subject range keeps it on the primary key: `shape:` rows only, never the whole table.
-    `DELETE FROM _substrat_tuples WHERE subject >= 'shape:' AND subject < 'shape;' AND relation = ?
+    `DELETE FROM _substrat_tuples WHERE subject >= 'shape:' AND subject < 'shape;' AND relation IN (?, ?, ?, ?)
         AND NOT EXISTS (SELECT 1 FROM json_each(?) r
                          WHERE json_extract(r.value, '$[0]') = _substrat_tuples.subject
-                           AND json_extract(r.value, '$[1]') = _substrat_tuples.object)`, GRANTEE_KEY_RELATION, JSON.stringify(rows));
-    for (const [subject, object] of rows) {
-        db.run('INSERT OR IGNORE INTO _substrat_tuples (subject, relation, object) VALUES (?, ?, ?)', subject, GRANTEE_KEY_RELATION, object);
-    }
+                           AND json_extract(r.value, '$[1]') = _substrat_tuples.relation
+                           AND json_extract(r.value, '$[2]') = _substrat_tuples.object)
+        -- A type that ever bound keeps its protected keys whatever this version declares (#1686):
+        -- its holders keep the shape until released, and a direct grant of a key of it made while
+        -- the shape is out of the registry would be a holder no marker, unbind or held check sees.
+        AND NOT (relation = ? AND EXISTS (SELECT 1 FROM _substrat_tuples b
+                                           WHERE b.subject = _substrat_tuples.subject AND b.relation = ?))`, SHAPE_PROTECTED_KEY_RELATION, BIND_HOLDER_RELATION, BIND_PERMISSION_RELATION, BIND_KEY_RELATION, json, SHAPE_PROTECTED_KEY_RELATION, BOUND_EVER_RELATION);
+    db.run(`INSERT OR IGNORE INTO _substrat_tuples (subject, relation, object)
+       SELECT json_extract(value, '$[0]'), json_extract(value, '$[1]'), json_extract(value, '$[2]') FROM json_each(?)`, json);
 }
 /**
  * The kernel's event for one (person, entity) a pass changed — a top-up or a retirement — the