@harperfast/harper 5.3.0-beta.3 → 5.3.0-beta.4

package/resources/DESIGN.md

@@ -4,8 +4,6 @@ The Resource layer is Harper's universal abstraction: all queryable/mutable thin
 
 **Read this when:** you're touching the read/write path, authorization, subscriptions, or table CRUD semantics.
 
-See also: `../DESIGN.md` for cross-cutting non-obvious internals (RecordObject prototype, `getFromSource` timing, blob orphan cleanup).
-
 > **Navigation convention.** This guide references code by **symbol name** (e.g. `_writeUpdate`) and by **section marker** (e.g. `// #section: write-path-internals`). Jump in your editor via go-to-symbol, or `grep` for the section marker. Line numbers drift; symbols and section markers don't.
 
 ---
@@ -61,23 +59,23 @@ Static methods are protocol entry points (each wrapped in `transactional()`); in
 
 One giant `makeTable()` factory that returns a `TableResource extends Resource` class. The file is divided into the sections below; each is anchored by a `// #section: <name>` marker — grep for the marker (or use VS Code's go-to-symbol within the section) to land directly.
 
-| Section marker                   | Contents                                                                                                                                                                                                                                                                                                                                                                                                         |
-| -------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `#section: setup-and-factory`    | `makeTable(options)` entry, attribute parsing & primary-key detection, replication wiring, `class Updatable` (RecordObject prototype: `getUpdatedTime`, `getExpiresAt`, `addTo`, `subtractFrom`). Ends where `class TableResource` opens.                                                                                                                                                                        |
-| `#section: static-config`        | Static configuration properties: `name`, `primaryStore`, `auditStore`, `primaryKey`, `indices`, `audit`, `databasePath`, `attributes`, `replicate`, `sealed`, `splitSegments`, `getResidencyById`, `dbisDB`, `schemaDefined`, `expirationMS`.                                                                                                                                                                    |
-| `#section: resource-registry`    | `sourcedFrom()` (cache/source hierarchy — the largest static), `isCaching`, `shouldRevalidateEvents`, `getResource()`, `_updateResource`, `ensureLoaded()`.                                                                                                                                                                                                                                                      |
-| `#section: lifecycle-admin`      | `getNewId()` (UUID / autoincrement / prefix / time-based strategies), `setTTLExpiration`, residency (`getResidencyRecord`, `setResidency`, `setResidencyById`, `getResidency`), `enableAuditing`, `coerceId`, `dropTable`.                                                                                                                                                                                       |
-| `#section: read-path`            | `get()` overloads & impl.                                                                                                                                                                                                                                                                                                                                                                                        |
-| `#section: authz-hooks`          | `allowRead`, `allowUpdate`, `allowCreate`, `allowDelete`.                                                                                                                                                                                                                                                                                                                                                        |
-| `#section: write-path-public`    | `update()`, `save()`, `addTo()`, `subtractFrom()`, `getMetadata`, `getRecord`, `getChanges`, `_setChanges`, `setRecord`, `invalidate()`, `operation()`, `put()`, `create()`, `patch()`.                                                                                                                                                                                                                          |
-| `#section: write-path-internals` | **`_writeUpdate()` — the central write routine** (versioning, conflict resolution, audit, residency, replication metadata, blob orphan tracking). The `write.skipped` flag mentioned in `../DESIGN.md` is set in this method's early-return paths. Also `_writeInvalidate`, `_writeRelocate`, `_recordRelocate`, `evict()`, `lock()`/`unlock()` (record locks — see `../DESIGN.md`), `delete()`, `_writeDelete`. |
-| `isPlainOptions`                 | _(after the class)_ argument-position helper for `lock()` (distinguishes `lock(options)` from `lock(target, options)`)                                                                                                                                                                                                                                                                                           |
-| `#section: search-query`         | `search()` (the query engine — index selection, filter evaluation), `transformToOrderedSelect` (select-clause ordering), `transformEntryForSelect` (record → response shape).                                                                                                                                                                                                                                    |
-| `#section: pub-sub`              | `subscribe()` (subscription request handling, replay, cursor management), `subscribeOnThisThread`, `doesExist()`, `publish()`, `_writePublish()`.                                                                                                                                                                                                                                                                |
-| `#section: validation`           | `validate(record, patch?)` — schema enforcement, computed attributes, attribute coercion.                                                                                                                                                                                                                                                                                                                        |
-| `#section: stats-admin`          | `getUpdatedTime`, `addAttributes`, `removeAttributes`, `getSize`, `getAuditSize`, `getStorageStats`, `getRecordCount`, `updatedAttributes` (schema diff machinery).                                                                                                                                                                                                                                              |
-| `#section: computed-history`     | `setComputedAttribute`, `deleteHistory`, `getHistory` (generator), `getHistoryOfRecord`, `clear`, `cleanup`, `_readTxnForContext`.                                                                                                                                                                                                                                                                               |
-| _(after the class)_              | `getFromSource()` — cache miss → source load (see `../DESIGN.md` for the resolve-before-commit timing trap); local helpers (`coerceType`, `isDescendantId`, etc.).                                                                                                                                                                                                                                               |
+| Section marker                   | Contents                                                                                                                                                                                                                                                                                                                                                                                                              |
+| -------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `#section: setup-and-factory`    | `makeTable(options)` entry, attribute parsing & primary-key detection, replication wiring, `class Updatable` (RecordObject prototype: `getUpdatedTime`, `getExpiresAt`, `addTo`, `subtractFrom`). Ends where `class TableResource` opens.                                                                                                                                                                             |
+| `#section: static-config`        | Static configuration properties: `name`, `primaryStore`, `auditStore`, `primaryKey`, `indices`, `audit`, `databasePath`, `attributes`, `replicate`, `sealed`, `splitSegments`, `getResidencyById`, `dbisDB`, `schemaDefined`, `expirationMS`.                                                                                                                                                                         |
+| `#section: resource-registry`    | `sourcedFrom()` (cache/source hierarchy — the largest static), `isCaching`, `shouldRevalidateEvents`, `getResource()`, `_updateResource`, `ensureLoaded()`.                                                                                                                                                                                                                                                           |
+| `#section: lifecycle-admin`      | `getNewId()` (UUID / autoincrement / prefix / time-based strategies), `setTTLExpiration`, residency (`getResidencyRecord`, `setResidency`, `setResidencyById`, `getResidency`), `enableAuditing`, `coerceId`, `dropTable`.                                                                                                                                                                                            |
+| `#section: read-path`            | `get()` overloads & impl.                                                                                                                                                                                                                                                                                                                                                                                             |
+| `#section: authz-hooks`          | `allowRead`, `allowUpdate`, `allowCreate`, `allowDelete`.                                                                                                                                                                                                                                                                                                                                                             |
+| `#section: write-path-public`    | `update()`, `save()`, `addTo()`, `subtractFrom()`, `getMetadata`, `getRecord`, `getChanges`, `_setChanges`, `setRecord`, `invalidate()`, `operation()`, `put()`, `create()`, `patch()`.                                                                                                                                                                                                                               |
+| `#section: write-path-internals` | **`_writeUpdate()` — the central write routine** (versioning, conflict resolution, audit, residency, replication metadata, blob orphan tracking). The `write.skipped` flag ("Blob orphan cleanup" below) is set in this method's early-return paths. Also `_writeInvalidate`, `_writeRelocate`, `_recordRelocate`, `evict()`, `lock()`/`unlock()` (record locks — see `record-locks.md`), `delete()`, `_writeDelete`. |
+| `isPlainOptions`                 | _(after the class)_ argument-position helper for `lock()` (distinguishes `lock(options)` from `lock(target, options)`)                                                                                                                                                                                                                                                                                                |
+| `#section: search-query`         | `search()` (the query engine — index selection, filter evaluation), `transformToOrderedSelect` (select-clause ordering), `transformEntryForSelect` (record → response shape).                                                                                                                                                                                                                                         |
+| `#section: pub-sub`              | `subscribe()` (subscription request handling, replay, cursor management), `subscribeOnThisThread`, `doesExist()`, `publish()`, `_writePublish()`.                                                                                                                                                                                                                                                                     |
+| `#section: validation`           | `validate(record, patch?)` — schema enforcement, computed attributes, attribute coercion.                                                                                                                                                                                                                                                                                                                             |
+| `#section: stats-admin`          | `getUpdatedTime`, `addAttributes`, `removeAttributes`, `getSize`, `getAuditSize`, `getStorageStats`, `getRecordCount`, `updatedAttributes` (schema diff machinery).                                                                                                                                                                                                                                                   |
+| `#section: computed-history`     | `setComputedAttribute`, `deleteHistory`, `getHistory` (generator), `getHistoryOfRecord`, `clear`, `cleanup`, `_readTxnForContext`.                                                                                                                                                                                                                                                                                    |
+| _(after the class)_              | `getFromSource()` — cache miss → source load (see "getFromSource() timing" below for the resolve-before-commit trap); local helpers (`coerceType`, `isDescendantId`, etc.).                                                                                                                                                                                                                                           |
 
 ---
 
@@ -94,7 +92,7 @@ One giant `makeTable()` factory that returns a `TableResource extends Resource`
 | Where is record-level TTL evaluated?                                           | `Table.ts → setTTLExpiration` (`#section: lifecycle-admin`); `Updatable.getExpiresAt` (`#section: setup-and-factory`). Stored expiry metadata is resolved in the `_writeUpdate` commit closure: `options.expiresAt ?? context.expiresAt ?? (record @expiresAt field, if finite &amp; ≥ 0) ?? table default`. This metadata drives read-hiding + the cleanup sweep. The `@expiresAt` attribute is authoritative for **direct** put/patch only; cache/source fills persist via `recordUpdater` and derive expiry from `sourceContext.expiresAt` (source freshness / table default), not the field.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
 | Why does `search()` hide a row that's past its TTL but not yet swept?          | `Table.ts → transformEntryForSelect` unconditionally treats `entry.expiresAt < Date.now()` as gone (lazy eviction on read) — correct for a SELECT, but a mutation locating rows to overwrite needs the opposite: pass `target.includeExpired = true` (read by the SQL engine's `runUpdate`/`runDelete` via `SqlEngineContext.includeExpiredRows`) to treat such a row as a live match, matching the leniency a direct by-id `put`/`patch` already has (they skip this check entirely, since `Resource.patch`'s static options don't request `ensureLoaded`).                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
 | How are residencies enforced (replication)?                                    | `Table.ts → #section: lifecycle-admin` (residency block: `getResidencyRecord`, `setResidency`, `setResidencyById`, `getResidency`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
-| How is the RecordObject prototype applied?                                     | `RecordEncoder.ts` (see `../DESIGN.md`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
+| How is the RecordObject prototype applied?                                     | `RecordEncoder.ts` (see "RecordObject prototype and entryMap" below)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     |
 | Where is the per-request transaction stored?                                   | `transaction.ts` + `contextStorage` (AsyncLocalStorage)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
 | What is in `context.transaction` when no scope owns one?                       | Whatever the last table call left there. A completed scope leaves `RELEASED_TRANSACTION` (`DatabaseTransaction.ts`), which reads latest and no-ops on commit. The next table call finds no transaction and `Table.ts → txnForContext` installs an `ImmediateTransaction` in the slot. It reports `open === OPEN`, but `transaction()` and `Resource`'s dispatcher gate on `isJoinableScope` — OPEN _and_ staging its writes for a later commit — so they never join it and start their own scope instead (#2292); `txnForContext` keeps returning it for reads and for writes that reach no wrapper, and those commit per write. It is also the only transaction that opens its native handle inside its own `commit()` (its `getReadTxn` never opens one), which is why `commit()` re-reads `this.transaction` after its save loop (#2288).                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
 | How do I find a transaction that is holding write intents and is never reaped? | Three log surfaces, joined by the native transaction id (`longLivedTransactions.ts`, #2471). `runLongLivedTransactionSweep()` runs on the **main thread only** — rocksdb-js's registry is process-global, so one sweep sees every worker's handles — and names any handle open past `storage.longTransactionReportThreshold` (default `5m`, `0` disables all three), including handles no `DatabaseTransaction` owns. Changing the threshold clears the accrued backoff, so a handle already under observation is re-measured against the new value. `reportIfLongLived()` in each worker's monitor reports every link holding its own handle under that link's own id, adding `startedFrom`, the table, and the state keeping it alive (`source-apply`, `replay`, `commit-phase`, `active` when written in the current or preceding monitor tick), which is what the reap branches below it deliberately do not do. Attribution lines are capped at 10 per monitor tick; a summary names deferred holders, whose due backoff remains intact for a later tick. Attribution suppression is keyed on database path, native id, and handle-open time, so a reattached descriptor id cannot inherit its predecessor's backoff. The two stuck-commit logs (`checkOverloaded()` and `abandonCommitAfterDeadline()`) append holder candidates from **every** database, the commit's own first and then by age — the verification table is one process-global slot array, so a holder in another database parks this commit at the same rate — never filtered by age, because a coordinated retry can park on a transaction younger than itself. |
@@ -209,13 +207,12 @@ first and a throw from it is what stops the prune.
 | `Table.deleteHistory`                                                        | LMDB (`RocksTransactionLogStore.remove()` is a no-op, so it must NOT raise) |
 | `delete_transaction_logs_before` whole-database branch (`ResourceBridge.ts`) | RocksDB                                                                     |
 
-Seven things that are easy to get wrong here:
+Things that are easy to get wrong here:
 
 - **The floor cannot be derived from the surviving log.** For four of the five paths the oldest
   surviving entry would do, because they prune a database-wide time prefix. `Table.deleteHistory`
-  does not: it removes one table's entries out of a database-scoped log, so a sibling's entry
-  survives _below_ the newest entry it removed. Measured, LMDB: highest removed `…147.797`, oldest
-  surviving `…143.309` — a log-derived floor would have certified a cursor at `…145`.
+  removes one table's entries from a database-scoped log, so a sibling's entry survives _below_ the
+  newest entry it removed, and a floor taken from that survivor certifies cursors over removed history.
 - **The record's presence is the trust marker.** `Symbol.for('audit-floor')` is a different key from
   `last-removed`, which is still live and still maintained by the LMDB retention loop (#2338 hardened
   its write path and added tests for the retry-carry — do not remove it). They coexist because they
@@ -368,7 +365,7 @@ Tests: `../unitTests/resources/defineResource.test.js`, `../unitTests/resources/
   - **The override makes the access-control decision.** No `allow*` predicate runs for it. An override that is not meant to be public must check authorization itself.
 - **FK-side relationship accessors (`relationship.from`) are a synchronous contract** (v4 parity). The resolvers built in `Table.updatedAttributes` must read through `getSync`/`getEntry` (which block on a RocksDB block-cache miss, like v4's LMDB page faults) — never `get()`, whose `async: true` path returns a Promise on a cache miss and would leak an intermittent value-or-Promise to `record.<relation>` in user code. Pinned by the "relationship property access is synchronous" test in `../unitTests/resources/query.test.js`, which stubs `store.get` to throw.
   - **Known gap, not covered by that test:** the `.to` side (one-to-many/many-to-many, resolved via `relatedTable.search(...).asArray`) is synchronous for a local read, but `asArray` can still hand back a Promise when the search lands on a **caching** table whose row needs source revalidation — `transformEntryForSelect` returns `loadingFromSource.then(transform)` for an expired/invalidated entry. Same bug class this PR closes on the FK side; not yet fixed here.
-- When adding a new early-return path inside a commit handler in `_writeUpdate`, follow the blob-cleanup protocol documented in `../DESIGN.md` ("Blob orphan cleanup").
+- When adding a new early-return path inside a commit handler in `_writeUpdate`, follow the blob-cleanup protocol in "Blob orphan cleanup" below.
 - If you add a new top-level section to `Table.ts`, drop a `// #section: <name>` marker at its start and add a row to the section map above.
 - Tests for this layer live in `../unitTests/resources/`.
 
@@ -564,28 +561,15 @@ Without this, every re-delivery appended another entry under the origin's log ke
 
 ## A second sequential save() on the same ImmediateTransaction context must chain on `operation.innerCommit`
 
-Two `update()`+`save()` cycles on the _same resource instance_, outside an explicit `transaction()`,
-reuse the same `ImmediateTransaction` object even after its first cycle has closed it (`this.open =
-CLOSED`). The second `save()` re-enters `ImmediateTransaction.save()` with `isCommitting` false, so it
-calls `this.commit()` again; that `commit()`'s own sweep loop calls `this.save(newWrite, ...)` — a
-**polymorphic re-dispatch to `ImmediateTransaction.save()`**, now with `isCommitting` true, which takes
-the `super.save(operation, transaction, true)` branch with no handle to forward (a CLOSED context has
-none) and so creates its own
-brand-new `RocksTransaction` and immediately commits it, stashing the real commit promise on
-`operation.innerCommit`. But the outer `commit()`'s sweep loop discards the return value of
-`this.save(operation, ...)` for every write it processes — that's fine when the write commits inline,
-but this reused-context write's real work happens in a _third_, more deeply nested `commit()` call
-(triggered by `immediateCommit` inside the nested `save()`), whose promise never propagates back through
-any of the enclosing calls. `Table.save()`'s ordinary `#savingOperation` path used to just return
-`#saveOperation(operation)`'s result directly — which can resolve before that nested native commit
-actually settles, so a caller's `await resource.save()` can return before the write is durable (a real,
-if narrow, race: `LockTest.get()` immediately after can read the pre-write value). The lock-writable hold
-branch already avoided this by explicitly returning `operation.innerCommit` after its own recursive
-`save()`; the ordinary path now does the same — `when(this.#saveOperation(operation), () =>
-operation.innerCommit)`. `operation.innerCommit` is `undefined` when a write commits inline (no
-immediateCommit), so this is safe for the common case. Found via record-lock scoped-lock staging
-(harper#483), which is what first made this reused-closed-context pattern reachable for an ordinary
-resource, but the gap is general to `Table.save()`, not lock-specific.
+Two `update()`+`save()` cycles on one resource instance outside `transaction()` reuse one
+`ImmediateTransaction` after its first cycle has closed it. The second write then commits through a
+nested `commit()` on a fresh native transaction (the `immediateCommit` branch of
+`DatabaseTransaction.save()`), whose promise the outer commit's sweep loop discards; that branch
+stashes it on `operation.innerCommit`, which is `undefined` for a write that commits inline. Both
+`Table.save()` branches — the ordinary `#savingOperation` path and the lock-writable hold path — wait
+on it, or `await resource.save()` returns before the write is durable and an immediate read sees the
+old value. The gap is general to `Table.save()`, not lock-specific. Exercised by `recordLock.test.js`
+"scoped lock outside transaction() persists across sequential saves until unlock()".
 
 ## A commit retry re-saves into the transaction it is retrying; `ImmediateTransaction.save()` must forward it
 
@@ -704,7 +688,7 @@ The additive-only rule above repairs the _consumer_ of a partial peer snapshot;
 
 ## Subscription version selection (`Table.ts`)
 
-`Table.subscribe` resolves audit events through `eventFromAudit` for both replay and live delivery: unless `includeSuperseded` (defaulting to `rawEvents`) is enabled, record mutations must match the current primary version; an absent primary entry cannot establish a current version and is skipped. Publishing advances that primary version too: a put followed by a publish replays the message but not the put under the default rule. Messages and control events are independent. Version filtering precedes historical reconstruction and `previousCount` acceptance, while audit cursors advance over rejected entries. Single-record replay retains its bounded history walk because it also contains independent published messages; only an accepted replay event suppresses the current snapshot, and `eventFilter` also applies to that fallback. Default replay reads the current primary entry for each mutation. Opting into superseded full records retains audit-history reconstruction costs and retention limits, including patch reconstruction on durable MQTT live delivery.
+`Table.subscribe` resolves audit events through `eventFromAudit` for both replay and live delivery: unless `includeSuperseded` (defaulting to `rawEvents`) is enabled, record mutations must match the current primary version; an absent primary entry cannot establish a current version and is skipped. Publishing advances that primary version too: a put followed by a publish replays the message but not the put under the default rule. Messages and control events are independent. Version filtering precedes historical reconstruction and `previousCount` acceptance, while audit cursors advance over rejected entries. Single-record replay retains its bounded history walk because it also contains independent published messages; only an accepted replay event suppresses the current snapshot, and `eventFilter` also applies to that fallback. Default replay reads the current primary entry for each mutation. Opting into superseded full records retains audit-history reconstruction costs and retention limits, including patch reconstruction on durable MQTT live delivery. Collection `startTime` replay leaves `subscription.startTime` — the exclusive resume cursor `DurableSubscriptionsSession.saveSubscriptions` persists — never past a key with an unhandled in-table record: RocksDB gives every record of a transaction one `txnLogKey`, so the cursor moves to a key only on reaching the next key or finishing, and an early return (failed send, close at a yield or drain) leaves it up to one transaction back, which the resume re-sends (`subscriptionReplay.test.js`, "cursor after replay stops early"). The cursor counts events handed to the subscription queue, not consumed ones; `close()` discards the queue.
 
 ## Audit-store `'committed'` notification batching (`transactionBroadcast.ts`)
 
@@ -718,20 +702,27 @@ The cross-thread subscription path (default `crossThreads`) drives every `Table.
 ## Audit-entry removal loops must track every `removeAuditEntry()`/`removeEntry()` promise
 
 `scheduleAuditCleanup` (`auditStore.ts`) and `Table.deleteHistory` (`Table.ts`, the LMDB path behind
-`delete_transaction_logs_before`) both iterate a range of audit records and remove each one. Both were
-originally written as `completion = removeAuditEntry(auditStore, auditRecord)` inside the loop, awaiting
-only the final iteration's promise afterward. Any rejection from a non-last iteration was silently
-discarded — the promise reference was overwritten before it could be awaited or caught — and surfaced
-later as an unhandled rejection instead, with no logging to explain it. Any loop that removes
-audit/primary-store entries in a batch must attach a rejection handler to every removal immediately
-and drain all tracked promises before returning — never stash a per-iteration promise in an outer
-variable to await only the last one. `Table.deleteHistory` allows up to 1,000 LMDB removals in flight
+`delete_transaction_logs_before`) both iterate a range of audit records and remove each one. Any loop
+that removes audit/primary-store entries in a batch must attach a rejection handler to every removal
+immediately and drain all tracked promises before returning — never stash a per-iteration promise in
+an outer variable to await only the last one: an overwritten promise's rejection is never awaited or
+caught, and surfaces later as an unhandled rejection with no log to explain it. `Table.deleteHistory`
+allows up to 1,000 LMDB removals in flight
 (ten for RocksDB) so storage writes batch without growing an unbounded pending
 set. Live removals are tracked in a `Set`, and each one removes itself and wakes at most one parked
 producer when it settles, so any completion releases the loop. In these removal loops, do not repeatedly
 race the live set: each race attaches another reaction to every long-pending removal. Both phases drain
 their tracked removals before settling, including when iteration throws. `scheduleAuditCleanup` remains
-sequential because it is an automatic background loop.
+sequential because it is an automatic background loop, and it ends a pass at its first failed removal
+rather than continuing past it: its `deleted` count (the backoff input) and the `last-removed` marker it
+writes then cover a contiguous removed prefix — nothing above that marker was removed by the pass — and
+the next pass retries the failed entry first. Past a failure there is no key the marker can truthfully
+record, and counting failures as progress re-armed an all-failing pass at 10 ms. The guarantee is
+pass-local: the marker is written after its removals, `deleteHistory` never writes it, and a marker an
+older build persisted past a failed entry is not repaired; completeness is the audit floor's question.
+Stopping costs liveness only for an entry that fails on every pass, which is why `removeAuditEntry` must
+fail only when `auditStore.remove()` does (below). Regression: `auditLog.test.js` "ends a pass at a failed
+removal".
 
 Individual removal failures are logged and excluded from the returned count, but a purge that attempted
 at least one removal and completed none rejects with the first error after both phases have drained.
@@ -749,9 +740,11 @@ followed by an unconditional remove: a record recreated between those operations
 `removeAuditEntry` has a second, nested version of the same hazard: for a `'delete'`-type audit record it
 also invokes a per-table delete callback (`addDeleteRemovalCallback`) that removes the corresponding
 primary-store tombstone. That callback's promise must be returned and joined with the audit-store
-removal (currently via `Promise.all`, with the callback's own rejection caught and logged separately so
-a failed tombstone cleanup doesn't get misreported as a failed audit-entry removal) — otherwise the
-tombstone removal is fire-and-forget and the same detached-rejection hazard reappears one level down.
+removal (currently via `Promise.all`, with the callback's own rejection — and a throwing tombstone
+lookup, such as lmdb-js `getEntry(undefined)` for an undecodable recordId — caught and logged through
+`warnContained`, so neither a failed tombstone cleanup nor a throwing log sink gets misreported as a failed
+audit-entry removal) — otherwise the tombstone removal is fire-and-forget and the same detached-rejection
+hazard reappears one level down.
 A tombstone whose cleanup fails this way is not swept automatically — `scheduleAuditCleanup`'s automatic
 pass never retries it, since the audit entry that would have triggered a retry is already gone. It sits
 in the primary store until an operator runs `delete_transaction_logs_before` with `cleanup_deleted_records: true`.
@@ -779,11 +772,9 @@ of the exported `scheduleAuditCleanup`, because a store-wide segment purge loopi
 duplicated work. If a future change passes `skipThreadCheck: true` at the registration site, that
 backstop — not the registration — becomes the thing keeping the loop single.
 
-Both re-arm guards are **Rocks-only**, deliberately: the LMDB arm keeps `origin/main`'s unconditional
-re-arm, so it neither yields to an already-pending pass (a pressure-armed 100ms pass can be cancelled
-and replaced by the idle backoff) nor restricts itself to one worker. Those are pre-existing LMDB
-behaviours, not invariants this section establishes — don't read the paragraphs above as
-engine-independent.
+Both re-arm guards are **Rocks-only**. The LMDB arm re-arms unconditionally, so it neither yields to
+an already-pending pass (a pressure-armed 100ms pass can be cancelled and replaced by the idle
+backoff) nor restricts itself to one worker — pre-existing LMDB behavior, not an invariant.
 
 Two things a purge does **not** need to coordinate, both load-bearing for the continuous cadence.
 Unlinking a segment a consumer has mapped is safe **on POSIX**: the inode outlives the unlink, and the
@@ -808,11 +799,11 @@ number into its write instruction synchronously and the native writer consumes i
 (`node_modules/lmdb/write.js`), so a pass suspended inside `await removeAuditEntry()` still has a
 delete pending against the primary and audit DBIs, and LMDB forbids closing a DBI an existing
 transaction has modified. `dropDatabase()` and the legacy arm of `Table.dropTable()` await it.
-`closeDatabase()` and branch `close()` are synchronous and cannot; what covers them is that every
-environment touch remaining in a resumed pass — cursor advance, cursor release, marker write, re-arm —
-re-checks `rootStore.status`, plus the fact that their production callers reach them only for RocksDB
-stores, whose pass is one synchronous `purgeLogs()` call with nothing suspended mid-removal.
-`resetDatabases()` closes LMDB roots with no retirement call at all, so that re-check is a routine
+`closeDatabase()` and branch `close()` close commit admission, then drain tracked Rocks transactions,
+table maintenance, audit cleanup, and derived-index work before closing stores. Every environment
+touch remaining in a resumed cleanup pass — cursor advance, cursor release, marker write, re-arm —
+also re-checks `rootStore.status`. `resetDatabases()` closes LMDB roots with no retirement call at all,
+so that re-check is a routine
 path rather than a defensive one.
 
 The last-removed marker is retained until it commits. A rejected write is logged and carried to the
@@ -824,13 +815,7 @@ were already removed.
 
 When a blob attribute is created from a Node `Readable` (e.g. `createBlob(stream)` then `row.payload_blob = blob; await table.put(row)`), the put does **not** wait for the underlying stream to fully drain into the file before resolving. Internally `saveBlob` kicks off a `writeBlobWithStream` pipeline whose `storageInfo.saving` promise is tracked separately. The put resolves once encoding has captured the blob reference; the bytes finish writing concurrently.
 
-Consequence for callers that wrap the source in a hashing `Transform`: calling `hash.digest('hex')` after `await table.put()` is unsafe — more `chunk.update()` calls can still fire as the stream drains, producing `Error [ERR_CRYPTO_HASH_FINALIZED]: Digest already called`. Options:
-
-- Buffer first, then hash + put (what `components/deploymentRecorder.ts` does for Slice A — small payloads only).
-- Hash via Transform while extraction reads the stream, and only finalize the hash on the Transform's `'end'` event before any second put with the final hash.
-- Await `storageInfo.saving` directly if you have a handle to the FileBackedBlob (the cleanest path for streaming).
-
-Future agents touching `components/deploymentRecorder.ts` for Slice B's streaming variant should pick one of the latter two patterns.
+Consequence for callers that wrap the source in a hashing `Transform`: calling `hash.digest('hex')` after `await table.put()` is unsafe — more `chunk.update()` calls can still fire as the stream drains, producing `Error [ERR_CRYPTO_HASH_FINALIZED]: Digest already called`. Finalize the hash only after the source has ended and the blob's `storageInfo.saving` has settled, as `components/deploymentRecorder.ts`'s streaming ingest does (`await Promise.all([putDone, tapDone, saving])` before `digest`). Its in-memory sources, and its capped fallback for a missing deployment table, buffer and hash before the put instead.
 
 ## Table drops, the `dropping` tombstone, and ghost tables
 
@@ -847,29 +832,24 @@ silently re-opened with create-if-missing on the next start, which resurrects "d
 
 ## The exclusive `update-attributes` lock is a bounded synchronous wait, and drop-then-recreate needs the column-family eviction fix (`Table.ts`)
 
-Two related traps: the create/schema-update path's exclusive `update-attributes` lock is a
-synchronous bounded wait (`acquireUpdateAttributesLock` in `Table.ts`: brief hot spin, then
-`Atomics.wait` backoff, retryable `ServerError` after the 10s `UPDATE_ATTRIBUTES_LOCK_TIMEOUT` — harper#2251; it used to be
-an unbounded `while (!tryLock()) {}` spin that pinned a worker core forever if the holder never
-released). Release is structural — `table()` releases in a single `finally` and `dropTable` uses
-`withUpdateAttributesLock` — so a throw inside the locked window cannot leak the lock (regression
-suite: `unitTests/resources/updateAttributesLock.test.js`). Because the acquire can now throw,
-`table()` takes the RocksDB lock _before_ it mutates the live `Table` (attributes, class metadata,
-index handles): losing the race then leaves this worker's in-memory schema exactly as it found it,
-and moving any mutation above that acquire reintroduces schema drift the catalog never saw. LMDB
-keeps the lazy acquire — its `exclusiveLock()` is an environment-wide write transaction that cannot
-time out, so taking it eagerly would stall every write to the database on an unchanged reload. A
-successful acquire that waited past `UPDATE_ATTRIBUTES_LOCK_SLOW_WAIT` (1s) warns once, since
-contention is otherwise invisible until it becomes a timeout. The locked
-sections MUST stay synchronous: the wait blocks the event loop, so an awaited operation inside
-one would stall a concurrent acquirer to its deadline. And dropping then recreating a
-same-named table within one process requires @harperfast/rocksdb-js >= the column-family
-eviction fix (2.1.0 / rocksdb-js#647): older bindings keep the dropped column family's
-by-name registry entry alive whenever other worker threads hold handles, so the recreate
-silently reuses a dangling handle and every write fails with "Invalid column family specified
-in write batch", poisoning the whole database env until restart. The regression suite for all
-of this is `unitTests/resources/dropTableGhost.test.js` (it fails by design on pre-fix
-bindings).
+The create/schema-update path's exclusive `update-attributes` lock is a synchronous bounded wait
+(`acquireUpdateAttributesLock` in `Table.ts`: brief hot spin, then `Atomics.wait` backoff, retryable
+`ServerError` after the 10s `UPDATE_ATTRIBUTES_LOCK_TIMEOUT`, harper#2251). Release is structural —
+`table()` releases in a single `finally` and `dropTable` uses `withUpdateAttributesLock` — so a throw
+inside the locked window cannot leak the lock (`unitTests/resources/updateAttributesLock.test.js`).
+Because the acquire can throw, `table()` takes the RocksDB lock _before_ it mutates the live `Table`
+(attributes, class metadata, index handles): losing the race then leaves this worker's in-memory
+schema exactly as it found it, and moving any mutation above that acquire reintroduces schema drift
+the catalog never saw. LMDB keeps the lazy acquire — its `exclusiveLock()` is an environment-wide
+write transaction that cannot time out, so taking it eagerly would stall every write to the database
+on an unchanged reload. A successful acquire that waited past `UPDATE_ATTRIBUTES_LOCK_SLOW_WAIT` (1s)
+warns once, since contention is otherwise invisible until it becomes a timeout. The locked sections
+MUST stay synchronous: the wait blocks the event loop, so an awaited operation inside one would stall
+a concurrent acquirer to its deadline. Dropping then recreating a same-named table within one process
+depends on rocksdb-js's column-family eviction fix (≥ 2.1.0, rocksdb-js#647; `package.json` pins a
+later release): without it the recreate reuses a dangling handle and every write fails with "Invalid
+column family specified in write batch". Regression suite: `unitTests/resources/dropTableGhost.test.js`,
+which fails by design on pre-fix bindings.
 
 ## RocksDB transaction log purges are database-wide only (`ResourceBridge.deleteTransactionLogsBefore`)
 

package/resources/DatabaseTransaction.ts

@@ -1,14 +1,19 @@
 import { cleanupUnusedBlobs, collectRetainedFileIds } from './blob.ts';
 import { Transaction as LMDBTransaction } from 'lmdb';
 import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts';
-import { ServerError, TransactionCommitConflictTimeoutError } from '../utility/errors/hdbError.ts';
+import {
+	DatabaseClosingError,
+	DatabaseDrainTimeoutError,
+	ServerError,
+	TransactionCommitConflictTimeoutError,
+} from '../utility/errors/hdbError.ts';
 import { lockNotHeldError, type RecordLockHandle } from './recordLock.ts';
 import * as harperLogger from '../utility/logging/harper_logger.ts';
 import type { Context, Id } from './ResourceInterface.ts';
 import * as envMngr from '../utility/environment/environmentManager.ts';
 import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
 import { convertToMS } from '../utility/common_utils.ts';
-import { when } from '../utility/when.ts';
+import { settleBeforeDeadline, when } from '../utility/when.ts';
 import { setTimeout as delay } from 'node:timers/promises';
 import { Transaction as RocksTransaction, type Store as RocksStore, constants } from '@harperfast/rocksdb-js';
 const RETRY_NOW_VALUE = constants.RETRY_NOW_VALUE;
@@ -65,6 +70,7 @@ interface OutstandingCommit {
 	start: number;
 	prev: OutstandingCommit | undefined;
 	next: OutstandingCommit | undefined;
+	commitResolution: Promise<number | void>;
 	// Identity for the one-time checkOverloaded() log below (harper#2001) — otherwise a stuck commit
 	// gives no indication of which database/table/resource to investigate. Snapshotted at arm time
 	// (not read from `this.writes`/`this.startedFrom` lazily off the DatabaseTransaction object)
@@ -74,6 +80,7 @@ interface OutstandingCommit {
 	// so that every outstanding commit carries its own identity and its own `logged` flag: a second
 	// commit that is still stuck once the first settles becomes the new oldest and logs on its own.
 	store: any;
+	rootStore: any;
 	startedFrom: { resourceName: string; method: string } | undefined;
 	nativeTransaction: any;
 	logged: boolean;
@@ -81,6 +88,10 @@ interface OutstandingCommit {
 let oldestOutstandingCommit: OutstandingCommit | undefined;
 let newestOutstandingCommit: OutstandingCommit | undefined;
 let outstandingCommitCount = 0;
+const suspendedDatabaseCommits = new WeakMap<object, number>();
+const permanentlySuspended = Symbol('permanentlySuspendedDatabaseCommits');
+let suspendedDatabaseRootCount = 0;
+let hasPermanentlySuspendedDatabaseRoots = false;
 // Caps the stuck-commit log (checkOverloaded() below) to at most one line per this interval across
 // the whole thread, regardless of how many distinct commits individually cross the threshold — see
 // the comment at the log site for why a per-commit-only dedup isn't enough under sustained overload.
@@ -101,11 +112,12 @@ function allowStuckCommitLog(site: 'shed' | 'abandon', now: number): boolean {
 function describeCommitIdentity(
 	store: any,
 	startedFrom: { resourceName: string; method: string } | undefined,
-	nativeTransaction: any
+	nativeTransaction: any,
+	rootStore = store?.rootStore
 ): string {
 	const nativeTransactionId = nativeTransaction?.id;
 	return (
-		`from table: ${store?.rootStore?.databaseName ?? '?'}.${store?.name ?? '?'}` +
+		`from table: ${rootStore?.databaseName ?? '?'}.${store?.name ?? '?'}` +
 		(nativeTransactionId !== undefined ? ` (transaction ${nativeTransactionId})` : '') +
 		(startedFrom?.resourceName
 			? `, started from ${startedFrom.resourceName}${startedFrom.method ? '.' + startedFrom.method : ''}`
@@ -121,9 +133,9 @@ function describeCommitIdentity(
 // from its own submission, keeping the overload window per-attempt rather than cumulative over a
 // retry ladder. `.then(untrack, untrack)` also marks an ERR_BUSY rejection handled, so this
 // tracking never surfaces as an unhandled rejection alongside the caller's own handler.
-// Exported only so unit tests can drive the list with controllable promises: the unlink order that
-// matters (a middle or tail node settling first) cannot be forced through real writes, and a node
-// left linked would 503 every write on this thread forever. commit() below is the sole caller.
+// Exported so raw RocksDB writers and unit tests share the same queue. The unlink order that matters
+// (a middle or tail node settling first) cannot be forced through real writes, and a node left linked
+// would 503 every write on this thread forever.
 // Also the single source for write-queue-depth accounting (getTransactionQueueDepths below): every
 // native commit this function tracks is, by definition, exactly the write-queue backlog — see the
 // comment there for why that used to be a second, separately-maintained counter.
@@ -133,7 +145,8 @@ export function trackOutstandingCommit(
 	commitResolution: Promise<number | void>,
 	store?: any,
 	startedFrom?: { resourceName: string; method: string },
-	nativeTransaction?: any
+	nativeTransaction?: any,
+	rootStore = store?.rootStore
 ): void {
 	// Guards against a future caller passing a non-Promise: today commit() always hands this a real
 	// Promise, but an unguarded link here would leave a node permanently wedged in the list (503ing
@@ -143,7 +156,9 @@ export function trackOutstandingCommit(
 		start: performance.now(),
 		prev: newestOutstandingCommit,
 		next: undefined,
+		commitResolution,
 		store,
+		rootStore,
 		startedFrom,
 		nativeTransaction,
 		logged: false,
@@ -172,6 +187,120 @@ export function trackOutstandingCommit(
 	commitResolution.then(untrack, untrack);
 }
 
+export function databaseCommitsSuspended(rootStore: object | undefined): boolean {
+	if (rootStore == null) return false;
+	if ((rootStore as any).status === 'closed') return true;
+	if (suspendedDatabaseRootCount === 0 && !hasPermanentlySuspendedDatabaseRoots) return false;
+	if ((rootStore as any)[permanentlySuspended]) return true;
+	return suspendedDatabaseRootCount > 0 && (suspendedDatabaseCommits.get(rootStore) ?? 0) > 0;
+}
+
+/** Keep abandoned root wrappers fenced without retaining the process-wide active-fence counter. */
+export function permanentlySuspendDatabaseCommits(rootStores: Iterable<object>): void {
+	for (const rootStore of rootStores) {
+		if ((rootStore as any)[permanentlySuspended]) continue;
+		const suspension = suspendedDatabaseCommits.get(rootStore) ?? 0;
+		if (suspension > 0) suspendedDatabaseRootCount--;
+		suspendedDatabaseCommits.delete(rootStore);
+		(rootStore as any)[permanentlySuspended] = true;
+		hasPermanentlySuspendedDatabaseRoots = true;
+	}
+}
+
+export function getSuspendedDatabaseRootCount(): number {
+	return suspendedDatabaseRootCount;
+}
+
+export function commitTrackedRocksTransaction(
+	transaction: RocksTransaction,
+	store: any,
+	rootStore = store?.rootStore
+): Promise<number | void> {
+	if (databaseCommitsSuspended(rootStore)) {
+		try {
+			transaction.abort();
+		} catch {}
+		return Promise.reject(new DatabaseClosingError(rootStore?.databaseName ?? rootStore?.path ?? 'unknown'));
+	}
+	let commitResolution: Promise<number | void>;
+	try {
+		commitResolution = transaction.commit() as Promise<number | void>;
+	} catch (error) {
+		return Promise.reject(error);
+	}
+	trackOutstandingCommit(commitResolution, store, undefined, transaction, rootStore);
+	return commitResolution;
+}
+
+let databaseCommitDrainTimeoutMilliseconds = 120_000;
+
+export function getDatabaseCommitDrainTimeoutMilliseconds(): number {
+	return databaseCommitDrainTimeoutMilliseconds;
+}
+
+export function setDatabaseCommitDrainTimeoutMilliseconds(timeoutMilliseconds: number): number {
+	const previous = databaseCommitDrainTimeoutMilliseconds;
+	databaseCommitDrainTimeoutMilliseconds = timeoutMilliseconds;
+	return previous;
+}
+
+/**
+ * Stop new native commits for these database roots and wait for submissions already in flight.
+ * The returned release is reference-counted and idempotent so overlapping lifecycle operations
+ * cannot reopen commit admission underneath each other.
+ */
+export function suspendDatabaseCommits(rootStores: Iterable<object>): {
+	waitForDrain(options?: { deadline?: number; databaseName?: string }): Promise<void>;
+	release(): void;
+} {
+	const roots = [...new Set(rootStores)];
+	const rootSet = new Set(roots);
+	for (const rootStore of roots) {
+		if ((rootStore as any)[permanentlySuspended]) continue;
+		const count = suspendedDatabaseCommits.get(rootStore) ?? 0;
+		if (count === 0) suspendedDatabaseRootCount++;
+		suspendedDatabaseCommits.set(rootStore, count + 1);
+	}
+	let released = false;
+	return {
+		async waitForDrain(options = {}) {
+			const timeoutMilliseconds =
+				options.deadline === undefined
+					? databaseCommitDrainTimeoutMilliseconds
+					: Math.max(0, options.deadline - Date.now());
+			const deadline = options.deadline ?? Date.now() + timeoutMilliseconds;
+			for (;;) {
+				const pending: Promise<number | void>[] = [];
+				for (let outstanding = oldestOutstandingCommit; outstanding; outstanding = outstanding.next) {
+					if (rootSet.has(outstanding.rootStore)) pending.push(outstanding.commitResolution);
+				}
+				if (pending.length === 0) return;
+				await settleBeforeDeadline(
+					pending,
+					deadline,
+					() =>
+						new DatabaseDrainTimeoutError(
+							options.databaseName ?? (roots[0] as any)?.databaseName ?? (roots[0] as any)?.path ?? 'unknown',
+							timeoutMilliseconds
+						)
+				);
+			}
+		},
+		release() {
+			if (released) return;
+			released = true;
+			for (const rootStore of roots) {
+				if ((rootStore as any)[permanentlySuspended]) continue;
+				const count = suspendedDatabaseCommits.get(rootStore) ?? 0;
+				if (count <= 1) {
+					suspendedDatabaseCommits.delete(rootStore);
+					suspendedDatabaseRootCount--;
+				} else suspendedDatabaseCommits.set(rootStore, count - 1);
+			}
+		},
+	};
+}
+
 /**
  * How many write commits are outstanding on this thread and how long the oldest has been waiting
  * (`oldestAgeMs` is undefined when none is). `oldestAgeMs` is exactly the value checkOverloaded()
@@ -1027,11 +1156,12 @@ export class DatabaseTransaction implements Transaction {
 						describeCommitIdentity(
 							oldestOutstandingCommit.store,
 							oldestOutstandingCommit.startedFrom,
-							oldestOutstandingCommit.nativeTransaction
+							oldestOutstandingCommit.nativeTransaction,
+							oldestOutstandingCommit.rootStore
 						) +
 						`.` +
 						describeHolderCandidates(
-							oldestOutstandingCommit.store?.rootStore?.path,
+							oldestOutstandingCommit.rootStore?.path,
 							oldestOutstandingCommit.nativeTransaction?.id
 						) +
 						` Further record updates and publishes from new application requests on this thread ` +
@@ -1398,6 +1528,7 @@ export class DatabaseTransaction implements Transaction {
 						}
 						// with options.transaction set this is a retry round — the save loop above already
 						// re-staged the writes into it
+						this.assertDatabaseCommitAllowed(transaction);
 						commitResolution = transaction.commit() as Promise<void>;
 						recordCommitLatency(commitResolution, performance.now());
 						// Write-queue-depth accounting for this replay commit happens uniformly below, via
@@ -1457,6 +1588,7 @@ export class DatabaseTransaction implements Transaction {
 							// getReadTxn), so commit() can resolve to RETRY_NOW_VALUE. That
 							// sentinel (a number) is why commitResolution is typed
 							// Promise<number | void>; it is handled in the resolve callback below.
+							this.assertDatabaseCommitAllowed(transaction);
 							commitResolution = transaction.commit();
 							// Record how long this commit stays outstanding (submit → settle) as a distribution
 							// metric. This is the same clock the overload check uses (trackOutstandingCommit
@@ -1795,6 +1927,20 @@ export class DatabaseTransaction implements Transaction {
 		this.writesAbandoned = false;
 	}
 
+	private assertDatabaseCommitAllowed(transaction: RocksTransaction): void {
+		const rootStore = this.writes[0]?.store?.rootStore;
+		if (!databaseCommitsSuspended(rootStore)) return;
+		const retryable = !this.root && !this.snapshotFree;
+		if (this.transaction === transaction) this.detachOwnedTransaction();
+		abortNativeTransaction(transaction, 'aborting a transaction while its database is closing');
+		try {
+			this.abort();
+		} catch (error) {
+			harperLogger.debug?.('cleaning up a transaction while its database is closing', error);
+		}
+		throw new DatabaseClosingError(rootStore?.databaseName ?? rootStore?.path ?? 'unknown', retryable);
+	}
+
 	abort(): void {
 		while (this.readTxnsUsed > 0) this.doneReadTxn(); // release the read snapshot when we abort, we assume we don't need it
 		// Defensively release any native handle whose reference bookkeeping was already consumed.
@@ -1967,6 +2113,7 @@ export class DatabaseTransaction implements Transaction {
 	}
 	directCommitSync(): void {
 		const transaction = this.transaction;
+		if (transaction) this.assertDatabaseCommitAllowed(transaction);
 		try {
 			transaction?.commitSync();
 		} catch (error) {