@harperfast/harper 5.2.10 → 5.3.0-alpha.1

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@harperfast/harper",
 	"description": "Harper is an open-source Node.js performance platform that unifies database, cache, application, and messaging layers into one in-memory process.",
-	"version": "5.2.10",
+	"version": "5.3.0-alpha.1",
 	"license": "Apache-2.0",
 	"homepage": "https://harper.fast",
 	"bugs": {
@@ -84,6 +84,7 @@
 		"test:unit:utility": "mocha \"unitTests/utility/**/*.js\" --exclude \"unitTests/utility/logging/**/*\"",
 		"test:unit:server": "mocha \"unitTests/server/**/*.js\"",
 		"test:unit:config": "mocha \"unitTests/config/**/*.js\"",
+		"test:unit:windows": "node unitTests/windowsGate.mjs",
 		"test:unit:typestrip": "env NODE_OPTIONS=\"--conditions=typestrip\" npm run test:unit",
 		"test:unit:typestrip:all": "npm run test:unit:typestrip unitTests"
 	},
@@ -150,7 +151,7 @@
 		"chai": "^6.2.0",
 		"chai-as-promised": "^8.0.2",
 		"chai-integer": "^0.1.0",
-		"eventsource": "^4.0.0",
+		"eventsource": "^5.0.0",
 		"globals": "^17.0.0",
 		"intercept-stdout": "0.1.2",
 		"mkcert": "^3.2.0",
@@ -172,14 +173,14 @@
 	},
 	"dependencies": {
 		"@aws-sdk/client-s3": "^3.1012.0",
-		"@aws-sdk/lib-storage": "3.1106.0",
+		"@aws-sdk/lib-storage": "3.1116.0",
 		"@endo/static-module-record": "^1.1.2",
 		"@fastify/autoload": "^6.3.1",
 		"@fastify/compress": "^8.3.1",
 		"@fastify/cors": "^11.2.0",
 		"@fastify/static": "^9.1.3",
 		"@harperfast/extended-iterable": "1.0.3",
-		"@harperfast/rocksdb-js": "2.9.0",
+		"@harperfast/rocksdb-js": "2.8.0",
 		"@harperfast/skills": "^1.10.8",
 		"@turf/area": "6.5.0",
 		"@turf/boolean-contains": "6.5.0",
@@ -190,7 +191,7 @@
 		"@turf/distance": "6.5.0",
 		"@turf/helpers": "6.5.0",
 		"@turf/length": "6.5.0",
-		"alasql": "4.17.3",
+		"alasql": "4.19.0",
 		"amaro": "^1.1.8",
 		"argon2": "0.45.1",
 		"asn1js": "3.0.10",
@@ -212,7 +213,7 @@
 		"human-readable-ids": "1.0.4",
 		"inquirer": "8.2.7",
 		"is-number": "7.0.0",
-		"joi": "17.13.4",
+		"joi": "17.13.6",
 		"json-bigint-fixes": "1.1.0",
 		"jsonata": "1.8.9",
 		"jsonwebtoken": "9.0.3",
@@ -230,7 +231,7 @@
 		"normalize-path": "^3.0.0",
 		"ora": "8.2.0",
 		"ordered-binary": "1.6.1",
-		"papaparse": "5.5.4",
+		"papaparse": "5.6.0",
 		"passport": "0.7.0",
 		"passport-http": "0.3.0",
 		"passport-local": "1.0.0",

package/resources/DESIGN.md

@@ -12,23 +12,24 @@ See also: `../DESIGN.md` for cross-cutting non-obvious internals (RecordObject p
 
 ## File overview
 
-| File                    | Purpose                                                                                                                              |
-| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------ |
-| `Resource.ts`           | Base class; `transactional()` wrapper; method routing                                                                                |
-| `Table.ts`              | Table-as-Resource implementation. Factory `makeTable()` returns a `TableResource` subclass per table. **See section markers below.** |
-| `Resources.ts`          | Registry mapping URL paths → Resource classes                                                                                        |
-| `RequestTarget.ts`      | Parses path/query into a structured target                                                                                           |
-| `ResourceInterface.ts`  | Type definitions (`Context`, `Record`, etc.)                                                                                         |
-| `RecordEncoder.ts`      | msgpack encoding + `entryMap` (record → storage entry)                                                                               |
-| `IterableEventQueue.ts` | Async iterable used for subscriptions and streaming responses                                                                        |
-| `transaction.ts`        | Per-request transaction object stored in `contextStorage`                                                                            |
-| `auditStore.ts`         | Append-only audit log records                                                                                                        |
-| `nodeIdMapping.ts`      | Maps node IDs ↔ timestamps for replication ordering                                                                                  |
-| `openApi.ts`            | Generates OpenAPI/JSON Schema from `@export` schemas                                                                                 |
-| `defineTable.ts`        | Code-first table authoring (`defineTable` + `types`) — a TS front-end to the canonical `table()` model                               |
-| `defineResource.ts`     | Per-method request contract (`defineResource` / `Resource.withSchema`, `t`, `schemaOf`) — typed handlers + edge validation           |
-| `jsonSchemaTypes.ts`    | Shared `JsonSchemaFragment` IR + `attributeToFragment` projector (one vocabulary for validation/OpenAPI/MCP)                         |
-| `analytics/`            | Telemetry recording (separate from monitoring)                                                                                       |
+| File                    | Purpose                                                                                                                                      |
+| ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- |
+| `Resource.ts`           | Base class; `transactional()` wrapper; method routing                                                                                        |
+| `Table.ts`              | Table-as-Resource implementation. Factory `makeTable()` returns a `TableResource` subclass per table. **See section markers below.**         |
+| `Resources.ts`          | Registry mapping URL paths → Resource classes                                                                                                |
+| `RequestTarget.ts`      | Parses path/query into a structured target                                                                                                   |
+| `ResourceInterface.ts`  | Type definitions (`Context`, `Record`, etc.)                                                                                                 |
+| `RecordEncoder.ts`      | msgpack encoding + `entryMap` (record → storage entry)                                                                                       |
+| `IterableEventQueue.ts` | Async iterable used for subscriptions and streaming responses                                                                                |
+| `transaction.ts`        | Per-request transaction object stored in `contextStorage`                                                                                    |
+| `auditStore.ts`         | Append-only audit log records                                                                                                                |
+| `recordLock.ts`         | Exclusive record locks (harper#483): option contract, native key lock primitives (`lockAttemptKey`, `makeKeyLockHandle`, `acquireRecordKey`) |
+| `nodeIdMapping.ts`      | Maps node IDs ↔ timestamps for replication ordering                                                                                          |
+| `openApi.ts`            | Generates OpenAPI/JSON Schema from `@export` schemas                                                                                         |
+| `defineTable.ts`        | Code-first table authoring (`defineTable` + `types`) — a TS front-end to the canonical `table()` model                                       |
+| `defineResource.ts`     | Per-method request contract (`defineResource` / `Resource.withSchema`, `t`, `schemaOf`) — typed handlers + edge validation                   |
+| `jsonSchemaTypes.ts`    | Shared `JsonSchemaFragment` IR + `attributeToFragment` projector (one vocabulary for validation/OpenAPI/MCP)                                 |
+| `analytics/`            | Telemetry recording (separate from monitoring)                                                                                               |
 
 ---
 
@@ -57,46 +58,93 @@ Static methods are protocol entry points (each wrapped in `transactional()`); in
 
 One giant `makeTable()` factory that returns a `TableResource extends Resource` class. The file is divided into the sections below; each is anchored by a `// #section: <name>` marker — grep for the marker (or use VS Code's go-to-symbol within the section) to land directly.
 
-| Section marker                   | Contents                                                                                                                                                                                                                                                                                                                                                          |
-| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `#section: setup-and-factory`    | `makeTable(options)` entry, attribute parsing & primary-key detection, replication wiring, `class Updatable` (RecordObject prototype: `getUpdatedTime`, `getExpiresAt`, `addTo`, `subtractFrom`). Ends where `class TableResource` opens.                                                                                                                         |
-| `#section: static-config`        | Static configuration properties: `name`, `primaryStore`, `auditStore`, `primaryKey`, `indices`, `audit`, `databasePath`, `attributes`, `replicate`, `sealed`, `splitSegments`, `getResidencyById`, `dbisDB`, `schemaDefined`, `expirationMS`.                                                                                                                     |
-| `#section: resource-registry`    | `sourcedFrom()` (cache/source hierarchy — the largest static), `isCaching`, `shouldRevalidateEvents`, `getResource()`, `_updateResource`, `ensureLoaded()`.                                                                                                                                                                                                       |
-| `#section: lifecycle-admin`      | `getNewId()` (UUID / autoincrement / prefix / time-based strategies), `setTTLExpiration`, residency (`getResidencyRecord`, `setResidency`, `setResidencyById`, `getResidency`), `enableAuditing`, `coerceId`, `dropTable`.                                                                                                                                        |
-| `#section: read-path`            | `get()` overloads & impl.                                                                                                                                                                                                                                                                                                                                         |
-| `#section: authz-hooks`          | `allowRead`, `allowUpdate`, `allowCreate`, `allowDelete`.                                                                                                                                                                                                                                                                                                         |
-| `#section: write-path-public`    | `update()`, `save()`, `addTo()`, `subtractFrom()`, `getMetadata`, `getRecord`, `getChanges`, `_setChanges`, `setRecord`, `invalidate()`, `operation()`, `put()`, `create()`, `patch()`.                                                                                                                                                                           |
-| `#section: write-path-internals` | **`_writeUpdate()` — the central write routine** (versioning, conflict resolution, audit, residency, replication metadata, blob orphan tracking). The `write.skipped` flag mentioned in `../DESIGN.md` is set in this method's early-return paths. Also `_writeInvalidate`, `_writeRelocate`, `_recordRelocate`, `evict()`, `lock()`, `delete()`, `_writeDelete`. |
-| `#section: search-query`         | `search()` (the query engine — index selection, filter evaluation), `transformToOrderedSelect` (select-clause ordering), `transformEntryForSelect` (record → response shape).                                                                                                                                                                                     |
-| `#section: pub-sub`              | `subscribe()` (subscription request handling, replay, cursor management), `subscribeOnThisThread`, `doesExist()`, `publish()`, `_writePublish()`.                                                                                                                                                                                                                 |
-| `#section: validation`           | `validate(record, patch?)` — schema enforcement, computed attributes, attribute coercion.                                                                                                                                                                                                                                                                         |
-| `#section: stats-admin`          | `getUpdatedTime`, `addAttributes`, `removeAttributes`, `getSize`, `getAuditSize`, `getStorageStats`, `getRecordCount`, `updatedAttributes` (schema diff machinery).                                                                                                                                                                                               |
-| `#section: computed-history`     | `setComputedAttribute`, `deleteHistory`, `getHistory` (generator), `getHistoryOfRecord`, `clear`, `cleanup`, `_readTxnForContext`.                                                                                                                                                                                                                                |
-| _(after the class)_              | `getFromSource()` — cache miss → source load (see `../DESIGN.md` for the resolve-before-commit timing trap); local helpers (`coerceType`, `isDescendantId`, etc.).                                                                                                                                                                                                |
+| Section marker                   | Contents                                                                                                                                                                                                                                                                                                                                                                                                         |
+| -------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `#section: setup-and-factory`    | `makeTable(options)` entry, attribute parsing & primary-key detection, replication wiring, `class Updatable` (RecordObject prototype: `getUpdatedTime`, `getExpiresAt`, `addTo`, `subtractFrom`). Ends where `class TableResource` opens.                                                                                                                                                                        |
+| `#section: static-config`        | Static configuration properties: `name`, `primaryStore`, `auditStore`, `primaryKey`, `indices`, `audit`, `databasePath`, `attributes`, `replicate`, `sealed`, `splitSegments`, `getResidencyById`, `dbisDB`, `schemaDefined`, `expirationMS`.                                                                                                                                                                    |
+| `#section: resource-registry`    | `sourcedFrom()` (cache/source hierarchy — the largest static), `isCaching`, `shouldRevalidateEvents`, `getResource()`, `_updateResource`, `ensureLoaded()`.                                                                                                                                                                                                                                                      |
+| `#section: lifecycle-admin`      | `getNewId()` (UUID / autoincrement / prefix / time-based strategies), `setTTLExpiration`, residency (`getResidencyRecord`, `setResidency`, `setResidencyById`, `getResidency`), `enableAuditing`, `coerceId`, `dropTable`.                                                                                                                                                                                       |
+| `#section: read-path`            | `get()` overloads & impl.                                                                                                                                                                                                                                                                                                                                                                                        |
+| `#section: authz-hooks`          | `allowRead`, `allowUpdate`, `allowCreate`, `allowDelete`.                                                                                                                                                                                                                                                                                                                                                        |
+| `#section: write-path-public`    | `update()`, `save()`, `addTo()`, `subtractFrom()`, `getMetadata`, `getRecord`, `getChanges`, `_setChanges`, `setRecord`, `invalidate()`, `operation()`, `put()`, `create()`, `patch()`.                                                                                                                                                                                                                          |
+| `#section: write-path-internals` | **`_writeUpdate()` — the central write routine** (versioning, conflict resolution, audit, residency, replication metadata, blob orphan tracking). The `write.skipped` flag mentioned in `../DESIGN.md` is set in this method's early-return paths. Also `_writeInvalidate`, `_writeRelocate`, `_recordRelocate`, `evict()`, `lock()`/`unlock()` (record locks — see `../DESIGN.md`), `delete()`, `_writeDelete`. |
+| `isPlainOptions`                 | _(after the class)_ argument-position helper for `lock()` (distinguishes `lock(options)` from `lock(target, options)`)                                                                                                                                                                                                                                                                                           |
+| `#section: search-query`         | `search()` (the query engine — index selection, filter evaluation), `transformToOrderedSelect` (select-clause ordering), `transformEntryForSelect` (record → response shape).                                                                                                                                                                                                                                    |
+| `#section: pub-sub`              | `subscribe()` (subscription request handling, replay, cursor management), `subscribeOnThisThread`, `doesExist()`, `publish()`, `_writePublish()`.                                                                                                                                                                                                                                                                |
+| `#section: validation`           | `validate(record, patch?)` — schema enforcement, computed attributes, attribute coercion.                                                                                                                                                                                                                                                                                                                        |
+| `#section: stats-admin`          | `getUpdatedTime`, `addAttributes`, `removeAttributes`, `getSize`, `getAuditSize`, `getStorageStats`, `getRecordCount`, `updatedAttributes` (schema diff machinery).                                                                                                                                                                                                                                              |
+| `#section: computed-history`     | `setComputedAttribute`, `deleteHistory`, `getHistory` (generator), `getHistoryOfRecord`, `clear`, `cleanup`, `_readTxnForContext`.                                                                                                                                                                                                                                                                               |
+| _(after the class)_              | `getFromSource()` — cache miss → source load (see `../DESIGN.md` for the resolve-before-commit timing trap); local helpers (`coerceType`, `isDescendantId`, etc.).                                                                                                                                                                                                                                               |
 
 ---
 
 ## "Where is X" cheat sheet
 
-| Question                                                              | Where                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
-| --------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| How is a CRUD request authorized?                                     | `Table.ts → #section: authz-hooks`; defaults in `Resource.ts` (`allowRead` etc.)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
-| Where does versioning / conflict resolution happen?                   | `Table.ts → _writeUpdate` (`#section: write-path-internals`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
-| How does `search()` choose an index?                                  | `Table.ts → search` (`#section: search-query`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
-| How are subscriptions replayed?                                       | `Table.ts → subscribe` (`#section: pub-sub`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
-| How is the response body shaped (select clause)?                      | `Table.ts → transformEntryForSelect` (`#section: search-query`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
-| Where is record-level TTL evaluated?                                  | `Table.ts → setTTLExpiration` (`#section: lifecycle-admin`); `Updatable.getExpiresAt` (`#section: setup-and-factory`). Stored expiry metadata is resolved in the `_writeUpdate` commit closure: `options.expiresAt ?? context.expiresAt ?? (record @expiresAt field, if finite &amp; ≥ 0) ?? table default`. This metadata drives read-hiding + the cleanup sweep. The `@expiresAt` attribute is authoritative for **direct** put/patch only; cache/source fills persist via `recordUpdater` and derive expiry from `sourceContext.expiresAt` (source freshness / table default), not the field.                                                                                                                                                                                                                                             |
-| Why does `search()` hide a row that's past its TTL but not yet swept? | `Table.ts → transformEntryForSelect` unconditionally treats `entry.expiresAt < Date.now()` as gone (lazy eviction on read) — correct for a SELECT, but a mutation locating rows to overwrite needs the opposite: pass `target.includeExpired = true` (read by the SQL engine's `runUpdate`/`runDelete` via `SqlEngineContext.includeExpiredRows`) to treat such a row as a live match, matching the leniency a direct by-id `put`/`patch` already has (they skip this check entirely, since `Resource.patch`'s static options don't request `ensureLoaded`).                                                                                                                                                                                                                                                                                 |
-| How are residencies enforced (replication)?                           | `Table.ts → #section: lifecycle-admin` (residency block: `getResidencyRecord`, `setResidency`, `setResidencyById`, `getResidency`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
-| How is the RecordObject prototype applied?                            | `RecordEncoder.ts` (see `../DESIGN.md`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
-| Where is the per-request transaction stored?                          | `transaction.ts` + `contextStorage` (AsyncLocalStorage)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
-| What is in `context.transaction` when no scope owns one?              | Whatever the last table call left there. A completed scope leaves `RELEASED_TRANSACTION` (`DatabaseTransaction.ts`), which reads latest and no-ops on commit. The next table call finds no transaction and `Table.ts → txnForContext` installs an `ImmediateTransaction` in the slot. It reports `open === OPEN`, but `transaction()` and `Resource`'s dispatcher gate on `isJoinableScope` — OPEN _and_ staging its writes for a later commit — so they never join it and start their own scope instead (#2292); `txnForContext` keeps returning it for reads and for writes that reach no wrapper, and those commit per write. It is also the only transaction that opens its native handle inside its own `commit()` (its `getReadTxn` never opens one), which is why `commit()` re-reads `this.transaction` after its save loop (#2288). |
-| How does a query opt out of a read snapshot?                          | Pass `snapshot: false` on the search request (e.g. `get_analytics`). `Table.ts → search` calls `txn.useReadTxn(snapshot === false)`; on RocksDB `DatabaseTransaction.getReadTxn` then builds the read txn with `{ disableSnapshot: true }` so a long scan reads latest without pinning a snapshot. No-op on LMDB (`LMDBTransaction.useReadTxn`).                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
-| How does a URL path map to a Resource?                                | `Resources.ts → getMatch` (exact/prefix fast path) then `matchParamRoute` (parameterised routes); see "Path routing" below                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
-| How does HNSW keep the graph connected on delete?                     | `indexes/HierarchicalNavigableSmallWorld.ts → index()` delete path: zero-degree orphans reindexed via `needsReindexing`; severed multi-node islands detected and reconnected by `repairSeveredNeighbors` (#1712)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
-| How is a filter applied _during_ a vector search?                     | Predicate-aware traversal (#1241): `search.ts → executeConditions` composes companion AND conditions with request `vectorFilter` / `rowFilter` predicates into one `(primaryKey) => boolean` (`composeRecordFilter`) and passes it to `HierarchicalNavigableSmallWorld.search(cond, ctx, filter)`. The filter gates result admission at layer 0 only (routing ignores it, ACORN-style); a visit budget (`filterExpansion`) bounds the under-filled/selective case. Very selective _condition_ filters are instead diverted to the exact brute-force path by the query planner's `estimateCountAsSort` ordering.                                                                                                                                                                                                                              |
-| How does post-ordering resolve vector distances safely?               | Each comparator owns its `Sort`, passes it directly to the custom-index resolver, and caches distances by that immutable per-query sort object.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
-| How is application row filtering applied?                             | Authorization admission happens in the resource operation before query work. The legacy `allow*` hook, when armed by the protocol, is evaluated once with its historical receiver semantics; overriding it never changes its scope. An operation override may add indexed conditions and/or attach the JavaScript-only synchronous `target.rowFilter(record, context)`. `Table.search` composes it with query filters and rechecks the final materialized cache/source record. `SubscriptionRequest.rowFilter` covers full-row events; `eventFilter(event, context)` explicitly handles tombstones/messages/raw events. Prefer indexed conditions because an opaque predicate may inspect every admitted candidate and `limit` applies after filtering.                                                                                      |
+| Question                                                                       | Where                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
+| ------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| How is a CRUD request authorized?                                              | `Table.ts → #section: authz-hooks`; defaults in `Resource.ts` (`allowRead` etc.)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| Where does versioning / conflict resolution happen?                            | `Table.ts → _writeUpdate` (`#section: write-path-internals`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| How does `search()` choose an index?                                           | `Table.ts → search` (`#section: search-query`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
+| How are subscriptions replayed?                                                | `Table.ts → subscribe` (`#section: pub-sub`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| How is the response body shaped (select clause)?                               | `Table.ts → transformEntryForSelect` (`#section: search-query`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| Where is record-level TTL evaluated?                                           | `Table.ts → setTTLExpiration` (`#section: lifecycle-admin`); `Updatable.getExpiresAt` (`#section: setup-and-factory`). Stored expiry metadata is resolved in the `_writeUpdate` commit closure: `options.expiresAt ?? context.expiresAt ?? (record @expiresAt field, if finite &amp; ≥ 0) ?? table default`. This metadata drives read-hiding + the cleanup sweep. The `@expiresAt` attribute is authoritative for **direct** put/patch only; cache/source fills persist via `recordUpdater` and derive expiry from `sourceContext.expiresAt` (source freshness / table default), not the field.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| Why does `search()` hide a row that's past its TTL but not yet swept?          | `Table.ts → transformEntryForSelect` unconditionally treats `entry.expiresAt < Date.now()` as gone (lazy eviction on read) — correct for a SELECT, but a mutation locating rows to overwrite needs the opposite: pass `target.includeExpired = true` (read by the SQL engine's `runUpdate`/`runDelete` via `SqlEngineContext.includeExpiredRows`) to treat such a row as a live match, matching the leniency a direct by-id `put`/`patch` already has (they skip this check entirely, since `Resource.patch`'s static options don't request `ensureLoaded`).                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| How are residencies enforced (replication)?                                    | `Table.ts → #section: lifecycle-admin` (residency block: `getResidencyRecord`, `setResidency`, `setResidencyById`, `getResidency`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
+| How is the RecordObject prototype applied?                                     | `RecordEncoder.ts` (see `../DESIGN.md`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
+| Where is the per-request transaction stored?                                   | `transaction.ts` + `contextStorage` (AsyncLocalStorage)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
+| What is in `context.transaction` when no scope owns one?                       | Whatever the last table call left there. A completed scope leaves `RELEASED_TRANSACTION` (`DatabaseTransaction.ts`), which reads latest and no-ops on commit. The next table call finds no transaction and `Table.ts → txnForContext` installs an `ImmediateTransaction` in the slot. It reports `open === OPEN`, but `transaction()` and `Resource`'s dispatcher gate on `isJoinableScope` — OPEN _and_ staging its writes for a later commit — so they never join it and start their own scope instead (#2292); `txnForContext` keeps returning it for reads and for writes that reach no wrapper, and those commit per write. It is also the only transaction that opens its native handle inside its own `commit()` (its `getReadTxn` never opens one), which is why `commit()` re-reads `this.transaction` after its save loop (#2288).                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| How do I find a transaction that is holding write intents and is never reaped? | Three log surfaces, joined by the native transaction id (`longLivedTransactions.ts`, #2471). `runLongLivedTransactionSweep()` runs on the **main thread only** — rocksdb-js's registry is process-global, so one sweep sees every worker's handles — and names any handle open past `storage.longTransactionReportThreshold` (default `5m`, `0` disables all three), including handles no `DatabaseTransaction` owns. Changing the threshold clears the accrued backoff, so a handle already under observation is re-measured against the new value. `reportIfLongLived()` in each worker's monitor reports every link holding its own handle under that link's own id, adding `startedFrom`, the table, and the state keeping it alive (`source-apply`, `replay`, `commit-phase`, `active` when written in the current or preceding monitor tick), which is what the reap branches below it deliberately do not do. Attribution lines are capped at 10 per monitor tick; a summary names deferred holders, whose due backoff remains intact for a later tick. Attribution suppression is keyed on database path, native id, and handle-open time, so a reattached descriptor id cannot inherit its predecessor's backoff. The two stuck-commit logs (`checkOverloaded()` and `abandonCommitAfterDeadline()`) append holder candidates from **every** database, the commit's own first and then by age — the verification table is one process-global slot array, so a holder in another database parks this commit at the same rate — never filtered by age, because a coordinated retry can park on a transaction younger than itself. |
+| How does a query opt out of a read snapshot?                                   | Pass `snapshot: false` on the search request (e.g. `get_analytics`). `Table.ts → search` calls `txn.useReadTxn(snapshot === false)`; on RocksDB `DatabaseTransaction.getReadTxn` then builds the read txn with `{ disableSnapshot: true }` so a long scan reads latest without pinning a snapshot. No-op on LMDB (`LMDBTransaction.useReadTxn`).                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| How does a URL path map to a Resource?                                         | `Resources.ts → getMatch` (exact/prefix fast path) then `matchParamRoute` (parameterised routes); see "Path routing" below                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
+| What do chained conditions (`a=ge=X&=le=Y`) mean over array values?            | SAME-ELEMENT scoping: `prepareConditions` (`Table.ts → search`) collapses the chain into one range comparator (`gele`/`gtlt`/…) before execution, so the indexed path (per-element index entries, one range scan) and the unindexed path (`search.ts → attributeComparator`, per-element `some` over the collapsed predicate) agree. Repeating the attribute as two independent conditions is independently existential (different elements may satisfy different legs). Only a single `and`-chained leg is supported — `\|=` and a second `&=` are rejected. An indexed scan whose range spans more than one indexed value collapses to one result per record before paging (`search.ts → distinctRecords`, #2434), so `limit`/`offset` count records rather than index entries; element equality stays uncollapsed because `[indexedValue, primaryKey]` is already unique. Pinned by `unitTests/resources/query-array-scoping.test.js`; known gaps: chained-leg values are never type-coerced (#2433), an undeclared (untyped) indexed attribute holding an array still repeats per entry, error paths (#2435).                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
+| How does HNSW keep the graph connected on delete?                              | `indexes/HierarchicalNavigableSmallWorld.ts → index()` delete path: zero-degree orphans reindexed via `needsReindexing`; severed multi-node islands detected and reconnected by `repairSeveredNeighbors` (#1712)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| How is a filter applied _during_ a vector search?                              | Predicate-aware traversal (#1241): `search.ts → executeConditions` composes companion AND conditions with request `vectorFilter` / `rowFilter` predicates into one `(primaryKey) => boolean` (`composeRecordFilter`) and passes it to `HierarchicalNavigableSmallWorld.search(cond, ctx, filter)`. The filter gates result admission at layer 0 only (routing ignores it, ACORN-style); a visit budget (`filterExpansion`) bounds the under-filled/selective case. Very selective _condition_ filters are instead diverted to the exact brute-force path by the query planner's `estimateCountAsSort` ordering.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| How does post-ordering resolve vector distances safely?                        | Each comparator owns its `Sort`, passes it directly to the custom-index resolver, and caches distances by that immutable per-query sort object.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| How is application row filtering applied?                                      | Authorization admission happens in the resource operation before query work. The legacy `allow*` hook, when armed by the protocol, is evaluated once with its historical receiver semantics; overriding it never changes its scope. An operation override may add indexed conditions and/or attach the JavaScript-only synchronous `target.rowFilter(record, context)`. `Table.search` composes it with query filters and rechecks the final materialized cache/source record. `SubscriptionRequest.rowFilter` covers full-row events; `eventFilter(event, context)` explicitly handles tombstones/messages/raw events. Prefer indexed conditions because an opaque predicate may inspect every admitted candidate and `limit` applies after filtering.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
+
+**An audit record carries two clocks; never substitute one for the other (harper#2412 stage 0b).**
+`AuditRecord.version` is the originating write's record version — LWW ordering in
+`precedesExistingVersion`, `@updatedTime`, ETag/`Last-Modified`. Historical audit-only entries may
+carry the surviving version instead. `AuditRecord.txnLogKey` is that
+entry's key in the per-origin transaction log: write identity, the record→log lookup
+(`auditStore.get(logKey, tableId, id, nodeId)`), and every resume cursor. On LMDB these have always
+been distinct fields; on RocksDB the read surface used to overwrite `version` with the log key, and
+`#2409`'s `recordVersion` alias is now absorbed back into `version`.
+
+They hold the same value for every write whose record version is its own commit timestamp, which is
+every ordinary local write, so a bug that confuses them stays invisible until a **source fill**
+(`getFromSource`, core #2065): the record is stored at the source-reported version while its log
+entry is keyed at the fill's commit. A RocksDB record stores its version in the existing word and,
+when the clocks diverge, keeps its audit head in `additionalAuditRefs`; stage 2 can replace that
+compatibility pointer with a dedicated log-key word.
+
+Consequences worth knowing:
+
+- **Identity is `(nodeId, log key)`, never a version.** `isAuditEntryWrite` (`auditStore.ts`) is the
+  single predicate; `removeAuditEntry`'s tombstone removal and `blob.ts`'s orphan sweep both gate on
+  it, and both retain rather than delete when identity is unknown. A version compare there would let
+  one write authorize destroying another's tombstone or blob.
+- **A RocksDB applied write carries its own record version.** `TransactionWrite.recordVersion`, set from
+  `options.version` by every `_write*` builder and read in `save()` only when the transaction is
+  `sourceApply` or `isReplay`, is how a replication receiver stores the origin's version while the
+  transaction commits under the origin's log key — so a peer's copy of an origin's log stays in the
+  origin's clock. New audit entries persist that write version even when an out-of-order merge leaves
+  a newer record version in the primary store. `getAppliedWriteVersion` also bounds historical
+  overloaded values by `txnLogKey`. One frame can carry writes at different record versions, so this
+  cannot be a per-transaction value. Deprecated LMDB keeps its legacy transaction-version apply
+  behavior.
+- **`additionalAuditRefs[].version` is a log key, not a version.** Every consumer follows it straight
+  into `auditStore.get` (`Table.ts`'s `auditRefsToVisit`). The list carries folded out-of-order
+  branches and, when a RocksDB record version differs from its log key, the record's own audit head.
+  Under stage 2 the stored reference field can be renamed; until then, "fixing" it to the record
+  version silently unaddresses the entry it points at.
+- **Crash replay uses both.** `replayLogs` delimits transactions by `txnLogKey` (which is also what
+  `CorruptFrameStop.truncatedVersions` records) and replays each write at its stored `version`.
+  Stamping a replayed record at its log key would move its version forward and make a later
+  legitimate write look stale.
+- **Compatibility surfaces still report `localTime` as the transaction timestamp.** Subscription,
+  history, and pro-to-core replication event shapes predate this internal name and remain unchanged;
+  no on-disk or on-wire identifier changes in this stage.
 
 **QUERY admission uses the body projection.** `Resource.transactional` resolves an asynchronous HTTP QUERY body before resource resolution, recursively clones away client-supplied `checkPermission`, and copies the body's `select` onto the operation admission target before `allowRead`. After authorization, `Resource.query` transfers only the narrowed projection to the body target. This ensures a body-only relationship select is checked before `Table.search`; permission-control fields from QUERY data must never reach a search target.
 
@@ -141,6 +189,20 @@ Tests: `../unitTests/resources/paramRoutes.test.js` (unit) and `../integrationTe
 
 ---
 
+## Persisted relationship catalog
+
+GraphQL `@relationship` attributes are live objects in component workers, but the operations API runs in a separate thread that does not load component schemas. `processGraphQLSchema` therefore records each relationship's target `database.table`, and `table()` persists a normalized, data-only relationship list on the table's primary catalog descriptor. The primary descriptor is deliberate: older Harper versions ignore the unknown field instead of loading a per-attribute row without the runtime table definition and silently returning `null`.
+
+`getDatabases()` waits until the complete table catalog is loaded, then hydrates catalog-owned relationship attributes with a stable target-class snapshot and calls `updatedAttributes()` to rebuild resolvers. The list is authoritative only when the GraphQL authoring path explicitly supplies `schemaRelationshipsDefined`; admin and replication callers that omit it cannot erase component-owned metadata. An empty list removes catalog-owned relationships. Live schema relationships and same-name runtime attributes always win, and `@enumerable` is not persisted, so operations queries traverse relationships only when explicitly selected.
+
+The primary-descriptor update shares the `update-attributes` serialization boundary with `dropTable()`, re-reads the row after locking, and refuses to replace a drop tombstone. Missing or malformed targets are skipped rather than installed without authorization metadata.
+
+Only the GraphQL authoring path persists relationships. `defineTable()` resolves its relation targets through a lazy thunk so forward references and cycles work, and the target class is not resolvable at registration time — so code-first relationships remain worker-only, and the operations API still rejects them as unknown attributes. Tightening the operations API's object/nested `get_attributes` form to reject names it cannot resolve is NOT an option: `Table.transformEntryForSelect` projects undeclared JSON sub-objects through that same form, so `validation/searchValidator.ts` deliberately leaves it unchecked.
+
+Tests: `../unitTests/resources/schemaMigrationFragility.test.js` (catalog round-trip and failure behavior) and `../integrationTests/apiTests/graphql.test.mjs` (operations API in both directions).
+
+---
+
 ## Typed, discoverable resources (code-first schema + request contract)
 
 Design record: the full RFC and its type-level design proofs live in the design PR (**HarperFast/harper#1503**); this section is the retained summary.