@harperfast/harper-pro 5.0.17 → 5.0.19

package/core/resources/RecordEncoder.ts

@@ -195,20 +195,23 @@ export class RecordEncoder extends Encoder {
 		const superGetStructures = this.getStructures;
 		this.saveStructures = function (structures, isCompatible): boolean | undefined {
 			if (this.isRocksDB) {
-				return this.rootStore.transactionSync((txn) => {
-					const sharedStructuresKey = [Symbol.for('structures'), this.name];
-					const existingStructuresBuffer = txn.getBinarySync(sharedStructuresKey);
-					const existingStructures = existingStructuresBuffer ? this.decode(existingStructuresBuffer) : undefined;
-					if (typeof isCompatible == 'function') {
-						if (!isCompatible(existingStructures)) {
+				return this.rootStore.transactionSync(
+					(txn) => {
+						const sharedStructuresKey = [Symbol.for('structures'), this.name];
+						const existingStructuresBuffer = txn.getBinarySync(sharedStructuresKey);
+						const existingStructures = existingStructuresBuffer ? this.decode(existingStructuresBuffer) : undefined;
+						if (typeof isCompatible == 'function') {
+							if (!isCompatible(existingStructures)) {
+								return false;
+							}
+						} else if (existingStructures && existingStructures.length !== isCompatible) {
 							return false;
 						}
-					} else if (existingStructures && existingStructures.length !== isCompatible) {
-						return false;
-					}
-					txn.putSync(sharedStructuresKey, structures);
-					this.structureUpdate = structures;
-				});
+						txn.putSync(sharedStructuresKey, structures);
+						this.structureUpdate = structures;
+					},
+					{ retryOnBusy: true }
+				);
 			} else {
 				const result = superSaveStructures.call(this, structures, isCompatible);
 				this.structureUpdate = structures;

package/core/resources/RocksTransactionLogStore.ts

@@ -5,6 +5,7 @@ import { Decoder, readAuditEntry, ENTRY_DATAVIEW, AuditRecord, createAuditEntry
 import { isMainThread } from 'node:worker_threads';
 import { EventEmitter } from 'node:events';
 import { asBinary } from 'lmdb';
+import * as harperLogger from '../utility/logging/harper_logger.ts';
 
 if (!process.env.HARPER_NO_FLUSH_ON_EXIT && isMainThread) {
 	// we want to be able to test log replay
@@ -288,29 +289,53 @@ export class RocksTransactionLogStore extends EventEmitter {
 			iterable.iterate = () => aggregateIterator;
 		}
 		const mappedAggregateIterable = iterable.map(({ timestamp, data, endTxn }: TransactionEntry) => {
-			const decoder = new Decoder(data.buffer, data.byteOffset, data.byteLength);
-			data.dataView = decoder;
-			// This represents the data that shouldn't be transferred for replication
-			let structureVersion = decoder.getUint32(0);
-			let position = 4;
-			let previousResidencyId: number;
-			let previousVersion: number;
-			if (structureVersion & HAS_PREVIOUS_RESIDENCY_ID) {
-				previousResidencyId = decoder.getUint32(position);
-				position += 4;
-			}
-			if (structureVersion & HAS_PREVIOUS_VERSION) {
-				// does previous residency id and version actually require separate flags?
-				previousVersion = decoder.getFloat64(position);
-				position += 8;
+			// Per-entry try/catch: a corrupt rocks prelude (first 4-16 bytes) would otherwise
+			// throw a raw `RangeError: Offset is outside the bounds of the DataView` out
+			// through `iterable.map`, escape the for-of consumer, and land as an
+			// uncaughtException on a later tick — stalling outgoing replication at the
+			// failing offset on every catch-up attempt. On error, yield a sentinel record
+			// with the timestamp preserved so iteration advances past the bad entry;
+			// downstream consumers already skip records with no `tableId`/`type`.
+			try {
+				const decoder = new Decoder(data.buffer, data.byteOffset, data.byteLength);
+				(data as any).dataView = decoder;
+				// This represents the data that shouldn't be transferred for replication
+				let structureVersion = decoder.getUint32(0);
+				let position = 4;
+				let previousResidencyId: number;
+				let previousVersion: number;
+				if (structureVersion & HAS_PREVIOUS_RESIDENCY_ID) {
+					previousResidencyId = decoder.getUint32(position);
+					position += 4;
+				}
+				if (structureVersion & HAS_PREVIOUS_VERSION) {
+					// does previous residency id and version actually require separate flags?
+					previousVersion = decoder.getFloat64(position);
+					position += 8;
+				}
+				const auditRecord = readAuditEntry(data, position, undefined);
+				auditRecord.version = timestamp;
+				auditRecord.endTxn = endTxn;
+				auditRecord.previousResidencyId = previousResidencyId;
+				auditRecord.previousVersion = previousVersion;
+				auditRecord.structureVersion = structureVersion & 0x00ffffff;
+				return auditRecord;
+			} catch (error) {
+				harperLogger.error('Failed to decode rocks transaction log entry; skipping', error, {
+					timestamp,
+					byteLength: data?.byteLength,
+				});
+				return {
+					version: timestamp,
+					endTxn,
+					type: undefined,
+					tableId: undefined,
+					recordId: undefined,
+					getValue: () => undefined,
+					getBinaryValue: () => undefined,
+					getBinaryRecordId: () => undefined,
+				} as unknown as AuditRecord;
 			}
-			const auditRecord = readAuditEntry(data, position, undefined, true);
-			auditRecord.version = timestamp;
-			auditRecord.endTxn = endTxn;
-			auditRecord.previousResidencyId = previousResidencyId;
-			auditRecord.previousVersion = previousVersion;
-			auditRecord.structureVersion = structureVersion & 0x00ffffff;
-			return auditRecord;
 		});
 		// Add methods to the mapped iterable if we have an aggregate iterator
 		if (aggregateIterator?.addLog) {

package/core/resources/Table.ts

@@ -6,7 +6,7 @@
 
 import { CONFIG_PARAMS, OPERATIONS_ENUM, SYSTEM_TABLE_NAMES, SYSTEM_SCHEMA_NAME } from '../utility/hdbTerms.ts';
 import { type Database } from 'lmdb';
-import { getIndexedValues } from '../utility/lmdb/commonUtility.js';
+import { getIndexedValues, getNextMonotonicTime } from '../utility/lmdb/commonUtility.js';
 import { getThisNodeId, exportIdMapping } from './nodeIdMapping.ts';
 import lodash from 'lodash';
 import { ExtendedIterable, SKIP } from '@harperfast/extended-iterable';
@@ -99,6 +99,7 @@ export const EVICTED = 8; // note that 2 is reserved for timestamps
 const TEST_WRITE_KEY_BUFFER = Buffer.allocUnsafeSlow(8192);
 const MAX_KEY_BYTES = 1978;
 const EVENT_HIGH_WATER_MARK = 100;
+const REPLAY_YIELD_INTERVAL = 100; // yield to the event loop every N records during subscription replay
 const FULL_PERMISSIONS = {
 	read: true,
 	insert: true,
@@ -805,23 +806,23 @@ export function makeTable(options) {
 		/**
 		 * Set TTL expiration for records in this table. On retrieval, record timestamps are checked for expiration.
 		 * This also informs the scheduling for record eviction.
-		 * @param expirationTime Time in seconds until records expire (are stale)
-		 * @param evictionTime Time in seconds until records are evicted (removed)
+		 * @param opts Time in seconds until records expire, or an options object with `expiration`, `eviction`,
+		 * and `scanInterval` (all in seconds, all optional). Number form preserves any previously configured
+		 * eviction/scanInterval; object form replaces all three.
 		 */
-		static setTTLExpiration(expiration: number | { expiration: number; eviction?: number; scanInterval?: number }) {
-			// we set up a timer to remove expired entries. we only want the timer/reaper to run in one thread,
-			// so we use the first one
-			if (typeof expiration === 'number') {
-				expirationMs = expiration * 1000;
-				if (!evictionMs) evictionMs = 0; // by default, no extra time for eviction
-			} else if (expiration && typeof expiration === 'object') {
-				// an object with expiration times/options specified
-				expirationMs = expiration.expiration * 1000;
-				evictionMs = (expiration.eviction || 0) * 1000;
-				cleanupInterval = expiration.scanInterval * 1000;
-			} else throw new Error('Invalid expiration value type');
+		static setTTLExpiration(opts: number | { expiration?: number; eviction?: number; scanInterval?: number }) {
+			if (opts == null || (typeof opts !== 'number' && typeof opts !== 'object'))
+				throw new Error('Invalid expiration value type');
+			if (typeof opts === 'number') {
+				expirationMs = opts * 1000;
+			} else {
+				// `??` so an explicit 0 is treated as the user's chosen value, not as "missing"
+				expirationMs = (opts.expiration ?? 0) * 1000;
+				evictionMs = (opts.eviction ?? 0) * 1000;
+				cleanupInterval = (opts.scanInterval ?? 0) * 1000;
+			}
 			if (expirationMs < 0) throw new Error('Expiration can not be negative');
-			// default to one quarter of the total eviction time, and make sure it fits into a 32-bit signed integer
+			// default to one quarter of the total expiration+eviction window
 			cleanupInterval = cleanupInterval || (expirationMs + evictionMs) / 4;
 			scheduleCleanup();
 		}
@@ -2641,12 +2642,21 @@ export function makeTable(options) {
 			}
 			if (!request) request = {};
 			const getFullRecord = !request.rawEvents;
-			let pendingRealTimeQueue = []; // while we are servicing a loop for older messages, we have to queue up real-time messages and deliver them in order
+			// While the count, !omitCurrent, and non-collection branches replay older messages, real-time
+			// messages from the listener accumulate here and are drained at the end of the IIFE so they
+			// arrive after the replayed history, in order. The startTime branch sets this to null and
+			// uses dropDuringReplay instead — its snapshot:false cursor picks up the live tail directly.
+			let pendingRealTimeQueue: any[] | null = [];
+			// Set during the startTime audit-log replay. The cursor iterates the audit log forward with
+			// snapshot:false, which catches any commits that land during yield points; dropping in the
+			// listener avoids duplicate delivery.
+			let dropDuringReplay = false;
 			const thisId = requestTargetToId(request) ?? null; // treat undefined and null as the root
 			const subscription = addSubscription(
 				TableResource,
 				thisId,
 				function (id: Id, auditRecord: any, localTime: number, beginTxn: boolean) {
+					if (dropDuringReplay) return;
 					try {
 						let type = auditRecord.type;
 						let value;
@@ -2689,6 +2699,11 @@ export function makeTable(options) {
 				request.startTime || 0,
 				request
 			);
+			// Attach the request.listener BEFORE invoking the IIFE so that sync sends from the
+			// IIFE's prologue go directly to the listener via emit('data') instead of accumulating
+			// in subscription.queue. Without this, the IIFE can fill the queue past
+			// EVENT_HIGH_WATER_MARK and hit waitForDrain before the consumer's listener exists.
+			if (request.listener) subscription!.on('data', request.listener);
 			const result = (async () => {
 				const isCollection = request.isCollection ?? thisId == null;
 				if (isCollection) {
@@ -2699,17 +2714,27 @@ export function makeTable(options) {
 				let count = request.previousCount;
 				if (count > 1000) count = 1000; // don't allow too many, we have to hold these in memory
 				let startTime = request.startTime;
+				let recordsSinceYield = 0;
+
 				if (isCollection) {
 					// a collection should retrieve all descendant ids
 					if (startTime) {
 						if (count)
 							throw new ClientError('startTime and previousCount can not be combined for a table level subscription');
-						// start time specified, get the audit history for this time range
+						// start time specified, get the audit history for this time range. We drop real-time
+						// messages during this loop because the snapshot:false cursor will pick them up itself.
+						pendingRealTimeQueue = null;
+						dropDuringReplay = true;
+
 						for (const auditRecord of auditStore.getRange({
 							start: startTime,
 							exclusiveStart: true,
 							snapshot: false, // no need for a snapshot, audits don't change
 						})) {
+							if (++recordsSinceYield >= REPLAY_YIELD_INTERVAL) {
+								recordsSinceYield = 0;
+								await rest();
+							}
 							if (auditRecord.tableId !== tableId) continue;
 							const id = auditRecord.recordId;
 							if (thisId == null || isDescendantId(thisId, id)) {
@@ -2727,14 +2752,33 @@ export function makeTable(options) {
 									if ((await subscription.waitForDrain()) === false) return;
 								}
 							}
-							// TODO: Would like to do this asynchronously, but would need to catch up on anything published during iteration
-							//await rest(); // yield for fairness
-							subscription.startTime = auditRecord.localTime; // update so we don't double send
+							subscription!.startTime = auditRecord.localTime ?? auditRecord.version; // update so we don't double send
 						}
+						// No catch-up sweep needed. With snapshot:false (lmdb), notifyFromTransactionData
+						// calls resetReadTxn before iterating, which bumps renewId; on the cursor's next
+						// .next() it renews to a fresh txn whose snapshot is at least as recent. With
+						// rocksdb, the audit-log iterator re-reads `_lastCommittedPosition` on each next()
+						// (live tail). Either way, at loop exit subscription.startTime is at or past
+						// lastTxnTime, and the gate in notifyFromTransactionData handles the handoff
+						// once dropDuringReplay flips back.
+						dropDuringReplay = false;
 					} else if (count) {
+						// Raise the listener's gate up front so that any in-flight 'committed' callbacks
+						// for records the cursor will capture in `history` get gated out of
+						// pendingRealTimeQueue rather than queued and re-emitted as duplicates after
+						// history is sent. getNextMonotonicTime() returns a strictly-greater value than
+						// any audit record's localTime issued so far — it's the same source Harper uses
+						// to assign localTimes — so this gates exactly the records the cursor's
+						// snapshot:true view can see. Anything committed strictly after this point will
+						// pass the gate and reach the queue.
+						subscription!.startTime = getNextMonotonicTime();
 						const history = [];
 						// we are collecting the history in reverse order to get the right count, then reversing to send
 						for (const auditRecord of auditStore.getRange({ start: 'z', end: false, reverse: true })) {
+							if (++recordsSinceYield >= REPLAY_YIELD_INTERVAL) {
+								recordsSinceYield = 0;
+								await rest();
+							}
 							try {
 								if (auditRecord.tableId !== tableId) continue;
 								const id = auditRecord.recordId;
@@ -2752,20 +2796,34 @@ export function makeTable(options) {
 							} catch (error) {
 								logger.error?.('Error getting history entry', auditRecord.localTime, error);
 							}
-							// TODO: Would like to do this asynchronously, but would need to catch up on anything published during iteration
-							//await rest(); // yield for fairness
 						}
 						for (let i = history.length; i > 0; ) {
 							send(history[--i]);
 						}
-						if (history[0]) subscription.startTime = history[0].localTime; // update so don't double send
 					} else if (!request.omitCurrent) {
+						// Raise the listener's gate up front so that any in-flight 'committed' callbacks
+						// for pre-subscribe commits (which haven't yet advanced lastTxnTime when subscribe
+						// is called) get gated out of the queue. Otherwise the listener fires for them
+						// during cursor yields and emits stale events the cursor either covered (current
+						// state) or correctly skipped (e.g., deletes via `if (!value) continue`).
+						// getNextMonotonicTime() is the same source Harper uses to assign audit record
+						// localTimes, so the gate cuts at a precise instant in the same time domain.
+						subscription!.startTime = getNextMonotonicTime();
+
+						// Retained-message semantics: subscriber may legitimately receive a record twice
+						// if a post-subscribe write hits a key the cursor also visits. This is
+						// idempotent for "current state then live updates" — both deliveries land at
+						// the same final state. We don't dedupe.
 						for (const { key: id, value, version, localTime, size } of primaryStore.getRange({
 							start: thisId ?? false,
 							end: thisId == null ? undefined : [thisId, MAXIMUM_KEY],
 							versions: true,
 							snapshot: false, // no need for a snapshot, just want the latest data
 						})) {
+							if (++recordsSinceYield >= REPLAY_YIELD_INTERVAL) {
+								recordsSinceYield = 0;
+								await rest();
+							}
 							if (!value) continue;
 							send({ id, localTime, value, version, type: 'put', size });
 							if (subscription.queue?.length > EVENT_HIGH_WATER_MARK) {
@@ -2791,13 +2849,19 @@ export function makeTable(options) {
 					}
 					logger.trace?.('Subscription from', startTime, 'from', thisId, localTime);
 					if (startTime < localTime) {
-						// start time specified, get the audit history for this record
+						// start time specified, get the audit history for this record. Set startTime up
+						// front so the listener gate skips any in-flight 'committed' for this version
+						// during the yields below — otherwise that event would be queued and drained as a
+						// duplicate of the entry send.
+						subscription!.startTime = localTime ?? entry?.version;
 						const history = [];
 						let nextTime = localTime;
 						let nodeId = entry?.nodeId;
 						do {
-							//TODO: Would like to do this asynchronously, but we will need to run catch after this to ensure we didn't miss anything
-							//await auditStore.prefetch([key]); // do it asynchronously for better fairness/concurrency and avoid page faults
+							if (++recordsSinceYield >= REPLAY_YIELD_INTERVAL) {
+								recordsSinceYield = 0;
+								await rest();
+							}
 							const auditRecord = auditStore.getSync(nextTime, tableId, thisId, nodeId);
 							if (auditRecord) {
 								if (startTime < nextTime) {
@@ -2819,7 +2883,6 @@ export function makeTable(options) {
 						for (let i = history.length; i > 0; ) {
 							send(history[--i]);
 						}
-						subscription.startTime = localTime; // make sure we don't re-broadcast the current version that we already sent
 					}
 					if (!request.omitCurrent && entry?.value) {
 						// if retain and it exists, send the current value first
@@ -2831,10 +2894,12 @@ export function makeTable(options) {
 					}
 				}
 				// now send any queued messages
-				for (const event of pendingRealTimeQueue) {
-					send(event);
+				if (pendingRealTimeQueue) {
+					for (const event of pendingRealTimeQueue) {
+						send(event);
+					}
+					pendingRealTimeQueue = null;
 				}
-				pendingRealTimeQueue = null;
 			})();
 			result.catch((error) => {
 				harperLogger.error?.('Error in real-time subscription:', error);
@@ -2846,7 +2911,6 @@ export function makeTable(options) {
 				}
 				subscription.send(event);
 			}
-			if (request.listener) subscription.on('data', request.listener);
 			return subscription;
 		}
 
@@ -4245,6 +4309,8 @@ export function makeTable(options) {
 									Boolean(invalidated),
 									auditRecord
 								);
+								// arm the eviction scanner, mirroring the .put() path
+								if (sourceContext.expiresAt) scheduleCleanup();
 							} else if (existingEntry) {
 								logger.trace?.(
 									`Deleting resolved record from source with id: ${id}, timestamp: ${new Date(txnTime).toISOString()}`

package/core/resources/auditStore.ts

@@ -50,6 +50,7 @@ export type AuditRecord = {
 	previousAdditionalAuditRefs?: Array<{ version: number; nodeId: number }>;
 	endTxn?: boolean;
 	structureVersion?: number;
+	getBinaryRecordId?: any;
 };
 
 const ENTRY_HEADER = Buffer.alloc(2816); // this is sized to be large enough for the maximum key size (1976) plus large usernames. We may want to consider some limits on usernames to ensure this all fits
@@ -73,6 +74,16 @@ export const transactionKeyEncoder = {
 		if (buffer[start] === 66) {
 			const dataView =
 				buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength));
+			// Without this bounds check, a truncated key buffer escapes as RangeError up
+			// through lmdb-js's iterator and lands as an uncaughtException on a later tick,
+			// stalling outgoing replication for the affected (peer, db) pair.
+			if (start + 8 > buffer.byteLength) {
+				harperLogger.warn('Audit key buffer too short for float64 read; returning NaN sentinel', {
+					start,
+					byteLength: buffer.byteLength,
+				});
+				return NaN;
+			}
 			return dataView.getFloat64(start);
 		} else {
 			return readKey(buffer, start, end);
@@ -439,6 +450,15 @@ export function readAuditEntry(buffer: Uint8Array, start = 0, end = undefined):
 		const nodeId = decoder.readInt();
 		const tableId = decoder.readInt();
 		let length = decoder.readInt();
+		// A corrupt length field (e.g., a 0xff-prefixed uint32) would otherwise push
+		// decoder.position hundreds of megabytes past the buffer; the next readFloat64
+		// then throws with the bogus position in the message. Failing fast here keeps
+		// the throw inside this try/catch so we surface a sentinel instead.
+		if (length < 0 || decoder.position + length > buffer.byteLength) {
+			throw new RangeError(
+				`Audit entry recordId length ${length} exceeds remaining buffer (position ${decoder.position}, byteLength ${buffer.byteLength})`
+			);
+		}
 		const recordIdStart = decoder.position;
 		const recordIdEnd = (decoder.position += length);
 		// TODO: Once we support multiple format versions, we can conditionally read the version (and the previousResidencyId)
@@ -469,6 +489,11 @@ export function readAuditEntry(buffer: Uint8Array, start = 0, end = undefined):
 			}
 		}
 		length = decoder.readInt();
+		if (length < 0 || decoder.position + length > buffer.byteLength) {
+			throw new RangeError(
+				`Audit entry username length ${length} exceeds remaining buffer (position ${decoder.position}, byteLength ${buffer.byteLength})`
+			);
+		}
 		const usernameStart = decoder.position;
 		const usernameEnd = (decoder.position += length);
 		let value: any;
@@ -477,8 +502,17 @@ export function readAuditEntry(buffer: Uint8Array, start = 0, end = undefined):
 			tableId,
 			nodeId,
 			get recordId() {
-				// use a subarray to protect against the underlying buffer being modified
-				return readKey(buffer.subarray(0, recordIdEnd), recordIdStart, recordIdEnd);
+				// The recordId is decoded lazily and lives outside readAuditEntry's try/catch,
+				// so a corrupt recordId region would otherwise escape as an uncaught RangeError
+				// on property access. Catch and return undefined; callers already treat missing
+				// recordId as a skip-eligible entry.
+				try {
+					// use a subarray to protect against the underlying buffer being modified
+					return readKey(buffer.subarray(0, recordIdEnd), recordIdStart, recordIdEnd);
+				} catch (error) {
+					harperLogger.warn('Failed to decode audit recordId; treating as corrupt', error);
+					return undefined;
+				}
 			},
 			getBinaryRecordId() {
 				return buffer.subarray(recordIdStart, recordIdEnd);
@@ -486,9 +520,14 @@ export function readAuditEntry(buffer: Uint8Array, start = 0, end = undefined):
 			version,
 			previousVersion,
 			get user() {
-				return usernameEnd > usernameStart
-					? readKey(buffer.subarray(0, usernameEnd), usernameStart, usernameEnd)
-					: undefined;
+				try {
+					return usernameEnd > usernameStart
+						? readKey(buffer.subarray(0, usernameEnd), usernameStart, usernameEnd)
+						: undefined;
+				} catch (error) {
+					harperLogger.warn('Failed to decode audit username; treating as corrupt', error);
+					return undefined;
+				}
 			},
 			get encoded() {
 				return start ? buffer.subarray(start, end) : buffer;
@@ -523,10 +562,52 @@ export function readAuditEntry(buffer: Uint8Array, start = 0, end = undefined):
 		};
 	} catch (error) {
 		harperLogger.error('Reading audit entry error', error, buffer);
-		return {};
+		return createCorruptAuditSentinel(buffer, start, end);
 	}
 }
 
+/**
+ * Build a structurally complete audit record for an entry that failed to decode. The fields
+ * mirror the happy-path shape so downstream consumers that access (e.g.) `getValue` or the
+ * `recordId` getter don't blow up with a `TypeError: not a function` / `undefined.is(...)`
+ * after the header decode already failed. Consumers identify these by the undefined
+ * `tableId`/`type` (the same signal lmdb has produced from this catch since before this
+ * change) and skip them — `classifyAuditEntryForReplay` calls them out as `corrupt-header`,
+ * and the dispatch loops in Table.ts / transactionBroadcast.ts filter via tableId guards.
+ */
+function createCorruptAuditSentinel(buffer: Uint8Array, start: number, end: number | undefined): AuditRecord {
+	return {
+		type: undefined,
+		tableId: undefined,
+		nodeId: undefined,
+		recordId: undefined,
+		version: undefined,
+		previousVersion: undefined,
+		user: undefined,
+		extendedType: undefined,
+		residencyId: undefined,
+		previousResidencyId: undefined,
+		expiresAt: undefined,
+		originatingOperation: undefined,
+		previousAdditionalAuditRefs: undefined,
+		get encoded() {
+			return start ? buffer.subarray(start, end) : buffer;
+		},
+		get size() {
+			return start !== undefined && end !== undefined ? end - start : buffer.byteLength;
+		},
+		getBinaryRecordId() {
+			return undefined;
+		},
+		getValue() {
+			return undefined;
+		},
+		getBinaryValue() {
+			return undefined;
+		},
+	} as any;
+}
+
 export class Decoder extends DataView<ArrayBufferLike> {
 	position = 0;
 	readInt() {

package/core/resources/databases.ts

@@ -1063,6 +1063,7 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 				const dbi = openIndex(dbiKey, rootStore, attribute);
 				if (
 					changed ||
+					attributeDescriptor.indexingFailed ||
 					(attributeDescriptor.indexingPID && attributeDescriptor.indexingPID !== process.pid) ||
 					attributeDescriptor.restartNumber < workerData?.restartNumber
 				) {
@@ -1071,6 +1072,7 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 					attributeDescriptor = attributesDbi.getSync(dbiKey);
 					if (
 						changed ||
+						attributeDescriptor.indexingFailed ||
 						(attributeDescriptor.indexingPID && attributeDescriptor.indexingPID !== process.pid) ||
 						attributeDescriptor.restartNumber < workerData?.restartNumber
 					) {
@@ -1084,14 +1086,20 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 						if (hasExistingData) {
 							attribute.lastIndexedKey = attributeDescriptor?.lastIndexedKey ?? undefined;
 							attribute.indexingPID = process.pid;
+							delete attribute.indexingFailed; // clear failure flag for the new run
 							dbi.isIndexing = true;
-							Object.defineProperty(attribute, 'dbi', { value: dbi });
+							Object.defineProperty(attribute, 'dbi', { value: dbi, configurable: true, enumerable: false });
 							// we only set indexing nulls to true if new or reindexing, we can't have partial indexing of null
 							attributesToIndex.push(attribute);
 						}
 					}
 					attributesDbi.put(dbiKey, attribute);
 				}
+				// If a migration is in progress (indexingPID set), any newly opened dbi must also
+				// reflect isIndexing = true. A resetDatabases() during an active runIndexing creates
+				// a new dbi object; without this, queries could use the new dbi (isIndexing = false)
+				// and return incomplete results while the backfill is still running.
+				if (attributeDescriptor?.indexingPID) dbi.isIndexing = true;
 				if (attributeDescriptor?.indexNulls && attribute.indexNulls === undefined) attribute.indexNulls = true;
 				dbi.indexNulls = attribute.indexNulls;
 				indices[attribute.name] = dbi;
@@ -1162,6 +1170,7 @@ async function runIndexing(Table, attributes, indicesToRemove) {
 			lastResolution = index.drop();
 		}
 		let interrupted;
+		let hadIndexingErrors = false;
 		const attributeErrorReported = {};
 		let indexed = 0;
 		const attributesLength = attributes.length;
@@ -1215,6 +1224,7 @@ async function runIndexing(Table, attributes, indicesToRemove) {
 							}
 						}
 					} catch (error) {
+						hadIndexingErrors = true;
 						if (!attributeErrorReported[property]) {
 							// just report an indexing error once per attribute so we don't spam the logs
 							attributeErrorReported[property] = true;
@@ -1227,6 +1237,7 @@ async function runIndexing(Table, attributes, indicesToRemove) {
 					() => outstanding--,
 					(error) => {
 						outstanding--;
+						hadIndexingErrors = true;
 						logger.error(error);
 					}
 				);
@@ -1244,20 +1255,69 @@ async function runIndexing(Table, attributes, indicesToRemove) {
 				if (outstanding > MAX_OUTSTANDING_INDEXING) await lastResolution;
 				else if (outstanding > MIN_OUTSTANDING_INDEXING) await new Promise((resolve) => setImmediate(resolve)); // yield event turn, don't want to use all computation
 			}
+		}
+		// Await the last pending put. If it rejects, that is also an indexing error.
+		// Note: the when() calls above already attach rejection handlers to each record's
+		// last-put promise; this try-catch specifically handles the case where lastResolution
+		// itself rejects (i.e. the very last put in the loop failed) which would otherwise
+		// throw past the hadIndexingErrors check to the outer catch. The broader issue of
+		// unhandled rejections from non-last puts in multi-value attributes is pre-existing
+		// and out of scope for this fix.
+		try {
+			await lastResolution;
+		} catch (error) {
+			hadIndexingErrors = true;
+			logger.error(error);
+		}
+		// Yield one more event turn so any queued when() error callbacks (which fire as
+		// microtasks when their tracked promise settles) have a chance to set hadIndexingErrors
+		// before we decide whether to mark indexing as complete.
+		await new Promise((resolve) => setImmediate(resolve));
+		if (hadIndexingErrors) {
+			// Some records failed to index. Persist the failure marker in the descriptor so
+			// the next call to table() (including after a restart with a fresh PID) re-triggers
+			// the backfill from the last checkpoint. Do NOT clear indexingPID or isIndexing —
+			// leave the index in its incomplete state so queries return 503 "not indexed yet"
+			// rather than silently returning partial results. This is the key fix for the
+			// serent-canopy issue #135 fingerprint: a completed migration with transient errors
+			// (e.g. ERR_BUSY from RocksDB under load) leaving gaps while appearing successful.
+			for (const attribute of attributes) {
+				attribute.indexingFailed = true;
+				// Preserve lastIndexedKey so the retry resumes from the last checkpoint.
+				lastResolution = Table.dbisDB.put(attribute.key, attribute);
+				// Keep isIndexing = true on both the attribute.dbi and the currently-active dbi
+				// in Table.indices (which may differ if resetDatabases() ran during this pass).
+				attribute.dbi.isIndexing = true;
+				const activeDbi = Table.indices[attribute.name];
+				if (activeDbi) activeDbi.isIndexing = true;
+			}
+			await lastResolution;
+			logger.warn(
+				`Indexing of ${Table.tableName} encountered errors on some records - index will remain incomplete. ` +
+					`On next restart the migration will be retried from the last checkpoint (indexingFailed=true). ` +
+					`Affected attributes: ${attributes.map((a) => a.name).join(', ')}`
+			);
+		} else {
 			// update the attributes to indicate that we are finished
 			for (const attribute of attributes) {
 				delete attribute.lastIndexedKey;
 				delete attribute.indexingPID;
+				delete attribute.indexingFailed;
 				attribute.dbi.isIndexing = false;
+				// Also clear isIndexing on the currently-active dbi in Table.indices, which may
+				// differ from attribute.dbi if a resetDatabases() call during this migration
+				// opened a new dbi and registered it there.
+				const activeDbi = Table.indices[attribute.name];
+				if (activeDbi) activeDbi.isIndexing = false;
 				lastResolution = Table.dbisDB.put(attribute.key, attribute);
 			}
+			await lastResolution;
+			// now notify all the threads that we are done and the index is ready to use
+			await signalling.signalSchemaChange(
+				new SchemaEventMsg(process.pid, 'indexing-finished', Table.databaseName, Table.tableName)
+			);
+			logger.info(`Finished indexing ${Table.tableName} attributes`, attributes);
 		}
-		await lastResolution;
-		// now notify all the threads that we are done and the index is ready to use
-		await signalling.signalSchemaChange(
-			new SchemaEventMsg(process.pid, 'indexing-finished', Table.databaseName, Table.tableName)
-		);
-		logger.info(`Finished indexing ${Table.tableName} attributes`, attributes);
 	} catch (error) {
 		logger.error('Error in indexing', error);
 	}