@harperfast/harper 5.2.0 → 5.2.2

package/resources/analytics/write.ts

@@ -397,26 +397,28 @@ function storeDBSizeMetrics(analyticsTable: Table, databases: Databases) {
 	}
 }
 
-function storeVolumeMetrics(analyticsTable: Table, databases: Databases) {
-	for (const [db, tables] of Object.entries(databases)) {
-		try {
-			const [firstTable] = Object.values(tables);
-			const storageStats = firstTable?.getStorageStats();
-			if (!storageStats) {
-				continue;
+async function storeVolumeMetrics(analyticsTable: Table, databases: Databases) {
+	await Promise.all(
+		Object.entries(databases).map(async ([db, tables]) => {
+			try {
+				const [firstTable] = Object.values(tables);
+				const storageStats = await firstTable?.getStorageStats();
+				if (!storageStats) {
+					return;
+				}
+				const metric = {
+					metric: METRIC.STORAGE_VOLUME,
+					database: db,
+					...storageStats,
+				};
+				storeMetric(analyticsTable, metric);
+				log.trace?.(`db ${db} storage volume metrics: ${JSON.stringify(metric)}`);
+			} catch (error) {
+				// a table or db was deleted, could get an error here
+				log.warn?.(`Error getting DB volume metrics`, error);
 			}
-			const metric = {
-				metric: METRIC.STORAGE_VOLUME,
-				database: db,
-				...storageStats,
-			};
-			storeMetric(analyticsTable, metric);
-			log.trace?.(`db ${db} storage volume metrics: ${JSON.stringify(metric)}`);
-		} catch (error) {
-			// a table or db was deleted, could get an error here
-			log.warn?.(`Error getting DB volume metrics`, error);
-		}
-	}
+		})
+	);
 }
 
 // RocksDB stat names are kebab-case with a "rocksdb." prefix (e.g. "rocksdb.block.cache.hit");
@@ -1047,7 +1049,7 @@ async function aggregation(fromPeriod, toPeriod = 60000) {
 	storeDBSizeMetrics(analyticsTable, databases);
 
 	// database storage volume metrics
-	storeVolumeMetrics(analyticsTable, databases);
+	await storeVolumeMetrics(analyticsTable, databases);
 
 	// rocksdb engine stats (only for RocksDB-backed databases)
 	const rocksDBPeriod = lastRocksDBStatsTime ? now - lastRocksDBStatsTime : undefined;

package/resources/auditStore.ts

@@ -111,6 +111,10 @@ const MAX_DELETES_PER_CLEANUP = 1000;
 // setTimeout silently falls back to 1ms for delays past this, which would turn the backoff into the
 // hot loop it is meant to avoid — a `logging.auditRetention` over ~248 days reaches it via retention/10
 const MAX_CLEANUP_DELAY = 2 ** 31 - 1;
+// separate mint/read latches so a legacy-entry read warn can't mask the still-minting signal;
+// mint latch keyed per (table, type) so one entry type can't silence another's producer stack
+const warnedBodylessMints = new Set<string>();
+const warnedBodylessTables = new Set<number>();
 const FLOAT_TARGET = new Float64Array(1);
 const FLOAT_BUFFER = new Uint8Array(FLOAT_TARGET.buffer);
 let DEFAULT_AUDIT_CLEANUP_DELAY = 10000; // default delay of 10 seconds
@@ -416,10 +420,24 @@ export function createAuditEntry(auditRecord: AuditRecord, start = 0) {
 		originatingOperation,
 		previousAdditionalAuditRefs,
 	} = auditRecord;
-	const action = EVENT_TYPES[type];
+	let action = EVENT_TYPES[type];
 	if (!action) {
 		throw new Error(`Invalid audit entry type ${type}`);
 	}
+	if (action & (HAS_RECORD | HAS_PARTIAL_RECORD) && !encodedRecord?.length) {
+		// Readers decode the remainder whenever HAS_RECORD is set, so an audit-only commit minted with
+		// no body must not advertise one (#2153). HAS_PARTIAL_RECORD is kept: it also drives
+		// record-history reconstruction, and the read path tolerates the empty body.
+		if (!warnedBodylessMints.has(`${tableId}:${type}`)) {
+			warnedBodylessMints.add(`${tableId}:${type}`);
+			// the Error's stack identifies which write path delivered the missing value
+			harperLogger.warn(
+				`Audit entry (${type}) for record ${recordId} in table ${tableId} has no record body`,
+				new Error('bodyless audit mint')
+			);
+		}
+		action &= ~HAS_RECORD;
+	}
 	let position = start + 1;
 	if (previousVersion) {
 		if (previousVersion > 1) ENTRY_DATAVIEW.setFloat64(start, previousVersion);
@@ -621,6 +639,19 @@ export function readAuditEntry(buffer: Uint8Array, start = 0, end = undefined):
 			},
 			getValue(store, fullRecord?, auditTime?) {
 				if (action & HAS_RECORD || (action & HAS_PARTIAL_RECORD && !fullRecord)) {
+					if (decoder.position >= (end ?? buffer.byteLength)) {
+						// Entry advertises a record but has no body (minted before #2153): nothing to decode, and
+						// return undefined rather than falling through — this branch means the caller asked for the
+						// entry's own content (full-record consumers with an auditTime never enter it for partials
+						// and still reconstruct below). Warn latched per table: this getter runs inside range scans.
+						if (!warnedBodylessTables.has(tableId)) {
+							warnedBodylessTables.add(tableId);
+							harperLogger.warn(
+								`Audit entry (${EVENT_TYPES[action & 0xf]}) for table ${tableId} advertises a record but has no body; treating as having no record`
+							);
+						}
+						return;
+					}
 					if (!value) {
 						value = decodeFromDatabase(
 							// the audit value has no on-disk timestamp/metadata prefix (the audit entry carries

package/resources/blob.ts

@@ -55,7 +55,6 @@ type StorageInfo = {
 	recordId?: number;
 	contentBuffer?: any;
 	source?: Readable;
-	storageBuffer?: Buffer;
 	compress?: boolean;
 	flush?: boolean;
 	start?: number;
@@ -829,7 +828,7 @@ class FileBackedBlob extends (Blob as unknown as { new (): Blob }) implements Bl
 			storageInfoForBlob.set(slicedBlob, slicedStorageInfo);
 			if (this.size != undefined)
 				slicedBlob.size = (end == undefined ? this.size : Math.min(end, this.size)) - (start ?? 0);
-		} else if (sourceStorageInfo?.contentBuffer && !sourceStorageInfo.storageBuffer) {
+		} else if (sourceStorageInfo?.contentBuffer) {
 			const slicedStorageInfo = {
 				...sourceStorageInfo,
 				contentBuffer: sourceStorageInfo.contentBuffer.subarray(start, end),
@@ -1340,8 +1339,11 @@ function getNextStorageIndex(blobStoragePaths: string[], fileId: number) {
 	}
 	if (((blobStoragePaths as any).lastUpdated ?? 0) + 60000 < now) {
 		(blobStoragePaths as any).lastUpdated = now;
-		// create a new frequency table based on the available space
-		createFrequencyTableForStoragePaths(blobStoragePaths);
+		// create a new frequency table based on the available space; fire-and-forget, so a
+		// transient stat/mkdir failure must not become an unhandled rejection
+		createFrequencyTableForStoragePaths(blobStoragePaths).catch((error) => {
+			logger.warn?.('Error creating storage path frequency table', error);
+		});
 	}
 	const nextIndex = (blobStoragePaths as any).frequencyTable[fileId % FREQUENCY_TABLE_SIZE];
 	return nextIndex;
@@ -1356,6 +1358,10 @@ async function createFrequencyTableForStoragePaths(blobStoragePaths: string[]) {
 	if (!statfs) return; // statfs is not available on all older node versions
 	const availableSpaces = await Promise.all(
 		blobStoragePaths.map(async (path) => {
+			// This compares MULTIPLE distinct paths against each other, so it always needs the
+			// real per-path number: quota-status.json (see getStorageSpaceStats, #1976) is a
+			// single instance-wide figure that can't tell two disks apart, and would collapse
+			// every path in a multi-volume STORAGE_BLOBPATHS config to the same "available" value.
 			let stats: StatsFs;
 			try {
 				stats = await statfs(path);
@@ -1733,10 +1739,14 @@ addExtension({
 				throw new Error('No store specified, cannot load blob from storage');
 			}
 		} else {
+			// contentBuffer is a *copy* (copyingUnpacker uses copyBuffers), so it stays valid for the
+			// lifetime of the blob. Do not retain the raw ext-body `buffer` here: it is a view into the
+			// store's read buffer, which is recycled by later reads. Keeping it and re-emitting it on
+			// re-encode (e.g. a read-modify-write / REST PATCH) serialized whatever foreign bytes had
+			// since overwritten that buffer, corrupting the record (harper#2103).
 			storageInfoForBlob.set(blob, {
 				storageIndex: 0,
 				fileId: null,
-				storageBuffer: buffer as any,
 				contentBuffer: blobInfo[1] as any,
 			});
 			blob.size = blobInfo[1]?.length;
@@ -1757,9 +1767,6 @@ addExtension({
 		if (blob.type) options.type = blob.type;
 		if (blob.size !== undefined) options.size = blob.size;
 		if (storageInfo) {
-			if (storageInfo.storageBuffer) {
-				return storageInfo.storageBuffer;
-			}
 			if (
 				storageInfo.contentBuffer &&
 				(storageInfo.contentBuffer?.length < FILE_STORAGE_THRESHOLD || blob.saveInRecord)

package/resources/databases.ts

@@ -33,7 +33,7 @@ import { handleLocalTimeForGets } from './RecordEncoder.ts';
 import { deleteRootBlobPathsForDB } from './blob.ts';
 import { CUSTOM_INDEXES } from './indexes/customIndexes.ts';
 import { OpenDBIObject } from '../utility/lmdb/OpenDBIObject.ts';
-import { RocksDatabase, type RocksDatabaseOptions } from '@harperfast/rocksdb-js';
+import { RocksDatabase, supportedCompression, type RocksDatabaseOptions } from '@harperfast/rocksdb-js';
 import { PrimaryRocksDatabase } from './PrimaryRocksDatabase.ts';
 import { replayLogs } from './replayLogs.ts';
 import { totalmem } from 'node:os';
@@ -98,6 +98,80 @@ const DEFAULT_DATABASE_NAME = 'data';
 const DEFINED_TABLES = Symbol('defined-tables');
 const DEFAULT_COMPRESSION_THRESHOLD = (envGet(CONFIG_PARAMS.STORAGE_PAGESIZE) || 4096) - 60; // larger than this requires multiple pages
 initSync();
+/**
+ * The RocksDB block/blob codec for every column family this process opens (`storage.rocks.compression`),
+ * or `undefined` to leave rocksdb-js on its own default (lz4 wherever the native build has it).
+ *
+ * Resolved on the first open and then frozen, deliberately. RocksDB fixes a column family's codec
+ * for as long as it is open and rejects a reopen that disagrees, and Harper's worker threads share
+ * one process-wide column-family registry — so every open, in every thread, has to resolve the same
+ * value. Re-reading config per open does not guarantee that: on a fresh install the system families
+ * are created by `mountHdb()` before the config file exists, so the main thread would resolve
+ * nothing and the workers would resolve the configured codec, after which `__dbis__` cannot be
+ * reopened and Harper fails with "The system database failed to load". The installer stages this
+ * value before `mountHdb()` (see utility/install/installer.ts) so that first open already sees it.
+ *
+ * Unset is NOT "use the build default" for a family that already exists — see toRocksCompression.
+ */
+let resolvedRocksCompression: string | undefined;
+let rocksCompressionResolved = false;
+
+export function getRocksCompression(): string | undefined {
+	if (!rocksCompressionResolved) {
+		resolvedRocksCompression = readDatabaseCodec();
+		rocksCompressionResolved = true;
+	}
+	return resolvedRocksCompression;
+}
+
+/**
+ * Test-only: un-freezes the resolved codec. Production code never calls this — the freeze is the
+ * invariant (see getRocksCompression above) — but a test process runs many unrelated test files in
+ * one process, so whichever file happens to open a RocksDatabase first freezes this for everyone
+ * after it. Tests that need to exercise config changes call this to get back to the unresolved state.
+ */
+export function resetRocksCompression(): void {
+	resolvedRocksCompression = undefined;
+	rocksCompressionResolved = false;
+}
+
+/**
+ * The codec every column family in this process opens under.
+ *
+ * Compression is a deployment setting, not a per-table one. RocksDB opens all of a database's
+ * column families in one call, so the codec has to be decided before the first open — which is
+ * before Harper has read any table's metadata (that catalog is itself one of the families being
+ * opened). Resolving one codec from configuration and applying it to every family is what makes
+ * that possible; it is passed with `compressionForAllColumnFamilies` so families this process
+ * never names individually adopt it too, which is what lets a database created before the codec
+ * existed start compressing.
+ *
+ * `storage.rocks.compression` names a codec outright. Otherwise `storage.compression` (default
+ * true) decides enabled-or-not and the build default fills in the algorithm. Per-table metadata
+ * still records the LMDB-era boolean, but no longer selects: a table persisted as disabled inside
+ * a deployment that enables compression would need its own codec, and it cannot have one.
+ */
+function readDatabaseCodec(): string | undefined {
+	const explicit = readRocksCompressionConfig();
+	if (explicit) return explicit;
+	return toRocksCompression(getDefaultCompression()) as string | undefined;
+}
+
+function readRocksCompressionConfig(): string | undefined {
+	const configured = envGet(CONFIG_PARAMS.STORAGE_ROCKS_COMPRESSION);
+	if (configured === undefined || configured === null || configured === '') return undefined;
+	const requested = String(configured).trim().toLowerCase();
+	if (!requested) return undefined;
+	// Rejected here rather than at the open: an unsupported name throws inside RocksDatabase.open,
+	// which surfaces as the system database failing to load partway through startup.
+	if (!supportedCompression.includes(requested)) {
+		throw new Error(
+			`storage.rocks.compression="${requested}" is not available in this build of @harperfast/rocksdb-js. Supported: ${supportedCompression.join(', ')}`
+		);
+	}
+	return requested;
+}
+
 // I don't know if this is the best place for this, but somewhere we need to specify which tables
 // replicate by default:
 export const NON_REPLICATING_SYSTEM_TABLES = [
@@ -177,22 +251,51 @@ export const tables: Tables = Object.create(null);
 export const databases: Databases = Object.create(null);
 
 /**
- * Map a persisted (LMDB-era) compression value to what rocksdb-js accepts. Table metadata
- * carries values where a defined falsy value (false, '') means compression was explicitly
- * disabled and true/{ threshold, ... } mean enabled with defaults. rocksdb-js >= 2.6
- * validates `compression` strictly and treats UNSET as "use the build default (lz4)", so
- * disabled must be mapped to 'none' explicitly, and true to unset.
+ * Codec used to honor an "enabled, unspecified" compression setting, or `undefined` where the
+ * native build cannot provide it (in which case the request degrades to the build default rather
+ * than throwing).
+ */
+const DEFAULT_ENABLED_CODEC = supportedCompression.includes('lz4') ? 'lz4' : undefined;
+
+/**
+ * Map a persisted (LMDB-era) compression value to what rocksdb-js accepts. Table metadata carries
+ * values where a defined falsy value (false, '') means compression was explicitly disabled, and
+ * `true` / `{ threshold, ... }` mean enabled with defaults — `storage.compression` defaults to
+ * `true` (defaultConfig.yaml), so essentially every pre-existing table asked for compression.
+ *
+ * "Enabled" resolves to an explicit codec rather than to unset. Unset is not equivalent: RocksDB
+ * persists the codec per column family and a reopen that requests nothing inherits what the family
+ * already has, applying the build default only when the family does not yet exist. Leaving these
+ * unset therefore silently ignores the operator's request on every database created before the
+ * native build carried codecs — it keeps writing uncompressed forever, while a brand-new database
+ * gets lz4. Naming the codec makes the setting mean the same thing in both cases.
+ *
+ * This governs newly written files; existing SSTs keep their codec until write traffic rewrites
+ * them (`db.compact()` will not — see getRocksCompression above).
  */
 export function toRocksCompression(compression: unknown): unknown {
-	if (compression === true) return undefined;
-	if (compression !== undefined && !compression) return 'none';
+	if (compression === undefined) return undefined;
+	if (!compression) return 'none';
+	// An object carrying an explicit `algorithm` is already a rocksdb-js request; anything else
+	// (`true`, or an LMDB descriptor like { startingOffset, threshold }) is "enabled, unspecified".
+	if (compression === true || (typeof compression === 'object' && !(compression as { algorithm?: unknown }).algorithm))
+		return DEFAULT_ENABLED_CODEC;
 	return compression;
 }
 
 function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSort?: boolean }) {
 	options.disableWAL ??= true;
 	const legacyOptions = options as { compression?: unknown };
-	legacyOptions.compression = toRocksCompression(legacyOptions.compression);
+	// A configured codec applies to every column family, overriding whatever per-table metadata
+	// carries — that metadata records the LMDB-era boolean, so without this there is no way to
+	// select a RocksDB codec for a deployment.
+	// One codec for every column family, and applied to every family this open touches — not just
+	// the one being named. RocksDB opens them all at once and a family's codec cannot change while
+	// it is open, so a family this process never names individually would otherwise stay on
+	// whatever it was created with, forever.
+	const databaseCodec = getRocksCompression();
+	legacyOptions.compression = databaseCodec;
+	if (databaseCodec) (options as { compressionForAllColumnFamilies?: boolean }).compressionForAllColumnFamilies = true;
 	// Apply read-only mode if enabled
 	if (isReadOnlyMode()) {
 		options.readOnly = true;
@@ -588,7 +691,7 @@ function initStores(
 				...internalDbiInit,
 				disableWAL: false,
 				name: INTERNAL_DBIS_NAME,
-			} as any) as RocksDatabaseEx;
+			} as any);
 		} else {
 			attributesDbi = rootStore.openDB(INTERNAL_DBIS_NAME, internalDbiInit as any);
 		}
@@ -1350,7 +1453,11 @@ function openIndex(dbiKey: string, rootStore: RootDatabaseKind, attribute: any)
 		// Enable cache (WeakLRUCache + VT) for all custom-object index stores so the VT is
 		// available before resolveIndexFormat decides the format. Versioned stores need the VT
 		// for cached traversal; legacy stores pay a small per-write cache.delete() overhead only.
-		dbi = openRocksDatabase(rootStore.path, { ...dbiInit, name: dbiKey, cache: isCustomObjectIndex } as any) as any;
+		dbi = openRocksDatabase(rootStore.path, {
+			...dbiInit,
+			name: dbiKey,
+			cache: isCustomObjectIndex,
+		} as any) as any;
 		(dbi as any).rootStore = rootStore;
 		// Custom-index object stores (e.g. HNSW) write graph nodes via plain put() with no staged
 		// transaction timestamp, so their values carry no version and the PrimaryRocksDatabase
@@ -1573,6 +1680,10 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 				clearInterruptedDropEntries(rootStore.path, tableName);
 			}
 			if (rootStore instanceof RocksDatabase) {
+				// Usually a genuinely new column family (existingTableMeta above found no catalog
+				// entry), but an interrupted drop just completed above can leave the physical CF
+				// behind under its old codec even though the catalog entry is gone — same fallback
+				// as the reconcile paths covers that remnant case too.
 				primaryStore = openRocksDatabase(rootStore.path, { ...dbiInit, name: dbiName, cache: true } as any);
 			} else {
 				primaryStore = (rootStore as any).openDB(dbiName, dbiInit as any);
@@ -2183,7 +2294,7 @@ function completeInterruptedDrop(rootStore, attributesDbi, databaseName: string,
 	if (rootStore instanceof RocksDatabase) {
 		for (const columnName of (rootStore as any).columns) {
 			if (columnName.startsWith(tableName + '/')) {
-				const columnStore = openRocksDatabase(rootStore.path, { name: columnName } as any);
+				const columnStore = openRocksDatabase(rootStore.path, { name: columnName });
 				try {
 					columnStore.dropSync();
 				} catch (error) {

package/resources/indexes/HierarchicalNavigableSmallWorld.ts

@@ -3,7 +3,6 @@ import { FLOAT32_OPTIONS } from 'msgpackr';
 import { loggerWithTag } from '../../utility/logging/logger.ts';
 import { ClientError } from '../../utility/errors/hdbError.ts';
 import type { Id } from '../../resources/ResourceInterface.ts';
-import { RocksDatabase } from '@harperfast/rocksdb-js';
 import { SKIP } from '@harperfast/extended-iterable';
 
 const logger = loggerWithTag('HNSW');
@@ -50,13 +49,35 @@ function dequantizeInt8(q: Int8Array, scale: number): number[] {
 // cap deliberately favors latency — apps wanting higher recall set efConstructionSearch or a per-query
 // ef. Tune as graph build quality / larger-N data improves.
 const AUTO_EF_BASE = 100;
-const AUTO_EF_REF = 1000;
+// The index store holds a graph node plus a primary-key mapping per record, so a key count is twice
+// the node count. Sizes here are in nodes; this converts back for the one consumer still calibrated
+// against keys.
+const INDEX_KEYS_PER_NODE = 2;
+// Graph nodes at which ef equals AUTO_EF_BASE. Originally expressed per index-store key against a
+// reference of 1000, so 500 nodes is the same point. The node count is a high-water mark rather than
+// a live count, so the resolved ef can differ from that formula by one at a rounding boundary.
+const AUTO_EF_REF = 500;
 const AUTO_EF_MAX = 512;
 function autoScaleEf(nodeCount: number): number {
 	const scaled = Math.round(AUTO_EF_BASE * Math.sqrt(Math.max(1, nodeCount / AUTO_EF_REF)));
 	return Math.min(AUTO_EF_MAX, Math.max(AUTO_EF_BASE, scaled));
 }
 
+// Candidate-list size used when searching a layer above 0. Those layers only supply the entry point
+// for the next layer down, so a greedy walk is enough; a larger value costs work proportional to the
+// layer's population without improving the entry point it hands off.
+const ROUTING_EF = 1;
+// Ceiling on the ef a query's own `offset + limit` can ask for. `limit` is unprivileged and set on
+// every request, so this bounds what a caller can make one thread do synchronously: layer 0 holds
+// `ef` candidates in a sorted array with an O(len) insert. Kept within a small multiple of
+// AUTO_EF_MAX so the worst case stays the same order as the index's own auto-scaled ceiling; a
+// caller who genuinely wants more sets an explicit `ef` and owns the cost.
+const LIMIT_EF_MAX = 4 * AUTO_EF_MAX;
+// How long a resolved graph size is reused before it is looked up again (see approximateNodeCount).
+// ef moves with the square root of the count and is capped, so a slightly stale size is immaterial;
+// this only has to be short enough that a table growing from empty picks up a larger ef promptly.
+const NODE_COUNT_TTL = 10_000;
+
 class MinHeap {
 	private data: Candidate[] = [];
 	get size() {
@@ -181,6 +202,8 @@ export class HierarchicalNavigableSmallWorld {
 	// frozen node the object store hands back. WeakMap so entries are collected when the store evicts
 	// the frozen node — without it, every cache hit on a frozen node would re-slice and re-clone.
 	private convertedNodes = new WeakMap<object, any>();
+	private nodeCount = 0;
+	private nodeCountAt = 0;
 	constructor(indexStore: any, options: any) {
 		this.indexStore = indexStore;
 		if (indexStore) {
@@ -334,17 +357,12 @@ export class HierarchicalNavigableSmallWorld {
 				this.indexStore.put(ENTRY_POINT, nodeId, options);
 			}
 
-			// For each level from top to bottom
+			// Pure descent — only neighbors[0] is used — so it runs greedily for the same reason
+			// search() does. The connection-building pass below keeps efConstruction: it selects the
+			// edges that get stored.
 			while (currentLevel > level) {
 				// Search for closest neighbors at current level
-				const neighbors = this.searchLayer(
-					vector,
-					entryPointId,
-					entryPoint,
-					this.efConstruction,
-					currentLevel,
-					options
-				);
+				const neighbors = this.searchLayer(vector, entryPointId, entryPoint, ROUTING_EF, currentLevel, options);
 
 				if (neighbors.length > 0) {
 					entryPointId = neighbors[0].id; // closest neighbor becomes new entry point
@@ -692,6 +710,34 @@ export class HierarchicalNavigableSmallWorld {
 		}
 	}
 
+	/**
+	 * Number of nodes in the graph, for the ef auto-scale. Must stay O(1): it runs per query, and an
+	 * exact count is a full key scan. RocksDB's `estimate-num-keys` is not a usable substitute — it
+	 * counts unreconciled overwrites, which graph construction produces in bulk. See DESIGN.md for the
+	 * measurements behind both. The memo is the only gate on how often the size is resolved; nothing on
+	 * the query path may bypass it, or the O(1) lookup becomes per-query work again.
+	 */
+	private approximateNodeCount(): number {
+		const now = Date.now();
+		if (this.nodeCountAt > 0 && now - this.nodeCountAt < NODE_COUNT_TTL) return this.nodeCount;
+		this.nodeCount = this.resolveNodeCount();
+		this.nodeCountAt = now;
+		return this.nodeCount;
+	}
+
+	/** O(1) node count — the shared id counter, else a single reverse seek to the largest node id. */
+	private resolveNodeCount(): number {
+		if (this.idIncrementer) return Number(Atomics.load(this.idIncrementer, 0));
+		try {
+			for (const key of this.indexStore.getKeys({ reverse: true, limit: 1, start: Infinity, end: 0 })) {
+				if (typeof key === 'number') return key + 1;
+			}
+		} catch (error) {
+			logger.debug?.('could not resolve node count from the largest node id', error);
+		}
+		return 0; // empty (or unreadable) graph — autoScaleEf falls back to its floor
+	}
+
 	private safeGetSync(key: any, options?: any): any {
 		try {
 			let node = this.indexStore.getSync(key, options);
@@ -945,7 +991,12 @@ export class HierarchicalNavigableSmallWorld {
 		// returns true are admitted to the result list at layer 0; routing is unaffected. Composed by
 		// search.ts from companion AND conditions and caller-supplied vector/row filters. Must be
 		// synchronous and side-effect free. JS-API only (never from a REST query string).
-		filter?: (primaryKey: Id) => boolean
+		filter?: (primaryKey: Id) => boolean,
+		// offset + limit for a bounded query. A layer-0 search returns at most `ef` candidates, so a
+		// query asking for more rows than that used to come back short with no error — capped at 512
+		// (AUTO_EF_MAX) however large the limit was. Raising ef to cover the request keeps `limit`
+		// meaningful; the caller pays for what it asked for.
+		minResults?: number
 	) {
 		let limit: number | undefined; // only set for threshold comparators; 0 is a valid threshold (e.g. dotProduct)
 		let limitInclusive = false; // true for `le`, false for `lt`
@@ -978,18 +1029,33 @@ export class HierarchicalNavigableSmallWorld {
 		// Resolve search ef: per-query ef wins; else an explicitly-configured efConstructionSearch;
 		// else auto-scale with the graph size so recall holds as the table grows.
 		let effectiveEf = this.efConstructionSearch;
-		if (ef !== undefined && ef > 0) effectiveEf = ef;
-		else if (!this.efSearchConfigured) {
-			const nodeCount = this.indexStore.getKeysCount
-				? this.indexStore.getKeysCount()
-				: (this.indexStore.getStats?.()?.entryCount ?? 0);
-			effectiveEf = autoScaleEf(nodeCount);
+		const explicitEf = ef !== undefined && ef > 0;
+		if (explicitEf) effectiveEf = ef;
+		else if (!this.efSearchConfigured) effectiveEf = autoScaleEf(this.approximateNodeCount());
+		// The ef the index chose for itself, before any limit-derived widening. The filter budget below
+		// stays calibrated against this rather than against what a caller's `limit` asked for.
+		const resolvedEf = effectiveEf;
+		// A bounded query must be able to come back full: layer 0 keeps at most `ef` candidates, so a
+		// limit above the resolved ef truncated the result set with no error. Widen the candidate list
+		// to cover the request, up to LIMIT_EF_MAX — `ef` sizes a synchronous traversal that holds every
+		// admitted candidate in a sorted array with an O(len) insert, so an unbounded one lets a plain
+		// `limit` stall the thread. Past the ceiling the result set is still short, as it was before.
+		// A per-query `ef` is left authoritative: it is an explicit cost ceiling, and a caller who sets
+		// one has said what they are willing to spend.
+		// The ceiling is the only bound: clamping to the graph size as well would need a count exact as
+		// of this query — the memo reads low while a table grows, truncating the very limit this
+		// honours — and an ef above the node count is free, the traversal ending at the graph, not ef.
+		if (minResults !== undefined && !explicitEf && minResults > effectiveEf) {
+			effectiveEf = Math.max(effectiveEf, Math.min(minResults, LIMIT_EF_MAX));
 		}
 		// Predicate-aware traversal budget (#1241): matches accrue slower than visits under a selective
 		// filter, so bound layer-0 work at ef * filterExpansion nodes. Only built when a filter is active.
+		// Deliberately `resolvedEf`, not the limit-widened ef: this budget is what stops a selective
+		// filter crawling the whole graph, and multiplying it by a caller's limit would turn a filtered
+		// vector query into a record-loading scan.
 		const filterState: FilterState | undefined = filter
 			? {
-					maxVisits: effectiveEf * (filterExpansion && filterExpansion > 0 ? filterExpansion : this.filterExpansion),
+					maxVisits: resolvedEf * (filterExpansion && filterExpansion > 0 ? filterExpansion : this.filterExpansion),
 					nodesVisited: 0,
 					filterEvaluations: 0,
 				}
@@ -1000,13 +1066,17 @@ export class HierarchicalNavigableSmallWorld {
 		let results: Candidate[] = [];
 		// For each level from top to bottom. The filter applies only at layer 0 (result admission);
 		// upper layers route unfiltered so non-matching hubs still guide the descent.
+		//
+		// Only layer 0 gets the full candidate list; the layers above it just hand down an entry point,
+		// so searching them at the full ef costs work proportional to the layer's population (~N/M
+		// nodes) rather than to ef. See DESIGN.md.
 		for (let l = entryPoint.level; l >= 0; l--) {
 			// Search for closest neighbors at current level
 			results = this.searchLayer(
 				target,
 				entryPointId,
 				entryPoint,
-				effectiveEf,
+				l === 0 ? effectiveEf : ROUTING_EF,
 				l,
 				options,
 				distanceFunction,
@@ -1040,10 +1110,8 @@ export class HierarchicalNavigableSmallWorld {
 	 * and $distance.
 	 */
 	exactDistance(searchCondition: { target: number[]; distance?: string }, recordVector: number[] | Int8Array): number {
-		if (recordVector == null) return Infinity; // missing vector sorts last
-		// distance fns require a plain Array (they guard on Array.isArray); records normally store a
-		// float[] vector, but convert defensively in case a typed array slips through.
-		const vec = Array.isArray(recordVector) ? recordVector : Array.from(recordVector);
+		if (!searchCondition.target) throw new ClientError('A target vector must be provided for an HNSW query');
+		if (!Array.isArray(searchCondition.target)) throw new ClientError('The target vector must be an array');
 		const fn =
 			searchCondition.distance === 'euclidean'
 				? euclideanDistance
@@ -1051,7 +1119,14 @@ export class HierarchicalNavigableSmallWorld {
 					? dotProductDistance
 					: searchCondition.distance === 'cosine'
 						? cosineDistance
-						: this.distance;
+						: searchCondition.distance
+							? null
+							: this.distance;
+		if (!fn) throw new ClientError('Unknown distance function');
+		if (recordVector == null) return Infinity; // missing vector sorts last
+		// distance fns require a plain Array (they guard on Array.isArray); records normally store a
+		// float[] vector, but convert defensively in case a typed array slips through.
+		const vec = Array.isArray(recordVector) ? recordVector : Array.from(recordVector);
 		return fn(searchCondition.target, vec);
 	}
 	/**
@@ -1223,9 +1298,11 @@ export class HierarchicalNavigableSmallWorld {
 	 * @returns
 	 */
 	estimateCountAsSort() {
-		const count =
-			this.indexStore instanceof RocksDatabase ? this.indexStore.getKeysCount() : this.indexStore.getStats().entryCount;
-		return Math.sqrt(count * this.efConstructionSearch);
+		// Same O(1) source search() uses — this runs per query whenever a vector sort is planned
+		// alongside another condition, where an exact getKeysCount() is a second full key scan on the
+		// query path. Scaled back to the index-store key count it used to be given, so the planner's
+		// condition ordering is unchanged by the switch to a node count.
+		return Math.sqrt(this.approximateNodeCount() * INDEX_KEYS_PER_NODE * this.efConstructionSearch);
 	}
 
 	/**
@@ -1234,27 +1311,24 @@ export class HierarchicalNavigableSmallWorld {
 	 * @param vector
 	 * @param context
 	 * @param entry
+	 * @param sortDefinition
 	 */
-	propertyResolver(vector: number[], context: any, entry: any) {
-		const sortDefinition = context?.sort;
+	propertyResolver(vector: number[], context: any, entry: any, sortDefinition?: any) {
 		if (sortDefinition) {
+			if (!context) return this.exactDistance(sortDefinition, vector);
 			// set up a cache for these so they can be accessed by $distance and not be recalculated during a sort
-			let vectorDistances = sortDefinition.vectorDistances;
-			if (vectorDistances) {
-				const difference = vectorDistances.get(entry);
-				if (difference) return difference;
-			} else vectorDistances = context.vectorDistances = sortDefinition.vectorDistances = new Map();
+			let vectorDistanceCaches = context.vectorDistanceCaches;
+			if (!vectorDistanceCaches) vectorDistanceCaches = context.vectorDistanceCaches = new WeakMap();
+			let vectorDistances = vectorDistanceCaches.get(sortDefinition);
+			const cacheKey =
+				entry && typeof entry === 'object' ? entry : vector && typeof vector === 'object' ? vector : null;
+			if (vectorDistances && cacheKey) {
+				const difference = vectorDistances.get(cacheKey);
+				if (difference !== undefined) return difference;
+			} else if (!vectorDistances) vectorDistanceCaches.set(sortDefinition, (vectorDistances = new WeakMap()));
 
-			let distanceFunction = this.distance;
-			if (sortDefinition.type)
-				distanceFunction =
-					sortDefinition.distance === 'euclidean'
-						? euclideanDistance
-						: sortDefinition.distance === 'dotProduct'
-							? dotProductDistance
-							: cosineDistance;
-			const distance = distanceFunction(sortDefinition.target, vector);
-			vectorDistances.set(entry, distance);
+			const distance = this.exactDistance(sortDefinition, vector);
+			if (cacheKey) vectorDistances.set(cacheKey, distance);
 			return distance;
 		}
 		return vector;