@harperfast/harper 5.2.12 → 5.2.14

package/dist/resources/Table.js

@@ -102,6 +102,20 @@ const EVICTION_BATCH_SIZE = 100;
 // letting an unbounded number of open transactions (and their snapshots) accumulate.
 const MAX_INFLIGHT_EVICTION_BATCHES = 4;
 const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]);
+// Smallest forward sample `getRecordCount` will extrapolate a record rate from; below it the scan runs
+// to completion and reports an exact count.
+const MIN_ESTIMATOR_SAMPLE = 1_000;
+// Budget intervals the forward scan may spend before it must estimate rather than keep scanning.
+const MAX_ESTIMATE_CHECKPOINTS = 20;
+// A store estimate's `count`, or 0 when the store answered with a shape that cannot be trusted --
+// DESIGN.md's invariant for this API family is that such an answer degrades rather than poisons.
+function usableCount(estimate) {
+    const { count, confidence } = estimate ?? {};
+    // `confidence` needs its own finiteness check, not just the range: `null >= 0 && null <= 1` is true
+    if (!Number.isFinite(count) || count < 0 || !Number.isFinite(confidence) || confidence < 0 || confidence > 1)
+        return 0;
+    return count;
+}
 envMngr.initSync();
 const LMDB_PREFETCH_WRITES = envMngr.get(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PREFETCHWRITES);
 const LOCK_TIMEOUT = 10000;
@@ -1534,7 +1548,7 @@ function makeTable(options) {
                     records: './', // an href to the records themselves
                     name: tableName,
                     database: databaseName,
-                    auditSize: auditStore instanceof rocksdb_js_1.RocksDatabase ? auditStore.getKeysCount() : auditStore?.getStats().entryCount,
+                    auditSize: auditStore?.getStats().entryCount,
                     attributes,
                     recordCount: undefined,
                     estimatedRecordRange: undefined,
@@ -2485,7 +2499,10 @@ function makeTable(options) {
                                     // getRange yields ascending by audit-log key, so the first entry is the oldest retained.
                                     // Mirror replicationConnection's retention check and the cleanup key basis (localTime ??
                                     // version). Fall back to the nominal time-based purge floor when the log is empty/unavailable.
-                                    for (const entry of auditStore.getRange({ start: 1, log: options?.nodeId })) {
+                                    for (const entry of auditStore.getRange({
+                                        start: 1,
+                                        log: auditStore.logFor(options?.nodeId, options?.viaNodeId).name,
+                                    })) {
                                         oldestRetainedAuditTime = entry.localTime ?? entry.version;
                                         break;
                                     }
@@ -2509,7 +2526,7 @@ function makeTable(options) {
                             // A recommit of the same transaction survived that skip only because the old write batch
                             // still carried the put; a fresh-transaction replay (ERR_TRY_AGAIN) would drop the write.
                             if (isRocksDB && !stagedOwnAuditEntry && dedupVersionCouldBeRetained(txnTime)) {
-                                const priorAudit = auditStore.get(txnTime, tableId, id, options?.nodeId);
+                                const priorAudit = auditStore.get(txnTime, tableId, id, options?.nodeId, options?.viaNodeId);
                                 if (priorAudit &&
                                     priorAudit.version === txnTime &&
                                     precedesExistingVersion(txnTime, { version: txnTime, localTime: txnTime, key: id, nodeId: priorAudit.nodeId }, options?.nodeId) === 0) {
@@ -2558,7 +2575,7 @@ function makeTable(options) {
                                     return false;
                                 if (!dedupVersionCouldBeRetained(txnTime))
                                     return false; // pre-retention version — skip the end-of-log scan (best-effort; see above)
-                                const duplicate = auditStore.get(txnTime, tableId, id, options?.nodeId);
+                                const duplicate = auditStore.get(txnTime, tableId, id, options?.nodeId, options?.viaNodeId);
                                 return (duplicate &&
                                     duplicate.version === txnTime &&
                                     precedesExistingVersion(txnTime, { version: txnTime, localTime: txnTime, key: id, nodeId: duplicate.nodeId }, options?.nodeId) === 0);
@@ -2966,6 +2983,7 @@ function makeTable(options) {
                 commit: (txnTime, existingEntry, retry, transaction) => {
                     write.stagedEntry = undefined; // reset per round; set below once the removal is applied
                     write.superseded = false; // reset per round, as in the update path
+                    write.skipped = false;
                     // what a preceding write in this transaction left for this key is what gets removed
                     // from the indices here, not the pre-transaction record (harper#1968)
                     const priorStagedOp = (0, DatabaseTransaction_ts_1.priorStagedWrite)(write);
@@ -2984,6 +3002,20 @@ function makeTable(options) {
                     if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0) {
                         return;
                     }
+                    const removalNodeId = options?.nodeId ?? 0; // this node's own id is 0
+                    const stagedRemoval = { value: undefined, version: txnTime, nodeId: removalNodeId };
+                    // re-delivered onto the removal it already made: writing again would append an audit entry
+                    // that every peer forwards and re-logs in turn. On a recommit the kept write batch shows this write's own
+                    // round-one tombstone as the existing entry, so a retry round repeats the first round's decision.
+                    const redelivered = retry
+                        ? write.redelivered
+                        : existingRecord == null && isSameRemoval(removalBefore(write, existingEntry), txnTime, removalNodeId);
+                    write.redelivered = redelivered;
+                    if (redelivered) {
+                        write.stagedEntry = stagedRemoval;
+                        write.skipped = true;
+                        return;
+                    }
                     updateIndices(id, existingRecord, null, transaction && { transaction });
                     if (audit || trackDeletes) {
                         updateRecord(id, null, existingEntry, txnTime, 0, audit, {
@@ -3000,7 +3032,7 @@ function makeTable(options) {
                         // Only RocksDB's remove() takes an options object; on LMDB the 2nd arg is ifVersion, and its writes already join the batched txn.
                         (0, RecordEncoder_ts_1.removeEntry)(primaryStore, existingEntry, isRocksDB && transaction ? { transaction } : undefined);
                     }
-                    write.stagedEntry = { value: undefined }; // the key holds no record for the rest of this transaction
+                    write.stagedEntry = stagedRemoval; // the key holds no record for the rest of this transaction
                     // the removal supersedes the nearest record an earlier write in this transaction stored
                     // (older ones were already marked by their staged successors), so its saved blobs are
                     // cleaned up post-commit unless its audit entry references them
@@ -4674,85 +4706,182 @@ function makeTable(options) {
             const exactCount = options?.exactCount;
             const TIME_LIMIT = options?.timeLimit ?? 1000 / 2; // one second time limit, enforced by seeing if we are halfway through at 500ms
             const start = performance.now();
-            // `entryCount` (the exact key count) is only needed once the scan blows the time budget --
-            // to decide whether to estimate and as the extrapolation base. On RocksDB it is a full
-            // key-only scan, so we defer it: tables that finish within budget (the common case) and
-            // `exact_count` requests never pay for it. `halfway`/`entryCount` stay 0 until first computed.
             let entryCount = 0;
-            let halfway = 0;
-            let counted = false;
-            let completeForExact = false;
+            let remainderPhysical = 0;
+            let estimator;
+            // feature-detected per DESIGN.md's invariant for this API family; LMDB stores do not implement it
+            const canEstimate = typeof primaryStore.createCountEstimator === 'function' && typeof primaryStore.estimateCount === 'function';
+            let estimatorFailed = false;
+            let warnedNoBase = false;
+            let checkpoints = 0;
+            let checkpointedEntries = 0;
             let recordCount = 0;
             let entriesScanned = 0;
+            let lastKey;
             let limit;
-            for (const { value } of primaryStore.getRange({ start: true, lazy: true, snapshot: false })) {
+            let nextCheckAt = start + TIME_LIMIT;
+            for (const { key, value } of primaryStore.getRange({ start: true, lazy: true, snapshot: false })) {
                 if (value != null)
                     recordCount++;
                 entriesScanned++;
+                lastKey = key;
                 await rest();
-                if (!exactCount && !completeForExact && performance.now() - start > TIME_LIMIT) {
-                    if (!counted) {
-                        counted = true;
-                        entryCount = isRocksDB
-                            ? primaryStore.getKeysCount({ start: undefined })
-                            : primaryStore.getStats().entryCount;
-                        halfway = Math.floor(entryCount / 2);
-                    }
-                    if (entriesScanned < halfway) {
-                        // it is taking too long, so we will just take this sample and a sample from the end to estimate
-                        limit = entriesScanned;
-                        break;
+                // a table too small to reach the floor is small enough to finish exactly
+                if (exactCount || entriesScanned < MIN_ESTIMATOR_SAMPLE)
+                    continue;
+                const now = performance.now();
+                if (now <= nextCheckAt)
+                    continue;
+                nextCheckAt = now + TIME_LIMIT;
+                checkpoints++;
+                if (canEstimate && !estimatorFailed) {
+                    try {
+                        estimator ??= primaryStore.createCountEstimator({ start: true });
+                        estimator.advance(lastKey, entriesScanned - checkpointedEntries);
+                        checkpointedEntries = entriesScanned;
+                        entryCount = usableCount(estimator.estimate());
+                    }
+                    catch (error) {
+                        // a store closing concurrently -- drop_table can, while this scan is parked in a yield
+                        logger_ts_1.logger.debug?.('Count estimator unavailable, falling back to an exact scan', error);
+                        estimatorFailed = true;
+                        estimator = undefined;
+                        entryCount = 0;
+                    }
+                }
+                else if (!canEstimate) {
+                    // `canEstimate` false is not the same as "LMDB": a RocksDB store whose native module predates
+                    // the estimator API lands here too, and `RocksDatabase.getStats()` carries no `entryCount`.
+                    try {
+                        const stats = primaryStore.getStats?.();
+                        entryCount = Number.isFinite(stats?.entryCount) && stats.entryCount > 0 ? stats.entryCount : 0;
+                    }
+                    catch {
+                        entryCount = 0;
+                    }
+                }
+                if (!entryCount && canEstimate && !estimatorFailed) {
+                    // Range estimates are block-granular and can report 0 for a store whose entries are still
+                    // in the memtable. Without a base the escape cannot fire at all, so fall back to the
+                    // whole-store property rather than silently walking the table.
+                    try {
+                        const wholeStore = primaryStore.getEstimatedKeyCount();
+                        entryCount = Number.isFinite(wholeStore) && wholeStore > 0 ? wholeStore : 0;
+                    }
+                    catch {
+                        entryCount = 0;
+                    }
+                    if (!entryCount && !warnedNoBase) {
+                        warnedNoBase = true;
+                        logger_ts_1.logger.debug?.(`No usable key-count estimate for ${tableName}; counting records by full scan`);
+                    }
+                }
+                // Zero is "no usable base": degrade to the exact scan. The checkpoint ceiling is what stops a
+                // base that keeps undershooting from holding the halfway test false forever and walking the
+                // whole table; the reverse sample is bounded by `limit` in turn.
+                if (entryCount > 0 &&
+                    (checkpoints >= MAX_ESTIMATE_CHECKPOINTS || entriesScanned < Math.floor(entryCount / 2))) {
+                    if (canEstimate) {
+                        try {
+                            const remaining = primaryStore.estimateCount({ start: lastKey, exclusiveStart: true });
+                            // widened by its own reported untrustworthiness: block-granular, so it can land below
+                            // the live count it is meant to bound
+                            const remainingCount = usableCount(remaining);
+                            remainderPhysical = remainingCount > 0 ? remainingCount * (2 - remaining.confidence) : 0;
+                        }
+                        catch {
+                            remainderPhysical = 0;
+                        }
+                        // A zero or unusable remainder is valid -- entries still in the memtable read as none
+                        // through range statistics -- but it would leave `baseMax` resting on the sampled ends
+                        // alone. The whole-store property is a separate, non-range source, so fall back to it.
+                        if (!remainderPhysical) {
+                            try {
+                                const wholeStore = primaryStore.getEstimatedKeyCount();
+                                if (Number.isFinite(wholeStore))
+                                    remainderPhysical = Math.max(wholeStore - entriesScanned, 0);
+                            }
+                            catch {
+                                remainderPhysical = 0;
+                            }
+                        }
                     }
-                    // Past the halfway point already: finishing the scan for an exact count is cheaper
-                    // than estimating. Set the flag so we stop re-evaluating the budget on each remaining iteration.
-                    completeForExact = true;
+                    limit = entriesScanned;
+                    break;
                 }
             }
             if (limit) {
                 // in this case we are going to make an estimate of the table count using the first thousand
                 // entries and last thousand entries
                 const firstRecordCount = recordCount;
+                const firstKey = lastKey;
                 recordCount = 0;
                 // Bound the reverse scan explicitly. The getRange `limit` option is honored by lmdb-js but
                 // ignored by rocksdb-js; without this break the scan reads the whole table, so `recordRate`
                 // blows up to ~entryCount/(2*limit) and the estimate scales with entryCount^2 -- the source
                 // of the wildly inflated `record_count` (e.g. 20,000,000 for ~105k rows) on large RocksDB
-                // tables. The early-exit above guarantees limit < entryCount/2, so the two samples stay disjoint.
+                // tables.
                 let reverseScanned = 0;
-                for (const { value } of primaryStore.getRange({
+                // Sized independently of the forward scan. `entriesScanned` is whatever the forward pass
+                // covered before it escaped, and the checkpoint ceiling lets that run twenty budget
+                // intervals when the base keeps undershooting; matching it here would read that same count
+                // again and double the wall clock of the call this path exists to bound.
+                const reverseLimit = Math.min(limit, MIN_ESTIMATOR_SAMPLE);
+                // Disjointness is enforced against the forward scan's own last key rather than inferred from
+                // the base, which is an estimate that can overshoot by more than 2x.
+                let sampledWholeTable = false;
+                for (const { key, value } of primaryStore.getRange({
                     start: '\uffff',
                     reverse: true,
                     lazy: true,
-                    limit,
+                    limit: reverseLimit,
                     snapshot: false,
                 })) {
+                    if ((0, ordered_binary_1.compareKeys)(key, firstKey) <= 0) {
+                        sampledWholeTable = true;
+                        break;
+                    }
                     if (value != null)
                         recordCount++;
                     reverseScanned++;
                     await rest();
-                    if (reverseScanned >= limit)
+                    if (reverseScanned >= reverseLimit)
                         break;
                 }
+                // the samples met, so between them they covered every entry
+                if (sampledWholeTable)
+                    return { recordCount: recordCount + firstRecordCount };
                 // Use the actual entries sampled, not limit*2: the reverse scan can yield fewer than `limit`
                 // (concurrent deletions under snapshot:false, or an overestimated entryCount), and counting
                 // those un-scanned slots would inflate the denominator and underestimate the rate.
-                const sampleSize = limit + reverseScanned;
-                const recordRate = (recordCount + firstRecordCount) / sampleSize;
-                const variance = Math.pow((recordCount - firstRecordCount + 1) / limit / 2, 2) + // variance between samples
-                    (recordRate * (1 - recordRate)) / sampleSize;
-                const sd = Math.max(Math.sqrt(variance) * entryCount, 1);
-                const estimatedRecordCount = Math.round(recordRate * entryCount);
-                // TODO: This uses a normal/Wald interval, but a binomial confidence interval is probably better calculated using
-                // Wilson score interval or Agresti-Coull interval (I think the latter is a little easier to calculate/implement).
-                const lowerCiLimit = Math.max(estimatedRecordCount - 1.96 * sd, recordCount + firstRecordCount);
-                const upperCiLimit = Math.min(estimatedRecordCount + 1.96 * sd, entryCount);
-                let significantUnit = Math.pow(10, Math.round(Math.log10(sd)));
-                if (significantUnit > estimatedRecordCount)
-                    significantUnit = significantUnit / 10;
-                recordCount = Math.round(estimatedRecordCount / significantUnit) * significantUnit;
+                const sampledRecords = recordCount + firstRecordCount;
+                const recordRate = sampledRecords / (limit + reverseScanned);
+                // Endpoints for the extrapolation base, spanning both ways an estimated base can be wrong:
+                // every remaining entry superseded (only what was sampled is live) through every remaining
+                // entry live (the uncalibrated physical count). Churn concentrated outside the sampled ends
+                // calibrates to nothing, so an interval derived from the estimator's confidence would sit
+                // narrowly around the wrong number. Both endpoints are themselves estimates on RocksDB, so
+                // this is a widened heuristic interval, not a guaranteed bound on the live count.
+                const baseMin = entriesScanned + reverseScanned;
+                const baseMax = Math.max(entriesScanned + remainderPhysical, entryCount, baseMin);
+                const estimatedRecordCount = Math.round(recordRate * Math.max(entryCount, baseMin));
+                // The samples counted these directly, and the entries between them can only add; everything
+                // outside the samples could be live. A statistical interval inside those endpoints would be
+                // narrowest exactly where the ends are least representative of the middle -- sampled ends
+                // that are all deletion entries give a rate of 0, collapsing an upper end to ~0 with live
+                // rows in between -- so the endpoints are the evidence itself.
+                const lower = sampledRecords;
+                const upper = Math.round(baseMax);
+                // Report only the precision the interval supports, but never so coarse a unit that the
+                // estimate rounds away: `baseMax` is physical and can exceed a calibrated estimate by
+                // orders of magnitude, which a single division cannot walk back.
+                let significantUnit = Math.pow(10, Math.round(Math.log10(Math.max((upper - lower) / 2, 1))));
+                while (significantUnit > estimatedRecordCount && significantUnit > 1)
+                    significantUnit /= 10;
+                recordCount = Math.min(Math.max(Math.round(estimatedRecordCount / significantUnit) * significantUnit, lower), upper);
                 return {
                     recordCount,
-                    estimatedRange: [Math.round(lowerCiLimit), Math.round(upperCiLimit)],
+                    estimatedRange: [lower, upper],
                 };
             }
             return {
@@ -5670,6 +5799,24 @@ function makeTable(options) {
         }
         return ids;
     }
+    /**
+     * What left a delete's key without a record: the nearest earlier write in the transaction that staged
+     * state, else the stored entry. Only writes marked skipped are passed over, because an invalidate or
+     * relocate stores a null stub without staging it.
+     */
+    function removalBefore(write, existingEntry) {
+        for (let prior = write.priorWrite; prior; prior = prior.priorWrite) {
+            if (prior.stagedEntry)
+                return prior.stagedEntry;
+            if (!prior.skipped)
+                return;
+        }
+        if (!(existingEntry?.metadataFlags & exports.INVALIDATED))
+            return existingEntry;
+    }
+    function isSameRemoval(entry, txnTime, nodeId) {
+        return entry != null && entry.value == null && entry.version === txnTime && (entry.nodeId ?? 0) === nodeId;
+    }
     function precedesExistingVersion(txnTime, existingEntry, nodeId) {
         if (nodeId === undefined) {
             nodeId = (0, nodeIdMapping_ts_1.getThisNodeId)(auditStore);
@@ -5903,25 +6050,35 @@ function makeTable(options) {
                     resolve(resolvedEntry);
                 }
                 catch (error) {
-                    error.message += ` while resolving record ${id} for ${tableName}`;
-                    if (existingRecord &&
-                        (((error.code === 'ECONNRESET' || error.code === 'ECONNREFUSED' || error.code === 'EAI_AGAIN') &&
-                            !context?.mustRevalidate) ||
-                            (context?.staleIfError &&
-                                (error.statusCode === 500 ||
-                                    error.statusCode === 502 ||
-                                    error.statusCode === 503 ||
-                                    error.statusCode === 504)))) {
-                        // these are conditions under which we can use stale data after an error
-                        resolve({
-                            key: id,
-                            version: existingVersion,
-                            value: existingRecord,
-                        });
-                        logger_ts_1.logger.trace?.(error.message, '(returned stale record)');
+                    // A source may reject with anything at all, so deciding how to settle is itself
+                    // fallible: `message` is not assignable on every error (a DOMException from
+                    // AbortSignal.timeout), and a nullish rejection makes the reads below throw.
+                    // Leaving this promise unsettled hangs the caller forever, so every path here
+                    // has to end in resolve() or reject().
+                    try {
+                        (0, hdbError_ts_1.appendErrorContext)(error, ` while resolving record ${id} for ${tableName}`);
+                        if (existingRecord &&
+                            (((error.code === 'ECONNRESET' || error.code === 'ECONNREFUSED' || error.code === 'EAI_AGAIN') &&
+                                !context?.mustRevalidate) ||
+                                (context?.staleIfError &&
+                                    (error.statusCode === 500 ||
+                                        error.statusCode === 502 ||
+                                        error.statusCode === 503 ||
+                                        error.statusCode === 504)))) {
+                            // these are conditions under which we can use stale data after an error
+                            resolve({
+                                key: id,
+                                version: existingVersion,
+                                value: existingRecord,
+                            });
+                            logger_ts_1.logger.trace?.(error?.message, '(returned stale record)');
+                        }
+                        else
+                            reject(error);
+                    }
+                    catch (settlingError) {
+                        reject(error ?? settlingError);
                     }
-                    else
-                        reject(error);
                     const resolveDuration = performance.now() - start;
                     (0, write_ts_1.recordAction)(resolveDuration, 'cache-resolution', tableName, null, 'fail');
                     if (responseHeaders)