@harperfast/harper 5.2.10 → 5.3.0-alpha.1

package/dist/resources/replayLogsGuards.js

@@ -9,12 +9,17 @@
 // running over potentially millions of entries, pinning CPU. These guards classify each
 // entry up front so the loop can skip cleanly.
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.REPLAY_WALL_CLOCK_LIMIT_MS = exports.REPLAY_NO_PROGRESS_TIME_SKIP_FLOOR = exports.REPLAY_NO_PROGRESS_TIME_LIMIT_MS = exports.REPLAY_NO_PROGRESS_COUNT_LIMIT = exports.RECORD_BEARING_FLAGS = void 0;
+exports.MAX_CORRUPT_FRAME_REPORTS = exports.REPLAY_WALL_CLOCK_LIMIT_MS = exports.REPLAY_NO_PROGRESS_TIME_SKIP_FLOOR = exports.REPLAY_NO_PROGRESS_TIME_LIMIT_MS = exports.REPLAY_NO_PROGRESS_COUNT_LIMIT = exports.RECORD_BEARING_FLAGS = void 0;
 exports.classifyAuditEntryForReplay = classifyAuditEntryForReplay;
 exports.isUndecodableValidatedWrite = isUndecodableValidatedWrite;
 exports.shouldAbortStalledReplay = shouldAbortStalledReplay;
 exports.shouldAbortSlowReplay = shouldAbortSlowReplay;
+exports.isMidLogBreak = isMidLogBreak;
 exports.endIteratorOnCorruptFrame = endIteratorOnCorruptFrame;
+exports.getCorruptFrameReports = getCorruptFrameReports;
+exports.getEvictedCorruptFrameReportCount = getEvictedCorruptFrameReportCount;
+exports.clearCorruptFrameReports = clearCorruptFrameReports;
+exports.createCorruptFrameReporter = createCorruptFrameReporter;
 // Mirrors `HAS_RECORD` (16) | `HAS_PARTIAL_RECORD` (32) from auditStore.ts — the action
 // bits the writer sets when an entry carries (or should carry) a record body. Redeclared
 // here so this module stays free of the Harper module graph for unit testing; a lock
@@ -122,12 +127,26 @@ function shouldAbortSlowReplay(totalElapsedMs, timeLimitMs = exports.REPLAY_WALL
     return totalElapsedMs >= timeLimitMs;
 }
 /**
- * Wraps a transaction-log query iterator so a corrupt/torn frame ends that log's iteration
- * cleanly instead of escaping as an uncaughtException. rocksdb-js throws a bounded RangeError
- * when an entry's framing is broken; framing loss means the next entry can't be located, so the
- * frame marks end-of-log (entries before it were already yielded) and startup replay /
- * replication broadcast continue. `onCorruptFrame` fires once, latched — kept a callback (not a
- * direct log) so this module stays out of the Harper module graph and is unit-testable.
+ * Whether a corrupt frame is a mid-log break — intact entries followed it, so entries were lost
+ * rather than the log merely being torn at its tail. Shared by the reporter below and by
+ * `RocksTransactionLogStore.getRange`'s `CorruptFrameStop.midLogBreak`, so the two can't drift on
+ * what counts as a mid-log break.
+ */
+function isMidLogBreak(error) {
+    return error.resyncPosition != null;
+}
+/**
+ * Wraps a transaction-log query iterator so a corrupt/torn frame ends that log's iteration cleanly
+ * instead of escaping as an uncaughtException.
+ *
+ * Iteration stops at the break whether or not intact entries follow it. Resuming past a mid-log
+ * break would recover those entries, but a transaction log carries no frame-level transaction
+ * boundaries: replay groups equal-version entries into one source transaction, so skipping a frame
+ * inside such a group applies and checkpoints the surviving subset of a transaction that never
+ * committed that way at the source. Availability loss is visible and recoverable; a silently torn
+ * transaction is neither. Recovery is deferred until the engine can resume at a proven boundary
+ * (harper#2016, harper#2063); `resyncPosition` is used only to report the break as lost entries
+ * rather than as a truncated tail.
  */
 function endIteratorOnCorruptFrame(iterator, onCorruptFrame) {
     let stopped = false;
@@ -167,4 +186,89 @@ function endIteratorOnCorruptFrame(iterator, onCorruptFrame) {
         },
     };
 }
+/** Distinct break sites retained. One physical corruption yields one site, so this is generous. */
+exports.MAX_CORRUPT_FRAME_REPORTS = 256;
+// Re-encounters move a site to the end, so the first key is the least recently seen. Full means
+// evicting that one rather than refusing a new site, which could not then be deduplicated.
+const corruptFrameReports = new Map();
+let evictedCorruptFrameReports = 0;
+/**
+ * Every corrupt frame seen by this worker. This is not yet consumed by cluster/health status;
+ * callers must aggregate it across worker threads before exposing a node-wide signal.
+ *
+ * Mid-log breaks are immediately logged at error level even without that consumer.
+ */
+function getCorruptFrameReports() {
+    return [...corruptFrameReports.values()];
+}
+/** Break sites evicted because {@link MAX_CORRUPT_FRAME_REPORTS} was reached. */
+function getEvictedCorruptFrameReportCount() {
+    return evictedCorruptFrameReports;
+}
+function clearCorruptFrameReports() {
+    corruptFrameReports.clear();
+    evictedCorruptFrameReports = 0;
+}
+// `logId`/`position` are absent on any rocksdb-js predating the resync support, and every break on
+// a stream would then collapse onto one key — folding genuinely different corruptions into a single
+// count that only ever logs once. The message carries the offset and file in text, so it separates
+// them when the fields can't.
+function corruptFrameKey(logName, error) {
+    const { logId, position } = error;
+    return logId != null && position != null
+        ? `${logName}\u0000${logId}:${position}`
+        : `${logName}\u0000${error.message}`;
+}
+/**
+ * Records a corrupt frame and logs it once per distinct break.
+ *
+ * A mid-log break is an `error`, not a `warn`: entries after it were acknowledged and are now
+ * quarantined behind it. Severity follows the break's own shape, never this pass's outcome — every
+ * pass stops, so keying on that would report the #2063 signature with the benign torn-tail `warn`.
+ * A site first seen as a torn tail can escalate later, since only a rocksdb-js that reports
+ * `resyncPosition` can tell the two apart.
+ */
+function createCorruptFrameReporter(logger) {
+    return (logName) => (error) => {
+        const midLog = isMidLogBreak(error);
+        const unreadableBytes = error.unreadableBytes ?? 0;
+        const now = Date.now();
+        const key = corruptFrameKey(logName, error);
+        const existing = corruptFrameReports.get(key);
+        if (existing) {
+            corruptFrameReports.delete(key);
+            corruptFrameReports.set(key, existing);
+            existing.occurrences++;
+            existing.lastSeen = now;
+            if (!midLog || existing.midLog)
+                return;
+            existing.midLog = true;
+            existing.unreadableBytes = unreadableBytes;
+        }
+        else {
+            if (corruptFrameReports.size >= exports.MAX_CORRUPT_FRAME_REPORTS) {
+                corruptFrameReports.delete(corruptFrameReports.keys().next().value);
+                evictedCorruptFrameReports++;
+            }
+            corruptFrameReports.set(key, {
+                log: logName,
+                logId: error.logId ?? undefined,
+                position: error.position ?? undefined,
+                midLog,
+                unreadableBytes,
+                firstSeen: now,
+                lastSeen: now,
+                occurrences: 1,
+            });
+        }
+        if (midLog) {
+            logger.error(`Corrupt entry in transaction log "${logName}"; ${unreadableBytes} byte(s) are unreadable and the entries within them are lost. ` +
+                'Intact entries follow the break, but reading stops there rather than skipping the frame, which could tear a source transaction: ' +
+                'they are quarantined until the log is repaired or this node is re-cloned, and they are neither replayed nor replicated meanwhile.', error);
+        }
+        else {
+            logger.warn(`Stopping transaction log "${logName}" at a corrupt entry during replay`, error);
+        }
+    };
+}
 //# sourceMappingURL=replayLogsGuards.js.map

package/dist/resources/replayLogsGuards.js.map

@@ -1 +1 @@
-{"version":3,"file":"replayLogsGuards.js","sourceRoot":"","sources":["../../resources/replayLogsGuards.ts"],"names":[],"mappings":";AAAA,qFAAqF;AACrF,uFAAuF;AACvF,EAAE;AACF,yFAAyF;AACzF,uFAAuF;AACvF,iFAAiF;AACjF,sFAAsF;AACtF,wFAAwF;AACxF,+CAA+C;;;AAuB/C,kEAUC;AAmBD,kEAEC;AAsCD,4DASC;AAoBD,sDAEC;AAUD,8DAmCC;AAtKD,wFAAwF;AACxF,yFAAyF;AACzF,qFAAqF;AACrF,oEAAoE;AACvD,QAAA,oBAAoB,GAAG,EAAE,GAAG,EAAE,CAAC;AAE5C;;;;;;;;;;;;;;GAcG;AACH,SAAgB,2BAA2B,CAC1C,MAA0B,EAC1B,OAA2B,EAC3B,SAAkB;IAElB,IAAI,MAAM,KAAK,SAAS,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,gBAAgB,CAAC;IAC3E,kFAAkF;IAClF,mFAAmF;IACnF,IAAI,CAAC,MAAM,GAAG,4BAAoB,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS;QAAE,OAAO,gBAAgB,CAAC;IACjF,OAAO,IAAI,CAAC;AACb,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,SAAgB,2BAA2B,CAAC,IAAwB,EAAE,MAAe;IACpF,OAAO,MAAM,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,SAAS,CAAC,CAAC;AACrF,CAAC;AAED,gGAAgG;AAChG,kGAAkG;AAClG,kGAAkG;AAClG,+FAA+F;AAC/F,mGAAmG;AACnG,gGAAgG;AAChG,gGAAgG;AAChG,0DAA0D;AAE1D,6FAA6F;AAC7F,mGAAmG;AACnG,mEAAmE;AACtD,QAAA,8BAA8B,GAAG,OAAO,CAAC;AAEtD,oGAAoG;AACpG,qGAAqG;AACrG,+DAA+D;AAClD,QAAA,gCAAgC,GAAG,MAAM,CAAC;AAEvD,oGAAoG;AACpG,gGAAgG;AAChG,oGAAoG;AACpG,kGAAkG;AACrF,QAAA,kCAAkC,GAAG,KAAK,CAAC;AAExD;;;;;;;;;;GAUG;AACH,SAAgB,wBAAwB,CACvC,aAAqB,EACrB,eAAuB,EACvB,UAAU,GAAG,sCAA8B,EAC3C,WAAW,GAAG,wCAAgC,EAC9C,aAAa,GAAG,0CAAkC;IAElD,IAAI,aAAa,IAAI,UAAU;QAAE,OAAO,IAAI,CAAC;IAC7C,OAAO,aAAa,IAAI,aAAa,IAAI,eAAe,IAAI,WAAW,CAAC;AACzE,CAAC;AAED,gGAAgG;AAChG,+FAA+F;AAC/F,gGAAgG;AAChG,iGAAiG;AACjG,+FAA+F;AAC/F,kGAAkG;AAClG,kCAAkC;AACrB,QAAA,0BAA0B,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAEzD;;;;;;;;GAQG;AACH,SAAgB,qBAAqB,CAAC,cAAsB,EAAE,WAAW,GAAG,kCAA0B;IACrG,OAAO,cAAc,IAAI,WAAW,CAAC;AACtC,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,yBAAyB,CACxC,QAAqB,EACrB,cAA2C;IAE3C,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,OAAO;QACN,CAAC,MAAM,CAAC,QAAQ,CAAC;YAChB,OAAO,IAAI,CAAC;QACb,CAAC;QACD,IAAI;YACH,IAAI,OAAO;gBAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;YACrD,IAAI,CAAC;gBACJ,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;YACxB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,yEAAyE;gBACzE,wEAAwE;gBACxE,IAAI,CAAC,CAAC,KAAK,YAAY,UAAU,CAAC;oBAAE,MAAM,KAAK,CAAC;gBAChD,OAAO,GAAG,IAAI,CAAC;gBACf,cAAc,CAAC,KAAK,CAAC,CAAC;gBACtB,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;YACzC,CAAC;QACF,CAAC;QACD,sFAAsF;QACtF,2FAA2F;QAC3F,MAAM,CAAC,KAAW;YACjB,OAAO,GAAG,IAAI,CAAC;YACf,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,UAAU;gBAAE,OAAO,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QAC9B,CAAC;QACD,KAAK,CAAC,KAAW;YAChB,OAAO,GAAG,IAAI,CAAC;YACf,IAAI,OAAO,QAAQ,CAAC,KAAK,KAAK,UAAU;gBAAE,OAAO,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACvE,MAAM,KAAK,CAAC;QACb,CAAC;KACD,CAAC;AACH,CAAC"}
+{"version":3,"file":"replayLogsGuards.js","sourceRoot":"","sources":["../../resources/replayLogsGuards.ts"],"names":[],"mappings":";AAAA,qFAAqF;AACrF,uFAAuF;AACvF,EAAE;AACF,yFAAyF;AACzF,uFAAuF;AACvF,iFAAiF;AACjF,sFAAsF;AACtF,wFAAwF;AACxF,+CAA+C;;;AAuB/C,kEAUC;AAmBD,kEAEC;AAsCD,4DASC;AAoBD,sDAEC;AA2CD,sCAEC;AAeD,8DAmCC;AAkCD,wDAEC;AAGD,8EAEC;AAED,4DAGC;AAsBD,gEA6CC;AAzUD,wFAAwF;AACxF,yFAAyF;AACzF,qFAAqF;AACrF,oEAAoE;AACvD,QAAA,oBAAoB,GAAG,EAAE,GAAG,EAAE,CAAC;AAE5C;;;;;;;;;;;;;;GAcG;AACH,SAAgB,2BAA2B,CAC1C,MAA0B,EAC1B,OAA2B,EAC3B,SAAkB;IAElB,IAAI,MAAM,KAAK,SAAS,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,gBAAgB,CAAC;IAC3E,kFAAkF;IAClF,mFAAmF;IACnF,IAAI,CAAC,MAAM,GAAG,4BAAoB,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS;QAAE,OAAO,gBAAgB,CAAC;IACjF,OAAO,IAAI,CAAC;AACb,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,SAAgB,2BAA2B,CAAC,IAAwB,EAAE,MAAe;IACpF,OAAO,MAAM,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,SAAS,CAAC,CAAC;AACrF,CAAC;AAED,gGAAgG;AAChG,kGAAkG;AAClG,kGAAkG;AAClG,+FAA+F;AAC/F,mGAAmG;AACnG,gGAAgG;AAChG,gGAAgG;AAChG,0DAA0D;AAE1D,6FAA6F;AAC7F,mGAAmG;AACnG,mEAAmE;AACtD,QAAA,8BAA8B,GAAG,OAAO,CAAC;AAEtD,oGAAoG;AACpG,qGAAqG;AACrG,+DAA+D;AAClD,QAAA,gCAAgC,GAAG,MAAM,CAAC;AAEvD,oGAAoG;AACpG,gGAAgG;AAChG,oGAAoG;AACpG,kGAAkG;AACrF,QAAA,kCAAkC,GAAG,KAAK,CAAC;AAExD;;;;;;;;;;GAUG;AACH,SAAgB,wBAAwB,CACvC,aAAqB,EACrB,eAAuB,EACvB,UAAU,GAAG,sCAA8B,EAC3C,WAAW,GAAG,wCAAgC,EAC9C,aAAa,GAAG,0CAAkC;IAElD,IAAI,aAAa,IAAI,UAAU;QAAE,OAAO,IAAI,CAAC;IAC7C,OAAO,aAAa,IAAI,aAAa,IAAI,eAAe,IAAI,WAAW,CAAC;AACzE,CAAC;AAED,gGAAgG;AAChG,+FAA+F;AAC/F,gGAAgG;AAChG,iGAAiG;AACjG,+FAA+F;AAC/F,kGAAkG;AAClG,kCAAkC;AACrB,QAAA,0BAA0B,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAEzD;;;;;;;;GAQG;AACH,SAAgB,qBAAqB,CAAC,cAAsB,EAAE,WAAW,GAAG,kCAA0B;IACrG,OAAO,cAAc,IAAI,WAAW,CAAC;AACtC,CAAC;AAqCD;;;;;GAKG;AACH,SAAgB,aAAa,CAAC,KAAwB;IACrD,OAAO,KAAK,CAAC,cAAc,IAAI,IAAI,CAAC;AACrC,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAgB,yBAAyB,CACxC,QAAqB,EACrB,cAAkD;IAElD,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,OAAO;QACN,CAAC,MAAM,CAAC,QAAQ,CAAC;YAChB,OAAO,IAAI,CAAC;QACb,CAAC;QACD,IAAI;YACH,IAAI,OAAO;gBAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;YACrD,IAAI,CAAC;gBACJ,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;YACxB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,yEAAyE;gBACzE,wEAAwE;gBACxE,IAAI,CAAC,CAAC,KAAK,YAAY,UAAU,CAAC;oBAAE,MAAM,KAAK,CAAC;gBAChD,OAAO,GAAG,IAAI,CAAC;gBACf,cAAc,CAAC,KAA0B,CAAC,CAAC;gBAC3C,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;YACzC,CAAC;QACF,CAAC;QACD,sFAAsF;QACtF,2FAA2F;QAC3F,MAAM,CAAC,KAAW;YACjB,OAAO,GAAG,IAAI,CAAC;YACf,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,UAAU;gBAAE,OAAO,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QAC9B,CAAC;QACD,KAAK,CAAC,KAAW;YAChB,OAAO,GAAG,IAAI,CAAC;YACf,IAAI,OAAO,QAAQ,CAAC,KAAK,KAAK,UAAU;gBAAE,OAAO,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACvE,MAAM,KAAK,CAAC;QACb,CAAC;KACD,CAAC;AACH,CAAC;AAoBD,mGAAmG;AACtF,QAAA,yBAAyB,GAAG,GAAG,CAAC;AAE7C,gGAAgG;AAChG,2FAA2F;AAC3F,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAA8B,CAAC;AAClE,IAAI,0BAA0B,GAAG,CAAC,CAAC;AAEnC;;;;;GAKG;AACH,SAAgB,sBAAsB;IACrC,OAAO,CAAC,GAAG,mBAAmB,CAAC,MAAM,EAAE,CAAC,CAAC;AAC1C,CAAC;AAED,iFAAiF;AACjF,SAAgB,iCAAiC;IAChD,OAAO,0BAA0B,CAAC;AACnC,CAAC;AAED,SAAgB,wBAAwB;IACvC,mBAAmB,CAAC,KAAK,EAAE,CAAC;IAC5B,0BAA0B,GAAG,CAAC,CAAC;AAChC,CAAC;AAED,mGAAmG;AACnG,oGAAoG;AACpG,mGAAmG;AACnG,8BAA8B;AAC9B,SAAS,eAAe,CAAC,OAAe,EAAE,KAAwB;IACjE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC;IAClC,OAAO,KAAK,IAAI,IAAI,IAAI,QAAQ,IAAI,IAAI;QACvC,CAAC,CAAC,GAAG,OAAO,SAAS,KAAK,IAAI,QAAQ,EAAE;QACxC,CAAC,CAAC,GAAG,OAAO,SAAS,KAAK,CAAC,OAAO,EAAE,CAAC;AACvC,CAAC;AAED;;;;;;;;GAQG;AACH,SAAgB,0BAA0B,CAAC,MAG1C;IACA,OAAO,CAAC,OAAe,EAAE,EAAE,CAAC,CAAC,KAAwB,EAAE,EAAE;QACxD,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;QACpC,MAAM,eAAe,GAAG,KAAK,CAAC,eAAe,IAAI,CAAC,CAAC;QACnD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,eAAe,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9C,IAAI,QAAQ,EAAE,CAAC;YACd,mBAAmB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAChC,mBAAmB,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;YACvC,QAAQ,CAAC,WAAW,EAAE,CAAC;YACvB,QAAQ,CAAC,QAAQ,GAAG,GAAG,CAAC;YACxB,IAAI,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM;gBAAE,OAAO;YACvC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC;YACvB,QAAQ,CAAC,eAAe,GAAG,eAAe,CAAC;QAC5C,CAAC;aAAM,CAAC;YACP,IAAI,mBAAmB,CAAC,IAAI,IAAI,iCAAyB,EAAE,CAAC;gBAC3D,mBAAmB,CAAC,MAAM,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC;gBACpE,0BAA0B,EAAE,CAAC;YAC9B,CAAC;YACD,mBAAmB,CAAC,GAAG,CAAC,GAAG,EAAE;gBAC5B,GAAG,EAAE,OAAO;gBACZ,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,SAAS;gBAC/B,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,SAAS;gBACrC,MAAM;gBACN,eAAe;gBACf,SAAS,EAAE,GAAG;gBACd,QAAQ,EAAE,GAAG;gBACb,WAAW,EAAE,CAAC;aACd,CAAC,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,CAAC,KAAK,CACX,qCAAqC,OAAO,MAAM,eAAe,gEAAgE;gBAChI,kIAAkI;gBAClI,mIAAmI,EACpI,KAAK,CACL,CAAC;QACH,CAAC;aAAM,CAAC;YACP,MAAM,CAAC,IAAI,CAAC,6BAA6B,OAAO,oCAAoC,EAAE,KAAK,CAAC,CAAC;QAC9F,CAAC;IACF,CAAC,CAAC;AACH,CAAC"}

package/dist/resources/search.js

@@ -18,6 +18,7 @@ const ordered_binary_1 = require("ordered-binary");
 const extended_iterable_1 = require("@harperfast/extended-iterable");
 const Table_ts_1 = require("./Table.js");
 const RecordEncoder_ts_1 = require("./RecordEncoder.js");
+const DatabaseTransaction_ts_1 = require("./DatabaseTransaction.js");
 const write_1 = require("./analytics/write");
 const rocksdb_js_1 = require("@harperfast/rocksdb-js");
 // these are ratios/percentages of overall table size
@@ -212,6 +213,41 @@ function composeRecordFilter(recordFilters, table, context) {
         return verdict;
     };
 }
+/**
+ * A record's index entries are not adjacent — the composite `[indexedValue, primaryKey]` key sorts
+ * on the indexed value first — so comparing neighbours cannot collapse them and the scan has to
+ * remember what it already yielded (#2434).
+ *
+ * Heap cost is one key per distinct record in the scanned range, held for the life of the scan, and
+ * a page window does not cap it: sibling-condition and row filters run downstream, so a selective
+ * filter keeps the scan running and the set growing while the page fills. On a multi-million-row
+ * range that is hundreds of MB where main streamed at O(1). Accepted for #2434 over the bounded
+ * alternative, which costs a primary read per index entry.
+ */
+function distinctRecords(entries) {
+    const distinct = new extended_iterable_1.ExtendedIterable();
+    distinct.iterate = (options) => {
+        // A non-scalar key decodes to a fresh instance per entry, so identity never matches it;
+        // `writeKeyId` is the store's own key equality. Its bytes get their own set because a scalar
+        // string id can legitimately carry exactly those bytes, and one set would fold the two —
+        // the collision `flattenKey` has, and the reason Table.ts prefixes its key ids.
+        const yieldedKeys = new Set();
+        const yieldedEncodedKeys = new Set();
+        // map rather than filter: filter clears `continueOnRecoverableError` for everything below it
+        return entries
+            .map((primaryKey) => {
+            const isEncoded = typeof primaryKey === 'object' && primaryKey !== null;
+            const yielded = isEncoded ? yieldedEncodedKeys : yieldedKeys;
+            const identity = isEncoded ? (0, DatabaseTransaction_ts_1.writeKeyId)(primaryKey) : primaryKey;
+            if (yielded.has(identity))
+                return extended_iterable_1.SKIP;
+            yielded.add(identity);
+            return primaryKey;
+        })
+            .iterate(options);
+    };
+    return distinct;
+}
 /**
  * Search for records or keys, based on the search condition, using an index if available
  * @param searchCondition
@@ -233,6 +269,9 @@ minResults) {
         throw new hdbError_ts_1.ClientError(`Search condition for ${attribute_name} must have a value`);
     }
     let needFullScan;
+    // `[indexedValue, primaryKey]` is unique, so a scan that stays inside one indexed value cannot
+    // reach a record twice and skips the collapse below
+    let scansOneIndexedValue;
     if (Array.isArray(attribute_name)) {
         const firstAttributeName = attribute_name[0];
         // get the potential relationship attribute
@@ -356,6 +395,7 @@ minResults) {
                 start = value;
                 end = value;
                 inclusiveEnd = true;
+                scansOneIndexedValue = true;
                 break;
             case 'in':
                 // Phase 1: route through filter — index-merge optimization is a Phase 2 follow-up.
@@ -497,7 +537,7 @@ minResults) {
             }
             return loaded;
         }
-        return index.getRange(rangeOptions).map(filter
+        const scanned = index.getRange(rangeOptions).map(filter
             ? function ({ key, value }) {
                 let recordMatcher;
                 if (typeof key === 'string' && key.length > terms_ts_1.MAX_SEARCH_KEY_LENGTH) {
@@ -520,6 +560,14 @@ minResults) {
                 }));
             }
             : ({ value }) => value);
+        // Collapsing AFTER the map is required, not incidental: the condition's own filter tests one
+        // element at a time, so a record whose first entry fails it and whose second passes must
+        // still be reached. Collapsing first would settle that record on the failing entry and drop
+        // it. Scoped to attributes the schema declares multi-valued; a runtime array under an
+        // undeclared attribute repeats the same way but stays outside this boundary.
+        return scansOneIndexedValue || !findAttribute(Table.attributes, attribute_name)?.elements
+            ? scanned
+            : distinctRecords(scanned);
     }
     else {
         return Table.primaryStore
@@ -1020,7 +1068,7 @@ function filterByType(searchCondition, Table, context, filtered, isPrimaryKey, e
         canUseIndex =
             canUseIndex && // is it a comparator that makes sense to use index
                 !isPrimaryKey && // no need to use index for primary keys, since we will be iterating over the primary keys
-                !!(Table && usableIndex(Table, attribute)) && // is there a usable (complete) index for this attribute
+                Table?.indices[attribute] && // is there an index for this attribute
                 estimatedIncomingCount > 3; // do we have a valid estimate of multiple incoming records (that is worth using an index for)
         if (canUseIndex) {
             if (searchCondition.estimated_count == undefined)
@@ -1077,27 +1125,95 @@ function filterByType(searchCondition, Table, context, filtered, isPrimaryKey, e
         return recordFilter;
     }
 }
-/** The index a condition can be driven by; searchByIndex refuses a rebuilding one, so it reads as absent. */
-function usableIndex(table, attributeName) {
-    const index = attributeName == null ? undefined : table.indices[attributeName];
-    return index?.isIndexing ? undefined : index;
-}
 /**
- * True when an index this attribute would have to be driven by is still being built, following a
- * relationship path to the local join index and on to the related table's leaf index.
+ * Estimates a range comparator's entry count from the store's statistical estimate, blended with
+ * the table-fraction heuristic by the estimate's confidence so low-confidence estimates degrade
+ * to the old behavior. Returns undefined (caller falls back to the heuristic) when the store
+ * cannot estimate, the shape is unexpected, or the executed range wouldn't match this one.
  */
-function drivesRebuildingIndex(table, attributeName, relationshipOffset = 0) {
-    if (!Array.isArray(attributeName))
-        return attributeName != null && attributeName !== table.primaryKey && !!table.indices[attributeName]?.isIndexing;
-    if (relationshipOffset >= attributeName.length - 1)
-        return drivesRebuildingIndex(table, attributeName[relationshipOffset]);
-    const attribute = findAttribute(table.attributes, attributeName[relationshipOffset]);
-    if (!attribute)
-        return false;
-    if (table.indices[attribute.relationship?.from]?.isIndexing)
-        return true;
-    const relatedTable = attribute.definition?.tableClass || attribute.elements?.definition?.tableClass;
-    return !!relatedTable && drivesRebuildingIndex(relatedTable, attributeName, relationshipOffset + 1);
+function estimateRangeCondition(table, condition, searchType, fraction) {
+    const attributeName = condition[0] ?? condition.attribute;
+    const isPrimaryKey = attributeName === table.primaryKey;
+    const store = isPrimaryKey ? table.primaryStore : table.indices[attributeName];
+    // LMDB-backed and custom index stores do not implement estimateCount
+    if (typeof store?.estimateCount !== 'function')
+        return undefined;
+    let value = condition[1] ?? condition.value;
+    if (value instanceof Date)
+        value = value.getTime();
+    let range;
+    switch (searchType) {
+        case 'lt':
+            // `start: true` mirrors searchByIndex: `true` sorts above `null`, so an indexNulls
+            // index's `[null, primaryKey]` entries are outside the executed range
+            range = { start: true, end: value };
+            break;
+        case 'le':
+            range = { start: true, end: value, inclusiveEnd: true };
+            break;
+        case 'gt':
+            range = { start: value, exclusiveStart: true };
+            break;
+        case 'ge':
+            range = { start: value };
+            break;
+        case 'between':
+        case 'gele':
+        case 'gelt':
+        case 'gtlt':
+        case 'gtle': {
+            if (!Array.isArray(value))
+                return undefined;
+            let [start, end] = value;
+            if (start instanceof Date)
+                start = start.getTime();
+            if (end instanceof Date)
+                end = end.getTime();
+            range = {
+                start,
+                end,
+                inclusiveEnd: searchType === 'between' || searchType === 'gele' || searchType === 'gtle',
+                exclusiveStart: searchType === 'gtlt' || searchType === 'gtle',
+            };
+            break;
+        }
+        case 'starts_with': {
+            const prefix = value?.toString();
+            if (!prefix)
+                return undefined;
+            range = { start: prefix, end: getStringPrefixUpperBound(prefix) };
+            break;
+        }
+        case 'prefix': {
+            let start = Array.isArray(value) ? value : [value, null];
+            if (start[start.length - 1] != null)
+                start = start.concat(null);
+            const end = start.slice(0);
+            end[end.length - 1] = ordered_binary_1.MAXIMUM_KEY;
+            range = { start, end };
+            break;
+        }
+        default:
+            return undefined;
+    }
+    // Long string bounds get truncated + filtered at execution (searchByIndex), so the
+    // executed range is wider than this one; don't estimate what won't be iterated.
+    if ((typeof range.start === 'string' && range.start.length > terms_ts_1.MAX_SEARCH_KEY_LENGTH) ||
+        (typeof range.end === 'string' && range.end.length > terms_ts_1.MAX_SEARCH_KEY_LENGTH)) {
+        return undefined;
+    }
+    let count, confidence;
+    try {
+        ({ count, confidence } = store.estimateCount(range) ?? {});
+    }
+    catch {
+        // a concurrently closing/dropped store must degrade the plan, not fail the query
+        return undefined;
+    }
+    if (!Number.isFinite(count) || count < 0 || !Number.isFinite(confidence) || confidence < 0 || confidence > 1)
+        return undefined;
+    const heuristic = fraction * estimatedEntryCount(table.primaryStore) + 1;
+    return Math.max(1, Math.round(confidence * count + (1 - confidence) * heuristic));
 }
 function estimateCondition(table) {
     function estimateConditionForTable(condition) {
@@ -1129,8 +1245,12 @@ function estimateCondition(table) {
             // skip if it is cached
             let searchType = condition.comparator || condition.search_type;
             searchType = ALTERNATE_COMPARATOR_NAMES[searchType] || searchType;
-            // before comparator dispatch: several branches fall back to a finite table-fraction heuristic
-            if (drivesRebuildingIndex(table, condition[0] ?? condition.attribute)) {
+            if (condition.negated) {
+                // a negated condition always executes as a full scan (searchByIndex forces
+                // needFullScan), so follow the filter-only convention used by contains/ends_with:
+                // estimate Infinity so its positive-range estimate can never win the driving-condition
+                // ordering. Short-circuit here rather than computing (and discarding) a positive-range
+                // estimate that would cross the native FFI boundary for nothing.
                 condition.estimated_count = Infinity;
             }
             else if (searchType === terms_ts_1.SEARCH_TYPES.EQUALS || !searchType) {
@@ -1162,13 +1282,13 @@ function estimateCondition(table) {
                 }
                 else {
                     // we only attempt to estimate count on equals operator because that's really all that LMDB supports (some other key-value stores like libmdbx could be considered if we need to do estimated counts of ranges at some point)
-                    const index = usableIndex(table, attribute_name);
+                    const index = table.indices[attribute_name];
                     condition.estimated_count = index ? index.getValuesCount(condition[1] ?? condition.value) : Infinity;
                 }
             }
             else if (searchType === 'contains' || searchType === 'ends_with' || searchType === 'ne') {
                 const attribute_name = condition[0] ?? condition.attribute;
-                const index = usableIndex(table, attribute_name);
+                const index = table.indices[attribute_name];
                 if (condition.value === null && searchType === 'ne') {
                     condition.estimated_count = Math.max(estimatedEntryCount(table.primaryStore) - (index ? index.getValuesCount(null) : 0), 0);
                 }
@@ -1177,7 +1297,7 @@ function estimateCondition(table) {
             }
             else if (searchType === 'in') {
                 const attribute_name = condition[0] ?? condition.attribute;
-                const index = usableIndex(table, attribute_name);
+                const index = table.indices[attribute_name];
                 if (Array.isArray(condition.value) && index) {
                     // Sum of per-value matches (over-counts duplicates but is a fine ceiling)
                     let estimate = 0;
@@ -1191,15 +1311,18 @@ function estimateCondition(table) {
                 }
                 else
                     condition.estimated_count = Infinity;
-                // for range queries (betweens, startsWith, greater, etc.), just arbitrarily guess
             }
             else if (searchType === 'starts_with' || searchType === 'prefix')
-                condition.estimated_count = STARTS_WITH_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
+                condition.estimated_count =
+                    estimateRangeCondition(table, condition, searchType, STARTS_WITH_ESTIMATE) ??
+                        STARTS_WITH_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
             else if (searchType === 'between')
-                condition.estimated_count = BETWEEN_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
+                condition.estimated_count =
+                    estimateRangeCondition(table, condition, searchType, BETWEEN_ESTIMATE) ??
+                        BETWEEN_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
             else if (searchType === 'sort') {
                 const attribute_name = condition[0] ?? condition.attribute;
-                const index = usableIndex(table, attribute_name);
+                const index = table.indices[attribute_name];
                 if (index?.customIndex?.estimateCountAsSort)
                     // allow custom index to define its own estimation of counts
                     condition.estimated_count = index.customIndex.estimateCountAsSort(condition);
@@ -1209,12 +1332,14 @@ function estimateCondition(table) {
             else {
                 // for the search types that use the broadest range, try do them last
                 const attribute_name = condition[0] ?? condition.attribute;
-                const index = usableIndex(table, attribute_name);
+                const index = table.indices[attribute_name];
                 if (index?.customIndex?.estimateCount)
                     // allow custom index to define its own estimation of counts
                     condition.estimated_count = index.customIndex.estimateCount(condition.value);
                 else
-                    condition.estimated_count = OPEN_RANGE_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
+                    condition.estimated_count =
+                        estimateRangeCondition(table, condition, searchType, OPEN_RANGE_ESTIMATE) ??
+                            OPEN_RANGE_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
             }
             // we give a condition significantly more weight/preference if we will be ordering by it
             if (typeof condition.descending === 'boolean')
@@ -1427,6 +1552,7 @@ function parseBlock(query, expectedEnd) {
                         break;
                     case 'group-by':
                         recordError('group by is not implemented yet');
+                        break;
                     case 'sort':
                         query.sort = toSortObject(args);
                         break;