@harperfast/harper-pro 5.0.17 → 5.0.19

package/dist/core/resources/Table.js

@@ -93,6 +93,7 @@ exports.EVICTED = 8; // note that 2 is reserved for timestamps
 const TEST_WRITE_KEY_BUFFER = Buffer.allocUnsafeSlow(8192);
 const MAX_KEY_BYTES = 1978;
 const EVENT_HIGH_WATER_MARK = 100;
+const REPLAY_YIELD_INTERVAL = 100; // yield to the event loop every N records during subscription replay
 const FULL_PERMISSIONS = {
     read: true,
     insert: true,
@@ -732,28 +733,25 @@ function makeTable(options) {
         /**
          * Set TTL expiration for records in this table. On retrieval, record timestamps are checked for expiration.
          * This also informs the scheduling for record eviction.
-         * @param expirationTime Time in seconds until records expire (are stale)
-         * @param evictionTime Time in seconds until records are evicted (removed)
+         * @param opts Time in seconds until records expire, or an options object with `expiration`, `eviction`,
+         * and `scanInterval` (all in seconds, all optional). Number form preserves any previously configured
+         * eviction/scanInterval; object form replaces all three.
          */
-        static setTTLExpiration(expiration) {
-            // we set up a timer to remove expired entries. we only want the timer/reaper to run in one thread,
-            // so we use the first one
-            if (typeof expiration === 'number') {
-                expirationMs = expiration * 1000;
-                if (!evictionMs)
-                    evictionMs = 0; // by default, no extra time for eviction
+        static setTTLExpiration(opts) {
+            if (opts == null || (typeof opts !== 'number' && typeof opts !== 'object'))
+                throw new Error('Invalid expiration value type');
+            if (typeof opts === 'number') {
+                expirationMs = opts * 1000;
             }
-            else if (expiration && typeof expiration === 'object') {
-                // an object with expiration times/options specified
-                expirationMs = expiration.expiration * 1000;
-                evictionMs = (expiration.eviction || 0) * 1000;
-                cleanupInterval = expiration.scanInterval * 1000;
+            else {
+                // `??` so an explicit 0 is treated as the user's chosen value, not as "missing"
+                expirationMs = (opts.expiration ?? 0) * 1000;
+                evictionMs = (opts.eviction ?? 0) * 1000;
+                cleanupInterval = (opts.scanInterval ?? 0) * 1000;
             }
-            else
-                throw new Error('Invalid expiration value type');
             if (expirationMs < 0)
                 throw new Error('Expiration can not be negative');
-            // default to one quarter of the total eviction time, and make sure it fits into a 32-bit signed integer
+            // default to one quarter of the total expiration+eviction window
             cleanupInterval = cleanupInterval || (expirationMs + evictionMs) / 4;
             scheduleCleanup();
         }
@@ -2467,9 +2465,19 @@ function makeTable(options) {
             if (!request)
                 request = {};
             const getFullRecord = !request.rawEvents;
-            let pendingRealTimeQueue = []; // while we are servicing a loop for older messages, we have to queue up real-time messages and deliver them in order
+            // While the count, !omitCurrent, and non-collection branches replay older messages, real-time
+            // messages from the listener accumulate here and are drained at the end of the IIFE so they
+            // arrive after the replayed history, in order. The startTime branch sets this to null and
+            // uses dropDuringReplay instead — its snapshot:false cursor picks up the live tail directly.
+            let pendingRealTimeQueue = [];
+            // Set during the startTime audit-log replay. The cursor iterates the audit log forward with
+            // snapshot:false, which catches any commits that land during yield points; dropping in the
+            // listener avoids duplicate delivery.
+            let dropDuringReplay = false;
             const thisId = requestTargetToId(request) ?? null; // treat undefined and null as the root
             const subscription = (0, transactionBroadcast_ts_1.addSubscription)(TableResource, thisId, function (id, auditRecord, localTime, beginTxn) {
+                if (dropDuringReplay)
+                    return;
                 try {
                     let type = auditRecord.type;
                     let value;
@@ -2514,6 +2522,12 @@ function makeTable(options) {
                     logger_ts_1.logger.error?.(error);
                 }
             }, request.startTime || 0, request);
+            // Attach the request.listener BEFORE invoking the IIFE so that sync sends from the
+            // IIFE's prologue go directly to the listener via emit('data') instead of accumulating
+            // in subscription.queue. Without this, the IIFE can fill the queue past
+            // EVENT_HIGH_WATER_MARK and hit waitForDrain before the consumer's listener exists.
+            if (request.listener)
+                subscription.on('data', request.listener);
             const result = (async () => {
                 const isCollection = request.isCollection ?? thisId == null;
                 if (isCollection) {
@@ -2527,17 +2541,25 @@ function makeTable(options) {
                 if (count > 1000)
                     count = 1000; // don't allow too many, we have to hold these in memory
                 let startTime = request.startTime;
+                let recordsSinceYield = 0;
                 if (isCollection) {
                     // a collection should retrieve all descendant ids
                     if (startTime) {
                         if (count)
                             throw new hdbError_js_1.ClientError('startTime and previousCount can not be combined for a table level subscription');
-                        // start time specified, get the audit history for this time range
+                        // start time specified, get the audit history for this time range. We drop real-time
+                        // messages during this loop because the snapshot:false cursor will pick them up itself.
+                        pendingRealTimeQueue = null;
+                        dropDuringReplay = true;
                         for (const auditRecord of auditStore.getRange({
                             start: startTime,
                             exclusiveStart: true,
                             snapshot: false, // no need for a snapshot, audits don't change
                         })) {
+                            if (++recordsSinceYield >= REPLAY_YIELD_INTERVAL) {
+                                recordsSinceYield = 0;
+                                await rest();
+                            }
                             if (auditRecord.tableId !== tableId)
                                 continue;
                             const id = auditRecord.recordId;
@@ -2557,15 +2579,34 @@ function makeTable(options) {
                                         return;
                                 }
                             }
-                            // TODO: Would like to do this asynchronously, but would need to catch up on anything published during iteration
-                            //await rest(); // yield for fairness
-                            subscription.startTime = auditRecord.localTime; // update so we don't double send
+                            subscription.startTime = auditRecord.localTime ?? auditRecord.version; // update so we don't double send
                         }
+                        // No catch-up sweep needed. With snapshot:false (lmdb), notifyFromTransactionData
+                        // calls resetReadTxn before iterating, which bumps renewId; on the cursor's next
+                        // .next() it renews to a fresh txn whose snapshot is at least as recent. With
+                        // rocksdb, the audit-log iterator re-reads `_lastCommittedPosition` on each next()
+                        // (live tail). Either way, at loop exit subscription.startTime is at or past
+                        // lastTxnTime, and the gate in notifyFromTransactionData handles the handoff
+                        // once dropDuringReplay flips back.
+                        dropDuringReplay = false;
                     }
                     else if (count) {
+                        // Raise the listener's gate up front so that any in-flight 'committed' callbacks
+                        // for records the cursor will capture in `history` get gated out of
+                        // pendingRealTimeQueue rather than queued and re-emitted as duplicates after
+                        // history is sent. getNextMonotonicTime() returns a strictly-greater value than
+                        // any audit record's localTime issued so far — it's the same source Harper uses
+                        // to assign localTimes — so this gates exactly the records the cursor's
+                        // snapshot:true view can see. Anything committed strictly after this point will
+                        // pass the gate and reach the queue.
+                        subscription.startTime = (0, commonUtility_js_1.getNextMonotonicTime)();
                         const history = [];
                         // we are collecting the history in reverse order to get the right count, then reversing to send
                         for (const auditRecord of auditStore.getRange({ start: 'z', end: false, reverse: true })) {
+                            if (++recordsSinceYield >= REPLAY_YIELD_INTERVAL) {
+                                recordsSinceYield = 0;
+                                await rest();
+                            }
                             try {
                                 if (auditRecord.tableId !== tableId)
                                     continue;
@@ -2586,22 +2627,34 @@ function makeTable(options) {
                             catch (error) {
                                 logger_ts_1.logger.error?.('Error getting history entry', auditRecord.localTime, error);
                             }
-                            // TODO: Would like to do this asynchronously, but would need to catch up on anything published during iteration
-                            //await rest(); // yield for fairness
                         }
                         for (let i = history.length; i > 0;) {
                             send(history[--i]);
                         }
-                        if (history[0])
-                            subscription.startTime = history[0].localTime; // update so don't double send
                     }
                     else if (!request.omitCurrent) {
+                        // Raise the listener's gate up front so that any in-flight 'committed' callbacks
+                        // for pre-subscribe commits (which haven't yet advanced lastTxnTime when subscribe
+                        // is called) get gated out of the queue. Otherwise the listener fires for them
+                        // during cursor yields and emits stale events the cursor either covered (current
+                        // state) or correctly skipped (e.g., deletes via `if (!value) continue`).
+                        // getNextMonotonicTime() is the same source Harper uses to assign audit record
+                        // localTimes, so the gate cuts at a precise instant in the same time domain.
+                        subscription.startTime = (0, commonUtility_js_1.getNextMonotonicTime)();
+                        // Retained-message semantics: subscriber may legitimately receive a record twice
+                        // if a post-subscribe write hits a key the cursor also visits. This is
+                        // idempotent for "current state then live updates" — both deliveries land at
+                        // the same final state. We don't dedupe.
                         for (const { key: id, value, version, localTime, size } of primaryStore.getRange({
                             start: thisId ?? false,
                             end: thisId == null ? undefined : [thisId, ordered_binary_1.MAXIMUM_KEY],
                             versions: true,
                             snapshot: false, // no need for a snapshot, just want the latest data
                         })) {
+                            if (++recordsSinceYield >= REPLAY_YIELD_INTERVAL) {
+                                recordsSinceYield = 0;
+                                await rest();
+                            }
                             if (!value)
                                 continue;
                             send({ id, localTime, value, version, type: 'put', size });
@@ -2632,13 +2685,19 @@ function makeTable(options) {
                     }
                     logger_ts_1.logger.trace?.('Subscription from', startTime, 'from', thisId, localTime);
                     if (startTime < localTime) {
-                        // start time specified, get the audit history for this record
+                        // start time specified, get the audit history for this record. Set startTime up
+                        // front so the listener gate skips any in-flight 'committed' for this version
+                        // during the yields below — otherwise that event would be queued and drained as a
+                        // duplicate of the entry send.
+                        subscription.startTime = localTime ?? entry?.version;
                         const history = [];
                         let nextTime = localTime;
                         let nodeId = entry?.nodeId;
                         do {
-                            //TODO: Would like to do this asynchronously, but we will need to run catch after this to ensure we didn't miss anything
-                            //await auditStore.prefetch([key]); // do it asynchronously for better fairness/concurrency and avoid page faults
+                            if (++recordsSinceYield >= REPLAY_YIELD_INTERVAL) {
+                                recordsSinceYield = 0;
+                                await rest();
+                            }
                             const auditRecord = auditStore.getSync(nextTime, tableId, thisId, nodeId);
                             if (auditRecord) {
                                 if (startTime < nextTime) {
@@ -2664,7 +2723,6 @@ function makeTable(options) {
                         for (let i = history.length; i > 0;) {
                             send(history[--i]);
                         }
-                        subscription.startTime = localTime; // make sure we don't re-broadcast the current version that we already sent
                     }
                     if (!request.omitCurrent && entry?.value) {
                         // if retain and it exists, send the current value first
@@ -2676,10 +2734,12 @@ function makeTable(options) {
                     }
                 }
                 // now send any queued messages
-                for (const event of pendingRealTimeQueue) {
-                    send(event);
+                if (pendingRealTimeQueue) {
+                    for (const event of pendingRealTimeQueue) {
+                        send(event);
+                    }
+                    pendingRealTimeQueue = null;
                 }
-                pendingRealTimeQueue = null;
             })();
             result.catch((error) => {
                 harper_logger_js_1.default.error?.('Error in real-time subscription:', error);
@@ -2691,8 +2751,6 @@ function makeTable(options) {
                 }
                 subscription.send(event);
             }
-            if (request.listener)
-                subscription.on('data', request.listener);
             return subscription;
         }
         /**
@@ -4061,6 +4119,9 @@ function makeTable(options) {
                                 transaction,
                                 tableToTrack: tableName,
                             }, 'put', Boolean(invalidated), auditRecord);
+                            // arm the eviction scanner, mirroring the .put() path
+                            if (sourceContext.expiresAt)
+                                scheduleCleanup();
                         }
                         else if (existingEntry) {
                             logger_ts_1.logger.trace?.(`Deleting resolved record from source with id: ${id}, timestamp: ${new Date(txnTime).toISOString()}`);