@peerbit/shared-log 16.0.6 → 16.0.7

package/dist/src/ranges.js

@@ -1373,13 +1373,13 @@ export const calculateCoverage = async (properties) => {
     return minCoverage === Infinity || minCoverage <= 0 ? 0 : minCoverage;
 };
 const getClosest = (direction, rects, point, includeStrict, numbers, options) => {
-    const createQueries = (p, equality) => {
+    const createQueries = (p, phase) => {
         let queries;
         if (direction === "below") {
             queries = [
                 new IntegerCompare({
                     key: "end2",
-                    compare: equality ? Compare.LessOrEqual : Compare.Less,
+                    compare: phase === "primary" ? Compare.LessOrEqual : Compare.Greater,
                     value: p,
                 }),
             ];
@@ -1388,7 +1388,7 @@ const getClosest = (direction, rects, point, includeStrict, numbers, options) =>
             queries = [
                 new IntegerCompare({
                     key: "start1",
-                    compare: equality ? Compare.GreaterOrEqual : Compare.Greater,
+                    compare: phase === "primary" ? Compare.GreaterOrEqual : Compare.Less,
                     value: p,
                 }),
             ];
@@ -1413,12 +1413,18 @@ const getClosest = (direction, rects, point, includeStrict, numbers, options) =>
         if (options?.hash) {
             queries.push(new StringMatch({ key: "hash", value: options.hash }));
         }
+        if (options?.excludeId) {
+            queries.push(new Not(new ByteMatchQuery({
+                key: "id",
+                value: options.excludeId,
+            })));
+        }
         return queries;
     };
     const sortByOldest = new Sort({ key: "timestamp", direction: "asc" });
     const sortByHash = new Sort({ key: "hash", direction: "asc" }); // when breaking even
     const iterator = rects.iterate({
-        query: createQueries(point, false),
+        query: createQueries(point, "primary"),
         sort: [
             direction === "below"
                 ? new Sort({ key: ["end2"], direction: "desc" })
@@ -1428,7 +1434,7 @@ const getClosest = (direction, rects, point, includeStrict, numbers, options) =>
         ],
     }, options);
     const iteratorWrapped = rects.iterate({
-        query: createQueries(direction === "below" ? numbers.maxValue : numbers.zero, true),
+        query: createQueries(point, "wrapped"),
         sort: [
             direction === "below"
                 ? new Sort({ key: ["end2"], direction: "desc" })
@@ -1531,10 +1537,10 @@ export const getCoveringRangesSameOwner = (rects, range) => {
 export function getDistance(from, to, direction, end) {
     const abs = (value) => value < 0 ? -value : value;
     const diff = (a, b) => abs(a - b);
+    if (from === to) {
+        return typeof from === "number" ? 0 : 0n;
+    }
     if (direction === "closest") {
-        if (from === to) {
-            return typeof from === "number" ? 0 : 0n; // returns 0 of the correct type
-        }
         return diff(from, to) < diff(end, diff(from, to))
             ? diff(from, to)
             : diff(end, diff(from, to));
@@ -1553,9 +1559,43 @@ export function getDistance(from, to, direction, end) {
     }
     throw new Error("Invalid direction");
 }
-const joinIterator = (iterators, point, direction, numbers) => {
+/** @internal */
+export const joinIterator = (iterators, point, direction, numbers) => {
+    const compareQueuedResults = (left, right) => {
+        if (left.dist < right.dist)
+            return -1;
+        if (left.dist > right.dist)
+            return 1;
+        // Match the native closest-range total order. The shaped iterator users do
+        // not always project every tie field, so only compare a field when both
+        // results carry it; getSamples uses complete ranges and therefore always
+        // follows distance, timestamp, owner hash, then range id.
+        const leftValue = left.result.value;
+        const rightValue = right.result.value;
+        if (leftValue.timestamp != null && rightValue.timestamp != null) {
+            if (leftValue.timestamp < rightValue.timestamp)
+                return -1;
+            if (leftValue.timestamp > rightValue.timestamp)
+                return 1;
+        }
+        if (leftValue.hash != null && rightValue.hash != null) {
+            if (leftValue.hash < rightValue.hash)
+                return -1;
+            if (leftValue.hash > rightValue.hash)
+                return 1;
+        }
+        const leftId = leftValue.id == null ? undefined : toBase64(leftValue.id);
+        const rightId = rightValue.id == null ? undefined : toBase64(rightValue.id);
+        if (leftId != null && rightId != null) {
+            if (leftId < rightId)
+                return -1;
+            if (leftId > rightId)
+                return 1;
+        }
+        return 0;
+    };
     let queues = [];
-    return {
+    const joined = {
         next: async (count) => {
             let results = [];
             for (let i = 0; i < iterators.length; i++) {
@@ -1589,13 +1629,14 @@ const joinIterator = (iterators, point, direction, numbers) => {
             // pull the 'count' the closest element from one of the queue
             for (let i = 0; i < count; i++) {
                 let closestQueue = -1;
-                let closestDist = Number.MAX_VALUE;
+                let selected;
                 for (let j = 0; j < queues.length; j++) {
                     let queue = queues[j];
                     if (queue && queue.elements.length > 0) {
-                        let closest = queue.elements[0];
-                        if (closest.dist < closestDist) {
-                            closestDist = closest.dist;
+                        const candidate = queue.elements[0];
+                        if (candidate &&
+                            (!selected || compareQueuedResults(candidate, selected) < 0)) {
+                            selected = candidate;
                             closestQueue = j;
                         }
                     }
@@ -1603,7 +1644,7 @@ const joinIterator = (iterators, point, direction, numbers) => {
                 if (closestQueue === -1) {
                     break;
                 }
-                let closest = queues[closestQueue]?.elements.shift();
+                const closest = queues[closestQueue]?.elements.shift();
                 if (closest) {
                     results.push(closest.result);
                 }
@@ -1612,23 +1653,33 @@ const joinIterator = (iterators, point, direction, numbers) => {
         },
         pending: async () => {
             let allPending = await Promise.all(iterators.map((x) => x.pending()));
-            return allPending.reduce((acc, x) => acc + x, 0);
+            // A child can be exhausted while this join still owns prefetched rows.
+            // Counting local queues makes the same contract compose through nested joins.
+            const buffered = queues.reduce((acc, queue) => acc + queue.elements.length, 0);
+            return buffered + allPending.reduce((acc, x) => acc + x, 0);
         },
-        done: () => iterators.every((x) => x.done() === true),
+        done: () => queues.every((queue) => queue.elements.length === 0) &&
+            iterators.every((x) => x.done() === true),
         close: async () => {
-            for (const iterator of iterators) {
-                await iterator.close();
+            queues.length = 0;
+            const settled = await Promise.allSettled(iterators.map(async (iterator) => iterator.close()));
+            const errors = settled.flatMap((result) => result.status === "rejected" ? [result.reason] : []);
+            if (errors.length === 1) {
+                throw errors[0];
+            }
+            if (errors.length > 1) {
+                throw new AggregateError(errors, "Failed to close joined iterators");
             }
         },
         all: async () => {
             let results = [];
-            for (const iterator of iterators) {
-                let res = await iterator.all();
-                results.push(...res);
+            while (!joined.done()) {
+                results.push(...(await joined.next(100)));
             }
             return results;
         },
     };
+    return joined;
 };
 const getClosestAroundOrContaining = (peers, point, includeStrictBelow, includeStrictAbove, numbers, options) => {
     const closestBelow = getClosest("below", peers, point, includeStrictBelow, numbers, options);
@@ -1639,23 +1690,24 @@ const getClosestAroundOrContaining = (peers, point, includeStrictBelow, includeS
 export const getAdjecentSameOwner = async (peers, range, numbers) => {
     const closestBelowIterator = getClosest("below", peers, range.start1, true, numbers, {
         hash: range.hash,
+        excludeId: range.id,
     });
     const closestBelow = await closestBelowIterator.next(1);
     closestBelowIterator.close();
     const closestAboveIterator = getClosest("above", peers, range.end2, true, numbers, {
         hash: range.hash,
+        excludeId: range.id,
     });
     const closestAbove = await closestAboveIterator.next(1);
     closestAboveIterator.close();
-    return {
-        below: range.idString === closestBelow[0]?.value.idString
-            ? undefined
-            : closestBelow[0]?.value,
-        above: closestBelow[0]?.id.primitive === closestAbove[0]?.id.primitive ||
-            range.idString === closestBelow[0]?.value.idString
-            ? undefined
-            : closestAbove[0]?.value,
-    };
+    const below = range.idString === closestBelow[0]?.value.idString
+        ? undefined
+        : closestBelow[0]?.value;
+    const above = range.idString === closestAbove[0]?.value.idString ||
+        below?.idString === closestAbove[0]?.value.idString
+        ? undefined
+        : closestAbove[0]?.value;
+    return { below, above };
 };
 export const getAllMergeCandiates = async (peers, range, numbers) => {
     const adjacent = await getAdjecentSameOwner(peers, range, numbers);
@@ -1706,7 +1758,7 @@ export const getSamples = async (cursor, peers, roleAge, numbers, options) => {
         return new Map();
     }
     const now = +new Date();
-    let matured = 0;
+    const matureOwners = new Set();
     let uniqueVisited = new Set();
     const peerFilter = options?.peerFilter;
     for (let i = 0; i < cursor.length; i++) {
@@ -1718,10 +1770,10 @@ export const getSamples = async (cursor, peers, roleAge, numbers, options) => {
             }
             uniqueVisited.add(rect.value.hash);
             let prev = leaders.get(rect.value.hash);
+            if (isMatured(rect.value, now, roleAge)) {
+                matureOwners.add(rect.value.hash);
+            }
             if (!prev) {
-                if (isMatured(rect.value, now, roleAge)) {
-                    matured++;
-                }
                 leaders.set(rect.value.hash, { intersecting: true });
             }
             else {
@@ -1734,7 +1786,7 @@ export const getSamples = async (cursor, peers, roleAge, numbers, options) => {
                 break; // nothing more to find
             }
         }
-        if (options?.onlyIntersecting || matured > i) {
+        if (options?.onlyIntersecting || matureOwners.size > i) {
             continue;
         }
         let foundOneUniqueMatured = false;
@@ -1745,11 +1797,11 @@ export const getSamples = async (cursor, peers, roleAge, numbers, options) => {
             uniqueVisited.add(rect.hash);
             const prev = leaders.get(rect.hash);
             if (m) {
+                matureOwners.add(rect.hash);
                 if (!prev) {
-                    matured++;
                     leaders.set(rect.hash, { intersecting: false });
                 }
-                if (matured > i) {
+                if (matureOwners.size > i) {
                     foundOneUniqueMatured = true;
                 }
             }
@@ -2195,44 +2247,78 @@ const replicationRangeGeometryKey = (range) => toSegmentsBigInt(range)
 // margin below SQLite's default expression-depth limit (1000), including the
 // nested comparisons needed for wrapped ranges.
 const MAX_REBALANCE_QUERY_RANGES = 128;
-const createRebalanceQueryPlan = (changes, includeBoundaryWhenEmpty) => {
-    const boundarySource = changes.find((change) => change.rebalanceBoundaryOnly ||
-        change.range.mode === ReplicationIntent.NonStrict);
-    const boundaryChanges = boundarySource || includeBoundaryWhenEmpty
-        ? boundarySource
-            ? [{ ...boundarySource, rebalanceBoundaryOnly: true }]
-            : []
-        : undefined;
+const MAX_REBALANCE_TASKS_PER_PAGE = 8;
+const buildStrictQueryRangeDTOs = (segments, template) => {
+    const merged = mergeBigIntSegments(segments
+        .filter(([start, end]) => start < end)
+        .map(([start, end]) => [start, end]));
+    if (merged.length === 0) {
+        return [];
+    }
+    const max = isU32Range(template) ? BigInt(MAX_U32) : MAX_U64;
+    const arcs = [];
+    if (merged.length > 1 &&
+        merged[0][0] === 0n &&
+        merged[merged.length - 1][1] === max) {
+        // Preserve the historical seam-first order. Apart from compatibility, this
+        // keeps the two linear pieces of a wrapped query range in one task.
+        arcs.push({
+            first: merged[merged.length - 1],
+            second: merged[0],
+        });
+        for (let i = 1; i + 1 < merged.length; i++) {
+            arcs.push({ first: merged[i] });
+        }
+    }
+    else {
+        for (const segment of merged) {
+            arcs.push({ first: segment });
+        }
+    }
+    return arcs.map(({ first, second }) => {
+        const [start1, end1] = toOriginalType(first, template);
+        const [start2, end2] = second
+            ? toOriginalType(second, template)
+            : [start1, end1];
+        return {
+            start1,
+            end1,
+            start2,
+            end2,
+            mode: ReplicationIntent.Strict,
+        };
+    });
+};
+const createRebalanceQueryGeometryPlan = (changes, includeBoundaryWhenEmpty) => {
+    const boundary = includeBoundaryWhenEmpty ||
+        changes.some((change) => change.rebalanceBoundaryOnly ||
+            change.range.mode === ReplicationIntent.NonStrict);
     const geometryChanges = changes.filter((change) => !change.rebalanceBoundaryOnly);
     if (geometryChanges.length === 0) {
-        return { boundaryChanges, geometryBatches: [], ownedIntervals: [] };
+        return {
+            boundary,
+            geometryRanges: [],
+            ownedIntervals: [],
+            taskCount: boundary ? 1 : 0,
+        };
     }
     const template = geometryChanges[geometryChanges.length - 1];
     const exactSegments = mergeBigIntSegments(geometryChanges
         .flatMap((change) => toSegmentsBigInt(change.range))
         .filter(([start, end]) => start < end)
         .map(([start, end]) => [start, end]));
-    const exactRanges = buildStrictQueryRanges(exactSegments, template.range);
-    const exactChanges = exactRanges.map((range) => ({
-        range,
-        type: "replaced",
-        timestamp: template.timestamp,
-    }));
-    const geometryBatches = [];
-    for (let offset = 0; offset < exactChanges.length; offset += MAX_REBALANCE_QUERY_RANGES) {
-        geometryBatches.push(exactChanges.slice(offset, offset + MAX_REBALANCE_QUERY_RANGES));
-    }
+    const geometryRanges = buildStrictQueryRangeDTOs(exactSegments, template.range);
     // A row can contain coordinates in several query batches. Assign every exact
     // linear interval to its batch up front so candidates can be de-duplicated with
     // O(component-count) plan memory instead of an unbounded set of emitted hashes.
     // The two linear pieces of a wrapped seam range inherit the same batch tag.
-    const ownedIntervals = exactRanges
+    const ownedIntervals = geometryRanges
         .flatMap((range, rangeIndex) => toSegmentsBigInt(range)
         .filter(([start, end]) => start < end)
         .map(([start, end]) => ({
         start,
         end,
-        batch: Math.floor(rangeIndex / MAX_REBALANCE_QUERY_RANGES),
+        geometryTask: Math.floor(rangeIndex / MAX_REBALANCE_QUERY_RANGES),
     })))
         .sort((a, b) => a.start < b.start
         ? -1
@@ -2248,9 +2334,47 @@ const createRebalanceQueryPlan = (changes, includeBoundaryWhenEmpty) => {
             throw new Error("Rebalance query intervals must not overlap");
         }
     }
-    return { boundaryChanges, geometryBatches, ownedIntervals };
+    return {
+        boundary,
+        geometryRanges,
+        ownedIntervals,
+        taskCount: (boundary ? 1 : 0) +
+            Math.ceil(geometryRanges.length / MAX_REBALANCE_QUERY_RANGES),
+    };
 };
-const findOwningRebalanceBatch = (coordinates, intervals) => {
+/**
+ * Bounds control-plane query construction only. A task can still match an
+ * arbitrary number of local rows, so executors must apply their own durable
+ * row/byte/time budgets while scanning it.
+ */
+export const getRebalanceScanTaskPage = (plan, cursor) => {
+    const start = cursor?.nextTask ?? 0;
+    if (!Number.isSafeInteger(start) || start < 0 || start > plan.taskCount) {
+        throw new Error("Invalid rebalance task cursor");
+    }
+    const end = Math.min(start + MAX_REBALANCE_TASKS_PER_PAGE, plan.taskCount);
+    const boundaryOffset = plan.boundary ? 1 : 0;
+    const tasks = [];
+    for (let ordinal = start; ordinal < end; ordinal++) {
+        if (plan.boundary && ordinal === 0) {
+            tasks.push({ kind: "boundary", ordinal });
+            continue;
+        }
+        const geometryTask = ordinal - boundaryOffset;
+        const rangeOffset = geometryTask * MAX_REBALANCE_QUERY_RANGES;
+        tasks.push({
+            kind: "geometry",
+            ordinal,
+            geometryTask,
+            ranges: plan.geometryRanges.slice(rangeOffset, rangeOffset + MAX_REBALANCE_QUERY_RANGES),
+        });
+    }
+    return {
+        tasks,
+        next: end < plan.taskCount ? { nextTask: end } : undefined,
+    };
+};
+export const findOwningRebalanceTask = (coordinates, intervals) => {
     let minimum;
     for (const value of coordinates) {
         const coordinate = typeof value === "number" ? BigInt(value) : value;
@@ -2267,7 +2391,9 @@ const findOwningRebalanceBatch = (coordinates, intervals) => {
             }
             else {
                 minimum =
-                    minimum == null ? interval.batch : Math.min(minimum, interval.batch);
+                    minimum == null
+                        ? interval.geometryTask
+                        : Math.min(minimum, interval.geometryTask);
                 break;
             }
         }
@@ -2289,7 +2415,13 @@ const rotateRebalanceHistory = (changes, rebalanceHistory) => {
         }
     }
 };
-export const mergeReplicationChanges = (changesOrChangesArr, rebalanceHistory, options) => {
+const snapshotRebalanceHistory = (rebalanceHistory) => {
+    // Expiration is a cache concern, not a planner concern. Resolve it once and
+    // give the pure planner a stable view for the complete plan.
+    rebalanceHistory.trim();
+    return new Set([...rebalanceHistory.map.keys()].filter((key) => typeof key === "string"));
+};
+const mergeReplicationChangesFromHistory = (changesOrChangesArr, rebalanceHistory) => {
     // Work from an immutable snapshot. In particular, synthetic difference ranges
     // must never become input events or rebalance-history entries.
     const changes = flattenReplicationChanges(changesOrChangesArr);
@@ -2451,70 +2583,99 @@ export const mergeReplicationChanges = (changesOrChangesArr, rebalanceHistory, o
             rebalanceBoundaryOnly: true,
         });
     }
+    return deduplicated;
+};
+export const mergeReplicationChanges = (changesOrChangesArr, rebalanceHistory, options) => {
+    const changes = flattenReplicationChanges(changesOrChangesArr);
+    const merged = mergeReplicationChangesFromHistory(changes, snapshotRebalanceHistory(rebalanceHistory));
     if (options?.updateHistory !== false) {
         rotateRebalanceHistory(changes, rebalanceHistory);
     }
-    return deduplicated;
+    return merged;
+};
+export const createRebalanceScanPlan = (input) => {
+    const sourceChanges = [...input.changes];
+    const queryChanges = input.forceFresh
+        ? sourceChanges
+        : mergeReplicationChangesFromHistory(sourceChanges, input.history);
+    const geometry = createRebalanceQueryGeometryPlan(queryChanges, sourceChanges.length === 0);
+    return {
+        ...geometry,
+        historyMutations: input.forceFresh
+            ? []
+            : sourceChanges.map((change) => ({
+                rangeHash: change.range.rangeHash,
+                present: change.type === "added",
+            })),
+    };
+};
+const applyRebalanceHistoryMutations = (mutations, rebalanceHistory) => {
+    for (const mutation of mutations) {
+        if (mutation.present) {
+            rebalanceHistory.add(mutation.rangeHash);
+        }
+        else {
+            rebalanceHistory.del(mutation.rangeHash);
+        }
+    }
 };
 const REBALANCE_ITERATOR_BATCH_SIZE = 1048;
 export const toRebalance = (changeOrChanges, index, rebalanceHistory, options) => {
     const sourceChanges = flattenReplicationChanges(changeOrChanges);
-    const queryChanges = options?.forceFresh
-        ? sourceChanges
-        : mergeReplicationChanges(sourceChanges, rebalanceHistory, {
-            updateHistory: false,
-        });
-    const plan = createRebalanceQueryPlan(queryChanges, sourceChanges.length === 0);
+    const plan = createRebalanceScanPlan({
+        changes: sourceChanges,
+        history: options?.forceFresh
+            ? new Set()
+            : snapshotRebalanceHistory(rebalanceHistory),
+        forceFresh: options?.forceFresh,
+    });
     return {
         [Symbol.asyncIterator]: async function* () {
             let completed = false;
             try {
-                const passes = [];
-                if (plan.boundaryChanges) {
-                    passes.push({ changes: plan.boundaryChanges });
-                }
-                for (let batch = 0; batch < plan.geometryBatches.length; batch++) {
-                    passes.push({
-                        changes: plan.geometryBatches[batch],
-                        geometryBatch: batch,
-                    });
-                }
-                for (const pass of passes) {
-                    const isGeometry = pass.geometryBatch != null;
-                    const iterator = index.iterate({
-                        query: createAssignedRangesQuery(pass.changes, {
-                            strict: isGeometry,
-                            includeBoundaryWhenEmpty: !isGeometry,
-                        }),
-                    });
-                    let passCompleted = false;
-                    try {
-                        while (iterator.done() !== true) {
-                            const entries = await iterator.next(REBALANCE_ITERATOR_BATCH_SIZE);
-                            if (entries.length === 0) {
-                                break;
-                            }
-                            for (const entry of entries) {
-                                if (isGeometry) {
-                                    if (plan.boundaryChanges &&
-                                        entry.value.assignedToRangeBoundary) {
-                                        continue;
-                                    }
-                                    if (findOwningRebalanceBatch(entry.value.coordinates, plan.ownedIntervals) !== pass.geometryBatch) {
-                                        continue;
+                let cursor;
+                for (;;) {
+                    const page = getRebalanceScanTaskPage(plan, cursor);
+                    for (const task of page.tasks) {
+                        const isGeometry = task.kind === "geometry";
+                        const iterator = index.iterate({
+                            query: createAssignedRangesQuery(isGeometry ? task.ranges.map((range) => ({ range })) : [], {
+                                strict: isGeometry,
+                                includeBoundaryWhenEmpty: !isGeometry,
+                            }),
+                        });
+                        let taskCompleted = false;
+                        try {
+                            while (iterator.done() !== true) {
+                                const entries = await iterator.next(REBALANCE_ITERATOR_BATCH_SIZE);
+                                if (entries.length === 0) {
+                                    break;
+                                }
+                                for (const entry of entries) {
+                                    if (isGeometry) {
+                                        if (plan.boundary && entry.value.assignedToRangeBoundary) {
+                                            continue;
+                                        }
+                                        if (findOwningRebalanceTask(entry.value.coordinates, plan.ownedIntervals) !== task.geometryTask) {
+                                            continue;
+                                        }
                                     }
+                                    yield entry.value;
                                 }
-                                yield entry.value;
                             }
+                            taskCompleted = iterator.done() === true;
+                        }
+                        finally {
+                            await iterator.close();
+                        }
+                        if (!taskCompleted) {
+                            return;
                         }
-                        passCompleted = iterator.done() === true;
-                    }
-                    finally {
-                        await iterator.close();
                     }
-                    if (!passCompleted) {
-                        return;
+                    if (!page.next) {
+                        break;
                     }
+                    cursor = page.next;
                 }
                 completed = true;
             }
@@ -2522,8 +2683,8 @@ export const toRebalance = (changeOrChanges, index, rebalanceHistory, options) =
                 // An early consumer return, iterator error, incomplete empty batch, or
                 // close failure in any sequential pass must leave history untouched so the
                 // next event retries the complete query. forceFresh preserves its bypass.
-                if (completed && !options?.forceFresh) {
-                    rotateRebalanceHistory(sourceChanges, rebalanceHistory);
+                if (completed) {
+                    applyRebalanceHistoryMutations(plan.historyMutations, rebalanceHistory);
                 }
             }
         },