@peerbit/shared-log 10.0.6 → 10.1.0-7d319f1

package/dist/src/ranges.js

@@ -9,13 +9,13 @@ var __metadata = (this && this.__metadata) || function (k, v) {
 };
 import { deserialize, field, serialize, variant, vec } from "@dao-xyz/borsh";
 import { PublicSignKey, equals, randomBytes, sha256Base64Sync, toBase64, } from "@peerbit/crypto";
-import { And, BoolQuery, ByteMatchQuery, Compare, IntegerCompare, Not, Or, Sort, SortDirection, StringMatch,
+import { And, BoolQuery, ByteMatchQuery, Compare, IntegerCompare, Not, Or, Sort, SortDirection, StringMatch, iteratorInSeries,
 /* 	iteratorInSeries, */
  } from "@peerbit/indexer-interface";
 import { id } from "@peerbit/indexer-interface";
 import { Meta, ShallowMeta } from "@peerbit/log";
+import { debounceAccumulator } from "./debounce.js";
 import { MAX_U32, MAX_U64, } from "./integers.js";
-import {} from "./replication-domain.js";
 export var ReplicationIntent;
 (function (ReplicationIntent) {
     ReplicationIntent[ReplicationIntent["NonStrict"] = 0] = "NonStrict";
@@ -193,7 +193,7 @@ let ReplicationRangeMessageU32 = class ReplicationRangeMessageU32 extends Replic
             id: this.id,
             publicKeyHash: key.hashcode(),
             offset: this.offset,
-            length: this.factor,
+            width: this.factor,
             timestamp: this.timestamp,
             mode: this.mode,
         });
@@ -250,7 +250,7 @@ let ReplicationRangeMessageU64 = class ReplicationRangeMessageU64 extends Replic
             id: this.id,
             publicKeyHash: key.hashcode(),
             offset: this.offset,
-            length: this.factor,
+            width: this.factor,
             timestamp: this.timestamp,
             mode: this.mode,
         });
@@ -349,6 +349,185 @@ __decorate([
     field({ type: "u8" }),
     __metadata("design:type", Number)
 ], HashableSegmentU64.prototype, "mode", void 0);
+/**
+ * Convert a GeneralRange<N> into one or two `[bigint, bigint]` segments.
+ * - If it’s not wrapped, there’s one segment: [start1, end1).
+ * - If it’s wrapped, there’s two: [start1, end1) and [start2, end2).
+ *
+ * We always do the conversion to bigints internally.
+ */
+export function toSegmentsBigInt(range) {
+    // Safely convert the numeric fields to bigint
+    const s1 = typeof range.start1 === "number" ? BigInt(range.start1) : range.start1;
+    const e1 = typeof range.end1 === "number" ? BigInt(range.end1) : range.end1;
+    const s2 = typeof range.start2 === "number" ? BigInt(range.start2) : range.start2;
+    const e2 = typeof range.end2 === "number" ? BigInt(range.end2) : range.end2;
+    const segments = [];
+    segments.push([s1, e1]);
+    if (s2 !== s1 && s2 !== e2) {
+        segments.push([s2, e2]);
+    }
+    return segments;
+}
+/**
+ * Build an array of new GeneralRange<N> objects from leftover `[bigint, bigint]` segments.
+ * We split them in pairs, each range can hold up to two segments:
+ *
+ * - [seg1Start, seg1End)
+ * - [seg2Start, seg2End) (if available)
+ *
+ * We convert bigints back to the correct numeric type, if needed.
+ */
+function buildRangesFromBigIntSegments(segments, templateRange) {
+    // Sort by start
+    segments.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
+    const result = [];
+    let i = 0;
+    const proto = Object.getPrototypeOf(templateRange);
+    while (i < segments.length) {
+        const seg1 = segments[i];
+        i++;
+        let seg2 = null;
+        if (i < segments.length) {
+            seg2 = segments[i];
+            i++;
+        }
+        // Convert back to the original numeric type
+        const [s1, e1] = toOriginalType(seg1, templateRange);
+        const [s2, e2] = seg2
+            ? toOriginalType(seg2, templateRange)
+            : [s1, e1];
+        // Build a new range object. You can clone or replicate metadata as needed.
+        const newRange = Object.assign(Object.create(proto), {
+            ...templateRange,
+            start1: s1,
+            end1: e1,
+            start2: s2,
+            end2: e2,
+        });
+        result.push(newRange);
+    }
+    return result;
+}
+/**
+ * Subtract one bigint segment [bStart, bEnd) from [aStart, aEnd).
+ * Returns 0..2 leftover segments in bigint form.
+ */
+function subtractBigIntSegment(aStart, aEnd, bStart, bEnd) {
+    const result = [];
+    // No overlap
+    if (bEnd <= aStart || bStart >= aEnd) {
+        result.push([aStart, aEnd]);
+        return result;
+    }
+    // Fully contained
+    if (bStart <= aStart && bEnd >= aEnd) {
+        return [];
+    }
+    // Partial overlaps
+    if (bStart > aStart) {
+        result.push([aStart, bStart]);
+    }
+    if (bEnd < aEnd) {
+        result.push([bEnd, aEnd]);
+    }
+    return result;
+}
+/**
+ * Helper: convert `[bigint, bigint]` to `[number, number]` if N is "u32",
+ * or keep as `[bigint, bigint]` if N is "u64".
+ */
+function toOriginalType(segment, templateRange) {
+    const [start, end] = segment;
+    if (isU32Range(templateRange)) {
+        // Convert back to number
+        return [Number(start), Number(end)];
+    }
+    else {
+        // Keep as bigint
+        return [start, end];
+    }
+}
+/**
+ * Merge any adjacent or overlapping `[bigint, bigint]` segments.
+ * E.g. [10,20) and [20,25) => [10,25)
+ */
+export function mergeBigIntSegments(segments) {
+    if (segments.length < 2)
+        return segments;
+    // Sort by start
+    segments.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
+    const merged = [];
+    let current = segments[0];
+    for (let i = 1; i < segments.length; i++) {
+        const next = segments[i];
+        // If current overlaps or touches next
+        if (current[1] >= next[0]) {
+            // Merge
+            current = [current[0], current[1] > next[1] ? current[1] : next[1]];
+        }
+        else {
+            merged.push(current);
+            current = next;
+        }
+    }
+    merged.push(current);
+    return merged;
+}
+/**
+ * Figure out if a given range is "u32" or "u64".
+ * You might also store this in the object itself if you prefer.
+ */
+function isU32Range(range) {
+    // If you store a separate `type: "u32" | "u64"` in the range, you can just check that:
+    // return range.type === "u32";
+    // or we do a hack by checking the type of start1, e.g.:
+    // If "start1" is a number (not a bigint), we treat it as u32
+    return typeof range.start1 === "number";
+}
+export function symmetricDifferenceRanges(rangeA, rangeB) {
+    const segmentsA = toSegmentsBigInt(rangeA);
+    const segmentsB = toSegmentsBigInt(rangeB);
+    const resultSegmentsA = [];
+    const resultSegmentsB = [];
+    // Compute symmetric difference for A
+    for (const [aStart, aEnd] of segmentsA) {
+        let leftover = [[aStart, aEnd]];
+        for (const [bStart, bEnd] of segmentsB) {
+            const newLeftover = [];
+            for (const [start, end] of leftover) {
+                newLeftover.push(...subtractBigIntSegment(start, end, bStart, bEnd));
+            }
+            leftover = newLeftover;
+        }
+        resultSegmentsA.push(...leftover);
+    }
+    // Compute symmetric difference for B
+    for (const [bStart, bEnd] of segmentsB) {
+        let leftover = [[bStart, bEnd]];
+        for (const [aStart, aEnd] of segmentsA) {
+            const newLeftover = [];
+            for (const [start, end] of leftover) {
+                newLeftover.push(...subtractBigIntSegment(start, end, aStart, aEnd));
+            }
+            leftover = newLeftover;
+        }
+        resultSegmentsB.push(...leftover);
+    }
+    // Remove zero-length or invalid segments
+    const validSegmentsA = resultSegmentsA.filter(([start, end]) => start < end);
+    const validSegmentsB = resultSegmentsB.filter(([start, end]) => start < end);
+    // Merge and deduplicate segments
+    const mergedSegmentsA = mergeBigIntSegments(validSegmentsA);
+    const mergedSegmentsB = mergeBigIntSegments(validSegmentsB);
+    // Build ranges
+    const rangesFromA = buildRangesFromBigIntSegments(mergedSegmentsA, rangeA);
+    const rangesFromB = buildRangesFromBigIntSegments(mergedSegmentsB, rangeB);
+    return {
+        rangesFromA,
+        rangesFromB,
+    };
+}
 export class ReplicationRangeIndexableU32 {
     id;
     hash;
@@ -364,12 +543,12 @@ export class ReplicationRangeIndexableU32 {
         this.hash =
             properties.publicKeyHash ||
                 properties.publicKey.hashcode();
-        this.transform({ length: properties.length, offset: properties.offset });
+        this.transform({ width: properties.width, offset: properties.offset });
         this.mode = properties.mode ?? ReplicationIntent.NonStrict;
         this.timestamp = properties.timestamp || BigInt(0);
     }
     transform(properties) {
-        const ranges = getSegmentsFromOffsetAndRange(properties.offset, properties.length, 0, MAX_U32);
+        const ranges = getSegmentsFromOffsetAndRange(properties.offset, properties.width, 0, MAX_U32);
         this.start1 = Math.round(ranges[0][0]);
         this.end1 = Math.round(ranges[0][1]);
         this.start2 = Math.round(ranges[1][0]);
@@ -390,6 +569,10 @@ export class ReplicationRangeIndexableU32 {
     get idString() {
         return toBase64(this.id);
     }
+    get rangeHash() {
+        const ser = serialize(this);
+        return sha256Base64Sync(ser);
+    }
     contains(point) {
         return ((point >= this.start1 && point < this.end1) ||
             (point >= this.start2 && point < this.end2));
@@ -436,7 +619,8 @@ export class ReplicationRangeIndexableU32 {
         return false;
     }
     equalRange(other) {
-        return (this.start1 === other.start1 &&
+        return (this.hash === other.hash &&
+            this.start1 === other.start1 &&
             this.end1 === other.end1 &&
             this.start2 === other.start2 &&
             this.end2 === other.end2);
@@ -495,7 +679,7 @@ __decorate([
 ], ReplicationRangeIndexableU32.prototype, "mode", void 0);
 export class ReplicationRangeIndexableU64 {
     id;
-    hash;
+    hash; // publickey hash
     timestamp;
     start1;
     end1;
@@ -508,12 +692,12 @@ export class ReplicationRangeIndexableU64 {
         this.hash =
             properties.publicKeyHash ||
                 properties.publicKey.hashcode();
-        this.transform({ length: properties.length, offset: properties.offset });
+        this.transform({ width: properties.width, offset: properties.offset });
         this.mode = properties.mode ?? ReplicationIntent.NonStrict;
         this.timestamp = properties.timestamp || BigInt(0);
     }
     transform(properties) {
-        const ranges = getSegmentsFromOffsetAndRange(BigInt(properties.offset), BigInt(properties.length), 0n, MAX_U64);
+        const ranges = getSegmentsFromOffsetAndRange(BigInt(properties.offset), BigInt(properties.width), 0n, MAX_U64);
         this.start1 = ranges[0][0];
         this.end1 = ranges[0][1];
         this.start2 = ranges[1][0];
@@ -538,6 +722,10 @@ export class ReplicationRangeIndexableU64 {
         return ((point >= this.start1 && point < this.end1) ||
             (point >= this.start2 && point < this.end2));
     }
+    get rangeHash() {
+        const ser = serialize(this);
+        return sha256Base64Sync(ser);
+    }
     overlaps(other, checkOther = true) {
         if (this.contains(other.start1) ||
             this.contains(other.start2) ||
@@ -580,7 +768,8 @@ export class ReplicationRangeIndexableU64 {
         return false;
     }
     equalRange(other) {
-        return (this.start1 === other.start1 &&
+        return (this.hash === other.hash &&
+            this.start1 === other.start1 &&
             this.end1 === other.end1 &&
             this.start2 === other.start2 &&
             this.end2 === other.end2);
@@ -649,18 +838,17 @@ export const mergeRanges = (segments, numbers) => {
     if (!sameHash) {
         throw new Error("Segments have different publicKeyHash");
     }
-    // only allow merging segments with length 1 (trivial)
-    const sameLength = segments.every((x) => x.width === 1 || x.width === 1n);
-    if (!sameLength) {
-        throw new Error("Segments have different length, only merging of segments length 1 is supported");
-    }
     const sorted = segments.sort((a, b) => Number(a.start1 - b.start1));
     let calculateLargeGap = () => {
         let last = sorted[sorted.length - 1];
         let largestArc = numbers.zero;
         let largestArcIndex = -1;
+        let mode = ReplicationIntent.NonStrict;
         for (let i = 0; i < sorted.length; i++) {
             const current = sorted[i];
+            if (current.mode === ReplicationIntent.Strict) {
+                mode = ReplicationIntent.Strict;
+            }
             if (current.start1 !== last.start1) {
                 let arc = numbers.zero;
                 if (current.start1 < last.end2) {
@@ -677,19 +865,28 @@ export const mergeRanges = (segments, numbers) => {
             }
             last = current;
         }
-        return [largestArc, largestArcIndex];
+        return [largestArc, largestArcIndex, mode];
     };
-    const [largestArc, largestArcIndex] = calculateLargeGap();
+    const [largestArc, largestArcIndex, mode] = calculateLargeGap();
     let totalLengthFinal = numbers.maxValue - largestArc;
+    const proto = segments[0].constructor;
     if (largestArcIndex === -1) {
+        if (mode !== segments[0].mode) {
+            return new proto({
+                width: segments[0].width,
+                offset: segments[0].start1,
+                publicKeyHash: segments[0].hash,
+                mode,
+            });
+        }
         return segments[0]; // all ranges are the same
     }
     // use segments[0] constructor to create a new object
-    const proto = segments[0].constructor;
     return new proto({
-        length: totalLengthFinal,
+        width: totalLengthFinal,
         offset: segments[largestArcIndex].start1,
         publicKeyHash: segments[0].hash,
+        mode,
     });
 };
 const createContainingPointQuery = (points, options) => {
@@ -815,7 +1012,7 @@ export const appromixateCoverage = async (properties) => {
     }
     return hits / properties.samples;
 };
-const getClosest = (direction, rects, point, roleAgeLimit, matured, now, includeStrict, numbers, options) => {
+const getClosest = (direction, rects, point, includeStrict, numbers, options) => {
     const createQueries = (p, equality) => {
         let queries;
         if (direction === "below") {
@@ -825,11 +1022,6 @@ const getClosest = (direction, rects, point, roleAgeLimit, matured, now, include
                     compare: equality ? Compare.LessOrEqual : Compare.Less,
                     value: p,
                 }),
-                new IntegerCompare({
-                    key: "timestamp",
-                    compare: matured ? Compare.LessOrEqual : Compare.GreaterOrEqual,
-                    value: BigInt(now - roleAgeLimit),
-                }),
             ];
         }
         else {
@@ -839,13 +1031,17 @@ const getClosest = (direction, rects, point, roleAgeLimit, matured, now, include
                     compare: equality ? Compare.GreaterOrEqual : Compare.Greater,
                     value: p,
                 }),
-                new IntegerCompare({
-                    key: "timestamp",
-                    compare: matured ? Compare.LessOrEqual : Compare.GreaterOrEqual,
-                    value: BigInt(now - roleAgeLimit),
-                }),
             ];
         }
+        if (options?.time) {
+            queries.push(new IntegerCompare({
+                key: "timestamp",
+                compare: options?.time?.matured
+                    ? Compare.LessOrEqual
+                    : Compare.GreaterOrEqual,
+                value: BigInt(options.time.now - options.time.roleAgeLimit),
+            }));
+        }
         queries.push(new IntegerCompare({ key: "width", compare: Compare.Greater, value: 0 }));
         if (!includeStrict) {
             queries.push(new IntegerCompare({
@@ -854,6 +1050,9 @@ const getClosest = (direction, rects, point, roleAgeLimit, matured, now, include
                 value: ReplicationIntent.NonStrict,
             }));
         }
+        if (options?.hash) {
+            queries.push(new StringMatch({ key: "hash", value: options.hash }));
+        }
         return queries;
     };
     const sortByOldest = new Sort({ key: "timestamp", direction: "asc" });
@@ -936,7 +1135,7 @@ export const getCoveringRangeQuery = (range) => {
         ]),
     ];
 };
-export const iHaveCoveringRange = async (rects, range) => {
+export const countCoveringRangesSameOwner = async (rects, range) => {
     return ((await rects.count({
         query: [
             ...getCoveringRangeQuery(range),
@@ -952,6 +1151,22 @@ export const iHaveCoveringRange = async (rects, range) => {
         ],
     })) > 0);
 };
+export const getCoveringRangesSameOwner = (rects, range) => {
+    return rects.iterate({
+        query: [
+            ...getCoveringRangeQuery(range),
+            new StringMatch({
+                key: "hash",
+                value: range.hash,
+            }),
+            // assume that we are looking for other ranges, not want to update an existing one
+            new Not(new ByteMatchQuery({
+                key: "id",
+                value: range.id,
+            })),
+        ],
+    });
+};
 // TODO
 export function getDistance(from, to, direction, end) {
     const abs = (value) => value < 0 ? -value : value;
@@ -1055,103 +1270,54 @@ const joinIterator = (iterators, point, direction, numbers) => {
         },
     };
 };
-const getClosestAround = (peers, point, roleAge, now, includeStrictBelow, includeStrictAbove, numbers, options) => {
-    const closestBelow = getClosest("below", peers, point, roleAge, true, now, includeStrictBelow, numbers, options);
-    const closestAbove = getClosest("above", peers, point, roleAge, true, now, includeStrictAbove, numbers, options);
-    /* const containing = iterateRangesContainingPoint<S, R>(
-        peers,
-        point,
-        {
-            time: {
-                roleAgeLimit: roleAge,
-                matured: true,
-                now,
-            }
-        }
-    );
-
-    return iteratorInSeries(
-        containing,
-        joinIterator<S, R>([closestBelow, closestAbove], point, "closest", numbers),
-    ); */
-    return joinIterator([closestBelow, closestAbove], point, "closest", numbers);
+const getClosestAroundOrContaining = (peers, point, includeStrictBelow, includeStrictAbove, numbers, options) => {
+    const closestBelow = getClosest("below", peers, point, includeStrictBelow, numbers, options);
+    const closestAbove = getClosest("above", peers, point, includeStrictAbove, numbers, options);
+    const containing = iterateRangesContainingPoint(peers, point, options);
+    return iteratorInSeries(containing, joinIterator([closestBelow, closestAbove], point, "closest", numbers));
 };
-export const isMatured = (segment, now, minAge) => {
-    return now - Number(segment.timestamp) >= minAge;
+export const getAdjecentSameOwner = async (peers, range, numbers) => {
+    const closestBelowIterator = getClosest("below", peers, range.start1, true, numbers, {
+        hash: range.hash,
+    });
+    const closestBelow = await closestBelowIterator.next(1);
+    closestBelowIterator.close();
+    const closestAboveIterator = getClosest("above", peers, range.end2, true, numbers, {
+        hash: range.hash,
+    });
+    const closestAbove = await closestAboveIterator.next(1);
+    closestAboveIterator.close();
+    return {
+        below: range.idString === closestBelow[0]?.value.idString
+            ? undefined
+            : closestBelow[0]?.value,
+        above: closestBelow[0]?.id.primitive === closestAbove[0]?.id.primitive ||
+            range.idString === closestBelow[0]?.value.idString
+            ? undefined
+            : closestAbove[0]?.value,
+    };
 };
-/*
-
-const collectNodesAroundPoint = async <R extends "u32" | "u64">(
-    roleAge: number,
-    peers: Index<ReplicationRangeIndexable<R>>,
-    collector: (
-        rect: { hash: string },
-        matured: boolean,
-        intersecting: boolean,
-    ) => void,
-    point: NumberFromType<R>,
-    now: number,
-    numbers: Numbers<R>,
-    done: () => boolean = () => true,
-) => {
-    const containing = iterateRangesContainingPoint<
-        { timestamp: true, hash: true },
-        R
-    >(peers, point, 0, true, now, { shape: { timestamp: true, hash: true } as const });
-    const allContaining = await containing.all();
-    for (const rect of allContaining) {
-        collector(rect.value, isMatured(rect.value, now, roleAge), true);
-    }
-
-    if (done()) {
-        return;
-    }
-
-    const closestBelow = getClosest<undefined, R>(
-        "below",
-        peers,
-        point,
-        0,
-        true,
-        now,
-        false,
-        numbers
-    );
-    const closestAbove = getClosest<undefined, R>(
-        "above",
-        peers,
-        point,
-        0,
-        true,
-        now,
-        false,
-        numbers
-    );
-    const aroundIterator = joinIterator<undefined, R>(
-        [closestBelow, closestAbove],
-        point,
-        "closest",
-        numbers,
-    );
-    while (aroundIterator.done() !== true && done() !== true) {
-        const res = await aroundIterator.next(1);
-        for (const rect of res) {
-            collector(rect.value, isMatured(rect.value, now, roleAge), false);
-            if (done()) {
-                return;
-            }
-        }
+export const getAllMergeCandiates = async (peers, range, numbers) => {
+    const adjacent = await getAdjecentSameOwner(peers, range, numbers);
+    const covering = await getCoveringRangesSameOwner(peers, range).all();
+    let ret = new Map();
+    if (adjacent.below) {
+        ret.set(adjacent.below.idString, adjacent.below);
+    }
+    if (adjacent.above) {
+        ret.set(adjacent.above.idString, adjacent.above);
     }
+    for (const range of covering) {
+        ret.set(range.value.idString, range.value);
+    }
+    return ret.values();
+};
+export const isMatured = (segment, now, minAge) => {
+    return now - Number(segment.timestamp) >= minAge;
 };
- */
 const collectClosestAround = async (roleAge, peers, collector, point, now, numbers, done = () => true) => {
-    const closestBelow = getClosest("below", peers, point, 0, true, now, false, numbers);
-    const closestAbove = getClosest("above", peers, point, 0, true, now, false, numbers);
-    /* 	const containingIterator = iterateRangesContainingPoint<undefined, R>(
-            peers,
-            point,
-        );
-     */
+    const closestBelow = getClosest("below", peers, point, false, numbers);
+    const closestAbove = getClosest("above", peers, point, false, numbers);
     const aroundIterator = joinIterator([/* containingIterator,  */ closestBelow, closestAbove], point, "closest", numbers);
     let visited = new Set();
     while (aroundIterator.done() !== true && done() !== true) {
@@ -1243,7 +1409,13 @@ export const minimumWidthToCover = async (minReplicas /* , replicatorCount: numb
 export const getCoverSet = async (properties) => {
     const { peers, start, widthToCoverScaled, roleAge } = properties;
     const now = Date.now();
-    const { startNode, startLocation, endLocation } = await getStartAndEnd(peers, start, widthToCoverScaled, roleAge, now, properties.numbers);
+    const { startNode, startLocation, endLocation } = await getStartAndEnd(peers, start, widthToCoverScaled, properties.numbers, {
+        time: {
+            roleAgeLimit: roleAge,
+            now,
+            matured: true,
+        },
+    });
     let ret = new Set();
     // if start node (assume is self) and not mature, ask all known remotes if limited
     // TODO consider a more robust stragety here in a scenario where there are many nodes, lets say
@@ -1288,7 +1460,13 @@ export const getCoverSet = async (properties) => {
     };
     const resolveNextAbove = async (nextLocation, roleAge) => {
         // if not get closest from above
-        let next = await fetchOne(getClosest("above", peers, nextLocation, roleAge, true, now, true, properties.numbers));
+        let next = await fetchOne(getClosest("above", peers, nextLocation, true, properties.numbers, {
+            time: {
+                matured: true,
+                roleAgeLimit: roleAge,
+                now,
+            },
+        }));
         return next;
     };
     const resolveNext = async (nextLocation, roleAge) => {
@@ -1362,21 +1540,6 @@ export const getCoverSet = async (properties) => {
     start instanceof PublicSignKey && ret.add(start.hashcode());
     return ret;
 };
-/* export const getReplicationDiff = (changes: ReplicationChange) => {
-    // reduce the change set to only regions that are changed for each peer
-    // i.e. subtract removed regions from added regions, and vice versa
-    const result = new Map<string, { range: ReplicationRangeIndexable, added: boolean }[]>();
-    
-    for (const addedChange of changes.added ?? []) {
-        let prev = result.get(addedChange.hash) ?? [];
-        for (const [_hash, ranges] of result.entries()) {
-            for (const r of ranges) {
-    
-            }
-        }
-    }
-}
- */
 export const matchEntriesInRangeQuery = (range) => {
     const c1 = new And([
         new IntegerCompare({
@@ -1410,33 +1573,146 @@ export const matchEntriesInRangeQuery = (range) => {
     ];
     return new Or(ors);
 };
-export const toRebalance = (changes, index) => {
+export const debounceAggregationChanges = (fn, delay) => {
+    return debounceAccumulator((result) => {
+        return fn([...result.values()]);
+    }, () => {
+        let aggregated = new Map();
+        return {
+            add: (change) => {
+                const prev = aggregated.get(change.range.idString);
+                if (prev) {
+                    if (prev.range.timestamp < change.range.timestamp) {
+                        aggregated.set(change.range.idString, change);
+                    }
+                }
+                else {
+                    aggregated.set(change.range.idString, change);
+                }
+            },
+            delete: (key) => {
+                aggregated.delete(key);
+            },
+            size: () => aggregated.size,
+            value: aggregated,
+        };
+    }, delay);
+};
+export const mergeReplicationChanges = (changesOrChangesArr, rebalanceHistory) => {
+    let first = changesOrChangesArr[0];
+    let changes;
+    if (!Array.isArray(first)) {
+        changes = changesOrChangesArr;
+    }
+    else {
+        changes = changesOrChangesArr.flat();
+    }
+    // group by hash so we can cancel out changes
+    const grouped = new Map();
+    for (const change of changes) {
+        const prev = grouped.get(change.range.hash);
+        if (prev) {
+            prev.push(change);
+        }
+        else {
+            grouped.set(change.range.hash, [change]);
+        }
+    }
+    let all = [];
+    for (const [_k, v] of grouped) {
+        if (v.length > 1) {
+            // sort by timestamp so newest is last
+            v.sort((a, b) => a.range.timestamp < b.range.timestamp
+                ? -1
+                : a.range.timestamp > b.range.timestamp
+                    ? 1
+                    : 0);
+            let results = [];
+            let consumed = new Set();
+            for (let i = 0; i < v.length; i++) {
+                // if segment is removed and we have previously processed it
+                // then go over each overlapping added segment add remove the removal,
+                // equivalent is that this would represent  (1 - 1 + 1) = 1
+                if (v[i].type === "removed" || v[i].type === "replaced") {
+                    if (rebalanceHistory.has(v[i].range.rangeHash)) {
+                        let vStart = v.length;
+                        for (let j = i + 1; j < vStart; j++) {
+                            const newer = v[j];
+                            if (newer.type === "added" && !newer.matured) {
+                                const { rangesFromA: updatedRemoved, rangesFromB: updatedNewer, } = symmetricDifferenceRanges(v[i].range, newer.range);
+                                for (const diff of updatedRemoved) {
+                                    results.push({
+                                        range: diff,
+                                        type: "removed",
+                                        timestamp: v[i].timestamp,
+                                    });
+                                }
+                                for (const diff of updatedNewer) {
+                                    v.push({
+                                        range: diff,
+                                        type: "added",
+                                        timestamp: newer.timestamp,
+                                    });
+                                }
+                                consumed.add(j);
+                            }
+                        }
+                        rebalanceHistory.del(v[i].range.rangeHash);
+                    }
+                    else {
+                        results.push(v[i]);
+                    }
+                }
+                else if (v[i].type === "added") {
+                    // TODO should the below clause be used?
+                    // after testing it seems that certain changes are not propagating as expected using this
+                    /* if (rebalanceHistory.has(v[i].range.rangeHash)) {
+                        continue;
+                    } */
+                    rebalanceHistory.add(v[i].range.rangeHash);
+                    if (!consumed.has(i)) {
+                        results.push(v[i]);
+                    }
+                }
+                else {
+                    results.push(v[i]);
+                }
+            }
+            all.push(...results);
+        }
+        else {
+            rebalanceHistory.add(v[0].range.rangeHash);
+            all.push(v[0]);
+        }
+    }
+    return all;
+};
+export const toRebalance = (changeOrChanges, index, rebalanceHistory) => {
+    const change = mergeReplicationChanges(changeOrChanges, rebalanceHistory);
     const assignedRangesQuery = (changes) => {
         let ors = [];
+        let onlyStrict = true;
         for (const change of changes) {
             const matchRange = matchEntriesInRangeQuery(change.range);
-            if (change.type === "updated") {
-                // assuming a range is to be removed, is this entry still enoughly replicated
-                const prevMatchRange = matchEntriesInRangeQuery(change.prev);
-                ors.push(prevMatchRange);
-                ors.push(matchRange);
-            }
-            else {
-                ors.push(matchRange);
+            ors.push(matchRange);
+            if (change.range.mode === ReplicationIntent.NonStrict) {
+                onlyStrict = false;
             }
         }
         // entry is assigned to a range boundary, meaning it is due to be inspected
-        ors.push(new BoolQuery({
-            key: "assignedToRangeBoundary",
-            value: true,
-        }));
+        if (!onlyStrict || changes.length === 0) {
+            ors.push(new BoolQuery({
+                key: "assignedToRangeBoundary",
+                value: true,
+            }));
+        }
         // entry is not sufficiently replicated, and we are to still keep it
         return new Or(ors);
     };
     return {
         [Symbol.asyncIterator]: async function* () {
             const iterator = index.iterate({
-                query: assignedRangesQuery(changes),
+                query: assignedRangesQuery(change),
             });
             while (iterator.done() !== true) {
                 const entries = await iterator.all(); // TODO choose right batch sizes here for optimal memory usage / speed
@@ -1451,7 +1727,7 @@ export const toRebalance = (changes, index) => {
         },
     };
 };
-export const fetchOneFromPublicKey = async (publicKey, index, roleAge, now, numbers, options) => {
+export const fetchOneFromPublicKey = async (publicKey, index, numbers, options) => {
     let iterator = index.iterate({
         query: [new StringMatch({ key: "hash", value: publicKey.hashcode() })],
     }, options);
@@ -1459,8 +1735,9 @@ export const fetchOneFromPublicKey = async (publicKey, index, roleAge, now, numb
     await iterator.close();
     let node = result[0]?.value;
     if (node) {
-        if (!isMatured(node, now, roleAge)) {
-            const matured = await fetchOne(getClosestAround(index, node.start1, roleAge, now, false, false, numbers, options));
+        if (options?.time &&
+            !isMatured(node, options.time.now, options.time.roleAgeLimit)) {
+            const matured = await fetchOne(getClosestAroundOrContaining(index, node.start1, false, false, numbers, options));
             if (matured) {
                 node = matured;
             }
@@ -1468,17 +1745,17 @@ export const fetchOneFromPublicKey = async (publicKey, index, roleAge, now, numb
     }
     return node;
 };
-export const getStartAndEnd = async (peers, start, widthToCoverScaled, roleAge, now, numbers, options) => {
+export const getStartAndEnd = async (peers, start, widthToCoverScaled, numbers, options) => {
     // find a good starting point
     let startNode = undefined;
     let startLocation = undefined;
     const nodeFromPoint = async (point = numbers.random()) => {
         startLocation = point;
-        startNode = await fetchOneClosest(peers, startLocation, roleAge, now, false, true, numbers, options);
+        startNode = await fetchOneClosest(peers, startLocation, false, true, numbers, options);
     };
     if (start instanceof PublicSignKey) {
         // start at our node (local first)
-        startNode = await fetchOneFromPublicKey(start, peers, roleAge, now, numbers, options);
+        startNode = await fetchOneFromPublicKey(start, peers, numbers, options);
         if (!startNode) {
             // fetch randomly
             await nodeFromPoint();
@@ -1529,7 +1806,7 @@ export const getStartAndEnd = async (peers, start, widthToCoverScaled, roleAge,
         endLocation,
     };
 };
-export const fetchOneClosest = (peers, point, roleAge, now, includeStrictBelow, includeStrictAbove, numbers, options) => {
-    return fetchOne(getClosestAround(peers, point, roleAge, now, includeStrictBelow, includeStrictAbove, numbers, options));
+export const fetchOneClosest = (peers, point, includeStrictBelow, includeStrictAbove, numbers, options) => {
+    return fetchOne(getClosestAroundOrContaining(peers, point, includeStrictBelow, includeStrictAbove, numbers, options));
 };
 //# sourceMappingURL=ranges.js.map