@peerbit/shared-log 9.1.2 → 9.2.0-0b8baa8

package/dist/src/ranges.js

@@ -1,7 +1,313 @@
-import { PublicSignKey, equals } from "@peerbit/crypto";
-import { And, ByteMatchQuery, Compare, IntegerCompare, Not, Or, SearchRequest, Sort, SortDirection, StringMatch, iterate, iteratorInSeries, } from "@peerbit/indexer-interface";
-import { ReplicationIntent, } from "./replication.js";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import { deserialize, field, serialize, variant } from "@dao-xyz/borsh";
+import { PublicSignKey, equals, randomBytes, toBase64 } from "@peerbit/crypto";
+import { And, BoolQuery, ByteMatchQuery, Compare, IntegerCompare, Not, Or, Sort, SortDirection, StringMatch, iteratorInSeries, } from "@peerbit/indexer-interface";
+import { id } from "@peerbit/indexer-interface";
+import { Meta, ShallowMeta } from "@peerbit/log";
+import {} from "./replication-domain.js";
 import { MAX_U32, scaleToU32 } from "./role.js";
+import { groupByGidSync } from "./utils.js";
+export var ReplicationIntent;
+(function (ReplicationIntent) {
+    ReplicationIntent[ReplicationIntent["NonStrict"] = 0] = "NonStrict";
+    ReplicationIntent[ReplicationIntent["Strict"] = 1] = "Strict";
+})(ReplicationIntent || (ReplicationIntent = {}));
+export const getSegmentsFromOffsetAndRange = (offset, factor) => {
+    let start1 = offset;
+    let end1Unscaled = offset + factor; // only add factor if it is not 1 to prevent numerical issues (like (0.9 + 1) % 1 => 0.8999999)
+    let end1 = Math.min(end1Unscaled, MAX_U32);
+    return [
+        [start1, end1],
+        end1Unscaled > MAX_U32
+            ? [0, (factor !== MAX_U32 ? offset + factor : offset) % MAX_U32]
+            : [start1, end1],
+    ];
+};
+export const shouldAssigneToRangeBoundary = (leaders) => {
+    let assignedToRangeBoundary = leaders === false || leaders.size === 0;
+    if (!assignedToRangeBoundary && leaders) {
+        for (const [_, { intersecting }] of leaders) {
+            if (!intersecting) {
+                assignedToRangeBoundary = true;
+                break;
+            }
+        }
+    }
+    return assignedToRangeBoundary;
+};
+export class EntryReplicated {
+    id; // hash + coordinate
+    hash;
+    gid;
+    coordinate;
+    wallTime;
+    assignedToRangeBoundary;
+    _meta;
+    _metaResolved;
+    constructor(properties) {
+        this.coordinate = properties.coordinate;
+        this.hash = properties.hash;
+        this.gid = properties.meta.gid;
+        this.id = this.hash + "-" + this.coordinate;
+        this.wallTime = properties.meta.clock.timestamp.wallTime;
+        const shallow = properties.meta instanceof Meta
+            ? new ShallowMeta(properties.meta)
+            : properties.meta;
+        this._meta = serialize(shallow);
+        this._metaResolved = deserialize(this._meta, ShallowMeta);
+        this._metaResolved = properties.meta;
+        this.assignedToRangeBoundary = properties.assignedToRangeBoundary;
+    }
+    get meta() {
+        if (!this._metaResolved) {
+            this._metaResolved = deserialize(this._meta, ShallowMeta);
+        }
+        return this._metaResolved;
+    }
+}
+__decorate([
+    id({ type: "string" }),
+    __metadata("design:type", String)
+], EntryReplicated.prototype, "id", void 0);
+__decorate([
+    field({ type: "string" }),
+    __metadata("design:type", String)
+], EntryReplicated.prototype, "hash", void 0);
+__decorate([
+    field({ type: "string" }),
+    __metadata("design:type", String)
+], EntryReplicated.prototype, "gid", void 0);
+__decorate([
+    field({ type: "u32" }),
+    __metadata("design:type", Number)
+], EntryReplicated.prototype, "coordinate", void 0);
+__decorate([
+    field({ type: "u64" }),
+    __metadata("design:type", BigInt)
+], EntryReplicated.prototype, "wallTime", void 0);
+__decorate([
+    field({ type: "bool" }),
+    __metadata("design:type", Boolean)
+], EntryReplicated.prototype, "assignedToRangeBoundary", void 0);
+__decorate([
+    field({ type: Uint8Array }),
+    __metadata("design:type", Uint8Array)
+], EntryReplicated.prototype, "_meta", void 0);
+let ReplicationRange = class ReplicationRange {
+    id;
+    timestamp;
+    _offset;
+    _factor;
+    mode;
+    constructor(properties) {
+        const { id, offset, factor, timestamp, mode } = properties;
+        this.id = id;
+        this._offset = offset;
+        this._factor = factor;
+        this.timestamp = timestamp;
+        this.mode = mode;
+    }
+    get factor() {
+        return this._factor;
+    }
+    get offset() {
+        return this._offset;
+    }
+    toReplicationRangeIndexable(key) {
+        return new ReplicationRangeIndexable({
+            id: this.id,
+            publicKeyHash: key.hashcode(),
+            offset: this.offset,
+            length: this.factor,
+            timestamp: this.timestamp,
+            mode: this.mode,
+        });
+    }
+};
+__decorate([
+    field({ type: Uint8Array }),
+    __metadata("design:type", Uint8Array)
+], ReplicationRange.prototype, "id", void 0);
+__decorate([
+    field({ type: "u64" }),
+    __metadata("design:type", BigInt)
+], ReplicationRange.prototype, "timestamp", void 0);
+__decorate([
+    field({ type: "u32" }),
+    __metadata("design:type", Number)
+], ReplicationRange.prototype, "_offset", void 0);
+__decorate([
+    field({ type: "u32" }),
+    __metadata("design:type", Number)
+], ReplicationRange.prototype, "_factor", void 0);
+__decorate([
+    field({ type: "u8" }),
+    __metadata("design:type", Number)
+], ReplicationRange.prototype, "mode", void 0);
+ReplicationRange = __decorate([
+    variant(0),
+    __metadata("design:paramtypes", [Object])
+], ReplicationRange);
+export { ReplicationRange };
+export class ReplicationRangeIndexable {
+    id;
+    hash;
+    timestamp;
+    start1;
+    end1;
+    start2;
+    end2;
+    width;
+    mode;
+    constructor(properties) {
+        this.id = properties.id ?? randomBytes(32);
+        this.hash =
+            properties.publicKeyHash ||
+                properties.publicKey.hashcode();
+        if (!properties.normalized) {
+            this.transform({ length: properties.length, offset: properties.offset });
+        }
+        else {
+            this.transform({
+                length: scaleToU32(properties.length),
+                offset: scaleToU32(properties.offset),
+            });
+        }
+        this.mode = properties.mode ?? ReplicationIntent.NonStrict;
+        this.timestamp = properties.timestamp || BigInt(0);
+    }
+    transform(properties) {
+        const ranges = getSegmentsFromOffsetAndRange(properties.offset, properties.length);
+        this.start1 = Math.round(ranges[0][0]);
+        this.end1 = Math.round(ranges[0][1]);
+        this.start2 = Math.round(ranges[1][0]);
+        this.end2 = Math.round(ranges[1][1]);
+        this.width =
+            this.end1 -
+                this.start1 +
+                (this.end2 < this.end1 ? this.end2 - this.start2 : 0);
+        if (this.start1 > 0xffffffff ||
+            this.end1 > 0xffffffff ||
+            this.start2 > 0xffffffff ||
+            this.end2 > 0xffffffff ||
+            this.width > 0xffffffff ||
+            this.width < 0) {
+            throw new Error("Segment coordinate out of bounds");
+        }
+    }
+    get idString() {
+        return toBase64(this.id);
+    }
+    contains(point) {
+        return ((point >= this.start1 && point < this.end1) ||
+            (point >= this.start2 && point < this.end2));
+    }
+    overlaps(other, checkOther = true) {
+        if (this.contains(other.start1) ||
+            this.contains(other.start2) ||
+            this.contains(other.end1 - 1) ||
+            this.contains(other.end2 - 1)) {
+            return true;
+        }
+        if (checkOther) {
+            return other.overlaps(this, false);
+        }
+        return false;
+    }
+    toReplicationRange() {
+        return new ReplicationRange({
+            id: this.id,
+            offset: this.start1,
+            factor: this.width,
+            timestamp: this.timestamp,
+            mode: this.mode,
+        });
+    }
+    distanceTo(point) {
+        let wrappedPoint = MAX_U32 - point;
+        return Math.min(Math.abs(this.start1 - point), Math.abs(this.end2 - point), Math.abs(this.start1 - wrappedPoint), Math.abs(this.end2 - wrappedPoint));
+    }
+    get wrapped() {
+        return this.end2 < this.end1;
+    }
+    get widthNormalized() {
+        return this.width / MAX_U32;
+    }
+    equals(other) {
+        if (equals(this.id, other.id) &&
+            this.hash === other.hash &&
+            this.timestamp === other.timestamp &&
+            this.mode === other.mode &&
+            this.start1 === other.start1 &&
+            this.end1 === other.end1 &&
+            this.start2 === other.start2 &&
+            this.end2 === other.end2 &&
+            this.width === other.width) {
+            return true;
+        }
+        return false;
+    }
+    equalRange(other) {
+        return (this.start1 === other.start1 &&
+            this.end1 === other.end1 &&
+            this.start2 === other.start2 &&
+            this.end2 === other.end2);
+    }
+    toString() {
+        let roundToTwoDecimals = (num) => Math.round(num * 100) / 100;
+        if (Math.abs(this.start1 - this.start2) < 0.0001) {
+            return `([${roundToTwoDecimals(this.start1 / MAX_U32)}, ${roundToTwoDecimals(this.end1 / MAX_U32)}])`;
+        }
+        return `([${roundToTwoDecimals(this.start1 / MAX_U32)}, ${roundToTwoDecimals(this.end1 / MAX_U32)}] [${roundToTwoDecimals(this.start2 / MAX_U32)}, ${roundToTwoDecimals(this.end2 / MAX_U32)}])`;
+    }
+    toStringDetailed() {
+        return `(hash ${this.hash} range: ${this.toString()})`;
+    }
+}
+__decorate([
+    id({ type: Uint8Array }),
+    __metadata("design:type", Uint8Array)
+], ReplicationRangeIndexable.prototype, "id", void 0);
+__decorate([
+    field({ type: "string" }),
+    __metadata("design:type", String)
+], ReplicationRangeIndexable.prototype, "hash", void 0);
+__decorate([
+    field({ type: "u64" }),
+    __metadata("design:type", BigInt)
+], ReplicationRangeIndexable.prototype, "timestamp", void 0);
+__decorate([
+    field({ type: "u32" }),
+    __metadata("design:type", Number)
+], ReplicationRangeIndexable.prototype, "start1", void 0);
+__decorate([
+    field({ type: "u32" }),
+    __metadata("design:type", Number)
+], ReplicationRangeIndexable.prototype, "end1", void 0);
+__decorate([
+    field({ type: "u32" }),
+    __metadata("design:type", Number)
+], ReplicationRangeIndexable.prototype, "start2", void 0);
+__decorate([
+    field({ type: "u32" }),
+    __metadata("design:type", Number)
+], ReplicationRangeIndexable.prototype, "end2", void 0);
+__decorate([
+    field({ type: "u32" }),
+    __metadata("design:type", Number)
+], ReplicationRangeIndexable.prototype, "width", void 0);
+__decorate([
+    field({ type: "u8" }),
+    __metadata("design:type", Number)
+], ReplicationRangeIndexable.prototype, "mode", void 0);
 const containingPoint = (rects, point, roleAgeLimit, matured, now, options) => {
     // point is between 0 and 1, and the range can start at any offset between 0 and 1 and have length between 0 and 1
     let queries = [
@@ -37,19 +343,12 @@ const containingPoint = (rects, point, roleAgeLimit, matured, now, options) => {
             value: BigInt(now - roleAgeLimit),
         }),
     ];
-    return iterate(rects, new SearchRequest({
+    return rects.iterate({
         query: queries,
         sort: options?.sort,
-        fetch: 0xffffffff,
-    }));
-    /* const results = await rects.query(new SearchRequest({
-        query: queries,
-        sort: options?.sort,
-        fetch: 0xffffffff
-    }))
-    return results.results.map(x => x.value) */
+    }, options);
 };
-const getClosest = (direction, rects, point, roleAgeLimit, matured, now, includeStrict) => {
+const getClosest = (direction, rects, point, roleAgeLimit, matured, now, includeStrict, options) => {
     const createQueries = (p, equality) => {
         let queries;
         if (direction === "below") {
@@ -90,22 +389,32 @@ const getClosest = (direction, rects, point, roleAgeLimit, matured, now, include
         }
         return queries;
     };
-    const iterator = iterate(rects, new SearchRequest({
+    const sortByOldest = new Sort({ key: "timestamp", direction: "asc" });
+    const sortByHash = new Sort({ key: "hash", direction: "asc" }); // when breaking even
+    const iterator = rects.iterate({
         query: createQueries(point, false),
-        sort: direction === "below"
-            ? new Sort({ key: ["end2"], direction: "desc" })
-            : new Sort({ key: ["start1"], direction: "asc" }),
-    }));
-    const iteratorWrapped = iterate(rects, new SearchRequest({
+        sort: [
+            direction === "below"
+                ? new Sort({ key: ["end2"], direction: "desc" })
+                : new Sort({ key: ["start1"], direction: "asc" }),
+            sortByOldest,
+            sortByHash,
+        ],
+    }, options);
+    const iteratorWrapped = rects.iterate({
         query: createQueries(direction === "below" ? MAX_U32 : 0, true),
-        sort: direction === "below"
-            ? new Sort({ key: ["end2"], direction: "desc" })
-            : new Sort({ key: ["start1"], direction: "asc" }),
-    }));
+        sort: [
+            direction === "below"
+                ? new Sort({ key: ["end2"], direction: "desc" })
+                : new Sort({ key: ["start1"], direction: "asc" }),
+            sortByOldest,
+            sortByHash,
+        ],
+    }, options);
     return joinIterator([iterator, iteratorWrapped], point, direction);
 };
 export const hasCoveringRange = async (rects, range) => {
-    return ((await rects.count(new SearchRequest({
+    return ((await rects.count({
         query: [
             new Or([
                 new And([
@@ -169,7 +478,7 @@ export const hasCoveringRange = async (rects, range) => {
                 value: range.id,
             })),
         ],
-    }))) > 0);
+    })) > 0);
 };
 export const getDistance = (from, to, direction, end = MAX_U32) => {
     // if direction is 'above' only measure distance from 'from to 'to' from above.
@@ -202,21 +511,17 @@ const joinIterator = (iterators, point, direction) => {
     let queues = [];
     return {
         next: async (count) => {
-            let results = {
-                kept: 0, // TODO
-                results: [],
-            };
+            let results = [];
             for (let i = 0; i < iterators.length; i++) {
                 let queue = queues[i];
                 if (!queue) {
-                    queue = { elements: [], kept: 0 };
+                    queue = { elements: [] };
                     queues[i] = queue;
                 }
                 let iterator = iterators[i];
-                if (queue.elements.length < count && iterator.done() === false) {
+                if (queue.elements.length < count && iterator.done() !== true) {
                     let res = await iterator.next(count);
-                    queue.kept = res.kept;
-                    for (const el of res.results) {
+                    for (const el of res) {
                         const closest = el.value;
                         let dist;
                         if (direction === "closest") {
@@ -254,15 +559,16 @@ const joinIterator = (iterators, point, direction) => {
                 }
                 let closest = queues[closestQueue]?.elements.shift();
                 if (closest) {
-                    results.results.push(closest.result);
+                    results.push(closest.result);
                 }
             }
-            for (let i = 0; i < queues.length; i++) {
-                results.kept += queues[i].elements.length + queues[i].kept;
-            }
             return results;
         },
-        done: () => iterators.every((x) => x.done()),
+        pending: async () => {
+            let allPending = await Promise.all(iterators.map((x) => x.pending()));
+            return allPending.reduce((acc, x) => acc + x, 0);
+        },
+        done: () => iterators.every((x) => x.done() === true),
         close: async () => {
             for (const iterator of iterators) {
                 await iterator.close();
@@ -278,71 +584,80 @@ const joinIterator = (iterators, point, direction) => {
         },
     };
 };
-const getClosestAround = (peers, point, roleAge, now, includeStrictBelow, includeStrictAbove) => {
-    const closestBelow = getClosest("below", peers, point, roleAge, true, now, includeStrictBelow);
-    const closestAbove = getClosest("above", peers, point, roleAge, true, now, includeStrictAbove);
-    const containing = containingPoint(peers, point, roleAge, true, now);
+const getClosestAround = (peers, point, roleAge, now, includeStrictBelow, includeStrictAbove, options) => {
+    const closestBelow = getClosest("below", peers, point, roleAge, true, now, includeStrictBelow, options);
+    const closestAbove = getClosest("above", peers, point, roleAge, true, now, includeStrictAbove, options);
+    const containing = containingPoint(peers, point, roleAge, true, now, options);
     return iteratorInSeries(containing, joinIterator([closestBelow, closestAbove], point, "closest"));
 };
 const collectNodesAroundPoint = async (roleAge, peers, collector, point, now, done = () => true) => {
-    const containing = containingPoint(peers, point, 0, true, now);
-    const allContaining = await containing.next(0xffffffff);
-    for (const rect of allContaining.results) {
-        collector(rect.value, isMatured(rect.value, now, roleAge));
+    /* let shape = { timestamp: true, hash: true } as const */
+    const containing = containingPoint(peers, point, 0, true, now /* , { shape } */);
+    const allContaining = await containing.all();
+    for (const rect of allContaining) {
+        collector(rect.value, isMatured(rect.value, now, roleAge), true);
     }
     if (done()) {
         return;
     }
-    const closestBelow = getClosest("below", peers, point, 0, true, now, false);
-    const closestAbove = getClosest("above", peers, point, 0, true, now, false);
+    const closestBelow = getClosest("below", peers, point, 0, true, now, false /* , { shape } */);
+    const closestAbove = getClosest("above", peers, point, 0, true, now, false /* , { shape } */);
     const aroundIterator = joinIterator([closestBelow, closestAbove], point, "closest");
-    while (aroundIterator.done() === false && done() === false) {
+    while (aroundIterator.done() !== true && done() !== true) {
         const res = await aroundIterator.next(1);
-        for (const rect of res.results) {
-            collector(rect.value, isMatured(rect.value, now, roleAge));
+        for (const rect of res) {
+            collector(rect.value, isMatured(rect.value, now, roleAge), false);
             if (done()) {
                 return;
             }
         }
     }
 };
+export const getEvenlySpacedU32 = (from, count) => {
+    let ret = new Array(count);
+    for (let i = 0; i < count; i++) {
+        ret[i] = Math.round(from + (i * MAX_U32) / count) % MAX_U32;
+    }
+    return ret;
+};
 export const isMatured = (segment, now, minAge) => {
     return now - Number(segment.timestamp) >= minAge;
 };
-export const getSamples = async (cursor, peers, amount, roleAge) => {
-    const leaders = new Set();
+// get peer sample that are responsible for the cursor point
+// will return a list of peers that want to replicate the data,
+// but also if necessary a list of peers that are responsible for the data
+// but have not explicitly replicating a range that cover the cursor point
+export const getSamples = async (cursor, peers, roleAge) => {
+    const leaders = new Map();
     if (!peers) {
-        return [];
-    }
-    const size = await peers.getSize();
-    amount = Math.min(amount, size);
-    if (amount === 0) {
-        return [];
+        return new Map();
     }
     const now = +new Date();
     const maturedLeaders = new Set();
-    for (let i = 0; i < amount; i++) {
+    for (let i = 0; i < cursor.length; i++) {
         // evenly distributed
-        const point = Math.round(cursor + (i * MAX_U32) / amount) % MAX_U32;
         // aquire at least one unique node for each point
-        await collectNodesAroundPoint(roleAge, peers, (rect, m) => {
+        await collectNodesAroundPoint(roleAge, peers, (rect, m, intersecting) => {
             if (m) {
                 maturedLeaders.add(rect.hash);
             }
-            leaders.add(rect.hash);
-        }, point, now, () => {
+            const prev = leaders.get(rect.hash);
+            if (!prev || (intersecting && !prev.intersecting)) {
+                leaders.set(rect.hash, { intersecting });
+            }
+        }, cursor[i], now, () => {
             if (maturedLeaders.size > i) {
                 return true;
             }
             return false;
         });
     }
-    return [...leaders];
+    return leaders;
 };
 const fetchOne = async (iterator) => {
     const value = await iterator.next(1);
     await iterator.close();
-    return value.results[0]?.value;
+    return value[0]?.value;
 };
 export const minimumWidthToCover = async (minReplicas /* , replicatorCount: number */) => {
     /* minReplicas = Math.min(minReplicas, replicatorCount); */ // TODO do we need this?
@@ -365,10 +680,9 @@ export const getCoverSet = async (properties) => {
     if (properties.eager) {
         const eagerFetch = properties.eager === true
             ? 1000
-            : properties.eager.unmaturedFetchCoverSize;
+            : (properties.eager.unmaturedFetchCoverSize ?? 1000);
         // pull all umatured
-        const rects = await peers.query(new SearchRequest({
-            fetch: eagerFetch,
+        const iterator = peers.iterate({
             query: [
                 new IntegerCompare({
                     key: "timestamp",
@@ -376,8 +690,10 @@ export const getCoverSet = async (properties) => {
                     value: BigInt(now - roleAge),
                 }),
             ],
-        }));
-        for (const rect of rects.results) {
+        });
+        const rects = await iterator.next(eagerFetch);
+        await iterator.close();
+        for (const rect of rects) {
             ret.add(rect.value.hash);
         }
     }
@@ -467,15 +783,114 @@ export const getCoverSet = async (properties) => {
     start instanceof PublicSignKey && ret.add(start.hashcode());
     return ret;
 };
-export const fetchOneFromPublicKey = async (publicKey, index, roleAge, now) => {
-    let result = await index.query(new SearchRequest({
+/* export const getReplicationDiff = (changes: ReplicationChange) => {
+    // reduce the change set to only regions that are changed for each peer
+    // i.e. subtract removed regions from added regions, and vice versa
+    const result = new Map<string, { range: ReplicationRangeIndexable, added: boolean }[]>();
+
+    for (const addedChange of changes.added ?? []) {
+        let prev = result.get(addedChange.hash) ?? [];
+        for (const [_hash, ranges] of result.entries()) {
+            for (const r of ranges) {
+
+            }
+        }
+    }
+}
+ */
+const matchRangeQuery = (range) => {
+    let ors = [];
+    ors.push(new And([
+        new IntegerCompare({
+            key: "coordinate",
+            compare: "gte",
+            value: range.start1,
+        }),
+        new IntegerCompare({
+            key: "coordinate",
+            compare: "lt",
+            value: range.end1,
+        }),
+    ]));
+    ors.push(new And([
+        new IntegerCompare({
+            key: "coordinate",
+            compare: "gte",
+            value: range.start2,
+        }),
+        new IntegerCompare({
+            key: "coordinate",
+            compare: "lt",
+            value: range.end2,
+        }),
+    ]));
+    return new Or(ors);
+};
+export const toRebalance = (changes, index) => {
+    const assignedRangesQuery = (changes) => {
+        let ors = [];
+        for (const change of changes) {
+            const matchRange = matchRangeQuery(change.range);
+            if (change.type === "updated") {
+                // assuming a range is to be removed, is this entry still enoughly replicated
+                const prevMatchRange = matchRangeQuery(change.prev);
+                ors.push(prevMatchRange);
+                ors.push(matchRange);
+                /* 	ors.push(
+                        new And([
+                            // not sufficiently replicated
+                            new IntegerCompare({
+                                key: "replicatorsMissingShifted",
+                                compare: Compare.Greater,
+                                value: HALF_MAX_U32 - 1, // + 1 since we are going to remove a replicator for this entry
+                            }),
+    
+                            // is not the current range
+                            new Not(matchRange),
+    
+                            // but was in the the previous range
+                            prevMatchRange,
+                        ]),
+                    ); */
+            }
+            else {
+                ors.push(matchRange);
+            }
+        }
+        // entry is assigned to a range boundary, meaning it is due to be inspected
+        ors.push(new BoolQuery({
+            key: "assignedToRangeBoundary",
+            value: true,
+        }));
+        // entry is not sufficiently replicated, and we are to still keep it
+        return new Or(ors);
+    };
+    return {
+        [Symbol.asyncIterator]: async function* () {
+            const iterator = index.iterate({
+                query: assignedRangesQuery(changes),
+            });
+            while (iterator.done() !== true) {
+                const entries = await iterator.next(1000); // TODO choose right batch sizes here for optimal memory usage / speed
+                // TODO do we need this
+                const grouped = await groupByGidSync(entries.map((x) => x.value));
+                for (const [gid, entries] of grouped.entries()) {
+                    yield { gid, entries };
+                }
+            }
+        },
+    };
+};
+export const fetchOneFromPublicKey = async (publicKey, index, roleAge, now, options) => {
+    let iterator = index.iterate({
         query: [new StringMatch({ key: "hash", value: publicKey.hashcode() })],
-        fetch: 1,
-    }));
-    let node = result.results[0]?.value;
+    }, options);
+    let result = await iterator.next(1);
+    await iterator.close();
+    let node = result[0]?.value;
     if (node) {
         if (!isMatured(node, now, roleAge)) {
-            const matured = await fetchOne(getClosestAround(index, node.start1, roleAge, now, false, false));
+            const matured = await fetchOne(getClosestAround(index, node.start1, roleAge, now, false, false, options));
             if (matured) {
                 node = matured;
             }
@@ -483,17 +898,17 @@ export const fetchOneFromPublicKey = async (publicKey, index, roleAge, now) => {
     }
     return node;
 };
-export const getStartAndEnd = async (peers, start, widthToCoverScaled, roleAge, now, intervalWidth) => {
+export const getStartAndEnd = async (peers, start, widthToCoverScaled, roleAge, now, intervalWidth, options) => {
     // find a good starting point
     let startNode = undefined;
     let startLocation = undefined;
     const nodeFromPoint = async (point = scaleToU32(Math.random())) => {
         startLocation = point;
-        startNode = await fetchOneClosest(peers, startLocation, roleAge, now, false, true);
+        startNode = await fetchOneClosest(peers, startLocation, roleAge, now, false, true, options);
     };
     if (start instanceof PublicSignKey) {
         // start at our node (local first)
-        startNode = await fetchOneFromPublicKey(start, peers, roleAge, now);
+        startNode = await fetchOneFromPublicKey(start, peers, roleAge, now, options);
         if (!startNode) {
             // fetch randomly
             await nodeFromPoint();
@@ -533,7 +948,7 @@ export const getStartAndEnd = async (peers, start, widthToCoverScaled, roleAge,
         endLocation: Math.round(endLocation),
     };
 };
-export const fetchOneClosest = (peers, point, roleAge, now, includeStrictBelow, includeStrictAbove) => {
-    return fetchOne(getClosestAround(peers, point, roleAge, now, includeStrictBelow, includeStrictAbove));
+export const fetchOneClosest = (peers, point, roleAge, now, includeStrictBelow, includeStrictAbove, options) => {
+    return fetchOne(getClosestAround(peers, point, roleAge, now, includeStrictBelow, includeStrictAbove, options));
 };
 //# sourceMappingURL=ranges.js.map