@peerbit/shared-log 9.1.2 → 9.2.0

package/src/ranges.ts

@@ -1,6 +1,8 @@
-import { PublicSignKey, equals } from "@peerbit/crypto";
+import { deserialize, field, serialize, variant } from "@dao-xyz/borsh";
+import { PublicSignKey, equals, randomBytes, toBase64 } from "@peerbit/crypto";
 import {
 	And,
+	BoolQuery,
 	ByteMatchQuery,
 	Compare,
 	type Index,
@@ -11,30 +13,416 @@ import {
 	Not,
 	Or,
 	type Query,
-	SearchRequest,
+	type ReturnTypeFromShape,
+	type Shape,
 	Sort,
 	SortDirection,
 	StringMatch,
-	iterate,
 	iteratorInSeries,
 } from "@peerbit/indexer-interface";
-import type { u32 } from "./replication-domain.js";
-import {
-	ReplicationIntent,
-	type ReplicationRangeIndexable,
-} from "./replication.js";
+import { id } from "@peerbit/indexer-interface";
+import { Meta, ShallowMeta } from "@peerbit/log";
+import { type ReplicationChanges, type u32 } from "./replication-domain.js";
 import { MAX_U32, scaleToU32 } from "./role.js";
+import { groupByGidSync } from "./utils.js";
+
+export enum ReplicationIntent {
+	NonStrict = 0, // indicates that the segment will be replicated and nearby data might be replicated as well
+	Strict = 1, // only replicate data in the segment to the specified replicator, not any other data
+}
+
+export const getSegmentsFromOffsetAndRange = (
+	offset: number,
+	factor: number,
+): [[number, number], [number, number]] => {
+	let start1 = offset;
+	let end1Unscaled = offset + factor; // only add factor if it is not 1 to prevent numerical issues (like (0.9 + 1) % 1 => 0.8999999)
+	let end1 = Math.min(end1Unscaled, MAX_U32);
+	return [
+		[start1, end1],
+		end1Unscaled > MAX_U32
+			? [0, (factor !== MAX_U32 ? offset + factor : offset) % MAX_U32]
+			: [start1, end1],
+	];
+};
+
+export const shouldAssigneToRangeBoundary = (
+	leaders:
+		| Map<
+				string,
+				{
+					intersecting: boolean;
+				}
+		  >
+		| false,
+) => {
+	let assignedToRangeBoundary = leaders === false || leaders.size === 0;
+	if (!assignedToRangeBoundary && leaders) {
+		for (const [_, { intersecting }] of leaders) {
+			if (!intersecting) {
+				assignedToRangeBoundary = true;
+				break;
+			}
+		}
+	}
+	return assignedToRangeBoundary;
+};
+export class EntryReplicated {
+	@id({ type: "string" })
+	id: string; // hash + coordinate
+
+	@field({ type: "string" })
+	hash: string;
+
+	@field({ type: "string" })
+	gid: string;
+
+	@field({ type: "u32" })
+	coordinate: number;
+
+	@field({ type: "u64" })
+	wallTime: bigint;
+
+	@field({ type: "bool" })
+	assignedToRangeBoundary: boolean;
+
+	@field({ type: Uint8Array })
+	private _meta: Uint8Array;
+
+	private _metaResolved: ShallowMeta;
+
+	constructor(properties: {
+		coordinate: number;
+		hash: string;
+		meta: Meta;
+		assignedToRangeBoundary: boolean;
+	}) {
+		this.coordinate = properties.coordinate;
+		this.hash = properties.hash;
+		this.gid = properties.meta.gid;
+		this.id = this.hash + "-" + this.coordinate;
+		this.wallTime = properties.meta.clock.timestamp.wallTime;
+		const shallow =
+			properties.meta instanceof Meta
+				? new ShallowMeta(properties.meta)
+				: properties.meta;
+		this._meta = serialize(shallow);
+		this._metaResolved = deserialize(this._meta, ShallowMeta);
+		this._metaResolved = properties.meta;
+		this.assignedToRangeBoundary = properties.assignedToRangeBoundary;
+	}
+
+	get meta(): ShallowMeta {
+		if (!this._metaResolved) {
+			this._metaResolved = deserialize(this._meta, ShallowMeta);
+		}
+		return this._metaResolved;
+	}
+}
+
+@variant(0)
+export class ReplicationRange {
+	@field({ type: Uint8Array })
+	id: Uint8Array;
+
+	@field({ type: "u64" })
+	timestamp: bigint;
+
+	@field({ type: "u32" })
+	private _offset: number;
+
+	@field({ type: "u32" })
+	private _factor: number;
+
+	@field({ type: "u8" })
+	mode: ReplicationIntent;
+
+	constructor(properties: {
+		id: Uint8Array;
+		offset: number;
+		factor: number;
+		timestamp: bigint;
+		mode: ReplicationIntent;
+	}) {
+		const { id, offset, factor, timestamp, mode } = properties;
+		this.id = id;
+		this._offset = offset;
+		this._factor = factor;
+		this.timestamp = timestamp;
+		this.mode = mode;
+	}
+
+	get factor(): number {
+		return this._factor;
+	}
+
+	get offset(): number {
+		return this._offset;
+	}
+
+	toReplicationRangeIndexable(key: PublicSignKey): ReplicationRangeIndexable {
+		return new ReplicationRangeIndexable({
+			id: this.id,
+			publicKeyHash: key.hashcode(),
+			offset: this.offset,
+			length: this.factor,
+			timestamp: this.timestamp,
+			mode: this.mode,
+		});
+	}
+}
+
+export class ReplicationRangeIndexable {
+	@id({ type: Uint8Array })
+	id: Uint8Array;
+
+	@field({ type: "string" })
+	hash: string;
+
+	@field({ type: "u64" })
+	timestamp: bigint;
+
+	@field({ type: "u32" })
+	start1!: number;
+
+	@field({ type: "u32" })
+	end1!: number;
+
+	@field({ type: "u32" })
+	start2!: number;
+
+	@field({ type: "u32" })
+	end2!: number;
+
+	@field({ type: "u32" })
+	width!: number;
+
+	@field({ type: "u8" })
+	mode: ReplicationIntent;
 
-const containingPoint = (
+	constructor(
+		properties: {
+			id?: Uint8Array;
+			normalized?: boolean;
+			offset: number;
+			length: number;
+			mode?: ReplicationIntent;
+			timestamp?: bigint;
+		} & ({ publicKeyHash: string } | { publicKey: PublicSignKey }),
+	) {
+		this.id = properties.id ?? randomBytes(32);
+		this.hash =
+			(properties as { publicKeyHash: string }).publicKeyHash ||
+			(properties as { publicKey: PublicSignKey }).publicKey.hashcode();
+		if (!properties.normalized) {
+			this.transform({ length: properties.length, offset: properties.offset });
+		} else {
+			this.transform({
+				length: scaleToU32(properties.length),
+				offset: scaleToU32(properties.offset),
+			});
+		}
+
+		this.mode = properties.mode ?? ReplicationIntent.NonStrict;
+		this.timestamp = properties.timestamp || BigInt(0);
+	}
+
+	private transform(properties: { offset: number; length: number }) {
+		const ranges = getSegmentsFromOffsetAndRange(
+			properties.offset,
+			properties.length,
+		);
+		this.start1 = Math.round(ranges[0][0]);
+		this.end1 = Math.round(ranges[0][1]);
+		this.start2 = Math.round(ranges[1][0]);
+		this.end2 = Math.round(ranges[1][1]);
+
+		this.width =
+			this.end1 -
+			this.start1 +
+			(this.end2 < this.end1 ? this.end2 - this.start2 : 0);
+
+		if (
+			this.start1 > 0xffffffff ||
+			this.end1 > 0xffffffff ||
+			this.start2 > 0xffffffff ||
+			this.end2 > 0xffffffff ||
+			this.width > 0xffffffff ||
+			this.width < 0
+		) {
+			throw new Error("Segment coordinate out of bounds");
+		}
+	}
+
+	get idString() {
+		return toBase64(this.id);
+	}
+
+	contains(point: number) {
+		return (
+			(point >= this.start1 && point < this.end1) ||
+			(point >= this.start2 && point < this.end2)
+		);
+	}
+
+	overlaps(other: ReplicationRangeIndexable, checkOther = true): boolean {
+		if (
+			this.contains(other.start1) ||
+			this.contains(other.start2) ||
+			this.contains(other.end1 - 1) ||
+			this.contains(other.end2 - 1)
+		) {
+			return true;
+		}
+
+		if (checkOther) {
+			return other.overlaps(this, false);
+		}
+		return false;
+	}
+	toReplicationRange() {
+		return new ReplicationRange({
+			id: this.id,
+			offset: this.start1,
+			factor: this.width,
+			timestamp: this.timestamp,
+			mode: this.mode,
+		});
+	}
+
+	distanceTo(point: number) {
+		let wrappedPoint = MAX_U32 - point;
+		return Math.min(
+			Math.abs(this.start1 - point),
+			Math.abs(this.end2 - point),
+			Math.abs(this.start1 - wrappedPoint),
+			Math.abs(this.end2 - wrappedPoint),
+		);
+	}
+	get wrapped() {
+		return this.end2 < this.end1;
+	}
+
+	get widthNormalized() {
+		return this.width / MAX_U32;
+	}
+
+	equals(other: ReplicationRangeIndexable) {
+		if (
+			equals(this.id, other.id) &&
+			this.hash === other.hash &&
+			this.timestamp === other.timestamp &&
+			this.mode === other.mode &&
+			this.start1 === other.start1 &&
+			this.end1 === other.end1 &&
+			this.start2 === other.start2 &&
+			this.end2 === other.end2 &&
+			this.width === other.width
+		) {
+			return true;
+		}
+
+		return false;
+	}
+
+	equalRange(other: ReplicationRangeIndexable) {
+		return (
+			this.start1 === other.start1 &&
+			this.end1 === other.end1 &&
+			this.start2 === other.start2 &&
+			this.end2 === other.end2
+		);
+	}
+
+	toString() {
+		let roundToTwoDecimals = (num: number) => Math.round(num * 100) / 100;
+
+		if (Math.abs(this.start1 - this.start2) < 0.0001) {
+			return `([${roundToTwoDecimals(this.start1 / MAX_U32)}, ${roundToTwoDecimals(this.end1 / MAX_U32)}])`;
+		}
+		return `([${roundToTwoDecimals(this.start1 / MAX_U32)}, ${roundToTwoDecimals(this.end1 / MAX_U32)}] [${roundToTwoDecimals(this.start2 / MAX_U32)}, ${roundToTwoDecimals(this.end2 / MAX_U32)}])`;
+	}
+
+	toStringDetailed() {
+		return `(hash ${this.hash} range: ${this.toString()})`;
+	}
+
+	/* removeRange(other: ReplicationRangeIndexable): ReplicationRangeIndexable | ReplicationRangeIndexable[] {
+		if (!this.overlaps(other)) {
+			return this
+		}
+
+		if (this.equalRange(other)) {
+			return []
+		}
+
+		let diff: ReplicationRangeIndexable[] = [];
+		let start1 = this.start1;
+		if (other.start1 > start1) {
+			diff.push(new ReplicationRangeIndexable({
+				id: this.id,
+				offset: this.start1,
+				length: other.start1 - this.start1,
+				mode: this.mode,
+				publicKeyHash: this.hash,
+				timestamp: this.timestamp,
+				normalized: false
+			}));
+
+			start1 = other.end2
+		}
+
+		if (other.end1 < this.end1) {
+			diff.push(new ReplicationRangeIndexable({
+				id: this.id,
+				offset: other.end1,
+				length: this.end1 - other.end1,
+				mode: this.mode,
+				publicKeyHash: this.hash,
+				timestamp: this.timestamp,
+				normalized: false
+			}));
+		}
+
+		if (other.start2 > this.start2) {
+			diff.push(new ReplicationRangeIndexable({
+				id: this.id,
+				offset: this.start2,
+				length: other.start2 - this.start2,
+				mode: this.mode,
+				publicKeyHash: this.hash,
+				timestamp: this.timestamp,
+				normalized: false
+			}));
+		}
+
+		if (other.end2 < this.end2) {
+			diff.push(new ReplicationRangeIndexable({
+				id: this.id,
+				offset: other.end2,
+				length: this.end2 - other.end2,
+				mode: this.mode,
+				publicKeyHash: this.hash,
+				timestamp: this.timestamp,
+				normalized: false
+			}));
+		}
+
+		return diff;
+	} */
+}
+
+const containingPoint = <S extends Shape | undefined = undefined>(
 	rects: Index<ReplicationRangeIndexable>,
 	point: number,
 	roleAgeLimit: number,
 	matured: boolean,
 	now: number,
 	options?: {
+		shape?: S;
 		sort?: Sort[];
 	},
-): IndexIterator<ReplicationRangeIndexable> => {
+): IndexIterator<ReplicationRangeIndexable, S> => {
 	// point is between 0 and 1, and the range can start at any offset between 0 and 1 and have length between 0 and 1
 
 	let queries = [
@@ -70,13 +458,12 @@ const containingPoint = (
 			value: BigInt(now - roleAgeLimit),
 		}),
 	];
-	return iterate(
-		rects,
-		new SearchRequest({
+	return rects.iterate(
+		{
 			query: queries,
 			sort: options?.sort,
-			fetch: 0xffffffff,
-		}),
+		},
+		options,
 	);
 	/* const results = await rects.query(new SearchRequest({
 		query: queries,
@@ -86,7 +473,7 @@ const containingPoint = (
 	return results.results.map(x => x.value) */
 };
 
-const getClosest = (
+const getClosest = <S extends Shape | undefined = undefined>(
 	direction: "above" | "below",
 	rects: Index<ReplicationRangeIndexable>,
 	point: number,
@@ -94,7 +481,8 @@ const getClosest = (
 	matured: boolean,
 	now: number,
 	includeStrict: boolean,
-): IndexIterator<ReplicationRangeIndexable> => {
+	options?: { shape?: S },
+): IndexIterator<ReplicationRangeIndexable, S> => {
 	const createQueries = (p: number, equality: boolean) => {
 		let queries: Query[];
 		if (direction === "below") {
@@ -140,28 +528,38 @@ const getClosest = (
 		return queries;
 	};
 
-	const iterator = iterate(
-		rects,
-		new SearchRequest({
+	const sortByOldest = new Sort({ key: "timestamp", direction: "asc" });
+	const sortByHash = new Sort({ key: "hash", direction: "asc" }); // when breaking even
+
+	const iterator = rects.iterate(
+		{
 			query: createQueries(point, false),
-			sort:
+			sort: [
 				direction === "below"
 					? new Sort({ key: ["end2"], direction: "desc" })
 					: new Sort({ key: ["start1"], direction: "asc" }),
-		}),
+				sortByOldest,
+				sortByHash,
+			],
+		},
+		options,
 	);
-	const iteratorWrapped = iterate(
-		rects,
-		new SearchRequest({
+
+	const iteratorWrapped = rects.iterate(
+		{
 			query: createQueries(direction === "below" ? MAX_U32 : 0, true),
-			sort:
+			sort: [
 				direction === "below"
 					? new Sort({ key: ["end2"], direction: "desc" })
 					: new Sort({ key: ["start1"], direction: "asc" }),
-		}),
+				sortByOldest,
+				sortByHash,
+			],
+		},
+		options,
 	);
 
-	return joinIterator([iterator, iteratorWrapped], point, direction);
+	return joinIterator<S>([iterator, iteratorWrapped], point, direction);
 };
 
 export const hasCoveringRange = async (
@@ -169,75 +567,73 @@ export const hasCoveringRange = async (
 	range: ReplicationRangeIndexable,
 ) => {
 	return (
-		(await rects.count(
-			new SearchRequest({
-				query: [
-					new Or([
-						new And([
-							new IntegerCompare({
-								key: "start1",
-								compare: Compare.LessOrEqual,
-								value: range.start1,
-							}),
-							new IntegerCompare({
-								key: "end1",
-								compare: Compare.GreaterOrEqual,
-								value: range.end1,
-							}),
-						]),
-						new And([
-							new IntegerCompare({
-								key: "start2",
-								compare: Compare.LessOrEqual,
-								value: range.start1,
-							}),
-							new IntegerCompare({
-								key: "end2",
-								compare: Compare.GreaterOrEqual,
-								value: range.end1,
-							}),
-						]),
+		(await rects.count({
+			query: [
+				new Or([
+					new And([
+						new IntegerCompare({
+							key: "start1",
+							compare: Compare.LessOrEqual,
+							value: range.start1,
+						}),
+						new IntegerCompare({
+							key: "end1",
+							compare: Compare.GreaterOrEqual,
+							value: range.end1,
+						}),
 					]),
-					new Or([
-						new And([
-							new IntegerCompare({
-								key: "start1",
-								compare: Compare.LessOrEqual,
-								value: range.start2,
-							}),
-							new IntegerCompare({
-								key: "end1",
-								compare: Compare.GreaterOrEqual,
-								value: range.end2,
-							}),
-						]),
-						new And([
-							new IntegerCompare({
-								key: "start2",
-								compare: Compare.LessOrEqual,
-								value: range.start2,
-							}),
-							new IntegerCompare({
-								key: "end2",
-								compare: Compare.GreaterOrEqual,
-								value: range.end2,
-							}),
-						]),
+					new And([
+						new IntegerCompare({
+							key: "start2",
+							compare: Compare.LessOrEqual,
+							value: range.start1,
+						}),
+						new IntegerCompare({
+							key: "end2",
+							compare: Compare.GreaterOrEqual,
+							value: range.end1,
+						}),
 					]),
-					new StringMatch({
-						key: "hash",
-						value: range.hash,
-					}),
-					// assume that we are looking for other ranges, not want to update an existing one
-					new Not(
-						new ByteMatchQuery({
-							key: "id",
-							value: range.id,
+				]),
+				new Or([
+					new And([
+						new IntegerCompare({
+							key: "start1",
+							compare: Compare.LessOrEqual,
+							value: range.start2,
 						}),
-					),
-				],
-			}),
-		)) > 0
+						new IntegerCompare({
+							key: "end1",
+							compare: Compare.GreaterOrEqual,
+							value: range.end2,
+						}),
+					]),
+					new And([
+						new IntegerCompare({
+							key: "start2",
+							compare: Compare.LessOrEqual,
+							value: range.start2,
+						}),
+						new IntegerCompare({
+							key: "end2",
+							compare: Compare.GreaterOrEqual,
+							value: range.end2,
+						}),
+					]),
+				]),
+				new StringMatch({
+					key: "hash",
+					value: range.hash,
+				}),
+				// assume that we are looking for other ranges, not want to update an existing one
+				new Not(
+					new ByteMatchQuery({
+						key: "id",
+						value: range.id,
+					}),
+				),
+			],
+		})) > 0
 	);
 };
 
@@ -280,15 +676,14 @@ export const getDistance = (
 	throw new Error("Invalid direction");
 };
 
-const joinIterator = (
-	iterators: IndexIterator<ReplicationRangeIndexable>[],
+const joinIterator = <S extends Shape | undefined = undefined>(
+	iterators: IndexIterator<ReplicationRangeIndexable, S>[],
 	point: number,
 	direction: "above" | "below" | "closest",
-) => {
+): IndexIterator<ReplicationRangeIndexable, S> => {
 	let queues: {
-		kept: number;
 		elements: {
-			result: IndexedResult<ReplicationRangeIndexable>;
+			result: IndexedResult<ReturnTypeFromShape<ReplicationRangeIndexable, S>>;
 			dist: number;
 		}[];
 	}[] = [];
@@ -296,23 +691,23 @@ const joinIterator = (
 	return {
 		next: async (
 			count: number,
-		): Promise<IndexedResults<ReplicationRangeIndexable>> => {
-			let results: IndexedResults<ReplicationRangeIndexable> = {
-				kept: 0, // TODO
-				results: [],
-			};
+		): Promise<
+			IndexedResults<ReturnTypeFromShape<ReplicationRangeIndexable, S>>
+		> => {
+			let results: IndexedResults<
+				ReturnTypeFromShape<ReplicationRangeIndexable, S>
+			> = [];
 			for (let i = 0; i < iterators.length; i++) {
 				let queue = queues[i];
 				if (!queue) {
-					queue = { elements: [], kept: 0 };
+					queue = { elements: [] };
 					queues[i] = queue;
 				}
 				let iterator = iterators[i];
-				if (queue.elements.length < count && iterator.done() === false) {
+				if (queue.elements.length < count && iterator.done() !== true) {
 					let res = await iterator.next(count);
-					queue.kept = res.kept;
 
-					for (const el of res.results) {
+					for (const el of res) {
 						const closest = el.value;
 
 						let dist: number;
@@ -356,24 +751,25 @@ const joinIterator = (
 
 				let closest = queues[closestQueue]?.elements.shift();
 				if (closest) {
-					results.results.push(closest.result);
+					results.push(closest.result);
 				}
 			}
-
-			for (let i = 0; i < queues.length; i++) {
-				results.kept += queues[i].elements.length + queues[i].kept;
-			}
-
 			return results;
 		},
-		done: () => iterators.every((x) => x.done()),
+		pending: async () => {
+			let allPending = await Promise.all(iterators.map((x) => x.pending()));
+			return allPending.reduce((acc, x) => acc + x, 0);
+		},
+		done: () => iterators.every((x) => x.done() === true),
 		close: async () => {
 			for (const iterator of iterators) {
 				await iterator.close();
 			}
 		},
 		all: async () => {
-			let results: IndexedResult<ReplicationRangeIndexable>[] = [];
+			let results: IndexedResult<
+				ReturnTypeFromShape<ReplicationRangeIndexable, S>
+			>[] = [];
 			for (const iterator of iterators) {
 				let res = await iterator.all();
 				results.push(...res);
@@ -383,15 +779,18 @@ const joinIterator = (
 	};
 };
 
-const getClosestAround = (
+const getClosestAround = <
+	S extends (Shape & { timestamp: true }) | undefined = undefined,
+>(
 	peers: Index<ReplicationRangeIndexable>,
 	point: number,
 	roleAge: number,
 	now: number,
 	includeStrictBelow: boolean,
 	includeStrictAbove: boolean,
+	options?: { shape?: S },
 ) => {
-	const closestBelow = getClosest(
+	const closestBelow = getClosest<S>(
 		"below",
 		peers,
 		point,
@@ -399,8 +798,9 @@ const getClosestAround = (
 		true,
 		now,
 		includeStrictBelow,
+		options,
 	);
-	const closestAbove = getClosest(
+	const closestAbove = getClosest<S>(
 		"above",
 		peers,
 		point,
@@ -408,45 +808,79 @@ const getClosestAround = (
 		true,
 		now,
 		includeStrictAbove,
+		options,
+	);
+	const containing = containingPoint<S>(
+		peers,
+		point,
+		roleAge,
+		true,
+		now,
+		options,
 	);
-	const containing = containingPoint(peers, point, roleAge, true, now);
 
 	return iteratorInSeries(
 		containing,
-		joinIterator([closestBelow, closestAbove], point, "closest"),
+		joinIterator<S>([closestBelow, closestAbove], point, "closest"),
 	);
 };
 
 const collectNodesAroundPoint = async (
 	roleAge: number,
 	peers: Index<ReplicationRangeIndexable>,
-	collector: (rect: ReplicationRangeIndexable, matured: boolean) => void,
+	collector: (
+		rect: { hash: string },
+		matured: boolean,
+		interescting: boolean,
+	) => void,
 	point: u32,
 	now: number,
 	done: () => boolean = () => true,
 ) => {
-	const containing = containingPoint(peers, point, 0, true, now);
-
-	const allContaining = await containing.next(0xffffffff);
-	for (const rect of allContaining.results) {
-		collector(rect.value, isMatured(rect.value, now, roleAge));
+	/* let shape = { timestamp: true, hash: true } as const */
+	const containing = containingPoint(
+		peers,
+		point,
+		0,
+		true,
+		now /* , { shape } */,
+	);
+	const allContaining = await containing.all();
+	for (const rect of allContaining) {
+		collector(rect.value, isMatured(rect.value, now, roleAge), true);
 	}
 
 	if (done()) {
 		return;
 	}
 
-	const closestBelow = getClosest("below", peers, point, 0, true, now, false);
-	const closestAbove = getClosest("above", peers, point, 0, true, now, false);
+	const closestBelow = getClosest(
+		"below",
+		peers,
+		point,
+		0,
+		true,
+		now,
+		false /* , { shape } */,
+	);
+	const closestAbove = getClosest(
+		"above",
+		peers,
+		point,
+		0,
+		true,
+		now,
+		false /* , { shape } */,
+	);
 	const aroundIterator = joinIterator(
 		[closestBelow, closestAbove],
 		point,
 		"closest",
 	);
-	while (aroundIterator.done() === false && done() === false) {
+	while (aroundIterator.done() !== true && done() !== true) {
 		const res = await aroundIterator.next(1);
-		for (const rect of res.results) {
-			collector(rect.value, isMatured(rect.value, now, roleAge));
+		for (const rect of res) {
+			collector(rect.value, isMatured(rect.value, now, roleAge), false);
 			if (done()) {
 				return;
 			}
@@ -454,6 +888,14 @@ const collectNodesAroundPoint = async (
 	}
 };
 
+export const getEvenlySpacedU32 = (from: number, count: number) => {
+	let ret: number[] = new Array(count);
+	for (let i = 0; i < count; i++) {
+		ret[i] = Math.round(from + (i * MAX_U32) / count) % MAX_U32;
+	}
+	return ret;
+};
+
 export const isMatured = (
 	segment: { timestamp: bigint },
 	now: number,
@@ -461,44 +903,42 @@ export const isMatured = (
 ) => {
 	return now - Number(segment.timestamp) >= minAge;
 };
-
+// get peer sample that are responsible for the cursor point
+// will return a list of peers that want to replicate the data,
+// but also if necessary a list of peers that are responsible for the data
+// but have not explicitly replicating a range that cover the cursor point
 export const getSamples = async (
-	cursor: u32,
+	cursor: u32[],
 	peers: Index<ReplicationRangeIndexable>,
-	amount: number,
 	roleAge: number,
-) => {
-	const leaders: Set<string> = new Set();
+): Promise<Map<string, { intersecting: boolean }>> => {
+	const leaders: Map<string, { intersecting: boolean }> = new Map();
 	if (!peers) {
-		return [];
-	}
-
-	const size = await peers.getSize();
-
-	amount = Math.min(amount, size);
-
-	if (amount === 0) {
-		return [];
+		return new Map();
 	}
 
 	const now = +new Date();
 
 	const maturedLeaders = new Set();
-	for (let i = 0; i < amount; i++) {
+	for (let i = 0; i < cursor.length; i++) {
 		// evenly distributed
-		const point = Math.round(cursor + (i * MAX_U32) / amount) % MAX_U32;
 
 		// aquire at least one unique node for each point
 		await collectNodesAroundPoint(
 			roleAge,
 			peers,
-			(rect, m) => {
+			(rect, m, intersecting) => {
 				if (m) {
 					maturedLeaders.add(rect.hash);
 				}
-				leaders.add(rect.hash);
+
+				const prev = leaders.get(rect.hash);
+
+				if (!prev || (intersecting && !prev.intersecting)) {
+					leaders.set(rect.hash, { intersecting });
+				}
 			},
-			point,
+			cursor[i],
 			now,
 			() => {
 				if (maturedLeaders.size > i) {
@@ -509,13 +949,15 @@ export const getSamples = async (
 		);
 	}
 
-	return [...leaders];
+	return leaders;
 };
 
-const fetchOne = async (iterator: IndexIterator<ReplicationRangeIndexable>) => {
+const fetchOne = async <S extends Shape | undefined>(
+	iterator: IndexIterator<ReplicationRangeIndexable, S>,
+) => {
 	const value = await iterator.next(1);
 	await iterator.close();
-	return value.results[0]?.value;
+	return value[0]?.value;
 };
 
 export const minimumWidthToCover = async (
@@ -566,22 +1008,21 @@ export const getCoverSet = async (properties: {
 		const eagerFetch =
 			properties.eager === true
 				? 1000
-				: properties.eager.unmaturedFetchCoverSize;
+				: (properties.eager.unmaturedFetchCoverSize ?? 1000);
 
 		// pull all umatured
-		const rects = await peers.query(
-			new SearchRequest({
-				fetch: eagerFetch,
-				query: [
-					new IntegerCompare({
-						key: "timestamp",
-						compare: Compare.GreaterOrEqual,
-						value: BigInt(now - roleAge),
-					}),
-				],
-			}),
-		);
-		for (const rect of rects.results) {
+		const iterator = peers.iterate({
+			query: [
+				new IntegerCompare({
+					key: "timestamp",
+					compare: Compare.GreaterOrEqual,
+					value: BigInt(now - roleAge),
+				}),
+			],
+		});
+		const rects = await iterator.next(eagerFetch);
+		await iterator.close();
+		for (const rect of rects) {
 			ret.add(rect.value.hash);
 		}
 	}
@@ -713,24 +1154,177 @@ export const getCoverSet = async (properties: {
 	start instanceof PublicSignKey && ret.add(start.hashcode());
 	return ret;
 };
+/* export const getReplicationDiff = (changes: ReplicationChange) => {
+	// reduce the change set to only regions that are changed for each peer
+	// i.e. subtract removed regions from added regions, and vice versa
+	const result = new Map<string, { range: ReplicationRangeIndexable, added: boolean }[]>();
+
+	for (const addedChange of changes.added ?? []) {
+		let prev = result.get(addedChange.hash) ?? [];
+		for (const [_hash, ranges] of result.entries()) {
+			for (const r of ranges) {
+
+			}
+		}
+	}
+}
+ */
+
+const matchRangeQuery = (range: ReplicationRangeIndexable) => {
+	let ors = [];
+	ors.push(
+		new And([
+			new IntegerCompare({
+				key: "coordinate",
+				compare: "gte",
+				value: range.start1,
+			}),
+			new IntegerCompare({
+				key: "coordinate",
+				compare: "lt",
+				value: range.end1,
+			}),
+		]),
+	);
 
-export const fetchOneFromPublicKey = async (
+	ors.push(
+		new And([
+			new IntegerCompare({
+				key: "coordinate",
+				compare: "gte",
+				value: range.start2,
+			}),
+			new IntegerCompare({
+				key: "coordinate",
+				compare: "lt",
+				value: range.end2,
+			}),
+		]),
+	);
+
+	return new Or(ors);
+};
+export const toRebalance = (
+	changes: ReplicationChanges,
+	index: Index<EntryReplicated>,
+): AsyncIterable<{ gid: string; entries: EntryReplicated[] }> => {
+	const assignedRangesQuery = (changes: ReplicationChanges) => {
+		let ors: Query[] = [];
+		for (const change of changes) {
+			const matchRange = matchRangeQuery(change.range);
+			if (change.type === "updated") {
+				// assuming a range is to be removed, is this entry still enoughly replicated
+				const prevMatchRange = matchRangeQuery(change.prev);
+				ors.push(prevMatchRange);
+				ors.push(matchRange);
+
+				/* 	ors.push(
+						new And([
+							// not sufficiently replicated
+							new IntegerCompare({
+								key: "replicatorsMissingShifted",
+								compare: Compare.Greater,
+								value: HALF_MAX_U32 - 1, // + 1 since we are going to remove a replicator for this entry
+							}),
+	
+							// is not the current range
+							new Not(matchRange),
+	
+							// but was in the the previous range
+							prevMatchRange,
+						]),
+					); */
+			} else {
+				ors.push(matchRange);
+			}
+		}
+
+		// entry is assigned to a range boundary, meaning it is due to be inspected
+		ors.push(
+			new BoolQuery({
+				key: "assignedToRangeBoundary",
+				value: true,
+			}),
+		);
+
+		// entry is not sufficiently replicated, and we are to still keep it
+		return new Or(ors);
+	};
+
+	return {
+		[Symbol.asyncIterator]: async function* () {
+			const iterator = index.iterate({
+				query: assignedRangesQuery(changes),
+			});
+
+			/* 	const iteratorFitlered = index.iterate({
+					query: assignedRangesQuery(changes),
+				}); */
+
+			while (iterator.done() !== true) {
+				const entries = await iterator.next(0xffffffff); // TODO batch sizes
+				/* const entriesFiltered = await iteratorFitlered.next(0xffffffff); // TODO batch sizes
+
+				console.log(
+					"SIZE",
+					await index.getSize(),
+					entries.results.length,
+					entriesFiltered.results.length,
+					Math.min(
+						...entries.results
+							.filter((x) => x.value.assignedToRangeBoundary === false)
+							.map((x) => x.value.coordinate / 0xffffffff),
+					),
+					Math.max(
+						...entries.results
+							.filter((x) => x.value.assignedToRangeBoundary === false)
+							.map((x) => x.value.coordinate / 0xffffffff),
+					),
+				); */
+
+				// TODO do we need this
+				const grouped = await groupByGidSync(entries.map((x) => x.value));
+
+				for (const [gid, entries] of grouped.entries()) {
+					yield { gid, entries };
+				}
+			}
+		},
+	};
+};
+
+export const fetchOneFromPublicKey = async <
+	S extends (Shape & { timestamp: true }) | undefined = undefined,
+>(
 	publicKey: PublicSignKey,
 	index: Index<ReplicationRangeIndexable>,
 	roleAge: number,
 	now: number,
+	options?: {
+		shape: S;
+	},
 ) => {
-	let result = await index.query(
-		new SearchRequest({
+	let iterator = index.iterate<S>(
+		{
 			query: [new StringMatch({ key: "hash", value: publicKey.hashcode() })],
-			fetch: 1,
-		}),
+		},
+		options,
 	);
-	let node = result.results[0]?.value;
+	let result = await iterator.next(1);
+	await iterator.close();
+	let node = result[0]?.value;
 	if (node) {
 		if (!isMatured(node, now, roleAge)) {
 			const matured = await fetchOne(
-				getClosestAround(index, node.start1, roleAge, now, false, false),
+				getClosestAround<S>(
+					index,
+					node.start1,
+					roleAge,
+					now,
+					false,
+					false,
+					options,
+				),
 			);
 			if (matured) {
 				node = matured;
@@ -740,16 +1334,24 @@ export const fetchOneFromPublicKey = async (
 	return node;
 };
 
-export const getStartAndEnd = async (
+export const getStartAndEnd = async <
+	S extends (Shape & { timestamp: true }) | undefined,
+>(
 	peers: Index<ReplicationRangeIndexable>,
 	start: number | PublicSignKey | undefined | undefined,
 	widthToCoverScaled: number,
 	roleAge: number,
 	now: number,
 	intervalWidth: number,
-) => {
+	options?: { shape: S },
+): Promise<{
+	startNode: ReturnTypeFromShape<ReplicationRangeIndexable, S> | undefined;
+	startLocation: number;
+	endLocation: number;
+}> => {
 	// find a good starting point
-	let startNode: ReplicationRangeIndexable | undefined = undefined;
+	let startNode: ReturnTypeFromShape<ReplicationRangeIndexable, S> | undefined =
+		undefined;
 	let startLocation: number | undefined = undefined;
 
 	const nodeFromPoint = async (point = scaleToU32(Math.random())) => {
@@ -761,12 +1363,19 @@ export const getStartAndEnd = async (
 			now,
 			false,
 			true,
+			options,
 		);
 	};
 
 	if (start instanceof PublicSignKey) {
 		// start at our node (local first)
-		startNode = await fetchOneFromPublicKey(start, peers, roleAge, now);
+		startNode = await fetchOneFromPublicKey(
+			start,
+			peers,
+			roleAge,
+			now,
+			options,
+		);
 		if (!startNode) {
 			// fetch randomly
 			await nodeFromPoint();
@@ -810,22 +1419,26 @@ export const getStartAndEnd = async (
 	};
 };
 
-export const fetchOneClosest = (
+export const fetchOneClosest = <
+	S extends (Shape & { timestamp: true }) | undefined = undefined,
+>(
 	peers: Index<ReplicationRangeIndexable>,
 	point: number,
 	roleAge: number,
 	now: number,
 	includeStrictBelow: boolean,
 	includeStrictAbove: boolean,
+	options?: { shape?: S },
 ) => {
 	return fetchOne(
-		getClosestAround(
+		getClosestAround<S>(
 			peers,
 			point,
 			roleAge,
 			now,
 			includeStrictBelow,
 			includeStrictAbove,
+			options,
 		),
 	);
 };