@peerbit/shared-log 9.1.2 → 9.2.0

package/src/replication-domain-hash.ts

@@ -1,6 +1,7 @@
 import { BinaryReader, BinaryWriter } from "@dao-xyz/borsh";
 import { sha256 } from "@peerbit/crypto";
 import type { ShallowOrFullEntry } from "@peerbit/log";
+import type { EntryReplicated } from "./ranges.js";
 import {
 	type Log,
 	type ReplicationDomain,
@@ -15,7 +16,7 @@ export const hashToU32 = (hash: Uint8Array) => {
 };
 
 const hashTransformer: ReplicationDomainMapper<any> = async (
-	entry: ShallowOrFullEntry<any>,
+	entry: ShallowOrFullEntry<any> | EntryReplicated,
 ) => {
 	// For a fixed set or members, the choosen leaders will always be the same (address invariant)
 	// This allows for that same content is always chosen to be distributed to same peers, to remove unecessary copies
@@ -29,6 +30,7 @@ const hashTransformer: ReplicationDomainMapper<any> = async (
 	// convert hash of slot to a number
 	return hashToU32(seed);
 };
+
 export type ReplicationDomainHash = ReplicationDomain<undefined, any>;
 export const createReplicationDomainHash: () => ReplicationDomainHash = () => {
 	return {

package/src/replication-domain-time.ts

@@ -1,4 +1,5 @@
 import type { ShallowOrFullEntry } from "@peerbit/log";
+import type { EntryReplicated } from "./ranges.js";
 import {
 	type ReplicationDomain,
 	type ReplicationDomainMapper,
@@ -27,7 +28,7 @@ export const fromEntry = (
 	const scalar = scalarNanoToUnit[unit];
 	const originTime = +origin / scalarMilliToUnit[unit];
 
-	const fn = (entry: ShallowOrFullEntry<any>) => {
+	const fn = (entry: ShallowOrFullEntry<any> | EntryReplicated) => {
 		const cursor = entry.meta.clock.timestamp.wallTime / scalar;
 		return Math.round(Number(cursor) - originTime);
 	};

package/src/replication-domain.ts

@@ -1,15 +1,14 @@
 import type { PublicSignKey } from "@peerbit/crypto";
 import { type Index } from "@peerbit/indexer-interface";
 import type { Entry, ShallowEntry } from "@peerbit/log";
-import type {
-	ReplicationLimits,
-	ReplicationRangeIndexable,
-} from "./replication.js";
+import { debounceAcculmulator } from "./debounce.js";
+import type { EntryReplicated, ReplicationRangeIndexable } from "./ranges.js";
+import type { ReplicationLimits } from "./replication.js";
 import { MAX_U32 } from "./role.js";
 
 export type u32 = number;
 export type ReplicationDomainMapper<T> = (
-	entry: Entry<T> | ShallowEntry,
+	entry: Entry<T> | ShallowEntry | EntryReplicated,
 ) => Promise<u32> | u32;
 
 export type Log = {
@@ -33,6 +32,64 @@ type CoverRange = {
 	offset: number | PublicSignKey;
 	length?: number;
 };
+export type ReplicationChanges = ReplicationChange[];
+export type ReplicationChange =
+	| {
+			type: "added";
+			range: ReplicationRangeIndexable;
+	  }
+	| {
+			type: "removed";
+			range: ReplicationRangeIndexable;
+	  }
+	| {
+			type: "updated";
+			range: ReplicationRangeIndexable;
+			prev: ReplicationRangeIndexable;
+	  };
+
+export const mergeReplicationChanges = (
+	changes: ReplicationChanges | ReplicationChanges[],
+): ReplicationChanges => {
+	let first = changes[0];
+	if (!Array.isArray(first)) {
+		return changes as ReplicationChanges;
+	}
+	return (changes as ReplicationChanges[]).flat();
+};
+
+export const debounceAggregationChanges = (
+	fn: (changeOrChanges: ReplicationChange[]) => void,
+	delay: number,
+) => {
+	return debounceAcculmulator(
+		(result) => {
+			return fn([...result.values()]);
+		},
+		() => {
+			let aggregated: Map<string, ReplicationChange> = new Map();
+			return {
+				add: (change: ReplicationChange) => {
+					const prev = aggregated.get(change.range.idString);
+					if (prev) {
+						if (prev.range.timestamp < change.range.timestamp) {
+							aggregated.set(change.range.idString, change);
+						}
+					} else {
+						aggregated.set(change.range.idString, change);
+					}
+				},
+				delete: (key: string) => {
+					aggregated.delete(key);
+				},
+				size: () => aggregated.size,
+				value: aggregated,
+			};
+		},
+		delay,
+	);
+};
+
 export type ReplicationDomain<Args, T> = {
 	type: string;
 	fromEntry: ReplicationDomainMapper<T>;
@@ -40,6 +97,12 @@ export type ReplicationDomain<Args, T> = {
 		args: Args | undefined,
 		log: Log,
 	) => Promise<CoverRange> | CoverRange;
+
+	// to rebalance will return an async iterator of objects that will be added to the log
+	/* toRebalance(
+		change: ReplicationChange,
+		index: Index<EntryWithCoordinate>
+	): AsyncIterable<{ gid: string, entries: { coordinate: number, hash: string }[] }> | Promise<AsyncIterable<{ gid: string, entries: EntryWithCoordinate[] }>>; */
 };
 
 export const uniformToU32 = (cursor: number) => {

package/src/replication.ts

@@ -6,244 +6,18 @@ import {
 	variant,
 	vec,
 } from "@dao-xyz/borsh";
-import { PublicSignKey, equals, randomBytes } from "@peerbit/crypto";
-import { type Index, id } from "@peerbit/indexer-interface";
+import { randomBytes } from "@peerbit/crypto";
+import { type Index } from "@peerbit/indexer-interface";
 import { TransportMessage } from "./message.js";
-import { MAX_U32, Observer, Replicator, Role, scaleToU32 } from "./role.js";
+import {
+	ReplicationIntent,
+	ReplicationRange,
+	type ReplicationRangeIndexable,
+} from "./ranges.js";
+import { Observer, Replicator, Role } from "./role.js";
 
 export type ReplicationLimits = { min: MinReplicas; max?: MinReplicas };
 
-export enum ReplicationIntent {
-	NonStrict = 0, // indicates that the segment will be replicated and nearby data might be replicated as well
-	Strict = 1, // only replicate data in the segment to the specified replicator, not any other data
-}
-
-export const getSegmentsFromOffsetAndRange = (
-	offset: number,
-	factor: number,
-): [[number, number], [number, number]] => {
-	let start1 = offset;
-	let end1Unscaled = offset + factor; // only add factor if it is not 1 to prevent numerical issues (like (0.9 + 1) % 1 => 0.8999999)
-	let end1 = Math.min(end1Unscaled, MAX_U32);
-	return [
-		[start1, end1],
-		end1Unscaled > MAX_U32
-			? [0, (factor !== MAX_U32 ? offset + factor : offset) % MAX_U32]
-			: [start1, end1],
-	];
-};
-
-@variant(0)
-export class ReplicationRange {
-	@field({ type: Uint8Array })
-	private id: Uint8Array;
-
-	@field({ type: "u64" })
-	timestamp: bigint;
-
-	@field({ type: "u32" })
-	private _offset: number;
-
-	@field({ type: "u32" })
-	private _factor: number;
-
-	@field({ type: "u8" })
-	mode: ReplicationIntent;
-
-	constructor(properties: {
-		id: Uint8Array;
-		offset: number;
-		factor: number;
-		timestamp: bigint;
-		mode: ReplicationIntent;
-	}) {
-		const { id, offset, factor, timestamp, mode } = properties;
-		this.id = id;
-		this._offset = offset;
-		this._factor = factor;
-		this.timestamp = timestamp;
-		this.mode = mode;
-	}
-
-	get factor(): number {
-		return this._factor;
-	}
-
-	get offset(): number {
-		return this._offset;
-	}
-
-	toReplicationRangeIndexable(key: PublicSignKey): ReplicationRangeIndexable {
-		return new ReplicationRangeIndexable({
-			id: this.id,
-			publicKeyHash: key.hashcode(),
-			offset: this.offset,
-			length: this.factor,
-			timestamp: this.timestamp,
-			mode: this.mode,
-		});
-	}
-}
-
-export class ReplicationRangeIndexable {
-	@id({ type: Uint8Array })
-	id: Uint8Array;
-
-	@field({ type: "string" })
-	hash: string;
-
-	@field({ type: "u64" })
-	timestamp: bigint;
-
-	@field({ type: "u32" })
-	start1: number;
-
-	@field({ type: "u32" })
-	end1: number;
-
-	@field({ type: "u32" })
-	start2: number;
-
-	@field({ type: "u32" })
-	end2: number;
-
-	@field({ type: "u32" })
-	width: number;
-
-	@field({ type: "u8" })
-	mode: ReplicationIntent;
-
-	constructor(
-		properties: {
-			id?: Uint8Array;
-			normalized?: boolean;
-			offset: number;
-			length: number;
-			mode?: ReplicationIntent;
-			timestamp?: bigint;
-		} & ({ publicKeyHash: string } | { publicKey: PublicSignKey }),
-	) {
-		this.id = properties.id ?? randomBytes(32);
-		this.hash =
-			(properties as { publicKeyHash: string }).publicKeyHash ||
-			(properties as { publicKey: PublicSignKey }).publicKey.hashcode();
-		if (!properties.normalized) {
-			this.transform({ length: properties.length, offset: properties.offset });
-		} else {
-			this.transform({
-				length: scaleToU32(properties.length),
-				offset: scaleToU32(properties.offset),
-			});
-		}
-
-		this.mode = properties.mode ?? ReplicationIntent.NonStrict;
-		this.timestamp = properties.timestamp || BigInt(0);
-	}
-
-	private transform(properties: { offset: number; length: number }) {
-		const ranges = getSegmentsFromOffsetAndRange(
-			properties.offset,
-			properties.length,
-		);
-		this.start1 = Math.round(ranges[0][0]);
-		this.end1 = Math.round(ranges[0][1]);
-		this.start2 = Math.round(ranges[1][0]);
-		this.end2 = Math.round(ranges[1][1]);
-
-		this.width =
-			this.end1 -
-			this.start1 +
-			(this.end2 < this.end1 ? this.end2 - this.start2 : 0);
-
-		if (
-			this.start1 > 0xffffffff ||
-			this.end1 > 0xffffffff ||
-			this.start2 > 0xffffffff ||
-			this.end2 > 0xffffffff ||
-			this.width > 0xffffffff ||
-			this.width < 0
-		) {
-			throw new Error("Segment coordinate out of bounds");
-		}
-	}
-
-	contains(point: number) {
-		return (
-			(point >= this.start1 && point < this.end1) ||
-			(point >= this.start2 && point < this.end2)
-		);
-	}
-
-	overlaps(other: ReplicationRangeIndexable, checkOther = true): boolean {
-		if (
-			this.contains(other.start1) ||
-			this.contains(other.start2) ||
-			this.contains(other.end1) ||
-			this.contains(other.end2)
-		) {
-			return true;
-		}
-
-		if (checkOther) {
-			return other.overlaps(this, false);
-		}
-		return false;
-	}
-	toReplicationRange() {
-		return new ReplicationRange({
-			id: this.id,
-			offset: this.start1,
-			factor: this.width,
-			timestamp: this.timestamp,
-			mode: this.mode,
-		});
-	}
-
-	distanceTo(point: number) {
-		let wrappedPoint = MAX_U32 - point;
-		return Math.min(
-			Math.abs(this.start1 - point),
-			Math.abs(this.end2 - point),
-			Math.abs(this.start1 - wrappedPoint),
-			Math.abs(this.end2 - wrappedPoint),
-		);
-	}
-	get wrapped() {
-		return this.end2 < this.end1;
-	}
-
-	get widthNormalized() {
-		return this.width / MAX_U32;
-	}
-
-	equals(other: ReplicationRangeIndexable) {
-		if (
-			equals(this.id, other.id) &&
-			this.hash === other.hash &&
-			this.timestamp === other.timestamp &&
-			this.mode === other.mode &&
-			this.start1 === other.start1 &&
-			this.end1 === other.end1 &&
-			this.start2 === other.start2 &&
-			this.end2 === other.end2 &&
-			this.width === other.width
-		) {
-			return true;
-		}
-
-		return false;
-	}
-
-	toString() {
-		let roundToTwoDecimals = (num: number) => Math.round(num * 100) / 100;
-
-		if (Math.abs(this.start1 - this.start2) < 0.0001) {
-			return `([${roundToTwoDecimals(this.start1 / MAX_U32)}, ${roundToTwoDecimals(this.end1 / MAX_U32)}])`;
-		}
-		return `([${roundToTwoDecimals(this.start1 / MAX_U32)}, ${roundToTwoDecimals(this.end1 / MAX_U32)}] [${roundToTwoDecimals(this.start2 / MAX_U32)}, ${roundToTwoDecimals(this.end2 / MAX_U32)}])`;
-	}
-}
-
 interface SharedLog {
 	replicas: Partial<ReplicationLimits>;
 	replicationIndex: Index<ReplicationRangeIndexable> | undefined;
@@ -343,7 +117,7 @@ export class StoppedReplicating extends TransportMessage {
 @variant(1)
 export class RelativeMinReplicas extends MinReplicas {
 	_value: number; // (0, 1]
-
+	
 	constructor(value: number) {
 		super();
 		this._value = value;

package/src/role.ts

@@ -6,6 +6,7 @@
 import { field, variant, vec } from "@dao-xyz/borsh";
 
 export const MAX_U32 = 4294967295;
+export const HALF_MAX_U32 = 2147483647; // rounded down
 export const scaleToU32 = (value: number) => Math.round(MAX_U32 * value);
 
 export const overlaps = (x1: number, x2: number, y1: number, y2: number) => {

package/src/utils.ts

@@ -0,0 +1,49 @@
+import { Entry, ShallowEntry } from "@peerbit/log";
+import type { EntryWithRefs } from "./exchange-heads.js";
+import { EntryReplicated } from "./ranges.js";
+
+export const groupByGid = async <
+	T extends ShallowEntry | Entry<any> | EntryWithRefs<any> | EntryReplicated,
+>(
+	entries: T[],
+): Promise<Map<string, T[]>> => {
+	const groupByGid: Map<string, T[]> = new Map();
+	for (const head of entries) {
+		const gid =
+			head instanceof Entry
+				? (await head.getMeta()).gid
+				: head instanceof ShallowEntry
+					? head.meta.gid
+					: head instanceof EntryReplicated
+						? head.gid
+						: (await head.entry.getMeta()).gid;
+		let value = groupByGid.get(gid);
+		if (!value) {
+			value = [];
+			groupByGid.set(gid, value);
+		}
+		value.push(head);
+	}
+	return groupByGid;
+};
+
+export const groupByGidSync = async <T extends ShallowEntry | EntryReplicated>(
+	entries: T[],
+): Promise<Map<string, T[]>> => {
+	const groupByGid: Map<string, T[]> = new Map();
+	for (const head of entries) {
+		const gid =
+			head instanceof Entry
+				? (await head.getMeta()).gid
+				: head instanceof ShallowEntry
+					? head.meta.gid
+					: head.gid;
+		let value = groupByGid.get(gid);
+		if (!value) {
+			value = [];
+			groupByGid.set(gid, value);
+		}
+		value.push(head);
+	}
+	return groupByGid;
+};