@peerbit/shared-log 9.0.10 → 9.1.0

package/src/replication-domain-hash.ts

@@ -0,0 +1,43 @@
+import { BinaryReader, BinaryWriter } from "@dao-xyz/borsh";
+import { sha256 } from "@peerbit/crypto";
+import type { ShallowOrFullEntry } from "@peerbit/log";
+import {
+	type Log,
+	type ReplicationDomain,
+	type ReplicationDomainMapper,
+} from "./replication-domain.js";
+
+export const hashToU32 = (hash: Uint8Array) => {
+	const seedNumber = new BinaryReader(
+		hash.subarray(hash.length - 4, hash.length),
+	).u32();
+	return seedNumber;
+};
+
+const hashTransformer: ReplicationDomainMapper<any> = async (
+	entry: ShallowOrFullEntry<any>,
+) => {
+	// For a fixed set or members, the choosen leaders will always be the same (address invariant)
+	// This allows for that same content is always chosen to be distributed to same peers, to remove unecessary copies
+
+	// Convert this thing we wan't to distribute to 8 bytes so we get can convert it into a u64
+	// modulus into an index
+	const utf8writer = new BinaryWriter();
+	utf8writer.string(entry.meta.gid);
+	const seed = await sha256(utf8writer.finalize());
+
+	// convert hash of slot to a number
+	return hashToU32(seed);
+};
+export type ReplicationDomainHash = ReplicationDomain<undefined, any>;
+export const createReplicationDomainHash: () => ReplicationDomainHash = () => {
+	return {
+		type: "hash",
+		fromEntry: hashTransformer,
+		fromArgs: async (args: undefined, log: Log) => {
+			return {
+				offset: log.node.identity.publicKey,
+			};
+		},
+	};
+};

package/src/replication-domain-time.ts

@@ -0,0 +1,85 @@
+import type { ShallowOrFullEntry } from "@peerbit/log";
+import {
+	type ReplicationDomain,
+	type ReplicationDomainMapper,
+	type u32,
+} from "./replication-domain.js";
+
+type TimeUnit = "seconds" | "milliseconds" | "microseconds" | "nanoseconds";
+
+const scalarNanoToUnit = {
+	seconds: BigInt(1e9),
+	milliseconds: BigInt(1e6),
+	microseconds: BigInt(1e3),
+	nanoseconds: BigInt(1),
+};
+const scalarMilliToUnit = {
+	seconds: 1e3,
+	milliseconds: 1,
+	microseconds: 1e-3,
+	nanoseconds: 1e-6,
+};
+
+export const fromEntry = (
+	origin: Date,
+	unit: TimeUnit = "milliseconds",
+): ReplicationDomainMapper<any> => {
+	const scalar = scalarNanoToUnit[unit];
+	const originTime = +origin / scalarMilliToUnit[unit];
+
+	const fn = (entry: ShallowOrFullEntry<any>) => {
+		const cursor = entry.meta.clock.timestamp.wallTime / scalar;
+		return Math.round(Number(cursor) - originTime);
+	};
+	return fn;
+};
+
+type TimeRange = { from: number; to: number };
+
+export type ReplicationDomainTime = ReplicationDomain<TimeRange, any> & {
+	fromTime: (time: number | Date) => u32;
+	fromDuration: (duration: number) => u32;
+};
+
+export const createReplicationDomainTime = (
+	origin: Date,
+	unit: TimeUnit = "milliseconds",
+): ReplicationDomainTime => {
+	const originScaled = +origin * scalarMilliToUnit[unit];
+	const fromMilliToUnit = scalarMilliToUnit[unit];
+	const fromTime = (time: number | Date): u32 => {
+		return (
+			(typeof time === "number" ? time : +time * fromMilliToUnit) - originScaled
+		);
+	};
+
+	const fromDuration = (duration: number): u32 => {
+		return duration;
+	};
+	return {
+		type: "time",
+		fromTime,
+		fromDuration,
+		fromEntry: fromEntry(origin, unit),
+		fromArgs: async (args: TimeRange | undefined, log) => {
+			if (!args) {
+				return {
+					offset: log.node.identity.publicKey,
+				};
+			}
+			return {
+				offset: fromTime(args.from),
+				length: fromDuration(args.to - args.from),
+			};
+			/* 	roleAge = roleAge ?? (await log.getDefaultMinRoleAge());
+				const ranges = await getCoverSet(
+					log.replicationIndex,
+					roleAge,
+					fromTime(args.from),
+					fromDuration(args.to - args.from),
+					undefined,
+				);
+				return [...ranges]; */
+		},
+	};
+};

package/src/replication-domain.ts

@@ -0,0 +1,50 @@
+import type { PublicSignKey } from "@peerbit/crypto";
+import { type Index } from "@peerbit/indexer-interface";
+import type { Entry, ShallowEntry } from "@peerbit/log";
+import type {
+	ReplicationLimits,
+	ReplicationRangeIndexable,
+} from "./replication.js";
+import { MAX_U32 } from "./role.js";
+
+export type u32 = number;
+export type ReplicationDomainMapper<T> = (
+	entry: Entry<T> | ShallowEntry,
+) => Promise<u32> | u32;
+
+export type Log = {
+	replicas: ReplicationLimits;
+	node: {
+		identity: {
+			publicKey: PublicSignKey;
+		};
+	};
+	syncInFlight: Map<string, Map<string, { timestamp: number }>>;
+	replicationIndex: Index<ReplicationRangeIndexable>;
+	getDefaultMinRoleAge: () => Promise<number>;
+};
+export type ReplicationDomainCoverSet<Args> = (
+	log: Log,
+	roleAge: number | undefined,
+	args: Args,
+) => Promise<string[]> | string[]; // minimum set of peers that covers all the data
+
+type CoverRange = {
+	offset: number | PublicSignKey;
+	length?: number;
+};
+export type ReplicationDomain<Args, T> = {
+	type: string;
+	fromEntry: ReplicationDomainMapper<T>;
+	fromArgs: (
+		args: Args | undefined,
+		log: Log,
+	) => Promise<CoverRange> | CoverRange;
+};
+
+export const uniformToU32 = (cursor: number) => {
+	return cursor * MAX_U32;
+};
+
+export type ExtractDomainArgs<T> =
+	T extends ReplicationDomain<infer Args, any> ? Args : never;

package/src/replication.ts

@@ -1,5 +1,4 @@
 import {
-	BinaryReader,
 	deserialize,
 	field,
 	option,
@@ -10,18 +9,13 @@ import {
 import { PublicSignKey, equals, randomBytes } from "@peerbit/crypto";
 import { type Index, id } from "@peerbit/indexer-interface";
 import { TransportMessage } from "./message.js";
-import {
-	Observer,
-	Replicator,
-	Role,
-	SEGMENT_COORDINATE_SCALE,
-} from "./role.js";
+import { MAX_U32, Observer, Replicator, Role, scaleToU32 } from "./role.js";
 
 export type ReplicationLimits = { min: MinReplicas; max?: MinReplicas };
 
 export enum ReplicationIntent {
-	Explicit = 0,
-	Automatic = 1,
+	NonStrict = 0, // indicates that the segment will be replicated and nearby data might be replicated as well
+	Strict = 1, // only replicate data in the segment to the specified replicator, not any other data
 }
 
 export const getSegmentsFromOffsetAndRange = (
@@ -30,15 +24,16 @@ export const getSegmentsFromOffsetAndRange = (
 ): [[number, number], [number, number]] => {
 	let start1 = offset;
 	let end1Unscaled = offset + factor; // only add factor if it is not 1 to prevent numerical issues (like (0.9 + 1) % 1 => 0.8999999)
-	let end1 = Math.min(end1Unscaled, 1);
+	let end1 = Math.min(end1Unscaled, MAX_U32);
 	return [
 		[start1, end1],
-		end1Unscaled > 1
-			? [0, (factor !== 1 ? offset + factor : offset) % 1]
+		end1Unscaled > MAX_U32
+			? [0, (factor !== MAX_U32 ? offset + factor : offset) % MAX_U32]
 			: [start1, end1],
 	];
 };
 
+@variant(0)
 export class ReplicationRange {
 	@field({ type: Uint8Array })
 	private id: Uint8Array;
@@ -52,25 +47,30 @@ export class ReplicationRange {
 	@field({ type: "u32" })
 	private _factor: number;
 
+	@field({ type: "u8" })
+	mode: ReplicationIntent;
+
 	constructor(properties: {
 		id: Uint8Array;
 		offset: number;
 		factor: number;
 		timestamp: bigint;
+		mode: ReplicationIntent;
 	}) {
-		const { id, offset, factor, timestamp } = properties;
+		const { id, offset, factor, timestamp, mode } = properties;
 		this.id = id;
-		this._offset = Math.round(offset * SEGMENT_COORDINATE_SCALE);
-		this._factor = Math.round(factor * SEGMENT_COORDINATE_SCALE);
+		this._offset = offset;
+		this._factor = factor;
 		this.timestamp = timestamp;
+		this.mode = mode;
 	}
 
 	get factor(): number {
-		return this._factor / SEGMENT_COORDINATE_SCALE;
+		return this._factor;
 	}
 
 	get offset(): number {
-		return this._offset / SEGMENT_COORDINATE_SCALE;
+		return this._offset;
 	}
 
 	toReplicationRangeIndexable(key: PublicSignKey): ReplicationRangeIndexable {
@@ -80,6 +80,7 @@ export class ReplicationRange {
 			offset: this.offset,
 			length: this.factor,
 			timestamp: this.timestamp,
+			mode: this.mode,
 		});
 	}
 }
@@ -110,15 +111,15 @@ export class ReplicationRangeIndexable {
 	width: number;
 
 	@field({ type: "u8" })
-	replicationIntent: ReplicationIntent;
+	mode: ReplicationIntent;
 
 	constructor(
 		properties: {
 			id?: Uint8Array;
-
+			normalized?: boolean;
 			offset: number;
 			length: number;
-			replicationIntent?: ReplicationIntent;
+			mode?: ReplicationIntent;
 			timestamp?: bigint;
 		} & ({ publicKeyHash: string } | { publicKey: PublicSignKey }),
 	) {
@@ -126,9 +127,16 @@ export class ReplicationRangeIndexable {
 		this.hash =
 			(properties as { publicKeyHash: string }).publicKeyHash ||
 			(properties as { publicKey: PublicSignKey }).publicKey.hashcode();
-		this.transform({ length: properties.length, offset: properties.offset });
-		this.replicationIntent =
-			properties.replicationIntent ?? ReplicationIntent.Explicit;
+		if (!properties.normalized) {
+			this.transform({ length: properties.length, offset: properties.offset });
+		} else {
+			this.transform({
+				length: scaleToU32(properties.length),
+				offset: scaleToU32(properties.offset),
+			});
+		}
+
+		this.mode = properties.mode ?? ReplicationIntent.NonStrict;
 		this.timestamp = properties.timestamp || BigInt(0);
 	}
 
@@ -137,10 +145,10 @@ export class ReplicationRangeIndexable {
 			properties.offset,
 			properties.length,
 		);
-		this.start1 = Math.round(ranges[0][0] * SEGMENT_COORDINATE_SCALE);
-		this.end1 = Math.round(ranges[0][1] * SEGMENT_COORDINATE_SCALE);
-		this.start2 = Math.round(ranges[1][0] * SEGMENT_COORDINATE_SCALE);
-		this.end2 = Math.round(ranges[1][1] * SEGMENT_COORDINATE_SCALE);
+		this.start1 = Math.round(ranges[0][0]);
+		this.end1 = Math.round(ranges[0][1]);
+		this.start2 = Math.round(ranges[1][0]);
+		this.end2 = Math.round(ranges[1][1]);
 
 		this.width =
 			this.end1 -
@@ -152,7 +160,8 @@ export class ReplicationRangeIndexable {
 			this.end1 > 0xffffffff ||
 			this.start2 > 0xffffffff ||
 			this.end2 > 0xffffffff ||
-			this.width > 0xffffffff
+			this.width > 0xffffffff ||
+			this.width < 0
 		) {
 			throw new Error("Segment coordinate out of bounds");
 		}
@@ -183,14 +192,15 @@ export class ReplicationRangeIndexable {
 	toReplicationRange() {
 		return new ReplicationRange({
 			id: this.id,
-			offset: this.start1 / SEGMENT_COORDINATE_SCALE,
-			factor: this.width / SEGMENT_COORDINATE_SCALE,
+			offset: this.start1,
+			factor: this.width,
 			timestamp: this.timestamp,
+			mode: this.mode,
 		});
 	}
 
 	distanceTo(point: number) {
-		let wrappedPoint = SEGMENT_COORDINATE_SCALE - point;
+		let wrappedPoint = MAX_U32 - point;
 		return Math.min(
 			Math.abs(this.start1 - point),
 			Math.abs(this.end2 - point),
@@ -203,7 +213,7 @@ export class ReplicationRangeIndexable {
 	}
 
 	get widthNormalized() {
-		return this.width / SEGMENT_COORDINATE_SCALE;
+		return this.width / MAX_U32;
 	}
 
 	equals(other: ReplicationRangeIndexable) {
@@ -211,7 +221,7 @@ export class ReplicationRangeIndexable {
 			equals(this.id, other.id) &&
 			this.hash === other.hash &&
 			this.timestamp === other.timestamp &&
-			this.replicationIntent === other.replicationIntent &&
+			this.mode === other.mode &&
 			this.start1 === other.start1 &&
 			this.end1 === other.end1 &&
 			this.start2 === other.start2 &&
@@ -228,9 +238,9 @@ export class ReplicationRangeIndexable {
 		let roundToTwoDecimals = (num: number) => Math.round(num * 100) / 100;
 
 		if (Math.abs(this.start1 - this.start2) < 0.0001) {
-			return `([${roundToTwoDecimals(this.start1 / SEGMENT_COORDINATE_SCALE)}, ${roundToTwoDecimals(this.end1 / SEGMENT_COORDINATE_SCALE)}])`;
+			return `([${roundToTwoDecimals(this.start1 / MAX_U32)}, ${roundToTwoDecimals(this.end1 / MAX_U32)}])`;
 		}
-		return `([${roundToTwoDecimals(this.start1 / SEGMENT_COORDINATE_SCALE)}, ${roundToTwoDecimals(this.end1 / SEGMENT_COORDINATE_SCALE)}] [${roundToTwoDecimals(this.start2 / SEGMENT_COORDINATE_SCALE)}, ${roundToTwoDecimals(this.end2 / SEGMENT_COORDINATE_SCALE)}])`;
+		return `([${roundToTwoDecimals(this.start1 / MAX_U32)}, ${roundToTwoDecimals(this.end1 / MAX_U32)}] [${roundToTwoDecimals(this.start2 / MAX_U32)}, ${roundToTwoDecimals(this.end2 / MAX_U32)}])`;
 	}
 }
 
@@ -278,8 +288,8 @@ export class ResponseRoleMessage extends TransportMessage {
 		this.role = properties.role;
 	}
 
-	toReplicationInfoMessage(): ResponseReplicationInfoMessage {
-		return new ResponseReplicationInfoMessage({
+	toReplicationInfoMessage(): AllReplicatingSegmentsMessage {
+		return new AllReplicatingSegmentsMessage({
 			segments:
 				this.role instanceof Replicator
 					? this.role.segments.map((x) => {
@@ -288,6 +298,7 @@ export class ResponseRoleMessage extends TransportMessage {
 								offset: x.offset,
 								factor: x.factor,
 								timestamp: x.timestamp,
+								mode: ReplicationIntent.NonStrict,
 							});
 						})
 					: [],
@@ -296,7 +307,7 @@ export class ResponseRoleMessage extends TransportMessage {
 }
 
 @variant([1, 2])
-export class ResponseReplicationInfoMessage extends TransportMessage {
+export class AllReplicatingSegmentsMessage extends TransportMessage {
 	@field({ type: vec(ReplicationRange) })
 	segments: ReplicationRange[];
 
@@ -307,7 +318,7 @@ export class ResponseReplicationInfoMessage extends TransportMessage {
 }
 
 @variant([1, 3])
-export class StartedReplicating extends TransportMessage {
+export class AddedReplicationSegmentMessage extends TransportMessage {
 	@field({ type: vec(ReplicationRange) })
 	segments: ReplicationRange[];
 
@@ -376,10 +387,3 @@ export const maxReplicas = (
 	const numberOfLeaders = Math.max(Math.min(higher, max), lower);
 	return numberOfLeaders;
 };
-
-export const hashToUniformNumber = (hash: Uint8Array) => {
-	const seedNumber = new BinaryReader(
-		hash.subarray(hash.length - 4, hash.length),
-	).u32();
-	return seedNumber / 0xffffffff; // bounded between 0 and 1
-};

package/src/role.ts

@@ -5,7 +5,8 @@
  */
 import { field, variant, vec } from "@dao-xyz/borsh";
 
-export const SEGMENT_COORDINATE_SCALE = 4294967295;
+export const MAX_U32 = 4294967295;
+export const scaleToU32 = (value: number) => Math.round(MAX_U32 * value);
 
 export const overlaps = (x1: number, x2: number, y1: number, y2: number) => {
 	if (x1 <= y2 && y1 <= x2) {
@@ -59,20 +60,20 @@ export class RoleReplicationSegment {
 		}
 
 		this.timestamp = timestamp ?? BigInt(+new Date());
-		this.factorNominator = Math.round(SEGMENT_COORDINATE_SCALE * factor);
+		this.factorNominator = Math.round(MAX_U32 * factor);
 
 		if (offset > 1 || offset < 0) {
 			throw new Error("Expecting offset to be between 0 and 1, got: " + offset);
 		}
-		this.offsetNominator = Math.round(SEGMENT_COORDINATE_SCALE * offset);
+		this.offsetNominator = Math.round(MAX_U32 * offset);
 	}
 
 	get factor(): number {
-		return this.factorNominator / SEGMENT_COORDINATE_SCALE;
+		return this.factorNominator / MAX_U32;
 	}
 
 	get offset(): number {
-		return this.offsetNominator / SEGMENT_COORDINATE_SCALE;
+		return this.offsetNominator / MAX_U32;
 	}
 }