@peerbit/shared-log 9.2.13 → 10.0.0-05f4bef

package/src/replication-domain-hash.ts

@@ -1,41 +1,66 @@
-import { BinaryReader, BinaryWriter } from "@dao-xyz/borsh";
+import { BinaryWriter } from "@dao-xyz/borsh";
 import { sha256 } from "@peerbit/crypto";
 import type { ShallowOrFullEntry } from "@peerbit/log";
-import type { EntryReplicated } from "./ranges.js";
+import { bytesToNumber } from "./integers.js";
+import { type EntryReplicated } from "./ranges.js";
 import {
 	type Log,
 	type ReplicationDomain,
 	type ReplicationDomainMapper,
 } from "./replication-domain.js";
 
-export const hashToU32 = (hash: Uint8Array) => {
+/* const hashToU32 = (hash: Uint8Array) => {
 	const seedNumber = new BinaryReader(
 		hash.subarray(hash.length - 4, hash.length),
 	).u32();
 	return seedNumber;
 };
 
-const hashTransformer: ReplicationDomainMapper<any> = async (
-	entry: ShallowOrFullEntry<any> | EntryReplicated,
-) => {
-	// For a fixed set or members, the choosen leaders will always be the same (address invariant)
-	// This allows for that same content is always chosen to be distributed to same peers, to remove unecessary copies
 
-	// Convert this thing we wan't to distribute to 8 bytes so we get can convert it into a u64
-	// modulus into an index
-	const utf8writer = new BinaryWriter();
-	utf8writer.string(entry.meta.gid);
-	const seed = await sha256(utf8writer.finalize());
+const hashToU64 = (hash: Uint8Array): bigint => {
+	const seedNumber = new BinaryReader(
+		hash.subarray(hash.length - 4, hash.length), //  
+	).u64();
+	return seedNumber;
+};
+ */
 
-	// convert hash of slot to a number
-	return hashToU32(seed);
+const hashTransformer = <R extends "u32" | "u64">(
+	resolution: R,
+): ReplicationDomainMapper<any, R> => {
+	const numberConverter = bytesToNumber(resolution);
+	if (resolution === "u32") {
+		return (async (entry: ShallowOrFullEntry<any> | EntryReplicated<R>) => {
+			const utf8writer = new BinaryWriter();
+			utf8writer.string(entry.meta.gid);
+			const seed = await sha256(utf8writer.finalize());
+			return numberConverter(seed);
+		}) as ReplicationDomainMapper<any, R>;
+	} else if (resolution === "u64") {
+		return (async (entry: ShallowOrFullEntry<any> | EntryReplicated<R>) => {
+			const utf8writer = new BinaryWriter();
+			utf8writer.string(entry.meta.gid);
+			const seed = await sha256(utf8writer.finalize());
+			return numberConverter(seed);
+		}) as ReplicationDomainMapper<any, R>;
+	} else {
+		throw new Error("Unsupported resolution");
+	}
 };
 
-export type ReplicationDomainHash = ReplicationDomain<undefined, any>;
-export const createReplicationDomainHash: () => ReplicationDomainHash = () => {
+export type ReplicationDomainHash<R extends "u32" | "u64"> = ReplicationDomain<
+	undefined,
+	any,
+	R
+>;
+
+export const createReplicationDomainHash = <R extends "u32" | "u64">(
+	resolution: R,
+): ReplicationDomainHash<R> => {
 	return {
+		resolution,
 		type: "hash",
-		fromEntry: hashTransformer,
+		fromEntry: hashTransformer<R>(resolution),
 		fromArgs: async (args: undefined, log: Log) => {
 			return {
 				offset: log.node.identity.publicKey,

package/src/replication-domain-time.ts

@@ -1,9 +1,8 @@
 import type { ShallowOrFullEntry } from "@peerbit/log";
-import type { EntryReplicated } from "./ranges.js";
+import { type EntryReplicated } from "./ranges.js";
 import {
 	type ReplicationDomain,
 	type ReplicationDomainMapper,
-	type u32,
 } from "./replication-domain.js";
 
 type TimeUnit = "seconds" | "milliseconds" | "microseconds" | "nanoseconds";
@@ -24,11 +23,11 @@ const scalarMilliToUnit = {
 export const fromEntry = (
 	origin: Date,
 	unit: TimeUnit = "milliseconds",
-): ReplicationDomainMapper<any> => {
+): ReplicationDomainMapper<any, "u32"> => {
 	const scalar = scalarNanoToUnit[unit];
 	const originTime = +origin / scalarMilliToUnit[unit];
 
-	const fn = (entry: ShallowOrFullEntry<any> | EntryReplicated) => {
+	const fn = (entry: ShallowOrFullEntry<any> | EntryReplicated<"u32">) => {
 		const cursor = entry.meta.clock.timestamp.wallTime / scalar;
 		return Math.round(Number(cursor) - originTime);
 	};
@@ -37,9 +36,9 @@ export const fromEntry = (
 
 type TimeRange = { from: number; to: number };
 
-export type ReplicationDomainTime = ReplicationDomain<TimeRange, any> & {
-	fromTime: (time: number | Date) => u32;
-	fromDuration: (duration: number) => u32;
+export type ReplicationDomainTime = ReplicationDomain<TimeRange, any, "u32"> & {
+	fromTime: (time: number | Date) => number;
+	fromDuration: (duration: number) => number;
 };
 
 export const createReplicationDomainTime = (
@@ -48,16 +47,17 @@ export const createReplicationDomainTime = (
 ): ReplicationDomainTime => {
 	const originScaled = +origin * scalarMilliToUnit[unit];
 	const fromMilliToUnit = scalarMilliToUnit[unit];
-	const fromTime = (time: number | Date): u32 => {
+	const fromTime = (time: number | Date): number => {
 		return (
 			(typeof time === "number" ? time : +time * fromMilliToUnit) - originScaled
 		);
 	};
 
-	const fromDuration = (duration: number): u32 => {
+	const fromDuration = (duration: number): number => {
 		return duration;
 	};
 	return {
+		resolution: "u32",
 		type: "time",
 		fromTime,
 		fromDuration,

package/src/replication-domain.ts

@@ -1,15 +1,15 @@
 import type { PublicSignKey } from "@peerbit/crypto";
 import { type Index } from "@peerbit/indexer-interface";
 import type { Entry, ShallowEntry } from "@peerbit/log";
-import { debounceAcculmulator } from "./debounce.js";
-import type { EntryReplicated, ReplicationRangeIndexable } from "./ranges.js";
+import { debounceAccumulator } from "./debounce.js";
+import type { ReplicationRangeIndexable } from "./index.js";
+import type { NumberFromType } from "./integers.js";
+import type { EntryReplicated } from "./ranges.js";
 import type { ReplicationLimits } from "./replication.js";
-import { MAX_U32 } from "./role.js";
 
-export type u32 = number;
-export type ReplicationDomainMapper<T> = (
-	entry: Entry<T> | ShallowEntry | EntryReplicated,
-) => Promise<u32> | u32;
+export type ReplicationDomainMapper<T, R extends "u32" | "u64"> = (
+	entry: Entry<T> | ShallowEntry | EntryReplicated<R>,
+) => Promise<NumberFromType<R>> | NumberFromType<R>;
 
 export type Log = {
 	replicas: ReplicationLimits;
@@ -18,8 +18,7 @@ export type Log = {
 			publicKey: PublicSignKey;
 		};
 	};
-	syncInFlight: Map<string, Map<string, { timestamp: number }>>;
-	replicationIndex: Index<ReplicationRangeIndexable>;
+	replicationIndex: Index<ReplicationRangeIndexable<any>>;
 	getDefaultMinRoleAge: () => Promise<number>;
 };
 export type ReplicationDomainCoverSet<Args> = (
@@ -28,24 +27,24 @@ export type ReplicationDomainCoverSet<Args> = (
 	args: Args,
 ) => Promise<string[]> | string[]; // minimum set of peers that covers all the data
 
-type CoverRange = {
-	offset: number | PublicSignKey;
-	length?: number;
+type CoverRange<T extends number | bigint> = {
+	offset: T | PublicSignKey;
+	length?: T;
 };
 export type ReplicationChanges = ReplicationChange[];
 export type ReplicationChange =
 	| {
 			type: "added";
-			range: ReplicationRangeIndexable;
+			range: ReplicationRangeIndexable<any>;
 	  }
 	| {
 			type: "removed";
-			range: ReplicationRangeIndexable;
+			range: ReplicationRangeIndexable<any>;
 	  }
 	| {
 			type: "updated";
-			range: ReplicationRangeIndexable;
-			prev: ReplicationRangeIndexable;
+			range: ReplicationRangeIndexable<any>;
+			prev: ReplicationRangeIndexable<any>;
 	  };
 
 export const mergeReplicationChanges = (
@@ -62,7 +61,7 @@ export const debounceAggregationChanges = (
 	fn: (changeOrChanges: ReplicationChange[]) => void,
 	delay: number,
 ) => {
-	return debounceAcculmulator(
+	return debounceAccumulator(
 		(result) => {
 			return fn([...result.values()]);
 		},
@@ -90,24 +89,15 @@ export const debounceAggregationChanges = (
 	);
 };
 
-export type ReplicationDomain<Args, T> = {
+export type ReplicationDomain<Args, T, R extends "u32" | "u64"> = {
+	resolution: R;
 	type: string;
-	fromEntry: ReplicationDomainMapper<T>;
+	fromEntry: ReplicationDomainMapper<T, R>;
 	fromArgs: (
 		args: Args | undefined,
 		log: Log,
-	) => Promise<CoverRange> | CoverRange;
-
-	// to rebalance will return an async iterator of objects that will be added to the log
-	/* toRebalance(
-		change: ReplicationChange,
-		index: Index<EntryWithCoordinate>
-	): AsyncIterable<{ gid: string, entries: { coordinate: number, hash: string }[] }> | Promise<AsyncIterable<{ gid: string, entries: EntryWithCoordinate[] }>>; */
-};
-
-export const uniformToU32 = (cursor: number) => {
-	return cursor * MAX_U32;
+	) => Promise<CoverRange<NumberFromType<R>>> | CoverRange<NumberFromType<R>>;
 };
 
 export type ExtractDomainArgs<T> =
-	T extends ReplicationDomain<infer Args, any> ? Args : never;
+	T extends ReplicationDomain<infer Args, any, any> ? Args : never;

package/src/replication.ts

@@ -11,8 +11,9 @@ import { type Index } from "@peerbit/indexer-interface";
 import { TransportMessage } from "./message.js";
 import {
 	ReplicationIntent,
-	ReplicationRange,
 	type ReplicationRangeIndexable,
+	ReplicationRangeMessage,
+	ReplicationRangeMessageU32,
 } from "./ranges.js";
 import { Observer, Replicator, Role } from "./role.js";
 
@@ -20,7 +21,7 @@ export type ReplicationLimits = { min: MinReplicas; max?: MinReplicas };
 
 interface SharedLog {
 	replicas: Partial<ReplicationLimits>;
-	replicationIndex: Index<ReplicationRangeIndexable> | undefined;
+	replicationIndex: Index<ReplicationRangeIndexable<any>> | undefined;
 }
 
 export class MinReplicas {
@@ -67,7 +68,7 @@ export class ResponseRoleMessage extends TransportMessage {
 			segments:
 				this.role instanceof Replicator
 					? this.role.segments.map((x) => {
-							return new ReplicationRange({
+							return new ReplicationRangeMessageU32({
 								id: randomBytes(32),
 								offset: x.offset,
 								factor: x.factor,
@@ -82,10 +83,10 @@ export class ResponseRoleMessage extends TransportMessage {
 
 @variant([1, 2])
 export class AllReplicatingSegmentsMessage extends TransportMessage {
-	@field({ type: vec(ReplicationRange) })
-	segments: ReplicationRange[];
+	@field({ type: vec(ReplicationRangeMessage) })
+	segments: ReplicationRangeMessage<any>[];
 
-	constructor(properties: { segments: ReplicationRange[] }) {
+	constructor(properties: { segments: ReplicationRangeMessage<any>[] }) {
 		super();
 		this.segments = properties.segments;
 	}
@@ -93,10 +94,10 @@ export class AllReplicatingSegmentsMessage extends TransportMessage {
 
 @variant([1, 3])
 export class AddedReplicationSegmentMessage extends TransportMessage {
-	@field({ type: vec(ReplicationRange) })
-	segments: ReplicationRange[];
+	@field({ type: vec(ReplicationRangeMessage) })
+	segments: ReplicationRangeMessage<any>[];
 
-	constructor(properties: { segments: ReplicationRange[] }) {
+	constructor(properties: { segments: ReplicationRangeMessage<any>[] }) {
 		super();
 		this.segments = properties.segments;
 	}

package/src/role.ts

@@ -4,10 +4,7 @@
  * Roles have been replaces with just replication segments.
  */
 import { field, variant, vec } from "@dao-xyz/borsh";
-
-export const MAX_U32 = 4294967295;
-export const HALF_MAX_U32 = 2147483647; // rounded down
-export const scaleToU32 = (value: number) => Math.round(MAX_U32 * value);
+import { MAX_U32, denormalizer } from "./integers.js";
 
 export const overlaps = (x1: number, x2: number, y1: number, y2: number) => {
 	if (x1 <= y2 && y1 <= x2) {
@@ -40,6 +37,7 @@ export class Observer extends Role {
 
 export const REPLICATOR_TYPE_VARIANT = new Uint8Array([2]);
 
+const denormalizeru32 = denormalizer("u32");
 export class RoleReplicationSegment {
 	@field({ type: "u64" })
 	timestamp: bigint;
@@ -61,12 +59,12 @@ export class RoleReplicationSegment {
 		}
 
 		this.timestamp = timestamp ?? BigInt(+new Date());
-		this.factorNominator = Math.round(MAX_U32 * factor);
+		this.factorNominator = denormalizeru32(factor);
 
 		if (offset > 1 || offset < 0) {
 			throw new Error("Expecting offset to be between 0 and 1, got: " + offset);
 		}
-		this.offsetNominator = Math.round(MAX_U32 * offset);
+		this.offsetNominator = denormalizeru32(factor);
 	}
 
 	get factor(): number {

package/src/sync/index.ts

@@ -0,0 +1,51 @@
+import type { Cache } from "@peerbit/cache";
+import type { PublicSignKey } from "@peerbit/crypto";
+import type { Index } from "@peerbit/indexer-interface";
+import type { Entry, Log } from "@peerbit/log";
+import type { RPC, RequestContext } from "@peerbit/rpc";
+import type { EntryWithRefs } from "../exchange-heads.js";
+import type { Numbers } from "../integers.js";
+import type { TransportMessage } from "../message.js";
+import type { EntryReplicated, ReplicationRangeIndexable } from "../ranges.js";
+
+export type SynchronizerComponents<R extends "u32" | "u64"> = {
+	rpc: RPC<TransportMessage, TransportMessage>;
+	rangeIndex: Index<ReplicationRangeIndexable<R>, any>;
+	entryIndex: Index<EntryReplicated<R>, any>;
+	log: Log<any>;
+	coordinateToHash: Cache<string>;
+	numbers: Numbers<R>;
+};
+export type SynchronizerConstructor<R extends "u32" | "u64"> = new (
+	properties: SynchronizerComponents<R>,
+) => Syncronizer<R>;
+
+export type SyncableKey = string | bigint; // hash or coordinate
+
+export interface Syncronizer<R extends "u32" | "u64"> {
+	onMaybeMissingEntries(properties: {
+		entries: Map<string, EntryReplicated<R>>;
+		targets: string[];
+	}): Promise<void> | void;
+
+	onMessage(
+		message: TransportMessage,
+		context: RequestContext,
+	): Promise<boolean> | boolean;
+
+	onReceivedEntries(properties: {
+		entries: EntryWithRefs<any>[];
+		from: PublicSignKey;
+	}): Promise<void> | void;
+
+	onEntryAdded(entry: Entry<any>): void;
+	onEntryRemoved(hash: string): void;
+	onPeerDisconnected(key: PublicSignKey): void;
+
+	open(): Promise<void> | void;
+	close(): Promise<void> | void;
+
+	get pending(): number;
+
+	get syncInFlight(): Map<string, Map<SyncableKey, { timestamp: number }>>;
+}