@peerbit/shared-log 9.0.5 → 9.0.6-7872365

package/src/index.ts

@@ -73,6 +73,7 @@ import {
 	ReplicationRangeIndexable,
 	RequestReplicationInfoMessage,
 	ResponseReplicationInfoMessage,
+	ResponseRoleMessage,
 	StartedReplicating,
 	StoppedReplicating,
 	decodeReplicas,
@@ -80,7 +81,7 @@ import {
 	hashToUniformNumber,
 	maxReplicas,
 } from "./replication.js";
-import { SEGMENT_COORDINATE_SCALE } from "./role.js";
+import { Observer, Replicator, SEGMENT_COORDINATE_SCALE } from "./role.js";
 
 export * from "./replication.js";
 
@@ -96,10 +97,10 @@ const groupByGid = async <
 	const groupByGid: Map<string, T[]> = new Map();
 	for (const head of entries) {
 		const gid = await (head instanceof Entry
-			? head.getGid()
+			? (await head.getMeta()).gid
 			: head instanceof ShallowEntry
 				? head.meta.gid
-				: head.entry.getGid());
+				: (await head.entry.getMeta()).gid);
 		let value = groupByGid.get(gid);
 		if (!value) {
 			value = [];
@@ -156,6 +157,7 @@ export type SharedLogOptions<T> = {
 	timeUntilRoleMaturity?: number;
 	waitForReplicatorTimeout?: number;
 	distributionDebounceTime?: number;
+	compatibility?: number;
 };
 
 export const DEFAULT_MIN_REPLICAS = 2;
@@ -205,7 +207,9 @@ export class SharedLog<T = Uint8Array> extends Program<
 		publicKey: PublicSignKey,
 	) => Promise<boolean> | boolean;
 
-	private _logProperties?: LogProperties<T> & LogEvents<T>;
+	private _logProperties?: LogProperties<T> &
+		LogEvents<T> &
+		SharedLogOptions<T>;
 	private _closeController!: AbortController;
 	private _gidParentCache!: Cache<Entry<any>[]>;
 	private _respondToIHaveTimeout!: any;
@@ -278,6 +282,32 @@ export class SharedLog<T = Uint8Array> extends Program<
 		return this._replicationSettings;
 	}
 
+	get compatibility(): number | undefined {
+		return this._logProperties?.compatibility;
+	}
+
+	private get v8Behaviour() {
+		return (this.compatibility ?? Number.MAX_VALUE) < 9;
+	}
+
+	// @deprecated
+	private getRole() {
+		let isFixedReplicationSettings =
+			(this._replicationSettings as FixedReplicationOptions).factor !==
+			undefined;
+		if (isFixedReplicationSettings) {
+			const fixedSettings = this
+				._replicationSettings as FixedReplicationOptions;
+			return new Replicator({
+				factor: fixedSettings.factor,
+				offset: fixedSettings.offset ?? 0,
+			});
+		}
+
+		// TODO this is not accurate but might be good enough
+		return new Observer();
+	}
+
 	async isReplicating() {
 		if (!this._replicationSettings) {
 			return false;
@@ -596,41 +626,15 @@ export class SharedLog<T = Uint8Array> extends Program<
 			logger.warn("Not allowed to replicate by canReplicate");
 		}
 
-		added &&
-			(await this.rpc.send(
+		if (added) {
+			await this.rpc.send(
 				new StartedReplicating({ segments: [range.toReplicationRange()] }),
 				{
 					priority: 1,
 				},
-			));
-	}
-
-	/* async updateRole(role: InitialReplicationOptions, onRoleChange = true) {
-		await this.setupReplicationSettings(role)
-		return this._updateRole(, onRoleChange);
-	} */
-
-	/* private async _updateRole(
-		role: Observer | Replicator = this._role,
-		onRoleChange = true
-	) {
-		this._role = role;
-		const { changed } = await this._modifyReplicators(
-			this.role,
-			this.node.identity.publicKey
-		);
-
-		await this.rpc.subscribe();
-		await this.rpc.send(new ResponseReplicationInfoMessage({ segments: await  }), {
-			priority: 1
-		});
-
-		if (onRoleChange && changed !== "none") {
-			await this.onRoleChange(this._role, this.node.identity.publicKey);
+			);
 		}
-
-		return changed;
-	} */
+	}
 
 	async append(
 		data: T,
@@ -1019,12 +1023,16 @@ export class SharedLog<T = Uint8Array> extends Program<
 	async _onMessage(
 		msg: TransportMessage,
 		context: RequestContext,
-	): Promise<TransportMessage | undefined> {
+	): Promise<void> {
 		try {
 			if (!context.from) {
 				throw new Error("Missing from in update role message");
 			}
 
+			if (msg instanceof ResponseRoleMessage) {
+				msg = msg.toReplicationInfoMessage(); // migration
+			}
+
 			if (msg instanceof ExchangeHeadsMessage) {
 				/**
 				 * I have received heads from someone else.
@@ -1132,7 +1140,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 
 								logger.debug(
 									`${this.node.identity.publicKey.hashcode()}: Dropping heads with gid: ${
-										entry.entry.gid
+										entry.entry.meta.gid
 									}. Because not leader`,
 								);
 							}
@@ -1300,6 +1308,8 @@ export class SharedLog<T = Uint8Array> extends Program<
 			} else if (msg instanceof BlocksMessage) {
 				await this.remoteBlocks.onMessage(msg.message);
 			} else if (msg instanceof RequestReplicationInfoMessage) {
+				// TODO this message type is never used, should we remove it?
+
 				if (context.from.equals(this.node.identity.publicKey)) {
 					return;
 				}
@@ -1313,6 +1323,28 @@ export class SharedLog<T = Uint8Array> extends Program<
 						mode: new SilentDelivery({ to: [context.from], redundancy: 1 }),
 					},
 				);
+
+				// for backwards compatibility (v8) remove this when we are sure that all nodes are v9+
+				if (this.v8Behaviour) {
+					const role = this.getRole();
+					if (role instanceof Replicator) {
+						const fixedSettings = this
+							._replicationSettings as FixedReplicationOptions;
+						if (fixedSettings.factor === 1) {
+							await this.rpc.send(
+								new ResponseRoleMessage({
+									role,
+								}),
+								{
+									mode: new SilentDelivery({
+										to: [context.from],
+										redundancy: 1,
+									}),
+								},
+							);
+						}
+					}
+				}
 			} else if (
 				msg instanceof ResponseReplicationInfoMessage ||
 				msg instanceof StartedReplicating
@@ -1321,6 +1353,10 @@ export class SharedLog<T = Uint8Array> extends Program<
 					return;
 				}
 
+				let replicationInfoMessage = msg as
+					| ResponseReplicationInfoMessage
+					| StartedReplicating;
+
 				// we have this statement because peers might have changed/announced their role,
 				// but we don't know them as "subscribers" yet. i.e. they are not online
 
@@ -1343,7 +1379,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 							await this.removeReplicator(context.from!);
 						}
 						let addedOnce = false;
-						for (const segment of msg.segments) {
+						for (const segment of replicationInfoMessage.segments) {
 							const added = await this.addReplicationRange(
 								segment.toReplicationRangeIndexable(context.from!),
 								context.from!,
@@ -1646,7 +1682,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 		},
 	) {
 		return this.isLeader(
-			entry.gid,
+			entry.meta.gid,
 			decodeReplicas(entry).getValue(this),
 			options,
 		);
@@ -1698,6 +1734,15 @@ export class SharedLog<T = Uint8Array> extends Program<
 						},
 					)
 					.catch((e) => logger.error(e.toString()));
+
+				if (this.v8Behaviour) {
+					// for backwards compatibility
+					this.rpc
+						.send(new ResponseRoleMessage({ role: this.getRole() }), {
+							mode: new SilentDelivery({ redundancy: 1, to: [publicKey] }),
+						})
+						.catch((e) => logger.error(e.toString()));
+				}
 			}
 		} else {
 			await this.removeReplicator(publicKey);

package/src/replication.ts

@@ -2,6 +2,7 @@ import {
 	BinaryReader,
 	deserialize,
 	field,
+	option,
 	serialize,
 	variant,
 	vec,
@@ -9,28 +10,14 @@ import {
 import { PublicSignKey, equals, randomBytes } from "@peerbit/crypto";
 import { type Index, id } from "@peerbit/indexer-interface";
 import { TransportMessage } from "./message.js";
-import { SEGMENT_COORDINATE_SCALE } from "./role.js";
+import {
+	Observer,
+	Replicator,
+	Role,
+	SEGMENT_COORDINATE_SCALE,
+} from "./role.js";
 
 export type ReplicationLimits = { min: MinReplicas; max?: MinReplicas };
-/* 
-export class ReplicatorRect {
-
-	@id({ type: Uint8Array })
-	id: Uint8Array;
-
-	@field({ type: 'string' })
-	hash: string;
-
-	@field({ type: vec(ReplicationSegment) })
-	segments: ReplicationSegment[];
-
-	constructor(properties: { hash: string; segments: ReplicationSegment[] }) {
-		this.id = randomBytes(32);
-		this.hash = properties.hash;
-		this.segments = properties.segments;
-	}
-};
- */
 
 export enum ReplicationIntent {
 	Explicit = 0,
@@ -280,7 +267,35 @@ export class RequestReplicationInfoMessage extends TransportMessage {
 	}
 }
 
+// @deprecated remove when possible
 @variant([1, 1])
+export class ResponseRoleMessage extends TransportMessage {
+	@field({ type: option(Role) })
+	role: Observer | Replicator;
+
+	constructor(properties: { role: Observer | Replicator }) {
+		super();
+		this.role = properties.role;
+	}
+
+	toReplicationInfoMessage(): ResponseReplicationInfoMessage {
+		return new ResponseReplicationInfoMessage({
+			segments:
+				this.role instanceof Replicator
+					? this.role.segments.map((x) => {
+							return new ReplicationRange({
+								id: randomBytes(32),
+								offset: x.offset,
+								factor: x.factor,
+								timestamp: x.timestamp,
+							});
+						})
+					: [],
+		});
+	}
+}
+
+@variant([1, 2])
 export class ResponseReplicationInfoMessage extends TransportMessage {
 	@field({ type: vec(ReplicationRange) })
 	segments: ReplicationRange[];
@@ -291,7 +306,7 @@ export class ResponseReplicationInfoMessage extends TransportMessage {
 	}
 }
 
-@variant([1, 2])
+@variant([1, 3])
 export class StartedReplicating extends TransportMessage {
 	@field({ type: vec(ReplicationRange) })
 	segments: ReplicationRange[];
@@ -302,7 +317,7 @@ export class StartedReplicating extends TransportMessage {
 	}
 }
 
-@variant([1, 3])
+@variant([1, 4])
 export class StoppedReplicating extends TransportMessage {
 	@field({ type: vec(Uint8Array) })
 	segmentIds: Uint8Array[];

package/src/role.ts

@@ -1,16 +1,23 @@
-/* import { field, variant, vec } from "@dao-xyz/borsh"; */
+/**
+ * @deprecated
+ * Code below is deprecated and will be removed in the future.
+ * Roles have been replaces with just replication segments.
+ */
+import { field, variant, vec } from "@dao-xyz/borsh";
+
+export const SEGMENT_COORDINATE_SCALE = 4294967295;
 
-/* export const overlaps = (x1: number, x2: number, y1: number, y2: number) => {
+export const overlaps = (x1: number, x2: number, y1: number, y2: number) => {
 	if (x1 <= y2 && y1 <= x2) {
 		return true;
 	}
 	return false;
-}; */
+};
 
-/* export abstract class Role {
-	abstract equals(other: Role): boolean; 
+export abstract class Role {
+	abstract equals(other: Role): boolean;
 }
- 
+
 export const NO_TYPE_VARIANT = new Uint8Array([0]);
 
 @variant(0)
@@ -29,83 +36,80 @@ export class Observer extends Role {
 	}
 }
 
-export const REPLICATOR_TYPE_VARIANT = new Uint8Array([2]);*/
-
-export const SEGMENT_COORDINATE_SCALE = 4294967295;
-/* export class ReplicationSegment {
+export const REPLICATOR_TYPE_VARIANT = new Uint8Array([2]);
 
+export class RoleReplicationSegment {
 	@field({ type: "u64" })
 	timestamp: bigint;
 
 	@field({ type: "u32" })
-	start: number;
-
-	@field({ type: 'u32' })
-	end: number;
-
+	private factorNominator: number;
 
+	@field({ type: "u32" })
+	private offsetNominator: number;
 
 	constructor(properties: {
-		start: number;
-		end: number;
-		timestamp: bigint;
+		factor: number;
+		offset: number;
+		timestamp?: bigint;
 	}) {
-		const { start, end, timestamp } = properties;
+		const { factor, timestamp, offset } = properties;
+		if (factor > 1 || factor < 0) {
+			throw new Error("Expecting factor to be between 0 and 1, got: " + factor);
+		}
 
-		if (start > end) {
-			throw new Error("Range 'start' needs to be lower or equal to 'end'")
+		this.timestamp = timestamp ?? BigInt(+new Date());
+		this.factorNominator = Math.round(SEGMENT_COORDINATE_SCALE * factor);
+
+		if (offset > 1 || offset < 0) {
+			throw new Error("Expecting offset to be between 0 and 1, got: " + offset);
 		}
-		this.start = Math.round(start * SEGMENT_COORDINATE_SCALE);
-		this.end = Math.round(end * SEGMENT_COORDINATE_SCALE);
-		this.timestamp = timestamp
+		this.offsetNominator = Math.round(SEGMENT_COORDINATE_SCALE * offset);
 	}
 
+	get factor(): number {
+		return this.factorNominator / SEGMENT_COORDINATE_SCALE;
+	}
 
-
+	get offset(): number {
+		return this.offsetNominator / SEGMENT_COORDINATE_SCALE;
+	}
 }
- */
-
-/* abstract class Capacity { }
 
 @variant(2)
 export class Replicator extends Role {
+	@field({ type: vec(RoleReplicationSegment) })
+	segments: RoleReplicationSegment[];
 
-	@field({ type: vec(Capacity) })
-	capacity: Capacity[];
-
-	constructor(properties?: { capacity: Capacity[] }) {
+	constructor(properties: {
+		factor: number;
+		timestamp?: bigint;
+		offset: number;
+	}) {
 		super();
-		this.capacity = properties?.capacity || [];
-	} */
-
-/* constructor(properties: {
-	timestamp?: bigint;
-	factor: number;
-	offset: number;
-}) {
-	super();
-	let timestamp = properties.timestamp || BigInt(+new Date);
-	let ranges = getSegmentsFromOffsetAndRange(properties.offset, properties.factor);
-	this.segments = ranges.map(x => new ReplicationSegment({ start: x[0], end: x[1], timestamp }));
-}
-*/
-/* get factor(): number {
-	return this.segments[0]!.factor;
-}
+		const segment: RoleReplicationSegment = new RoleReplicationSegment(
+			properties,
+		);
+		this.segments = [segment];
+	}
 
-get offset(): number {
-	return this.segments[0]!.offset;
-}
+	get factor(): number {
+		return this.segments[0]!.factor;
+	}
 
-get timestamp(): bigint {
-	return this.segments[0]!.timestamp;
-} */
-
-/* equals(other: Role) {
-	return (
-		other instanceof Replicator &&
-		other.factor === this.factor &&
-		other.offset === this.offset
-	);
-} */
-/* } */
+	get offset(): number {
+		return this.segments[0]!.offset;
+	}
+
+	get timestamp(): bigint {
+		return this.segments[0]!.timestamp;
+	}
+
+	equals(other: Role) {
+		return (
+			other instanceof Replicator &&
+			other.factor === this.factor &&
+			other.offset === this.offset
+		);
+	}
+}