@peerbit/shared-log 1.0.2

package/src/exchange-heads.ts

@@ -0,0 +1,109 @@
+import { variant, option, field, vec, fixedArray } from "@dao-xyz/borsh";
+import { Entry } from "@peerbit/log";
+import { Log } from "@peerbit/log";
+import { logger as loggerFn } from "@peerbit/logger";
+import { TransportMessage } from "./message.js";
+
+const logger = loggerFn({ module: "exchange-heads" });
+
+export class MinReplicas {
+	get value(): number {
+		throw new Error("Not implemented");
+	}
+}
+
+@variant(0)
+export class AbsolutMinReplicas extends MinReplicas {
+	_value: number;
+	constructor(value: number) {
+		super();
+		this._value = value;
+	}
+	get value() {
+		return this._value;
+	}
+}
+
+/**
+ * This thing allows use to faster sync since we can provide
+ * references that can be read concurrently to
+ * the entry when doing Log.fromEntry or Log.fromEntryHash
+ */
+@variant(0)
+export class EntryWithRefs<T> {
+	@field({ type: Entry })
+	entry: Entry<T>;
+
+	@field({ type: vec(Entry) })
+	references: Entry<T>[]; // are some parents to the entry
+
+	constructor(properties: { entry: Entry<T>; references: Entry<T>[] }) {
+		this.entry = properties.entry;
+		this.references = properties.references;
+	}
+}
+
+@variant([0, 0])
+export class ExchangeHeadsMessage<T> extends TransportMessage {
+	@field({ type: vec(EntryWithRefs) })
+	heads: EntryWithRefs<T>[];
+
+	@field({ type: option(MinReplicas) })
+	minReplicas?: MinReplicas;
+
+	@field({ type: fixedArray("u8", 4) })
+	reserved: Uint8Array = new Uint8Array(4);
+
+	constructor(props: {
+		logId: Uint8Array;
+		heads: EntryWithRefs<T>[];
+		minReplicas?: MinReplicas;
+	}) {
+		super();
+		this.heads = props.heads;
+		this.minReplicas = props.minReplicas;
+	}
+}
+
+@variant([0, 1])
+export class RequestHeadsMessage extends TransportMessage {
+	@field({ type: "string" })
+	address: string;
+
+	constructor(props: { topic: string; address: string }) {
+		super();
+		if (props) {
+			this.address = props.address;
+		}
+	}
+}
+
+export const createExchangeHeadsMessage = async (
+	log: Log<any>,
+	heads: Entry<any>[],
+	includeReferences: boolean
+) => {
+	const headsSet = new Set(heads);
+	const headsWithRefs = await Promise.all(
+		heads.map(async (head) => {
+			const refs = !includeReferences
+				? []
+				: (
+						await log.getReferenceSamples(head, {
+							pointerCount: 8,
+							memoryLimit: 1e6 / heads.length,
+						})
+				  ) // 1mb total limit split on all heads
+						.filter((r) => !headsSet.has(r)); // pick a proportional amount of refs so we can efficiently load the log. TODO should be equidistant for good performance?
+			return new EntryWithRefs({
+				entry: head,
+				references: refs,
+			});
+		})
+	);
+	logger.debug(`Send latest heads of '${log.id}'`);
+	return new ExchangeHeadsMessage({
+		logId: log.id!,
+		heads: headsWithRefs,
+	});
+};

package/src/exchange-replication.ts

@@ -0,0 +1,206 @@
+import { variant, field, option, vec } from "@dao-xyz/borsh";
+import { TransportMessage } from "./message.js";
+
+export const WAIT_FOR_PEERS_TIME = 5000;
+
+@variant([2, 0])
+export class ReplicatorInfo extends TransportMessage {
+	@field({ type: option("string") })
+	fromId?: string;
+
+	@field({ type: "string" })
+	topic: string;
+
+	@field({ type: "u32" })
+	store: number; // address
+
+	@field({ type: vec("string") })
+	heads?: string[]; // address
+
+	constructor(props?: {
+		fromId?: string;
+		topic: string;
+		store: number;
+		heads?: string[];
+	}) {
+		super();
+		if (props) {
+			this.fromId = props.fromId;
+			this.topic = props.topic;
+			this.store = props.store;
+			this.heads = props.heads;
+			/*  this.allowForks = props.allowForks; */
+		}
+	}
+}
+
+/* @variant([2, 1])
+export class RequestReplicatorInfo extends ProtocolMessage {
+	@field({ type: "string" })
+	id: string;
+
+	@field({ type: "string" })
+	topic: string;
+
+	@field({ type: "string" })
+	address: string; // address
+
+	@field({ type: vec("string") })
+	heads: string[];
+
+	constructor(props?: {
+		topic: string;
+		address: Address | string;
+		heads: string[];
+	}) {
+		super();
+		if (props) {
+			this.id = uuid();
+			this.topic = props.topic;
+			this.address =
+				typeof props.address === "string"
+					? props.address
+					: props.address.toString();
+			this.heads = props.heads;
+		}
+	}
+}
+ */
+/* export interface PeerInfoWithMeta {
+	peerInfo: ReplicatorInfo;
+	publicKey: PublicSignKey;
+} */
+/* return new PeerInfo({
+	key: this._shard.peer.orbitDB.identity,
+	addresses: (await this._shard.peer.node.id()).addresses.map(x => x.toString()),
+	memoryLeft: v8.getHeapStatistics().total_available_size//v8
+}) */
+
+/* export const createEmitHealthCheckJob = (properties: { stores: () => string[] | undefined, subscribingForReplication: (topic: string) => boolean }, replicationTopic: string, publish: (topic: string, message: Uint8Array) => Promise<void>, isOnline: () => boolean, controller: AbortController, sign: (bytes: Uint8Array) => Promise<{ signature: Uint8Array, publicKey: PublicKey }>, encryption: PublicKeyEncryption) => {
+
+	const emitHealthcheck = async (): Promise<void> => {
+		const s = properties.stores();
+		if (!s || s.length === 0) {
+			return;
+		}
+		const signedMessage = await new MaybeSigned({
+			data: serialize(new PeerInfo({
+				replicationTopic,
+				stores: s,
+				subscribingForReplication: properties.subscribingForReplication(replicationTopic),
+				memoryLeft: v8.getHeapStatistics().total_available_size//v8
+
+			}))
+		}).sign(sign)
+		const decryptedMessage = new DecryptedThing({
+			data: serialize(signedMessage)
+		})// TODO add encryption  .init(encryption).encrypt(lala)
+
+		return publish(replicationTopic, serialize(decryptedMessage))
+	}
+
+	const task = async () => {
+		await emitHealthcheck();
+	}
+
+	const cron = async () => {
+		let stop = false;
+		let promise: Promise<any> = undefined;
+		let delayStopper: () => void | undefined = undefined;
+		controller.signal.addEventListener("abort", async () => {
+			stop = true;
+			if (delayStopper)
+				delayStopper();
+			await promise;
+		});
+		while (isOnline() && !stop) { // 
+			promise = task();
+			await promise;
+			await delay(EMIT_HEALTHCHECK_INTERVAL, { stopperCallback: (stopper) => { delayStopper = stopper } }); // some delay
+		}
+	}
+	return cron;
+}
+ */
+
+/* export const requestPeerInfo = async (
+	serializedRequest: Uint8Array,
+	replicationTopic: string,
+	publish: (topic: string, message: Uint8Array) => Promise<void>,
+	identity: Identity
+) => {
+	const signedMessage = await new MaybeSigned({
+		data: serializedRequest,
+	}).sign(async (data) => {
+		return {
+			publicKey: identity.publicKey,
+			signature: await identity.sign(data),
+		};
+	});
+	const decryptedMessage = new DecryptedThing({
+		data: serialize(signedMessage),
+	}); // TODO add encryption  .init(encryption).encrypt(lala)
+
+	return publish(replicationTopic, serialize(decryptedMessage));
+}; */
+
+/* export const exchangePeerInfo = async (
+	fromId: string,
+	replicationTopic: string,
+	store: Store<any>,
+	heads: string[] | undefined,
+	publish: (message: Uint8Array) => Promise<void>,
+	sign: (
+		bytes: Uint8Array
+	) => Promise<{ signature: Uint8Array; publicKey: PublicSignKey }>
+) => {
+	const signedMessage = await new MaybeSigned({
+		data: serialize(
+			new ReplicatorInfo({
+				fromId,
+				replicationTopic,
+				store: store._storeIndex,
+				heads,
+			})
+		),
+	}).sign(sign);
+
+	const decryptedMessage = new DecryptedThing({
+		data: serialize(signedMessage),
+	}); // TODO add encryption  .init(encryption).encrypt(lala)
+
+	return publish(serialize(decryptedMessage));
+};
+
+export class ResourceRequirement {
+	async ok(_orbitdb: Peerbit): Promise<boolean> {
+		throw new Error("Not implemented");
+	}
+}
+
+@variant(0)
+export class NoResourceRequirement extends ResourceRequirement {} */
+
+/* @variant(1)
+export class HeapSizeRequirement extends ResourceRequirement {
+
+	@field({ type: 'u64' })
+	heapSize: bigint
+
+	constructor(properties?: { heapSize: bigint }) {
+		super();
+		if (properties) {
+			this.heapSize = properties.heapSize;
+		}
+	}
+
+	async ok(orbitdb: OrbitDB): Promise<boolean> {
+		if (!v8 || typeof orbitdb.heapsizeLimitForForks !== 'number') {
+			return true;
+		}
+		const usedHeap: number = v8.getHeapStatistics().used_heap_size;
+		return BigInt(usedHeap) + this.heapSize < orbitdb.heapsizeLimitForForks;
+	}
+
+
+} */