@peerbit/stream 1.0.20 → 2.0.1

package/src/logger.ts

@@ -1,2 +1,2 @@
 import { logger as logFn } from "@peerbit/logger";
-export const logger = logFn({ module: "direct-stream", level: "warn" });
+export const logger = logFn({ module: "lazystream", level: "warn" });

package/src/metrics.ts

@@ -1,70 +1,18 @@
 import { PublicSignKey } from "@peerbit/crypto";
 
-export class Frequency {
-	private interval: ReturnType<typeof setInterval>;
-	private lastFrequency: number;
-	private events = 0;
-	private intervalTime: number;
-	constructor(properties: { intervalTime } = { intervalTime: 10 * 1000 }) {
-		this.intervalTime = properties.intervalTime;
-		this.interval = setInterval(() => {
-			this.lastFrequency = this.events / this.intervalTime;
-			this.events = 0;
-		}, properties.intervalTime);
-	}
-
-	count(event: number) {
-		this.events += event;
-	}
-
-	close() {
-		clearInterval(this.interval);
-		this.events = 0;
-	}
+export class MovingAverageTracker {
+	private lastTS = 0;
 
-	get frequency() {
-		return this.lastFrequency;
-	}
-}
+	value = 0;
 
-export class RouteFrequency {
-	private interval: ReturnType<typeof setInterval>;
-	private lastFrequency: Map<string, number>;
-	private events: Map<string, number>;
-	private intervalTime: number;
-	constructor(properties: { intervalTime } = { intervalTime: 10 * 1000 }) {
-		this.intervalTime = properties.intervalTime;
-		this.events = new Map();
+	constructor(readonly tau = 10) {
+		this.lastTS = +new Date();
 	}
-
-	start() {
-		this.interval = setInterval(() => {
-			this.lastFrequency = this.events;
-			this.events = new Map();
-		}, this.intervalTime);
-	}
-
-	increment(to: string, bytes: Uint8Array) {
-		let value = this.events.get(to);
-		if (value == null) {
-			value = 1;
-			this.events.set(to, bytes.length);
-		} else {
-			this.events.set(to, value + bytes.length);
-		}
-	}
-
-	close() {
-		clearInterval(this.interval);
-		this.events.clear();
-	}
-
-	getFrequency(to: PublicSignKey) {
-		const count = this.lastFrequency.get(to.hashcode());
-		if (count) {
-			return count / this.intervalTime;
-		}
-
-		return undefined;
+	add(number: number) {
+		const now = +new Date();
+		const dt = (now - this.lastTS) / 1000;
+		const alpha_t = 1 - Math.exp(-dt / this.tau);
+		this.value = (1 - alpha_t) * this.value + (alpha_t * number) / dt;
+		this.lastTS = now;
 	}
 }

package/src/routes.ts

@@ -1,283 +1,334 @@
-import Graphs from "graphology";
-import type { MultiUndirectedGraph } from "graphology";
-import { dijkstra, unweighted } from "graphology-shortest-path";
-import { logger } from "./logger.js";
-import { MinimalEdgeMapper } from "graphology-utils/getters";
-
-interface EdgeData {
-	weight: number;
-	time: number;
-}
+import { PublicSignKey } from "@peerbit/crypto";
+
+export const MAX_ROUTE_DISTANCE = Number.MAX_SAFE_INTEGER;
 export class Routes {
-	graph: MultiUndirectedGraph<any, EdgeData>;
-	private peerId: string;
-	constructor(peerId: string) {
-		this.peerId = peerId;
-		this.graph = new (Graphs as any).UndirectedGraph();
-	}
+	// END receiver -> Neighbour
 
-	get linksCount() {
-		return this.graph.edges().length;
+	routes: Map<
+		string,
+		Map<string, { session: number; list: { hash: string; distance: number }[] }>
+	> = new Map();
+
+	pendingRoutes: Map<
+		number,
+		Map<
+			string,
+			{
+				from: string;
+				neighbour: string;
+				distance: number;
+			}[]
+		>
+	> = new Map();
+	latestSession: number;
+
+	constructor(readonly me: string) {
+		this.latestSession = 0;
 	}
 
-	get nodeCount() {
-		return this.graph.nodes().length;
+	clear() {
+		this.routes.clear();
+		this.pendingRoutes.clear();
 	}
 
-	/**
-	 *
-	 * @param from
-	 * @param to
-	 * @returns new nodes
-	 */
-	addLink(
+	add(
 		from: string,
-		to: string,
-		weight: number,
-		origin: string = this.peerId
-	): string[] {
-		const linkExisted = this.hasLink(from, to);
-		const newReachableNodesFromOrigin: string[] = [];
-		if (!linkExisted) {
-			const currentTime = +new Date();
-			const fromWasReachable =
-				origin == from ||
-				this.getPath(origin, from, { unweighted: true }).length;
-			const toWasReachable =
-				origin === to || this.getPath(origin, to, { unweighted: true }).length;
-			const fromIsNowReachable = toWasReachable;
-			const toIsNowReachable = fromWasReachable;
-
-			const visited = new Set<string | number>();
-			const newReachableNodes: string[] = [];
-			if (fromIsNowReachable) {
-				newReachableNodes.push(from);
-			}
-			if (toIsNowReachable) {
-				newReachableNodes.push(to);
-			}
-			if (fromWasReachable) {
-				visited.add(from);
-			}
-			if (toWasReachable) {
-				visited.add(to);
+		neighbour: string,
+		target: string,
+		distance: number,
+		session: number
+	) {
+		let fromMap = this.routes.get(from);
+		if (!fromMap) {
+			fromMap = new Map();
+			this.routes.set(from, fromMap);
+		}
+
+		let prev = fromMap.get(target) || {
+			session: session ?? +new Date(),
+			list: [] as { hash: string; distance: number }[]
+		};
+
+		this.latestSession = Math.max(this.latestSession, session);
+
+		if (session != null) {
+			// this condition means that when we add new routes in a session that is newer
+			if (prev.session < session) {
+				prev = { session, list: [] }; // reset route info how to reach this target
+			} else if (prev.session > session) {
+				return; // new routing information superseedes this
 			}
+		}
 
-			if (!this.graph.hasNode(from)) {
-				this.graph.addNode(from);
+		if (from === this.me && neighbour === target) {
+			// force distance to neighbour as targets to always favor directly sending to them
+			// i.e. if target is our neighbour, always assume the shortest path to them is the direct path
+			distance = -1;
+		}
+
+		for (const route of prev.list) {
+			if (route.hash === neighbour) {
+				route.distance = Math.min(route.distance, distance);
+				prev.list.sort((a, b) => a.distance - b.distance);
+				return;
 			}
-			if (!this.graph.hasNode(to)) {
-				this.graph.addNode(to);
+		}
+		prev.list.push({ distance, hash: neighbour });
+		prev.list.sort((a, b) => a.distance - b.distance);
+		fromMap.set(target, prev);
+	}
+
+	removeTarget(target: string) {
+		this.routes.delete(target);
+		for (const [fromMapKey, fromMap] of this.routes) {
+			// delete target
+			fromMap.delete(target);
+			if (fromMap.size === 0) {
+				this.routes.delete(fromMapKey);
 			}
+		}
+		return [target];
+	}
 
-			this.graph.addUndirectedEdge(from, to, { weight, time: currentTime });
+	removeNeighbour(target: string) {
+		this.routes.delete(target);
+		const maybeUnreachable: Set<string> = new Set([target]);
+		for (const [fromMapKey, fromMap] of this.routes) {
+			// delete target
+			fromMap.delete(target);
 
-			for (const newReachableNode of newReachableNodes) {
-				// get all nodes from this and add them to the new reachable set of nodes one can access from origin
+			// delete this as neighbour
+			for (const [remote, neighbours] of fromMap) {
+				neighbours.list = neighbours.list.filter((x) => x.hash !== target);
+				if (neighbours.list.length === 0) {
+					fromMap.delete(remote);
+					maybeUnreachable.add(remote);
+				}
+			}
 
-				const stack = [newReachableNode]; // iterate from the not reachable node
-				while (stack.length > 0) {
-					const node = stack.shift();
-					if (!node) {
-						continue;
-					}
-					if (visited.has(node)) {
-						continue;
-					}
+			if (fromMap.size === 0) {
+				this.routes.delete(fromMapKey);
+			}
+		}
+		return [...maybeUnreachable].filter((x) => !this.isReachable(this.me, x));
+	}
 
-					visited.add(node);
-					const neighbors = this.graph.neighbors(node);
-					for (const neighbor of neighbors) {
-						const edge = this.graph.undirectedEdge(node, neighbor);
-						if (!edge) {
-							logger.warn(`Missing edge between: ${node} - ${neighbor}`);
-							continue;
-						}
+	findNeighbor(from: string, target: string) {
+		return this.routes.get(from)?.get(target);
+	}
 
-						const attributes = this.graph.getEdgeAttributes(edge);
-						if (attributes.time > currentTime) {
-							continue; // a new link has been added while we are iterating, dont follow this path
-						}
+	isReachable(from: string, target: string) {
+		return (
+			(this.routes.get(from)?.get(target)?.list[0]?.distance ??
+				Number.MAX_SAFE_INTEGER) < MAX_ROUTE_DISTANCE
+		);
+	}
 
-						if (visited.has(neighbor)) {
-							continue;
-						}
+	hasShortestPath(target: string) {
+		const path = this.routes.get(this.me)?.get(target);
+		if (!path) {
+			return false;
+		}
+		return path.list[0].distance <= 0;
+	}
 
-						stack.push(neighbor);
-					}
-					newReachableNodesFromOrigin.push(node);
-				}
+	hasTarget(target: string) {
+		for (const [k, v] of this.routes) {
+			if (v.has(target)) {
+				return true;
 			}
-		} else {
-			// update weight
-			const edge = this.graph.undirectedEdge(from, to);
-			this.graph.setEdgeAttribute(edge, "weight", weight);
-			this.graph.setEdgeAttribute(edge, "time", +new Date());
 		}
+		return false;
+	}
 
-		return newReachableNodesFromOrigin;
+	getDependent(target: string) {
+		const dependent: string[] = [];
+		for (const [fromMapKey, fromMap] of this.routes) {
+			if (fromMapKey !== this.me && fromMap.has(target)) {
+				dependent.push(fromMapKey);
+			}
+		}
+		return dependent;
 	}
 
-	/**
-	 *
-	 * @param from
-	 * @param to
-	 * @param origin
-	 * @returns nodes that are no longer reachable from origin
-	 */
-	deleteLink(from: string, to: string, origin: string = this.peerId): string[] {
-		const link = this.getLink(from, to);
-		if (link) {
-			const date = +new Date();
-			const fromWasReachable =
-				origin == from ||
-				this.getPath(origin, from, { unweighted: true }).length;
-			const toWasReachable =
-				origin === to || this.getPath(origin, to, { unweighted: true }).length;
-			this.graph.dropEdge(link);
-
-			const unreachableNodesFromOrigin: string[] = [];
-			if (
-				fromWasReachable &&
-				origin !== from &&
-				this.getPath(origin, from, { unweighted: true }).length === 0
-			) {
-				unreachableNodesFromOrigin.push(from);
+	count(from = this.me) {
+		const set: Set<string> = new Set();
+		const map = this.routes.get(from);
+		if (map) {
+			for (const [k, v] of map) {
+				set.add(k);
+				for (const peer of v.list) {
+					set.add(peer.hash);
+				}
 			}
-			if (
-				toWasReachable &&
-				origin !== to &&
-				this.getPath(origin, to, { unweighted: true }).length === 0
-			) {
-				unreachableNodesFromOrigin.push(to);
+		}
+		return set.size;
+	}
+
+	countAll() {
+		let size = 0;
+		for (const [from, map] of this.routes) {
+			for (const [k, v] of map) {
+				size += v.list.length;
 			}
+		}
+		return size;
+	}
 
-			// remove subgraphs that are now disconnected from me
-			for (const disconnected of [...unreachableNodesFromOrigin]) {
-				const node = disconnected;
-				if (!this.graph.hasNode(node)) {
-					continue;
-				}
+	// for all tos if
+	getFanout(
+		from: PublicSignKey,
+		tos: string[],
+		redundancy: number
+	): Map<string, { to: string; timestamp: number }[]> | undefined {
+		if (tos.length === 0) {
+			return undefined;
+		}
 
-				const stack = [disconnected];
-				const visited = new Set<string | number>();
-				while (stack.length > 0) {
-					const node = stack.shift();
-					const nodeId = node;
-					if (!nodeId || !this.graph.hasNode(nodeId)) {
-						continue;
-					}
-					if (visited.has(nodeId)) {
-						continue;
-					}
+		let fanoutMap:
+			| Map<string, { to: string; timestamp: number }[]>
+			| undefined = undefined;
 
-					visited.add(nodeId);
+		const fromKey = from.hashcode();
 
-					const neighbors = this.graph.neighbors(node);
+		// Message to > 0
+		if (tos.length > 0) {
+			for (const to of tos) {
+				if (to === this.me || fromKey === to) {
+					continue; // don't send to me or backwards
+				}
 
-					for (const neighbor of neighbors) {
-						const edge = this.graph.undirectedEdge(node, neighbor);
-						if (!edge) {
-							logger.warn(`Missing edge between: ${node} - ${neighbor}`);
-							continue;
+				const neighbour = this.findNeighbor(fromKey, to);
+				if (neighbour) {
+					let foundClosest = false;
+					for (
+						let i = 0;
+						i < Math.min(neighbour.list.length, redundancy);
+						i++
+					) {
+						const distance = neighbour.list[i].distance;
+						if (distance >= redundancy) {
+							break; // because neighbour listis sorted
 						}
-						const attributes = this.graph.getEdgeAttributes(edge);
-						if (attributes.time > date) {
-							continue; // don't follow path because this is a new link that might provide some new connectivity
+						if (distance <= 0) {
+							foundClosest = true;
 						}
-
-						if (visited.has(neighbor)) {
-							continue;
+						const fanout: { to: string; timestamp: number }[] = (
+							fanoutMap || (fanoutMap = new Map())
+						).get(neighbour.list[i].hash);
+						if (!fanout) {
+							fanoutMap.set(neighbour.list[i].hash, [
+								{ to, timestamp: neighbour.session }
+							]);
+						} else {
+							fanout.push({ to, timestamp: neighbour.session });
 						}
-
-						stack.push(neighbor);
 					}
-					this.graph.dropNode(nodeId);
-					if (disconnected !== nodeId) {
-						unreachableNodesFromOrigin.push(nodeId.toString());
+					if (!foundClosest && from.hashcode() === this.me) {
+						return undefined; // we dont have the shortest path to our target (yet). Send to all
 					}
+
+					continue;
 				}
+
+				// we can't find path, send message to all peers
+				return undefined;
 			}
-			return unreachableNodesFromOrigin;
 		}
-		return [];
+		return fanoutMap || (fanoutMap = new Map());
 	}
 
-	getLink(from: string, to: string): string | undefined {
-		if (!this.graph.hasNode(from) || !this.graph.hasNode(to)) {
-			return undefined;
+	/**
+	 * Returns a list of a prunable nodes that are not needed to reach all remote nodes
+	 */
+	getPrunable(neighbours: string[]): string[] {
+		const map = this.routes.get(this.me);
+		if (map) {
+			// check if all targets can be reached without it
+			return neighbours.filter((candidate) => {
+				for (const [target, neighbours] of map) {
+					if (
+						target !== candidate &&
+						neighbours.list.length === 1 &&
+						neighbours.list[0].hash === candidate
+					) {
+						return false;
+					}
+				}
+				return true;
+			});
 		}
+		return [];
+	}
 
-		const edges = this.graph.edges(from, to);
-		if (edges.length > 1) {
-			throw new Error("Unexpected edge count: " + edges.length);
+	public addPendingRouteConnection(
+		session: number,
+		route: {
+			from: string;
+			neighbour: string;
+			target: string;
+			distance: number;
 		}
-		if (edges.length > 0) {
-			return edges[0];
+	) {
+		let map = this.pendingRoutes.get(session);
+		if (!map) {
+			map = new Map();
+			this.pendingRoutes.set(session, map);
+		}
+		let arr = map.get(route.target);
+		if (!arr) {
+			arr = [];
+			map.set(route.target, arr);
 		}
-		return undefined;
-	}
 
-	getLinkData(from: string, to: string): EdgeData | undefined {
-		const edgeId = this.getLink(from, to);
-		if (edgeId) return this.graph.getEdgeAttributes(edgeId);
-		return undefined;
-	}
+		arr.push(route);
 
-	hasLink(from: string, to: string): boolean {
-		return this.graph.hasEdge(from, to);
-	}
-	hasNode(node: string): boolean {
-		return this.graph.hasNode(node);
+		const neighbour = this.findNeighbor(route.from, route.target);
+		if (!neighbour || neighbour.session === session) {
+			// Commit directly since we dont have any data at all (better have something than nothing)
+			this.commitPendingRouteConnection(session, route.target);
+		}
 	}
 
-	getPath(
-		from: string,
-		to: string,
-		options?: { unweighted?: boolean } | { block?: string }
-	): unweighted.ShortestPath | dijkstra.BidirectionalDijstraResult {
-		try {
-			let getEdgeWeight:
-				| keyof EdgeData
-				| MinimalEdgeMapper<number, EdgeData> = (edge) =>
-				this.graph.getEdgeAttribute(edge, "weight");
-			const blockId = (options as { block?: string })?.block;
-			if (blockId) {
-				const neighBourEdges = new Set(
-					this.graph
-						.inboundNeighbors(blockId)
-						.map((x) => this.graph.edges(x, blockId))
-						.flat()
-				);
-				getEdgeWeight = (edge) => {
-					if (neighBourEdges.has(edge)) {
-						return Number.MAX_SAFE_INTEGER;
-					}
-					return this.graph.getEdgeAttribute(edge, "weight");
-				};
-			}
+	// always commit if we dont know the peer yet
+	// do pending commits per remote (?)
 
-			// TODO catching for network changes and resuse last result
-			const path =
-				((options as { unweighted?: boolean })?.unweighted
-					? unweighted.bidirectional(this.graph, from, to)
-					: dijkstra.bidirectional(this.graph, from, to, getEdgeWeight)) || [];
-			if (path?.length > 0 && path[0] !== from) {
-				path.reverse();
+	public commitPendingRouteConnection(session: number, target?: string) {
+		const map = this.pendingRoutes.get(session);
+		if (!map) {
+			return;
+		}
+		if (target) {
+			const routes = map.get(target);
+			if (routes) {
+				for (const route of routes) {
+					this.add(
+						route.from,
+						route.neighbour,
+						target,
+						route.distance,
+						session
+					);
+				}
+				map.delete(target);
+				return;
+			} else {
+				return;
 			}
-
-			if (blockId) {
-				if (path.includes(blockId)) {
-					return []; // Path does not exist, as we go through a blocked node with inifite weight
+		} else {
+			for (const [target, routes] of map) {
+				for (const route of routes) {
+					this.add(
+						route.from,
+						route.neighbour,
+						target,
+						route.distance,
+						session
+					);
 				}
 			}
-
-			return path as any; // TODO fix types
-		} catch (error) {
-			return [];
 		}
-	}
-	clear() {
-		this.graph.clear();
+
+		this.pendingRoutes.delete(session);
 	}
 }

package/lib/esm/peer-map.d.ts

@@ -1 +0,0 @@
-export type PeerMap<T> = Map<string, T>;

package/lib/esm/peer-map.js

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=peer-map.js.map

package/lib/esm/peer-map.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"peer-map.js","sourceRoot":"","sources":["../../src/peer-map.ts"],"names":[],"mappings":""}

package/src/peer-map.ts

@@ -1 +0,0 @@
-export type PeerMap<T> = Map<string, T>;