@peerbit/shared-log 13.2.22 → 13.2.24

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peerbit/shared-log",
-  "version": "13.2.22",
+  "version": "13.2.24",
   "description": "Shared log",
   "sideEffects": false,
   "type": "module",
@@ -24,6 +24,7 @@
   "files": [
     "src",
     "dist",
+    "!dist/benchmark",
     "!dist/e2e",
     "!dist/test",
     "!**/*.tsbuildinfo"
@@ -61,36 +62,36 @@
     "pidusage": "^4.0.1",
     "pino": "^9.4.0",
     "uint8arrays": "^5.1.0",
-    "@peerbit/blocks": "4.2.8",
-    "@peerbit/blocks-interface": "2.1.4",
+    "@peerbit/any-store": "2.2.14",
+    "@peerbit/blocks-interface": "2.1.5",
     "@peerbit/cache": "3.1.1",
-    "@peerbit/crypto": "3.1.4",
-    "@peerbit/any-store": "2.2.13",
-    "@peerbit/indexer-interface": "3.0.8",
+    "@peerbit/blocks": "4.2.10",
+    "@peerbit/crypto": "3.1.5",
     "@peerbit/diagnostics": "0.0.1",
-    "@peerbit/indexer-sqlite3": "3.0.11",
+    "@peerbit/indexer-interface": "3.0.9",
+    "@peerbit/indexer-sqlite3": "3.0.12",
+    "@peerbit/log": "6.2.14",
     "@peerbit/logger": "2.0.1",
-    "@peerbit/log": "6.2.12",
-    "@peerbit/program": "6.0.46",
-    "@peerbit/pubsub": "5.3.11",
-    "@peerbit/pubsub-interface": "5.1.9",
-    "@peerbit/rpc": "6.1.14",
-    "@peerbit/stream-interface": "6.0.14",
+    "@peerbit/program": "6.0.48",
+    "@peerbit/pubsub-interface": "5.2.0",
+    "@peerbit/riblt": "1.2.0",
+    "@peerbit/stream-interface": "6.0.15",
     "@peerbit/time": "3.0.1",
-    "@peerbit/riblt": "1.2.0"
+    "@peerbit/pubsub": "5.4.0",
+    "@peerbit/rpc": "6.1.16"
   },
   "optionalDependencies": {
-    "@peerbit/native-backbone": "0.2.4",
-    "@peerbit/shared-log-rust": "0.1.3"
+    "@peerbit/native-backbone": "0.2.5",
+    "@peerbit/shared-log-rust": "0.1.4"
   },
   "devDependencies": {
     "@types/libsodium-wrappers": "^0.7.14",
     "@types/pidusage": "^2.0.5",
     "uuid": "^11.1.1",
-    "@peerbit/indexer-rust": "1.0.6",
-    "@peerbit/test-utils": "3.1.18",
-    "peerbit": "5.3.18",
-    "@peerbit/any-store-rust": "0.1.3"
+    "@peerbit/indexer-rust": "1.0.7",
+    "@peerbit/test-utils": "3.1.20",
+    "@peerbit/any-store-rust": "0.1.4",
+    "peerbit": "5.3.20"
   },
   "repository": {
     "type": "git",

package/src/index.ts

@@ -186,6 +186,7 @@ import {
 	type Numbers,
 	createNumbers,
 } from "./integers.js";
+import { LeaderPlanCache } from "./leader-plan-cache.js";
 import { TransportMessage } from "./message.js";
 import { PIDReplicationController } from "./pid.js";
 import {
@@ -1631,6 +1632,20 @@ type WaitForReplicatorsOptions<R extends "u32" | "u64"> =
 
 type WaitForReplicator = { key: string; replicator: boolean };
 
+type ReceiveLeaderObservation = {
+	isLeader: boolean;
+	fromIsLeader: boolean;
+};
+
+// Keep long-lived role callbacks outside the receive activation so event-target
+// wrappers cannot retain decoded entries through the callback's closure context.
+const createReceiveLeaderObserver =
+	(observation: ReceiveLeaderObservation, localKey: string, fromKey: string) =>
+	(key: string) => {
+		observation.isLeader ||= localKey === key;
+		observation.fromIsLeader ||= fromKey === key;
+	};
+
 type PendingMaturityRecord<R extends "u32" | "u64"> = {
 	range: ReplicationChange<ReplicationRangeIndexable<R>>;
 	timeout: ReturnType<typeof setTimeout>;
@@ -2638,6 +2653,14 @@ const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_MEMORY_LIMIT = 0.001;
 const RECALCULATE_PARTICIPATION_RELATIVE_DENOMINATOR_FLOOR = 1e-3;
 const TOPIC_SUBSCRIBERS_CACHE_TTL_MS = 250;
 const LEADER_SELECTION_CONTEXT_CACHE_TTL_MS = 50;
+// Leader-plan results are invalidated structurally (replication:change,
+// replicator:mature, subscriber changes). The TTL bounds wall-clock maturity
+// flips that have no timer at all — a range re-crossing maturity after the
+// dynamic default roleAge regrew is only visible via expiry — so it is
+// load-bearing for that edge and must stay at the same order as the
+// leader-selection context TTL.
+const LEADER_PLAN_CACHE_TTL_MS = 50;
+const LEADER_PLAN_CACHE_MAX = 10_000;
 const ADAPTIVE_REBALANCE_IDLE_INTERVAL_MULTIPLIER = 5;
 const ADAPTIVE_REBALANCE_MIN_IDLE_AFTER_LOCAL_APPEND_MS = 10_000;
 
@@ -4536,6 +4559,7 @@ export class SharedLog<
 		expiresAt: number;
 		context: LeaderSelectionContext;
 	};
+	private _leaderPlanCache!: LeaderPlanCache;
 	// Sync capability bits advertised by peers (SyncCapabilitiesMessage), keyed
 	// by public key hash. Entries are dropped on unsubscribe/disconnect.
 	private _peerSyncCapabilities!: Map<string, number>;
@@ -5931,6 +5955,34 @@ export class SharedLog<
 	private invalidateLeaderSelectionContextCache() {
 		this._receiveOwnershipRevision++;
 		this._leaderSelectionContextCache = undefined;
+		this._leaderPlanCache?.invalidate();
+	}
+
+	/**
+	 * Cache key for a leader-plan lookup, or undefined when the call is
+	 * uncacheable: explicit freshness demands, per-call candidate filters,
+	 * roleAge outside the two hot buckets (dynamic default and 0), or an
+	 * in-flight replication-range mutation window (a plan observed
+	 * mid-transition must not outlive the transition).
+	 */
+	private leaderPlanCacheBucket(options?: {
+		roleAge?: number;
+		candidates?: Iterable<string>;
+		freshLeaderPlan?: boolean;
+	}): string | undefined {
+		if (options?.freshLeaderPlan || options?.candidates != null) {
+			return undefined;
+		}
+		if (this._receiveOwnershipMutationAdmissions !== 0) {
+			return undefined;
+		}
+		if (options?.roleAge == null) {
+			return "d";
+		}
+		if (options.roleAge === 0) {
+			return "0";
+		}
+		return undefined;
 	}
 
 	private isReceiveOwnershipSnapshotStable(revision: number): boolean {
@@ -7027,6 +7079,9 @@ export class SharedLog<
 		}
 
 		this._isReplicating = true;
+		// Flipping replication mode changes getDefaultMinRoleAge and the
+		// self-replicating context input even when no range is admitted.
+		this.invalidateLeaderSelectionContextCache();
 
 		if (isAdaptiveReplicatorOption(options!)) {
 			this._isAdaptiveReplicating = true;
@@ -10967,9 +11022,12 @@ export class SharedLog<
 
 		try {
 			throwIfInactive();
+			// Ownership decisions here feed prune/delete outcomes; never serve
+			// them from the leader-plan cache.
 			const currentLeaders = await this.findLeadersFromEntry(
 				args.entry,
 				decodeReplicas(args.entry).getValue(this),
+				{ freshLeaderPlan: true },
 			);
 			throwIfInactive();
 			if (currentLeaders.size > 0) {
@@ -15817,6 +15875,10 @@ export class SharedLog<
 		this._repairMetrics = createRepairMetrics();
 		this._topicSubscribersCache = new Map();
 		this._leaderSelectionContextCache = undefined;
+		this._leaderPlanCache = new LeaderPlanCache({
+			max: LEADER_PLAN_CACHE_MAX,
+			ttl: LEADER_PLAN_CACHE_TTL_MS,
+		});
 		this._peerSyncCapabilities = new Map();
 		this._liveRawGossipBatches = new Map();
 		this._liveRawGossipFlushScheduled = false;
@@ -17300,7 +17362,10 @@ export class SharedLog<
 									) {
 										return;
 									}
-									this.uniqueReplicators.add(keyHash);
+									if (!this.uniqueReplicators.has(keyHash)) {
+										this.uniqueReplicators.add(keyHash);
+										this.invalidateLeaderSelectionContextCache();
+									}
 
 									if (!this._replicatorJoinEmitted.has(keyHash)) {
 										this._replicatorJoinEmitted.add(keyHash);
@@ -20422,6 +20487,10 @@ export class SharedLog<
 										isLeader = true;
 										fromIsLeader = leaders.has(contextFromHash);
 									} else {
+										const leaderObservation: ReceiveLeaderObservation = {
+											isLeader: false,
+											fromIsLeader: false,
+										};
 										leaders = await this._waitForEntryReplicators(
 											latestEntry,
 											maxMaxReplicas,
@@ -20435,15 +20504,15 @@ export class SharedLog<
 												// Let raw receive confirm immediate leadership against the current replicator set.
 												roleAge: 0,
 												timeout: 2e4,
-												onLeader: (key) => {
-													isLeader =
-														isLeader ||
-														this.node.identity.publicKey.hashcode() === key;
-													fromIsLeader =
-														fromIsLeader || contextFromHash === key;
-												},
+												onLeader: createReceiveLeaderObserver(
+													leaderObservation,
+													this.node.identity.publicKey.hashcode(),
+													contextFromHash,
+												),
 											},
 										);
+										isLeader = leaderObservation.isLeader;
+										fromIsLeader = leaderObservation.fromIsLeader;
 									}
 								} else {
 									if (plannedLeaders) {
@@ -22229,8 +22298,20 @@ export class SharedLog<
 			timeout: this.waitForReplicatorTimeout,
 		},
 	): Promise<LeaderMap | false> {
+		// Leader rechecks can outlive the receive call while replication roles
+		// settle. Keep the metadata they need, not the decoded entry's serialized
+		// backing buffer (which can be as large as the transferred block).
+		const leaderSelectionEntry =
+			entry instanceof Entry
+				? deserialize(
+						Uint8Array.from(serialize(entry.toShallow(true))),
+						ShallowEntry,
+					)
+				: entry instanceof ShallowEntry
+					? deserialize(Uint8Array.from(serialize(entry)), ShallowEntry)
+					: entry;
 		if (
-			this.canPlanNativeHashGid(entry) &&
+			this.canPlanNativeHashGid(leaderSelectionEntry) &&
 			(this._nativeBackbone ??
 				this._nativeSharedLogState ??
 				this._nativeRangePlanner)
@@ -22240,7 +22321,7 @@ export class SharedLog<
 				options,
 				async (checkOptions) => {
 					const plan = await this.planEntryLeaders(
-						entry,
+						leaderSelectionEntry,
 						replicas,
 						checkOptions,
 					);
@@ -22250,8 +22331,8 @@ export class SharedLog<
 		}
 
 		return this._waitForReplicators(
-			await this.createCoordinates(entry, replicas),
-			entry,
+			await this.createCoordinates(leaderSelectionEntry, replicas),
+			leaderSelectionEntry,
 			waitFor,
 			options,
 		);
@@ -24903,6 +24984,43 @@ export class SharedLog<
 	}
 
 	private async _findLeaders(
+		cursors: NumberFromType<R>[],
+		options?: {
+			roleAge?: number;
+			candidates?: Iterable<string>;
+			freshLeaderPlan?: boolean;
+		},
+		ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
+	): Promise<Map<string, { intersecting: boolean }>> {
+		const cacheBucket = this.leaderPlanCacheBucket(options);
+		if (cacheBucket == null) {
+			return this._findLeadersUncached(
+				cursors,
+				options,
+				ownershipLifecycleController,
+			);
+		}
+		const cacheKey = `c:${cursors.join(",")}|${cacheBucket}`;
+		const cached = this._leaderPlanCache.get(cacheKey);
+		if (cached) {
+			this.throwIfReplicationOwnershipLifecycleInactive(
+				ownershipLifecycleController,
+			);
+			return cached;
+		}
+		const capturedVersion = this._leaderPlanCache.capture();
+		const leaders = await this._findLeadersUncached(
+			cursors,
+			options,
+			ownershipLifecycleController,
+		);
+		if (this._receiveOwnershipMutationAdmissions === 0) {
+			this._leaderPlanCache.put(cacheKey, leaders, capturedVersion);
+		}
+		return leaders;
+	}
+
+	private async _findLeadersUncached(
 		cursors: NumberFromType<R>[],
 		options?: {
 			roleAge?: number;
@@ -25353,6 +25471,46 @@ export class SharedLog<
 	}
 
 	private async _findLeadersFromHashGid(
+		gid: string,
+		replicas: number,
+		options?: {
+			roleAge?: number;
+			candidates?: Iterable<string>;
+			freshLeaderPlan?: boolean;
+		},
+		ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
+	): Promise<Map<string, { intersecting: boolean }> | undefined> {
+		const cacheBucket = this.leaderPlanCacheBucket(options);
+		if (cacheBucket == null) {
+			return this._findLeadersFromHashGidUncached(
+				gid,
+				replicas,
+				options,
+				ownershipLifecycleController,
+			);
+		}
+		const cacheKey = `g:${gid}:${replicas}|${cacheBucket}`;
+		const cached = this._leaderPlanCache.get(cacheKey);
+		if (cached) {
+			this.throwIfReplicationOwnershipLifecycleInactive(
+				ownershipLifecycleController,
+			);
+			return cached;
+		}
+		const capturedVersion = this._leaderPlanCache.capture();
+		const leaders = await this._findLeadersFromHashGidUncached(
+			gid,
+			replicas,
+			options,
+			ownershipLifecycleController,
+		);
+		if (leaders && this._receiveOwnershipMutationAdmissions === 0) {
+			this._leaderPlanCache.put(cacheKey, leaders, capturedVersion);
+		}
+		return leaders;
+	}
+
+	private async _findLeadersFromHashGidUncached(
 		gid: string,
 		replicas: number,
 		options?: {
@@ -25480,6 +25638,7 @@ export class SharedLog<
 		replicas: number,
 		options?: {
 			roleAge?: number;
+			freshLeaderPlan?: boolean;
 		},
 		ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
 	): Promise<Map<string, { intersecting: boolean }>> {

package/src/leader-plan-cache.ts

@@ -0,0 +1,61 @@
+import { Cache } from "@peerbit/cache";
+
+type LeaderMap = Map<string, { intersecting: boolean }>;
+
+const cloneLeaders = (leaders: LeaderMap): LeaderMap => {
+	const cloned: LeaderMap = new Map();
+	for (const [hash, meta] of leaders) {
+		cloned.set(hash, { intersecting: meta.intersecting });
+	}
+	return cloned;
+};
+
+/**
+ * Memoizes leader-election plans between replication-topology changes.
+ *
+ * Coherence is structural, not temporal: a private monotonic version is
+ * bumped (and the store wiped) on every invalidation, and an async fill is
+ * only stored when the version captured before the computation still holds —
+ * a plan computed across a topology change is returned to its caller but
+ * never cached. The TTL bounds wall-clock maturity flips that fire no event
+ * (a range re-crossing maturity after the dynamic roleAge regrew has no
+ * pending timer), so it is load-bearing for that edge.
+ *
+ * Callers mutate returned maps and their value objects, so every hit and
+ * every store clones.
+ */
+export class LeaderPlanCache {
+	private cache: Cache<{ leaders: LeaderMap }>;
+	private version = 0;
+
+	constructor(options: { max: number; ttl: number }) {
+		this.cache = new Cache(options);
+	}
+
+	invalidate(): void {
+		this.version++;
+		this.cache.clear();
+	}
+
+	get(key: string): LeaderMap | undefined {
+		const entry = this.cache.get(key);
+		return entry ? cloneLeaders(entry.leaders) : undefined;
+	}
+
+	/** Capture the version before an awaited plan computation. */
+	capture(): number {
+		return this.version;
+	}
+
+	/** Store only if no invalidation happened since `capturedVersion`. */
+	put(key: string, leaders: LeaderMap, capturedVersion: number): void {
+		if (capturedVersion !== this.version) {
+			return;
+		}
+		this.cache.add(
+			key,
+			{ leaders: cloneLeaders(leaders) },
+			Math.max(1, leaders.size),
+		);
+	}
+}

package/src/ranges.ts

@@ -126,7 +126,7 @@ export class EntryReplicatedU32 implements EntryReplicated<"u32"> {
 	@field({ type: Uint8Array })
 	private _meta: Uint8Array;
 
-	private _metaResolved: ShallowMeta;
+	private _metaResolved?: ShallowMeta;
 
 	constructor(properties: {
 		coordinates: number[];
@@ -154,13 +154,16 @@ export class EntryReplicatedU32 implements EntryReplicated<"u32"> {
 			properties.meta?.clock.timestamp.wallTime ?? properties.wallTime!;
 		this.hashNumber = properties.hashNumber;
 		this._meta =
-			properties.metaBytes ??
-			serialize(
-				properties.meta instanceof Meta
-					? new ShallowMeta(properties.meta)
-					: properties.meta!,
-			);
-		this._metaResolved = properties.meta as ShallowMeta;
+			properties.metaBytes != null
+				? Uint8Array.from(properties.metaBytes)
+				: Uint8Array.from(
+						serialize(
+							properties.meta instanceof Meta
+								? new ShallowMeta(properties.meta)
+								: properties.meta!,
+						),
+					);
+		this._metaResolved = undefined;
 		this.assignedToRangeBoundary = properties.assignedToRangeBoundary;
 	}
 
@@ -199,7 +202,7 @@ export class EntryReplicatedU64 implements EntryReplicated<"u64"> {
 	@field({ type: Uint8Array })
 	private _meta: Uint8Array;
 
-	private _metaResolved: ShallowMeta;
+	private _metaResolved?: ShallowMeta;
 
 	constructor(properties: {
 		coordinates: bigint[];
@@ -227,13 +230,16 @@ export class EntryReplicatedU64 implements EntryReplicated<"u64"> {
 		this.wallTime =
 			properties.meta?.clock.timestamp.wallTime ?? properties.wallTime!;
 		this._meta =
-			properties.metaBytes ??
-			serialize(
-				properties.meta instanceof Meta
-					? new ShallowMeta(properties.meta)
-					: properties.meta!,
-			);
-		this._metaResolved = properties.meta as ShallowMeta;
+			properties.metaBytes != null
+				? Uint8Array.from(properties.metaBytes)
+				: Uint8Array.from(
+						serialize(
+							properties.meta instanceof Meta
+								? new ShallowMeta(properties.meta)
+								: properties.meta!,
+						),
+					);
+		this._metaResolved = undefined;
 		this.assignedToRangeBoundary = properties.assignedToRangeBoundary;
 	}
 

package/dist/benchmark/adaptive-ingest.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=adaptive-ingest.d.ts.map

package/dist/benchmark/adaptive-ingest.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"adaptive-ingest.d.ts","sourceRoot":"","sources":["../../benchmark/adaptive-ingest.ts"],"names":[],"mappings":""}