@peerbit/shared-log 7.0.10 → 8.0.0

package/lib/esm/role.d.ts

@@ -1,7 +1,4 @@
-export declare const containsPoint: (rect: {
-    offset: number;
-    factor: number;
-}, point: number) => boolean;
+export declare const overlaps: (x1: number, x2: number, y1: number, y2: number) => boolean;
 export declare abstract class Role {
     abstract equals(other: Role): any;
 }

package/lib/esm/role.js

@@ -9,23 +9,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
 };
 var NoType_1, Observer_1, Replicator_1;
 import { field, variant, vec } from "@dao-xyz/borsh";
-export const containsPoint = (rect, point) => {
-    if (rect.factor === 0) {
-        return false;
-    }
-    const start = rect.offset;
-    const endUnwrapped = rect.offset + rect.factor;
-    let end = endUnwrapped;
-    let wrapped = false;
-    if (endUnwrapped > 1) {
-        end = endUnwrapped % 1;
-        wrapped = true;
-    }
-    const inFirstInterval = point >= start && point < Math.min(endUnwrapped, 1);
-    const inSecondInterval = !inFirstInterval && wrapped && point >= 0 && point < end;
-    return inFirstInterval || inSecondInterval;
-};
-const overlaps = (x1, x2, y1, y2) => {
+export const overlaps = (x1, x2, y1, y2) => {
     if (x1 <= y2 && y1 <= x2) {
         return true;
     }

package/lib/esm/role.js.map

@@ -1 +1 @@
-{"version":3,"file":"role.js","sourceRoot":"","sources":["../../src/role.ts"],"names":[],"mappings":";;;;;;;;;;AAAA,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AAErD,MAAM,CAAC,MAAM,aAAa,GAAG,CAC5B,IAAwC,EACxC,KAAa,EACZ,EAAE;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,KAAK,CAAC;IACd,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;IAC1B,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC/C,IAAI,GAAG,GAAG,YAAY,CAAC;IACvB,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QACtB,GAAG,GAAG,YAAY,GAAG,CAAC,CAAC;QACvB,OAAO,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,MAAM,eAAe,GAAG,KAAK,IAAI,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,gBAAgB,GACrB,CAAC,eAAe,IAAI,OAAO,IAAI,KAAK,IAAI,CAAC,IAAI,KAAK,GAAG,GAAG,CAAC;IAE1D,OAAO,eAAe,IAAI,gBAAgB,CAAC;AAC5C,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAG,CAAC,EAAU,EAAE,EAAU,EAAE,EAAU,EAAE,EAAU,EAAE,EAAE;IACnE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AACF,MAAM,OAAgB,IAAI;CAEzB;AAED,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAG5C,IAAM,MAAM,cAAZ,MAAM,MAAO,SAAQ,IAAI;IAC/B,MAAM,CAAC,KAAW;QACjB,OAAO,KAAK,YAAY,QAAM,CAAC;IAChC,CAAC;CACD,CAAA;AAJY,MAAM;IADlB,OAAO,CAAC,CAAC,CAAC;GACE,MAAM,CAIlB;;AAED,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAGlD,IAAM,QAAQ,gBAAd,MAAM,QAAS,SAAQ,IAAI;IACjC,MAAM,CAAC,KAAW;QACjB,OAAO,KAAK,YAAY,UAAQ,CAAC;IAClC,CAAC;CACD,CAAA;AAJY,QAAQ;IADpB,OAAO,CAAC,CAAC,CAAC;GACE,QAAQ,CAIpB;;AAED,MAAM,CAAC,MAAM,uBAAuB,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAE3D,MAAM,OAAO,kBAAkB;IAE9B,SAAS,CAAS;IAGV,eAAe,CAAS;IAGxB,eAAe,CAAS;IAEhC,YAAY,UAIX;QACA,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC;QACjD,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,+CAA+C,GAAG,MAAM,CAAC,CAAC;QAC3E,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,MAAM,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QAClD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,MAAM,CAAC,CAAC;QAEvD,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,+CAA+C,GAAG,MAAM,CAAC,CAAC;QAC3E,CAAC;QACD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,MAAM,CAAC,CAAC;IACxD,CAAC;IAED,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC;IAC1C,CAAC;IAED,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC;IAC1C,CAAC;IAED,QAAQ,CAAC,KAAyB;QACjC,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QACrB,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QACnC,IAAI,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC;QACtB,IAAI,EAAE,GAAG,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QACrC,IAAI,QAAQ,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC;QACb,CAAC;QAED,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;YACtB,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;gBACZ,EAAE,GAAG,CAAC,CAAC;gBACP,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YACb,CAAC;YACD,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;gBACZ,EAAE,GAAG,CAAC,CAAC;gBACP,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YACb,CAAC;YACD,IAAI,QAAQ,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC;gBAC9B,OAAO,IAAI,CAAC;YACb,CAAC;QACF,CAAC;QACD,OAAO,KAAK,CAAC;IACd,CAAC;CACD;AA3DA;IADC,KAAK,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;;qDACL;AAGV;IADP,KAAK,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;;2DACS;AAGxB;IADP,KAAK,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;;2DACS;AAwD1B,IAAM,UAAU,kBAAhB,MAAM,UAAW,SAAQ,IAAI;IAEnC,QAAQ,CAAuB;IAE/B,YAAY,UAIX;QACA,KAAK,EAAE,CAAC;QACR,MAAM,OAAO,GAAuB,IAAI,kBAAkB,CAAC,UAAU,CAAC,CAAC;QACvE,IAAI,CAAC,QAAQ,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC;IACjC,CAAC;IAED,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC;IACjC,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC;IACpC,CAAC;IAED,MAAM,CAAC,KAAW;QACjB,OAAO,CACN,KAAK,YAAY,YAAU;YAC3B,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM;YAC5B,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAC5B,CAAC;IACH,CAAC;CACD,CAAA;AA/BA;IADC,KAAK,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,kBAAkB,CAAC,EAAE,CAAC;;4CACV;AAFnB,UAAU;IADtB,OAAO,CAAC,CAAC,CAAC;;GACE,UAAU,CAiCtB"}
+{"version":3,"file":"role.js","sourceRoot":"","sources":["../../src/role.ts"],"names":[],"mappings":";;;;;;;;;;AAAA,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AAErD,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAU,EAAE,EAAU,EAAE,EAAU,EAAE,EAAU,EAAE,EAAE;IAC1E,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,OAAgB,IAAI;CAEzB;AAED,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAG5C,IAAM,MAAM,cAAZ,MAAM,MAAO,SAAQ,IAAI;IAC/B,MAAM,CAAC,KAAW;QACjB,OAAO,KAAK,YAAY,QAAM,CAAC;IAChC,CAAC;CACD,CAAA;AAJY,MAAM;IADlB,OAAO,CAAC,CAAC,CAAC;GACE,MAAM,CAIlB;;AAED,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAGlD,IAAM,QAAQ,gBAAd,MAAM,QAAS,SAAQ,IAAI;IACjC,MAAM,CAAC,KAAW;QACjB,OAAO,KAAK,YAAY,UAAQ,CAAC;IAClC,CAAC;CACD,CAAA;AAJY,QAAQ;IADpB,OAAO,CAAC,CAAC,CAAC;GACE,QAAQ,CAIpB;;AAED,MAAM,CAAC,MAAM,uBAAuB,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAE3D,MAAM,OAAO,kBAAkB;IAE9B,SAAS,CAAS;IAGV,eAAe,CAAS;IAGxB,eAAe,CAAS;IAEhC,YAAY,UAIX;QACA,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC;QACjD,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,+CAA+C,GAAG,MAAM,CAAC,CAAC;QAC3E,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,MAAM,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QAClD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,MAAM,CAAC,CAAC;QAEvD,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,+CAA+C,GAAG,MAAM,CAAC,CAAC;QAC3E,CAAC;QACD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,MAAM,CAAC,CAAC;IACxD,CAAC;IAED,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC;IAC1C,CAAC;IAED,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC;IAC1C,CAAC;IAED,QAAQ,CAAC,KAAyB;QACjC,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QACrB,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QACnC,IAAI,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC;QACtB,IAAI,EAAE,GAAG,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QACrC,IAAI,QAAQ,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC;QACb,CAAC;QAED,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;YACtB,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;gBACZ,EAAE,GAAG,CAAC,CAAC;gBACP,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YACb,CAAC;YACD,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;gBACZ,EAAE,GAAG,CAAC,CAAC;gBACP,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YACb,CAAC;YACD,IAAI,QAAQ,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC;gBAC9B,OAAO,IAAI,CAAC;YACb,CAAC;QACF,CAAC;QACD,OAAO,KAAK,CAAC;IACd,CAAC;CACD;AA3DA;IADC,KAAK,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;;qDACL;AAGV;IADP,KAAK,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;;2DACS;AAGxB;IADP,KAAK,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;;2DACS;AAwD1B,IAAM,UAAU,kBAAhB,MAAM,UAAW,SAAQ,IAAI;IAEnC,QAAQ,CAAuB;IAE/B,YAAY,UAIX;QACA,KAAK,EAAE,CAAC;QACR,MAAM,OAAO,GAAuB,IAAI,kBAAkB,CAAC,UAAU,CAAC,CAAC;QACvE,IAAI,CAAC,QAAQ,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC;IACjC,CAAC;IAED,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC;IACjC,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC;IACpC,CAAC;IAED,MAAM,CAAC,KAAW;QACjB,OAAO,CACN,KAAK,YAAY,YAAU;YAC3B,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM;YAC5B,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAC5B,CAAC;IACH,CAAC;CACD,CAAA;AA/BA;IADC,KAAK,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,kBAAkB,CAAC,EAAE,CAAC;;4CACV;AAFnB,UAAU;IADtB,OAAO,CAAC,CAAC,CAAC;;GACE,UAAU,CAiCtB"}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@peerbit/shared-log",
-	"version": "7.0.10",
+	"version": "8.0.0",
 	"description": "Shared log",
 	"type": "module",
 	"sideEffects": false,
@@ -31,15 +31,15 @@
 	"license": "MIT",
 	"dependencies": {
 		"@dao-xyz/borsh": "^5.2.1",
-		"@peerbit/log": "3.0.25",
+		"@peerbit/log": "3.0.27",
 		"@peerbit/logger": "1.0.2",
-		"@peerbit/program": "3.0.20",
-		"@peerbit/rpc": "3.0.24",
+		"@peerbit/program": "4.0.0",
+		"@peerbit/rpc": "4.0.0",
 		"@peerbit/time": "2.0.6",
 		"p-debounce": "^4.0.0"
 	},
 	"devDependencies": {
-		"@peerbit/test-utils": "^2.0.24"
+		"@peerbit/test-utils": "^2.0.26"
 	},
-	"gitHead": "2ba4c5e5d682895614713cbef6c779f44508291d"
+	"gitHead": "9536898b1f74d42c4f97183b864945fa29422d3e"
 }

package/src/exchange-heads.ts

@@ -86,7 +86,7 @@ export class ResponseIPrune extends TransportMessage {
 	}
 }
 
-const MAX_EXCHANGE_MESSAGE_SIZE = 5e6; // 5mb (since stream limits are 10mb)
+const MAX_EXCHANGE_MESSAGE_SIZE = 1e5; // 100kb. Too large size might not be faster (even if we can do 5mb)
 
 export const createExchangeHeadsMessages = async (
 	log: Log<any>,

package/src/index.ts

@@ -51,7 +51,6 @@ import yallist from "yallist";
 import {
 	AcknowledgeDelivery,
 	DeliveryMode,
-	SeekDelivery,
 	SilentDelivery,
 	NotStartedError
 } from "@peerbit/stream-interface";
@@ -62,7 +61,7 @@ import { PIDReplicationController } from "./pid.js";
 export * from "./replication.js";
 import PQueue from "p-queue";
 import { CPUUsage, CPUUsageIntervalLag } from "./cpu.js";
-import { collectNodesAroundPoint, getCover } from "./ranges.js";
+import { getCoverSet, getSamples, isMatured } from "./ranges.js";
 export { type CPUUsage, CPUUsageIntervalLag };
 export { Observer, Replicator, Role };
 
@@ -92,20 +91,20 @@ export type ReplicationLimitsOptions =
 
 type StringRoleOptions = "observer" | "replicator";
 
-type AdaptiveReplicatorOptions = {
+export type AdaptiveReplicatorOptions = {
 	type: "replicator";
 	limits?: {
-		memory?: number;
+		storage?: number;
 		cpu?: number | { max: number; monitor?: CPUUsage };
 	};
 };
 
-type FixedReplicatorOptions = {
+export type FixedReplicatorOptions = {
 	type: "replicator";
 	factor: number;
 };
 
-type ObserverType = {
+export type ObserverType = {
 	type: "observer";
 };
 
@@ -167,7 +166,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 
 	// options
 	private _role: Observer | Replicator;
-	private _roleOptions: AdaptiveReplicatorOptions | Observer | Replicator;
+	private _roleConfig: AdaptiveReplicatorOptions | Observer | Replicator;
 	private _sortedPeersCache: yallist<ReplicatorRect> | undefined;
 	private _totalParticipation: number;
 	private _gidPeersHistory: Map<string, Set<string>>;
@@ -204,6 +203,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 	private remoteBlocks: RemoteBlocks;
 
 	private openTime: number;
+	private oldestOpenTime: number;
 
 	private sync?: (entry: Entry<T>) => boolean;
 
@@ -219,6 +219,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 	private syncMoreInterval?: ReturnType<typeof setTimeout>;
 	private syncInFlightQueue: Map<string, PublicSignKey[]>;
 	private syncInFlightQueueInverted: Map<string, Set<string>>;
+	private syncInFlight: Map<string, Map<string, { timestamp: number }>>;
 
 	replicas: ReplicationLimits;
 
@@ -233,10 +234,20 @@ export class SharedLog<T = Uint8Array> extends Program<
 		this.rpc = new RPC();
 	}
 
+	/**
+	 * Returns the current role
+	 */
 	get role(): Observer | Replicator {
 		return this._role;
 	}
 
+	/**
+	 * Return the
+	 */
+	get roleConfig(): Observer | Replicator | AdaptiveReplicatorOptions {
+		return this._roleConfig;
+	}
+
 	get totalParticipation(): number {
 		return this._totalParticipation;
 	}
@@ -261,9 +272,9 @@ export class SharedLog<T = Uint8Array> extends Program<
 			this.replicationController = new PIDReplicationController(
 				this.node.identity.publicKey.hashcode(),
 				{
-					memory:
-						options?.limits?.memory != null
-							? { max: options?.limits?.memory }
+					storage:
+						options?.limits?.storage != null
+							? { max: options?.limits?.storage }
 							: undefined,
 					cpu:
 						options?.limits?.cpu != null
@@ -289,49 +300,49 @@ export class SharedLog<T = Uint8Array> extends Program<
 		if (options instanceof Observer || options instanceof Replicator) {
 			throw new Error("Unsupported role option type");
 		} else if (options === "observer") {
-			this._roleOptions = new Observer();
+			this._roleConfig = new Observer();
 		} else if (options === "replicator") {
 			setupDebouncedRebalancing();
-			this._roleOptions = { type: options };
+			this._roleConfig = { type: options };
 		} else if (options) {
 			if (options.type === "replicator") {
 				if (isAdaptiveReplicatorOption(options)) {
 					setupDebouncedRebalancing(options);
-					this._roleOptions = options;
+					this._roleConfig = options;
 				} else {
-					this._roleOptions = new Replicator({
+					this._roleConfig = new Replicator({
 						factor: options.factor,
-						offset: hashToUniformNumber(this.node.identity.publicKey.bytes)
+						offset: this.getReplicationOffset()
 					});
 				}
 			} else {
-				this._roleOptions = new Observer();
+				this._roleConfig = new Observer();
 			}
 		} else {
 			// Default option
 			setupDebouncedRebalancing();
-			this._roleOptions = { type: "replicator" };
+			this._roleConfig = { type: "replicator" };
 		}
 
 		// setup the initial role
 
 		if (
-			this._roleOptions instanceof Replicator ||
-			this._roleOptions instanceof Observer
+			this._roleConfig instanceof Replicator ||
+			this._roleConfig instanceof Observer
 		) {
-			this._role = this._roleOptions as Replicator | Observer;
+			this._role = this._roleConfig as Replicator | Observer;
 		} else {
 			// initial role in a dynamic setup
 
-			if (this._roleOptions?.limits) {
+			if (this._roleConfig?.limits) {
 				this._role = new Replicator({
 					factor: this._role instanceof Replicator ? this._role.factor : 0,
-					offset: hashToUniformNumber(this.node.identity.publicKey.bytes)
+					offset: this.getReplicationOffset()
 				});
 			} else {
 				this._role = new Replicator({
 					factor: this._role instanceof Replicator ? this._role.factor : 1,
-					offset: hashToUniformNumber(this.node.identity.publicKey.bytes)
+					offset: this.getReplicationOffset()
 				});
 			}
 		}
@@ -354,12 +365,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 		);
 
 		await this.rpc.subscribe();
-
-		await this.rpc.send(new ResponseRoleMessage({ role: this._role }), {
-			mode: new SeekDelivery({
-				redundancy: 1
-			})
-		});
+		await this.rpc.send(new ResponseRoleMessage({ role: this._role }));
 
 		if (onRoleChange && changed !== "none") {
 			this.onRoleChange(this._role, this.node.identity.publicKey);
@@ -418,6 +424,15 @@ export class SharedLog<T = Uint8Array> extends Program<
 					}
 					leaders = newAndOldLeaders;
 				}
+				let set = this._gidPeersHistory.get(result.entry.meta.gid);
+				if (!set) {
+					set = new Set(leaders);
+					this._gidPeersHistory.set(result.entry.meta.gid, set);
+				} else {
+					for (const receiver of leaders) {
+						set.add(receiver);
+					}
+				}
 				mode = isLeader
 					? new SilentDelivery({ redundancy: 1, to: leaders })
 					: new AcknowledgeDelivery({ redundancy: 1, to: leaders });
@@ -453,7 +468,9 @@ export class SharedLog<T = Uint8Array> extends Program<
 		this.latestRoleMessages = new Map();
 		this.syncInFlightQueue = new Map();
 		this.syncInFlightQueueInverted = new Map();
+		this.syncInFlight = new Map();
 		this.openTime = +new Date();
+		this.oldestOpenTime = this.openTime;
 		this.timeUntilRoleMaturity =
 			options?.timeUntilRoleMaturity || WAIT_FOR_ROLE_MATURITY;
 		this.waitForReplicatorTimeout =
@@ -467,7 +484,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 		this.setupRole(options?.role);
 
 		const id = sha256Base64Sync(this.log.id);
-		const storage = await this.node.memory.sublevel(id);
+		const storage = await this.node.storage.sublevel(id);
 
 		const localBlocks = await new AnyBlockStore(
 			await storage.sublevel("blocks")
@@ -590,21 +607,25 @@ export class SharedLog<T = Uint8Array> extends Program<
 					this.syncInFlightQueue.delete(key);
 				}
 			}
-			this.rpc
-				.send(
-					new ResponseMaybeSync({
-						hashes: requestHashes
-					}),
-					{
-						mode: new SilentDelivery({ to: from, redundancy: 1 })
-					}
-				)
-				.finally(() => {
-					if (this.closed) {
-						return;
+
+			const nowMin10s = +new Date() - 1e4;
+			for (const [key, map] of this.syncInFlight) {
+				// cleanup "old" missing syncs
+				for (const [hash, { timestamp }] of map) {
+					if (timestamp < nowMin10s) {
+						map.delete(hash);
 					}
-					this.syncMoreInterval = setTimeout(requestSync, 1e4);
-				});
+				}
+				if (map.size === 0) {
+					this.syncInFlight.delete(key);
+				}
+			}
+			this.requestSync(requestHashes, from).finally(() => {
+				if (this.closed) {
+					return;
+				}
+				this.syncMoreInterval = setTimeout(requestSync, 1e4);
+			});
 		};
 		requestSync();
 	}
@@ -668,6 +689,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 		this._pendingIHave.clear();
 		this.syncInFlightQueue.clear();
 		this.syncInFlightQueueInverted.clear();
+		this.syncInFlight.clear();
 		this.latestRoleMessages.clear();
 		this._gidPeersHistory.clear();
 
@@ -824,6 +846,17 @@ export class SharedLog<T = Uint8Array> extends Program<
 						this.rebalanceParticipationDebounced?.();
 					}
 
+					/// we clear sync in flight here because we want to join before that, so that entries are totally accounted for
+					for (const head of heads) {
+						const set = this.syncInFlight.get(context.from.hashcode());
+						if (set) {
+							set.delete(head.entry.hash);
+							if (set?.size === 0) {
+								this.syncInFlight.delete(context.from.hashcode());
+							}
+						}
+					}
+
 					if (maybeDelete) {
 						for (const entries of maybeDelete as EntryWithRefs<any>[][]) {
 							const headsWithGid = this.log.headsIndex.gids.get(
@@ -930,18 +963,11 @@ export class SharedLog<T = Uint8Array> extends Program<
 						inverted.add(hash);
 					} else if (!this.log.has(hash)) {
 						this.syncInFlightQueue.set(hash, []);
-						requestHashes.push(hash);
+						requestHashes.push(hash); // request immediately (first time we have seen this hash)
 					}
 				}
 
-				await this.rpc.send(
-					new ResponseMaybeSync({
-						hashes: requestHashes
-					}),
-					{
-						mode: new SilentDelivery({ to: [context.from], redundancy: 1 })
-					}
-				);
+				await this.requestSync(requestHashes, [context.from.hashcode()]);
 			} else if (msg instanceof ResponseMaybeSync) {
 				// TODO better choice of step size
 				const entries = (
@@ -1042,10 +1068,12 @@ export class SharedLog<T = Uint8Array> extends Program<
 	async waitForReplicator(...keys: PublicSignKey[]) {
 		const check = () => {
 			for (const k of keys) {
+				const rect = this.getReplicatorsSorted()
+					?.toArray()
+					?.find((x) => x.publicKey.equals(k));
 				if (
-					!this.getReplicatorsSorted()
-						?.toArray()
-						?.find((x) => x.publicKey.equals(k))
+					!rect ||
+					!isMatured(rect.role, +new Date(), this.getDefaultMinRoleAge())
 				) {
 					return false;
 				}
@@ -1069,6 +1097,10 @@ export class SharedLog<T = Uint8Array> extends Program<
 		return !!isLeader;
 	}
 
+	private getReplicationOffset() {
+		return hashToUniformNumber(this.node.identity.publicKey.bytes);
+	}
+
 	private async waitForIsLeader(
 		slot: { toString(): string },
 		numberOfLeaders: number,
@@ -1137,6 +1169,17 @@ export class SharedLog<T = Uint8Array> extends Program<
 		return this.findLeadersFromUniformNumber(cursor, numberOfLeaders, options);
 	}
 
+	getDefaultMinRoleAge(): number {
+		const now = +new Date();
+		const replLength = this.getReplicatorsSorted()!.length;
+		const diffToOldest =
+			replLength > 1 ? now - this.oldestOpenTime - 1 : Number.MAX_SAFE_INTEGER;
+		return Math.min(
+			this.timeUntilRoleMaturity,
+			diffToOldest,
+			(this.timeUntilRoleMaturity * Math.log(replLength)) / 3
+		); // / 3 so that if 2 replicators and timeUntilRoleMaturity = 1e4 the result will be 1
+	}
 	private findLeadersFromUniformNumber(
 		cursor: number,
 		numberOfLeaders: number,
@@ -1144,33 +1187,26 @@ export class SharedLog<T = Uint8Array> extends Program<
 			roleAge?: number;
 		}
 	) {
-		const leaders: Set<string> = new Set();
-		const width = 1;
-		const peers = this.getReplicatorsSorted();
-		if (!peers || peers?.length === 0) {
-			return [];
-		}
-		numberOfLeaders = Math.min(numberOfLeaders, peers.length);
+		const roleAge = options?.roleAge ?? this.getDefaultMinRoleAge(); // TODO -500 as is added so that i f someone else is just as new as us, then we treat them as mature as us. without -500 we might be slower syncing if two nodes starts almost at the same time
 
-		const t = +new Date();
-		const roleAge =
-			options?.roleAge ??
-			Math.min(this.timeUntilRoleMaturity, +new Date() - this.openTime - 500); // TODO -500 as is added so that i f someone else is just as new as us, then we treat them as mature as us. without -500 we might be slower syncing if two nodes starts almost at the same time
-
-		for (let i = 0; i < numberOfLeaders; i++) {
-			const point = ((cursor + i / numberOfLeaders) % 1) * width;
-			const currentNode = peers.head;
-			collectNodesAroundPoint(t, roleAge, peers, currentNode, leaders, point);
-		}
+		const sampes = getSamples(
+			cursor,
+			this.getReplicatorsSorted()!,
+			numberOfLeaders,
+			roleAge,
+			this.role instanceof Replicator && this.role.factor === 0
+				? this.node.identity.publicKey.hashcode()
+				: undefined
+		);
 
-		return [...leaders];
+		return sampes;
 	}
 
 	/**
 	 *
 	 * @returns groups where at least one in any group will have the entry you are looking for
 	 */
-	getReplicatorUnion(roleAge: number = this.timeUntilRoleMaturity) {
+	getReplicatorUnion(roleAge: number = this.getDefaultMinRoleAge()) {
 		if (this.closed === true) {
 			throw new Error("Closed");
 		}
@@ -1192,12 +1228,18 @@ export class SharedLog<T = Uint8Array> extends Program<
 		// the entry we are looking for
 		const coveringWidth = width / minReplicas;
 
-		return getCover(
+		const set = getCoverSet(
 			coveringWidth,
 			peers,
 			roleAge,
 			this.role instanceof Replicator ? this.node.identity.publicKey : undefined
 		);
+
+		// add all in flight
+		for (const [key, _] of this.syncInFlight) {
+			set.add(key);
+		}
+		return [...set];
 	}
 
 	async replicator(
@@ -1255,7 +1297,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 			await this.rebalanceParticipationDebounced?.(); /* await this.rebalanceParticipation(false); */
 			if (update.changed === "added") {
 				await this.rpc.send(new ResponseRoleMessage({ role: this._role }), {
-					mode: new SeekDelivery({
+					mode: new SilentDelivery({
 						to: [publicKey.hashcode()],
 						redundancy: 1
 					})
@@ -1303,6 +1345,10 @@ export class SharedLog<T = Uint8Array> extends Program<
 				// TODO should we remove replicators if they are already added?
 				return { changed: "none" };
 			}
+			this.oldestOpenTime = Math.min(
+				this.oldestOpenTime,
+				Number(role.timestamp)
+			);
 
 			// insert or if already there do nothing
 			const rect: ReplicatorRect = {
@@ -1380,13 +1426,29 @@ export class SharedLog<T = Uint8Array> extends Program<
 			for (const [_a, b] of this._gidPeersHistory) {
 				b.delete(publicKey.hashcode());
 			}
+			this.syncInFlight.delete(publicKey.hashcode());
+			const waitingHashes = this.syncInFlightQueueInverted.get(
+				publicKey.hashcode()
+			);
+			if (waitingHashes) {
+				for (const hash of waitingHashes) {
+					let arr = this.syncInFlightQueue.get(hash);
+					if (arr) {
+						arr = arr.filter((x) => !x.equals(publicKey));
+					}
+					if (this.syncInFlightQueue.size === 0) {
+						this.syncInFlightQueue.delete(hash);
+					}
+				}
+			}
+			this.syncInFlightQueueInverted.delete(publicKey.hashcode());
 		}
 
 		if (subscribed) {
 			if (this.role instanceof Replicator) {
 				this.rpc
 					.send(new ResponseRoleMessage({ role: this._role }), {
-						mode: new SeekDelivery({ redundancy: 1, to: [publicKey] })
+						mode: new SilentDelivery({ redundancy: 1, to: [publicKey] })
 					})
 					.catch((e) => logger.error(e.toString()));
 			}
@@ -1637,6 +1699,29 @@ export class SharedLog<T = Uint8Array> extends Program<
 		return changed;
 	}
 
+	private async requestSync(hashes: string[], to: Set<string> | string[]) {
+		const now = +new Date();
+		for (const node of to) {
+			let map = this.syncInFlight.get(node);
+			if (!map) {
+				map = new Map();
+				this.syncInFlight.set(node, map);
+			}
+			for (const hash of hashes) {
+				map.set(hash, { timestamp: now });
+			}
+		}
+
+		await this.rpc.send(
+			new ResponseMaybeSync({
+				hashes: hashes
+			}),
+			{
+				mode: new SilentDelivery({ to, redundancy: 1 })
+			}
+		);
+	}
+
 	async _onUnsubscription(evt: CustomEvent<UnsubcriptionEvent>) {
 		logger.debug(
 			`Peer disconnected '${evt.detail.from.hashcode()}' from '${JSON.stringify(
@@ -1715,13 +1800,13 @@ export class SharedLog<T = Uint8Array> extends Program<
 		}
 
 		// The role is fixed (no changes depending on memory usage or peer count etc)
-		if (this._roleOptions instanceof Role) {
+		if (this._roleConfig instanceof Role) {
 			return false;
 		}
 
 		// TODO second condition: what if the current role is Observer?
 		if (
-			this._roleOptions.type == "replicator" &&
+			this._roleConfig.type == "replicator" &&
 			this._role instanceof Replicator
 		) {
 			const peers = this.getReplicatorsSorted();

package/src/pid.ts

@@ -1,10 +1,3 @@
-type ReplicationErrorFunction = (objectives: {
-	coverage: number;
-	balance: number;
-	memory: number;
-	cpu: number;
-}) => number;
-
 export class PIDReplicationController {
 	integral: number;
 	prevError: number;
@@ -18,14 +11,14 @@ export class PIDReplicationController {
 	constructor(
 		readonly id: string,
 		options: {
-			memory?: { max: number };
+			storage?: { max: number };
 			cpu?: { max: number };
 			kp?: number;
 			ki?: number;
 			kd?: number;
 		} = {}
 	) {
-		const { memory, cpu, kp = 0.7, ki = 0.025, kd = 0.05 } = options;
+		const { storage: memory, cpu, kp = 0.7, ki = 0.025, kd = 0.05 } = options;
 		this.kp = kp;
 		this.ki = ki;
 		this.kd = kd;