@peerbit/shared-log 12.2.0-e38a67b → 12.3.0

package/dist/benchmark/pid-convergence.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=pid-convergence.d.ts.map

package/dist/benchmark/pid-convergence.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pid-convergence.d.ts","sourceRoot":"","sources":["../../benchmark/pid-convergence.ts"],"names":[],"mappings":""}

package/dist/benchmark/pid-convergence.js

@@ -0,0 +1,138 @@
+import { Bench } from "tinybench";
+import { PIDReplicationController } from "../src/pid.js";
+// Benchmarks PID convergence speed in a simplified simulation model.
+//
+// Run with:
+//   cd packages/programs/data/shared-log
+//   PID_WARMUP=2 PID_ITERATIONS=50 PID_RUNS=200 node --loader ts-node/esm ./benchmark/pid-convergence.ts
+//
+// Notes:
+// - This is a *model* benchmark: it does not simulate network, pruning, or real memory measurements.
+// - It is intended to catch regressions/improvements in the controller dynamics and step cost.
+const parseNumberList = (value, defaults) => {
+    if (!value)
+        return defaults;
+    const parsed = value
+        .split(",")
+        .map((x) => Number.parseInt(x.trim(), 10))
+        .filter((x) => Number.isFinite(x) && x > 0);
+    return parsed.length > 0 ? parsed : defaults;
+};
+const warmupIterations = Number.parseInt(process.env.PID_WARMUP || "2", 10);
+const iterations = Number.parseInt(process.env.PID_ITERATIONS || "50", 10);
+const runsPerTask = Number.parseInt(process.env.PID_RUNS || "200", 10);
+const maxSteps = Number.parseInt(process.env.PID_MAX_STEPS || "200", 10);
+const peersList = parseNumberList(process.env.PID_PEERS, [2, 3, 5]);
+const clamp01 = (value) => Math.max(0, Math.min(1, value));
+const simulateConvergence = (properties) => {
+    const { controllers, initialFactors, cpuUsage, memoryUsage, expectedFactor, epsilon, } = properties;
+    for (const controller of controllers) {
+        controller.reset();
+    }
+    let factors = initialFactors.slice();
+    const peerCount = factors.length;
+    for (let step = 0; step < maxSteps; step++) {
+        const totalFactor = clamp01(factors.reduce((acc, f) => acc + f, 0));
+        const next = [];
+        for (let peerIndex = 0; peerIndex < peerCount; peerIndex++) {
+            next[peerIndex] = controllers[peerIndex].step({
+                memoryUsage: memoryUsage(peerIndex, factors[peerIndex]),
+                currentFactor: factors[peerIndex],
+                totalFactor,
+                peerCount,
+                cpuUsage: cpuUsage(peerIndex),
+            });
+        }
+        factors = next;
+        let ok = true;
+        for (let peerIndex = 0; peerIndex < peerCount; peerIndex++) {
+            const target = expectedFactor(peerIndex, peerCount);
+            if (Math.abs(factors[peerIndex] - target) > epsilon) {
+                ok = false;
+                break;
+            }
+        }
+        if (ok && Math.abs(factors.reduce((acc, f) => acc + f, 0) - 1) <= epsilon) {
+            return { converged: true, steps: step + 1 };
+        }
+    }
+    return { converged: false, steps: maxSteps };
+};
+const suite = new Bench({
+    name: "pid-convergence",
+    warmupIterations: Number.isFinite(warmupIterations) ? warmupIterations : 0,
+    iterations: Number.isFinite(iterations) ? iterations : undefined,
+});
+const makeControllers = (peerCount, options) => Array.from({ length: peerCount }, (_, i) => new PIDReplicationController(`p${i}`, options(i)));
+for (const peerCount of peersList) {
+    const controllers = makeControllers(peerCount, () => ({}));
+    const epsilon = 0.02;
+    const target = (_i, n) => 1 / n;
+    const initialFactors = Array.from({ length: peerCount }, () => 0);
+    suite.add(`even (peers=${peerCount})`, () => {
+        for (let i = 0; i < runsPerTask; i++) {
+            const res = simulateConvergence({
+                controllers,
+                initialFactors,
+                cpuUsage: () => 0,
+                memoryUsage: () => 0,
+                expectedFactor: target,
+                epsilon,
+            });
+            if (!res.converged) {
+                throw new Error(`did not converge (peers=${peerCount})`);
+            }
+        }
+    });
+}
+// A constrained scenario (2 peers): peer 0 has a memory cap (ratio), peer 1 is unconstrained.
+{
+    const peerCount = 2;
+    const totalSize = 1_000_000; // bytes
+    const memoryLimit = 250_000; // 25% of total
+    const limitRatio = memoryLimit / totalSize;
+    const epsilon = 0.03;
+    const controllers = makeControllers(peerCount, (peerIndex) => peerIndex === 0 ? { storage: { max: memoryLimit } } : {});
+    const initialFactors = [0.5, 0.5];
+    const expectedFactor = (peerIndex) => peerIndex === 0 ? limitRatio : 1 - limitRatio;
+    suite.add("memory-limited (2 peers, 25% cap)", () => {
+        for (let i = 0; i < runsPerTask; i++) {
+            const res = simulateConvergence({
+                controllers,
+                initialFactors,
+                cpuUsage: () => 0,
+                memoryUsage: (peerIndex, factor) => peerIndex === 0 ? totalSize * factor : 0,
+                expectedFactor,
+                epsilon,
+            });
+            if (!res.converged) {
+                throw new Error("did not converge (memory-limited)");
+            }
+        }
+    });
+}
+await suite.run();
+if (process.env.BENCH_JSON === "1") {
+    const tasks = suite.tasks.map((task) => ({
+        name: task.name,
+        hz: task.result?.hz ?? null,
+        mean_ms: task.result?.mean ?? null,
+        rme: task.result?.rme ?? null,
+        samples: task.result?.samples?.length ?? null,
+    }));
+    process.stdout.write(JSON.stringify({
+        name: suite.name,
+        tasks,
+        meta: {
+            peersList,
+            warmupIterations,
+            iterations,
+            runsPerTask,
+            maxSteps,
+        },
+    }, null, 2));
+}
+else {
+    console.table(suite.table());
+}
+//# sourceMappingURL=pid-convergence.js.map

package/dist/benchmark/pid-convergence.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pid-convergence.js","sourceRoot":"","sources":["../../benchmark/pid-convergence.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAC;AAEzD,qEAAqE;AACrE,EAAE;AACF,YAAY;AACZ,yCAAyC;AACzC,yGAAyG;AACzG,EAAE;AACF,SAAS;AACT,qGAAqG;AACrG,+FAA+F;AAE/F,MAAM,eAAe,GAAG,CAAC,KAAyB,EAAE,QAAkB,EAAE,EAAE;IACzE,IAAI,CAAC,KAAK;QAAE,OAAO,QAAQ,CAAC;IAC5B,MAAM,MAAM,GAAG,KAAK;SAClB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;SACzC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7C,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC9C,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;AAC5E,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;AAC3E,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,KAAK,EAAE,EAAE,CAAC,CAAC;AACvE,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,KAAK,EAAE,EAAE,CAAC,CAAC;AACzE,MAAM,SAAS,GAAG,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAEpE,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AAEnE,MAAM,mBAAmB,GAAG,CAAC,UAO5B,EAAE,EAAE;IACJ,MAAM,EACL,WAAW,EACX,cAAc,EACd,QAAQ,EACR,WAAW,EACX,cAAc,EACd,OAAO,GACP,GAAG,UAAU,CAAC;IAEf,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACtC,UAAU,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC;IAED,IAAI,OAAO,GAAG,cAAc,CAAC,KAAK,EAAE,CAAC;IACrC,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC;IAEjC,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;QAC5C,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACpE,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,SAAS,EAAE,SAAS,EAAE,EAAE,CAAC;YAC5D,IAAI,CAAC,SAAS,CAAC,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC;gBAC7C,WAAW,EAAE,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;gBACvD,aAAa,EAAE,OAAO,CAAC,SAAS,CAAC;gBACjC,WAAW;gBACX,SAAS;gBACT,QAAQ,EAAE,QAAQ,CAAC,SAAS,CAAC;aAC7B,CAAC,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,IAAI,CAAC;QAEf,IAAI,EAAE,GAAG,IAAI,CAAC;QACd,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,SAAS,EAAE,SAAS,EAAE,EAAE,CAAC;YAC5D,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YACpD,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC;gBACrD,EAAE,GAAG,KAAK,CAAC;gBACX,MAAM;YACP,CAAC;QACF,CAAC;QACD,IAAI,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC;YAC3E,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,GAAG,CAAC,EAAE,CAAC;QAC7C,CAAC;IACF,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AAC9C,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC;IACvB,IAAI,EAAE,iBAAiB;IACvB,gBAAgB,EAAE,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;IAC1E,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;CAChE,CAAC,CAAC;AAEH,MAAM,eAAe,GAAG,CACvB,SAAiB,EACjB,OAAyF,EACxF,EAAE,CACH,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,wBAAwB,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAEhG,KAAK,MAAM,SAAS,IAAI,SAAS,EAAE,CAAC;IACnC,MAAM,WAAW,GAAG,eAAe,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,IAAI,CAAC;IACrB,MAAM,MAAM,GAAG,CAAC,EAAU,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;IAChD,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAElE,KAAK,CAAC,GAAG,CAAC,eAAe,SAAS,GAAG,EAAE,GAAG,EAAE;QAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,GAAG,GAAG,mBAAmB,CAAC;gBAC/B,WAAW;gBACX,cAAc;gBACd,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;gBACjB,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;gBACpB,cAAc,EAAE,MAAM;gBACtB,OAAO;aACP,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,2BAA2B,SAAS,GAAG,CAAC,CAAC;YAC1D,CAAC;QACF,CAAC;IACF,CAAC,CAAC,CAAC;AACJ,CAAC;AAED,8FAA8F;AAC9F,CAAC;IACA,MAAM,SAAS,GAAG,CAAC,CAAC;IACpB,MAAM,SAAS,GAAG,SAAS,CAAC,CAAC,QAAQ;IACrC,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,eAAe;IAC5C,MAAM,UAAU,GAAG,WAAW,GAAG,SAAS,CAAC;IAC3C,MAAM,OAAO,GAAG,IAAI,CAAC;IAErB,MAAM,WAAW,GAAG,eAAe,CAAC,SAAS,EAAE,CAAC,SAAS,EAAE,EAAE,CAC5D,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CACxD,CAAC;IAEF,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAClC,MAAM,cAAc,GAAG,CAAC,SAAiB,EAAE,EAAE,CAC5C,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC;IAE/C,KAAK,CAAC,GAAG,CAAC,mCAAmC,EAAE,GAAG,EAAE;QACnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,GAAG,GAAG,mBAAmB,CAAC;gBAC/B,WAAW;gBACX,cAAc;gBACd,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;gBACjB,WAAW,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,CAClC,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACzC,cAAc;gBACd,OAAO;aACP,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YACtD,CAAC;QACF,CAAC;IACF,CAAC,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC;AAElB,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACxC,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,IAAI;QAC3B,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,IAAI;QAClC,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,IAAI;QAC7B,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,IAAI;KAC7C,CAAC,CAAC,CAAC;IACJ,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,IAAI,CAAC,SAAS,CACb;QACC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,KAAK;QACL,IAAI,EAAE;YACL,SAAS;YACT,gBAAgB;YAChB,UAAU;YACV,WAAW;YACX,QAAQ;SACR;KACD,EACD,IAAI,EACJ,CAAC,CACD,CACD,CAAC;AACH,CAAC;KAAM,CAAC;IACP,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;AAC9B,CAAC"}

package/dist/benchmark/rateless-iblt-sender-startsync.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=rateless-iblt-sender-startsync.d.ts.map

package/dist/benchmark/rateless-iblt-sender-startsync.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rateless-iblt-sender-startsync.d.ts","sourceRoot":"","sources":["../../benchmark/rateless-iblt-sender-startsync.ts"],"names":[],"mappings":""}

package/dist/benchmark/rateless-iblt-sender-startsync.js

@@ -0,0 +1,104 @@
+import { Cache } from "@peerbit/cache";
+import { ready as ribltReady } from "@peerbit/riblt";
+import { Bench } from "tinybench";
+import { createNumbers } from "../src/integers.js";
+import { RatelessIBLTSynchronizer } from "../src/sync/rateless-iblt.js";
+// Benchmarks sender-side StartSync setup cost (sort + encoder build + initial symbols),
+// by timing `RatelessIBLTSynchronizer.onMaybeMissingEntries()` for large batches.
+//
+// Run with:
+//   cd packages/programs/data/shared-log
+//   RIBLT_SIZES="1000,10000,50000" RIBLT_WARMUP=5 RIBLT_ITERATIONS=20 \\
+//     node --loader ts-node/esm ./benchmark/rateless-iblt-sender-startsync.ts
+const parseNumberList = (value, defaults) => {
+    if (!value)
+        return defaults;
+    const parsed = value
+        .split(",")
+        .map((x) => Number.parseInt(x.trim(), 10))
+        .filter((x) => Number.isFinite(x) && x > 0);
+    return parsed.length > 0 ? parsed : defaults;
+};
+const sizes = parseNumberList(process.env.RIBLT_SIZES, [1_000, 10_000, 50_000]);
+const warmupIterations = Number.parseInt(process.env.RIBLT_WARMUP || "5", 10);
+const iterations = Number.parseInt(process.env.RIBLT_ITERATIONS || "20", 10);
+const useRandomHashes = process.env.RIBLT_RANDOM === "1";
+const U64_MASK = (1n << 64n) - 1n;
+const createXorShift64Star = (seed) => {
+    let x = seed & U64_MASK;
+    return () => {
+        x ^= x >> 12n;
+        x ^= (x << 25n) & U64_MASK;
+        x ^= x >> 27n;
+        return (x * 2685821657736338717n) & U64_MASK;
+    };
+};
+const createEntries = (size) => {
+    const entries = new Map();
+    const rand = useRandomHashes
+        ? createXorShift64Star(88172645463393265n ^ BigInt(size))
+        : undefined;
+    for (let i = 0; i < size; i++) {
+        const hash = `h${i}`;
+        entries.set(hash, {
+            hash,
+            hashNumber: useRandomHashes ? rand() : BigInt(i + 1),
+            assignedToRangeBoundary: false,
+        });
+    }
+    return entries;
+};
+const createSync = (numbers) => {
+    const send = async () => { };
+    const rpc = { send };
+    return new RatelessIBLTSynchronizer({
+        rpc,
+        rangeIndex: {},
+        entryIndex: {},
+        log: {},
+        coordinateToHash: new Cache({ max: 10 }),
+        numbers,
+    });
+};
+const clearOutgoing = async (sync) => {
+    const ids = Array.from(sync.outgoingSyncProcesses.keys());
+    for (const id of ids) {
+        sync.outgoingSyncProcesses.get(id)?.free();
+    }
+};
+await ribltReady;
+const numbers = createNumbers("u64");
+const suite = new Bench({
+    name: "rateless-iblt-sender-startsync",
+    warmupIterations: Number.isFinite(warmupIterations) ? warmupIterations : 0,
+    iterations: Number.isFinite(iterations) ? iterations : undefined,
+});
+const sync = createSync(numbers);
+for (const size of sizes) {
+    const entries = createEntries(size);
+    suite.add(`onMaybeMissingEntries (rateless IBLT, n=${size})`, async () => {
+        await clearOutgoing(sync);
+        await sync.onMaybeMissingEntries({ entries, targets: ["t"] });
+        await clearOutgoing(sync);
+    });
+}
+await suite.run();
+if (process.env.BENCH_JSON === "1") {
+    const tasks = suite.tasks.map((task) => ({
+        name: task.name,
+        hz: task.result?.hz ?? null,
+        mean_ms: task.result?.mean ?? null,
+        rme: task.result?.rme ?? null,
+        samples: task.result?.samples?.length ?? null,
+    }));
+    process.stdout.write(JSON.stringify({
+        name: suite.name,
+        tasks,
+        meta: { sizes, warmupIterations, iterations },
+    }, null, 2));
+}
+else {
+    console.table(suite.table());
+}
+await sync.close();
+//# sourceMappingURL=rateless-iblt-sender-startsync.js.map

package/dist/benchmark/rateless-iblt-sender-startsync.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rateless-iblt-sender-startsync.js","sourceRoot":"","sources":["../../benchmark/rateless-iblt-sender-startsync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,KAAK,IAAI,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,aAAa,EAAgB,MAAM,oBAAoB,CAAC;AACjE,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AAExE,wFAAwF;AACxF,kFAAkF;AAClF,EAAE;AACF,YAAY;AACZ,yCAAyC;AACzC,yEAAyE;AACzE,8EAA8E;AAE9E,MAAM,eAAe,GAAG,CAAC,KAAyB,EAAE,QAAkB,EAAE,EAAE;IACzE,IAAI,CAAC,KAAK;QAAE,OAAO,QAAQ,CAAC;IAC5B,MAAM,MAAM,GAAG,KAAK;SAClB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;SACzC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7C,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC9C,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;AAChF,MAAM,gBAAgB,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;AAC9E,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;AAE7E,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,GAAG,CAAC;AACzD,MAAM,QAAQ,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC;AAClC,MAAM,oBAAoB,GAAG,CAAC,IAAY,EAAE,EAAE;IAC7C,IAAI,CAAC,GAAG,IAAI,GAAG,QAAQ,CAAC;IACxB,OAAO,GAAG,EAAE;QACX,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC;QACd,CAAC,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC;QAC3B,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC;QACd,OAAO,CAAC,CAAC,GAAG,oBAAoB,CAAC,GAAG,QAAQ,CAAC;IAC9C,CAAC,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAA0B,IAAY,EAAE,EAAE;IAC/D,MAAM,OAAO,GAAG,IAAI,GAAG,EAAe,CAAC;IACvC,MAAM,IAAI,GAAG,eAAe;QAC3B,CAAC,CAAC,oBAAoB,CAAC,kBAAkB,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACzD,CAAC,CAAC,SAAS,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE;YACjB,IAAI;YACJ,UAAU,EAAE,eAAe,CAAC,CAAC,CAAC,IAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;YACrD,uBAAuB,EAAE,KAAK;SAC9B,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,OAA2B,CAAC;AACpC,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAA0B,OAAmB,EAAE,EAAE;IACnE,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE,GAAE,CAAC,CAAC;IAC5B,MAAM,GAAG,GAAG,EAAE,IAAI,EAAS,CAAC;IAE5B,OAAO,IAAI,wBAAwB,CAAI;QACtC,GAAG;QACH,UAAU,EAAE,EAAS;QACrB,UAAU,EAAE,EAAS;QACrB,GAAG,EAAE,EAAS;QACd,gBAAgB,EAAE,IAAI,KAAK,CAAS,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC;QAChD,OAAO;KACP,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,KAAK,EAAE,IAAmC,EAAE,EAAE;IACnE,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC,CAAC;IAC1D,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC;IAC5C,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,UAAU,CAAC;AAEjB,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAmB,CAAC;AAEvD,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC;IACvB,IAAI,EAAE,gCAAgC;IACtC,gBAAgB,EAAE,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;IAC1E,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;CAChE,CAAC,CAAC;AAEH,MAAM,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;AAEjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;IAC1B,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IAEpC,KAAK,CAAC,GAAG,CAAC,2CAA2C,IAAI,GAAG,EAAE,KAAK,IAAI,EAAE;QACxE,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;QAC1B,MAAM,IAAI,CAAC,qBAAqB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9D,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC;AAElB,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACxC,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,IAAI;QAC3B,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,IAAI;QAClC,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,IAAI;QAC7B,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,IAAI;KAC7C,CAAC,CAAC,CAAC;IACJ,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,IAAI,CAAC,SAAS,CACb;QACC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,KAAK;QACL,IAAI,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE;KAC7C,EACD,IAAI,EACJ,CAAC,CACD,CACD,CAAC;AACH,CAAC;KAAM,CAAC;IACP,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC"}

package/dist/benchmark/rateless-iblt-startsync-cache.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=rateless-iblt-startsync-cache.d.ts.map

package/dist/benchmark/rateless-iblt-startsync-cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rateless-iblt-startsync-cache.d.ts","sourceRoot":"","sources":["../../benchmark/rateless-iblt-startsync-cache.ts"],"names":[],"mappings":""}

package/dist/benchmark/rateless-iblt-startsync-cache.js

@@ -0,0 +1,112 @@
+import { Cache } from "@peerbit/cache";
+import { ready as ribltReady } from "@peerbit/riblt";
+import { Bench } from "tinybench";
+import { createNumbers } from "../src/integers.js";
+import { RatelessIBLTSynchronizer } from "../src/sync/rateless-iblt.js";
+// Run with:
+//   cd packages/programs/data/shared-log
+//   RIBLT_SIZES="1000,10000,50000" RIBLT_WARMUP=5 RIBLT_ITERATIONS=20 \
+//     node --loader ts-node/esm ./benchmark/rateless-iblt-startsync-cache.ts
+const parseNumberList = (value, defaults) => {
+    if (!value)
+        return defaults;
+    const parsed = value
+        .split(",")
+        .map((x) => Number.parseInt(x.trim(), 10))
+        .filter((x) => Number.isFinite(x) && x > 0);
+    return parsed.length > 0 ? parsed : defaults;
+};
+const sizes = parseNumberList(process.env.RIBLT_SIZES, [1_000, 10_000, 50_000]);
+const warmupIterations = Number.parseInt(process.env.RIBLT_WARMUP || "5", 10);
+const iterations = Number.parseInt(process.env.RIBLT_ITERATIONS || "20", 10);
+const createEntries = (size) => {
+    const entries = new Array(size);
+    for (let i = 0; i < size; i++) {
+        entries[i] = { value: { hash: `h${i}`, hashNumber: BigInt(i + 1) } };
+    }
+    return entries;
+};
+const createSync = (entries, numbers) => {
+    const entryIndex = {
+        iterate: () => ({
+            all: async () => entries,
+        }),
+    };
+    const send = async () => { };
+    const rpc = { send };
+    return new RatelessIBLTSynchronizer({
+        rpc,
+        rangeIndex: {},
+        entryIndex,
+        log: {},
+        coordinateToHash: new Cache({ max: 10 }),
+        numbers,
+    });
+};
+await ribltReady;
+const numbers = createNumbers("u64");
+const suite = new Bench({
+    name: "rateless-iblt-startsync-cache",
+    warmupIterations: Number.isFinite(warmupIterations) ? warmupIterations : 0,
+    iterations: Number.isFinite(iterations) ? iterations : undefined,
+});
+const syncsToClose = [];
+for (const size of sizes) {
+    const entries = createEntries(size);
+    const range = {
+        start1: 0n,
+        end1: numbers.maxValue,
+        start2: 0n,
+        end2: 0n,
+    };
+    const coldSync = createSync(entries, numbers);
+    const warmSync = createSync(entries, numbers);
+    const warmDecoder = await warmSync.getLocalDecoderForRange(range);
+    if (warmDecoder) {
+        warmDecoder.free();
+    }
+    syncsToClose.push(() => coldSync.close());
+    syncsToClose.push(() => warmSync.close());
+    suite.add(`StartSync local decoder (cold, n=${size})`, async () => {
+        coldSync.invalidateLocalRangeEncoderCache();
+        const decoder = await coldSync.getLocalDecoderForRange(range);
+        if (decoder) {
+            decoder.free();
+        }
+    });
+    suite.add(`StartSync local decoder (warm, n=${size})`, async () => {
+        const decoder = await warmSync.getLocalDecoderForRange(range);
+        if (decoder) {
+            decoder.free();
+        }
+    });
+    suite.add(`StartSync local decoder (after invalidation, n=${size})`, async () => {
+        warmSync.invalidateLocalRangeEncoderCache();
+        const decoder = await warmSync.getLocalDecoderForRange(range);
+        if (decoder) {
+            decoder.free();
+        }
+    });
+}
+await suite.run();
+if (process.env.BENCH_JSON === "1") {
+    const tasks = suite.tasks.map((task) => ({
+        name: task.name,
+        hz: task.result?.hz ?? null,
+        mean_ms: task.result?.mean ?? null,
+        rme: task.result?.rme ?? null,
+        samples: task.result?.samples?.length ?? null,
+    }));
+    process.stdout.write(JSON.stringify({
+        name: suite.name,
+        tasks,
+        meta: { sizes, warmupIterations, iterations },
+    }, null, 2));
+}
+else {
+    console.table(suite.table());
+}
+for (const closeSync of syncsToClose) {
+    await closeSync();
+}
+//# sourceMappingURL=rateless-iblt-startsync-cache.js.map

package/dist/benchmark/rateless-iblt-startsync-cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rateless-iblt-startsync-cache.js","sourceRoot":"","sources":["../../benchmark/rateless-iblt-startsync-cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,KAAK,IAAI,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,aAAa,EAAgB,MAAM,oBAAoB,CAAC;AACjE,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AAExE,YAAY;AACZ,yCAAyC;AACzC,wEAAwE;AACxE,6EAA6E;AAE7E,MAAM,eAAe,GAAG,CAAC,KAAyB,EAAE,QAAkB,EAAE,EAAE;IACzE,IAAI,CAAC,KAAK;QAAE,OAAO,QAAQ,CAAC;IAC5B,MAAM,MAAM,GAAG,KAAK;SAClB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;SACzC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7C,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC9C,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;AAChF,MAAM,gBAAgB,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;AAC9E,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;AAE7E,MAAM,aAAa,GAAG,CAAC,IAAY,EAAE,EAAE;IACtC,MAAM,OAAO,GACZ,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;IACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC;IACtE,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAClB,OAA+D,EAC/D,OAAuB,EACtB,EAAE;IACH,MAAM,UAAU,GAAG;QAClB,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;YACf,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,OAAO;SACxB,CAAC;KACK,CAAC;IAET,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE,GAAE,CAAC,CAAC;IAC5B,MAAM,GAAG,GAAG,EAAE,IAAI,EAAS,CAAC;IAE5B,OAAO,IAAI,wBAAwB,CAAQ;QAC1C,GAAG;QACH,UAAU,EAAE,EAAS;QACrB,UAAU;QACV,GAAG,EAAE,EAAS;QACd,gBAAgB,EAAE,IAAI,KAAK,CAAS,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC;QAChD,OAAO;KACP,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,UAAU,CAAC;AAEjB,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAmB,CAAC;AAEvD,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC;IACvB,IAAI,EAAE,+BAA+B;IACrC,gBAAgB,EAAE,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;IAC1E,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;CAChE,CAAC,CAAC;AAEH,MAAM,YAAY,GAA+B,EAAE,CAAC;AAEpD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;IAC1B,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,KAAK,GAAG;QACb,MAAM,EAAE,EAAE;QACV,IAAI,EAAE,OAAO,CAAC,QAAQ;QACtB,MAAM,EAAE,EAAE;QACV,IAAI,EAAE,EAAE;KACR,CAAC;IAEF,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAE9C,MAAM,WAAW,GAAG,MAAO,QAAgB,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;IAC3E,IAAI,WAAW,EAAE,CAAC;QACjB,WAAW,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAED,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAmB,CAAC,CAAC;IAC3D,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAmB,CAAC,CAAC;IAE3D,KAAK,CAAC,GAAG,CAAC,oCAAoC,IAAI,GAAG,EAAE,KAAK,IAAI,EAAE;QAChE,QAAgB,CAAC,gCAAgC,EAAE,CAAC;QACrD,MAAM,OAAO,GAAG,MAAO,QAAgB,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;QACvE,IAAI,OAAO,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,EAAE,CAAC;QAChB,CAAC;IACF,CAAC,CAAC,CAAC;IAEH,KAAK,CAAC,GAAG,CAAC,oCAAoC,IAAI,GAAG,EAAE,KAAK,IAAI,EAAE;QACjE,MAAM,OAAO,GAAG,MAAO,QAAgB,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;QACvE,IAAI,OAAO,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,EAAE,CAAC;QAChB,CAAC;IACF,CAAC,CAAC,CAAC;IAEH,KAAK,CAAC,GAAG,CAAC,kDAAkD,IAAI,GAAG,EAAE,KAAK,IAAI,EAAE;QAC9E,QAAgB,CAAC,gCAAgC,EAAE,CAAC;QACrD,MAAM,OAAO,GAAG,MAAO,QAAgB,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;QACvE,IAAI,OAAO,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,EAAE,CAAC;QAChB,CAAC;IACF,CAAC,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC;AAClB,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACxC,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,IAAI;QAC3B,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,IAAI;QAClC,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,IAAI;QAC7B,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,IAAI;KAC7C,CAAC,CAAC,CAAC;IACJ,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,IAAI,CAAC,SAAS,CACb;QACC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,KAAK;QACL,IAAI,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE;KAC7C,EACD,IAAI,EACJ,CAAC,CACD,CACD,CAAC;AACH,CAAC;KAAM,CAAC;IACP,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;AAC9B,CAAC;AAED,KAAK,MAAM,SAAS,IAAI,YAAY,EAAE,CAAC;IACtC,MAAM,SAAS,EAAE,CAAC;AACnB,CAAC"}

package/dist/benchmark/sync-catchup.d.ts

@@ -0,0 +1,3 @@
+import type { TestSetupConfig } from "../test/utils.js";
+export declare const testSetups: TestSetupConfig<any>[];
+//# sourceMappingURL=sync-catchup.d.ts.map

package/dist/benchmark/sync-catchup.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sync-catchup.d.ts","sourceRoot":"","sources":["../../benchmark/sync-catchup.ts"],"names":[],"mappings":"AAoBA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAOxD,eAAO,MAAM,UAAU,EAAE,eAAe,CAAC,GAAG,CAAC,EAa5C,CAAC"}

package/dist/benchmark/sync-catchup.js

@@ -0,0 +1,109 @@
+// Benchmarks full catch-up sync time: one peer has N entries, the other starts empty.
+//
+// Run with:
+//   cd packages/programs/data/shared-log
+//   CATCHUP_COUNT=5000 CATCHUP_TIMEOUT=60000 node --loader ts-node/esm ./benchmark/sync-catchup.ts
+//
+// Notes:
+// - This is an integration benchmark (network + sync + indexing). It is more variable than pure
+//   algorithmic benches; prefer running it a few times and comparing medians.
+// - For CI regression tracking, consider smaller `CATCHUP_COUNT` values and low run counts.
+import { keys } from "@libp2p/crypto";
+import { TestSession } from "@peerbit/test-utils";
+import { waitForResolved } from "@peerbit/time";
+import { expect } from "chai";
+import { performance } from "node:perf_hooks";
+import { v4 as uuid } from "uuid";
+import { createReplicationDomainHash } from "../src/replication-domain-hash.js";
+import { RatelessIBLTSynchronizer } from "../src/sync/rateless-iblt.js";
+import { SimpleSyncronizer } from "../src/sync/simple.js";
+import { EventStore } from "../test/utils/stores/event-store.js";
+const entryCount = Number.parseInt(process.env.CATCHUP_COUNT || "5000", 10);
+const timeoutMs = Number.parseInt(process.env.CATCHUP_TIMEOUT || "60000", 10);
+const runs = Number.parseInt(process.env.CATCHUP_RUNS || "1", 10);
+export const testSetups = [
+    {
+        domain: createReplicationDomainHash("u32"),
+        type: "u32",
+        syncronizer: SimpleSyncronizer,
+        name: "u32-simple",
+    },
+    {
+        domain: createReplicationDomainHash("u64"),
+        type: "u64",
+        syncronizer: RatelessIBLTSynchronizer,
+        name: "u64-iblt",
+    },
+];
+const fixedKeys = [
+    {
+        libp2p: {
+            privateKey: keys.privateKeyFromRaw(new Uint8Array([
+                204, 234, 187, 172, 226, 232, 70, 175, 62, 211, 147, 91, 229, 157,
+                168, 15, 45, 242, 144, 98, 75, 58, 208, 9, 223, 143, 251, 52, 252,
+                159, 64, 83, 52, 197, 24, 246, 24, 234, 141, 183, 151, 82, 53, 142,
+                57, 25, 148, 150, 26, 209, 223, 22, 212, 40, 201, 6, 191, 72, 148,
+                82, 66, 138, 199, 185,
+            ])),
+        },
+    },
+    {
+        libp2p: {
+            privateKey: keys.privateKeyFromRaw(new Uint8Array([
+                237, 55, 205, 86, 40, 44, 73, 169, 196, 118, 36, 69, 214, 122, 28,
+                157, 208, 163, 15, 215, 104, 193, 151, 177, 62, 231, 253, 120, 122,
+                222, 174, 242, 120, 50, 165, 97, 8, 235, 97, 186, 148, 251, 100, 168,
+                49, 10, 119, 71, 246, 246, 174, 163, 198, 54, 224, 6, 174, 212, 159,
+                187, 2, 137, 47, 192,
+            ])),
+        },
+    },
+];
+const runOnce = async (setup) => {
+    const session = await TestSession.disconnected(2, fixedKeys);
+    const store = new EventStore();
+    const db1 = await session.peers[0].open(store.clone(), {
+        args: { replicate: { factor: 1 }, setup },
+    });
+    const db2 = await session.peers[1].open(store.clone(), {
+        args: { replicate: { factor: 1 }, setup },
+    });
+    for (let i = 0; i < entryCount; i++) {
+        await db1.add(uuid(), { meta: { next: [] } });
+    }
+    expect(db1.log.log.length).to.equal(entryCount);
+    await waitForResolved(() => session.peers[0].dial(session.peers[1].getMultiaddrs()));
+    const t0 = performance.now();
+    await waitForResolved(() => {
+        expect(db2.log.log.length).to.equal(entryCount);
+    }, { timeout: timeoutMs, delayInterval: 250 });
+    const dt = performance.now() - t0;
+    await session.stop();
+    return dt;
+};
+const tasks = [];
+for (const setup of testSetups) {
+    const samples = [];
+    for (let i = 0; i < runs; i++) {
+        samples.push(await runOnce(setup));
+    }
+    const mean_ms = samples.reduce((acc, x) => acc + x, 0) / samples.length;
+    tasks.push({
+        name: `${setup.name} catchup (n=${entryCount})`,
+        mean_ms,
+        hz: mean_ms > 0 ? 1000 / mean_ms : 0,
+        rme: null,
+        samples: samples.length,
+    });
+}
+if (process.env.BENCH_JSON === "1") {
+    process.stdout.write(JSON.stringify({
+        name: "shared-log-sync-catchup",
+        tasks,
+        meta: { entryCount, timeoutMs, runs },
+    }, null, 2));
+}
+else {
+    console.table(tasks);
+}
+//# sourceMappingURL=sync-catchup.js.map

package/dist/benchmark/sync-catchup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sync-catchup.js","sourceRoot":"","sources":["../../benchmark/sync-catchup.ts"],"names":[],"mappings":"AAAA,sFAAsF;AACtF,EAAE;AACF,YAAY;AACZ,yCAAyC;AACzC,mGAAmG;AACnG,EAAE;AACF,SAAS;AACT,gGAAgG;AAChG,8EAA8E;AAC9E,4FAA4F;AAE5F,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAC9B,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,EAAE,IAAI,IAAI,EAAE,MAAM,MAAM,CAAC;AAClC,OAAO,EAAE,2BAA2B,EAAE,MAAM,mCAAmC,CAAC;AAChF,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAE1D,OAAO,EAAE,UAAU,EAAE,MAAM,qCAAqC,CAAC;AAEjE,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,MAAM,EAAE,EAAE,CAAC,CAAC;AAC5E,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,OAAO,EAAE,EAAE,CAAC,CAAC;AAC9E,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;AAElE,MAAM,CAAC,MAAM,UAAU,GAA2B;IACjD;QACC,MAAM,EAAE,2BAA2B,CAAC,KAAK,CAAC;QAC1C,IAAI,EAAE,KAAK;QACX,WAAW,EAAE,iBAAiB;QAC9B,IAAI,EAAE,YAAY;KAClB;IACD;QACC,MAAM,EAAE,2BAA2B,CAAC,KAAK,CAAC;QAC1C,IAAI,EAAE,KAAK;QACX,WAAW,EAAE,wBAAwB;QACrC,IAAI,EAAE,UAAU;KAChB;CACD,CAAC;AAEF,MAAM,SAAS,GAAG;IACjB;QACC,MAAM,EAAE;YACP,UAAU,EAAE,IAAI,CAAC,iBAAiB,CACjC,IAAI,UAAU,CAAC;gBACd,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG;gBACjE,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG;gBACjE,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG;gBAClE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG;gBACjE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;aACrB,CAAC,CACF;SACD;KACD;IACD;QACC,MAAM,EAAE;YACP,UAAU,EAAE,IAAI,CAAC,iBAAiB,CACjC,IAAI,UAAU,CAAC;gBACd,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE;gBACjE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;gBAClE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;gBACpE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;gBACnE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG;aACpB,CAAC,CACF;SACD;KACD;CACD,CAAC;AAEF,MAAM,OAAO,GAAG,KAAK,EAAE,KAA2B,EAAE,EAAE;IACrD,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,YAAY,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG,IAAI,UAAU,EAAe,CAAC;IAE5C,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE;QACtD,IAAI,EAAE,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE;KACzC,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE;QACtD,IAAI,EAAE,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE;KACzC,CAAC,CAAC;IAEH,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAEhD,MAAM,eAAe,CAAC,GAAG,EAAE,CAC1B,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC,CACvD,CAAC;IAEF,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,eAAe,CACpB,GAAG,EAAE;QACJ,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACjD,CAAC,EACD,EAAE,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,GAAG,EAAE,CAC1C,CAAC;IACF,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;IAElC,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;IAErB,OAAO,EAAE,CAAC;AACX,CAAC,CAAC;AAEF,MAAM,KAAK,GAMN,EAAE,CAAC;AAER,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;IAChC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,OAAO,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IACpC,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IACxE,KAAK,CAAC,IAAI,CAAC;QACV,IAAI,EAAE,GAAG,KAAK,CAAC,IAAI,eAAe,UAAU,GAAG;QAC/C,OAAO;QACP,EAAE,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACpC,GAAG,EAAE,IAAI;QACT,OAAO,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAC;AACJ,CAAC;AAED,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;IACpC,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,IAAI,CAAC,SAAS,CACb;QACC,IAAI,EAAE,yBAAyB;QAC/B,KAAK;QACL,IAAI,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE;KACrC,EACD,IAAI,EACJ,CAAC,CACD,CACD,CAAC;AACH,CAAC;KAAM,CAAC;IACP,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AACtB,CAAC"}

package/dist/src/index.d.ts

@@ -16,7 +16,7 @@ import { type ReplicationDomainHash, createReplicationDomainHash } from "./repli
 import { type ReplicationDomainTime, createReplicationDomainTime } from "./replication-domain-time.js";
 import { type CoverRange, type ExtractDomainArgs, type ReplicationDomain, type ReplicationDomainConstructor } from "./replication-domain.js";
 import { AbsoluteReplicas, AddedReplicationSegmentMessage, AllReplicatingSegmentsMessage, type ReplicationLimits } from "./replication.js";
-import type { SynchronizerConstructor, Syncronizer } from "./sync/index.js";
+import type { SyncOptions, SynchronizerConstructor, Syncronizer } from "./sync/index.js";
 export { type ReplicationDomain, type ReplicationDomainHash, type ReplicationDomainTime, createReplicationDomainHash, createReplicationDomainTime, };
 export { type CPUUsage, CPUUsageIntervalLag };
 export * from "./replication.js";
@@ -88,9 +88,12 @@ export type SharedLogOptions<T, D extends ReplicationDomain<any, T, R>, R extend
     respondToIHaveTimeout?: number;
     canReplicate?: (publicKey: PublicSignKey) => Promise<boolean> | boolean;
     keep?: (entry: ShallowOrFullEntry<T> | EntryReplicated<R>) => Promise<boolean> | boolean;
+    sync?: SyncOptions<R>;
     syncronizer?: SynchronizerConstructor<R>;
     timeUntilRoleMaturity?: number;
     waitForReplicatorTimeout?: number;
+    waitForReplicatorRequestIntervalMs?: number;
+    waitForReplicatorRequestMaxAttempts?: number;
     waitForPruneDelay?: number;
     distributionDebounceTime?: number;
     compatibility?: number;
@@ -102,7 +105,9 @@ export type SharedLogOptions<T, D extends ReplicationDomain<any, T, R>, R extend
 export declare const DEFAULT_MIN_REPLICAS = 2;
 export declare const WAIT_FOR_REPLICATOR_TIMEOUT = 9000;
 export declare const WAIT_FOR_ROLE_MATURITY = 5000;
-export declare const WAIT_FOR_PRUNE_DELAY = 5000;
+export declare const WAIT_FOR_REPLICATOR_REQUEST_INTERVAL = 1000;
+export declare const WAIT_FOR_REPLICATOR_REQUEST_MIN_ATTEMPTS = 3;
+export declare const WAIT_FOR_PRUNE_DELAY = 0;
 export type Args<T, D extends ReplicationDomain<any, T, R>, R extends "u32" | "u64" = D extends ReplicationDomain<any, T, infer I> ? I : "u32"> = LogProperties<T> & LogEvents<T> & SharedLogOptions<T, D, R>;
 export type DeliveryOptions = {
     settle?: true | {
@@ -146,6 +151,7 @@ export declare class SharedLog<T, D extends ReplicationDomain<any, T, R> = any,
     private coordinateToHash;
     private recentlyRebalanced;
     uniqueReplicators: Set<string>;
+    private _replicatorsReconciled;
     _gidPeersHistory: Map<string, Set<string>>;
     private _onSubscriptionFn;
     private _onUnsubscriptionFn;
@@ -179,6 +185,8 @@ export declare class SharedLog<T, D extends ReplicationDomain<any, T, R> = any,
     private cpuUsage?;
     timeUntilRoleMaturity: number;
     waitForReplicatorTimeout: number;
+    waitForReplicatorRequestIntervalMs: number;
+    waitForReplicatorRequestMaxAttempts?: number;
     waitForPruneDelay: number;
     distributionDebounceTime: number;
     replicationController: PIDReplicationController;
@@ -355,7 +363,12 @@ export declare class SharedLog<T, D extends ReplicationDomain<any, T, R> = any,
     rebalanceAll(options?: {
         clearCache?: boolean;
     }): Promise<false | undefined>;
-    waitForPruned(): Promise<void>;
+    waitForPruned(options?: {
+        timeout?: number;
+        signal?: AbortSignal;
+        delayInterval?: number;
+        timeoutMessage?: string;
+    }): Promise<void>;
     onReplicationChange(changeOrChanges: ReplicationChanges<ReplicationRangeIndexable<R>> | ReplicationChanges<ReplicationRangeIndexable<R>>[]): Promise<false | undefined>;
     _onUnsubscription(evt: CustomEvent<UnsubcriptionEvent>): Promise<void>;
     _onSubscription(evt: CustomEvent<SubscriptionEvent>): Promise<void>;