@peerbit/shared-log 13.1.16 → 13.1.17

package/dist/src/sync/rateless-iblt.js

@@ -32,7 +32,7 @@ var __runInitializers = (this && this.__runInitializers) || function (thisArg, i
     }
     return useValue ? value : void 0;
 };
-import { field, variant, vec } from "@dao-xyz/borsh";
+import { field, variant, } from "@dao-xyz/borsh";
 import { Cache } from "@peerbit/cache";
 import { randomBytes, toBase64 } from "@peerbit/crypto";
 import { And, IntegerCompare, Or, } from "@peerbit/indexer-interface";
@@ -42,7 +42,8 @@ import { SilentDelivery } from "@peerbit/stream-interface";
 import {} from "../exchange-heads.js";
 import { TransportMessage } from "../message.js";
 import {} from "../ranges.js";
-import { SimpleSyncronizer } from "./simple.js";
+import { emitSyncProfileDuration, emitSyncProfileEvent, syncProfileStart, } from "./profile.js";
+import { SYNC_MESSAGE_PRIORITY, SimpleSyncronizer, } from "./simple.js";
 export const logger = loggerFn("peerbit:shared-log:rateless");
 const coerceBigInt = (value) => typeof value === "bigint" ? value : BigInt(value);
 let SymbolSerialized = (() => {
@@ -77,12 +78,285 @@ let SymbolSerialized = (() => {
         }
     };
 })();
+const CODED_SYMBOL_WORDS = 3;
+const CODED_SYMBOL_WORD_BYTES = 8;
+const CODED_SYMBOL_BYTES = CODED_SYMBOL_WORDS * CODED_SYMBOL_WORD_BYTES;
+const BIG_UINT64_ARRAY_IS_LITTLE_ENDIAN = typeof BigUint64Array !== "undefined" &&
+    new Uint8Array(new BigUint64Array([1n]).buffer)[0] === 1;
+const assertValidFlatCodedSymbols = (flat) => {
+    if (flat.length % CODED_SYMBOL_WORDS !== 0) {
+        throw new Error("Invalid RIBLT coded symbol batch");
+    }
+};
+export class CodedSymbolBatch {
+    flat;
+    symbols;
+    constructor(properties) {
+        this.flat = properties.flat;
+        this.symbols = properties.symbols;
+    }
+    static from(symbols) {
+        if (symbols instanceof CodedSymbolBatch) {
+            return symbols;
+        }
+        if (typeof BigUint64Array !== "undefined" &&
+            symbols instanceof BigUint64Array) {
+            return CodedSymbolBatch.fromFlat(symbols);
+        }
+        return CodedSymbolBatch.fromSymbols(symbols);
+    }
+    static fromFlat(flat) {
+        assertValidFlatCodedSymbols(flat);
+        return new CodedSymbolBatch({ flat });
+    }
+    static fromSymbols(symbols) {
+        return new CodedSymbolBatch({ symbols });
+    }
+    get length() {
+        return this.flat
+            ? this.flat.length / CODED_SYMBOL_WORDS
+            : (this.symbols?.length ?? 0);
+    }
+    toFlat() {
+        if (this.flat) {
+            return this.flat;
+        }
+        const symbols = this.symbols ?? [];
+        const flat = new BigUint64Array(symbols.length * CODED_SYMBOL_WORDS);
+        for (let i = 0; i < symbols.length; i++) {
+            const offset = i * CODED_SYMBOL_WORDS;
+            const symbol = symbols[i];
+            flat[offset] = coerceBigInt(symbol.count);
+            flat[offset + 1] = coerceBigInt(symbol.hash);
+            flat[offset + 2] = coerceBigInt(symbol.symbol);
+        }
+        this.flat = flat;
+        return flat;
+    }
+    toSymbols() {
+        const symbols = [];
+        for (const symbol of this) {
+            symbols.push(symbol instanceof SymbolSerialized
+                ? symbol
+                : new SymbolSerialized({
+                    count: symbol.count,
+                    hash: symbol.hash,
+                    symbol: symbol.symbol,
+                }));
+        }
+        return symbols;
+    }
+    *[Symbol.iterator]() {
+        if (this.symbols) {
+            yield* this.symbols;
+            return;
+        }
+        const flat = this.flat;
+        if (!flat) {
+            return;
+        }
+        for (let i = 0; i < flat.length; i += CODED_SYMBOL_WORDS) {
+            yield {
+                count: flat[i],
+                hash: flat[i + 1],
+                symbol: flat[i + 2],
+            };
+        }
+    }
+}
+const codedSymbolBatchField = {
+    serialize: (symbols, writer) => {
+        const batch = CodedSymbolBatch.from(symbols);
+        const length = batch.length;
+        writer.u32(length);
+        if (length === 0) {
+            return;
+        }
+        if (typeof BigUint64Array !== "undefined") {
+            const flat = batch.toFlat();
+            if (BIG_UINT64_ARRAY_IS_LITTLE_ENDIAN) {
+                writer.set(new Uint8Array(flat.buffer, flat.byteOffset, flat.byteLength));
+                return;
+            }
+            for (let i = 0; i < flat.length; i++) {
+                writer.u64(flat[i]);
+            }
+            return;
+        }
+        for (const symbol of batch) {
+            writer.u64(symbol.count);
+            writer.u64(symbol.hash);
+            writer.u64(symbol.symbol);
+        }
+    },
+    deserialize: (reader) => {
+        const length = reader.u32();
+        const wordLength = length * CODED_SYMBOL_WORDS;
+        const byteLength = length * CODED_SYMBOL_BYTES;
+        if (typeof BigUint64Array !== "undefined" &&
+            BIG_UINT64_ARRAY_IS_LITTLE_ENDIAN) {
+            if (reader._offset + byteLength > reader._buf.length) {
+                throw new Error("Invalid RIBLT coded symbol batch length");
+            }
+            const bytes = reader.buffer(byteLength);
+            const flat = new BigUint64Array(wordLength);
+            new Uint8Array(flat.buffer).set(bytes);
+            return CodedSymbolBatch.fromFlat(flat);
+        }
+        const symbols = [];
+        for (let i = 0; i < length; i++) {
+            symbols.push(new SymbolSerialized({
+                count: reader.u64(),
+                hash: reader.u64(),
+                symbol: reader.u64(),
+            }));
+        }
+        return CodedSymbolBatch.fromSymbols(symbols);
+    },
+};
+const addSymbolsToRiblt = (target, symbols) => {
+    if (typeof BigUint64Array !== "undefined" &&
+        typeof target.add_symbols === "function") {
+        target.add_symbols(symbols instanceof BigUint64Array
+            ? symbols
+            : BigUint64Array.from(symbols, coerceBigInt));
+        return;
+    }
+    for (const symbol of symbols) {
+        target.add_symbol(coerceBigInt(symbol));
+    }
+};
+const produceNextCodedSymbols = (encoder, count) => {
+    const produceBatch = encoder
+        .produce_next_coded_symbols;
+    if (typeof BigUint64Array !== "undefined" && produceBatch) {
+        return CodedSymbolBatch.fromFlat(produceBatch.call(encoder, count));
+    }
+    const symbols = [];
+    for (let i = 0; i < count; i++) {
+        symbols.push(new SymbolSerialized(encoder.produce_next_coded_symbol()));
+    }
+    return CodedSymbolBatch.fromSymbols(symbols);
+};
+const flatFromCodedSymbols = (symbols) => {
+    if (typeof BigUint64Array !== "undefined" &&
+        symbols instanceof BigUint64Array) {
+        assertValidFlatCodedSymbols(symbols);
+        return symbols;
+    }
+    return CodedSymbolBatch.from(symbols).toFlat();
+};
+const getRemoteSymbolValues = (decoder) => {
+    const getBatch = decoder.get_remote_symbol_values;
+    if (typeof BigUint64Array !== "undefined" && getBatch) {
+        return Array.from(getBatch.call(decoder));
+    }
+    const symbols = [];
+    for (const missingSymbol of decoder.get_remote_symbols()) {
+        symbols.push(coerceBigInt(missingSymbol));
+    }
+    return symbols;
+};
+const prepareStartSyncEncoder = (coordinates, maxValue, initialSymbolCount) => {
+    const encoder = new EncoderWrapper();
+    let complete = false;
+    try {
+        const prepareAndProduceNative = encoder
+            .add_symbols_sorted_find_range_and_produce;
+        if (typeof BigUint64Array !== "undefined" && prepareAndProduceNative) {
+            const prepared = prepareAndProduceNative.call(encoder, BigUint64Array.from(coordinates), coerceBigInt(maxValue), initialSymbolCount);
+            if (prepared.length < 2 ||
+                (prepared.length - 2) % CODED_SYMBOL_WORDS !== 0) {
+                throw new Error("Invalid RIBLT prepared encoder result");
+            }
+            complete = true;
+            return {
+                encoder,
+                start: prepared[0],
+                end: prepared[1],
+                initialSymbols: CodedSymbolBatch.fromFlat(prepared.subarray(2)),
+            };
+        }
+        const prepareNative = encoder
+            .add_symbols_sorted_and_find_range;
+        if (typeof BigUint64Array !== "undefined" && prepareNative) {
+            const range = prepareNative.call(encoder, BigUint64Array.from(coordinates), coerceBigInt(maxValue));
+            if (range.length !== 2) {
+                throw new Error("Invalid RIBLT range result");
+            }
+            complete = true;
+            return { encoder, start: range[0], end: range[1] };
+        }
+        let sortedEntries;
+        if (typeof BigUint64Array !== "undefined") {
+            const typed = new BigUint64Array(coordinates.length);
+            for (let i = 0; i < coordinates.length; i++) {
+                typed[i] = coordinates[i];
+            }
+            typed.sort();
+            sortedEntries = typed;
+        }
+        else {
+            sortedEntries = [...coordinates].sort((a, b) => {
+                if (a > b) {
+                    return 1;
+                }
+                else if (a < b) {
+                    return -1;
+                }
+                else {
+                    return 0;
+                }
+            });
+        }
+        // assume sorted, and find the largest gap
+        let largestGap = 0n;
+        let largestGapIndex = 0;
+        for (let i = 0; i < sortedEntries.length; i++) {
+            const current = sortedEntries[i];
+            const next = sortedEntries[(i + 1) % sortedEntries.length];
+            const gap = next >= current
+                ? next - current
+                : coerceBigInt(maxValue) - current + next;
+            if (gap > largestGap) {
+                largestGap = gap;
+                largestGapIndex = i;
+            }
+        }
+        const smallestRangeStartIndex = (largestGapIndex + 1) % sortedEntries.length;
+        const smallestRangeEndIndex = largestGapIndex; /// === (smallRangeStartIndex + 1) % sortedEntries.length
+        let smallestRangeStart = sortedEntries[smallestRangeStartIndex];
+        let smallestRangeEnd = sortedEntries[smallestRangeEndIndex];
+        let start, end;
+        if (smallestRangeEnd === smallestRangeStart) {
+            start = smallestRangeEnd;
+            end = smallestRangeEnd + 1n;
+            if (end > maxValue) {
+                end = 0n;
+            }
+        }
+        else {
+            start = smallestRangeStart;
+            end = smallestRangeEnd;
+        }
+        addSymbolsToRiblt(encoder, sortedEntries);
+        complete = true;
+        return { encoder, start, end };
+    }
+    finally {
+        if (!complete) {
+            encoder.free();
+        }
+    }
+};
 const getSyncIdString = (message) => {
     return toBase64(message.syncId);
 };
 const DEFAULT_CONVERGENT_REPAIR_TIMEOUT_MS = 30_000;
 const DEFAULT_CONVERGENT_RETRY_INTERVALS_MS = [0, 1_000, 3_000, 7_000];
 const DEFAULT_MAX_CONVERGENT_TRACKED_HASHES = 4_096;
+const MIN_MORE_SYMBOLS_BATCH_SIZE = 64;
+const MAX_MORE_SYMBOLS_BATCH_SIZE = 1_024;
 let StartSync = (() => {
     let _classDecorators = [variant([3, 0])];
     let _classDescriptor;
@@ -108,7 +382,7 @@ let StartSync = (() => {
             _syncId_decorators = [field({ type: Uint8Array })];
             _start_decorators = [field({ type: "u64" })];
             _end_decorators = [field({ type: "u64" })];
-            _symbols_decorators = [field({ type: vec(SymbolSerialized) })];
+            _symbols_decorators = [field({ type: codedSymbolBatchField })];
             __esDecorate(null, null, _syncId_decorators, { kind: "field", name: "syncId", static: false, private: false, access: { has: obj => "syncId" in obj, get: obj => obj.syncId, set: (obj, value) => { obj.syncId = value; } }, metadata: _metadata }, _syncId_initializers, _syncId_extraInitializers);
             __esDecorate(null, null, _start_decorators, { kind: "field", name: "start", static: false, private: false, access: { has: obj => "start" in obj, get: obj => obj.start, set: (obj, value) => { obj.start = value; } }, metadata: _metadata }, _start_initializers, _start_extraInitializers);
             __esDecorate(null, null, _end_decorators, { kind: "field", name: "end", static: false, private: false, access: { has: obj => "end" in obj, get: obj => obj.end, set: (obj, value) => { obj.end = value; } }, metadata: _metadata }, _end_initializers, _end_extraInitializers);
@@ -128,7 +402,7 @@ let StartSync = (() => {
             this.syncId = randomBytes(32);
             this.start = coerceBigInt(props.from);
             this.end = coerceBigInt(props.to);
-            this.symbols = props.symbols;
+            this.symbols = CodedSymbolBatch.from(props.symbols);
         }
     };
     return StartSync = _classThis;
@@ -155,7 +429,7 @@ let MoreSymbols = (() => {
             const _metadata = typeof Symbol === "function" && Symbol.metadata ? Object.create(_classSuper[Symbol.metadata] ?? null) : void 0;
             _syncId_decorators = [field({ type: Uint8Array })];
             _seqNo_decorators = [field({ type: "u64" })];
-            _symbols_decorators = [field({ type: vec(SymbolSerialized) })];
+            _symbols_decorators = [field({ type: codedSymbolBatchField })];
             __esDecorate(null, null, _syncId_decorators, { kind: "field", name: "syncId", static: false, private: false, access: { has: obj => "syncId" in obj, get: obj => obj.syncId, set: (obj, value) => { obj.syncId = value; } }, metadata: _metadata }, _syncId_initializers, _syncId_extraInitializers);
             __esDecorate(null, null, _seqNo_decorators, { kind: "field", name: "seqNo", static: false, private: false, access: { has: obj => "seqNo" in obj, get: obj => obj.seqNo, set: (obj, value) => { obj.seqNo = value; } }, metadata: _metadata }, _seqNo_initializers, _seqNo_extraInitializers);
             __esDecorate(null, null, _symbols_decorators, { kind: "field", name: "symbols", static: false, private: false, access: { has: obj => "symbols" in obj, get: obj => obj.symbols, set: (obj, value) => { obj.symbols = value; } }, metadata: _metadata }, _symbols_initializers, _symbols_extraInitializers);
@@ -172,7 +446,7 @@ let MoreSymbols = (() => {
             __runInitializers(this, _symbols_extraInitializers);
             this.syncId = props.syncId;
             this.seqNo = props.lastSeqNo + 1n;
-            this.symbols = props.symbols;
+            this.symbols = CodedSymbolBatch.from(props.symbols);
         }
     };
     return MoreSymbols = _classThis;
@@ -279,9 +553,10 @@ const matchEntriesByHashNumberInRangeQuery = (range) => {
         ]),
     ]);
 };
-const buildEncoderOrDecoderFromRange = async (ranges, entryIndex, type) => {
+const buildEncoderOrDecoderFromRange = async (ranges, entryIndex, type, profile) => {
     await ribltReady;
     const encoder = type === "encoder" ? new EncoderWrapper() : new DecoderWrapper();
+    const rangeQueryStartedAt = syncProfileStart(profile);
     const entries = await entryIndex
         .iterate({
         // Range sync for IBLT is done in hashNumber space.
@@ -298,11 +573,37 @@ const buildEncoderOrDecoderFromRange = async (ranges, entryIndex, type) => {
         },
     })
         .all();
+    if (profile) {
+        emitSyncProfileDuration(profile, rangeQueryStartedAt, {
+            name: "rateless.rangeQuery",
+            entries: entries.length,
+            details: { type },
+        });
+    }
     if (entries.length === 0) {
         return false;
     }
-    for (const entry of entries) {
-        encoder.add_symbol(coerceBigInt(entry.value.hashNumber));
+    const addSymbolsStartedAt = syncProfileStart(profile);
+    if (typeof BigUint64Array !== "undefined" &&
+        typeof encoder.add_symbols === "function") {
+        const symbols = new BigUint64Array(entries.length);
+        for (let i = 0; i < entries.length; i++) {
+            symbols[i] = coerceBigInt(entries[i].value.hashNumber);
+        }
+        addSymbolsToRiblt(encoder, symbols);
+    }
+    else {
+        for (const entry of entries) {
+            encoder.add_symbol(coerceBigInt(entry.value.hashNumber));
+        }
+    }
+    if (profile) {
+        emitSyncProfileDuration(profile, addSymbolsStartedAt, {
+            name: "rateless.rangeAddSymbols",
+            entries: entries.length,
+            symbols: entries.length,
+            details: { type },
+        });
     }
     return encoder;
 };
@@ -489,14 +790,34 @@ export class RatelessIBLTSynchronizer {
         return decoder;
     }
     async getLocalDecoderForRange(ranges) {
+        const profile = this.properties.sync?.profile;
         const key = this.localRangeEncoderCacheKey(ranges);
         const cached = this.localRangeEncoderCache.get(key);
         if (cached && cached.version === this.localRangeEncoderCacheVersion) {
+            const startedAt = syncProfileStart(profile);
             cached.lastUsed = Date.now();
-            return this.decoderFromCachedEncoder(cached.encoder);
+            try {
+                return this.decoderFromCachedEncoder(cached.encoder);
+            }
+            finally {
+                if (profile) {
+                    emitSyncProfileDuration(profile, startedAt, {
+                        name: "rateless.localDecoder",
+                        cacheHit: true,
+                    });
+                }
+            }
         }
-        const encoder = (await buildEncoderOrDecoderFromRange(ranges, this.properties.entryIndex, "encoder"));
+        const startedAt = syncProfileStart(profile);
+        const encoder = (await buildEncoderOrDecoderFromRange(ranges, this.properties.entryIndex, "encoder", profile));
         if (!encoder) {
+            if (profile) {
+                emitSyncProfileDuration(profile, startedAt, {
+                    name: "rateless.localDecoder",
+                    cacheHit: false,
+                    entries: 0,
+                });
+            }
             return false;
         }
         const now = Date.now();
@@ -527,9 +848,21 @@ export class RatelessIBLTSynchronizer {
             }
             this.localRangeEncoderCache.delete(oldestKey);
         }
-        return this.decoderFromCachedEncoder(encoder);
+        try {
+            return this.decoderFromCachedEncoder(encoder);
+        }
+        finally {
+            if (profile) {
+                emitSyncProfileDuration(profile, startedAt, {
+                    name: "rateless.localDecoder",
+                    cacheHit: false,
+                });
+            }
+        }
     }
     async onMaybeMissingEntries(properties) {
+        const profile = this.properties.sync?.profile;
+        const startedAt = syncProfileStart(profile);
         // NOTE: this method is best-effort dispatch, not a per-hash convergence API.
         // It may require follow-up repair rounds under churn/loss to fully close all gaps.
         // Strategy:
@@ -542,12 +875,33 @@ export class RatelessIBLTSynchronizer {
         let maxSyncWithSimpleMethod = 1e3;
         // Small batch => use simple synchronizer entirely
         if (properties.entries.size <= minSyncIbltSize) {
-            await this.simple.onMaybeMissingEntries({
-                entries: properties.entries,
-                targets: properties.targets,
-            });
+            if (profile) {
+                emitSyncProfileEvent(profile, {
+                    name: "rateless.dispatchMode",
+                    entries: properties.entries.size,
+                    targets: properties.targets.length,
+                    details: { mode: "simple-small" },
+                });
+            }
+            try {
+                await this.simple.onMaybeMissingEntries({
+                    entries: properties.entries,
+                    targets: properties.targets,
+                });
+            }
+            finally {
+                if (profile) {
+                    emitSyncProfileDuration(profile, startedAt, {
+                        name: "rateless.onMaybeMissingEntries",
+                        entries: properties.entries.size,
+                        targets: properties.targets.length,
+                        details: { mode: "simple-small" },
+                    });
+                }
+            }
             return;
         }
+        const selectStartedAt = syncProfileStart(profile);
         const nonBoundaryEntries = [];
         for (const entry of properties.entries.values()) {
             if (entry.assignedToRangeBoundary) {
@@ -604,91 +958,84 @@ export class RatelessIBLTSynchronizer {
                 allCoordinatesToSyncWithIblt.push(coerceBigInt(entry.hashNumber));
             }
         }
-        if (allCoordinatesToSyncWithIblt.length === 0) {
-            return;
-        }
-        await ribltReady;
-        let sortedEntries;
-        if (typeof BigUint64Array !== "undefined") {
-            const typed = new BigUint64Array(allCoordinatesToSyncWithIblt.length);
-            for (let i = 0; i < allCoordinatesToSyncWithIblt.length; i++) {
-                typed[i] = allCoordinatesToSyncWithIblt[i];
-            }
-            typed.sort();
-            sortedEntries = typed;
-        }
-        else {
-            sortedEntries = allCoordinatesToSyncWithIblt.sort((a, b) => {
-                if (a > b) {
-                    return 1;
-                }
-                else if (a < b) {
-                    return -1;
-                }
-                else {
-                    return 0;
-                }
+        if (profile) {
+            emitSyncProfileDuration(profile, selectStartedAt, {
+                name: "rateless.selectEntries",
+                entries: properties.entries.size,
+                symbols: allCoordinatesToSyncWithIblt.length,
+                targets: properties.targets.length,
+                details: {
+                    naiveEntries: entriesToSyncNaively.size,
+                    priority: priorityFn != null,
+                },
             });
         }
-        // assume sorted, and find the largest gap
-        let largestGap = 0n;
-        let largestGapIndex = 0;
-        for (let i = 0; i < sortedEntries.length; i++) {
-            const current = sortedEntries[i];
-            const next = sortedEntries[(i + 1) % sortedEntries.length];
-            const gap = next >= current
-                ? next - current
-                : coerceBigInt(this.properties.numbers.maxValue) - current + next;
-            if (gap > largestGap) {
-                largestGap = gap;
-                largestGapIndex = i;
-            }
-        }
-        const smallestRangeStartIndex = (largestGapIndex + 1) % sortedEntries.length;
-        const smallestRangeEndIndex = largestGapIndex; /// === (smallRangeStartIndex + 1) % sortedEntries.length
-        let smallestRangeStart = sortedEntries[smallestRangeStartIndex];
-        let smallestRangeEnd = sortedEntries[smallestRangeEndIndex];
-        let start, end;
-        if (smallestRangeEnd === smallestRangeStart) {
-            start = smallestRangeEnd;
-            end = smallestRangeEnd + 1n;
-            if (end > this.properties.numbers.maxValue) {
-                end = 0n;
-            }
-        }
-        else {
-            start = smallestRangeStart;
-            end = smallestRangeEnd;
-        }
-        const startSync = new StartSync({ from: start, to: end, symbols: [] });
-        const encoder = new EncoderWrapper();
-        if (typeof BigUint64Array !== "undefined" &&
-            sortedEntries instanceof BigUint64Array) {
-            encoder.add_symbols(sortedEntries);
-        }
-        else {
-            for (const entry of sortedEntries) {
-                encoder.add_symbol(coerceBigInt(entry));
+        if (allCoordinatesToSyncWithIblt.length === 0) {
+            if (profile) {
+                emitSyncProfileEvent(profile, {
+                    name: "rateless.dispatchMode",
+                    entries: properties.entries.size,
+                    targets: properties.targets.length,
+                    details: { mode: "simple-only" },
+                });
+                emitSyncProfileDuration(profile, startedAt, {
+                    name: "rateless.onMaybeMissingEntries",
+                    entries: properties.entries.size,
+                    targets: properties.targets.length,
+                    details: { mode: "simple-only" },
+                });
             }
+            return;
         }
+        await ribltReady;
         // For smaller sets, the original `sqrt(n)` heuristic can occasionally under-provision
         // low-degree symbols early, causing an unnecessary `MoreSymbols` round-trip. Use a
         // small floor to make small-delta syncs more reliable without affecting large-n behavior.
-        let initialSymbols = Math.round(Math.sqrt(allCoordinatesToSyncWithIblt.length)); // TODO choose better
-        initialSymbols = Math.max(64, initialSymbols);
-        for (let i = 0; i < initialSymbols; i++) {
-            startSync.symbols.push(new SymbolSerialized(encoder.produce_next_coded_symbol()));
+        let initialSymbolCount = Math.round(Math.sqrt(allCoordinatesToSyncWithIblt.length)); // TODO choose better
+        initialSymbolCount = Math.max(64, initialSymbolCount);
+        const prepareStartedAt = syncProfileStart(profile);
+        const { encoder, start, end, initialSymbols } = prepareStartSyncEncoder(allCoordinatesToSyncWithIblt, this.properties.numbers.maxValue, initialSymbolCount);
+        if (profile) {
+            emitSyncProfileDuration(profile, prepareStartedAt, {
+                name: "rateless.prepareStartSyncEncoder",
+                entries: allCoordinatesToSyncWithIblt.length,
+                symbols: initialSymbols?.length,
+                details: {
+                    initialSymbolCount,
+                    includesInitialSymbols: initialSymbols != null,
+                },
+            });
         }
+        let startSyncSymbols = initialSymbols;
+        if (!startSyncSymbols) {
+            const produceStartedAt = syncProfileStart(profile);
+            startSyncSymbols = produceNextCodedSymbols(encoder, initialSymbolCount);
+            if (profile) {
+                emitSyncProfileDuration(profile, produceStartedAt, {
+                    name: "rateless.produceStartSyncSymbols",
+                    symbols: startSyncSymbols.length,
+                });
+            }
+        }
+        const startSync = new StartSync({
+            from: start,
+            to: end,
+            symbols: startSyncSymbols,
+        });
+        const syncId = getSyncIdString(startSync);
         const clear = () => {
             encoder.free();
-            clearTimeout(this.outgoingSyncProcesses.get(getSyncIdString(startSync))?.timeout);
-            this.outgoingSyncProcesses.delete(getSyncIdString(startSync));
+            clearTimeout(this.outgoingSyncProcesses.get(syncId)?.timeout);
+            this.outgoingSyncProcesses.delete(syncId);
         };
         const createTimeout = () => {
             return setTimeout(clear, 1e4); // TODO arg
         };
         let lastSeqNo = -1n;
-        let nextBatch = 1e4;
+        // Keep follow-up symbol payloads bounded. Each symbol is serialized as an
+        // object with three bigint fields, so very large batches can dominate heap under
+        // concurrent churn even though the native RIBLT encoder itself is compact.
+        const nextBatch = Math.max(MIN_MORE_SYMBOLS_BATCH_SIZE, Math.min(MAX_MORE_SYMBOLS_BATCH_SIZE, Math.ceil(allCoordinatesToSyncWithIblt.length / 4)));
         const obj = {
             encoder,
             timeout: createTimeout(),
@@ -701,26 +1048,73 @@ export class RatelessIBLTSynchronizer {
             },
             next: (properties) => {
                 if (properties.lastSeqNo <= lastSeqNo) {
-                    return [];
+                    return CodedSymbolBatch.fromSymbols([]);
                 }
                 lastSeqNo++;
                 obj.refresh(); // TODO use timestamp instead and collective pruning/refresh
-                let result = [];
-                for (let i = 0; i < nextBatch; i++) {
-                    result.push(encoder.produce_next_coded_symbol());
+                const produceStartedAt = syncProfileStart(profile);
+                const symbols = produceNextCodedSymbols(encoder, nextBatch);
+                if (profile) {
+                    emitSyncProfileDuration(profile, produceStartedAt, {
+                        name: "rateless.produceMoreSymbols",
+                        syncId,
+                        symbols: symbols.length,
+                    });
                 }
-                return result;
+                return symbols;
             },
             free: clear,
             outgoing: properties.entries,
         };
-        this.outgoingSyncProcesses.set(getSyncIdString(startSync), obj);
-        this.simple.rpc.send(startSync, {
+        this.outgoingSyncProcesses.set(syncId, obj);
+        if (profile) {
+            emitSyncProfileEvent(profile, {
+                name: "rateless.dispatchMode",
+                entries: properties.entries.size,
+                symbols: startSyncSymbols.length,
+                targets: properties.targets.length,
+                syncId,
+                details: { mode: "rateless" },
+            });
+        }
+        const sendStartedAt = syncProfileStart(profile);
+        const sendResult = this.simple.rpc.send(startSync, {
             mode: new SilentDelivery({ to: properties.targets, redundancy: 1 }),
-            priority: 1,
+            priority: SYNC_MESSAGE_PRIORITY,
         });
+        if (profile) {
+            void Promise.resolve(sendResult).then(() => emitSyncProfileDuration(profile, sendStartedAt, {
+                name: "rateless.sendStartSync",
+                messages: 1,
+                symbols: startSyncSymbols.length,
+                targets: properties.targets.length,
+                syncId,
+            }), () => emitSyncProfileDuration(profile, sendStartedAt, {
+                name: "rateless.sendStartSync",
+                messages: 1,
+                symbols: startSyncSymbols.length,
+                targets: properties.targets.length,
+                syncId,
+                details: { rejected: true },
+            }));
+        }
+        if (profile) {
+            emitSyncProfileDuration(profile, startedAt, {
+                name: "rateless.onMaybeMissingEntries",
+                entries: properties.entries.size,
+                messages: 1,
+                symbols: startSyncSymbols.length,
+                targets: properties.targets.length,
+                details: {
+                    mode: "rateless",
+                    ibltEntries: allCoordinatesToSyncWithIblt.length,
+                    naiveEntries: entriesToSyncNaively.size,
+                },
+            });
+        }
     }
     async onMessage(message, context) {
+        const profile = this.properties.sync?.profile;
         if (message instanceof StartSync) {
             const syncId = getSyncIdString(message);
             if (this.ingoingSyncProcesses.has(syncId)) {
@@ -731,19 +1125,36 @@ export class RatelessIBLTSynchronizer {
             }
             this.startedOrCompletedSynchronizations.add(syncId);
             const wrapped = message.end < message.start;
+            const decoderStartedAt = syncProfileStart(profile);
             const decoder = await this.getLocalDecoderForRange({
                 start1: message.start,
                 end1: wrapped ? this.properties.numbers.maxValue : message.end,
                 start2: 0n,
                 end2: wrapped ? message.end : 0n,
             });
+            if (profile) {
+                emitSyncProfileDuration(profile, decoderStartedAt, {
+                    name: "rateless.getLocalDecoderForRange",
+                    syncId,
+                    details: { wrapped, found: decoder !== false },
+                });
+            }
             if (!decoder) {
+                const sendStartedAt = syncProfileStart(profile);
                 await this.simple.rpc.send(new RequestAll({
                     syncId: message.syncId,
                 }), {
                     mode: new SilentDelivery({ to: [context.from], redundancy: 1 }),
-                    priority: 1,
+                    priority: SYNC_MESSAGE_PRIORITY,
                 });
+                if (profile) {
+                    emitSyncProfileDuration(profile, sendStartedAt, {
+                        name: "rateless.sendRequestAll",
+                        messages: 1,
+                        targets: 1,
+                        syncId,
+                    });
+                }
                 return true;
             }
             const createTimeout = () => {
@@ -778,9 +1189,15 @@ export class RatelessIBLTSynchronizer {
                         if (!decoder.decoded()) {
                             return false;
                         }
-                        const allMissingSymbolsInRemote = [];
-                        for (const missingSymbol of decoder.get_remote_symbols()) {
-                            allMissingSymbolsInRemote.push(missingSymbol);
+                        const remoteStartedAt = syncProfileStart(profile);
+                        const allMissingSymbolsInRemote = getRemoteSymbolValues(decoder);
+                        if (profile) {
+                            emitSyncProfileDuration(profile, remoteStartedAt, {
+                                name: "rateless.remoteSymbols",
+                                entries: allMissingSymbolsInRemote.length,
+                                symbols: allMissingSymbolsInRemote.length,
+                                syncId,
+                            });
                         }
                         // The IBLT decoder is based on a local snapshot. Entries can arrive via
                         // overlapping repair before we issue the follow-up simple request, so
@@ -796,7 +1213,37 @@ export class RatelessIBLTSynchronizer {
                             break;
                         }
                         lastSeqNo = symbolMessage.seqNo;
-                        for (const symbol of symbolMessage.symbols) {
+                        const addBatchAndDecode = decoder
+                            .add_coded_symbols_and_try_decode;
+                        if (typeof BigUint64Array !== "undefined" && addBatchAndDecode) {
+                            const flatStartedAt = syncProfileStart(profile);
+                            const flatSymbols = flatFromCodedSymbols(symbolMessage.symbols);
+                            if (profile) {
+                                emitSyncProfileDuration(profile, flatStartedAt, {
+                                    name: "rateless.symbolBatchToFlat",
+                                    symbols: flatSymbols.length / CODED_SYMBOL_WORDS,
+                                    syncId,
+                                });
+                            }
+                            const decodeStartedAt = syncProfileStart(profile);
+                            const decoded = addBatchAndDecode.call(decoder, flatSymbols);
+                            if (profile) {
+                                emitSyncProfileDuration(profile, decodeStartedAt, {
+                                    name: "rateless.decodeBatch",
+                                    symbols: flatSymbols.length / CODED_SYMBOL_WORDS,
+                                    syncId,
+                                    details: { decoded },
+                                });
+                            }
+                            if (decoded && finalizeIfDecoded()) {
+                                return true;
+                            }
+                            continue;
+                        }
+                        const decodeLoopStartedAt = syncProfileStart(profile);
+                        let symbolsProcessed = 0;
+                        for (const symbol of CodedSymbolBatch.from(symbolMessage.symbols)) {
+                            symbolsProcessed += 1;
                             const normalizedSymbol = symbol instanceof SymbolSerialized
                                 ? symbol
                                 : new SymbolSerialized({
@@ -820,6 +1267,13 @@ export class RatelessIBLTSynchronizer {
                                 throw error;
                             }
                         }
+                        if (profile) {
+                            emitSyncProfileDuration(profile, decodeLoopStartedAt, {
+                                name: "rateless.decodeSymbolLoop",
+                                symbols: symbolsProcessed,
+                                syncId,
+                            });
+                        }
                     }
                     return false;
                 },
@@ -834,17 +1288,27 @@ export class RatelessIBLTSynchronizer {
                 return true;
             }
             // not done, request more symbols
+            const sendStartedAt = syncProfileStart(profile);
             await this.simple.rpc.send(new RequestMoreSymbols({
                 lastSeqNo: 0n,
                 syncId: message.syncId,
             }), {
                 mode: new SilentDelivery({ to: [context.from], redundancy: 1 }),
-                priority: 1,
+                priority: SYNC_MESSAGE_PRIORITY,
             });
+            if (profile) {
+                emitSyncProfileDuration(profile, sendStartedAt, {
+                    name: "rateless.sendRequestMoreSymbols",
+                    messages: 1,
+                    targets: 1,
+                    syncId,
+                });
+            }
             return true;
         }
         else if (message instanceof MoreSymbols) {
-            const obj = this.ingoingSyncProcesses.get(getSyncIdString(message));
+            const syncId = getSyncIdString(message);
+            const obj = this.ingoingSyncProcesses.get(syncId);
             if (!obj) {
                 return true;
             }
@@ -856,28 +1320,55 @@ export class RatelessIBLTSynchronizer {
                 return true; // we don't have enough information, or received information that is redundant
             }
             // we are not done
-            this.simple.rpc.send(new RequestMoreSymbols({
+            const sendStartedAt = syncProfileStart(profile);
+            const sendResult = this.simple.rpc.send(new RequestMoreSymbols({
                 lastSeqNo: message.seqNo,
                 syncId: message.syncId,
             }), {
                 mode: new SilentDelivery({ to: [context.from], redundancy: 1 }),
-                priority: 1,
+                priority: SYNC_MESSAGE_PRIORITY,
             });
+            if (profile) {
+                void Promise.resolve(sendResult).then(() => emitSyncProfileDuration(profile, sendStartedAt, {
+                    name: "rateless.sendRequestMoreSymbols",
+                    messages: 1,
+                    targets: 1,
+                    syncId,
+                }), () => emitSyncProfileDuration(profile, sendStartedAt, {
+                    name: "rateless.sendRequestMoreSymbols",
+                    messages: 1,
+                    targets: 1,
+                    syncId,
+                    details: { rejected: true },
+                }));
+            }
             return true;
         }
         else if (message instanceof RequestMoreSymbols) {
-            const obj = this.outgoingSyncProcesses.get(getSyncIdString(message));
+            const syncId = getSyncIdString(message);
+            const obj = this.outgoingSyncProcesses.get(syncId);
             if (!obj) {
                 return true;
             }
+            const symbols = obj.next(message);
+            const sendStartedAt = syncProfileStart(profile);
             await this.properties.rpc.send(new MoreSymbols({
                 lastSeqNo: message.lastSeqNo,
                 syncId: message.syncId,
-                symbols: obj.next(message).map((x) => new SymbolSerialized(x)),
+                symbols,
             }), {
                 mode: new SilentDelivery({ to: [context.from], redundancy: 1 }),
-                priority: 1,
+                priority: SYNC_MESSAGE_PRIORITY,
             });
+            if (profile) {
+                emitSyncProfileDuration(profile, sendStartedAt, {
+                    name: "rateless.sendMoreSymbols",
+                    messages: 1,
+                    symbols: symbols.length,
+                    targets: 1,
+                    syncId,
+                });
+            }
             return true;
         }
         else if (message instanceof RequestAll) {