@peerbit/shared-log 13.1.17 → 13.2.0

package/dist/src/sync/simple.js

@@ -37,7 +37,7 @@ import { Cache } from "@peerbit/cache";
 import { Compare, IntegerCompare, Or, } from "@peerbit/indexer-interface";
 import { Entry, Log } from "@peerbit/log";
 import { CONVERGENCE_MESSAGE_PRIORITY, SilentDelivery, } from "@peerbit/stream-interface";
-import { EntryWithRefs, createExchangeHeadsMessages, } from "../exchange-heads.js";
+import { EntryWithRefs, createExchangeHeadsMessages, createRawExchangeHeadsMessages, } from "../exchange-heads.js";
 import { TransportMessage } from "../message.js";
 import { emitSyncProfileDuration, syncProfileStart } from "./profile.js";
 let RequestMaybeSync = (() => {
@@ -160,10 +160,99 @@ let ConfirmEntriesMessage = (() => {
     return ConfirmEntriesMessage = _classThis;
 })();
 export { ConfirmEntriesMessage };
-const getHashesFromSymbols = async (symbols, entryIndex, coordinateToHash) => {
+export const SIMPLE_SYNC_RAW_EXCHANGE_HEADS_CAPABILITY = 1;
+let ResponseMaybeSyncCapabilities = (() => {
+    let _classDecorators = [variant([0, 8])];
+    let _classDescriptor;
+    let _classExtraInitializers = [];
+    let _classThis;
+    let _classSuper = TransportMessage;
+    let _hashes_decorators;
+    let _hashes_initializers = [];
+    let _hashes_extraInitializers = [];
+    let _capabilities_decorators;
+    let _capabilities_initializers = [];
+    let _capabilities_extraInitializers = [];
+    var ResponseMaybeSyncCapabilities = class extends _classSuper {
+        static { _classThis = this; }
+        static {
+            const _metadata = typeof Symbol === "function" && Symbol.metadata ? Object.create(_classSuper[Symbol.metadata] ?? null) : void 0;
+            _hashes_decorators = [field({ type: vec("string") })];
+            _capabilities_decorators = [field({ type: "u32" })];
+            __esDecorate(null, null, _hashes_decorators, { kind: "field", name: "hashes", static: false, private: false, access: { has: obj => "hashes" in obj, get: obj => obj.hashes, set: (obj, value) => { obj.hashes = value; } }, metadata: _metadata }, _hashes_initializers, _hashes_extraInitializers);
+            __esDecorate(null, null, _capabilities_decorators, { kind: "field", name: "capabilities", static: false, private: false, access: { has: obj => "capabilities" in obj, get: obj => obj.capabilities, set: (obj, value) => { obj.capabilities = value; } }, metadata: _metadata }, _capabilities_initializers, _capabilities_extraInitializers);
+            __esDecorate(null, _classDescriptor = { value: _classThis }, _classDecorators, { kind: "class", name: _classThis.name, metadata: _metadata }, null, _classExtraInitializers);
+            ResponseMaybeSyncCapabilities = _classThis = _classDescriptor.value;
+            if (_metadata) Object.defineProperty(_classThis, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
+            __runInitializers(_classThis, _classExtraInitializers);
+        }
+        hashes = __runInitializers(this, _hashes_initializers, void 0);
+        capabilities = (__runInitializers(this, _hashes_extraInitializers), __runInitializers(this, _capabilities_initializers, void 0));
+        constructor(props) {
+            super();
+            __runInitializers(this, _capabilities_extraInitializers);
+            this.hashes = props.hashes;
+            this.capabilities =
+                props.capabilities ?? SIMPLE_SYNC_RAW_EXCHANGE_HEADS_CAPABILITY;
+        }
+    };
+    return ResponseMaybeSyncCapabilities = _classThis;
+})();
+export { ResponseMaybeSyncCapabilities };
+let RequestMaybeSyncCoordinateCapabilities = (() => {
+    let _classDecorators = [variant([0, 9])];
+    let _classDescriptor;
+    let _classExtraInitializers = [];
+    let _classThis;
+    let _classSuper = TransportMessage;
+    let _hashNumbers_decorators;
+    let _hashNumbers_initializers = [];
+    let _hashNumbers_extraInitializers = [];
+    let _capabilities_decorators;
+    let _capabilities_initializers = [];
+    let _capabilities_extraInitializers = [];
+    var RequestMaybeSyncCoordinateCapabilities = class extends _classSuper {
+        static { _classThis = this; }
+        static {
+            const _metadata = typeof Symbol === "function" && Symbol.metadata ? Object.create(_classSuper[Symbol.metadata] ?? null) : void 0;
+            _hashNumbers_decorators = [field({ type: vec("u64") })];
+            _capabilities_decorators = [field({ type: "u32" })];
+            __esDecorate(null, null, _hashNumbers_decorators, { kind: "field", name: "hashNumbers", static: false, private: false, access: { has: obj => "hashNumbers" in obj, get: obj => obj.hashNumbers, set: (obj, value) => { obj.hashNumbers = value; } }, metadata: _metadata }, _hashNumbers_initializers, _hashNumbers_extraInitializers);
+            __esDecorate(null, null, _capabilities_decorators, { kind: "field", name: "capabilities", static: false, private: false, access: { has: obj => "capabilities" in obj, get: obj => obj.capabilities, set: (obj, value) => { obj.capabilities = value; } }, metadata: _metadata }, _capabilities_initializers, _capabilities_extraInitializers);
+            __esDecorate(null, _classDescriptor = { value: _classThis }, _classDecorators, { kind: "class", name: _classThis.name, metadata: _metadata }, null, _classExtraInitializers);
+            RequestMaybeSyncCoordinateCapabilities = _classThis = _classDescriptor.value;
+            if (_metadata) Object.defineProperty(_classThis, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
+            __runInitializers(_classThis, _classExtraInitializers);
+        }
+        hashNumbers = __runInitializers(this, _hashNumbers_initializers, void 0);
+        capabilities = (__runInitializers(this, _hashNumbers_extraInitializers), __runInitializers(this, _capabilities_initializers, void 0));
+        constructor(props) {
+            super();
+            __runInitializers(this, _capabilities_extraInitializers);
+            this.hashNumbers = props.hashNumbers;
+            this.capabilities =
+                props.capabilities ?? SIMPLE_SYNC_RAW_EXCHANGE_HEADS_CAPABILITY;
+        }
+    };
+    return RequestMaybeSyncCoordinateCapabilities = _classThis;
+})();
+export { RequestMaybeSyncCoordinateCapabilities };
+const canReceiveRawExchangeHeads = (message) => (message instanceof ResponseMaybeSyncCapabilities ||
+    message instanceof RequestMaybeSyncCoordinateCapabilities) &&
+    (message.capabilities & SIMPLE_SYNC_RAW_EXCHANGE_HEADS_CAPABILITY) !== 0;
+const getHashesFromSymbols = async (symbols, entryIndex, coordinateToHash, resolveHashesForSymbols, resolveHashListForSymbols) => {
     let queries = [];
     let batchSize = 128; // TODO arg
     let results = new Set();
+    let missingSymbols = [];
+    const addMissingUnlessCached = (symbol) => {
+        const fromCache = coordinateToHash.get(symbol);
+        if (fromCache) {
+            results.add(fromCache);
+            return;
+        }
+        missingSymbols.push(symbol);
+    };
     const handleBatch = async (end = false) => {
         if (queries.length >= batchSize || (end && queries.length > 0)) {
             const entries = await entryIndex
@@ -176,16 +265,63 @@ const getHashesFromSymbols = async (symbols, entryIndex, coordinateToHash) => {
             }
         }
     };
-    for (let i = 0; i < symbols.length; i++) {
-        const fromCache = coordinateToHash.get(symbols[i]);
-        if (fromCache) {
-            results.add(fromCache);
-            continue;
+    if (resolveHashListForSymbols) {
+        const resolvedHashes = await resolveHashListForSymbols(symbols);
+        if (resolvedHashes) {
+            const resolvedHashList = Array.isArray(resolvedHashes)
+                ? resolvedHashes
+                : [...resolvedHashes];
+            let mergedHashes;
+            for (const symbol of symbols) {
+                const fromCache = coordinateToHash.get(symbol);
+                if (fromCache) {
+                    mergedHashes ??= new Set(resolvedHashList);
+                    mergedHashes.add(fromCache);
+                }
+            }
+            return mergedHashes ?? resolvedHashList;
+        }
+    }
+    if (resolveHashesForSymbols) {
+        const resolved = await resolveHashesForSymbols(symbols);
+        if (resolved) {
+            for (const symbol of symbols) {
+                const hashes = resolved.get(symbol);
+                if (!hashes) {
+                    addMissingUnlessCached(symbol);
+                    continue;
+                }
+                let singleHash;
+                let count = 0;
+                for (const hash of hashes) {
+                    results.add(hash);
+                    singleHash = hash;
+                    count += 1;
+                }
+                if (count === 0) {
+                    addMissingUnlessCached(symbol);
+                }
+                else if (count === 1) {
+                    coordinateToHash.add(symbol, singleHash);
+                }
+            }
+        }
+        else {
+            for (const symbol of symbols) {
+                addMissingUnlessCached(symbol);
+            }
+        }
+    }
+    else {
+        for (const symbol of symbols) {
+            addMissingUnlessCached(symbol);
         }
+    }
+    for (const symbol of missingSymbols) {
         const matchQuery = new IntegerCompare({
             key: "hashNumber",
             compare: Compare.Equal,
-            value: symbols[i],
+            value: symbol,
         });
         queries.push(matchQuery);
         await handleBatch();
@@ -193,6 +329,7 @@ const getHashesFromSymbols = async (symbols, entryIndex, coordinateToHash) => {
     await handleBatch(true);
     return results;
 };
+const hashLookupResultSize = (hashes) => Array.isArray(hashes) ? hashes.length : hashes.size;
 const DEFAULT_CONVERGENT_REPAIR_TIMEOUT_MS = 30_000;
 const DEFAULT_CONVERGENT_RETRY_INTERVALS_MS = [0, 1_000, 3_000, 7_000];
 const DEFAULT_BEST_EFFORT_RETRY_INTERVALS_MS = [0];
@@ -207,6 +344,8 @@ export const SYNC_MESSAGE_PRIORITY = CONVERGENCE_MESSAGE_PRIORITY;
 // pubsub stream warmup). Keep it coarse-grained so we do not hammer the network under
 // large historical backfills.
 const SIMPLE_SYNC_RETRY_AFTER_MS = 10_000;
+const EXCHANGE_HEAD_RESPONSE_DEDUPE_TTL_MS = SIMPLE_SYNC_RETRY_AFTER_MS - 1_000;
+const RECENT_KNOWN_EXCHANGE_HEAD_SUPPRESSION_MS = 30_000;
 const createDeferred = () => {
     let resolve;
     let reject;
@@ -226,7 +365,12 @@ export class SimpleSyncronizer {
     log;
     entryIndex;
     coordinateToHash;
+    resolveHashesForSymbols;
+    resolveHashListForSymbols;
     syncOptions;
+    isEntryRecentlyKnownByPeer;
+    sendRawExchangeHeads;
+    recentlySentExchangeHeads;
     repairSessionCounter;
     repairSessions;
     // Syncing and dedeplucation work
@@ -240,7 +384,12 @@ export class SimpleSyncronizer {
         this.log = properties.log;
         this.entryIndex = properties.entryIndex;
         this.coordinateToHash = properties.coordinateToHash;
+        this.resolveHashesForSymbols = properties.resolveHashesForSymbols;
+        this.resolveHashListForSymbols = properties.resolveHashListForSymbols;
         this.syncOptions = properties.sync;
+        this.isEntryRecentlyKnownByPeer = properties.isEntryRecentlyKnownByPeer;
+        this.sendRawExchangeHeads = properties.sendRawExchangeHeads;
+        this.recentlySentExchangeHeads = new Map();
         this.repairSessionCounter = 0;
         this.repairSessions = new Map();
     }
@@ -303,6 +452,37 @@ export class SimpleSyncronizer {
         }
         return out;
     }
+    filterRecentlySentExchangeHeads(hashes, peer) {
+        const peerHash = peer.hashcode();
+        const now = Date.now();
+        let recentlySent = this.recentlySentExchangeHeads.get(peerHash);
+        if (!recentlySent) {
+            recentlySent = new Map();
+            this.recentlySentExchangeHeads.set(peerHash, recentlySent);
+        }
+        for (const [hash, timestamp] of recentlySent) {
+            if (now - timestamp > EXCHANGE_HEAD_RESPONSE_DEDUPE_TTL_MS) {
+                recentlySent.delete(hash);
+            }
+        }
+        const out = [];
+        const seen = new Set();
+        for (const hash of hashes) {
+            if (seen.has(hash)) {
+                continue;
+            }
+            seen.add(hash);
+            if (recentlySent.has(hash)) {
+                continue;
+            }
+            if (this.isEntryRecentlyKnownByPeer?.(hash, peerHash, RECENT_KNOWN_EXCHANGE_HEAD_SUPPRESSION_MS)) {
+                continue;
+            }
+            recentlySent.set(hash, now);
+            out.push(hash);
+        }
+        return out;
+    }
     isRepairSessionComplete(session) {
         for (const state of session.targets.values()) {
             if (state.unresolved.size > 0) {
@@ -348,12 +528,22 @@ export class SimpleSyncronizer {
             return;
         }
         for (const state of session.targets.values()) {
-            for (const hash of [...state.unresolved]) {
-                if (await this.log.has(hash)) {
-                    state.unresolved.delete(hash);
-                }
+            const resolved = typeof this.log.hasMany === "function"
+                ? await this.log.hasMany(state.unresolved)
+                : await this.getExistingRepairHashes(state.unresolved);
+            for (const hash of resolved) {
+                state.unresolved.delete(hash);
+            }
+        }
+    }
+    async getExistingRepairHashes(hashes) {
+        const resolved = new Set();
+        for (const hash of hashes) {
+            if (await this.log.has(hash)) {
+                resolved.add(hash);
             }
         }
+        return resolved;
     }
     markRepairSessionResolvedHashes(hashes) {
         if (hashes.length === 0 || this.repairSessions.size === 0) {
@@ -370,6 +560,19 @@ export class SimpleSyncronizer {
             }
         }
     }
+    markRepairSessionResolvedHash(hash) {
+        if (this.repairSessions.size === 0) {
+            return;
+        }
+        for (const [sessionId, session] of this.repairSessions) {
+            for (const state of session.targets.values()) {
+                state.unresolved.delete(hash);
+            }
+            if (this.isRepairSessionComplete(session)) {
+                this.finalizeRepairSession(sessionId, true);
+            }
+        }
+    }
     async runRepairSession(sessionId) {
         const session = this.repairSessions.get(sessionId);
         if (!session) {
@@ -535,9 +738,15 @@ export class SimpleSyncronizer {
         };
     }
     async onMaybeMissingEntries(properties) {
+        await this.onMaybeMissingHashes({
+            hashes: this.getPrioritizedHashes(properties.entries),
+            targets: properties.targets,
+        });
+    }
+    async onMaybeMissingHashes(properties) {
         const profile = this.syncOptions?.profile;
         const startedAt = syncProfileStart(profile);
-        const hashes = this.getPrioritizedHashes(properties.entries);
+        const hashes = [...properties.hashes];
         const chunks = this.chunk(hashes, this.maxHashesPerMessage);
         try {
             await chunks.reduce((promise, chunk) => promise.then(() => this.rpc.send(new RequestMaybeSync({ hashes: chunk }), {
@@ -559,69 +768,92 @@ export class SimpleSyncronizer {
             }
         }
     }
+    /**
+     * Ship exchange heads to one peer: the fused native raw path when the
+     * peer advertised raw capability and the shared log provided a fused
+     * sender, otherwise the TS message path (raw or plain by capability).
+     * Returns the number of messages sent and whether the fused path ran.
+     */
+    async shipExchangeHeads(hashes, to, canReceiveRaw) {
+        if (canReceiveRaw && this.sendRawExchangeHeads) {
+            const sentMessages = await this.sendRawExchangeHeads(hashes, [
+                to.hashcode(),
+            ]);
+            if (sentMessages !== undefined) {
+                return { messages: sentMessages, fused: true };
+            }
+        }
+        let messages = 0;
+        const messageGenerator = canReceiveRaw
+            ? createRawExchangeHeadsMessages(this.log, hashes, this.syncOptions?.profile)
+            : createExchangeHeadsMessages(this.log, hashes);
+        for await (const message of messageGenerator) {
+            messages += 1;
+            await this.rpc.send(message, {
+                mode: new SilentDelivery({ to: [to], redundancy: 1 }),
+            });
+        }
+        return { messages, fused: false };
+    }
     async onMessage(msg, context) {
         const from = context.from;
         if (msg instanceof RequestMaybeSync) {
             await this.queueSync(msg.hashes, from);
             return true;
         }
-        else if (msg instanceof ResponseMaybeSync) {
+        else if (msg instanceof ResponseMaybeSync ||
+            msg instanceof ResponseMaybeSyncCapabilities) {
             // TODO perhaps send less messages to more receivers for performance reasons?
             // TODO wait for previous send to target before trying to send more?
             const profile = this.syncOptions?.profile;
             const startedAt = syncProfileStart(profile);
+            const hashes = this.filterRecentlySentExchangeHeads(msg.hashes, from);
             let messages = 0;
+            let fused = false;
             try {
-                for await (const message of createExchangeHeadsMessages(this.log, msg.hashes)) {
-                    messages += 1;
-                    await this.rpc.send(message, {
-                        mode: new SilentDelivery({ to: [context.from], redundancy: 1 }),
-                    });
-                }
+                ({ messages, fused } = await this.shipExchangeHeads(hashes, context.from, canReceiveRawExchangeHeads(msg)));
             }
             finally {
                 if (profile) {
                     emitSyncProfileDuration(profile, startedAt, {
                         name: "simple.exchangeHeads",
-                        entries: msg.hashes.length,
+                        entries: hashes.length,
                         messages,
                         targets: 1,
-                        details: { source: "responseMaybeSync" },
+                        details: { source: "responseMaybeSync", fused },
                     });
                 }
             }
             return true;
         }
-        else if (msg instanceof RequestMaybeSyncCoordinate) {
+        else if (msg instanceof RequestMaybeSyncCoordinate ||
+            msg instanceof RequestMaybeSyncCoordinateCapabilities) {
             const profile = this.syncOptions?.profile;
             const lookupStartedAt = syncProfileStart(profile);
-            const hashes = await getHashesFromSymbols(msg.hashNumbers, this.entryIndex, this.coordinateToHash);
+            const hashes = await getHashesFromSymbols(msg.hashNumbers, this.entryIndex, this.coordinateToHash, this.resolveHashesForSymbols, this.resolveHashListForSymbols);
             if (profile) {
                 emitSyncProfileDuration(profile, lookupStartedAt, {
                     name: "simple.coordinateLookup",
-                    entries: hashes.size,
+                    entries: hashLookupResultSize(hashes),
                     symbols: msg.hashNumbers.length,
                 });
             }
             const exchangeStartedAt = syncProfileStart(profile);
+            const hashesToSend = this.filterRecentlySentExchangeHeads(hashes, from);
             let messages = 0;
+            let fused = false;
             try {
-                for await (const message of createExchangeHeadsMessages(this.log, hashes)) {
-                    messages += 1;
-                    await this.rpc.send(message, {
-                        mode: new SilentDelivery({ to: [context.from], redundancy: 1 }),
-                        // dont set priority 1 here because this will block other messages that should higher priority
-                    });
-                }
+                // dont set priority 1 here because this will block other messages that should higher priority
+                ({ messages, fused } = await this.shipExchangeHeads(hashesToSend, context.from, canReceiveRawExchangeHeads(msg)));
             }
             finally {
                 if (profile) {
                     emitSyncProfileDuration(profile, exchangeStartedAt, {
                         name: "simple.exchangeHeads",
-                        entries: hashes.size,
+                        entries: hashesToSend.length,
                         messages,
                         targets: 1,
-                        details: { source: "requestMaybeSyncCoordinate" },
+                        details: { source: "requestMaybeSyncCoordinate", fused },
                     });
                 }
             }
@@ -632,47 +864,105 @@ export class SimpleSyncronizer {
         }
     }
     onReceivedEntries(properties) {
-        const resolvedHashes = [];
-        for (const entry of properties.entries) {
-            resolvedHashes.push(entry.entry.hash);
-            this.clearSyncInFlightForPeer(properties.from.hashcode(), entry.entry.hash);
-        }
-        this.markRepairSessionResolvedHashes(resolvedHashes);
+        return this.onReceivedEntryHashes({
+            hashes: properties.entries.map((entry) => entry.entry.hash),
+            from: properties.from,
+        });
+    }
+    onReceivedEntryHashes(properties) {
+        this.clearSyncInFlightForPeerHashes(properties.from.hashcode(), properties.hashes);
+        this.markRepairSessionResolvedHashes(properties.hashes);
     }
     async queueSync(keys, from, options) {
         const requestHashes = [];
         const profile = this.syncOptions?.profile;
         const startedAt = syncProfileStart(profile);
+        const resolveKnownStartedAt = syncProfileStart(profile);
+        const knownKeys = options?.skipCheck === true
+            ? undefined
+            : await this.resolveKnownSyncKeys(keys);
+        if (profile) {
+            emitSyncProfileDuration(profile, resolveKnownStartedAt, {
+                name: "simple.queueSync.resolveKnown",
+                entries: keys.length,
+                count: knownKeys?.keys.size ?? 0,
+                details: {
+                    checkedCoordinates: knownKeys?.checkedCoordinates === true,
+                    checkedHashes: knownKeys?.checkedHashes === true,
+                    skipCheck: options?.skipCheck === true,
+                },
+            });
+        }
+        const fromHash = from.hashcode();
+        let queuedHashAliases;
+        const getQueuedSyncKeyForBatch = (key) => {
+            if (this.syncInFlightQueue.has(key)) {
+                return key;
+            }
+            if (typeof key === "string") {
+                if (!queuedHashAliases) {
+                    queuedHashAliases = new Map();
+                    for (const queuedKey of this.syncInFlightQueue.keys()) {
+                        if (typeof queuedKey !== "bigint") {
+                            continue;
+                        }
+                        const hash = this.coordinateToHash.get(queuedKey);
+                        if (hash) {
+                            queuedHashAliases.set(hash, queuedKey);
+                        }
+                    }
+                }
+                return queuedHashAliases.get(key);
+            }
+            const hash = this.coordinateToHash.get(key);
+            return hash && this.syncInFlightQueue.has(hash) ? hash : undefined;
+        };
         try {
+            const loopStartedAt = syncProfileStart(profile);
             for (const key of keys) {
-                const coordinateOrHash = this.getQueuedSyncKey(key) ?? key;
+                const coordinateOrHash = getQueuedSyncKeyForBatch(key) ?? key;
                 const inFlight = this.syncInFlightQueue.get(coordinateOrHash);
                 if (inFlight) {
-                    if (!inFlight.find((x) => x.hashcode() === from.hashcode())) {
+                    if (!inFlight.find((x) => x.hashcode() === fromHash)) {
                         inFlight.push(from);
-                        let inverted = this.syncInFlightQueueInverted.get(from.hashcode());
+                        let inverted = this.syncInFlightQueueInverted.get(fromHash);
                         if (!inverted) {
                             inverted = new Set();
-                            this.syncInFlightQueueInverted.set(from.hashcode(), inverted);
+                            this.syncInFlightQueueInverted.set(fromHash, inverted);
                         }
                         inverted.add(coordinateOrHash);
                     }
                 }
                 else if (options?.skipCheck ||
-                    !(await this.checkHasCoordinateOrHash(coordinateOrHash))) {
+                    !(await this.checkHasCoordinateOrHash(coordinateOrHash, knownKeys))) {
                     // Track the initial sender so we can retry if the first request is lost.
                     this.syncInFlightQueue.set(coordinateOrHash, [from]);
-                    let inverted = this.syncInFlightQueueInverted.get(from.hashcode());
+                    let inverted = this.syncInFlightQueueInverted.get(fromHash);
                     if (!inverted) {
                         inverted = new Set();
-                        this.syncInFlightQueueInverted.set(from.hashcode(), inverted);
+                        this.syncInFlightQueueInverted.set(fromHash, inverted);
                     }
                     inverted.add(coordinateOrHash);
                     requestHashes.push(coordinateOrHash); // request immediately (first time we have seen this hash)
+                    if (queuedHashAliases &&
+                        typeof coordinateOrHash === "bigint") {
+                        const hash = this.coordinateToHash.get(coordinateOrHash);
+                        if (hash) {
+                            queuedHashAliases.set(hash, coordinateOrHash);
+                        }
+                    }
                 }
             }
+            if (profile) {
+                emitSyncProfileDuration(profile, loopStartedAt, {
+                    name: "simple.queueSync.plan",
+                    entries: keys.length,
+                    count: requestHashes.length,
+                    targets: 1,
+                });
+            }
             requestHashes.length > 0 &&
-                (await this.requestSync(requestHashes, [from.hashcode()]));
+                (await this.requestSync(requestHashes, [fromHash]));
         }
         finally {
             if (profile) {
@@ -726,7 +1016,11 @@ export class SimpleSyncronizer {
                 const chunks = this.chunk(coordinateHashes, this.maxCoordinatesPerMessage);
                 coordinateMessages = chunks.length;
                 for (const chunk of chunks) {
-                    await this.rpc.send(new RequestMaybeSyncCoordinate({ hashNumbers: chunk }), {
+                    await this.rpc.send(this.syncOptions?.rawExchangeHeads
+                        ? new RequestMaybeSyncCoordinateCapabilities({
+                            hashNumbers: chunk,
+                        })
+                        : new RequestMaybeSyncCoordinate({ hashNumbers: chunk }), {
                         mode: new SilentDelivery({ to, redundancy: 1 }),
                         priority: SYNC_MESSAGE_PRIORITY,
                     });
@@ -736,7 +1030,9 @@ export class SimpleSyncronizer {
                 const chunks = this.chunk(stringHashes, this.maxHashesPerMessage);
                 stringMessages = chunks.length;
                 for (const chunk of chunks) {
-                    await this.rpc.send(new ResponseMaybeSync({ hashes: chunk }), {
+                    await this.rpc.send(this.syncOptions?.rawExchangeHeads
+                        ? new ResponseMaybeSyncCapabilities({ hashes: chunk })
+                        : new ResponseMaybeSync({ hashes: chunk }), {
                         mode: new SilentDelivery({ to, redundancy: 1 }),
                         priority: SYNC_MESSAGE_PRIORITY,
                     });
@@ -758,7 +1054,58 @@ export class SimpleSyncronizer {
             }
         }
     }
-    async checkHasCoordinateOrHash(key) {
+    async resolveKnownSyncKeys(keys) {
+        const hashes = [];
+        const coordinates = [];
+        for (const key of keys) {
+            if (typeof key === "bigint") {
+                coordinates.push(key);
+            }
+            else {
+                hashes.push(key);
+            }
+        }
+        const known = {
+            keys: new Set(),
+            checkedCoordinates: false,
+            checkedHashes: false,
+        };
+        if (hashes.length > 0) {
+            for (const hash of await this.log.hasMany(hashes)) {
+                known.keys.add(hash);
+            }
+            known.checkedHashes = true;
+        }
+        if (coordinates.length > 0 && this.resolveHashesForSymbols) {
+            const resolved = await this.resolveHashesForSymbols(coordinates);
+            if (resolved) {
+                for (const coordinate of coordinates) {
+                    const hashes = resolved.get(coordinate);
+                    if (!hashes) {
+                        continue;
+                    }
+                    for (const _hash of hashes) {
+                        known.keys.add(coordinate);
+                        break;
+                    }
+                }
+                known.checkedCoordinates = true;
+            }
+        }
+        return known.checkedCoordinates || known.checkedHashes ? known : undefined;
+    }
+    async checkHasCoordinateOrHash(key, knownKeys) {
+        if (knownKeys) {
+            if (knownKeys.keys.has(key)) {
+                return true;
+            }
+            if (typeof key === "bigint" && knownKeys.checkedCoordinates) {
+                return false;
+            }
+            if (typeof key === "string" && knownKeys.checkedHashes) {
+                return false;
+            }
+        }
         return typeof key === "bigint"
             ? (await this.entryIndex.count({ query: { hashNumber: key } })) > 0
             : this.log.has(key);
@@ -834,18 +1181,49 @@ export class SimpleSyncronizer {
         this.syncInFlightQueue.clear();
         this.syncInFlightQueueInverted.clear();
         this.syncInFlight.clear();
+        this.recentlySentExchangeHeads.clear();
         for (const sessionId of [...this.repairSessions.keys()]) {
             this.finalizeRepairSession(sessionId, false);
         }
         clearTimeout(this.syncMoreInterval);
     }
     onEntryAdded(entry) {
-        this.clearSyncProcess(entry.hash);
-        this.markRepairSessionResolvedHashes([entry.hash]);
+        this.onEntryAddedHash(entry.hash);
+    }
+    onEntryAddedHashes(hashes) {
+        if (hashes.length === 0 || !this.hasEntryAddedState()) {
+            return;
+        }
+        this.clearSyncProcesses(hashes);
+        this.markRepairSessionResolvedHashes(hashes);
+    }
+    onEntryAddedHash(hash) {
+        if (!this.hasEntryAddedState()) {
+            return;
+        }
+        this.clearSyncProcess(hash);
+        this.markRepairSessionResolvedHash(hash);
     }
     onEntryRemoved(hash) {
+        if (!this.hasSyncProcessState()) {
+            return;
+        }
         return this.clearSyncProcess(hash);
     }
+    onEntryRemovedHashes(hashes) {
+        if (hashes.length === 0 || !this.hasSyncProcessState()) {
+            return;
+        }
+        return this.clearSyncProcesses(hashes);
+    }
+    hasEntryAddedState() {
+        return this.hasSyncProcessState() || this.repairSessions.size > 0;
+    }
+    hasSyncProcessState() {
+        return (this.syncInFlightQueue.size > 0 ||
+            this.syncInFlightQueueInverted.size > 0 ||
+            this.syncInFlight.size > 0);
+    }
     clearSyncProcessKey(key) {
         const inflight = this.syncInFlightQueue.get(key);
         if (inflight) {
@@ -870,24 +1248,52 @@ export class SimpleSyncronizer {
             }
         }
     }
-    getKnownAliases(hash) {
-        const aliases = new Set([hash]);
-        for (const key of [
-            ...this.syncInFlightQueue.keys(),
-            ...[...this.syncInFlight.values()].flatMap((map) => [...map.keys()]),
-        ]) {
+    forEachKnownAlias(hash, callback) {
+        callback(hash);
+        if (this.syncInFlightQueue.size === 0 && this.syncInFlight.size === 0) {
+            return;
+        }
+        for (const key of this.syncInFlightQueue.keys()) {
             if (typeof key === "bigint" && this.coordinateToHash.get(key) === hash) {
-                aliases.add(key);
+                callback(key);
+            }
+        }
+        for (const map of this.syncInFlight.values()) {
+            for (const key of map.keys()) {
+                if (typeof key === "bigint" &&
+                    this.coordinateToHash.get(key) === hash) {
+                    callback(key);
+                }
             }
         }
-        return [...aliases];
     }
     clearSyncInFlightForPeer(publicKeyHash, hash) {
         const map = this.syncInFlight.get(publicKeyHash);
         if (!map) {
             return;
         }
-        for (const key of this.getKnownAliases(hash)) {
+        this.forEachKnownAlias(hash, (key) => map.delete(key));
+        if (map.size === 0) {
+            this.syncInFlight.delete(publicKeyHash);
+        }
+    }
+    clearSyncInFlightForPeerHashes(publicKeyHash, hashes) {
+        const map = this.syncInFlight.get(publicKeyHash);
+        if (!map || hashes.length === 0) {
+            return;
+        }
+        const keys = new Set(hashes);
+        const hashSet = new Set(hashes);
+        for (const key of map.keys()) {
+            if (typeof key !== "bigint") {
+                continue;
+            }
+            const hash = this.coordinateToHash.get(key);
+            if (hash != null && hashSet.has(hash)) {
+                keys.add(key);
+            }
+        }
+        for (const key of keys) {
             map.delete(key);
         }
         if (map.size === 0) {
@@ -895,7 +1301,32 @@ export class SimpleSyncronizer {
         }
     }
     clearSyncProcess(hash) {
-        for (const key of this.getKnownAliases(hash)) {
+        this.forEachKnownAlias(hash, (key) => this.clearSyncProcessKey(key));
+    }
+    clearSyncProcesses(hashes) {
+        if (hashes.length === 0) {
+            return;
+        }
+        const keys = new Set(hashes);
+        const hashSet = new Set(hashes);
+        const maybeAddAlias = (key) => {
+            if (typeof key !== "bigint") {
+                return;
+            }
+            const hash = this.coordinateToHash.get(key);
+            if (hash != null && hashSet.has(hash)) {
+                keys.add(key);
+            }
+        };
+        for (const key of this.syncInFlightQueue.keys()) {
+            maybeAddAlias(key);
+        }
+        for (const map of this.syncInFlight.values()) {
+            for (const key of map.keys()) {
+                maybeAddAlias(key);
+            }
+        }
+        for (const key of keys) {
             this.clearSyncProcessKey(key);
         }
     }
@@ -905,6 +1336,7 @@ export class SimpleSyncronizer {
     }
     clearSyncProcessPublicKeyHash(publicKeyHash) {
         this.syncInFlight.delete(publicKeyHash);
+        this.recentlySentExchangeHeads.delete(publicKeyHash);
         const map = this.syncInFlightQueueInverted.get(publicKeyHash);
         if (map) {
             for (const hash of map) {