@helia/bitswap 1.0.0 → 1.0.1-0a528bb

package/src/utils/merge-messages.ts

@@ -0,0 +1,70 @@
+import { base64 } from 'multiformats/bases/base64'
+import type { BitswapMessage, Block, BlockPresence, WantlistEntry } from '../pb/message.js'
+
+export function mergeMessages (messageA: BitswapMessage, messageB: BitswapMessage): BitswapMessage {
+  const wantListEntries = new Map<string, WantlistEntry>(
+    (messageA.wantlist?.entries ?? []).map(entry => ([
+      base64.encode(entry.cid),
+      entry
+    ]))
+  )
+
+  for (const entry of messageB.wantlist?.entries ?? []) {
+    const key = base64.encode(entry.cid)
+    const existingEntry = wantListEntries.get(key)
+
+    if (existingEntry != null) {
+      // take highest priority
+      if (existingEntry.priority > entry.priority) {
+        entry.priority = existingEntry.priority
+      }
+
+      // take later values if passed, otherwise use earlier ones
+      entry.cancel = entry.cancel ?? existingEntry.cancel
+      entry.wantType = entry.wantType ?? existingEntry.wantType
+      entry.sendDontHave = entry.sendDontHave ?? existingEntry.sendDontHave
+    }
+
+    wantListEntries.set(key, entry)
+  }
+
+  const blockPresences = new Map<string, BlockPresence>(
+    messageA.blockPresences.map(presence => ([
+      base64.encode(presence.cid),
+      presence
+    ]))
+  )
+
+  for (const blockPresence of messageB.blockPresences) {
+    const key = base64.encode(blockPresence.cid)
+
+    // override earlier block presence with later one as if duplicated it is
+    // likely to be more accurate since it is more recent
+    blockPresences.set(key, blockPresence)
+  }
+
+  const blocks = new Map<string, Block>(
+    messageA.blocks.map(block => ([
+      base64.encode(block.data),
+      block
+    ]))
+  )
+
+  for (const block of messageB.blocks) {
+    const key = base64.encode(block.data)
+
+    blocks.set(key, block)
+  }
+
+  const output: BitswapMessage = {
+    wantlist: {
+      full: messageA.wantlist?.full ?? messageB.wantlist?.full ?? false,
+      entries: [...wantListEntries.values()]
+    },
+    blockPresences: [...blockPresences.values()],
+    blocks: [...blocks.values()],
+    pendingBytes: messageA.pendingBytes + messageB.pendingBytes
+  }
+
+  return output
+}

package/src/utils/split-message.ts

@@ -0,0 +1,133 @@
+/* eslint-disable complexity */
+import { CodeError } from '@libp2p/interface'
+import { encodingLength } from 'uint8-varint'
+import { BitswapMessage, Block, BlockPresence, WantlistEntry } from '../pb/message.js'
+
+/**
+ * https://github.com/ipfs/kubo/issues/4473#issuecomment-350390693
+ */
+export const MAX_BLOCK_SIZE = 4193648
+const MAX_ENCODED_BLOCK_SIZE = MAX_BLOCK_SIZE + 16
+
+/**
+ * Split the passed Bitswap message into multiple smaller messages that when
+ * serialized will be under the maximum message size.
+ *
+ * Since blocks are the largest thing to send, we first try to fit as many
+ * blocks as possible into the message, then add (smaller) block presences and
+ * wants until the max size is reached.
+ *
+ * If a block is encountered that is larger than the max message size an error
+ * will be thrown.
+ */
+export async function * splitMessage (message: BitswapMessage, maxSize: number): AsyncGenerator<Uint8Array> {
+  const wantListEntries = message.wantlist?.entries ?? []
+  const blockPresences = message.blockPresences
+  const blocks = message.blocks
+
+  let wantListIndex = 0
+  let blockPresencesIndex = 0
+  let blocksIndex = 0
+  let doneSending = false
+
+  while (true) {
+    const subMessage: Required<BitswapMessage> = {
+      wantlist: {
+        full: message.wantlist?.full ?? false,
+        entries: []
+      },
+      blockPresences: [],
+      blocks: [],
+      pendingBytes: 0
+    }
+
+    let size = BitswapMessage.encode(subMessage).byteLength
+
+    let { added, hasMore, newSize } = addToMessage(blocks, subMessage.blocks, blocksIndex, maxSize, size, calculateEncodedBlockSize)
+
+    blocksIndex += added
+    size = newSize
+    const haveMoreBlocks = hasMore
+
+    ;({ added, hasMore, newSize } = addToMessage(blockPresences, subMessage.blockPresences, blockPresencesIndex, maxSize, size, calculateEncodedBlockPresenceSize))
+
+    blockPresencesIndex += added
+    size = newSize
+    const haveMorePresences = hasMore
+
+    ;({ added, hasMore, newSize } = addToMessage(wantListEntries, subMessage.wantlist.entries, wantListIndex, maxSize, size, calculateEncodedWantlistEntrySize))
+
+    wantListIndex += added
+    size = newSize
+    const haveMoreWantlistEntries = hasMore
+
+    doneSending = !haveMoreBlocks && !haveMorePresences && !haveMoreWantlistEntries
+
+    // if we're sending multiple messages this is no longer the full wantlist
+    if (!doneSending) {
+      subMessage.wantlist.full = false
+    }
+
+    yield BitswapMessage.encode(subMessage)
+
+    if (doneSending) {
+      break
+    }
+  }
+}
+
+interface AddResult {
+  hasMore: boolean
+  added: number
+  newSize: number
+}
+
+function addToMessage <T> (input: T[], output: T[], start: number, maxSize: number, size: number, calculateSize: (arg: T) => number): AddResult {
+  let added = 0
+  let hasMore = false
+
+  // try to send as many blocks as possible
+  for (let i = start; i < input.length; i++) {
+    const item = input[i]
+    const itemSize = calculateSize(item)
+
+    if (itemSize > MAX_ENCODED_BLOCK_SIZE) {
+      throw new CodeError('Cannot send block as after encoding it is over the max message size', 'ERR_BLOCK_TOO_LARGE')
+    }
+
+    const newSize = size + itemSize
+
+    if (newSize > maxSize) {
+      hasMore = true
+      break
+    }
+
+    output.push(item)
+    added++
+    size = newSize
+  }
+
+  return { hasMore, added, newSize: size }
+}
+
+function calculateEncodedBlockSize (block: Block): number {
+  // 3 is the "blocks" field number in message.proto
+  return calculateLength(3, Block.encode(block))
+}
+
+function calculateEncodedBlockPresenceSize (blockPresence: BlockPresence): number {
+  // 4 is the "blockPresences" field number in message.proto
+  return calculateLength(4, BlockPresence.encode(blockPresence))
+}
+
+function calculateEncodedWantlistEntrySize (entry: WantlistEntry): number {
+  // 1 is the "entries" field number in message.proto
+  return calculateLength(1, WantlistEntry.encode(entry))
+}
+
+function calculateLength (fieldNumber: number, data: Uint8Array): number {
+  const fieldNumberLength = encodingLength(fieldNumber)
+  const dataLengthLength = encodingLength(data.byteLength)
+
+  return fieldNumberLength + dataLengthLength + data.byteLength
+}

package/src/want-list.ts

@@ -17,7 +17,7 @@ import vd from './utils/varint-decoder.js'
 import type { BitswapNotifyProgressEvents, MultihashHasherLoader } from './index.js'
 import type { BitswapNetworkWantProgressEvents, Network } from './network.js'
 import type { BitswapMessage } from './pb/message.js'
-import type { ComponentLogger, PeerId, Startable, AbortOptions, Libp2p, TypedEventTarget } from '@libp2p/interface'
+import type { ComponentLogger, PeerId, Startable, AbortOptions, Libp2p, TypedEventTarget, Metrics } from '@libp2p/interface'
 import type { Logger } from '@libp2p/logger'
 import type { PeerMap } from '@libp2p/peer-collections'
 import type { DeferredPromise } from 'p-defer'
@@ -27,6 +27,7 @@ export interface WantListComponents {
   network: Network
   logger: ComponentLogger
   libp2p: Libp2p
+  metrics?: Metrics
 }
 
 export interface WantListInit {
@@ -114,11 +115,11 @@ export class WantList extends TypedEventEmitter<WantListEvents> implements Start
     setMaxListeners(Infinity, this)
     this.peers = trackedPeerMap({
       name: 'helia_bitswap_peers',
-      metrics: components.libp2p.metrics
+      metrics: components.metrics
     })
     this.wants = trackedMap({
       name: 'helia_bitswap_wantlist',
-      metrics: components.libp2p.metrics
+      metrics: components.metrics
     })
     this.network = components.network
     this.sendMessagesDelay = init.sendMessagesDelay ?? DEFAULT_MESSAGE_SEND_DELAY
@@ -375,7 +376,7 @@ export class WantList extends TypedEventEmitter<WantListEvents> implements Start
    * Invoked when a message is received from a bitswap peer
    */
   private async receiveMessage (sender: PeerId, message: BitswapMessage): Promise<void> {
-    this.log('received message from %p', sender)
+    this.log('received message %d from %p with %d blocks', sender, message.blocks.length)
     let blocksCancelled = false
 
     // process blocks
@@ -397,7 +398,12 @@ export class WantList extends TypedEventEmitter<WantListEvents> implements Start
         continue
       }
 
-      const hash = await hasher.digest(block.data)
+      let hash: any = hasher.digest(block.data)
+
+      if (hash.then != null) {
+        hash = await hash
+      }
+
       const cid = CID.create(cidVersion === 0 ? 0 : 1, multicodec, hash)
 
       this.log('received block from %p for %c', sender, cid)
@@ -423,7 +429,7 @@ export class WantList extends TypedEventEmitter<WantListEvents> implements Start
       const entry = this.wants.get(cidStr)
 
       if (entry == null) {
-        return
+        continue
       }
 
       // since we received the block, flip the cancel flag to send cancels to

package/dist/typedoc-urls.json

@@ -1,20 +0,0 @@
-{
-  "Bitswap": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.Bitswap.html",
-  ".:Bitswap": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.Bitswap.html",
-  "BitswapComponents": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapComponents.html",
-  ".:BitswapComponents": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapComponents.html",
-  "BitswapOptions": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapOptions.html",
-  ".:BitswapOptions": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapOptions.html",
-  "MultihashHasherLoader": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.MultihashHasherLoader.html",
-  ".:MultihashHasherLoader": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.MultihashHasherLoader.html",
-  "WantListEntry": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.WantListEntry.html",
-  ".:WantListEntry": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.WantListEntry.html",
-  "BitswapNotifyProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapNotifyProgressEvents.html",
-  ".:BitswapNotifyProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapNotifyProgressEvents.html",
-  "BitswapWantBlockProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapWantBlockProgressEvents.html",
-  ".:BitswapWantBlockProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapWantBlockProgressEvents.html",
-  "BitswapWantProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapWantProgressEvents.html",
-  ".:BitswapWantProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapWantProgressEvents.html",
-  "createBitswap": "https://ipfs.github.io/helia/functions/_helia_bitswap.createBitswap.html",
-  ".:createBitswap": "https://ipfs.github.io/helia/functions/_helia_bitswap.createBitswap.html"
-}