@helia/bitswap 1.0.0 → 1.0.1-52dbcf2

package/src/utils/split-message.ts

@@ -0,0 +1,133 @@
+/* eslint-disable complexity */
+import { CodeError } from '@libp2p/interface'
+import { encodingLength } from 'uint8-varint'
+import { BitswapMessage, Block, BlockPresence, WantlistEntry } from '../pb/message.js'
+
+/**
+ * https://github.com/ipfs/kubo/issues/4473#issuecomment-350390693
+ */
+export const MAX_BLOCK_SIZE = 4193648
+const MAX_ENCODED_BLOCK_SIZE = MAX_BLOCK_SIZE + 16
+
+/**
+ * Split the passed Bitswap message into multiple smaller messages that when
+ * serialized will be under the maximum message size.
+ *
+ * Since blocks are the largest thing to send, we first try to fit as many
+ * blocks as possible into the message, then add (smaller) block presences and
+ * wants until the max size is reached.
+ *
+ * If a block is encountered that is larger than the max message size an error
+ * will be thrown.
+ */
+export async function * splitMessage (message: BitswapMessage, maxSize: number): AsyncGenerator<Uint8Array> {
+  const wantListEntries = message.wantlist?.entries ?? []
+  const blockPresences = message.blockPresences
+  const blocks = message.blocks
+
+  let wantListIndex = 0
+  let blockPresencesIndex = 0
+  let blocksIndex = 0
+  let doneSending = false
+
+  while (true) {
+    const subMessage: Required<BitswapMessage> = {
+      wantlist: {
+        full: message.wantlist?.full ?? false,
+        entries: []
+      },
+      blockPresences: [],
+      blocks: [],
+      pendingBytes: 0
+    }
+
+    let size = BitswapMessage.encode(subMessage).byteLength
+
+    let { added, hasMore, newSize } = addToMessage(blocks, subMessage.blocks, blocksIndex, maxSize, size, calculateEncodedBlockSize)
+
+    blocksIndex += added
+    size = newSize
+    const haveMoreBlocks = hasMore
+
+    ;({ added, hasMore, newSize } = addToMessage(blockPresences, subMessage.blockPresences, blockPresencesIndex, maxSize, size, calculateEncodedBlockPresenceSize))
+
+    blockPresencesIndex += added
+    size = newSize
+    const haveMorePresences = hasMore
+
+    ;({ added, hasMore, newSize } = addToMessage(wantListEntries, subMessage.wantlist.entries, wantListIndex, maxSize, size, calculateEncodedWantlistEntrySize))
+
+    wantListIndex += added
+    size = newSize
+    const haveMoreWantlistEntries = hasMore
+
+    doneSending = !haveMoreBlocks && !haveMorePresences && !haveMoreWantlistEntries
+
+    // if we're sending multiple messages this is no longer the full wantlist
+    if (!doneSending) {
+      subMessage.wantlist.full = false
+    }
+
+    yield BitswapMessage.encode(subMessage)
+
+    if (doneSending) {
+      break
+    }
+  }
+}
+
+interface AddResult {
+  hasMore: boolean
+  added: number
+  newSize: number
+}
+
+function addToMessage <T> (input: T[], output: T[], start: number, maxSize: number, size: number, calculateSize: (arg: T) => number): AddResult {
+  let added = 0
+  let hasMore = false
+
+  // try to send as many blocks as possible
+  for (let i = start; i < input.length; i++) {
+    const item = input[i]
+    const itemSize = calculateSize(item)
+
+    if (itemSize > MAX_ENCODED_BLOCK_SIZE) {
+      throw new CodeError('Cannot send block as after encoding it is over the max message size', 'ERR_BLOCK_TOO_LARGE')
+    }
+
+    const newSize = size + itemSize
+
+    if (newSize > maxSize) {
+      hasMore = true
+      break
+    }
+
+    output.push(item)
+    added++
+    size = newSize
+  }
+
+  return { hasMore, added, newSize: size }
+}
+
+function calculateEncodedBlockSize (block: Block): number {
+  // 3 is the "blocks" field number in message.proto
+  return calculateLength(3, Block.encode(block))
+}
+
+function calculateEncodedBlockPresenceSize (blockPresence: BlockPresence): number {
+  // 4 is the "blockPresences" field number in message.proto
+  return calculateLength(4, BlockPresence.encode(blockPresence))
+}
+
+function calculateEncodedWantlistEntrySize (entry: WantlistEntry): number {
+  // 1 is the "entries" field number in message.proto
+  return calculateLength(1, WantlistEntry.encode(entry))
+}
+
+function calculateLength (fieldNumber: number, data: Uint8Array): number {
+  const fieldNumberLength = encodingLength(fieldNumber)
+  const dataLengthLength = encodingLength(data.byteLength)
+
+  return fieldNumberLength + dataLengthLength + data.byteLength
+}

package/src/want-list.ts

@@ -375,7 +375,7 @@ export class WantList extends TypedEventEmitter<WantListEvents> implements Start
    * Invoked when a message is received from a bitswap peer
    */
   private async receiveMessage (sender: PeerId, message: BitswapMessage): Promise<void> {
-    this.log('received message from %p', sender)
+    this.log('received message %d from %p with %d blocks', sender, message.blocks.length)
     let blocksCancelled = false
 
     // process blocks
@@ -397,7 +397,12 @@ export class WantList extends TypedEventEmitter<WantListEvents> implements Start
         continue
       }
 
-      const hash = await hasher.digest(block.data)
+      let hash: any = hasher.digest(block.data)
+
+      if (hash.then != null) {
+        hash = await hash
+      }
+
       const cid = CID.create(cidVersion === 0 ? 0 : 1, multicodec, hash)
 
       this.log('received block from %p for %c', sender, cid)
@@ -423,7 +428,7 @@ export class WantList extends TypedEventEmitter<WantListEvents> implements Start
       const entry = this.wants.get(cidStr)
 
       if (entry == null) {
-        return
+        continue
       }
 
       // since we received the block, flip the cancel flag to send cancels to

package/dist/typedoc-urls.json

@@ -1,20 +0,0 @@
-{
-  "Bitswap": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.Bitswap.html",
-  ".:Bitswap": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.Bitswap.html",
-  "BitswapComponents": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapComponents.html",
-  ".:BitswapComponents": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapComponents.html",
-  "BitswapOptions": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapOptions.html",
-  ".:BitswapOptions": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapOptions.html",
-  "MultihashHasherLoader": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.MultihashHasherLoader.html",
-  ".:MultihashHasherLoader": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.MultihashHasherLoader.html",
-  "WantListEntry": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.WantListEntry.html",
-  ".:WantListEntry": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.WantListEntry.html",
-  "BitswapNotifyProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapNotifyProgressEvents.html",
-  ".:BitswapNotifyProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapNotifyProgressEvents.html",
-  "BitswapWantBlockProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapWantBlockProgressEvents.html",
-  ".:BitswapWantBlockProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapWantBlockProgressEvents.html",
-  "BitswapWantProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapWantProgressEvents.html",
-  ".:BitswapWantProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapWantProgressEvents.html",
-  "createBitswap": "https://ipfs.github.io/helia/functions/_helia_bitswap.createBitswap.html",
-  ".:createBitswap": "https://ipfs.github.io/helia/functions/_helia_bitswap.createBitswap.html"
-}