@gmod/bbi 9.0.9 → 9.0.12

package/src/block-view.ts

@@ -7,10 +7,11 @@ import {
   decompressAndParseSummaryBlocks,
   unzipBatch,
 } from './unzip.ts'
-import { groupBlocks } from './util.ts'
+import { getDataView, groupBlocks } from './util.ts'
 
 import type { Feature } from './types.ts'
 import type { BigWigFeatureArrays, SummaryFeatureArrays } from './unzip.ts'
+import type { Block } from './util.ts'
 import type { GenericFilehandle } from 'generic-filehandle2'
 
 const decoder = new TextDecoder('utf8')
@@ -22,11 +23,6 @@ interface CoordRequest {
   end: number
 }
 
-interface ReadData {
-  offset: number
-  length: number
-}
-
 interface Options {
   signal?: AbortSignal
   request?: CoordRequest
@@ -71,7 +67,7 @@ function parseSummaryBlock(
         maxScore,
         minScore,
         summary: true,
-        score: sumData / (validCnt || 1),
+        score: validCnt ? sumData / validCnt : 0,
       })
     }
   }
@@ -298,6 +294,85 @@ function parseBigWigBlockAsArrays(
   return { starts, ends, scores }
 }
 
+function parseSummaryBlockAsArrays(
+  b: Uint8Array,
+  startOffset: number,
+  request?: CoordRequest,
+): {
+  starts: Int32Array
+  ends: Int32Array
+  scores: Float32Array
+  minScores: Float32Array
+  maxScores: Float32Array
+} {
+  const dataView = new DataView(b.buffer, b.byteOffset, b.length)
+  const maxItems = Math.floor((b.byteLength - startOffset) / 32)
+  const starts = new Int32Array(maxItems)
+  const ends = new Int32Array(maxItems)
+  const scores = new Float32Array(maxItems)
+  const minScores = new Float32Array(maxItems)
+  const maxScores = new Float32Array(maxItems)
+  let idx = 0
+  let offset = startOffset
+  while (offset < b.byteLength) {
+    const chromId = dataView.getUint32(offset, true)
+    offset += 4
+    const start = dataView.getUint32(offset, true)
+    offset += 4
+    const end = dataView.getUint32(offset, true)
+    offset += 4
+    const validCnt = dataView.getUint32(offset, true)
+    offset += 4
+    const minScore = dataView.getFloat32(offset, true)
+    offset += 4
+    const maxScore = dataView.getFloat32(offset, true)
+    offset += 4
+    const sumData = dataView.getFloat32(offset, true)
+    offset += 8
+    if (
+      !request ||
+      (chromId === request.chrId &&
+        coordFilter(start, end, request.start, request.end))
+    ) {
+      starts[idx] = start
+      ends[idx] = end
+      scores[idx] = validCnt ? sumData / validCnt : 0
+      minScores[idx] = minScore
+      maxScores[idx] = maxScore
+      idx++
+    }
+  }
+  if (idx < maxItems) {
+    return {
+      starts: starts.subarray(0, idx),
+      ends: ends.subarray(0, idx),
+      scores: scores.subarray(0, idx),
+      minScores: minScores.subarray(0, idx),
+      maxScores: maxScores.subarray(0, idx),
+    }
+  }
+  return { starts, ends, scores, minScores, maxScores }
+}
+
+function parseBlock(
+  blockType: string,
+  data: Uint8Array,
+  blockOffset: number,
+  request?: CoordRequest,
+): Feature[] {
+  switch (blockType) {
+    case 'summary':
+      return parseSummaryBlock(data, 0, request)
+    case 'bigwig':
+      return parseBigWigBlock(data, 0, request)
+    case 'bigbed':
+      return parseBigBedBlock(data, 0, blockOffset * (1 << 8), request)
+    default:
+      console.warn(`Don't know what to do with ${blockType}`)
+      return []
+  }
+}
+
 /**
  * View into a subset of the data in a BigWig file.
  *
@@ -310,7 +385,7 @@ export class BlockView {
   // efficiently query genomic intervals by chromosome and position
   private rTreePromise?: Promise<Uint8Array>
 
-  private featureCache = new AbortablePromiseCache<ReadData, Uint8Array>({
+  private rTreeNodeCache = new AbortablePromiseCache<Block, Uint8Array>({
     cache: new QuickLRU({ maxSize: 1000 }),
 
     fill: async ({ length, offset }, signal) =>
@@ -337,20 +412,21 @@ export class BlockView {
     start: number,
     end: number,
     opts?: Options,
-  ): Promise<{ blocks: ReadData[]; chrId: number } | undefined> {
+  ): Promise<{ blocks: Block[]; chrId: number } | undefined> {
     const chrId = this.refsByName[chrName]
     if (chrId === undefined) {
       return undefined
     }
     if (!this.rTreePromise) {
-      this.rTreePromise = this.bbi.read(48, this.rTreeOffset, opts)
+      this.rTreePromise = this.bbi
+        .read(48, this.rTreeOffset, opts)
+        .catch((e: unknown) => {
+          this.rTreePromise = undefined
+          throw e
+        })
     }
     const buffer = await this.rTreePromise
-    const dataView = new DataView(
-      buffer.buffer,
-      buffer.byteOffset,
-      buffer.length,
-    )
+    const dataView = getDataView(buffer)
     const magic = dataView.getUint32(0, true)
     if (magic !== CIR_TREE_MAGIC) {
       throw new Error(
@@ -370,7 +446,7 @@ export class BlockView {
       (startChrom < chrId || (startChrom === chrId && startBase <= end)) &&
       (endChrom > chrId || (endChrom === chrId && endBase >= start))
 
-    const blocks: ReadData[] = []
+    const blocks: Block[] = []
     let currentOffsets = [this.rTreeOffset + 48]
 
     while (currentOffsets.length > 0) {
@@ -381,7 +457,7 @@ export class BlockView {
       for (const { min, max } of spans) {
         const length = max - min
         const offset = min
-        const resultBuffer = await this.featureCache.get(
+        const resultBuffer = await this.rTreeNodeCache.get(
           `${length}_${offset}`,
           { length, offset },
           opts?.signal,
@@ -389,42 +465,36 @@ export class BlockView {
         for (const element of currentOffsets) {
           if (min <= element && element <= max) {
             const data = resultBuffer.subarray(element - offset)
-            const dv = new DataView(data.buffer, data.byteOffset, data.length)
+            const dv = getDataView(data)
             const isLeaf = dv.getUint8(0)
             const count = dv.getUint16(2, true)
-            let nodeOffset = 4
-            if (isLeaf === 1) {
+            if (isLeaf === 1 || isLeaf === 0) {
+              const entrySize = isLeaf === 1 ? 32 : 24
+              let nodeOffset = 4
               for (let i = 0; i < count; i++) {
                 const startChrom = dv.getUint32(nodeOffset, true)
                 const startBase = dv.getUint32(nodeOffset + 4, true)
                 const endChrom = dv.getUint32(nodeOffset + 8, true)
                 const endBase = dv.getUint32(nodeOffset + 12, true)
-                const blockOffset = Number(
-                  dv.getBigUint64(nodeOffset + 16, true),
-                )
-                const blockSize = Number(dv.getBigUint64(nodeOffset + 24, true))
-                nodeOffset += 32
                 if (
                   blockIntersectsQuery(startChrom, startBase, endChrom, endBase)
                 ) {
-                  blocks.push({ offset: blockOffset, length: blockSize })
-                }
-              }
-            } else if (isLeaf === 0) {
-              for (let i = 0; i < count; i++) {
-                const startChrom = dv.getUint32(nodeOffset, true)
-                const startBase = dv.getUint32(nodeOffset + 4, true)
-                const endChrom = dv.getUint32(nodeOffset + 8, true)
-                const endBase = dv.getUint32(nodeOffset + 12, true)
-                const childOffset = Number(
-                  dv.getBigUint64(nodeOffset + 16, true),
-                )
-                nodeOffset += 24
-                if (
-                  blockIntersectsQuery(startChrom, startBase, endChrom, endBase)
-                ) {
-                  nextOffsets.push(childOffset)
+                  const childOrBlockOffset = Number(
+                    dv.getBigUint64(nodeOffset + 16, true),
+                  )
+                  if (isLeaf === 1) {
+                    const blockSize = Number(
+                      dv.getBigUint64(nodeOffset + 24, true),
+                    )
+                    blocks.push({
+                      offset: childOrBlockOffset,
+                      length: blockSize,
+                    })
+                  } else {
+                    nextOffsets.push(childOrBlockOffset)
+                  }
                 }
+                nodeOffset += entrySize
               }
             }
           }
@@ -460,15 +530,34 @@ export class BlockView {
     opts?: Options,
   ): Promise<BigWigFeatureArrays | SummaryFeatureArrays> {
     const collected = await this._collectBlocks(chrName, start, end, opts)
-    const blocks = collected?.blocks ?? []
-    const request = collected
-      ? { chrId: collected.chrId, start, end }
-      : undefined
-    const optsWithReq = { ...opts, request }
     if (this.blockType === 'summary') {
-      return this._readSummaryFeaturesAsArrays(blocks, optsWithReq)
+      return collected
+        ? this._readSummaryFeaturesAsArrays(
+            collected.blocks,
+            { chrId: collected.chrId, start, end },
+            opts,
+          )
+        : {
+            starts: new Int32Array(0),
+            ends: new Int32Array(0),
+            scores: new Float32Array(0),
+            minScores: new Float32Array(0),
+            maxScores: new Float32Array(0),
+            isSummary: true as const,
+          }
     }
-    return this._readBigWigFeaturesAsArrays(blocks, optsWithReq)
+    return collected
+      ? this._readBigWigFeaturesAsArrays(
+          collected.blocks,
+          { chrId: collected.chrId, start, end },
+          opts,
+        )
+      : {
+          starts: new Int32Array(0),
+          ends: new Int32Array(0),
+          scores: new Float32Array(0),
+          isSummary: false as const,
+        }
   }
 
   public async readFeatures(
@@ -486,55 +575,44 @@ export class BlockView {
       const groupOffset = blockGroup.offset
       const subBlocks = blockGroup.blocks
 
-      let decompressedData: Uint8Array
-      let decompressedOffsets: number[]
-
       if (uncompressBufSize > 0) {
-        const localBlocks: { offset: number; length: number }[] = []
-        for (const block of subBlocks) {
-          localBlocks.push({
-            offset: block.offset - groupOffset,
-            length: block.length,
-          })
+        const localBlocks = subBlocks.map(block => ({
+          offset: block.offset - groupOffset,
+          length: block.length,
+        }))
+        const { data: decompressedData, offsets } = await unzipBatch(
+          data,
+          localBlocks,
+          uncompressBufSize,
+        )
+        for (let i = 0; i < subBlocks.length; i++) {
+          const resultData = decompressedData.subarray(
+            offsets[i],
+            offsets[i + 1],
+          )
+          const features = parseBlock(
+            blockType,
+            resultData,
+            subBlocks[i]!.offset,
+            request,
+          )
+          for (const f of features) {
+            allFeatures.push(f)
+          }
         }
-        const result = await unzipBatch(data, localBlocks, uncompressBufSize)
-        decompressedData = result.data
-        decompressedOffsets = result.offsets
       } else {
-        decompressedData = data
-        decompressedOffsets = []
         for (const block of subBlocks) {
-          decompressedOffsets.push(block.offset - groupOffset)
-        }
-        decompressedOffsets.push(data.length)
-      }
-
-      for (let i = 0; i < subBlocks.length; i++) {
-        const start = decompressedOffsets[i]!
-        const end = decompressedOffsets[i + 1]!
-        const resultData = decompressedData.subarray(start, end)
-        let features: Feature[]
-        switch (blockType) {
-          case 'summary':
-            features = parseSummaryBlock(resultData, 0, request)
-            break
-          case 'bigwig':
-            features = parseBigWigBlock(resultData, 0, request)
-            break
-          case 'bigbed':
-            features = parseBigBedBlock(
-              resultData,
-              0,
-              subBlocks[i]!.offset * (1 << 8),
-              request,
-            )
-            break
-          default:
-            features = []
-            console.warn(`Don't know what to do with ${blockType}`)
-        }
-        for (const f of features) {
-          allFeatures.push(f)
+          const start = block.offset - groupOffset
+          const resultData = data.subarray(start, start + block.length)
+          const features = parseBlock(
+            blockType,
+            resultData,
+            block.offset,
+            request,
+          )
+          for (const f of features) {
+            allFeatures.push(f)
+          }
         }
       }
     }
@@ -543,10 +621,11 @@ export class BlockView {
 
   private async _readBigWigFeaturesAsArrays(
     blocks: { offset: number; length: number }[],
+    request: CoordRequest,
     opts: Options = {},
   ): Promise<BigWigFeatureArrays> {
     const { uncompressBufSize } = this
-    const { signal, request } = opts
+    const { signal } = opts
     const blockGroupsToFetch = groupBlocks(blocks)
 
     const allStarts: Int32Array[] = []
@@ -568,8 +647,8 @@ export class BlockView {
           data,
           localBlocks,
           uncompressBufSize,
-          request?.start ?? 0,
-          request?.end ?? 0,
+          request.start,
+          request.end,
         )
         if (result.starts.length > 0) {
           allStarts.push(result.starts)
@@ -628,10 +707,11 @@ export class BlockView {
 
   private async _readSummaryFeaturesAsArrays(
     blocks: { offset: number; length: number }[],
+    request: CoordRequest,
     opts: Options = {},
   ): Promise<SummaryFeatureArrays> {
     const { uncompressBufSize } = this
-    const { signal, request } = opts
+    const { signal } = opts
     const blockGroupsToFetch = groupBlocks(blocks)
 
     const allStarts: Int32Array[] = []
@@ -655,9 +735,9 @@ export class BlockView {
           data,
           localBlocks,
           uncompressBufSize,
-          request?.chrId ?? 0,
-          request?.start ?? 0,
-          request?.end ?? 0,
+          request.chrId,
+          request.start,
+          request.end,
         )
         if (result.starts.length > 0) {
           allStarts.push(result.starts)
@@ -673,27 +753,14 @@ export class BlockView {
             block.offset,
             block.offset + block.length,
           )
-          const features = parseSummaryBlock(blockData, 0, request)
-          if (features.length > 0) {
-            const starts = new Int32Array(features.length)
-            const ends = new Int32Array(features.length)
-            const scores = new Float32Array(features.length)
-            const minScores = new Float32Array(features.length)
-            const maxScores = new Float32Array(features.length)
-            for (let i = 0; i < features.length; i++) {
-              const f = features[i]!
-              starts[i] = f.start
-              ends[i] = f.end
-              scores[i] = f.score ?? 0
-              minScores[i] = f.minScore ?? 0
-              maxScores[i] = f.maxScore ?? 0
-            }
-            allStarts.push(starts)
-            allEnds.push(ends)
-            allScores.push(scores)
-            allMinScores.push(minScores)
-            allMaxScores.push(maxScores)
-            totalCount += features.length
+          const result = parseSummaryBlockAsArrays(blockData, 0, request)
+          if (result.starts.length > 0) {
+            allStarts.push(result.starts)
+            allEnds.push(result.ends)
+            allScores.push(result.scores)
+            allMinScores.push(result.minScores)
+            allMaxScores.push(result.maxScores)
+            totalCount += result.starts.length
           }
         }
       }

package/src/range.ts

@@ -6,9 +6,10 @@ export interface IRange {
 // Merges overlapping or adjacent byte ranges so that nearby R-tree nodes can
 // be fetched in a single read rather than many small reads
 export function mergeRanges(ranges: IRange[]) {
-  const sorted = [...ranges].sort((a, b) =>
-    a.min !== b.min ? a.min - b.min : a.max - b.max,
-  )
+  if (ranges.length === 0) {
+    return []
+  }
+  const sorted = ranges.toSorted((a, b) => a.min - b.min)
 
   const merged: IRange[] = []
   let current = sorted[0]!

package/src/types.ts

@@ -1,6 +1,5 @@
 export interface ZoomLevel {
   reductionLevel: number
-  reserved: number
   dataOffset: number
   indexOffset: number
 }

package/src/unzip.ts

@@ -9,20 +9,23 @@ export interface UnzipBatchResult {
   offsets: number[]
 }
 
-export async function unzipBatch(
-  data: Uint8Array,
-  blocks: { offset: number; length: number }[],
-  maxOutputSize: number,
-): Promise<UnzipBatchResult> {
+function blocksToTypedArrays(blocks: { offset: number; length: number }[]) {
   const inputOffsets = new Uint32Array(blocks.length)
   const inputLengths = new Uint32Array(blocks.length)
-
   for (let i = 0; i < blocks.length; i++) {
     const block = blocks[i]!
     inputOffsets[i] = block.offset
     inputLengths[i] = block.length
   }
+  return { inputOffsets, inputLengths }
+}
 
+export async function unzipBatch(
+  data: Uint8Array,
+  blocks: { offset: number; length: number }[],
+  maxOutputSize: number,
+): Promise<UnzipBatchResult> {
+  const { inputOffsets, inputLengths } = blocksToTypedArrays(blocks)
   return inflateRawBatch(data, inputOffsets, inputLengths, maxOutputSize)
 }
 
@@ -33,15 +36,7 @@ export async function decompressAndParseBigWigBlocks(
   reqStart: number,
   reqEnd: number,
 ) {
-  const inputOffsets = new Uint32Array(blocks.length)
-  const inputLengths = new Uint32Array(blocks.length)
-
-  for (let i = 0; i < blocks.length; i++) {
-    const block = blocks[i]!
-    inputOffsets[i] = block.offset
-    inputLengths[i] = block.length
-  }
-
+  const { inputOffsets, inputLengths } = blocksToTypedArrays(blocks)
   return decompressAndParseBigWig(
     data,
     inputOffsets,
@@ -60,15 +55,7 @@ export async function decompressAndParseSummaryBlocks(
   reqStart: number,
   reqEnd: number,
 ) {
-  const inputOffsets = new Uint32Array(blocks.length)
-  const inputLengths = new Uint32Array(blocks.length)
-
-  for (let i = 0; i < blocks.length; i++) {
-    const block = blocks[i]!
-    inputOffsets[i] = block.offset
-    inputLengths[i] = block.length
-  }
-
+  const { inputOffsets, inputLengths } = blocksToTypedArrays(blocks)
   return decompressAndParseSummary(
     data,
     inputOffsets,

package/src/util.ts

@@ -1,7 +1,26 @@
-interface Block {
+const decoder = new TextDecoder('utf8')
+
+export interface Block {
   offset: number
   length: number
 }
+
+export function getDataView(buffer: Uint8Array, byteOffset = 0) {
+  return new DataView(
+    buffer.buffer,
+    buffer.byteOffset + byteOffset,
+    buffer.length - byteOffset,
+  )
+}
+
+// Decode a null-terminated fixed-width key from a B+ tree node
+export function parseKey(buffer: Uint8Array, offset: number, keySize: number) {
+  const nullPos = buffer.indexOf(0, offset)
+  const end =
+    nullPos !== -1 && nullPos < offset + keySize ? nullPos : offset + keySize
+  return decoder.decode(buffer.subarray(offset, end))
+}
+
 // sort blocks by file offset and
 // group blocks that are within 2KB of eachother
 export function groupBlocks(blocks: Block[]) {
@@ -11,7 +30,7 @@ export function groupBlocks(blocks: Block[]) {
   let lastBlock: (Block & { blocks: Block[] }) | undefined
   let lastBlockEnd: number | undefined
   for (const block of blocks) {
-    if (lastBlock && lastBlockEnd && block.offset - lastBlockEnd <= 2000) {
+    if (lastBlock && block.offset - lastBlockEnd! <= 2000) {
       lastBlock.length = block.offset + block.length - lastBlock.offset
       lastBlock.blocks.push(block)
       lastBlockEnd = block.offset + block.length

package/src/wasm/inflate-wasm-inlined.d.ts

@@ -0,0 +1,21 @@
+export interface BatchResult {
+    data: Uint8Array;
+    offsets: number[];
+}
+export interface WasmBigWigArrays {
+    starts: Int32Array;
+    ends: Int32Array;
+    scores: Float32Array;
+}
+export interface WasmSummaryArrays {
+    starts: Int32Array;
+    ends: Int32Array;
+    scores: Float32Array;
+    minScores: Float32Array;
+    maxScores: Float32Array;
+}
+export declare function inflateRaw(input: Uint8Array, outputSize: number): Promise<Uint8Array>;
+export declare function inflateRawUnknownSize(input: Uint8Array): Promise<Uint8Array>;
+export declare function inflateRawBatch(inputs: Uint8Array, inputOffsets: Uint32Array, inputLengths: Uint32Array, maxOutputSize: number): Promise<BatchResult>;
+export declare function decompressAndParseBigWig(inputs: Uint8Array, inputOffsets: Uint32Array, inputLengths: Uint32Array, maxBlockSize: number, reqStart: number, reqEnd: number): Promise<WasmBigWigArrays>;
+export declare function decompressAndParseSummary(inputs: Uint8Array, inputOffsets: Uint32Array, inputLengths: Uint32Array, maxBlockSize: number, reqChrId: number, reqStart: number, reqEnd: number): Promise<WasmSummaryArrays>;