@gmod/bbi 7.0.5 → 8.0.0

package/src/bbi.ts

@@ -5,6 +5,7 @@ import { toArray } from 'rxjs/operators'
 import { BlockView } from './block-view.ts'
 
 import type {
+  BigWigFeatureArrays,
   BigWigHeader,
   BigWigHeaderWithRefNames,
   Feature,
@@ -12,6 +13,7 @@ import type {
   RequestOptions2,
   RequestOptions,
   Statistics,
+  SummaryFeatureArrays,
   ZoomLevel,
 } from './types.ts'
 import type { GenericFilehandle } from 'generic-filehandle2'
@@ -19,6 +21,8 @@ import type { GenericFilehandle } from 'generic-filehandle2'
 const BIG_WIG_MAGIC = -2003829722
 const BIG_BED_MAGIC = -2021002517
 
+const decoder = new TextDecoder('utf8')
+
 function getDataView(buffer: Uint8Array) {
   return new DataView(buffer.buffer, buffer.byteOffset, buffer.length)
 }
@@ -71,7 +75,7 @@ export abstract class BBI {
 
   private async _getHeader(opts?: RequestOptions) {
     const header = await this._getMainHeader(opts)
-    const chroms = await this._readChromTree(header, opts)
+    const chroms = await this._readChromosomeTree(header, opts)
     return {
       ...header,
       ...chroms,
@@ -96,7 +100,8 @@ export abstract class BBI {
     offset += 2
     const numZoomLevels = dataView.getUint16(offset, true)
     offset += 2
-    const chromTreeOffset = Number(dataView.getBigUint64(offset, true))
+    // Offset to the B+ tree that maps chromosome names to integer IDs
+    const chromosomeTreeOffset = Number(dataView.getBigUint64(offset, true))
     offset += 8
     const unzoomedDataOffset = Number(dataView.getBigUint64(offset, true))
     offset += 8
@@ -142,7 +147,7 @@ export abstract class BBI {
 
     let totalSummary: Statistics
     if (totalSummaryOffset) {
-      const b2 = b.subarray(Number(totalSummaryOffset))
+      const b2 = b.subarray(totalSummaryOffset)
       let offset = 0
       const dataView = getDataView(b2)
       const basesCovered = Number(dataView.getBigUint64(offset, true))
@@ -166,7 +171,6 @@ export abstract class BBI {
     } else {
       throw new Error('no stats')
     }
-    const decoder = new TextDecoder('utf8')
 
     return {
       zoomLevels,
@@ -178,7 +182,7 @@ export abstract class BBI {
       definedFieldCount,
       uncompressBufSize,
       asOffset,
-      chromTreeOffset,
+      chromosomeTreeOffset,
       totalSummaryOffset,
       unzoomedDataOffset,
       unzoomedIndexOffset,
@@ -190,16 +194,21 @@ export abstract class BBI {
     }
   }
 
-  private async _readChromTree(
+  // Reads the B+ tree that maps chromosome names to integer IDs
+  // This is part of the "cirTree" (combined ID R-tree) structure, which uses
+  // integer chromosome IDs instead of strings for more efficient spatial indexing
+  private async _readChromosomeTree(
     header: BigWigHeader,
     opts?: { signal?: AbortSignal },
   ) {
-    const refsByNumber: Record<number, RefInfo> = []
-    const refsByName: Record<string, number> = {}
+    const refsByNumber: RefInfo[] = []
+    const refsByName = {} as Record<string, number>
 
-    const chromTreeOffset = Number(header.chromTreeOffset)
+    const chromosomeTreeOffset = header.chromosomeTreeOffset
 
-    const dataView = getDataView(await this.bbi.read(32, chromTreeOffset, opts))
+    const dataView = getDataView(
+      await this.bbi.read(32, chromosomeTreeOffset, opts),
+    )
     let offset = 0
     // const magic = dataView.getUint32(offset, true) // unused
     offset += 4
@@ -212,9 +221,8 @@ export abstract class BBI {
     // const itemCount = dataView.getBigUint64(offset, true) // unused
     offset += 8
 
-    const decoder = new TextDecoder('utf8')
-
-    const bptReadNode = async (currentOffset: number) => {
+    // Recursively traverses the B+ tree to populate chromosome name-to-ID mappings
+    const readBPlusTreeNode = async (currentOffset: number) => {
       const b = await this.bbi.read(4, currentOffset)
       const dataView = getDataView(b)
       let offset = 0
@@ -225,6 +233,7 @@ export abstract class BBI {
       const count = dataView.getUint16(offset, true)
       offset += 2
 
+      // Leaf nodes contain the actual chromosome name-to-ID mappings
       if (isLeafNode) {
         const b = await this.bbi.read(
           count * (keySize + valSize),
@@ -234,9 +243,10 @@ export abstract class BBI {
         offset = 0
 
         for (let n = 0; n < count; n++) {
-          const key = decoder
-            .decode(b.subarray(offset, offset + keySize))
-            .replaceAll('\0', '')
+          const keyEnd = b.indexOf(0, offset)
+          const key = decoder.decode(
+            b.subarray(offset, keyEnd !== -1 ? keyEnd : offset + keySize),
+          )
           offset += keySize
           const refId = dataView.getUint32(offset, true)
           offset += 4
@@ -251,6 +261,7 @@ export abstract class BBI {
           }
         }
       } else {
+        // Non-leaf nodes contain pointers to child nodes
         const nextNodes = []
         const dataView = getDataView(
           await this.bbi.read(count * (keySize + 8), currentOffset + offset),
@@ -261,12 +272,12 @@ export abstract class BBI {
           offset += keySize
           const childOffset = Number(dataView.getBigUint64(offset, true))
           offset += 8
-          nextNodes.push(bptReadNode(childOffset))
+          nextNodes.push(readBPlusTreeNode(childOffset))
         }
         await Promise.all(nextNodes)
       }
     }
-    await bptReadNode(chromTreeOffset + 32)
+    await readBPlusTreeNode(chromosomeTreeOffset + 32)
     return {
       refsByName,
       refsByNumber,
@@ -284,7 +295,7 @@ export abstract class BBI {
       this.bbi,
       refsByName,
       unzoomedIndexOffset,
-      uncompressBufSize > 0,
+      uncompressBufSize,
       fileType,
     )
   }
@@ -345,7 +356,76 @@ export abstract class BBI {
   ) {
     const ob = await this.getFeatureStream(refName, start, end, opts)
 
-    const ret = await firstValueFrom(ob.pipe(toArray()))
-    return ret.flat()
+    const arrays = await firstValueFrom(ob.pipe(toArray()))
+    const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0)
+    const result = new Array(totalLength)
+    let index = 0
+    for (const arr of arrays) {
+      for (const item of arr) {
+        result[index++] = item
+      }
+    }
+    return result
+  }
+
+  /**
+   * Gets features from a BigWig file as typed arrays (more efficient than getFeatures)
+   *
+   * @param refName - The chromosome name
+   * @param start - The start of a region
+   * @param end - The end of a region
+   * @param opts - Options including basesPerSpan or scale
+   * @returns Promise with typed arrays: starts, ends, scores (and minScores/maxScores for summary data)
+   */
+  public async getFeaturesAsArrays(
+    refName: string,
+    start: number,
+    end: number,
+    opts?: RequestOptions2,
+  ): Promise<BigWigFeatureArrays | SummaryFeatureArrays> {
+    const features = await this.getFeatures(refName, start, end, opts)
+    const count = features.length
+
+    if (count === 0) {
+      return {
+        starts: new Int32Array(0),
+        ends: new Int32Array(0),
+        scores: new Float32Array(0),
+      }
+    }
+
+    const hasSummary = features[0]?.summary === true
+
+    if (hasSummary) {
+      const starts = new Int32Array(count)
+      const ends = new Int32Array(count)
+      const scores = new Float32Array(count)
+      const minScores = new Float32Array(count)
+      const maxScores = new Float32Array(count)
+
+      for (let i = 0; i < count; i++) {
+        const f = features[i]!
+        starts[i] = f.start
+        ends[i] = f.end
+        scores[i] = f.score ?? 0
+        minScores[i] = f.minScore ?? 0
+        maxScores[i] = f.maxScore ?? 0
+      }
+
+      return { starts, ends, scores, minScores, maxScores }
+    }
+
+    const starts = new Int32Array(count)
+    const ends = new Int32Array(count)
+    const scores = new Float32Array(count)
+
+    for (let i = 0; i < count; i++) {
+      const f = features[i]!
+      starts[i] = f.start
+      ends[i] = f.end
+      scores[i] = f.score ?? 0
+    }
+
+    return { starts, ends, scores }
   }
 }

package/src/bigbed.ts

@@ -6,6 +6,9 @@ import { map, reduce } from 'rxjs/operators'
 import { BBI } from './bbi.ts'
 
 import type { Feature, RequestOptions } from './types.ts'
+import type { GenericFilehandle } from 'generic-filehandle2'
+
+const decoder = new TextDecoder('utf8')
 
 interface Loc {
   key: string
@@ -25,6 +28,133 @@ export function filterUndef<T>(ts: (T | undefined)[]): T[] {
   return ts.filter((t: T | undefined): t is T => !!t)
 }
 
+function getTabField(str: string, fieldIndex: number) {
+  if (fieldIndex < 0) {
+    return undefined
+  }
+  let start = 0
+  for (let i = 0; i < fieldIndex; i++) {
+    start = str.indexOf('\t', start)
+    if (start === -1) {
+      return undefined
+    }
+    start++
+  }
+  const end = str.indexOf('\t', start)
+  return end === -1 ? str.slice(start) : str.slice(start, end)
+}
+
+// Parses a null-terminated string key from a B+ tree node
+function parseKey(buffer: Uint8Array, offset: number, keySize: number) {
+  const keyEnd = buffer.indexOf(0, offset)
+  return decoder.decode(
+    buffer.subarray(offset, keyEnd !== -1 ? keyEnd : offset + keySize),
+  )
+}
+
+// Recursively traverses a B+ tree to search for a specific name in the BigBed extraIndex
+// B+ trees are balanced tree structures optimized for disk-based searches
+async function readBPlusTreeNode(
+  bbi: GenericFilehandle,
+  nodeOffset: number,
+  blockSize: number,
+  keySize: number,
+  valSize: number,
+  name: string,
+  field: number,
+  opts: RequestOptions,
+): Promise<Loc | undefined> {
+  const len = 4 + blockSize * (keySize + valSize)
+  const buffer = await bbi.read(len, nodeOffset, opts)
+  const dataView = new DataView(buffer.buffer, buffer.byteOffset, buffer.length)
+  let offset = 0
+  const nodeType = dataView.getInt8(offset)
+  offset += 2 // skip nodeType byte + 1 reserved byte
+  const cnt = dataView.getInt16(offset, true)
+  offset += 2
+
+  // Non-leaf node (nodeType === 0): contains keys and child node pointers for navigation
+  if (nodeType === 0) {
+    const leafkeys = []
+    for (let i = 0; i < cnt; i++) {
+      const key = parseKey(buffer, offset, keySize)
+      offset += keySize
+      const dataOffset = Number(dataView.getBigUint64(offset, true))
+      offset += 8
+      leafkeys.push({
+        key,
+        offset: dataOffset,
+      })
+    }
+
+    // Binary search to find the appropriate child node
+    let left = 0
+    let right = leafkeys.length - 1
+    let targetIndex = leafkeys.length - 1
+
+    while (left <= right) {
+      const mid = Math.floor((left + right) / 2)
+      const cmp = name.localeCompare(leafkeys[mid]!.key)
+
+      if (cmp < 0) {
+        targetIndex = mid - 1
+        right = mid - 1
+      } else {
+        left = mid + 1
+      }
+    }
+
+    const childOffset =
+      targetIndex >= 0 ? leafkeys[targetIndex]!.offset : leafkeys[0]!.offset
+    return readBPlusTreeNode(
+      bbi,
+      childOffset,
+      blockSize,
+      keySize,
+      valSize,
+      name,
+      field,
+      opts,
+    )
+  } else if (nodeType === 1) {
+    // Leaf node (nodeType === 1): contains actual key-value data
+    const keys = []
+    for (let i = 0; i < cnt; i++) {
+      const key = parseKey(buffer, offset, keySize)
+      offset += keySize
+      const dataOffset = Number(dataView.getBigUint64(offset, true))
+      offset += 8
+      const length = dataView.getUint32(offset, true)
+      offset += 4
+      offset += 4 // skip reserved
+      keys.push({
+        key,
+        offset: dataOffset,
+        length,
+      })
+    }
+
+    // Binary search for exact key match in sorted leaf node
+    let left = 0
+    let right = keys.length - 1
+
+    while (left <= right) {
+      const mid = Math.floor((left + right) / 2)
+      const cmp = name.localeCompare(keys[mid]!.key)
+
+      if (cmp === 0) {
+        return { ...keys[mid]!, field }
+      } else if (cmp < 0) {
+        right = mid - 1
+      } else {
+        left = mid + 1
+      }
+    }
+
+    return undefined
+  }
+}
+
 export class BigBed extends BBI {
   public readIndicesCache = new AbortablePromiseCache<RequestOptions, Index[]>({
     cache: new QuickLRU({ maxSize: 1 }),
@@ -53,7 +183,7 @@ export class BigBed extends BBI {
    */
   private async _readIndices(opts: RequestOptions) {
     const { extHeaderOffset } = await this.getHeader(opts)
-    const b = await this.bbi.read(64, Number(extHeaderOffset))
+    const b = await this.bbi.read(64, extHeaderOffset)
 
     const dataView = new DataView(b.buffer, b.byteOffset, b.length)
     let offset = 0
@@ -71,9 +201,9 @@ export class BigBed extends BBI {
 
     const blocklen = 20
     const len = blocklen * count
-    const buffer = await this.bbi.read(len, Number(dataOffset))
+    const buffer = await this.bbi.read(len, dataOffset)
 
-    const indices = [] as Index[]
+    const indices: Index[] = []
 
     for (let i = 0; i < count; i += 1) {
       const b = buffer.subarray(i * blocklen)
@@ -84,9 +214,14 @@ export class BigBed extends BBI {
       const fieldcount = dataView.getInt16(offset, true)
       offset += 2
       const dataOffset = Number(dataView.getBigUint64(offset, true))
-      offset += 8 + 4 //4 skip
+      offset += 8 + 4 // skip 8-byte offset + 4 reserved bytes
       const field = dataView.getInt16(offset, true)
-      indices.push({ type, fieldcount, offset: Number(dataOffset), field })
+      indices.push({
+        type,
+        fieldcount,
+        offset: dataOffset,
+        field,
+      })
     }
     return indices
   }
@@ -109,7 +244,6 @@ export class BigBed extends BBI {
     if (indices.length === 0) {
       return []
     }
-    const decoder = new TextDecoder('utf8')
     const locs = indices.map(async index => {
       const { offset: offset2, field } = index
       const b = await this.bbi.read(32, offset2, opts)
@@ -127,74 +261,16 @@ export class BigBed extends BBI {
       // const _itemCount = Number(dataView.getBigUint64(offset, true))
       offset += 8
 
-      const bptReadNode = async (nodeOffset: number) => {
-        const val = Number(nodeOffset)
-        const len = 4 + blockSize * (keySize + valSize)
-        const buffer = await this.bbi.read(len, val, opts)
-        const b = buffer
-        const dataView = new DataView(b.buffer, b.byteOffset, b.length)
-        let offset = 0
-        const nodeType = dataView.getInt8(offset)
-        offset += 2 //skip 1
-        const cnt = dataView.getInt16(offset, true)
-        offset += 2
-        const keys = []
-        if (nodeType === 0) {
-          const leafkeys = []
-          for (let i = 0; i < cnt; i++) {
-            const key = decoder
-              .decode(b.subarray(offset, offset + keySize))
-              .replaceAll('\0', '')
-            offset += keySize
-            const dataOffset = Number(dataView.getBigUint64(offset, true))
-            offset += 8
-            leafkeys.push({
-              key,
-              offset: dataOffset,
-            })
-          }
-
-          let lastOffset = 0
-          for (const { key, offset } of leafkeys) {
-            if (name.localeCompare(key) < 0 && lastOffset) {
-              return bptReadNode(lastOffset)
-            }
-            lastOffset = offset
-          }
-          return bptReadNode(lastOffset)
-        } else if (nodeType === 1) {
-          for (let i = 0; i < cnt; i++) {
-            const key = decoder
-              .decode(b.subarray(offset, offset + keySize))
-              .replaceAll('\0', '')
-            offset += keySize
-            const dataOffset = Number(dataView.getBigUint64(offset, true))
-            offset += 8
-            const length = dataView.getUint32(offset, true)
-            offset += 4
-            const reserved = dataView.getUint32(offset, true)
-            offset += 4
-            keys.push({
-              key,
-              offset: dataOffset,
-              length,
-              reserved,
-            })
-          }
-
-          for (const n of keys) {
-            if (n.key === name) {
-              return {
-                ...n,
-                field,
-              }
-            }
-          }
-
-          return undefined
-        }
-      }
-      return bptReadNode(offset2 + 32)
+      return readBPlusTreeNode(
+        this.bbi,
+        offset2 + 32,
+        blockSize,
+        keySize,
+        valSize,
+        name,
+        field,
+        opts,
+      )
     })
     return filterUndef(await Promise.all(locs))
   }
@@ -206,7 +282,7 @@ export class BigBed extends BBI {
    *
    * @param name - the name to search for
    *
-   * @param opts - options object with optional AboutSignal
+   * @param opts - options object with optional AbortSignal
    *
    * @return array of Feature
    */
@@ -222,16 +298,22 @@ export class BigBed extends BBI {
           observer.error(e)
         })
       }).pipe(
-        reduce((acc, curr) => acc.concat(curr)),
-        map(x => {
-          for (const element of x) {
-            element.field = block.field
-          }
-          return x
-        }),
+        reduce((acc, curr) => {
+          acc.push(...curr)
+          return acc
+        }, [] as Feature[]),
+        map(features => features.map(f => ({ ...f, field: block.field }))),
       )
     })
     const ret = await firstValueFrom(merge(...res))
-    return ret.filter(f => f.rest?.split('\t')[(f.field || 0) - 3] === name)
+    // Filter to features where the indexed field matches the search name
+    // field offset is adjusted by -3 to account for chrom, chromStart, chromEnd columns
+    return ret.filter(f => {
+      if (!f.rest) {
+        return false
+      }
+      const fieldIndex = (f.field || 0) - 3
+      return getTabField(f.rest, fieldIndex) === name
+    })
   }
 }

package/src/bigwig.ts

@@ -20,13 +20,12 @@ export class BigWig extends BBI {
 
     for (let i = maxLevel; i >= 0; i -= 1) {
       const zh = zoomLevels[i]
-      // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
       if (zh && zh.reductionLevel <= 2 * basesPerPx) {
         return new BlockView(
           this.bbi,
           refsByName,
           zh.indexOffset,
-          uncompressBufSize > 0,
+          uncompressBufSize,
           'summary',
         )
       }