@gmod/bbi 4.0.6 → 5.0.1

package/src/bbi.ts

@@ -1,5 +1,4 @@
 import { Buffer } from 'buffer'
-import { Parser } from 'binary-parser'
 import { LocalFile, RemoteFile, GenericFilehandle } from 'generic-filehandle'
 import { firstValueFrom, Observable } from 'rxjs'
 import { toArray } from 'rxjs/operators'
@@ -8,14 +7,19 @@ import { BlockView } from './block-view'
 const BIG_WIG_MAGIC = -2003829722
 const BIG_BED_MAGIC = -2021002517
 
-function toString(arr: Uint8Array) {
-  return new TextDecoder().decode(arr)
+interface ZoomLevel {
+  reductionLevel: number
+  reserved: number
+  dataOffset: number
+  indexOffset: number
 }
 
 export interface Feature {
+  offset?: number
+  chromId: number
   start: number
   end: number
-  score: number
+  score?: number
   rest?: string // for bigbed line
   minScore?: number // for summary line
   maxScore?: number // for summary line
@@ -27,6 +31,8 @@ interface Statistics {
   scoreSum: number
   basesCovered: number
   scoreSumSquares: number
+  scoreMin: number
+  scoreMax: number
 }
 
 interface RefInfo {
@@ -34,87 +40,31 @@ interface RefInfo {
   id: number
   length: number
 }
-export interface Header {
+
+export interface MainHeader {
+  magic: number
   version: number
   autoSql: string
   totalSummary: Statistics
-  zoomLevels: any
+  asOffset: number
+  zoomLevels: ZoomLevel[]
+  fieldCount: number
+  numZoomLevels: number
   unzoomedIndexOffset: number
+  totalSummaryOffset: number
   unzoomedDataOffset: number
   definedFieldCount: number
   uncompressBufSize: number
   chromTreeOffset: number
-  fileSize: number
   extHeaderOffset: number
   isBigEndian: boolean
   fileType: string
+}
+export interface Header extends MainHeader {
   refsByName: Record<string, number>
   refsByNumber: Record<number, RefInfo>
 }
 
-/**
- * get the compiled parsers for different sections of the bigwig file
- *
- * @param isBE - is big endian, typically false
- * @return an object with compiled parsers
- */
-function getParsers(isBE: boolean) {
-  const le = isBE ? 'big' : 'little'
-  const headerParser = new Parser()
-    .endianess(le)
-    .int32('magic')
-    .uint16('version')
-    .uint16('numZoomLevels')
-    .uint64('chromTreeOffset')
-    .uint64('unzoomedDataOffset')
-    .uint64('unzoomedIndexOffset')
-    .uint16('fieldCount')
-    .uint16('definedFieldCount')
-    .uint64('asOffset') // autoSql offset, used in bigbed
-    .uint64('totalSummaryOffset')
-    .uint32('uncompressBufSize')
-    .uint64('extHeaderOffset') // name index offset, used in bigbed
-    .array('zoomLevels', {
-      length: 'numZoomLevels',
-      type: new Parser()
-        .endianess(le)
-        .uint32('reductionLevel')
-        .uint32('reserved')
-        .uint64('dataOffset')
-        .uint64('indexOffset'),
-    })
-
-  const totalSummaryParser = new Parser()
-    .endianess(le)
-    .uint64('basesCovered')
-    .doublele('scoreMin')
-    .doublele('scoreMax')
-    .doublele('scoreSum')
-    .doublele('scoreSumSquares')
-
-  const chromTreeParser = new Parser()
-    .endianess(le)
-    .uint32('magic')
-    .uint32('blockSize')
-    .uint32('keySize')
-    .uint32('valSize')
-    .uint64('itemCount')
-
-  const isLeafNode = new Parser()
-    .endianess(le)
-    .uint8('isLeafNode')
-    .skip(1)
-    .uint16('cnt')
-    .saveOffset('offset')
-
-  return {
-    chromTreeParser,
-    totalSummaryParser,
-    headerParser,
-    isLeafNode,
-  }
-}
-
 export interface RequestOptions {
   signal?: AbortSignal
   headers?: Record<string, string>
@@ -139,7 +89,8 @@ export abstract class BBI {
   }
 
   /*
-   * @param filehandle - a filehandle from generic-filehandle or implementing something similar to the node10 fs.promises API
+   * @param filehandle - a filehandle from generic-filehandle or implementing
+   * something similar to the node10 fs.promises API
    *
    * @param path - a Local file path as a string
    *
@@ -176,7 +127,8 @@ export abstract class BBI {
   private async _getMainHeader(
     opts?: RequestOptions,
     requestSize = 2000,
-  ): Promise<Header> {
+  ): Promise<MainHeader> {
+    const le = true
     const { buffer } = await this.bbi.read(
       Buffer.alloc(requestSize),
       0,
@@ -185,30 +137,102 @@ export abstract class BBI {
       opts,
     )
     const isBigEndian = this._isBigEndian(buffer)
-    const ret = getParsers(isBigEndian)
-    const header = ret.headerParser.parse(buffer)
-    const { magic, asOffset, totalSummaryOffset } = header
-    header.fileType = magic === BIG_BED_MAGIC ? 'bigbed' : 'bigwig'
-    if (asOffset > requestSize || totalSummaryOffset > requestSize) {
-      return this._getMainHeader(opts, requestSize * 2)
-    }
-    if (asOffset) {
-      const off = Number(header.asOffset)
-      header.autoSql = toString(buffer.subarray(off, buffer.indexOf(0, off)))
+    const b = buffer
+    const dataView = new DataView(b.buffer, b.byteOffset, b.length)
+    let offset = 0
+    const magic = dataView.getInt32(offset, le)
+    offset += 4
+    const version = dataView.getUint16(offset, le)
+    offset += 2
+    const numZoomLevels = dataView.getUint16(offset, le)
+    offset += 2
+    const chromTreeOffset = Number(dataView.getBigUint64(offset, le))
+    offset += 8
+    const unzoomedDataOffset = Number(dataView.getBigUint64(offset, le))
+    offset += 8
+    const unzoomedIndexOffset = Number(dataView.getBigUint64(offset, le))
+    offset += 8
+    const fieldCount = dataView.getUint16(offset, le)
+    offset += 2
+    const definedFieldCount = dataView.getUint16(offset, le)
+    offset += 2
+    const asOffset = Number(dataView.getBigUint64(offset, le))
+    offset += 8
+    const totalSummaryOffset = Number(dataView.getBigUint64(offset, le))
+    offset += 8
+    const uncompressBufSize = dataView.getUint32(offset, le)
+    offset += 4
+    const extHeaderOffset = Number(dataView.getBigUint64(offset, le))
+    offset += 8
+    const zoomLevels = [] as ZoomLevel[]
+    for (let i = 0; i < numZoomLevels; i++) {
+      const reductionLevel = dataView.getUint32(offset, le)
+      offset += 4
+      const reserved = dataView.getUint32(offset, le)
+      offset += 4
+      const dataOffset = Number(dataView.getBigUint64(offset, le))
+      offset += 8
+      const indexOffset = Number(dataView.getBigUint64(offset, le))
+      offset += 8
+      zoomLevels.push({ reductionLevel, reserved, dataOffset, indexOffset })
     }
 
+    const fileType = magic === BIG_BED_MAGIC ? 'bigbed' : 'bigwig'
+
     // refetch header if it is too large on first pass,
     // 8*5 is the sizeof the totalSummary struct
-    if (header.totalSummaryOffset > requestSize - 8 * 5) {
+    if (asOffset > requestSize || totalSummaryOffset > requestSize - 8 * 5) {
       return this._getMainHeader(opts, requestSize * 2)
     }
 
-    if (header.totalSummaryOffset) {
-      const tail = buffer.subarray(Number(header.totalSummaryOffset))
-      const sum = ret.totalSummaryParser.parse(tail)
-      header.totalSummary = { ...sum, basesCovered: Number(sum.basesCovered) }
+    let totalSummary: Statistics
+    if (totalSummaryOffset) {
+      const b = buffer.subarray(Number(totalSummaryOffset))
+      let offset = 0
+      const dataView = new DataView(b.buffer, b.byteOffset, b.length)
+      const basesCovered = Number(dataView.getBigUint64(offset, le))
+      offset += 8
+      const scoreMin = dataView.getFloat64(offset, le)
+      offset += 8
+      const scoreMax = dataView.getFloat64(offset, le)
+      offset += 8
+      const scoreSum = dataView.getFloat64(offset, le)
+      offset += 8
+      const scoreSumSquares = dataView.getFloat64(offset, le)
+      offset += 8
+
+      totalSummary = {
+        scoreMin,
+        scoreMax,
+        scoreSum,
+        scoreSumSquares,
+        basesCovered,
+      }
+    } else {
+      throw new Error('no stats')
+    }
+
+    return {
+      zoomLevels,
+      magic,
+      extHeaderOffset,
+      numZoomLevels,
+      fieldCount,
+      totalSummary,
+      definedFieldCount,
+      uncompressBufSize,
+      asOffset,
+      chromTreeOffset,
+      totalSummaryOffset,
+      unzoomedDataOffset,
+      unzoomedIndexOffset,
+      fileType,
+      version,
+      isBigEndian,
+      autoSql: asOffset
+        ? buffer.subarray(asOffset, buffer.indexOf(0, asOffset)).toString()
+        : '',
     }
-    return { ...header, isBigEndian }
   }
 
   private _isBigEndian(buffer: Buffer) {
@@ -225,19 +249,19 @@ export abstract class BBI {
 
   // todo: add progress if long running
   private async _readChromTree(
-    header: Header,
+    header: MainHeader,
     opts?: { signal?: AbortSignal },
   ) {
     const isBE = header.isBigEndian
-    const le = isBE ? 'big' : 'little'
+    const le = !isBE
     const refsByNumber: Record<
       number,
       { name: string; id: number; length: number }
     > = []
     const refsByName: Record<string, number> = {}
 
-    let unzoomedDataOffset = Number(header.unzoomedDataOffset)
-    const chromTreeOffset = Number(header.chromTreeOffset)
+    let unzoomedDataOffset = header.unzoomedDataOffset
+    const chromTreeOffset = header.chromTreeOffset
     while (unzoomedDataOffset % 4 !== 0) {
       unzoomedDataOffset += 1
     }
@@ -250,33 +274,42 @@ export abstract class BBI {
       opts,
     )
 
-    const p = getParsers(isBE)
-    const { keySize } = p.chromTreeParser.parse(buffer)
-    const leafNodeParser = new Parser()
-      .endianess(le)
-      .string('key', { stripNull: true, length: keySize })
-      .uint32('refId')
-      .uint32('refSize')
-      .saveOffset('offset')
-    const nonleafNodeParser = new Parser()
-      .endianess(le)
-      .skip(keySize)
-      .uint64('childOffset')
-      .saveOffset('offset')
+    const b = buffer
+    const dataView = new DataView(b.buffer, b.byteOffset, b.length)
+    let offset = 0
+    //    const magic = dataView.getUint32(offset, le)
+    offset += 4
+    //   const blockSize = dataView.getUint32(offset, le)
+    offset += 4
+    const keySize = dataView.getUint32(offset, le)
+    offset += 4
+    //  const valSize = dataView.getUint32(offset, le)
+    offset += 4
+    // const itemCount = dataView.getBigUint64(offset, le)
+    offset += 8
+
     const rootNodeOffset = 32
     const bptReadNode = async (currentOffset: number) => {
       let offset = currentOffset
       if (offset >= buffer.length) {
         throw new Error('reading beyond end of buffer')
       }
-      const ret = p.isLeafNode.parse(buffer.subarray(offset))
-      const { isLeafNode, cnt } = ret
-      offset += ret.offset
+      const isLeafNode = dataView.getUint8(offset)
+      offset += 2 //skip 1
+      const cnt = dataView.getUint16(offset, le)
+      offset += 2
       if (isLeafNode) {
-        for (let n = 0; n < cnt; n += 1) {
-          const leafRet = leafNodeParser.parse(buffer.subarray(offset))
-          offset += leafRet.offset
-          const { key, refId, refSize } = leafRet
+        for (let n = 0; n < cnt; n++) {
+          const key = buffer
+            .subarray(offset, offset + keySize)
+            .toString()
+            .replaceAll('\0', '')
+          offset += keySize
+          const refId = dataView.getUint32(offset, le)
+          offset += 4
+          const refSize = dataView.getUint32(offset, le)
+          offset += 4
+
           const refRec = { name: key, id: refId, length: refSize }
           refsByName[this.renameRefSeqs(key)] = refId
           refsByNumber[refId] = refRec
@@ -284,10 +317,10 @@ export abstract class BBI {
       } else {
         // parse index node
         const nextNodes = []
-        for (let n = 0; n < cnt; n += 1) {
-          const nonleafRet = nonleafNodeParser.parse(buffer.subarray(offset))
-          const { childOffset } = nonleafRet
-          offset += nonleafRet.offset
+        for (let n = 0; n < cnt; n++) {
+          offset += keySize
+          const childOffset = Number(dataView.getBigUint64(offset, le))
+          offset += 8
           nextNodes.push(
             bptReadNode(Number(childOffset) - Number(chromTreeOffset)),
           )

package/src/bigbed.ts

@@ -1,16 +1,16 @@
 import { Buffer } from 'buffer'
-import { Parser } from 'binary-parser'
 import { Observable, merge, firstValueFrom } from 'rxjs'
 import { map, reduce } from 'rxjs/operators'
 import AbortablePromiseCache from '@gmod/abortable-promise-cache'
 import QuickLRU from 'quick-lru'
 
+// locals
 import { BBI, Feature, RequestOptions } from './bbi'
 
 interface Loc {
   key: string
-  offset: bigint
-  length: bigint
+  offset: number
+  length: number
   field?: number
 }
 
@@ -46,8 +46,10 @@ export class BigBed extends BBI {
 
   /*
    * parse the bigbed extraIndex fields
-   * @param abortSignal to abort operation
-   * @return a Promise for an array of Index data structure since there can be multiple extraIndexes in a bigbed, see bedToBigBed documentation
+   *
+   *
+   * @return a Promise for an array of Index data structure since there can be
+   * multiple extraIndexes in a bigbed, see bedToBigBed documentation
    */
   private async _readIndices(opts: RequestOptions) {
     const { extHeaderOffset, isBigEndian } = await this.getHeader(opts)
@@ -57,15 +59,17 @@ export class BigBed extends BBI {
       64,
       Number(extHeaderOffset),
     )
-    const le = isBigEndian ? 'big' : 'little'
-    const ret = new Parser()
-      .endianess(le)
-      .uint16('size')
-      .uint16('count')
-      .uint64('offset')
-      .parse(data)
-
-    const { count, offset } = ret
+    const le = !isBigEndian
+
+    const b = data
+    const dataView = new DataView(b.buffer, b.byteOffset, b.length)
+    let offset = 0
+    // const _size = dataView.getUint16(offset, le)
+    offset += 2
+    const count = dataView.getUint16(offset, le)
+    offset += 2
+    const dataOffset = Number(dataView.getBigUint64(offset, le))
+    offset += 8
 
     // no extra index is defined if count==0
     if (count === 0) {
@@ -78,26 +82,30 @@ export class BigBed extends BBI {
       Buffer.alloc(len),
       0,
       len,
-      Number(offset),
+      Number(dataOffset),
     )
-    const extParser = new Parser()
-      .endianess(le)
-      .int16('type')
-      .int16('fieldcount')
-      .uint64('offset')
-      .skip(4)
-      .int16('field')
+
     const indices = [] as Index[]
 
     for (let i = 0; i < count; i += 1) {
-      indices.push(extParser.parse(buffer.subarray(i * blocklen)))
+      const b = buffer.subarray(i * blocklen)
+      const dataView = new DataView(b.buffer, b.byteOffset, b.length)
+      let offset = 0
+      const type = dataView.getInt16(offset, le)
+      offset += 2
+      const fieldcount = dataView.getInt16(offset, le)
+      offset += 2
+      const dataOffset = Number(dataView.getBigUint64(offset, le))
+      offset += 8 + 4 //4 skip
+      const field = dataView.getInt16(offset, le)
+      indices.push({ type, fieldcount, offset: Number(dataOffset), field })
     }
     return indices
   }
 
   /*
-   * perform a search in the bigbed extraIndex to find which blocks in the bigbed data to look for the
-   * actual feature data
+   * perform a search in the bigbed extraIndex to find which blocks in the
+   * bigbed data to look for the actual feature data
    *
    * @param name - the name to search for
    * @param opts - a SearchOptions argument with optional signal
@@ -112,56 +120,32 @@ export class BigBed extends BBI {
     if (indices.length === 0) {
       return []
     }
-    const locs = indices.map(async (index: any): Promise<Loc | undefined> => {
-      const { offset, field } = index
+    const locs = indices.map(async (index): Promise<Loc | undefined> => {
+      const { offset: offset2, field } = index
       const { buffer: data } = await this.bbi.read(
         Buffer.alloc(32),
         0,
         32,
-        Number(offset),
+        offset2,
         opts,
       )
-      const le = isBigEndian ? 'big' : 'little'
-      const p = new Parser()
-        .endianess(le)
-        .int32('magic')
-        .int32('blockSize')
-        .int32('keySize')
-        .int32('valSize')
-        .uint64('itemCount')
-
-      const { blockSize, keySize, valSize } = p.parse(data)
-      // console.log({blockSize,keySize,valSize})
-      const bpt = new Parser()
-        .endianess(le)
-        .int8('nodeType')
-        .skip(1)
-        .int16('cnt')
-        .choice({
-          tag: 'nodeType',
-          choices: {
-            0: new Parser().array('leafkeys', {
-              length: 'cnt',
-              type: new Parser()
-                .endianess(le)
-                .string('key', { length: keySize, stripNull: true })
-                .uint64('offset'),
-            }),
-            1: new Parser().array('keys', {
-              length: 'cnt',
-              type: new Parser()
-                .endianess(le)
-                .string('key', { length: keySize, stripNull: true })
-                .uint64('offset')
-                .uint32('length')
-                .uint32('reserved'),
-            }),
-          },
-        })
-
-      const bptReadNode = async (
-        nodeOffset: number,
-      ): Promise<Loc | undefined> => {
+      const le = !isBigEndian
+      const b = data
+
+      const dataView = new DataView(b.buffer, b.byteOffset, b.length)
+      let offset = 0
+      // const _magic = dataView.getInt32(offset, le)
+      offset += 4
+      const blockSize = dataView.getInt32(offset, le)
+      offset += 4
+      const keySize = dataView.getInt32(offset, le)
+      offset += 4
+      const valSize = dataView.getInt32(offset, le)
+      offset += 4
+      // const _itemCount = Number(dataView.getBigUint64(offset, le))
+      offset += 8
+
+      const bptReadNode = async (nodeOffset: number) => {
         const val = Number(nodeOffset)
         const len = 4 + blockSize * (keySize + valSize)
         const { buffer } = await this.bbi.read(
@@ -171,27 +155,62 @@ export class BigBed extends BBI {
           val,
           opts,
         )
-        const node = bpt.parse(buffer)
-        if (node.leafkeys) {
-          let lastOffset
-          for (const { key, offset } of node.leafkeys) {
+        const b = buffer
+        const dataView = new DataView(b.buffer, b.byteOffset, b.length)
+        let offset = 0
+        const nodeType = dataView.getInt8(offset)
+        offset += 2 //skip 1
+        const cnt = dataView.getInt16(offset, le)
+        offset += 2
+        const keys = []
+        if (nodeType === 0) {
+          const leafkeys = []
+          for (let i = 0; i < cnt; i++) {
+            const key = b
+              .subarray(offset, offset + keySize)
+              .toString()
+              .replaceAll('\0', '')
+            offset += keySize
+            const dataOffset = Number(dataView.getBigUint64(offset, le))
+            offset += 8
+            leafkeys.push({ key, offset: dataOffset })
+          }
+
+          let lastOffset = 0
+          for (const { key, offset } of leafkeys) {
             if (name.localeCompare(key) < 0 && lastOffset) {
               return bptReadNode(lastOffset)
             }
             lastOffset = offset
           }
           return bptReadNode(lastOffset)
-        }
-        for (const n of node.keys) {
-          if (n.key === name) {
-            return { ...n, field }
+        } else if (nodeType === 1) {
+          for (let i = 0; i < cnt; i++) {
+            const key = b
+              .subarray(offset, offset + keySize)
+              .toString()
+              .replaceAll('\0', '')
+            offset += keySize
+            const dataOffset = Number(dataView.getBigUint64(offset, le))
+            offset += 8
+            const length = dataView.getUint32(offset, le)
+            offset += 4
+            const reserved = dataView.getUint32(offset, le)
+            offset += 4
+            keys.push({ key, offset: dataOffset, length, reserved })
+          }
+
+          for (const n of keys) {
+            if (n.key === name) {
+              return { ...n, field }
+            }
           }
-        }
 
-        return undefined
+          return undefined
+        }
       }
       const rootNodeOffset = 32
-      return bptReadNode(Number(offset) + rootNodeOffset)
+      return bptReadNode(offset2 + rootNodeOffset)
     })
     return filterUndef(await Promise.all(locs))
   }

package/src/bigint-polyfill/polyfill.ts

@@ -1,4 +1,4 @@
-import { getBigInt64, getBigUint64, setBigInt64, setBigUint64 } from './pure'
+import { getBigInt64, getBigUint64 } from './pure'
 
 if (!('getBigInt64' in DataView)) {
   DataView.prototype.getBigInt64 = function (byteOffset, littleEndian) {
@@ -11,15 +11,3 @@ if (!('getBigUint64' in DataView)) {
     return getBigUint64(this, byteOffset, littleEndian)
   }
 }
-
-if (!('setBigInt64' in DataView)) {
-  DataView.prototype.setBigInt64 = function (byteOffset, value, littleEndian) {
-    setBigInt64(this, byteOffset, value, littleEndian)
-  }
-}
-
-if (!('setBigUint64' in DataView)) {
-  DataView.prototype.setBigUint64 = function (byteOffset, value, littleEndian) {
-    setBigUint64(this, byteOffset, value, littleEndian)
-  }
-}

package/src/bigint-polyfill/pure.ts

@@ -40,39 +40,3 @@ export function getBigUint64(
     BigInt(a * littleEndianMask + b * bigEndianMask)
   )
 }
-
-export function setBigInt64(
-  dataView: DataView,
-  byteOffset: number,
-  value: bigint,
-  littleEndian: boolean | undefined,
-) {
-  const hi = Number(value >> BigInt32)
-  const lo = Number(value & BigInt(0xffffffff))
-
-  if (littleEndian) {
-    dataView.setInt32(byteOffset + 4, hi, littleEndian)
-    dataView.setUint32(byteOffset, lo, littleEndian)
-  } else {
-    dataView.setInt32(byteOffset, hi, littleEndian)
-    dataView.setUint32(byteOffset + 4, lo, littleEndian)
-  }
-}
-
-export function setBigUint64(
-  dataView: DataView,
-  byteOffset: number,
-  value: bigint,
-  littleEndian: boolean | undefined,
-) {
-  const hi = Number(value >> BigInt32)
-  const lo = Number(value & BigInt(0xffffffff))
-
-  if (littleEndian) {
-    dataView.setUint32(byteOffset + 4, hi, littleEndian)
-    dataView.setUint32(byteOffset, lo, littleEndian)
-  } else {
-    dataView.setUint32(byteOffset, hi, littleEndian)
-    dataView.setUint32(byteOffset + 4, lo, littleEndian)
-  }
-}