@gmod/bbi 1.0.35 → 2.0.0

package/src/bbi.ts

@@ -1,9 +1,7 @@
-import { Parser } from '@gmod/binary-parser'
+import { Parser } from 'binary-parser'
 import { LocalFile, RemoteFile, GenericFilehandle } from 'generic-filehandle'
 import { Observable, Observer } from 'rxjs'
 import { reduce } from 'rxjs/operators'
-import AbortablePromiseCache from 'abortable-promise-cache'
-import QuickLRU from 'quick-lru'
 import { BlockView } from './blockView'
 
 const BIG_WIG_MAGIC = -2003829722
@@ -53,7 +51,7 @@ export interface Header {
  * @param isBE - is big endian, typically false
  * @return an object with compiled parsers
  */
-function getParsers(isBE: boolean): any {
+function getParsers(isBE: boolean) {
   const le = isBE ? 'big' : 'little'
   const headerParser = new Parser()
     .endianess(le)
@@ -72,6 +70,7 @@ function getParsers(isBE: boolean): any {
     .array('zoomLevels', {
       length: 'numZoomLevels',
       type: new Parser()
+        .endianess(le)
         .uint32('reductionLevel')
         .uint32('reserved')
         .uint64('dataOffset')
@@ -81,10 +80,10 @@ function getParsers(isBE: boolean): any {
   const totalSummaryParser = new Parser()
     .endianess(le)
     .uint64('basesCovered')
-    .double('scoreMin')
-    .double('scoreMax')
-    .double('scoreSum')
-    .double('scoreSumSquares')
+    .doublele('scoreMin')
+    .doublele('scoreMax')
+    .doublele('scoreSum')
+    .doublele('scoreSumSquares')
 
   const chromTreeParser = new Parser()
     .endianess(le)
@@ -99,6 +98,7 @@ function getParsers(isBE: boolean): any {
     .uint8('isLeafNode')
     .skip(1)
     .uint16('cnt')
+    .saveOffset('offset')
 
   return {
     chromTreeParser,
@@ -117,12 +117,7 @@ export interface RequestOptions {
 export abstract class BBI {
   protected bbi: GenericFilehandle
 
-  protected headerCache = new AbortablePromiseCache({
-    cache: new QuickLRU({ maxSize: 1 }),
-    fill: async (params: any, signal?: AbortSignal) => {
-      return this._getHeader({ ...params, signal })
-    },
-  })
+  private headerP?: Promise<Header>
 
   protected renameRefSeqs: (a: string) => string
 
@@ -131,12 +126,14 @@ export abstract class BBI {
    * @return a Header object
    */
   public getHeader(opts: RequestOptions | AbortSignal = {}) {
-    const options = 'aborted' in opts ? { signal: opts } : opts
-    return this.headerCache.get(
-      JSON.stringify(options),
-      options,
-      options.signal,
-    )
+    const options = 'aborted' in opts ? { signal: opts as AbortSignal } : opts
+    if (!this.headerP) {
+      this.headerP = this._getHeader(options).catch(e => {
+        this.headerP = undefined
+        throw e
+      })
+    }
+    return this.headerP
   }
 
   /*
@@ -153,8 +150,8 @@ export abstract class BBI {
       renameRefSeqs?: (a: string) => string
     } = {},
   ) {
-    const { filehandle, renameRefSeqs, path, url } = options
-    this.renameRefSeqs = renameRefSeqs || ((s: string): string => s)
+    const { filehandle, renameRefSeqs = s => s, path, url } = options
+    this.renameRefSeqs = renameRefSeqs
     if (filehandle) {
       this.bbi = filehandle
     } else if (url) {
@@ -185,25 +182,24 @@ export abstract class BBI {
     )
     const isBigEndian = this._isBigEndian(buffer)
     const ret = getParsers(isBigEndian)
-    const header = ret.headerParser.parse(buffer).result
-    header.fileType = header.magic === BIG_BED_MAGIC ? 'bigbed' : 'bigwig'
-    if (
-      header.asOffset > requestSize ||
-      header.totalSummaryOffset > requestSize
-    ) {
+    const header = ret.headerParser.parse(buffer)
+    const { magic, asOffset, totalSummaryOffset } = header
+    header.fileType = magic === BIG_BED_MAGIC ? 'bigbed' : 'bigwig'
+    if (asOffset > requestSize || totalSummaryOffset > requestSize) {
       return this._getMainHeader(opts, requestSize * 2)
     }
-    if (header.asOffset) {
+    if (asOffset) {
+      const off = Number(header.asOffset)
       header.autoSql = buffer
-        .slice(header.asOffset, buffer.indexOf(0, header.asOffset))
+        .subarray(off, buffer.indexOf(0, off))
         .toString('utf8')
     }
     if (header.totalSummaryOffset > requestSize) {
       return this._getMainHeader(opts, requestSize * 2)
     }
     if (header.totalSummaryOffset) {
-      const tail = buffer.slice(header.totalSummaryOffset)
-      header.totalSummary = ret.totalSummaryParser.parse(tail).result
+      const tail = buffer.subarray(Number(header.totalSummaryOffset))
+      header.totalSummary = ret.totalSummaryParser.parse(tail)
     }
     return { ...header, isBigEndian }
   }
@@ -228,46 +224,48 @@ export abstract class BBI {
       [key: number]: { name: string; id: number; length: number }
     } = []
     const refsByName: { [key: string]: number } = {}
-    const { chromTreeOffset } = header
-    let { unzoomedDataOffset } = header
 
+    let unzoomedDataOffset = Number(header.unzoomedDataOffset)
+    const chromTreeOffset = Number(header.chromTreeOffset)
     while (unzoomedDataOffset % 4 !== 0) {
       unzoomedDataOffset += 1
     }
-
-    const { buffer: data } = await this.bbi.read(
-      Buffer.alloc(unzoomedDataOffset - chromTreeOffset),
+    const off = unzoomedDataOffset - chromTreeOffset
+    const { buffer } = await this.bbi.read(
+      Buffer.alloc(off),
       0,
-      unzoomedDataOffset - chromTreeOffset,
-      chromTreeOffset,
+      off,
+      Number(chromTreeOffset),
       opts,
     )
 
     const p = getParsers(isBE)
-    const { keySize } = p.chromTreeParser.parse(data).result
+    const { keySize } = p.chromTreeParser.parse(buffer)
     const leafNodeParser = new Parser()
       .endianess(le)
       .string('key', { stripNull: true, length: keySize })
       .uint32('refId')
       .uint32('refSize')
+      .saveOffset('offset')
     const nonleafNodeParser = new Parser()
       .endianess(le)
       .skip(keySize)
       .uint64('childOffset')
+      .saveOffset('offset')
     const rootNodeOffset = 32
-    const bptReadNode = async (currentOffset: number): Promise<void> => {
+    const bptReadNode = async (currentOffset: number) => {
       let offset = currentOffset
-      if (offset >= data.length) {
+      if (offset >= buffer.length) {
         throw new Error('reading beyond end of buffer')
       }
-      const ret = p.isLeafNode.parse(data.slice(offset))
-      const { isLeafNode, cnt } = ret.result
+      const ret = p.isLeafNode.parse(buffer.subarray(offset))
+      const { isLeafNode, cnt } = ret
       offset += ret.offset
       if (isLeafNode) {
         for (let n = 0; n < cnt; n += 1) {
-          const leafRet = leafNodeParser.parse(data.slice(offset))
+          const leafRet = leafNodeParser.parse(buffer.subarray(offset))
           offset += leafRet.offset
-          const { key, refId, refSize } = leafRet.result
+          const { key, refId, refSize } = leafRet
           const refRec = { name: key, id: refId, length: refSize }
           refsByName[this.renameRefSeqs(key)] = refId
           refsByNumber[refId] = refRec
@@ -276,11 +274,12 @@ export abstract class BBI {
         // parse index node
         const nextNodes = []
         for (let n = 0; n < cnt; n += 1) {
-          const nonleafRet = nonleafNodeParser.parse(data.slice(offset))
-          let { childOffset } = nonleafRet.result
+          const nonleafRet = nonleafNodeParser.parse(buffer.subarray(offset))
+          const { childOffset } = nonleafRet
           offset += nonleafRet.offset
-          childOffset -= chromTreeOffset
-          nextNodes.push(bptReadNode(childOffset))
+          nextNodes.push(
+            bptReadNode(Number(childOffset) - Number(chromTreeOffset)),
+          )
         }
         await Promise.all(nextNodes)
       }
@@ -299,19 +298,15 @@ export abstract class BBI {
   protected async getUnzoomedView(opts: RequestOptions): Promise<BlockView> {
     const {
       unzoomedIndexOffset,
-      zoomLevels,
       refsByName,
       uncompressBufSize,
       isBigEndian,
       fileType,
     } = await this.getHeader(opts)
-    const nzl = zoomLevels[0]
-    const cirLen = nzl ? nzl.dataOffset - unzoomedIndexOffset : 4000
     return new BlockView(
       this.bbi,
       refsByName,
       unzoomedIndexOffset,
-      cirLen,
       isBigEndian,
       uncompressBufSize > 0,
       fileType,

package/src/bigbed.ts

@@ -1,5 +1,5 @@
-import { Parser } from '@gmod/binary-parser'
-import { Observable, Observer, merge } from 'rxjs'
+import { Parser } from 'binary-parser'
+import { Observable, merge } from 'rxjs'
 import { map, reduce } from 'rxjs/operators'
 import AbortablePromiseCache from 'abortable-promise-cache'
 import QuickLRU from 'quick-lru'
@@ -9,8 +9,8 @@ import { BlockView } from './blockView'
 
 interface Loc {
   key: string
-  offset: number
-  length: number
+  offset: bigint
+  length: bigint
   field?: number
 }
 
@@ -33,10 +33,6 @@ export class BigBed extends BBI {
     },
   })
 
-  public constructor(opts?: any) {
-    super(opts)
-  }
-
   public readIndices(opts: AbortSignal | RequestOptions = {}) {
     const options = 'aborted' in opts ? { signal: opts } : opts
     return this.readIndicesCache.get(
@@ -53,7 +49,7 @@ export class BigBed extends BBI {
    * @return promise for a BlockView
    */
   protected async getView(
-    scale: number,
+    _scale: number,
     opts: RequestOptions,
   ): Promise<BlockView> {
     return this.getUnzoomedView(opts)
@@ -64,13 +60,13 @@ export class BigBed extends BBI {
    * @param abortSignal to abort operation
    * @return a Promise for an array of Index data structure since there can be multiple extraIndexes in a bigbed, see bedToBigBed documentation
    */
-  private async _readIndices(opts: RequestOptions): Promise<Index[]> {
+  private async _readIndices(opts: RequestOptions) {
     const { extHeaderOffset, isBigEndian } = await this.getHeader(opts)
     const { buffer: data } = await this.bbi.read(
       Buffer.alloc(64),
       0,
       64,
-      extHeaderOffset,
+      Number(extHeaderOffset),
     )
     const le = isBigEndian ? 'big' : 'little'
     const ret = new Parser()
@@ -78,7 +74,8 @@ export class BigBed extends BBI {
       .uint16('size')
       .uint16('count')
       .uint64('offset')
-      .parse(data).result
+      .parse(data)
+
     const { count, offset } = ret
 
     // no extra index is defined if count==0
@@ -88,7 +85,12 @@ export class BigBed extends BBI {
 
     const blocklen = 20
     const len = blocklen * count
-    const { buffer } = await this.bbi.read(Buffer.alloc(len), 0, len, offset)
+    const { buffer } = await this.bbi.read(
+      Buffer.alloc(len),
+      0,
+      len,
+      Number(offset),
+    )
     const extParser = new Parser()
       .endianess(le)
       .int16('type')
@@ -96,10 +98,10 @@ export class BigBed extends BBI {
       .uint64('offset')
       .skip(4)
       .int16('field')
-    const indices = []
+    const indices = [] as Index[]
 
     for (let i = 0; i < count; i += 1) {
-      indices.push(extParser.parse(buffer.slice(i * blocklen)).result)
+      indices.push(extParser.parse(buffer.subarray(i * blocklen)))
     }
     return indices
   }
@@ -127,20 +129,22 @@ export class BigBed extends BBI {
         Buffer.alloc(32),
         0,
         32,
-        offset,
+        Number(offset),
         opts,
       )
+      const le = isBigEndian ? 'big' : 'little'
       const p = new Parser()
-        .endianess(isBigEndian ? 'big' : 'little')
+        .endianess(le)
         .int32('magic')
         .int32('blockSize')
         .int32('keySize')
         .int32('valSize')
         .uint64('itemCount')
 
-      const { blockSize, keySize, valSize } = p.parse(data).result
+      const { blockSize, keySize, valSize } = p.parse(data)
+      // console.log({blockSize,keySize,valSize})
       const bpt = new Parser()
-        .endianess(isBigEndian ? 'big' : 'little')
+        .endianess(le)
         .int8('nodeType')
         .skip(1)
         .int16('cnt')
@@ -150,12 +154,14 @@ export class BigBed extends BBI {
             0: new Parser().array('leafkeys', {
               length: 'cnt',
               type: new Parser()
+                .endianess(le)
                 .string('key', { length: keySize, stripNull: true })
                 .uint64('offset'),
             }),
             1: new Parser().array('keys', {
               length: 'cnt',
               type: new Parser()
+                .endianess(le)
                 .string('key', { length: keySize, stripNull: true })
                 .uint64('offset')
                 .uint32('length')
@@ -167,15 +173,16 @@ export class BigBed extends BBI {
       const bptReadNode = async (
         nodeOffset: number,
       ): Promise<Loc | undefined> => {
+        const val = Number(nodeOffset)
         const len = 4 + blockSize * (keySize + valSize)
         const { buffer } = await this.bbi.read(
           Buffer.alloc(len),
           0,
           len,
-          nodeOffset,
+          val,
           opts,
         )
-        const node = bpt.parse(buffer).result
+        const node = bpt.parse(buffer)
         if (node.leafkeys) {
           let lastOffset
           for (let i = 0; i < node.leafkeys.length; i += 1) {
@@ -196,7 +203,7 @@ export class BigBed extends BBI {
         return undefined
       }
       const rootNodeOffset = 32
-      return bptReadNode(offset + rootNodeOffset)
+      return bptReadNode(Number(offset) + rootNodeOffset)
     })
     return filterUndef(await Promise.all(locs))
   }
@@ -219,7 +226,7 @@ export class BigBed extends BBI {
     }
     const view = await this.getUnzoomedView(opts)
     const res = blocks.map(block => {
-      return new Observable((observer: Observer<Feature[]>) => {
+      return new Observable<Feature[]>(observer => {
         view.readFeatures(observer, [block], opts)
       }).pipe(
         reduce((acc, curr) => acc.concat(curr)),
@@ -232,8 +239,6 @@ export class BigBed extends BBI {
       )
     })
     const ret = await merge(...res).toPromise()
-    return ret.filter((f: any) => {
-      return f.rest.split('\t')[f.field - 3] === name
-    })
+    return ret.filter(f => f.rest?.split('\t')[(f.field || 0) - 3] === name)
   }
 }

package/src/bigwig.ts

@@ -5,15 +5,10 @@ export class BigWig extends BBI {
   /**
    * Retrieves a BlockView of a specific zoomLevel
    *
-   * @param refName - The chromosome name
-   * @param start - The start of a region
-   * @param end - The end of a region
+   * @param scale - number
    * @param opts - An object containing basesPerSpan (e.g. pixels per basepair) or scale used to infer the zoomLevel to use
    */
-  protected async getView(
-    scale: number,
-    opts: RequestOptions,
-  ): Promise<BlockView> {
+  protected async getView(scale: number, opts: RequestOptions) {
     const { zoomLevels, refsByName, fileSize, isBigEndian, uncompressBufSize } =
       await this.getHeader(opts)
     const basesPerPx = 1 / scale
@@ -26,15 +21,12 @@ export class BigWig extends BBI {
     for (let i = maxLevel; i >= 0; i -= 1) {
       const zh = zoomLevels[i]
       if (zh && zh.reductionLevel <= 2 * basesPerPx) {
-        const indexLength =
-          i < zoomLevels.length - 1
-            ? zoomLevels[i + 1].dataOffset - zh.indexOffset
-            : fileSize - 4 - zh.indexOffset
+        const indexOffset = Number(zh.indexOffset)
+
         return new BlockView(
           this.bbi,
           refsByName,
-          zh.indexOffset,
-          indexLength,
+          indexOffset,
           isBigEndian,
           uncompressBufSize > 0,
           'summary',