@gmod/bam 1.1.17 → 2.0.0

package/src/bamFile.ts

@@ -1,80 +1,76 @@
+import { Buffer } from 'buffer'
 import crc32 from 'buffer-crc32'
 import { unzip, unzipChunkSlice } from '@gmod/bgzf-filehandle'
-import entries from 'object.entries-ponyfill'
 import { LocalFile, RemoteFile, GenericFilehandle } from 'generic-filehandle'
 import AbortablePromiseCache from 'abortable-promise-cache'
 import QuickLRU from 'quick-lru'
-//locals
+
+// locals
 import BAI from './bai'
 import CSI from './csi'
 import Chunk from './chunk'
 import BAMFeature from './record'
-import IndexFile from './indexFile'
 import { parseHeaderText } from './sam'
-import {
-  abortBreakPoint,
-  checkAbortSignal,
-  timeout,
-  makeOpts,
-  BamOpts,
-  BaseOpts,
-} from './util'
+import { checkAbortSignal, timeout, makeOpts, BamOpts, BaseOpts } from './util'
 
 export const BAM_MAGIC = 21840194
 
 const blockLen = 1 << 16
 
-function flat<T>(arr: T[][]) {
-  return ([] as T[]).concat(...arr)
-}
-
-async function gen2array<T>(gen: AsyncIterable<T>): Promise<T[]> {
-  const out: T[] = []
+async function gen2array<T>(gen: AsyncIterable<T[]>): Promise<T[]> {
+  let out: T[] = []
   for await (const x of gen) {
-    out.push(x)
+    out = out.concat(x)
   }
   return out
 }
 
+interface Args {
+  chunk: Chunk
+  opts: BaseOpts
+}
+
+class NullFilehandle {
+  public read(): Promise<any> {
+    throw new Error('never called')
+  }
+  public stat(): Promise<any> {
+    throw new Error('never called')
+  }
+
+  public readFile(): Promise<any> {
+    throw new Error('never called')
+  }
+
+  public close(): Promise<any> {
+    throw new Error('never called')
+  }
+}
 export default class BamFile {
-  private renameRefSeq: (a: string) => string
-  private bam: GenericFilehandle
-  private index: IndexFile
-  private chunkSizeLimit: number
-  private fetchSizeLimit: number
-  private header: any
-  protected chrToIndex: any
-  protected indexToChr: any
-  private yieldThreadTime: number
-
-  private featureCache = new AbortablePromiseCache({
-    //@ts-ignore
+  public renameRefSeq: (a: string) => string
+  public bam: GenericFilehandle
+  public header?: string
+  public chrToIndex?: Record<string, number>
+  public indexToChr?: { refName: string; length: number }[]
+  public yieldThreadTime: number
+  public index?: BAI | CSI
+  public htsget = false
+  public headerP?: ReturnType<BamFile['getHeaderPre']>
+
+  private featureCache = new AbortablePromiseCache<Args, BAMFeature[]>({
     cache: new QuickLRU({
       maxSize: 50,
     }),
-    //@ts-ignore
-    fill: async ({ chunk, opts }, signal) => {
+    fill: async (args: Args, signal) => {
+      const { chunk, opts } = args
       const { data, cpositions, dpositions } = await this._readChunk({
         chunk,
         opts: { ...opts, signal },
       })
-      const feats = await this.readBamFeatures(
-        data,
-        cpositions,
-        dpositions,
-        chunk,
-      )
-      return feats
+      return this.readBamFeatures(data, cpositions, dpositions, chunk)
     },
   })
 
-  /**
-   * @param {object} args
-   * @param {string} [args.bamPath]
-   * @param {FileHandle} [args.bamFilehandle]
-   * @param {string} [args.baiPath]
-   * @param {FileHandle} [args.baiFilehandle]
-   */
   constructor({
     bamFilehandle,
     bamPath,
@@ -85,8 +81,7 @@ export default class BamFile {
     csiPath,
     csiFilehandle,
     csiUrl,
-    fetchSizeLimit,
-    chunkSizeLimit,
+    htsget,
     yieldThreadTime = 100,
     renameRefSeqs = n => n,
   }: {
@@ -99,10 +94,9 @@ export default class BamFile {
     csiPath?: string
     csiFilehandle?: GenericFilehandle
     csiUrl?: string
-    fetchSizeLimit?: number
-    chunkSizeLimit?: number
     renameRefSeqs?: (a: string) => string
     yieldThreadTime?: number
+    htsget?: boolean
   }) {
     this.renameRefSeq = renameRefSeqs
 
@@ -112,6 +106,9 @@ export default class BamFile {
       this.bam = new LocalFile(bamPath)
     } else if (bamUrl) {
       this.bam = new RemoteFile(bamUrl)
+    } else if (htsget) {
+      this.htsget = true
+      this.bam = new NullFilehandle()
     } else {
       throw new Error('unable to initialize bam')
     }
@@ -131,40 +128,31 @@ export default class BamFile {
       this.index = new BAI({ filehandle: new LocalFile(`${bamPath}.bai`) })
     } else if (bamUrl) {
       this.index = new BAI({ filehandle: new RemoteFile(`${bamUrl}.bai`) })
+    } else if (htsget) {
+      this.htsget = true
     } else {
       throw new Error('unable to infer index format')
     }
-    this.fetchSizeLimit = fetchSizeLimit || 500000000 // 500MB
-    this.chunkSizeLimit = chunkSizeLimit || 300000000 // 300MB
     this.yieldThreadTime = yieldThreadTime
   }
 
-  async getHeader(origOpts: AbortSignal | BaseOpts = {}) {
+  async getHeaderPre(origOpts?: BaseOpts) {
     const opts = makeOpts(origOpts)
+    if (!this.index) {
+      return
+    }
     const indexData = await this.index.parse(opts)
     const ret = indexData.firstDataLine
       ? indexData.firstDataLine.blockPosition + 65535
       : undefined
     let buffer
     if (ret) {
-      const res = await this.bam.read(
-        Buffer.alloc(ret + blockLen),
-        0,
-        ret + blockLen,
-        0,
-        opts,
-      )
-
-      const { bytesRead } = res
-      ;({ buffer } = res)
-      if (!bytesRead) {
+      const s = ret + blockLen
+      const res = await this.bam.read(Buffer.alloc(s), 0, s, 0, opts)
+      if (!res.bytesRead) {
         throw new Error('Error reading header')
       }
-      if (bytesRead < ret) {
-        buffer = buffer.subarray(0, bytesRead)
-      } else {
-        buffer = buffer.subarray(0, ret)
-      }
+      buffer = res.buffer.subarray(0, Math.min(res.bytesRead, ret))
     } else {
       buffer = (await this.bam.readFile(opts)) as Buffer
     }
@@ -188,6 +176,16 @@ export default class BamFile {
     return parseHeaderText(this.header)
   }
 
+  getHeader(opts?: BaseOpts) {
+    if (!this.headerP) {
+      this.headerP = this.getHeaderPre(opts).catch(e => {
+        this.headerP = undefined
+        throw e
+      })
+    }
+    return this.headerP
+  }
+
   async getHeaderText(opts: BaseOpts = {}) {
     await this.getHeader(opts)
     return this.header
@@ -198,7 +196,7 @@ export default class BamFile {
   async _readRefSeqs(
     start: number,
     refSeqBytes: number,
-    opts: BaseOpts = {},
+    opts?: BaseOpts,
   ): Promise<{
     chrToIndex: { [key: string]: number }
     indexToChr: { refName: string; length: number }[]
@@ -249,55 +247,25 @@ export default class BamFile {
     chr: string,
     min: number,
     max: number,
-    opts: BamOpts = {
-      viewAsPairs: false,
-      pairAcrossChr: false,
-      maxInsertSize: 200000,
-    },
+    opts?: BamOpts,
   ) {
-    return flat(
-      await gen2array(this.streamRecordsForRange(chr, min, max, opts)),
-    )
+    return gen2array(this.streamRecordsForRange(chr, min, max, opts))
   }
 
   async *streamRecordsForRange(
     chr: string,
     min: number,
     max: number,
-    opts: BamOpts = {},
+    opts?: BamOpts,
   ) {
-    const { signal } = opts
-    const chrId = this.chrToIndex && this.chrToIndex[chr]
-    let chunks: Chunk[]
-    if (!(chrId >= 0)) {
-      chunks = []
+    await this.getHeader(opts)
+    const chrId = this.chrToIndex?.[chr]
+    if (chrId === undefined || !this.index) {
+      yield []
     } else {
-      chunks = await this.index.blocksForRange(chrId, min - 1, max, opts)
-
-      if (!chunks) {
-        throw new Error('Error in index fetch')
-      }
-    }
-
-    for (let i = 0; i < chunks.length; i += 1) {
-      await abortBreakPoint(signal)
-      const size = chunks[i].fetchedSize()
-      if (size > this.chunkSizeLimit) {
-        throw new Error(
-          `Too many BAM features. BAM chunk size ${size} bytes exceeds chunkSizeLimit of ${this.chunkSizeLimit}`,
-        )
-      }
-    }
-
-    const totalSize = chunks
-      .map(s => s.fetchedSize())
-      .reduce((a, b) => a + b, 0)
-    if (totalSize > this.fetchSizeLimit) {
-      throw new Error(
-        `data size of ${totalSize.toLocaleString()} bytes exceeded fetch size limit of ${this.fetchSizeLimit.toLocaleString()} bytes`,
-      )
+      const chunks = await this.index.blocksForRange(chrId, min - 1, max, opts)
+      yield* this._fetchChunkFeatures(chunks, chrId, min, max, opts)
     }
-    yield* this._fetchChunkFeatures(chunks, chrId, min, max, opts)
   }
 
   async *_fetchChunkFeatures(
@@ -305,26 +273,21 @@ export default class BamFile {
     chrId: number,
     min: number,
     max: number,
-    opts: BamOpts,
+    opts: BamOpts = {},
   ) {
-    const { viewAsPairs = false } = opts
-    const feats = []
+    const { viewAsPairs } = opts
+    const feats = [] as BAMFeature[][]
     let done = false
 
-    for (let i = 0; i < chunks.length; i++) {
-      const c = chunks[i]
-      const records = (await this.featureCache.get(
-        c.toString(),
-        {
-          chunk: c,
-          opts,
-        },
+    for (const chunk of chunks) {
+      const records = await this.featureCache.get(
+        chunk.toString(),
+        { chunk, opts },
         opts.signal,
-      )) as BAMFeature[]
+      )
 
-      const recs = []
-      for (let i = 0; i < records.length; i += 1) {
-        const feature = records[i]
+      const recs = [] as BAMFeature[]
+      for (const feature of records) {
         if (feature.seq_id() === chrId) {
           if (feature.get('start') >= max) {
             // past end of range, can stop iterating
@@ -350,36 +313,36 @@ export default class BamFile {
   }
 
   async fetchPairs(chrId: number, feats: BAMFeature[][], opts: BamOpts) {
-    const { pairAcrossChr = false, maxInsertSize = 200000 } = opts
+    const { pairAcrossChr, maxInsertSize = 200000 } = opts
     const unmatedPairs: { [key: string]: boolean } = {}
     const readIds: { [key: string]: number } = {}
     feats.map(ret => {
       const readNames: { [key: string]: number } = {}
-      for (let i = 0; i < ret.length; i++) {
-        const name = ret[i].name()
-        const id = ret[i].id()
+      for (const element of ret) {
+        const name = element.name()
+        const id = element.id()
         if (!readNames[name]) {
           readNames[name] = 0
         }
         readNames[name]++
         readIds[id] = 1
       }
-      entries(readNames).forEach(([k, v]: [string, number]) => {
+      for (const [k, v] of Object.entries(readNames)) {
         if (v === 1) {
           unmatedPairs[k] = true
         }
-      })
+      }
     })
 
     const matePromises: Promise<Chunk[]>[] = []
     feats.map(ret => {
-      for (let i = 0; i < ret.length; i++) {
-        const f = ret[i]
+      for (const f of ret) {
         const name = f.name()
         const start = f.get('start')
         const pnext = f._next_pos()
         const rnext = f._next_refid()
         if (
+          this.index &&
           unmatedPairs[name] &&
           (pairAcrossChr ||
             (rnext === chrId && Math.abs(start - pnext) < maxInsertSize))
@@ -393,50 +356,53 @@ export default class BamFile {
 
     // filter out duplicate chunks (the blocks are lists of chunks, blocks are
     // concatenated, then filter dup chunks)
-    const mateChunks = flat(await Promise.all(matePromises))
-      .sort()
-      .filter(
-        (item, pos, ary) => !pos || item.toString() !== ary[pos - 1].toString(),
-      )
-
-    const mateTotalSize = mateChunks
-      .map(s => s.fetchedSize())
-      .reduce((a, b) => a + b, 0)
-    if (mateTotalSize > this.fetchSizeLimit) {
-      throw new Error(
-        `data size of ${mateTotalSize.toLocaleString()} bytes exceeded fetch size limit of ${this.fetchSizeLimit.toLocaleString()} bytes`,
-      )
+    const map = new Map<string, Chunk>()
+    const res = await Promise.all(matePromises)
+    for (const m of res.flat()) {
+      if (!map.has(m.toString())) {
+        map.set(m.toString(), m)
+      }
     }
-    const mateFeatPromises = mateChunks.map(async c => {
-      const { data, cpositions, dpositions, chunk } = await this._readChunk({
-        chunk: c,
-        opts,
-      })
-      const feats = await this.readBamFeatures(
-        data,
-        cpositions,
-        dpositions,
-        chunk,
-      )
-      const mateRecs = []
-      for (let i = 0; i < feats.length; i += 1) {
-        const feature = feats[i]
-        if (unmatedPairs[feature.get('name')] && !readIds[feature.id()]) {
-          mateRecs.push(feature)
+
+    const mateFeatPromises = await Promise.all(
+      [...map.values()].map(async c => {
+        const { data, cpositions, dpositions, chunk } = await this._readChunk({
+          chunk: c,
+          opts,
+        })
+        const mateRecs = [] as BAMFeature[]
+        for (const feature of await this.readBamFeatures(
+          data,
+          cpositions,
+          dpositions,
+          chunk,
+        )) {
+          if (unmatedPairs[feature.get('name')] && !readIds[feature.id()]) {
+            mateRecs.push(feature)
+          }
         }
-      }
-      return mateRecs
-    })
-    return flat(await Promise.all(mateFeatPromises))
+        return mateRecs
+      }),
+    )
+    return mateFeatPromises.flat()
   }
 
-  async _readChunk({ chunk, opts }: { chunk: Chunk; opts: BaseOpts }) {
-    const size = chunk.fetchedSize()
-    const { buffer, bytesRead } = await this.bam.read(
+  async _readRegion(position: number, size: number, opts: BaseOpts = {}) {
+    const { bytesRead, buffer } = await this.bam.read(
       Buffer.alloc(size),
       0,
       size,
+      position,
+      opts,
+    )
+
+    return buffer.subarray(0, Math.min(bytesRead, size))
+  }
+
+  async _readChunk({ chunk, opts }: { chunk: Chunk; opts: BaseOpts }) {
+    const buffer = await this._readRegion(
       chunk.minv.blockPosition,
+      chunk.fetchedSize(),
       opts,
     )
 
@@ -444,10 +410,7 @@ export default class BamFile {
       buffer: data,
       cpositions,
       dpositions,
-    } = await unzipChunkSlice(
-      buffer.subarray(0, Math.min(bytesRead, size)),
-      chunk,
-    )
+    } = await unzipChunkSlice(buffer, chunk)
     return { data, cpositions, dpositions, chunk }
   }
 
@@ -458,7 +421,7 @@ export default class BamFile {
     chunk: Chunk,
   ) {
     let blockStart = 0
-    const sink = []
+    const sink = [] as BAMFeature[]
     let pos = 0
     let last = +Date.now()
 
@@ -480,29 +443,34 @@ export default class BamFile {
             start: blockStart,
             end: blockEnd,
           },
-          // the below results in an automatically calculated file-offset based ID
-          // if the info for that is available, otherwise crc32 of the features
+          // the below results in an automatically calculated file-offset based
+          // ID if the info for that is available, otherwise crc32 of the
+          // features
           //
-          // cpositions[pos] refers to actual file offset of a bgzip block boundaries
+          // cpositions[pos] refers to actual file offset of a bgzip block
+          // boundaries
           //
-          // we multiply by (1 <<8) in order to make sure each block has a "unique"
-          // address space so that data in that block could never overlap
+          // we multiply by (1 <<8) in order to make sure each block has a
+          // "unique" address space so that data in that block could never
+          // overlap
           //
           // then the blockStart-dpositions is an uncompressed file offset from
-          // that bgzip block boundary, and since the cpositions are multiplied by
-          // (1 << 8) these uncompressed offsets get a unique space
+          // that bgzip block boundary, and since the cpositions are multiplied
+          // by (1 << 8) these uncompressed offsets get a unique space
           //
-          // this has an extra chunk.minv.dataPosition added on because it blockStart
-          // starts at 0 instead of chunk.minv.dataPosition
+          // this has an extra chunk.minv.dataPosition added on because it
+          // blockStart starts at 0 instead of chunk.minv.dataPosition
           //
-          // the +1 is just to avoid any possible uniqueId 0 but this does not realistically happen
-          fileOffset: cpositions
-            ? cpositions[pos] * (1 << 8) +
-              (blockStart - dpositions[pos]) +
-              chunk.minv.dataPosition +
-              1
-            : // must be slice, not subarray for buffer polyfill on web
-              crc32.signed(ba.slice(blockStart, blockEnd)),
+          // the +1 is just to avoid any possible uniqueId 0 but this does not
+          // realistically happen
+          fileOffset:
+            cpositions.length > 0
+              ? cpositions[pos] * (1 << 8) +
+                (blockStart - dpositions[pos]) +
+                chunk.minv.dataPosition +
+                1
+              : // must be slice, not subarray for buffer polyfill on web
+                crc32.signed(ba.slice(blockStart, blockEnd)),
         })
 
         sink.push(feature)
@@ -518,19 +486,22 @@ export default class BamFile {
   }
 
   async hasRefSeq(seqName: string) {
-    const refId = this.chrToIndex && this.chrToIndex[seqName]
-    return this.index.hasRefSeq(refId)
+    const seqId = this.chrToIndex?.[seqName]
+    return seqId === undefined ? false : this.index?.hasRefSeq(seqId)
   }
 
   async lineCount(seqName: string) {
-    const refId = this.chrToIndex && this.chrToIndex[seqName]
-    return this.index.lineCount(refId)
+    const seqId = this.chrToIndex?.[seqName]
+    return seqId === undefined || !this.index ? 0 : this.index.lineCount(seqId)
   }
 
   async indexCov(seqName: string, start?: number, end?: number) {
+    if (!this.index) {
+      return []
+    }
     await this.index.parse()
-    const seqId = this.chrToIndex && this.chrToIndex[seqName]
-    return this.index.indexCov(seqId, start, end)
+    const seqId = this.chrToIndex?.[seqName]
+    return seqId === undefined ? [] : this.index.indexCov(seqId, start, end)
   }
 
   async blocksForRange(
@@ -539,8 +510,13 @@ export default class BamFile {
     end: number,
     opts?: BaseOpts,
   ) {
+    if (!this.index) {
+      return []
+    }
     await this.index.parse()
-    const seqId = this.chrToIndex && this.chrToIndex[seqName]
-    return this.index.blocksForRange(seqId, start, end, opts)
+    const seqId = this.chrToIndex?.[seqName]
+    return seqId === undefined
+      ? []
+      : this.index.blocksForRange(seqId, start, end, opts)
   }
 }

package/src/chunk.ts

@@ -2,28 +2,14 @@ import VirtualOffset from './virtualOffset'
 
 // little class representing a chunk in the index
 export default class Chunk {
-  public minv: VirtualOffset
-  public maxv: VirtualOffset
-  public bin: number
-  public _fetchedSize?: number
+  public buffer?: Buffer
 
-  /**
-   * @param {VirtualOffset} minv
-   * @param {VirtualOffset} maxv
-   * @param {number} bin
-   * @param {number} [fetchedSize]
-   */
   constructor(
-    minv: VirtualOffset,
-    maxv: VirtualOffset,
-    bin: number,
-    fetchedSize = undefined,
-  ) {
-    this.minv = minv
-    this.maxv = maxv
-    this.bin = bin
-    this._fetchedSize = fetchedSize
-  }
+    public minv: VirtualOffset,
+    public maxv: VirtualOffset,
+    public bin: number,
+    public _fetchedSize?: number,
+  ) {}
 
   toUniqueString() {
     return `${this.minv}..${this.maxv} (bin ${