@gmod/tabix 1.5.14 → 1.6.0

package/src/tabixIndexedFile.ts

@@ -1,7 +1,7 @@
 import AbortablePromiseCache from '@gmod/abortable-promise-cache'
 import LRU from 'quick-lru'
 import { Buffer } from 'buffer'
-import { GenericFilehandle, LocalFile } from 'generic-filehandle'
+import { GenericFilehandle, RemoteFile, LocalFile } from 'generic-filehandle'
 import { unzip, unzipChunkSlice } from '@gmod/bgzf-filehandle'
 import { checkAbortSignal } from './util'
 import IndexFile, { Options, IndexData } from './indexFile'
@@ -10,6 +10,11 @@ import Chunk from './chunk'
 import TBI from './tbi'
 import CSI from './csi'
 
+function isASCII(str: string) {
+  // eslint-disable-next-line no-control-regex
+  return /^[\u0000-\u007F]*$/.test(str)
+}
+
 type GetLinesCallback = (line: string, fileOffset: number) => void
 
 const decoder =
@@ -27,50 +32,60 @@ interface ReadChunk {
   dpositions: number[]
 }
 
-function timeout(time: number) {
-  return new Promise(resolve => setTimeout(resolve, time))
-}
 export default class TabixIndexedFile {
   private filehandle: GenericFilehandle
   private index: IndexFile
-  private chunkSizeLimit: number
-  private yieldTime: number
   private renameRefSeq: (n: string) => string
   private chunkCache: AbortablePromiseCache<Chunk, ReadChunk>
 
   /**
    * @param {object} args
+   *
    * @param {string} [args.path]
+   *
    * @param {filehandle} [args.filehandle]
+   *
    * @param {string} [args.tbiPath]
+   *
    * @param {filehandle} [args.tbiFilehandle]
+   *
    * @param {string} [args.csiPath]
+   *
    * @param {filehandle} [args.csiFilehandle]
-   * @param {number} [args.yieldTime] yield to main thread after N milliseconds if reading features is taking a long time to avoid hanging main thread
-   * @param {function} [args.renameRefSeqs] optional function with sig `string => string` to transform
-   * reference sequence names for the purpose of indexing and querying. note that the data that is returned is
-   * not altered, just the names of the reference sequences that are used for querying.
+   *
+   * @param {url} [args.url]
+   *
+   * @param {csiUrl} [args.csiUrl]
+   *
+   * @param {tbiUrl} [args.tbiUrl]
+   *
+   * @param {function} [args.renameRefSeqs] optional function with sig `string
+   * => string` to transform reference sequence names for the purpose of
+   * indexing and querying. note that the data that is returned is not altered,
+   * just the names of the reference sequences that are used for querying.
    */
   constructor({
     path,
     filehandle,
+    url,
     tbiPath,
+    tbiUrl,
     tbiFilehandle,
     csiPath,
+    csiUrl,
     csiFilehandle,
-    yieldTime = 500,
-    chunkSizeLimit = 50000000,
     renameRefSeqs = n => n,
     chunkCacheSize = 5 * 2 ** 20,
   }: {
     path?: string
     filehandle?: GenericFilehandle
+    url?: string
     tbiPath?: string
+    tbiUrl?: string
     tbiFilehandle?: GenericFilehandle
     csiPath?: string
+    csiUrl?: string
     csiFilehandle?: GenericFilehandle
-    yieldTime?: number
-    chunkSizeLimit?: number
     renameRefSeqs?: (n: string) => string
     chunkCacheSize?: number
   }) {
@@ -78,6 +93,8 @@ export default class TabixIndexedFile {
       this.filehandle = filehandle
     } else if (path) {
       this.filehandle = new LocalFile(path)
+    } else if (url) {
+      this.filehandle = new RemoteFile(url)
     } else {
       throw new TypeError('must provide either filehandle or path')
     }
@@ -107,15 +124,25 @@ export default class TabixIndexedFile {
         filehandle: new LocalFile(`${path}.tbi`),
         renameRefSeqs,
       })
+    } else if (csiUrl) {
+      this.index = new CSI({
+        filehandle: new RemoteFile(csiUrl),
+      })
+    } else if (tbiUrl) {
+      this.index = new TBI({
+        filehandle: new RemoteFile(tbiUrl),
+      })
+    } else if (url) {
+      this.index = new TBI({
+        filehandle: new RemoteFile(`${url}.tbi`),
+      })
     } else {
       throw new TypeError(
-        'must provide one of tbiFilehandle, tbiPath, csiFilehandle, or csiPath',
+        'must provide one of tbiFilehandle, tbiPath, csiFilehandle, csiPath, tbiUrl, csiUrl',
       )
     }
 
-    this.chunkSizeLimit = chunkSizeLimit
     this.renameRefSeq = renameRefSeqs
-    this.yieldTime = yieldTime
     this.chunkCache = new AbortablePromiseCache<Chunk, ReadChunk>({
       cache: new LRU({ maxSize: Math.floor(chunkCacheSize / (1 << 16)) }),
       fill: (args: Chunk, signal?: AbortSignal) =>
@@ -125,10 +152,16 @@ export default class TabixIndexedFile {
 
   /**
    * @param refName name of the reference sequence
+   *
    * @param start start of the region (in 0-based half-open coordinates)
+   *
    * @param end end of the region (in 0-based half-open coordinates)
-   * @param opts callback called for each line in the region. can also pass a object param containing obj.lineCallback, obj.signal, etc
-   * @returns promise that is resolved when the whole read is finished, rejected on error
+   *
+   * @param opts callback called for each line in the region. can also pass a
+   * object param containing obj.lineCallback, obj.signal, etc
+   *
+   * @returns promise that is resolved when the whole read is finished,
+   * rejected on error
    */
   async getLines(
     refName: string,
@@ -139,9 +172,7 @@ export default class TabixIndexedFile {
     let signal: AbortSignal | undefined
     let options: Options = {}
     let callback: (line: string, lineOffset: number) => void
-    if (opts === undefined) {
-      throw new TypeError('line callback must be provided')
-    }
+
     if (typeof opts === 'function') {
       callback = opts
     } else {
@@ -149,12 +180,6 @@ export default class TabixIndexedFile {
       callback = opts.lineCallback
       signal = opts.signal
     }
-    if (refName === undefined) {
-      throw new TypeError('must provide a reference sequence name')
-    }
-    if (!callback) {
-      throw new TypeError('line callback must be provided')
-    }
 
     const metadata = await this.index.getMetadata(options)
     checkAbortSignal(signal)
@@ -172,21 +197,8 @@ export default class TabixIndexedFile {
     const chunks = await this.index.blocksForRange(refName, start, end, options)
     checkAbortSignal(signal)
 
-    // check the chunks for any that are over the size limit.  if
-    // any are, don't fetch any of them
-    for (const chunk of chunks) {
-      const size = chunk.fetchedSize()
-      if (size > this.chunkSizeLimit) {
-        throw new Error(
-          `Too much data. Chunk size ${size.toLocaleString()} bytes exceeds chunkSizeLimit of ${this.chunkSizeLimit.toLocaleString()}.`,
-        )
-      }
-    }
-
     // now go through each chunk and parse and filter the lines out of it
-    let last = Date.now()
     for (const c of chunks) {
-      let previousStartCoordinate: number | undefined
       const { buffer, cpositions, dpositions } = await this.chunkCache.get(
         c.toString(),
         c,
@@ -196,16 +208,33 @@ export default class TabixIndexedFile {
       checkAbortSignal(signal)
       let blockStart = 0
       let pos = 0
-      while (blockStart < buffer.length) {
-        const n = buffer.indexOf('\n', blockStart)
-        if (n === -1) {
-          break
+
+      const str = decoder?.decode(buffer) ?? buffer.toString()
+      // fast path, Buffer is just ASCII chars and not gigantor, can be
+      // converted to string and processed directly. if it is not ASCII or
+      // gigantic (chrome max str len is 512Mb), we have to decode line by line
+      const strIsASCII = buffer.length < 500_000_000 && isASCII(str)
+      while (blockStart < str.length) {
+        let line: string
+        let n: number
+        if (strIsASCII) {
+          n = str.indexOf('\n', blockStart)
+          if (n === -1) {
+            break
+          }
+          line = str.slice(blockStart, n)
+        } else {
+          n = buffer.indexOf('\n', blockStart)
+          if (n === -1) {
+            break
+          }
+          const b = buffer.slice(blockStart, n)
+          line = decoder?.decode(b) ?? b.toString()
         }
-        const b = buffer.slice(blockStart, n)
-        const line = decoder?.decode(b) ?? b.toString()
 
+        // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
         if (dpositions) {
-          while (blockStart + c.minv.dataPosition >= dpositions[pos++]) {}
+          while (blockStart + c.minv.dataPosition >= dpositions[pos++]!) {}
           pos--
         }
 
@@ -218,48 +247,31 @@ export default class TabixIndexedFile {
           line,
         )
 
-        // do a small check just to make sure that the lines are really sorted
-        // by start coordinate
-        if (
-          previousStartCoordinate !== undefined &&
-          startCoordinate !== undefined &&
-          previousStartCoordinate > startCoordinate
-        ) {
-          throw new Error(
-            `Lines not sorted by start coordinate (${previousStartCoordinate} > ${startCoordinate}), this file is not usable with Tabix.`,
-          )
-        }
-        previousStartCoordinate = startCoordinate
-
         if (overlaps) {
           callback(
-            line.trim(),
-            // cpositions[pos] refers to actual file offset of a bgzip block boundaries
+            line,
+            // cpositions[pos] refers to actual file offset of a bgzip block
+            // boundaries
             //
-            // we multiply by (1 <<8) in order to make sure each block has a "unique"
-            // address space so that data in that block could never overlap
+            // we multiply by (1 <<8) in order to make sure each block has a
+            // "unique" address space so that data in that block could never
+            // overlap
             //
-            // then the blockStart-dpositions is an uncompressed file offset from
-            // that bgzip block boundary, and since the cpositions are multiplied by
-            // (1 << 8) these uncompressed offsets get a unique space
-            cpositions[pos] * (1 << 8) +
-              (blockStart - dpositions[pos]) +
+            // then the blockStart-dpositions is an uncompressed file offset
+            // from that bgzip block boundary, and since the cpositions are
+            // multiplied by (1 << 8) these uncompressed offsets get a unique
+            // space
+            cpositions[pos]! * (1 << 8) +
+              (blockStart - dpositions[pos]!) +
               c.minv.dataPosition +
               1,
           )
         } else if (startCoordinate !== undefined && startCoordinate >= end) {
-          // the lines were overlapping the region, but now have stopped, so
-          // we must be at the end of the relevant data and we can stop
-          // processing data now
+          // the lines were overlapping the region, but now have stopped, so we
+          // must be at the end of the relevant data and we can stop processing
+          // data now
           return
         }
-
-        // yield if we have emitted beyond the yield limit
-        if (this.yieldTime && last - Date.now() > this.yieldTime) {
-          last = Date.now()
-          checkAbortSignal(signal)
-          await timeout(1)
-        }
         blockStart = n + 1
       }
     }
@@ -270,14 +282,16 @@ export default class TabixIndexedFile {
   }
 
   /**
-   * get a buffer containing the "header" region of
-   * the file, which are the bytes up to the first
-   * non-meta line
+   * get a buffer containing the "header" region of the file, which are the
+   * bytes up to the first non-meta line
    */
   async getHeaderBuffer(opts: Options = {}) {
     const { firstDataLine, metaChar, maxBlockSize } =
       await this.getMetadata(opts)
+
     checkAbortSignal(opts.signal)
+
+    // eslint-disable-next-line @typescript-eslint/restrict-plus-operands
     const maxFetch = (firstDataLine?.blockPosition || 0) + maxBlockSize
     // TODO: what if we don't have a firstDataLine, and the header
     // actually takes up more than one block? this case is not covered here
@@ -299,14 +313,14 @@ export default class TabixIndexedFile {
           lastNewline = i
         }
       }
-      return bytes.slice(0, lastNewline + 1)
+      return bytes.subarray(0, lastNewline + 1)
     }
     return bytes
   }
 
   /**
-   * get a string containing the "header" region of the
-   * file, is the portion up to the first non-meta line
+   * get a string containing the "header" region of the file, is the portion up
+   * to the first non-meta line
    *
    * @returns {Promise} for a string
    */
@@ -316,9 +330,8 @@ export default class TabixIndexedFile {
   }
 
   /**
-   * get an array of reference sequence names, in the order in which
-   * they occur in the file. reference sequence renaming is not applied
-   * to these names.
+   * get an array of reference sequence names, in the order in which they occur
+   * in the file. reference sequence renaming is not applied to these names.
    */
   async getReferenceSequenceNames(opts: Options = {}) {
     const metadata = await this.getMetadata(opts)
@@ -326,12 +339,17 @@ export default class TabixIndexedFile {
   }
 
   /**
-   * @param {object} metadata metadata object from the parsed index,
-   * containing columnNumbers, metaChar, and format
+   * @param {object} metadata metadata object from the parsed index, containing
+   * columnNumbers, metaChar, and format
+   *
    * @param {string} regionRefName
+   *
    * @param {number} regionStart region start coordinate (0-based-half-open)
+   *
    * @param {number} regionEnd region end coordinate (0-based-half-open)
+   *
    * @param {array[string]} line
+   *
    * @returns {object} like `{startCoordinate, overlaps}`. overlaps is boolean,
    * true if line is a data line that overlaps the given region
    */
@@ -364,22 +382,25 @@ export default class TabixIndexedFile {
     }
     const maxColumn = Math.max(ref, start, end)
 
-    // this code is kind of complex, but it is fairly fast.
-    // basically, we want to avoid doing a split, because if the lines are really long
-    // that could lead to us allocating a bunch of extra memory, which is slow
+    // this code is kind of complex, but it is fairly fast. basically, we want
+    // to avoid doing a split, because if the lines are really long that could
+    // lead to us allocating a bunch of extra memory, which is slow
 
     let currentColumnNumber = 1 // cols are numbered starting at 1 in the index metadata
     let currentColumnStart = 0
     let refSeq = ''
     let startCoordinate = -Infinity
-    for (let i = 0; i < line.length + 1; i += 1) {
-      if (line[i] === '\t' || i === line.length) {
+    const l = line.length
+    for (let i = 0; i < l + 1; i++) {
+      if (line[i] === '\t' || i === l) {
         if (currentColumnNumber === ref) {
           if (
             this.renameRefSeq(line.slice(currentColumnStart, i)) !==
             regionRefName
           ) {
-            return { overlaps: false }
+            return {
+              overlaps: false,
+            }
           }
         } else if (currentColumnNumber === start) {
           startCoordinate = parseInt(line.slice(currentColumnStart, i), 10)
@@ -388,12 +409,18 @@ export default class TabixIndexedFile {
             startCoordinate -= 1
           }
           if (startCoordinate >= regionEnd) {
-            return { startCoordinate, overlaps: false }
+            return {
+              startCoordinate,
+              overlaps: false,
+            }
           }
           if (end === 0 || end === start) {
             // if we have no end, we assume the feature is 1 bp long
             if (startCoordinate + 1 <= regionStart) {
-              return { startCoordinate, overlaps: false }
+              return {
+                startCoordinate,
+                overlaps: false,
+              }
             }
           }
         } else if (format === 'VCF' && currentColumnNumber === 4) {
@@ -407,9 +434,11 @@ export default class TabixIndexedFile {
                   refSeq,
                   line.slice(currentColumnStart, i),
                 )
-              : parseInt(line.slice(currentColumnStart, i), 10)
+              : Number.parseInt(line.slice(currentColumnStart, i), 10)
           if (endCoordinate <= regionStart) {
-            return { overlaps: false }
+            return {
+              overlaps: false,
+            }
           }
         }
         currentColumnStart = i + 1
@@ -419,7 +448,10 @@ export default class TabixIndexedFile {
         }
       }
     }
-    return { startCoordinate, overlaps: true }
+    return {
+      startCoordinate,
+      overlaps: true,
+    }
   }
 
   _getVcfEnd(startCoordinate: number, refSeq: string, info: any) {
@@ -450,8 +482,11 @@ export default class TabixIndexedFile {
   }
 
   /**
-   * return the approximate number of data lines in the given reference sequence
+   * return the approximate number of data lines in the given reference
+   * sequence
+   *
    * @param refSeq reference sequence name
+   *
    * @returns number of data lines present on that reference sequence
    */
   async lineCount(refName: string, opts: Options = {}) {
@@ -468,7 +503,7 @@ export default class TabixIndexedFile {
       opts,
     )
 
-    return buffer.slice(0, bytesRead)
+    return buffer.subarray(0, bytesRead)
   }
 
   /**
@@ -476,8 +511,8 @@ export default class TabixIndexedFile {
    * contiguous bgzip blocks) of the file
    */
   async readChunk(c: Chunk, opts: Options = {}) {
-    // fetch the uncompressed data, uncompress carefully a block at a time,
-    // and stop when done
+    // fetch the uncompressed data, uncompress carefully a block at a time, and
+    // stop when done
 
     const data = await this._readRegion(
       c.minv.blockPosition,

package/src/tbi.ts

@@ -10,7 +10,8 @@ const TBI_MAGIC = 21578324 // TBI\1
 const TAD_LIDX_SHIFT = 14
 
 /**
- * calculate the list of bins that may overlap with region [beg,end) (zero-based half-open)
+ * calculate the list of bins that may overlap with region [beg,end)
+ * (zero-based half-open)
  */
 function reg2bins(beg: number, end: number) {
   beg += 1 // < convert to 1-based closed
@@ -22,16 +23,16 @@ function reg2bins(beg: number, end: number) {
     [73 + (beg >> 20), 73 + (end >> 20)],
     [585 + (beg >> 17), 585 + (end >> 17)],
     [4681 + (beg >> 14), 4681 + (end >> 14)],
-  ]
+  ] as const
 }
 
 export default class TabixIndex extends IndexFile {
   async lineCount(refName: string, opts: Options = {}) {
     const indexData = await this.parse(opts)
-    if (!indexData) {
+    const refId = indexData.refNameToId[refName]
+    if (refId === undefined) {
       return -1
     }
-    const refId = indexData.refNameToId[refName]
     const idx = indexData.indices[refId]
     if (!idx) {
       return -1
@@ -194,10 +195,10 @@ export default class TabixIndex extends IndexFile {
     }
 
     const indexData = await this.parse(opts)
-    if (!indexData) {
+    const refId = indexData.refNameToId[refName]
+    if (refId === undefined) {
       return []
     }
-    const refId = indexData.refNameToId[refName]
     const ba = indexData.indices[refId]
     if (!ba) {
       return []

package/src/util.ts

@@ -15,14 +15,15 @@ class AbortError extends Error {
   public code: string | undefined
 }
 /**
- * Properly check if the given AbortSignal is aborted.
- * Per the standard, if the signal reads as aborted,
- * this function throws either a DOMException AbortError, or a regular error
- * with a `code` attribute set to `ERR_ABORTED`.
+ * Properly check if the given AbortSignal is aborted. Per the standard, if the
+ * signal reads as aborted, this function throws either a DOMException
+ * AbortError, or a regular error with a `code` attribute set to `ERR_ABORTED`.
  *
  * For convenience, passing `undefined` is a no-op
  *
- * @param {AbortSignal} [signal] an AbortSignal, or anything with an `aborted` attribute
+ * @param {AbortSignal} [signal] an AbortSignal, or anything with an `aborted`
+ * attribute
+ *
  * @returns nothing
  */
 export function checkAbortSignal(signal?: AbortSignal) {
@@ -59,7 +60,7 @@ export function canMergeBlocks(chunk1: Chunk, chunk2: Chunk) {
   )
 }
 
-export function optimizeChunks(chunks: Chunk[], lowest: VirtualOffset) {
+export function optimizeChunks(chunks: Chunk[], lowest?: VirtualOffset) {
   const mergedChunks: Chunk[] = []
   let lastChunk: Chunk | null = null
 

package/src/virtualOffset.ts

@@ -16,20 +16,6 @@ export default class VirtualOffset {
       this.blockPosition - b.blockPosition || this.dataPosition - b.dataPosition
     )
   }
-
-  static min(...args: VirtualOffset[]) {
-    let min
-    let i = 0
-    for (; !min; i += 1) {
-      min = args[i]
-    }
-    for (; i < args.length; i += 1) {
-      if (min.compareTo(args[i]) > 0) {
-        min = args[i]
-      }
-    }
-    return min
-  }
 }
 export function fromBytes(bytes: Buffer, offset = 0, bigendian = false) {
   if (bigendian) {
@@ -37,12 +23,12 @@ export function fromBytes(bytes: Buffer, offset = 0, bigendian = false) {
   }
 
   return new VirtualOffset(
-    bytes[offset + 7] * 0x10000000000 +
-      bytes[offset + 6] * 0x100000000 +
-      bytes[offset + 5] * 0x1000000 +
-      bytes[offset + 4] * 0x10000 +
-      bytes[offset + 3] * 0x100 +
-      bytes[offset + 2],
-    (bytes[offset + 1] << 8) | bytes[offset],
+    bytes[offset + 7]! * 0x10000000000 +
+      bytes[offset + 6]! * 0x100000000 +
+      bytes[offset + 5]! * 0x1000000 +
+      bytes[offset + 4]! * 0x10000 +
+      bytes[offset + 3]! * 0x100 +
+      bytes[offset + 2]!,
+    (bytes[offset + 1]! << 8) | bytes[offset]!,
   )
 }