@gmod/cram 3.0.7 → 4.0.0

package/src/craiIndex.ts

@@ -1,8 +1,9 @@
-import { unzip } from './unzip'
-import { open } from './io'
-import { CramMalformedError } from './errors'
 import { CramFileSource } from './cramFile/file'
-import { Filehandle } from './cramFile/filehandle'
+import { CramMalformedError } from './errors'
+import { open } from './io'
+import { unzip } from './unzip'
+
+import type { GenericFilehandle } from 'generic-filehandle2'
 
 const BAI_MAGIC = 21_578_050 // BAI\1
 
@@ -33,11 +34,8 @@ function addRecordToIndex(index: ParsedIndex, record: number[]) {
   })
 }
 
-function maybeUnzip(data: Buffer) {
-  if (data[0] === 31 && data[1] === 139) {
-    return unzip(data)
-  }
-  return data
+function maybeUnzip(data: Uint8Array) {
+  return data[0] === 31 && data[1] === 139 ? unzip(data) : data
 }
 
 export default class CraiIndex {
@@ -50,10 +48,12 @@ export default class CraiIndex {
   // 4. Container start byte position in the file
   // 5. Slice start byte position in the container data (‘blocks’)
   // 6. Slice size in bytes
-  // Each line represents a slice in the CRAM file. Please note that all slices must be listed in index file.
+  //
+  // Each line represents a slice in the CRAM file. Please note that all slices
+  // must be listed in index file.
   private parseIndexP?: Promise<ParsedIndex>
 
-  private filehandle: Filehandle
+  private filehandle: GenericFilehandle
 
   /**
    *
@@ -69,18 +69,19 @@ export default class CraiIndex {
   async parseIndex() {
     const index: ParsedIndex = {}
     const uncompressedBuffer = maybeUnzip(await this.filehandle.readFile())
+    const dataView = new DataView(uncompressedBuffer.buffer)
     if (
       uncompressedBuffer.length > 4 &&
-      uncompressedBuffer.readUInt32LE(0) === BAI_MAGIC
+      dataView.getUint32(0, true) === BAI_MAGIC
     ) {
       throw new CramMalformedError(
         'invalid .crai index file. note: file appears to be a .bai index. this is technically legal but please open a github issue if you need support',
       )
     }
-    // interpret the text as regular ascii, since it is
-    // supposed to be only digits and whitespace characters
-    // this is written in a deliberately low-level fashion for performance,
-    // because some .crai files can be pretty large.
+    // interpret the text as regular ascii, since it is supposed to be only
+    // digits and whitespace characters this is written in a deliberately
+    // low-level fashion for performance, because some .crai files can be
+    // pretty large.
     let currentRecord: number[] = []
     let currentString = ''
     for (const charCode of uncompressedBuffer) {

package/src/cramFile/codecs/_base.ts

@@ -1,5 +1,5 @@
-import CramSlice from '../slice'
 import { CramFileBlock } from '../file'
+import CramSlice from '../slice'
 import { DataType } from './dataSeriesTypes'
 
 export interface DataTypeMapping {

package/src/cramFile/codecs/beta.ts

@@ -1,9 +1,9 @@
-import { CramUnimplementedError } from '../../errors'
 import CramCodec, { Cursors } from './_base'
 import { getBits } from './getBits'
-import CramSlice from '../slice'
-import { CramFileBlock } from '../file'
+import { CramUnimplementedError } from '../../errors'
 import { BetaEncoding } from '../encoding'
+import { CramFileBlock } from '../file'
+import CramSlice from '../slice'
 
 export default class BetaCodec extends CramCodec<
   'int',

package/src/cramFile/codecs/byteArrayLength.ts

@@ -1,7 +1,7 @@
 import CramCodec, { Cursors } from './_base'
 import { ByteArrayLengthEncoding, CramEncoding } from '../encoding'
-import CramSlice from '../slice'
 import { CramFileBlock } from '../file'
+import CramSlice from '../slice'
 import { DataType } from './dataSeriesTypes'
 import { tinyMemoize } from '../util'
 

package/src/cramFile/codecs/byteArrayStop.ts

@@ -1,9 +1,8 @@
-import { CramMalformedError } from '../../errors'
-
 import CramCodec, { Cursor, Cursors } from './_base'
-import CramSlice from '../slice'
-import { CramFileBlock } from '../file'
+import { CramMalformedError } from '../../errors'
 import { ByteArrayStopCramEncoding } from '../encoding'
+import { CramFileBlock } from '../file'
+import CramSlice from '../slice'
 import { CramBufferOverrunError } from './getBits'
 
 export default class ByteArrayStopCodec extends CramCodec<

package/src/cramFile/codecs/external.ts

@@ -1,8 +1,8 @@
-import { CramMalformedError, CramUnimplementedError } from '../../errors'
 import CramCodec, { Cursor, Cursors } from './_base'
-import { parseItf8 } from '../util'
-import CramSlice from '../slice'
+import { CramMalformedError, CramUnimplementedError } from '../../errors'
 import { CramFileBlock } from '../file'
+import CramSlice from '../slice'
+import { parseItf8 } from '../util'
 import { CramBufferOverrunError } from './getBits'
 import { ExternalCramEncoding } from '../encoding'
 

package/src/cramFile/codecs/gamma.ts

@@ -1,9 +1,9 @@
-import { CramUnimplementedError } from '../../errors'
 import CramCodec, { Cursors } from './_base'
 import { getBits } from './getBits'
+import { CramUnimplementedError } from '../../errors'
 import { GammaEncoding } from '../encoding'
-import CramSlice from '../slice'
 import { CramFileBlock } from '../file'
+import CramSlice from '../slice'
 
 export default class GammaCodec extends CramCodec<
   'int',

package/src/cramFile/codecs/getBits.ts

@@ -1,7 +1,7 @@
 export class CramBufferOverrunError extends Error {}
 
 export function getBits(
-  data: Buffer,
+  data: Uint8Array,
   cursor: { bytePosition: number; bitPosition: number },
   numBits: number,
 ) {

package/src/cramFile/codecs/huffman.ts

@@ -1,10 +1,9 @@
-import { CramMalformedError } from '../../errors'
 import CramCodec, { Cursor, Cursors } from './_base'
 import { getBits } from './getBits'
+import { CramMalformedError } from '../../errors'
 import { HuffmanEncoding } from '../encoding'
-
-import CramSlice from '../slice'
 import { CramFileBlock } from '../file'
+import CramSlice from '../slice'
 
 function numberOfSetBits(ii: number) {
   let i = (ii - (ii >> 1)) & 0x55555555

package/src/cramFile/codecs/index.ts

@@ -1,15 +1,14 @@
-import { CramUnimplementedError } from '../../errors'
-
-import HuffmanIntCodec from './huffman'
-import ExternalCodec from './external'
-import ByteArrayStopCodec from './byteArrayStop'
-import ByteArrayLengthCodec from './byteArrayLength'
+import CramCodec from './_base'
 import BetaCodec from './beta'
+import ByteArrayLengthCodec from './byteArrayLength'
+import ByteArrayStopCodec from './byteArrayStop'
+import { DataType } from './dataSeriesTypes'
+import ExternalCodec from './external'
 import GammaCodec from './gamma'
+import HuffmanIntCodec from './huffman'
 import SubexpCodec from './subexp'
-import CramCodec from './_base'
+import { CramUnimplementedError } from '../../errors'
 import { CramEncoding } from '../encoding'
-import { DataType } from './dataSeriesTypes'
 
 const codecClasses = {
   1: ExternalCodec,

package/src/cramFile/codecs/subexp.ts

@@ -1,9 +1,9 @@
-import { CramUnimplementedError } from '../../errors'
 import CramCodec, { Cursors } from './_base'
 import { getBits } from './getBits'
-import CramSlice from '../slice'
-import { CramFileBlock } from '../file'
+import { CramUnimplementedError } from '../../errors'
 import { SubexpEncoding } from '../encoding'
+import { CramFileBlock } from '../file'
+import CramSlice from '../slice'
 
 export default class SubexpCodec extends CramCodec<
   'int',

package/src/cramFile/container/compressionScheme.ts

@@ -1,12 +1,12 @@
+import { CramMalformedError } from '../../errors'
 import { instantiateCodec } from '../codecs'
 import CramCodec from '../codecs/_base'
-import { CramCompressionHeader } from '../sectionParsers'
-import { CramEncoding } from '../encoding'
-import { CramMalformedError } from '../../errors'
 import {
   DataSeriesEncodingKey,
   DataSeriesEncodingMap,
 } from '../codecs/dataSeriesTypes'
+import { CramEncoding } from '../encoding'
+import { CramCompressionHeader } from '../sectionParsers'
 
 // the hardcoded data type to be decoded for each core
 // data field

package/src/cramFile/container/index.ts

@@ -1,8 +1,7 @@
-import { Buffer } from 'buffer'
 import { CramMalformedError } from '../../errors'
 // locals
-import { itf8Size, parseItem, tinyMemoize } from '../util'
 import CramSlice from '../slice'
+import { itf8Size, parseItem, tinyMemoize } from '../util'
 import CramContainerCompressionScheme from './compressionScheme'
 import CramFile from '../file'
 import { getSectionParsers } from '../sectionParsers'
@@ -88,8 +87,10 @@ export default class CramContainer {
 
     // parse the container header. do it in 2 pieces because you cannot tell
     // how much to buffer until you read numLandmarks
-    const bytes1 = Buffer.allocUnsafe(cramContainerHeader1.maxLength)
-    await this.file.read(bytes1, 0, cramContainerHeader1.maxLength, position)
+    const bytes1 = await this.file.read(
+      cramContainerHeader1.maxLength,
+      position,
+    )
     const header1 = parseItem(bytes1, cramContainerHeader1.parser)
     const numLandmarksSize = itf8Size(header1.numLandmarks)
     if (position + header1.length >= fileSize) {
@@ -99,12 +100,8 @@ export default class CramContainer {
       )
       return undefined
     }
-    const bytes2 = Buffer.allocUnsafe(
-      cramContainerHeader2.maxLength(header1.numLandmarks),
-    )
-    await this.file.read(
-      bytes2,
-      0,
+
+    const bytes2 = await this.file.read(
       cramContainerHeader2.maxLength(header1.numLandmarks),
       position + header1._size - numLandmarksSize,
     )

package/src/cramFile/file.ts

@@ -1,27 +1,27 @@
-import { Buffer } from 'buffer'
-import crc32 from 'crc/crc32'
-import QuickLRU from 'quick-lru'
-import htscodecs from '../htscodecs'
 import bzip2 from 'bzip2'
+import crc32 from 'crc/calculators/crc32'
+import QuickLRU from 'quick-lru'
 import { XzReadableStream } from 'xz-decompress'
+
 import { CramMalformedError, CramUnimplementedError } from '../errors'
-// locals
-import { unzip } from '../unzip'
+import htscodecs from '../htscodecs'
+import { open } from '../io'
 import ransuncompress from '../rans'
+import { parseHeaderText } from '../sam'
+import { unzip } from '../unzip'
+import CramContainer from './container'
+import CramRecord from './record'
 import {
   BlockHeader,
   CompressionMethod,
   cramFileDefinition,
   getSectionParsers,
 } from './sectionParsers'
-import CramContainer from './container'
-import CramRecord from './record'
-import { open } from '../io'
-import { parseItem, tinyMemoize } from './util'
-import { parseHeaderText } from '../sam'
-import { Filehandle } from './filehandle'
+import { concatUint8Array, parseItem, tinyMemoize } from './util'
+
+import type { GenericFilehandle } from 'generic-filehandle2'
 
-function bufferToStream(buf: Buffer) {
+function bufferToStream(buf: Uint8Array) {
   return new ReadableStream({
     start(controller) {
       controller.enqueue(buf)
@@ -45,7 +45,7 @@ function getEndianness() {
 }
 
 export interface CramFileSource {
-  filehandle?: Filehandle
+  filehandle?: GenericFilehandle
   url?: string
   path?: string
 }
@@ -66,12 +66,12 @@ export type CramFileBlock = BlockHeader & {
   _endPosition: number
   contentPosition: number
   _size: number
-  content: Buffer
+  content: Uint8Array
   crc32?: number
 }
 
 export default class CramFile {
-  private file: Filehandle
+  private file: GenericFilehandle
   public validateChecksums: boolean
   public fetchReferenceSequenceCallback?: SeqFetch
   public options: {
@@ -101,21 +101,20 @@ export default class CramFile {
     }
   }
 
-  // can just read this object like a filehandle
-  read(buffer: Buffer, offset: number, length: number, position: number) {
-    return this.file.read(buffer, offset, length, position)
-  }
-
   // can just stat this object like a filehandle
   stat() {
     return this.file.stat()
   }
 
+  // can just stat this object like a filehandle
+  read(length: number, position: number) {
+    return this.file.read(length, position)
+  }
+
   // memoized
   async getDefinition() {
     const { maxLength, parser } = cramFileDefinition()
-    const headbytes = Buffer.allocUnsafe(maxLength)
-    await this.file.read(headbytes, 0, maxLength, 0)
+    const headbytes = await this.file.read(maxLength, 0)
     const definition = parser(headbytes).value
     if (definition.majorVersion !== 2 && definition.majorVersion !== 3) {
       throw new CramUnimplementedError(
@@ -135,13 +134,18 @@ export default class CramFile {
     const firstBlock = await firstContainer.getFirstBlock()
     if (firstBlock === undefined) {
       return parseHeaderText('')
+    } else {
+      const content = firstBlock.content
+      const dataView = new DataView(content.buffer)
+      const headerLength = dataView.getInt32(0, true)
+      const textStart = 4
+      const decoder = new TextDecoder('utf8')
+      const text = decoder.decode(
+        content.subarray(textStart, textStart + headerLength),
+      )
+      this.header = text
+      return parseHeaderText(text)
     }
-    const content = firstBlock.content
-    const headerLength = content.readInt32LE(0)
-    const textStart = 4
-    const text = content.toString('utf8', textStart, textStart + headerLength)
-    this.header = text
-    return parseHeaderText(text)
   }
 
   async getHeaderText() {
@@ -200,9 +204,11 @@ export default class CramFile {
     recordedCrc32: number,
     description: string,
   ) {
-    const b = Buffer.allocUnsafe(length)
-    await this.file.read(b, 0, length, position)
-    const calculatedCrc32 = crc32.unsigned(b)
+    const b = await this.file.read(length, position)
+    // this shift >>> 0 is equivalent to crc32(b).unsigned but uses the
+    // internal calculator of crc32 to avoid accidentally importing buffer
+    // https://github.com/alexgorbatchev/crc/blob/31fc3853e417b5fb5ec83335428805842575f699/src/define_crc.ts#L5
+    const calculatedCrc32 = crc32(b) >>> 0
     if (calculatedCrc32 !== recordedCrc32) {
       throw new CramMalformedError(
         `crc mismatch in ${description}: recorded CRC32 = ${recordedCrc32}, but calculated CRC32 = ${calculatedCrc32}`,
@@ -264,21 +270,23 @@ export default class CramFile {
       return undefined
     }
 
-    const buffer = Buffer.allocUnsafe(cramBlockHeader.maxLength)
-    await this.file.read(buffer, 0, cramBlockHeader.maxLength, position)
+    const buffer = await this.file.read(cramBlockHeader.maxLength, position)
     return parseItem(buffer, cramBlockHeader.parser, 0, position)
   }
 
   async _parseSection<T>(
     section: {
       maxLength: number
-      parser: (buffer: Buffer, offset: number) => { offset: number; value: T }
+      parser: (
+        buffer: Uint8Array,
+        offset: number,
+      ) => { offset: number; value: T }
     },
     position: number,
     size = section.maxLength,
-    preReadBuffer?: Buffer,
+    preReadBuffer?: Uint8Array,
   ) {
-    let buffer: Buffer
+    let buffer: Uint8Array
     if (preReadBuffer) {
       buffer = preReadBuffer
     } else {
@@ -286,8 +294,7 @@ export default class CramFile {
       if (position + size >= fileSize) {
         return undefined
       }
-      buffer = Buffer.allocUnsafe(size)
-      await this.file.read(buffer, 0, size, position)
+      buffer = await this.file.read(size, position)
     }
     const data = parseItem(buffer, section.parser, 0, position)
     if (data._size !== size) {
@@ -300,43 +307,43 @@ export default class CramFile {
 
   async _uncompress(
     compressionMethod: CompressionMethod,
-    inputBuffer: Buffer,
-    outputBuffer: Buffer,
+    inputBuffer: Uint8Array,
+    uncompressedSize: number,
   ) {
     if (compressionMethod === 'gzip') {
-      const result = unzip(inputBuffer)
-      result.copy(outputBuffer)
+      return unzip(inputBuffer)
     } else if (compressionMethod === 'bzip2') {
       const bits = bzip2.array(inputBuffer)
       let size = bzip2.header(bits)
-      let j = 0
       let chunk: Uint8Array | -1
+      const chunks = []
       do {
         chunk = bzip2.decompress(bits, size)
         if (chunk !== -1) {
-          Buffer.from(chunk).copy(outputBuffer, j)
-          j += chunk.length
+          chunks.push(chunk)
           size -= chunk.length
         }
       } while (chunk !== -1)
+      return concatUint8Array(chunks)
     } else if (compressionMethod === 'lzma') {
       const decompressedResponse = new Response(
         new XzReadableStream(bufferToStream(inputBuffer)),
       )
-      const ret = Buffer.from(await decompressedResponse.arrayBuffer())
-      ret.copy(outputBuffer)
+      return new Uint8Array(await decompressedResponse.arrayBuffer())
     } else if (compressionMethod === 'rans') {
+      const outputBuffer = new Uint8Array(uncompressedSize)
       ransuncompress(inputBuffer, outputBuffer)
+      return outputBuffer
       // htscodecs r4x8 is slower, but compatible.
       // htscodecs.r4x8_uncompress(inputBuffer, outputBuffer);
     } else if (compressionMethod === 'rans4x16') {
-      htscodecs.r4x16_uncompress(inputBuffer, outputBuffer)
+      return htscodecs.r4x16_uncompress(inputBuffer)
     } else if (compressionMethod === 'arith') {
-      htscodecs.arith_uncompress(inputBuffer, outputBuffer)
+      return htscodecs.arith_uncompress(inputBuffer)
     } else if (compressionMethod === 'fqzcomp') {
-      htscodecs.fqzcomp_uncompress(inputBuffer, outputBuffer)
+      return htscodecs.fqzcomp_uncompress(inputBuffer)
     } else if (compressionMethod === 'tok3') {
-      htscodecs.tok3_uncompress(inputBuffer, outputBuffer)
+      return htscodecs.tok3_uncompress(inputBuffer)
     } else {
       throw new CramUnimplementedError(
         `${compressionMethod} decompression not yet implemented`,
@@ -353,7 +360,18 @@ export default class CramFile {
     }
     const blockContentPosition = blockHeader._endPosition
 
-    const uncompressedData = Buffer.allocUnsafe(blockHeader.uncompressedSize)
+    const d = await this.file.read(
+      blockHeader.compressedSize,
+      blockContentPosition,
+    )
+    const uncompressedData =
+      blockHeader.compressionMethod !== 'raw'
+        ? await this._uncompress(
+            blockHeader.compressionMethod,
+            d,
+            blockHeader.uncompressedSize,
+          )
+        : d
 
     const block: CramFileBlock = {
       ...blockHeader,
@@ -361,30 +379,6 @@ export default class CramFile {
       contentPosition: blockContentPosition,
       content: uncompressedData,
     }
-
-    if (blockHeader.compressionMethod !== 'raw') {
-      const compressedData = Buffer.allocUnsafe(blockHeader.compressedSize)
-      await this.read(
-        compressedData,
-        0,
-        blockHeader.compressedSize,
-        blockContentPosition,
-      )
-
-      await this._uncompress(
-        blockHeader.compressionMethod,
-        compressedData,
-        uncompressedData,
-      )
-    } else {
-      await this.read(
-        uncompressedData,
-        0,
-        blockHeader.uncompressedSize,
-        blockContentPosition,
-      )
-    }
-
     if (majorVersion >= 3) {
       // parse the crc32
       const crc = await this._parseSection(

package/src/cramFile/record.ts

@@ -1,5 +1,6 @@
 import Constants from './constants'
 import CramContainerCompressionScheme from './container/compressionScheme'
+
 import type decodeRecord from './slice/decodeRecord'
 
 export interface RefRegion {
@@ -13,7 +14,6 @@ export interface ReadFeature {
   pos: number
   refPos: number
   data: any
-
   ref?: string
   sub?: string
 }
@@ -458,8 +458,8 @@ export default class CramRecord {
     compressionScheme: CramContainerCompressionScheme,
   ) {
     if (this.readFeatures) {
-      // use the reference bases to decode the bases
-      // substituted in each base substitution
+      // use the reference bases to decode the bases substituted in each base
+      // substitution
       this.readFeatures.forEach(readFeature => {
         if (readFeature.code === 'X') {
           decodeBaseSubstitution(