@gmod/cram 3.0.6 → 4.0.0

package/src/cramFile/file.ts

@@ -1,27 +1,27 @@
-import { Buffer } from 'buffer'
-import crc32 from 'crc/crc32'
-import QuickLRU from 'quick-lru'
-import htscodecs from '@jkbonfield/htscodecs'
 import bzip2 from 'bzip2'
+import crc32 from 'crc/calculators/crc32'
+import QuickLRU from 'quick-lru'
 import { XzReadableStream } from 'xz-decompress'
+
 import { CramMalformedError, CramUnimplementedError } from '../errors'
-// locals
-import { unzip } from '../unzip'
+import htscodecs from '../htscodecs'
+import { open } from '../io'
 import ransuncompress from '../rans'
+import { parseHeaderText } from '../sam'
+import { unzip } from '../unzip'
+import CramContainer from './container'
+import CramRecord from './record'
 import {
   BlockHeader,
   CompressionMethod,
   cramFileDefinition,
   getSectionParsers,
 } from './sectionParsers'
-import CramContainer from './container'
-import CramRecord from './record'
-import { open } from '../io'
-import { parseItem, tinyMemoize } from './util'
-import { parseHeaderText } from '../sam'
-import { Filehandle } from './filehandle'
+import { concatUint8Array, parseItem, tinyMemoize } from './util'
+
+import type { GenericFilehandle } from 'generic-filehandle2'
 
-function bufferToStream(buf: Buffer) {
+function bufferToStream(buf: Uint8Array) {
   return new ReadableStream({
     start(controller) {
       controller.enqueue(buf)
@@ -45,7 +45,7 @@ function getEndianness() {
 }
 
 export interface CramFileSource {
-  filehandle?: Filehandle
+  filehandle?: GenericFilehandle
   url?: string
   path?: string
 }
@@ -66,12 +66,12 @@ export type CramFileBlock = BlockHeader & {
   _endPosition: number
   contentPosition: number
   _size: number
-  content: Buffer
+  content: Uint8Array
   crc32?: number
 }
 
 export default class CramFile {
-  private file: Filehandle
+  private file: GenericFilehandle
   public validateChecksums: boolean
   public fetchReferenceSequenceCallback?: SeqFetch
   public options: {
@@ -101,21 +101,20 @@ export default class CramFile {
     }
   }
 
-  // can just read this object like a filehandle
-  read(buffer: Buffer, offset: number, length: number, position: number) {
-    return this.file.read(buffer, offset, length, position)
-  }
-
   // can just stat this object like a filehandle
   stat() {
     return this.file.stat()
   }
 
+  // can just stat this object like a filehandle
+  read(length: number, position: number) {
+    return this.file.read(length, position)
+  }
+
   // memoized
   async getDefinition() {
     const { maxLength, parser } = cramFileDefinition()
-    const headbytes = Buffer.allocUnsafe(maxLength)
-    await this.file.read(headbytes, 0, maxLength, 0)
+    const headbytes = await this.file.read(maxLength, 0)
     const definition = parser(headbytes).value
     if (definition.majorVersion !== 2 && definition.majorVersion !== 3) {
       throw new CramUnimplementedError(
@@ -135,17 +134,18 @@ export default class CramFile {
     const firstBlock = await firstContainer.getFirstBlock()
     if (firstBlock === undefined) {
       return parseHeaderText('')
+    } else {
+      const content = firstBlock.content
+      const dataView = new DataView(content.buffer)
+      const headerLength = dataView.getInt32(0, true)
+      const textStart = 4
+      const decoder = new TextDecoder('utf8')
+      const text = decoder.decode(
+        content.subarray(textStart, textStart + headerLength),
+      )
+      this.header = text
+      return parseHeaderText(text)
     }
-    const content = firstBlock.content
-    // find the end of the trailing zeros in the header text
-    const headerLength = content.readInt32LE(0)
-    const textStart = 4
-    // let textEnd = content.length - 1
-    // while (textEnd >= textStart && !content[textEnd]) textEnd -= 1
-    // trim off the trailing zeros
-    const text = content.toString('utf8', textStart, textStart + headerLength)
-    this.header = text
-    return parseHeaderText(text)
   }
 
   async getHeaderText() {
@@ -189,7 +189,8 @@ export default class CramFile {
           position = block._endPosition
         }
       } else {
-        // otherwise, just traverse to the next container using the container's length
+        // otherwise, just traverse to the next container using the container's
+        // length
         position += currentHeader._size + currentHeader.length
       }
     }
@@ -203,9 +204,11 @@ export default class CramFile {
     recordedCrc32: number,
     description: string,
   ) {
-    const b = Buffer.allocUnsafe(length)
-    await this.file.read(b, 0, length, position)
-    const calculatedCrc32 = crc32.unsigned(b)
+    const b = await this.file.read(length, position)
+    // this shift >>> 0 is equivalent to crc32(b).unsigned but uses the
+    // internal calculator of crc32 to avoid accidentally importing buffer
+    // https://github.com/alexgorbatchev/crc/blob/31fc3853e417b5fb5ec83335428805842575f699/src/define_crc.ts#L5
+    const calculatedCrc32 = crc32(b) >>> 0
     if (calculatedCrc32 !== recordedCrc32) {
       throw new CramMalformedError(
         `crc mismatch in ${description}: recorded CRC32 = ${recordedCrc32}, but calculated CRC32 = ${calculatedCrc32}`,
@@ -230,9 +233,9 @@ export default class CramFile {
       if (!currentHeader) {
         break
       }
-      // if this is the first container, read all the blocks in the
-      // container, because we cannot trust the container
-      // header's given length due to a bug somewhere in htslib
+      // if this is the first container, read all the blocks in the container,
+      // because we cannot trust the container header's given length due to a
+      // bug somewhere in htslib
       if (containerCount === 0) {
         position = currentHeader._endPosition
         for (let j = 0; j < currentHeader.numBlocks; j++) {
@@ -243,7 +246,8 @@ export default class CramFile {
           position = block._endPosition
         }
       } else {
-        // otherwise, just traverse to the next container using the container's length
+        // otherwise, just traverse to the next container using the container's
+        // length
         position += currentHeader._size + currentHeader.length
       }
       containerCount += 1
@@ -266,21 +270,23 @@ export default class CramFile {
       return undefined
     }
 
-    const buffer = Buffer.allocUnsafe(cramBlockHeader.maxLength)
-    await this.file.read(buffer, 0, cramBlockHeader.maxLength, position)
+    const buffer = await this.file.read(cramBlockHeader.maxLength, position)
     return parseItem(buffer, cramBlockHeader.parser, 0, position)
   }
 
   async _parseSection<T>(
     section: {
       maxLength: number
-      parser: (buffer: Buffer, offset: number) => { offset: number; value: T }
+      parser: (
+        buffer: Uint8Array,
+        offset: number,
+      ) => { offset: number; value: T }
     },
     position: number,
     size = section.maxLength,
-    preReadBuffer?: Buffer,
+    preReadBuffer?: Uint8Array,
   ) {
-    let buffer: Buffer
+    let buffer: Uint8Array
     if (preReadBuffer) {
       buffer = preReadBuffer
     } else {
@@ -288,8 +294,7 @@ export default class CramFile {
       if (position + size >= fileSize) {
         return undefined
       }
-      buffer = Buffer.allocUnsafe(size)
-      await this.file.read(buffer, 0, size, position)
+      buffer = await this.file.read(size, position)
     }
     const data = parseItem(buffer, section.parser, 0, position)
     if (data._size !== size) {
@@ -302,43 +307,43 @@ export default class CramFile {
 
   async _uncompress(
     compressionMethod: CompressionMethod,
-    inputBuffer: Buffer,
-    outputBuffer: Buffer,
+    inputBuffer: Uint8Array,
+    uncompressedSize: number,
   ) {
     if (compressionMethod === 'gzip') {
-      const result = unzip(inputBuffer)
-      result.copy(outputBuffer)
+      return unzip(inputBuffer)
     } else if (compressionMethod === 'bzip2') {
       const bits = bzip2.array(inputBuffer)
       let size = bzip2.header(bits)
-      let j = 0
       let chunk: Uint8Array | -1
+      const chunks = []
       do {
         chunk = bzip2.decompress(bits, size)
         if (chunk !== -1) {
-          Buffer.from(chunk).copy(outputBuffer, j)
-          j += chunk.length
+          chunks.push(chunk)
           size -= chunk.length
         }
       } while (chunk !== -1)
+      return concatUint8Array(chunks)
     } else if (compressionMethod === 'lzma') {
       const decompressedResponse = new Response(
         new XzReadableStream(bufferToStream(inputBuffer)),
       )
-      const ret = Buffer.from(await decompressedResponse.arrayBuffer())
-      ret.copy(outputBuffer)
+      return new Uint8Array(await decompressedResponse.arrayBuffer())
     } else if (compressionMethod === 'rans') {
+      const outputBuffer = new Uint8Array(uncompressedSize)
       ransuncompress(inputBuffer, outputBuffer)
+      return outputBuffer
       // htscodecs r4x8 is slower, but compatible.
       // htscodecs.r4x8_uncompress(inputBuffer, outputBuffer);
     } else if (compressionMethod === 'rans4x16') {
-      htscodecs.r4x16_uncompress(inputBuffer, outputBuffer)
+      return htscodecs.r4x16_uncompress(inputBuffer)
     } else if (compressionMethod === 'arith') {
-      htscodecs.arith_uncompress(inputBuffer, outputBuffer)
+      return htscodecs.arith_uncompress(inputBuffer)
     } else if (compressionMethod === 'fqzcomp') {
-      htscodecs.fqzcomp_uncompress(inputBuffer, outputBuffer)
+      return htscodecs.fqzcomp_uncompress(inputBuffer)
     } else if (compressionMethod === 'tok3') {
-      htscodecs.tok3_uncompress(inputBuffer, outputBuffer)
+      return htscodecs.tok3_uncompress(inputBuffer)
     } else {
       throw new CramUnimplementedError(
         `${compressionMethod} decompression not yet implemented`,
@@ -355,7 +360,18 @@ export default class CramFile {
     }
     const blockContentPosition = blockHeader._endPosition
 
-    const uncompressedData = Buffer.allocUnsafe(blockHeader.uncompressedSize)
+    const d = await this.file.read(
+      blockHeader.compressedSize,
+      blockContentPosition,
+    )
+    const uncompressedData =
+      blockHeader.compressionMethod !== 'raw'
+        ? await this._uncompress(
+            blockHeader.compressionMethod,
+            d,
+            blockHeader.uncompressedSize,
+          )
+        : d
 
     const block: CramFileBlock = {
       ...blockHeader,
@@ -363,30 +379,6 @@ export default class CramFile {
       contentPosition: blockContentPosition,
       content: uncompressedData,
     }
-
-    if (blockHeader.compressionMethod !== 'raw') {
-      const compressedData = Buffer.allocUnsafe(blockHeader.compressedSize)
-      await this.read(
-        compressedData,
-        0,
-        blockHeader.compressedSize,
-        blockContentPosition,
-      )
-
-      await this._uncompress(
-        blockHeader.compressionMethod,
-        compressedData,
-        uncompressedData,
-      )
-    } else {
-      await this.read(
-        uncompressedData,
-        0,
-        blockHeader.uncompressedSize,
-        blockContentPosition,
-      )
-    }
-
     if (majorVersion >= 3) {
       // parse the crc32
       const crc = await this._parseSection(

package/src/cramFile/record.ts

@@ -1,5 +1,6 @@
 import Constants from './constants'
 import CramContainerCompressionScheme from './container/compressionScheme'
+
 import type decodeRecord from './slice/decodeRecord'
 
 export interface RefRegion {
@@ -13,7 +14,6 @@ export interface ReadFeature {
   pos: number
   refPos: number
   data: any
-
   ref?: string
   sub?: string
 }
@@ -458,8 +458,8 @@ export default class CramRecord {
     compressionScheme: CramContainerCompressionScheme,
   ) {
     if (this.readFeatures) {
-      // use the reference bases to decode the bases
-      // substituted in each base substitution
+      // use the reference bases to decode the bases substituted in each base
+      // substitution
       this.readFeatures.forEach(readFeature => {
         if (readFeature.code === 'X') {
           decodeBaseSubstitution(

package/src/cramFile/sectionParsers.ts

@@ -1,23 +1,22 @@
 import { TupleOf } from '../typescript'
-import { parseItf8, parseLtf8 } from './util'
 import { DataSeriesEncodingMap } from './codecs/dataSeriesTypes'
 import { CramEncoding } from './encoding'
+import { parseItf8, parseLtf8 } from './util'
 
 export function cramFileDefinition() {
   return {
-    parser: (buffer: Buffer, _startOffset = 0) => {
-      const b = buffer
+    parser: (b: Uint8Array, _startOffset = 0) => {
       const dataView = new DataView(b.buffer, b.byteOffset, b.length)
+      const decoder = new TextDecoder('utf8')
       let offset = 0
-      const magic = buffer.subarray(offset, offset + 4).toString()
+      const magic = decoder.decode(b.subarray(offset, offset + 4))
       offset += 4
       const majorVersion = dataView.getUint8(offset)
       offset += 1
       const minorVersion = dataView.getUint8(offset)
       offset += 1
-      const fileId = b
-        .subarray(offset, offset + 20)
-        .toString()
+      const fileId = decoder
+        .decode(b.subarray(offset, offset + 20))
         .replaceAll('\0', '')
       offset += 20
       return {
@@ -34,7 +33,7 @@ export function cramFileDefinition() {
   }
 }
 export function cramBlockHeader() {
-  const parser = (buffer: Buffer, _startOffset = 0) => {
+  const parser = (buffer: Uint8Array, _startOffset = 0) => {
     const b = buffer
     const dataView = new DataView(b.buffer, b.byteOffset, b.length)
     let offset = 0
@@ -97,7 +96,7 @@ export function cramBlockHeader() {
 
 export function cramBlockCrc32() {
   return {
-    parser: (buffer: Buffer, offset: number) => {
+    parser: (buffer: Uint8Array, offset: number) => {
       const b = buffer
       const dataView = new DataView(b.buffer, b.byteOffset, b.length)
       const crc32 = dataView.getUint32(offset, true)
@@ -115,8 +114,13 @@ export function cramBlockCrc32() {
 
 export type CramTagDictionary = string[][]
 
-function makeTagSet(buffer: Buffer, stringStart: number, stringEnd: number) {
-  const str = buffer.toString('utf8', stringStart, stringEnd)
+function makeTagSet(
+  buffer: Uint8Array,
+  stringStart: number,
+  stringEnd: number,
+) {
+  const decoder = new TextDecoder('utf8')
+  const str = decoder.decode(buffer.subarray(stringStart, stringEnd))
   const tags = []
   for (let i = 0; i < str.length; i += 3) {
     tags.push(str.slice(i, i + 3))
@@ -126,7 +130,7 @@ function makeTagSet(buffer: Buffer, stringStart: number, stringEnd: number) {
 
 export function cramTagDictionary() {
   return {
-    parser: (buffer: Buffer, offset: number) => {
+    parser: (buffer: Uint8Array, offset: number) => {
       const [size, newOffset1] = parseItf8(buffer, offset)
       offset += newOffset1
       const subbuf = buffer.subarray(offset, offset + size)
@@ -169,7 +173,7 @@ export interface CramPreservationMap {
 
 export function cramPreservationMap() {
   return {
-    parser: (buffer: Buffer, offset: number) => {
+    parser: (buffer: Uint8Array, offset: number) => {
       const b = buffer
       const dataView = new DataView(b.buffer, b.byteOffset, b.length)
       const [mapSize, newOffset1] = parseItf8(buffer, offset)
@@ -284,7 +288,7 @@ function cramUnmappedSliceHeader(majorVersion: number) {
   maxLength += 5 * 2
   maxLength += 16
 
-  const parser = (buffer: Buffer, offset: number) => {
+  const parser = (buffer: Uint8Array, offset: number) => {
     const [numRecords, newOffset1] = parseItf8(buffer, offset)
     offset += newOffset1
     let recordCounter = 0
@@ -348,7 +352,7 @@ function cramMappedSliceHeader(majorVersion: number) {
   maxLength += 16 // MD5
 
   return {
-    parser: (buffer: Buffer, offset: number) => {
+    parser: (buffer: Uint8Array, offset: number) => {
       // L0
       const [refSeqId, newOffset1] = parseItf8(buffer, offset)
       offset += newOffset1
@@ -419,12 +423,13 @@ function cramMappedSliceHeader(majorVersion: number) {
 
 function cramEncoding() {
   return {
-    parser: (buffer: Buffer, offset: number) => cramEncodingSub(buffer, offset),
+    parser: (buffer: Uint8Array, offset: number) =>
+      cramEncodingSub(buffer, offset),
   }
 }
 
 function cramEncodingSub(
-  buffer: Buffer,
+  buffer: Uint8Array,
   offset: number,
 ): { value: Value; offset: number } {
   const b = buffer
@@ -542,7 +547,7 @@ function cramEncodingSub(
 
 function cramDataSeriesEncodingMap() {
   return {
-    parser: (buffer: Buffer, offset: number) => {
+    parser: (buffer: Uint8Array, offset: number) => {
       const [mapSize, newOffset1] = parseItf8(buffer, offset)
       offset += newOffset1
       const [mapCount, newOffset2] = parseItf8(buffer, offset)
@@ -572,7 +577,7 @@ function cramDataSeriesEncodingMap() {
 
 function cramTagEncodingMap() {
   return {
-    parser: (buffer: Buffer, offset: number) => {
+    parser: (buffer: Uint8Array, offset: number) => {
       const [mapSize, newOffset1] = parseItf8(buffer, offset)
       offset += newOffset1
       const [mapCount, newOffset2] = parseItf8(buffer, offset)
@@ -604,7 +609,7 @@ function cramTagEncodingMap() {
 
 function cramCompressionHeader() {
   return {
-    parser: (buffer: Buffer, offset: number) => {
+    parser: (buffer: Uint8Array, offset: number) => {
       // TODO: if we want to support CRAM v1, we will need to refactor
       // compression header into 2 parts to parse the landmarks, like the
       // container header
@@ -644,7 +649,7 @@ function cramContainerHeader1(majorVersion: number) {
   maxLength += 5 + 5
   return {
     maxLength,
-    parser: (buffer: Buffer, offset: number) => {
+    parser: (buffer: Uint8Array, offset: number) => {
       const b = buffer
       const dataView = new DataView(b.buffer, b.byteOffset, b.length)
       // byte size of the container data (blocks)
@@ -704,7 +709,7 @@ function cramContainerHeader1(majorVersion: number) {
 
 function cramContainerHeader2(majorVersion: number) {
   return {
-    parser: (buffer: Buffer, offset: number) => {
+    parser: (buffer: Uint8Array, offset: number) => {
       const b = buffer
       const dataView = new DataView(b.buffer, b.byteOffset, b.length)
       const [numLandmarks, newOffset1] = parseItf8(buffer, offset)

package/src/cramFile/slice/decodeRecord.ts

@@ -1,4 +1,9 @@
 import { CramMalformedError } from '../../errors'
+import { Cursors, DataTypeMapping } from '../codecs/_base'
+import { DataSeriesEncodingKey } from '../codecs/dataSeriesTypes'
+import CramContainerCompressionScheme, {
+  DataSeriesTypes,
+} from '../container/compressionScheme'
 import {
   BamFlagsDecoder,
   CramFlagsDecoder,
@@ -6,13 +11,8 @@ import {
   ReadFeature,
 } from '../record'
 import CramSlice, { SliceHeader } from './index'
-import { isMappedSliceHeader } from '../sectionParsers'
-import CramContainerCompressionScheme, {
-  DataSeriesTypes,
-} from '../container/compressionScheme'
 import { CramFileBlock } from '../file'
-import { Cursors, DataTypeMapping } from '../codecs/_base'
-import { DataSeriesEncodingKey } from '../codecs/dataSeriesTypes'
+import { isMappedSliceHeader } from '../sectionParsers'
 
 /**
  * given a Buffer, read a string up to the first null character
@@ -42,37 +42,37 @@ function parseTagValueArray(buffer: Uint8Array) {
 
   if (arrayType === 'c') {
     const arr = new Int8Array(buffer.buffer)
-    for (let i = 0; i < length; i += 1) {
+    for (let i = 0; i < length; i++) {
       array[i] = arr[i]!
     }
   } else if (arrayType === 'C') {
     const arr = new Uint8Array(buffer.buffer)
-    for (let i = 0; i < length; i += 1) {
+    for (let i = 0; i < length; i++) {
       array[i] = arr[i]!
     }
   } else if (arrayType === 's') {
     const arr = new Int16Array(buffer.buffer)
-    for (let i = 0; i < length; i += 1) {
+    for (let i = 0; i < length; i++) {
       array[i] = arr[i]!
     }
   } else if (arrayType === 'S') {
     const arr = new Uint16Array(buffer.buffer)
-    for (let i = 0; i < length; i += 1) {
+    for (let i = 0; i < length; i++) {
       array[i] = arr[i]!
     }
   } else if (arrayType === 'i') {
     const arr = new Int32Array(buffer.buffer)
-    for (let i = 0; i < length; i += 1) {
+    for (let i = 0; i < length; i++) {
       array[i] = arr[i]!
     }
   } else if (arrayType === 'I') {
     const arr = new Uint32Array(buffer.buffer)
-    for (let i = 0; i < length; i += 1) {
+    for (let i = 0; i < length; i++) {
       array[i] = arr[i]!
     }
   } else if (arrayType === 'f') {
     const arr = new Float32Array(buffer.buffer)
-    for (let i = 0; i < length; i += 1) {
+    for (let i = 0; i < length; i++) {
       array[i] = arr[i]!
     }
   } else {
@@ -158,7 +158,7 @@ function decodeReadFeatures(
     return data
   }
 
-  for (let i = 0; i < readFeatureCount; i += 1) {
+  for (let i = 0; i < readFeatureCount; i++) {
     const code = String.fromCharCode(decodeDataSeries('FC'))
 
     const readPosDelta = decodeDataSeries('FP')
@@ -315,7 +315,7 @@ export default function decodeRecord(
   // TN = tag names
   const TN = compressionScheme.getTagNames(TLindex)!
   const ntags = TN.length
-  for (let i = 0; i < ntags; i += 1) {
+  for (let i = 0; i < ntags; i++) {
     const tagId = TN[i]!
     const tagName = tagId.slice(0, 2)
     const tagType = tagId.slice(2, 3)
@@ -381,14 +381,14 @@ export default function decodeRecord(
     qualityScores = null
   } else {
     const bases = new Array(readLength) as number[]
-    for (let i = 0; i < bases.length; i += 1) {
+    for (let i = 0; i < bases.length; i++) {
       bases[i] = decodeDataSeries('BA')
     }
     readBases = String.fromCharCode(...bases)
 
     if (CramFlagsDecoder.isPreservingQualityScores(cramFlags)) {
       qualityScores = new Array(readLength)
-      for (let i = 0; i < bases.length; i += 1) {
+      for (let i = 0; i < bases.length; i++) {
         qualityScores[i] = decodeDataSeries('QS')
       }
     }

package/src/cramFile/slice/index.ts

@@ -1,21 +1,20 @@
 import { CramArgumentError, CramMalformedError } from '../../errors'
-import { parseItem, sequenceMD5, tinyMemoize } from '../util'
-
+import { Cursors, DataTypeMapping } from '../codecs/_base'
+import { CramBufferOverrunError } from '../codecs/getBits'
 import Constants from '../constants'
 import decodeRecord, { DataSeriesDecoder } from './decodeRecord'
-import CramRecord from '../record'
+import { DataSeriesEncodingKey } from '../codecs/dataSeriesTypes'
 import CramContainer from '../container'
+import { DataSeriesTypes } from '../container/compressionScheme'
 import CramFile, { CramFileBlock } from '../file'
+import CramRecord from '../record'
 import {
-  getSectionParsers,
-  isMappedSliceHeader,
   MappedSliceHeader,
   UnmappedSliceHeader,
+  getSectionParsers,
+  isMappedSliceHeader,
 } from '../sectionParsers'
-import { CramBufferOverrunError } from '../codecs/getBits'
-import { Cursors, DataTypeMapping } from '../codecs/_base'
-import { DataSeriesEncodingKey } from '../codecs/dataSeriesTypes'
-import { DataSeriesTypes } from '../container/compressionScheme'
+import { parseItem, sequenceMD5, tinyMemoize } from '../util'
 
 export type SliceHeader = CramFileBlock & {
   parsedContent: MappedSliceHeader | UnmappedSliceHeader