@gmod/cram 1.6.3 → 1.7.0

package/src/cramFile/codecs/dataSeriesTypes.ts

@@ -0,0 +1,39 @@
+import { CramEncoding } from '../encoding'
+
+export type DataType = 'int' | 'byte' | 'long' | 'byteArray'
+
+export type DataSeriesEncodingKey =
+  | 'BF'
+  | 'CF'
+  | 'RI'
+  | 'RL'
+  | 'AP'
+  | 'RG'
+  | 'RN'
+  | 'MF'
+  | 'NS'
+  | 'NP'
+  | 'TS'
+  | 'NF'
+  | 'TL'
+  | 'FN'
+  | 'FC'
+  | 'FP'
+  | 'DL'
+  | 'BB'
+  | 'QQ'
+  | 'BS'
+  | 'IN'
+  | 'RS'
+  | 'PD'
+  | 'HC'
+  | 'SC'
+  | 'MQ'
+  | 'BA'
+  | 'QS'
+  | 'TC'
+  | 'TN'
+// | 'TM'
+// | 'TV'
+
+export type DataSeriesEncodingMap = Record<DataSeriesEncodingKey, CramEncoding>

package/src/cramFile/codecs/{external.js → external.ts}

@@ -1,13 +1,24 @@
-import {
-  CramUnimplementedError,
-  CramMalformedError,
-  CramBufferOverrunError,
-} from '../../errors'
-import CramCodec from './_base'
+import { CramMalformedError, CramUnimplementedError } from '../../errors'
+import CramCodec, { Cursor, Cursors } from './_base'
 import { parseItf8 } from '../util'
+import CramSlice from '../slice'
+import { CramFileBlock } from '../file'
+import { CramBufferOverrunError } from './getBits'
+import { ExternalCramEncoding } from '../encoding'
 
-export default class ExternalCodec extends CramCodec {
-  constructor(parameters = {}, dataType) {
+export default class ExternalCodec extends CramCodec<
+  'int' | 'byte',
+  ExternalCramEncoding['parameters']
+> {
+  private readonly _decodeData: (
+    contentBlock: CramFileBlock,
+    cursor: Cursor,
+  ) => number
+
+  constructor(
+    parameters: ExternalCramEncoding['parameters'],
+    dataType: 'int' | 'byte',
+  ) {
     super(parameters, dataType)
     if (this.dataType === 'int') {
       this._decodeData = this._decodeInt
@@ -20,7 +31,12 @@ export default class ExternalCodec extends CramCodec {
     }
   }
 
-  decode(slice, coreDataBlock, blocksByContentId, cursors) {
+  decode(
+    slice: CramSlice,
+    coreDataBlock: CramFileBlock,
+    blocksByContentId: Record<number, CramFileBlock>,
+    cursors: Cursors,
+  ) {
     const { blockContentId } = this.parameters
     const contentBlock = blocksByContentId[blockContentId]
     if (!contentBlock) {
@@ -32,16 +48,16 @@ export default class ExternalCodec extends CramCodec {
     return this._decodeData(contentBlock, cursor)
   }
 
-  _decodeInt(contentBlock, cursor) {
+  _decodeInt(contentBlock: CramFileBlock, cursor: Cursor) {
     const [result, bytesRead] = parseItf8(
       contentBlock.content,
       cursor.bytePosition,
     )
-    cursor.bytePosition += bytesRead
+    cursor.bytePosition = cursor.bytePosition + bytesRead
     return result
   }
 
-  _decodeByte(contentBlock, cursor) {
+  _decodeByte(contentBlock: CramFileBlock, cursor: Cursor) {
     if (cursor.bytePosition >= contentBlock.content.length) {
       throw new CramBufferOverrunError(
         'attempted to read beyond end of block. this file seems truncated.',

package/src/cramFile/codecs/gamma.ts

@@ -0,0 +1,42 @@
+import { CramUnimplementedError } from '../../errors'
+import CramCodec, { Cursors } from './_base'
+import { getBits } from './getBits'
+import { GammaEncoding } from '../encoding'
+import CramSlice from '../slice'
+import { CramFileBlock } from '../file'
+
+export default class GammaCodec extends CramCodec<
+  'int',
+  GammaEncoding['parameters']
+> {
+  constructor(parameters: GammaEncoding['parameters'], dataType: 'int') {
+    super(parameters, dataType)
+    if (this.dataType !== 'int') {
+      throw new CramUnimplementedError(
+        `${this.dataType} decoding not yet implemented by GAMMA codec`,
+      )
+    }
+  }
+
+  decode(
+    slice: CramSlice,
+    coreDataBlock: CramFileBlock,
+    blocksByContentId: Record<number, CramFileBlock>,
+    cursors: Cursors,
+  ) {
+    let length = 1
+
+    while (getBits(coreDataBlock.content, cursors.coreBlock, 1) === 0) {
+      length = length + 1
+    }
+
+    const readBits = getBits(
+      coreDataBlock.content,
+      cursors.coreBlock,
+      length - 1,
+    )
+
+    const value = readBits | (1 << (length - 1))
+    return value - this.parameters.offset
+  }
+}

package/src/cramFile/codecs/getBits.ts

@@ -0,0 +1,28 @@
+export class CramBufferOverrunError extends Error {}
+
+export function getBits(
+  data: Buffer,
+  cursor: { bytePosition: number; bitPosition: number },
+  numBits: number,
+) {
+  let val = 0
+  if (
+    cursor.bytePosition + (7 - cursor.bitPosition + numBits) / 8 >
+    data.length
+  ) {
+    throw new CramBufferOverrunError(
+      'read error during decoding. the file seems to be truncated.',
+    )
+  }
+  for (let dlen = numBits; dlen; dlen--) {
+    // get the next `dlen` bits in the input, put them in val
+    val <<= 1
+    val |= (data[cursor.bytePosition] >> cursor.bitPosition) & 1
+    cursor.bitPosition -= 1
+    if (cursor.bitPosition < 0) {
+      cursor.bytePosition += 1
+    }
+    cursor.bitPosition &= 7
+  }
+  return val
+}

package/src/cramFile/codecs/{huffman.js → huffman.ts}

@@ -1,14 +1,36 @@
 import { CramMalformedError } from '../../errors'
-import CramCodec from './_base'
+import CramCodec, { Cursor, Cursors } from './_base'
+import { getBits } from './getBits'
+import { HuffmanEncoding } from '../encoding'
 
-function numberOfSetBits(ii) {
+import CramSlice from '../slice'
+import { CramFileBlock } from '../file'
+
+function numberOfSetBits(ii: number) {
   let i = (ii - (ii >> 1)) & 0x55555555
   i = (i & 0x33333333) + ((i >> 2) & 0x33333333)
   return (((i + (i >> 4)) & 0x0f0f0f0f) * 0x01010101) >> 24
 }
 
-export default class HuffmanIntCodec extends CramCodec {
-  constructor(parameters = {}, dataType) {
+type Code = { bitLength: number; value: number; bitCode: number }
+
+export default class HuffmanIntCodec extends CramCodec<
+  'byte' | 'int',
+  HuffmanEncoding['parameters']
+> {
+  private codes: Record<number, Code> = {}
+  private codeBook: Record<number, number[]> = {}
+  private sortedByValue: Code[] = []
+  private sortedCodes: Code[] = []
+  private sortedValuesByBitCode: number[] = []
+  private sortedBitCodes: number[] = []
+  private sortedBitLengthsByBitCode: number[] = []
+  private bitCodeToValue: number[] = []
+
+  constructor(
+    parameters: HuffmanEncoding['parameters'],
+    dataType: 'byte' | 'int',
+  ) {
     super(parameters, dataType)
     if (!['byte', 'int'].includes(this.dataType)) {
       throw new TypeError(
@@ -28,7 +50,9 @@ export default class HuffmanIntCodec extends CramCodec {
 
   buildCodeBook() {
     // parse the parameters together into a `codes` data structure
-    let codes = new Array(this.parameters.numCodes)
+    let codes: Array<{ symbol: number; bitLength: number }> = new Array(
+      this.parameters.numCodes,
+    )
     for (let i = 0; i < this.parameters.numCodes; i += 1) {
       codes[i] = {
         symbol: this.parameters.symbols[i],
@@ -54,16 +78,20 @@ export default class HuffmanIntCodec extends CramCodec {
     let codeLength = 0
     let codeValue = -1
     Object.entries(this.codeBook).forEach(([bitLength, symbols]) => {
-      bitLength = parseInt(bitLength, 10)
+      const bitLengthInt = parseInt(bitLength, 10)
       symbols.forEach(symbol => {
-        const code = { bitLength, value: symbol }
-        codeValue += 1
-        const delta = bitLength - codeLength // new length?
-        codeValue <<= delta // pad with 0's
+        const code = {
+          bitLength: bitLengthInt,
+          value: symbol,
+          bitCode: 0,
+        }
+        codeValue = codeValue + 1
+        const delta = bitLengthInt - codeLength // new length?
+        codeValue = codeValue << delta // pad with 0's
         code.bitCode = codeValue // calculated: huffman code
-        codeLength += delta // adjust current code length
+        codeLength = codeLength + delta // adjust current code length
 
-        if (numberOfSetBits(codeValue) > bitLength) {
+        if (numberOfSetBits(codeValue) > bitLengthInt) {
           throw new CramMalformedError('Symbol out of range')
         }
 
@@ -93,7 +121,12 @@ export default class HuffmanIntCodec extends CramCodec {
     }
   }
 
-  decode(slice, coreDataBlock, blocksByContentId, cursors) {
+  decode(
+    slice: CramSlice,
+    coreDataBlock: CramFileBlock,
+    blocksByContentId: Record<number, CramFileBlock>,
+    cursors: Cursors,
+  ) {
     return this._decode(slice, coreDataBlock, cursors.coreBlock)
   }
 
@@ -106,7 +139,7 @@ export default class HuffmanIntCodec extends CramCodec {
     return this.sortedCodes[0].value
   }
 
-  _decode(slice, coreDataBlock, coreCursor) {
+  _decode(slice: CramSlice, coreDataBlock: CramFileBlock, coreCursor: Cursor) {
     const input = coreDataBlock.content
 
     let prevLen = 0
@@ -114,7 +147,7 @@ export default class HuffmanIntCodec extends CramCodec {
     for (let i = 0; i < this.sortedCodes.length; i += 1) {
       const length = this.sortedCodes[i].bitLength
       bits <<= length - prevLen
-      bits |= this._getBits(input, coreCursor, length - prevLen)
+      bits |= getBits(input, coreCursor, length - prevLen)
       prevLen = length
       {
         const index = this.bitCodeToValue[bits]

package/src/cramFile/codecs/{index.js → index.ts}

@@ -7,6 +7,9 @@ import ByteArrayLengthCodec from './byteArrayLength'
 import BetaCodec from './beta'
 import GammaCodec from './gamma'
 import SubexpCodec from './subexp'
+import CramCodec from './_base'
+import { CramEncoding } from '../encoding'
+import { DataType } from './dataSeriesTypes'
 
 const codecClasses = {
   1: ExternalCodec,
@@ -20,11 +23,14 @@ const codecClasses = {
   9: GammaCodec,
 }
 
-export function getCodecClassWithId(id) {
-  return codecClasses[id]
+function getCodecClassWithId(id: number) {
+  return (codecClasses as any)[id]
 }
 
-export function instantiateCodec(encodingData, dataType) {
+export function instantiateCodec<TResult extends DataType = DataType>(
+  encodingData: CramEncoding,
+  dataType: DataType | 'ignore',
+): CramCodec<TResult> {
   const CodecClass = getCodecClassWithId(
     dataType === 'ignore' ? 0 : encodingData.codecId,
   )

package/src/cramFile/codecs/subexp.ts

@@ -0,0 +1,45 @@
+import { CramUnimplementedError } from '../../errors'
+import CramCodec, { Cursors } from './_base'
+import { getBits } from './getBits'
+import CramSlice from '../slice'
+import { CramFileBlock } from '../file'
+import { SubexpEncoding } from '../encoding'
+
+export default class SubexpCodec extends CramCodec<
+  'int',
+  SubexpEncoding['parameters']
+> {
+  constructor(parameters: SubexpEncoding['parameters'], dataType: 'int') {
+    super(parameters, dataType)
+    if (this.dataType !== 'int') {
+      throw new CramUnimplementedError(
+        `${this.dataType} decoding not yet implemented by SUBEXP codec`,
+      )
+    }
+  }
+
+  decode(
+    slice: CramSlice,
+    coreDataBlock: CramFileBlock,
+    blocksByContentId: Record<number, CramFileBlock>,
+    cursors: Cursors,
+  ) {
+    let numLeadingOnes = 0
+    while (getBits(coreDataBlock.content, cursors.coreBlock, 1)) {
+      numLeadingOnes = numLeadingOnes + 1
+    }
+
+    let b
+    let n
+    if (numLeadingOnes === 0) {
+      b = this.parameters.K
+      n = getBits(coreDataBlock.content, cursors.coreBlock, b)
+    } else {
+      b = numLeadingOnes + this.parameters.K - 1
+      const bits = getBits(coreDataBlock.content, cursors.coreBlock, b)
+      n = (1 << b) | bits
+    }
+
+    return n - this.parameters.offset
+  }
+}

package/src/cramFile/container/{compressionScheme.js → compressionScheme.ts}

@@ -1,5 +1,12 @@
-import { CramMalformedError } from '../../errors'
 import { instantiateCodec } from '../codecs'
+import CramCodec from '../codecs/_base'
+import { CramCompressionHeader, CramPreservationMap } from '../sectionParsers'
+import { CramEncoding } from '../encoding'
+import { CramMalformedError } from '../../errors'
+import {
+  DataSeriesEncodingKey,
+  DataSeriesEncodingMap,
+} from '../codecs/dataSeriesTypes'
 
 // the hardcoded data type to be decoded for each core
 // data field
@@ -34,12 +41,14 @@ const dataSeriesTypes = {
   QS: 'byte',
   QQ: 'byteArray',
   TL: 'int',
-  TM: 'ignore',
-  TV: 'ignore',
-}
+  // TM: 'ignore',
+  // TV: 'ignore',
+} as const
+
+export type DataSeriesTypes = typeof dataSeriesTypes
 
-function parseSubstitutionMatrix(byteArray) {
-  const matrix = new Array(5)
+function parseSubstitutionMatrix(byteArray: number[]) {
+  const matrix: string[][] = new Array(5)
   for (let i = 0; i < 5; i += 1) {
     matrix[i] = new Array(4)
   }
@@ -72,25 +81,44 @@ function parseSubstitutionMatrix(byteArray) {
   return matrix
 }
 
+type DataSeriesCache = {
+  [K in DataSeriesEncodingKey]?: CramCodec<DataSeriesTypes[K]>
+}
+
 export default class CramContainerCompressionScheme {
-  constructor(content) {
-    Object.assign(this, content)
+  public readNamesIncluded: boolean
+  public APdelta: boolean
+  public referenceRequired: boolean
+  public tagIdsDictionary: Record<number, string[]>
+  public substitutionMatrix: string[][]
+  public dataSeriesCodecCache: DataSeriesCache = {}
+  public tagCodecCache: Record<string, CramCodec> = {}
+  public tagEncoding: Record<string, CramEncoding> = {}
+  public dataSeriesEncoding: DataSeriesEncodingMap
+  private preservation: CramPreservationMap
+  private _endPosition: number
+  private _size: number
+
+  constructor(content: CramCompressionHeader) {
+    // Object.assign(this, content)
     // interpret some of the preservation map tags for convenient use
     this.readNamesIncluded = content.preservation.RN
     this.APdelta = content.preservation.AP
     this.referenceRequired = !!content.preservation.RR
     this.tagIdsDictionary = content.preservation.TD
     this.substitutionMatrix = parseSubstitutionMatrix(content.preservation.SM)
-
-    this.dataSeriesCodecCache = new Map()
-    this.tagCodecCache = {}
+    this.dataSeriesEncoding = content.dataSeriesEncoding
+    this.tagEncoding = content.tagEncoding
+    this.preservation = content.preservation
+    this._size = content._size
+    this._endPosition = content._endPosition
   }
 
   /**
    * @param {string} tagName three-character tag name
    * @private
    */
-  getCodecForTag(tagName) {
+  getCodecForTag(tagName: string): CramCodec {
     if (!this.tagCodecCache[tagName]) {
       const encodingData = this.tagEncoding[tagName]
       if (encodingData) {
@@ -108,12 +136,15 @@ export default class CramContainerCompressionScheme {
    * @param {number} tagListId ID of the tag list to fetch from the tag dictionary
    * @private
    */
-  getTagNames(tagListId) {
+  getTagNames(tagListId: number) {
     return this.tagIdsDictionary[tagListId]
   }
 
-  getCodecForDataSeries(dataSeriesName) {
-    let r = this.dataSeriesCodecCache.get(dataSeriesName)
+  getCodecForDataSeries<TDataSeries extends DataSeriesEncodingKey>(
+    dataSeriesName: TDataSeries,
+  ): CramCodec<DataSeriesTypes[TDataSeries]> | undefined {
+    let r: CramCodec<DataSeriesTypes[TDataSeries]> | undefined =
+      this.dataSeriesCodecCache[dataSeriesName]
     if (r === undefined) {
       const encodingData = this.dataSeriesEncoding[dataSeriesName]
       if (encodingData) {
@@ -124,19 +155,20 @@ export default class CramContainerCompressionScheme {
           )
         }
         r = instantiateCodec(encodingData, dataType)
-        this.dataSeriesCodecCache.set(dataSeriesName, r)
+        // didn't find a way to make TS understand this
+        this.dataSeriesCodecCache[dataSeriesName] = r as CramCodec<any>
       }
     }
     return r
   }
 
   toJSON() {
-    const data = {}
+    const data: any = {}
     Object.keys(this).forEach(k => {
       if (/Cache$/.test(k)) {
         return
       }
-      data[k] = this[k]
+      data[k] = (this as any)[k]
     })
     return data
   }

package/src/cramFile/container/{index.js → index.ts}

@@ -3,15 +3,10 @@ import { CramMalformedError } from '../../errors'
 import { itf8Size, parseItem, tinyMemoize } from '../util'
 import CramSlice from '../slice'
 import CramContainerCompressionScheme from './compressionScheme'
+import CramFile from '../file'
 
 export default class CramContainer {
-  constructor(cramFile, position) {
-    // cram file this container comes from
-    this.file = cramFile
-    // position of this container in the file
-    this.filePosition = position
-    // console.log(`container: ${this.filePosition}`)
-  }
+  constructor(public file: CramFile, public filePosition: number) {}
 
   // memoize
   getHeader() {
@@ -28,6 +23,9 @@ export default class CramContainer {
     }
     const sectionParsers = await this.file.getSectionParsers()
     const block = await this.getFirstBlock()
+    if (block === undefined) {
+      return undefined
+    }
     if (block.contentType !== 'COMPRESSION_HEADER') {
       throw new CramMalformedError(
         `invalid content type ${block.contentType} in what is supposed to be the compression header block`,
@@ -39,8 +37,10 @@ export default class CramContainer {
       0,
       block.contentPosition,
     )
-    block.content = content
-    return block
+    return {
+      ...block,
+      parsedContent: content,
+    }
   }
 
   async getFirstBlock() {
@@ -55,16 +55,16 @@ export default class CramContainer {
     if (!header) {
       return undefined
     }
-    return new CramContainerCompressionScheme(header.content)
+    return new CramContainerCompressionScheme(header.parsedContent)
   }
 
-  getSlice(slicePosition, sliceSize) {
+  getSlice(slicePosition: number, sliceSize: number) {
     // note: slicePosition is relative to the end of the container header
     // TODO: perhaps we should cache slices?
     return new CramSlice(this, slicePosition, sliceSize)
   }
 
-  async _readContainerHeader(position) {
+  async _readContainerHeader(position: number) {
     const sectionParsers = await this.file.getSectionParsers()
     const { cramContainerHeader1, cramContainerHeader2 } = sectionParsers
     const { size: fileSize } = await this.file.stat()
@@ -77,7 +77,7 @@ export default class CramContainer {
     // how much to buffer until you read numLandmarks
     const bytes1 = Buffer.allocUnsafe(cramContainerHeader1.maxLength)
     await this.file.read(bytes1, 0, cramContainerHeader1.maxLength, position)
-    const header1 = parseItem(bytes1, cramContainerHeader1.parser)
+    const header1 = parseItem(bytes1, cramContainerHeader1.parser) as any
     const numLandmarksSize = itf8Size(header1.numLandmarks)
     if (position + header1.length >= fileSize) {
       console.warn(

package/src/cramFile/encoding.ts

@@ -0,0 +1,98 @@
+export type NullEncoding = {
+  codecId: 0
+  parametersBytes: number
+  parameters: Record<string, never>
+}
+
+export type ExternalCramEncoding = {
+  codecId: 1
+  parametersBytes: number
+  parameters: {
+    blockContentId: number
+  }
+}
+
+export type GolombEncoding = {
+  codecId: 2
+  parametersBytes: number
+  parameters: {
+    offset: number
+    M: number
+  }
+}
+
+export type HuffmanEncoding = {
+  codecId: 3
+  parametersBytes: number
+  parameters: {
+    numCodes: number
+    symbols: number[]
+    numLengths: number
+    bitLengths: number[]
+  }
+}
+
+export type ByteArrayLengthEncoding = {
+  codecId: 4
+  parametersBytes: number
+  parameters: {
+    lengthsEncoding: CramEncoding
+    valuesEncoding: CramEncoding
+  }
+}
+
+export type ByteArrayStopCramEncoding = {
+  codecId: 5
+  parametersBytes: number
+  parameters: {
+    stopByte: number
+    blockContentId: number
+  }
+}
+
+export type BetaEncoding = {
+  codecId: 6
+  parametersBytes: number
+  parameters: {
+    offset: number
+    length: number // int number of bits
+  }
+}
+
+export type SubexpEncoding = {
+  codecId: 7
+  parametersBytes: number
+  parameters: {
+    offset: number
+    K: number
+  }
+}
+
+export type GolombRiceEncoding = {
+  codecId: 8
+  parametersBytes: number
+  parameters: {
+    offset: number
+    log2m: number
+  }
+}
+
+export type GammaEncoding = {
+  codecId: 9
+  parametersBytes: number
+  parameters: {
+    offset: number
+  }
+}
+
+export type CramEncoding =
+  | NullEncoding
+  | ExternalCramEncoding
+  | GolombEncoding
+  | HuffmanEncoding
+  | ByteArrayLengthEncoding
+  | ByteArrayStopCramEncoding
+  | BetaEncoding
+  | SubexpEncoding
+  | GolombRiceEncoding
+  | GammaEncoding