@gmod/cram 1.6.3 → 1.7.0

package/src/cramFile/slice/{index.js → index.ts}

@@ -1,12 +1,24 @@
-import {
-  CramMalformedError,
-  CramBufferOverrunError,
-  CramArgumentError,
-} from '../../errors'
-import { parseItem, tinyMemoize, sequenceMD5 } from '../util'
+import { CramArgumentError, CramMalformedError } from '../../errors'
+import { parseItem, sequenceMD5, tinyMemoize } from '../util'
 
 import Constants from '../constants'
-import decodeRecord from './decodeRecord'
+import decodeRecord, { DataSeriesDecoder } from './decodeRecord'
+import CramRecord from '../record'
+import CramContainer from '../container'
+import CramFile, { CramFileBlock } from '../file'
+import {
+  isMappedSliceHeader,
+  MappedSliceHeader,
+  UnmappedSliceHeader,
+} from '../sectionParsers'
+import { CramBufferOverrunError } from '../codecs/getBits'
+import { Cursors, DataTypeMapping } from '../codecs/_base'
+import { DataSeriesEncodingKey } from '../codecs/dataSeriesTypes'
+import { DataSeriesTypes } from '../container/compressionScheme'
+
+export type SliceHeader = CramFileBlock & {
+  parsedContent: MappedSliceHeader | UnmappedSliceHeader
+}
 
 /**
  * @private
@@ -16,13 +28,16 @@ import decodeRecord from './decodeRecord'
  * @param {CramRecord} thisRecord
  */
 function calculateMultiSegmentMatedTemplateLength(
-  allRecords,
-  currentRecordNumber,
-  thisRecord,
+  allRecords: CramRecord[],
+  currentRecordNumber: number,
+  thisRecord: CramRecord,
 ) {
-  function getAllMatedRecords(startRecord) {
+  function getAllMatedRecords(startRecord: CramRecord) {
     const records = [startRecord]
-    if (startRecord.mateRecordNumber >= 0) {
+    if (
+      startRecord.mateRecordNumber !== undefined &&
+      startRecord.mateRecordNumber >= 0
+    ) {
       const mateRecord = allRecords[startRecord.mateRecordNumber]
       if (!mateRecord) {
         throw new CramMalformedError(
@@ -57,7 +72,10 @@ function calculateMultiSegmentMatedTemplateLength(
  * @param {CramRecord} thisRecord
  * @param {CramRecord} mateRecord
  */
-function calculateIntraSliceMatePairTemplateLength(thisRecord, mateRecord) {
+function calculateIntraSliceMatePairTemplateLength(
+  thisRecord: CramRecord,
+  mateRecord: CramRecord,
+) {
   // this just estimates the template length by using the simple (non-gapped) end coordinate of each
   // read, because gapping in the alignment doesn't mean the template is longer or shorter
   const start = Math.min(thisRecord.alignmentStart, mateRecord.alignmentStart)
@@ -76,10 +94,10 @@ function calculateIntraSliceMatePairTemplateLength(thisRecord, mateRecord) {
  * just one record having the index in the slice of its mate
  */
 function associateIntraSliceMate(
-  allRecords,
-  currentRecordNumber,
-  thisRecord,
-  mateRecord,
+  allRecords: CramRecord[],
+  currentRecordNumber: number,
+  thisRecord: CramRecord,
+  mateRecord: CramRecord,
 ) {
   if (!mateRecord) {
     throw new CramMalformedError(
@@ -162,40 +180,48 @@ function associateIntraSliceMate(
 }
 
 export default class CramSlice {
-  constructor(container, position) {
-    this.container = container
+  private file: CramFile
+
+  constructor(
+    public container: CramContainer,
+    public containerPosition: number,
+    _unused: number,
+  ) {
     this.file = container.file
-    this.containerPosition = position
   }
 
   // memoize
-  async getHeader() {
+  async getHeader(): Promise<SliceHeader> {
     // fetch and parse the slice header
     const sectionParsers = await this.file.getSectionParsers()
     const containerHeader = await this.container.getHeader()
     const header = await this.file.readBlock(
       containerHeader._endPosition + this.containerPosition,
     )
+    if (header === undefined) {
+      throw new Error()
+    }
     if (header.contentType === 'MAPPED_SLICE_HEADER') {
-      header.content = parseItem(
+      const content = parseItem(
         header.content,
         sectionParsers.cramMappedSliceHeader.parser,
         0,
         containerHeader._endPosition,
       )
+      return { ...header, parsedContent: content }
     } else if (header.contentType === 'UNMAPPED_SLICE_HEADER') {
-      header.content = parseItem(
+      const content = parseItem(
         header.content,
         sectionParsers.cramUnmappedSliceHeader.parser,
         0,
         containerHeader._endPosition,
       )
+      return { ...header, parsedContent: content }
     } else {
       throw new CramMalformedError(
-        `error reading slice header block, invalid content type ${header._contentType}`,
+        `error reading slice header block, invalid content type ${header.contentType}`,
       )
     }
-    return header
   }
 
   // memoize
@@ -203,9 +229,13 @@ export default class CramSlice {
     const header = await this.getHeader()
     // read all the blocks into memory and store them
     let blockPosition = header._endPosition
-    const blocks = new Array(header.content.numBlocks)
+    const blocks: CramFileBlock[] = new Array(header.parsedContent.numBlocks)
     for (let i = 0; i < blocks.length; i += 1) {
-      blocks[i] = await this.file.readBlock(blockPosition)
+      const block = await this.file.readBlock(blockPosition)
+      if (block === undefined) {
+        throw new Error()
+      }
+      blocks[i] = block
       blockPosition = blocks[i]._endPosition
     }
 
@@ -220,9 +250,9 @@ export default class CramSlice {
   }
 
   // memoize
-  async _getBlocksContentIdIndex() {
+  async _getBlocksContentIdIndex(): Promise<Record<number, CramFileBlock>> {
     const blocks = await this.getBlocks()
-    const blocksByContentId = {}
+    const blocksByContentId: Record<number, CramFileBlock> = {}
     blocks.forEach(block => {
       if (block.contentType === 'EXTERNAL_DATA') {
         blocksByContentId[block.contentId] = block
@@ -231,37 +261,47 @@ export default class CramSlice {
     return blocksByContentId
   }
 
-  async getBlockByContentId(id) {
+  async getBlockByContentId(id: number) {
     const blocksByContentId = await this._getBlocksContentIdIndex()
     return blocksByContentId[id]
   }
 
   async getReferenceRegion() {
     // read the slice header
-    const sliceHeader = (await this.getHeader()).content
+    const sliceHeader = (await this.getHeader()).parsedContent
+    if (!isMappedSliceHeader(sliceHeader)) {
+      throw new Error()
+    }
 
     if (sliceHeader.refSeqId < 0) {
       return undefined
     }
 
     const compressionScheme = await this.container.getCompressionScheme()
+    if (compressionScheme === undefined) {
+      throw new Error()
+    }
 
     // console.log(JSON.stringify(sliceHeader, null, '  '))
 
     if (sliceHeader.refBaseBlockId >= 0) {
-      const refBlock = this.getBlockByContentId(sliceHeader.refBaseBlockId)
+      const refBlock = await this.getBlockByContentId(
+        sliceHeader.refBaseBlockId,
+      )
       if (!refBlock) {
         throw new CramMalformedError(
           'embedded reference specified, but reference block does not exist',
         )
       }
 
-      if (sliceHeader.span > refBlock.uncompressedSize) {
-        throw new CramMalformedError('Embedded reference is too small')
-      }
+      // TODO: we do not read anything named 'span'
+      // if (sliceHeader.span > refBlock.uncompressedSize) {
+      //   throw new CramMalformedError('Embedded reference is too small')
+      // }
 
+      // TODO verify
       return {
-        seq: refBlock.data.toString('utf8'),
+        seq: (refBlock as any).data.toString('utf8'),
         start: sliceHeader.refSeqStart,
         end: sliceHeader.refSeqStart + sliceHeader.refSeqSpan - 1,
         span: sliceHeader.refSeqSpan,
@@ -308,8 +348,14 @@ export default class CramSlice {
     const { majorVersion } = await this.file.getDefinition()
 
     const compressionScheme = await this.container.getCompressionScheme()
+    if (compressionScheme === undefined) {
+      throw new Error()
+    }
 
     const sliceHeader = await this.getHeader()
+    if (sliceHeader === undefined) {
+      throw new Error()
+    }
 
     const blocksByContentId = await this._getBlocksContentIdIndex()
 
@@ -317,19 +363,20 @@ export default class CramSlice {
     if (
       majorVersion > 1 &&
       this.file.options.checkSequenceMD5 &&
-      sliceHeader.content.refSeqId >= 0 &&
-      sliceHeader.content.md5.join('') !== '0000000000000000'
+      isMappedSliceHeader(sliceHeader.parsedContent) &&
+      sliceHeader.parsedContent.refSeqId >= 0 &&
+      sliceHeader.parsedContent.md5.join('') !== '0000000000000000'
     ) {
       const refRegion = await this.getReferenceRegion()
       if (refRegion) {
         const { seq, start, end } = refRegion
         const seqMd5 = sequenceMD5(seq)
-        const storedMd5 = sliceHeader.content.md5
+        const storedMd5 = sliceHeader.parsedContent.md5
           .map(byte => (byte < 16 ? '0' : '') + byte.toString(16))
           .join('')
         if (seqMd5 !== storedMd5) {
           throw new CramMalformedError(
-            `MD5 checksum reference mismatch for ref ${sliceHeader.content.refSeqId} pos ${start}..${end}. recorded MD5: ${storedMd5}, calculated MD5: ${seqMd5}`,
+            `MD5 checksum reference mismatch for ref ${sliceHeader.parsedContent.refSeqId} pos ${start}..${end}. recorded MD5: ${storedMd5}, calculated MD5: ${seqMd5}`,
           )
         }
       }
@@ -340,12 +387,14 @@ export default class CramSlice {
     // data note that we are only decoding a single block here, the core
     // data block
     const coreDataBlock = await this.getCoreDataBlock()
-    const cursors = {
-      lastAlignmentStart: sliceHeader.content.refSeqStart || 0,
+    const cursors: Cursors = {
+      lastAlignmentStart: isMappedSliceHeader(sliceHeader.parsedContent)
+        ? sliceHeader.parsedContent.refSeqStart
+        : 0,
       coreBlock: { bitPosition: 7, bytePosition: 0 },
       externalBlocks: {
         map: new Map(),
-        getCursor(contentId) {
+        getCursor(contentId: number) {
           let r = this.map.get(contentId)
           if (r === undefined) {
             r = { bitPosition: 7, bytePosition: 0 }
@@ -356,7 +405,11 @@ export default class CramSlice {
       },
     }
 
-    const decodeDataSeries = dataSeriesName => {
+    const decodeDataSeries: DataSeriesDecoder = <
+      T extends DataSeriesEncodingKey,
+    >(
+      dataSeriesName: T,
+    ): DataTypeMapping[DataSeriesTypes[T]] => {
       const codec = compressionScheme.getCodecForDataSeries(dataSeriesName)
       if (!codec) {
         throw new CramMalformedError(
@@ -364,12 +417,18 @@ export default class CramSlice {
         )
       }
       // console.log(dataSeriesName, Object.getPrototypeOf(codec))
-      return codec.decode(this, coreDataBlock, blocksByContentId, cursors)
+      const decoded = codec.decode(
+        this,
+        coreDataBlock,
+        blocksByContentId,
+        cursors,
+      )
+      return decoded
     }
-    let records = new Array(sliceHeader.content.numRecords)
+    let records: CramRecord[] = new Array(sliceHeader.parsedContent.numRecords)
     for (let i = 0; i < records.length; i += 1) {
       try {
-        records[i] = decodeRecord(
+        const init = decodeRecord(
           this,
           decodeDataSeries,
           compressionScheme,
@@ -380,11 +439,14 @@ export default class CramSlice {
           majorVersion,
           i,
         )
-        records[i].uniqueId =
-          sliceHeader.contentPosition +
-          sliceHeader.content.recordCounter +
-          i +
-          1
+        records[i] = new CramRecord({
+          ...init,
+          uniqueId:
+            sliceHeader.contentPosition +
+            sliceHeader.parsedContent.recordCounter +
+            i +
+            1,
+        })
       } catch (e) {
         if (e instanceof CramBufferOverrunError) {
           console.warn(
@@ -402,7 +464,7 @@ export default class CramSlice {
     // objects Resolve mate pair cross-references between records in this slice
     for (let i = 0; i < records.length; i += 1) {
       const { mateRecordNumber } = records[i]
-      if (mateRecordNumber >= 0) {
+      if (mateRecordNumber !== undefined && mateRecordNumber >= 0) {
         associateIntraSliceMate(
           records,
           i,
@@ -415,30 +477,38 @@ export default class CramSlice {
     return records
   }
 
-  async getRecords(filterFunction) {
+  async getRecords(filterFunction: (r: CramRecord) => boolean) {
     // fetch the features if necessary, using the file-level feature cache
     const cacheKey = this.container.filePosition + this.containerPosition
-    let recordsPromise = this.file.featureCache.get(cacheKey)
+    let recordsPromise = this.file.featureCache.get(cacheKey.toString())
     if (!recordsPromise) {
       recordsPromise = this._fetchRecords()
-      this.file.featureCache.set(cacheKey, recordsPromise)
+      this.file.featureCache.set(cacheKey.toString(), recordsPromise)
     }
 
-    const records = (await recordsPromise).filter(filterFunction)
+    const unfiltered = await recordsPromise
+    const records = unfiltered.filter(filterFunction)
 
     // if we can fetch reference sequence, add the reference sequence to the records
     if (records.length && this.file.fetchReferenceSequenceCallback) {
       const sliceHeader = await this.getHeader()
       if (
-        sliceHeader.content.refSeqId >= 0 || // single-ref slice
-        sliceHeader.content.refSeqId === -2 // multi-ref slice
+        isMappedSliceHeader(sliceHeader.parsedContent) &&
+        (sliceHeader.parsedContent.refSeqId >= 0 || // single-ref slice
+          sliceHeader.parsedContent.refSeqId === -2) // multi-ref slice
       ) {
         const singleRefId =
-          sliceHeader.content.refSeqId >= 0
-            ? sliceHeader.content.refSeqId
+          sliceHeader.parsedContent.refSeqId >= 0
+            ? sliceHeader.parsedContent.refSeqId
             : undefined
         const compressionScheme = await this.container.getCompressionScheme()
-        const refRegions = {} // seqId => { start, end, seq }
+        if (compressionScheme === undefined) {
+          throw new Error()
+        }
+        const refRegions: Record<
+          string,
+          { id: number; start: number; end: number; seq: string | null }
+        > = {} // seqId => { start, end, seq }
 
         // iterate over the records to find the spans of the reference sequences we need to fetch
         for (let i = 0; i < records.length; i += 1) {
@@ -450,6 +520,7 @@ export default class CramSlice {
               id: seqId,
               start: records[i].alignmentStart,
               end: -Infinity,
+              seq: null,
             }
             refRegions[seqId] = refRegion
           }
@@ -485,7 +556,11 @@ export default class CramSlice {
             singleRefId !== undefined ? singleRefId : records[i].sequenceId
           const refRegion = refRegions[seqId]
           if (refRegion && refRegion.seq) {
-            records[i].addReferenceSequence(refRegion, compressionScheme)
+            const seq = refRegion.seq
+            records[i].addReferenceSequence(
+              { ...refRegion, seq },
+              compressionScheme,
+            )
           }
         }
       }

package/src/cramFile/{util.js → util.ts}

@@ -1,7 +1,8 @@
 import md5 from 'md5'
-import { CramBufferOverrunError } from '../errors'
+import { Parser } from '@gmod/binary-parser'
+import { CramBufferOverrunError } from './codecs/getBits'
 
-export function itf8Size(v) {
+export function itf8Size(v: number) {
   if (!(v & ~0x7f)) {
     return 1
   }
@@ -17,21 +18,24 @@ export function itf8Size(v) {
   return 5
 }
 
-export function parseItf8(buffer, initialOffset) {
+export function parseItf8(
+  buffer: Uint8Array,
+  initialOffset: number,
+): [number, number] {
   let offset = initialOffset
   const countFlags = buffer[offset]
   let result
   if (countFlags < 0x80) {
     result = countFlags
-    offset += 1
+    offset = offset + 1
   } else if (countFlags < 0xc0) {
     result = ((countFlags << 8) | buffer[offset + 1]) & 0x3fff
-    offset += 2
+    offset = offset + 2
   } else if (countFlags < 0xe0) {
     result =
       ((countFlags << 16) | (buffer[offset + 1] << 8) | buffer[offset + 2]) &
       0x1fffff
-    offset += 3
+    offset = offset + 3
   } else if (countFlags < 0xf0) {
     result =
       ((countFlags << 24) |
@@ -39,7 +43,7 @@ export function parseItf8(buffer, initialOffset) {
         (buffer[offset + 2] << 8) |
         buffer[offset + 3]) &
       0x0fffffff
-    offset += 4
+    offset = offset + 4
   } else {
     result =
       ((countFlags & 0x0f) << 28) |
@@ -49,7 +53,7 @@ export function parseItf8(buffer, initialOffset) {
       (buffer[offset + 4] & 0x0f)
     // x=((0xff & 0x0f)<<28) | (0xff<<20) | (0xff<<12) | (0xff<<4) | (0x0f & 0x0f);
     // TODO *val_p = uv < 0x80000000UL ? uv : -((int32_t) (0xffffffffUL - uv)) - 1;
-    offset += 5
+    offset = offset + 5
   }
   if (offset > buffer.length) {
     throw new CramBufferOverrunError(
@@ -133,23 +137,30 @@ export function parseItf8(buffer, initialOffset) {
 //   return [result, offset - initialOffset]
 // },
 
-export function parseItem(
-  buffer,
-  parser,
+export type ParsedItem<T> = T & {
+  _endPosition: number
+  _size: number
+}
+
+export function parseItem<T>(
+  buffer: Buffer,
+  parser: Parser<T>,
   startBufferPosition = 0,
   startFilePosition = 0,
-) {
+): ParsedItem<T> {
   const { offset, result } = parser.parse(buffer)
-  result._endPosition = offset + startFilePosition
-  result._size = offset - startBufferPosition
-  return result
+  return {
+    ...result,
+    _endPosition: offset + startFilePosition,
+    _size: offset - startBufferPosition,
+  }
 }
 
 // this would be nice as a decorator, but i'm a little worried about
 // babel support for it going away or changing.
 // memoizes a method in the stupidest possible way, with no regard for the
 // arguments.  actually, this only works on methods that take no arguments
-export function tinyMemoize(_class, methodName) {
+export function tinyMemoize(_class: any, methodName: any) {
   const method = _class.prototype[methodName]
   const memoAttrName = `_memo_${methodName}`
   _class.prototype[methodName] = function _tinyMemoized() {
@@ -164,6 +175,6 @@ export function tinyMemoize(_class, methodName) {
   }
 }
 
-export function sequenceMD5(seq) {
+export function sequenceMD5(seq: string) {
   return md5(seq.toUpperCase().replace(/[^\x21-\x7e]/g, ''))
 }

package/src/{errors.js → errors.ts}

@@ -6,11 +6,6 @@ export class CramUnimplementedError extends Error {}
 /** An error caused by malformed data.  */
 export class CramMalformedError extends CramError {}
 
-/**
- * An error caused by attempting to read beyond the end of the defined data.
- */
-export class CramBufferOverrunError extends CramMalformedError {}
-
 /**
  * An error caused by data being too big, exceeding a size limit.
  */