@gmod/cram 2.0.3 → 3.0.0

package/src/cramFile/slice/index.ts

@@ -7,6 +7,7 @@ import CramRecord from '../record'
 import CramContainer from '../container'
 import CramFile, { CramFileBlock } from '../file'
 import {
+  getSectionParsers,
   isMappedSliceHeader,
   MappedSliceHeader,
   UnmappedSliceHeader,
@@ -191,10 +192,15 @@ export default class CramSlice {
   }
 
   // memoize
-  async getHeader(): Promise<SliceHeader> {
+  async getHeader() {
     // fetch and parse the slice header
-    const sectionParsers = await this.file.getSectionParsers()
+    const { majorVersion } = await this.file.getDefinition()
+    const sectionParsers = getSectionParsers(majorVersion)
     const containerHeader = await this.container.getHeader()
+    if (!containerHeader) {
+      throw new Error('no container header detected')
+    }
+
     const header = await this.file.readBlock(
       containerHeader._endPosition + this.containerPosition,
     )
@@ -230,7 +236,7 @@ export default class CramSlice {
     // read all the blocks into memory and store them
     let blockPosition = header._endPosition
     const blocks: CramFileBlock[] = new Array(header.parsedContent.numBlocks)
-    for (let i = 0; i < blocks.length; i += 1) {
+    for (let i = 0; i < blocks.length; i++) {
       const block = await this.file.readBlock(blockPosition)
       if (block === undefined) {
         throw new Error('block undefined')
@@ -363,14 +369,14 @@ export default class CramSlice {
       this.file.options.checkSequenceMD5 &&
       isMappedSliceHeader(sliceHeader.parsedContent) &&
       sliceHeader.parsedContent.refSeqId >= 0 &&
-      sliceHeader.parsedContent.md5.join('') !== '0000000000000000'
+      sliceHeader.parsedContent.md5?.join('') !== '0000000000000000'
     ) {
       const refRegion = await this.getReferenceRegion()
       if (refRegion) {
         const { seq, start, end } = refRegion
         const seqMd5 = sequenceMD5(seq)
         const storedMd5 = sliceHeader.parsedContent.md5
-          .map(byte => (byte < 16 ? '0' : '') + byte.toString(16))
+          ?.map(byte => (byte < 16 ? '0' : '') + byte.toString(16))
           .join('')
         if (seqMd5 !== storedMd5) {
           throw new CramMalformedError(

package/src/cramFile/util.ts

@@ -1,5 +1,5 @@
 import md5 from 'md5'
-import { Parser } from '@gmod/binary-parser'
+import Long from 'long'
 import { CramBufferOverrunError } from './codecs/getBits'
 
 export function itf8Size(v: number) {
@@ -18,10 +18,7 @@ export function itf8Size(v: number) {
   return 5
 }
 
-export function parseItf8(
-  buffer: Uint8Array,
-  initialOffset: number,
-): [number, number] {
+export function parseItf8(buffer: Uint8Array, initialOffset: number) {
   let offset = initialOffset
   const countFlags = buffer[offset]
   let result
@@ -60,106 +57,108 @@ export function parseItf8(
       'Attempted to read beyond end of buffer; this file seems truncated.',
     )
   }
-  return [result, offset - initialOffset]
+  return [result, offset - initialOffset] as const
 }
 
-// parseLtf8(buffer, initialOffset) {
-//   let offset = initialOffset
-//   const countFlags = buffer[offset]
-//   let result
-//   if (countFlags < 0x80) {
-//     result = countFlags
-//     offset += 1
-//   } else if (countFlags < 0xc0) {
-//     result = ((buffer[offset] << 8) | buffer[offset + 1]) & 0x3fff
-//     offset += 2
-//   } else if (countFlags < 0xe0) {
-//     result =
-//       ((buffer[offset] << 16) |
-//         (buffer[offset + 1] << 8) |
-//         buffer[offset + 2]) &
-//       0x1fffff
-//     offset += 3
-//   } else if (countFlags < 0xf0) {
-//     result =
-//       ((buffer[offset] << 24) |
-//         (buffer[offset + 1] << 16) |
-//         (buffer[offset + 2] << 8) |
-//         buffer[offset + 3]) &
-//       0x0fffffff
-//     offset += 4
-//   } else if (countFlags < 0xf8) {
-//     result =
-//       ((buffer[offset] & 15) * Math.pow(2,32) + (buffer[offset + 1] << 24)) |
-//       ((buffer[offset + 2] << 16) |
-//         (buffer[offset + 3] << 8) |
-//         buffer[offset + 4])
-//     // TODO *val_p = uv < 0x80000000UL ? uv : -((int32_t) (0xffffffffUL - uv)) - 1;
-//     offset += 5
-//   } else if (countFlags < 0xfc) {
-//     result =
-//       ((((buffer[offset] & 7) << 8) | buffer[offset + 1]) * Math.pow(2,32) +
-//         (buffer[offset + 2] << 24)) |
-//       ((buffer[offset + 3] << 16) |
-//         (buffer[offset + 4] << 8) |
-//         buffer[offset + 5])
-//     offset += 6
-//   } else if (countFlags < 0xfe) {
-//     result =
-//       ((((buffer[offset] & 3) << 16) |
-//         (buffer[offset + 1] << 8) |
-//         buffer[offset + 2]) *
-//         Math.pow(2,32) +
-//         (buffer[offset + 3] << 24)) |
-//       ((buffer[offset + 4] << 16) |
-//         (buffer[offset + 5] << 8) |
-//         buffer[offset + 6])
-//     offset += 7
-//   } else if (countFlags < 0xff) {
-//     result = Long.fromBytesBE(buffer.slice(offset + 1, offset + 8))
-//     if (
-//       result.greaterThan(Number.MAX_SAFE_INTEGER) ||
-//       result.lessThan(Number.MIN_SAFE_INTEGER)
-//     )
-//       throw new CramUnimplementedError('integer overflow')
-//     result = result.toNumber()
-//     offset += 8
-//   } else {
-//     result = Long.fromBytesBE(buffer.slice(offset + 1, offset + 9))
-//     if (
-//       result.greaterThan(Number.MAX_SAFE_INTEGER) ||
-//       result.lessThan(Number.MIN_SAFE_INTEGER)
-//     )
-//       throw new CramUnimplementedError('integer overflow')
-//     result = result.toNumber()
-//     offset += 9
-//   }
-//   return [result, offset - initialOffset]
-// },
-
-export type ParsedItem<T> = T & {
-  _endPosition: number
-  _size: number
+export function parseLtf8(buffer: Buffer, initialOffset: number) {
+  let offset = initialOffset
+  const countFlags = buffer[offset]
+  let n
+  if (countFlags < 0x80) {
+    n = countFlags
+    offset += 1
+  } else if (countFlags < 0xc0) {
+    n = ((buffer[offset] << 8) | buffer[offset + 1]) & 0x3fff
+    offset += 2
+  } else if (countFlags < 0xe0) {
+    n =
+      ((buffer[offset] << 16) |
+        (buffer[offset + 1] << 8) |
+        buffer[offset + 2]) &
+      0x1fffff
+    n = ((countFlags & 63) << 16) | buffer.readUInt16LE(offset + 1)
+    offset += 3
+  } else if (countFlags < 0xf0) {
+    n =
+      ((buffer[offset] << 24) |
+        (buffer[offset + 1] << 16) |
+        (buffer[offset + 2] << 8) |
+        buffer[offset + 3]) &
+      0x0fffffff
+    offset += 4
+  } else if (countFlags < 0xf8) {
+    n =
+      ((buffer[offset] & 15) * Math.pow(2, 32) + (buffer[offset + 1] << 24)) |
+      ((buffer[offset + 2] << 16) |
+        (buffer[offset + 3] << 8) |
+        buffer[offset + 4])
+    // TODO *val_p = uv < 0x80000000UL ? uv : -((int32_t) (0xffffffffUL - uv)) - 1;
+    offset += 5
+  } else if (countFlags < 0xfc) {
+    n =
+      ((((buffer[offset] & 7) << 8) | buffer[offset + 1]) * Math.pow(2, 32) +
+        (buffer[offset + 2] << 24)) |
+      ((buffer[offset + 3] << 16) |
+        (buffer[offset + 4] << 8) |
+        buffer[offset + 5])
+    offset += 6
+  } else if (countFlags < 0xfe) {
+    n =
+      ((((buffer[offset] & 3) << 16) |
+        (buffer[offset + 1] << 8) |
+        buffer[offset + 2]) *
+        Math.pow(2, 32) +
+        (buffer[offset + 3] << 24)) |
+      ((buffer[offset + 4] << 16) |
+        (buffer[offset + 5] << 8) |
+        buffer[offset + 6])
+    offset += 7
+  } else if (countFlags < 0xff) {
+    n = Long.fromBytesBE(
+      buffer.slice(offset + 1, offset + 8) as unknown as number[],
+    )
+    if (
+      n.greaterThan(Number.MAX_SAFE_INTEGER) ||
+      n.lessThan(Number.MIN_SAFE_INTEGER)
+    ) {
+      throw new Error('integer overflow')
+    }
+    n = n.toNumber()
+    offset += 8
+  } else {
+    n = Long.fromBytesBE(
+      buffer.slice(offset + 1, offset + 9) as unknown as number[],
+    )
+    if (
+      n.greaterThan(Number.MAX_SAFE_INTEGER) ||
+      n.lessThan(Number.MIN_SAFE_INTEGER)
+    ) {
+      throw new Error('integer overflow')
+    }
+    n = n.toNumber()
+    offset += 9
+  }
+  return [n, offset - initialOffset] as const
 }
 
 export function parseItem<T>(
   buffer: Buffer,
-  parser: Parser<T>,
+  parser: (buffer: Buffer, offset: number) => { offset: number; value: T },
   startBufferPosition = 0,
   startFilePosition = 0,
-): ParsedItem<T> {
-  const { offset, result } = parser.parse(buffer)
+) {
+  const { offset, value } = parser(buffer, startBufferPosition)
   return {
-    ...result,
+    ...value,
     _endPosition: offset + startFilePosition,
     _size: offset - startBufferPosition,
   }
 }
 
-// this would be nice as a decorator, but i'm a little worried about
-// babel support for it going away or changing.
-// memoizes a method in the stupidest possible way, with no regard for the
-// arguments.  actually, this only works on methods that take no arguments
+// this would be nice as a decorator, but i'm a little worried about babel
+// support for it going away or changing. memoizes a method in the stupidest
+// possible way, with no regard for the arguments.  actually, this only works
+// on methods that take no arguments
 export function tinyMemoize(_class: any, methodName: any) {
   const method = _class.prototype[methodName]
   const memoAttrName = `_memo_${methodName}`

package/src/indexedCramFile.ts

@@ -1,5 +1,4 @@
-import { CramSizeLimitError, CramUnimplementedError } from './errors'
-
+import { CramUnimplementedError } from './errors'
 import CramFile from './cramFile'
 import CramRecord from './cramFile/record'
 import { SeqFetch } from './cramFile/file'
@@ -24,7 +23,6 @@ export interface CramIndexLike {
 export default class IndexedCramFile {
   public cram: CramFile
   public index: CramIndexLike
-  private fetchSizeLimit: number
 
   /**
    *
@@ -32,14 +30,12 @@ export default class IndexedCramFile {
    * @param {CramFile} args.cram
    * @param {Index-like} args.index object that supports getEntriesForRange(seqId,start,end) -> Promise[Array[index entries]]
    * @param {number} [args.cacheSize] optional maximum number of CRAM records to cache.  default 20,000
-   * @param {number} [args.fetchSizeLimit] optional maximum number of bytes to fetch in a single getRecordsForRange call.  Default 3 MiB.
    * @param {boolean} [args.checkSequenceMD5] - default true. if false, disables verifying the MD5
    * checksum of the reference sequence underlying a slice. In some applications, this check can cause an inconvenient amount (many megabases) of sequences to be fetched.
    */
   constructor(
     args: {
       index: CramIndexLike
-      fetchSizeLimit?: number
     } & (
       | { cram: CramFile }
       | ({
@@ -70,8 +66,6 @@ export default class IndexedCramFile {
     if (!this.index.getEntriesForRange) {
       throw new Error('invalid arguments: not an index')
     }
-
-    this.fetchSizeLimit = args.fetchSizeLimit || 3000000
   }
 
   /**
@@ -103,12 +97,6 @@ export default class IndexedCramFile {
     }
     const seqId = seq
     const slices = await this.index.getEntriesForRange(seqId, start, end)
-    const totalSize = slices.map(s => s.sliceBytes).reduce((a, b) => a + b, 0)
-    if (totalSize > this.fetchSizeLimit) {
-      throw new CramSizeLimitError(
-        `data size of ${totalSize.toLocaleString()} bytes exceeded fetch size limit of ${this.fetchSizeLimit.toLocaleString()} bytes`,
-      )
-    }
 
     // TODO: do we need to merge or de-duplicate the blocks?
 
@@ -180,17 +168,7 @@ export default class IndexedCramFile {
 
       const mateRecordPromises = []
       const mateFeatPromises: Promise<CramRecord[]>[] = []
-
-      const mateTotalSize = mateChunks
-        .map(s => s.sliceBytes)
-        .reduce((a, b) => a + b, 0)
-      if (mateTotalSize > this.fetchSizeLimit) {
-        throw new Error(
-          `mate data size of ${mateTotalSize.toLocaleString()} bytes exceeded fetch size limit of ${this.fetchSizeLimit.toLocaleString()} bytes`,
-        )
-      }
-
-      mateChunks.forEach(c => {
+      for (const c of mateChunks) {
         let recordPromise = this.cram.featureCache.get(c.toString())
         if (!recordPromise) {
           recordPromise = this.getRecordsInSlice(c, () => true)
@@ -210,7 +188,7 @@ export default class IndexedCramFile {
           return mateRecs
         })
         mateFeatPromises.push(featPromise)
-      })
+      }
       const newMateFeats = await Promise.all(mateFeatPromises)
       if (newMateFeats.length) {
         const newMates = newMateFeats.reduce((result, current) =>

package/src/io/index.ts

@@ -1,16 +1,6 @@
-import url from 'url'
-import { ensureNotNullish } from '../typescript'
 import { Filehandle } from '../cramFile/filehandle'
 import { LocalFile, RemoteFile } from 'generic-filehandle'
 
-function fromUrl(source: string) {
-  const { protocol, pathname } = url.parse(source)
-  if (protocol === 'file:') {
-    return new LocalFile(unescape(ensureNotNullish(pathname)))
-  }
-  return new RemoteFile(source)
-}
-
 function open(
   maybeUrl?: string,
   maybePath?: string,
@@ -20,7 +10,7 @@ function open(
     return maybeFilehandle
   }
   if (maybeUrl) {
-    return fromUrl(maybeUrl)
+    return new RemoteFile(maybeUrl)
   }
   if (maybePath) {
     return new LocalFile(maybePath)
@@ -28,6 +18,6 @@ function open(
   throw new Error('no url, path, or filehandle provided, cannot open')
 }
 
-export { fromUrl, open }
+export { open }
 
 export { LocalFile, RemoteFile } from 'generic-filehandle'

package/src/typescript.ts

@@ -1,10 +1,3 @@
-export function ensureNotNullish<T>(x: T | null | undefined): T {
-  if (x === null || x === undefined) {
-    throw new Error('Value must not be nullish.')
-  }
-  return x
-}
-
 export type TupleOf<T, N extends number> = N extends N
   ? number extends N
     ? T[]

package/src/typings/binary-parser.d.ts

@@ -1,44 +0,0 @@
-declare module '@gmod/binary-parser' {
-  export interface Options {
-    stripNull?: boolean
-    formatter?: (item: any) => any
-    length?: number | string | ((this: { $parent: unknown }) => void)
-  }
-
-  export class Parser<T = unknown> {
-    public static start<TStart>(): Parser<TStart>
-
-    public uint8(name?: string | null, options?: Options): Parser
-
-    public itf8(name?: string | null, options?: Options): Parser
-
-    public ltf8(name?: string | null, options?: Options): Parser
-
-    public uint32(name?: string | null, options?: Options): Parser
-
-    public int32(name?: string | null, options?: Options): Parser
-
-    public buffer(name?: string | null, options?: Options): Parser
-
-    public string(name?: string | null, options?: Options): Parser
-
-    public namely(name: string): Parser
-
-    public nest(
-      name?: string | null,
-      options?: { type: Parser | string } & Options,
-    ): Parser
-
-    public choice(
-      name?: string | null,
-      options?: { tag: string; choices: any } & Options,
-    ): Parser
-
-    public array(
-      name?: string | null,
-      options?: { type: string | Parser } & Options,
-    ): Parser
-
-    parse(bytes: Buffer): { result: T; offset: number }
-  }
-}