@gmod/bam 7.6.1 → 7.7.0

package/src/htsget.ts

@@ -2,42 +2,111 @@ import { unzip } from '@gmod/bgzf-filehandle'
 
 import BamFile, { BAM_MAGIC } from './bamFile.ts'
 import Chunk from './chunk.ts'
-import { parseHeaderText } from './sam.ts'
-import { appendInRange, concatUint8Array } from './util.ts'
+import { appendInRange, concatUint8Array, parseRefSeqs } from './util.ts'
 import { VirtualOffset } from './virtualOffset.ts'
 
 import type { BamRecordClass, BamRecordLike } from './bamFile.ts'
 import type BamRecord from './record.ts'
 import type { BamOpts, BaseOpts } from './util.ts'
+import type { Fetcher } from 'generic-filehandle2'
 
 interface HtsgetChunk {
   url: string
   headers?: Record<string, string>
+  // present on either all of a ticket's urls or none; only a hint, since the
+  // blocks still have to be concatenated in ticket order either way
+  class?: 'header' | 'body'
 }
 
-async function fetchOk(url: string, opts?: RequestInit) {
-  const res = await fetch(url, opts)
+interface HtsgetTicket {
+  htsget: { urls: HtsgetChunk[] }
+}
+
+interface HtsgetErrorBody {
+  htsget?: { error?: string; message?: string }
+}
+
+// Errors carry a JSON body naming the error type, e.g. {"htsget":
+// {"error":"InvalidAuthentication","message":"..."}}. Surface that rather than
+// the raw payload, since 401/403 are the first thing a misconfigured token hits.
+function htsgetErrorMessage(text: string) {
+  let parsed: HtsgetErrorBody | undefined
+  try {
+    parsed = JSON.parse(text)
+  } catch {
+    // not JSON, fall through to the raw body
+  }
+  const err = parsed?.htsget
+  return err?.error === undefined
+    ? text
+    : err.message === undefined
+      ? err.error
+      : `${err.error}: ${err.message}`
+}
+
+/**
+ * Offset of the first alignment record in the concatenated ticket blocks.
+ *
+ * The spec has the client concatenate the blocks in ticket order to get a
+ * complete data stream, so whether the header is its own block is up to the
+ * server: htsnexus splits it into a leading `data:` url, while htsget-rs
+ * returns header and records together in a single url. Either way the
+ * concatenation starts with the BAM header, so seek past it rather than
+ * dropping blocks — a url count or class attribute can't tell the two layouts
+ * apart. Returns 0 when the stream has no header, which is what a server
+ * sending body-only blocks produces.
+ */
+function recordsOffset(uncba: Uint8Array) {
+  const dataView = new DataView(
+    uncba.buffer,
+    uncba.byteOffset,
+    uncba.byteLength,
+  )
+  if (uncba.byteLength < 8 || dataView.getInt32(0, true) !== BAM_MAGIC) {
+    return 0
+  }
+  const refs = parseRefSeqs(uncba, 8 + dataView.getInt32(4, true), n => n)
+  if (!refs) {
+    throw new Error('truncated BAM header in htsget response')
+  }
+  return refs.end
+}
+
+async function fetchOk(fetchFn: Fetcher, url: string, opts?: RequestInit) {
+  const res = await fetchFn(url, opts)
   if (!res.ok) {
-    throw new Error(`HTTP ${res.status} fetching ${url}: ${await res.text()}`)
+    throw new Error(
+      `HTTP ${res.status} fetching ${url}: ${htsgetErrorMessage(await res.text())}`,
+    )
   }
   return res
 }
 
-async function fetchChunk({ url, headers }: HtsgetChunk, opts?: RequestInit) {
+async function fetchChunk(
+  fetchFn: Fetcher,
+  { url, headers }: HtsgetChunk,
+  opts?: RequestInit,
+) {
   // pass base64 data URLs straight to fetch; otherwise apply headers (minus
   // referer, which isn't a permitted client-set header).
   // https://stackoverflow.com/a/54123275/2129219
   const { referer: _referer, ...rest } = headers ?? {}
   const res = url.startsWith('data:')
-    ? await fetchOk(url)
-    : await fetchOk(url, { ...opts, headers: rest })
+    ? await fetchOk(fetchFn, url)
+    : await fetchOk(fetchFn, url, { ...opts, headers: rest })
   return new Uint8Array(await res.arrayBuffer())
 }
 
-async function fetchAndConcat(arr: HtsgetChunk[], opts?: RequestInit) {
+async function fetchAndConcat(
+  fetchFn: Fetcher,
+  arr: HtsgetChunk[],
+  opts?: RequestInit,
+) {
   // Pipeline unzip after each fetch so decompression overlaps later fetches.
   return concatUint8Array(
-    await Promise.all(arr.map(async c => unzip(await fetchChunk(c, opts)))),
+    await Promise.all(
+      arr.map(async c => unzip(await fetchChunk(fetchFn, c, opts))),
+    ),
   )
 }
 
@@ -48,14 +117,39 @@ export default class HtsgetFile<
 
   private trackId: string
 
+  private fetchFn: Fetcher
+
   constructor(args: {
     trackId: string
     baseUrl: string
     recordClass?: BamRecordClass<T>
+    /**
+     * fetch implementation used for every request, so an `Authorization: Bearer
+     * <token>` header can be added for servers that require one. It is also
+     * called with the data-block urls from the ticket, which may point at
+     * third-party hosts, so only attach credentials to hosts you trust — the
+     * spec has servers put whatever a data block needs in that url's own
+     * `headers` field, which is applied either way.
+     */
+    fetch?: Fetcher
   }) {
     super({ htsget: true, recordClass: args.recordClass })
     this.baseUrl = args.baseUrl
     this.trackId = args.trackId
+    this.fetchFn = args.fetch ?? ((input, init) => fetch(input, init))
+  }
+
+  /**
+   * Requests a ticket and returns its data blocks decompressed and
+   * concatenated, which per the spec is a complete BAM stream.
+   */
+  private async fetchTicket(query: string, opts?: BaseOpts) {
+    const url = `${this.baseUrl}/${this.trackId}?${query}`
+    const res = await fetchOk(this.fetchFn, url, { signal: opts?.signal })
+    const ticket: HtsgetTicket = await res.json()
+    return fetchAndConcat(this.fetchFn, ticket.htsget.urls, {
+      signal: opts?.signal,
+    })
   }
 
   async getRecordsForRange(
@@ -65,71 +159,32 @@ export default class HtsgetFile<
     opts?: BamOpts,
   ) {
     await this.getHeader(opts)
-    const base = `${this.baseUrl}/${this.trackId}`
-    const url = `${base}?referenceName=${chr}&start=${min}&end=${max}&format=BAM`
     const chrId = this.chrToIndex?.[chr]
     if (chrId === undefined) {
       return []
     }
-    const result = await fetchOk(url, opts)
-    const data = await result.json()
-    const uncba = await fetchAndConcat(data.htsget.urls.slice(1), {
-      signal: opts?.signal,
-    })
-
+    const uncba = await this.fetchTicket(
+      `referenceName=${chr}&start=${min}&end=${max}&format=BAM`,
+      opts,
+    )
     const zero = new VirtualOffset(0, 0)
-    const allRecords = this.readBamFeatures(
-      uncba,
+    const records = this.readBamFeatures(
+      uncba.subarray(recordsOffset(uncba)),
       [],
       [],
       new Chunk(zero, zero, 0),
     )
-
-    return appendInRange(allRecords, chrId, min, max)
+    return appendInRange(records, chrId, min, max)
   }
 
   async getHeaderPre(opts: BaseOpts = {}) {
-    const url = `${this.baseUrl}/${this.trackId}?referenceName=na&class=header`
-    const result = await fetchOk(url, opts)
-    const data = await result.json()
-    const uncba = await fetchAndConcat(data.htsget.urls, {
-      signal: opts.signal,
-    })
-    const dataView = new DataView(
-      uncba.buffer,
-      uncba.byteOffset,
-      uncba.byteLength,
-    )
-
-    if (dataView.getInt32(0, true) !== BAM_MAGIC) {
-      throw new Error('Not a BAM file')
-    }
-    const headLen = dataView.getInt32(4, true)
-
-    const decoder = new TextDecoder()
-    const headerText = decoder.decode(uncba.subarray(8, 8 + headLen))
-    const samHeader = parseHeaderText(headerText)
-
-    // use the @SQ lines in the header to figure out the
-    // mapping between ref ref ID numbers and names
-    const idToName: { refName: string; length: number }[] = []
-    const nameToId: Record<string, number> = {}
-    const sqLines = samHeader.filter(l => l.tag === 'SQ')
-    for (const [refId, sqLine] of sqLines.entries()) {
-      let refName = ''
-      let length = 0
-      for (const item of sqLine.data) {
-        if (item.tag === 'SN') {
-          refName = item.value
-        } else if (item.tag === 'LN') {
-          length = +item.value
-        }
-      }
-      nameToId[refName] = refId
-      idToName[refId] = { refName, length }
+    // format is the only parameter the spec permits alongside class=header;
+    // servers SHOULD reject anything else with InvalidInput
+    const uncba = await this.fetchTicket('class=header&format=BAM', opts)
+    const samHeader = this.applyHeader(uncba)
+    if (!samHeader) {
+      throw new Error('Insufficient data for reference sequences')
     }
-    this.chrToIndex = nameToId
-    this.indexToChr = idToName
     return samHeader
   }
 }

package/src/index.ts

@@ -6,3 +6,5 @@ export { default as HtsgetFile } from './htsget.ts'
 
 export type { Bytes } from './record.ts'
 export type { BamRecordClass, BamRecordLike } from './bamFile.ts'
+// for typing the HtsgetFile `fetch` option
+export type { Fetcher } from 'generic-filehandle2'

package/src/indexFile.ts

@@ -4,7 +4,7 @@ import { optimizeChunks } from './util.ts'
 
 import type Chunk from './chunk.ts'
 import type { BaseOpts } from './util.ts'
-import type { Offset, VirtualOffset } from './virtualOffset.ts'
+import type { OffsetCoords, VirtualOffset } from './virtualOffset.ts'
 import type { GenericFilehandle } from 'generic-filehandle2'
 
 export interface Region {
@@ -21,7 +21,6 @@ export interface RefIndex {
 export interface ParsedIndexBase<R extends RefIndex = RefIndex> {
   firstDataLine: VirtualOffset | undefined
   refCount: number
-  maxBlockSize: number
   indices: (refId: number) => R | undefined
 }
 
@@ -84,7 +83,7 @@ export default abstract class IndexFile<
   protected abstract getLowestChunk(
     refIndex: RefIndex,
     min: number,
-  ): Offset | undefined
+  ): OffsetCoords | undefined
 
   async blocksForRange(
     refId: number,

package/src/sam.ts

@@ -8,9 +8,14 @@ export function parseHeaderText(text: string) {
         tag: tag.slice(1),
         data: fields.map(f => {
           const r = f.indexOf(':')
-          const fieldTag = f.slice(0, r)
-          const value = f.slice(r + 1)
-          return { tag: fieldTag, value }
+          // A field with no colon is not a TAG:VALUE pair — @CO lines are free
+          // text, one comment per line (`samtools view -H c2#pad.3.0.cram`).
+          // Without this branch `slice(0, -1)` silently drops the comment's last
+          // character into the tag. Same shape @gmod/cram's parseHeaderText
+          // returns, so both feed a consumer's header parser identically.
+          return r === -1
+            ? { tag: f, value: '' }
+            : { tag: f.slice(0, r), value: f.slice(r + 1) }
         }),
       })
     }

package/src/util.ts

@@ -1,8 +1,7 @@
 import Chunk from './chunk.ts'
-import { longFromBytesToUnsigned } from './long.ts'
 import { VirtualOffset } from './virtualOffset.ts'
 
-import type { Offset } from './virtualOffset.ts'
+import type { OffsetCoords } from './virtualOffset.ts'
 
 export interface BamOpts {
   viewAsPairs?: boolean
@@ -30,7 +29,7 @@ export interface BaseOpts {
   onProgress?: (bytesDownloaded: number, totalBytes?: number) => void
 }
 
-export function optimizeChunks(chunks: Chunk[], lowest?: Offset) {
+export function optimizeChunks(chunks: Chunk[], lowest?: OffsetCoords) {
   const n = chunks.length
   if (n === 0) {
     return chunks
@@ -102,9 +101,22 @@ export function optimizeChunks(chunks: Chunk[], lowest?: Offset) {
   return mergedChunks
 }
 
+// The pseudo-bin's mapped-record count, a little-endian uint64. Read as two
+// 32-bit halves rather than via BigInt: exact up to Number.MAX_SAFE_INTEGER,
+// which is far past any real record count, and allocates nothing.
 export function parsePseudoBin(bytes: Uint8Array, offset: number) {
+  const low =
+    bytes[offset]! |
+    (bytes[offset + 1]! << 8) |
+    (bytes[offset + 2]! << 16) |
+    (bytes[offset + 3]! << 24)
+  const high =
+    bytes[offset + 4]! |
+    (bytes[offset + 5]! << 8) |
+    (bytes[offset + 6]! << 16) |
+    (bytes[offset + 7]! << 24)
   return {
-    lineCount: longFromBytesToUnsigned(bytes, offset),
+    lineCount: (high >>> 0) * 2 ** 32 + (low >>> 0),
   }
 }
 
@@ -117,13 +129,21 @@ export function parsePseudoBin(bytes: Uint8Array, offset: number) {
 // Shrinks both the byte estimate and the actual fetch with no extra I/O.
 export function clampChunkEnds(
   chunks: Chunk[],
-  extraBoundaries: number[] = [],
+  extraBoundaries: ArrayLike<number> = [],
 ) {
-  const boundaries = [...extraBoundaries]
+  // Float64Array rather than number[]: `extraBoundaries` is the linear index,
+  // which runs to tens of thousands of entries on a human-sized reference, and
+  // this is the only place they get copied.
+  const boundaries = new Float64Array(
+    extraBoundaries.length + chunks.length * 2,
+  )
+  boundaries.set(extraBoundaries)
+  let n = extraBoundaries.length
   for (const c of chunks) {
-    boundaries.push(c.minv.blockPosition, c.maxv.blockPosition)
+    boundaries[n++] = c.minv.blockPosition
+    boundaries[n++] = c.maxv.blockPosition
   }
-  boundaries.sort((a, b) => a - b)
+  boundaries.sort()
 
   for (const c of chunks) {
     const max = c.maxv.blockPosition
@@ -184,14 +204,24 @@ export function parseRefSeqs(
     indexToChr.push({ refName, length: lRef })
     p += 8 + lName
   }
-  return { chrToIndex, indexToChr }
+  // end is the offset just past the header, i.e. where alignment records start
+  return { chrToIndex, indexToChr, end: p }
 }
 
-// SYNC: ~/src/gmod/tabix-js/src/util.ts minVirtualOffset
+// SYNC: ~/src/gmod/tabix-js/src/util.ts minVirtualOffset — but NOT the 0:0
+// skip below, which is only sound for BAM. A tabix'd file with no header lines
+// really does have its first record at 0:0.
 /**
  * The smallest of `current` and the `count` packed virtual offsets starting at
  * `offset`, allocating at most one VirtualOffset rather than one per entry.
  *
+ * 0:0 is skipped rather than treated as the minimum. No BAM record can live
+ * there — the magic and header occupy the start of the file — so it is the
+ * "unset" placeholder htslib leaves in linear-index windows ahead of a
+ * reference's first read (see test/data/HG00096_illumina_lowcov.bam.bai, whose
+ * first three windows are 0). Counting it collapses firstDataLine to 0:0 and
+ * makes callers size a header read from nothing.
+ *
  * The index first pass exists only to find this minimum, and it visits every
  * linear-index entry in the file to do it. Building a VirtualOffset per entry
  * to compare and discard it is the bulk of that pass.
@@ -215,7 +245,10 @@ export function minVirtualOffset(
       bytes[p + 3]! * 0x100 +
       bytes[p + 2]!
     const data = (bytes[p + 1]! << 8) | bytes[p]!
-    if (block < minBlock || (block === minBlock && data < minData)) {
+    if (
+      (block !== 0 || data !== 0) &&
+      (block < minBlock || (block === minBlock && data < minData))
+    ) {
       minBlock = block
       minData = data
       found = true

package/src/virtualOffset.ts

@@ -1,8 +1,15 @@
-export interface Offset {
+/**
+ * The two coordinates a virtual offset packs, without the `blockPos:dataPos`
+ * string form. What every consumer that only compares positions needs — the
+ * BAI linear index stores these as raw numbers and materializes no object.
+ */
+export interface OffsetCoords {
   blockPosition: number
   dataPosition: number
+}
+
+export interface Offset extends OffsetCoords {
   toString(): string
-  compareTo(arg: Offset): number
 }
 
 export class VirtualOffset {
@@ -16,12 +23,6 @@ export class VirtualOffset {
   toString() {
     return `${this.blockPosition}:${this.dataPosition}`
   }
-
-  compareTo(b: VirtualOffset) {
-    return (
-      this.blockPosition - b.blockPosition || this.dataPosition - b.dataPosition
-    )
-  }
 }
 export function fromBytes(bytes: Uint8Array, offset = 0) {
   return new VirtualOffset(

package/dist/long.d.ts

@@ -1 +0,0 @@
-export declare function longFromBytesToUnsigned(source: Uint8Array, i?: number): number;

package/dist/long.js

@@ -1,17 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.longFromBytesToUnsigned = longFromBytesToUnsigned;
-const TWO_PWR_32_DBL = 2 ** 32;
-// avoids dependency on long.js
-function longFromBytesToUnsigned(source, i = 0) {
-    const low = source[i] |
-        (source[i + 1] << 8) |
-        (source[i + 2] << 16) |
-        (source[i + 3] << 24);
-    const high = source[i + 4] |
-        (source[i + 5] << 8) |
-        (source[i + 6] << 16) |
-        (source[i + 7] << 24);
-    return (high >>> 0) * TWO_PWR_32_DBL + (low >>> 0);
-}
-//# sourceMappingURL=long.js.map

package/dist/long.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"long.js","sourceRoot":"","sources":["../src/long.ts"],"names":[],"mappings":";;AAGA,0DAYC;AAfD,MAAM,cAAc,GAAG,CAAC,IAAI,EAAE,CAAA;AAE9B,+BAA+B;AAC/B,SAAgB,uBAAuB,CAAC,MAAkB,EAAE,CAAC,GAAG,CAAC;IAC/D,MAAM,GAAG,GACP,MAAM,CAAC,CAAC,CAAE;QACV,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,IAAI,CAAC,CAAC;QACrB,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,IAAI,EAAE,CAAC;QACtB,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,IAAI,EAAE,CAAC,CAAA;IACxB,MAAM,IAAI,GACR,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE;QACd,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,IAAI,CAAC,CAAC;QACrB,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,IAAI,EAAE,CAAC;QACtB,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,IAAI,EAAE,CAAC,CAAA;IACxB,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,GAAG,cAAc,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAA;AACpD,CAAC"}

package/esm/long.d.ts

@@ -1 +0,0 @@
-export declare function longFromBytesToUnsigned(source: Uint8Array, i?: number): number;

package/esm/long.js

@@ -1,14 +0,0 @@
-const TWO_PWR_32_DBL = 2 ** 32;
-// avoids dependency on long.js
-export function longFromBytesToUnsigned(source, i = 0) {
-    const low = source[i] |
-        (source[i + 1] << 8) |
-        (source[i + 2] << 16) |
-        (source[i + 3] << 24);
-    const high = source[i + 4] |
-        (source[i + 5] << 8) |
-        (source[i + 6] << 16) |
-        (source[i + 7] << 24);
-    return (high >>> 0) * TWO_PWR_32_DBL + (low >>> 0);
-}
-//# sourceMappingURL=long.js.map

package/esm/long.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"long.js","sourceRoot":"","sources":["../src/long.ts"],"names":[],"mappings":"AAAA,MAAM,cAAc,GAAG,CAAC,IAAI,EAAE,CAAA;AAE9B,+BAA+B;AAC/B,MAAM,UAAU,uBAAuB,CAAC,MAAkB,EAAE,CAAC,GAAG,CAAC;IAC/D,MAAM,GAAG,GACP,MAAM,CAAC,CAAC,CAAE;QACV,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,IAAI,CAAC,CAAC;QACrB,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,IAAI,EAAE,CAAC;QACtB,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,IAAI,EAAE,CAAC,CAAA;IACxB,MAAM,IAAI,GACR,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE;QACd,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,IAAI,CAAC,CAAC;QACrB,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,IAAI,EAAE,CAAC;QACtB,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,IAAI,EAAE,CAAC,CAAA;IACxB,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,GAAG,cAAc,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAA;AACpD,CAAC"}

package/src/long.ts

@@ -1,16 +0,0 @@
-const TWO_PWR_32_DBL = 2 ** 32
-
-// avoids dependency on long.js
-export function longFromBytesToUnsigned(source: Uint8Array, i = 0) {
-  const low =
-    source[i]! |
-    (source[i + 1]! << 8) |
-    (source[i + 2]! << 16) |
-    (source[i + 3]! << 24)
-  const high =
-    source[i + 4]! |
-    (source[i + 5]! << 8) |
-    (source[i + 6]! << 16) |
-    (source[i + 7]! << 24)
-  return (high >>> 0) * TWO_PWR_32_DBL + (low >>> 0)
-}