@gmod/gbz-base 0.0.1 → 1.0.0

package/package.json

@@ -1,9 +1,16 @@
 {
   "name": "@gmod/gbz-base",
-  "version": "0.0.1",
+  "version": "1.0.0",
   "description": "Pure TypeScript reader for gbz-base pangenome databases (.gbz.db) with random access over HTTP range requests",
   "license": "MIT",
+  "packageManager": "pnpm@11.15.1",
   "type": "module",
+  "sideEffects": false,
+  "author": {
+    "name": "Colin Diesh",
+    "email": "colin.diesh@gmail.com",
+    "url": "https://github.com/cmdcolin"
+  },
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "exports": {
@@ -18,21 +25,44 @@
   },
   "files": [
     "dist",
+    "src",
     "bin",
     "tools/haplotype-index/Cargo.toml",
     "tools/haplotype-index/src"
   ],
+  "scripts": {
+    "test": "vitest",
+    "coverage": "pnpm test --coverage",
+    "clean": "rimraf dist",
+    "format": "prettier --write .",
+    "format:check": "prettier --check .",
+    "lint": "eslint --report-unused-disable-directives --max-warnings 0",
+    "typecheck": "tsc --noEmit -p tsconfig.lint.json",
+    "prebuild": "pnpm clean",
+    "build": "tsc",
+    "preversion": "pnpm lint && pnpm format:check && pnpm typecheck && pnpm test --run && pnpm build",
+    "version": "git-cliff --tag v$npm_package_version --unreleased --prepend CHANGELOG.md && git add CHANGELOG.md",
+    "postversion": "git push --follow-tags"
+  },
   "dependencies": {
     "generic-filehandle2": "^2.0.0"
   },
   "devDependencies": {
-    "@types/node": "^24.0.0",
-    "typescript": "^5.9.0",
-    "vitest": "^3.2.0"
+    "@eslint/js": "^10.0.1",
+    "@types/node": "^25.9.5",
+    "@vitest/coverage-v8": "^4.1.10",
+    "eslint": "^10.8.0",
+    "eslint-plugin-import-x": "^4.17.1",
+    "git-cliff": "^2.13.1",
+    "prettier": "^3.9.6",
+    "rimraf": "^6.1.3",
+    "typescript": "^6.0.3",
+    "typescript-eslint": "^8.65.0",
+    "vitest": "^4.1.10"
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/GMOD/gbz-base-js.git"
+    "url": "git+https://github.com/GMOD/gbz-base-js.git"
   },
   "keywords": [
     "pangenome",
@@ -44,10 +74,5 @@
   ],
   "publishConfig": {
     "access": "public"
-  },
-  "scripts": {
-    "build": "tsc -p tsconfig.build.json",
-    "test": "vitest run",
-    "typecheck": "tsc -p tsconfig.json --noEmit"
   }
-}
+}

package/src/cli.ts

@@ -0,0 +1,203 @@
+import { LocalFile, RemoteFile } from 'generic-filehandle2'
+
+import { GBZBase, formatPathName } from './db.ts'
+import {
+  subgraphAroundNodes,
+  subgraphAtOffset,
+  subgraphInInterval,
+} from './query.ts'
+
+import type { HaplotypeOutput } from './subgraph.ts'
+
+const USAGE = `Usage: gbz-base-query [options] graph.gbz.db
+
+  --sample STR         sample name (default: generic path)
+  --contig STR         contig name (required for --offset and --interval)
+  --haplotype INT      haplotype number (default: 0)
+  -o, --offset INT     sequence offset
+  -i, --interval A..B  half-open sequence interval
+  -n, --node INT       node identifier (may repeat)
+  --context INT        context length in bp (default: 100)
+  --limit INT          safety limit for the number of nodes
+  --haplotypes SEL     all, distinct, reference-only or none (default: all)
+  --cigar              output CIGAR strings for the haplotypes
+  --resolve            name haplotypes from the HaplotypeSamples table
+  --alignments         print one alignment record per haplotype fragment instead of the subgraph
+  --block-size INT     bytes fetched per range request (default: 65536)
+  --stats              print fetch statistics to stderr
+`
+
+interface Args {
+  file: string
+  sample?: string
+  contig?: string
+  haplotype: number
+  offset?: number
+  interval?: [number, number]
+  nodes: number[]
+  context: number
+  limit?: number
+  haplotypes: HaplotypeOutput
+  cigar: boolean
+  resolve: boolean
+  alignments: boolean
+  blockSize: number
+  stats: boolean
+}
+
+function parseArgs(argv: string[]): Args {
+  const args: Args = {
+    file: '',
+    haplotype: 0,
+    nodes: [],
+    context: 100,
+    haplotypes: 'all',
+    cigar: false,
+    resolve: false,
+    alignments: false,
+    blockSize: 65536,
+    stats: false,
+  }
+  const next = (i: number) => {
+    const value = argv[i + 1]
+    if (value === undefined) {
+      throw new Error(`${argv[i]} needs a value`)
+    }
+    return value
+  }
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i]!
+    switch (arg) {
+      case '--sample':
+        args.sample = next(i++)
+        break
+      case '--contig':
+        args.contig = next(i++)
+        break
+      case '--haplotype':
+        args.haplotype = Number(next(i++))
+        break
+      case '-o':
+      case '--offset':
+        args.offset = Number(next(i++))
+        break
+      case '-i':
+      case '--interval': {
+        const [a, b] = next(i++).split('..')
+        args.interval = [Number(a), Number(b)]
+        break
+      }
+      case '-n':
+      case '--node':
+        args.nodes.push(Number(next(i++)))
+        break
+      case '--context':
+        args.context = Number(next(i++))
+        break
+      case '--limit':
+        args.limit = Number(next(i++))
+        break
+      case '--haplotypes':
+        args.haplotypes = next(i++) as HaplotypeOutput
+        break
+      case '--cigar':
+        args.cigar = true
+        break
+      case '--resolve':
+        args.resolve = true
+        break
+      case '--alignments':
+        args.alignments = true
+        args.resolve = true
+        break
+      case '--block-size':
+        args.blockSize = Number(next(i++))
+        break
+      case '--stats':
+        args.stats = true
+        break
+      case '--format':
+        next(i++)
+        break
+      case '-h':
+      case '--help':
+        process.stdout.write(USAGE)
+        process.exit(0)
+        break
+      default:
+        if (arg.startsWith('-')) {
+          throw new Error(`Unknown option ${arg}`)
+        }
+        args.file = arg
+    }
+  }
+  if (!args.file) {
+    throw new Error(USAGE)
+  }
+  return args
+}
+
+export async function main(argv: string[]) {
+  const args = parseArgs(argv)
+  const source = /^https?:\/\//.test(args.file)
+    ? new RemoteFile(args.file)
+    : new LocalFile(args.file)
+  const db = await GBZBase.open(source, { blockSize: args.blockSize })
+  const opts = {
+    context: args.context,
+    haplotypes: args.haplotypes,
+    ...(args.limit === undefined ? {} : { limit: args.limit }),
+  }
+  const query = {
+    contig: args.contig ?? '',
+    haplotype: args.haplotype,
+    ...(args.sample === undefined ? {} : { sample: args.sample }),
+  }
+  const subgraph =
+    args.nodes.length > 0
+      ? await subgraphAroundNodes(db, args.nodes, opts)
+      : args.interval
+        ? await subgraphInInterval(
+            db,
+            query,
+            args.interval[0],
+            args.interval[1],
+            opts,
+          )
+        : args.offset !== undefined
+          ? await subgraphAtOffset(db, query, args.offset, opts)
+          : undefined
+  if (!subgraph) {
+    throw new Error(
+      'Query type must be specified using --offset, --interval or --node',
+    )
+  }
+  if (args.resolve) {
+    await subgraph.identifyPaths()
+  }
+  const output = args.alignments
+    ? subgraph.alignments().map(a => ({
+        ...a,
+        name:
+          a.name && a.hapStart !== undefined && a.hapEnd !== undefined
+            ? formatPathName({ ...a.name, fragment: a.hapStart }, a.hapEnd)
+            : undefined,
+        start: undefined,
+      }))
+    : subgraph.toJSON(args.cigar, {
+        names: args.resolve ? 'resolved' : 'anonymous',
+      })
+  process.stdout.write(`${JSON.stringify(output)}\n`)
+  if (args.stats) {
+    const { fetches, bytesFetched } = db.sqlite.pager
+    const {
+      orderedAlignments,
+      lcsAlignments,
+      identificationSteps,
+      identificationFetches,
+    } = subgraph.stats
+    process.stderr.write(
+      `Subgraph contains ${subgraph.nodeCount} nodes and ${subgraph.pathCount} paths; ${fetches} fetches, ${bytesFetched} bytes; ${orderedAlignments} ordered + ${lcsAlignments} lcs alignments; identification ${identificationSteps} steps, ${identificationFetches} lookups\n`,
+    )
+  }
+}

package/src/db.ts

@@ -0,0 +1,322 @@
+import { ENDMARKER, nodeId, nodeOrientation } from './gbwt/node.ts'
+import { GbwtRecord, decompressEdges } from './gbwt/record.ts'
+import { decodeSequence, encodedSequenceLength } from './gbwt/sequence.ts'
+import { SqliteDatabase } from './sqlite/database.ts'
+
+import type { ByteSource } from './filehandle.ts'
+import type { Pos } from './gbwt/record.ts'
+import type { PagerOptions } from './sqlite/pager.ts'
+import type { SqlValue } from './sqlite/record.ts'
+
+export interface PathName {
+  sample: string
+  contig: string
+  haplotype: number
+  fragment: number
+}
+
+export const GENERIC_SAMPLE = '_gbwt_ref'
+export const SCHEMA_VERSION = 'GBZ-base version 4'
+
+export class SchemaVersionError extends Error {
+  override name = 'SchemaVersionError'
+
+  readonly found: string | undefined
+
+  constructor(found: string | undefined) {
+    super(
+      found === undefined
+        ? `not a gbz-base database: its Tags table has no version`
+        : `unsupported database schema "${found}"; this reader understands "${SCHEMA_VERSION}"`,
+    )
+    this.found = found
+  }
+}
+
+export function formatPathName(name: PathName, end: number) {
+  return `${name.sample}#${name.haplotype}#${name.contig}[${name.fragment}-${end}]`
+}
+
+export interface HaplotypeSample {
+  node: number
+  offset: number
+  pathHandle: number
+  orientation: 'forward' | 'reverse'
+  pathOffset: number
+}
+
+export interface GbzPath {
+  handle: number
+  fwStart: Pos
+  revStart: Pos
+  name: PathName
+  isIndexed: boolean
+}
+
+export class GbzRecord {
+  readonly sequenceLen: number
+  readonly handle: number
+  readonly edges: Pos[]
+  readonly bwt: Uint8Array
+  readonly encodedSequence: Uint8Array
+  readonly next: number | undefined
+  private decoded: string | undefined
+
+  constructor(
+    handle: number,
+    edges: Pos[],
+    bwt: Uint8Array,
+    encodedSequence: Uint8Array,
+    next: number | undefined,
+  ) {
+    this.handle = handle
+    this.edges = edges
+    this.bwt = bwt
+    this.encodedSequence = encodedSequence
+    this.next = next
+    this.sequenceLen = encodedSequenceLength(encodedSequence)
+  }
+
+  get id() {
+    return nodeId(this.handle)
+  }
+
+  get orientation() {
+    return nodeOrientation(this.handle)
+  }
+
+  get sequence() {
+    this.decoded ??= decodeSequence(this.encodedSequence)
+    return this.decoded
+  }
+
+  successors() {
+    return this.edges.filter(e => e.node !== ENDMARKER).map(e => e.node)
+  }
+
+  gbwt() {
+    if (this.edges.length === 0) {
+      throw new Error(`GBWT record for handle ${this.handle} is empty`)
+    }
+    return new GbwtRecord(this.edges, this.bwt)
+  }
+}
+
+function num(value: SqlValue | undefined, what: string) {
+  if (typeof value !== 'number') {
+    throw new Error(`${what} is not a number in the database`)
+  }
+  return value
+}
+
+function str(value: SqlValue | undefined, what: string) {
+  if (typeof value !== 'string') {
+    throw new Error(`${what} is not text in the database`)
+  }
+  return value
+}
+
+function blob(value: SqlValue | undefined, what: string) {
+  if (!(value instanceof Uint8Array)) {
+    throw new Error(`${what} is not a blob in the database`)
+  }
+  return value
+}
+
+function rowToPath(rowid: number, values: SqlValue[]): GbzPath {
+  return {
+    handle: rowid,
+    fwStart: {
+      node: num(values[1], 'Paths.fw_node'),
+      offset: num(values[2], 'Paths.fw_offset'),
+    },
+    revStart: {
+      node: num(values[3], 'Paths.rev_node'),
+      offset: num(values[4], 'Paths.rev_offset'),
+    },
+    name: {
+      sample: str(values[5], 'Paths.sample'),
+      contig: str(values[6], 'Paths.contig'),
+      haplotype: num(values[7], 'Paths.haplotype'),
+      fragment: num(values[8], 'Paths.fragment'),
+    },
+    isIndexed: num(values[9], 'Paths.is_indexed') !== 0,
+  }
+}
+
+export class GBZBase {
+  private tagCache: Promise<Map<string, string>> | undefined
+  private pathCache: Promise<GbzPath[]> | undefined
+
+  readonly sqlite: SqliteDatabase
+
+  private constructor(sqlite: SqliteDatabase) {
+    this.sqlite = sqlite
+  }
+
+  static async open(source: ByteSource, opts: PagerOptions = {}) {
+    const sqlite = await SqliteDatabase.open(source, opts)
+    for (const table of ['Tags', 'Nodes', 'Paths', 'ReferenceIndex']) {
+      sqlite.rootPage(table)
+    }
+    const db = new GBZBase(sqlite)
+    const version = await db.tag('version')
+    if (version !== SCHEMA_VERSION) {
+      throw new SchemaVersionError(version)
+    }
+    return db
+  }
+
+  tags() {
+    this.tagCache ??= (async () => {
+      const tags = new Map<string, string>()
+      for await (const { values } of this.sqlite.scan('Tags')) {
+        tags.set(str(values[0], 'Tags.key'), str(values[1], 'Tags.value'))
+      }
+      return tags
+    })()
+    return this.tagCache
+  }
+
+  async tag(key: string) {
+    return (await this.tags()).get(key)
+  }
+
+  async getRecord(handle: number) {
+    const row = await this.sqlite.byRowid('Nodes', handle)
+    if (!row) {
+      return undefined
+    }
+    const next = row[4]
+    return new GbzRecord(
+      handle,
+      decompressEdges(blob(row[1], 'Nodes.edges')),
+      blob(row[2], 'Nodes.bwt'),
+      blob(row[3], 'Nodes.sequence'),
+      typeof next === 'number' ? next : undefined,
+    )
+  }
+
+  paths() {
+    this.pathCache ??= (async () => {
+      const paths: GbzPath[] = []
+      for await (const { rowid, values } of this.sqlite.scan('Paths')) {
+        paths.push(rowToPath(rowid, values))
+      }
+      return paths
+    })()
+    return this.pathCache
+  }
+
+  async getPath(handle: number) {
+    const row = await this.sqlite.byRowid('Paths', handle)
+    return row ? rowToPath(handle, row) : undefined
+  }
+
+  async findPath(name: PathName) {
+    const candidates = (await this.paths()).filter(
+      p =>
+        p.name.sample === name.sample &&
+        p.name.contig === name.contig &&
+        p.name.haplotype === name.haplotype &&
+        p.name.fragment <= name.fragment,
+    )
+    return candidates.sort((a, b) => b.name.fragment - a.name.fragment)[0]
+  }
+
+  async pathsForSample(sample: string) {
+    return (await this.paths()).filter(p => p.name.sample === sample)
+  }
+
+  get hasHaplotypeIndex() {
+    return (
+      this.sqlite.has('HaplotypeSamples') && this.sqlite.has('HaplotypeLengths')
+    )
+  }
+
+  async haplotypeSampleInterval() {
+    const value = await this.tag('haplotype_index_interval')
+    return value === undefined ? undefined : Number(value)
+  }
+
+  private sampleFromRow(values: SqlValue[]): HaplotypeSample {
+    return {
+      node: num(values[0], 'HaplotypeSamples.node_handle'),
+      offset: num(values[1], 'HaplotypeSamples.node_offset'),
+      pathHandle: num(values[2], 'HaplotypeSamples.path_handle'),
+      orientation:
+        num(values[3], 'HaplotypeSamples.orientation') === 0
+          ? 'forward'
+          : 'reverse',
+      pathOffset: num(values[4], 'HaplotypeSamples.path_offset'),
+    }
+  }
+
+  async haplotypeSamplesInRange(minHandle: number, maxHandle: number) {
+    const samples: HaplotypeSample[] = []
+    for await (const key of this.sqlite.indexScanFrom('HaplotypeSamples', [
+      minHandle,
+      0,
+    ])) {
+      const node = num(key[0], 'HaplotypeSamples.node_handle')
+      if (node > maxHandle) {
+        break
+      }
+      const row = await this.sqlite.byRowid(
+        'HaplotypeSamples',
+        num(key[2], 'HaplotypeSamples rowid'),
+      )
+      if (row) {
+        samples.push(this.sampleFromRow(row))
+      }
+    }
+    return samples
+  }
+
+  async haplotypeSampleAt(node: number, offset: number) {
+    const key = await this.sqlite.indexSeekLE('HaplotypeSamples', [
+      node,
+      offset,
+    ])
+    if (key?.[0] !== node || key[1] !== offset) {
+      return undefined
+    }
+    const row = await this.sqlite.byRowid(
+      'HaplotypeSamples',
+      num(key[2], 'HaplotypeSamples rowid'),
+    )
+    return row ? this.sampleFromRow(row) : undefined
+  }
+
+  async haplotypeLength(pathHandle: number) {
+    const row = await this.sqlite.byRowid('HaplotypeLengths', pathHandle)
+    return row ? num(row[1], 'HaplotypeLengths.length') : undefined
+  }
+
+  async indexedPosition(
+    pathHandle: number,
+    pathOffset: number,
+  ): Promise<{ pathOffset: number; pos: Pos } | undefined> {
+    const key = await this.sqlite.indexSeekLE('ReferenceIndex', [
+      pathHandle,
+      pathOffset,
+    ])
+    if (key?.[0] !== pathHandle) {
+      return undefined
+    }
+    const row = await this.sqlite.byRowid(
+      'ReferenceIndex',
+      num(key[2], 'ReferenceIndex rowid'),
+    )
+    if (!row) {
+      throw new Error('ReferenceIndex row referenced by its index is missing')
+    }
+    return {
+      pathOffset: num(row[1], 'ReferenceIndex.path_offset'),
+      pos: {
+        node: num(row[2], 'ReferenceIndex.node_handle'),
+        offset: num(row[3], 'ReferenceIndex.node_offset'),
+      },
+    }
+  }
+}

package/src/filehandle.ts

@@ -0,0 +1,4 @@
+export interface ByteSource {
+  read(length: number, position: number): Promise<Uint8Array>
+  stat(): Promise<{ size: number }>
+}

package/src/gbwt/bytecode.ts

@@ -0,0 +1,87 @@
+export class ByteCodeReader {
+  offset = 0
+  private bytes: Uint8Array
+
+  constructor(bytes: Uint8Array) {
+    this.bytes = bytes
+  }
+
+  get done() {
+    return this.offset >= this.bytes.length
+  }
+
+  byte() {
+    const value = this.bytes[this.offset]
+    if (value === undefined) {
+      return undefined
+    }
+    this.offset += 1
+    return value
+  }
+
+  int() {
+    let shift = 1
+    let result = 0
+    while (this.offset < this.bytes.length) {
+      const value = this.bytes[this.offset]!
+      this.offset += 1
+      result += (value & 0x7f) * shift
+      shift *= 128
+      if ((value & 0x80) === 0) {
+        return result
+      }
+    }
+    return undefined
+  }
+}
+
+export interface Run {
+  value: number
+  len: number
+}
+
+const RLE_THRESHOLD = 255
+const RLE_UNIVERSE = 256
+
+export class RunReader {
+  private source: ByteCodeReader
+  private sigma: number
+  private threshold: number
+
+  constructor(bytes: Uint8Array, sigma: number) {
+    this.source = new ByteCodeReader(bytes)
+    this.sigma = sigma === 0 ? Number.MAX_SAFE_INTEGER : sigma
+    this.threshold =
+      this.sigma < RLE_THRESHOLD ? Math.floor(RLE_UNIVERSE / this.sigma) : 0
+  }
+
+  next(): Run | undefined {
+    if (this.sigma >= RLE_THRESHOLD) {
+      const value = this.source.int()
+      const len = this.source.int()
+      return value === undefined || len === undefined
+        ? undefined
+        : { value, len: len + 1 }
+    }
+    const byte = this.source.byte()
+    if (byte === undefined) {
+      return undefined
+    }
+    const value = byte % this.sigma
+    let len = Math.floor(byte / this.sigma) + 1
+    if (len === this.threshold) {
+      const extra = this.source.int()
+      if (extra === undefined) {
+        return undefined
+      }
+      len += extra
+    }
+    return { value, len }
+  }
+
+  *[Symbol.iterator]() {
+    for (let run = this.next(); run !== undefined; run = this.next()) {
+      yield run
+    }
+  }
+}