@gmod/gbz-base 0.0.1

package/dist/sqlite/btree.js

@@ -0,0 +1,201 @@
+import { decodeRecord, readVarint } from "./record.js";
+const INTERIOR_INDEX = 0x02;
+const INTERIOR_TABLE = 0x05;
+const LEAF_INDEX = 0x0a;
+const LEAF_TABLE = 0x0d;
+function readHeader(page, start) {
+    const view = new DataView(page.buffer, page.byteOffset, page.byteLength);
+    const type = page[start];
+    if (type !== INTERIOR_INDEX && type !== INTERIOR_TABLE && type !== LEAF_INDEX && type !== LEAF_TABLE) {
+        throw new Error(`SQLite page has unknown b-tree type ${type}`);
+    }
+    const interior = type === INTERIOR_INDEX || type === INTERIOR_TABLE;
+    return {
+        type,
+        cellCount: view.getUint16(start + 3),
+        rightChild: interior ? view.getUint32(start + 8) : 0,
+        cellPointers: start + (interior ? 12 : 8),
+    };
+}
+function cellOffset(page, header, index) {
+    const view = new DataView(page.buffer, page.byteOffset, page.byteLength);
+    return view.getUint16(header.cellPointers + 2 * index);
+}
+function readUint32(page, offset) {
+    const view = new DataView(page.buffer, page.byteOffset, page.byteLength);
+    return view.getUint32(offset);
+}
+export class BTree {
+    pager;
+    usable;
+    constructor(pager, reservedBytes) {
+        this.pager = pager;
+        this.usable = pager.pageSize - reservedBytes;
+    }
+    localPayloadSize(total, isIndex) {
+        const usable = this.usable;
+        const maxLocal = isIndex ? Math.floor(((usable - 12) * 64) / 255) - 23 : usable - 35;
+        if (total <= maxLocal) {
+            return total;
+        }
+        const minLocal = Math.floor(((usable - 12) * 32) / 255) - 23;
+        const spill = minLocal + ((total - minLocal) % (usable - 4));
+        return spill <= maxLocal ? spill : minLocal;
+    }
+    async payload(page, offset, total, isIndex) {
+        const local = this.localPayloadSize(total, isIndex);
+        if (local === total) {
+            return page.subarray(offset, offset + total);
+        }
+        const result = new Uint8Array(total);
+        result.set(page.subarray(offset, offset + local), 0);
+        let filled = local;
+        let next = readUint32(page, offset + local);
+        while (next !== 0 && filled < total) {
+            const overflow = await this.pager.page(next);
+            const chunk = overflow.subarray(4, 4 + Math.min(this.usable - 4, total - filled));
+            result.set(chunk, filled);
+            filled += chunk.length;
+            next = readUint32(overflow, 0);
+        }
+        if (filled !== total) {
+            throw new Error('SQLite overflow chain ended before the payload was complete');
+        }
+        return result;
+    }
+    async pageAt(pageNumber) {
+        const page = await this.pager.page(pageNumber);
+        return { page, header: readHeader(page, pageNumber === 1 ? 100 : 0) };
+    }
+    async tableRowid(root, rowid) {
+        let pageNumber = root;
+        for (;;) {
+            const { page, header } = await this.pageAt(pageNumber);
+            if (header.type === LEAF_TABLE) {
+                let low = 0;
+                let high = header.cellCount;
+                while (low < high) {
+                    const mid = (low + high) >> 1;
+                    const offset = cellOffset(page, header, mid);
+                    const [size, afterSize] = readVarint(page, offset);
+                    const [key, afterKey] = readVarint(page, afterSize);
+                    if (key === rowid) {
+                        return decodeRecord(await this.payload(page, afterKey, size, false));
+                    }
+                    if (key < rowid) {
+                        low = mid + 1;
+                    }
+                    else {
+                        high = mid;
+                    }
+                }
+                return undefined;
+            }
+            if (header.type !== INTERIOR_TABLE) {
+                throw new Error('SQLite table b-tree contains an index page');
+            }
+            let low = 0;
+            let high = header.cellCount;
+            while (low < high) {
+                const mid = (low + high) >> 1;
+                const offset = cellOffset(page, header, mid);
+                const [key] = readVarint(page, offset + 4);
+                if (key < rowid) {
+                    low = mid + 1;
+                }
+                else {
+                    high = mid;
+                }
+            }
+            pageNumber = low === header.cellCount ? header.rightChild : readUint32(page, cellOffset(page, header, low));
+        }
+    }
+    async *tableScan(root) {
+        const { page, header } = await this.pageAt(root);
+        if (header.type === LEAF_TABLE) {
+            for (let i = 0; i < header.cellCount; i++) {
+                const offset = cellOffset(page, header, i);
+                const [size, afterSize] = readVarint(page, offset);
+                const [rowid, afterKey] = readVarint(page, afterSize);
+                yield { rowid, values: decodeRecord(await this.payload(page, afterKey, size, false)) };
+            }
+        }
+        else {
+            for (let i = 0; i < header.cellCount; i++) {
+                yield* this.tableScan(readUint32(page, cellOffset(page, header, i)));
+            }
+            yield* this.tableScan(header.rightChild);
+        }
+    }
+    async indexCell(page, header, index) {
+        const offset = cellOffset(page, header, index);
+        const interior = header.type === INTERIOR_INDEX;
+        const [size, afterSize] = readVarint(page, interior ? offset + 4 : offset);
+        const values = decodeRecord(await this.payload(page, afterSize, size, true));
+        return { values, leftChild: interior ? readUint32(page, offset) : 0 };
+    }
+    async *indexScanFrom(root, low) {
+        const { page, header } = await this.pageAt(root);
+        let first = 0;
+        let high = header.cellCount;
+        while (first < high) {
+            const mid = (first + high) >> 1;
+            const cell = await this.indexCell(page, header, mid);
+            if (compareKey(cell.values, low) < 0) {
+                first = mid + 1;
+            }
+            else {
+                high = mid;
+            }
+        }
+        for (let i = first; i < header.cellCount; i++) {
+            const cell = await this.indexCell(page, header, i);
+            if (header.type === INTERIOR_INDEX) {
+                yield* this.indexScanFrom(cell.leftChild, low);
+            }
+            yield cell.values;
+        }
+        if (header.type === INTERIOR_INDEX) {
+            yield* this.indexScanFrom(header.rightChild, low);
+        }
+    }
+    async indexSeekLE(root, key) {
+        let best;
+        let pageNumber = root;
+        for (;;) {
+            const { page, header } = await this.pageAt(pageNumber);
+            let low = 0;
+            let high = header.cellCount;
+            let child = 0;
+            while (low < high) {
+                const mid = (low + high) >> 1;
+                const cell = await this.indexCell(page, header, mid);
+                if (compareKey(cell.values, key) <= 0) {
+                    best = cell.values;
+                    low = mid + 1;
+                }
+                else {
+                    child = cell.leftChild;
+                    high = mid;
+                }
+            }
+            if (header.type === LEAF_INDEX) {
+                return best;
+            }
+            pageNumber = low === header.cellCount ? header.rightChild : child;
+        }
+    }
+}
+function compareKey(values, key) {
+    for (let i = 0; i < key.length; i++) {
+        const a = values[i];
+        const b = key[i];
+        if (typeof a !== 'number' || b === undefined) {
+            throw new Error('SQLite index key is not numeric');
+        }
+        if (a !== b) {
+            return a < b ? -1 : 1;
+        }
+    }
+    return 0;
+}

package/dist/sqlite/database.d.ts

@@ -0,0 +1,29 @@
+import type { ByteSource } from '../filehandle.ts';
+import { BTree } from './btree.ts';
+import { Pager } from './pager.ts';
+import type { PagerOptions } from './pager.ts';
+import type { SqlValue } from './record.ts';
+export interface SqliteObject {
+    type: string;
+    name: string;
+    tableName: string;
+    rootPage: number;
+    sql: string;
+}
+export declare class SqliteDatabase {
+    readonly pager: Pager;
+    readonly btree: BTree;
+    readonly objects: Map<string, SqliteObject>;
+    private constructor();
+    static open(source: ByteSource, opts?: PagerOptions): Promise<SqliteDatabase>;
+    rootPage(name: string): number;
+    indexOn(tableName: string): number;
+    byRowid(table: string, rowid: number): Promise<SqlValue[] | undefined>;
+    scan(table: string): AsyncGenerator<{
+        rowid: number;
+        values: SqlValue[];
+    }, any, any>;
+    indexSeekLE(table: string, key: number[]): Promise<SqlValue[] | undefined>;
+    indexScanFrom(table: string, low: number[]): AsyncGenerator<SqlValue[], any, any>;
+    has(name: string): boolean;
+}

package/dist/sqlite/database.js

@@ -0,0 +1,71 @@
+import { BTree } from "./btree.js";
+import { Pager } from "./pager.js";
+export class SqliteDatabase {
+    pager;
+    btree;
+    objects;
+    constructor(pager, btree, objects) {
+        this.pager = pager;
+        this.btree = btree;
+        this.objects = objects;
+    }
+    static async open(source, opts = {}) {
+        const { size } = await source.stat();
+        const firstBlock = await source.read(Math.min(opts.blockSize ?? 65536, size), 0);
+        const header = firstBlock.subarray(0, 100);
+        const magic = new TextDecoder().decode(header.subarray(0, 15));
+        if (magic !== 'SQLite format 3') {
+            throw new Error('Not a SQLite database');
+        }
+        const view = new DataView(header.buffer, header.byteOffset, header.byteLength);
+        const rawPageSize = view.getUint16(16);
+        const pageSize = rawPageSize === 1 ? 65536 : rawPageSize;
+        const reserved = header[20] ?? 0;
+        const encoding = view.getUint32(56);
+        if (encoding !== 1) {
+            throw new Error(`SQLite text encoding ${encoding} is not UTF-8`);
+        }
+        const pager = new Pager(source, pageSize, size, opts);
+        pager.seed(0, firstBlock);
+        pager.fetches += 1;
+        pager.bytesFetched += firstBlock.length;
+        const btree = new BTree(pager, reserved);
+        const objects = new Map();
+        for await (const { values } of btree.tableScan(1)) {
+            const [type, name, tableName, rootPage, sql] = values;
+            if (typeof type === 'string' && typeof name === 'string' && typeof tableName === 'string' && typeof rootPage === 'number') {
+                objects.set(name, { type, name, tableName, rootPage, sql: typeof sql === 'string' ? sql : '' });
+            }
+        }
+        return new SqliteDatabase(pager, btree, objects);
+    }
+    rootPage(name) {
+        const object = this.objects.get(name);
+        if (!object) {
+            throw new Error(`SQLite database has no object named ${name}`);
+        }
+        return object.rootPage;
+    }
+    indexOn(tableName) {
+        const index = [...this.objects.values()].find(o => o.type === 'index' && o.tableName === tableName);
+        if (!index) {
+            throw new Error(`SQLite table ${tableName} has no index`);
+        }
+        return index.rootPage;
+    }
+    byRowid(table, rowid) {
+        return this.btree.tableRowid(this.rootPage(table), rowid);
+    }
+    scan(table) {
+        return this.btree.tableScan(this.rootPage(table));
+    }
+    indexSeekLE(table, key) {
+        return this.btree.indexSeekLE(this.indexOn(table), key);
+    }
+    indexScanFrom(table, low) {
+        return this.btree.indexScanFrom(this.indexOn(table), low);
+    }
+    has(name) {
+        return this.objects.has(name);
+    }
+}

package/dist/sqlite/pager.d.ts

@@ -0,0 +1,19 @@
+import type { ByteSource } from '../filehandle.ts';
+export interface PagerOptions {
+    blockSize?: number;
+    maxBlocks?: number;
+}
+export declare class Pager {
+    private source;
+    readonly pageSize: number;
+    private fileSize;
+    private blocks;
+    private readonly blockSize;
+    private readonly maxBlocks;
+    bytesFetched: number;
+    fetches: number;
+    constructor(source: ByteSource, pageSize: number, fileSize: number, opts?: PagerOptions);
+    seed(index: number, bytes: Uint8Array): void;
+    private block;
+    page(pageNumber: number): Promise<Uint8Array<ArrayBufferLike>>;
+}

package/dist/sqlite/pager.js

@@ -0,0 +1,55 @@
+export class Pager {
+    source;
+    pageSize;
+    fileSize;
+    blocks = new Map();
+    blockSize;
+    maxBlocks;
+    bytesFetched = 0;
+    fetches = 0;
+    constructor(source, pageSize, fileSize, opts = {}) {
+        this.source = source;
+        this.pageSize = pageSize;
+        this.fileSize = fileSize;
+        const requested = opts.blockSize ?? 65536;
+        this.blockSize = Math.max(pageSize, Math.ceil(requested / pageSize) * pageSize);
+        this.maxBlocks = opts.maxBlocks ?? 256;
+    }
+    seed(index, bytes) {
+        if (bytes.length === Math.min(this.blockSize, this.fileSize - index * this.blockSize)) {
+            this.blocks.set(index, Promise.resolve(bytes));
+        }
+    }
+    block(index) {
+        const cached = this.blocks.get(index);
+        if (cached) {
+            this.blocks.delete(index);
+            this.blocks.set(index, cached);
+            return cached;
+        }
+        const start = index * this.blockSize;
+        const length = Math.min(this.blockSize, this.fileSize - start);
+        const pending = this.source.read(length, start).then(bytes => {
+            this.bytesFetched += bytes.length;
+            return bytes;
+        });
+        this.fetches += 1;
+        this.blocks.set(index, pending);
+        if (this.blocks.size > this.maxBlocks) {
+            const oldest = this.blocks.keys().next().value;
+            if (oldest !== undefined) {
+                this.blocks.delete(oldest);
+            }
+        }
+        return pending;
+    }
+    async page(pageNumber) {
+        const offset = (pageNumber - 1) * this.pageSize;
+        if (pageNumber < 1 || offset + this.pageSize > this.fileSize) {
+            throw new Error(`SQLite page ${pageNumber} is outside the file`);
+        }
+        const bytes = await this.block(Math.floor(offset / this.blockSize));
+        const within = offset % this.blockSize;
+        return bytes.subarray(within, within + this.pageSize);
+    }
+}

package/dist/sqlite/record.d.ts

@@ -0,0 +1,3 @@
+export type SqlValue = null | number | string | Uint8Array;
+export declare function readVarint(bytes: Uint8Array, offset: number): [value: number, next: number];
+export declare function decodeRecord(payload: Uint8Array): SqlValue[];

package/dist/sqlite/record.js

@@ -0,0 +1,85 @@
+export function readVarint(bytes, offset) {
+    let value = 0;
+    for (let i = 0; i < 8; i++) {
+        const byte = bytes[offset + i];
+        if (byte === undefined) {
+            throw new Error('SQLite varint runs past the end of the buffer');
+        }
+        value = value * 128 + (byte & 0x7f);
+        if ((byte & 0x80) === 0) {
+            return [value, offset + i + 1];
+        }
+    }
+    const last = bytes[offset + 8];
+    if (last === undefined) {
+        throw new Error('SQLite varint runs past the end of the buffer');
+    }
+    value = value * 256 + last;
+    if (!Number.isSafeInteger(value)) {
+        throw new Error('SQLite varint exceeds the safe integer range');
+    }
+    return [value, offset + 9];
+}
+function readSignedInt(view, offset, width) {
+    switch (width) {
+        case 1:
+            return view.getInt8(offset);
+        case 2:
+            return view.getInt16(offset);
+        case 3:
+            return (view.getInt8(offset) << 16) | view.getUint16(offset + 1);
+        case 4:
+            return view.getInt32(offset);
+        case 6:
+            return view.getInt16(offset) * 4294967296 + view.getUint32(offset + 2);
+        default: {
+            const big = view.getBigInt64(offset);
+            if (big > BigInt(Number.MAX_SAFE_INTEGER) || big < BigInt(Number.MIN_SAFE_INTEGER)) {
+                throw new Error(`SQLite integer ${big} exceeds the safe integer range`);
+            }
+            return Number(big);
+        }
+    }
+}
+const decoder = new TextDecoder();
+export function decodeRecord(payload) {
+    const [headerSize, firstType] = readVarint(payload, 0);
+    const types = [];
+    let offset = firstType;
+    while (offset < headerSize) {
+        const [type, next] = readVarint(payload, offset);
+        types.push(type);
+        offset = next;
+    }
+    const view = new DataView(payload.buffer, payload.byteOffset, payload.byteLength);
+    let body = headerSize;
+    return types.map(type => {
+        if (type === 0) {
+            return null;
+        }
+        if (type >= 1 && type <= 6) {
+            const width = type <= 4 ? type : type === 5 ? 6 : 8;
+            const value = readSignedInt(view, body, width);
+            body += width;
+            return value;
+        }
+        if (type === 7) {
+            const value = view.getFloat64(body);
+            body += 8;
+            return value;
+        }
+        if (type === 8) {
+            return 0;
+        }
+        if (type === 9) {
+            return 1;
+        }
+        if (type >= 12) {
+            const length = (type - 12) >> 1;
+            const slice = payload.subarray(body, body + length);
+            body += length;
+            return type % 2 === 0 ? slice : decoder.decode(slice);
+        }
+        throw new Error(`SQLite serial type ${type} is reserved`);
+    });
+}

package/dist/subgraph.d.ts

@@ -0,0 +1,122 @@
+import type { GBZBase, PathName } from './db.ts';
+import type { Pos } from './gbwt/record.ts';
+import type { Orientation } from './gbwt/node.ts';
+export type HaplotypeOutput = 'all' | 'distinct' | 'reference-only' | 'none';
+export interface PathPosition {
+    seqOffset: number;
+    handle: number;
+    nodeOffset: number;
+    gbwtOffset: number;
+}
+export interface ReferencePath {
+    position: PathPosition;
+    name: PathName;
+    handle: number;
+}
+export interface PathIdentity {
+    pathHandle: number;
+    name: PathName;
+    orientation: Orientation;
+    hapStart: number;
+    hapEnd: number;
+}
+export interface SubgraphPath {
+    name: string;
+    weight?: number;
+    cigar?: string;
+    path: {
+        id: string;
+        is_reverse: boolean;
+    }[];
+}
+export interface SubgraphJson {
+    nodes: {
+        id: string;
+        sequence: string;
+    }[];
+    edges: {
+        from: string;
+        from_is_reverse: boolean;
+        to: string;
+        to_is_reverse: boolean;
+    }[];
+    paths: SubgraphPath[];
+}
+export interface HaplotypeAlignment {
+    pathHandle: number | undefined;
+    name: PathName | undefined;
+    strand: '+' | '-';
+    hapStart: number | undefined;
+    hapEnd: number | undefined;
+    refStart: number;
+    refEnd: number;
+    cigar: string;
+    weight: number | undefined;
+    path: number[];
+    start: Pos;
+}
+export interface ToJsonOptions {
+    names?: 'anonymous' | 'resolved';
+}
+export declare class Subgraph {
+    private db;
+    private records;
+    private paths;
+    private refId;
+    private refPath;
+    private refHandle;
+    private refInterval;
+    private refIndexCache;
+    private refPrefixCache;
+    limit: number | undefined;
+    readonly stats: {
+        orderedAlignments: number;
+        lcsAlignments: number;
+        identificationSteps: number;
+        identificationFetches: number;
+    };
+    constructor(db: GBZBase);
+    get nodeCount(): number;
+    get pathCount(): number;
+    get referenceInterval(): {
+        name: PathName;
+        start: number;
+        end: number;
+    } | undefined;
+    hasNode(id: number): boolean;
+    hasHandle(handle: number): boolean;
+    private record;
+    private sortedHandles;
+    private addNode;
+    private ensureNode;
+    private clearPaths;
+    pathPosition(query: PathName): Promise<ReferencePath>;
+    private findPathPosition;
+    aroundPosition(handle: number, nodeOffset: number, context: number): Promise<{
+        inserted: number;
+        removed: number;
+    }>;
+    aroundInterval(start: PathPosition, len: number, context: number): Promise<{
+        inserted: number;
+        removed: number;
+    }>;
+    aroundNodes(nodes: Iterable<number>, context: number): Promise<{
+        inserted: number;
+        removed: number;
+    }>;
+    private insertContext;
+    extractPaths(reference: ReferencePath | undefined, output: HaplotypeOutput): void;
+    private distinctPaths;
+    identifyPaths(): Promise<void>;
+    private refIndex;
+    private refPrefix;
+    private orderedMatches;
+    private pathLen;
+    private prefixMatches;
+    private suffixMatches;
+    private align;
+    private edits;
+    alignToRef(pathIndex: number): string | undefined;
+    alignments(): HaplotypeAlignment[];
+    toJSON(cigar: boolean, opts?: ToJsonOptions): SubgraphJson;
+}