@jbrowse/graph-walks 0.0.0-stage → 0.1.0

package/esm/walkReader.js

@@ -0,0 +1,250 @@
+import { decodeChunk, pieceLinks } from "./walkFormat2.js";
+import { WalkGraph, byteBudgetError, chunkQueryStart, joinPieces, keptStepsError, lookbackChunks, parseWalkRow, stepBudgetError, walkCut, walkHeader, walkRowName, } from "./walkRows.js";
+// A window reads whole chunks and keeps the steps on it, so a narrow window
+// reads several times what it keeps: 464 haplotypes across the 1.5 kb ABCA7
+// repeat read 5.0 M steps and keep 1.16 M, and the cut takes 0.4 s from warm
+// files. The step budget bounds what the cut keeps, and rows are read to this
+// multiple of it.
+const READ_STEPS_PER_KEPT = 8;
+// A walk the read cuts off goes on in the chunk its row names, out to which a
+// cut reads on, in at most this many rounds and this many chunks from the read
+const FOLLOW_ROUNDS = 3;
+const FOLLOW_CHUNKS = 4;
+// bp of reference a cut keeps past each end of the window
+const WALK_CONTEXT = 1000;
+// @gmod/tabix loops forever on a NaN end and reads nothing for a NaN start
+// (GMOD/tabix-js#157), so every query refuses a range that is not finite first
+function checkRange(refName, start, end) {
+    if (!Number.isFinite(start) || !Number.isFinite(end)) {
+        throw new Error(`${refName}:${start}-${end} is not a finite range to query the graph's index for`);
+    }
+}
+function getLines(file, refName, start, end, opts) {
+    checkRange(refName, start, end);
+    return file.getLines(refName, start, end, opts);
+}
+// Settles like `promise` unless `signal` aborts first
+function raceAbort(promise, signal) {
+    if (!signal) {
+        return promise;
+    }
+    signal.throwIfAborted();
+    return new Promise((resolve, reject) => {
+        const onAbort = () => {
+            reject(signal.reason instanceof Error
+                ? signal.reason
+                : new Error(String(signal.reason)));
+        };
+        signal.addEventListener('abort', onAbort, { once: true });
+        promise.then(resolve, reject).finally(() => {
+            signal.removeEventListener('abort', onAbort);
+        });
+    });
+}
+/**
+ * A walk-indexed graph's three files, read a window at a time: one read of
+ * each over whole chunks, from the lookback chunks before the window to its
+ * end. The indexes first estimate the bytes those reads fetch, and a window
+ * past the byte budget reads no row. Only the walks `keep` accepts are
+ * decoded. The step budget bounds the steps the cut keeps, and rows holding
+ * READ_STEPS_PER_KEPT times as many stop the reads, which go out together.
+ */
+export class WalkReader {
+    constructor(files, chunkWithoutHeader = 65536) {
+        this.files = files;
+        this.chunkWithoutHeader = chunkWithoutHeader;
+    }
+    // The header, read once for every caller: one caller giving up stops only
+    // its own wait, and a read that fails is tried again by the next caller
+    header(opts = {}) {
+        if (!this.headerRead) {
+            const read = this.files.walks.getHeaderLines().then(lines => {
+                var _a;
+                const header = walkHeader(lines);
+                const chunk = (_a = header.chunk) !== null && _a !== void 0 ? _a : this.chunkWithoutHeader;
+                return {
+                    ...header,
+                    chunk,
+                    lookback: lookbackChunks(header.maxNode, chunk),
+                };
+            });
+            read.catch(() => {
+                if (this.headerRead === read) {
+                    this.headerRead = undefined;
+                }
+            });
+            this.headerRead = read;
+        }
+        return raceAbort(this.headerRead, opts.signal);
+    }
+    // The reference sequences the walk file holds windows on
+    referenceSequences(opts = {}) {
+        return this.files.walks.getReferenceSequenceNames(opts);
+    }
+    async queryStart(start, opts = {}) {
+        const { chunk, lookback } = await this.header(opts);
+        return chunkQueryStart(start, chunk, lookback);
+    }
+    async cut(refName, region, opts = {}) {
+        const { signal, keep, context = WALK_CONTEXT, stepBudget = Infinity, byteBudget = Infinity, } = opts;
+        const { chunk, lookback, format } = await this.header({ signal });
+        // format 2 holds the whole cut in its walk file
+        const format2 = format === 2;
+        const nodeFile = this.files.nodes;
+        const linkFile = opts.links === false ? undefined : this.files.links;
+        if (!format2 && !nodeFile) {
+            throw new Error('A format 1 walk file needs its node file: open it with `nodes`');
+        }
+        const links = format2 ? opts.links !== false : linkFile !== undefined;
+        const from = chunkQueryStart(region.start, chunk, lookback);
+        const files = format2
+            ? [this.files.walks]
+            : [this.files.walks, nodeFile, ...(linkFile ? [linkFile] : [])];
+        const budgeted = Number.isFinite(byteBudget) &&
+            files.every(file => file.bytesForRegions !== undefined);
+        const bytesFor = async (spans) => {
+            if (!budgeted) {
+                return 0;
+            }
+            const sizes = await Promise.all(files.map(file => file.bytesForRegions(spans.map(span => ({ refName, ...span })), { signal })));
+            return sizes.reduce((sum, bytes) => sum + bytes, 0);
+        };
+        const tooLarge = budgeted
+            ? await byteBudgetError(async (end) => {
+                checkRange(refName, from, end);
+                return bytesFor([{ start: from, end }]);
+            }, byteBudget, region, chunk, opts.budgetNames)
+            : undefined;
+        if (tooLarge) {
+            throw tooLarge;
+        }
+        const reads = new AbortController();
+        const stop = () => {
+            reads.abort(signal === null || signal === void 0 ? void 0 : signal.reason);
+        };
+        signal === null || signal === void 0 ? void 0 : signal.throwIfAborted();
+        signal === null || signal === void 0 ? void 0 : signal.addEventListener('abort', stop, { once: true });
+        const rows = [];
+        let cut;
+        const graph = new WalkGraph({
+            refName,
+            start: region.start,
+            end: region.end,
+            context,
+            sequences: opts.sequences,
+        });
+        const read = (file, lineCallback, start = from, end = region.end) => getLines(file, refName, start, end, {
+            signal: reads.signal,
+            lineCallback,
+        });
+        const addRow = (line) => {
+            if (keep === undefined || keep(walkRowName(line))) {
+                rows.push(parseWalkRow(line));
+            }
+        };
+        const addNode = (line) => {
+            graph.addNode(line);
+        };
+        const addLink = (line) => {
+            graph.addLink(line);
+        };
+        const decoded = new Set();
+        const ends = [];
+        // Format 2: each chunk read is decoded once, into the rows, nodes and
+        // links format 1 would have filed, and the links between pieces read
+        const readChunks = async (start = from, end = region.end) => {
+            const byChunk = new Map();
+            await read(this.files.walks, line => {
+                const cols = line.split('\t');
+                const cs = +cols[1];
+                const held = byChunk.get(cs);
+                if (held) {
+                    held.push(cols);
+                }
+                else {
+                    byChunk.set(cs, [cols]);
+                }
+            }, start, end);
+            for (const [cs, chunkRows] of byChunk) {
+                if (decoded.has(cs)) {
+                    continue;
+                }
+                decoded.add(cs);
+                const chunkCut = decodeChunk(chunkRows, keep);
+                rows.push(...chunkCut.rows);
+                for (const [id, node] of chunkCut.nodes) {
+                    graph.placeNode(id, node);
+                }
+                graph.addFormat2Chunk(chunkCut.chunk);
+                ends.push(...chunkCut.ends);
+            }
+            if (links) {
+                for (const [a, b] of pieceLinks(ends)) {
+                    graph.addLinkEnds(a, b);
+                }
+            }
+        };
+        const checkSteps = () => {
+            const error = stepBudgetError(rows, stepBudget, READ_STEPS_PER_KEPT, region, chunk, keep !== undefined, opts.budgetNames);
+            if (error) {
+                throw error;
+            }
+        };
+        try {
+            await (format2
+                ? readChunks().then(checkSteps)
+                : Promise.all([
+                    read(this.files.walks, addRow).then(checkSteps),
+                    read(nodeFile, addNode),
+                    ...(linkFile ? [read(linkFile, addLink)] : []),
+                ]));
+            // the chunks read, first to last, which a follow widens to the
+            // farthest chunk a cut-off walk goes on in, so the reference between
+            // is read too
+            let first = from;
+            let last = Math.floor((region.end - 1) / chunk) * chunk;
+            const reach = FOLLOW_CHUNKS * chunk;
+            const [lo, hi] = [first - reach, last + reach];
+            cut = walkCut(joinPieces(rows), graph.nodes, refName);
+            let spent = await bytesFor([{ start: from, end: region.end }]);
+            for (let round = 0; round < FOLLOW_ROUNDS; round++) {
+                const starts = cut.open
+                    .filter(c => c.refName === refName && c.start >= lo && c.start <= hi)
+                    .map(c => c.start);
+                const spans = [
+                    { start: Math.min(first, ...starts), end: first },
+                    { start: last + chunk, end: Math.max(last, ...starts) + 1 },
+                ].filter(span => span.end > span.start);
+                if (spans.length === 0) {
+                    break;
+                }
+                spent += await bytesFor(spans);
+                if (spent > byteBudget) {
+                    break;
+                }
+                first = Math.min(first, ...starts);
+                last = Math.max(last, ...starts);
+                await Promise.all(spans.flatMap(({ start, end }) => format2
+                    ? [readChunks(start, end)]
+                    : [
+                        read(this.files.walks, addRow, start, end),
+                        read(nodeFile, addNode, start, end),
+                        ...(linkFile ? [read(linkFile, addLink, start, end)] : []),
+                    ]));
+                cut = walkCut(joinPieces(rows), graph.nodes, refName);
+            }
+        }
+        catch (error) {
+            reads.abort();
+            throw error;
+        }
+        finally {
+            signal === null || signal === void 0 ? void 0 : signal.removeEventListener('abort', stop);
+        }
+        const tooMany = keptStepsError(cut.fragments, stepBudget, region, keep !== undefined);
+        if (tooMany) {
+            throw tooMany;
+        }
+        return { graph, ...cut };
+    }
+}

package/esm/walkRows.d.ts

@@ -0,0 +1,172 @@
+import { RegionTooLargeError } from './util.ts';
+import type { GraphTables } from './util.ts';
+import type { Format2Chunk } from './walkFormat2.ts';
+export interface WalkRow {
+    name: string;
+    fragStart: number;
+    hapOffset: number;
+    piece: number;
+    n: number;
+    enc: string;
+    steps?: {
+        ids: Int32Array;
+        rev: Uint8Array;
+    };
+    chunkStart: number;
+    prev?: WalkChunk;
+    next?: WalkChunk;
+}
+export interface WalkChunk {
+    refName: string;
+    start: number;
+}
+export interface WalkNode {
+    refName: string;
+    start: number;
+    end: number;
+    rank: number;
+    onWindow: boolean;
+}
+export interface WalkWindow {
+    refName: string;
+    start: number;
+    end: number;
+    context: number;
+    sequences?: boolean;
+}
+export declare function walkRowName(line: string): string;
+export declare function parseWalkRow(line: string): WalkRow;
+export interface WalkHeader {
+    format?: number;
+    chunk?: number;
+    maxNode?: number;
+    references: string[];
+    haplotypes: string[];
+}
+/**
+ * What a walk file's header states: its chunk size in a
+ * `#walks\tchunk:i:65536` line, from gfa-to-tabix 0.4.0 a
+ * `#reference\tGRCh38` line per reference sample and a
+ * `#haplotype\tHG002#1` line per other haplotype with rows, and from 0.5.0
+ * the longest node in bp as `maxnode:i:1024` beside the chunk size
+ */
+export declare function walkHeader(lines: string[]): WalkHeader;
+/**
+ * How many chunks before the window's own a cut reads. A node is filed under
+ * the chunk holding its start, so a node `maxNode` bp long reaching into the
+ * window starts at most ceil(maxNode / chunk) chunks back. At least one, since
+ * the chunk before carries the detour a haplotype takes into the window's
+ * first reference node; one when the header gives no `maxnode:i:`, which holds
+ * for vg's graphs, chopped to nodes of at most 1,024 bp.
+ */
+export declare function lookbackChunks(maxNode: number | undefined, chunk: number): number;
+export declare function chunkQueryStart(start: number, chunk: number, lookback?: number): number;
+export declare function walkNameFilter(wanted: string[] | undefined, referenceHaplotype: string | undefined): ((name: string) => boolean) | undefined;
+export declare function decodeSteps(enc: string, n: number, ids?: Int32Array<ArrayBuffer>, rev?: Uint8Array<ArrayBuffer>, at?: number): {
+    ids: Int32Array<ArrayBuffer>;
+    rev: Uint8Array<ArrayBuffer>;
+};
+export interface WalkRun {
+    name: string;
+    hapOffset: number;
+    ids: Int32Array;
+    rev: Uint8Array;
+    prev?: WalkChunk;
+    next?: WalkChunk;
+}
+export declare function joinPieces(rows: WalkRow[]): WalkRun[];
+export interface WalkFragment {
+    name: string;
+    hapStart: number;
+    hapEnd: number;
+    ids: Int32Array;
+    rev: Uint8Array;
+}
+/**
+ * The cut a window draws from its walks: the reference inside the window plus
+ * its context, and each run from its first to its last step on that stretch.
+ * A run that leaves the reference there, through steps off it or across a
+ * deletion, is followed to the reference step where it rejoins, when the read
+ * holds it, and otherwise through the steps off the reference it holds: at
+ * the ABCA7 VNTR four haplotypes leave inside the repeat and rejoin 1.8 kb
+ * past the window, and cut at the context they measured as partial walks. The reference is cut out as far as those rejoins reach, and
+ * a run then runs on outward while the next step's node is already in the
+ * cut. Without the run-on a haplotype whose alternate allele straddles the
+ * edge stopped short of a node other walks had brought into the cut, and that
+ * node drew without it. The run-on adds no node, so one pass is enough, but
+ * it reaches through nodes other walks brought in, so with a window edge inside
+ * a collapsed array a row's length depends on which haplotypes are cut.
+ */
+export declare function walkCut(runs: WalkRun[], nodes: Map<number, WalkNode>, refName: string): {
+    kept: Set<number>;
+    fragments: WalkFragment[];
+    open: WalkChunk[];
+};
+/**
+ * The node and link rows a walk cut reads, parsed as they arrive, and the
+ * tables of the cut they make with its walks. A node filed under two chunks
+ * arrives twice and is read once; columns past a node row's eighth (LN:i:,
+ * SQ:Z: and whatever follows) are read only for the bases a window asks for.
+ */
+export declare class WalkGraph {
+    readonly nodes: Map<number, WalkNode>;
+    readonly sequences: Map<number, string>;
+    private links;
+    private readonly format2;
+    private readonly t;
+    private readonly window;
+    constructor(window: WalkWindow);
+    addNode(line: string): void;
+    placeNode(id: number, node: Omit<WalkNode, 'onWindow'>): void;
+    addLinkEnds(source: number, target: number): void;
+    addFormat2Chunk(chunk: Format2Chunk): void;
+    addLink(line: string): void;
+    /**
+     * The cut as typed arrays, which the worker transfers rather than copies:
+     * nodes by reference position, each link once in the order its L line
+     * sorts, then a walk per fragment, the reference's first and the rest by
+     * name. graphTablesGFA writes them as the GFA they stand for, the same bytes
+     * for the same rows.
+     */
+    tables(kept: Set<number>, fragments: WalkFragment[]): GraphTables;
+    private referenceFirst;
+}
+/**
+ * The zoom-in notice for a cut whose walk rows hold more steps than
+ * `readPerKept` times `stepBudget`, or undefined when they fit. A window reads whole chunks, those before it
+ * included, so the span that fits is counted in chunks from the window's
+ * start; when the chunks a window there always reads are over on their own,
+ * no zoom fits and the notice asks for fewer haplotypes. A cut for every
+ * haplotype is also offered fewer where the reference and one haplotype, at
+ * the walks' average steps, would fit the whole window.
+ */
+export interface BudgetNames {
+    steps: string;
+    bytes: string;
+}
+export declare const BUDGET_NAMES: BudgetNames;
+export declare function stepBudgetError(rows: WalkRow[], stepBudget: number, readPerKept: number, window: {
+    start: number;
+    end: number;
+}, chunk: number, filtered: boolean, names?: BudgetNames): RegionTooLargeError | undefined;
+/**
+ * The zoom-in notice for a cut that keeps more steps than `budget`, or
+ * undefined when it fits. The span that fits is the window scaled by how far
+ * over the cut is.
+ */
+export declare function keptStepsError(fragments: WalkFragment[], budget: number, window: {
+    start: number;
+    end: number;
+}, filtered: boolean): RegionTooLargeError | undefined;
+/**
+ * The zoom-in notice for a cut whose three files would fetch more than
+ * `budget` compressed bytes, or undefined when they fit. `bytesTo(end)` is the
+ * indexes' estimate for a read from the window's first query base to `end`;
+ * the span that fits grows a chunk at a time from the chunk holding the
+ * window's start, and when that chunk and those before it are over on their
+ * own, no zoom fits.
+ */
+export declare function byteBudgetError(bytesTo: (end: number) => Promise<number>, budget: number, window: {
+    start: number;
+    end: number;
+}, chunk: number, names?: BudgetNames): Promise<RegionTooLargeError | undefined>;