@jbrowse/graph-walks 0.0.0-stage → 1.0.0

package/esm/walkReader.js

@@ -0,0 +1,218 @@
+import { decodeChunk, pieceLinks } from "./walkChunk.js";
+import { WalkGraph, byteBudgetError, chunkQueryStart, joinPieces, keptStepsError, lookbackChunks, stepBudgetError, walkCut, walkHeader, } from "./walkRows.js";
+// A window reads whole chunks and keeps the steps on it, so a narrow window
+// reads several times what it keeps: 464 haplotypes across the 1.5 kb ABCA7
+// repeat read 5.0 M steps and keep 1.16 M, and the cut takes 0.4 s from warm
+// files. The step budget bounds what the cut keeps, and rows are read to this
+// multiple of it.
+const READ_STEPS_PER_KEPT = 8;
+// A walk the read cuts off goes on in the chunk its row names, out to which a
+// cut reads on, in at most this many rounds and this many chunks from the read
+const FOLLOW_ROUNDS = 3;
+const FOLLOW_CHUNKS = 4;
+// bp of reference a cut keeps past each end of the window
+const WALK_CONTEXT = 1000;
+// A walk file gfa-to-tabix 0.12 or earlier wrote, which the reader refuses
+export class WalkFormatError extends Error {
+    constructor() {
+        super(...arguments);
+        this.name = 'WalkFormatError';
+    }
+}
+// @gmod/tabix loops forever on a NaN end and reads nothing for a NaN start
+// (GMOD/tabix-js#157), so every query refuses a range that is not finite first
+function checkRange(refName, start, end) {
+    if (!Number.isFinite(start) || !Number.isFinite(end)) {
+        throw new Error(`${refName}:${start}-${end} is not a finite range to query the graph's index for`);
+    }
+}
+// Settles like `promise` unless `signal` aborts first
+function raceAbort(promise, signal) {
+    if (!signal) {
+        return promise;
+    }
+    signal.throwIfAborted();
+    return new Promise((resolve, reject) => {
+        const onAbort = () => {
+            reject(signal.reason instanceof Error
+                ? signal.reason
+                : new Error(String(signal.reason)));
+        };
+        signal.addEventListener('abort', onAbort, { once: true });
+        promise.then(resolve, reject).finally(() => {
+            signal.removeEventListener('abort', onAbort);
+        });
+    });
+}
+/**
+ * A `gfa-to-tabix --walks` file, `<prefix>.<sample>.walks.bed.gz`, read a
+ * window at a time: one read over whole chunks, from the lookback chunks
+ * before the window to its end, each chunk decoded once. The index first
+ * estimates the bytes the read fetches, and a window past the byte budget
+ * reads no row. Only the walks `keep` accepts are decoded. The step budget
+ * bounds the steps the cut keeps, and rows holding READ_STEPS_PER_KEPT times
+ * as many stop the read.
+ */
+export class WalkReader {
+    constructor(walks) {
+        this.walks = walks;
+    }
+    // The header, read once for every caller: one caller giving up stops only
+    // its own wait, and a read that fails is tried again by the next caller
+    header(opts = {}) {
+        if (!this.headerRead) {
+            const read = this.walks.getHeaderLines().then(lines => {
+                var _a;
+                const header = walkHeader(lines);
+                if (header.format !== 2) {
+                    throw new WalkFormatError('This walk file is format 1, from gfa-to-tabix 0.12 or earlier; rebuild it with gfa-to-tabix 1.0 or later');
+                }
+                const chunk = (_a = header.chunk) !== null && _a !== void 0 ? _a : 65536;
+                return {
+                    ...header,
+                    chunk,
+                    lookback: lookbackChunks(header.maxNode, chunk),
+                };
+            });
+            read.catch(() => {
+                if (this.headerRead === read) {
+                    this.headerRead = undefined;
+                }
+            });
+            this.headerRead = read;
+        }
+        return raceAbort(this.headerRead, opts.signal);
+    }
+    // The reference sequences the walk file holds windows on
+    referenceSequences(opts = {}) {
+        return this.walks.getReferenceSequenceNames(opts);
+    }
+    async queryStart(start, opts = {}) {
+        const { chunk, lookback } = await this.header(opts);
+        return chunkQueryStart(start, chunk, lookback);
+    }
+    async cut(refName, region, opts = {}) {
+        var _a;
+        const { signal, keep, context = WALK_CONTEXT, stepBudget = Infinity, byteBudget = Infinity, links = true, } = opts;
+        const { chunk, lookback } = await this.header({ signal });
+        const from = chunkQueryStart(region.start, chunk, lookback);
+        const bytesForRegions = (_a = this.walks.bytesForRegions) === null || _a === void 0 ? void 0 : _a.bind(this.walks);
+        const budgeted = Number.isFinite(byteBudget) && bytesForRegions !== undefined;
+        const bytesFor = async (spans) => budgeted
+            ? bytesForRegions(spans.map(span => ({ refName, ...span })), { signal })
+            : 0;
+        const tooLarge = budgeted
+            ? await byteBudgetError(async (end) => {
+                checkRange(refName, from, end);
+                return bytesFor([{ start: from, end }]);
+            }, byteBudget, region, chunk, opts.budgetNames)
+            : undefined;
+        if (tooLarge) {
+            throw tooLarge;
+        }
+        const reads = new AbortController();
+        const stop = () => {
+            reads.abort(signal === null || signal === void 0 ? void 0 : signal.reason);
+        };
+        signal === null || signal === void 0 ? void 0 : signal.throwIfAborted();
+        signal === null || signal === void 0 ? void 0 : signal.addEventListener('abort', stop, { once: true });
+        const rows = [];
+        let cut;
+        const graph = new WalkGraph({
+            refName,
+            start: region.start,
+            end: region.end,
+            context,
+            sequences: opts.sequences,
+        });
+        const decoded = new Set();
+        const ends = [];
+        // each chunk read is decoded once, into its rows, nodes and the links
+        // between pieces
+        const readChunks = async (start, end) => {
+            checkRange(refName, start, end);
+            const byChunk = new Map();
+            await this.walks.getLines(refName, start, end, {
+                signal: reads.signal,
+                lineCallback: line => {
+                    const cols = line.split('\t');
+                    const cs = +cols[1];
+                    const held = byChunk.get(cs);
+                    if (held) {
+                        held.push(cols);
+                    }
+                    else {
+                        byChunk.set(cs, [cols]);
+                    }
+                },
+            });
+            for (const [cs, chunkRows] of byChunk) {
+                if (decoded.has(cs)) {
+                    continue;
+                }
+                decoded.add(cs);
+                const chunkCut = decodeChunk(chunkRows, keep, opts.sequences);
+                rows.push(...chunkCut.rows);
+                for (const [id, node] of chunkCut.nodes) {
+                    graph.placeNode(id, node);
+                }
+                graph.addSequences(chunkCut.sequences);
+                graph.addChunk(chunkCut.chunk);
+                ends.push(...chunkCut.ends);
+            }
+            if (links) {
+                for (const [a, b] of pieceLinks(ends)) {
+                    graph.addLinkEnds(a, b);
+                }
+            }
+        };
+        try {
+            await readChunks(from, region.end);
+            const tooManyRead = stepBudgetError(rows, stepBudget, READ_STEPS_PER_KEPT, region, chunk, keep !== undefined, opts.budgetNames);
+            if (tooManyRead) {
+                throw tooManyRead;
+            }
+            // the chunks read, first to last, which a follow widens to the
+            // farthest chunk a cut-off walk goes on in, so the reference between
+            // is read too
+            let first = from;
+            let last = Math.floor((region.end - 1) / chunk) * chunk;
+            const reach = FOLLOW_CHUNKS * chunk;
+            const [lo, hi] = [first - reach, last + reach];
+            cut = walkCut(joinPieces(rows), graph.nodes, refName);
+            let spent = await bytesFor([{ start: from, end: region.end }]);
+            for (let round = 0; round < FOLLOW_ROUNDS; round++) {
+                const starts = cut.open
+                    .filter(c => c.refName === refName && c.start >= lo && c.start <= hi)
+                    .map(c => c.start);
+                const spans = [
+                    { start: Math.min(first, ...starts), end: first },
+                    { start: last + chunk, end: Math.max(last, ...starts) + 1 },
+                ].filter(span => span.end > span.start);
+                if (spans.length === 0) {
+                    break;
+                }
+                spent += await bytesFor(spans);
+                if (spent > byteBudget) {
+                    break;
+                }
+                first = Math.min(first, ...starts);
+                last = Math.max(last, ...starts);
+                await Promise.all(spans.map(({ start, end }) => readChunks(start, end)));
+                cut = walkCut(joinPieces(rows), graph.nodes, refName);
+            }
+        }
+        catch (error) {
+            reads.abort();
+            throw error;
+        }
+        finally {
+            signal === null || signal === void 0 ? void 0 : signal.removeEventListener('abort', stop);
+        }
+        const tooMany = keptStepsError(cut.fragments, stepBudget, region, keep !== undefined);
+        if (tooMany) {
+            throw tooMany;
+        }
+        return { graph, ...cut };
+    }
+}

package/esm/walkRows.d.ts

@@ -0,0 +1,160 @@
+import { RegionTooLargeError } from './util.ts';
+import type { GraphTables } from './util.ts';
+import type { Chunk } from './walkChunk.ts';
+export interface WalkRow {
+    name: string;
+    fragStart: number;
+    hapOffset: number;
+    piece: number;
+    n: number;
+    steps: {
+        ids: Int32Array;
+        rev: Uint8Array;
+    };
+    chunkStart: number;
+    prev?: WalkChunk;
+    next?: WalkChunk;
+}
+export interface WalkChunk {
+    refName: string;
+    start: number;
+}
+export interface WalkNode {
+    refName: string;
+    start: number;
+    end: number;
+    rank: number;
+    onWindow: boolean;
+}
+export interface WalkWindow {
+    refName: string;
+    start: number;
+    end: number;
+    context: number;
+    sequences?: boolean;
+}
+export interface WalkHeader {
+    format?: number;
+    chunk?: number;
+    maxNode?: number;
+    references: string[];
+    haplotypes: string[];
+}
+/**
+ * What a walk file's header states: a `#walks` line with the format, the
+ * chunk size and the longest node in bp, `format:i:2\tchunk:i:65536\tmaxnode:i:1024`,
+ * a `#reference\tGRCh38` line naming the reference sample and a
+ * `#haplotype\tHG002#1` line per other haplotype with rows
+ */
+export declare function walkHeader(lines: string[]): WalkHeader;
+/**
+ * How many chunks before the window's own a cut reads. A node is filed under
+ * the chunk holding its start, so a node `maxNode` bp long reaching into the
+ * window starts at most ceil(maxNode / chunk) chunks back. At least one, since
+ * the chunk before carries the detour a haplotype takes into the window's
+ * first reference node; one when the header gives no `maxnode:i:`, which holds
+ * for vg's graphs, chopped to nodes of at most 1,024 bp.
+ */
+export declare function lookbackChunks(maxNode: number | undefined, chunk: number): number;
+export declare function chunkQueryStart(start: number, chunk: number, lookback?: number): number;
+export declare function walkNameFilter(wanted: string[] | undefined, referenceHaplotype: string | undefined): ((name: string) => boolean) | undefined;
+export interface WalkRun {
+    name: string;
+    hapOffset: number;
+    ids: Int32Array;
+    rev: Uint8Array;
+    prev?: WalkChunk;
+    next?: WalkChunk;
+}
+export declare function joinPieces(rows: WalkRow[]): WalkRun[];
+export interface WalkFragment {
+    name: string;
+    hapStart: number;
+    hapEnd: number;
+    ids: Int32Array;
+    rev: Uint8Array;
+}
+/**
+ * The cut a window draws from its walks: the reference inside the window plus
+ * its context, and each run from its first to its last step on that stretch.
+ * A run that leaves the reference there, through steps off it or across a
+ * deletion, is followed to the reference step where it rejoins, when the read
+ * holds it, and otherwise through the steps off the reference it holds: at
+ * the ABCA7 VNTR four haplotypes leave inside the repeat and rejoin 1.8 kb
+ * past the window, and cut at the context they measured as partial walks. The reference is cut out as far as those rejoins reach, and
+ * a run then runs on outward while the next step's node is already in the
+ * cut. Without the run-on a haplotype whose alternate allele straddles the
+ * edge stopped short of a node other walks had brought into the cut, and that
+ * node drew without it. The run-on adds no node, so one pass is enough, but
+ * it reaches through nodes other walks brought in, so with a window edge inside
+ * a collapsed array a row's length depends on which haplotypes are cut.
+ */
+export declare function walkCut(runs: WalkRun[], nodes: Map<number, WalkNode>, refName: string): {
+    kept: Set<number>;
+    fragments: WalkFragment[];
+    open: WalkChunk[];
+};
+/**
+ * The nodes and links of the chunks a cut reads, and the tables of the cut
+ * they make with its walks. A node in two chunks is placed once.
+ */
+export declare class WalkGraph {
+    readonly nodes: Map<number, WalkNode>;
+    readonly sequences: Map<number, string>;
+    private links;
+    private readonly chunks;
+    private readonly window;
+    constructor(window: WalkWindow);
+    placeNode(id: number, node: Omit<WalkNode, 'onWindow'>): void;
+    addLinkEnds(source: number, target: number): void;
+    addChunk(chunk: Chunk): void;
+    addSequences(sequences: Map<number, string>): void;
+    /**
+     * The cut as typed arrays, which the worker transfers rather than copies:
+     * nodes by reference position, each link once in the order its L line
+     * sorts, then a walk per fragment, the reference's first and the rest by
+     * name. graphTablesGFA writes them as the GFA they stand for, the same bytes
+     * for the same rows.
+     */
+    tables(kept: Set<number>, fragments: WalkFragment[]): GraphTables;
+    private referenceFirst;
+}
+/**
+ * The zoom-in notice for a cut whose walk rows hold more steps than
+ * `readPerKept` times `stepBudget`, or undefined when they fit. A window reads whole chunks, those before it
+ * included, so the span that fits is counted in chunks from the window's
+ * start; when the chunks a window there always reads are over on their own,
+ * no zoom fits and the notice asks for fewer haplotypes. A cut for every
+ * haplotype is also offered fewer where the reference and one haplotype, at
+ * the walks' average steps, would fit the whole window.
+ */
+export interface BudgetNames {
+    steps: string;
+    bytes: string;
+}
+export declare const BUDGET_NAMES: BudgetNames;
+export declare function stepBudgetError(rows: WalkRow[], stepBudget: number, readPerKept: number, window: {
+    start: number;
+    end: number;
+}, chunk: number, filtered: boolean, names?: BudgetNames): RegionTooLargeError | undefined;
+/**
+ * The zoom-in notice for a cut that keeps more steps than `budget`, or
+ * undefined when it fits. The span that fits is the window scaled by how far
+ * over the cut is.
+ */
+export declare function keptStepsError(fragments: WalkFragment[], budget: number, window: {
+    start: number;
+    end: number;
+}, filtered: boolean): RegionTooLargeError | undefined;
+/**
+ * The zoom-in notice for a cut whose three files would fetch more than
+ * `budget` compressed bytes, or undefined when they fit. `bytesTo(end)` is the
+ * indexes' estimate for a read from the window's first query base to `end`;
+ * the span that fits grows a chunk at a time from the chunk holding the
+ * window's start, and when that chunk and those before it are over on their
+ * own, no zoom fits.
+ */
+export declare function byteBudgetError(bytesTo: (end: number) => Promise<number>, budget: number, window: {
+    start: number;
+    end: number;
+}, chunk: number, names?: BudgetNames): Promise<RegionTooLargeError | undefined>;