@jbrowse/graph-walks 0.0.0-stage → 0.1.0

package/esm/walkRows.js

@@ -0,0 +1,579 @@
+import { RegionTooLargeError, bpString, panSNMatchesPrefix } from "./util.js";
+import { placeAndLink } from "./walkFormat2.js";
+// The name column, read without splitting the steps, so a row the cut does
+// not want costs a few indexOf calls
+export function walkRowName(line) {
+    let at = -1;
+    for (let k = 0; k < 3; k++) {
+        at = line.indexOf('\t', at + 1);
+    }
+    return line.slice(at + 1, line.indexOf('\t', at + 1));
+}
+export function parseWalkRow(line) {
+    const c = line.split('\t');
+    const row = {
+        chunkStart: +c[1],
+        name: c[3],
+        fragStart: +c[4],
+        hapOffset: +c[5],
+        piece: +c[6],
+        n: +c[7],
+        enc: c[8],
+    };
+    for (let k = 9; k < c.length; k++) {
+        const tag = c[k];
+        const side = tag.startsWith('pv:')
+            ? 'prev'
+            : tag.startsWith('nx:')
+                ? 'next'
+                : undefined;
+        if (side) {
+            const at = tag.lastIndexOf(':');
+            row[side] = {
+                refName: tag[3] === 'Z' ? tag.slice(5, at) : c[0],
+                start: +tag.slice(at + 1),
+            };
+        }
+    }
+    return row;
+}
+function integerTag(line, tag) {
+    const match = new RegExp(`(?:^|\\t)${tag}:i:(\\d+)`).exec(line);
+    return match ? +match[1] : undefined;
+}
+/**
+ * What a walk file's header states: its chunk size in a
+ * `#walks\tchunk:i:65536` line, from gfa-to-tabix 0.4.0 a
+ * `#reference\tGRCh38` line per reference sample and a
+ * `#haplotype\tHG002#1` line per other haplotype with rows, and from 0.5.0
+ * the longest node in bp as `maxnode:i:1024` beside the chunk size
+ */
+export function walkHeader(lines) {
+    var _a, _b, _c;
+    const header = { references: [], haplotypes: [] };
+    for (const line of lines) {
+        const [key, value = ''] = line.split('\t');
+        if (key === '#reference') {
+            header.references.push(value);
+        }
+        else if (key === '#haplotype') {
+            header.haplotypes.push(value);
+        }
+        else {
+            (_a = header.format) !== null && _a !== void 0 ? _a : (header.format = integerTag(line, 'format'));
+            (_b = header.chunk) !== null && _b !== void 0 ? _b : (header.chunk = integerTag(line, 'chunk'));
+            (_c = header.maxNode) !== null && _c !== void 0 ? _c : (header.maxNode = integerTag(line, 'maxnode'));
+        }
+    }
+    return header;
+}
+/**
+ * How many chunks before the window's own a cut reads. A node is filed under
+ * the chunk holding its start, so a node `maxNode` bp long reaching into the
+ * window starts at most ceil(maxNode / chunk) chunks back. At least one, since
+ * the chunk before carries the detour a haplotype takes into the window's
+ * first reference node; one when the header gives no `maxnode:i:`, which holds
+ * for vg's graphs, chopped to nodes of at most 1,024 bp.
+ */
+export function lookbackChunks(maxNode, chunk) {
+    return maxNode === undefined ? 1 : Math.max(1, Math.ceil(maxNode / chunk));
+}
+// The first base a cut queries: the start of the chunk `lookback` chunks
+// before the one holding the window's start
+export function chunkQueryStart(start, chunk, lookback = 1) {
+    return Math.max(0, (Math.floor(start / chunk) - lookback) * chunk);
+}
+// Which walks a cut decodes: the reference's, and those `wanted` names by
+// PanSN prefix at sample or haplotype depth; undefined decodes every one
+export function walkNameFilter(wanted, referenceHaplotype) {
+    return wanted === undefined || wanted.length === 0
+        ? undefined
+        : (name) => panSNMatchesPrefix(name, referenceHaplotype) ||
+            wanted.some(prefix => panSNMatchesPrefix(name, prefix));
+}
+export function decodeSteps(enc, n, ids = new Int32Array(n), rev = new Uint8Array(n), at = 0) {
+    let prev = 0;
+    let i = at;
+    let v = 0;
+    let neg = false;
+    for (let p = 0; p <= enc.length; p++) {
+        const ch = p < enc.length ? enc.charCodeAt(p) : 44;
+        if (ch === 44) {
+            const d = neg ? -v : v;
+            const r = d & 1;
+            rev[i] = r;
+            prev += (d - r) / 2;
+            ids[i++] = prev;
+            v = 0;
+            neg = false;
+        }
+        else if (ch === 45) {
+            neg = true;
+        }
+        else {
+            v = v * 10 + (ch - 48);
+        }
+    }
+    return { ids, rev };
+}
+export function joinPieces(rows) {
+    const byPath = new Map();
+    for (const row of rows) {
+        const key = `${row.name}\t${row.fragStart}`;
+        const list = byPath.get(key);
+        if (list) {
+            list.push(row);
+        }
+        else {
+            byPath.set(key, [row]);
+        }
+    }
+    const runs = [];
+    for (const pieces of byPath.values()) {
+        pieces.sort((a, b) => a.piece - b.piece);
+        let from = 0;
+        for (let i = 1; i <= pieces.length; i++) {
+            if (i === pieces.length ||
+                pieces[i].piece !== pieces[i - 1].piece + 1) {
+                runs.push(decodeRun(pieces.slice(from, i)));
+                from = i;
+            }
+        }
+    }
+    return runs;
+}
+function decodeRun(pieces) {
+    const n = pieces.reduce((sum, p) => sum + p.n, 0);
+    const ids = new Int32Array(n);
+    const rev = new Uint8Array(n);
+    let at = 0;
+    for (const p of pieces) {
+        if (p.steps) {
+            ids.set(p.steps.ids, at);
+            rev.set(p.steps.rev, at);
+        }
+        else {
+            decodeSteps(p.enc, p.n, ids, rev, at);
+        }
+        at += p.n;
+    }
+    const first = pieces[0];
+    return {
+        name: first.name,
+        hapOffset: first.hapOffset,
+        ids,
+        rev,
+        prev: first.prev,
+        next: pieces.at(-1).next,
+    };
+}
+/**
+ * The cut a window draws from its walks: the reference inside the window plus
+ * its context, and each run from its first to its last step on that stretch.
+ * A run that leaves the reference there, through steps off it or across a
+ * deletion, is followed to the reference step where it rejoins, when the read
+ * holds it, and otherwise through the steps off the reference it holds: at
+ * the ABCA7 VNTR four haplotypes leave inside the repeat and rejoin 1.8 kb
+ * past the window, and cut at the context they measured as partial walks. The reference is cut out as far as those rejoins reach, and
+ * a run then runs on outward while the next step's node is already in the
+ * cut. Without the run-on a haplotype whose alternate allele straddles the
+ * edge stopped short of a node other walks had brought into the cut, and that
+ * node drew without it. The run-on adds no node, so one pass is enough, but
+ * it reaches through nodes other walks brought in, so with a window edge inside
+ * a collapsed array a row's length depends on which haplotypes are cut.
+ */
+export function walkCut(runs, nodes, refName) {
+    const onWindow = (id) => { var _a; return ((_a = nodes.get(id)) === null || _a === void 0 ? void 0 : _a.onWindow) === true; };
+    const reference = (id) => {
+        const node = nodes.get(id);
+        return (node === null || node === void 0 ? void 0 : node.rank) === 0 ? node : undefined;
+    };
+    // the chunks walks cut off at the end of a run go on in
+    const open = new Map();
+    const goesOn = (chunk) => {
+        if (chunk) {
+            open.set(`${chunk.refName}:${chunk.start}`, chunk);
+        }
+    };
+    // Where a walk stops being followed from step `at`, going by `dir`: the
+    // reference step it rejoins at past any steps off the reference, or across
+    // a deletion; else its last step off the reference before the run ends,
+    // noting the chunk the path goes on in, or reaches another reference
+    // sequence
+    const rejoin = (run, at, dir) => {
+        const { ids } = run;
+        let i = at + dir;
+        while (i >= 0 && i < ids.length && reference(ids[i]) === undefined) {
+            i += dir;
+        }
+        if (i < 0 || i >= ids.length) {
+            goesOn(dir < 0 ? run.prev : run.next);
+            return i - dir;
+        }
+        const from = reference(ids[at]);
+        const to = reference(ids[i]);
+        if (to.refName !== refName) {
+            return i - dir;
+        }
+        const adjacent = i === at + dir && (to.start === from.end || to.end === from.start);
+        return adjacent ? at : i;
+    };
+    let reachStart = Infinity;
+    let reachEnd = -Infinity;
+    const spans = [];
+    for (const run of runs) {
+        const { ids } = run;
+        let first = -1;
+        let last = -1;
+        for (let i = 0; i < ids.length; i++) {
+            if (onWindow(ids[i])) {
+                if (first < 0) {
+                    first = i;
+                }
+                last = i;
+            }
+        }
+        if (first >= 0) {
+            if (run.name !== refName) {
+                first = rejoin(run, first, -1);
+                last = rejoin(run, last, 1);
+                for (const i of [first, last]) {
+                    const node = reference(ids[i]);
+                    if (node) {
+                        reachStart = Math.min(reachStart, node.start);
+                        reachEnd = Math.max(reachEnd, node.end);
+                    }
+                }
+            }
+            spans.push({ run, first, last });
+        }
+    }
+    const inReach = (id) => {
+        const node = reference(id);
+        return (onWindow(id) ||
+            ((node === null || node === void 0 ? void 0 : node.refName) === refName &&
+                node.end > reachStart &&
+                node.start < reachEnd));
+    };
+    const kept = new Set();
+    for (const [id, node] of nodes) {
+        if (node.onWindow) {
+            kept.add(id);
+        }
+    }
+    for (const span of spans) {
+        const { ids } = span.run;
+        if (span.run.name === refName) {
+            while (span.first > 0 && inReach(ids[span.first - 1])) {
+                span.first--;
+            }
+            while (span.last < ids.length - 1 && inReach(ids[span.last + 1])) {
+                span.last++;
+            }
+        }
+        for (let i = span.first; i <= span.last; i++) {
+            kept.add(ids[i]);
+        }
+    }
+    const length = (id) => {
+        const node = nodes.get(id);
+        return node === undefined ? 0 : node.end - node.start;
+    };
+    const fragments = [];
+    for (const { run, first: f, last: l } of spans) {
+        const { ids, rev } = run;
+        let first = f;
+        let last = l;
+        while (first > 0 && kept.has(ids[first - 1])) {
+            first--;
+        }
+        while (last < ids.length - 1 && kept.has(ids[last + 1])) {
+            last++;
+        }
+        let hapStart = run.hapOffset;
+        for (let i = 0; i < first; i++) {
+            hapStart += length(ids[i]);
+        }
+        let hapEnd = hapStart;
+        for (let i = first; i <= last; i++) {
+            hapEnd += length(ids[i]);
+        }
+        fragments.push({
+            name: run.name,
+            hapStart,
+            hapEnd,
+            ids: ids.subarray(first, last + 1),
+            rev: rev.subarray(first, last + 1),
+        });
+    }
+    return { kept, fragments, open: [...open.values()] };
+}
+// The positions of a line's first `n` tabs, -1 past the last
+function tabs(line, n, out) {
+    let at = -1;
+    for (let k = 0; k < n; k++) {
+        at = at === -1 && k > 0 ? -1 : line.indexOf('\t', at + 1);
+        out[k] = at;
+    }
+    return out;
+}
+/**
+ * The node and link rows a walk cut reads, parsed as they arrive, and the
+ * tables of the cut they make with its walks. A node filed under two chunks
+ * arrives twice and is read once; columns past a node row's eighth (LN:i:,
+ * SQ:Z: and whatever follows) are read only for the bases a window asks for.
+ */
+export class WalkGraph {
+    constructor(window) {
+        this.nodes = new Map();
+        this.sequences = new Map();
+        this.links = [];
+        // format 2's chunks, whose nodes off R and links tables() fills in
+        this.format2 = [];
+        this.t = [];
+        this.window = window;
+    }
+    addNode(line) {
+        const t = tabs(line, 8, this.t);
+        const id = +line.slice(t[2] + 1, t[3]);
+        if (!this.nodes.has(id)) {
+            if (this.window.sequences) {
+                const at = line.indexOf('\tSQ:Z:');
+                if (at >= 0) {
+                    const end = line.indexOf('\t', at + 1);
+                    this.sequences.set(id, line.slice(at + 6, end < 0 ? undefined : end));
+                }
+            }
+            this.placeNode(id, {
+                rank: +line.slice(t[3] + 1, t[4]),
+                refName: line.slice(t[4] + 1, t[5]),
+                start: +line.slice(t[5] + 1, t[6]),
+                end: +line.slice(t[6] + 1, t[7] === -1 ? undefined : t[7]),
+            });
+        }
+    }
+    // A node with its position, unless the cut already holds it
+    placeNode(id, node) {
+        if (this.nodes.has(id)) {
+            return;
+        }
+        const { window } = this;
+        this.nodes.set(id, {
+            ...node,
+            onWindow: node.rank === 0 &&
+                node.refName === window.refName &&
+                node.end > window.start - window.context &&
+                node.start < window.end + window.context,
+        });
+    }
+    // A link between two endpoints, each id * 2 + 1 when reversed
+    addLinkEnds(source, target) {
+        this.links.push(source, target);
+    }
+    addFormat2Chunk(chunk) {
+        this.format2.push(chunk);
+    }
+    // An endpoint is held as id * 2 + 1 when reversed
+    addLink(line) {
+        const t = tabs(line, 5, this.t);
+        this.links.push(endpoint(line, t[2] + 1, t[3]), endpoint(line, t[3] + 1, t[4] === -1 ? line.length : t[4]));
+    }
+    /**
+     * The cut as typed arrays, which the worker transfers rather than copies:
+     * nodes by reference position, each link once in the order its L line
+     * sorts, then a walk per fragment, the reference's first and the rest by
+     * name. graphTablesGFA writes them as the GFA they stand for, the same bytes
+     * for the same rows.
+     */
+    tables(kept, fragments) {
+        if (this.format2.length > 0) {
+            placeAndLink(this.format2, kept, (id, node) => {
+                const held = this.nodes.get(id);
+                if (held) {
+                    this.nodes.set(id, { ...held, ...node });
+                }
+            }, (a, b) => {
+                this.links.push(a, b);
+            });
+        }
+        const held = [];
+        for (const id of kept) {
+            const node = this.nodes.get(id);
+            if (node) {
+                held.push({ ...node, id });
+            }
+        }
+        held.sort((x, y) => x.refName < y.refName
+            ? -1
+            : x.refName > y.refName
+                ? 1
+                : x.start - y.start || x.id - y.id);
+        const index = new Map();
+        const names = [];
+        const refNames = [];
+        const refIndex = new Map();
+        const lengths = new Int32Array(held.length);
+        const refs = new Int32Array(held.length);
+        const starts = new Float64Array(held.length);
+        const ranks = new Int32Array(held.length);
+        held.forEach(({ id, refName, start, end, rank }, i) => {
+            index.set(id, i);
+            names.push(String(id));
+            let ref = refIndex.get(refName);
+            if (ref === undefined) {
+                ref = refNames.length;
+                refIndex.set(refName, ref);
+                refNames.push(refName);
+            }
+            lengths[i] = end - start;
+            refs[i] = ref;
+            starts[i] = start;
+            ranks[i] = rank;
+        });
+        // a kept node with no row of its own is named, and not drawn
+        const nodeIndex = (id) => {
+            let i = index.get(id);
+            if (i === undefined) {
+                i = names.length;
+                index.set(id, i);
+                names.push(String(id));
+            }
+            return i;
+        };
+        const byLine = new Map();
+        for (let i = 0; i < this.links.length; i += 2) {
+            const a = this.links[i];
+            const b = this.links[i + 1];
+            const source = Math.floor(a / 2);
+            const target = Math.floor(b / 2);
+            if (kept.has(source) && kept.has(target)) {
+                byLine.set(`${source}\t${a % 2 ? '-' : '+'}\t${target}\t${b % 2 ? '-' : '+'}`, i);
+            }
+        }
+        const lines = [...byLine.keys()].sort();
+        const from = new Int32Array(lines.length);
+        const to = new Int32Array(lines.length);
+        const strands = new Uint8Array(lines.length);
+        lines.forEach((line, k) => {
+            const i = byLine.get(line);
+            const a = this.links[i];
+            const b = this.links[i + 1];
+            from[k] = nodeIndex(Math.floor(a / 2));
+            to[k] = nodeIndex(Math.floor(b / 2));
+            strands[k] = (a % 2) | ((b % 2) << 1);
+        });
+        const walks = this.referenceFirst(fragments);
+        const offsets = new Int32Array(walks.length + 1);
+        walks.forEach(({ ids }, p) => {
+            offsets[p + 1] = offsets[p] + ids.length;
+        });
+        const steps = new Int32Array(offsets[walks.length]);
+        const reversed = new Uint8Array(steps.length);
+        walks.forEach(({ ids, rev }, p) => {
+            const at = offsets[p];
+            for (let j = 0; j < ids.length; j++) {
+                steps[at + j] = nodeIndex(ids[j]);
+            }
+            reversed.set(rev, at);
+        });
+        return {
+            nodes: { names, lengths, refs, starts, ranks, refNames },
+            links: { from, to, strands },
+            walks: {
+                names: walks.map(f => f.name),
+                starts: Float64Array.from(walks, f => f.hapStart),
+                ends: Float64Array.from(walks, f => f.hapEnd),
+                offsets,
+                steps,
+                reversed,
+            },
+        };
+    }
+    // The view anchors on the first walk when no walk names the assembly, as
+    // gbz-base's cut has it; in file order that was whichever haplotype the
+    // chunk listed first, CHM13 on a GRCh38 window
+    referenceFirst(fragments) {
+        const { refName } = this.window;
+        return [...fragments].sort((a, b) => Number(b.name === refName) - Number(a.name === refName) ||
+            (a.name < b.name ? -1 : a.name > b.name ? 1 : a.hapStart - b.hapStart));
+    }
+}
+function endpoint(line, from, to) {
+    return (+line.slice(from, to - 1) * 2 + (line.charCodeAt(to - 1) === 45 ? 1 : 0));
+}
+export const BUDGET_NAMES = {
+    steps: 'the step budget',
+    bytes: 'a byte budget of',
+};
+export function stepBudgetError(rows, stepBudget, readPerKept, window, chunk, filtered, names = BUDGET_NAMES) {
+    var _a;
+    let total = 0;
+    const perChunk = new Map();
+    const walks = new Set();
+    for (const row of rows) {
+        total += row.n;
+        perChunk.set(row.chunkStart, ((_a = perChunk.get(row.chunkStart)) !== null && _a !== void 0 ? _a : 0) + row.n);
+        walks.add(row.name);
+    }
+    const budget = stepBudget * readPerKept;
+    if (total <= budget) {
+        return undefined;
+    }
+    const windowBp = window.end - window.start;
+    let read = 0;
+    let fitsBp = 0;
+    for (const cs of [...perChunk.keys()].sort((a, b) => a - b)) {
+        read += perChunk.get(cs);
+        if (read > budget) {
+            break;
+        }
+        fitsBp = Math.max(fitsBp, cs + chunk - window.start);
+    }
+    const fewerFit = !filtered && walks.size > 2 && (2 * total) / walks.size <= budget;
+    return new RegionTooLargeError(fitsBp > 0
+        ? `Zoom in to about ${bpString(fitsBp)}${fewerFit ? ', or choose fewer haplotypes,' : ''} to see the graph`
+        : `Too many haplotype steps here to draw ${filtered ? 'these haplotypes' : 'every haplotype'} (${read.toLocaleString()} steps read, over ${readPerKept} × ${names.steps} of ${stepBudget.toLocaleString()}); choose fewer haplotypes`, budget, windowBp, Math.max(fitsBp, 1));
+}
+/**
+ * The zoom-in notice for a cut that keeps more steps than `budget`, or
+ * undefined when it fits. The span that fits is the window scaled by how far
+ * over the cut is.
+ */
+export function keptStepsError(fragments, budget, window, filtered) {
+    const total = fragments.reduce((sum, f) => sum + f.ids.length, 0);
+    if (total <= budget) {
+        return undefined;
+    }
+    const windowBp = window.end - window.start;
+    const fitsBp = Math.max(1, Math.floor((windowBp * budget) / total));
+    return new RegionTooLargeError(`Zoom in to about ${bpString(fitsBp)}${filtered ? '' : ', or choose fewer haplotypes,'} to see the graph`, budget, windowBp, fitsBp);
+}
+/**
+ * The zoom-in notice for a cut whose three files would fetch more than
+ * `budget` compressed bytes, or undefined when they fit. `bytesTo(end)` is the
+ * indexes' estimate for a read from the window's first query base to `end`;
+ * the span that fits grows a chunk at a time from the chunk holding the
+ * window's start, and when that chunk and those before it are over on their
+ * own, no zoom fits.
+ */
+export async function byteBudgetError(bytesTo, budget, window, chunk, names = BUDGET_NAMES) {
+    const bytes = await bytesTo(window.end);
+    if (bytes <= budget) {
+        return undefined;
+    }
+    let fitsBp = 0;
+    let least;
+    for (let cs = Math.floor(window.start / chunk) * chunk; cs < window.end; cs += chunk) {
+        const read = await bytesTo(cs + 1);
+        least !== null && least !== void 0 ? least : (least = read);
+        if (read > budget) {
+            break;
+        }
+        fitsBp = cs + chunk - window.start;
+    }
+    const windowBp = window.end - window.start;
+    return new RegionTooLargeError(fitsBp > 0
+        ? `Zoom in to about ${bpString(fitsBp)} to see the graph`
+        : `Too much graph here to fetch (${megabytes(least !== null && least !== void 0 ? least : bytes)} against ${names.bytes} ${megabytes(budget)})`, budget, windowBp, Math.max(fitsBp, 1));
+}
+const megabytes = (bytes) => `${(bytes / 1e6).toFixed(1)} MB`;

package/package.json

@@ -1,6 +1,64 @@
 {
   "name": "@jbrowse/graph-walks",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "Read gfa-to-tabix walk files by range requests and cut a window of a pangenome graph: the reference, every haplotype's walk through it, and the nodes and links they visit",
+  "license": "MIT",
+  "author": {
+    "name": "Colin Diesh",
+    "email": "colin.diesh@gmail.com",
+    "url": "https://github.com/cmdcolin"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/GMOD/graph-walks.git"
+  },
+  "keywords": [
+    "pangenome",
+    "graph",
+    "gfa",
+    "tabix",
+    "haplotype"
+  ],
+  "type": "module",
+  "packageManager": "pnpm@11.20.0",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "publishConfig": {
+    "access": "public",
+    "exports": {
+      ".": {
+        "types": "./esm/index.d.ts",
+        "default": "./esm/index.js"
+      }
+    }
+  },
+  "files": [
+    "esm"
+  ],
+  "scripts": {
+    "build": "rm -rf esm && tsc -p tsconfig.build.json",
+    "typecheck": "tsc --noEmit",
+    "lint": "eslint --report-unused-disable-directives --max-warnings 0",
+    "format": "prettier --write .",
+    "format:check": "prettier --check .",
+    "test": "vitest run",
+    "prepack": "pnpm build",
+    "preversion": "pnpm lint && pnpm format:check && pnpm typecheck && pnpm test && pnpm build",
+    "postversion": "git push --follow-tags"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "devDependencies": {
+    "@eslint/js": "^10.0.1",
+    "@gmod/tabix": "^3.10.1",
+    "eslint": "^10.12.0",
+    "eslint-plugin-import-x": "^4.17.1",
+    "generic-filehandle2": "^2.5.0",
+    "prettier": "^3.9.10",
+    "typescript": "^6.0.3",
+    "typescript-eslint": "^8.71.1",
+    "vitest": "^5.0.3"
+  }
+}