@jbrowse/core 5.0.0-beta.10 → 5.0.0-beta.11

package/esm/util/cachedSetup.d.ts

@@ -1,6 +1,12 @@
 import type { BaseOptions } from '../data_adapters/BaseAdapter/types.ts';
 /** byte-granularity progress reporter to hand an index reader's `onProgress` */
 type OnProgress = (current: number, total?: number) => void;
+/**
+ * How long a setup with no live waiter keeps running before its signal aborts:
+ * `createAbortRotation.begin()` aborts the superseded call before it posts the
+ * replacement, which joins within this window instead of restarting from byte 0.
+ */
+export declare const ABANDONED_SETUP_GRACE_MS = 100;
 /**
  * Memoize an adapter's one-time setup (open the file, read the header, build a
  * parser, download and parse a whole file) into a loader every method can
@@ -17,11 +23,13 @@ type OnProgress = (current: number, total?: number) => void;
  *   moment that fetch was superseded — the display's latest-wins guard gates the
  *   old callback off — so the fetch replacing it awaited a multi-GB parse behind
  *   a blank loading overlay.
- * - **`signal` is withheld from the shared work.** Honoring one caller's
- *   cancel would abort a parse the caller replacing it is already waiting on,
- *   and reject them both. A superseded fetch just stops listening. Cancellation
- *   stays with the per-call work (indexed range queries), which no one else is
- *   waiting on.
+ * - **Cancellation is shared, not per caller.** The setup gets one signal that
+ *   aborts once its last live waiter has aborted and nobody rejoins within
+ *   {@link ABANDONED_SETUP_GRACE_MS}, so a superseded fetch cannot cancel a
+ *   parse its replacement is waiting on, yet a download nobody awaits stops.
+ *   Each caller's own await rejects the moment its own signal aborts. An
+ *   abandoned setup leaves the memo first, so the next call starts afresh and
+ *   an abandoned run's late statuses reach no one.
  * - **`label` is shown only while the *first* attempt is in flight.** Re-entry
  *   on pan/zoom (every getFeatures and byte estimate awaits the loader) would
  *   otherwise re-flash "Downloading index" over an index that is already
@@ -32,7 +40,9 @@ type OnProgress = (current: number, total?: number) => void;
  * a whole-file fetch driving its own download bar — since two labels for one
  * download is worse than one, and phases nest (see `openPhase` in progress.ts).
  * `onProgress` is for handing to an index reader that can upgrade the label to a
- * determinate bar.
+ * determinate bar; like the status, it fans out to every live waiter. It is
+ * undefined without a `label`, so a setup that narrates itself never opts its
+ * reads into streamed progress.
  */
 export declare function cachedSetup<T>({ setup, label, }: {
     setup: (opts: BaseOptions, onProgress?: OnProgress) => Promise<T>;

package/esm/util/cachedSetup.js

@@ -1,4 +1,11 @@
+import { checkAbortSignal, makeAbortError } from './aborting.js';
 import { downloadStatus } from './progress.js';
+/**
+ * How long a setup with no live waiter keeps running before its signal aborts:
+ * `createAbortRotation.begin()` aborts the superseded call before it posts the
+ * replacement, which joins within this window instead of restarting from byte 0.
+ */
+export const ABANDONED_SETUP_GRACE_MS = 100;
 /**
  * Memoize an adapter's one-time setup (open the file, read the header, build a
  * parser, download and parse a whole file) into a loader every method can
@@ -15,11 +22,13 @@ import { downloadStatus } from './progress.js';
  *   moment that fetch was superseded — the display's latest-wins guard gates the
  *   old callback off — so the fetch replacing it awaited a multi-GB parse behind
  *   a blank loading overlay.
- * - **`signal` is withheld from the shared work.** Honoring one caller's
- *   cancel would abort a parse the caller replacing it is already waiting on,
- *   and reject them both. A superseded fetch just stops listening. Cancellation
- *   stays with the per-call work (indexed range queries), which no one else is
- *   waiting on.
+ * - **Cancellation is shared, not per caller.** The setup gets one signal that
+ *   aborts once its last live waiter has aborted and nobody rejoins within
+ *   {@link ABANDONED_SETUP_GRACE_MS}, so a superseded fetch cannot cancel a
+ *   parse its replacement is waiting on, yet a download nobody awaits stops.
+ *   Each caller's own await rejects the moment its own signal aborts. An
+ *   abandoned setup leaves the memo first, so the next call starts afresh and
+ *   an abandoned run's late statuses reach no one.
  * - **`label` is shown only while the *first* attempt is in flight.** Re-entry
  *   on pan/zoom (every getFeatures and byte estimate awaits the loader) would
  *   otherwise re-flash "Downloading index" over an index that is already
@@ -30,48 +39,109 @@ import { downloadStatus } from './progress.js';
  * a whole-file fetch driving its own download bar — since two labels for one
  * download is worse than one, and phases nest (see `openPhase` in progress.ts).
  * `onProgress` is for handing to an index reader that can upgrade the label to a
- * determinate bar.
+ * determinate bar; like the status, it fans out to every live waiter. It is
+ * undefined without a `label`, so a setup that narrates itself never opts its
+ * reads into streamed progress.
  */
 export function cachedSetup({ setup, label, }) {
-    const waiting = new Set();
-    let cached;
-    let ready = false;
-    const run = (opts, onProgress) => {
-        cached ??= setup({
-            ...opts,
-            signal: undefined,
-            statusCallback: status => {
-                for (const cb of waiting) {
-                    cb(status);
+    let flight;
+    let resident;
+    const launch = (started, opts) => setup({
+        ...opts,
+        signal: started.controller.signal,
+        statusCallback: status => {
+            for (const cb of started.waiting.keys()) {
+                cb(status);
+            }
+        },
+    }, label === undefined
+        ? undefined
+        : (current, total) => {
+            for (const report of started.reporters.keys()) {
+                report(current, total);
+            }
+        }).then(value => {
+        clearTimeout(started.grace);
+        if (flight === started) {
+            resident = { value };
+        }
+        return value;
+    }, (e) => {
+        clearTimeout(started.grace);
+        if (flight === started) {
+            flight = undefined;
+        }
+        throw e;
+    });
+    const abandon = (started) => {
+        if (flight === started) {
+            flight = undefined;
+            started.controller.abort();
+        }
+    };
+    const join = (opts, onProgress) => {
+        const { signal, statusCallback } = opts;
+        const started = (flight ??= {
+            controller: new AbortController(),
+            waiting: new Map(),
+            reporters: new Map(),
+            live: 0,
+        });
+        clearTimeout(started.grace);
+        started.live++;
+        count(started.waiting, statusCallback);
+        count(started.reporters, onProgress);
+        const promise = (started.promise ??= launch(started, opts));
+        return new Promise((resolve, reject) => {
+            let joined = true;
+            const leave = () => {
+                if (joined) {
+                    joined = false;
+                    signal?.removeEventListener('abort', onAbort);
+                    started.live--;
+                    uncount(started.waiting, statusCallback);
+                    uncount(started.reporters, onProgress);
+                }
+            };
+            const onAbort = () => {
+                leave();
+                reject(makeAbortError());
+                if (started.live === 0) {
+                    started.grace = setTimeout(() => {
+                        abandon(started);
+                    }, ABANDONED_SETUP_GRACE_MS);
                 }
-            },
-        }, onProgress)
-            .then(result => {
-            ready = true;
-            return result;
-        })
-            .catch((e) => {
-            cached = undefined;
-            throw e;
+            };
+            signal?.addEventListener('abort', onAbort, { once: true });
+            promise.finally(leave).then(resolve, reject);
         });
-        return cached;
     };
     return async (opts = {}) => {
-        const { statusCallback } = opts;
-        if (statusCallback) {
-            waiting.add(statusCallback);
+        if (resident) {
+            return resident.value;
         }
-        try {
-            return await (label === undefined || ready
-                ? run(opts)
-                : downloadStatus(label, statusCallback, onProgress => run(opts, onProgress)));
+        checkAbortSignal(opts.signal);
+        const wait = (onProgress) => join(opts, onProgress);
+        return label === undefined
+            ? wait()
+            : downloadStatus(label, opts.statusCallback, wait);
+    };
+}
+function count(counts, key) {
+    if (key) {
+        counts.set(key, (counts.get(key) ?? 0) + 1);
+    }
+}
+function uncount(counts, key) {
+    if (key) {
+        const left = counts.get(key) - 1;
+        if (left > 0) {
+            counts.set(key, left);
         }
-        finally {
-            if (statusCallback) {
-                waiting.delete(statusCallback);
-            }
+        else {
+            counts.delete(key);
         }
-    };
+    }
 }
 /**
  * {@link cachedSetup} taking a bare function instead of an options object.

package/esm/util/cellMatches.d.ts

@@ -0,0 +1,23 @@
+import type { AggregateStep, BinStep, CellsStep, TransformStep } from './markEncodingTypes.ts';
+import type { Staged } from './stepTables.ts';
+/**
+ * Whether a `cells` step, an interval `bin` and an `aggregate` run as one
+ * walk: the bin cuts the runs' own extents at whole-base edges it writes as
+ * `start` and `end`, the aggregate groups by those, and every op is the sum or
+ * mean of `match` weighted by `overlap`, so a bin's answer is its bases
+ * matched and compared.
+ */
+export declare function fusesCellMatches(_cells: CellsStep, bin: BinStep, agg: AggregateStep): boolean;
+/** Whether a step list starts with the bin and aggregate a `cells` fuses with. */
+export declare function fusesAfterCells(cells: CellsStep, steps: readonly TransformStep[] | undefined): boolean;
+/**
+ * `cells`, then a `bin` over the runs' extents and the `aggregate` of their
+ * `match` weighted by `overlap`, in one walk with no run table: each bin's
+ * bases matched and compared, counted straight off the texts. Every run
+ * meets its bins in the order its lanes would hold it, so the groups come out
+ * where the three steps put them, and a weighted sum of 0s and 1s is an
+ * integer, so each sum is theirs to the bit. Undefined where a row's start is
+ * not a whole base inside a Uint32 or the bins spread too thinly for a dense
+ * index, and the steps run as written.
+ */
+export declare function binnedCellMatches({ table, bounds }: Staged, cells: CellsStep, bin: BinStep, agg: AggregateStep): Staged | undefined;

package/esm/util/cellMatches.js

@@ -0,0 +1,264 @@
+// A `cells` step, the interval `bin` behind it and the `aggregate` of its
+// matches as one walk over the rows' bytes, which `runSteps` in
+// featureTransforms.ts reaches by relative path only.
+import { fusesBinAggregate } from './binnedAggregate.js';
+import { DASH, DEFAULT_CELLS_FIELD, MATCH, MISMATCH, NO_STATE, SPACE, cellState, firstDrawn, isGapByte, lastDrawn, walkedTexts, } from './cellsStep.js';
+import { numberReaderOf, readerOf } from './featureTable.js';
+import { BIN_OVERLAP_FIELD, DEFAULT_BIN_AS, OpSums, groupKey, madeGroups, } from './stepTables.js';
+const isEdges = (pair) => pair?.length === 2 && pair[0] === 'start' && pair[1] === 'end';
+/**
+ * Whether a `cells` step, an interval `bin` and an `aggregate` run as one
+ * walk: the bin cuts the runs' own extents at whole-base edges it writes as
+ * `start` and `end`, the aggregate groups by those, and every op is the sum or
+ * mean of `match` weighted by `overlap`, so a bin's answer is its bases
+ * matched and compared.
+ */
+export function fusesCellMatches(_cells, bin, agg) {
+    return (fusesBinAggregate(bin, agg) &&
+        isEdges(bin.fields) &&
+        isEdges(bin.as ?? DEFAULT_BIN_AS) &&
+        Number.isInteger(bin.step) &&
+        bin.step > 0 &&
+        agg.ops.length > 0 &&
+        agg.ops.every(op => (op.op === 'mean' || op.op === 'sum') &&
+            op.field === 'match' &&
+            op.weight === BIN_OVERLAP_FIELD));
+}
+/** Whether a step list starts with the bin and aggregate a `cells` fuses with. */
+export function fusesAfterCells(cells, steps) {
+    const [bin, agg] = steps ?? [];
+    return (bin?.type === 'bin' &&
+        agg?.type === 'aggregate' &&
+        fusesCellMatches(cells, bin, agg));
+}
+// Where the widest section spans more bins than this over the bins all the
+// rows could reach, its dense index costs more than the runs would, and the
+// steps run as written.
+const SPARSE_BINS_SLACK = 65536;
+// A run's start is a Uint32 lane in the unfused `cells`.
+const MAX_POSITION = 2 ** 32;
+/**
+ * The groups the pieces of a section's runs meet, numbered in the order they
+ * meet them, over one dense index of the section's bins that each section
+ * reuses; with each bin's bases matched and compared.
+ */
+class SectionBins {
+    groupOfBin;
+    matched;
+    compared;
+    groupRow;
+    groupBin;
+    groupSection;
+    groupMatched;
+    groupCompared;
+    groups = 0;
+    low = 0;
+    section = 0;
+    sectionStart = 0;
+    constructor(span, capacity) {
+        this.groupOfBin = new Int32Array(span).fill(-1);
+        this.matched = new Float64Array(span);
+        this.compared = new Float64Array(span);
+        this.groupRow = new Uint32Array(capacity);
+        this.groupBin = new Float64Array(capacity);
+        this.groupSection = new Uint32Array(capacity);
+        this.groupMatched = new Float64Array(capacity);
+        this.groupCompared = new Float64Array(capacity);
+    }
+    open(section, low) {
+        this.section = section;
+        this.low = low;
+        this.sectionStart = this.groups;
+    }
+    // A piece's bin from `from` to `to`, a new group where no piece met it.
+    touch(from, to, r) {
+        const { groupOfBin, low } = this;
+        for (let b = from; b <= to; b++) {
+            if (groupOfBin[b - low] < 0) {
+                const g = this.groups++;
+                groupOfBin[b - low] = g;
+                this.groupRow[g] = r;
+                this.groupBin[g] = b;
+                this.groupSection[g] = this.section;
+            }
+        }
+    }
+    // The section's counts onto its groups, and its bins cleared for the next.
+    close() {
+        const { groupOfBin, matched, compared, low } = this;
+        for (let g = this.sectionStart; g < this.groups; g++) {
+            const k = this.groupBin[g] - low;
+            this.groupMatched[g] = matched[k];
+            this.groupCompared[g] = compared[k];
+            matched[k] = 0;
+            compared[k] = 0;
+            groupOfBin[k] = -1;
+        }
+    }
+    /**
+     * One row's runs as `cells` writes them, each met by its bins as the bin
+     * step cuts it and in the order the lanes hold them: a run when the next
+     * column's state differs, an insertion at the reference base after it,
+     * ahead of the run still open. The bases a run of matches or mismatches
+     * holds are counted in their bins as the walk passes them.
+     */
+    walk(r, startPos, size, rowBytes, rowAt, rowLen, refBytes, refAt, refLen) {
+        const { matched, compared, low } = this;
+        const first = firstDrawn(rowBytes, rowAt, rowLen);
+        const last = lastDrawn(rowBytes, rowAt, rowLen, first);
+        let pos = startPos;
+        let bin = Math.floor(pos / size);
+        let edge = bin * size + size;
+        let m = 0;
+        let c = 0;
+        let runOpen = false;
+        let runBin = 0;
+        let runState = NO_STATE;
+        let runBase = -1;
+        let inserting = false;
+        for (let col = 0; col < refLen; col++) {
+            const refByte = refBytes[refAt + col];
+            const rowByte = col < rowLen ? rowBytes[rowAt + col] : SPACE;
+            if (refByte === DASH) {
+                if (!inserting && col < rowLen && !isGapByte(rowByte)) {
+                    inserting = true;
+                }
+                continue;
+            }
+            if (inserting) {
+                this.touch(bin, bin, r);
+                inserting = false;
+            }
+            const st = cellState(refByte, rowByte, col >= first && col <= last && col < rowLen);
+            const base = st === MISMATCH ? rowByte : -1;
+            if (st !== runState || base !== runBase) {
+                if (runOpen) {
+                    this.touch(runBin, pos > edge - size ? bin : bin - 1, r);
+                }
+                runOpen = st !== NO_STATE;
+                runBin = bin;
+                runState = st;
+                runBase = base;
+            }
+            if (st === MATCH) {
+                m++;
+                c++;
+            }
+            else if (st === MISMATCH) {
+                c++;
+            }
+            pos++;
+            if (pos === edge) {
+                if (c > 0) {
+                    matched[bin - low] += m;
+                    compared[bin - low] += c;
+                    m = 0;
+                    c = 0;
+                }
+                bin++;
+                edge += size;
+            }
+        }
+        if (runOpen) {
+            this.touch(runBin, pos > edge - size ? bin : bin - 1, r);
+        }
+        if (c > 0) {
+            matched[bin - low] += m;
+            compared[bin - low] += c;
+        }
+    }
+}
+/**
+ * `cells`, then a `bin` over the runs' extents and the `aggregate` of their
+ * `match` weighted by `overlap`, in one walk with no run table: each bin's
+ * bases matched and compared, counted straight off the texts. Every run
+ * meets its bins in the order its lanes would hold it, so the groups come out
+ * where the three steps put them, and a weighted sum of 0s and 1s is an
+ * integer, so each sum is theirs to the bit. Undefined where a row's start is
+ * not a whole base inside a Uint32 or the bins spread too thinly for a dense
+ * index, and the steps run as written.
+ */
+export function binnedCellMatches({ table, bounds }, cells, bin, agg) {
+    const { field = DEFAULT_CELLS_FIELD } = cells;
+    const texts = walkedTexts(table, field);
+    const readStart = numberReaderOf(table.column('start'));
+    const size = bin.step;
+    const sections = bounds.length - 1;
+    const lowBin = new Float64Array(sections);
+    let widest = 0;
+    let reach = 0;
+    let spans = 0;
+    for (let s = 0; s < sections; s++) {
+        let low = Infinity;
+        let high = -Infinity;
+        for (let r = bounds[s]; r < bounds[s + 1]; r++) {
+            const refLen = texts.reference.length[texts.refOf(r)];
+            const pos = readStart(r);
+            if (refLen > 0 && pos >= 0) {
+                if (!Number.isInteger(pos) || pos + refLen >= MAX_POSITION) {
+                    return undefined;
+                }
+                const lo = Math.floor(pos / size);
+                const hi = Math.floor((pos + refLen) / size);
+                low = Math.min(low, lo);
+                high = Math.max(high, hi);
+                reach += hi - lo + 1;
+            }
+        }
+        lowBin[s] = low;
+        const span = high >= low ? high - low + 1 : 0;
+        widest = Math.max(widest, span);
+        spans += span;
+    }
+    if (widest > 2 * reach + SPARSE_BINS_SLACK) {
+        return undefined;
+    }
+    const { bytes: rowBytes, offset: rowOffset, length: rowLength } = texts.row;
+    const { bytes: refBytes, offset: refOffset, length: refLength, } = texts.reference;
+    const walk = new SectionBins(widest, Math.min(spans, reach));
+    for (let s = 0; s < sections; s++) {
+        walk.open(s, lowBin[s]);
+        for (let r = bounds[s]; r < bounds[s + 1]; r++) {
+            const f = texts.refOf(r);
+            const refLen = refLength[f];
+            const pos = readStart(r);
+            if (refLen > 0 && pos >= 0) {
+                const k = texts.rowOf(r);
+                walk.walk(r, pos, size, rowBytes, rowOffset[k], rowLength[k], refBytes, refOffset[f], refLen);
+            }
+        }
+        walk.close();
+    }
+    const { groups, groupRow, groupBin, groupSection } = walk;
+    const ops = agg.ops.map(op => {
+        const sums = new OpSums(op, groups);
+        sums.sum.set(walk.groupMatched.subarray(0, groups));
+        sums.weight.set(walk.groupCompared.subarray(0, groups));
+        return sums.column(groups);
+    });
+    const start = new Float64Array(groups);
+    const end = new Float64Array(groups);
+    for (let g = 0; g < groups; g++) {
+        start[g] = groupBin[g] * size;
+        end[g] = start[g] + size;
+    }
+    const readRef = readerOf(table.column('refName'));
+    const refNames = new Array(groups);
+    for (let g = 0; g < groups; g++) {
+        refNames[g] = readRef(groupRow[g]);
+    }
+    return madeGroups({
+        groups,
+        keys: (agg.groupby ?? []).map((field) => [
+            field,
+            Array.from(field === 'start' ? start : end, groupKey),
+        ]),
+        ops,
+        step: agg,
+        refNames,
+        start,
+        end,
+        sectionOf: groupSection,
+        sections: bounds.length,
+    });
+}

package/esm/util/cellsStep.d.ts

@@ -0,0 +1,29 @@
+import type { FeatureTable, TextColumn } from './featureTable.ts';
+import type { CellsStep } from './markEncodingTypes.ts';
+import type { Staged } from './stepTables.ts';
+export declare const DEFAULT_CELLS_FIELD = "seq";
+export declare const DASH = 45;
+export declare const SPACE = 32;
+export declare const MATCH = 0;
+export declare const MISMATCH = 1;
+export declare const NO_STATE = 255;
+export declare function isGapByte(b: number): b is 32 | 45;
+export declare function firstDrawn(bytes: Uint8Array, at: number, len: number): number;
+export declare function lastDrawn(bytes: Uint8Array, at: number, len: number, first: number): number;
+export declare function cellState(refByte: number, rowByte: number, drawn: boolean): 0 | 1 | 2 | 255;
+/**
+ * The texts a `cells` walk reads, by the row of the table it walked: each
+ * row's own, and the reference it is read against, which a row fanned out of
+ * a container reads off the container.
+ */
+declare class WalkedTexts {
+    readonly row: TextColumn;
+    readonly reference: TextColumn;
+    private readonly refRow;
+    constructor(row: TextColumn, ref: TextColumn, refRow?: Uint32Array);
+    rowOf(r: number): number;
+    refOf(r: number): number;
+}
+export declare function walkedTexts(table: FeatureTable, field: string): WalkedTexts;
+export declare function cells({ table, bounds }: Staged, step: CellsStep): Staged;
+export {};