@genome-spy/core 0.78.0 → 0.79.1

package/dist/src/data/sources/lazy/bigBedSource.js

@@ -2,19 +2,28 @@ import {
     activateExprRefProps,
     withoutExprRef,
 } from "../../../paramRuntime/paramUtils.js";
-import addBaseUrl from "../../../utils/addBaseUrl.js";
+import { createDescriptorFieldAttacher } from "../urlDescriptor.js";
+import UrlDescriptorController from "../urlDescriptorController.js";
+import UrlDescriptorState, {
+    updateUrlDescriptorState,
+} from "../urlDescriptorState.js";
 import { registerBuiltInLazyDataSource } from "./lazyDataSourceRegistry.js";
 import SingleAxisWindowedSource from "./singleAxisWindowedSource.js";
 
 export default class BigBedSource extends SingleAxisWindowedSource {
-    /** @type {import("@gmod/bed").default} */
-    parser;
+    /**
+     * @typedef {object} BigBedHandle
+     * @prop {(datum: Record<string, any>) => Record<string, any>} attachFields
+     * @prop {import("@gmod/bbi").BigBed} bbi
+     * @prop {(chrom: string, fields: { start: number, end: number, rest?: string }) => Record<string, any>} parseLine
+     * @prop {string} url
+     */
 
-    /** @type {import("@gmod/bbi").BigBed} */
-    bbi;
+    /** @type {UrlDescriptorState<BigBedHandle>} */
+    #descriptorState = new UrlDescriptorState();
 
-    /** @type {(chrom: string, fields: { start: number, end: number, rest?: string }) => Record<string, any>} */
-    parseLine;
+    /** @type {UrlDescriptorController} */
+    #urlDescriptors;
 
     /**
      * @param {import("../../../spec/data.js").BigBedData} params
@@ -38,7 +47,7 @@ export default class BigBedSource extends SingleAxisWindowedSource {
             paramsWithDefaults,
             (props) => {
                 if (props.has("url")) {
-                    this.#initialize().then(() => this.reloadLastDomain());
+                    this.#reloadIfCurrentDomainNeedsData();
                 } else if (props.has("windowSize")) {
                     this.reloadLastDomain();
                 }
@@ -47,6 +56,11 @@ export default class BigBedSource extends SingleAxisWindowedSource {
             { batchMode: "whenPropagated" }
         );
 
+        this.#urlDescriptors = new UrlDescriptorController(this, {
+            getUrl: () => this.params.url,
+            onChange: () => this.#reloadIfCurrentDomainNeedsData(),
+        });
+
         if (!this.params.url) {
             throw new Error("No URL provided for BigBedSource");
         }
@@ -61,78 +75,120 @@ export default class BigBedSource extends SingleAxisWindowedSource {
     }
 
     #initialize() {
-        this.initializedPromise = new Promise((resolve, reject) => {
-            Promise.all([
-                import("@gmod/bed"),
-                import("@gmod/bbi"),
-                import("generic-filehandle2"),
-            ]).then(([bed, { BigBed }, { RemoteFile }]) => {
-                const BED = bed.default;
-
-                this.bbi = new BigBed({
-                    filehandle: new RemoteFile(
-                        addBaseUrl(
-                            withoutExprRef(this.params.url),
-                            this.view.getBaseUrl()
-                        )
-                    ),
-                });
-
-                this.setLoadingStatus("loading");
-                this.bbi
-                    .getHeader()
-                    .then(async (header) => {
-                        // @ts-ignore TODO: Fix
-                        this.parser = new BED({ autoSql: header.autoSql });
-                        try {
-                            const fastParser = makeFastParser(this.parser);
-                            this.parseLine = (chrom, f) =>
-                                fastParser(chrom, f.start, f.end, f.rest);
-                        } catch {
-                            this.parseLine = (chrom, f) =>
-                                this.parser.parseLine(
-                                    `${chrom}\t${f.start}\t${f.end}\t${f.rest}`
-                                );
-                        }
-
-                        this.setLoadingStatus("complete");
-                        resolve();
-                    })
-                    .catch((e) => {
-                        // Load empty data
-                        this.load();
-                        this.setLoadingStatus(
-                            "error",
-                            `${withoutExprRef(this.params.url)}: ${e.message}`
-                        );
-                        reject(e);
-                    });
-            });
+        const initializePromise = this.#doInitialize();
+        this.initializedPromise = initializePromise;
+        return initializePromise;
+    }
+
+    /**
+     * Refreshes active descriptors and reloads the current domain only if the
+     * current loaded data does not cover the new active descriptor set.
+     */
+    async #reloadIfCurrentDomainNeedsData() {
+        try {
+            await this.#initialize();
+
+            if (
+                !this.isDataReadyForDomain({
+                    [this.channel]: this.scaleResolution.getDomain(),
+                })
+            ) {
+                this.reloadLastDomain();
+            }
+        } catch {
+            // Initialization has already updated the loading status.
+        }
+    }
+
+    async #doInitialize() {
+        await updateUrlDescriptorState({
+            controller: this.#urlDescriptors,
+            state: this.#descriptorState,
+            clearData: () => this.load(),
+            setLoadingStatus: (status, detail) =>
+                this.setLoadingStatus(status, detail),
+            loadModules: loadBigBedModules,
+            createHandle: (descriptor, { BigBed, RemoteFile, BED }) =>
+                this.#createHandle(descriptor, BigBed, RemoteFile, BED),
+        });
+    }
+
+    /**
+     * @param {import("../urlDescriptor.js").UrlDescriptor} descriptor
+     * @param {typeof import("@gmod/bbi").BigBed} BigBed
+     * @param {typeof import("generic-filehandle2").RemoteFile} RemoteFile
+     * @param {typeof import("@gmod/bed").default} BED
+     * @returns {Promise<BigBedHandle>}
+     */
+    async #createHandle(descriptor, BigBed, RemoteFile, BED) {
+        const bbi = new BigBed({
+            filehandle: new RemoteFile(descriptor.url),
         });
+        const header = await bbi.getHeader();
+
+        // @ts-ignore TODO: Fix
+        const parser = new BED({ autoSql: header.autoSql });
+        /** @type {BigBedHandle["parseLine"]} */
+        let parseLine;
+        try {
+            const fastParser = makeFastParser(parser);
+            parseLine = (chrom, f) => fastParser(chrom, f.start, f.end, f.rest);
+        } catch {
+            parseLine = (chrom, f) =>
+                parser.parseLine(`${chrom}\t${f.start}\t${f.end}\t${f.rest}`);
+        }
 
-        return this.initializedPromise;
+        return {
+            attachFields: createDescriptorFieldAttacher(descriptor.fields),
+            bbi,
+            parseLine,
+            url: descriptor.url,
+        };
     }
 
     /**
      * @param {number[]} interval linearized domain
      */
     async loadInterval(interval) {
+        const handles = this.#descriptorState.handles;
         const features = await this.discretizeAndLoad(
             interval,
             async (d, signal) =>
-                this.bbi
-                    .getFeatures(d.chrom, d.startPos, d.endPos, {
-                        signal,
-                    })
-                    .then((features) =>
-                        features.map((f) => this.parseLine(d.chrom, f))
+                (
+                    await Promise.all(
+                        handles.map((handle) =>
+                            handle.bbi
+                                .getFeatures(d.chrom, d.startPos, d.endPos, {
+                                    signal,
+                                })
+                                .then((features) =>
+                                    features.map((f) =>
+                                        handle.attachFields(
+                                            handle.parseLine(d.chrom, f)
+                                        )
+                                    )
+                                )
+                        )
                     )
+                ).flat()
         );
 
         if (features) {
             this.publishData(features);
+            this.#descriptorState.markLoaded();
         }
     }
+
+    /**
+     * @param {import("./singleAxisLazySource.js").DataReadinessRequest} request
+     * @returns {boolean}
+     */
+    isDataReadyForDomain(request) {
+        return (
+            this.#descriptorState.activeSetLoaded &&
+            super.isDataReadyForDomain(request)
+        );
+    }
 }
 
 /**
@@ -310,6 +366,15 @@ function isBigBedSource(params) {
 
 registerBuiltInLazyDataSource(isBigBedSource, BigBedSource);
 
+async function loadBigBedModules() {
+    const [bed, { BigBed }, { RemoteFile }] = await Promise.all([
+        import("@gmod/bed"),
+        import("@gmod/bbi"),
+        import("generic-filehandle2"),
+    ]);
+    return { BigBed, RemoteFile, BED: bed.default };
+}
+
 /**
  * @param {T[]} arr
  * @param {number} size

package/dist/src/data/sources/lazy/bigWigSource.d.ts

@@ -16,9 +16,9 @@ export default class BigWigSource extends SingleAxisWindowedSource {
     onDomainChanged(domain: number[]): Promise<void>;
     /**
      * @param {number[]} interval linearized domain
-     * @param {number} reductionLevel
+     * @param {number[]} selectedReductionLevels
      */
-    loadInterval(interval: number[], reductionLevel: number): Promise<void>;
+    loadInterval(interval: number[], selectedReductionLevels: number[]): Promise<void>;
     #private;
 }
 import SingleAxisWindowedSource from "./singleAxisWindowedSource.js";

package/dist/src/data/sources/lazy/bigWigSource.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"bigWigSource.d.ts","sourceRoot":"","sources":["../../../../../src/data/sources/lazy/bigWigSource.js"],"names":[],"mappings":"AAQA;;GAEG;AACH;IAOI;;;OAGG;IACH,oBAHW,OAAO,uBAAuB,EAAE,UAAU,QAC1C,OAAO,uBAAuB,EAAE,OAAO,EAoCjD;IArBG,mDAYC;IA+DL;;;;OAIG;IACH,wBAFW,MAAM,EAAE,iBAqBlB;IAED;;;OAGG;IAEH,uBAJW,MAAM,EAAE,kBACR,MAAM,iBAkChB;;CACJ;qCAvKoC,+BAA+B"}
+{"version":3,"file":"bigWigSource.d.ts","sourceRoot":"","sources":["../../../../../src/data/sources/lazy/bigWigSource.js"],"names":[],"mappings":"AAYA;;GAEG;AACH;IAeI;;;OAGG;IACH,oBAHW,OAAO,uBAAuB,EAAE,UAAU,QAC1C,OAAO,uBAAuB,EAAE,OAAO,EAyCjD;IA1BG,mDAYC;IA+FL;;;;OAIG;IACH,wBAFW,MAAM,EAAE,iBA+BlB;IAED;;;OAGG;IAEH,uBAJW,MAAM,EAAE,2BACR,MAAM,EAAE,iBAqBlB;;CA8FJ;qCA9SoC,+BAA+B"}

package/dist/src/data/sources/lazy/bigWigSource.js

@@ -2,19 +2,31 @@ import {
     activateExprRefProps,
     withoutExprRef,
 } from "../../../paramRuntime/paramUtils.js";
-import addBaseUrl from "../../../utils/addBaseUrl.js";
 import { registerBuiltInLazyDataSource } from "./lazyDataSourceRegistry.js";
 import SingleAxisWindowedSource from "./singleAxisWindowedSource.js";
+import { createDescriptorFieldAttacher } from "../urlDescriptor.js";
+import UrlDescriptorController from "../urlDescriptorController.js";
+import UrlDescriptorState, {
+    updateUrlDescriptorState,
+} from "../urlDescriptorState.js";
 
 /**
  *
  */
 export default class BigWigSource extends SingleAxisWindowedSource {
-    /** @type {number[]} */
-    #reductionLevels = [];
+    /**
+     * @typedef {object} BigWigHandle
+     * @prop {import("@gmod/bbi").BigWig} bbi
+     * @prop {(datum: Record<string, any>) => Record<string, any>} attachFields
+     * @prop {number[]} reductionLevels
+     * @prop {string} url
+     */
+
+    /** @type {UrlDescriptorState<BigWigHandle>} */
+    #descriptorState = new UrlDescriptorState();
 
-    /** @type {import("@gmod/bbi").BigWig} */
-    #bbi;
+    /** @type {UrlDescriptorController} */
+    #urlDescriptors;
 
     /**
      * @param {import("../../../spec/data.js").BigWigData} params
@@ -38,7 +50,7 @@ export default class BigWigSource extends SingleAxisWindowedSource {
             paramsWithDefaults,
             (props) => {
                 if (props.has("url")) {
-                    this.#initialize().then(() => this.reloadLastDomain());
+                    this.#reloadIfCurrentDomainNeedsData();
                 } else if (props.has("pixelsPerBin")) {
                     this.reloadLastDomain();
                 }
@@ -47,6 +59,11 @@ export default class BigWigSource extends SingleAxisWindowedSource {
             { batchMode: "whenPropagated" }
         );
 
+        this.#urlDescriptors = new UrlDescriptorController(this, {
+            getUrl: () => this.params.url,
+            onChange: () => this.#reloadIfCurrentDomainNeedsData(),
+        });
+
         if (!this.params.url) {
             throw new Error("No URL provided for BigWigSource");
         }
@@ -60,52 +77,79 @@ export default class BigWigSource extends SingleAxisWindowedSource {
         return "bigWigSource";
     }
 
+    /**
+     * @returns {Promise<void>}
+     */
     #initialize() {
-        this.initializedPromise = new Promise((resolve, reject) => {
-            Promise.all([
-                import("@gmod/bbi"),
-                import("generic-filehandle2"),
-            ]).then(([{ BigWig }, { RemoteFile }]) => {
-                this.#bbi = new BigWig({
-                    filehandle: new RemoteFile(
-                        addBaseUrl(
-                            withoutExprRef(this.params.url),
-                            this.view.getBaseUrl()
-                        )
-                    ),
-                });
-
-                this.setLoadingStatus("loading");
-                this.#bbi
-                    .getHeader()
-                    .then((header) => {
-                        this.#reductionLevels =
-                            /** @type {{reductionLevel: number}[]} */ (
-                                header.zoomLevels
-                            )
-                                .map((z) => z.reductionLevel)
-                                .reverse();
-
-                        // Add the non-reduced level. Not sure if this is the best way to do it.
-                        // Afaik, the non-reduced bin size is not available in the header.
-                        this.#reductionLevels.push(1);
-
-                        this.setLoadingStatus("complete");
-                        resolve();
-                    })
-                    .catch((e) => {
-                        // Load empty data
-                        this.load();
-                        this.setLoadingStatus(
-                            "error",
-                            `${withoutExprRef(this.params.url)}: ${e.message}`
-                        );
-                        reject(e);
-                    });
-            });
+        const initializePromise = this.#doInitialize();
+        this.initializedPromise = initializePromise;
+        return initializePromise;
+    }
+
+    /**
+     * Refreshes active descriptors and reloads the current domain only if the
+     * current loaded data does not cover the new active descriptor set.
+     */
+    async #reloadIfCurrentDomainNeedsData() {
+        try {
+            await this.#initialize();
+
+            if (
+                !this.isDataReadyForDomain({
+                    [this.channel]: this.scaleResolution.getDomain(),
+                })
+            ) {
+                this.reloadLastDomain();
+            }
+        } catch {
+            // Initialization has already updated the loading status.
+        }
+    }
+
+    /**
+     * @returns {Promise<void>}
+     */
+    async #doInitialize() {
+        await updateUrlDescriptorState({
+            controller: this.#urlDescriptors,
+            state: this.#descriptorState,
+            clearData: () => this.load(),
+            setLoadingStatus: (status, detail) =>
+                this.setLoadingStatus(status, detail),
+            loadModules: loadBigWigModules,
+            createHandle: (descriptor, { BigWig, RemoteFile }) =>
+                this.#createHandle(descriptor, BigWig, RemoteFile),
         });
+    }
 
-        return this.initializedPromise;
+    /**
+     * @param {import("../urlDescriptor.js").UrlDescriptor} descriptor
+     * @param {typeof import("@gmod/bbi").BigWig} BigWig
+     * @param {typeof import("generic-filehandle2").RemoteFile} RemoteFile
+     * @returns {Promise<BigWigHandle>}
+     */
+    async #createHandle(descriptor, BigWig, RemoteFile) {
+        const bbi = new BigWig({
+            filehandle: new RemoteFile(descriptor.url),
+        });
+        const header = await bbi.getHeader();
+        const reductionLevels = /** @type {{reductionLevel: number}[]} */ (
+            header.zoomLevels
+        )
+            .map((z) => z.reductionLevel)
+            .reverse();
+
+        // Add the non-reduced level. Not sure if this is the best way to do it.
+        // Afaik, the non-reduced bin size is not available in the header.
+        reductionLevels.push(1);
+
+        const handle = {
+            bbi,
+            attachFields: createDescriptorFieldAttacher(descriptor.fields),
+            reductionLevels,
+            url: descriptor.url,
+        };
+        return handle;
     }
 
     /**
@@ -116,42 +160,116 @@ export default class BigWigSource extends SingleAxisWindowedSource {
     async onDomainChanged(domain) {
         await this.initializedPromise;
 
+        const handles = this.#descriptorState.handles;
+        if (!handles.length) {
+            this.publishData([]);
+            this.#descriptorState.markLoaded();
+            return;
+        }
+
         // TODO: Postpone the initial load until layout is computed and remove 700.
         const length = this.scaleResolution.getAxisLength() || 700;
 
-        const reductionLevel = findReductionLevel(
-            domain,
-            length,
-            this.#reductionLevels
+        const selectedReductionLevels = handles.map((handle) =>
+            findReductionLevel(domain, length, handle.reductionLevels)
         );
 
         // The sensible minimum window size actually depends on the non-reduced data density...
         // Using 5000 as a default to avoid too many requests.
-        const windowSize = Math.max(reductionLevel * length, 5000);
+        const windowSize = Math.max(
+            ...selectedReductionLevels.map(
+                (reductionLevel) => reductionLevel * length
+            ),
+            5000
+        );
 
         this.callIfWindowsChanged(domain, windowSize, (quantizedInterval) =>
-            this.loadInterval(quantizedInterval, reductionLevel)
+            this.loadInterval(quantizedInterval, selectedReductionLevels)
         );
     }
 
     /**
      * @param {number[]} interval linearized domain
-     * @param {number} reductionLevel
+     * @param {number[]} selectedReductionLevels
      */
     // @ts-expect-error
-    async loadInterval(interval, reductionLevel) {
-        const scale =
-            1 / 2 / reductionLevel / withoutExprRef(this.params.pixelsPerBin);
+    async loadInterval(interval, selectedReductionLevels) {
+        const handles = this.#descriptorState.handles;
         const featureChunks = await this.discretizeAndLoad(interval, {
             load: (d, signal) =>
-                this.#bbi
-                    .getFeatures(d.chrom, d.startPos, d.endPos, {
-                        scale,
-                        signal,
-                    })
-                    .then((features) => mapFeatures(d.chrom, features)),
+                this.#loadFeatures(d, handles, selectedReductionLevels, signal),
             loadBatch: (intervals, signal) =>
-                this.#bbi
+                this.#loadFeatureBatches(
+                    intervals,
+                    handles,
+                    selectedReductionLevels,
+                    signal
+                ),
+        });
+
+        if (featureChunks) {
+            this.publishData(featureChunks);
+            this.#descriptorState.markLoaded();
+        }
+    }
+
+    /**
+     * @param {import("@genome-spy/core/genome/genome.js").DiscreteChromosomeInterval} interval
+     * @param {BigWigHandle[]} handles
+     * @param {number[]} selectedReductionLevels
+     * @param {AbortSignal} signal
+     */
+    async #loadFeatures(interval, handles, selectedReductionLevels, signal) {
+        const featuresByHandle = await Promise.all(
+            handles.map((handle, i) => {
+                const scale = bigWigScale(
+                    selectedReductionLevels[i],
+                    withoutExprRef(this.params.pixelsPerBin)
+                );
+
+                return handle.bbi
+                    .getFeatures(
+                        interval.chrom,
+                        interval.startPos,
+                        interval.endPos,
+                        {
+                            scale,
+                            signal,
+                        }
+                    )
+                    .then((features) =>
+                        mapFeatures(
+                            interval.chrom,
+                            features,
+                            handle.attachFields
+                        )
+                    );
+            })
+        );
+
+        return featuresByHandle.flat();
+    }
+
+    /**
+     * @param {import("@genome-spy/core/genome/genome.js").DiscreteChromosomeInterval[]} intervals
+     * @param {BigWigHandle[]} handles
+     * @param {number[]} selectedReductionLevels
+     * @param {AbortSignal} signal
+     */
+    async #loadFeatureBatches(
+        intervals,
+        handles,
+        selectedReductionLevels,
+        signal
+    ) {
+        const chunksByHandle = await Promise.all(
+            handles.map((handle, handleIndex) => {
+                const scale = bigWigScale(
+                    selectedReductionLevels[handleIndex],
+                    withoutExprRef(this.params.pixelsPerBin)
+                );
+
+                return handle.bbi
                     .getFeaturesMulti(
                         intervals.map((d) => ({
                             refName: d.chrom,
@@ -160,16 +278,32 @@ export default class BigWigSource extends SingleAxisWindowedSource {
                         })),
                         { scale, signal }
                     )
-                    .then((featureChunks) =>
-                        featureChunks.map((features, i) =>
-                            mapFeatures(intervals[i].chrom, features)
+                    .then((chunks) =>
+                        chunks.map((features, intervalIndex) =>
+                            mapFeatures(
+                                intervals[intervalIndex].chrom,
+                                features,
+                                handle.attachFields
+                            )
                         )
-                    ),
-        });
+                    );
+            })
+        );
 
-        if (featureChunks) {
-            this.publishData(featureChunks);
-        }
+        return intervals.map((_, intervalIndex) =>
+            chunksByHandle.flatMap((chunks) => chunks[intervalIndex])
+        );
+    }
+
+    /**
+     * @param {import("./singleAxisLazySource.js").DataReadinessRequest} request
+     * @returns {boolean}
+     */
+    isDataReadyForDomain(request) {
+        return (
+            this.#descriptorState.activeSetLoaded &&
+            super.isDataReadyForDomain(request)
+        );
     }
 }
 
@@ -183,17 +317,28 @@ function isBigWigSource(params) {
 
 registerBuiltInLazyDataSource(isBigWigSource, BigWigSource);
 
+async function loadBigWigModules() {
+    const [{ BigWig }, { RemoteFile }] = await Promise.all([
+        import("@gmod/bbi"),
+        import("generic-filehandle2"),
+    ]);
+    return { BigWig, RemoteFile };
+}
+
 /**
  * @param {string} chrom
  * @param {import("@gmod/bbi").Feature[]} features
+ * @param {(datum: Record<string, any>) => Record<string, any>} attachFields
  */
-function mapFeatures(chrom, features) {
-    return features.map((f) => ({
-        chrom,
-        start: f.start,
-        end: f.end,
-        score: f.score,
-    }));
+function mapFeatures(chrom, features, attachFields) {
+    return features.map((f) =>
+        attachFields({
+            chrom,
+            start: f.start,
+            end: f.end,
+            score: f.score,
+        })
+    );
 }
 
 /**
@@ -207,3 +352,11 @@ function findReductionLevel(domain, widthInPixels, reductionLevels) {
         reductionLevels.find((r) => r < bpPerPixel) ?? reductionLevels.at(-1)
     );
 }
+
+/**
+ * @param {number} reductionLevel
+ * @param {number} pixelsPerBin
+ */
+function bigWigScale(reductionLevel, pixelsPerBin) {
+    return 1 / 2 / reductionLevel / pixelsPerBin;
+}