@genome-spy/core 0.48.1 → 0.49.0

package/dist/src/data/collector.d.ts

@@ -12,25 +12,27 @@ export default class Collector extends FlowNode {
     params: import("../spec/transform.js").CollectParams;
     /** @type {(function(Collector):void)[]} */
     observers: ((arg0: Collector) => void)[];
-    /** @type {Map<any | any[], import("./flowNode.js").Data>} TODO: proper type for key */
-    facetBatches: Map<any | any[], import("./flowNode.js").Data>;
-    _init(): void;
-    /** @type {import("./flowNode.js").Data} */
-    _data: import("./flowNode.js").Data;
+    /** @type {Map<import("../spec/channel.js").Scalar[], Data>} TODO: proper type for key */
+    facetBatches: Map<import("../spec/channel.js").Scalar[], import("./flowNode.js").Data>;
     /**
-     * @returns {Iterable<import("./flowNode.js").Datum>}
+     * @returns {Iterable<Datum>}
      */
     getData(): Iterable<import("./flowNode.js").Datum>;
     /**
      *
-     * @param {(datum: import("./flowNode.js").Datum) => void} visitor
+     * @param {(datum: Datum) => void} visitor
      */
     visitData(visitor: (datum: import("./flowNode.js").Datum) => void): void;
     /**
      * Returns the total number of data items collected.
      */
     getItemCount(): number;
-    _checkStatus(): void;
+    /**
+     * Use an index to find a datum by its unique id.
+     *
+     * @param {number} uniqueId
+     */
+    findDatumByUniqueId(uniqueId: number): import("./flowNode.js").Datum;
     #private;
 }
 import FlowNode from "./flowNode.js";

package/dist/src/data/collector.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"collector.d.ts","sourceRoot":"","sources":["../../../src/data/collector.js"],"names":[],"mappings":"AAQA;;;;;GAKG;AACH;IAKI;;OAEG;IACH,qBAFW,OAAO,sBAAsB,EAAE,aAAa,EActD;IATG,qDAA2C;IAE3C,2CAA2C;IAC3C,WADW,QAAU,SAAS,KAAE,IAAI,CAAC,EAAE,CACpB;IAEnB,uFAAuF;IACvF,cADW,IAAI,GAAG,GAAG,GAAG,EAAE,EAAE,OAAO,eAAe,EAAE,IAAI,CAAC,CAC5B;IAKjC,cAOC;IANG,2CAA2C;IAC3C,OADW,OAAO,eAAe,EAAE,IAAI,CACxB;IA6GnB;;OAEG;IACH,WAFa,SAAS,OAAO,eAAe,EAAE,KAAK,CAAC,CAuBnD;IAED;;;OAGG;IACH,2BAFmB,OAAO,eAAe,EAAE,KAAK,KAAK,IAAI,QAUxD;IAED;;OAEG;IACH,uBAMC;IAED,qBAMC;;CACJ;qBAzMyD,eAAe"}
+{"version":3,"file":"collector.d.ts","sourceRoot":"","sources":["../../../src/data/collector.js"],"names":[],"mappings":"AAUA;;;;;GAKG;AACH;IA8BI;;OAEG;IACH,qBAFW,OAAO,sBAAsB,EAAE,aAAa,EAetD;IAVG,qDAA2C;IAE3C,2CAA2C;IAC3C,WADW,QAAU,SAAS,KAAE,IAAI,CAAC,EAAE,CACpB;IAGnB,yFAAyF;IACzF,cADW,IAAI,OAAO,oBAAoB,EAAE,MAAM,EAAE,+BAAO,CACN;IAuHzD;;OAEG;IACH,mDAmBC;IAED;;;OAGG;IACH,6DAF6B,IAAI,QAUhC;IAED;;OAEG;IACH,uBAMC;IA8CD;;;;OAIG;IACH,8BAFW,MAAM,iCA4BhB;;CACJ;qBA3SyD,eAAe"}

package/dist/src/data/collector.js

@@ -1,10 +1,12 @@
 import { InternMap } from "internmap";
-import { group } from "d3-array";
+import { bisector, group } from "d3-array";
 import { compare } from "vega-util";
 import iterateNestedMaps from "../utils/iterateNestedMaps.js";
 import FlowNode, { BEHAVIOR_COLLECTS, isFacetBatch } from "./flowNode.js";
 import { field } from "../utils/field.js";
 import { asArray } from "../utils/arrayUtils.js";
+import { radixSortIntoLookupArray } from "../utils/radixSort.js";
+import { UNIQUE_ID_KEY } from "./transforms/identifier.js";
 
 /**
  * Collects (materializes) the data that flows through this node.
@@ -13,6 +15,31 @@ import { asArray } from "../utils/arrayUtils.js";
  * Grouping is primarily intended for handling faceted data.
  */
 export default class Collector extends FlowNode {
+    /**
+     * @typedef {import("./flowNode.js").Datum} Datum
+     * @typedef {import("./flowNode.js").Data} Data
+     */
+
+    /**
+     * Current batch that is being collected.
+     * @type {Data}
+     */
+    #buffer = [];
+
+    #uniqueIdAccessor = field(UNIQUE_ID_KEY);
+
+    /**
+     * @type {number[]}
+     */
+    #uniqueIdIndex = [];
+
+    /**
+     * Start and end indices of all facets if they are concatenated into a single array.
+     * Used together with the uniqueIdIndex for looking up data items by their unique id.
+     * @type {{start: number, stop: number, facetId: import("../spec/channel.js").Scalar[]}[]}
+     */
+    #facetIndices;
+
     get behavior() {
         return BEHAVIOR_COLLECTS;
     }
@@ -28,32 +55,30 @@ export default class Collector extends FlowNode {
         /** @type {(function(Collector):void)[]} */
         this.observers = [];
 
-        /** @type {Map<any | any[], import("./flowNode.js").Data>} TODO: proper type for key */
-        this.facetBatches = undefined;
+        // TODO: Consider nested maps instead of InternMap
+        /** @type {Map<import("../spec/channel.js").Scalar[], Data>} TODO: proper type for key */
+        this.facetBatches = new InternMap([], JSON.stringify);
 
-        this._init();
+        this.#init();
     }
 
-    _init() {
-        /** @type {import("./flowNode.js").Data} */
-        this._data = [];
+    #init() {
+        this.#buffer = [];
 
-        // TODO: Consider nested maps
-        this.facetBatches = new InternMap([], JSON.stringify);
-        this.facetBatches.set(undefined, this._data);
+        this.facetBatches.clear();
+        this.facetBatches.set(undefined, this.#buffer);
     }
 
     reset() {
         super.reset();
-        this._init();
+        this.#init();
     }
 
     /**
-     *
-     * @param {import("./flowNode.js").Datum} datum
+     * @param {Datum} datum
      */
     handle(datum) {
-        this._data.push(datum);
+        this.#buffer.push(datum);
     }
 
     /**
@@ -61,12 +86,15 @@ export default class Collector extends FlowNode {
      */
     beginBatch(flowBatch) {
         if (isFacetBatch(flowBatch)) {
-            this._data = [];
-            this.facetBatches.set(asArray(flowBatch.facetId), this._data);
+            this.#buffer = [];
+            this.facetBatches.set(asArray(flowBatch.facetId), this.#buffer);
         }
     }
 
     complete() {
+        // Free some memory
+        this.#buffer = [];
+
         const sort = this.params?.sort;
         // Vega's "compare" function is incredibly slow (uses megamorphic field accessor)
         // TODO: Implement a replacement for static data types
@@ -84,6 +112,8 @@ export default class Collector extends FlowNode {
                 throw new Error("TODO: Support faceted data!");
             }
 
+            const data = this.facetBatches.get(undefined);
+
             const accessors = this.params.groupby.map((fieldName) =>
                 field(fieldName)
             );
@@ -91,10 +121,10 @@ export default class Collector extends FlowNode {
                 accessors.length > 1
                     ? // There's something strange in d3-array's typings
                       /** @type {Map<any, any>} */ /** @type {any} */ (
-                          group(this._data, ...accessors)
+                          group(data, ...accessors)
                       )
                     : // D3's group is SLOW!
-                      groupBy(this._data, accessors[0]);
+                      groupBy(data, accessors[0]);
 
             this.facetBatches.clear();
             for (const [key, data] of iterateNestedMaps(groups)) {
@@ -107,6 +137,7 @@ export default class Collector extends FlowNode {
             sortData(data);
         }
 
+        this.#buildUniqueIdIndex();
         this.#propagateToChildren();
 
         super.complete();
@@ -118,16 +149,16 @@ export default class Collector extends FlowNode {
 
     #propagateToChildren() {
         if (this.children.length) {
-            for (const [key, data] of this.facetBatches.entries()) {
-                if (key) {
+            for (const [facetId, data] of this.facetBatches.entries()) {
+                if (facetId) {
                     /** @type {import("../types/flowBatch.js").FacetBatch} */
-                    const facetBatch = { type: "facet", facetId: key };
+                    const facetBatch = { type: "facet", facetId };
                     for (const child of this.children) {
                         child.beginBatch(facetBatch);
                     }
                 }
-                for (const datum of data) {
-                    this._propagate(datum);
+                for (let i = 0, n = data.length; i < n; i++) {
+                    this._propagate(data[i]);
                 }
             }
         }
@@ -146,10 +177,10 @@ export default class Collector extends FlowNode {
     }
 
     /**
-     * @returns {Iterable<import("./flowNode.js").Datum>}
+     * @returns {Iterable<Datum>}
      */
     getData() {
-        this._checkStatus();
+        this.#checkStatus();
 
         switch (this.facetBatches.size) {
             case 0:
@@ -161,9 +192,7 @@ export default class Collector extends FlowNode {
                 return {
                     [Symbol.iterator]: function* generator() {
                         for (const data of groups.values()) {
-                            for (let i = 0; i < data.length; i++) {
-                                yield data[i];
-                            }
+                            yield* data;
                         }
                     },
                 };
@@ -173,10 +202,10 @@ export default class Collector extends FlowNode {
 
     /**
      *
-     * @param {(datum: import("./flowNode.js").Datum) => void} visitor
+     * @param {(datum: Datum) => void} visitor
      */
     visitData(visitor) {
-        this._checkStatus();
+        this.#checkStatus();
 
         for (const data of this.facetBatches.values()) {
             for (let i = 0; i < data.length; i++) {
@@ -196,21 +225,90 @@ export default class Collector extends FlowNode {
         return count;
     }
 
-    _checkStatus() {
+    #checkStatus() {
         if (!this.completed) {
             throw new Error(
                 "Data propagation is not completed! No data are available."
             );
         }
     }
+
+    /**
+     * Builds an index for looking up data items by their unique id.
+     * Using a sorted index and binary search for O(log n) complexity.
+     */
+    #buildUniqueIdIndex() {
+        this.#facetIndices = [];
+
+        /** @type {Datum} */
+        const obj = this.facetBatches.values().next().value?.[0];
+        if (obj == null || !(UNIQUE_ID_KEY in obj)) {
+            return; // No unique ids in the data
+        }
+
+        let cumulativePos = 0;
+
+        /** @type {number[]} */
+        const ids = [];
+
+        const a = this.#uniqueIdAccessor;
+
+        for (const [facetId, data] of this.facetBatches) {
+            this.#facetIndices.push({
+                start: cumulativePos,
+                stop: cumulativePos + data.length,
+                facetId,
+            });
+            cumulativePos += data.length;
+
+            for (let i = 0, n = data.length; i < n; i++) {
+                ids.push(a(data[i]));
+            }
+        }
+
+        this.#uniqueIdIndex = radixSortIntoLookupArray(ids);
+    }
+
+    /**
+     * Use an index to find a datum by its unique id.
+     *
+     * @param {number} uniqueId
+     */
+    findDatumByUniqueId(uniqueId) {
+        if (!this.#uniqueIdIndex.length) {
+            return;
+        }
+
+        const facetBisector = bisector((f) => f.start).right;
+        const a = this.#uniqueIdAccessor;
+        const indexBisector = bisector((i) => a(getDatum(i))).left;
+
+        const getDatum = (/** @type {number} */ i) => {
+            const fi = facetBisector(this.#facetIndices, i);
+            const facet = this.#facetIndices[fi - 1];
+            if (!facet || i >= facet.stop) {
+                return;
+            }
+            const data = this.facetBatches.get(facet.facetId);
+            return data[i - facet.start];
+        };
+
+        const index = indexBisector(this.#uniqueIdIndex, uniqueId);
+        if (index >= 0) {
+            const datum = getDatum(this.#uniqueIdIndex[index]);
+            if (datum && a(datum) === uniqueId) {
+                return datum;
+            }
+        }
+    }
 }
 
 /**
  * Like D3's group but without InternMap, which is slow.
  * TODO: Implement multi-level grouping
  *
- * @param {import("./flowNode.js").Datum[]} data
- * @param {(data: import("./flowNode.js").Datum) => import("../spec/channel.js").Scalar} accessor
+ * @param {Datum[]} data
+ * @param {(data: Datum) => import("../spec/channel.js").Scalar} accessor
  */
 function groupBy(data, accessor) {
     const groups = new Map();

package/dist/src/data/collector.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=collector.test.d.ts.map

package/dist/src/data/collector.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"collector.test.d.ts","sourceRoot":"","sources":["../../../src/data/collector.test.js"],"names":[],"mappings":""}

package/dist/src/data/collector.test.js

@@ -1,6 +1,7 @@
-import { expect, test } from "vitest";
+import { describe, expect, test } from "vitest";
 
 import Collector from "./collector.js";
+import { UNIQUE_ID_KEY } from "./transforms/identifier.js";
 
 const data = [1, 5, 2, 4, 3].map((x) => ({ x }));
 
@@ -82,3 +83,56 @@ test("Collector throws on incomplete flow", () => {
 
     expect(() => collector.getData()).toThrow();
 });
+
+describe("Indexing unique ids", () => {
+    test("Collector builds a working index when ids are available", () => {
+        const collector = new Collector({
+            type: "collect",
+            groupby: ["a"],
+        });
+
+        const data = [
+            { a: 1, x: 1, [UNIQUE_ID_KEY]: 8 },
+            { a: 1, x: 2, [UNIQUE_ID_KEY]: 2 },
+            { a: 1, x: 3, [UNIQUE_ID_KEY]: 4 },
+            { a: 1, x: 4, [UNIQUE_ID_KEY]: 6 },
+            { a: 2, x: 5, [UNIQUE_ID_KEY]: 9 },
+            { a: 2, x: 6, [UNIQUE_ID_KEY]: 7 },
+            { a: 2, x: 7, [UNIQUE_ID_KEY]: 3 },
+            { a: 2, x: 8, [UNIQUE_ID_KEY]: 1 },
+        ];
+
+        for (const d of data) {
+            collector.handle(d);
+        }
+        collector.complete();
+
+        expect(collector.findDatumByUniqueId(8)).toEqual(data[0]);
+        expect(collector.findDatumByUniqueId(2)).toEqual(data[1]);
+        expect(collector.findDatumByUniqueId(4)).toEqual(data[2]);
+        expect(collector.findDatumByUniqueId(6)).toEqual(data[3]);
+        expect(collector.findDatumByUniqueId(9)).toEqual(data[4]);
+        expect(collector.findDatumByUniqueId(7)).toEqual(data[5]);
+        expect(collector.findDatumByUniqueId(3)).toEqual(data[6]);
+        expect(collector.findDatumByUniqueId(1)).toEqual(data[7]);
+    });
+
+    test("Collector returns undefined when ids are not available", () => {
+        const collector = new Collector({
+            type: "collect",
+            groupby: ["a"],
+        });
+
+        const data = [
+            { a: 1, x: 1 },
+            { a: 2, x: 5 },
+        ];
+
+        for (const d of data) {
+            collector.handle(d);
+        }
+        collector.complete();
+
+        expect(collector.findDatumByUniqueId(0)).toBeUndefined();
+    });
+});

package/dist/src/data/dataFlow.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=dataFlow.test.d.ts.map

package/dist/src/data/dataFlow.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataFlow.test.d.ts","sourceRoot":"","sources":["../../../src/data/dataFlow.test.js"],"names":[],"mappings":""}

package/dist/src/data/flow.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=flow.test.d.ts.map

package/dist/src/data/flow.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flow.test.d.ts","sourceRoot":"","sources":["../../../src/data/flow.test.js"],"names":[],"mappings":""}

package/dist/src/data/flow.test.js

@@ -2,7 +2,10 @@ import { describe, expect, test } from "vitest";
 import FilterTransform from "./transforms/filter.js";
 import FormulaTransform from "./transforms/formula.js";
 import Collector from "./collector.js";
-import { SynchronousSequenceSource } from "./flowTestUtils.js";
+import {
+    SynchronousSequenceSource,
+    makeParamMediatorProvider,
+} from "./flowTestUtils.js";
 
 describe("Test flow graphs", () => {
     test("Trivial graph: sequence to collector", () => {
@@ -28,8 +31,6 @@ describe("Test flow graphs", () => {
 
         source.dispatch();
 
-        expect(collector1.getData()).not.toBe(collector2._data);
-
         expect(collector1.getData()).toEqual(
             [0, 1, 2, 3, 4].map((d) => ({
                 data: d,
@@ -45,18 +46,23 @@ describe("Test flow graphs", () => {
 
     test.skip("Implement stub for ParamMediator");
 
-    /*
     test("Longer chain of nodes", () => {
         const source = new SynchronousSequenceSource(10);
-        const filter = new FilterTransform({
-            type: "filter",
-            expr: "datum.data < 5",
-        });
-        const formula = new FormulaTransform({
-            type: "formula",
-            expr: "datum.data * 2",
-            as: "data",
-        });
+        const filter = new FilterTransform(
+            {
+                type: "filter",
+                expr: "datum.data < 5",
+            },
+            makeParamMediatorProvider()
+        );
+        const formula = new FormulaTransform(
+            {
+                type: "formula",
+                expr: "datum.data * 2",
+                as: "data",
+            },
+            makeParamMediatorProvider()
+        );
         const collector = new Collector();
 
         source.addChild(filter);
@@ -72,5 +78,4 @@ describe("Test flow graphs", () => {
             }))
         );
     });
-    */
 });

package/dist/src/data/flowNode.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=flowNode.test.d.ts.map

package/dist/src/data/flowNode.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flowNode.test.d.ts","sourceRoot":"","sources":["../../../src/data/flowNode.test.js"],"names":[],"mappings":""}

package/dist/src/data/flowOptimizer.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=flowOptimizer.test.d.ts.map

package/dist/src/data/flowOptimizer.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flowOptimizer.test.d.ts","sourceRoot":"","sources":["../../../src/data/flowOptimizer.test.js"],"names":[],"mappings":""}

package/dist/src/data/flowOptimizer.test.js

@@ -10,6 +10,7 @@ import DataFlow from "./dataFlow.js";
 import { combineIdenticalDataSources } from "./flowOptimizer.js";
 import InlineSource from "./sources/inlineSource.js";
 import UrlSource from "./sources/urlSource.js";
+import { makeParamMediatorProvider } from "./flowTestUtils.js";
 
 test("validateLinks() detects broken graph", () => {
     const root = new FlowNode();
@@ -132,14 +133,12 @@ describe("removeRedundantCloneTransforms", () => {
     });
 });
 
-const viewStub = {
-    paramMediator: {
-        registerParam: () => {},
-        allocateSetter: () => {},
-        createExpression: () => {},
-    },
-    getBaseUrl: () => "",
-};
+/** @type {import("../view/view.js").default} */
+const viewStub = /** @type {any} */ (
+    Object.assign(makeParamMediatorProvider(), {
+        getBaseUrl: () => "",
+    })
+);
 
 describe("Merge indentical data sources", () => {
     test("Merges correctly", () => {
@@ -191,8 +190,8 @@ describe("Merge indentical data sources", () => {
         /** @type {DataFlow<string>} */
         const dataFlow = new DataFlow();
 
-        const a = new InlineSource({ values: [1, 2, 3] });
-        const b = new InlineSource({ values: [1, 2, 3] });
+        const a = new InlineSource({ values: [1, 2, 3] }, viewStub);
+        const b = new InlineSource({ values: [1, 2, 3] }, viewStub);
 
         dataFlow.addDataSource(a, "a");
         dataFlow.addDataSource(b, "b");

package/dist/src/data/formats/fasta.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=fasta.test.d.ts.map

package/dist/src/data/formats/fasta.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fasta.test.d.ts","sourceRoot":"","sources":["../../../../src/data/formats/fasta.test.js"],"names":[],"mappings":""}

package/dist/src/data/sources/inlineSource.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=inlineSource.test.d.ts.map

package/dist/src/data/sources/inlineSource.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"inlineSource.test.d.ts","sourceRoot":"","sources":["../../../../src/data/sources/inlineSource.test.js"],"names":[],"mappings":""}

package/dist/src/data/sources/inlineSource.test.js

@@ -15,33 +15,37 @@ async function collectSource(source) {
 }
 
 test("InlineSource propagates an object", async () => {
-    expect(await collectSource(new InlineSource({ values: { x: 1 } }))).toEqual(
-        [{ x: 1 }]
-    );
+    expect(
+        await collectSource(new InlineSource({ values: { x: 1 } }, undefined))
+    ).toEqual([{ x: 1 }]);
 });
 
 test("InlineSource propagates an array of objects", async () => {
     expect(
-        await collectSource(new InlineSource({ values: [{ x: 1 }, { x: 2 }] }))
+        await collectSource(
+            new InlineSource({ values: [{ x: 1 }, { x: 2 }] }, undefined)
+        )
     ).toEqual([{ x: 1 }, { x: 2 }]);
 });
 
 test("InlineSource wraps scalars to objects", async () => {
-    expect(await collectSource(new InlineSource({ values: [1, 2] }))).toEqual([
-        { data: 1 },
-        { data: 2 },
-    ]);
+    expect(
+        await collectSource(new InlineSource({ values: [1, 2] }, undefined))
+    ).toEqual([{ data: 1 }, { data: 2 }]);
 });
 
 test("InlineSource parses a string", async () => {
     expect(
         await collectSource(
-            new InlineSource({
-                values: "a\n1\n2\n3",
-                format: {
-                    type: "csv",
+            new InlineSource(
+                {
+                    values: "a\n1\n2\n3",
+                    format: {
+                        type: "csv",
+                    },
                 },
-            })
+                undefined
+            )
         )
     ).toEqual([{ a: 1 }, { a: 2 }, { a: 3 }]);
 });
@@ -49,8 +53,11 @@ test("InlineSource parses a string", async () => {
 test("InlineSource throws on a string and a missing format specifier", () => {
     expect(
         () =>
-            new InlineSource({
-                values: "a\n1\n2\n3",
-            })
+            new InlineSource(
+                {
+                    values: "a\n1\n2\n3",
+                },
+                undefined
+            )
     ).toThrow();
 });

package/dist/src/data/sources/sequenceSource.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=sequenceSource.test.d.ts.map

package/dist/src/data/sources/sequenceSource.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sequenceSource.test.d.ts","sourceRoot":"","sources":["../../../../src/data/sources/sequenceSource.test.js"],"names":[],"mappings":""}