@genome-spy/core 0.36.0 → 0.37.0

package/dist/src/data/collector.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Collects (materializes) the data that flows through this node.
+ * The collected data can be optionally grouped and sorted.
+ *
+ * Grouping is primarily intended for handling faceted data.
+ */
+export default class Collector extends FlowNode {
+    /**
+     * @param {import("../spec/transform").CollectParams} [params]
+     */
+    constructor(params?: import("../spec/transform").CollectParams);
+    params: import("../spec/transform").CollectParams;
+    /** @type {(function(Collector):void)[]} */
+    observers: ((arg0: Collector) => void)[];
+    /** @type {Map<any | any[], import("./flowNode").Data>} TODO: proper type for key */
+    facetBatches: Map<any | any[], import("./flowNode").Data>;
+    _init(): void;
+    /** @type {import("./flowNode").Data} */
+    _data: import("./flowNode").Data;
+    /**
+     * @returns {Iterable<import("./flowNode").Datum>}
+     */
+    getData(): Iterable<import("./flowNode").Datum>;
+    /**
+     *
+     * @param {(datum: import("./flowNode").Datum) => void} visitor
+     */
+    visitData(visitor: (datum: import("./flowNode").Datum) => void): void;
+    /**
+     * Returns the total number of data items collected.
+     */
+    getItemCount(): number;
+    _checkStatus(): void;
+}
+import FlowNode from "./flowNode";
+//# sourceMappingURL=collector.d.ts.map

package/dist/src/data/collector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"collector.d.ts","sourceRoot":"","sources":["../../../src/data/collector.js"],"names":[],"mappings":"AAQA;;;;;GAKG;AACH;IAKI;;OAEG;IACH,qBAFW,OAAO,mBAAmB,EAAE,aAAa,EAcnD;IATG,kDAA2C;IAE3C,2CAA2C;IAC3C,mBADqB,SAAS,KAAE,IAAI,IACjB;IAEnB,oFAAoF;IACpF,cADW,IAAI,GAAG,GAAG,GAAG,EAAE,EAAE,OAAO,YAAY,EAAE,IAAI,CAAC,CACzB;IAKjC,cAOC;IANG,wCAAwC;IACxC,OADW,OAAO,YAAY,EAAE,IAAI,CACrB;IAuFnB;;OAEG;IACH,WAFa,SAAS,OAAO,YAAY,EAAE,KAAK,CAAC,CAuBhD;IAED;;;OAGG;IACH,2BAFmB,OAAO,YAAY,EAAE,KAAK,KAAK,IAAI,QAUrD;IAED;;OAEG;IACH,uBAMC;IAED,qBAMC;CACJ;qBAnLyD,YAAY"}

package/dist/src/data/collector.js

@@ -0,0 +1,184 @@
+import { InternMap } from "internmap";
+import { group } from "d3-array";
+import { compare } from "vega-util";
+import iterateNestedMaps from "../utils/iterateNestedMaps";
+import FlowNode, { BEHAVIOR_COLLECTS, isFacetBatch } from "./flowNode";
+import { field } from "../utils/field";
+import { asArray } from "../utils/arrayUtils";
+
+/**
+ * Collects (materializes) the data that flows through this node.
+ * The collected data can be optionally grouped and sorted.
+ *
+ * Grouping is primarily intended for handling faceted data.
+ */
+export default class Collector extends FlowNode {
+    get behavior() {
+        return BEHAVIOR_COLLECTS;
+    }
+
+    /**
+     * @param {import("../spec/transform").CollectParams} [params]
+     */
+    constructor(params) {
+        super();
+
+        this.params = params ?? { type: "collect" };
+
+        /** @type {(function(Collector):void)[]} */
+        this.observers = [];
+
+        /** @type {Map<any | any[], import("./flowNode").Data>} TODO: proper type for key */
+        this.facetBatches = undefined;
+
+        this._init();
+    }
+
+    _init() {
+        /** @type {import("./flowNode").Data} */
+        this._data = [];
+
+        // TODO: Consider nested maps
+        this.facetBatches = new InternMap([], JSON.stringify);
+        this.facetBatches.set(undefined, this._data);
+    }
+
+    reset() {
+        super.reset();
+        this._init();
+    }
+
+    /**
+     *
+     * @param {import("./flowNode").Datum} datum
+     */
+    handle(datum) {
+        this._data.push(datum);
+    }
+
+    /**
+     * @param {import("../types/flowBatch").FlowBatch} flowBatch
+     */
+    beginBatch(flowBatch) {
+        if (isFacetBatch(flowBatch)) {
+            this._data = [];
+            this.facetBatches.set(asArray(flowBatch.facetId), this._data);
+        }
+    }
+
+    complete() {
+        const sort = this.params?.sort;
+        // Vega's "compare" function is incredibly slow (uses megamorphic field accessor)
+        // TODO: Implement a replacement for static data types
+        const comparator = sort ? compare(sort.field, sort.order) : undefined;
+
+        /** @param {any[]} data */
+        const sortData = (data) => {
+            if (comparator) {
+                data.sort(comparator);
+            }
+        };
+
+        if (this.params.groupby?.length) {
+            if (this.facetBatches.size > 1) {
+                throw new Error("TODO: Support faceted data!");
+            }
+
+            const accessors = this.params.groupby.map((fieldName) =>
+                field(fieldName)
+            );
+            // @ts-ignore
+            const groups = group(this._data, ...accessors);
+
+            this.facetBatches.clear();
+            for (const [key, data] of iterateNestedMaps(groups)) {
+                this.facetBatches.set(key, data);
+            }
+        }
+
+        for (const data of this.facetBatches.values()) {
+            // TODO: Only sort if not already sorted
+            sortData(data);
+        }
+
+        if (this.children.length) {
+            for (const [key, data] of this.facetBatches.entries()) {
+                if (key) {
+                    /** @type {import("../types/flowBatch").FacetBatch} */
+                    const facetBatch = { type: "facet", facetId: key };
+                    for (const child of this.children) {
+                        child.beginBatch(facetBatch);
+                    }
+                }
+                for (const datum of data) {
+                    this._propagate(datum);
+                }
+            }
+        }
+
+        super.complete();
+
+        for (const observer of this.observers) {
+            observer(this);
+        }
+    }
+
+    /**
+     * @returns {Iterable<import("./flowNode").Datum>}
+     */
+    getData() {
+        this._checkStatus();
+
+        switch (this.facetBatches.size) {
+            case 0:
+                return [];
+            case 1:
+                return [...this.facetBatches.values()][0];
+            default: {
+                const groups = this.facetBatches;
+                return {
+                    [Symbol.iterator]: function* generator() {
+                        for (const data of groups.values()) {
+                            for (let i = 0; i < data.length; i++) {
+                                yield data[i];
+                            }
+                        }
+                    },
+                };
+            }
+        }
+    }
+
+    /**
+     *
+     * @param {(datum: import("./flowNode").Datum) => void} visitor
+     */
+    visitData(visitor) {
+        this._checkStatus();
+
+        for (const data of this.facetBatches.values()) {
+            for (let i = 0; i < data.length; i++) {
+                visitor(data[i]);
+            }
+        }
+    }
+
+    /**
+     * Returns the total number of data items collected.
+     */
+    getItemCount() {
+        let count = 0;
+        for (const data of this.facetBatches.values()) {
+            count += data.length;
+        }
+        return count;
+    }
+
+    _checkStatus() {
+        if (!this.completed) {
+            throw new Error(
+                "Data propagation is not completed! No data are available."
+            );
+        }
+    }
+}

package/dist/src/data/collector.test.js

@@ -0,0 +1,84 @@
+import { expect, test } from "vitest";
+
+import Collector from "./collector";
+
+const data = [1, 5, 2, 4, 3].map((x) => ({ x }));
+
+test("Collector collects data", () => {
+    const collector = new Collector();
+
+    for (const d of data) {
+        collector.handle(d);
+    }
+    collector.complete();
+
+    expect(collector.getData()).toEqual(data);
+});
+
+test("Collector collects and sorts data", () => {
+    const collector = new Collector({
+        type: "collect",
+        sort: { field: ["x"] },
+    });
+
+    for (const d of data) {
+        collector.handle(d);
+    }
+    collector.complete();
+
+    expect([...collector.getData()]).toEqual(
+        [1, 2, 3, 4, 5].map((x) => ({ x }))
+    );
+});
+
+test("Collector collects, groups, and sorts data", () => {
+    const collector = new Collector({
+        type: "collect",
+        sort: { field: ["x"] },
+        groupby: ["a", "b"],
+    });
+
+    const data = [
+        { a: 1, b: 1, x: 1 },
+        { a: 1, b: 2, x: 2 },
+        { a: 1, b: 2, x: 3 },
+        { a: 2, b: 1, x: 4 },
+        { a: 2, b: 1, x: 5 },
+        { a: 2, b: 2, x: 6 },
+    ];
+
+    for (const d of data) {
+        collector.handle(d);
+    }
+    collector.complete();
+
+    const cd = [...collector.getData()];
+
+    expect(cd.map((d) => ({ x: d.x }))).toEqual(
+        [1, 2, 3, 4, 5, 6].map((x) => ({ x }))
+    );
+
+    /** @param {any[]} group*/
+    const getGroupX = (group) =>
+        collector.facetBatches.get(group).map((d) => d.x);
+
+    expect(getGroupX([1, 1])).toEqual([1]);
+    expect(getGroupX([1, 2])).toEqual([2, 3]);
+    expect(getGroupX([2, 1])).toEqual([4, 5]);
+    expect(getGroupX([2, 2])).toEqual([6]);
+
+    expect(new Set(collector.facetBatches.keys())).toEqual(
+        new Set([
+            [1, 1],
+            [1, 2],
+            [2, 1],
+            [2, 2],
+        ])
+    );
+});
+
+test("Collector throws on incomplete flow", () => {
+    const collector = new Collector();
+
+    expect(() => collector.getData()).toThrow();
+});

package/dist/src/data/dataFlow.d.ts

@@ -0,0 +1,65 @@
+/**
+ * @template H A key (string, object, whatever) that is used to retrieve
+ *      data sources and collectors.
+ */
+export default class DataFlow<H> {
+    /** @type {Map<H, import("./sources/dataSource").default>} */
+    _dataSourcesByHost: Map<H, import("./sources/dataSource").default>;
+    /** @type {Map<H, import("./collector").default>} */
+    _collectorsByHost: Map<H, import("./collector").default>;
+    /** @type {Map<H, (function(import("./collector").default):void)[]>} */
+    _observers: Map<H, ((arg0: import("./collector").default) => void)[]>;
+    get dataSources(): import("./sources/dataSource").default[];
+    get collectors(): import("./collector").default[];
+    /**
+     * Adds a callback function that will be called when a collector has completed.
+     *
+     * @param {function(import("./collector").default):void} callback
+     * @param {H} key
+     */
+    addObserver(callback: (arg0: import("./collector").default) => void, key: H): void;
+    /**
+     *
+     * @param {import("./collector").default} collector
+     * @param {H} key
+     */
+    _relayObserverCallback(collector: import("./collector").default, key: H): void;
+    /**
+     *
+     * @param {import("./sources/dataSource").default} dataSource
+     * @param {H} key
+     */
+    addDataSource(dataSource: import("./sources/dataSource").default, key: H): void;
+    /**
+     *
+     * @param {H} key
+     */
+    findDataSourceByKey(key: H): import("./sources/dataSource").default;
+    /**
+     *
+     * @param {string} name
+     */
+    findNamedDataSource(name: string): {
+        dataSource: NamedSource;
+        hosts: H[];
+    };
+    /**
+     *
+     * @param {import("./collector").default} collector
+     * @param {H} key
+     */
+    addCollector(collector: import("./collector").default, key: H): void;
+    /**
+     *
+     * @param {H} key
+     */
+    findCollectorByKey(key: H): import("./collector").default;
+    /**
+     * Allows the flow nodes to perform final initialization after the flow
+     * structure has been built and optimized.
+     * Must be called before any data are to be propagated.
+     */
+    initialize(): void;
+}
+import NamedSource from "./sources/namedSource";
+//# sourceMappingURL=dataFlow.d.ts.map

package/dist/src/data/dataFlow.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataFlow.d.ts","sourceRoot":"","sources":["../../../src/data/dataFlow.js"],"names":[],"mappings":"AAEA;;;GAGG;AACH;IAEQ,6DAA6D;IAC7D,oBADW,IAAI,CAAC,EAAE,OAAO,sBAAsB,EAAE,OAAO,CAAC,CACtB;IAEnC,oDAAoD;IACpD,mBADW,IAAI,CAAC,EAAE,OAAO,aAAa,EAAE,OAAO,CAAC,CACd;IAElC,uEAAuE;IACvE,2BAD4B,OAAO,aAAa,EAAE,OAAO,KAAE,IAAI,KACpC;IAG/B,4DAEC;IAED,kDAEC;IAED;;;;;OAKG;IACH,6BAHoB,OAAO,aAAa,EAAE,OAAO,KAAE,IAAI,OAC5C,CAAC,QAUX;IAED;;;;OAIG;IACH,kCAHW,OAAO,aAAa,EAAE,OAAO,OAC7B,CAAC,QAUX;IAED;;;;OAIG;IACH,0BAHW,OAAO,sBAAsB,EAAE,OAAO,OACtC,CAAC,QAIX;IAED;;;OAGG;IACH,yBAFW,CAAC,0CAIX;IAED;;;OAGG;IACH,0BAFW,MAAM;;;MAiChB;IAED;;;;OAIG;IACH,wBAHW,OAAO,aAAa,EAAE,OAAO,OAC7B,CAAC,QAOX;IAED;;;OAGG;IACH,wBAFW,CAAC,iCAIX;IAED;;;;OAIG;IACH,mBAIC;CACJ;wBA7IuB,uBAAuB"}

package/dist/src/data/dataFlow.js

@@ -0,0 +1,142 @@
+import NamedSource from "./sources/namedSource";
+
+/**
+ * @template H A key (string, object, whatever) that is used to retrieve
+ *      data sources and collectors.
+ */
+export default class DataFlow {
+    constructor() {
+        /** @type {Map<H, import("./sources/dataSource").default>} */
+        this._dataSourcesByHost = new Map();
+
+        /** @type {Map<H, import("./collector").default>} */
+        this._collectorsByHost = new Map();
+
+        /** @type {Map<H, (function(import("./collector").default):void)[]>} */
+        this._observers = new Map();
+    }
+
+    get dataSources() {
+        return [...new Set(this._dataSourcesByHost.values()).values()];
+    }
+
+    get collectors() {
+        return [...this._collectorsByHost.values()];
+    }
+
+    /**
+     * Adds a callback function that will be called when a collector has completed.
+     *
+     * @param {function(import("./collector").default):void} callback
+     * @param {H} key
+     */
+    addObserver(callback, key) {
+        let arr = this._observers.get(key);
+        if (!arr) {
+            arr = [];
+            this._observers.set(key, arr);
+        }
+
+        arr.push(callback);
+    }
+
+    /**
+     *
+     * @param {import("./collector").default} collector
+     * @param {H} key
+     */
+    _relayObserverCallback(collector, key) {
+        const arr = this._observers.get(key);
+        if (arr) {
+            for (const callback of arr) {
+                // eslint-disable-next-line callback-return
+                callback(collector);
+            }
+        }
+    }
+
+    /**
+     *
+     * @param {import("./sources/dataSource").default} dataSource
+     * @param {H} key
+     */
+    addDataSource(dataSource, key) {
+        this._dataSourcesByHost.set(key, dataSource);
+    }
+
+    /**
+     *
+     * @param {H} key
+     */
+    findDataSourceByKey(key) {
+        return this._dataSourcesByHost.get(key);
+    }
+
+    /**
+     *
+     * @param {string} name
+     */
+    findNamedDataSource(name) {
+        /** @type {NamedSource} */
+        let namedSource;
+        /** @type {H[]} */
+        let hosts = [];
+
+        // Note: If a named sources with the same name are present at multiple locations in the
+        // view hierarchy, the should actually be exactly the same instance. It's arranged that
+        // way in the data flow optimization phase.
+
+        for (const [host, dataSource] of this._dataSourcesByHost.entries()) {
+            if (dataSource instanceof NamedSource) {
+                if (name == dataSource.identifier) {
+                    if (namedSource && namedSource !== dataSource) {
+                        // TODO: Write a test case for this and remove the runtime check.
+                        throw new Error(
+                            `Found multiple instances of named data: ${name}. Data flow optimization is broken (it's a bug).`
+                        );
+                    }
+                    namedSource = dataSource;
+                    hosts.push(host);
+                }
+            }
+        }
+
+        if (namedSource) {
+            return {
+                dataSource: namedSource,
+                hosts,
+            };
+        }
+    }
+
+    /**
+     *
+     * @param {import("./collector").default} collector
+     * @param {H} key
+     */
+    addCollector(collector, key) {
+        this._collectorsByHost.set(key, collector);
+        collector.observers.push((collector) =>
+            this._relayObserverCallback(collector, key)
+        );
+    }
+
+    /**
+     *
+     * @param {H} key
+     */
+    findCollectorByKey(key) {
+        return this._collectorsByHost.get(key);
+    }
+
+    /**
+     * Allows the flow nodes to perform final initialization after the flow
+     * structure has been built and optimized.
+     * Must be called before any data are to be propagated.
+     */
+    initialize() {
+        for (const ds of this.dataSources) {
+            ds.visit((node) => node.initialize());
+        }
+    }
+}

package/dist/src/data/dataFlow.test.js

@@ -0,0 +1,5 @@
+import { describe, test } from "vitest";
+
+describe("DataFlow", () => {
+    test.todo("TODO");
+});

package/dist/src/data/facetNode.d.ts

@@ -0,0 +1,17 @@
+/**
+ * @typedef {object} FacetParams
+ * @prop {string[]} groupby
+ */
+export default class FacetNode extends FlowNode {
+    /**
+     * @param {FacetParams} params
+     */
+    constructor(params: FacetParams);
+    /** @type {Map<any | any[], FlowNode>} */
+    subflows: Map<any | any[], FlowNode>;
+}
+export type FacetParams = {
+    groupby: string[];
+};
+import FlowNode from "./flowNode";
+//# sourceMappingURL=facetNode.d.ts.map

package/dist/src/data/facetNode.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"facetNode.d.ts","sourceRoot":"","sources":["../../../src/data/facetNode.js"],"names":[],"mappings":"AAEA;;;GAGG;AACH;IACI;;OAEG;IACH,oBAFW,WAAW,EAOrB;IAFG,yCAAyC;IACzC,UADW,IAAI,GAAG,GAAG,GAAG,EAAE,EAAE,QAAQ,CAAC,CACZ;CAEhC;;aAZS,MAAM,EAAE;;qBAJG,YAAY"}

package/dist/src/data/facetNode.js

@@ -0,0 +1,17 @@
+import FlowNode from "./flowNode";
+
+/**
+ * @typedef {object} FacetParams
+ * @prop {string[]} groupby
+ */
+export default class FacetNode extends FlowNode {
+    /**
+     * @param {FacetParams} params
+     */
+    constructor(params) {
+        super();
+
+        /** @type {Map<any | any[], FlowNode>} */
+        this.subflows = new Map();
+    }
+}

package/dist/src/data/flow.test.js

@@ -0,0 +1,72 @@
+import { describe, expect, test } from "vitest";
+import FilterTransform from "./transforms/filter";
+import FormulaTransform from "./transforms/formula";
+import Collector from "./collector";
+import { SynchronousSequenceSource } from "./flowTestUtils";
+
+describe("Test flow graphs", () => {
+    test("Trivial graph: sequence to collector", () => {
+        const source = new SynchronousSequenceSource(5);
+        const collector = new Collector();
+        source.addChild(collector);
+
+        source.dispatch();
+
+        expect(collector.getData()).toEqual(
+            [0, 1, 2, 3, 4].map((d) => ({
+                data: d,
+            }))
+        );
+    });
+
+    test("Trivial branching: sequence to two collectors", () => {
+        const source = new SynchronousSequenceSource(5);
+        const collector1 = new Collector();
+        source.addChild(collector1);
+        const collector2 = new Collector();
+        source.addChild(collector2);
+
+        source.dispatch();
+
+        expect(collector1.getData()).not.toBe(collector2._data);
+
+        expect(collector1.getData()).toEqual(
+            [0, 1, 2, 3, 4].map((d) => ({
+                data: d,
+            }))
+        );
+
+        expect(collector2.getData()).toEqual(
+            [0, 1, 2, 3, 4].map((d) => ({
+                data: d,
+            }))
+        );
+    });
+
+    test("Longer chain of nodes", () => {
+        const source = new SynchronousSequenceSource(10);
+        const filter = new FilterTransform({
+            type: "filter",
+            expr: "datum.data < 5",
+        });
+        const formula = new FormulaTransform({
+            type: "formula",
+            expr: "datum.data * 2",
+            as: "data",
+        });
+        const collector = new Collector();
+
+        source.addChild(filter);
+        filter.addChild(formula);
+        formula.addChild(collector);
+
+        source.visit((node) => node.initialize());
+        source.dispatch();
+
+        expect(collector.getData()).toEqual(
+            [0, 2, 4, 6, 8].map((d) => ({
+                data: d,
+            }))
+        );
+    });
+});