@genome-spy/core 0.36.0 → 0.37.0

package/dist/src/data/flowNode.d.ts

@@ -0,0 +1,136 @@
+/**
+ * @param {import("../types/flowBatch").FlowBatch} flowBatch
+ * @returns {flowBatch is import("../types/flowBatch").FileBatch}
+ */
+export function isFileBatch(flowBatch: import("../types/flowBatch").FlowBatch): flowBatch is import("../types/flowBatch").FileBatch;
+/**
+ * @param {import("../types/flowBatch").FlowBatch} flowBatch
+ * @returns {flowBatch is import("../types/flowBatch").FacetBatch}
+ */
+export function isFacetBatch(flowBatch: import("../types/flowBatch").FlowBatch): flowBatch is import("../types/flowBatch").FacetBatch;
+/**
+ * The FlowNode clones the data objects passing through or creates entirely
+ * new objects.
+ */
+export const BEHAVIOR_CLONES: number;
+/**
+ * FlowNode modifies the objects that pass through. Creating defensive copies
+ * (clones) in an upstream node may be appropriate, depending on branching.
+ */
+export const BEHAVIOR_MODIFIES: number;
+/**
+ * The flow node collects data objects and may emit the to its children.
+ * The collected data objects must not be modified by downstream transforms.
+ */
+export const BEHAVIOR_COLLECTS: number;
+/**
+ * This is heavily inspired by Vega's and Vega-Lite's data flow system.
+ *
+ * @typedef {Record<string, any>} Datum
+ * @typedef {Datum[]} Data
+ */
+export default class FlowNode {
+    get behavior(): number;
+    /** @type {FlowNode[]} */
+    children: FlowNode[];
+    /** @type {FlowNode} */
+    parent: FlowNode;
+    /** True if all data have been processed */
+    completed: boolean;
+    /**
+     * Resets the node and all its descendants to their initial state, i.e., preparing
+     * for a new batch of data.
+     */
+    reset(): void;
+    /**
+     * Allows for doing final initialization after the flow structure has been
+     * built and optimized. Must be called before any data are to be propagated.
+     */
+    initialize(): void;
+    /**
+     * Dynamically updates the propagator method to allow the JavaScript engine
+     * to employ optimizations such as inlining.
+     */
+    _updatePropagator(): void;
+    /**
+     *
+     * @param {any} datum
+     */
+    _propagate(datum: any): void;
+    /**
+     *
+     * @param {FlowNode} parent
+     */
+    setParent(parent: FlowNode): void;
+    /**
+     *
+     * @param {FlowNode} child
+     */
+    addChild(child: FlowNode): FlowNode;
+    /**
+     * @param {FlowNode} node
+     */
+    adopt(node: FlowNode): void;
+    /**
+     * @param {FlowNode} otherParent
+     */
+    adoptChildrenOf(otherParent: FlowNode): void;
+    /**
+     * @param {FlowNode} newParent
+     */
+    insertAsParent(newParent: FlowNode): void;
+    /**
+     *
+     * @param {FlowNode} child
+     */
+    removeChild(child: FlowNode): void;
+    /**
+     * Removes this node and ligates (connects) the preceding and succeeding nodes.
+     */
+    excise(): void;
+    isRoot(): boolean;
+    isBranching(): boolean;
+    isTerminal(): boolean;
+    /**
+     * Visits child nodes in depth-first order.
+     *
+     * @param {(function(FlowNode):void) & { afterChildren?: function(FlowNode):void}} visitor
+     */
+    visit(visitor: ((arg0: FlowNode) => void) & {
+        afterChildren?: (arg0: FlowNode) => void;
+    }): void;
+    /**
+     * @param {number} [depth]
+     * @returns {string}
+     */
+    subtreeToString(depth?: number): string;
+    /**
+     * The global object for expressions (in formula and filter transforms).
+     * Nodes in the hierarchy may extend the object using Object.create to
+     * introduce variables that are visible downstream the flow.
+     *
+     * @returns {Record<string, any>}
+     */
+    getGlobalObject(): Record<string, any>;
+    /**
+     *
+     * @param {Datum} datum
+     */
+    handle(datum: Datum): void;
+    complete(): void;
+    /**
+     * Signals that a new batch of data will be propagated.
+     *
+     * @param {import("../types/flowBatch").FlowBatch} flowBatch
+     */
+    beginBatch(flowBatch: import("../types/flowBatch").FlowBatch): void;
+}
+/**
+ * This is heavily inspired by Vega's and Vega-Lite's data flow system.
+ */
+export type Datum = Record<string, any>;
+/**
+ * This is heavily inspired by Vega's and Vega-Lite's data flow system.
+ */
+export type Data = Datum[];
+//# sourceMappingURL=flowNode.d.ts.map

package/dist/src/data/flowNode.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flowNode.d.ts","sourceRoot":"","sources":["../../../src/data/flowNode.js"],"names":[],"mappings":"AA+QA;;;GAGG;AACH,uCAHW,OAAO,oBAAoB,EAAE,SAAS,uDAKhD;AAED;;;GAGG;AACH,wCAHW,OAAO,oBAAoB,EAAE,SAAS,wDAKhD;AA3RD;;;GAGG;AACH,qCAAsC;AAEtC;;;GAGG;AACH,uCAAwC;AAExC;;;GAGG;AACH,uCAAwC;AAIxC;;;;;GAKG;AACH;IACI,uBAEC;IAGG,yBAAyB;IACzB,UADW,QAAQ,EAAE,CACH;IAElB,uBAAuB;IACvB,QADW,QAAQ,CACI;IAEvB,2CAA2C;IAC3C,mBAAsB;IAG1B;;;OAGG;IACH,cAMC;IAED;;;OAGG;IACH,mBAEC;IAED;;;OAGG;IACH,0BAUC;IAwLD;;;OAGG;IACH,kBAFW,GAAG,QAIb;IA5LD;;;OAGG;IACH,kBAFW,QAAQ,QAIlB;IAED;;;OAGG;IACH,gBAFW,QAAQ,YAUlB;IAED;;OAEG;IACH,YAFW,QAAQ,QAQlB;IAED;;OAEG;IACH,6BAFW,QAAQ,QAMlB;IAED;;OAEG;IACH,0BAFW,QAAQ,QAalB;IAED;;;OAGG;IACH,mBAFW,QAAQ,QAWlB;IAED;;OAEG;IACH,eAiBC;IAED,kBAEC;IAED,uBAEC;IAED,sBAEC;IAED;;;;OAIG;IACH,eAFW,QAAU,QAAQ,KAAE,IAAI,CAAC,GAAG;QAAE,aAAa,CAAC,SAAW,QAAQ,KAAE,IAAI,CAAA;KAAC,QAchF;IAED;;;OAGG;IACH,wBAHW,MAAM,GACJ,MAAM,CAWlB;IAED;;;;;;OAMG;IACH,mBAFa,OAAO,MAAM,EAAE,GAAG,CAAC,CAM/B;IAED;;;OAGG;IACH,cAFW,KAAK,QAKf;IAED,iBAMC;IAED;;;;OAIG;IACH,sBAFW,OAAO,oBAAoB,EAAE,SAAS,QAMhD;CASJ;;;;oBApPY,OAAO,MAAM,EAAE,GAAG,CAAC;;;;mBACnB,KAAK,EAAE"}

package/dist/src/data/flowNode.js

@@ -0,0 +1,286 @@
+import { range } from "d3-array";
+
+/**
+ * The FlowNode clones the data objects passing through or creates entirely
+ * new objects.
+ */
+export const BEHAVIOR_CLONES = 1 << 0;
+
+/**
+ * FlowNode modifies the objects that pass through. Creating defensive copies
+ * (clones) in an upstream node may be appropriate, depending on branching.
+ */
+export const BEHAVIOR_MODIFIES = 1 << 1;
+
+/**
+ * The flow node collects data objects and may emit the to its children.
+ * The collected data objects must not be modified by downstream transforms.
+ */
+export const BEHAVIOR_COLLECTS = 1 << 2;
+
+const ROOT_CONTEXT_OBJECT = {};
+
+/**
+ * This is heavily inspired by Vega's and Vega-Lite's data flow system.
+ *
+ * @typedef {Record<string, any>} Datum
+ * @typedef {Datum[]} Data
+ */
+export default class FlowNode {
+    get behavior() {
+        return 0;
+    }
+
+    constructor() {
+        /** @type {FlowNode[]} */
+        this.children = [];
+
+        /** @type {FlowNode} */
+        this.parent = undefined;
+
+        /** True if all data have been processed */
+        this.completed = false;
+    }
+
+    /**
+     * Resets the node and all its descendants to their initial state, i.e., preparing
+     * for a new batch of data.
+     */
+    reset() {
+        this.completed = false;
+
+        for (const child of this.children) {
+            child.reset();
+        }
+    }
+
+    /**
+     * Allows for doing final initialization after the flow structure has been
+     * built and optimized. Must be called before any data are to be propagated.
+     */
+    initialize() {
+        // override
+    }
+
+    /**
+     * Dynamically updates the propagator method to allow the JavaScript engine
+     * to employ optimizations such as inlining.
+     */
+    _updatePropagator() {
+        this._propagate = Function(
+            "children",
+            range(this.children.length)
+                .map((i) => `const child${i} = children[${i}];`)
+                .join("\n") +
+                `return function propagate(datum) {${range(this.children.length)
+                    .map((i) => `child${i}.handle(datum);`)
+                    .join("\n")}}`
+        )(this.children);
+    }
+
+    /**
+     *
+     * @param {FlowNode} parent
+     */
+    setParent(parent) {
+        this.parent = parent;
+    }
+
+    /**
+     *
+     * @param {FlowNode} child
+     */
+    addChild(child) {
+        if (child.parent) {
+            throw new Error("Cannot add the child! It already has a parent.");
+        }
+        this.children.push(child);
+        child.setParent(this);
+        this._updatePropagator();
+        return this;
+    }
+
+    /**
+     * @param {FlowNode} node
+     */
+    adopt(node) {
+        if (node.parent) {
+            node.parent.removeChild(node);
+        }
+
+        this.addChild(node);
+    }
+
+    /**
+     * @param {FlowNode} otherParent
+     */
+    adoptChildrenOf(otherParent) {
+        for (const child of otherParent.children) {
+            this.adopt(child);
+        }
+    }
+
+    /**
+     * @param {FlowNode} newParent
+     */
+    insertAsParent(newParent) {
+        if (this.isRoot()) {
+            // Root should always be a data source
+            throw new Error("Cannot insert a new parent for a root node!");
+        }
+
+        newParent.parent = this.parent;
+        this.parent.children[this.parent.children.indexOf(this)] = newParent;
+        this.parent._updatePropagator();
+        this.parent = undefined;
+        newParent.addChild(this);
+    }
+
+    /**
+     *
+     * @param {FlowNode} child
+     */
+    removeChild(child) {
+        const index = this.children.indexOf(child);
+        if (index > -1) {
+            this.children.splice(index, 1);
+            child.parent = undefined;
+            this._updatePropagator();
+        } else {
+            throw new Error("Trying to remove an unknown child node!");
+        }
+    }
+
+    /**
+     * Removes this node and ligates (connects) the preceding and succeeding nodes.
+     */
+    excise() {
+        if (this.isRoot()) {
+            // TODO: Implement
+            throw new Error("Cannot excise root node!");
+        } else if (this.isTerminal()) {
+            this.parent.removeChild(this);
+        } else if (this.children.length == 1) {
+            const child = this.children[0];
+            child.setParent(this.parent);
+            this.parent.children[this.parent.children.indexOf(this)] = child;
+            this.parent._updatePropagator();
+            this.setParent(undefined);
+            this.children.length = 0;
+        } else {
+            // TODO: Implement
+            throw new Error("Cannot excise a node that has multiple children!");
+        }
+    }
+
+    isRoot() {
+        return !this.parent;
+    }
+
+    isBranching() {
+        return this.children.length > 1;
+    }
+
+    isTerminal() {
+        return this.children.length == 0;
+    }
+
+    /**
+     * Visits child nodes in depth-first order.
+     *
+     * @param {(function(FlowNode):void) & { afterChildren?: function(FlowNode):void}} visitor
+     */
+    visit(visitor) {
+        // pre-order
+        visitor(this);
+
+        for (const child of this.children) {
+            child.visit(visitor);
+        }
+
+        // post-oder
+        if (visitor.afterChildren) {
+            visitor.afterChildren(this);
+        }
+    }
+
+    /**
+     * @param {number} [depth]
+     * @returns {string}
+     */
+    subtreeToString(depth = 0) {
+        const childTree = this.children
+            .map((child) => child.subtreeToString(depth + 1))
+            .join("");
+        return `${" ".repeat(depth * 2)}* ${this.constructor.name}${
+            ("identifier" in this && this.identifier
+                ? ": " + this.identifier
+                : "") ?? ""
+        } \n${childTree}`;
+    }
+
+    /**
+     * The global object for expressions (in formula and filter transforms).
+     * Nodes in the hierarchy may extend the object using Object.create to
+     * introduce variables that are visible downstream the flow.
+     *
+     * @returns {Record<string, any>}
+     */
+    getGlobalObject() {
+        return this.parent
+            ? this.parent.getGlobalObject()
+            : ROOT_CONTEXT_OBJECT;
+    }
+
+    /**
+     *
+     * @param {Datum} datum
+     */
+    handle(datum) {
+        // Default implementation just passes through
+        this._propagate(datum);
+    }
+
+    complete() {
+        this.completed = true;
+
+        for (const child of this.children) {
+            child.complete();
+        }
+    }
+
+    /**
+     * Signals that a new batch of data will be propagated.
+     *
+     * @param {import("../types/flowBatch").FlowBatch} flowBatch
+     */
+    beginBatch(flowBatch) {
+        for (const child of this.children) {
+            child.beginBatch(flowBatch);
+        }
+    }
+
+    /**
+     *
+     * @param {any} datum
+     */
+    _propagate(datum) {
+        // Implementation is set dynamically in add/removeChild
+    }
+}
+
+/**
+ * @param {import("../types/flowBatch").FlowBatch} flowBatch
+ * @returns {flowBatch is import("../types/flowBatch").FileBatch}
+ */
+export function isFileBatch(flowBatch) {
+    return flowBatch.type == "file";
+}
+
+/**
+ * @param {import("../types/flowBatch").FlowBatch} flowBatch
+ * @returns {flowBatch is import("../types/flowBatch").FacetBatch}
+ */
+export function isFacetBatch(flowBatch) {
+    return flowBatch.type == "facet";
+}

package/dist/src/data/flowNode.test.js

@@ -0,0 +1,50 @@
+import { describe, expect, test } from "vitest";
+import FlowNode from "./flowNode";
+import { validateLinks } from "./flowOptimizer";
+
+describe("Flow mutation", () => {
+    test("Excise a terminal node", () => {
+        const a = new FlowNode();
+        const b = new FlowNode();
+
+        a.addChild(b);
+        b.excise();
+
+        expect(a.children[0]).toBeUndefined();
+        expect(b.parent).toBeUndefined();
+
+        expect(validateLinks(a)).toBeTruthy();
+    });
+
+    test("Excise a node in the middle", () => {
+        const a = new FlowNode();
+        const b = new FlowNode();
+        const c = new FlowNode();
+
+        a.addChild(b);
+        b.addChild(c);
+        b.excise();
+
+        expect(a.children[0]).toBe(c);
+        expect(c.parent).toBe(a);
+
+        expect(validateLinks(a)).toBeTruthy();
+    });
+
+    test("Insert as parent", () => {
+        const a = new FlowNode();
+        const b = new FlowNode();
+        const c = new FlowNode();
+        const d = new FlowNode();
+
+        a.addChild(c);
+        a.addChild(d);
+        c.insertAsParent(b);
+
+        expect(a.children[0]).toBe(b);
+        expect(a.children[1]).toBe(d);
+        expect(a.children[0].children[0]).toBe(c);
+
+        expect(validateLinks(a)).toBeTruthy();
+    });
+});

package/dist/src/data/flowOptimizer.d.ts

@@ -0,0 +1,27 @@
+/**
+ * @param {import("./flowNode").default} node
+ * @param {import("./flowNode").default} [parent]
+ */
+export function validateLinks(node: import("./flowNode").default, parent?: import("./flowNode").default): boolean;
+/**
+ * Removes possible redundant CloneTransforms that were added during graph construction.
+ *
+ * @param {import("./flowNode").default} node
+ */
+export function removeRedundantCloneTransforms(node: import("./flowNode").default, cloneRequired?: boolean): void;
+export function removeRedundantCollectors(): void;
+export function combineAndPullCollectorsUp(): void;
+/**
+ * @param {import("./dataFlow").default<any>} dataFlow
+ */
+export function combineIdenticalDataSources(dataFlow: import("./dataFlow").default<any>): void;
+/**
+ *
+ * @param {import("./flowNode").default} root
+ */
+export function optimizeFlowGraph(root: import("./flowNode").default): void;
+/**
+ * @param {import("./dataFlow").default<any>} dataFlow
+ */
+export function optimizeDataFlow(dataFlow: import("./dataFlow").default<any>): void;
+//# sourceMappingURL=flowOptimizer.d.ts.map

package/dist/src/data/flowOptimizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flowOptimizer.d.ts","sourceRoot":"","sources":["../../../src/data/flowOptimizer.js"],"names":[],"mappings":"AAGA;;;GAGG;AACH,oCAHW,OAAO,YAAY,EAAE,OAAO,WAC5B,OAAO,YAAY,EAAE,OAAO,WActC;AAED;;;;GAIG;AACH,qDAFW,OAAO,YAAY,EAAE,OAAO,iCAgCtC;AAED,kDAEC;AAED,mDAEC;AAmBD;;GAEG;AACH,sDAFW,OAAO,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,QAwB3C;AAED;;;GAGG;AACH,wCAFW,OAAO,YAAY,EAAE,OAAO,QAStC;AAED;;GAEG;AACH,2CAFW,OAAO,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,QAQ3C"}

package/dist/src/data/flowOptimizer.js

@@ -0,0 +1,133 @@
+import { BEHAVIOR_CLONES, BEHAVIOR_COLLECTS } from "./flowNode";
+import CloneTransform from "./transforms/clone";
+
+/**
+ * @param {import("./flowNode").default} node
+ * @param {import("./flowNode").default} [parent]
+ */
+export function validateLinks(node, parent = undefined) {
+    if (node.parent !== parent) {
+        return false;
+    }
+
+    for (const child of node.children) {
+        if (!validateLinks(child, node)) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+/**
+ * Removes possible redundant CloneTransforms that were added during graph construction.
+ *
+ * @param {import("./flowNode").default} node
+ */
+export function removeRedundantCloneTransforms(node, cloneRequired = false) {
+    if (node.behavior & BEHAVIOR_COLLECTS) {
+        // If an object is modified downstream of a collector, it must be cloned
+        cloneRequired = true;
+    }
+
+    if (node instanceof CloneTransform) {
+        if (cloneRequired) {
+            cloneRequired = false;
+        } else {
+            const child = node.children[0];
+            node.excise();
+            if (child) {
+                removeRedundantCloneTransforms(child, cloneRequired);
+            }
+            return;
+        }
+    }
+
+    if (node.behavior & BEHAVIOR_CLONES) {
+        cloneRequired = false;
+    }
+
+    for (let i = 0, n = node.children.length; i < n; i++) {
+        // Clone transforms must be preserved at branches
+        removeRedundantCloneTransforms(
+            node.children[i],
+            cloneRequired || n > 1
+        );
+    }
+}
+
+export function removeRedundantCollectors() {
+    // TODO: Remove chained collectors, e.g., first collect and sort, then just collect
+}
+
+export function combineAndPullCollectorsUp() {
+    // TODO:
+}
+
+// TODO: Combine identical branches to prevent redundant work
+// Example:
+//
+//      --B--C--D
+//     /
+// --A----B--C--E
+//     \
+//      --F--G
+//
+// should become:
+//
+//              --D
+//             /
+// --A----B--C----E
+//     \
+//      --F--G
+
+/**
+ * @param {import("./dataFlow").default<any>} dataFlow
+ */
+export function combineIdenticalDataSources(dataFlow) {
+    const dataSourceEntries = [...dataFlow._dataSourcesByHost.entries()];
+
+    /** @type {Map<string, import("./sources/dataSource").default>} */
+    const sourcesByIdentifiers = new Map();
+    for (const e of dataSourceEntries) {
+        const ds = e[1];
+        if (ds.identifier && !sourcesByIdentifiers.has(ds.identifier)) {
+            sourcesByIdentifiers.set(ds.identifier, ds);
+        }
+    }
+
+    dataFlow._dataSourcesByHost.clear();
+
+    for (let [key, dataSource] of dataSourceEntries) {
+        const target = sourcesByIdentifiers.get(dataSource.identifier);
+        if (target) {
+            target.adoptChildrenOf(dataSource);
+            dataSource = target;
+        }
+        dataFlow.addDataSource(dataSource, key);
+    }
+}
+
+/**
+ *
+ * @param {import("./flowNode").default} root
+ */
+export function optimizeFlowGraph(root) {
+    removeRedundantCloneTransforms(root);
+    if (!validateLinks(root)) {
+        throw new Error(
+            "Encountered a bug! There's a problem in the data flow structure."
+        );
+    }
+}
+
+/**
+ * @param {import("./dataFlow").default<any>} dataFlow
+ */
+export function optimizeDataFlow(dataFlow) {
+    for (const dataSource of dataFlow.dataSources) {
+        optimizeFlowGraph(dataSource);
+    }
+
+    combineIdenticalDataSources(dataFlow);
+}