@genome-spy/core 0.14.0

package/dist/style.css

@@ -0,0 +1 @@
+.genome-spy{font-family:system-ui,"Segoe UI",Roboto,Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol";position:relative}.genome-spy canvas{transform:scale(1);opacity:1;transition:transform .6s,opacity .6s}.genome-spy .loading-message{position:absolute;top:0;bottom:0;left:0;right:0;display:flex;align-items:center;justify-content:center}.genome-spy .loading-message .message{color:#666;opacity:0;transition:opacity .7s}.genome-spy.loading canvas{transform:scale(.95);opacity:0}.genome-spy.loading .loading-message .message{opacity:1}.genome-spy.loading .ellipsis{animation:blinker 1s linear infinite}@keyframes blinker{50%{opacity:0}}.genome-spy .tooltip{position:absolute;max-width:450px;overflow:hidden;background:#f6f6f6;padding:10px;font-size:13px;box-shadow:0 3px 15px #00000036;pointer-events:none;z-index:100}.genome-spy .tooltip>:last-child{margin-bottom:0}.genome-spy .tooltip>.title{padding-bottom:5px;margin-bottom:5px;border-bottom:1px dashed #b6b6b6}.genome-spy .tooltip .summary{font-size:12px}.genome-spy .tooltip table{border-collapse:collapse}.genome-spy .tooltip table:first-child{margin-top:0}.genome-spy .tooltip table th,.genome-spy .tooltip table td{padding:2px .4em;vertical-align:top}.genome-spy .tooltip table th:first-child,.genome-spy .tooltip table td:first-child{padding-left:0}.genome-spy .tooltip table th{text-align:left;font-weight:bold}.genome-spy .tooltip .color-legend{display:inline-block;width:.8em;height:.8em;margin-left:.4em;box-shadow:0 0 3px 1px #fff}.genome-spy .tooltip .attributes .hovered{background-color:#e0e0e0}.genome-spy .tooltip .na{color:#aaa;font-style:italic;font-size:80%}.genome-spy .gene-track-tooltip .summary{font-size:90%}.genome-spy .message-box{display:flex;align-items:center;justify-content:center;position:absolute;top:0;height:100%;width:100%}.genome-spy .message-box>div{border:1px solid red;padding:10px;background:#fff0f0}

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@genome-spy/core",
+  "description": "GenomeSpy, a visualization grammar and a GPU-accelerated rendering engine for genomic (and other) data",
+  "author": {
+    "name": "Kari Lavikka",
+    "email": "kari.lavikka@helsinki.fi"
+  },
+  "contributors": [],
+  "license": "BSD-2-Clause",
+  "version": "0.14.0",
+  "main": "dist/index.js",
+  "module": "src/index.js",
+  "exports": {
+    ".": "./src/index.js",
+    "./*": "./src/*"
+  },
+  "files": [
+    "dist/",
+    "src/"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "github:tuner/genome-spy",
+    "directory": "packages/core"
+  },
+  "scripts": {
+    "dev": "node dev-server.js",
+    "build": "vite build",
+    "prepublishOnly": "npm run build",
+    "checkSpec": "tsc --allowJs --checkJs --strict --noEmit --moduleResolution node --target es6 src/spec/root.d.ts",
+    "buildSchema": "ts-json-schema-generator --path 'src/spec/*.ts' --type ViewSpec > dist/genome-spy-schema.json"
+  },
+  "dependencies": {
+    "@types/d3-array": "^3.0.2",
+    "@types/d3-dsv": "^3.0.0",
+    "@types/d3-ease": "^3.0.0",
+    "@types/d3-format": "^3.0.1",
+    "@types/d3-interpolate": "^3.0.1",
+    "@types/d3-scale": "^4.0.2",
+    "d3-array": "^3.1.1",
+    "d3-color": "^3.0.1",
+    "d3-ease": "^3.0.1",
+    "d3-format": "^3.0.1",
+    "flatqueue": "^1.2.1",
+    "internmap": "^2.0.3",
+    "lit-html": "^2.0.2",
+    "twgl.js": "^4.19.1",
+    "vega-expression": "^5.0.0",
+    "vega-loader": "^4.4.0",
+    "vega-scale": "^7.1.1",
+    "vega-util": "^1.16.0"
+  },
+  "gitHead": "f7b1df1d4279575e6a1c0a4ced4a1fb0fe05db0e"
+}

package/src/data/collector.js

@@ -0,0 +1,178 @@
+import { InternMap } from "internmap";
+import { group } from "d3-array";
+import { compare } from "vega-util";
+import iterateNestedMaps from "../utils/iterateNestedMaps";
+import FlowNode, { isFacetBatch } from "./flowNode";
+import { field } from "../utils/field";
+import { asArray } from "../utils/arrayUtils";
+
+/**
+ * Collects (materializes) the data that flows through this node.
+ * The collected data can be optionally grouped and sorted.
+ *
+ * Grouping is primarily intended for handling faceted data.
+ *
+ * @typedef {import("../spec/transform").CollectParams} CollectParams
+ * @typedef {import("./flowNode").Data} Data
+ */
+export default class Collector extends FlowNode {
+    /**
+     * @param {CollectParams} [params]
+     */
+    constructor(params) {
+        super();
+
+        this.params = params ?? { type: "collect" };
+
+        /** @type {(function(Collector):void)[]} */
+        this.observers = [];
+
+        /** @type {Map<any | any[], Data>} */
+        this.facetBatches = undefined;
+
+        this._init();
+    }
+
+    _init() {
+        /** @type {Data} */
+        this._data = [];
+
+        // TODO: Consider nested maps
+        this.facetBatches = new InternMap([], JSON.stringify);
+        this.facetBatches.set(undefined, this._data);
+    }
+
+    reset() {
+        super.reset();
+        this._init();
+    }
+
+    /**
+     *
+     * @param {import("./flowNode").Datum} datum
+     */
+    handle(datum) {
+        this._data.push(datum);
+    }
+
+    /**
+     * @param {import("./flowBatch").FlowBatch} flowBatch
+     */
+    beginBatch(flowBatch) {
+        // TODO: Propagate batches to children(?)
+
+        if (isFacetBatch(flowBatch)) {
+            this._data = [];
+            this.facetBatches.set(asArray(flowBatch.facetId), this._data);
+        }
+    }
+
+    complete() {
+        const sort = this.params?.sort;
+        // Vega's "compare" function is incredibly slow (uses megamorphic field accessor)
+        // TODO: Implement a replacement for static data types
+        const comparator = sort ? compare(sort.field, sort.order) : undefined;
+
+        /** @param {any[]} data */
+        const sortData = (data) => {
+            if (comparator) {
+                data.sort(comparator);
+            }
+        };
+
+        if (this.params.groupby?.length) {
+            if (this.facetBatches.size > 1) {
+                throw new Error("TODO: Support faceted data!");
+            }
+
+            const accessors = this.params.groupby.map((fieldName) =>
+                field(fieldName)
+            );
+            // @ts-ignore
+            const groups = group(this._data, ...accessors);
+
+            this.facetBatches.clear();
+            for (const [key, data] of iterateNestedMaps(groups)) {
+                this.facetBatches.set(key, data);
+            }
+        }
+
+        for (const data of this.facetBatches.values()) {
+            // TODO: Only sort if not already sorted
+            sortData(data);
+        }
+
+        if (this.children.length) {
+            for (const data of this.facetBatches.values()) {
+                for (const datum of data) {
+                    this._propagate(datum);
+                }
+            }
+        }
+
+        super.complete();
+
+        for (const observer of this.observers) {
+            observer(this);
+        }
+    }
+
+    /**
+     * @returns {Iterable<import("./flowNode").Datum>}
+     */
+    getData() {
+        this._checkStatus();
+
+        switch (this.facetBatches.size) {
+            case 0:
+                return [];
+            case 1:
+                return [...this.facetBatches.values()][0];
+            default: {
+                const groups = this.facetBatches;
+                return {
+                    [Symbol.iterator]: function* generator() {
+                        for (const data of groups.values()) {
+                            for (let i = 0; i < data.length; i++) {
+                                yield data[i];
+                            }
+                        }
+                    },
+                };
+            }
+        }
+    }
+
+    /**
+     *
+     * @param {(datum: import("./flowNode").Datum) => void} visitor
+     */
+    visitData(visitor) {
+        this._checkStatus();
+
+        for (const data of this.facetBatches.values()) {
+            for (let i = 0; i < data.length; i++) {
+                visitor(data[i]);
+            }
+        }
+    }
+
+    /**
+     * Returns the total number of data items collected.
+     */
+    getItemCount() {
+        let count = 0;
+        for (const data of this.facetBatches.values()) {
+            count += data.length;
+        }
+        return count;
+    }
+
+    _checkStatus() {
+        if (!this.completed) {
+            throw new Error(
+                "Data propagation is not completed! No data are available."
+            );
+        }
+    }
+}

package/src/data/collector.test.js

@@ -0,0 +1,82 @@
+import Collector from "./collector";
+
+const data = [1, 5, 2, 4, 3].map((x) => ({ x }));
+
+test("Collector collects data", () => {
+    const collector = new Collector();
+
+    for (const d of data) {
+        collector.handle(d);
+    }
+    collector.complete();
+
+    expect(collector.getData()).toEqual(data);
+});
+
+test("Collector collects and sorts data", () => {
+    const collector = new Collector({
+        type: "collect",
+        sort: { field: ["x"] },
+    });
+
+    for (const d of data) {
+        collector.handle(d);
+    }
+    collector.complete();
+
+    expect([...collector.getData()]).toEqual(
+        [1, 2, 3, 4, 5].map((x) => ({ x }))
+    );
+});
+
+test("Collector collects, groups, and sorts data", () => {
+    const collector = new Collector({
+        type: "collect",
+        sort: { field: ["x"] },
+        groupby: ["a", "b"],
+    });
+
+    const data = [
+        { a: 1, b: 1, x: 1 },
+        { a: 1, b: 2, x: 2 },
+        { a: 1, b: 2, x: 3 },
+        { a: 2, b: 1, x: 4 },
+        { a: 2, b: 1, x: 5 },
+        { a: 2, b: 2, x: 6 },
+    ];
+
+    for (const d of data) {
+        collector.handle(d);
+    }
+    collector.complete();
+
+    const cd = [...collector.getData()];
+
+    expect(cd.map((d) => ({ x: d.x }))).toEqual(
+        [1, 2, 3, 4, 5, 6].map((x) => ({ x }))
+    );
+
+    /** @param {any[]} group*/
+    const getGroupX = (group) =>
+        collector.facetBatches.get(group).map((d) => d.x);
+
+    expect(getGroupX([1, 1])).toEqual([1]);
+    expect(getGroupX([1, 2])).toEqual([2, 3]);
+    expect(getGroupX([2, 1])).toEqual([4, 5]);
+    expect(getGroupX([2, 2])).toEqual([6]);
+
+    expect(new Set(collector.facetBatches.keys())).toEqual(
+        new Set([
+            [1, 1],
+            [1, 2],
+            [2, 1],
+            [2, 2],
+        ])
+    );
+});
+
+test("Collector throws on incomplete flow", () => {
+    const collector = new Collector();
+
+    expect(() => collector.getData()).toThrow();
+});

package/src/data/dataFlow.js

@@ -0,0 +1,109 @@
+/**
+ *
+ * @typedef {import("./sources/dataSource").default} DataSource
+ * @typedef {import("./collector").default} Collector
+ */
+
+/**
+ * @template H A key (string, object, whatever) that is used to retrieve
+ *      data sources and collectors.
+ */
+export default class DataFlow {
+    constructor() {
+        /** @type {Map<H, DataSource>} */
+        this._dataSourcesByHost = new Map();
+
+        /** @type {Map<H, Collector>} */
+        this._collectorsByHost = new Map();
+
+        /** @type {Map<H, (function(Collector):void)[]>} */
+        this._observers = new Map();
+    }
+
+    get dataSources() {
+        return [...new Set(this._dataSourcesByHost.values()).values()];
+    }
+
+    get collectors() {
+        return [...this._collectorsByHost.values()];
+    }
+
+    /**
+     * Adds a callback function that will be called when a collector has completed.
+     *
+     * @param {function(Collector):void} callback
+     * @param {H} key
+     */
+    addObserver(callback, key) {
+        let arr = this._observers.get(key);
+        if (!arr) {
+            arr = [];
+            this._observers.set(key, arr);
+        }
+
+        arr.push(callback);
+    }
+
+    /**
+     *
+     * @param {Collector} collector
+     * @param {H} key
+     */
+    _relayObserverCallback(collector, key) {
+        const arr = this._observers.get(key);
+        if (arr) {
+            for (const callback of arr) {
+                // eslint-disable-next-line callback-return
+                callback(collector);
+            }
+        }
+    }
+
+    /**
+     *
+     * @param {DataSource} dataSource
+     * @param {H} key
+     */
+    addDataSource(dataSource, key) {
+        this._dataSourcesByHost.set(key, dataSource);
+    }
+
+    /**
+     *
+     * @param {H} key
+     */
+    findDataSourceByKey(key) {
+        return this._dataSourcesByHost.get(key);
+    }
+
+    /**
+     *
+     * @param {Collector} collector
+     * @param {H} key
+     */
+    addCollector(collector, key) {
+        this._collectorsByHost.set(key, collector);
+        collector.observers.push((collector) =>
+            this._relayObserverCallback(collector, key)
+        );
+    }
+
+    /**
+     *
+     * @param {H} key
+     */
+    findCollectorByKey(key) {
+        return this._collectorsByHost.get(key);
+    }
+
+    /**
+     * Allows the flow nodes to perform final initialization after the flow
+     * structure has been built and optimized.
+     * Must be called before any data are to be propagated.
+     */
+    initialize() {
+        for (const ds of this.dataSources) {
+            ds.visit((node) => node.initialize());
+        }
+    }
+}

package/src/data/dataFlow.test.js

@@ -0,0 +1,3 @@
+describe("DataFlow", () => {
+    test.todo("TODO");
+});

package/src/data/facetNode.js

@@ -0,0 +1,17 @@
+import FlowNode from "./flowNode";
+
+/**
+ * @typedef {object} FacetParams
+ * @prop {string[]} groupby
+ */
+export default class FacetNode extends FlowNode {
+    /**
+     * @param {FacetParams} params
+     */
+    constructor(params) {
+        super();
+
+        /** @type {Map<any | any[], FlowNode>} */
+        this.subflows = new Map();
+    }
+}

package/src/data/flow.test.js

@@ -0,0 +1,71 @@
+import FilterTransform from "./transforms/filter";
+import FormulaTransform from "./transforms/formula";
+import Collector from "./collector";
+import { SynchronousSequenceSource } from "./flowTestUtils";
+
+describe("Test flow graphs", () => {
+    test("Trivial graph: sequence to collector", () => {
+        const source = new SynchronousSequenceSource(5);
+        const collector = new Collector();
+        source.addChild(collector);
+
+        source.dispatch();
+
+        expect(collector.getData()).toEqual(
+            [0, 1, 2, 3, 4].map((d) => ({
+                data: d,
+            }))
+        );
+    });
+
+    test("Trivial branching: sequence to two collectors", () => {
+        const source = new SynchronousSequenceSource(5);
+        const collector1 = new Collector();
+        source.addChild(collector1);
+        const collector2 = new Collector();
+        source.addChild(collector2);
+
+        source.dispatch();
+
+        expect(collector1.getData()).not.toBe(collector2._data);
+
+        expect(collector1.getData()).toEqual(
+            [0, 1, 2, 3, 4].map((d) => ({
+                data: d,
+            }))
+        );
+
+        expect(collector2.getData()).toEqual(
+            [0, 1, 2, 3, 4].map((d) => ({
+                data: d,
+            }))
+        );
+    });
+
+    test("Longer chain of nodes", () => {
+        const source = new SynchronousSequenceSource(10);
+        const filter = new FilterTransform({
+            type: "filter",
+            expr: "datum.data < 5",
+        });
+        const formula = new FormulaTransform({
+            type: "formula",
+            expr: "datum.data * 2",
+            as: "data",
+        });
+        const collector = new Collector();
+
+        source.addChild(filter);
+        filter.addChild(formula);
+        formula.addChild(collector);
+
+        source.visit((node) => node.initialize());
+        source.dispatch();
+
+        expect(collector.getData()).toEqual(
+            [0, 2, 4, 6, 8].map((d) => ({
+                data: d,
+            }))
+        );
+    });
+});

package/src/data/flowBatch.d.ts

@@ -0,0 +1,40 @@
+import { Field } from "../spec/transform";
+
+export interface FlowBatchBase {
+    type: string;
+}
+
+/**
+ * Indicates that the contents of a new file will be propagated. The fields of
+ * the file may or may not differ from the previous file. The data items within
+ * a single batch must have homogenous fields.
+ *
+ * FlowNodes can react to FileBatches and, for example, clear their internal
+ * states that expect specific fields in a specific layout.
+ */
+export interface FileBatch {
+    type: "file";
+
+    /**
+     * An absolute or relative url where the file was loaded from.
+     */
+    url?: string;
+}
+
+/**
+ * Indicates that a new group/facet will be propagated. All data items that
+ * belong to a specific facet must be propagated within a single batch.
+ */
+export interface FacetBatch {
+    type: "facet";
+
+    facetId: any | any[];
+
+    /**
+     * The field or fields that were used for partitioning the data.
+     * May be missing if partitioning is not based on the fields.
+     */
+    facetField?: Field | Field[];
+}
+
+export type FlowBatch = FileBatch | FacetBatch;