@genome-spy/core 0.14.0

package/src/data/formats/fasta.js

@@ -0,0 +1,32 @@
+/**
+ * A bare-bones FASTA-parser
+ *
+ * @typedef {object} SequenceEntry
+ * @prop {string} identifier
+ * @prop {string} sequence
+ *
+ * @param {string} data
+ * @param {any} options
+ * @returns {SequenceEntry[]}
+ */
+export default function fasta(data, options) {
+    /** @type {SequenceEntry[]} */
+    const sequences = [];
+
+    /** @type {SequenceEntry} */
+    let currentEntry;
+
+    for (const line of data.split("\n")) {
+        if (line.startsWith(">")) {
+            const identifier = line.match(/>(\S+)/)[1];
+            currentEntry = { identifier, sequence: "" };
+            sequences.push(currentEntry);
+        } else if (currentEntry) {
+            currentEntry.sequence += line.trim();
+        } else {
+            throw new Error("Invalid fasta file!");
+        }
+    }
+
+    return sequences;
+}

package/src/data/formats/fasta.test.js

@@ -0,0 +1,26 @@
+import fasta from "./fasta";
+
+test("fasta", () => {
+    const fileContent = `>A stuff
+--------------------------AGAGTTTGATCCTGGCTCAGGGTGAACGCTGGCG
+GCGTGC----TTAAGACATGCAAGTCGAACGG-CCT------TCTTCG-G-AAGGC-AGT
+---------------------------------
+>B other stuff
+--------------------------AGAGTTTGATCATGGCTCAGGGTGAACGCTGGCG
+GCGTGC----TTAAGACATGCAAGTCGGACGA-TCG------GCTTCG---GCCGGTAGT
+---------------------------------
+`;
+
+    expect(fasta(fileContent)).toEqual([
+        {
+            identifier: "A",
+            sequence:
+                "--------------------------AGAGTTTGATCCTGGCTCAGGGTGAACGCTGGCGGCGTGC----TTAAGACATGCAAGTCGAACGG-CCT------TCTTCG-G-AAGGC-AGT---------------------------------",
+        },
+        {
+            identifier: "B",
+            sequence:
+                "--------------------------AGAGTTTGATCATGGCTCAGGGTGAACGCTGGCGGCGTGC----TTAAGACATGCAAGTCGGACGA-TCG------GCTTCG---GCCGGTAGT---------------------------------",
+        },
+    ]);
+});

package/src/data/sources/dataSource.js

@@ -0,0 +1,22 @@
+import FlowNode from "../flowNode";
+
+export default class DataSource extends FlowNode {
+    /**
+     * @return {string}
+     */
+    get identifier() {
+        return undefined;
+    }
+
+    /**
+     *
+     * @param {import("../flowNode").Datum} datum
+     */
+    handle(datum) {
+        throw new Error("Source does not handle incoming data!");
+    }
+
+    async load() {
+        // override
+    }
+}

package/src/data/sources/dataSourceFactory.js

@@ -0,0 +1,24 @@
+import InlineSource, { isInlineData } from "./inlineSource";
+import UrlSource, { isUrlData } from "./urlSource";
+import SequenceSource, { isSequenceGenerator } from "./sequenceSource";
+import DynamicSource, { isDynamicData } from "./dynamicSource";
+
+/**
+ * @param {Partial<import("../../spec/data").Data>} params
+ * @param {string} [baseUrl]
+ */
+export default function createDataSource(params, baseUrl) {
+    if (isInlineData(params)) {
+        return new InlineSource(params);
+    } else if (isUrlData(params)) {
+        return new UrlSource(params, baseUrl);
+    } else if (isSequenceGenerator(params)) {
+        return new SequenceSource(params);
+    } else if (isDynamicData(params)) {
+        return new DynamicSource();
+    }
+
+    throw new Error(
+        "Cannot figure out the data source type: " + JSON.stringify(params)
+    );
+}

package/src/data/sources/dataUtils.js

@@ -0,0 +1,31 @@
+/**
+ * Validates data source params, infers format if not specified explicitly,
+ * returns a complete DataSource params object.
+ *
+ * @param {import("../../spec/data").DataSource} params
+ *      DataSource parameters
+ */
+export function getFormat(params) {
+    const format = { ...params.format };
+
+    format.type = format.type || extractTypeFromUrl(params.url);
+    format.parse = format.parse || "auto";
+
+    if (!format.type) {
+        throw new Error(
+            "Format for the data source was not defined and it could not be inferred: " +
+                JSON.stringify(params)
+        );
+    }
+
+    return format;
+}
+
+/**
+ * @param {string} url
+ */
+export function extractTypeFromUrl(url) {
+    if (url) {
+        return url.match(/\.(csv|tsv|json)/)?.[1];
+    }
+}

package/src/data/sources/dynamicCallbackSource.js

@@ -0,0 +1,56 @@
+import DataSource from "./dataSource";
+
+/**
+ * @param {Partial<import("../../spec/data").Data>} data
+ * @returns {data is import("../../spec/data").DynamicCallbackData}
+ */
+export function isDynamicCallbackData(data) {
+    return "dynamicCallbackSource" in data;
+}
+
+/**
+ * A data source that retrieves the data from a callback that returns an iterable.
+ */
+export default class DynamicCallbackSource extends DataSource {
+    /**
+     * @param {function():Iterable<any>} [callback] Function that provides the data
+     */
+    constructor(callback) {
+        super();
+
+        this.callback = callback;
+    }
+
+    loadSynchronously() {
+        if (!this.callback) {
+            return;
+        }
+
+        const iterable = this.callback();
+
+        if (!iterable || typeof iterable[Symbol.iterator] !== "function") {
+            throw new Error(
+                "Dynamic data callback didn't return iterable data!"
+            );
+        }
+
+        this.reset();
+        this.beginBatch({ type: "file" });
+
+        let wrap;
+
+        for (const d of iterable) {
+            if (!wrap) {
+                wrap = typeof d != "object" ? (x) => ({ data: x }) : (x) => x;
+            }
+
+            this._propagate(wrap(d));
+        }
+
+        this.complete();
+    }
+
+    async load() {
+        this.loadSynchronously();
+    }
+}

package/src/data/sources/dynamicSource.js

@@ -0,0 +1,36 @@
+import DataSource from "./dataSource";
+
+/**
+ * @param {Partial<import("../../spec/data").Data>} data
+ * @returns {data is import("../../spec/data").DynamicData}
+ */
+export function isDynamicData(data) {
+    return "dynamicSource" in data;
+}
+
+export default class DynamicSource extends DataSource {
+    /**
+     *
+     * @param {Iterable<any>} iterable
+     */
+    publishData(iterable) {
+        this.reset();
+        this.beginBatch({ type: "file" });
+
+        let wrap;
+
+        for (const d of iterable) {
+            if (!wrap) {
+                wrap = typeof d != "object" ? (x) => ({ data: x }) : (x) => x;
+            }
+
+            this._propagate(wrap(d));
+        }
+
+        this.complete();
+    }
+
+    async load() {
+        // nop
+    }
+}

package/src/data/sources/inlineSource.js

@@ -0,0 +1,69 @@
+import { read } from "vega-loader";
+import { getFormat } from "./dataUtils";
+import DataSource from "./dataSource";
+
+/**
+ * @param {Partial<import("../../spec/data").Data>} data
+ * @returns {data is import("../../spec/data").InlineData}
+ */
+export function isInlineData(data) {
+    return "values" in data;
+}
+
+export default class InlineSource extends DataSource {
+    /**
+     * @param {import("../../spec/data").InlineData} params
+     */
+    constructor(params) {
+        super();
+
+        this.params = params;
+
+        if (typeof params.values == "string" && !params?.format?.type) {
+            throw new Error(
+                "Data format type (csv, dsv, ...) must be specified if a string is provided!"
+            );
+        }
+    }
+
+    loadSynchronously() {
+        const values = this.params.values;
+
+        let data = [];
+
+        let wrap = (x) => x;
+
+        if (Array.isArray(values)) {
+            if (values.length > 0) {
+                data = values;
+                // TODO: Should check the whole array and abort if types are heterogeneous
+                if (typeof values[0] != "object") {
+                    // Wrap scalars to objects
+                    wrap = (d) => ({ data: d });
+                }
+            }
+        } else if (typeof values == "object") {
+            data = [values];
+        } else if (typeof values == "string") {
+            // It's a string that needs to be parsed
+            data = read(values, getFormat(this.params));
+        } else {
+            throw new Error(
+                '"values" in data configuration is not an array, object, or a string!'
+            );
+        }
+
+        this.reset();
+        this.beginBatch({ type: "file" });
+
+        for (const d of data) {
+            this._propagate(wrap(d));
+        }
+
+        this.complete();
+    }
+
+    async load() {
+        this.loadSynchronously();
+    }
+}

package/src/data/sources/inlineSource.test.js

@@ -0,0 +1,55 @@
+import Collector from "../collector";
+import InlineSource from "./inlineSource";
+
+/**
+ * @param {InlineSource} source
+ */
+async function collectSource(source) {
+    const collector = new Collector();
+    source.addChild(collector);
+
+    await source.load();
+
+    return [...collector.getData()];
+}
+
+test("InlineSource propagates an object", async () => {
+    expect(await collectSource(new InlineSource({ values: { x: 1 } }))).toEqual(
+        [{ x: 1 }]
+    );
+});
+
+test("InlineSource propagates an array of objects", async () => {
+    expect(
+        await collectSource(new InlineSource({ values: [{ x: 1 }, { x: 2 }] }))
+    ).toEqual([{ x: 1 }, { x: 2 }]);
+});
+
+test("InlineSource wraps scalars to objects", async () => {
+    expect(await collectSource(new InlineSource({ values: [1, 2] }))).toEqual([
+        { data: 1 },
+        { data: 2 },
+    ]);
+});
+
+test("InlineSource parses a string", async () => {
+    expect(
+        await collectSource(
+            new InlineSource({
+                values: "a\n1\n2\n3",
+                format: {
+                    type: "csv",
+                },
+            })
+        )
+    ).toEqual([{ a: 1 }, { a: 2 }, { a: 3 }]);
+});
+
+test("InlineSource throws on a string and a missing format specifier", () => {
+    expect(
+        () =>
+            new InlineSource({
+                values: "a\n1\n2\n3",
+            })
+    ).toThrow();
+});

package/src/data/sources/namedSource.js

@@ -0,0 +1,74 @@
+import DataSource from "./dataSource";
+
+/**
+ * @param {Partial<import("../../spec/data").Data>} data
+ * @returns {data is import("../../spec/data").NamedData}
+ */
+export function isNamedData(data) {
+    return "name" in data;
+}
+
+export default class NamedSource extends DataSource {
+    /**
+     * @param {import("../../spec/data").NamedData} params
+     * @param {function(string):any[]} getNamedData
+     */
+    constructor(params, getNamedData) {
+        super();
+
+        this.getNamedData = getNamedData;
+        this.params = params;
+    }
+
+    /**
+     * @return {string}
+     */
+    get identifier() {
+        return this.params.name;
+    }
+
+    _getValues() {
+        const data = this.getNamedData(this.params.name);
+        if (data) {
+            return data;
+        } else {
+            throw new Error("Cannot find named data: " + this.params.name);
+        }
+    }
+
+    loadSynchronously() {
+        const data = this._getValues();
+
+        /**
+         * @param {any} x
+         */
+        let wrap = (x) => x;
+
+        if (Array.isArray(data)) {
+            if (data.length > 0) {
+                // TODO: Should check the whole array and abort if types are heterogeneous
+                if (typeof data[0] != "object") {
+                    // Wrap scalars to objects
+                    wrap = (d) => ({ data: d });
+                }
+            }
+        } else {
+            throw new Error(
+                `Named data "${this.params.name}" is not an array!`
+            );
+        }
+
+        this.reset();
+        this.beginBatch({ type: "file" });
+
+        for (const d of data) {
+            this._propagate(wrap(d));
+        }
+
+        this.complete();
+    }
+
+    async load() {
+        this.loadSynchronously();
+    }
+}

package/src/data/sources/sequenceSource.js

@@ -0,0 +1,46 @@
+import DataSource from "./dataSource";
+
+/**
+ * @param {Partial<import("../../spec/data").Data>} data
+ * @returns {data is import("../../spec/data").SequenceGenerator}
+ */
+export function isSequenceGenerator(data) {
+    return "sequence" in data;
+}
+
+export default class SequenceSource extends DataSource {
+    /**
+     *
+     * @param {import("../../spec/data").SequenceGenerator} params
+     */
+    constructor(params) {
+        super();
+        this.sequence = params.sequence;
+
+        if (!("start" in this.sequence)) {
+            throw new Error("'start' is missing from sequence parameters!");
+        }
+        if (!("stop" in this.sequence)) {
+            throw new Error("'stop' is missing from sequence parameters!");
+        }
+    }
+
+    loadSynchronously() {
+        const as = this.sequence.as || "data";
+        const step = this.sequence.step || 1;
+        const stop = this.sequence.stop;
+
+        this.reset();
+        this.beginBatch({ type: "file" });
+
+        for (let x = this.sequence.start; x < stop; x += step) {
+            this._propagate({ [as]: x });
+        }
+
+        this.complete();
+    }
+
+    async load() {
+        this.loadSynchronously();
+    }
+}

package/src/data/sources/sequenceSource.test.js

@@ -0,0 +1,45 @@
+import Collector from "../collector";
+import SequenceSource from "./sequenceSource";
+
+/**
+ * @param {SequenceSource} source
+ */
+async function collectSource(source) {
+    const collector = new Collector();
+    source.addChild(collector);
+
+    await source.load();
+
+    return [...collector.getData()];
+}
+
+test("SequenceSource generates a sequence", async () => {
+    expect(
+        await collectSource(
+            new SequenceSource({ sequence: { start: 0, stop: 3 } })
+        )
+    ).toEqual([{ data: 0 }, { data: 1 }, { data: 2 }]);
+});
+
+test("SequenceSource generates a sequence with a custom step", async () => {
+    expect(
+        await collectSource(
+            new SequenceSource({ sequence: { start: 0, stop: 5, step: 2 } })
+        )
+    ).toEqual([{ data: 0 }, { data: 2 }, { data: 4 }]);
+});
+
+test("SequenceSource generates a sequence with a custom field name", async () => {
+    expect(
+        await collectSource(
+            new SequenceSource({ sequence: { start: 0, stop: 3, as: "x" } })
+        )
+    ).toEqual([{ x: 0 }, { x: 1 }, { x: 2 }]);
+});
+
+test("SequenceSource throws on missing 'start' parameter", () => {
+    expect(() => new SequenceSource({ sequence: { stop: 3 } })).toThrow();
+});
+test("SequenceSource throws on missing 'stop' parameter", () => {
+    expect(() => new SequenceSource({ sequence: { start: 0 } })).toThrow();
+});

package/src/data/sources/urlSource.js

@@ -0,0 +1,74 @@
+import { loader as vegaLoader, read } from "vega-loader";
+import { getFormat } from "./dataUtils";
+import DataSource from "./dataSource";
+
+/**
+ * @param {Partial<import("../../spec/data").Data>} data
+ * @returns {data is import("../../spec/data").UrlData}
+ */
+export function isUrlData(data) {
+    return "url" in data;
+}
+
+export default class UrlSource extends DataSource {
+    /**
+     * @param {import("../../spec/data").UrlData} params
+     * @param {string} [baseUrl]
+     */
+    constructor(params, baseUrl) {
+        super();
+
+        this.params = params;
+        this.baseUrl = baseUrl;
+    }
+
+    get identifier() {
+        return JSON.stringify({ params: this.params, baseUrl: this.baseUrl });
+    }
+
+    async load() {
+        const url = this.params.url;
+
+        /** @type {string[]} */
+        const urls = Array.isArray(url) ? url : [url];
+
+        /** @param {string} url */
+        const load = async (url) =>
+            // TODO: Support chunked loading
+            /** @type {string} */ (
+                vegaLoader({
+                    baseURL: this.baseUrl,
+                })
+                    .load(url)
+                    .catch((e) => {
+                        // TODO: Include baseurl in the error message. Should be normalized, however.
+                        throw new Error(
+                            `Cannot fetch: ${this.baseUrl}${url}: ${e.message}`
+                        );
+                    })
+            );
+
+        /**
+         * @param {string} text
+         * @param {string} [url]
+         */
+        const readAndParse = (text, url) => {
+            try {
+                /** @type {any[]} */
+                const data = read(text, getFormat(this.params));
+                this.beginBatch({ type: "file", url: url });
+                for (const d of data) {
+                    this._propagate(d);
+                }
+            } catch (e) {
+                throw new Error(`Cannot parse: ${url}: ${e.message}`);
+            }
+        };
+
+        this.reset();
+
+        await Promise.all(urls.map((url) => load(url).then(readAndParse)));
+
+        this.complete();
+    }
+}

package/src/data/transforms/aggregate.js

@@ -0,0 +1,69 @@
+import { group as d3group } from "d3-array";
+import FlowNode, { BEHAVIOR_CLONES } from "../flowNode";
+import { field } from "../../utils/field";
+import iterateNestedMaps from "../../utils/iterateNestedMaps";
+
+/**
+ * A minimal aggregate transform that just counts grouped (by a single field) data items.
+ * Work in progress.
+ *
+ * Eventually this will implement the most of Vega's aggregate transform:
+ * https://vega.github.io/vega/docs/transforms/aggregate/
+ *
+ * @typedef {import("../../spec/transform").AggregateParams} AggregateParams
+ */
+
+export default class AggregateTransform extends FlowNode {
+    get behavior() {
+        return BEHAVIOR_CLONES;
+    }
+
+    /**
+     * @param {AggregateParams} params
+     */
+    constructor(params) {
+        super();
+        this.params = params;
+
+        /** @type {any[]} */
+        this.buffer = [];
+    }
+
+    reset() {
+        this.buffer = [];
+    }
+
+    /**
+     *
+     * @param {import("../flowNode").Datum} datum
+     */
+    handle(datum) {
+        this.buffer.push(datum);
+    }
+
+    complete() {
+        const params = this.params;
+
+        const groupby = params.groupby;
+
+        const groupFieldAccessors = groupby.map((f) => field(f));
+
+        // TODO: Fix case where no group fields are specified
+        const groups = d3group(this.buffer, ...groupFieldAccessors);
+
+        for (const [group, data] of iterateNestedMaps(groups)) {
+            /** @type {any} */
+            const datum = {
+                count: data.length,
+            };
+
+            for (let i = 0; i < groupby.length; i++) {
+                datum[groupby[i]] = group[i];
+            }
+
+            this._propagate(datum);
+        }
+
+        super.complete();
+    }
+}