@genome-spy/core 0.14.0

package/src/data/transforms/pileup.test.js

@@ -0,0 +1,69 @@
+import PileupTransform from "./pileup";
+import { processData } from "../flowTestUtils";
+
+// TODO: Test for lane preferences
+
+/**
+ * @typedef {import("../../spec/transform").PileupParams} PileupParams
+ */
+
+const reads = [
+    [0, 4],
+    [1, 3],
+    [2, 6],
+    [4, 8],
+    [8, 10],
+    [11, 14],
+    [11, 13],
+    [11, 12],
+    [15, 18],
+    [16, 18],
+    [17, 18],
+].map((d) => ({
+    start: d[0],
+    end: d[1],
+}));
+
+const lanes = [0, 1, 2, 1, 0, 0, 1, 2, 0, 1, 2];
+
+/** @type {PileupParams} */
+const params = {
+    type: "pileup",
+    start: "start",
+    end: "end",
+};
+
+/**
+ * @param {PileupParams} params
+ * @param {any[]} data
+ */
+function pileupTransform(params, data) {
+    const t = new PileupTransform(params);
+    t.initialize();
+    return processData(t, data);
+}
+
+test("Pileup transform produces correct pileup", () => {
+    const piledUp = lanes.map((d, i) => ({
+        ...reads[i],
+        lane: d,
+    }));
+
+    expect(pileupTransform(params, reads)).toEqual(piledUp);
+});
+
+test("Pileup transform produces correct pileup with consecutive contigs", () => {
+    // Simulate data having multiple chromosomes, sorted by [chrom, pos].
+    // Piling should handle suddenly decreasing start positions by freeing all
+    // reserved lanes.
+
+    const repeatedReads = [...reads, ...reads];
+    const repeatedLanes = [...lanes, ...lanes];
+
+    const piledUp = repeatedLanes.map((d, i) => ({
+        ...repeatedReads[i],
+        lane: d,
+    }));
+
+    expect(pileupTransform(params, repeatedReads)).toEqual(piledUp);
+});

package/src/data/transforms/project.js

@@ -0,0 +1,41 @@
+import { accessorName } from "vega-util";
+import { field } from "../../utils/field";
+import FlowNode, { BEHAVIOR_CLONES } from "../flowNode";
+
+/**
+ * @typedef {import("../../spec/transform").ProjectParams} ProjectParams
+ */
+export default class ProjectTransform extends FlowNode {
+    get behavior() {
+        return BEHAVIOR_CLONES;
+    }
+
+    /**
+     *
+     * @param {ProjectParams} params
+     */
+    constructor(params) {
+        super();
+
+        if (params.as && params.as.length != params.fields.length) {
+            throw new Error(`"fields" and "as" have unequal lengths!`);
+        }
+
+        // TODO: "If unspecified, all fields will be copied using their existing names."
+
+        const accessors = params.fields.map((f) => field(f));
+        const as = params.as ? params.as : accessors.map(accessorName);
+
+        /**
+         * @param {any} datum
+         */
+        this.handle = (datum) => {
+            /** @type {Record<string, any>} */
+            const projected = {};
+            for (let i = 0; i < accessors.length; i++) {
+                projected[as[i]] = accessors[i](datum);
+            }
+            this._propagate(projected);
+        };
+    }
+}

package/src/data/transforms/project.test.js

@@ -0,0 +1,31 @@
+import { processData } from "../flowTestUtils";
+import ProjectTransform from "./project";
+
+test("Project", () => {
+    const data = [
+        {
+            foo: "FOO",
+            bar: "BAR",
+            baz: { a: "A" },
+        },
+    ];
+
+    /** @param {import("./project").ProjectParams} params */
+    const p = (params) => processData(new ProjectTransform(params), data);
+
+    expect(p({ type: "project", fields: ["bar"] })).toEqual([{ bar: "BAR" }]);
+
+    expect(
+        p({ type: "project", fields: ["bar", "foo"], as: ["xBar", "xFoo"] })
+    ).toEqual([{ xBar: "BAR", xFoo: "FOO" }]);
+
+    expect(p({ type: "project", fields: ["baz.a"] })).toEqual([
+        { "baz.a": "A" },
+    ]);
+
+    expect(p({ type: "project", fields: ["baz.a"], as: ["a"] })).toEqual([
+        { a: "A" },
+    ]);
+
+    expect(() => p({ type: "project", fields: ["bar"], as: [] })).toThrow();
+});

package/src/data/transforms/regexExtract.js

@@ -0,0 +1,61 @@
+import { isString } from "vega-util";
+import { field } from "../../utils/field";
+import FlowNode, { BEHAVIOR_MODIFIES } from "../flowNode";
+
+/**
+ * @typedef {import("../../spec/transform").RegexExtractParams} RegexExtractParams
+ */
+
+export default class RegexExtractTransform extends FlowNode {
+    get behavior() {
+        return BEHAVIOR_MODIFIES;
+    }
+
+    /**
+     * @param {RegexExtractParams} params
+     */
+    constructor(params) {
+        super();
+
+        const re = new RegExp(params.regex);
+        const as = typeof params.as == "string" ? [params.as] : params.as;
+        const accessor = field(params.field);
+
+        /**
+         *
+         * @param {any} datum
+         */
+        this.handle = (datum) => {
+            const value = accessor(datum);
+            if (isString(value)) {
+                const m = value.match(re);
+
+                if (m) {
+                    if (m.length - 1 != as.length) {
+                        throw new Error(
+                            'The number of RegEx groups and the length of "as" do not match!'
+                        );
+                    }
+
+                    for (let i = 0; i < as.length; i++) {
+                        datum[as[i]] = m[i + 1];
+                    }
+                } else if (params.skipInvalidInput) {
+                    for (let i = 0; i < as.length; i++) {
+                        datum[as[i]] = undefined;
+                    }
+                } else {
+                    throw new Error(
+                        `"${value}" does not match the given regex: ${re.toString()}`
+                    );
+                }
+            } else if (!params.skipInvalidInput) {
+                throw new Error(
+                    `Trying to match a non-string field. Encountered type: ${typeof value}, field content: "${value}".`
+                );
+            }
+
+            this._propagate(datum);
+        };
+    }
+}

package/src/data/transforms/regexExtract.test.js

@@ -0,0 +1,66 @@
+import { processData } from "../flowTestUtils";
+
+import RegexExtractTransform from "./regexExtract";
+
+/**
+ * @param {import("./regexExtract").RegexExtractParams} params
+ * @param {any[]} data
+ */
+function transform(params, data) {
+    return processData(new RegexExtractTransform(params), data);
+}
+
+describe("RegexExtractTransform", () => {
+    const rows = [{ a: "12-34" }, { a: "23-45" }];
+
+    /** @type {import("./regexExtract").RegexExtractParams} */
+    const params = {
+        type: "regexExtract",
+        regex: "^(\\d+)-(\\d+)$",
+        field: "a",
+        as: ["b", "c"],
+    };
+
+    test("Valid config and input", () => {
+        expect(transform(params, rows)).toEqual([
+            { a: "12-34", b: "12", c: "34" },
+            { a: "23-45", b: "23", c: "45" },
+        ]);
+    });
+
+    test("Invalid config", () => {
+        /** @type {import("./regexExtract").RegexExtractParams} */
+        const config2 = {
+            type: "regexExtract",
+            regex: "^(\\d+)-(\\d+)$",
+            field: "a",
+            as: ["b", "c", "d"],
+        };
+
+        expect(() => transform(config2, rows)).toThrow();
+    });
+
+    test("Invalid data", () => {
+        const rows2 = [{ a: "12--34" }];
+
+        expect(() => transform(params, rows2)).toThrow();
+    });
+
+    test("Invalid, non-string data", () => {
+        const rows2 = [{ a: 123 }];
+
+        expect(() => transform(params, rows2)).toThrow();
+    });
+
+    test("Skip invalid or non-string data", () => {
+        const rows2 = [{ a: 123 }, { a: "xyzzy" }, { a: "12-34" }];
+
+        expect(transform({ ...params, skipInvalidInput: true }, rows2)).toEqual(
+            [
+                { a: 123, b: undefined, c: undefined },
+                { a: "xyzzy", b: undefined, c: undefined },
+                { a: "12-34", b: "12", c: "34" },
+            ]
+        );
+    });
+});

package/src/data/transforms/regexFold.js

@@ -0,0 +1,141 @@
+import { asArray } from "../../utils/arrayUtils";
+import FlowNode, { BEHAVIOR_CLONES, isFileBatch } from "../flowNode";
+
+/**
+ * Folds fields using a regex
+ *
+ * See: https://vega.github.io/vega/docs/transforms/fold/
+ *
+ * @typedef {import("../../spec/transform").RegexFoldParams} RegexFoldParams
+ */
+export default class RegexFoldTransform extends FlowNode {
+    get behavior() {
+        return BEHAVIOR_CLONES;
+    }
+
+    /**
+     * @param {RegexFoldParams} params
+     */
+    constructor(params) {
+        super();
+
+        const columnRegex = asArray(params.columnRegex).map(
+            (re) => new RegExp(re)
+        );
+        // TODO: Consider using named groups
+        const as = asArray(params.asValue);
+
+        if (columnRegex.length != as.length) {
+            throw new Error('Lengths of "columnRegex" and "as" are not equal!');
+        }
+
+        const skipRegex = params.skipRegex
+            ? new RegExp(params.skipRegex)
+            : undefined;
+
+        const sampleKey = params.asKey || "sample";
+
+        /** @type {[string, string[]][]} */
+        let sampleAttrs;
+
+        /** @type {string[]} */
+        let includedColumns;
+
+        /** @type {(datum: any, sampleId: string) => Record<string, any>} */
+        let create;
+
+        /**
+         * @param {any} datum
+         */
+        const detectColumns = (datum) => {
+            const colNames = Object.keys(datum);
+
+            /** @type {Map<string, string[]>} */
+            const sampleColMap = new Map();
+
+            for (const [i, re] of columnRegex.entries()) {
+                for (const colName of colNames) {
+                    const sampleId = re.exec(colName)?.[1];
+                    if (sampleId !== undefined) {
+                        let attrs = sampleColMap.get(sampleId);
+                        if (!attrs) {
+                            attrs = [];
+                            sampleColMap.set(sampleId, attrs);
+                        }
+
+                        attrs[i] = colName;
+                    }
+                }
+            }
+
+            sampleAttrs = [...sampleColMap.entries()];
+
+            includedColumns = colNames.filter(
+                (colName) =>
+                    !columnRegex.some((re) => re.test(colName)) &&
+                    !(skipRegex && skipRegex.test(colName))
+            );
+
+            const props = [
+                ...includedColumns.map(
+                    (prop) =>
+                        JSON.stringify(prop) +
+                        ": datum[" +
+                        JSON.stringify(prop) +
+                        "]"
+                ),
+                JSON.stringify(sampleKey) + ": sampleId",
+                ...as.map((a) => JSON.stringify(a) + ": null"),
+            ];
+
+            // eslint-disable-next-line no-new-func
+            create = /** @type {any} */ (
+                new Function(
+                    "datum",
+                    "sampleId",
+                    "return {\n" + props.join(",\n") + "\n};"
+                )
+            );
+        };
+
+        /**
+         * @param {any} datum
+         */
+        const doRegexFold = (datum) => {
+            if (!sampleAttrs) {
+                detectColumns(datum);
+            }
+
+            for (const [sampleId, attrs] of sampleAttrs) {
+                const tidyRow = create(datum, sampleId);
+                for (let i = 0; i < attrs.length; i++) {
+                    tidyRow[as[i]] = datum[attrs[i]];
+                }
+
+                this._propagate(tidyRow);
+            }
+        };
+
+        /**
+         * @param {any} datum
+         */
+        const detectAndHandle = (datum) => {
+            detectColumns(datum);
+            doRegexFold(datum);
+            this.handle = doRegexFold;
+        };
+
+        this.handle = detectAndHandle;
+
+        /**
+         *
+         * @param {import("../flowNode").FlowBatch} flowBatch
+         */
+        this.beginBatch = (flowBatch) => {
+            if (isFileBatch(flowBatch)) {
+                this.handle = detectAndHandle;
+            }
+            super.beginBatch(flowBatch);
+        };
+    }
+}

package/src/data/transforms/regexFold.test.js

@@ -0,0 +1,159 @@
+import { processData } from "../flowTestUtils";
+import RegexFoldTransform from "./regexFold";
+
+describe("RegexFold", () => {
+    test("Transform single variable", () => {
+        const sampleData = [
+            {
+                row: 1,
+                sample1_a: "r1s1a",
+                sample2_a: "r1s2a",
+            },
+            {
+                row: 2,
+                sample1_a: "r2s1a",
+                sample2_a: "r2s2a",
+            },
+        ];
+
+        /** @type { import("../../spec/transform").RegexFoldParams } */
+        const singleGatherConfig = {
+            type: "regexFold",
+            columnRegex: "^(.*)_a$",
+            asValue: "a",
+        };
+
+        const result = processData(
+            new RegexFoldTransform(singleGatherConfig),
+            sampleData
+        );
+
+        expect(result).toEqual([
+            {
+                row: 1,
+                sample: "sample1",
+                a: "r1s1a",
+            },
+            {
+                row: 1,
+                sample: "sample2",
+                a: "r1s2a",
+            },
+            {
+                row: 2,
+                sample: "sample1",
+                a: "r2s1a",
+            },
+            {
+                row: 2,
+                sample: "sample2",
+                a: "r2s2a",
+            },
+        ]);
+    });
+
+    test("Transform single variable and skip specific columns", () => {
+        const sampleData = [
+            {
+                row: 1,
+                sample1_a: "r1s1a",
+                sample2_a: "r1s2a",
+            },
+            {
+                row: 2,
+                sample1_a: "r2s1a",
+                sample2_a: "r2s2a",
+            },
+        ];
+
+        /** @type { import("../../spec/transform").RegexFoldParams } */
+        const singleGatherConfig = {
+            type: "regexFold",
+            columnRegex: "^(.*)_a$",
+            asValue: "a",
+            skipRegex: "^row$",
+        };
+
+        const result = processData(
+            new RegexFoldTransform(singleGatherConfig),
+            sampleData
+        );
+
+        expect(result).toEqual([
+            {
+                sample: "sample1",
+                a: "r1s1a",
+            },
+            {
+                sample: "sample2",
+                a: "r1s2a",
+            },
+            {
+                sample: "sample1",
+                a: "r2s1a",
+            },
+            {
+                sample: "sample2",
+                a: "r2s2a",
+            },
+        ]);
+    });
+
+    test("Transform multiple variables", () => {
+        const sampleData = [
+            {
+                row: 1,
+                sample1_a: "r1s1a",
+                sample2_a: "r1s2a",
+                sample1_b: "r1s1b",
+                sample2_b: "r1s2b",
+            },
+            {
+                row: 2,
+                sample1_a: "r2s1a",
+                sample2_a: "r2s2a",
+                sample1_b: "r2s1b",
+                sample2_b: "r2s2b",
+            },
+        ];
+
+        /** @type { import("../../spec/transform").RegexFoldParams } */
+        const singleGatherConfig = {
+            type: "regexFold",
+            columnRegex: ["^(.*)_a$", "^(.*)_b$"],
+            asValue: ["a", "b"],
+        };
+
+        const result = processData(
+            new RegexFoldTransform(singleGatherConfig),
+            sampleData
+        );
+
+        expect(result).toEqual([
+            {
+                row: 1,
+                sample: "sample1",
+                a: "r1s1a",
+                b: "r1s1b",
+            },
+            {
+                row: 1,
+                sample: "sample2",
+                a: "r1s2a",
+                b: "r1s2b",
+            },
+            {
+                row: 2,
+                sample: "sample1",
+                a: "r2s1a",
+                b: "r2s1b",
+            },
+            {
+                row: 2,
+                sample: "sample2",
+                a: "r2s2a",
+                b: "r2s2b",
+            },
+        ]);
+    });
+});

package/src/data/transforms/sample.js

@@ -0,0 +1,101 @@
+import FlowNode from "../flowNode";
+
+/**
+ * A reservoir sampler, based on: https://www.wikiwand.com/en/Reservoir_sampling
+ *
+ * @typedef {import("../../spec/transform").SampleParams} SampleParams
+ */
+export default class SampleTransform extends FlowNode {
+    /**
+     *
+     * @param {SampleParams} params
+     */
+    constructor(params) {
+        super();
+
+        this.k = params.size || 500;
+        this.reset();
+    }
+
+    reset() {
+        super.reset();
+
+        /** @type {any[]} */
+        this.reservoir = [];
+        /** @type {number} */
+        this.W = undefined;
+        this.ingester = this._initialIngester;
+    }
+
+    /**
+     *
+     * @param {any} item
+     */
+    _initialIngester(item) {
+        this.reservoir.push(item);
+
+        if (this.reservoir.length == this.k) {
+            this.W = Math.exp(Math.log(Math.random()) / this.k);
+            this.i = this.k;
+            this.next = this.i;
+            this.ingester = this._finalIngester;
+            this._setNextStop();
+        }
+    }
+
+    /**
+     *
+     * @param {any} item
+     */
+    _finalIngester(item) {
+        if (++this.i == this.next) {
+            this.reservoir[Math.floor(Math.random() * this.k)] = item;
+            this.W *= Math.exp(Math.log(Math.random()) / this.k);
+            this._setNextStop();
+        }
+    }
+
+    _setNextStop() {
+        this.next +=
+            Math.floor(Math.log(Math.random()) / Math.log(1 - this.W)) + 1;
+    }
+
+    /**
+     *
+     * @param {import("../flowNode").Datum} datum
+     */
+    handle(datum) {
+        this.ingester(datum);
+    }
+
+    complete() {
+        for (const datum of this.reservoir) {
+            this._propagate(datum);
+        }
+
+        super.complete();
+    }
+}
+
+/**
+ * A convenience function that uses SampleTransform to sample an iterable.
+ *
+ * @param {number} n
+ * @param {Iterable<T>} iterable
+ * @param {function(T):R} accessor
+ * @returns {R[]}
+ * @template T
+ * @template R
+ */
+export function sampleIterable(n, iterable, accessor) {
+    const sampler = new SampleTransform({
+        type: "sample",
+        size: n,
+    });
+    for (const d of iterable) {
+        sampler.handle(accessor(d));
+    }
+    sampler.complete();
+
+    return sampler.reservoir;
+}