@genome-spy/core 0.43.1 → 0.43.3

package/dist/src/data/sources/lazy/bigBedSource.js

@@ -92,12 +92,24 @@ export default class BigBedSource extends SingleAxisWindowedSource {
 }
 
 /**
- * A specific optimization for Hautaniemi Lab's Methylation project, where
- * we have hundreds of columns having small integers (0-100).
- * This parser avoids generating piles of garbage to be collected by the GC.
- * We don't split the line into an array of strings, but instead parse the
- * integer fields directly from the original string.
- * This parser doesn't support arrays, etc. at the moment.
+ * An optimized parser for Hautaniemi Lab's Methylation project, where
+ * we have hundreds of columns having small integers (0-100). This is over 5x
+ * faster than @gmod/bed's parser.
+ *
+ * Techniques used:
+ *
+ * 1. Avoid generating garbage by parsing integers directly from the string,
+ *    i.e., without splitting the line into an array of strings.
+ * 2. Use a template object to avoid hidden class changes after each property
+ *    assignment. Avoids garbage generation.
+ * 3. Generate and compile code that uses constants to access object properties,
+ *    avoiding Map lookups during assignment.
+ * 4. Input chrom, startPos, and endPos as parameters so that @gmod/bbi's
+ *    output doesn't first need to be converted to a string just to be parsed
+ *    again.
+ *
+ * This parser doesn't support arrays, etc. at the moment. This could, however,
+ * be extended into a fully-featured parser.
  *
  * @param {import("@gmod/bed").default} bed
  */
@@ -151,26 +163,69 @@ function makeFastParser(bed) {
         return value * sign;
     }
 
+    const templateFields = fields.map(
+        (field) =>
+            `${JSON.stringify(field.name)}: ${
+                field.isNumeric ? "0" : "emptyString"
+            }`
+    );
+
+    /**
+     * Make a template object with all fields to avoid the JavaScript VM's
+     * hidden class to be changed after each property assignment. Transitions
+     * between hidden classes generate plenty of garbage to be collected.
+     *
+     * Ideally, the parsed values would be assigned directly in this function,
+     * but for some reason, it results in abysmally slow performance on Chrome,
+     * but not on Firefox, where it would be super fast.
+     */
+    const makeTemplate = new Function(`
+        const emptyString = "";
+        return function makeTemplate(chrom, chromStart, chromEnd) {
+            return {
+                chrom,
+                chromStart,
+                chromEnd,
+                ${templateFields.join(",\n")}
+            }
+        };`)();
+
+    /*
+     * Generate setter code that uses constant field names to access the
+     * object's properties. This avoids Map lookups and allows for efficient
+     * machine code to be generated by the VM.
+     */
     const fieldParsers = fields.map((field) => {
-        const { name, type } = field;
+        const type = field.type;
+        const name = JSON.stringify(field.name);
 
         if (["ubyte", "int", "uint"].includes(type)) {
-            return () => {
-                currentObject[name] = parseInt();
-            };
+            return `d[${name}] = parseInt();`;
         } else if (field.isNumeric) {
-            return () => {
-                currentObject[name] = Number(parseString());
-            };
+            return `d[${name}] = Number(parseString());`;
         } else if (["char", "string", "lstring"].includes(type)) {
-            return () => {
-                currentObject[name] = parseString();
-            };
+            return `d[${name}] = parseString();`;
         } else {
             throw new Error("Unsupported type: " + type);
+            // TODO: Implement them
         }
     });
 
+    /*
+     * Split the field parsers into chunks to avoid creating so large
+     * functions that the JavaScript VM would decline to optimize it.
+     * Not sure if this is really necessary, but the added cost is minimal.
+     */
+    const chunckedFieldParsers = chunk(fieldParsers, 50).map((chunk, i) =>
+        Function(
+            "parseInt",
+            "parseString",
+            `return function parseFieldChunk${i}(d) {
+            ${chunk.join("\n")}
+        }`
+        )(parseInt, parseString)
+    );
+
     /**
      * @param {string} line
      */
@@ -189,14 +244,10 @@ function makeFastParser(bed) {
     function parseLine(chrom, chromStart, chromEnd, rest) {
         setLine(rest);
 
-        currentObject = {
-            chrom,
-            chromStart,
-            chromEnd,
-        };
+        currentObject = makeTemplate(chrom, chromStart, chromEnd);
 
-        for (let j = 0, n = fieldParsers.length; j < n; j++) {
-            fieldParsers[j]();
+        for (const parser of chunckedFieldParsers) {
+            parser(currentObject);
         }
 
         return currentObject;
@@ -204,3 +255,15 @@ function makeFastParser(bed) {
 
     return parseLine;
 }
+
+/**
+ * @param {T[]} arr
+ * @param {number} size
+ * @template T
+ */
+function chunk(arr, size) {
+    // https://www.30secondsofcode.org/js/s/split-array-into-chunks/
+    return Array.from({ length: Math.ceil(arr.length / size) }, (_v, i) =>
+        arr.slice(i * size, i * size + size)
+    );
+}

package/dist/src/data/transforms/coverage.d.ts

@@ -2,7 +2,10 @@
 /**
  * Computes coverage for sorted segments
  *
- * TODO: Binned coverage
+ * TODO: Binned coverage, e.g., don't emit a new segment for every
+ * coverage change, but only every n bases or so. The most straightforward
+ * way to implement it is a separate transform that bins the coverage
+ * segments and calculates weighted averages.
  */
 export default class CoverageTransform extends FlowNode {
     /**
@@ -23,7 +26,12 @@ export default class CoverageTransform extends FlowNode {
         chrom: string;
     };
     createSegment: Function;
-    ends: FlatQueue<any>;
+    /**
+     * End pos as priority, weight as value
+     *
+     * @type {FlatQueue<number>}
+     */
+    ends: FlatQueue<number>;
 }
 import FlowNode from "../flowNode.js";
 import FlatQueue from "flatqueue";

package/dist/src/data/transforms/coverage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"coverage.d.ts","sourceRoot":"","sources":["../../../../src/data/transforms/coverage.js"],"names":[],"mappings":";AAKA;;;;GAIG;AACH;IAKI;;OAEG;IACH,oBAFW,OAAO,yBAAyB,EAAE,cAAc,EA4C1D;IAxCG,yDAAoB;IAEpB,mDAAwC;IACxC,iDAAoC;IAEpC,mCAAmC;IACnC,sBADoB,GAAG,KAAE,MAAM,CAGT;IACtB,mCAAmC;IACnC,uBADoB,GAAG,KAAE,MAAM,CACsC;IAErE;;;;;MAKC;IAGD,wBAgBC;IAGD,qBAA2B;CA2HlC;qBAnLyC,gBAAgB;sBAHpC,WAAW"}
+{"version":3,"file":"coverage.d.ts","sourceRoot":"","sources":["../../../../src/data/transforms/coverage.js"],"names":[],"mappings":";AAKA;;;;;;;GAOG;AACH;IAKI;;OAEG;IACH,oBAFW,OAAO,yBAAyB,EAAE,cAAc,EAgD1D;IA5CG,yDAAoB;IAEpB,mDAAwC;IACxC,iDAAoC;IAEpC,mCAAmC;IACnC,sBADoB,GAAG,KAAE,MAAM,CAGT;IACtB,mCAAmC;IACnC,uBADoB,GAAG,KAAE,MAAM,CACsC;IAErE;;;;;MAKC;IAGD,wBAgBC;IAED;;;;OAIG;IACH,MAFU,UAAU,MAAM,CAAC,CAEA;CAiIlC;qBAhMyC,gBAAgB;sBAHpC,WAAW"}

package/dist/src/data/transforms/coverage.js

@@ -6,7 +6,10 @@ import FlowNode, { BEHAVIOR_CLONES } from "../flowNode.js";
 /**
  * Computes coverage for sorted segments
  *
- * TODO: Binned coverage
+ * TODO: Binned coverage, e.g., don't emit a new segment for every
+ * coverage change, but only every n bases or so. The most straightforward
+ * way to implement it is a separate transform that bins the coverage
+ * segments and calculates weighted averages.
  */
 export default class CoverageTransform extends FlowNode {
     get behavior() {
@@ -56,7 +59,11 @@ export default class CoverageTransform extends FlowNode {
             )
         );
 
-        // End pos as priority, weight as value
+        /**
+         * End pos as priority, weight as value
+         *
+         * @type {FlatQueue<number>}
+         */
         this.ends = new FlatQueue();
     }
 
@@ -75,7 +82,7 @@ export default class CoverageTransform extends FlowNode {
         const chromAccessor = this.chromAccessor;
         const weightAccessor = this.weightAccessor;
 
-        /** @type {Record<string, number|string>} used for merging adjacent segment */
+        /** @type {import("../flowNode.js").Datum} used for merging adjacent segment */
         let bufferedSegment;
 
         /** @type {string} */
@@ -91,7 +98,7 @@ export default class CoverageTransform extends FlowNode {
         /** @type {number} */
         let prevEdge;
 
-        // End pos as priority, weight as value
+        /** End pos as priority, weight as value */
         const ends = this.ends;
         ends.clear();
 
@@ -127,9 +134,7 @@ export default class CoverageTransform extends FlowNode {
         };
 
         const flushQueue = () => {
-            // Flush queue
-            /** @type {number} */
-            let edge;
+            let edge = 0;
             while ((edge = ends.peekValue()) !== undefined) {
                 pushSegment(prevEdge, edge, coverage);
                 prevEdge = edge;
@@ -147,8 +152,7 @@ export default class CoverageTransform extends FlowNode {
         this.handle = (datum) => {
             const start = startAccessor(datum);
 
-            /** @type {number} */
-            let edge;
+            let edge = 0;
             while ((edge = ends.peekValue()) !== undefined && edge < start) {
                 pushSegment(prevEdge, edge, coverage);
                 prevEdge = edge;
@@ -179,5 +183,14 @@ export default class CoverageTransform extends FlowNode {
             flushQueue();
             super.complete();
         };
+
+        /**
+         * @param {import("../../types/flowBatch.js").FlowBatch} flowBatch
+         */
+        this.beginBatch = (flowBatch) => {
+            flushQueue();
+            prevChrom = undefined;
+            super.beginBatch(flowBatch);
+        };
     }
 }

package/dist/src/data/transforms/coverage.test.js

@@ -1,4 +1,4 @@
-import { expect, test } from "vitest";
+import { describe, expect, test } from "vitest";
 import CoverageTransform from "./coverage.js";
 import { processData } from "../flowTestUtils.js";
 
@@ -16,108 +16,223 @@ function transform(params, data) {
     return processData(t, data);
 }
 
-test("Coverage transform produces correct coverage segments", () => {
-    const reads = [
-        [0, 4],
-        [1, 3],
-        [2, 6],
-        [4, 8],
-        [8, 10],
-        [11, 14],
-        [11, 13],
-        [11, 12],
-        [15, 18],
-        [16, 18],
-        [17, 18],
-    ].map((d) => ({
-        start: d[0],
-        end: d[1],
-    }));
-
-    const coverageSegments = [
-        [0, 1, 1],
-        [1, 2, 2],
-        [2, 3, 3],
-        [3, 6, 2],
-        [6, 10, 1],
-        [11, 12, 3],
-        [12, 13, 2],
-        [13, 14, 1],
-        [15, 16, 1],
-        [16, 17, 2],
-        [17, 18, 3],
-    ].map((d) => ({
-        start: d[0],
-        end: d[1],
-        coverage: d[2],
-    }));
-
+/**
+ *
+ * @param {[number, number][]} reads Start, end
+ * @param {[number, number, number][]} coverageSegments Start, end, coverage
+ */
+function testSimpleCoverage(reads, coverageSegments) {
     /** @type {CoverageParams} */
     const coverageConfig = {
         type: "coverage",
         start: "start",
         end: "end",
     };
-    expect(transform(coverageConfig, reads)).toEqual(coverageSegments);
-});
-
-test("Coverage transform handles chromosomes", () => {
-    const reads = [
-        { chrom: "chr1", start: 0, end: 1 },
-        { chrom: "chr2", start: 0, end: 1 },
-        { chrom: "chr3", start: 1, end: 3 },
-    ];
-
-    const coverageSegments = [
-        { chrom: "chr1", start: 0, end: 1, coverage: 1 },
-        { chrom: "chr2", start: 0, end: 1, coverage: 1 },
-        { chrom: "chr3", start: 1, end: 3, coverage: 1 },
-    ];
+    expect(
+        transform(
+            coverageConfig,
+            reads.map((d) => ({
+                start: d[0],
+                end: d[1],
+            }))
+        )
+    ).toEqual(
+        coverageSegments.map((d) => ({
+            start: d[0],
+            end: d[1],
+            coverage: d[2],
+        }))
+    );
+}
 
+/**
+ *
+ * @param {[number, number, number][]} reads Start, end, weight
+ * @param {[number, number, number][]} coverageSemgments Start, end, coverage
+ */
+function testWeightedCoverage(reads, coverageSegments) {
     /** @type {CoverageParams} */
     const coverageConfig = {
         type: "coverage",
-        chrom: "chrom",
         start: "start",
         end: "end",
+        weight: "weight",
     };
+    expect(
+        transform(
+            coverageConfig,
+            reads.map((d) => ({
+                start: d[0],
+                end: d[1],
+                weight: d[2],
+            }))
+        )
+    ).toEqual(
+        coverageSegments.map((d) => ({
+            start: d[0],
+            end: d[1],
+            coverage: d[2],
+        }))
+    );
+}
 
-    expect(transform(coverageConfig, reads)).toEqual(coverageSegments);
-});
+describe("Coverage transform", () => {
+    test("Typical case", () =>
+        testSimpleCoverage(
+            [
+                [0, 4],
+                [1, 3],
+                [2, 6],
+                [4, 8],
+                [8, 10],
+                [11, 14],
+                [11, 13],
+                [11, 12],
+                [15, 18],
+                [16, 18],
+                [17, 18],
+            ],
+            [
+                [0, 1, 1],
+                [1, 2, 2],
+                [2, 3, 3],
+                [3, 6, 2],
+                [6, 10, 1],
+                [11, 12, 3],
+                [12, 13, 2],
+                [13, 14, 1],
+                [15, 16, 1],
+                [16, 17, 2],
+                [17, 18, 3],
+            ]
+        ));
 
-test("Coverage transform handles weights", () => {
-    const reads = [
-        [0, 4, 1],
-        [1, 3, 2],
-        [2, 6, 3],
-        [8, 10, -1],
-    ].map((d) => ({
-        start: d[0],
-        end: d[1],
-        weight: d[2],
-    }));
+    test("Multiple identical overlapping segments", () =>
+        testSimpleCoverage(
+            [
+                [1, 2],
+                [3, 4],
+                [3, 4],
+                [5, 6],
+                [5, 6],
+                [5, 6],
+            ],
+            [
+                [1, 2, 1],
+                [3, 4, 2],
+                [5, 6, 3],
+            ]
+        ));
 
-    const coverageSegments = [
-        [0, 1, 1],
-        [1, 2, 3],
-        [2, 3, 6],
-        [3, 4, 4],
-        [4, 6, 3],
-        [8, 10, -1],
-    ].map((d) => ({
-        start: d[0],
-        end: d[1],
-        coverage: d[2],
-    }));
+    test("Adjacent segments with equal coverage are merged", () =>
+        testSimpleCoverage(
+            [
+                [1, 2],
+                [2, 3],
+                [3, 4],
+                [5, 6],
+                [6, 7],
+                [7, 8],
+            ],
+            [
+                [1, 4, 1],
+                [5, 8, 1],
+            ]
+        ));
 
-    /** @type {CoverageParams} */
-    const coverageConfig = {
-        type: "coverage",
-        chrom: "chrom",
-        start: "start",
-        end: "end",
-        weight: "weight",
-    };
+    test("Chromosomes pass through", () => {
+        const reads = [
+            { chrom: "chr1", start: 0, end: 1 },
+            { chrom: "chr2", start: 0, end: 1 },
+            { chrom: "chr3", start: 1, end: 3 },
+        ];
+
+        const coverageSegments = [
+            { chrom: "chr1", start: 0, end: 1, coverage: 1 },
+            { chrom: "chr2", start: 0, end: 1, coverage: 1 },
+            { chrom: "chr3", start: 1, end: 3, coverage: 1 },
+        ];
+
+        /** @type {CoverageParams} */
+        const coverageConfig = {
+            type: "coverage",
+            chrom: "chrom",
+            start: "start",
+            end: "end",
+        };
+
+        expect(transform(coverageConfig, reads)).toEqual(coverageSegments);
+    });
+
+    test("Typical weighted coverage", () =>
+        testWeightedCoverage(
+            [
+                [0, 4, 1],
+                [1, 3, 2],
+                [2, 6, 3],
+                [8, 10, -1],
+            ],
+            [
+                [0, 1, 1],
+                [1, 2, 3],
+                [2, 3, 6],
+                [3, 4, 4],
+                [4, 6, 3],
+                [8, 10, -1],
+            ]
+        ));
+
+    test("Multiple weights at a single locus", () =>
+        testWeightedCoverage(
+            [
+                // -- Locus 1
+                [1, 2, 1],
+                [1, 2, 2],
+                [1, 2, 3],
+                [1, 2, 4],
+                [1, 2, 5],
+                // -- Locus 2
+                [2, 3, 2],
+                [2, 3, 3],
+                [2, 3, 4],
+                [2, 3, 5],
+                [2, 3, 6],
+            ],
+            [
+                // -- Locus 1
+                [1, 2, 15],
+                // -- Locus 2
+                [2, 3, 20],
+            ]
+        ));
 
-    expect(transform(coverageConfig, reads)).toEqual(coverageSegments);
+    test("Adjacent segments with different weights produce separated segments", () =>
+        testWeightedCoverage(
+            [
+                // -- Cluster 1
+                [1, 2, 2],
+                [2, 3, 1],
+                [3, 4, 1],
+                // -- Cluster 2
+                [5, 6, 1],
+                [6, 7, 2],
+                [7, 8, 1],
+                // -- Cluster 3
+                [9, 10, 1],
+                [10, 11, 1],
+                [11, 12, 2],
+            ],
+            [
+                // -- Cluster 1
+                [1, 2, 2],
+                [2, 4, 1],
+                // -- Cluster 2
+                [5, 6, 1],
+                [6, 7, 2],
+                [7, 8, 1],
+                // -- Cluster 3
+                [9, 11, 1],
+                [11, 12, 2],
+            ]
+        ));
 });

package/dist/src/utils/kWayMerge.d.ts

@@ -1,9 +1,10 @@
 /**
- * Returns an iterator that merges multiple sorted arrays.
+ * Merges multiple sorted arrays.
  *
  * @param {T[][]} arrays
+ * @param {function(T):void} handler a function that will be called for each element
  * @param {function(T):number} [accessor]
  * @template T
  */
-export default function kWayMerge<T>(arrays: T[][], accessor?: (arg0: T) => number): Generator<T, void, unknown>;
+export default function kWayMerge<T>(arrays: T[][], handler: (arg0: T) => void, accessor?: (arg0: T) => number): void;
 //# sourceMappingURL=kWayMerge.d.ts.map

package/dist/src/utils/kWayMerge.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"kWayMerge.d.ts","sourceRoot":"","sources":["../../../src/utils/kWayMerge.js"],"names":[],"mappings":"AAEA;;;;;;GAMG;AACH,4EAHuB,MAAM,+BAmC5B"}
+{"version":3,"file":"kWayMerge.d.ts","sourceRoot":"","sources":["../../../src/utils/kWayMerge.js"],"names":[],"mappings":"AAEA;;;;;;;GAOG;AACH,0EAJuB,IAAI,0BACJ,MAAM,QAmC5B"}

package/dist/src/utils/kWayMerge.js

@@ -1,13 +1,14 @@
 import FlatQueue from "flatqueue";
 
 /**
- * Returns an iterator that merges multiple sorted arrays.
+ * Merges multiple sorted arrays.
  *
  * @param {T[][]} arrays
+ * @param {function(T):void} handler a function that will be called for each element
  * @param {function(T):number} [accessor]
  * @template T
  */
-export default function* kWayMerge(arrays, accessor = (x) => +x) {
+export default function kWayMerge(arrays, handler, accessor = (x) => +x) {
     // https://www.wikiwand.com/en/K-way_merge_algorithm
 
     // This could be optimized by implementing a tournament tree or
@@ -31,7 +32,7 @@ export default function* kWayMerge(arrays, accessor = (x) => +x) {
         let pointer = pointers[i];
         const element = array[pointer++];
 
-        yield element;
+        handler(element);
 
         if (pointer < array.length) {
             const newValue = accessor(array[pointer]);

package/dist/src/utils/kWayMerge.test.js

@@ -22,5 +22,9 @@ test("k-way merge merges multiple sorted arrays", () => {
     /** @type {function(any):number} */
     const accessor = (d) => d.a;
 
-    expect([...kWayMerge(arrays, accessor)]).toEqual(sorted);
+    /** @type {{a: number}[]} */
+    const result = [];
+    kWayMerge(arrays, (d) => result.push(d), accessor);
+
+    expect(result).toEqual(sorted);
 });

package/package.json

@@ -7,7 +7,7 @@
   },
   "contributors": [],
   "license": "MIT",
-  "version": "0.43.1",
+  "version": "0.43.3",
   "jsdelivr": "dist/bundle/index.js",
   "unpkg": "dist/bundle/index.js",
   "browser": "dist/bundle/index.js",
@@ -65,5 +65,5 @@
     "vega-scale": "^7.1.1",
     "vega-util": "^1.16.0"
   },
-  "gitHead": "e47791753d5467ab83b5ef073976eb2f2484000b"
+  "gitHead": "0c6e0418a7461b40da98896bfaf5e05732384e85"
 }