@jbrowse/core 5.0.0-beta.8 → 5.0.0-beta.9

package/esm/util/markEncoding.js

@@ -1,46 +1,29 @@
+import { makeScoreNormalizer, scaleTypeCode, } from '@jbrowse/render-core/scoreScale';
 import { GLYPH_DISC } from '@jbrowse/render-core/shaders/pointMarkConsts';
-import { categoricalPalette, categoricalValueColor } from '../ui/colors.js';
-import { cssColorToABGR, cssColorToRgba, packAbgr } from './colorBits.js';
-import { VIRIDIS_STOPS, buildColorRampLut } from './colorRamp.js';
+import { categoricalScale } from '../ui/colors.js';
+import { categoricalField } from './categoricalField.js';
+import { MISCONFIGURED_COLOR } from './color/index.js';
+import { cssColorToABGR, packAbgr } from './colorBits.js';
+import { buildColorRampLut, colorRampStops, rampDomain } from './colorRamp.js';
+import { fieldReader } from './fieldReader.js';
 import Flatbush from './flatbush/index.js';
 import { GLYPH_CODES, GLYPH_NAMES } from './glyphNames.js';
 import { isJexl, stringToJexlExpression } from './jexlStrings.js';
+import { numericValue } from './numericValue.js';
 import { buildJexlContext } from './simpleFeature.js';
+import { thresholdCuts, thresholdIndex, thresholdLabels, thresholdPalette, } from './thresholdScale.js';
+export { COLOR_SCHEMES } from './colorSchemes.js';
+export { NO_VALUE_LABEL } from './categoricalField.js';
 export const DEFAULT_MARK_COLOR = '#0068d1';
-// What a feature paints when its colour field is missing or a `jexl:` colour
-// yields a non-string: visible, so a misconfiguration surfaces rather than
-// vanishing.
-const FALLBACK_COLOR = cssColorToABGR('#808080');
-/**
- * #api
- * The key row a feature with nothing in a categorical field lands on, so the
- * legend says why a mark is grey, or a disc, rather than listing a blank
- * value.
- */
-export const NO_VALUE_LABEL = '(no value)';
+const FALLBACK_COLOR = cssColorToABGR(MISCONFIGURED_COLOR);
 function jexlExpression(ref, jexl) {
     if (!jexl) {
         throw new Error(`a jexl: channel needs a jexl instance (${ref})`);
     }
     return stringToJexlExpression(ref, jexl);
 }
-function fieldReader(ref, jexl) {
-    if (typeof ref === 'function') {
-        return ref;
-    }
-    if (isJexl(ref)) {
-        const expr = jexlExpression(ref, jexl);
-        return feature => expr.eval(buildJexlContext({ feature }));
-    }
-    return feature => feature.get(ref);
-}
-/**
- * #api
- * The field a `y` declaration reads, whichever of its two forms it is
- * written in.
- */
-export function valueField(y) {
-    return typeof y === 'string' ? y : y.field;
+function channelReader(ref, jexl) {
+    return typeof ref === 'function' ? ref : fieldReader(ref, jexl);
 }
 function isGlyphName(glyph) {
     return glyph in GLYPH_CODES;
@@ -62,90 +45,55 @@ function glyphReader(glyph, jexl) {
         return typeof v === 'string' && isGlyphName(v) ? GLYPH_CODES[v] : GLYPH_DISC;
     };
 }
-function rampStops(ramp) {
-    if (ramp === undefined || ramp === 'viridis') {
-        return VIRIDIS_STOPS;
-    }
-    return ramp.map(c => cssColorToRgba(c));
-}
 function lutColorAt(lut, t) {
     const entries = lut.length / 4;
     const i = Math.min(entries - 1, Math.max(0, Math.round(t * (entries - 1))));
     const o = i * 4;
     return packAbgr(lut[o], lut[o + 1], lut[o + 2], lut[o + 3]);
 }
-// The `[0, 1]` fraction a value sits at in the domain: the scale half of the
-// y-channel split (ADR-097), spelled once here for colour. The log arm floors
-// at the domain's own minimum so a domain below 1 still spreads.
-function normalizer(scale, domain) {
-    const [min, max] = domain;
-    if (scale === 'log') {
-        const floorV = min > 0 ? min : 1;
-        const logMin = Math.log2(floorV);
-        const logRange = Math.log2(Math.max(max, floorV)) - logMin;
-        return (v) => logRange <= 0
-            ? 0
-            : Math.min(1, Math.max(0, (Math.log2(Math.max(v, floorV)) - logMin) / logRange));
-    }
-    const range = max - min;
-    return (v) => range <= 0 ? 0 : Math.min(1, Math.max(0, (v - min) / range));
-}
-function categoryOrder(seen, domain) {
-    const listed = (domain ?? []).map(String);
-    const rest = [...seen.keys()].filter(k => !listed.includes(k));
-    const numeric = rest.every(k => k !== '' && Number.isFinite(Number(k)));
-    rest.sort(numeric ? (a, b) => Number(a) - Number(b) : (a, b) => a.localeCompare(b));
-    return [...listed, ...rest];
-}
 // The categorical arm `color` and `glyph` share: the walk records which
-// distinct value each admitted instance carried, and `resolve` hands every
-// value its range entry once the region's table is known. A listed domain
-// is the author's order and walks the range; without one the entry derives
-// from the value itself, so two regions that met different value sets still
-// agree on every value they share.
-function categoricalChannel(read, n) {
-    const categories = new Map();
-    const indexOf = new Int32Array(n);
+// key each admitted instance carried, `''` for none, and `resolve` hands
+// every key met its entry in the field's order, so two regions that met
+// different key sets still agree on every key they share.
+function categoricalChannel(read, categories, n) {
+    const keys = new Map();
+    const indexOf = new Uint32Array(n);
     return {
         indexOf,
         collect(f, at) {
-            const raw = read(f);
-            if (raw === undefined || raw === null || raw === '') {
-                indexOf[at] = -1;
-                return;
-            }
-            const label = String(raw);
-            let index = categories.get(label);
+            const key = categories.key(read(f));
+            let index = keys.get(key);
             if (index === undefined) {
-                index = categories.size;
-                categories.set(label, index);
+                index = keys.size;
+                keys.set(key, index);
             }
             indexOf[at] = index;
         },
-        resolve(domain, range) {
-            const ofIndex = Array.from({ length: categories.size });
+        resolve(entryOf) {
+            const ofIndex = Array.from({ length: keys.size });
             const entries = [];
-            categoryOrder(categories, domain).forEach((label, slot) => {
-                const value = domain
-                    ? range[slot % range.length]
-                    : categoricalValueColor(label, range);
-                const index = categories.get(label);
-                if (index !== undefined) {
-                    ofIndex[index] = value;
-                }
-                entries.push({ label, value });
-            });
+            for (const value of [...keys.keys()].sort(categories.compare)) {
+                const entry = entryOf(value);
+                ofIndex[keys.get(value)] = entry;
+                entries.push({ value, entry });
+            }
             return { ofIndex, entries };
         },
     };
 }
-function hasMissing(indexOf, count) {
+/**
+ * #api
+ * The hit index over `count` instances: each a box from `x` to `x2` at its
+ * `y`, or at 0 for a mark with no value.
+ */
+export function hitIndexOf(x, x2, y, count = x.length) {
+    const fb = new Flatbush(count, undefined, Float64Array);
     for (let i = 0; i < count; i++) {
-        if (indexOf[i] < 0) {
-            return true;
-        }
+        const v = y ? y[i] : 0;
+        fb.add(x[i], v, x2[i], v);
     }
-    return false;
+    fb.finish();
+    return fb;
 }
 /**
  * #api
@@ -162,14 +110,16 @@ export function encodeFeatures(features, encoding, lanes, ctx = {}) {
     const { jexl, report } = ctx;
     const n = features.length;
     const has = (lane) => lanes.includes(lane);
-    const readX = fieldReader(encoding.x ?? 'start', jexl);
-    const readX2 = fieldReader(encoding.x2 ?? 'end', jexl);
+    const readX = channelReader(encoding.x ?? 'start', jexl);
+    const readX2 = channelReader(encoding.x2 ?? 'end', jexl);
     const { y: yEncoding } = encoding;
     const readY = has('y') && yEncoding !== undefined
-        ? fieldReader(typeof yEncoding === 'object' ? valueField(yEncoding) : yEncoding, jexl)
+        ? channelReader(yEncoding, jexl)
         : undefined;
-    const readRow = has('row') && encoding.row !== undefined
-        ? fieldReader(encoding.row, jexl)
+    const { row: rowEncoding } = encoding;
+    const rowValues = has('row') && typeof rowEncoding === 'object' ? rowEncoding : undefined;
+    const readRow = has('row') && rowEncoding !== undefined && typeof rowEncoding !== 'object'
+        ? channelReader(rowEncoding, jexl)
         : undefined;
     const x = new Uint32Array(n);
     const x2 = new Uint32Array(n);
@@ -180,9 +130,11 @@ export function encodeFeatures(features, encoding, lanes, ctx = {}) {
     let yMin = Infinity;
     let yMax = -Infinity;
     let count = 0;
+    let skippedPosition = 0;
     const colorEncoding = encoding.color ?? DEFAULT_MARK_COLOR;
     const declaredScale = typeof colorEncoding === 'object' ? colorEncoding : undefined;
-    const rampEncoding = declaredScale && declaredScale.scale !== 'categorical'
+    const rampEncoding = declaredScale &&
+        (declaredScale.scale === 'linear' || declaredScale.scale === 'log')
         ? declaredScale
         : undefined;
     // Which side of the wire a ramp resolves on is the caller's lane choice: a
@@ -199,19 +151,35 @@ export function encodeFeatures(features, encoding, lanes, ctx = {}) {
             ? colorEncoding
             : scaled === undefined
                 ? colorEvaluator(colorEncoding, jexl)
-                : fieldReader(scaled.field, jexl);
+                : channelReader(scaled.field, jexl);
     // A scaled channel resolves after the walk, once the table is known: the
     // category per admitted instance, or a ramp's raw value, kept here.
-    const colorCategories = scaled?.scale === 'categorical' && readColor
-        ? categoricalChannel(readColor, n)
+    const colorField = scaled?.scale === 'categorical'
+        ? categoricalField(scaled.field, {
+            domain: scaled.domain?.map(String),
+            range: scaled.range,
+        })
+        : undefined;
+    const colorCategories = colorField && readColor
+        ? categoricalChannel(readColor, colorField, n)
         : undefined;
-    const rampValues = scaled && scaled.scale !== 'categorical'
-        ? (colorValue ?? new Float32Array(n))
+    const rampValues = scaled && rampEncoding ? (colorValue ?? new Float32Array(n)) : undefined;
+    const thresholdEncoding = scaled?.scale === 'threshold' ? scaled : undefined;
+    const cuts = thresholdEncoding
+        ? thresholdCuts(thresholdEncoding.domain ?? [])
+        : undefined;
+    const binColors = thresholdEncoding && cuts
+        ? Uint32Array.from(thresholdPalette(cuts.length + 1, thresholdEncoding.range), c => cssColorToABGR(c))
         : undefined;
     const { glyph: glyphEncoding } = encoding;
     const glyphScaled = glyph && typeof glyphEncoding === 'object' ? glyphEncoding : undefined;
-    const glyphCategories = glyphScaled
-        ? categoricalChannel(fieldReader(glyphScaled.field, jexl), n)
+    const glyphField = glyphScaled
+        ? categoricalField(glyphScaled.field, {
+            domain: glyphScaled.domain?.map(String),
+        })
+        : undefined;
+    const glyphCategories = glyphScaled && glyphField
+        ? categoricalChannel(channelReader(glyphScaled.field, jexl), glyphField, n)
         : undefined;
     const readGlyph = glyph
         ? glyphReader(typeof glyphEncoding === 'object' ? undefined : glyphEncoding, jexl)
@@ -219,10 +187,13 @@ export function encodeFeatures(features, encoding, lanes, ctx = {}) {
     for (let i = 0; i < n; i++) {
         report?.(i);
         const f = features[i];
-        const xv = Number(readX(f));
-        const x2v = Number(readX2(f));
-        const yv = readY ? Number(readY(f)) : 0;
+        const xv = numericValue(readX(f));
+        const x2v = numericValue(readX2(f));
+        const yv = readY ? numericValue(readY(f)) : 0;
         if (!Number.isFinite(xv) || !Number.isFinite(x2v) || !Number.isFinite(yv)) {
+            if (!Number.isFinite(xv) || !Number.isFinite(x2v)) {
+                skippedPosition++;
+            }
             continue;
         }
         x[count] = xv;
@@ -242,6 +213,10 @@ export function encodeFeatures(features, encoding, lanes, ctx = {}) {
             const rv = Number(readRow(f));
             row[count] = rv > 0 ? rv : 0;
         }
+        else if (row && rowValues) {
+            const rv = rowValues[i];
+            row[count] = rv > 0 ? rv : 0;
+        }
         if (glyphCategories) {
             glyphCategories.collect(f, count);
         }
@@ -253,7 +228,11 @@ export function encodeFeatures(features, encoding, lanes, ctx = {}) {
             colorCategories.collect(f, count);
         }
         else if (rampValues && readColor) {
-            rampValues[count] = Number(readColor(f));
+            rampValues[count] = numericValue(readColor(f));
+        }
+        else if (binColors && cuts && color && readColor) {
+            const bin = thresholdIndex(readColor(f), cuts);
+            color[count] = bin < 0 ? FALLBACK_COLOR : binColors[bin];
         }
         else if (color && readColor) {
             color[count] = readColor(f);
@@ -261,31 +240,43 @@ export function encodeFeatures(features, encoding, lanes, ctx = {}) {
         count++;
     }
     let scale;
-    if (scaled?.scale === 'categorical' && colorCategories && color) {
-        const palette = (scaled.palette ?? categoricalPalette).map(cssColorToABGR);
-        const { ofIndex, entries } = colorCategories.resolve(scaled.domain, palette);
+    if (scaled?.scale === 'categorical' &&
+        colorField &&
+        colorCategories &&
+        color) {
+        const { ofIndex, entries } = colorCategories.resolve(key => cssColorToABGR(colorField.color(key)));
         const { indexOf } = colorCategories;
         for (let i = 0; i < count; i++) {
-            const index = indexOf[i];
-            color[i] = index < 0 ? FALLBACK_COLOR : ofIndex[index];
+            color[i] = ofIndex[indexOf[i]];
         }
         scale = {
             kind: 'categorical',
             field: scaled.field,
-            entries: [
-                ...entries.map(e => ({ label: e.label, color: e.value })),
-                ...(hasMissing(indexOf, count)
-                    ? [{ label: NO_VALUE_LABEL, color: FALLBACK_COLOR }]
-                    : []),
-            ],
+            domain: [...colorField.domain],
+            ...(scaled.range ? { range: [...scaled.range] } : {}),
+            ...(keysAreNumeric(entries) ? { numericKeys: true } : {}),
+            entries: entries.map(e => ({ value: e.value, color: e.entry })),
         };
     }
-    else if (scaled && scaled.scale !== 'categorical' && rampValues) {
+    else if (thresholdEncoding && cuts && binColors) {
+        scale = {
+            kind: 'threshold',
+            field: thresholdEncoding.field,
+            domain: cuts,
+            entries: thresholdLabels(cuts).map((value, i) => ({
+                value,
+                color: binColors[i],
+            })),
+        };
+    }
+    else if (rampEncoding && rampValues) {
         const extent = finiteExtremes(rampValues, count);
-        const domain = scaled.domain ?? extent;
-        const lut = buildColorRampLut(rampStops(scaled.ramp));
+        const { domainMin, domainMax, domainMid } = rampEncoding;
+        const domain = rampDomain(domainMin, domainMax, extent);
+        const norm = makeScoreNormalizer(domain[0], domain[1], scaleTypeCode(rampEncoding.scale), 1);
+        const stops = colorRampStops(rampEncoding);
+        const lut = rampLut(stops, rampEncoding.scale, domain, domainMid);
         if (color) {
-            const norm = normalizer(scaled.scale, domain);
             for (let i = 0; i < count; i++) {
                 const v = rampValues[i];
                 color[i] = Number.isFinite(v)
@@ -295,47 +286,37 @@ export function encodeFeatures(features, encoding, lanes, ctx = {}) {
         }
         scale = {
             kind: 'ramp',
-            field: scaled.field,
-            scale: scaled.scale,
+            field: rampEncoding.field,
+            scale: rampEncoding.scale,
             domain,
-            pinned: scaled.domain !== undefined,
+            pinned: [domainMin !== undefined, domainMax !== undefined],
+            ...(domainMid === undefined ? {} : { domainMid, stops }),
             extent,
             lut,
         };
     }
     let glyphScale;
-    if (glyphScaled && glyphCategories && glyph) {
-        const { ofIndex, entries } = glyphCategories.resolve(glyphScaled.domain, glyphScaled.range ?? GLYPH_NAMES);
+    if (glyphScaled && glyphField && glyphCategories && glyph) {
+        const glyphOf = categoricalScale(glyphField.domain, glyphScaled.range ?? GLYPH_NAMES);
+        const { ofIndex, entries } = glyphCategories.resolve((key) => key === '' ? 'disc' : glyphOf(key));
         const codeOfIndex = Uint8Array.from(ofIndex, name => GLYPH_CODES[name]);
         const { indexOf } = glyphCategories;
         for (let i = 0; i < count; i++) {
-            const index = indexOf[i];
-            glyph[i] = index < 0 ? GLYPH_DISC : codeOfIndex[index];
+            glyph[i] = codeOfIndex[indexOf[i]];
         }
         glyphScale = {
             kind: 'glyph',
             field: glyphScaled.field,
-            entries: [
-                ...entries.map(e => ({ label: e.label, glyph: e.value })),
-                ...(hasMissing(indexOf, count)
-                    ? [{ label: NO_VALUE_LABEL, glyph: 'disc' }]
-                    : []),
-            ],
+            domain: [...glyphField.domain],
+            ...(glyphScaled.range ? { range: [...glyphScaled.range] } : {}),
+            entries: entries.map(e => ({ value: e.value, glyph: e.entry })),
         };
     }
-    let flatbushData;
-    if (has('index') && count > 0) {
-        const fb = new Flatbush(count, undefined, Float64Array);
-        for (let i = 0; i < count; i++) {
-            const v = y ? y[i] : 0;
-            fb.add(x[i], v, x2[i], v);
-        }
-        fb.finish();
-        flatbushData = fb.data;
-    }
+    const flatbushData = has('index') && count > 0 ? hitIndexOf(x, x2, y, count).data : undefined;
     const encoded = {
         count,
         skipped: n - count,
+        skippedPosition,
         x: x.subarray(0, count),
         x2: x2.subarray(0, count),
         featureIndex: featureIndex.subarray(0, count),
@@ -368,6 +349,52 @@ export function encodeFeatures(features, encoding, lanes, ctx = {}) {
     }
     return encoded;
 }
+function rampMid(scale, domain, domainMid) {
+    return domainMid === undefined
+        ? 0.5
+        : makeScoreNormalizer(domain[0], domain[1], scaleTypeCode(scale), 1)(domainMid);
+}
+function rampLut(stops, scale, domain, domainMid) {
+    return buildColorRampLut(stops, rampMid(scale, domain, domainMid));
+}
+const MAX_BAKED_RAMPS = 16;
+const bakedRamps = new Map();
+/**
+ * #api
+ * A ramp table over `extent`, the union a display took across the regions it
+ * loaded: each open end of the domain moved to the union's, the pinned ends
+ * kept, and the table baked again where a `domainMid` places its middle stop
+ * by that domain. Each region baked its own, so keeping the first region's
+ * put the middle colour at a value none of them declared.
+ *
+ * One table per stop list and middle position, so a display asking again over
+ * an extent that has not moved gets the bytes it already uploaded: a backend
+ * re-uploads a ramp on identity.
+ */
+export function rampOverExtent(table, extent) {
+    const { stops, pinned } = table;
+    const domain = rampDomain(pinned[0] ? table.domain[0] : undefined, pinned[1] ? table.domain[1] : undefined, extent);
+    if (!stops) {
+        return { ...table, extent, domain };
+    }
+    const mid = rampMid(table.scale, domain, table.domainMid);
+    const key = `${mid}|${stops.join(';')}`;
+    let lut = bakedRamps.get(key);
+    if (!lut) {
+        if (bakedRamps.size >= MAX_BAKED_RAMPS) {
+            bakedRamps.delete(bakedRamps.keys().next().value);
+        }
+        lut = buildColorRampLut(stops, mid);
+        bakedRamps.set(key, lut);
+    }
+    return { ...table, extent, domain, lut };
+}
+function keysAreNumeric(entries) {
+    const named = entries.filter(e => e.value !== '');
+    return named.length > 0 && named.every(e => Number.isFinite(Number(e.value)));
+}
+// `[Infinity, -Infinity]` where no value is finite, which a union of regions'
+// extents passes over.
 function finiteExtremes(values, count) {
     let min = Infinity;
     let max = -Infinity;
@@ -382,7 +409,7 @@ function finiteExtremes(values, count) {
             }
         }
     }
-    return min === Infinity ? [0, 1] : [min, max];
+    return [min, max];
 }
 /**
  * #api