@mhkeller/vgplot-dot-gl 1.0.0

package/src/prepare.js

@@ -0,0 +1,300 @@
+/**
+ * One pass over a query result. Works out everything the drawing needs that
+ * doesn't depend on the plot's scales:
+ *
+ * - which rows can be drawn (x, y and radius are numbers or known category codes, the fill isn't missing),
+ * - the lowest and highest values, which Plot uses to set up the scales,
+ * - which categories occur, so axes and legends list only those, as they do for Plot's own dot,
+ * - a small integer per row for the fill color,
+ * - the draw order (biggest dots first, like Plot's dot mark).
+ *
+ * The lowest and highest values come from every usable value in a column, not
+ * only from drawable rows, because that is how Plot sets a scale from a column.
+ *
+ * Columns from the database arrive already turned into numbers: doubles with NaN
+ * for null, epoch milliseconds for dates (the `dates` flags say which), and
+ * category codes for text and boolean columns (the `*Cats` lists say which). The
+ * lists come from the whole table, so a filtered result can use only some of them.
+ * Array data arrives as it was given, and dates are then Date objects. Everything
+ * here takes both.
+ */
+
+/** Sort the same way Plot sorts a list of categories. */
+const ascending = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
+
+const BINS = 1024;
+
+/** True when a column holds Date objects. The hints then stay Dates, so Plot picks a time scale. */
+function holdsDates(column) {
+  for (let i = 0; i < column.length; ++i) {
+    const v = column[i];
+    if (v != null) return v instanceof Date;
+  }
+  return false;
+}
+
+/**
+ * The categories that occur in a column of category codes, in list order, and for each code its row in
+ * that shorter list (-1 when it doesn't occur). Plot works a scale's domain out from every value of a
+ * column, drawn or not, so every row counts. The loop stops once every category has turned up, and then
+ * the full list comes back with no rows.
+ */
+function occurring(codes, cats) {
+  const n = cats.length;
+  const seen = new Uint8Array(n);
+  let left = n;
+  for (let i = 0; i < codes.length && left > 0; ++i) {
+    const c = codes[i];
+    if (c < n && !seen[c]) {
+      seen[c] = 1;
+      --left;
+    }
+  }
+  if (left === 0) return { list: cats, rows: null };
+  const rows = new Int32Array(n).fill(-1);
+  const list = [];
+  for (let c = 0; c < n; ++c) if (seen[c]) rows[c] = list.push(cats[c]) - 1;
+  return { list, rows };
+}
+
+/** Number columns are split into this many colors. Codes 254 and 255 are not colors: 255 marks a hidden dot. */
+export const CONTINUOUS_LEVELS = 254;
+
+/**
+ * @param {object} input
+ * @param {ArrayLike<any>} input.x
+ * @param {ArrayLike<any>} input.y
+ * @param {ArrayLike<any>} [input.r]         radius column, or leave out for a fixed radius
+ * @param {ArrayLike<any>} [input.fill]      fill column: category codes with `fillCats`, numbers with `continuous`,
+ *                                           otherwise values that are grouped into categories here
+ * @param {any[]} [input.xCats]              x is codes into this category list (sorted the way Plot sorts, null last)
+ * @param {any[]} [input.yCats]              y is codes into this category list
+ * @param {any[]} [input.fillCats]           fill is codes into this category list
+ * @param {boolean} [input.continuous]       treat `fill` as numbers and split them into CONTINUOUS_LEVELS steps
+ * @param {{x?: string|boolean, y?: string|boolean, r?: string|boolean, fill?: string|boolean}} [input.dates]
+ *                                           these columns hold epoch milliseconds that stand for dates. The value is
+ *                                           the SQL type ('DATE', 'TIMESTAMP', 'TIME', ...) when it is known, so the
+ *                                           tooltip can format each kind its own way, or just true when it isn't.
+ * @param {'-r'|null} [input.sort]           draw order: '-r' draws big dots first
+ * @param {number} [input.maxCategories]     most categories grouped here from plain values (254 at most, one byte)
+ * @param {boolean} [input.wantP25]          also pass along the 25th percentile of the radii, so Plot's
+ *                                           default dot size range comes out the same as with all rows
+ */
+export function prepare({ x, y, r = null, fill = null, xCats = null, yCats = null, fillCats = null, continuous = false, dates = {}, sort = '-r', maxCategories = 254, wantP25 = false }) {
+  const total = x.length;
+  const valid = new Uint8Array(total);
+  const factorize = !!fill && !fillCats && !continuous;
+  const temp = factorize ? new Uint16Array(total) : null;
+  const seen = factorize ? new Map() : null;
+  let fmin = Infinity, fmax = -Infinity;
+  // Array data has no SQL type, so a column of Date objects counts as dates with no kind.
+  const xDates = dates.x || holdsDates(x);
+  const yDates = dates.y || holdsDates(y);
+  const rDates = r ? dates.r || holdsDates(r) : false;
+  maxCategories = Math.min(254, maxCategories);
+
+  let count = 0;
+  let xmin = Infinity, xmax = -Infinity, xpos = Infinity;
+  let ymin = Infinity, ymax = -Infinity, ypos = Infinity;
+  let rmin = Infinity, rmax = -Infinity;
+
+  for (let i = 0; i < total; ++i) {
+    let xok, yok;
+    if (xCats) {
+      xok = x[i] < xCats.length;
+    } else {
+      const xv = x[i];
+      const xn = xv == null ? NaN : +xv;
+      xok = Number.isFinite(xn);
+      if (xok) {
+        if (xn < xmin) xmin = xn;
+        if (xn > xmax) xmax = xn;
+        if (xn > 0 && xn < xpos) xpos = xn;
+      }
+    }
+    if (yCats) {
+      yok = y[i] < yCats.length;
+    } else {
+      const yv = y[i];
+      const yn = yv == null ? NaN : +yv;
+      yok = Number.isFinite(yn);
+      if (yok) {
+        if (yn < ymin) ymin = yn;
+        if (yn > ymax) ymax = yn;
+        if (yn > 0 && yn < ypos) ypos = yn;
+      }
+    }
+    let rok = true;
+    if (r) {
+      const rv = r[i];
+      const rn = rv == null ? NaN : +rv;
+      rok = Number.isFinite(rn);
+      if (rok) {
+        if (rn < rmin) rmin = rn;
+        if (rn > rmax) rmax = rn;
+      }
+    }
+    // The fill's range and categories count rows that aren't drawn too, as the other columns' ranges do.
+    let fok = true;
+    if (fillCats) {
+      fok = fill[i] < fillCats.length;
+    } else if (continuous) {
+      const fv = fill[i];
+      const fn = fv == null ? NaN : +fv;
+      fok = Number.isFinite(fn);
+      if (fok) {
+        if (fn < fmin) fmin = fn;
+        if (fn > fmax) fmax = fn;
+      }
+    } else if (fill) {
+      const fv = fill[i];
+      fok = fv != null;
+      if (fok) {
+        let code = seen.get(fv);
+        if (code === undefined) {
+          code = seen.size;
+          if (code >= maxCategories) {
+            throw new Error(`dotGL: the fill column has more than ${maxCategories} distinct values`);
+          }
+          seen.set(fv, code);
+        }
+        temp[i] = code;
+      }
+    }
+    if (!xok || !yok || !rok || !fok) continue;
+    valid[i] = 1;
+    ++count;
+  }
+
+  // Each category's number is its place in the sorted list (the table's whole list when `fillCats` is given).
+  // When every category is in the result, number i is also hint row i; otherwise `fillRows` gives each number's row.
+  // Number columns get a step number over their range instead.
+  // Up to 254 colors fit one byte with 255 as the hidden code; more take two bytes with 65535 as the hidden code.
+  const cats = fillCats ? fillCats : seen ? Array.from(seen.keys()).sort(ascending) : [];
+  const levels = continuous ? CONTINUOUS_LEVELS : cats.length;
+  const hidden = levels > 254 ? 65535 : 255;
+  const codes = (hidden === 255 ? new Uint8Array(total) : new Uint16Array(total)).fill(hidden);
+  if (fillCats) {
+    for (let i = 0; i < total; ++i) if (valid[i]) codes[i] = fill[i];
+  } else if (continuous) {
+    const span = fmax - fmin;
+    for (let i = 0; i < total; ++i) {
+      if (!valid[i]) continue;
+      codes[i] = span > 0 ? Math.min(CONTINUOUS_LEVELS - 1, Math.floor(((+fill[i] - fmin) / span) * CONTINUOUS_LEVELS)) : 0;
+    }
+  } else if (fill) {
+    const remap = new Uint8Array(seen.size);
+    cats.forEach((c, i) => { remap[seen.get(c)] = i; });
+    for (let i = 0; i < total; ++i) if (valid[i]) codes[i] = remap[temp[i]];
+  } else {
+    for (let i = 0; i < total; ++i) if (valid[i]) codes[i] = 0;
+  }
+
+  // Axes and legends list only the categories in this result. Codes keep pointing into the full lists.
+  const xShown = xCats ? occurring(x, xCats).list : null;
+  const yShown = yCats ? occurring(y, yCats).list : null;
+  const fillShown = fillCats ? occurring(fill, fillCats) : { list: cats, rows: null };
+
+  // Radius histogram. It gives the draw order (a stable counting sort, biggest
+  // first) and the 25th percentile of the positive radii, which Plot's default
+  // dot size range depends on.
+  const perm = new Uint32Array(count);
+  let p25 = rmin;
+  const histogram = r && rmax > rmin && (sort === '-r' || wantP25);
+  if (histogram) {
+    const scale = (BINS - 1) / (rmax - rmin);
+    const bin = new Uint16Array(total);
+    const counts = new Uint32Array(BINS + 1);
+    const positive = new Uint32Array(BINS);
+    let positives = 0;
+    for (let i = 0; i < total; ++i) {
+      const rv = r[i];
+      const rn = rv == null ? NaN : +rv;
+      if (!Number.isFinite(rn)) continue;
+      const b = Math.round((rmax - rn) * scale); // 0 = largest radius
+      if (rn > 0) { ++positive[b]; ++positives; }
+      if (!valid[i]) continue;
+      bin[i] = b;
+      ++counts[b + 1];
+    }
+    let seenSoFar = 0;
+    for (let b = BINS - 1; b >= 0; --b) {
+      seenSoFar += positive[b];
+      if (seenSoFar >= positives * 0.25) { p25 = Math.min(rmax, Math.max(rmin, rmax - b / scale)); break; }
+    }
+    if (sort === '-r') {
+      for (let b = 0; b < BINS; ++b) counts[b + 1] += counts[b];
+      for (let i = 0; i < total; ++i) if (valid[i]) perm[counts[bin[i]]++] = i;
+    }
+  }
+  if (!histogram || sort !== '-r') {
+    let k = 0;
+    for (let i = 0; i < total; ++i) if (valid[i]) perm[k++] = i;
+  }
+
+  // Hints: short arrays of length k (or empty) that make Plot set up the same
+  // scales it would from the full columns. For x and y the smallest positive value
+  // goes first and the true minimum last: Plot's log scale looks at the first
+  // non-zero value and then keeps only values with that sign, while a linear scale
+  // just takes the lowest and highest. A category axis or legend gets the
+  // categories in the result, in list order, which is the domain Plot works out
+  // from the text column itself. For r, the spare hint slots are all the
+  // 25th-percentile radius, because Plot reads that value to pick its default
+  // dot size range.
+  const needSlot = (!xCats && xmin <= 0) || (!yCats && ymin <= 0);
+  const k = Math.max(2, fillShown.list.length, xShown?.length ?? 0, yShown?.length ?? 0, wantP25 && r ? 8 : 0, needSlot ? 3 : 0);
+  const asDate = (flag, v) => (flag ? new Date(v) : v);
+  const position = (min, max, pos, isDate) => {
+    // No usable value at all: pass `undefined`, which Plot treats like an empty column.
+    if (!Number.isFinite(min)) return new Array(k).fill(undefined);
+    const out = new Array(k).fill(asDate(isDate, max));
+    out[0] = asDate(isDate, pos < Infinity ? pos : min);
+    if (k > 2) out[k - 1] = asDate(isDate, min);
+    return out;
+  };
+  // The list padded to length k by repeating its last entry. An empty list stays empty: Plot would add
+  // `undefined` to a category axis another mark shares.
+  const padded = list => (list.length ? Array.from({ length: k }, (_, i) => list[Math.min(i, list.length - 1)]) : []);
+  const radius = () => {
+    if (!Number.isFinite(rmin)) return new Array(k).fill(undefined);
+    const out = new Array(k).fill(wantP25 ? p25 : rmax);
+    out[0] = rmin;
+    out[1] = rmax;
+    return out;
+  };
+  const hints = {
+    x: xCats ? padded(xShown) : position(xmin, xmax, xpos, xDates),
+    y: yCats ? padded(yShown) : position(ymin, ymax, ypos, yDates),
+    r: r ? radius() : null,
+    fill: !fill ? null
+      : continuous ? position(fmin, fmax, Infinity, !!dates.fill)
+      : padded(fillShown.list)
+  };
+
+  return {
+    n: count,
+    total,
+    perm,
+    codes,
+    hidden,
+    cats,
+    // For each fill code, its row in the fill hints (-1 when the result doesn't have it); null when every row is its code.
+    fillRows: fillShown.rows,
+    xCats,
+    yCats,
+    continuous,
+    dates: { x: xDates, y: yDates, r: rDates, fill: dates.fill || false },
+    levels,
+    hints,
+    k,
+    p25,
+    extent: {
+      x: xCats ? null : [xmin, xmax],
+      y: yCats ? null : [ymin, ymax],
+      r: r ? [rmin, rmax] : null,
+      fill: continuous ? [fmin, fmax] : null,
+      xpos,
+      ypos
+    }
+  };
+}

package/src/scale-map.js

@@ -0,0 +1,129 @@
+/**
+ * Turns one of Observable Plot's scale objects (what `svg.scale('x')` returns,
+ * and what a render function gets as `scales.scales.x`) into two parts:
+ *
+ * 1. `transformFor(scale)`: the curved part `T` (log, square root, power,
+ *    symlog; nothing for linear and time scales). Applied once per value when
+ *    the data is sent to the graphics card.
+ * 2. `affine(scale, center, shift)`: the straight-line part, worked out again
+ *    for every frame: `px = a * (T(v) - center) + b`. Panning and zooming only
+ *    change `a` and `b`.
+ *
+ * Both depend only on the scale object, so the tests compare them with the d3
+ * scales Plot builds from the same domain and range.
+ *
+ * A category axis (a point or band scale) is uploaded as category codes.
+ * `categoryAxis` gives each code its place in the scale's domain and the straight
+ * line through those places, `axisTransform` looks the places up, and `axisAffine`
+ * picks between the two kinds of line.
+ */
+
+const identity = v => v;
+
+/** Power that keeps the sign, like d3's pow and sqrt scales do for negative numbers. */
+const power = e => v => (v < 0 ? -Math.pow(-v, e) : Math.pow(v, e));
+
+/** The curved part of a scale, or a clear error for scale types the mark can't draw. */
+export function transformFor(scale, name = 'position') {
+  switch (scale?.type) {
+    case undefined:
+    case 'identity':
+    case 'linear':
+    case 'time':
+    case 'utc':
+      return identity;
+    case 'point':
+    case 'band':
+      // The uploaded values are category codes; `categoryAxis` places them.
+      return identity;
+    case 'log':
+      // The base only multiplies log values by a constant, and `affine` divides it out.
+      return Math.log;
+    case 'sqrt':
+      return power(0.5);
+    case 'pow':
+      return power(scale.exponent ?? 1);
+    case 'symlog': {
+      const c = scale.constant ?? 1;
+      return v => (v < 0 ? -Math.log1p(-v / c) : Math.log1p(v / c));
+    }
+    default:
+      throw new Error(`dotGL: the ${name} scale type "${scale.type}" is not supported`);
+  }
+}
+
+/**
+ * The straight-line part. It takes a value that has been through `T` and had `center` subtracted, and gives the plot pixel.
+ * `shift` moves the origin (for example to the top-left of the clip frame).
+ */
+export function affine(scale, center = 0, shift = 0, name = 'position') {
+  if (!scale || scale.type === 'identity') return { a: 1, b: shift - center };
+  const { domain, range } = scale;
+  if (!domain || domain.length !== 2 || !range || range.length !== 2) {
+    throw new Error(`dotGL: the ${name} scale must have a two-value domain and range`);
+  }
+  const T = transformFor(scale, name);
+  const t0 = T(+domain[0]);
+  const t1 = T(+domain[1]);
+  const [r0, r1] = range;
+  // When both ends of the domain are equal, d3 puts every value in the middle of the range.
+  if (t1 === t0) return { a: 0, b: (r0 + r1) / 2 + shift };
+  const a = (r1 - r0) / (t1 - t0);
+  return { a, b: r0 + (center - t0) * a + shift };
+}
+
+/**
+ * Where a point or band scale puts each category of a column drawn as category codes. `pos[code]` is the
+ * category's place in the scale's domain, and place i is at pixel `a * i + b`, because these scales space
+ * their domain evenly. A band scale's dots sit in the middle of the band. The domain is usually the
+ * categories in the data, but it can list more (an explicit domain, `vg.Fixed`, or another mark on the
+ * same axis) or leave some out; a category that isn't in the domain gets NaN and its dots aren't drawn,
+ * as with Plot's dot. A number scale places no categories. Plot picks one when the only category left is
+ * the empty value, and Plot's dot then draws nothing too.
+ */
+export function categoryAxis(scale, cats) {
+  const pos = new Float64Array(cats.length).fill(NaN);
+  if (scale.type !== 'point' && scale.type !== 'band') return { a: 0, b: 0, pos };
+  // d3 keeps the first of repeated values in a domain, so places count distinct values.
+  const place = new Map();
+  for (const v of scale.domain) if (!place.has(v)) place.set(v, place.size);
+  for (let i = 0; i < cats.length; ++i) pos[i] = place.get(cats[i]) ?? NaN;
+  const values = Array.from(place.keys());
+  const n = values.length;
+  if (n === 0) return { a: 0, b: 0, pos };
+  const first = scale.apply(values[0]);
+  const a = n > 1 ? (scale.apply(values[n - 1]) - first) / (n - 1) : 0;
+  return { a, b: first + (scale.bandwidth ?? 0) / 2, pos };
+}
+
+/**
+ * The function the painters and the pick index run on an axis value before its line: the place lookup
+ * for a category axis, the scale's curve otherwise. Codes past the end of the list (the hidden code) give NaN.
+ */
+export function axisTransform(scale, line, name = 'position') {
+  if (!line) return transformFor(scale, name);
+  const { pos } = line;
+  return code => (code < pos.length ? pos[code] : NaN);
+}
+
+/** True when two category place tables are the same, or both absent. */
+export function samePlaces(a, b) {
+  if (a === b) return true;
+  if (!a || !b || a.length !== b.length) return false;
+  for (let i = 0; i < a.length; ++i) if (!Object.is(a[i], b[i])) return false;
+  return true;
+}
+
+/**
+ * The per-frame line for an x or y axis, in the same form as `affine`: from the category axis when
+ * the axis is drawn from category codes, otherwise from the scale.
+ */
+export function axisAffine(scale, line, center = 0, shift = 0, name = 'position') {
+  return line ? { a: line.a, b: line.b + line.a * center + shift } : affine(scale, center, shift, name);
+}
+
+/** The pixel for one data value, worked out the same way the shader does it. */
+export function project(scale, v, center = 0, shift = 0) {
+  const { a, b } = affine(scale, center, shift);
+  return a * (transformFor(scale)(+v) - center) + b;
+}

package/src/shaders.js

@@ -0,0 +1,66 @@
+/**
+ * Each dot is a small square made of two triangles, and the fragment shader cuts
+ * a circle out of it with a one-pixel soft edge. The vertex shader places the
+ * square in screen pixels from the per-dot values plus the per-frame scale
+ * numbers. Colors are written with the alpha already multiplied in, so
+ * overlapping dots add up the way see-through SVG circles do.
+ *
+ * Attribute slots: 0 corner (per vertex), 1 x, 2 y, 3 r, 4 color code (per dot).
+ * The color code picks a texel of the 256×256 palette: column code % 256, row
+ * code / 256. The code `u_hidden` (255 for one-byte codes, 65535 for two-byte
+ * codes) hides the dot, and so does a transparent palette entry (a category the
+ * color scale leaves out).
+ */
+
+export const VERTEX = `#version 300 es
+precision highp float;
+layout(location = 0) in vec2 a_corner;
+layout(location = 1) in float a_x;
+layout(location = 2) in float a_y;
+layout(location = 3) in float a_r;
+layout(location = 4) in float a_cat;
+
+uniform vec2 u_ax;         // px = a_x * u_ax.x + u_ax.y  (CSS pixels, measured from the frame's top-left)
+uniform vec2 u_ay;
+uniform vec2 u_res;        // canvas size in device pixels
+uniform float u_dpr;
+uniform float u_offset;    // Plot's half-pixel nudge on 1x displays
+uniform vec4 u_r;          // x: 0 = one radius for all, 1 = per dot; y: a, z: b; w: the shared radius (CSS pixels)
+uniform int u_colorMode;   // 0 = one color for all, 1 = look the color up in the palette
+uniform vec4 u_color;
+uniform sampler2D u_palette;
+uniform float u_hidden;    // the color code that hides a dot
+
+out vec2 v_p;
+out float v_rpx;
+out vec4 v_color;
+
+void main() {
+  float px = a_x * u_ax.x + u_ax.y + u_offset;
+  float py = a_y * u_ay.x + u_ay.y + u_offset;
+  float r = u_r.x < 0.5 ? u_r.w : (u_r.y * a_r + u_r.z);
+  v_rpx = r * u_dpr;
+  v_p = a_corner * (v_rpx + 1.0);
+  vec2 c = vec2(px, py) * u_dpr + v_p;
+  v_color = u_colorMode == 1 ? texelFetch(u_palette, ivec2(int(mod(a_cat, 256.0)), int(a_cat / 256.0)), 0) : u_color;
+  bool hide = r <= 0.0 || a_cat >= u_hidden - 0.5 || v_color.a <= 0.0;
+  gl_Position = hide
+    ? vec4(2.0, 2.0, 0.0, 1.0)
+    : vec4(c.x / u_res.x * 2.0 - 1.0, 1.0 - c.y / u_res.y * 2.0, 0.0, 1.0);
+}`;
+
+export const FRAGMENT = `#version 300 es
+precision mediump float;
+in vec2 v_p;
+in float v_rpx;
+in vec4 v_color;
+uniform float u_opacity;
+out vec4 o;
+
+void main() {
+  float d = length(v_p);
+  float a = (1.0 - smoothstep(v_rpx - 0.5, v_rpx + 0.5, d)) * u_opacity * v_color.a;
+  o = vec4(v_color.rgb * a, a);
+}`;
+
+export const UNIFORMS = ['u_ax', 'u_ay', 'u_res', 'u_dpr', 'u_offset', 'u_r', 'u_colorMode', 'u_color', 'u_palette', 'u_opacity', 'u_hidden'];