@uwdata/mosaic-plot 0.5.0 → 0.6.1

package/src/marks/RasterMark.js

@@ -1,8 +1,22 @@
+import { ascending } from 'd3';
 import { scale } from '@observablehq/plot';
+import { gridDomainContinuous, gridDomainDiscrete } from './util/grid.js';
 import { isColor } from './util/is-color.js';
-import { createCanvas, raster, opacityMap, palette } from './util/raster.js';
-import { Grid2DMark } from './Grid2DMark.js';
-
+import { alphaScheme, alphaConstant, colorConstant, colorCategory, colorScheme, createCanvas } from './util/raster.js';
+import { DENSITY, Grid2DMark } from './Grid2DMark.js';
+import { Fixed, Transient } from '../symbols.js';
+
+/**
+ * Raster image mark. Data is binned to a grid based on the x and y options.
+ * The grid cells are then colored to form an image.
+ * The raster grid size defaults to the pixel width/height of the
+ * plot. The pixelSize option (default 1) changes the grid cell to pixel
+ * ratio. For example, a pixelSize of 0.5 will create a larger raster
+ * for higher resolution images on retina displays. The width and height
+ * options set the grid dimensions directly, overriding other options.
+ * The raster grid can optionally be smoothed (blurred) by setting
+ * the bandwidth option.
+ */
 export class RasterMark extends Grid2DMark {
   constructor(source, options) {
     super('image', source, options);
@@ -19,23 +33,19 @@ export class RasterMark extends Grid2DMark {
   }
 
   rasterize() {
-    const { bins, kde, groupby } = this;
+    const { bins, kde } = this;
     const [ w, h ] = bins;
 
     // raster data
     const { canvas, ctx, img } = imageData(this, w, h);
 
-    // scale function to map densities to [0, 1]
-    const s = imageScale(this);
-
-    // gather color domain as needed
-    const idx = groupby.indexOf(this.channelField('fill')?.as);
-    const domain = idx < 0 ? [] : kde.map(({ key }) => key[idx]);
+    // color + opacity encodings
+    const { alpha, alphaProp, color, colorProp } = rasterEncoding(this);
 
-    // generate raster images
-    this.data = kde.map(grid => {
-      const palette = imagePalette(this, domain, grid.key?.[idx]);
-      raster(grid, img.data, w, h, s, palette);
+    // generate rasters
+    this.data = kde.map(cell => {
+      color?.(img.data, w, h, cell[colorProp]);
+      alpha?.(img.data, w, h, cell[alphaProp]);
       ctx.putImageData(img, 0, 0);
       return { src: canvas.toDataURL() };
     });
@@ -57,65 +67,179 @@ export class RasterMark extends Grid2DMark {
   }
 }
 
-function imageData(mark, w, h) {
-  if (!mark.image || mark.image.w !== w || mark.image.h !== h) {
-    const canvas = createCanvas(w, h);
-    const ctx = canvas.getContext('2d', { willReadFrequently: true });
-    const img = ctx.getImageData(0, 0, w, h);
-    mark.image = { canvas, ctx, img, w, h };
+/**
+ * Density heatmap image.
+ * This is just a raster mark with default options for
+ * accurate binning and smoothing for density estimation.
+ */
+export class HeatmapMark extends RasterMark {
+  constructor(source, options) {
+    super(source, {
+      bandwidth: 20,
+      interpolate: 'linear',
+      pixelSize: 2,
+      ...options
+    });
   }
-  return mark.image;
 }
 
-function imageScale(mark) {
-  const { densityMap, kde, plot } = mark;
-  let domain = densityMap.fill && plot.getAttribute('colorDomain');
-
-  // compute kde grid extents if no explicit domain
-  if (!domain) {
-    let lo = 0, hi = 0;
-    kde.forEach(grid => {
-      for (const v of grid) {
-        if (v < lo) lo = v;
-        if (v > hi) hi = v;
-      }
-    });
-    domain = (lo === 0 && hi === 0) ? [0, 1] : [lo, hi];
+/**
+ * Utility method to generate color and alpha encoding helpers.
+ * The returned methods can write directly to a pixel raster.
+ */
+export function rasterEncoding(mark) {
+  const { aggr, densityMap, groupby, plot } = mark;
+  const hasDensity = aggr.includes(DENSITY);
+  const hasFillOpacity = aggr.includes('fillOpacity');
+  const fillEntry = mark.channel('fill');
+  const opacEntry = mark.channel('fillOpacity');
+
+  // check constraints, raise errors
+  if (aggr.length > 2 || (hasDensity && hasFillOpacity)) {
+    throw new Error('Invalid raster encodings. Try dropping an aggregate?');
+  }
+  if (groupby.includes(opacEntry?.as)) {
+    throw new Error('Raster fillOpacity must be an aggregate or constant.');
+  }
+
+  // determine fill encoding channel use
+  const fill = densityMap.fill || aggr.includes('fill') ? 'grid'
+    : groupby.includes(fillEntry?.as) ? 'group' // groupby
+    : isColor(fillEntry?.value) ? fillEntry.value // constant
+    : hasDensity && plot.getAttribute('colorScheme') ? 'grid'
+    : undefined;
+
+  // determine fill opacity encoding channel use
+  const opac = densityMap.fillOpacity || aggr.includes('fillOpacity') ? 'grid'
+    : typeof opacEntry?.value === 'number' ? opacEntry.value // constant
+    : hasDensity && fill !== 'grid' ? 'grid'
+    : undefined;
+
+  if (fill !== 'grid' && opac !== 'grid') {
+    // TODO: use a threshold-based encoding?
+    throw new Error('Raster mark missing density values.');
   }
 
-  const type = plot.getAttribute('colorScale');
-  return scale({ x: { type, domain, range: [0, 1] } }).apply;
+  const colorProp = fillEntry?.as ?? (fill === 'grid' ? DENSITY : null);
+  const alphaProp = opacEntry?.as ?? (opac === 'grid' ? DENSITY : null);
+  const color = fill !== 'grid' && fill !== 'group'
+    ? colorConstant(fill)
+    : colorScale(mark, colorProp);
+  const alpha = opac !== 'grid'
+    ? alphaConstant(opac)
+    : alphaScale(mark, alphaProp);
+
+  return { alphaProp, colorProp, alpha, color };
 }
 
-function imagePalette(mark, domain, value, steps = 1024) {
-  const { densityMap, plot } = mark;
-  const scheme = plot.getAttribute('colorScheme');
-
-  // initialize color to constant fill, if specified
-  const fill = mark.channel('fill');
-  let color = isColor(fill?.value) ? fill.value : undefined;
-
-  if (densityMap.fill || (scheme && !color)) {
-    if (scheme) {
-      try {
-        return palette(
-          steps,
-          scale({color: { scheme, domain: [0, 1] }}).interpolate
-        );
-      } catch (err) {
-        console.warn(err);
-      }
+function alphaScale(mark, prop) {
+  const { plot, kde: grids } = mark;
+
+  // determine scale domain
+  const domainAttr = plot.getAttribute('opacityDomain');
+  const domainFixed = domainAttr === Fixed;
+  const domainTransient = domainAttr?.[Transient];
+  const domain = (!domainFixed && !domainTransient && domainAttr)
+    || gridDomainContinuous(grids, prop);
+  if (domainFixed || domainTransient) {
+    if (domainTransient) domain[Transient] = true;
+    plot.setAttribute('colorDomain', domain);
+  }
+
+  // generate opacity scale
+  const s = scale({
+    opacity: {
+      type: plot.getAttribute('opacityScale'),
+      domain,
+      clamp: plot.getAttribute('opacityClamp'),
+      nice: plot.getAttribute('opacityNice'),
+      reverse: plot.getAttribute('opacityReverse'),
+      zero: plot.getAttribute('opacityZero'),
+      base: plot.getAttribute('opacityBase'),
+      exponent: plot.getAttribute('opacityExponent'),
+      constant: plot.getAttribute('opacityConstant')
     }
-  } else if (domain.length) {
-    // fill is based on data values
-    const range = plot.getAttribute('colorRange');
-    const spec = {
+  });
+  return alphaScheme(s);
+}
+
+function colorScale(mark, prop) {
+  const { plot, kde: grids } = mark;
+  const flat = !grids[0][prop]?.map; // not array-like
+  const discrete = flat || Array.isArray(grids[0][prop]);
+
+  // determine scale domain
+  const domainAttr = plot.getAttribute('colorDomain');
+  const domainFixed = domainAttr === Fixed;
+  const domainTransient = domainAttr?.[Transient];
+  const domain = (!domainFixed && !domainTransient && domainAttr) || (
+    flat ? grids.map(cell => cell[prop]).sort(ascending)
+      : discrete ? gridDomainDiscrete(grids, prop)
+      : gridDomainContinuous(grids, prop)
+  );
+  if (domainFixed || domainTransient) {
+    if (domainTransient) domain[Transient] = true;
+    plot.setAttribute('colorDomain', domain);
+  }
+
+  // generate color scale
+  const s = scale({
+    color: {
+      type: plot.getAttribute('colorScale'),
       domain,
-      range,
-      scheme: scheme || (range ? undefined : 'tableau10')
-    };
-    color = scale({ color: spec }).apply(value);
+      range: plot.getAttribute('colorRange'),
+      clamp: plot.getAttribute('colorClamp'),
+      n: plot.getAttribute('colorN'),
+      nice: plot.getAttribute('colorNice'),
+      reverse: plot.getAttribute('colorReverse'),
+      scheme: plot.getAttribute('colorScheme'),
+      interpolate: plot.getAttribute('colorInterpolate'),
+      pivot: plot.getAttribute('colorPivot'),
+      symmetric: plot.getAttribute('colorSymmetric'),
+      zero: plot.getAttribute('colorZero'),
+      base: plot.getAttribute('colorBase'),
+      exponent: plot.getAttribute('colorExponent'),
+      constant: plot.getAttribute('colorConstant')
+    }
+  });
+
+  // TODO: add support for threshold scales?
+  if (discrete) {
+    return colorCategory(s);
+  } else {
+    // Plot scales do not expose intermediate transformation of
+    // values to [0, 1] fractions. So we hobble together our own.
+    const frac = scale({
+      x: {
+        type: inferScaleType(s.type),
+        domain: s.domain,
+        reverse: s.reverse,
+        range: [0, 1],
+        clamp: s.clamp,
+        base: s.base,
+        exponent: s.exponent,
+        constant: s.constant
+      }
+    });
+    return colorScheme(1024, s, frac.apply);
   }
+}
 
-  return palette(steps, opacityMap(color));
+function inferScaleType(type) {
+  if (type.endsWith('symlog')) return 'symlog';
+  if (type.endsWith('log')) return 'log';
+  if (type.endsWith('pow')) return 'pow';
+  if (type.endsWith('sqrt')) return 'sqrt';
+  if (type === 'diverging') return 'linear';
+  return type;
+}
+
+export function imageData(mark, w, h) {
+  if (!mark.image || mark.image.w !== w || mark.image.h !== h) {
+    const canvas = createCanvas(w, h);
+    const ctx = canvas.getContext('2d', { willReadFrequently: true });
+    const img = ctx.getImageData(0, 0, w, h);
+    mark.image = { canvas, ctx, img, w, h };
+  }
+  return mark.image;
 }

package/src/marks/RasterTileMark.js

@@ -1,11 +1,10 @@
 import { coordinator } from '@uwdata/mosaic-core';
 import { Query, count, isBetween, lt, lte, neq, sql, sum } from '@uwdata/mosaic-sql';
-import { scale } from '@observablehq/plot';
+import { binExpr } from './util/bin-expr.js';
 import { extentX, extentY } from './util/extent.js';
-import { isColor } from './util/is-color.js';
-import { createCanvas, raster, opacityMap, palette } from './util/raster.js';
+import { createCanvas } from './util/raster.js';
 import { Grid2DMark } from './Grid2DMark.js';
-import { binField } from './util/bin-field.js';
+import { rasterEncoding } from './RasterMark.js';
 
 export class RasterTileMark extends Grid2DMark {
   constructor(source, options) {
@@ -35,46 +34,66 @@ export class RasterTileMark extends Grid2DMark {
   }
 
   tileQuery(extent) {
-    const { plot, binType, binPad, channels, densityMap, source } = this;
+    const { binType, binPad, channels, densityMap, source } = this;
     const [[x0, x1], [y0, y1]] = extent;
     const [nx, ny] = this.bins;
-    const bx = binField(this, 'x');
-    const by = binField(this, 'y');
-    const rx = !!plot.getAttribute('xReverse');
-    const ry = !!plot.getAttribute('yReverse');
-    const x = bin1d(bx, x0, x1, nx, rx, binPad);
-    const y = bin1d(by, y0, y1, ny, ry, binPad);
+    const [x, bx] = binExpr(this, 'x', nx, [x0, x1], binPad);
+    const [y, by] = binExpr(this, 'y', ny, [y0, y1], binPad);
 
     // with padded bins, include the entire domain extent
     // if the bins are flush, exclude the extent max
     const bounds = binPad
-      ? [isBetween(bx, [x0, x1]), isBetween(by, [y0, y1])]
-      : [lte(x0, bx), lt(bx, x1), lte(y0, by), lt(by, y1)];
+      ? [isBetween(bx, [+x0, +x1]), isBetween(by, [+y0, +y1])]
+      : [lte(+x0, bx), lt(bx, +x1), lte(+y0, by), lt(by, +y1)];
 
     const q = Query
       .from(source.table)
       .where(bounds);
 
     const groupby = this.groupby = [];
-    let agg = count();
+    const aggrMap = {};
     for (const c of channels) {
       if (Object.hasOwn(c, 'field')) {
-        const { channel, field } = c;
+        const { as, channel, field } = c;
         if (field.aggregate) {
-          agg = field;
+          // include custom aggregate
+          aggrMap[channel] = field;
           densityMap[channel] = true;
         } else if (channel === 'weight') {
-          agg = sum(field);
+          // compute weighted density
+          aggrMap.density = sum(field);
         } else if (channel !== 'x' && channel !== 'y') {
-          q.select({ [channel]: field });
-          groupby.push(channel);
+          // add groupby field
+          q.select({ [as]: field });
+          groupby.push(as);
         }
       }
     }
+    const aggr = this.aggr = Object.keys(aggrMap);
 
-    return binType === 'linear'
-      ? binLinear2d(q, x, y, agg, nx, groupby)
-      : bin2d(q, x, y, agg, nx, groupby);
+    // check for incompatible encodings
+    if (aggrMap.density && aggr.length > 1) {
+      throw new Error('Weight option can not be used with custom aggregates.');
+    }
+
+    // if no aggregates, default to count density
+    if (!aggr.length) {
+      aggr.push('density');
+      aggrMap.density = count();
+    }
+
+    // generate grid binning query
+    if (binType === 'linear') {
+      if (aggr.length > 1) {
+        throw new Error('Linear binning not applicable to multiple aggregates.');
+      }
+      if (!aggrMap.density) {
+        throw new Error('Linear binning not applicable to custom aggregates.');
+      }
+      return binLinear2d(q, x, y, aggrMap.density, nx, groupby);
+    } else {
+      return bin2d(q, x, y, aggrMap, nx, groupby);
+    }
   }
 
   async requestTiles() {
@@ -136,7 +155,7 @@ export class RasterTileMark extends Grid2DMark {
 
     // wait for tile queries to complete, then update
     const tiles = await Promise.all(queries);
-    this.grids = [{ grid: processTiles(m, n, xx, yy, coords, tiles) }];
+    this.grids = [{ density: processTiles(m, n, xx, yy, coords, tiles) }];
     this.convolve().update();
   }
 
@@ -145,23 +164,19 @@ export class RasterTileMark extends Grid2DMark {
   }
 
   rasterize() {
-    const { bins, kde, groupby } = this;
+    const { bins, kde } = this;
     const [ w, h ] = bins;
 
     // raster data
     const { canvas, ctx, img } = imageData(this, w, h);
 
-    // scale function to map densities to [0, 1]
-    const s = imageScale(this);
-
-    // gather color domain as needed
-    const idx = groupby.indexOf(this.channelField('fill')?.as);
-    const domain = idx < 0 ? [] : kde.map(({ key }) => key[idx]);
+    // color + opacity encodings
+    const { alpha, alphaProp, color, colorProp } = rasterEncoding(this);
 
-    // generate raster images
-    this.data = kde.map(grid => {
-      const palette = imagePalette(this, domain, grid.key?.[idx]);
-      raster(grid, img.data, w, h, s, palette);
+    // generate rasters
+    this.data = kde.map(cell => {
+      color?.(img.data, w, h, cell[colorProp]);
+      alpha?.(img.data, w, h, cell[alphaProp]);
       ctx.putImageData(img, 0, 0);
       return { src: canvas.toDataURL() };
     });
@@ -199,7 +214,7 @@ function copy(m, n, grid, values, tx, ty) {
   const num = values.numRows;
   if (num === 0) return;
   const index = values.getChild('index').toArray();
-  const value = values.getChild('value').toArray();
+  const value = values.getChild('density').toArray();
   for (let row = 0; row < num; ++row) {
     const idx = index[row];
     const i = tx + (idx % m);
@@ -220,72 +235,11 @@ function imageData(mark, w, h) {
   return mark.image;
 }
 
-function imageScale(mark) {
-  const { densityMap, kde, plot } = mark;
-  let domain = densityMap.fill && plot.getAttribute('colorDomain');
-
-  // compute kde grid extents if no explicit domain
-  if (!domain) {
-    let lo = 0, hi = 0;
-    kde.forEach(grid => {
-      for (const v of grid) {
-        if (v < lo) lo = v;
-        if (v > hi) hi = v;
-      }
-    });
-    domain = (lo === 0 && hi === 0) ? [0, 1] : [lo, hi];
-  }
-
-  const type = plot.getAttribute('colorScale');
-  return scale({ x: { type, domain, range: [0, 1] } }).apply;
-}
-
-function imagePalette(mark, domain, value, steps = 1024) {
-  const { densityMap, plot } = mark;
-  const scheme = plot.getAttribute('colorScheme');
-
-  // initialize color to constant fill, if specified
-  const fill = mark.channel('fill');
-  let color = isColor(fill?.value) ? fill.value : undefined;
-
-  if (densityMap.fill || (scheme && !color)) {
-    if (scheme) {
-      try {
-        return palette(
-          steps,
-          scale({color: { scheme, domain: [0, 1] }}).interpolate
-        );
-      } catch (err) {
-        console.warn(err);
-      }
-    }
-  } else if (domain.length) {
-    // fill is based on data values
-    const range = plot.getAttribute('colorRange');
-    const spec = {
-      domain,
-      range,
-      scheme: scheme || (range ? undefined : 'tableau10')
-    };
-    color = scale({ color: spec }).apply(value);
-  }
-
-  return palette(steps, opacityMap(color));
-}
-
-function bin1d(x, x0, x1, n, reverse, pad) {
-  const d = (n - pad) / (x1 - x0);
-  const f = d !== 1 ? ` * ${d}::DOUBLE` : '';
-  return reverse
-    ? sql`(${x1} - ${x}::DOUBLE)${f}`
-    : sql`(${x}::DOUBLE - ${x0})${f}`;
-}
-
-function bin2d(q, xp, yp, value, xn, groupby) {
+function bin2d(q, xp, yp, aggs, xn, groupby) {
   return q
     .select({
       index: sql`FLOOR(${xp})::INTEGER + FLOOR(${yp})::INTEGER * ${xn}`,
-      value
+      ...aggs
     })
     .groupby('index', groupby);
 }
@@ -320,9 +274,9 @@ function binLinear2d(q, xp, yp, value, xn, groupby) {
 
   return Query
     .from(Query.unionAll(a, b, c, d))
-    .select({ index: 'i', value: sum('w') }, groupby)
+    .select({ index: 'i', density: sum('w') }, groupby)
     .groupby('index', groupby)
-    .having(neq('value', 0));
+    .having(neq('density', 0));
 }
 
 function tileFloor(value) {

package/src/marks/util/arrow.js

@@ -0,0 +1,55 @@
+export const INTEGER = 2;
+export const FLOAT = 3;
+export const DECIMAL = 7;
+export const TIMESTAMP = 10;
+
+export function isArrowTable(values) {
+  return typeof values?.getChild === 'function';
+}
+
+export function convertArrowType(type) {
+  switch (type.typeId) {
+    case INTEGER:
+    case FLOAT:
+    case DECIMAL:
+      return Float64Array;
+    default:
+      return Array;
+  }
+}
+
+export function convertArrow(type) {
+  const { typeId } = type;
+
+  // map timestamp numbers to date objects
+  if (typeId === TIMESTAMP) {
+    return v => v == null ? v : new Date(v);
+  }
+
+  // map bignum to number
+  if (typeId === INTEGER && type.bitWidth >= 64) {
+    return v => v == null ? v : Number(v);
+  }
+
+  // otherwise use Arrow JS defaults
+  return v => v;
+}
+
+export function convertArrowColumn(column) {
+  const { type } = column;
+  const { typeId } = type;
+
+  // map bignum to number
+  if (typeId === INTEGER && type.bitWidth >= 64) {
+    const size = column.length;
+    const array = new Float64Array(size);
+    for (let row = 0; row < size; ++row) {
+      const v = column.get(row);
+      array[row] = v == null ? NaN : Number(v);
+    }
+    return array;
+  }
+
+  // otherwise use Arrow JS defaults
+  return column.toArray();
+}

package/src/marks/util/bin-expr.js

@@ -0,0 +1,30 @@
+import { sql } from '@uwdata/mosaic-sql';
+import { channelScale } from './channel-scale.js';
+
+/**
+ * Generates a SQL expression for 1D pixel-level binning.
+ * Adjusts for scale transformations (log, sqrt, ...).
+ * Returns a [binExpression, field] array, where field is the
+ * input value that is binned. Often the field is just a column
+ * name. For time data, fields are mapped to numerical timestamps.
+ */
+export function binExpr(mark, channel, n, extent, pad = 1, expr) {
+  // get base expression, the channel field unless otherwise given
+  const { field } = mark.channelField(channel);
+  expr = expr ?? field;
+
+  // extract scale information
+  const { type, apply, sqlApply } = channelScale(mark, channel);
+  const reverse = !!mark.plot.getAttribute(`${channel}Reverse`);
+
+  // return expressions for (unrounded) bin index and field
+  const [lo, hi] = extent.map(v => apply(v));
+  const v = sqlApply(expr);
+  const f = type === 'time' || type === 'utc' ? v : expr;
+  const d = hi === lo ? 0 : (n - pad) / (hi - lo);
+  const s = d !== 1 ? ` * ${d}::DOUBLE` : '';
+  const bin = reverse
+    ? sql`(${hi} - ${v}::DOUBLE)${s}`
+    : sql`(${v}::DOUBLE - ${lo})${s}`;
+  return [bin, f];
+}

package/src/marks/util/channel-scale.js

@@ -0,0 +1,27 @@
+import { scaleTransform } from '@uwdata/mosaic-sql';
+
+export function channelScale(mark, channel) {
+  const { plot } = mark;
+
+  let scaleType = plot.getAttribute(`${channel}Scale`);
+  if (!scaleType) {
+    const { field } = mark.channelField(channel, `${channel}1`, `${channel}2`);
+    const { type } = mark.stats[field.column];
+    scaleType = type === 'date' ? 'time' : 'linear';
+  }
+
+  const options = { type: scaleType };
+  switch (scaleType) {
+    case 'log':
+      options.base = plot.getAttribute(`${channel}Base`) ?? 10;
+      break;
+    case 'pow':
+      options.exponent = plot.getAttribute(`${channel}Exponent`) ?? 1;
+      break;
+    case 'symlog':
+      options.constant = plot.getAttribute(`${channel}Constant`) ?? 1;
+      break;
+  }
+
+  return scaleTransform(options);
+}