@uwdata/mosaic-core 0.8.0 → 0.10.0

package/src/DataCubeIndexer.js

@@ -1,252 +1,268 @@
-import { Query, and, create, isBetween, scaleTransform, sql } from '@uwdata/mosaic-sql';
+import {
+  Query, and, asColumn, create, isBetween, scaleTransform, sql
+} from '@uwdata/mosaic-sql';
+import { indexColumns } from './util/index-columns.js';
 import { fnv_hash } from './util/hash.js';
 
+const Skip = { skip: true, result: null };
+
 /**
  * Build and query optimized indices ("data cubes") for fast computation of
  * groupby aggregate queries over compatible client queries and selections.
  * A data cube contains pre-aggregated data for a Mosaic client, subdivided
- * by possible query values from an active view. Indexes are realized as
- * as temporary database tables that can be queried for rapid updates.
- * Compatible client queries must pull data from the same backing table and
- * must consist of only groupby dimensions and supported aggregates.
- * Compatible selections must contain an active clause that exposes a schema
- * for an interval or point value predicate.
+ * by possible query values from an active selection clause. These cubes are
+ * realized as as database tables that can be queried for rapid updates.
+ * Compatible client queries must consist of only groupby dimensions and
+ * supported aggregate functions. Compatible selections must contain an active
+ * clause that exposes metadata for an interval or point value predicate.
  */
 export class DataCubeIndexer {
   /**
-   *
-   * @param {import('./Coordinator.js').Coordinator} mc a Mosaic coordinator
-   * @param {*} options Options hash to configure the data cube indexes and pass selections to the coordinator.
+   * Create a new data cube index table manager.
+   * @param {import('./Coordinator.js').Coordinator} coordinator A Mosaic coordinator.
+   * @param {object} [options] Indexer options.
+   * @param {boolean} [options.enabled=true] Flag to enable/disable indexer.
+   * @param {boolean} [options.temp=true] Flag to indicate if generated data
+   *  cube index tables should be temporary tables.
    */
-  constructor(mc, { selection, temp = true }) {
-    /** @type import('./Coordinator.js').Coordinator */
-    this.mc = mc;
-    this.selection = selection;
+  constructor(coordinator, {
+    enabled = true,
+    temp = true
+  } = {}) {
+    /** @type {Map<import('./MosaicClient.js').MosaicClient, DataCubeInfo | Skip | null>} */
+    this.indexes = new Map();
+    this.active = null;
     this.temp = temp;
-    this.reset();
+    this.mc = coordinator;
+    this._enabled = enabled;
   }
 
-  reset() {
-    this.enabled = false;
-    this.clients = null;
-    this.indices = null;
-    this.activeView = null;
+  /**
+   * Set the enabled state of this indexer. If false, any cached state is
+   * cleared and subsequent index calls will return null until re-enabled.
+   * @param {boolean} state The enabled state.
+   */
+  enabled(state) {
+    if (state === undefined) {
+      return this._enabled;
+    } else if (this._enabled !== state) {
+      if (!state) this.clear();
+      this._enabled = state;
+    }
   }
 
+  /**
+   * Clear the cache of data cube index table entries for the current active
+   * selection clause. This method will also cancel any queued data cube table
+   * creation queries that have not yet been submitted to the database. This
+   * method does _not_ drop any existing data cube tables.
+   */
   clear() {
-    if (this.indices) {
-      this.mc.cancel(Array.from(this.indices.values(), index => index.result));
-      this.indices = null;
-    }
+    this.mc.cancel(Array.from(this.indexes.values(), info => info?.result));
+    this.indexes.clear();
+    this.active = null;
   }
 
-  index(clients, active) {
-    if (this.clients !== clients) {
-      // test client views for compatibility
-      const cols = Array.from(clients, getIndexColumns);
-      const from = cols[0]?.from;
-      this.enabled = cols.every(c => c && c.from === from);
-      this.clients = clients;
-      this.activeView = null;
-      this.clear();
-    }
-    if (!this.enabled) return false; // client views are not indexable
-
-    active = active || this.selection.active;
-    const { source } = active;
-    // exit early if indexes already set up for active view
-    if (source && source === this.activeView?.source) return true;
-
-    this.clear();
-    if (!source) return false; // nothing to work with
-    const activeView = this.activeView = getActiveView(active);
-    if (!activeView) return false; // active selection clause not compatible
-
-    this.mc.logger().warn('DATA CUBE INDEX CONSTRUCTION');
-
-    // create a selection with the active source removed
-    const sel = this.selection.remove(source);
-
-    // generate data cube indices
-    const indices = this.indices = new Map;
-    const { mc, temp } = this;
-    for (const client of clients) {
-      if (sel.skip(client, active)) continue;
-      const index = getIndexColumns(client);
-
-      // build index construction query
-      const query = client.query(sel.predicate(client))
-        .select({ ...activeView.columns, ...index.aux })
-        .groupby(Object.keys(activeView.columns));
-
-      // ensure active view columns are selected by subqueries
-      const [subq] = query.subqueries;
-      if (subq) {
-        const cols = Object.values(activeView.columns).map(c => c.columns[0]);
-        subqueryPushdown(subq, cols);
-      }
-
-      // push orderby criteria to later cube queries
-      const order = query.orderby();
-      query.query.orderby = [];
-
-      const sql = query.toString();
-      const id = (fnv_hash(sql) >>> 0).toString(16);
-      const table = `cube_index_${id}`;
-      const result = mc.exec(create(table, sql, { temp }));
-      indices.set(client, { table, result, order, ...index });
+  /**
+   * Return data cube index table information for the active state of a
+   * client-selection pair, or null if the client is not indexable. This
+   * method has multiple possible side effects, including data cube table
+   * generation and updating internal caches.
+   * @param {import('./MosaicClient.js').MosaicClient} client A Mosaic client.
+   * @param {import('./Selection.js').Selection} selection A Mosaic selection
+   *  to filter the client by.
+   * @param {import('./util/selection-types.js').SelectionClause} activeClause
+   *  A representative active selection clause for which to (possibly) generate
+   *  data cube index tables.
+   * @returns {DataCubeInfo | Skip | null} Data cube index table
+   *  information and query generator, or null if the client is not indexable.
+   */
+  index(client, selection, activeClause) {
+    // if not enabled, do nothing
+    if (!this._enabled) return null;
+
+    const { indexes, mc, temp } = this;
+    const { source } = activeClause;
+
+    // if there is no clause source to track, do nothing
+    if (!source) return null;
+
+    // if we have cached active columns, check for updates or exit
+    if (this.active) {
+      // if the active clause source has changed, clear indexer state
+      // this cancels outstanding requests and clears the index cache
+      // a clear also sets this.active to null
+      if (this.active.source !== source) this.clear();
+      // if we've seen this source and it's not indexable, do nothing
+      if (this.active?.source === null) return null;
     }
 
-    // index creation successful
-    return true;
-  }
+    // the current active columns cache value
+    let { active } = this;
 
-  async update() {
-    const { clients, selection, activeView } = this;
-    const filter = activeView.predicate(selection.active.predicate);
-    return Promise.all(
-      Array.from(clients).map(client => this.updateClient(client, filter))
-    );
-  }
+    // if cached active columns are unset, analyze the active clause
+    if (!active) {
+      // generate active data cube dimension columns to select over
+      // will return an object with null source if not indexable
+      this.active = active = activeColumns(activeClause);
+      // if the active clause is not indexable, exit now
+      if (active.source === null) return null;
+    }
+
+    // if we have cached data cube index table info, return that
+    if (indexes.has(client)) {
+      return indexes.get(client);
+    }
 
-  async updateClient(client, filter) {
-    const index = this.indices.get(client);
-    if (!index) return;
+    // get non-active data cube index table columns
+    const indexCols = indexColumns(client);
 
-    if (!filter) {
-      filter = this.activeView.predicate(this.selection.active.predicate);
+    let info;
+    if (!indexCols) {
+      // if client is not indexable, record null index
+      info = null;
+    } else if (selection.skip(client, activeClause)) {
+      // skip client if untouched by cross-filtering
+      info = Skip;
+    } else {
+      // generate data cube index table
+      const filter = selection.remove(source).predicate(client);
+      info = dataCubeInfo(client.query(filter), active, indexCols);
+      info.result = mc.exec(create(info.table, info.create, { temp }));
+      info.result.catch(e => mc.logger().error(e));
     }
 
-    const { table, dims, aggr, order = [] } = index;
-    const query = Query
-      .select(dims, aggr)
-      .from(table)
-      .groupby(dims)
-      .where(filter)
-      .orderby(order);
-    return this.mc.updateClient(client, query);
+    indexes.set(client, info);
+    return info;
   }
 }
 
-function getActiveView(clause) {
-  const { source, schema } = clause;
-  let columns = clause.predicate?.columns;
-  if (!schema || !columns) return null;
-  const { type, scales, pixelSize = 1 } = schema;
+/**
+ * Determines the active data cube dimension columns to select over. Returns
+ * an object with the clause source, column definitions, and a predicate
+ * generator function for the active dimensions of a data cube index table. If
+ * the active clause is not indexable or is missing metadata, this method
+ * returns an object with a null source property.
+ * @param {import('./util/selection-types.js').SelectionClause} clause The
+ *  active selection clause to analyze.
+ */
+function activeColumns(clause) {
+  const { source, meta } = clause;
+  const clausePred = clause.predicate;
+  const clauseCols = clausePred?.columns;
   let predicate;
+  let columns;
+
+  if (!meta || !clauseCols) {
+    return { source: null, columns, predicate };
+  }
 
-  if (type === 'interval' && scales) {
-    const bins = scales.map(s => binInterval(s, pixelSize));
-    if (bins.some(b => b == null)) return null; // unsupported scale type
+  // @ts-ignore
+  const { type, scales, bin, pixelSize = 1 } = meta;
 
-    if (bins.length === 1) {
+  if (type === 'point') {
+    predicate = x => x;
+    columns = Object.fromEntries(
+      clauseCols.map(col => [`${col}`, asColumn(col)])
+    );
+  } else if (type === 'interval' && scales) {
+    // determine pixel-level binning
+    const bins = scales.map(s => binInterval(s, pixelSize, bin));
+
+    if (bins.some(b => !b)) {
+      // bail if a scale type is unsupported
+    } else if (bins.length === 1) {
+      // single interval selection
       predicate = p => p ? isBetween('active0', p.range.map(bins[0])) : [];
-      columns = { active0: bins[0](clause.predicate.field) };
+      // @ts-ignore
+      columns = { active0: bins[0](clausePred.field) };
     } else {
+      // multiple interval selection
       predicate = p => p
-        ? and(p.children.map(({ range }, i) => isBetween(`active${i}`, range.map(bins[i]))))
+        ? and(p.children.map(
+            ({ range }, i) => isBetween(`active${i}`, range.map(bins[i]))
+          ))
         : [];
       columns = Object.fromEntries(
-        clause.predicate.children.map((p, i) => [`active${i}`, bins[i](p.field)])
+        // @ts-ignore
+        clausePred.children.map((p, i) => [`active${i}`, bins[i](p.field)])
       );
     }
-  } else if (type === 'point') {
-    predicate = x => x;
-    columns = Object.fromEntries(columns.map(col => [col.toString(), col]));
-  } else {
-    return null; // unsupported type
   }
 
-  return { source, columns, predicate };
+  return { source: columns ? source : null, columns, predicate };
 }
 
-function binInterval(scale, pixelSize) {
-  const { apply, sqlApply } = scaleTransform(scale);
-  if (apply) {
-    const { domain, range } = scale;
-    const lo = apply(Math.min(...domain));
-    const hi = apply(Math.max(...domain));
-    const a = (Math.abs(range[1] - range[0]) / (hi - lo)) / pixelSize;
-    const s = pixelSize === 1 ? '' : `${pixelSize}::INTEGER * `;
-    return value => sql`${s}FLOOR(${a}::DOUBLE * (${sqlApply(value)} - ${lo}::DOUBLE))::INTEGER`;
-  }
+const BIN = { ceil: 'CEIL', round: 'ROUND' };
+
+/**
+ * Returns a bin function generator to discretize a selection interval domain.
+ * @param {import('./util/selection-types.js').Scale} scale A scale that maps
+ *  domain values to the output range (typically pixels).
+ * @param {number} pixelSize The interactive pixel size. This value indicates
+ *  the bin step size and may be greater than an actual screen pixel.
+ * @param {import('./util/selection-types.js').BinMethod} bin The binning
+ *  method to apply, one of `floor`, `ceil', or `round`.
+ * @returns {(value: any) => import('@uwdata/mosaic-sql').SQLExpression}
+ *  A bin function generator.
+ */
+function binInterval(scale, pixelSize, bin) {
+  const { type, domain, range, apply, sqlApply } = scaleTransform(scale);
+  if (!apply) return; // unsupported scale type
+  const fn = BIN[`${bin}`.toLowerCase()] || 'FLOOR';
+  const lo = apply(Math.min(...domain));
+  const hi = apply(Math.max(...domain));
+  const a = type === 'identity' ? 1 : Math.abs(range[1] - range[0]) / (hi - lo);
+  const s = a / pixelSize === 1 ? '' : `${a / pixelSize}::DOUBLE * `;
+  const d = lo === 0 ? '' : ` - ${lo}::DOUBLE`;
+  return value => sql`${fn}(${s}(${sqlApply(value)}${d}))::INTEGER`;
 }
 
-const NO_INDEX = { from: NaN };
-
-function getIndexColumns(client) {
-  if (!client.filterIndexable) return NO_INDEX;
-  const q = client.query();
-  const from = getBaseTable(q);
-  if (!from || !q.groupby) return NO_INDEX;
-  const g = new Set(q.groupby().map(c => c.column));
-
-  const aggr = [];
-  const dims = [];
-  const aux = {}; // auxiliary columns needed by aggregates
-  let auxAs;
-
-  for (const entry of q.select()) {
-    const { as, expr: { aggregate, args } } = entry;
-    const op = aggregate?.toUpperCase?.();
-    switch (op) {
-      case 'COUNT':
-      case 'SUM':
-        aggr.push({ [as]: sql`SUM("${as}")::DOUBLE` });
-        break;
-      case 'AVG':
-        aux[auxAs = '__count__'] = sql`COUNT(*)`;
-        aggr.push({ [as]: sql`(SUM("${as}" * ${auxAs}) / SUM(${auxAs}))::DOUBLE` });
-        break;
-      case 'ARG_MAX':
-        aux[auxAs = `__max_${as}__`] = sql`MAX(${args[1]})`;
-        aggr.push({ [as]: sql`ARG_MAX("${as}", ${auxAs})` });
-        break;
-      case 'ARG_MIN':
-        aux[auxAs = `__min_${as}__`] = sql`MIN(${args[1]})`;
-        aggr.push({ [as]: sql`ARG_MIN("${as}", ${auxAs})` });
-        break;
-
-      // aggregates that commute directly
-      case 'MAX':
-      case 'MIN':
-      case 'BIT_AND':
-      case 'BIT_OR':
-      case 'BIT_XOR':
-      case 'BOOL_AND':
-      case 'BOOL_OR':
-      case 'PRODUCT':
-        aggr.push({ [as]: sql`${op}("${as}")` });
-        break;
-      default:
-        if (g.has(as)) dims.push(as);
-        else return null;
-    }
+/**
+ * Generate data cube table query information.
+ * @param {Query} clientQuery The original client query.
+ * @param {*} active Active (selected) column definitions.
+ * @param {*} indexCols Data cube index column definitions.
+ * @returns {DataCubeInfo}
+ */
+function dataCubeInfo(clientQuery, active, indexCols) {
+  const { dims, aggr, aux } = indexCols;
+  const { columns } = active;
+
+  // build index table construction query
+  const query = clientQuery
+    .select({ ...columns, ...aux })
+    .groupby(Object.keys(columns));
+
+  // ensure active clause columns are selected by subqueries
+  const [subq] = query.subqueries;
+  if (subq) {
+    const cols = Object.values(columns).flatMap(c => c.columns);
+    subqueryPushdown(subq, cols);
   }
 
-  return { aggr, dims, aux, from };
-}
+  // push orderby criteria to later cube queries
+  const order = query.orderby();
+  query.query.orderby = [];
 
-function getBaseTable(query) {
-  const subq = query.subqueries;
+  // generate creation query string and hash id
+  const create = query.toString();
+  const id = (fnv_hash(create) >>> 0).toString(16);
+  const table = `cube_index_${id}`;
 
-  // select query
-  if (query.select) {
-    const from = query.from();
-    if (!from.length) return undefined;
-    if (subq.length === 0) return from[0].from.table;
-  }
+  // generate data cube select query
+  const select = Query
+    .select(dims, aggr)
+    .from(table)
+    .groupby(dims)
+    .orderby(order);
 
-  // handle set operations / subqueries
-  const base = getBaseTable(subq[0]);
-  for (let i = 1; i < subq.length; ++i) {
-    const from = getBaseTable(subq[i]);
-    if (from === undefined) continue;
-    if (from !== base) return NaN;
-  }
-  return base;
+  return new DataCubeInfo({ table, create, active, select });
 }
 
+/**
+ * Push column selections down to subqueries.
+ */
 function subqueryPushdown(query, cols) {
   const memo = new Set;
   const pushdown = q => {
@@ -259,3 +275,46 @@ function subqueryPushdown(query, cols) {
   };
   pushdown(query);
 }
+
+/**
+ * Metadata and query generator for a data cube index table. This
+ * object provides the information needed to generate and query
+ * a data cube index table for a client-selection pair relative to
+ * a specific active clause and selection state.
+ */
+export class DataCubeInfo {
+  /**
+   * Create a new DataCubeInfo instance.
+   * @param {object} options
+   */
+  constructor({ table, create, active, select } = {}) {
+    /** The name of the data cube index table. */
+    this.table = table;
+    /** The SQL query used to generate the data cube index table. */
+    this.create = create;
+    /** A result promise returned for the data cube creation query. */
+    this.result = null;
+    /**
+     * Definitions and predicate function for the active columns,
+     * which are dynamically filtered by the active clause.
+     */
+    this.active = active;
+    /** Select query (sans where clause) for data cube tables. */
+    this.select = select;
+    /**
+     * Boolean flag indicating a client that should be skipped.
+     * This value is always false for completed data cube info.
+     */
+    this.skip = false;
+  }
+
+  /**
+   * Generate a data cube index table query for the given predicate.
+   * @param {import('@uwdata/mosaic-sql').SQLExpression} predicate The current
+   *  active clause predicate.
+   * @returns {Query} A data cube index table query.
+   */
+  query(predicate) {
+    return this.select.clone().where(this.active.predicate(predicate));
+  }
+}

package/src/MosaicClient.js

@@ -94,8 +94,8 @@ export class MosaicClient {
    * @param {*} error
    * @returns {this}
    */
-  queryError(error) {
-    console.error(error);
+  queryError(error) { // eslint-disable-line no-unused-vars
+    // do nothing, the coordinator logs the error
     return this;
   }
 
@@ -122,7 +122,7 @@ export class MosaicClient {
   /**
    * Requests a client update.
    * For example to (re-)render an interface component.
-   * 
+   *
    * @returns {this | Promise<any>}
    */
   update() {

package/src/QueryConsolidator.js

@@ -1,5 +1,5 @@
 import { Query, Ref, isDescribeQuery } from '@uwdata/mosaic-sql';
-import { queryResult } from './util/query-result.js';
+import { QueryResult } from './util/query-result.js';
 
 function wait(callback) {
   const method = typeof requestAnimationFrame !== 'undefined'
@@ -133,7 +133,7 @@ function consolidate(group, enqueue, record) {
         record: false,
         query: (group.query = consolidatedQuery(group, record))
       },
-      result: (group.result = queryResult())
+      result: (group.result = new QueryResult())
     });
   } else {
     // issue queries directly