@uwdata/mosaic-core 0.9.0 → 0.10.0

package/src/DataCubeIndexer.js

@@ -1,194 +1,211 @@
-import { Query, and, asColumn, create, isBetween, scaleTransform, sql } from '@uwdata/mosaic-sql';
-import { fnv_hash } from './util/hash.js';
+import {
+  Query, and, asColumn, create, isBetween, scaleTransform, sql
+} from '@uwdata/mosaic-sql';
 import { indexColumns } from './util/index-columns.js';
+import { fnv_hash } from './util/hash.js';
+
+const Skip = { skip: true, result: null };
 
 /**
  * Build and query optimized indices ("data cubes") for fast computation of
  * groupby aggregate queries over compatible client queries and selections.
  * A data cube contains pre-aggregated data for a Mosaic client, subdivided
- * by possible query values from an active view. Indexes are realized as
- * as temporary database tables that can be queried for rapid updates.
- * Compatible client queries must pull data from the same backing table and
- * must consist of only groupby dimensions and supported aggregates.
- * Compatible selections must contain an active clause that exposes metadata
- * for an interval or point value predicate.
+ * by possible query values from an active selection clause. These cubes are
+ * realized as as database tables that can be queried for rapid updates.
+ * Compatible client queries must consist of only groupby dimensions and
+ * supported aggregate functions. Compatible selections must contain an active
+ * clause that exposes metadata for an interval or point value predicate.
  */
 export class DataCubeIndexer {
   /**
-   *
-   * @param {import('./Coordinator.js').Coordinator} mc a Mosaic coordinator
-   * @param {*} options Options hash to configure the data cube indexes and pass selections to the coordinator.
+   * Create a new data cube index table manager.
+   * @param {import('./Coordinator.js').Coordinator} coordinator A Mosaic coordinator.
+   * @param {object} [options] Indexer options.
+   * @param {boolean} [options.enabled=true] Flag to enable/disable indexer.
+   * @param {boolean} [options.temp=true] Flag to indicate if generated data
+   *  cube index tables should be temporary tables.
    */
-  constructor(mc, { selection, temp = true }) {
-    /** @type import('./Coordinator.js').Coordinator */
-    this.mc = mc;
-    this.selection = selection;
+  constructor(coordinator, {
+    enabled = true,
+    temp = true
+  } = {}) {
+    /** @type {Map<import('./MosaicClient.js').MosaicClient, DataCubeInfo | Skip | null>} */
+    this.indexes = new Map();
+    this.active = null;
     this.temp = temp;
-    this.reset();
+    this.mc = coordinator;
+    this._enabled = enabled;
   }
 
-  reset() {
-    this.enabled = false;
-    this.clients = null;
-    this.indices = null;
-    this.active = null;
+  /**
+   * Set the enabled state of this indexer. If false, any cached state is
+   * cleared and subsequent index calls will return null until re-enabled.
+   * @param {boolean} state The enabled state.
+   */
+  enabled(state) {
+    if (state === undefined) {
+      return this._enabled;
+    } else if (this._enabled !== state) {
+      if (!state) this.clear();
+      this._enabled = state;
+    }
   }
 
+  /**
+   * Clear the cache of data cube index table entries for the current active
+   * selection clause. This method will also cancel any queued data cube table
+   * creation queries that have not yet been submitted to the database. This
+   * method does _not_ drop any existing data cube tables.
+   */
   clear() {
-    if (this.indices) {
-      this.mc.cancel(Array.from(this.indices.values(), index => index?.result));
-      this.indices = null;
-    }
+    this.mc.cancel(Array.from(this.indexes.values(), info => info?.result));
+    this.indexes.clear();
+    this.active = null;
   }
 
-  index(clients, activeClause) {
-    if (this.clients !== clients) {
-      // test client views for compatibility
-      const cols = Array.from(clients, indexColumns).filter(x => x);
-      const from = cols[0]?.from;
-      this.enabled = cols.length && cols.every(c => c.from === from);
-      this.clients = clients;
-      this.active = null;
-      this.clear();
-    }
-    if (!this.enabled) return false; // client views are not indexable
+  /**
+   * Return data cube index table information for the active state of a
+   * client-selection pair, or null if the client is not indexable. This
+   * method has multiple possible side effects, including data cube table
+   * generation and updating internal caches.
+   * @param {import('./MosaicClient.js').MosaicClient} client A Mosaic client.
+   * @param {import('./Selection.js').Selection} selection A Mosaic selection
+   *  to filter the client by.
+   * @param {import('./util/selection-types.js').SelectionClause} activeClause
+   *  A representative active selection clause for which to (possibly) generate
+   *  data cube index tables.
+   * @returns {DataCubeInfo | Skip | null} Data cube index table
+   *  information and query generator, or null if the client is not indexable.
+   */
+  index(client, selection, activeClause) {
+    // if not enabled, do nothing
+    if (!this._enabled) return null;
 
-    activeClause = activeClause || this.selection.active;
+    const { indexes, mc, temp } = this;
     const { source } = activeClause;
-    // exit early if indexes already set up for active view
-    if (source && source === this.active?.source) return true;
-
-    this.clear();
-    if (!source) return false; // nothing to work with
-    const active = this.active = activeColumns(activeClause);
-    if (!active) return false; // active selection clause not compatible
-
-    const logger = this.mc.logger();
-    logger.warn('DATA CUBE INDEX CONSTRUCTION');
-
-    // create a selection with the active source removed
-    const sel = this.selection.remove(source);
-
-    // generate data cube indices
-    const indices = this.indices = new Map;
-    const { mc, temp } = this;
-    for (const client of clients) {
-      // determine if client should be skipped due to cross-filtering
-      if (sel.skip(client, activeClause)) {
-        indices.set(client, null);
-        continue;
-      }
-
-      // generate column definitions for data cube and cube queries
-      const index = indexColumns(client);
-
-      // skip if client is not indexable
-      if (!index) {
-        continue;
-      }
-
-      // build index table construction query
-      const query = client.query(sel.predicate(client))
-        .select({ ...active.columns, ...index.aux })
-        .groupby(Object.keys(active.columns));
-
-      // ensure active view columns are selected by subqueries
-      const [subq] = query.subqueries;
-      if (subq) {
-        const cols = Object.values(active.columns).flatMap(c => c.columns);
-        subqueryPushdown(subq, cols);
-      }
-
-      // push orderby criteria to later cube queries
-      const order = query.orderby();
-      query.query.orderby = [];
-
-      const sql = query.toString();
-      const id = (fnv_hash(sql) >>> 0).toString(16);
-      const table = `cube_index_${id}`;
-      const result = mc.exec(create(table, sql, { temp }));
-      result.catch(e => logger.error(e));
-      indices.set(client, { table, result, order, ...index });
+
+    // if there is no clause source to track, do nothing
+    if (!source) return null;
+
+    // if we have cached active columns, check for updates or exit
+    if (this.active) {
+      // if the active clause source has changed, clear indexer state
+      // this cancels outstanding requests and clears the index cache
+      // a clear also sets this.active to null
+      if (this.active.source !== source) this.clear();
+      // if we've seen this source and it's not indexable, do nothing
+      if (this.active?.source === null) return null;
     }
 
-    // index creation successful
-    return true;
-  }
+    // the current active columns cache value
+    let { active } = this;
 
-  async update() {
-    const { clients, selection, active } = this;
-    const filter = active.predicate(selection.active.predicate);
-    return Promise.all(
-      Array.from(clients).map(client => this.updateClient(client, filter))
-    );
-  }
+    // if cached active columns are unset, analyze the active clause
+    if (!active) {
+      // generate active data cube dimension columns to select over
+      // will return an object with null source if not indexable
+      this.active = active = activeColumns(activeClause);
+      // if the active clause is not indexable, exit now
+      if (active.source === null) return null;
+    }
+
+    // if we have cached data cube index table info, return that
+    if (indexes.has(client)) {
+      return indexes.get(client);
+    }
+
+    // get non-active data cube index table columns
+    const indexCols = indexColumns(client);
 
-  async updateClient(client, filter) {
-    const { mc, indices, selection } = this;
-
-    // if client has no index, perform a standard update
-    if (!indices.has(client)) {
-      filter = selection.predicate(client);
-      return mc.updateClient(client, client.query(filter));
-    };
-
-    const index = this.indices.get(client);
-
-    // skip update if cross-filtered
-    if (!index) return;
-
-    // otherwise, query a data cube index table
-    const { table, dims, aggr, order = [] } = index;
-    const query = Query
-      .select(dims, aggr)
-      .from(table)
-      .groupby(dims)
-      .where(filter)
-      .orderby(order);
-    return mc.updateClient(client, query);
+    let info;
+    if (!indexCols) {
+      // if client is not indexable, record null index
+      info = null;
+    } else if (selection.skip(client, activeClause)) {
+      // skip client if untouched by cross-filtering
+      info = Skip;
+    } else {
+      // generate data cube index table
+      const filter = selection.remove(source).predicate(client);
+      info = dataCubeInfo(client.query(filter), active, indexCols);
+      info.result = mc.exec(create(info.table, info.create, { temp }));
+      info.result.catch(e => mc.logger().error(e));
+    }
+
+    indexes.set(client, info);
+    return info;
   }
 }
 
+/**
+ * Determines the active data cube dimension columns to select over. Returns
+ * an object with the clause source, column definitions, and a predicate
+ * generator function for the active dimensions of a data cube index table. If
+ * the active clause is not indexable or is missing metadata, this method
+ * returns an object with a null source property.
+ * @param {import('./util/selection-types.js').SelectionClause} clause The
+ *  active selection clause to analyze.
+ */
 function activeColumns(clause) {
   const { source, meta } = clause;
-  let columns = clause.predicate?.columns;
-  if (!meta || !columns) return null;
-  const { type, scales, bin, pixelSize = 1 } = meta;
+  const clausePred = clause.predicate;
+  const clauseCols = clausePred?.columns;
   let predicate;
+  let columns;
+
+  if (!meta || !clauseCols) {
+    return { source: null, columns, predicate };
+  }
 
-  if (type === 'interval' && scales) {
+  // @ts-ignore
+  const { type, scales, bin, pixelSize = 1 } = meta;
+
+  if (type === 'point') {
+    predicate = x => x;
+    columns = Object.fromEntries(
+      clauseCols.map(col => [`${col}`, asColumn(col)])
+    );
+  } else if (type === 'interval' && scales) {
     // determine pixel-level binning
     const bins = scales.map(s => binInterval(s, pixelSize, bin));
 
-    // bail if the scale type is unsupported
-    if (bins.some(b => b == null)) return null;
-
-    if (bins.length === 1) {
+    if (bins.some(b => !b)) {
+      // bail if a scale type is unsupported
+    } else if (bins.length === 1) {
       // single interval selection
       predicate = p => p ? isBetween('active0', p.range.map(bins[0])) : [];
-      columns = { active0: bins[0](clause.predicate.field) };
+      // @ts-ignore
+      columns = { active0: bins[0](clausePred.field) };
     } else {
       // multiple interval selection
       predicate = p => p
-        ? and(p.children.map(({ range }, i) => isBetween(`active${i}`, range.map(bins[i]))))
+        ? and(p.children.map(
+            ({ range }, i) => isBetween(`active${i}`, range.map(bins[i]))
+          ))
         : [];
       columns = Object.fromEntries(
-        clause.predicate.children.map((p, i) => [`active${i}`, bins[i](p.field)])
+        // @ts-ignore
+        clausePred.children.map((p, i) => [`active${i}`, bins[i](p.field)])
       );
     }
-  } else if (type === 'point') {
-    predicate = x => x;
-    columns = Object.fromEntries(columns.map(col => [`${col}`, asColumn(col)]));
-  } else {
-    // unsupported selection type
-    return null;
   }
 
-  return { source, columns, predicate };
+  return { source: columns ? source : null, columns, predicate };
 }
 
 const BIN = { ceil: 'CEIL', round: 'ROUND' };
 
+/**
+ * Returns a bin function generator to discretize a selection interval domain.
+ * @param {import('./util/selection-types.js').Scale} scale A scale that maps
+ *  domain values to the output range (typically pixels).
+ * @param {number} pixelSize The interactive pixel size. This value indicates
+ *  the bin step size and may be greater than an actual screen pixel.
+ * @param {import('./util/selection-types.js').BinMethod} bin The binning
+ *  method to apply, one of `floor`, `ceil', or `round`.
+ * @returns {(value: any) => import('@uwdata/mosaic-sql').SQLExpression}
+ *  A bin function generator.
+ */
 function binInterval(scale, pixelSize, bin) {
   const { type, domain, range, apply, sqlApply } = scaleTransform(scale);
   if (!apply) return; // unsupported scale type
@@ -201,6 +218,51 @@ function binInterval(scale, pixelSize, bin) {
   return value => sql`${fn}(${s}(${sqlApply(value)}${d}))::INTEGER`;
 }
 
+/**
+ * Generate data cube table query information.
+ * @param {Query} clientQuery The original client query.
+ * @param {*} active Active (selected) column definitions.
+ * @param {*} indexCols Data cube index column definitions.
+ * @returns {DataCubeInfo}
+ */
+function dataCubeInfo(clientQuery, active, indexCols) {
+  const { dims, aggr, aux } = indexCols;
+  const { columns } = active;
+
+  // build index table construction query
+  const query = clientQuery
+    .select({ ...columns, ...aux })
+    .groupby(Object.keys(columns));
+
+  // ensure active clause columns are selected by subqueries
+  const [subq] = query.subqueries;
+  if (subq) {
+    const cols = Object.values(columns).flatMap(c => c.columns);
+    subqueryPushdown(subq, cols);
+  }
+
+  // push orderby criteria to later cube queries
+  const order = query.orderby();
+  query.query.orderby = [];
+
+  // generate creation query string and hash id
+  const create = query.toString();
+  const id = (fnv_hash(create) >>> 0).toString(16);
+  const table = `cube_index_${id}`;
+
+  // generate data cube select query
+  const select = Query
+    .select(dims, aggr)
+    .from(table)
+    .groupby(dims)
+    .orderby(order);
+
+  return new DataCubeInfo({ table, create, active, select });
+}
+
+/**
+ * Push column selections down to subqueries.
+ */
 function subqueryPushdown(query, cols) {
   const memo = new Set;
   const pushdown = q => {
@@ -213,3 +275,46 @@ function subqueryPushdown(query, cols) {
   };
   pushdown(query);
 }
+
+/**
+ * Metadata and query generator for a data cube index table. This
+ * object provides the information needed to generate and query
+ * a data cube index table for a client-selection pair relative to
+ * a specific active clause and selection state.
+ */
+export class DataCubeInfo {
+  /**
+   * Create a new DataCubeInfo instance.
+   * @param {object} options
+   */
+  constructor({ table, create, active, select } = {}) {
+    /** The name of the data cube index table. */
+    this.table = table;
+    /** The SQL query used to generate the data cube index table. */
+    this.create = create;
+    /** A result promise returned for the data cube creation query. */
+    this.result = null;
+    /**
+     * Definitions and predicate function for the active columns,
+     * which are dynamically filtered by the active clause.
+     */
+    this.active = active;
+    /** Select query (sans where clause) for data cube tables. */
+    this.select = select;
+    /**
+     * Boolean flag indicating a client that should be skipped.
+     * This value is always false for completed data cube info.
+     */
+    this.skip = false;
+  }
+
+  /**
+   * Generate a data cube index table query for the given predicate.
+   * @param {import('@uwdata/mosaic-sql').SQLExpression} predicate The current
+   *  active clause predicate.
+   * @returns {Query} A data cube index table query.
+   */
+  query(predicate) {
+    return this.select.clone().where(this.active.predicate(predicate));
+  }
+}

package/src/MosaicClient.js

@@ -94,8 +94,8 @@ export class MosaicClient {
    * @param {*} error
    * @returns {this}
    */
-  queryError(error) {
-    console.error(error);
+  queryError(error) { // eslint-disable-line no-unused-vars
+    // do nothing, the coordinator logs the error
     return this;
   }
 
@@ -122,7 +122,7 @@ export class MosaicClient {
   /**
    * Requests a client update.
    * For example to (re-)render an interface component.
-   * 
+   *
    * @returns {this | Promise<any>}
    */
   update() {

package/src/QueryConsolidator.js

@@ -1,5 +1,5 @@
 import { Query, Ref, isDescribeQuery } from '@uwdata/mosaic-sql';
-import { queryResult } from './util/query-result.js';
+import { QueryResult } from './util/query-result.js';
 
 function wait(callback) {
   const method = typeof requestAnimationFrame !== 'undefined'
@@ -133,7 +133,7 @@ function consolidate(group, enqueue, record) {
         record: false,
         query: (group.query = consolidatedQuery(group, record))
       },
-      result: (group.result = queryResult())
+      result: (group.result = new QueryResult())
     });
   } else {
     // issue queries directly

package/src/QueryManager.js

@@ -1,7 +1,7 @@
 import { consolidator } from './QueryConsolidator.js';
 import { lruCache, voidCache } from './util/cache.js';
 import { priorityQueue } from './util/priority-queue.js';
-import { queryResult } from './util/query-result.js';
+import { QueryResult } from './util/query-result.js';
 
 export const Priority = { High: 0, Normal: 1, Low: 2 };
 
@@ -96,7 +96,7 @@ export class QueryManager {
   }
 
   request(request, priority = Priority.Normal) {
-    const result = queryResult();
+    const result = new QueryResult();
     const entry = { request, result };
     if (this._consolidate) {
       this._consolidate.add(entry, priority);
@@ -108,7 +108,9 @@ export class QueryManager {
 
   cancel(requests) {
     const set = new Set(requests);
-    this.queue.remove(({ result }) => set.has(result));
+    if (set.size) {
+      this.queue.remove(({ result }) => set.has(result));
+    }
   }
 
   clear() {

package/src/Selection.js

@@ -22,10 +22,13 @@ export class Selection extends Param {
    * @param {boolean} [options.cross=false] Boolean flag indicating
    *  cross-filtered resolution. If true, selection clauses will not
    *  be applied to the clients they are associated with.
+   * @param {boolean} [options.empty=false] Boolean flag indicating if a lack
+   *  of clauses should correspond to an empty selection with no records. This
+   *  setting determines the default selection state.
    * @returns {Selection} The new Selection instance.
    */
-  static intersect({ cross = false } = {}) {
-    return new Selection(new SelectionResolver({ cross }));
+  static intersect({ cross = false, empty = false } = {}) {
+    return new Selection(new SelectionResolver({ cross, empty }));
   }
 
   /**
@@ -35,10 +38,13 @@ export class Selection extends Param {
    * @param {boolean} [options.cross=false] Boolean flag indicating
    *  cross-filtered resolution. If true, selection clauses will not
    *  be applied to the clients they are associated with.
+   * @param {boolean} [options.empty=false] Boolean flag indicating if a lack
+   *  of clauses should correspond to an empty selection with no records. This
+   *  setting determines the default selection state.
    * @returns {Selection} The new Selection instance.
    */
-  static union({ cross = false } = {}) {
-    return new Selection(new SelectionResolver({ cross, union: true }));
+  static union({ cross = false, empty = false } = {}) {
+    return new Selection(new SelectionResolver({ cross, empty, union: true }));
   }
 
   /**
@@ -48,19 +54,26 @@ export class Selection extends Param {
    * @param {boolean} [options.cross=false] Boolean flag indicating
    *  cross-filtered resolution. If true, selection clauses will not
    *  be applied to the clients they are associated with.
+   * @param {boolean} [options.empty=false] Boolean flag indicating if a lack
+   *  of clauses should correspond to an empty selection with no records. This
+   *  setting determines the default selection state.
    * @returns {Selection} The new Selection instance.
    */
-  static single({ cross = false } = {}) {
-    return new Selection(new SelectionResolver({ cross, single: true }));
+  static single({ cross = false, empty = false } = {}) {
+    return new Selection(new SelectionResolver({ cross, empty, single: true }));
   }
 
   /**
    * Create a new Selection instance with a
    * cross-filtered intersect resolution strategy.
+   * @param {object} [options] The selection options.
+   * @param {boolean} [options.empty=false] Boolean flag indicating if a lack
+   *  of clauses should correspond to an empty selection with no records. This
+   *  setting determines the default selection state.
    * @returns {Selection} The new Selection instance.
    */
-  static crossfilter() {
-    return new Selection(new SelectionResolver({ cross: true }));
+  static crossfilter({ empty = false } = {}) {
+    return new Selection(new SelectionResolver({ cross: true, empty }));
   }
 
   /**
@@ -232,11 +245,15 @@ export class SelectionResolver {
    *  If false, an intersection strategy is used.
    * @param {boolean} [options.cross=false] Boolean flag to indicate cross-filtering.
    * @param {boolean} [options.single=false] Boolean flag to indicate single clauses only.
+   * @param {boolean} [options.empty=false] Boolean flag indicating if a lack
+   *  of clauses should correspond to an empty selection with no records. This
+   *  setting determines the default selection state.
    */
-  constructor({ union, cross, single } = {}) {
+  constructor({ union, cross, single, empty } = {}) {
     this.union = !!union;
     this.cross = !!cross;
     this.single = !!single;
+    this.empty = !!empty;
   }
 
   /**
@@ -274,7 +291,11 @@ export class SelectionResolver {
    *  based on the current state of this selection.
    */
   predicate(clauseList, active, client) {
-    const { union } = this;
+    const { empty, union } = this;
+
+    if (empty && !clauseList.length) {
+      return ['FALSE'];
+    }
 
     // do nothing if cross-filtering and client is currently active
     if (this.skip(client, active)) return undefined;