@uwdata/mosaic-core 0.9.0 → 0.11.0

package/src/DataCubeIndexer.js

@@ -1,194 +1,269 @@
-import { Query, and, asColumn, create, isBetween, scaleTransform, sql } from '@uwdata/mosaic-sql';
-import { fnv_hash } from './util/hash.js';
+import {
+  Query, and, asColumn, create, isBetween, scaleTransform, sql
+} from '@uwdata/mosaic-sql';
 import { indexColumns } from './util/index-columns.js';
+import { fnv_hash } from './util/hash.js';
+
+const Skip = { skip: true, result: null };
+
+/**
+ * @typedef {object} DataCubeIndexerOptions
+ * @property {string} [schema] Database schema (namespace) in which to write
+ *  data cube index tables (default 'mosaic').
+ * @property {boolean} [options.enabled=true] Flag to enable or disable the
+ *  indexer. This setting can later be updated via the `enabled` method.
+ */
 
 /**
  * Build and query optimized indices ("data cubes") for fast computation of
  * groupby aggregate queries over compatible client queries and selections.
  * A data cube contains pre-aggregated data for a Mosaic client, subdivided
- * by possible query values from an active view. Indexes are realized as
- * as temporary database tables that can be queried for rapid updates.
- * Compatible client queries must pull data from the same backing table and
- * must consist of only groupby dimensions and supported aggregates.
- * Compatible selections must contain an active clause that exposes metadata
- * for an interval or point value predicate.
+ * by possible query values from an active selection clause. These cubes are
+ * realized as as database tables that can be queried for rapid updates.
+ *
+ * Compatible client queries must consist of only groupby dimensions and
+ * supported aggregate functions. Compatible selections must contain an active
+ * clause that exposes metadata for an interval or point value predicate.
+ *
+ * Data cube index tables are written to a dedicated schema (namespace) that
+ * can be set using the *schema* constructor option. This schema acts as a
+ * persistent cache, and index tables may be used across sessions. The
+ * `dropIndexTables` method issues a query to remove *all* tables within
+ * this schema. This may be needed if the original tables have updated data,
+ * but should be used with care.
  */
 export class DataCubeIndexer {
   /**
-   *
-   * @param {import('./Coordinator.js').Coordinator} mc a Mosaic coordinator
-   * @param {*} options Options hash to configure the data cube indexes and pass selections to the coordinator.
+   * Create a new data cube index table manager.
+   * @param {import('./Coordinator.js').Coordinator} coordinator A Mosaic coordinator.
+   * @param {DataCubeIndexerOptions} [options] Data cube indexer options.
    */
-  constructor(mc, { selection, temp = true }) {
-    /** @type import('./Coordinator.js').Coordinator */
-    this.mc = mc;
-    this.selection = selection;
-    this.temp = temp;
-    this.reset();
-  }
-
-  reset() {
-    this.enabled = false;
-    this.clients = null;
-    this.indices = null;
+  constructor(coordinator, {
+    schema = 'mosaic',
+    enabled = true
+  } = {}) {
+    /** @type {Map<import('./MosaicClient.js').MosaicClient, DataCubeInfo | Skip | null>} */
+    this.indexes = new Map();
     this.active = null;
+    this.mc = coordinator;
+    this._schema = schema;
+    this._enabled = enabled;
   }
 
-  clear() {
-    if (this.indices) {
-      this.mc.cancel(Array.from(this.indices.values(), index => index?.result));
-      this.indices = null;
+  /**
+   * Set the enabled state of this indexer. If false, any local state is
+   * cleared and subsequent index calls will return null until re-enabled.
+   * This method has no effect on any index tables already in the database.
+   * @param {boolean} [state] The enabled state to set.
+   */
+  set enabled(state) {
+    if (this._enabled !== state) {
+      if (!state) this.clear();
+      this._enabled = state;
     }
   }
 
-  index(clients, activeClause) {
-    if (this.clients !== clients) {
-      // test client views for compatibility
-      const cols = Array.from(clients, indexColumns).filter(x => x);
-      const from = cols[0]?.from;
-      this.enabled = cols.length && cols.every(c => c.from === from);
-      this.clients = clients;
-      this.active = null;
+  /**
+   * Get the enabled state of this indexer.
+   * @returns {boolean} The current enabled state.
+   */
+  get enabled() {
+    return this._enabled;
+  }
+
+  /**
+   * Set the database schema used by this indexer. Upon changes, any local
+   * state is cleared. This method does _not_ drop any existing data cube
+   * tables, use `dropIndexTables` before changing the schema to also remove
+   * existing index tables in the database.
+   * @param {string} [schema] The schema name to set.
+   */
+  set schema(schema) {
+    if (this._schema !== schema) {
       this.clear();
+      this._schema = schema;
     }
-    if (!this.enabled) return false; // client views are not indexable
+  }
 
-    activeClause = activeClause || this.selection.active;
-    const { source } = activeClause;
-    // exit early if indexes already set up for active view
-    if (source && source === this.active?.source) return true;
+  /**
+   * Get the database schema used by this indexer.
+   * @returns {string} The current schema name.
+   */
+  get schema() {
+    return this._schema;
+  }
 
+  /**
+   * Issues a query through the coordinator to drop the current index table
+   * schema. *All* tables in the schema will be removed and local state is
+   * cleared. Call this method if the underlying base tables have been updated,
+   * causing derived index tables to become stale and inaccurate. Use this
+   * method with care! Once dropped, the schema will be repopulated by future
+   * data cube indexer requests.
+   * @returns A query result promise.
+   */
+  dropIndexTables() {
     this.clear();
-    if (!source) return false; // nothing to work with
-    const active = this.active = activeColumns(activeClause);
-    if (!active) return false; // active selection clause not compatible
-
-    const logger = this.mc.logger();
-    logger.warn('DATA CUBE INDEX CONSTRUCTION');
-
-    // create a selection with the active source removed
-    const sel = this.selection.remove(source);
-
-    // generate data cube indices
-    const indices = this.indices = new Map;
-    const { mc, temp } = this;
-    for (const client of clients) {
-      // determine if client should be skipped due to cross-filtering
-      if (sel.skip(client, activeClause)) {
-        indices.set(client, null);
-        continue;
-      }
-
-      // generate column definitions for data cube and cube queries
-      const index = indexColumns(client);
-
-      // skip if client is not indexable
-      if (!index) {
-        continue;
-      }
-
-      // build index table construction query
-      const query = client.query(sel.predicate(client))
-        .select({ ...active.columns, ...index.aux })
-        .groupby(Object.keys(active.columns));
-
-      // ensure active view columns are selected by subqueries
-      const [subq] = query.subqueries;
-      if (subq) {
-        const cols = Object.values(active.columns).flatMap(c => c.columns);
-        subqueryPushdown(subq, cols);
-      }
-
-      // push orderby criteria to later cube queries
-      const order = query.orderby();
-      query.query.orderby = [];
-
-      const sql = query.toString();
-      const id = (fnv_hash(sql) >>> 0).toString(16);
-      const table = `cube_index_${id}`;
-      const result = mc.exec(create(table, sql, { temp }));
-      result.catch(e => logger.error(e));
-      indices.set(client, { table, result, order, ...index });
-    }
-
-    // index creation successful
-    return true;
+    return this.mc.exec(`DROP SCHEMA IF EXISTS "${this.schema}" CASCADE`);
   }
 
-  async update() {
-    const { clients, selection, active } = this;
-    const filter = active.predicate(selection.active.predicate);
-    return Promise.all(
-      Array.from(clients).map(client => this.updateClient(client, filter))
-    );
+  /**
+   * Clear the cache of data cube index table entries for the current active
+   * selection clause. This method does _not_ drop any existing data cube
+   * tables. Use `dropIndexTables` to remove existing index tables from the
+   * database.
+   */
+  clear() {
+    this.indexes.clear();
+    this.active = null;
   }
 
-  async updateClient(client, filter) {
-    const { mc, indices, selection } = this;
-
-    // if client has no index, perform a standard update
-    if (!indices.has(client)) {
-      filter = selection.predicate(client);
-      return mc.updateClient(client, client.query(filter));
-    };
-
-    const index = this.indices.get(client);
-
-    // skip update if cross-filtered
-    if (!index) return;
-
-    // otherwise, query a data cube index table
-    const { table, dims, aggr, order = [] } = index;
-    const query = Query
-      .select(dims, aggr)
-      .from(table)
-      .groupby(dims)
-      .where(filter)
-      .orderby(order);
-    return mc.updateClient(client, query);
+  /**
+   * Return data cube index table information for the active state of a
+   * client-selection pair, or null if the client is not indexable. This
+   * method has multiple possible side effects, including data cube table
+   * generation and updating internal caches.
+   * @param {import('./MosaicClient.js').MosaicClient} client A Mosaic client.
+   * @param {import('./Selection.js').Selection} selection A Mosaic selection
+   *  to filter the client by.
+   * @param {import('./util/selection-types.js').SelectionClause} activeClause
+   *  A representative active selection clause for which to (possibly) generate
+   *  data cube index tables.
+   * @returns {DataCubeInfo | Skip | null} Data cube index table
+   *  information and query generator, or null if the client is not indexable.
+   */
+  index(client, selection, activeClause) {
+    // if not enabled, do nothing
+    if (!this.enabled) return null;
+
+    const { indexes, mc, schema } = this;
+    const { source } = activeClause;
+
+    // if there is no clause source to track, do nothing
+    if (!source) return null;
+
+    // if we have cached active columns, check for updates or exit
+    if (this.active) {
+      // if the active clause source has changed, clear indexer state
+      // this cancels outstanding requests and clears the index cache
+      // a clear also sets this.active to null
+      if (this.active.source !== source) this.clear();
+      // if we've seen this source and it's not indexable, do nothing
+      if (this.active?.source === null) return null;
+    }
+
+    // the current active columns cache value
+    let { active } = this;
+
+    // if cached active columns are unset, analyze the active clause
+    if (!active) {
+      // generate active data cube dimension columns to select over
+      // will return an object with null source if not indexable
+      this.active = active = activeColumns(activeClause);
+      // if the active clause is not indexable, exit now
+      if (active.source === null) return null;
+    }
+
+    // if we have cached data cube index table info, return that
+    if (indexes.has(client)) {
+      return indexes.get(client);
+    }
+
+    // get non-active data cube index table columns
+    const indexCols = indexColumns(client);
+
+    let info;
+    if (!indexCols) {
+      // if client is not indexable, record null index
+      info = null;
+    } else if (selection.skip(client, activeClause)) {
+      // skip client if untouched by cross-filtering
+      info = Skip;
+    } else {
+      // generate data cube index table
+      const filter = selection.remove(source).predicate(client);
+      info = dataCubeInfo(client.query(filter), active, indexCols, schema);
+      info.result = mc.exec([
+        `CREATE SCHEMA IF NOT EXISTS ${schema}`,
+        create(info.table, info.create, { temp: false })
+      ]);
+      info.result.catch(e => mc.logger().error(e));
+    }
+
+    indexes.set(client, info);
+    return info;
   }
 }
 
+/**
+ * Determines the active data cube dimension columns to select over. Returns
+ * an object with the clause source, column definitions, and a predicate
+ * generator function for the active dimensions of a data cube index table. If
+ * the active clause is not indexable or is missing metadata, this method
+ * returns an object with a null source property.
+ * @param {import('./util/selection-types.js').SelectionClause} clause The
+ *  active selection clause to analyze.
+ */
 function activeColumns(clause) {
   const { source, meta } = clause;
-  let columns = clause.predicate?.columns;
-  if (!meta || !columns) return null;
-  const { type, scales, bin, pixelSize = 1 } = meta;
+  const clausePred = clause.predicate;
+  const clauseCols = clausePred?.columns;
   let predicate;
+  let columns;
 
-  if (type === 'interval' && scales) {
+  if (!meta || !clauseCols) {
+    return { source: null, columns, predicate };
+  }
+
+  // @ts-ignore
+  const { type, scales, bin, pixelSize = 1 } = meta;
+
+  if (type === 'point') {
+    predicate = x => x;
+    columns = Object.fromEntries(
+      clauseCols.map(col => [`${col}`, asColumn(col)])
+    );
+  } else if (type === 'interval' && scales) {
     // determine pixel-level binning
     const bins = scales.map(s => binInterval(s, pixelSize, bin));
 
-    // bail if the scale type is unsupported
-    if (bins.some(b => b == null)) return null;
-
-    if (bins.length === 1) {
+    if (bins.some(b => !b)) {
+      // bail if a scale type is unsupported
+    } else if (bins.length === 1) {
       // single interval selection
       predicate = p => p ? isBetween('active0', p.range.map(bins[0])) : [];
-      columns = { active0: bins[0](clause.predicate.field) };
+      // @ts-ignore
+      columns = { active0: bins[0](clausePred.field) };
     } else {
       // multiple interval selection
       predicate = p => p
-        ? and(p.children.map(({ range }, i) => isBetween(`active${i}`, range.map(bins[i]))))
+        ? and(p.children.map(
+            ({ range }, i) => isBetween(`active${i}`, range.map(bins[i]))
+          ))
         : [];
       columns = Object.fromEntries(
-        clause.predicate.children.map((p, i) => [`active${i}`, bins[i](p.field)])
+        // @ts-ignore
+        clausePred.children.map((p, i) => [`active${i}`, bins[i](p.field)])
       );
     }
-  } else if (type === 'point') {
-    predicate = x => x;
-    columns = Object.fromEntries(columns.map(col => [`${col}`, asColumn(col)]));
-  } else {
-    // unsupported selection type
-    return null;
   }
 
-  return { source, columns, predicate };
+  return { source: columns ? source : null, columns, predicate };
 }
 
 const BIN = { ceil: 'CEIL', round: 'ROUND' };
 
+/**
+ * Returns a bin function generator to discretize a selection interval domain.
+ * @param {import('./util/selection-types.js').Scale} scale A scale that maps
+ *  domain values to the output range (typically pixels).
+ * @param {number} pixelSize The interactive pixel size. This value indicates
+ *  the bin step size and may be greater than an actual screen pixel.
+ * @param {import('./util/selection-types.js').BinMethod} bin The binning
+ *  method to apply, one of `floor`, `ceil', or `round`.
+ * @returns {(value: any) => import('@uwdata/mosaic-sql').SQLExpression}
+ *  A bin function generator.
+ */
 function binInterval(scale, pixelSize, bin) {
   const { type, domain, range, apply, sqlApply } = scaleTransform(scale);
   if (!apply) return; // unsupported scale type
@@ -201,6 +276,51 @@ function binInterval(scale, pixelSize, bin) {
   return value => sql`${fn}(${s}(${sqlApply(value)}${d}))::INTEGER`;
 }
 
+/**
+ * Generate data cube table query information.
+ * @param {Query} clientQuery The original client query.
+ * @param {*} active Active (selected) column definitions.
+ * @param {*} indexCols Data cube index column definitions.
+ * @returns {DataCubeInfo}
+ */
+function dataCubeInfo(clientQuery, active, indexCols, schema) {
+  const { dims, aggr, aux } = indexCols;
+  const { columns } = active;
+
+  // build index table construction query
+  const query = clientQuery
+    .select({ ...columns, ...aux })
+    .groupby(Object.keys(columns));
+
+  // ensure active clause columns are selected by subqueries
+  const [subq] = query.subqueries;
+  if (subq) {
+    const cols = Object.values(columns).flatMap(c => c.columns);
+    subqueryPushdown(subq, cols);
+  }
+
+  // push orderby criteria to later cube queries
+  const order = query.orderby();
+  query.query.orderby = [];
+
+  // generate creation query string and hash id
+  const create = query.toString();
+  const id = (fnv_hash(create) >>> 0).toString(16);
+  const table = `${schema}.cube_${id}`;
+
+  // generate data cube select query
+  const select = Query
+    .select(dims, aggr)
+    .from(table)
+    .groupby(dims)
+    .orderby(order);
+
+  return new DataCubeInfo({ id, table, create, active, select });
+}
+
+/**
+ * Push column selections down to subqueries.
+ */
 function subqueryPushdown(query, cols) {
   const memo = new Set;
   const pushdown = q => {
@@ -213,3 +333,46 @@ function subqueryPushdown(query, cols) {
   };
   pushdown(query);
 }
+
+/**
+ * Metadata and query generator for a data cube index table. This
+ * object provides the information needed to generate and query
+ * a data cube index table for a client-selection pair relative to
+ * a specific active clause and selection state.
+ */
+export class DataCubeInfo {
+  /**
+   * Create a new DataCubeInfo instance.
+   * @param {object} options
+   */
+  constructor({ table, create, active, select } = {}) {
+    /** The name of the data cube index table. */
+    this.table = table;
+    /** The SQL query used to generate the data cube index table. */
+    this.create = create;
+    /** A result promise returned for the data cube creation query. */
+    this.result = null;
+    /**
+     * Definitions and predicate function for the active columns,
+     * which are dynamically filtered by the active clause.
+     */
+    this.active = active;
+    /** Select query (sans where clause) for data cube tables. */
+    this.select = select;
+    /**
+     * Boolean flag indicating a client that should be skipped.
+     * This value is always false for completed data cube info.
+     */
+    this.skip = false;
+  }
+
+  /**
+   * Generate a data cube index table query for the given predicate.
+   * @param {import('@uwdata/mosaic-sql').SQLExpression} predicate The current
+   *  active clause predicate.
+   * @returns {Query} A data cube index table query.
+   */
+  query(predicate) {
+    return this.select.clone().where(this.active.predicate(predicate));
+  }
+}

package/src/MosaicClient.js

@@ -94,8 +94,8 @@ export class MosaicClient {
    * @param {*} error
    * @returns {this}
    */
-  queryError(error) {
-    console.error(error);
+  queryError(error) { // eslint-disable-line no-unused-vars
+    // do nothing, the coordinator logs the error
     return this;
   }
 
@@ -103,6 +103,7 @@ export class MosaicClient {
    * Request the coordinator to execute a query for this client.
    * If an explicit query is not provided, the client query method will
    * be called, filtered by the current filterBy selection.
+   * @returns {Promise}
    */
   requestQuery(query) {
     const q = query || this.query(this.filterBy?.predicate(this));
@@ -119,10 +120,18 @@ export class MosaicClient {
     this._requestUpdate();
   }
 
+  /**
+   * Reset this client, initiating new field info and query requests.
+   * @returns {Promise}
+   */
+  initialize() {
+    return this._coordinator.initializeClient(this);
+  }
+
   /**
    * Requests a client update.
    * For example to (re-)render an interface component.
-   * 
+   *
    * @returns {this | Promise<any>}
    */
   update() {

package/src/QueryConsolidator.js

@@ -1,5 +1,5 @@
 import { Query, Ref, isDescribeQuery } from '@uwdata/mosaic-sql';
-import { queryResult } from './util/query-result.js';
+import { QueryResult } from './util/query-result.js';
 
 function wait(callback) {
   const method = typeof requestAnimationFrame !== 'undefined'
@@ -108,6 +108,12 @@ function consolidationKey(query, cache) {
       // @ts-ignore
       q.$groupby(groupby.map(e => (e instanceof Ref && map[e.column]) || e));
     }
+    // @ts-ignore
+    else if (query.select().some(({ expr }) => expr.aggregate)) {
+      // if query is an ungrouped aggregate, add an explicit groupby to
+      // prevent improper consolidation with non-aggregate queries
+      q.$groupby('ALL');
+    }
 
     // key is just the transformed query as SQL
     return `${q}`;
@@ -133,7 +139,7 @@ function consolidate(group, enqueue, record) {
         record: false,
         query: (group.query = consolidatedQuery(group, record))
       },
-      result: (group.result = queryResult())
+      result: (group.result = new QueryResult())
     });
   } else {
     // issue queries directly
@@ -244,22 +250,20 @@ async function processResults(group, cache) {
 
 /**
  * Project a consolidated result to a client result
- * @param {*} data Consolidated query result, as an Apache Arrow Table
- * @param {*} map Column name map as [source, target] pairs
+ * @param {import('@uwdata/flechette').Table} data
+ *  Consolidated query result, as an Arrow Table
+ * @param {[string, string][]} map Column name map as [source, target] pairs
  * @returns the projected Apache Arrow table
  */
 function projectResult(data, map) {
-  const cols = {};
-  for (const [name, as] of map) {
-    cols[as] = data.getChild(name);
-  }
-  return new data.constructor(cols);
+  return data.select(map.map(x => x[0]), map.map(x => x[1]));
 }
 
 /**
  * Filter a consolidated describe query result to a client result
- * @param {*} data Consolidated query result
- * @param {*} map Column name map as [source, target] pairs
+ * @param {import('@uwdata/flechette').Table} data
+ *  Consolidated query result, as an Arrow Table
+ * @param {[string, string][]} map Column name map as [source, target] pairs
  * @returns the filtered table data
  */
 function filterResult(data, map) {

package/src/QueryManager.js

@@ -1,13 +1,13 @@
 import { consolidator } from './QueryConsolidator.js';
 import { lruCache, voidCache } from './util/cache.js';
-import { priorityQueue } from './util/priority-queue.js';
-import { queryResult } from './util/query-result.js';
+import { PriorityQueue } from './util/priority-queue.js';
+import { QueryResult } from './util/query-result.js';
 
 export const Priority = { High: 0, Normal: 1, Low: 2 };
 
 export class QueryManager {
   constructor() {
-    this.queue = priorityQueue(3);
+    this.queue = new PriorityQueue(3);
     this.db = null;
     this.clientCache = null;
     this._logger = null;
@@ -96,7 +96,7 @@ export class QueryManager {
   }
 
   request(request, priority = Priority.Normal) {
-    const result = queryResult();
+    const result = new QueryResult();
     const entry = { request, result };
     if (this._consolidate) {
       this._consolidate.add(entry, priority);
@@ -108,7 +108,15 @@ export class QueryManager {
 
   cancel(requests) {
     const set = new Set(requests);
-    this.queue.remove(({ result }) => set.has(result));
+    if (set.size) {
+      this.queue.remove(({ result }) => {
+        if (set.has(result)) {
+          result.reject('Canceled');
+          return true;
+        }
+        return false;
+      });
+    }
   }
 
   clear() {