@uwdata/mosaic-core 0.7.1 → 0.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uwdata/mosaic-core",
-  "version": "0.7.1",
+  "version": "0.9.0",
   "description": "Scalable and extensible linked data views.",
   "keywords": [
     "mosaic",
@@ -23,14 +23,17 @@
   "scripts": {
     "prebuild": "rimraf dist && mkdir dist",
     "build": "node ../../esbuild.js mosaic-core",
-    "lint": "eslint src test --ext .js",
+    "lint": "eslint src test",
     "test": "mocha 'test/**/*-test.js'",
     "prepublishOnly": "npm run test && npm run lint && npm run build"
   },
   "dependencies": {
-    "@duckdb/duckdb-wasm": "^1.28.1-dev109.0",
-    "@uwdata/mosaic-sql": "^0.7.0",
-    "apache-arrow": "^15.0.0"
+    "@duckdb/duckdb-wasm": "^1.28.1-dev195.0",
+    "@uwdata/mosaic-sql": "^0.9.0",
+    "apache-arrow": "^15.0.2"
   },
-  "gitHead": "7e6f3ea9b3011ea2c9201c1aa16e8e5664621a4c"
+  "devDependencies": {
+    "@uwdata/mosaic-duckdb": "^0.9.0"
+  },
+  "gitHead": "89bb9b0dfa747aed691eaeba35379525a6764c61"
 }

package/src/Coordinator.js

@@ -8,8 +8,7 @@ let _instance;
 
 /**
  * Set or retrieve the coordinator instance.
- *
- * @param {Coordinator} instance the coordinator instance to set
+ * @param {Coordinator} [instance] the coordinator instance to set
  * @returns {Coordinator} the coordinator instance
  */
 export function coordinator(instance) {
@@ -25,7 +24,7 @@ export class Coordinator {
   constructor(db = socketConnector(), options = {}) {
     const {
       logger = console,
-      manager = QueryManager()
+      manager = new QueryManager()
     } = options;
     this.manager = manager;
     this.logger(logger);
@@ -42,7 +41,15 @@ export class Coordinator {
     return this._logger;
   }
 
-  configure({ cache = true, consolidate = true, indexes = true }) {
+  /**
+   * Set configuration options for this coordinator.
+   * @param {object} [options] Configration options.
+   * @param {boolean} [options.cache=true] Boolean flag to enable/disable query caching.
+   * @param {boolean} [options.consolidate=true] Boolean flag to enable/disable query consolidation.
+   * @param {boolean|object} [options.indexes=true] Boolean flag to enable/disable
+   *  automatic data cube indexes or an index options object.
+   */
+  configure({ cache = true, consolidate = true, indexes = true } = {}) {
     this.manager.cache(cache);
     this.manager.consolidate(consolidate);
     this.indexes = indexes;
@@ -116,7 +123,6 @@ export class Coordinator {
 
   /**
    * Connect a client to the coordinator.
-   *
    * @param {import('./MosaicClient.js').MosaicClient} client the client to disconnect
    */
   async connect(client) {

package/src/DataCubeIndexer.js

@@ -1,5 +1,6 @@
-import { Query, and, create, isBetween, scaleTransform, sql } from '@uwdata/mosaic-sql';
+import { Query, and, asColumn, create, isBetween, scaleTransform, sql } from '@uwdata/mosaic-sql';
 import { fnv_hash } from './util/hash.js';
+import { indexColumns } from './util/index-columns.js';
 
 /**
  * Build and query optimized indices ("data cubes") for fast computation of
@@ -9,7 +10,7 @@ import { fnv_hash } from './util/hash.js';
  * as temporary database tables that can be queried for rapid updates.
  * Compatible client queries must pull data from the same backing table and
  * must consist of only groupby dimensions and supported aggregates.
- * Compatible selections must contain an active clause that exposes a schema
+ * Compatible selections must contain an active clause that exposes metadata
  * for an interval or point value predicate.
  */
 export class DataCubeIndexer {
@@ -30,38 +31,40 @@ export class DataCubeIndexer {
     this.enabled = false;
     this.clients = null;
     this.indices = null;
-    this.activeView = null;
+    this.active = null;
   }
 
   clear() {
     if (this.indices) {
-      this.mc.cancel(Array.from(this.indices.values(), index => index.result));
+      this.mc.cancel(Array.from(this.indices.values(), index => index?.result));
       this.indices = null;
     }
   }
 
-  index(clients, active) {
+  index(clients, activeClause) {
     if (this.clients !== clients) {
       // test client views for compatibility
-      const cols = Array.from(clients, getIndexColumns);
+      const cols = Array.from(clients, indexColumns).filter(x => x);
       const from = cols[0]?.from;
-      this.enabled = cols.every(c => c && c.from === from);
+      this.enabled = cols.length && cols.every(c => c.from === from);
       this.clients = clients;
-      this.activeView = null;
+      this.active = null;
       this.clear();
     }
     if (!this.enabled) return false; // client views are not indexable
 
-    active = active || this.selection.active;
-    const { source } = active;
-    if (source && source === this.activeView?.source) return true; // we're good!
+    activeClause = activeClause || this.selection.active;
+    const { source } = activeClause;
+    // exit early if indexes already set up for active view
+    if (source && source === this.active?.source) return true;
 
     this.clear();
     if (!source) return false; // nothing to work with
-    const activeView = this.activeView = getActiveView(active);
-    if (!activeView) return false; // active selection clause not compatible
+    const active = this.active = activeColumns(activeClause);
+    if (!active) return false; // active selection clause not compatible
 
-    this.mc.logger().warn('DATA CUBE INDEX CONSTRUCTION');
+    const logger = this.mc.logger();
+    logger.warn('DATA CUBE INDEX CONSTRUCTION');
 
     // create a selection with the active source removed
     const sel = this.selection.remove(source);
@@ -70,18 +73,29 @@ export class DataCubeIndexer {
     const indices = this.indices = new Map;
     const { mc, temp } = this;
     for (const client of clients) {
-      if (sel.skip(client, active)) continue;
-      const index = getIndexColumns(client);
+      // determine if client should be skipped due to cross-filtering
+      if (sel.skip(client, activeClause)) {
+        indices.set(client, null);
+        continue;
+      }
+
+      // generate column definitions for data cube and cube queries
+      const index = indexColumns(client);
+
+      // skip if client is not indexable
+      if (!index) {
+        continue;
+      }
 
-      // build index construction query
+      // build index table construction query
       const query = client.query(sel.predicate(client))
-        .select({ ...activeView.columns, ...index.count })
-        .groupby(Object.keys(activeView.columns));
+        .select({ ...active.columns, ...index.aux })
+        .groupby(Object.keys(active.columns));
 
       // ensure active view columns are selected by subqueries
       const [subq] = query.subqueries;
       if (subq) {
-        const cols = Object.values(activeView.columns).map(c => c.columns[0]);
+        const cols = Object.values(active.columns).flatMap(c => c.columns);
         subqueryPushdown(subq, cols);
       }
 
@@ -93,26 +107,37 @@ export class DataCubeIndexer {
       const id = (fnv_hash(sql) >>> 0).toString(16);
       const table = `cube_index_${id}`;
       const result = mc.exec(create(table, sql, { temp }));
+      result.catch(e => logger.error(e));
       indices.set(client, { table, result, order, ...index });
     }
+
+    // index creation successful
+    return true;
   }
 
   async update() {
-    const { clients, selection, activeView } = this;
-    const filter = activeView.predicate(selection.active.predicate);
+    const { clients, selection, active } = this;
+    const filter = active.predicate(selection.active.predicate);
     return Promise.all(
       Array.from(clients).map(client => this.updateClient(client, filter))
     );
   }
 
   async updateClient(client, filter) {
+    const { mc, indices, selection } = this;
+
+    // if client has no index, perform a standard update
+    if (!indices.has(client)) {
+      filter = selection.predicate(client);
+      return mc.updateClient(client, client.query(filter));
+    };
+
     const index = this.indices.get(client);
-    if (!index) return;
 
-    if (!filter) {
-      filter = this.activeView.predicate(this.selection.active.predicate);
-    }
+    // skip update if cross-filtered
+    if (!index) return;
 
+    // otherwise, query a data cube index table
     const { table, dims, aggr, order = [] } = index;
     const query = Query
       .select(dims, aggr)
@@ -120,25 +145,30 @@ export class DataCubeIndexer {
       .groupby(dims)
       .where(filter)
       .orderby(order);
-    return this.mc.updateClient(client, query);
+    return mc.updateClient(client, query);
   }
 }
 
-function getActiveView(clause) {
-  const { source, schema } = clause;
+function activeColumns(clause) {
+  const { source, meta } = clause;
   let columns = clause.predicate?.columns;
-  if (!schema || !columns) return null;
-  const { type, scales, pixelSize = 1 } = schema;
+  if (!meta || !columns) return null;
+  const { type, scales, bin, pixelSize = 1 } = meta;
   let predicate;
 
   if (type === 'interval' && scales) {
-    const bins = scales.map(s => binInterval(s, pixelSize));
-    if (bins.some(b => b == null)) return null; // unsupported scale type
+    // determine pixel-level binning
+    const bins = scales.map(s => binInterval(s, pixelSize, bin));
+
+    // bail if the scale type is unsupported
+    if (bins.some(b => b == null)) return null;
 
     if (bins.length === 1) {
+      // single interval selection
       predicate = p => p ? isBetween('active0', p.range.map(bins[0])) : [];
       columns = { active0: bins[0](clause.predicate.field) };
     } else {
+      // multiple interval selection
       predicate = p => p
         ? and(p.children.map(({ range }, i) => isBetween(`active${i}`, range.map(bins[i]))))
         : [];
@@ -148,87 +178,27 @@ function getActiveView(clause) {
     }
   } else if (type === 'point') {
     predicate = x => x;
-    columns = Object.fromEntries(columns.map(col => [col.toString(), col]));
+    columns = Object.fromEntries(columns.map(col => [`${col}`, asColumn(col)]));
   } else {
-    return null; // unsupported type
+    // unsupported selection type
+    return null;
   }
 
   return { source, columns, predicate };
 }
 
-function binInterval(scale, pixelSize) {
-  const { apply, sqlApply } = scaleTransform(scale);
-  if (apply) {
-    const { domain, range } = scale;
-    const lo = apply(Math.min(...domain));
-    const hi = apply(Math.max(...domain));
-    const a = (Math.abs(range[1] - range[0]) / (hi - lo)) / pixelSize;
-    const s = pixelSize === 1 ? '' : `${pixelSize}::INTEGER * `;
-    return value => sql`${s}FLOOR(${a}::DOUBLE * (${sqlApply(value)} - ${lo}::DOUBLE))::INTEGER`;
-  }
-}
-
-const NO_INDEX = { from: NaN };
-
-function getIndexColumns(client) {
-  if (!client.filterIndexable) return NO_INDEX;
-  const q = client.query();
-  const from = getBaseTable(q);
-  if (!from || !q.groupby) return NO_INDEX;
-  const g = new Set(q.groupby().map(c => c.column));
-
-  const aggr = [];
-  const dims = [];
-  let count;
-
-  for (const { as, expr: { aggregate } } of q.select()) {
-    switch (aggregate?.toUpperCase?.()) {
-      case 'COUNT':
-      case 'SUM':
-        aggr.push({ [as]: sql`SUM("${as}")::DOUBLE` });
-        break;
-      case 'AVG':
-        count = '_count_';
-        aggr.push({ [as]: sql`(SUM("${as}" * ${count}) / SUM(${count}))::DOUBLE` });
-        break;
-      case 'MAX':
-        aggr.push({ [as]: sql`MAX("${as}")` });
-        break;
-      case 'MIN':
-        aggr.push({ [as]: sql`MIN("${as}")` });
-        break;
-      default:
-        if (g.has(as)) dims.push(as);
-        else return null;
-    }
-  }
-
-  return {
-    aggr,
-    dims,
-    count: count ? { [count]: sql`COUNT(*)` } : {},
-    from
-  };
-}
-
-function getBaseTable(query) {
-  const subq = query.subqueries;
-
-  // select query
-  if (query.select) {
-    const from = query.from();
-    if (!from.length) return undefined;
-    if (subq.length === 0) return from[0].from.table;
-  }
-
-  // handle set operations / subqueries
-  const base = getBaseTable(subq[0]);
-  for (let i = 1; i < subq.length; ++i) {
-    const from = getBaseTable(subq[i]);
-    if (from === undefined) continue;
-    if (from !== base) return NaN;
-  }
-  return base;
+const BIN = { ceil: 'CEIL', round: 'ROUND' };
+
+function binInterval(scale, pixelSize, bin) {
+  const { type, domain, range, apply, sqlApply } = scaleTransform(scale);
+  if (!apply) return; // unsupported scale type
+  const fn = BIN[`${bin}`.toLowerCase()] || 'FLOOR';
+  const lo = apply(Math.min(...domain));
+  const hi = apply(Math.max(...domain));
+  const a = type === 'identity' ? 1 : Math.abs(range[1] - range[0]) / (hi - lo);
+  const s = a / pixelSize === 1 ? '' : `${a / pixelSize}::DOUBLE * `;
+  const d = lo === 0 ? '' : ` - ${lo}::DOUBLE`;
+  return value => sql`${fn}(${s}(${sqlApply(value)}${d}))::INTEGER`;
 }
 
 function subqueryPushdown(query, cols) {

package/src/FilterGroup.js

@@ -1,24 +1,27 @@
+import { Coordinator } from './Coordinator.js';
 import { DataCubeIndexer } from './DataCubeIndexer.js';
+import { MosaicClient } from './MosaicClient.js';
+import { Selection } from './Selection.js';
 
 export class FilterGroup {
   /**
-   * @param {import('./Coordinator.js').Coordinator} coordinator The Mosaic coordinator.
-   * @param {*} selection The shared filter selection.
-   * @param {*} index Boolean flag or options hash for data cube indexer.
-   *  Falsy values disable indexing.
+   * @param {Coordinator} coordinator The Mosaic coordinator.
+   * @param {Selection} selection The shared filter selection.
+   * @param {object|boolean} index Boolean flag or options hash for
+   *  a data cube indexer. Falsy values disable indexing.
    */
   constructor(coordinator, selection, index = true) {
-    /** @type import('./Coordinator.js').Coordinator */
     this.mc = coordinator;
     this.selection = selection;
+    /** @type {Set<MosaicClient>} */
     this.clients = new Set();
-    this.indexer = index
-      ? new DataCubeIndexer(this.mc, { ...index, selection })
-      : null;
+    /** @type {DataCubeIndexer | null} */
+    this.indexer = null;
+    this.index(index);
 
     const { value, activate } = this.handlers = {
       value: () => this.update(),
-      activate: clause => this.indexer?.index(this.clients, clause)
+      activate: clause => { this.indexer?.index(this.clients, clause); }
     };
     selection.addEventListener('value', value);
     selection.addEventListener('activate', activate);
@@ -30,6 +33,14 @@ export class FilterGroup {
     this.selection.removeEventListener('activate', activate);
   }
 
+  index(state) {
+    const { selection } = this;
+    const { resolver } = selection;
+    this.indexer = state && (resolver.single || !resolver.union)
+      ? new DataCubeIndexer(this.mc, { ...state, selection })
+      : null;
+  }
+
   reset() {
     this.indexer?.reset();
   }
@@ -46,9 +57,15 @@ export class FilterGroup {
     return this;
   }
 
+  /**
+   * Internal method to process a selection update.
+   * The return value is passed as a selection callback value.
+   * @returns {Promise} A Promise that resolves when the update completes.
+   */
   update() {
     const { mc, indexer, clients, selection } = this;
-    return indexer?.index(clients)
+    const hasIndex = indexer?.index(clients);
+    return hasIndex
       ? indexer.update()
       : defaultUpdate(mc, clients, selection);
   }

package/src/MosaicClient.js

@@ -48,6 +48,7 @@ export class MosaicClient {
 
   /**
    * Return an array of fields queried by this client.
+   * @returns {object[]|null} The fields to retrieve info for.
    */
   fields() {
     return null;
@@ -55,21 +56,25 @@ export class MosaicClient {
 
   /**
    * Called by the coordinator to set the field info for this client.
+   * @param {*} info The field info result.
    * @returns {this}
    */
-  fieldInfo() {
+  fieldInfo(info) { // eslint-disable-line no-unused-vars
     return this;
   }
 
   /**
    * Return a query specifying the data needed by this client.
+   * @param {*} [filter] The filtering criteria to apply in the query.
+   * @returns {*} The client query
    */
-  query() {
+  query(filter) { // eslint-disable-line no-unused-vars
     return null;
   }
 
   /**
    * Called by the coordinator to inform the client that a query is pending.
+   * @returns {this}
    */
   queryPending() {
     return this;
@@ -77,16 +82,17 @@ export class MosaicClient {
 
   /**
    * Called by the coordinator to return a query result.
-   *
-   * @param {*} data the query result
+   * @param {*} data The query result.
    * @returns {this}
    */
-  queryResult() {
+  queryResult(data) { // eslint-disable-line no-unused-vars
     return this;
   }
 
   /**
    * Called by the coordinator to report a query execution error.
+   * @param {*} error
+   * @returns {this}
    */
   queryError(error) {
     console.error(error);
@@ -116,6 +122,8 @@ export class MosaicClient {
   /**
    * Requests a client update.
    * For example to (re-)render an interface component.
+   * 
+   * @returns {this | Promise<any>}
    */
   update() {
     return this;

package/src/Param.js

@@ -4,7 +4,7 @@ import { distinct } from './util/distinct.js';
 /**
  * Test if a value is a Param instance.
  * @param {*} x The value to test.
- * @returns {boolean} True if the input is a Param, false otherwise.
+ * @returns {x is Param} True if the input is a Param, false otherwise.
  */
 export function isParam(x) {
   return x instanceof Param;
@@ -43,7 +43,9 @@ export class Param extends AsyncDispatch {
   static array(values) {
     if (values.some(v => isParam(v))) {
       const p = new Param();
-      const update = () => p.update(values.map(v => isParam(v) ? v.value : v));
+      const update = () => {
+        p.update(values.map(v => isParam(v) ? v.value : v));
+      };
       update();
       values.forEach(v => isParam(v) ? v.addEventListener('value', update) : 0);
       return p;

package/src/QueryConsolidator.js

@@ -5,6 +5,7 @@ function wait(callback) {
   const method = typeof requestAnimationFrame !== 'undefined'
     ? requestAnimationFrame
     : typeof setImmediate !== 'undefined' ? setImmediate : setTimeout;
+  // @ts-ignore
   return method(callback);
 }
 
@@ -82,7 +83,9 @@ function consolidationKey(query, cache) {
   const sql = `${query}`;
   if (query instanceof Query && !cache.get(sql)) {
     if (
+      // @ts-ignore
       query.orderby().length || query.where().length ||
+      // @ts-ignore
       query.qualify().length || query.having().length
     ) {
       // do not try to analyze if query includes clauses
@@ -97,9 +100,12 @@ function consolidationKey(query, cache) {
     // queries may refer to *derived* columns as group by criteria
     // we resolve these against the true grouping expressions
     const groupby = query.groupby();
+    // @ts-ignore
     if (groupby.length) {
       const map = {}; // expression map (as -> expr)
+      // @ts-ignore
       query.select().forEach(({ as, expr }) => map[as] = expr);
+      // @ts-ignore
       q.$groupby(groupby.map(e => (e instanceof Ref && map[e.column]) || e));
     }