@uwdata/mosaic-core 0.8.0 → 0.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uwdata/mosaic-core",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "description": "Scalable and extensible linked data views.",
   "keywords": [
     "mosaic",
@@ -28,12 +28,12 @@
     "prepublishOnly": "npm run test && npm run lint && npm run build"
   },
   "dependencies": {
-    "@duckdb/duckdb-wasm": "^1.28.1-dev194.0",
-    "@uwdata/mosaic-sql": "^0.8.0",
+    "@duckdb/duckdb-wasm": "^1.28.1-dev195.0",
+    "@uwdata/mosaic-sql": "^0.9.0",
     "apache-arrow": "^15.0.2"
   },
   "devDependencies": {
-    "@uwdata/mosaic-duckdb": "^0.8.0"
+    "@uwdata/mosaic-duckdb": "^0.9.0"
   },
-  "gitHead": "a24b4c9f7dfa1c38c6af96ec17e075326c1af9b0"
+  "gitHead": "89bb9b0dfa747aed691eaeba35379525a6764c61"
 }

package/src/Coordinator.js

@@ -24,7 +24,7 @@ export class Coordinator {
   constructor(db = socketConnector(), options = {}) {
     const {
       logger = console,
-      manager = QueryManager()
+      manager = new QueryManager()
     } = options;
     this.manager = manager;
     this.logger(logger);

package/src/DataCubeIndexer.js

@@ -1,5 +1,6 @@
-import { Query, and, create, isBetween, scaleTransform, sql } from '@uwdata/mosaic-sql';
+import { Query, and, asColumn, create, isBetween, scaleTransform, sql } from '@uwdata/mosaic-sql';
 import { fnv_hash } from './util/hash.js';
+import { indexColumns } from './util/index-columns.js';
 
 /**
  * Build and query optimized indices ("data cubes") for fast computation of
@@ -9,7 +10,7 @@ import { fnv_hash } from './util/hash.js';
  * as temporary database tables that can be queried for rapid updates.
  * Compatible client queries must pull data from the same backing table and
  * must consist of only groupby dimensions and supported aggregates.
- * Compatible selections must contain an active clause that exposes a schema
+ * Compatible selections must contain an active clause that exposes metadata
  * for an interval or point value predicate.
  */
 export class DataCubeIndexer {
@@ -30,39 +31,40 @@ export class DataCubeIndexer {
     this.enabled = false;
     this.clients = null;
     this.indices = null;
-    this.activeView = null;
+    this.active = null;
   }
 
   clear() {
     if (this.indices) {
-      this.mc.cancel(Array.from(this.indices.values(), index => index.result));
+      this.mc.cancel(Array.from(this.indices.values(), index => index?.result));
       this.indices = null;
     }
   }
 
-  index(clients, active) {
+  index(clients, activeClause) {
     if (this.clients !== clients) {
       // test client views for compatibility
-      const cols = Array.from(clients, getIndexColumns);
+      const cols = Array.from(clients, indexColumns).filter(x => x);
       const from = cols[0]?.from;
-      this.enabled = cols.every(c => c && c.from === from);
+      this.enabled = cols.length && cols.every(c => c.from === from);
       this.clients = clients;
-      this.activeView = null;
+      this.active = null;
       this.clear();
     }
     if (!this.enabled) return false; // client views are not indexable
 
-    active = active || this.selection.active;
-    const { source } = active;
+    activeClause = activeClause || this.selection.active;
+    const { source } = activeClause;
     // exit early if indexes already set up for active view
-    if (source && source === this.activeView?.source) return true;
+    if (source && source === this.active?.source) return true;
 
     this.clear();
     if (!source) return false; // nothing to work with
-    const activeView = this.activeView = getActiveView(active);
-    if (!activeView) return false; // active selection clause not compatible
+    const active = this.active = activeColumns(activeClause);
+    if (!active) return false; // active selection clause not compatible
 
-    this.mc.logger().warn('DATA CUBE INDEX CONSTRUCTION');
+    const logger = this.mc.logger();
+    logger.warn('DATA CUBE INDEX CONSTRUCTION');
 
     // create a selection with the active source removed
     const sel = this.selection.remove(source);
@@ -71,18 +73,29 @@ export class DataCubeIndexer {
     const indices = this.indices = new Map;
     const { mc, temp } = this;
     for (const client of clients) {
-      if (sel.skip(client, active)) continue;
-      const index = getIndexColumns(client);
+      // determine if client should be skipped due to cross-filtering
+      if (sel.skip(client, activeClause)) {
+        indices.set(client, null);
+        continue;
+      }
+
+      // generate column definitions for data cube and cube queries
+      const index = indexColumns(client);
+
+      // skip if client is not indexable
+      if (!index) {
+        continue;
+      }
 
-      // build index construction query
+      // build index table construction query
       const query = client.query(sel.predicate(client))
-        .select({ ...activeView.columns, ...index.aux })
-        .groupby(Object.keys(activeView.columns));
+        .select({ ...active.columns, ...index.aux })
+        .groupby(Object.keys(active.columns));
 
       // ensure active view columns are selected by subqueries
       const [subq] = query.subqueries;
       if (subq) {
-        const cols = Object.values(activeView.columns).map(c => c.columns[0]);
+        const cols = Object.values(active.columns).flatMap(c => c.columns);
         subqueryPushdown(subq, cols);
       }
 
@@ -94,6 +107,7 @@ export class DataCubeIndexer {
       const id = (fnv_hash(sql) >>> 0).toString(16);
       const table = `cube_index_${id}`;
       const result = mc.exec(create(table, sql, { temp }));
+      result.catch(e => logger.error(e));
       indices.set(client, { table, result, order, ...index });
     }
 
@@ -102,21 +116,28 @@ export class DataCubeIndexer {
   }
 
   async update() {
-    const { clients, selection, activeView } = this;
-    const filter = activeView.predicate(selection.active.predicate);
+    const { clients, selection, active } = this;
+    const filter = active.predicate(selection.active.predicate);
     return Promise.all(
       Array.from(clients).map(client => this.updateClient(client, filter))
     );
   }
 
   async updateClient(client, filter) {
+    const { mc, indices, selection } = this;
+
+    // if client has no index, perform a standard update
+    if (!indices.has(client)) {
+      filter = selection.predicate(client);
+      return mc.updateClient(client, client.query(filter));
+    };
+
     const index = this.indices.get(client);
-    if (!index) return;
 
-    if (!filter) {
-      filter = this.activeView.predicate(this.selection.active.predicate);
-    }
+    // skip update if cross-filtered
+    if (!index) return;
 
+    // otherwise, query a data cube index table
     const { table, dims, aggr, order = [] } = index;
     const query = Query
       .select(dims, aggr)
@@ -124,25 +145,30 @@ export class DataCubeIndexer {
       .groupby(dims)
       .where(filter)
       .orderby(order);
-    return this.mc.updateClient(client, query);
+    return mc.updateClient(client, query);
   }
 }
 
-function getActiveView(clause) {
-  const { source, schema } = clause;
+function activeColumns(clause) {
+  const { source, meta } = clause;
   let columns = clause.predicate?.columns;
-  if (!schema || !columns) return null;
-  const { type, scales, pixelSize = 1 } = schema;
+  if (!meta || !columns) return null;
+  const { type, scales, bin, pixelSize = 1 } = meta;
   let predicate;
 
   if (type === 'interval' && scales) {
-    const bins = scales.map(s => binInterval(s, pixelSize));
-    if (bins.some(b => b == null)) return null; // unsupported scale type
+    // determine pixel-level binning
+    const bins = scales.map(s => binInterval(s, pixelSize, bin));
+
+    // bail if the scale type is unsupported
+    if (bins.some(b => b == null)) return null;
 
     if (bins.length === 1) {
+      // single interval selection
       predicate = p => p ? isBetween('active0', p.range.map(bins[0])) : [];
       columns = { active0: bins[0](clause.predicate.field) };
     } else {
+      // multiple interval selection
       predicate = p => p
         ? and(p.children.map(({ range }, i) => isBetween(`active${i}`, range.map(bins[i]))))
         : [];
@@ -152,99 +178,27 @@ function getActiveView(clause) {
     }
   } else if (type === 'point') {
     predicate = x => x;
-    columns = Object.fromEntries(columns.map(col => [col.toString(), col]));
+    columns = Object.fromEntries(columns.map(col => [`${col}`, asColumn(col)]));
   } else {
-    return null; // unsupported type
+    // unsupported selection type
+    return null;
   }
 
   return { source, columns, predicate };
 }
 
-function binInterval(scale, pixelSize) {
-  const { apply, sqlApply } = scaleTransform(scale);
-  if (apply) {
-    const { domain, range } = scale;
-    const lo = apply(Math.min(...domain));
-    const hi = apply(Math.max(...domain));
-    const a = (Math.abs(range[1] - range[0]) / (hi - lo)) / pixelSize;
-    const s = pixelSize === 1 ? '' : `${pixelSize}::INTEGER * `;
-    return value => sql`${s}FLOOR(${a}::DOUBLE * (${sqlApply(value)} - ${lo}::DOUBLE))::INTEGER`;
-  }
-}
-
-const NO_INDEX = { from: NaN };
-
-function getIndexColumns(client) {
-  if (!client.filterIndexable) return NO_INDEX;
-  const q = client.query();
-  const from = getBaseTable(q);
-  if (!from || !q.groupby) return NO_INDEX;
-  const g = new Set(q.groupby().map(c => c.column));
-
-  const aggr = [];
-  const dims = [];
-  const aux = {}; // auxiliary columns needed by aggregates
-  let auxAs;
-
-  for (const entry of q.select()) {
-    const { as, expr: { aggregate, args } } = entry;
-    const op = aggregate?.toUpperCase?.();
-    switch (op) {
-      case 'COUNT':
-      case 'SUM':
-        aggr.push({ [as]: sql`SUM("${as}")::DOUBLE` });
-        break;
-      case 'AVG':
-        aux[auxAs = '__count__'] = sql`COUNT(*)`;
-        aggr.push({ [as]: sql`(SUM("${as}" * ${auxAs}) / SUM(${auxAs}))::DOUBLE` });
-        break;
-      case 'ARG_MAX':
-        aux[auxAs = `__max_${as}__`] = sql`MAX(${args[1]})`;
-        aggr.push({ [as]: sql`ARG_MAX("${as}", ${auxAs})` });
-        break;
-      case 'ARG_MIN':
-        aux[auxAs = `__min_${as}__`] = sql`MIN(${args[1]})`;
-        aggr.push({ [as]: sql`ARG_MIN("${as}", ${auxAs})` });
-        break;
-
-      // aggregates that commute directly
-      case 'MAX':
-      case 'MIN':
-      case 'BIT_AND':
-      case 'BIT_OR':
-      case 'BIT_XOR':
-      case 'BOOL_AND':
-      case 'BOOL_OR':
-      case 'PRODUCT':
-        aggr.push({ [as]: sql`${op}("${as}")` });
-        break;
-      default:
-        if (g.has(as)) dims.push(as);
-        else return null;
-    }
-  }
-
-  return { aggr, dims, aux, from };
-}
-
-function getBaseTable(query) {
-  const subq = query.subqueries;
-
-  // select query
-  if (query.select) {
-    const from = query.from();
-    if (!from.length) return undefined;
-    if (subq.length === 0) return from[0].from.table;
-  }
-
-  // handle set operations / subqueries
-  const base = getBaseTable(subq[0]);
-  for (let i = 1; i < subq.length; ++i) {
-    const from = getBaseTable(subq[i]);
-    if (from === undefined) continue;
-    if (from !== base) return NaN;
-  }
-  return base;
+const BIN = { ceil: 'CEIL', round: 'ROUND' };
+
+function binInterval(scale, pixelSize, bin) {
+  const { type, domain, range, apply, sqlApply } = scaleTransform(scale);
+  if (!apply) return; // unsupported scale type
+  const fn = BIN[`${bin}`.toLowerCase()] || 'FLOOR';
+  const lo = apply(Math.min(...domain));
+  const hi = apply(Math.max(...domain));
+  const a = type === 'identity' ? 1 : Math.abs(range[1] - range[0]) / (hi - lo);
+  const s = a / pixelSize === 1 ? '' : `${a / pixelSize}::DOUBLE * `;
+  const d = lo === 0 ? '' : ` - ${lo}::DOUBLE`;
+  return value => sql`${fn}(${s}(${sqlApply(value)}${d}))::INTEGER`;
 }
 
 function subqueryPushdown(query, cols) {

package/src/FilterGroup.js

@@ -1,24 +1,27 @@
+import { Coordinator } from './Coordinator.js';
 import { DataCubeIndexer } from './DataCubeIndexer.js';
+import { MosaicClient } from './MosaicClient.js';
+import { Selection } from './Selection.js';
 
 export class FilterGroup {
   /**
-   * @param {import('./Coordinator.js').Coordinator} coordinator The Mosaic coordinator.
-   * @param {*} selection The shared filter selection.
-   * @param {*} index Boolean flag or options hash for data cube indexer.
-   *  Falsy values disable indexing.
+   * @param {Coordinator} coordinator The Mosaic coordinator.
+   * @param {Selection} selection The shared filter selection.
+   * @param {object|boolean} index Boolean flag or options hash for
+   *  a data cube indexer. Falsy values disable indexing.
    */
   constructor(coordinator, selection, index = true) {
-    /** @type import('./Coordinator.js').Coordinator */
     this.mc = coordinator;
     this.selection = selection;
+    /** @type {Set<MosaicClient>} */
     this.clients = new Set();
-    this.indexer = index
-      ? new DataCubeIndexer(this.mc, { ...index, selection })
-      : null;
+    /** @type {DataCubeIndexer | null} */
+    this.indexer = null;
+    this.index(index);
 
     const { value, activate } = this.handlers = {
       value: () => this.update(),
-      activate: clause => this.indexer?.index(this.clients, clause)
+      activate: clause => { this.indexer?.index(this.clients, clause); }
     };
     selection.addEventListener('value', value);
     selection.addEventListener('activate', activate);
@@ -30,6 +33,14 @@ export class FilterGroup {
     this.selection.removeEventListener('activate', activate);
   }
 
+  index(state) {
+    const { selection } = this;
+    const { resolver } = selection;
+    this.indexer = state && (resolver.single || !resolver.union)
+      ? new DataCubeIndexer(this.mc, { ...state, selection })
+      : null;
+  }
+
   reset() {
     this.indexer?.reset();
   }

package/src/QueryManager.js

@@ -5,48 +5,51 @@ import { queryResult } from './util/query-result.js';
 
 export const Priority = { High: 0, Normal: 1, Low: 2 };
 
-export function QueryManager() {
-  const queue = priorityQueue(3);
-  let db;
-  let clientCache;
-  let logger;
-  let recorders = [];
-  let pending = null;
-  let consolidate;
-
-  function next() {
-    if (pending || queue.isEmpty()) return;
-    const { request, result } = queue.next();
-    pending = submit(request, result);
-    pending.finally(() => { pending = null; next(); });
+export class QueryManager {
+  constructor() {
+    this.queue = priorityQueue(3);
+    this.db = null;
+    this.clientCache = null;
+    this._logger = null;
+    this._logQueries = false;
+    this.recorders = [];
+    this.pending = null;
+    this._consolidate = null;
   }
 
-  function enqueue(entry, priority = Priority.Normal) {
-    queue.insert(entry, priority);
-    next();
+  next() {
+    if (this.pending || this.queue.isEmpty()) return;
+    const { request, result } = this.queue.next();
+    this.pending = this.submit(request, result);
+    this.pending.finally(() => { this.pending = null; this.next(); });
   }
 
-  function recordQuery(sql) {
-    if (recorders.length && sql) {
-      recorders.forEach(rec => rec.add(sql));
+  enqueue(entry, priority = Priority.Normal) {
+    this.queue.insert(entry, priority);
+    this.next();
+  }
+
+  recordQuery(sql) {
+    if (this.recorders.length && sql) {
+      this.recorders.forEach(rec => rec.add(sql));
     }
   }
 
-  async function submit(request, result) {
+  async submit(request, result) {
     try {
       const { query, type, cache = false, record = true, options } = request;
       const sql = query ? `${query}` : null;
 
       // update recorders
       if (record) {
-        recordQuery(sql);
+        this.recordQuery(sql);
       }
 
       // check query cache
       if (cache) {
-        const cached = clientCache.get(sql);
+        const cached = this.clientCache.get(sql);
         if (cached) {
-          logger.debug('Cache');
+          this._logger.debug('Cache');
           result.fulfill(cached);
           return;
         }
@@ -54,80 +57,85 @@ export function QueryManager() {
 
       // issue query, potentially cache result
       const t0 = performance.now();
-      const data = await db.query({ type, sql, ...options });
-      if (cache) clientCache.set(sql, data);
-      logger.debug(`Request: ${(performance.now() - t0).toFixed(1)}`);
+      if (this._logQueries) {
+        this._logger.debug('Query', { type, sql, ...options });
+      }
+      const data = await this.db.query({ type, sql, ...options });
+      if (cache) this.clientCache.set(sql, data);
+      this._logger.debug(`Request: ${(performance.now() - t0).toFixed(1)}`);
       result.fulfill(data);
     } catch (err) {
       result.reject(err);
     }
   }
 
-  return {
-    cache(value) {
-      return value !== undefined
-        ? (clientCache = value === true ? lruCache() : (value || voidCache()))
-        : clientCache;
-    },
-
-    logger(value) {
-      return value ? (logger = value) : logger;
-    },
-
-    connector(connector) {
-      return connector ? (db = connector) : db;
-    },
-
-    consolidate(flag) {
-      if (flag && !consolidate) {
-        consolidate = consolidator(enqueue, clientCache, recordQuery);
-      } else if (!flag && consolidate) {
-        consolidate = null;
-      }
-    },
-
-    request(request, priority = Priority.Normal) {
-      const result = queryResult();
-      const entry = { request, result };
-      if (consolidate) {
-        consolidate.add(entry, priority);
-      } else {
-        enqueue(entry, priority);
-      }
-      return result;
-    },
-
-    cancel(requests) {
-      const set = new Set(requests);
-      queue.remove(({ result }) => set.has(result));
-    },
-
-    clear() {
-      queue.remove(({ result }) => {
-        result.reject('Cleared');
-        return true;
-      });
-    },
-
-    record() {
-      let state = [];
-      const recorder = {
-        add(query) {
-          state.push(query);
-        },
-        reset() {
-          state = [];
-        },
-        snapshot() {
-          return state.slice();
-        },
-        stop() {
-          recorders = recorders.filter(x => x !== recorder);
-          return state;
-        }
-      };
-      recorders.push(recorder);
-      return recorder;
+  cache(value) {
+    return value !== undefined
+      ? (this.clientCache = value === true ? lruCache() : (value || voidCache()))
+      : this.clientCache;
+  }
+
+  logger(value) {
+    return value ? (this._logger = value) : this._logger;
+  }
+
+  logQueries(value) {
+    return value !== undefined ? this._logQueries = !!value : this._logQueries;
+  }
+
+  connector(connector) {
+    return connector ? (this.db = connector) : this.db;
+  }
+
+  consolidate(flag) {
+    if (flag && !this._consolidate) {
+      this._consolidate = consolidator(this.enqueue.bind(this), this.clientCache, this.recordQuery.bind(this));
+    } else if (!flag && this._consolidate) {
+      this._consolidate = null;
+    }
+  }
+
+  request(request, priority = Priority.Normal) {
+    const result = queryResult();
+    const entry = { request, result };
+    if (this._consolidate) {
+      this._consolidate.add(entry, priority);
+    } else {
+      this.enqueue(entry, priority);
     }
-  };
+    return result;
+  }
+
+  cancel(requests) {
+    const set = new Set(requests);
+    this.queue.remove(({ result }) => set.has(result));
+  }
+
+  clear() {
+    this.queue.remove(({ result }) => {
+      result.reject('Cleared');
+      return true;
+    });
+  }
+
+  record() {
+    let state = [];
+    const recorder = {
+      add(query) {
+        state.push(query);
+      },
+      reset() {
+        state = [];
+      },
+      snapshot() {
+        return state.slice();
+      },
+      stop() {
+        this.recorders = this.recorders.filter(x => x !== recorder);
+        return state;
+      }
+    };
+    this.recorders.push(recorder);
+    return recorder;
+  }
 }

package/src/Selection.js

@@ -98,19 +98,17 @@ export class Selection extends Param {
   }
 
   /**
-   * The current active (most recently updated) selection clause.
+   * The selection clause resolver.
    */
-  get active() {
-    return this.clauses.active;
+  get resolver() {
+    return this._resolver;
   }
 
   /**
-   * The value corresponding to the current active selection clause.
-   * This method ensures compatibility where a normal Param is expected.
+   * Indicate if this selection has a single resolution strategy.
    */
-  get value() {
-    // return value of the active clause
-    return this.active?.value;
+  get single() {
+    return this._resolver.single;
   }
 
   /**
@@ -121,10 +119,27 @@ export class Selection extends Param {
   }
 
   /**
-   * Indicate if this selection has a single resolution strategy.
+   * The current active (most recently updated) selection clause.
    */
-  get single() {
-    return this._resolver.single;
+  get active() {
+    return this.clauses.active;
+  }
+
+  /**
+   * The value corresponding to the current active selection clause.
+   * This method ensures compatibility where a normal Param is expected.
+   */
+  get value() {
+    return this.active?.value;
+  }
+
+  /**
+   * The value corresponding to a given source. Returns undefined if
+   * this selection does not include a clause from this source.
+   * @param {*} source The clause source to look up the value for.
+   */
+  valueFor(source) {
+    return this.clauses.find(c => c.source === source)?.value;
   }
 
   /**