@uwdata/mosaic-core 0.7.1 → 0.8.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uwdata/mosaic-core",
-  "version": "0.7.1",
+  "version": "0.8.0",
   "description": "Scalable and extensible linked data views.",
   "keywords": [
     "mosaic",
@@ -23,14 +23,17 @@
   "scripts": {
     "prebuild": "rimraf dist && mkdir dist",
     "build": "node ../../esbuild.js mosaic-core",
-    "lint": "eslint src test --ext .js",
+    "lint": "eslint src test",
     "test": "mocha 'test/**/*-test.js'",
     "prepublishOnly": "npm run test && npm run lint && npm run build"
   },
   "dependencies": {
-    "@duckdb/duckdb-wasm": "^1.28.1-dev109.0",
-    "@uwdata/mosaic-sql": "^0.7.0",
-    "apache-arrow": "^15.0.0"
+    "@duckdb/duckdb-wasm": "^1.28.1-dev194.0",
+    "@uwdata/mosaic-sql": "^0.8.0",
+    "apache-arrow": "^15.0.2"
   },
-  "gitHead": "7e6f3ea9b3011ea2c9201c1aa16e8e5664621a4c"
+  "devDependencies": {
+    "@uwdata/mosaic-duckdb": "^0.8.0"
+  },
+  "gitHead": "a24b4c9f7dfa1c38c6af96ec17e075326c1af9b0"
 }

package/src/Coordinator.js

@@ -8,8 +8,7 @@ let _instance;
 
 /**
  * Set or retrieve the coordinator instance.
- *
- * @param {Coordinator} instance the coordinator instance to set
+ * @param {Coordinator} [instance] the coordinator instance to set
  * @returns {Coordinator} the coordinator instance
  */
 export function coordinator(instance) {
@@ -42,7 +41,15 @@ export class Coordinator {
     return this._logger;
   }
 
-  configure({ cache = true, consolidate = true, indexes = true }) {
+  /**
+   * Set configuration options for this coordinator.
+   * @param {object} [options] Configration options.
+   * @param {boolean} [options.cache=true] Boolean flag to enable/disable query caching.
+   * @param {boolean} [options.consolidate=true] Boolean flag to enable/disable query consolidation.
+   * @param {boolean|object} [options.indexes=true] Boolean flag to enable/disable
+   *  automatic data cube indexes or an index options object.
+   */
+  configure({ cache = true, consolidate = true, indexes = true } = {}) {
     this.manager.cache(cache);
     this.manager.consolidate(consolidate);
     this.indexes = indexes;
@@ -116,7 +123,6 @@ export class Coordinator {
 
   /**
    * Connect a client to the coordinator.
-   *
    * @param {import('./MosaicClient.js').MosaicClient} client the client to disconnect
    */
   async connect(client) {

package/src/DataCubeIndexer.js

@@ -54,7 +54,8 @@ export class DataCubeIndexer {
 
     active = active || this.selection.active;
     const { source } = active;
-    if (source && source === this.activeView?.source) return true; // we're good!
+    // exit early if indexes already set up for active view
+    if (source && source === this.activeView?.source) return true;
 
     this.clear();
     if (!source) return false; // nothing to work with
@@ -75,7 +76,7 @@ export class DataCubeIndexer {
 
       // build index construction query
       const query = client.query(sel.predicate(client))
-        .select({ ...activeView.columns, ...index.count })
+        .select({ ...activeView.columns, ...index.aux })
         .groupby(Object.keys(activeView.columns));
 
       // ensure active view columns are selected by subqueries
@@ -95,6 +96,9 @@ export class DataCubeIndexer {
       const result = mc.exec(create(table, sql, { temp }));
       indices.set(client, { table, result, order, ...index });
     }
+
+    // index creation successful
+    return true;
   }
 
   async update() {
@@ -179,23 +183,40 @@ function getIndexColumns(client) {
 
   const aggr = [];
   const dims = [];
-  let count;
+  const aux = {}; // auxiliary columns needed by aggregates
+  let auxAs;
 
-  for (const { as, expr: { aggregate } } of q.select()) {
-    switch (aggregate?.toUpperCase?.()) {
+  for (const entry of q.select()) {
+    const { as, expr: { aggregate, args } } = entry;
+    const op = aggregate?.toUpperCase?.();
+    switch (op) {
       case 'COUNT':
       case 'SUM':
         aggr.push({ [as]: sql`SUM("${as}")::DOUBLE` });
         break;
       case 'AVG':
-        count = '_count_';
-        aggr.push({ [as]: sql`(SUM("${as}" * ${count}) / SUM(${count}))::DOUBLE` });
+        aux[auxAs = '__count__'] = sql`COUNT(*)`;
+        aggr.push({ [as]: sql`(SUM("${as}" * ${auxAs}) / SUM(${auxAs}))::DOUBLE` });
         break;
-      case 'MAX':
-        aggr.push({ [as]: sql`MAX("${as}")` });
+      case 'ARG_MAX':
+        aux[auxAs = `__max_${as}__`] = sql`MAX(${args[1]})`;
+        aggr.push({ [as]: sql`ARG_MAX("${as}", ${auxAs})` });
+        break;
+      case 'ARG_MIN':
+        aux[auxAs = `__min_${as}__`] = sql`MIN(${args[1]})`;
+        aggr.push({ [as]: sql`ARG_MIN("${as}", ${auxAs})` });
         break;
+
+      // aggregates that commute directly
+      case 'MAX':
       case 'MIN':
-        aggr.push({ [as]: sql`MIN("${as}")` });
+      case 'BIT_AND':
+      case 'BIT_OR':
+      case 'BIT_XOR':
+      case 'BOOL_AND':
+      case 'BOOL_OR':
+      case 'PRODUCT':
+        aggr.push({ [as]: sql`${op}("${as}")` });
         break;
       default:
         if (g.has(as)) dims.push(as);
@@ -203,12 +224,7 @@ function getIndexColumns(client) {
     }
   }
 
-  return {
-    aggr,
-    dims,
-    count: count ? { [count]: sql`COUNT(*)` } : {},
-    from
-  };
+  return { aggr, dims, aux, from };
 }
 
 function getBaseTable(query) {

package/src/FilterGroup.js

@@ -46,9 +46,15 @@ export class FilterGroup {
     return this;
   }
 
+  /**
+   * Internal method to process a selection update.
+   * The return value is passed as a selection callback value.
+   * @returns {Promise} A Promise that resolves when the update completes.
+   */
   update() {
     const { mc, indexer, clients, selection } = this;
-    return indexer?.index(clients)
+    const hasIndex = indexer?.index(clients);
+    return hasIndex
       ? indexer.update()
       : defaultUpdate(mc, clients, selection);
   }

package/src/MosaicClient.js

@@ -48,6 +48,7 @@ export class MosaicClient {
 
   /**
    * Return an array of fields queried by this client.
+   * @returns {object[]|null} The fields to retrieve info for.
    */
   fields() {
     return null;
@@ -55,21 +56,25 @@ export class MosaicClient {
 
   /**
    * Called by the coordinator to set the field info for this client.
+   * @param {*} info The field info result.
    * @returns {this}
    */
-  fieldInfo() {
+  fieldInfo(info) { // eslint-disable-line no-unused-vars
     return this;
   }
 
   /**
    * Return a query specifying the data needed by this client.
+   * @param {*} [filter] The filtering criteria to apply in the query.
+   * @returns {*} The client query
    */
-  query() {
+  query(filter) { // eslint-disable-line no-unused-vars
     return null;
   }
 
   /**
    * Called by the coordinator to inform the client that a query is pending.
+   * @returns {this}
    */
   queryPending() {
     return this;
@@ -77,16 +82,17 @@ export class MosaicClient {
 
   /**
    * Called by the coordinator to return a query result.
-   *
-   * @param {*} data the query result
+   * @param {*} data The query result.
    * @returns {this}
    */
-  queryResult() {
+  queryResult(data) { // eslint-disable-line no-unused-vars
     return this;
   }
 
   /**
    * Called by the coordinator to report a query execution error.
+   * @param {*} error
+   * @returns {this}
    */
   queryError(error) {
     console.error(error);
@@ -116,6 +122,8 @@ export class MosaicClient {
   /**
    * Requests a client update.
    * For example to (re-)render an interface component.
+   * 
+   * @returns {this | Promise<any>}
    */
   update() {
     return this;

package/src/Param.js

@@ -4,7 +4,7 @@ import { distinct } from './util/distinct.js';
 /**
  * Test if a value is a Param instance.
  * @param {*} x The value to test.
- * @returns {boolean} True if the input is a Param, false otherwise.
+ * @returns {x is Param} True if the input is a Param, false otherwise.
  */
 export function isParam(x) {
   return x instanceof Param;
@@ -43,7 +43,9 @@ export class Param extends AsyncDispatch {
   static array(values) {
     if (values.some(v => isParam(v))) {
       const p = new Param();
-      const update = () => p.update(values.map(v => isParam(v) ? v.value : v));
+      const update = () => {
+        p.update(values.map(v => isParam(v) ? v.value : v));
+      };
       update();
       values.forEach(v => isParam(v) ? v.addEventListener('value', update) : 0);
       return p;

package/src/QueryConsolidator.js

@@ -5,6 +5,7 @@ function wait(callback) {
   const method = typeof requestAnimationFrame !== 'undefined'
     ? requestAnimationFrame
     : typeof setImmediate !== 'undefined' ? setImmediate : setTimeout;
+  // @ts-ignore
   return method(callback);
 }
 
@@ -82,7 +83,9 @@ function consolidationKey(query, cache) {
   const sql = `${query}`;
   if (query instanceof Query && !cache.get(sql)) {
     if (
+      // @ts-ignore
       query.orderby().length || query.where().length ||
+      // @ts-ignore
       query.qualify().length || query.having().length
     ) {
       // do not try to analyze if query includes clauses
@@ -97,9 +100,12 @@ function consolidationKey(query, cache) {
     // queries may refer to *derived* columns as group by criteria
     // we resolve these against the true grouping expressions
     const groupby = query.groupby();
+    // @ts-ignore
     if (groupby.length) {
       const map = {}; // expression map (as -> expr)
+      // @ts-ignore
       query.select().forEach(({ as, expr }) => map[as] = expr);
+      // @ts-ignore
       q.$groupby(groupby.map(e => (e instanceof Ref && map[e.column]) || e));
     }
 

package/src/Selection.js

@@ -4,7 +4,7 @@ import { Param } from './Param.js';
 /**
  * Test if a value is a Selection instance.
  * @param {*} x The value to test.
- * @returns {boolean} True if the input is a Selection, false otherwise.
+ * @returns {x is Selection} True if the input is a Selection, false otherwise.
  */
 export function isSelection(x) {
   return x instanceof Selection;
@@ -76,7 +76,7 @@ export class Selection extends Param {
 
   /**
    * Create a cloned copy of this Selection instance.
-   * @returns {this} A clone of this selection.
+   * @returns {Selection} A clone of this selection.
    */
   clone() {
     const s = new Selection(this._resolver);
@@ -88,7 +88,7 @@ export class Selection extends Param {
    * Create a clone of this Selection with clauses corresponding
    * to the provided source removed.
    * @param {*} source The clause source to remove.
-   * @returns {this} A cloned and updated Selection.
+   * @returns {Selection} A cloned and updated Selection.
    */
   remove(source) {
     const s = this.clone();
@@ -168,9 +168,9 @@ export class Selection extends Param {
    * Upon value-typed updates, returns a dispatch queue filter function.
    * The return value depends on the selection resolution strategy.
    * @param {string} type The event type.
-   * @param {*} value The input event value.
-   * @returns {*} For value-typed events, returns a dispatch queue filter
-   *  function. Otherwise returns null.
+   * @param {*} value The new event value that will be enqueued.
+   * @returns {(value: *) => boolean|null} For value-typed events,
+   *  returns a dispatch queue filter function. Otherwise returns null.
    */
   emitQueueFilter(type, value) {
     return type === 'value'
@@ -285,5 +285,6 @@ export class SelectionResolver {
       const source = value.active?.source;
       return clauses => clauses.active?.source !== source;
     }
+    return null;
   }
 }

package/src/connectors/wasm.js

@@ -22,7 +22,7 @@ export function wasmConnector(options = {}) {
   /**
    * Get the backing DuckDB-WASM instance.
    * Will lazily initialize DuckDB-WASM if not already loaded.
-   * @returns {duckdb.AsyncDuckDB} The DuckDB-WASM instance.
+   * @returns {Promise<duckdb.AsyncDuckDB>} The DuckDB-WASM instance.
    */
   async function getDuckDB() {
     if (!db) await load();
@@ -32,7 +32,7 @@ export function wasmConnector(options = {}) {
   /**
    * Get the backing DuckDB-WASM connection.
    * Will lazily initialize DuckDB-WASM if not already loaded.
-   * @returns {duckdb.AsyncDuckDBConnection} The DuckDB-WASM connection.
+   * @returns {Promise<duckdb.AsyncDuckDBConnection>} The DuckDB-WASM connection.
    */
   async function getConnection() {
     if (!con) await load();

package/src/util/AsyncDispatch.js

@@ -16,7 +16,7 @@ export class AsyncDispatch {
   /**
    * Add an event listener callback for the provided event type.
    * @param {string} type The event type.
-   * @param {(value: *) => Promise?} callback The event handler
+   * @param {(value: *) => void | Promise} callback The event handler
    *  callback function to add. If the callback has already been
    *  added for the event type, this method has no effect.
    */
@@ -35,7 +35,7 @@ export class AsyncDispatch {
   /**
    * Remove an event listener callback for the provided event type.
    * @param {string} type The event type.
-   * @param {(value: *) => Promise?} callback The event handler
+   * @param {(value: *) => void | Promise} callback The event handler
    *  callback function to remove.
    */
   removeEventListener(type, callback) {
@@ -64,11 +64,12 @@ export class AsyncDispatch {
    * This default implementation simply returns null, indicating that
    * any other unemitted event values should be dropped (that is, all
    * queued events are filtered)
+   * @param {string} type The event type.
    * @param {*} value The new event value that will be enqueued.
    * @returns {(value: *) => boolean|null} A dispatch queue filter
    *  function, or null if all unemitted event values should be filtered.
    */
-  emitQueueFilter() {
+  emitQueueFilter(type, value) { // eslint-disable-line no-unused-vars
     // removes all pending items
     return null;
   }
@@ -94,20 +95,22 @@ export class AsyncDispatch {
   emit(type, value) {
     const entry = this._callbacks.get(type) || {};
     if (entry.pending) {
+      // an earlier emit is still processing
+      // enqueue the current update, possibly filtering other pending updates
       entry.queue.enqueue(value, this.emitQueueFilter(type, value));
     } else {
       const event = this.willEmit(type, value);
       const { callbacks, queue } = entry;
       if (callbacks?.size) {
-        const promise = Promise
-          .allSettled(Array.from(callbacks, callback => callback(event)))
-          .then(() => {
-            entry.pending = null;
-            if (!queue.isEmpty()) {
-              this.emit(type, queue.dequeue());
-            }
-          });
-        entry.pending = promise;
+        // broadcast update to callbacks, which may return promises
+        // wait until promises resolve, then process pending updates
+        const callbackValues = Array.from(callbacks, cb => cb(event));
+        entry.pending = Promise.allSettled(callbackValues).then(() => {
+          entry.pending = null;
+          if (!queue.isEmpty()) {
+            this.emit(type, queue.dequeue());
+          }
+        });
       }
     }
   }

package/src/util/convert-arrow.js

@@ -1,34 +1,25 @@
-// arrow type ids
-const INTEGER = 2;
-const FLOAT = 3;
-const DECIMAL = 7;
-const TIMESTAMP = 10;
+import { DataType } from 'apache-arrow';
 
 /**
  * Test if a value is an Apache Arrow table.
  * As sometimes multiple Arrow versions may be used simultaneously,
  * we use a "duck typing" approach and check for a getChild function.
  * @param {*} values The value to test
- * @returns true if the value duck types as Apache Arrow data
+ * @returns {values is import('apache-arrow').Table} true if the value duck types as Apache Arrow data
  */
 export function isArrowTable(values) {
   return typeof values?.getChild === 'function';
 }
 
 /**
- * Return a JavaScript array type for an Apache Arrow column type.
- * @param {*} type an Apache Arrow column type
- * @returns a JavaScript array constructor
- */
+  * Return a JavaScript array type for an Apache Arrow column type.
+  * @param {DataType} type an Apache Arrow column type
+  * @returns a JavaScript array constructor
+  */
 export function convertArrowArrayType(type) {
-  switch (type.typeId) {
-    case INTEGER:
-    case FLOAT:
-    case DECIMAL:
-      return Float64Array;
-    default:
-      return Array;
-  }
+  return DataType.isInt(type) || DataType.isFloat(type) || DataType.isDecimal(type)
+    ? Float64Array
+    : Array;
 }
 
 /**
@@ -37,24 +28,22 @@ export function convertArrowArrayType(type) {
  * Large integers (BigInt) are converted to Float64 numbers.
  * Fixed-point decimal values are convert to Float64 numbers.
  * Otherwise, the default Arrow values are used.
- * @param {*} type an Apache Arrow column type
+ * @param {DataType} type an Apache Arrow column type
  * @returns a value conversion function
  */
 export function convertArrowValue(type) {
-  const { typeId } = type;
-
   // map timestamp numbers to date objects
-  if (typeId === TIMESTAMP) {
+  if (DataType.isTimestamp(type)) {
     return v => v == null ? v : new Date(v);
   }
 
   // map bigint to number
-  if (typeId === INTEGER && type.bitWidth >= 64) {
+  if (DataType.isInt(type) && type.bitWidth >= 64) {
     return v => v == null ? v : Number(v);
   }
 
   // map decimal to number
-  if (typeId === DECIMAL) {
+  if (DataType.isDecimal(type)) {
     const scale = 1 / Math.pow(10, type.scale);
     return v => v == null ? v : decimalToNumber(v, scale);
   }
@@ -74,10 +63,9 @@ export function convertArrowValue(type) {
  */
 export function convertArrowColumn(column) {
   const { type } = column;
-  const { typeId } = type;
 
   // map timestamp numbers to date objects
-  if (typeId === TIMESTAMP) {
+  if (DataType.isTimestamp(type)) {
     const size = column.length;
     const array = new Array(size);
     for (let row = 0; row < size; ++row) {
@@ -88,7 +76,7 @@ export function convertArrowColumn(column) {
   }
 
   // map bigint to number
-  if (typeId === INTEGER && type.bitWidth >= 64) {
+  if (DataType.isInt(type) && type.bitWidth >= 64) {
     const size = column.length;
     const array = new Float64Array(size);
     for (let row = 0; row < size; ++row) {
@@ -99,7 +87,7 @@ export function convertArrowColumn(column) {
   }
 
   // map decimal to number
-  if (typeId === DECIMAL) {
+  if (DataType.isDecimal(type)) {
     const scale = 1 / Math.pow(10, type.scale);
     const size = column.length;
     const array = new Float64Array(size);
@@ -124,10 +112,10 @@ const BASE32 = Array.from(
 /**
  * Convert a fixed point decimal value to a double precision number.
  * Note: if the value is sufficiently large the conversion may be lossy!
- * @param {Uint32Array} v a fixed decimal value
+ * @param {Uint32Array & { signed: boolean }} v a fixed decimal value
  * @param {number} scale a scale factor, corresponding to the
  *  number of fractional decimal digits in the fixed point value
- * @returns the resulting number
+ * @returns {number} the resulting number
  */
 function decimalToNumber(v, scale) {
   const n = v.length;

package/src/util/js-type.js

@@ -24,6 +24,7 @@ export function jsType(type) {
       return 'boolean';
     case 'VARCHAR':
     case 'UUID':
+    case 'JSON':
       return 'string';
     case 'ARRAY':
     case 'LIST':

package/src/util/query-result.js

@@ -2,7 +2,8 @@ export function queryResult() {
   let resolve;
   let reject;
   const p = new Promise((r, e) => { resolve = r; reject = e; });
-  p.fulfill = value => (resolve(value), p);
-  p.reject = err => (reject(err), p);
-  return p;
+  return Object.assign(p, {
+    fulfill: value => (resolve(value), p),
+    reject: err => (reject(err), p)
+  });
 }