@uwdata/mosaic-core 0.9.0 → 0.11.0

package/src/Selection.js

@@ -10,6 +10,13 @@ export function isSelection(x) {
   return x instanceof Selection;
 }
 
+function create(options, include) {
+  return new Selection(
+    new SelectionResolver(options),
+    include ? [include].flat() : include
+  );
+}
+
 /**
  * Represents a dynamic set of query filter predicates.
  */
@@ -22,10 +29,16 @@ export class Selection extends Param {
    * @param {boolean} [options.cross=false] Boolean flag indicating
    *  cross-filtered resolution. If true, selection clauses will not
    *  be applied to the clients they are associated with.
+   * @param {boolean} [options.empty=false] Boolean flag indicating if a lack
+   *  of clauses should correspond to an empty selection with no records. This
+   *  setting determines the default selection state.
+   * @param {Selection|Selection[]} [options.include] Upstream selections whose
+   *  clauses should be included as part of the new selection. Any clauses
+   *  published to upstream selections will be relayed to the new selection.
    * @returns {Selection} The new Selection instance.
    */
-  static intersect({ cross = false } = {}) {
-    return new Selection(new SelectionResolver({ cross }));
+  static intersect({ cross = false, empty = false, include = [] } = {}) {
+    return create({ cross, empty }, include);
   }
 
   /**
@@ -35,10 +48,16 @@ export class Selection extends Param {
    * @param {boolean} [options.cross=false] Boolean flag indicating
    *  cross-filtered resolution. If true, selection clauses will not
    *  be applied to the clients they are associated with.
+   * @param {boolean} [options.empty=false] Boolean flag indicating if a lack
+   *  of clauses should correspond to an empty selection with no records. This
+   *  setting determines the default selection state.
+   * @param {Selection|Selection[]} [options.include] Upstream selections whose
+   *  clauses should be included as part of the new selection. Any clauses
+   *  published to upstream selections will be relayed to the new selection.
    * @returns {Selection} The new Selection instance.
    */
-  static union({ cross = false } = {}) {
-    return new Selection(new SelectionResolver({ cross, union: true }));
+  static union({ cross = false, empty = false, include = [] } = {}) {
+    return create({ cross, empty, union: true }, include);
   }
 
   /**
@@ -48,30 +67,53 @@ export class Selection extends Param {
    * @param {boolean} [options.cross=false] Boolean flag indicating
    *  cross-filtered resolution. If true, selection clauses will not
    *  be applied to the clients they are associated with.
+   * @param {boolean} [options.empty=false] Boolean flag indicating if a lack
+   *  of clauses should correspond to an empty selection with no records. This
+   *  setting determines the default selection state.
+   * @param {Selection|Selection[]} [options.include] Upstream selections whose
+   *  clauses should be included as part of the new selection. Any clauses
+   *  published to upstream selections will be relayed to the new selection.
    * @returns {Selection} The new Selection instance.
    */
-  static single({ cross = false } = {}) {
-    return new Selection(new SelectionResolver({ cross, single: true }));
+  static single({ cross = false, empty = false, include = [] } = {}) {
+    return create({ cross, empty, single: true }, include);
   }
 
   /**
    * Create a new Selection instance with a
    * cross-filtered intersect resolution strategy.
+   * @param {object} [options] The selection options.
+   * @param {boolean} [options.empty=false] Boolean flag indicating if a lack
+   *  of clauses should correspond to an empty selection with no records. This
+   *  setting determines the default selection state.
+   * @param {Selection|Selection[]} [options.include] Upstream selections whose
+   *  clauses should be included as part of the new selection. Any clauses
+   *  published to upstream selections will be relayed to the new selection.
    * @returns {Selection} The new Selection instance.
    */
-  static crossfilter() {
-    return new Selection(new SelectionResolver({ cross: true }));
+  static crossfilter({ empty = false, include = [] } = {}) {
+    return create({ cross: true, empty }, include);
   }
 
   /**
    * Create a new Selection instance.
-   * @param {SelectionResolver} resolver The selection resolution
+   * @param {SelectionResolver} [resolver] The selection resolution
    *  strategy to apply.
+   * @param {Selection[]} [include] Upstream selections whose clauses
+   * should be included as part of this selection. Any clauses published
+   * to these upstream selections will be relayed to this selection.
    */
-  constructor(resolver = new SelectionResolver()) {
+  constructor(resolver = new SelectionResolver(), include = []) {
     super([]);
     this._resolved = this._value;
     this._resolver = resolver;
+    /** @type {Set<Selection>} */
+    this._relay = new Set;
+    if (Array.isArray(include)) {
+      for (const sel of include) {
+        sel._relay.add(this);
+      }
+    }
   }
 
   /**
@@ -148,6 +190,7 @@ export class Selection extends Param {
    */
   activate(clause) {
     this.emit('activate', clause);
+    this._relay.forEach(sel => sel.activate(clause));
   }
 
   /**
@@ -160,6 +203,7 @@ export class Selection extends Param {
     // this ensures consistent clause state across unemitted event values
     this._resolved = this._resolver.resolve(this._resolved, clause, true);
     this._resolved.active = clause;
+    this._relay.forEach(sel => sel.update(clause));
     return super.update(this._resolved);
   }
 
@@ -232,11 +276,15 @@ export class SelectionResolver {
    *  If false, an intersection strategy is used.
    * @param {boolean} [options.cross=false] Boolean flag to indicate cross-filtering.
    * @param {boolean} [options.single=false] Boolean flag to indicate single clauses only.
+   * @param {boolean} [options.empty=false] Boolean flag indicating if a lack
+   *  of clauses should correspond to an empty selection with no records. This
+   *  setting determines the default selection state.
    */
-  constructor({ union, cross, single } = {}) {
+  constructor({ union, cross, single, empty } = {}) {
     this.union = !!union;
     this.cross = !!cross;
     this.single = !!single;
+    this.empty = !!empty;
   }
 
   /**
@@ -274,7 +322,11 @@ export class SelectionResolver {
    *  based on the current state of this selection.
    */
   predicate(clauseList, active, client) {
-    const { union } = this;
+    const { empty, union } = this;
+
+    if (empty && !clauseList.length) {
+      return ['FALSE'];
+    }
 
     // do nothing if cross-filtering and client is currently active
     if (this.skip(client, active)) return undefined;

package/src/SelectionClause.js

@@ -24,7 +24,10 @@ import { MosaicClient } from './MosaicClient.js';
  *  cross-filtering contexts.
  * @returns {SelectionClause} The generated selection clause.
  */
-export function point(field, value, { source, clients = undefined }) {
+export function clausePoint(field, value, {
+  source,
+  clients = source ? new Set([source]) : undefined
+}) {
   /** @type {SQLExpression | null} */
   const predicate = value !== undefined
     ? isNotDistinct(field, literal(value))
@@ -50,7 +53,10 @@ export function point(field, value, { source, clients = undefined }) {
  *  cross-filtering contexts.
  * @returns {SelectionClause} The generated selection clause.
  */
-export function points(fields, value, { source, clients = undefined }) {
+export function clausePoints(fields, value, {
+  source,
+  clients = source ? new Set([source]) : undefined
+}) {
   /** @type {SQLExpression | null} */
   let predicate = null;
   if (value) {
@@ -83,13 +89,17 @@ export function points(fields, value, { source, clients = undefined }) {
  * @param {number} [options.pixelSize=1] The interactive pixel size.
  * @returns {SelectionClause} The generated selection clause.
  */
-export function interval(field, value, {
-  source, clients, bin, scale, pixelSize = 1
+export function clauseInterval(field, value, {
+  source,
+  clients = source ? new Set([source]) : undefined,
+  bin,
+  scale,
+  pixelSize = 1
 }) {
   /** @type {SQLExpression | null} */
   const predicate = value != null ? isBetween(field, value) : null;
   /** @type {import('./util/selection-types.js').IntervalMetadata} */
-  const meta = { type: 'interval', scales: [scale], bin, pixelSize };
+  const meta = { type: 'interval', scales: scale && [scale], bin, pixelSize };
   return { meta, source, clients, value, predicate };
 }
 
@@ -108,8 +118,12 @@ export function interval(field, value, {
  * @param {number} [options.pixelSize=1] The interactive pixel size.
  * @returns {SelectionClause} The generated selection clause.
  */
-export function intervals(fields, value, {
-  source, clients, bin, scales = [], pixelSize = 1
+export function clauseIntervals(fields, value, {
+  source,
+  clients = source ? new Set([source]) : undefined,
+  bin,
+  scales = [],
+  pixelSize = 1
 }) {
   /** @type {SQLExpression | null} */
   const predicate = value != null
@@ -135,7 +149,7 @@ const MATCH_METHODS = { contains, prefix, suffix, regexp: regexp_matches };
  *  text matching method to use. Defaults to `'contains'`.
  * @returns {SelectionClause} The generated selection clause.
  */
-export function match(field, value, {
+export function clauseMatch(field, value, {
   source, clients = undefined, method = 'contains'
 }) {
   let fn = MATCH_METHODS[method];

package/src/connectors/rest.js

@@ -1,11 +1,11 @@
-import { tableFromIPC } from 'apache-arrow';
+import { decodeIPC } from '../util/decode-ipc.js';
 
 export function restConnector(uri = 'http://localhost:3000/') {
   return {
     /**
      * Query the DuckDB server.
      * @param {object} query
-     * @param {'exec' | 'arrow' | 'json'} [query.type] The query type: 'exec', 'arrow', or 'json'.
+     * @param {'exec' | 'arrow' | 'json' | 'create-bundle' | 'load-bundle'} [query.type] The query type.
      * @param {string} query.sql A SQL query string.
      * @returns the query result
      */
@@ -20,7 +20,7 @@ export function restConnector(uri = 'http://localhost:3000/') {
       });
 
       return query.type === 'exec' ? req
-        : query.type === 'arrow' ? tableFromIPC(req)
+        : query.type === 'arrow' ? decodeIPC(await (await req).arrayBuffer())
         : (await req).json();
     }
   };

package/src/connectors/socket.js

@@ -1,4 +1,4 @@
-import { tableFromIPC } from 'apache-arrow';
+import { decodeIPC } from '../util/decode-ipc.js';
 
 export function socketConnector(uri = 'ws://localhost:3000/') {
   const queue = [];
@@ -47,7 +47,7 @@ export function socketConnector(uri = 'ws://localhost:3000/') {
         } else if (query.type === 'exec') {
           resolve();
         } else if (query.type === 'arrow') {
-          resolve(tableFromIPC(data.arrayBuffer()));
+          resolve(decodeIPC(data));
         } else {
           throw new Error(`Unexpected socket data: ${data}`);
         }
@@ -59,6 +59,7 @@ export function socketConnector(uri = 'ws://localhost:3000/') {
 
   function init() {
     ws = new WebSocket(uri);
+    ws.binaryType = 'arraybuffer';
     for (const type in events) {
       ws.addEventListener(type, events[type]);
     }
@@ -84,7 +85,7 @@ export function socketConnector(uri = 'ws://localhost:3000/') {
     /**
      * Query the DuckDB server.
      * @param {object} query
-     * @param {'exec' | 'arrow' | 'json'} [query.type] The query type: 'exec', 'arrow', or 'json'.
+     * @param {'exec' | 'arrow' | 'json' | 'create-bundle' | 'load-bundle'} [query.type] The query type.
      * @param {string} query.sql A SQL query string.
      * @returns the query result
      */

package/src/connectors/wasm.js

@@ -1,4 +1,20 @@
 import * as duckdb from '@duckdb/duckdb-wasm';
+import { decodeIPC } from '../util/decode-ipc.js';
+
+// bypass duckdb-wasm query method to get Arrow IPC bytes directly
+// https://github.com/duckdb/duckdb-wasm/issues/267#issuecomment-2252749509
+function getArrowIPC(con, query) {
+  return new Promise((resolve, reject) => {
+    con.useUnsafe(async (bindings, conn) => {
+      try {
+        const buffer = await bindings.runQuery(conn, query);
+        resolve(buffer);
+      } catch (error) {
+        reject(error);
+      }
+    });
+  });
+}
 
 export function wasmConnector(options = {}) {
   const { duckdb, connection, ...opts } = options;
@@ -45,17 +61,17 @@ export function wasmConnector(options = {}) {
     /**
      * Query the DuckDB-WASM instance.
      * @param {object} query
-     * @param {'exec' | 'arrow' | 'json'} [query.type] The query type: 'exec', 'arrow', or 'json'.
+     * @param {'exec' | 'arrow' | 'json' | 'create-bundle' | 'load-bundle'} [query.type] The query type.
      * @param {string} query.sql A SQL query string.
      * @returns the query result
      */
     query: async query => {
       const { type, sql } = query;
       const con = await getConnection();
-      const result = await con.query(sql);
+      const result = await getArrowIPC(con, sql);
       return type === 'exec' ? undefined
-        : type === 'arrow' ? result
-        : result.toArray();
+        : type === 'arrow' ? decodeIPC(result)
+        : decodeIPC(result).toArray();
     }
   };
 }

package/src/index.js

@@ -3,18 +3,22 @@ export { Coordinator, coordinator } from './Coordinator.js';
 export { Selection, isSelection } from './Selection.js';
 export { Param, isParam } from './Param.js';
 export { Priority } from './QueryManager.js';
-export { point, points, interval, intervals, match } from './SelectionClause.js';
 
 export { restConnector } from './connectors/rest.js';
 export { socketConnector } from './connectors/socket.js';
 export { wasmConnector } from './connectors/wasm.js';
 
 export {
-  isArrowTable,
-  convertArrowArrayType,
-  convertArrowValue,
-  convertArrowColumn
-} from './util/convert-arrow.js'
+  clauseInterval,
+  clauseIntervals,
+  clausePoint,
+  clausePoints,
+  clauseMatch
+} from './SelectionClause.js';
+
+export { decodeIPC } from './util/decode-ipc.js';
 export { distinct } from './util/distinct.js';
+export { isArrowTable } from './util/is-arrow-table.js';
 export { synchronizer } from './util/synchronizer.js';
 export { throttle } from './util/throttle.js';
+export { toDataColumns } from './util/to-data-columns.js';

package/src/util/AsyncDispatch.js

@@ -1,8 +1,7 @@
 /**
- * Event dispatcher supporting asynchronous updates.
- * If an event handler callback returns a Promise, this dispatcher will
- * wait for all such Promises to settle before dispatching future events
- * of the same type.
+ * Event dispatcher supporting asynchronous updates. If an event handler
+ * callback returns a Promise, the dispatcher waits for all such Promises
+ * to settle before dispatching future events of the same type.
  */
 export class AsyncDispatch {
 
@@ -63,7 +62,7 @@ export class AsyncDispatch {
    * queue of unemitted event values prior to enqueueing a new value.
    * This default implementation simply returns null, indicating that
    * any other unemitted event values should be dropped (that is, all
-   * queued events are filtered)
+   * queued events are filtered).
    * @param {string} type The event type.
    * @param {*} value The new event value that will be enqueued.
    * @returns {(value: *) => boolean|null} A dispatch queue filter
@@ -83,6 +82,17 @@ export class AsyncDispatch {
     entry?.queue.clear();
   }
 
+  /**
+   * Returns a promise that resolves when any pending updates complete for
+   * the event of the given type currently being processed. The Promise will
+   * resolve immediately if the queue for the given event type is empty.
+   * @param {string} type The event type to wait for.
+   * @returns {Promise} A pending event promise.
+   */
+  async pending(type) {
+    await this._callbacks.get(type)?.pending;
+  }
+
   /**
    * Emit an event value to listeners for the given event type.
    * If a previous emit has not yet resolved, the event value

package/src/util/decode-ipc.js

@@ -0,0 +1,11 @@
+import { tableFromIPC } from '@uwdata/flechette';
+
+/**
+ * Decode Arrow IPC bytes to a table instance, with an option to map date and
+ * timestamp values to JS Date objects.
+ * @param {ArrayBuffer | Uint8Array} data Arrow IPC bytes.
+ * @returns {import('@uwdata/flechette').Table} A table instance.
+ */
+export function decodeIPC(data) {
+  return tableFromIPC(data, { useDate: true });
+}

package/src/util/field-info.js

@@ -1,6 +1,5 @@
 import { Query, asRelation, count, isNull, max, min, sql } from '@uwdata/mosaic-sql';
 import { jsType } from './js-type.js';
-import { convertArrowValue } from './convert-arrow.js';
 
 export const Count = 'count';
 export const Nulls = 'nulls';
@@ -52,20 +51,13 @@ async function getFieldInfo(mc, { table, column, stats }) {
   if (!(stats?.length || stats?.size)) return info;
 
   // query for summary stats
-  const result = await mc.query(
+  const [result] = await mc.query(
     summarize(table, column, stats),
     { persist: true }
   );
 
-  // extract summary stats, copy to field info
-  for (let i = 0; i < result.numCols; ++i) {
-    const { name } = result.schema.fields[i];
-    const child = result.getChildAt(i);
-    const convert = convertArrowValue(child.type);
-    info[name] = convert(child.get(0));
-  }
-
-  return info;
+  // extract summary stats, copy to field info, and return
+  return Object.assign(info, result);
 }
 
 async function getTableInfo(mc, table) {