@uwdata/mosaic-core 0.9.0 → 0.11.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uwdata/mosaic-core",
-  "version": "0.9.0",
+  "version": "0.11.0",
   "description": "Scalable and extensible linked data views.",
   "keywords": [
     "mosaic",
@@ -10,7 +10,7 @@
     "interface"
   ],
   "license": "BSD-3-Clause",
-  "author": "Jeffrey Heer (http://idl.cs.washington.edu)",
+  "author": "Jeffrey Heer (https://idl.uw.edu)",
   "type": "module",
   "main": "src/index.js",
   "module": "src/index.js",
@@ -24,16 +24,16 @@
     "prebuild": "rimraf dist && mkdir dist",
     "build": "node ../../esbuild.js mosaic-core",
     "lint": "eslint src test",
-    "test": "mocha 'test/**/*-test.js'",
+    "test": "vitest run --dangerouslyIgnoreUnhandledErrors",
     "prepublishOnly": "npm run test && npm run lint && npm run build"
   },
   "dependencies": {
-    "@duckdb/duckdb-wasm": "^1.28.1-dev195.0",
-    "@uwdata/mosaic-sql": "^0.9.0",
-    "apache-arrow": "^15.0.2"
+    "@duckdb/duckdb-wasm": "^1.28.1-dev278.0",
+    "@uwdata/flechette": "^1.0.2",
+    "@uwdata/mosaic-sql": "^0.11.0"
   },
   "devDependencies": {
-    "@uwdata/mosaic-duckdb": "^0.9.0"
+    "@uwdata/mosaic-duckdb": "^0.11.0"
   },
-  "gitHead": "89bb9b0dfa747aed691eaeba35379525a6764c61"
+  "gitHead": "861d616f39926a1d2aee83b59dbdd70b0b3caf12"
 }

package/src/Coordinator.js

@@ -1,9 +1,15 @@
 import { socketConnector } from './connectors/socket.js';
-import { FilterGroup } from './FilterGroup.js';
+import { DataCubeIndexer } from './DataCubeIndexer.js';
+import { MosaicClient } from './MosaicClient.js';
 import { QueryManager, Priority } from './QueryManager.js';
 import { queryFieldInfo } from './util/field-info.js';
+import { QueryResult } from './util/query-result.js';
 import { voidLogger } from './util/void-logger.js';
 
+/**
+ * The singleton Coordinator instance.
+ * @type {Coordinator}
+ */
 let _instance;
 
 /**
@@ -20,67 +26,119 @@ export function coordinator(instance) {
   return _instance;
 }
 
+/**
+ * @typedef {import('@uwdata/mosaic-sql').Query | string} QueryType
+ */
+
+/**
+ * A Mosaic Coordinator manages all database communication for clients and
+ * handles selection updates. The Coordinator also performs optimizations
+ * including query caching, consolidation, and data cube indexing.
+ * @param {*} [db] Database connector. Defaults to a web socket connection.
+ * @param {object} [options] Coordinator options.
+ * @param {*} [options.logger=console] The logger to use, defaults to `console`.
+ * @param {*} [options.manager] The query manager to use.
+ * @param {boolean} [options.cache=true] Boolean flag to enable/disable query caching.
+ * @param {boolean} [options.consolidate=true] Boolean flag to enable/disable query consolidation.
+ * @param {import('./DataCubeIndexer.js').DataCubeIndexerOptions} [options.indexes]
+ *  Data cube indexer options.
+ */
 export class Coordinator {
-  constructor(db = socketConnector(), options = {}) {
-    const {
-      logger = console,
-      manager = new QueryManager()
-    } = options;
+  constructor(db = socketConnector(), {
+    logger = console,
+    manager = new QueryManager(),
+    cache = true,
+    consolidate = true,
+    indexes = {}
+  } = {}) {
+    /** @type {QueryManager} */
     this.manager = manager;
-    this.logger(logger);
-    this.configure(options);
+    this.manager.cache(cache);
+    this.manager.consolidate(consolidate);
     this.databaseConnector(db);
+    this.logger(logger);
     this.clear();
-  }
-
-  logger(logger) {
-    if (arguments.length) {
-      this._logger = logger || voidLogger();
-      this.manager.logger(this._logger);
-    }
-    return this._logger;
+    this.dataCubeIndexer = new DataCubeIndexer(this, indexes);
   }
 
   /**
-   * Set configuration options for this coordinator.
-   * @param {object} [options] Configration options.
-   * @param {boolean} [options.cache=true] Boolean flag to enable/disable query caching.
-   * @param {boolean} [options.consolidate=true] Boolean flag to enable/disable query consolidation.
-   * @param {boolean|object} [options.indexes=true] Boolean flag to enable/disable
-   *  automatic data cube indexes or an index options object.
+   * Clear the coordinator state.
+   * @param {object} [options] Options object.
+   * @param {boolean} [options.clients=true] If true, disconnect all clients.
+   * @param {boolean} [options.cache=true] If true, clear the query cache.
    */
-  configure({ cache = true, consolidate = true, indexes = true } = {}) {
-    this.manager.cache(cache);
-    this.manager.consolidate(consolidate);
-    this.indexes = indexes;
-  }
-
   clear({ clients = true, cache = true } = {}) {
     this.manager.clear();
     if (clients) {
+      this.filterGroups?.forEach(group => group.disconnect());
+      this.filterGroups = new Map;
       this.clients?.forEach(client => this.disconnect(client));
-      this.filterGroups?.forEach(group => group.finalize());
       this.clients = new Set;
-      this.filterGroups = new Map;
     }
     if (cache) this.manager.cache().clear();
   }
 
+  /**
+   * Get or set the database connector.
+   * @param {*} [db] The database connector to use.
+   * @returns The current database connector.
+   */
   databaseConnector(db) {
     return this.manager.connector(db);
   }
 
+  /**
+   * Get or set the logger.
+   * @param {*} logger  The logger to use.
+   * @returns The current logger
+   */
+  logger(logger) {
+    if (arguments.length) {
+      this._logger = logger || voidLogger();
+      this.manager.logger(this._logger);
+    }
+    return this._logger;
+  }
+
   // -- Query Management ----
 
+  /**
+   * Cancel previosuly submitted query requests. These queries will be
+   * canceled if they are queued but have not yet been submitted.
+   * @param {QueryResult[]} requests An array
+   *  of query result objects, such as those returned by the `query` method.
+   */
   cancel(requests) {
     this.manager.cancel(requests);
   }
 
+  /**
+   * Issue a query for which no result (return value) is needed.
+   * @param {QueryType | QueryType[]} query The query or an array of queries.
+   *  Each query should be either a Query builder object or a SQL string.
+   * @param {object} [options] An options object.
+   * @param {number} [options.priority] The query priority, defaults to
+   *  `Priority.Normal`.
+   * @returns {QueryResult} A query result
+   *  promise.
+   */
   exec(query, { priority = Priority.Normal } = {}) {
-    query = Array.isArray(query) ? query.join(';\n') : query;
+    query = Array.isArray(query) ? query.filter(x => x).join(';\n') : query;
     return this.manager.request({ type: 'exec', query }, priority);
   }
 
+  /**
+   * Issue a query to the backing database. The submitted query may be
+   * consolidate with other queries and its results may be cached.
+   * @param {QueryType} query The query as either a Query builder object
+   *  or a SQL string.
+   * @param {object} [options] An options object.
+   * @param {'arrow' | 'json'} [options.type] The query result format type.
+   * @param {boolean} [options.cache=true] If true, cache the query result.
+   * @param {number} [options.priority] The query priority, defaults to
+   *  `Priority.Normal`.
+   * @returns {QueryResult} A query result promise.
+   */
   query(query, {
     type = 'arrow',
     cache = true,
@@ -90,15 +148,38 @@ export class Coordinator {
     return this.manager.request({ type, query, cache, options }, priority);
   }
 
+  /**
+   * Issue a query to prefetch data for later use. The query result is cached
+   * for efficient future access.
+   * @param {QueryType} query The query as either a Query builder object
+   *  or a SQL string.
+   * @param {object} [options] An options object.
+   * @param {'arrow' | 'json'} [options.type] The query result format type.
+   * @returns {QueryResult} A query result promise.
+   */
   prefetch(query, options = {}) {
     return this.query(query, { ...options, cache: true, priority: Priority.Low });
   }
 
+  /**
+   * Create a bundle of queries that can be loaded into the cache.
+   *
+   * @param {string} name The name of the bundle.
+   * @param {[string | {sql: string},  {alias: string}]} queries The queries to save into the bundle.
+   * @param {number} priority Request priority.
+   * @returns {QueryResult} A query result promise.
+   */
   createBundle(name, queries, priority = Priority.Low) {
-    const options = { name, queries };
+    const options = { name, queries: queries.map(q => typeof q == 'string' ? {sql: q} : q) };
     return this.manager.request({ type: 'create-bundle', options }, priority);
   }
 
+  /**
+   * Load a bundle into the cache.
+   * @param {string} name The name of the bundle.
+   * @param {number} priority Request priority.
+   * @returns {QueryResult} A query result promise.
+   */
   loadBundle(name, priority = Priority.High) {
     const options = { name };
     return this.manager.request({ type: 'load-bundle', options }, priority);
@@ -106,27 +187,44 @@ export class Coordinator {
 
   // -- Client Management ----
 
+  /**
+   * Update client data by submitting the given query and returning the
+   * data (or error) to the client.
+   * @param {MosaicClient} client A Mosaic client.
+   * @param {QueryType} query The data query.
+   * @param {number} [priority] The query priority.
+   * @returns {Promise} A Promise that resolves upon completion of the update.
+   */
   updateClient(client, query, priority = Priority.Normal) {
     client.queryPending();
-    return this.query(query, { priority }).then(
-      data => client.queryResult(data).update(),
-      err => { client.queryError(err); this._logger.error(err); }
-    );
+    return this.query(query, { priority })
+      .then(
+        data => client.queryResult(data).update(),
+        err => { this._logger.error(err); client.queryError(err); }
+      )
+      .catch(err => this._logger.error(err));
   }
 
+  /**
+   * Issue a query request for a client. If the query is null or undefined,
+   * the client is simply updated. Otherwise `updateClient` is called. As a
+   * side effect, this method clears the current data cube indexer state.
+   * @param {MosaicClient} client The client to update.
+   * @param {QueryType | null} [query] The query to issue.
+   */
   requestQuery(client, query) {
-    this.filterGroups.get(client.filterBy)?.reset();
+    this.dataCubeIndexer.clear();
     return query
       ? this.updateClient(client, query)
-      : client.update();
+      : Promise.resolve(client.update());
   }
 
   /**
    * Connect a client to the coordinator.
-   * @param {import('./MosaicClient.js').MosaicClient} client the client to disconnect
+   * @param {MosaicClient} client The Mosaic client to connect.
    */
   async connect(client) {
-    const { clients, filterGroups, indexes } = this;
+    const { clients } = this;
 
     if (clients.has(client)) {
       throw new Error('Client already connected.');
@@ -134,36 +232,108 @@ export class Coordinator {
     clients.add(client); // mark as connected
     client.coordinator = this;
 
+    // initialize client lifecycle
+    this.initializeClient(client);
+
+    // connect filter selection
+    connectSelection(this, client.filterBy, client);
+  }
+
+  async initializeClient(client) {
     // retrieve field statistics
     const fields = client.fields();
     if (fields?.length) {
       client.fieldInfo(await queryFieldInfo(this, fields));
     }
 
-    // connect filters
-    const filter = client.filterBy;
-    if (filter) {
-      if (filterGroups.has(filter)) {
-        filterGroups.get(filter).add(client);
-      } else {
-        const group = new FilterGroup(this, filter, indexes);
-        filterGroups.set(filter, group.add(client));
-      }
-    }
-
-    client.requestQuery();
+    // request data query
+    return client.requestQuery();
   }
 
   /**
    * Disconnect a client from the coordinator.
-   *
-   * @param {import('./MosaicClient.js').MosaicClient} client the client to disconnect
+   * @param {MosaicClient} client The Mosaic client to disconnect.
    */
   disconnect(client) {
     const { clients, filterGroups } = this;
     if (!clients.has(client)) return;
     clients.delete(client);
-    filterGroups.get(client.filterBy)?.remove(client);
     client.coordinator = null;
+
+    const group = filterGroups.get(client.filterBy);
+    if (group) {
+      group.clients.delete(client);
+    }
   }
 }
+
+/**
+ * Connect a selection-client pair to the coordinator to process updates.
+ * @param {Coordinator} mc The Mosaic coordinator.
+ * @param {import('./Selection.js').Selection} selection A selection.
+ * @param {MosaicClient} client A Mosiac client that is filtered by the
+ *  given selection.
+ */
+function connectSelection(mc, selection, client) {
+  if (!selection) return;
+  let entry = mc.filterGroups.get(selection);
+  if (!entry) {
+    const activate = clause => activateSelection(mc, selection, clause);
+    const value = () => updateSelection(mc, selection);
+
+    selection.addEventListener('activate', activate);
+    selection.addEventListener('value', value);
+
+    entry = {
+      selection,
+      clients: new Set,
+      disconnect() {
+        selection.removeEventListener('activate', activate);
+        selection.removeEventListener('value', value);
+      }
+    };
+    mc.filterGroups.set(selection, entry);
+  }
+  entry.clients.add(client);
+}
+
+/**
+ * Activate a selection, providing a clause indicative of potential
+ * next updates. Activation provides a preview of likely next events,
+ * enabling potential precomputation to optimize updates.
+ * @param {Coordinator} mc The Mosaic coordinator.
+ * @param {import('./Selection.js').Selection} selection A selection.
+ * @param {import('./util/selection-types.js').SelectionClause} clause A
+ *  selection clause representative of the activation.
+ */
+function activateSelection(mc, selection, clause) {
+  const { dataCubeIndexer, filterGroups } = mc;
+  const { clients } = filterGroups.get(selection);
+  for (const client of clients) {
+    dataCubeIndexer.index(client, selection, clause);
+  }
+}
+
+/**
+ * Process an updated selection value, querying filtered data for any
+ * associated clients.
+ * @param {Coordinator} mc The Mosaic coordinator.
+ * @param {import('./Selection.js').Selection} selection A selection.
+ * @returns {Promise} A Promise that resolves when the update completes.
+ */
+function updateSelection(mc, selection) {
+  const { dataCubeIndexer, filterGroups } = mc;
+  const { clients } = filterGroups.get(selection);
+  const { active } = selection;
+  return Promise.allSettled(Array.from(clients, client => {
+    const info = dataCubeIndexer.index(client, selection, active);
+    const filter = info ? null : selection.predicate(client);
+
+    // skip due to cross-filtering
+    if (info?.skip || (!info && !filter)) return;
+
+    // @ts-ignore
+    const query = info?.query(active.predicate) ?? client.query(filter);
+    return mc.updateClient(client, query);
+  }));
+}