@uwdata/mosaic-core 0.10.0 → 0.12.0

package/dist/types/util/selection-types.d.ts

@@ -0,0 +1,114 @@
+import { ExprNode } from '@uwdata/mosaic-sql';
+import { MosaicClient } from '../MosaicClient.js';
+/**
+ * Selection clause metadata to guide possible query optimizations.
+ * Sub-interfaces provide more information about the specifics of a
+ * given selection based on the selection type.
+ */
+export interface ClauseMetadata {
+    /**
+     * The selection type, such as `'point'`, `'interval'`, or `'match'`.
+     */
+    type: string;
+}
+/**
+ * Selection clause metadata indicating selection of one or more discrete
+ * point values, typically based on equality or is distinctiveness checks.
+ */
+export interface PointMetadata extends ClauseMetadata {
+    type: 'point';
+}
+/** Text search matching methods. */
+export type MatchMethod = 'contains' | 'prefix' | 'suffix' | 'regexp' | (string & {});
+/**
+ * Selection clause metadata indicating text search matching.
+ */
+export interface MatchMetadata extends ClauseMetadata {
+    type: MatchMethod;
+    /** The text search matching method used. */
+    method?: 'contains' | 'prefix' | 'suffix' | 'regexp' | (string & {});
+}
+/** Quantitative scale types. */
+export type ScaleType = 'identity' | 'linear' | 'log' | 'sqrt' | 'pow' | 'symlog' | 'time' | 'utc';
+/** A data value interval extent. */
+export type Extent = [number, number] | [Date, Date];
+/**
+ * Descriptor for a scale that maps a data domain to screen pixels.
+ */
+export interface Scale {
+    /** The scale type, such as `'linear'`, `'log'`, etc. */
+    type: ScaleType;
+    /** The scale domain, as an array of start and end data values. */
+    domain: Extent;
+    /**
+     * The scale range, as an array of start and end screen pixels.
+     * The range may be omitted for *identity* scales.
+     */
+    range?: [number, number];
+    /** The base of the logarithm. For `'log'` scales only. */
+    base?: number;
+    /** The constant parameter. For `'symlog'` scales only. */
+    constant?: number;
+    /** The exponent parameter. For `'pow'` scales only. */
+    exponent?: number;
+}
+/** A binning method name. */
+export type BinMethod = 'floor' | 'ceil' | 'round';
+/**
+ * Selection clause metadata for one or more selected intervals. This
+ * metadata can be used to determine appropriate data-space binning
+ * schemes that correspond to pixel-level bins in screen space.
+ */
+export interface IntervalMetadata extends ClauseMetadata {
+    type: 'interval';
+    /**
+     * The interactive pixel size used by the generating component.
+     * Values larger than one indicate intervals that "snap-to" values
+     * greater than a single pixel. If unspecified, assumed to be `1`.
+     */
+    pixelSize?: number;
+    /**
+     * An array of one or more scale descriptors that describe the
+     * mapping from data values to screen pixels.
+     */
+    scales?: Scale[];
+    /**
+     * A hint for the binning method to use when discretizing the
+     * interval domain. If unspecified, the default is `'floor'`.
+     */
+    bin?: BinMethod;
+}
+/**
+ * A selection clause representing filtering criteria
+ * to apply within a Mosiac Selection.
+ */
+export interface SelectionClause {
+    /**
+     * A unique identifier (according to object equality) for the source
+     * component that generated this clause. In many cases, this is a
+     * reference to the originating component itself.
+     */
+    source: any;
+    /**
+     * A set of Mosaic clients associated with this clause that should not
+     * be updated when this clause is applied in a cross-filtering context.
+     */
+    clients?: Set<MosaicClient>;
+    /**
+     * A selected value associated with this clause. For example, for a 1D
+     * interval selection clause the value may be a [lo, hi] array.
+     */
+    value: any;
+    /**
+     * A predicate SQL expression suitable for use in a query WHERE clause.
+     * The predicate should apply filtering criteria consistent with this
+     * clause's *value* property.
+     */
+    predicate: ExprNode | null;
+    /**
+     * Optional clause metadata that varies based on the selection type.
+     * The metadata can be used to optimize selection queries, for example
+     * by creating materialized views of pre-aggregated data when applicable.
+     */
+    meta?: ClauseMetadata;
+}

package/dist/types/util/synchronizer.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Create a new synchronizer instance to aid synchronization
+ * of updates on multiple pending operations.
+ */
+export function synchronizer(): {
+    /**
+     * Mark an item as pending.
+     * @param {*} item An item to synchronize on.
+     */
+    pending(item: any): void;
+    /**
+     * Mark a pending item as ready, indicating it is
+     * ready for a synchronized update.
+     * @param {*} item An item to synchronize on.
+     * @returns {boolean} True if the synchronizer is ready to
+     *  resolve, false otherwise.
+     */
+    ready(item: any): boolean;
+    /**
+     * Resolve the current synchronization cycle, causing the synchronize
+     * promise to resolve and thereby trigger downstream updates.
+     */
+    resolve(): void;
+    /**
+     * The promise for the current synchronization cycle.
+     * @return {Promise} The synchronization promise.
+     */
+    readonly promise: Promise<any>;
+};

package/dist/types/util/throttle.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Throttle invocations of a callback function. The callback must return
+ * a Promise. Upon repeated invocation, the callback will not be invoked
+ * until a prior Promise resolves. If multiple invocations occurs while
+ * waiting, only the most recent invocation will be pending.
+ * @param {(event: *) => Promise} callback The callback function.
+ * @param {boolean} [debounce=true] Flag indicating if invocations
+ *  should also be debounced within the current animation frame.
+ * @returns A new function that throttles access to the callback.
+ */
+export function throttle(callback: (event: any) => Promise<any>, debounce?: boolean): (event: any) => void;

package/dist/types/util/to-data-columns.d.ts

@@ -0,0 +1,29 @@
+/**
+ * @typedef {Array | Int8Array | Uint8Array | Uint8ClampedArray
+ *  | Int16Array | Uint16Array | Int32Array | Uint32Array
+ *  | Float32Array | Float64Array
+ * } Arrayish - an Array or TypedArray
+ */
+/**
+ * @typedef {
+ *  | { numRows: number, columns: Record<string,Arrayish> }
+ *  | { numRows: number, values: Arrayish; }
+ * } DataColumns
+*/
+/**
+ * Convert input data to a set of column arrays.
+ * @param {any} data The input data.
+ * @returns {DataColumns} An object with named column arrays.
+ */
+export function toDataColumns(data: any): DataColumns;
+/**
+ * - an Array or TypedArray
+ */
+export type Arrayish = any[] | Int8Array | Uint8Array | Uint8ClampedArray | Int16Array | Uint16Array | Int32Array | Uint32Array | Float32Array | Float64Array;
+export type DataColumns = any | {
+    numRows: number;
+    columns: Record<string, Arrayish>;
+} | {
+    numRows: number;
+    values: Arrayish;
+};

package/dist/types/util/void-logger.d.ts

@@ -0,0 +1,7 @@
+export function voidLogger(): {
+    debug(..._: any[]): void;
+    info(..._: any[]): void;
+    log(..._: any[]): void;
+    warn(..._: any[]): void;
+    error(..._: any[]): void;
+};

package/jsconfig.json

@@ -0,0 +1,11 @@
+{
+  "include": ["src/**/*"],
+  "compilerOptions": {
+    "checkJs": true,
+    "noEmit": true,
+    "noImplicitAny": false,
+    "module": "node16",
+    "skipLibCheck": true,
+    "types": []
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uwdata/mosaic-core",
-  "version": "0.10.0",
+  "version": "0.12.0",
   "description": "Scalable and extensible linked data views.",
   "keywords": [
     "mosaic",
@@ -16,24 +16,26 @@
   "module": "src/index.js",
   "jsdelivr": "dist/mosaic-core.min.js",
   "unpkg": "dist/mosaic-core.min.js",
+  "types": "dist/types/index.d.ts",
   "repository": {
     "type": "git",
     "url": "https://github.com/uwdata/mosaic.git"
   },
   "scripts": {
     "prebuild": "rimraf dist && mkdir dist",
-    "build": "node ../../esbuild.js mosaic-core",
+    "build": "npm run types && node ../../esbuild.js mosaic-core",
+    "types": "tsc",
     "lint": "eslint src test",
-    "test": "mocha 'test/**/*-test.js'",
+    "test": "vitest run && tsc -p jsconfig.json",
     "prepublishOnly": "npm run test && npm run lint && npm run build"
   },
   "dependencies": {
-    "@duckdb/duckdb-wasm": "^1.28.1-dev232.0",
-    "@uwdata/mosaic-sql": "^0.10.0",
-    "apache-arrow": "^16.1.0"
+    "@duckdb/duckdb-wasm": "^1.29.0",
+    "@uwdata/flechette": "^1.1.1",
+    "@uwdata/mosaic-sql": "^0.12.0"
   },
   "devDependencies": {
-    "@uwdata/mosaic-duckdb": "^0.10.0"
+    "@uwdata/mosaic-duckdb": "^0.12.0"
   },
-  "gitHead": "94fc4f0d4efc622001f6afd6714d1e9dda745be2"
+  "gitHead": "523b1afe2a0880291c92f81e4a7b91829362d285"
 }

package/src/Coordinator.js

@@ -1,8 +1,10 @@
 import { socketConnector } from './connectors/socket.js';
-import { DataCubeIndexer } from './DataCubeIndexer.js';
-import { QueryManager, Priority } from './QueryManager.js';
+import { PreAggregator } from './preagg/PreAggregator.js';
 import { queryFieldInfo } from './util/field-info.js';
+import { QueryResult } from './util/query-result.js';
 import { voidLogger } from './util/void-logger.js';
+import { MosaicClient } from './MosaicClient.js';
+import { QueryManager, Priority } from './QueryManager.js';
 
 /**
  * The singleton Coordinator instance.
@@ -24,17 +26,22 @@ export function coordinator(instance) {
   return _instance;
 }
 
+/**
+ * @typedef {import('@uwdata/mosaic-sql').Query | string} QueryType
+ */
+
 /**
  * A Mosaic Coordinator manages all database communication for clients and
  * handles selection updates. The Coordinator also performs optimizations
- * including query caching, consolidation, and data cube indexing.
+ * including query caching, consolidation, and pre-aggregation.
  * @param {*} [db] Database connector. Defaults to a web socket connection.
  * @param {object} [options] Coordinator options.
  * @param {*} [options.logger=console] The logger to use, defaults to `console`.
  * @param {*} [options.manager] The query manager to use.
  * @param {boolean} [options.cache=true] Boolean flag to enable/disable query caching.
  * @param {boolean} [options.consolidate=true] Boolean flag to enable/disable query consolidation.
- * @param {object} [options.indexes] Data cube indexer options.
+ * @param {import('./preagg/PreAggregator.js').PreAggregateOptions} [options.preagg]
+ *  Options for the Pre-aggregator.
  */
 export class Coordinator {
   constructor(db = socketConnector(), {
@@ -42,15 +49,16 @@ export class Coordinator {
     manager = new QueryManager(),
     cache = true,
     consolidate = true,
-    indexes = {}
+    preagg = {}
   } = {}) {
+    /** @type {QueryManager} */
     this.manager = manager;
     this.manager.cache(cache);
     this.manager.consolidate(consolidate);
-    this.dataCubeIndexer = new DataCubeIndexer(this, indexes);
-    this.logger(logger);
     this.databaseConnector(db);
+    this.logger(logger);
     this.clear();
+    this.preaggregator = new PreAggregator(this, preagg);
   }
 
   /**
@@ -97,7 +105,7 @@ export class Coordinator {
   /**
    * Cancel previosuly submitted query requests. These queries will be
    * canceled if they are queued but have not yet been submitted.
-   * @param {import('./util/query-result.js').QueryResult[]} requests An array
+   * @param {QueryResult[]} requests An array
    *  of query result objects, such as those returned by the `query` method.
    */
   cancel(requests) {
@@ -106,29 +114,30 @@ export class Coordinator {
 
   /**
    * Issue a query for which no result (return value) is needed.
-   * @param {import('@uwdata/mosaic-sql').Query | string} query The query.
+   * @param {QueryType | QueryType[]} query The query or an array of queries.
+   *  Each query should be either a Query builder object or a SQL string.
    * @param {object} [options] An options object.
    * @param {number} [options.priority] The query priority, defaults to
    *  `Priority.Normal`.
-   * @returns {import('./util/query-result.js').QueryResult} A query result
+   * @returns {QueryResult} A query result
    *  promise.
    */
   exec(query, { priority = Priority.Normal } = {}) {
-    query = Array.isArray(query) ? query.join(';\n') : query;
+    query = Array.isArray(query) ? query.filter(x => x).join(';\n') : query;
     return this.manager.request({ type: 'exec', query }, priority);
   }
 
   /**
    * Issue a query to the backing database. The submitted query may be
    * consolidate with other queries and its results may be cached.
-   * @param {import('@uwdata/mosaic-sql').Query | string} query The query.
+   * @param {QueryType} query The query as either a Query builder object
+   *  or a SQL string.
    * @param {object} [options] An options object.
    * @param {'arrow' | 'json'} [options.type] The query result format type.
    * @param {boolean} [options.cache=true] If true, cache the query result.
    * @param {number} [options.priority] The query priority, defaults to
    *  `Priority.Normal`.
-   * @returns {import('./util/query-result.js').QueryResult} A query result
-   *  promise.
+   * @returns {QueryResult} A query result promise.
    */
   query(query, {
     type = 'arrow',
@@ -142,21 +151,35 @@ export class Coordinator {
   /**
    * Issue a query to prefetch data for later use. The query result is cached
    * for efficient future access.
-   * @param {import('@uwdata/mosaic-sql').Query | string} query The query.
+   * @param {QueryType} query The query as either a Query builder object
+   *  or a SQL string.
    * @param {object} [options] An options object.
    * @param {'arrow' | 'json'} [options.type] The query result format type.
-   * @returns {import('./util/query-result.js').QueryResult} A query result
-   *  promise.
+   * @returns {QueryResult} A query result promise.
    */
   prefetch(query, options = {}) {
     return this.query(query, { ...options, cache: true, priority: Priority.Low });
   }
 
+  /**
+   * Create a bundle of queries that can be loaded into the cache.
+   *
+   * @param {string} name The name of the bundle.
+   * @param {[string | {sql: string},  {alias: string}]} queries The queries to save into the bundle.
+   * @param {number} priority Request priority.
+   * @returns {QueryResult} A query result promise.
+   */
   createBundle(name, queries, priority = Priority.Low) {
-    const options = { name, queries };
+    const options = { name, queries: queries.map(q => typeof q == 'string' ? {sql: q} : q) };
     return this.manager.request({ type: 'create-bundle', options }, priority);
   }
 
+  /**
+   * Load a bundle into the cache.
+   * @param {string} name The name of the bundle.
+   * @param {number} priority Request priority.
+   * @returns {QueryResult} A query result promise.
+   */
   loadBundle(name, priority = Priority.High) {
     const options = { name };
     return this.manager.request({ type: 'load-bundle', options }, priority);
@@ -167,8 +190,8 @@ export class Coordinator {
   /**
    * Update client data by submitting the given query and returning the
    * data (or error) to the client.
-   * @param {import('./MosaicClient.js').MosaicClient} client A Mosaic client.
-   * @param {import('@uwdata/mosaic-sql').Query | string} query The data query.
+   * @param {MosaicClient} client A Mosaic client.
+   * @param {QueryType} query The data query.
    * @param {number} [priority] The query priority.
    * @returns {Promise} A Promise that resolves upon completion of the update.
    */
@@ -185,23 +208,20 @@ export class Coordinator {
   /**
    * Issue a query request for a client. If the query is null or undefined,
    * the client is simply updated. Otherwise `updateClient` is called. As a
-   * side effect, this method clears the current data cube indexer state.
-   * @param {import('./MosaicClient.js').MosaicClient} client The client
-   *  to update.
-   * @param {import('@uwdata/mosaic-sql').Query | string | null} [query]
-   *  The query to issue.
+   * side effect, this method clears the current preaggregator state.
+   * @param {MosaicClient} client The client to update.
+   * @param {QueryType | null} [query] The query to issue.
    */
   requestQuery(client, query) {
-    this.dataCubeIndexer.clear();
+    this.preaggregator.clear();
     return query
       ? this.updateClient(client, query)
-      : client.update();
+      : Promise.resolve(client.update());
   }
 
   /**
    * Connect a client to the coordinator.
-   * @param {import('./MosaicClient.js').MosaicClient} client The Mosaic
-   *  client to connect.
+   * @param {MosaicClient} client The Mosaic client to connect.
    */
   async connect(client) {
     const { clients } = this;
@@ -212,22 +232,27 @@ export class Coordinator {
     clients.add(client); // mark as connected
     client.coordinator = this;
 
+    // initialize client lifecycle
+    this.initializeClient(client);
+
+    // connect filter selection
+    connectSelection(this, client.filterBy, client);
+  }
+
+  async initializeClient(client) {
     // retrieve field statistics
     const fields = client.fields();
     if (fields?.length) {
       client.fieldInfo(await queryFieldInfo(this, fields));
     }
 
-    // connect filter selection
-    connectSelection(this, client.filterBy, client);
-
-    client.requestQuery();
+    // request data query
+    return client.requestQuery();
   }
 
   /**
    * Disconnect a client from the coordinator.
-   * @param {import('./MosaicClient.js').MosaicClient} client The Mosaic
-   *  client to disconnect.
+   * @param {MosaicClient} client The Mosaic client to disconnect.
    */
   disconnect(client) {
     const { clients, filterGroups } = this;
@@ -246,8 +271,8 @@ export class Coordinator {
  * Connect a selection-client pair to the coordinator to process updates.
  * @param {Coordinator} mc The Mosaic coordinator.
  * @param {import('./Selection.js').Selection} selection A selection.
- * @param {import('./MosaicClient.js').MosaicClient} client A Mosiac
- *  client that is filtered by the given selection.
+ * @param {MosaicClient} client A Mosiac client that is filtered by the
+ *  given selection.
  */
 function connectSelection(mc, selection, client) {
   if (!selection) return;
@@ -282,10 +307,10 @@ function connectSelection(mc, selection, client) {
  *  selection clause representative of the activation.
  */
 function activateSelection(mc, selection, clause) {
-  const { dataCubeIndexer, filterGroups } = mc;
+  const { preaggregator, filterGroups } = mc;
   const { clients } = filterGroups.get(selection);
   for (const client of clients) {
-    dataCubeIndexer.index(client, selection, clause);
+    preaggregator.request(client, selection, clause);
   }
 }
 
@@ -297,11 +322,11 @@ function activateSelection(mc, selection, clause) {
  * @returns {Promise} A Promise that resolves when the update completes.
  */
 function updateSelection(mc, selection) {
-  const { dataCubeIndexer, filterGroups } = mc;
+  const { preaggregator, filterGroups } = mc;
   const { clients } = filterGroups.get(selection);
   const { active } = selection;
   return Promise.allSettled(Array.from(clients, client => {
-    const info = dataCubeIndexer.index(client, selection, active);
+    const info = preaggregator.request(client, selection, active);
     const filter = info ? null : selection.predicate(client);
 
     // skip due to cross-filtering

package/src/MosaicClient.js

@@ -38,11 +38,12 @@ export class MosaicClient {
   }
 
   /**
-   * Return a boolean indicating if the client query can be indexed. Should
-   * return true if changes to the filterBy selection does not change the
-   * groupby domain of the client query.
+   * Return a boolean indicating if the client query can be sped up with
+   * materialized views of pre-aggregated data. Should return true if changes to
+   * the filterBy selection does not change the groupby domain of the client
+   * query.
    */
-  get filterIndexable() {
+  get filterStable() {
     return true;
   }
 
@@ -103,6 +104,7 @@ export class MosaicClient {
    * Request the coordinator to execute a query for this client.
    * If an explicit query is not provided, the client query method will
    * be called, filtered by the current filterBy selection.
+   * @returns {Promise}
    */
   requestQuery(query) {
     const q = query || this.query(this.filterBy?.predicate(this));
@@ -119,6 +121,14 @@ export class MosaicClient {
     this._requestUpdate();
   }
 
+  /**
+   * Reset this client, initiating new field info and query requests.
+   * @returns {Promise}
+   */
+  initialize() {
+    return this._coordinator.initializeClient(this);
+  }
+
   /**
    * Requests a client update.
    * For example to (re-)render an interface component.