@uwdata/mosaic-core 0.11.0 → 0.12.1

package/dist/types/util/selection-types.d.ts

@@ -0,0 +1,114 @@
+import { ExprNode } from '@uwdata/mosaic-sql';
+import { MosaicClient } from '../MosaicClient.js';
+/**
+ * Selection clause metadata to guide possible query optimizations.
+ * Sub-interfaces provide more information about the specifics of a
+ * given selection based on the selection type.
+ */
+export interface ClauseMetadata {
+    /**
+     * The selection type, such as `'point'`, `'interval'`, or `'match'`.
+     */
+    type: string;
+}
+/**
+ * Selection clause metadata indicating selection of one or more discrete
+ * point values, typically based on equality or is distinctiveness checks.
+ */
+export interface PointMetadata extends ClauseMetadata {
+    type: 'point';
+}
+/** Text search matching methods. */
+export type MatchMethod = 'contains' | 'prefix' | 'suffix' | 'regexp' | (string & {});
+/**
+ * Selection clause metadata indicating text search matching.
+ */
+export interface MatchMetadata extends ClauseMetadata {
+    type: MatchMethod;
+    /** The text search matching method used. */
+    method?: 'contains' | 'prefix' | 'suffix' | 'regexp' | (string & {});
+}
+/** Quantitative scale types. */
+export type ScaleType = 'identity' | 'linear' | 'log' | 'sqrt' | 'pow' | 'symlog' | 'time' | 'utc';
+/** A data value interval extent. */
+export type Extent = [number, number] | [Date, Date];
+/**
+ * Descriptor for a scale that maps a data domain to screen pixels.
+ */
+export interface Scale {
+    /** The scale type, such as `'linear'`, `'log'`, etc. */
+    type: ScaleType;
+    /** The scale domain, as an array of start and end data values. */
+    domain: Extent;
+    /**
+     * The scale range, as an array of start and end screen pixels.
+     * The range may be omitted for *identity* scales.
+     */
+    range?: [number, number];
+    /** The base of the logarithm. For `'log'` scales only. */
+    base?: number;
+    /** The constant parameter. For `'symlog'` scales only. */
+    constant?: number;
+    /** The exponent parameter. For `'pow'` scales only. */
+    exponent?: number;
+}
+/** A binning method name. */
+export type BinMethod = 'floor' | 'ceil' | 'round';
+/**
+ * Selection clause metadata for one or more selected intervals. This
+ * metadata can be used to determine appropriate data-space binning
+ * schemes that correspond to pixel-level bins in screen space.
+ */
+export interface IntervalMetadata extends ClauseMetadata {
+    type: 'interval';
+    /**
+     * The interactive pixel size used by the generating component.
+     * Values larger than one indicate intervals that "snap-to" values
+     * greater than a single pixel. If unspecified, assumed to be `1`.
+     */
+    pixelSize?: number;
+    /**
+     * An array of one or more scale descriptors that describe the
+     * mapping from data values to screen pixels.
+     */
+    scales?: Scale[];
+    /**
+     * A hint for the binning method to use when discretizing the
+     * interval domain. If unspecified, the default is `'floor'`.
+     */
+    bin?: BinMethod;
+}
+/**
+ * A selection clause representing filtering criteria
+ * to apply within a Mosiac Selection.
+ */
+export interface SelectionClause {
+    /**
+     * A unique identifier (according to object equality) for the source
+     * component that generated this clause. In many cases, this is a
+     * reference to the originating component itself.
+     */
+    source: any;
+    /**
+     * A set of Mosaic clients associated with this clause that should not
+     * be updated when this clause is applied in a cross-filtering context.
+     */
+    clients?: Set<MosaicClient>;
+    /**
+     * A selected value associated with this clause. For example, for a 1D
+     * interval selection clause the value may be a [lo, hi] array.
+     */
+    value: any;
+    /**
+     * A predicate SQL expression suitable for use in a query WHERE clause.
+     * The predicate should apply filtering criteria consistent with this
+     * clause's *value* property.
+     */
+    predicate: ExprNode | null;
+    /**
+     * Optional clause metadata that varies based on the selection type.
+     * The metadata can be used to optimize selection queries, for example
+     * by creating materialized views of pre-aggregated data when applicable.
+     */
+    meta?: ClauseMetadata;
+}

package/dist/types/util/synchronizer.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Create a new synchronizer instance to aid synchronization
+ * of updates on multiple pending operations.
+ */
+export function synchronizer(): {
+    /**
+     * Mark an item as pending.
+     * @param {*} item An item to synchronize on.
+     */
+    pending(item: any): void;
+    /**
+     * Mark a pending item as ready, indicating it is
+     * ready for a synchronized update.
+     * @param {*} item An item to synchronize on.
+     * @returns {boolean} True if the synchronizer is ready to
+     *  resolve, false otherwise.
+     */
+    ready(item: any): boolean;
+    /**
+     * Resolve the current synchronization cycle, causing the synchronize
+     * promise to resolve and thereby trigger downstream updates.
+     */
+    resolve(): void;
+    /**
+     * The promise for the current synchronization cycle.
+     * @return {Promise} The synchronization promise.
+     */
+    readonly promise: Promise<any>;
+};

package/dist/types/util/throttle.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Throttle invocations of a callback function. The callback must return
+ * a Promise. Upon repeated invocation, the callback will not be invoked
+ * until a prior Promise resolves. If multiple invocations occurs while
+ * waiting, only the most recent invocation will be pending.
+ * @param {(event: *) => Promise} callback The callback function.
+ * @param {boolean} [debounce=true] Flag indicating if invocations
+ *  should also be debounced within the current animation frame.
+ * @returns A new function that throttles access to the callback.
+ */
+export function throttle(callback: (event: any) => Promise<any>, debounce?: boolean): (event: any) => void;

package/dist/types/util/to-data-columns.d.ts

@@ -0,0 +1,29 @@
+/**
+ * @typedef {Array | Int8Array | Uint8Array | Uint8ClampedArray
+ *  | Int16Array | Uint16Array | Int32Array | Uint32Array
+ *  | Float32Array | Float64Array
+ * } Arrayish - an Array or TypedArray
+ */
+/**
+ * @typedef {
+ *  | { numRows: number, columns: Record<string,Arrayish> }
+ *  | { numRows: number, values: Arrayish; }
+ * } DataColumns
+*/
+/**
+ * Convert input data to a set of column arrays.
+ * @param {any} data The input data.
+ * @returns {DataColumns} An object with named column arrays.
+ */
+export function toDataColumns(data: any): DataColumns;
+/**
+ * - an Array or TypedArray
+ */
+export type Arrayish = any[] | Int8Array | Uint8Array | Uint8ClampedArray | Int16Array | Uint16Array | Int32Array | Uint32Array | Float32Array | Float64Array;
+export type DataColumns = any | {
+    numRows: number;
+    columns: Record<string, Arrayish>;
+} | {
+    numRows: number;
+    values: Arrayish;
+};

package/dist/types/util/void-logger.d.ts

@@ -0,0 +1,7 @@
+export function voidLogger(): {
+    debug(..._: any[]): void;
+    info(..._: any[]): void;
+    log(..._: any[]): void;
+    warn(..._: any[]): void;
+    error(..._: any[]): void;
+};

package/jsconfig.json

@@ -0,0 +1,11 @@
+{
+  "include": ["src/**/*"],
+  "compilerOptions": {
+    "checkJs": true,
+    "noEmit": true,
+    "noImplicitAny": false,
+    "module": "node16",
+    "skipLibCheck": true,
+    "types": []
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uwdata/mosaic-core",
-  "version": "0.11.0",
+  "version": "0.12.1",
   "description": "Scalable and extensible linked data views.",
   "keywords": [
     "mosaic",
@@ -16,24 +16,26 @@
   "module": "src/index.js",
   "jsdelivr": "dist/mosaic-core.min.js",
   "unpkg": "dist/mosaic-core.min.js",
+  "types": "dist/types/index.d.ts",
   "repository": {
     "type": "git",
     "url": "https://github.com/uwdata/mosaic.git"
   },
   "scripts": {
     "prebuild": "rimraf dist && mkdir dist",
-    "build": "node ../../esbuild.js mosaic-core",
+    "build": "npm run types && node ../../esbuild.js mosaic-core",
+    "types": "tsc",
     "lint": "eslint src test",
-    "test": "vitest run --dangerouslyIgnoreUnhandledErrors",
+    "test": "vitest run && tsc -p jsconfig.json",
     "prepublishOnly": "npm run test && npm run lint && npm run build"
   },
   "dependencies": {
-    "@duckdb/duckdb-wasm": "^1.28.1-dev278.0",
-    "@uwdata/flechette": "^1.0.2",
-    "@uwdata/mosaic-sql": "^0.11.0"
+    "@duckdb/duckdb-wasm": "^1.29.0",
+    "@uwdata/flechette": "^1.1.1",
+    "@uwdata/mosaic-sql": "^0.12.1"
   },
   "devDependencies": {
-    "@uwdata/mosaic-duckdb": "^0.11.0"
+    "@uwdata/mosaic-duckdb": "^0.12.1"
   },
-  "gitHead": "861d616f39926a1d2aee83b59dbdd70b0b3caf12"
+  "gitHead": "fe3a7c34352da54ec36a1ebf557846f46a649782"
 }

package/src/Coordinator.js

@@ -1,10 +1,10 @@
 import { socketConnector } from './connectors/socket.js';
-import { DataCubeIndexer } from './DataCubeIndexer.js';
-import { MosaicClient } from './MosaicClient.js';
-import { QueryManager, Priority } from './QueryManager.js';
+import { PreAggregator } from './preagg/PreAggregator.js';
 import { queryFieldInfo } from './util/field-info.js';
 import { QueryResult } from './util/query-result.js';
 import { voidLogger } from './util/void-logger.js';
+import { MosaicClient } from './MosaicClient.js';
+import { QueryManager, Priority } from './QueryManager.js';
 
 /**
  * The singleton Coordinator instance.
@@ -33,15 +33,15 @@ export function coordinator(instance) {
 /**
  * A Mosaic Coordinator manages all database communication for clients and
  * handles selection updates. The Coordinator also performs optimizations
- * including query caching, consolidation, and data cube indexing.
+ * including query caching, consolidation, and pre-aggregation.
  * @param {*} [db] Database connector. Defaults to a web socket connection.
  * @param {object} [options] Coordinator options.
  * @param {*} [options.logger=console] The logger to use, defaults to `console`.
  * @param {*} [options.manager] The query manager to use.
  * @param {boolean} [options.cache=true] Boolean flag to enable/disable query caching.
  * @param {boolean} [options.consolidate=true] Boolean flag to enable/disable query consolidation.
- * @param {import('./DataCubeIndexer.js').DataCubeIndexerOptions} [options.indexes]
- *  Data cube indexer options.
+ * @param {import('./preagg/PreAggregator.js').PreAggregateOptions} [options.preagg]
+ *  Options for the Pre-aggregator.
  */
 export class Coordinator {
   constructor(db = socketConnector(), {
@@ -49,7 +49,7 @@ export class Coordinator {
     manager = new QueryManager(),
     cache = true,
     consolidate = true,
-    indexes = {}
+    preagg = {}
   } = {}) {
     /** @type {QueryManager} */
     this.manager = manager;
@@ -58,7 +58,7 @@ export class Coordinator {
     this.databaseConnector(db);
     this.logger(logger);
     this.clear();
-    this.dataCubeIndexer = new DataCubeIndexer(this, indexes);
+    this.preaggregator = new PreAggregator(this, preagg);
   }
 
   /**
@@ -208,12 +208,12 @@ export class Coordinator {
   /**
    * Issue a query request for a client. If the query is null or undefined,
    * the client is simply updated. Otherwise `updateClient` is called. As a
-   * side effect, this method clears the current data cube indexer state.
+   * side effect, this method clears the current preaggregator state.
    * @param {MosaicClient} client The client to update.
    * @param {QueryType | null} [query] The query to issue.
    */
   requestQuery(client, query) {
-    this.dataCubeIndexer.clear();
+    this.preaggregator.clear();
     return query
       ? this.updateClient(client, query)
       : Promise.resolve(client.update());
@@ -307,10 +307,10 @@ function connectSelection(mc, selection, client) {
  *  selection clause representative of the activation.
  */
 function activateSelection(mc, selection, clause) {
-  const { dataCubeIndexer, filterGroups } = mc;
+  const { preaggregator, filterGroups } = mc;
   const { clients } = filterGroups.get(selection);
   for (const client of clients) {
-    dataCubeIndexer.index(client, selection, clause);
+    preaggregator.request(client, selection, clause);
   }
 }
 
@@ -322,11 +322,11 @@ function activateSelection(mc, selection, clause) {
  * @returns {Promise} A Promise that resolves when the update completes.
  */
 function updateSelection(mc, selection) {
-  const { dataCubeIndexer, filterGroups } = mc;
+  const { preaggregator, filterGroups } = mc;
   const { clients } = filterGroups.get(selection);
   const { active } = selection;
   return Promise.allSettled(Array.from(clients, client => {
-    const info = dataCubeIndexer.index(client, selection, active);
+    const info = preaggregator.request(client, selection, active);
     const filter = info ? null : selection.predicate(client);
 
     // skip due to cross-filtering

package/src/MosaicClient.js

@@ -38,11 +38,12 @@ export class MosaicClient {
   }
 
   /**
-   * Return a boolean indicating if the client query can be indexed. Should
-   * return true if changes to the filterBy selection does not change the
-   * groupby domain of the client query.
+   * Return a boolean indicating if the client query can be sped up with
+   * materialized views of pre-aggregated data. Should return true if changes to
+   * the filterBy selection does not change the groupby domain of the client
+   * query.
    */
-  get filterIndexable() {
+  get filterStable() {
     return true;
   }
 

package/src/QueryConsolidator.js

@@ -1,4 +1,4 @@
-import { Query, Ref, isDescribeQuery } from '@uwdata/mosaic-sql';
+import { DescribeQuery, isAggregateExpression, isColumnRef, isDescribeQuery, isSelectQuery, Query } from '@uwdata/mosaic-sql';
 import { QueryResult } from './util/query-result.js';
 
 function wait(callback) {
@@ -13,10 +13,9 @@ function wait(callback) {
  * Create a consolidator to combine structurally compatible queries.
  * @param {*} enqueue Query manager enqueue method
  * @param {*} cache Client-side query cache (sql -> data)
- * @param {*} record Query recorder function
  * @returns A consolidator object
  */
-export function consolidator(enqueue, cache, record) {
+export function consolidator(enqueue, cache) {
   let pending = [];
   let id = 0;
 
@@ -28,7 +27,7 @@ export function consolidator(enqueue, cache, record) {
 
     // build and issue consolidated queries
     for (const group of groups) {
-      consolidate(group, enqueue, record);
+      consolidate(group, enqueue);
       processResults(group, cache);
     }
   }
@@ -75,18 +74,16 @@ function entryGroups(entries, cache) {
  * Queries with matching keys are conosolidation-compatible.
  * If a query is found in the cache, it is exempted from consolidation,
  * which is indicated by returning the precise query SQL as the key.
- * @param {*} query The input query.
+ * @param {Query | DescribeQuery} query The input query.
  * @param {*} cache The query cache (sql -> data).
  * @returns a key string
  */
 function consolidationKey(query, cache) {
   const sql = `${query}`;
-  if (query instanceof Query && !cache.get(sql)) {
+  if (isSelectQuery(query) && !cache.get(sql)) {
     if (
-      // @ts-ignore
-      query.orderby().length || query.where().length ||
-      // @ts-ignore
-      query.qualify().length || query.having().length
+      query._orderby.length || query._where.length ||
+      query._qualify.length || query._having.length
     ) {
       // do not try to analyze if query includes clauses
       // that may refer to *derived* columns we can't resolve
@@ -94,25 +91,21 @@ function consolidationKey(query, cache) {
     }
 
     // create a derived query stripped of selections
-    const q = query.clone().$select('*');
+    const q = query.clone().setSelect('*');
 
     // check group by criteria for compatibility
     // queries may refer to *derived* columns as group by criteria
     // we resolve these against the true grouping expressions
-    const groupby = query.groupby();
-    // @ts-ignore
+    const groupby = query._groupby;
     if (groupby.length) {
-      const map = {}; // expression map (as -> expr)
-      // @ts-ignore
-      query.select().forEach(({ as, expr }) => map[as] = expr);
-      // @ts-ignore
-      q.$groupby(groupby.map(e => (e instanceof Ref && map[e.column]) || e));
+      const map = {}; // expression map (alias -> expr)
+      query._select.forEach(({ alias, expr }) => map[alias] = expr);
+      q.setGroupby(groupby.map(e => (isColumnRef(e) && map[e.column]) || e));
     }
-    // @ts-ignore
-    else if (query.select().some(({ expr }) => expr.aggregate)) {
+    else if (query._select.some(e => isAggregateExpression(e.expr))) {
       // if query is an ungrouped aggregate, add an explicit groupby to
       // prevent improper consolidation with non-aggregate queries
-      q.$groupby('ALL');
+      q.setGroupby('ALL');
     }
 
     // key is just the transformed query as SQL
@@ -127,17 +120,15 @@ function consolidationKey(query, cache) {
  * Issue queries, consolidating where possible.
  * @param {*} group Array of bundled query entries
  * @param {*} enqueue Add entry to query queue
- * @param {*} record Query recorder function
  */
-function consolidate(group, enqueue, record) {
+function consolidate(group, enqueue) {
   if (shouldConsolidate(group)) {
     // issue a single consolidated query
     enqueue({
       request: {
         type: 'arrow',
         cache: false,
-        record: false,
-        query: (group.query = consolidatedQuery(group, record))
+        query: (group.query = consolidatedQuery(group))
       },
       result: (group.result = new QueryResult())
     });
@@ -170,10 +161,9 @@ function shouldConsolidate(group) {
 /**
  * Create a consolidated query for a group.
  * @param {*} group Array of bundled query entries
- * @param {*} record Query recorder function
  * @returns A consolidated Query instance
  */
-function consolidatedQuery(group, record) {
+function consolidatedQuery(group) {
   const maps = group.maps = [];
   const fields = new Map;
 
@@ -182,30 +172,29 @@ function consolidatedQuery(group, record) {
     const { query } = item.entry.request;
     const fieldMap = [];
     maps.push(fieldMap);
-    for (const { as, expr } of query.select()) {
+    for (const { alias, expr } of query._select) {
       const e = `${expr}`;
       if (!fields.has(e)) {
         fields.set(e, [`col${fields.size}`, expr]);
       }
       const [name] = fields.get(e);
-      fieldMap.push([name, as]);
+      fieldMap.push([name, alias]);
     }
-    record(`${query}`);
   }
 
   // use a cloned query as a starting point
   const query = group[0].entry.request.query.clone();
 
   // update group by statement as needed
-  const groupby = query.groupby();
+  const groupby = query._groupby;
   if (groupby.length) {
     const map = {};
     group.maps[0].forEach(([name, as]) => map[as] = name);
-    query.$groupby(groupby.map(e => (e instanceof Ref && map[e.column]) || e));
+    query.setGroupby(groupby.map(e => (isColumnRef(e) && map[e.column]) || e));
   }
 
   // update select statement and return
-  return query.$select(Array.from(fields.values()));
+  return query.setSelect(Array.from(fields.values()));
 }
 
 /**