@uwdata/mosaic-core 0.14.1 → 0.16.1

package/src/connectors/wasm.js

@@ -1,81 +1,144 @@
+/** @import { ExtractionOptions, Table } from '@uwdata/flechette' */
+/** @import { ArrowQueryRequest, Connector, ExecQueryRequest, JSONQueryRequest } from './Connector.js' */
 import * as duckdb from '@duckdb/duckdb-wasm';
 import { decodeIPC } from '../util/decode-ipc.js';
 
-// bypass duckdb-wasm query method to get Arrow IPC bytes directly
-// https://github.com/duckdb/duckdb-wasm/issues/267#issuecomment-2252749509
-function getArrowIPC(con, query) {
-  return new Promise((resolve, reject) => {
-    con.useUnsafe(async (bindings, conn) => {
-      try {
-        const buffer = await bindings.runQuery(conn, query);
-        resolve(buffer);
-      } catch (error) {
-        reject(error);
-      }
-    });
-  });
-}
+/**
+ * @typedef {object} DuckDBWASMOptions
+ * @property {boolean} [log] Flag to enable logging.
+ */
+
+/**
+ * @typedef {object} DuckDBWASMConnectorOptions
+ * @property {boolean} [log] Flag to enable logging.
+ * @property {ExtractionOptions} [ipc]
+ *  Arrow IPC extraction options.
+ * @property {duckdb.AsyncDuckDB} [duckdb]
+ *  Optional pre-existing DuckDB-WASM instance.
+ * @property {duckdb.AsyncDuckDBConnection} [connection]
+ *  Optional pre-existing DuckDB-WASM connection.
+ */
 
+/**
+ * Connect to a DuckDB-WASM instance.
+ * @param {DuckDBWASMConnectorOptions} [options] Connector options.
+ * @returns {DuckDBWASMConnector} A connector instance.
+ */
 export function wasmConnector(options = {}) {
-  const { duckdb, connection, ...opts } = options;
-  let db = duckdb;
-  let con = connection;
-  let loadPromise;
+  return new DuckDBWASMConnector(options);
+}
 
-  function load() {
-    if (!loadPromise) {
-      // use a loading promise to avoid race conditions
-      // synchronizes multiple callees on the same load
-      loadPromise = (db
-        ? Promise.resolve(db)
-        : initDatabase(opts).then(result => db = result))
-        .then(db => db.connect())
-        .then(result => con = result);
-    }
-    return loadPromise;
+/**
+ * DuckDB-WASM connector.
+ * @implements {Connector}
+ */
+export class DuckDBWASMConnector {
+  /**
+   * Create a new DuckDB-WASM connector instance.
+   * @param {DuckDBWASMConnectorOptions} [options]
+   */
+  constructor(options = {}) {
+    const { ipc, duckdb, connection, ...opts } = options;
+    /** @type {ExtractionOptions} */
+    this._ipc = ipc;
+    /** @type {DuckDBWASMOptions} */
+    this._options = opts;
+    /** @type {duckdb.AsyncDuckDB} */
+    this._db = duckdb;
+    /** @type {duckdb.AsyncDuckDBConnection} */
+    this._con = connection;
+    /** @type {Promise<unknown>} */
+    this._loadPromise;
   }
 
   /**
    * Get the backing DuckDB-WASM instance.
-   * Will lazily initialize DuckDB-WASM if not already loaded.
+   * Lazily initializes DuckDB-WASM if not already loaded.
    * @returns {Promise<duckdb.AsyncDuckDB>} The DuckDB-WASM instance.
    */
-  async function getDuckDB() {
-    if (!db) await load();
-    return db;
+  async getDuckDB() {
+    if (!this._db) await connect(this);
+    return this._db;
   }
 
   /**
    * Get the backing DuckDB-WASM connection.
-   * Will lazily initialize DuckDB-WASM if not already loaded.
+   * Lazily initializes DuckDB-WASM if not already loaded.
    * @returns {Promise<duckdb.AsyncDuckDBConnection>} The DuckDB-WASM connection.
    */
-  async function getConnection() {
-    if (!con) await load();
-    return con;
+  async getConnection() {
+    if (!this._con) await connect(this);
+    return this._con;
   }
 
-  return {
-    getDuckDB,
-    getConnection,
-    /**
-     * Query the DuckDB-WASM instance.
-     * @param {object} query
-     * @param {'exec' | 'arrow' | 'json'} [query.type] The query type.
-     * @param {string} query.sql A SQL query string.
-     * @returns the query result
-     */
-    query: async query => {
-      const { type, sql } = query;
-      const con = await getConnection();
-      const result = await getArrowIPC(con, sql);
-      return type === 'exec' ? undefined
-        : type === 'arrow' ? decodeIPC(result)
-        : decodeIPC(result).toArray();
-    }
+  /**
+   * @overload
+   * @param {ArrowQueryRequest} query
+   * @returns {Promise<Table>}
+   *
+   * @overload
+   * @param {ExecQueryRequest} query
+   * @returns {Promise<void>}
+   *
+   * @overload
+   * @param {JSONQueryRequest} query
+   * @returns {Promise<Record<string, any>[]>}
+   *
+   * @param {ArrowQueryRequest | ExecQueryRequest | JSONQueryRequest} query
+   * @returns {Promise<Table | void | Record<string, any>[]>}}
+   */
+  async query(query) {
+    const { type, sql } = query;
+    const con = await this.getConnection();
+    const result = await getArrowIPC(con, sql);
+    return type === 'exec' ? undefined
+      : type === 'arrow' ? decodeIPC(result, this._ipc)
+      : decodeIPC(result).toArray();
   };
 }
 
+/**
+ * Bypass duckdb-wasm query method to get Arrow IPC bytes directly.
+ * https://github.com/duckdb/duckdb-wasm/issues/267#issuecomment-2252749509
+ * @param {duckdb.AsyncDuckDBConnection} con The DuckDB-WASM connection.
+ * @param {string} query The SQL query to run.
+ */
+function getArrowIPC(con, query) {
+  return new Promise((resolve, reject) => {
+    con.useUnsafe(async (bindings, conn) => {
+      try {
+        const buffer = await bindings.runQuery(conn, query);
+        resolve(buffer);
+      } catch (error) {
+        reject(error);
+      }
+    });
+  });
+}
+
+/**
+ * Establish a new database connection for the given connector.
+ * @param {DuckDBWASMConnector} c The connector.
+ * @returns {Promise<unknown>}
+ */
+function connect(c) {
+  if (!c._loadPromise) {
+    // use a loading promise to avoid race conditions
+    // synchronizes multiple callees on the same load
+    c._loadPromise = (
+      c._db
+        ? Promise.resolve(c._db)
+        : initDatabase(c._options).then(result => c._db = result))
+      .then(db => db.connect())
+      .then(result => c._con = result);
+  }
+  return c._loadPromise;
+}
+
+/**
+ * Initialize a new DuckDB-WASM instance.
+ * @param {DuckDBWASMOptions} options
+ */
 async function initDatabase({
   log = false
 } = {}) {

package/src/index-types.ts

@@ -1,3 +1,4 @@
 export * from './index.js';
 export * from './types.js';
+export * from './connectors/Connector.js';
 export * from './util/selection-types.js';

package/src/make-client.js

@@ -1,64 +1,101 @@
-import { MosaicClient } from "./MosaicClient.js";
-import {
-  coordinator as defaultCoordinator,
-} from "./Coordinator.js";
+/** @import { Coordinator } from './Coordinator.js' */
+/** @import { Selection } from './Selection.js' */
+import { MosaicClient } from './MosaicClient.js';
+import { coordinator as defaultCoordinator } from './Coordinator.js';
 
 /**
  * @typedef {Object} MakeClientOptions
- * @property {import('./Coordinator.js').Coordinator} [coordinator] - Mosaic coordinator. Default to the global coordinator.
- * @property {import('./Selection.js').Selection|null} [selection] - A selection whose predicates will be fed into the query function to produce the SQL query.
- * @property {function(): Promise<void>} [prepare] - An async function to prepare the client before running queries.
- * @property {function(any): any} query - A function that returns a query from a list of selection predicates.
- * @property {function(any): void} [queryResult] - Called by the coordinator to return a query result.
- * @property {function(): void} [queryPending] - Called by the coordinator to report a query execution error.
- * @property {function(any): void} [queryError] - Called by the coordinator to inform the client that a query is pending.
+ * @property {Coordinator} [coordinator] Mosaic coordinator.
+ *  Defaults to the global coordinator.
+ * @property {Selection|null} [selection] A selection whose predicates are
+ *  fed into the query function to produce the SQL query.
+ * @property {boolean} [enabled] A flag (default `true`) indicating if the
+ *  client should initially be enabled or not.
+ * @property {boolean} [filterStable] A flag (default `true`) indicating if the
+ *  if client queries can be sped up using pre-aggregated data. Should be set
+ *  to `false` if filtering changes the groupby domain of the query.
+ * @property {function(): Promise<void>} [prepare]
+ *  An async function to prepare the client before running queries.
+ * @property {function(any): any} [query]
+ *  A function that returns a query from a list of selection predicates.
+ * @property {function(any): void} [queryResult]
+ *  Called by the coordinator to return a query result.
+ * @property {function(): void} [queryPending]
+ *  Called by the coordinator to report a query execution error.
+ * @property {function(any): void} [queryError]
+ *  Called by the coordinator to inform the client that a query is pending.
  */
 
-/** Make a new client with the given options, and connect the client to the provided coordinator.
- * @param {MakeClientOptions} options - The options for making the client
- * @returns {MosaicClient & { destroy: () => void }} - The result object with methods to request an update or destroy the client.
+/**
+ * Make a new client with the given options, and connect the client to the
+ * provided coordinator.
+ * @param {MakeClientOptions} options The options for making the client.
+ * @returns {MosaicClient & { destroy: () => void }} The resulting client,
+ *  along with a method to destroy the client when no longer needed.
  */
 export function makeClient(options) {
-  const coordinator = options.coordinator ?? defaultCoordinator();
-  const client = new ProxyClient({ ...options, coordinator });
+  const {
+    coordinator = defaultCoordinator(),
+    ...clientOptions
+  } = options;
+  const client = new ProxyClient(clientOptions);
   coordinator.connect(client);
   return client;
 }
 
-/** An internal class used to implement the makeClient API */
+/**
+ * An internal class used to implement the makeClient API.
+ */
 class ProxyClient extends MosaicClient {
-  /** @param {MakeClientOptions} options */
-  constructor(options) {
-    super(options.selection);
+  /**
+   * @param {MakeClientOptions} options The options for making the client.
+   */
+  constructor({
+    selection = undefined,
+    enabled = true,
+    filterStable = true,
+    ...methods
+  }) {
+    super(selection);
+    this.enabled = enabled;
+
+    /**
+     * @type {MakeClientOptions}
+     * @readonly
+     */
+    this._methods = methods;
 
-    /** @type {MakeClientOptions} */
-    this._options = { ...options };
+    /**
+     * @type {boolean}
+     * @readonly
+     */
+    this._filterStable = filterStable;
+  }
+
+  get filterStable() {
+    return this._filterStable;
   }
 
   async prepare() {
-    await this._options.prepare?.();
+    await this._methods.prepare?.();
   }
 
   query(filter) {
-    return this._options.query(filter);
+    return this._methods.query?.(filter) ?? null;
   }
 
   queryResult(data) {
-    this._options.queryResult?.(data);
+    this._methods.queryResult?.(data);
     return this;
   }
 
   queryPending() {
-    this._options.queryPending?.();
+    this._methods.queryPending?.();
     return this;
   }
 
   queryError(error) {
-    this._options.queryError?.(error);
+    this._methods.queryError?.(error);
     return this;
   }
-
-  destroy() {
-    this._options.coordinator.disconnect(this);
-  }
 }

package/src/preagg/PreAggregator.js

@@ -1,4 +1,11 @@
-import { Query, and, asNode, ceil, collectColumns, createTable, float64, floor, isBetween, int32, mul, round, scaleTransform, sub, isSelectQuery, ExprNode, SelectQuery, isAggregateExpression, ColumnNameRefNode } from '@uwdata/mosaic-sql';
+/**
+ * @import { ExprNode, SelectQuery } from '@uwdata/mosaic-sql'
+ * @import { Coordinator } from '../Coordinator.js'
+ * @import { MosaicClient } from '../MosaicClient.js'
+ * @import { Selection } from '../Selection.js'
+ * @import { BinMethod, Scale, SelectionClause } from '../util/selection-types.js'
+ */
+import { Query, and, asNode, ceil, collectColumns, createTable, float64, floor, isBetween, int32, mul, round, scaleTransform, sub, isSelectQuery, isAggregateExpression, ColumnNameRefNode } from '@uwdata/mosaic-sql';
 import { preaggColumns } from './preagg-columns.js';
 import { fnv_hash } from '../util/hash.js';
 
@@ -34,14 +41,14 @@ const Skip = { skip: true, result: null };
 export class PreAggregator {
   /**
    * Create a new manager of materialized views of pre-aggregated data.
-   * @param {import('../Coordinator.js').Coordinator} coordinator A Mosaic coordinator.
+   * @param {Coordinator} coordinator A Mosaic coordinator.
    * @param {PreAggregateOptions} [options] Pre-aggregation options.
    */
   constructor(coordinator, {
     schema = 'mosaic',
     enabled = true
   } = {}) {
-    /** @type {Map<import('../MosaicClient.js').MosaicClient, PreAggregateInfo | Skip | null>} */
+    /** @type {Map<MosaicClient, PreAggregateInfo | Skip | null>} */
     this.entries = new Map();
     this.active = null;
     this.mc = coordinator;
@@ -123,12 +130,10 @@ export class PreAggregator {
    * client-selection pair, or null if the client has unstable filters.
    * This method has multiple possible side effects, including materialized
    * view creation and updating internal caches.
-   * @param {import('../MosaicClient.js').MosaicClient} client A Mosaic client.
-   * @param {import('../Selection.js').Selection} selection A Mosaic selection
-   *  to filter the client by.
-   * @param {import('../util/selection-types.js').SelectionClause} activeClause
-   *  A representative active selection clause for which to (possibly) generate
-   *  materialized views of pre-aggregates.
+   * @param {MosaicClient} client A Mosaic client.
+   * @param {Selection} selection A Mosaic selection to filter the client by.
+   * @param {SelectionClause} activeClause A representative active selection
+   *  clause for which to generate materialized views of pre-aggregates.
    * @returns {PreAggregateInfo | Skip | null} Information and query generator
    * for pre-aggregated tables, or null if the client has unstable filters.
    */
@@ -205,8 +210,7 @@ export class PreAggregator {
  * function for the active dimensions of a pre-aggregated materialized view.
  * If the active clause is not indexable or is missing metadata, this method
  * returns an object with a null source property.
- * @param {import('../util/selection-types.js').SelectionClause} clause
- *  The active selection clause to analyze.
+ * @param {SelectionClause} clause The active selection clause to analyze.
  */
 function activeColumns(clause) {
   const { source, meta } = clause;
@@ -261,12 +265,12 @@ const BIN = { ceil, round };
 
 /**
  * Returns a bin function generator to discretize a selection interval domain.
- * @param {import('../util/selection-types.js').Scale} scale A scale that maps
- *  domain values to the output range (typically pixels).
+ * @param {Scale} scale A scale that maps domain values to the output range
+ *  (typically pixels).
  * @param {number} pixelSize The interactive pixel size. This value indicates
  *  the bin step size and may be greater than an actual screen pixel.
- * @param {import('../util/selection-types.js').BinMethod} bin The binning
- *  method to apply, one of `floor`, `ceil', or `round`.
+ * @param {BinMethod} bin The binning method to apply, one of `floor`,
+ *  `ceil', or `round`.
  * @returns {(value: any) => ExprNode} A bin function generator.
  */
 function binInterval(scale, pixelSize, bin) {
@@ -426,8 +430,7 @@ export class PreAggregateInfo {
 
   /**
    * Generate a materialized view query for the given predicate.
-   * @param {import('@uwdata/mosaic-sql').ExprNode} predicate The current
-   *  active clause predicate.
+   * @param {ExprNode} predicate The current active clause predicate.
    * @returns {SelectQuery} A materialized view query.
    */
   query(predicate) {

package/src/preagg/preagg-columns.js

@@ -1,5 +1,8 @@
-import { AggregateNode, ExprNode, Query, SelectQuery, collectAggregates, isAggregateExpression, isSelectQuery, isTableRef, rewrite, sql } from '@uwdata/mosaic-sql';
-import { MosaicClient } from '../MosaicClient.js';
+/**
+ * @import { AggregateNode, ExprNode, Query, SelectQuery } from '@uwdata/mosaic-sql'
+ * @import { MosaicClient } from '../MosaicClient.js'
+ */
+import { collectAggregates, isAggregateExpression, isSelectQuery, isTableRef, rewrite, sql } from '@uwdata/mosaic-sql';
 import { sufficientStatistics } from './sufficient-statistics.js';
 
 /**

package/src/preagg/sufficient-statistics.js

@@ -1,4 +1,5 @@
-import { AggregateNode, and, argmax, argmin, coalesce, count, div, exp, ExprNode, isNotNull, ln, max, min, mul, pow, regrAvgX, regrAvgY, regrCount, sql, sqrt, sub, sum } from '@uwdata/mosaic-sql';
+/** @import { AggregateNode, ExprNode } from '@uwdata/mosaic-sql' */
+import { and, argmax, argmin, coalesce, count, div, exp, isNotNull, ln, max, min, mul, pow, regrAvgX, regrAvgY, regrCount, sql, sqrt, sub, sum } from '@uwdata/mosaic-sql';
 import { fnv_hash } from '../util/hash.js';
 
 /**

package/src/types.ts

@@ -62,3 +62,26 @@ export interface Activatable {
    */
   activate(): void;
 }
+
+/**
+ * Interface for cache implementations.
+ */
+export interface Cache {
+  get(key: string): any;
+  set(key: string, value: any): any;
+  clear(): void;
+}
+
+/**
+ * Interface for logger implementations
+ */
+export interface Logger {
+  debug(...args: any[]): void;
+  info(...args: any[]): void;
+  log(...args: any[]): void;
+  warn(...args: any[]): void;
+  error(...args: any[]): void;
+  group(label?: any): void;
+  groupCollapsed(label?: any): void;
+  groupEnd(): void;
+}

package/src/util/cache.js

@@ -1,13 +1,28 @@
+/** @import { Cache } from '../types.js' */
+
 const requestIdle = typeof requestIdleCallback !== 'undefined'
   ? requestIdleCallback
   : setTimeout;
 
-export const voidCache = () => ({
-  get: () => undefined,
-  set: (key, value) => value,
-  clear: () => {}
-});
+/**
+ * Create a new cache that ignores all values.
+ * @returns {Cache}
+ */
+export function voidCache() {
+  return {
+    get: () => undefined,
+    set: (key, value) => value,
+    clear: () => {}
+  };
+}
 
+/**
+ * Create a new cache that uses an LRU eviction policy.
+ * @param {object} [options] Cache options.
+ * @param {number} [options.max] Maximum number of cache entries.
+ * @param {number} [options.ttl] Time-to-live for cache entries.
+ * @returns {Cache}
+ */
 export function lruCache({
   max = 1000, // max entries
   ttl = 3 * 60 * 60 * 1000 // time-to-live, default 3 hours

package/src/util/decode-ipc.js

@@ -1,11 +1,15 @@
+/** @import { ExtractionOptions, Table } from '@uwdata/flechette' */
 import { tableFromIPC } from '@uwdata/flechette';
 
 /**
- * Decode Arrow IPC bytes to a table instance, with an option to map date and
- * timestamp values to JS Date objects.
+ * Decode Arrow IPC bytes to a table instance.
+ * The default options map date and timestamp values to JS Date objects.
  * @param {ArrayBuffer | Uint8Array} data Arrow IPC bytes.
- * @returns {import('@uwdata/flechette').Table} A table instance.
+ * @param {ExtractionOptions} [options] Arrow IPC extraction options.
+ *  If unspecified, the default options will extract date and timestamp
+ *  values to JS Date objects.
+ * @returns {Table} A table instance.
  */
-export function decodeIPC(data) {
-  return tableFromIPC(data, { useDate: true });
+export function decodeIPC(data, options = { useDate: true }) {
+  return tableFromIPC(data, options);
 }

package/src/util/field-info.js

@@ -1,4 +1,5 @@
-import { AggregateNode, Query, asTableRef, count, isAggregateExpression, isNode, isNull, max, min, sql } from '@uwdata/mosaic-sql';
+/** @import { AggregateNode } from '@uwdata/mosaic-sql' */
+import { Query, asTableRef, count, isAggregateExpression, isNode, isNull, max, min, sql } from '@uwdata/mosaic-sql';
 import { jsType } from './js-type.js';
 
 export const Count = 'count';

package/src/util/js-type.js

@@ -41,7 +41,9 @@ export function jsType(type) {
     case 'GEOMETRY':
       return 'object';
     default:
-      if (type.startsWith('DECIMAL')) {
+      if (type.startsWith('ENUM')) {
+        return 'string';
+      } else if (type.startsWith('DECIMAL')) {
         return 'number';
       } else if (type.startsWith('STRUCT') || type.startsWith('MAP')) {
         return 'object';

package/src/util/void-logger.js

@@ -5,6 +5,9 @@ export function voidLogger() {
     info(..._) {},
     log(..._) {},
     warn(..._) {},
-    error(..._) {}
+    error(..._) {},
+    group(label) {},
+    groupCollapsed(label) {},
+    groupEnd() {}
   };
 }