@uwdata/mosaic-core 0.11.0 → 0.12.0

package/src/QueryManager.js

@@ -1,56 +1,85 @@
 import { consolidator } from './QueryConsolidator.js';
 import { lruCache, voidCache } from './util/cache.js';
 import { PriorityQueue } from './util/priority-queue.js';
-import { QueryResult } from './util/query-result.js';
+import { QueryResult, QueryState } from './util/query-result.js';
+import { voidLogger } from './util/void-logger.js';
 
-export const Priority = { High: 0, Normal: 1, Low: 2 };
+export const Priority = Object.freeze({ High: 0, Normal: 1, Low: 2 });
 
 export class QueryManager {
-  constructor() {
+  constructor(
+    maxConcurrentRequests = 32
+  ) {
     this.queue = new PriorityQueue(3);
     this.db = null;
     this.clientCache = null;
-    this._logger = null;
+    this._logger = voidLogger();
     this._logQueries = false;
-    this.recorders = [];
-    this.pending = null;
     this._consolidate = null;
+    /**
+     * Requests pending with the query manager.
+     * 
+     * @type {QueryResult[]}
+     */
+    this.pendingResults = [];
+    this.maxConcurrentRequests = maxConcurrentRequests;
+    this.pendingExec = false;
   }
 
   next() {
-    if (this.pending || this.queue.isEmpty()) return;
+    if (this.queue.isEmpty() || this.pendingResults.length > this.maxConcurrentRequests || this.pendingExec) {
+      return;
+    }
+
     const { request, result } = this.queue.next();
-    this.pending = this.submit(request, result);
-    this.pending.finally(() => { this.pending = null; this.next(); });
+
+    this.pendingResults.push(result);
+    if (request.type === 'exec') this.pendingExec = true;
+
+    this.submit(request, result).finally(() => {
+      // return from the queue all requests that are ready
+      while (this.pendingResults.length && this.pendingResults[0].state !== QueryState.pending) {
+        const result = this.pendingResults.shift();
+        if (result.state === QueryState.ready) {
+          result.fulfill();
+        } else if (result.state === QueryState.done) {
+          this._logger.warn('Found resolved query in pending results.');
+        }
+      }
+      if (request.type === 'exec') this.pendingExec = false;
+      this.next();
+    });
   }
 
+  /**
+   * Add an entry to the query queue with a priority.
+   * @param {object} entry The entry to add.
+   * @param {*} [entry.request] The query request.
+   * @param {QueryResult} [entry.result] The query result.
+   * @param {number} priority The query priority, defaults to `Priority.Normal`.
+   */
   enqueue(entry, priority = Priority.Normal) {
     this.queue.insert(entry, priority);
     this.next();
   }
 
-  recordQuery(sql) {
-    if (this.recorders.length && sql) {
-      this.recorders.forEach(rec => rec.add(sql));
-    }
-  }
-
+  /**
+   * Submit the query to the connector.
+   * @param {*} request The request.
+   * @param {QueryResult} result The query result.
+   */
   async submit(request, result) {
     try {
-      const { query, type, cache = false, record = true, options } = request;
+      const { query, type, cache = false, options } = request;
       const sql = query ? `${query}` : null;
 
-      // update recorders
-      if (record) {
-        this.recordQuery(sql);
-      }
-
       // check query cache
       if (cache) {
         const cached = this.clientCache.get(sql);
         if (cached) {
+          const data = await cached;
           this._logger.debug('Cache');
-          result.fulfill(cached);
+          result.ready(data);
           return;
         }
       }
@@ -60,10 +89,16 @@ export class QueryManager {
       if (this._logQueries) {
         this._logger.debug('Query', { type, sql, ...options });
       }
-      const data = await this.db.query({ type, sql, ...options });
+
+      const promise = this.db.query({ type, sql, ...options });
+      if (cache) this.clientCache.set(sql, promise);
+
+      const data = await promise;
+
       if (cache) this.clientCache.set(sql, data);
+
       this._logger.debug(`Request: ${(performance.now() - t0).toFixed(1)}`);
-      result.fulfill(data);
+      result.ready(type === 'exec' ? null : data);
     } catch (err) {
       result.reject(err);
     }
@@ -89,12 +124,18 @@ export class QueryManager {
 
   consolidate(flag) {
     if (flag && !this._consolidate) {
-      this._consolidate = consolidator(this.enqueue.bind(this), this.clientCache, this.recordQuery.bind(this));
+      this._consolidate = consolidator(this.enqueue.bind(this), this.clientCache);
     } else if (!flag && this._consolidate) {
       this._consolidate = null;
     }
   }
 
+  /**
+   * Request a query result.
+   * @param {*} request The request.
+   * @param {number} priority The query priority, defaults to `Priority.Normal`.
+   * @returns {QueryResult} A query result promise.
+   */
   request(request, priority = Priority.Normal) {
     const result = new QueryResult();
     const entry = { request, result };
@@ -116,6 +157,12 @@ export class QueryManager {
         }
         return false;
       });
+
+      for (const result of this.pendingResults) {
+        if (set.has(result)) {
+          result.reject('Canceled');
+        }
+      }
     }
   }
 
@@ -124,26 +171,10 @@ export class QueryManager {
       result.reject('Cleared');
       return true;
     });
-  }
 
-  record() {
-    let state = [];
-    const recorder = {
-      add(query) {
-        state.push(query);
-      },
-      reset() {
-        state = [];
-      },
-      snapshot() {
-        return state.slice();
-      },
-      stop() {
-        this.recorders = this.recorders.filter(x => x !== recorder);
-        return state;
-      }
-    };
-    this.recorders.push(recorder);
-    return recorder;
+    for (const result of this.pendingResults) {
+      result.reject('Cleared');
+    }
+    this.pendingResults = [];
   }
 }

package/src/Selection.js

@@ -1,5 +1,6 @@
-import { or } from '@uwdata/mosaic-sql';
+import { literal, or } from '@uwdata/mosaic-sql';
 import { Param } from './Param.js';
+import { MosaicClient } from './MosaicClient.js';
 
 /**
  * Test if a value is a Selection instance.
@@ -315,9 +316,11 @@ export class SelectionResolver {
 
   /**
    * Return a selection query predicate for the given client.
-   * @param {*[]} clauseList An array of selection clauses.
-   * @param {*} active The current active selection clause.
-   * @param {*} client The client whose data may be filtered.
+   * @param {import('./util/selection-types.js').SelectionClause[]} clauseList
+   *  An array of selection clauses.
+   * @param {import('./util/selection-types.js').SelectionClause} active
+   *  The current active selection clause.
+   * @param {MosaicClient} client The client whose data may be filtered.
    * @returns {*} The query predicate for filtering client data,
    *  based on the current state of this selection.
    */
@@ -325,7 +328,7 @@ export class SelectionResolver {
     const { empty, union } = this;
 
     if (empty && !clauseList.length) {
-      return ['FALSE'];
+      return [literal(false)];
     }
 
     // do nothing if cross-filtering and client is currently active

package/src/SelectionClause.js

@@ -1,7 +1,4 @@
-import {
-  SQLExpression, and, contains, isBetween, isNotDistinct, literal,
-  or, prefix, regexp_matches, suffix
-} from '@uwdata/mosaic-sql';
+import { ExprNode, and, contains, isBetween, isIn, isNotDistinct, literal, or, prefix, regexp_matches, suffix } from '@uwdata/mosaic-sql';
 import { MosaicClient } from './MosaicClient.js';
 
 /**
@@ -10,12 +7,11 @@ import { MosaicClient } from './MosaicClient.js';
  * @typedef {import('./util/selection-types.js').Extent} Extent
  * @typedef {import('./util/selection-types.js').MatchMethod} MatchMethod
  * @typedef {import('./util/selection-types.js').BinMethod} BinMethod
- * @typedef {SQLExpression | string} Field
  */
 
 /**
  * Generate a selection clause for a single selected point value.
- * @param {Field} field The table column or expression to select.
+ * @param {import('@uwdata/mosaic-sql').ExprValue} field The table column or expression to select.
  * @param {*} value The selected value.
  * @param {object} options Additional clause properties.
  * @param {*} options.source The source component generating this clause.
@@ -28,7 +24,7 @@ export function clausePoint(field, value, {
   source,
   clients = source ? new Set([source]) : undefined
 }) {
-  /** @type {SQLExpression | null} */
+  /** @type {ExprNode | null} */
   const predicate = value !== undefined
     ? isNotDistinct(field, literal(value))
     : null;
@@ -43,9 +39,9 @@ export function clausePoint(field, value, {
 
 /**
  * Generate a selection clause for multiple selected point values.
- * @param {Field[]} fields The table columns or expressions to select.
- * @param {any[][]} value The selected values, as an array of arrays where
- *  each subarray contains values corresponding to each *fields* entry.
+ * @param {import('@uwdata/mosaic-sql').ExprValue[]} fields The table columns or expressions to select.
+ * @param {any[][] | undefined} value The selected values, as an array of
+ *  arrays. Each subarray contains values for each *fields* entry.
  * @param {object} options Additional clause properties.
  * @param {*} options.source The source component generating this clause.
  * @param {Set<MosaicClient>} [options.clients] The Mosaic clients associated
@@ -57,14 +53,15 @@ export function clausePoints(fields, value, {
   source,
   clients = source ? new Set([source]) : undefined
 }) {
-  /** @type {SQLExpression | null} */
+  /** @type {ExprNode | null} */
   let predicate = null;
   if (value) {
-    const clauses = value.map(vals => {
-      const list = vals.map((v, i) => isNotDistinct(fields[i], literal(v)));
-      return list.length > 1 ? and(list) : list[0];
-    });
-    predicate = clauses.length > 1 ? or(clauses) : clauses[0];
+    const clauses = value.length && fields.length === 1
+      ? [isIn(fields[0], value.map(v => literal(v[0])))]
+      : value.map(v => and(v.map((_, i) => isNotDistinct(fields[i], literal(_)))));
+    predicate = value.length === 0 ? literal(false)
+      : clauses.length > 1 ? or(clauses)
+      : clauses[0];
   }
   return {
     meta: { type: 'point' },
@@ -77,7 +74,7 @@ export function clausePoints(fields, value, {
 
 /**
  * Generate a selection clause for a selected 1D interval.
- * @param {Field} field The table column or expression to select.
+ * @param {import('@uwdata/mosaic-sql').ExprValue} field The table column or expression to select.
  * @param {Extent} value The selected interval as a [lo, hi] array.
  * @param {object} options Additional clause properties.
  * @param {*} options.source The source component generating this clause.
@@ -96,7 +93,7 @@ export function clauseInterval(field, value, {
   scale,
   pixelSize = 1
 }) {
-  /** @type {SQLExpression | null} */
+  /** @type {ExprNode | null} */
   const predicate = value != null ? isBetween(field, value) : null;
   /** @type {import('./util/selection-types.js').IntervalMetadata} */
   const meta = { type: 'interval', scales: scale && [scale], bin, pixelSize };
@@ -105,7 +102,7 @@ export function clauseInterval(field, value, {
 
 /**
  * Generate a selection clause for multiple selected intervals.
- * @param {Field[]} fields The table columns or expressions to select.
+ * @param {import('@uwdata/mosaic-sql').ExprValue[]} fields The table columns or expressions to select.
  * @param {Extent[]} value The selected intervals, as an array of extents.
  * @param {object} options Additional clause properties.
  * @param {*} options.source The source component generating this clause.
@@ -125,7 +122,7 @@ export function clauseIntervals(fields, value, {
   scales = [],
   pixelSize = 1
 }) {
-  /** @type {SQLExpression | null} */
+  /** @type {ExprNode | null} */
   const predicate = value != null
     ? and(fields.map((f, i) => isBetween(f, value[i])))
     : null;
@@ -138,7 +135,7 @@ const MATCH_METHODS = { contains, prefix, suffix, regexp: regexp_matches };
 
 /**
  * Generate a selection clause for text search matching.
- * @param {Field} field The table column or expression to select.
+ * @param {import('@uwdata/mosaic-sql').ExprValue} field The table column or expression to select.
  * @param {string} value The selected text search query string.
  * @param {object} options Additional clause properties.
  * @param {*} options.source The source component generating this clause.
@@ -153,7 +150,7 @@ export function clauseMatch(field, value, {
   source, clients = undefined, method = 'contains'
 }) {
   let fn = MATCH_METHODS[method];
-  /** @type {SQLExpression | null} */
+  /** @type {ExprNode | null} */
   const predicate = value ? fn(field, literal(value)) : null;
   /** @type {import('./util/selection-types.js').MatchMetadata} */
   const meta = { type: 'match', method };

package/src/connectors/rest.js

@@ -6,7 +6,9 @@ export function restConnector(uri = 'http://localhost:3000/') {
      * Query the DuckDB server.
      * @param {object} query
      * @param {'exec' | 'arrow' | 'json' | 'create-bundle' | 'load-bundle'} [query.type] The query type.
-     * @param {string} query.sql A SQL query string.
+     * @param {string} [query.sql] A SQL query string.
+     * @param {string[]} [query.queries] The queries used to create a bundle.
+     * @param {string} [query.name] The name of a bundle to create or load.
      * @returns the query result
      */
     async query(query) {

package/src/connectors/socket.js

@@ -86,7 +86,9 @@ export function socketConnector(uri = 'ws://localhost:3000/') {
      * Query the DuckDB server.
      * @param {object} query
      * @param {'exec' | 'arrow' | 'json' | 'create-bundle' | 'load-bundle'} [query.type] The query type.
-     * @param {string} query.sql A SQL query string.
+     * @param {string} [query.sql] A SQL query string.
+     * @param {string[]} [query.queries] The queries used to create a bundle.
+     * @param {string} [query.name] The name of a bundle to create or load.
      * @returns the query result
      */
     query(query) {

package/src/connectors/wasm.js

@@ -61,7 +61,7 @@ export function wasmConnector(options = {}) {
     /**
      * Query the DuckDB-WASM instance.
      * @param {object} query
-     * @param {'exec' | 'arrow' | 'json' | 'create-bundle' | 'load-bundle'} [query.type] The query type.
+     * @param {'exec' | 'arrow' | 'json'} [query.type] The query type.
      * @param {string} query.sql A SQL query string.
      * @returns the query result
      */

package/src/index.js

@@ -22,3 +22,16 @@ export { isArrowTable } from './util/is-arrow-table.js';
 export { synchronizer } from './util/synchronizer.js';
 export { throttle } from './util/throttle.js';
 export { toDataColumns } from './util/to-data-columns.js';
+
+/**
+ * @typedef {import('./util/selection-types.js').ClauseMetadata} ClauseMetadata
+ * @typedef {import('./util/selection-types.js').PointMetadata} PointMetadata
+ * @typedef {import('./util/selection-types.js').MatchMethod} MatchMethod
+ * @typedef {import('./util/selection-types.js').MatchMetadata} MatchMetadata
+ * @typedef {import('./util/selection-types.js').ScaleType} ScaleType
+ * @typedef {import('./util/selection-types.js').Extent} Extent
+ * @typedef {import('./util/selection-types.js').Scale} Scale
+ * @typedef {import('./util/selection-types.js').BinMethod} BinMethod
+ * @typedef {import('./util/selection-types.js').IntervalMetadata} IntervalMetadata
+ * @typedef {import('./util/selection-types.js').SelectionClause} SelectionClause
+ */