@uwdata/mosaic-core 0.1.0 → 0.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uwdata/mosaic-core",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Scalable and extensible linked data views.",
   "keywords": [
     "mosaic",
@@ -28,9 +28,9 @@
     "prepublishOnly": "npm run test && npm run lint && npm run build"
   },
   "dependencies": {
-    "@duckdb/duckdb-wasm": "^1.20.0",
-    "@uwdata/mosaic-sql": "^0.1.0",
-    "apache-arrow": "^11.0.0"
+    "@duckdb/duckdb-wasm": "^1.25.0",
+    "@uwdata/mosaic-sql": "^0.2.0",
+    "apache-arrow": "^12.0.0"
   },
-  "gitHead": "a7967c35349bdf7f00abb113ce1dd9abb233cd62"
+  "gitHead": "e53cd914c807f99aabe78dcbe618dd9543e2f438"
 }

package/src/Catalog.js

@@ -49,8 +49,20 @@ export class Catalog {
     // no need for summary statistics
     if (!stats?.length) return colInfo;
 
-    const result = await this.mc.query(summarize(colInfo, stats));
+    const result = await this.mc.query(
+      summarize(colInfo, stats),
+      { persist: true }
+    );
     const info = { ...colInfo, ...(Array.from(result)[0]) };
+
+    // coerce bigint to number
+    for (const key in info) {
+      const value = info[key];
+      if (typeof value === 'bigint') {
+        info[key] = Number(value);
+      }
+    }
+
     return info;
   }
 

package/src/Coordinator.js

@@ -1,7 +1,7 @@
-import { socketClient } from './clients/socket.js';
+import { socketConnector } from './connectors/socket.js';
 import { Catalog } from './Catalog.js';
 import { FilterGroup } from './FilterGroup.js';
-import { QueryCache, voidCache } from './QueryCache.js';
+import { QueryManager, Priority } from './QueryManager.js';
 import { voidLogger } from './util/void-logger.js';
 
 let _instance;
@@ -16,22 +16,26 @@ export function coordinator(instance) {
 }
 
 export class Coordinator {
-  constructor(db = socketClient(), options = {}) {
+  constructor(db = socketConnector(), options = {}) {
     this.catalog = new Catalog(this);
+    this.manager = options.manager || QueryManager();
     this.logger(options.logger || console);
     this.configure(options);
-    this.databaseClient(db);
+    this.databaseConnector(db);
     this.clear();
+    this._recorders = [];
   }
 
   logger(logger) {
-    return arguments.length
-      ? (this._logger = logger || voidLogger())
-      : this._logger;
+    if (arguments.length) {
+      this._logger = logger || voidLogger();
+      this.manager.logger(this._logger);
+    }
+    return this._logger;
   }
 
   configure({ cache = true, indexes = true }) {
-    this.cache = cache ? new QueryCache() : voidCache();
+    this.manager.cache(cache);
     this.indexes = indexes;
   }
 
@@ -42,58 +46,58 @@ export class Coordinator {
       this.clients = new Set;
       this.filterGroups = new Map;
     }
-    if (cache) this.cache.clear();
+    if (cache) this.manager.cache().clear();
     if (catalog) this.catalog.clear();
   }
 
-  databaseClient(db) {
-    if (arguments.length > 0) {
-      this.db = db;
-    }
-    return this.db;
+  databaseConnector(db) {
+    return this.manager.connector(db);
   }
 
-  async exec(sql) {
-    try {
-      await this.db.query({ type: 'exec', sql });
-    } catch (err) {
-      this._logger.error(err);
-    }
+  // -- Query Management ----
+
+  cancel(requests) {
+    this.manager.cancel(requests);
   }
 
-  query(query, { type = 'arrow', cache = true } = {}) {
-    const sql = String(query);
-    const t0 = performance.now();
-    const cached = this.cache.get(sql);
-    if (cached) {
-      this._logger.debug('Cache');
-      return cached;
-    } else {
-      const request = this.db.query({ type, sql });
-      const result = cache ? this.cache.set(sql, request) : request;
-      result.then(() => this._logger.debug(`Query: ${performance.now() - t0}`));
-      return result;
-    }
+  exec(query, { priority = Priority.Normal } = {}) {
+    return this.manager.request({ type: 'exec', query }, priority);
   }
 
-  async updateClient(client, query) {
-    let result;
-    try {
-      client.queryPending();
-      result = await this.query(query);
-    } catch (err) {
-      this._logger.error(err);
-      client.queryError(err);
-      return;
-    }
-    try {
-      client.queryResult(result).update();
-    } catch (err) {
-      this._logger.error(err);
-    }
+  query(query, {
+    type = 'arrow',
+    cache = true,
+    priority = Priority.Normal,
+    ...options
+  } = {}) {
+    return this.manager.request({ type, query, cache, options }, priority);
+  }
+
+  prefetch(query, options = {}) {
+    return this.query(query, { ...options, cache: true, priority: Priority.Low });
+  }
+
+  createBundle(name, queries, priority = Priority.Low) {
+    const options = { name, queries };
+    return this.manager.request({ type: 'create-bundle', options }, priority);
+  }
+
+  loadBundle(name, priority = Priority.High) {
+    const options = { name };
+    return this.manager.request({ type: 'load-bundle', options }, priority);
+  }
+
+  // -- Client Management ----
+
+  updateClient(client, query, priority = Priority.Normal) {
+    client.queryPending();
+    return this.query(query, { priority }).then(
+      data => client.queryResult(data).update(),
+      err => { client.queryError(err); this._logger.error(err); }
+    );
   }
 
-  async requestQuery(client, query) {
+  requestQuery(client, query) {
     this.filterGroups.get(client.filterBy)?.reset();
     return query
       ? this.updateClient(client, query)
@@ -111,7 +115,7 @@ export class Coordinator {
     // retrieve field statistics
     const fields = client.fields();
     if (fields?.length) {
-      client.fieldStats(await catalog.queryFields(fields));
+      client.fieldInfo(await catalog.queryFields(fields));
     }
 
     // connect filters

package/src/DataTileIndexer.js

@@ -1,6 +1,5 @@
-import { Query, expr, and, isBetween, asColumn, epoch_ms } from '@uwdata/mosaic-sql';
+import { Query, and, asColumn, epoch_ms, isBetween, sql } from '@uwdata/mosaic-sql';
 import { fnv_hash } from './util/hash.js';
-import { skipClient } from './util/skip-client.js';
 
 const identity = x => x;
 
@@ -30,35 +29,43 @@ export class DataTileIndexer {
     this.activeView = null;
   }
 
+  clear() {
+    if (this.indices) {
+      this.mc.cancel(Array.from(this.indices.values(), index => index.result));
+      this.indices = null;
+    }
+  }
+
   index(clients, active) {
     if (this.clients !== clients) {
       // test client views for compatibility
-      const cols = Array.from(clients).map(getIndexColumns);
+      const cols = Array.from(clients, getIndexColumns);
       const from = cols[0]?.from;
       this.enabled = cols.every(c => c && c.from === from);
       this.clients = clients;
-      this.indices = null;
       this.activeView = null;
+      this.clear();
     }
     if (!this.enabled) return false; // client views are not indexable
 
     active = active || this.selection.active;
     const { source } = active;
+    if (source && source === this.activeView?.source) return true; // we're good!
+
+    this.clear();
     if (!source) return false; // nothing to work with
-    if (source === this.activeView?.source) return true; // we're good!
     const activeView = this.activeView = getActiveView(active);
     if (!activeView) return false; // active selection clause not compatible
 
     this.mc.logger().warn('DATA TILE INDEX CONSTRUCTION');
 
-    // create a selection with the active client removed
-    const sel = this.selection.clone().update({ source });
+    // create a selection with the active source removed
+    const sel = this.selection.remove(source);
 
     // generate data tile indices
     const indices = this.indices = new Map;
-    const promises = [];
     for (const client of clients) {
-      if (sel.cross && skipClient(client, active)) continue;
+      if (sel.skip(client, active)) continue;
       const index = getIndexColumns(client);
 
       // build index construction query
@@ -76,11 +83,9 @@ export class DataTileIndexer {
       const sql = query.toString();
       const id = (fnv_hash(sql) >>> 0).toString(16);
       const table = `tile_index_${id}`;
-      indices.set(client, { table, ...index });
-      promises.push(createIndex(this.mc, table, sql));
+      const result = createIndex(this.mc, table, sql);
+      indices.set(client, { table, result, ...index });
     }
-
-    return promises;
   }
 
   async update() {
@@ -100,12 +105,12 @@ export class DataTileIndexer {
     }
 
     const { table, dims, aggr } = index;
-    return this.mc.updateClient(client, Query
+    const query = Query
       .select(dims, aggr)
       .from(table)
       .groupby(dims)
-      .where(filter)
-    );
+      .where(filter);
+    return this.mc.updateClient(client, query);
   }
 }
 
@@ -113,22 +118,22 @@ function getActiveView(clause) {
   const { source, schema } = clause;
   let columns = clause.predicate?.columns;
   if (!schema || !columns) return null;
-  const { type, scales } = schema;
+  const { type, scales, pixelSize = 1 } = schema;
   let predicate;
 
   if (type === 'interval' && scales) {
-    const bins = scales.map(s => binInterval(s));
+    const bins = scales.map(s => binInterval(s, pixelSize));
     if (bins.some(b => b == null)) return null; // unsupported scale type
 
     if (bins.length === 1) {
-      predicate = p => p ? isBetween('active0', p.value.map(bins[0])) : [];
-      columns = { active0: bins[0](clause.predicate.expr) };
+      predicate = p => p ? isBetween('active0', p.range.map(bins[0])) : [];
+      columns = { active0: bins[0](clause.predicate.field) };
     } else {
       predicate = p => p
-        ? and(p.value.map(({ value }, i) => isBetween(`active${i}`, value.map(bins[i]))))
+        ? and(p.children.map(({ range }, i) => isBetween(`active${i}`, range.map(bins[i]))))
         : [];
       columns = Object.fromEntries(
-        clause.predicate.value.map((p, i) => [`active${i}`, bins[i](p.expr)])
+        clause.predicate.children.map((p, i) => [`active${i}`, bins[i](p.field)])
       );
     }
   } else if (type === 'point') {
@@ -141,53 +146,47 @@ function getActiveView(clause) {
   return { source, columns, predicate };
 }
 
-function binInterval(scale) {
+function binInterval(scale, pixelSize) {
   const { type, domain, range } = scale;
-  let lift, sql;
+  let lift, toSql;
 
   switch (type) {
     case 'linear':
       lift = identity;
-      sql = asColumn;
+      toSql = asColumn;
       break;
     case 'log':
       lift = Math.log;
-      sql = c => `LN(${asColumn(c)})`;
+      toSql = c => sql`LN(${asColumn(c)})`;
       break;
     case 'symlog':
       // TODO: support log constants other than 1?
       lift = x => Math.sign(x) * Math.log1p(Math.abs(x));
-      sql = c => (c = asColumn(c), `SIGN(${c}) * LN(1 + ABS(${c}))`);
+      toSql = c => (c = asColumn(c), sql`SIGN(${c}) * LN(1 + ABS(${c}))`);
       break;
     case 'sqrt':
       lift = Math.sqrt;
-      sql = c => `SQRT(${asColumn(c)})`;
+      toSql = c => sql`SQRT(${asColumn(c)})`;
       break;
     case 'utc':
     case 'time':
       lift = x => +x;
-      sql = c => c instanceof Date ? +c : epoch_ms(asColumn(c));
+      toSql = c => c instanceof Date ? +c : epoch_ms(asColumn(c));
       break;
   }
-  return lift ? binFunction(domain, range, lift, sql) : null;
+  return lift ? binFunction(domain, range, pixelSize, lift, toSql) : null;
 }
 
-function binFunction(domain, range, lift, sql) {
+function binFunction(domain, range, pixelSize, lift, toSql) {
   const lo = lift(Math.min(domain[0], domain[1]));
   const hi = lift(Math.max(domain[0], domain[1]));
-  const a = Math.abs(lift(range[1]) - lift(range[0])) / (hi - lo);
-  return value => expr(
-    `FLOOR(${a}::DOUBLE * (${sql(value)} - ${lo}::DOUBLE))`,
-    asColumn(value).columns
-  );
+  const a = (Math.abs(lift(range[1]) - lift(range[0])) / (hi - lo)) / pixelSize;
+  const s = pixelSize === 1 ? '' : `${pixelSize}::INTEGER * `;
+  return value => sql`${s}FLOOR(${a}::DOUBLE * (${toSql(value)} - ${lo}::DOUBLE))::INTEGER`;
 }
 
-async function createIndex(mc, table, query) {
-  try {
-    await mc.exec(`CREATE TEMP TABLE IF NOT EXISTS ${table} AS ${query}`);
-  } catch (err) {
-    mc.logger().error(err);
-  }
+function createIndex(mc, table, query) {
+  return mc.exec(`CREATE TEMP TABLE IF NOT EXISTS ${table} AS ${query}`);
 }
 
 const NO_INDEX = { from: NaN };
@@ -199,25 +198,25 @@ function getIndexColumns(client) {
   if (!from || !q.groupby) return NO_INDEX;
   const g = new Set(q.groupby().map(c => c.column));
 
-  let aggr = [];
-  let dims = [];
+  const aggr = [];
+  const dims = [];
   let count;
 
   for (const { as, expr: { aggregate } } of q.select()) {
     switch (aggregate?.toUpperCase()) {
       case 'COUNT':
       case 'SUM':
-        aggr.push({ [as]: expr(`SUM("${as}")::DOUBLE`) });
+        aggr.push({ [as]: sql`SUM("${as}")::DOUBLE` });
         break;
       case 'AVG':
         count = '_count_';
-        aggr.push({ [as]: expr(`(SUM("${as}" * ${count}) / SUM(${count}))::DOUBLE`) });
+        aggr.push({ [as]: sql`(SUM("${as}" * ${count}) / SUM(${count}))::DOUBLE` });
         break;
       case 'MAX':
-        aggr.push({ [as]: expr(`MAX("${as}")`) });
+        aggr.push({ [as]: sql`MAX("${as}")` });
         break;
       case 'MIN':
-        aggr.push({ [as]: expr(`MIN("${as}")`) });
+        aggr.push({ [as]: sql`MIN("${as}")` });
         break;
       default:
         if (g.has(as)) dims.push(as);
@@ -228,7 +227,7 @@ function getIndexColumns(client) {
   return {
     aggr,
     dims,
-    count: count ? { [count]: expr('COUNT(*)') } : {},
+    count: count ? { [count]: sql`COUNT(*)` } : {},
     from
   };
 }
@@ -244,7 +243,7 @@ function getBaseTable(query) {
   }
 
   // handle set operations / subqueries
-  let base = getBaseTable(subq[0]);
+  const base = getBaseTable(subq[0]);
   for (let i = 1; i < subq.length; ++i) {
     const from = getBaseTable(subq[i]);
     if (from === undefined) continue;

package/src/FilterGroup.js

@@ -1,5 +1,4 @@
 import { DataTileIndexer } from './DataTileIndexer.js';
-import { throttle } from './util/throttle.js';
 
 export class FilterGroup {
   constructor(coordinator, selection, index = true) {
@@ -9,10 +8,8 @@ export class FilterGroup {
     this.indexer = index ? new DataTileIndexer(this.mc, selection) : null;
 
     const { value, activate } = this.handlers = {
-      value: throttle(() => this.update()),
-      activate: clause => {
-        this.indexer?.index(this.clients, clause);
-      }
+      value: () => this.update(),
+      activate: clause => this.indexer?.index(this.clients, clause)
     };
     selection.addEventListener('value', value);
     selection.addEventListener('activate', activate);
@@ -40,7 +37,7 @@ export class FilterGroup {
     return this;
   }
 
-  async update() {
+  update() {
     const { mc, indexer, clients, selection } = this;
     return indexer?.index(clients)
       ? indexer.update()

package/src/Param.js

@@ -1,46 +1,92 @@
+import { AsyncDispatch } from './util/AsyncDispatch.js';
 import { distinct } from './util/distinct.js';
 
+/**
+ * Test if a value is a Param instance.
+ * @param {*} x The value to test.
+ * @returns {boolean} True if the input is a Param, false otherwise.
+ */
 export function isParam(x) {
   return x instanceof Param;
 }
 
-export class Param {
+/**
+ * Represents a dynamic parameter that dispatches updates
+ * upon parameter changes.
+ */
+export class Param extends AsyncDispatch {
+
+  /**
+   * Create a new Param instance.
+   * @param {*} value The initial value of the Param.
+   */
   constructor(value) {
+    super();
     this._value = value;
-    this._listeners = new Map;
   }
 
+  /**
+   * Create a new Param instance with the given initial value.
+   * @param {*} value The initial value of the Param.
+   * @returns {Param} The new Param instance.
+   */
   static value(value) {
     return new Param(value);
   }
 
+  /**
+   * Create a new Param instance over an array of initial values,
+   * which may contain nested Params.
+   * @param {*} values The initial values of the Param.
+   * @returns {Param} The new Param instance.
+   */
+  static array(values) {
+    if (values.some(v => isParam(v))) {
+      const p = new Param();
+      const update = () => p.update(values.map(v => isParam(v) ? v.value : v));
+      update();
+      values.forEach(v => isParam(v) ? v.addEventListener('value', update) : 0);
+      return p;
+    }
+    return new Param(values);
+  }
+
+  /**
+   * The current value of the Param.
+   */
   get value() {
     return this._value;
   }
 
+  /**
+   * Update the Param value
+   * @param {*} value The new value of the Param.
+   * @param {object} [options] The update options.
+   * @param {boolean} [options.force] A boolean flag indicating if the Param
+   *  should emit a 'value' event even if the internal value is unchanged.
+   * @returns {this} This Param instance.
+   */
   update(value, { force } = {}) {
-    const changed = distinct(this._value, value);
-    if (changed) this._value = value;
-    if (changed || force) this.emit('value', this.value);
-    return this;
-  }
-
-  addEventListener(type, callback) {
-    let list = this._listeners.get(type) || [];
-    if (!list.includes(callback)) {
-      list = list.concat(callback);
+    const shouldEmit = distinct(this._value, value) || force;
+    if (shouldEmit) {
+      this.emit('value', value);
+    } else {
+      this.cancel('value');
     }
-    this._listeners.set(type, list);
+    return this;
   }
 
-  removeEventListener(type, callback) {
-    const list = this._listeners.get(type);
-    if (list?.length) {
-      this._listeners.set(type, list.filter(x => x !== callback));
+  /**
+   * Upon value-typed updates, sets the current value to the input value
+   * immediately prior to the event value being emitted to listeners.
+   * @param {string} type The event type.
+   * @param {*} value The input event value.
+   * @returns {*} The input event value.
+   */
+  willEmit(type, value) {
+    if (type === 'value') {
+      this._value = value;
     }
-  }
-
-  emit(type, event) {
-    this._listeners.get(type)?.forEach(l => l(event));
+    return value;
   }
 }

package/src/QueryManager.js

@@ -0,0 +1,109 @@
+import { lruCache, voidCache } from './util/cache.js';
+import { priorityQueue } from './util/priority-queue.js';
+
+export const Priority = { High: 0, Normal: 1, Low: 2 };
+
+export function QueryManager() {
+  const queue = priorityQueue(3);
+  let db;
+  let clientCache;
+  let logger;
+  let recorders = [];
+  let pending = null;
+
+  function next() {
+    if (pending || queue.isEmpty()) return;
+    const { request, result } = queue.next();
+    pending = submit(request, result);
+    pending.finally(() => { pending = null; next(); });
+  }
+
+  async function submit(request, result) {
+    try {
+      const { query, type, cache = false, options } = request;
+      const sql = query ? String(query) : null;
+
+      // update recorders
+      if (recorders.length && sql) {
+        recorders.forEach(rec => rec.add(sql));
+      }
+
+      // check query cache
+      if (cache) {
+        const cached = clientCache.get(sql);
+        if (cached) {
+          logger.debug('Cache');
+          result.fulfill(cached);
+          return;
+        }
+      }
+
+      // issue query, potentially cache result
+      const t0 = performance.now();
+      const data = await db.query({ type, sql, ...options });
+      if (cache) clientCache.set(sql, data);
+      logger.debug(`Request: ${(performance.now() - t0).toFixed(1)}`);
+      result.fulfill(data);
+    } catch (err) {
+      result.reject(err);
+    }
+  }
+
+  return {
+    cache(value) {
+      return value !== undefined
+        ? (clientCache = value === true ? lruCache() : (value || voidCache()))
+        : clientCache;
+    },
+
+    logger(value) {
+      return value ? (logger = value) : logger;
+    },
+
+    connector(connector) {
+      return connector ? (db = connector) : db;
+    },
+
+    request(request, priority = Priority.Normal) {
+      const result = queryResult();
+      queue.insert({ request, result }, priority);
+      next();
+      return result;
+    },
+
+    cancel(requests) {
+      const set = new Set(requests);
+      queue.remove(({ result }) => set.has(result));
+    },
+
+    record() {
+      let state = [];
+      const recorder = {
+        add(query) {
+          state.push(query);
+        },
+        reset() {
+          state = [];
+        },
+        snapshot() {
+          return state.slice();
+        },
+        stop() {
+          recorders = recorders.filter(x => x !== recorder);
+          return state;
+        }
+      };
+      recorders.push(recorder);
+      return recorder;
+    }
+  };
+}
+
+function queryResult() {
+  let resolve;
+  let reject;
+  const p = new Promise((r, e) => { resolve = r; reject = e; });
+  p.fulfill = value => (resolve(value), p);
+  p.reject = err => (reject(err), p);
+  return p;
+}