@uwdata/mosaic-core 0.1.0 → 0.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uwdata/mosaic-core",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "Scalable and extensible linked data views.",
   "keywords": [
     "mosaic",
@@ -28,9 +28,9 @@
     "prepublishOnly": "npm run test && npm run lint && npm run build"
   },
   "dependencies": {
-    "@duckdb/duckdb-wasm": "^1.20.0",
-    "@uwdata/mosaic-sql": "^0.1.0",
+    "@duckdb/duckdb-wasm": "^1.27.0",
+    "@uwdata/mosaic-sql": "^0.3.0",
     "apache-arrow": "^11.0.0"
   },
-  "gitHead": "a7967c35349bdf7f00abb113ce1dd9abb233cd62"
+  "gitHead": "a8dd23fed4c7a24c0a2ee5261d1aabe4239ce574"
 }

package/src/Catalog.js

@@ -13,30 +13,16 @@ export class Catalog {
     this.tables = object();
   }
 
-  async tableInfo(table) {
+  tableInfo(table) {
     const cache = this.tables;
     if (cache[table]) {
       return cache[table];
     }
 
-    const q = this.mc.query(
-      `DESCRIBE "${table}"`,
-      { type: 'json', cache: false }
-    );
+    const infoPromise = getTableInfo(this.mc, table)
+      .catch(err => { cache[table] = null; throw err; });
 
-    return (cache[table] = q.then(result => {
-      const columns = object();
-      for (const entry of result) {
-        columns[entry.column_name] = {
-          table,
-          column: entry.column_name,
-          sqlType: entry.column_type,
-          type: jsType(entry.column_type),
-          nullable: entry.null === 'YES'
-        };
-      }
-      return columns;
-    }));
+    return (cache[table] = infoPromise);
   }
 
   async fieldInfo({ table, column, stats }) {
@@ -49,8 +35,20 @@ export class Catalog {
     // no need for summary statistics
     if (!stats?.length) return colInfo;
 
-    const result = await this.mc.query(summarize(colInfo, stats));
+    const result = await this.mc.query(
+      summarize(colInfo, stats),
+      { persist: true }
+    );
     const info = { ...colInfo, ...(Array.from(result)[0]) };
+
+    // coerce bigint to number
+    for (const key in info) {
+      const value = info[key];
+      if (typeof value === 'bigint') {
+        info[key] = Number(value);
+      }
+    }
+
     return info;
   }
 
@@ -61,6 +59,26 @@ export class Catalog {
   }
 }
 
+async function getTableInfo(mc, table) {
+  const result = await mc.query(
+    `DESCRIBE "${table}"`,
+    { type: 'json', cache: false }
+  );
+
+  const columns = object();
+  for (const entry of result) {
+    columns[entry.column_name] = {
+      table,
+      column: entry.column_name,
+      sqlType: entry.column_type,
+      type: jsType(entry.column_type),
+      nullable: entry.null === 'YES'
+    };
+  }
+
+  return columns;
+}
+
 async function resolveFields(catalog, list) {
   return list.length === 1 && list[0].column === '*'
     ? Object.values(await catalog.tableInfo(list[0].table))

package/src/Coordinator.js

@@ -1,11 +1,17 @@
-import { socketClient } from './clients/socket.js';
+import { socketConnector } from './connectors/socket.js';
 import { Catalog } from './Catalog.js';
 import { FilterGroup } from './FilterGroup.js';
-import { QueryCache, voidCache } from './QueryCache.js';
+import { QueryManager, Priority } from './QueryManager.js';
 import { voidLogger } from './util/void-logger.js';
 
 let _instance;
 
+/**
+ * Set or retrieve the coordinator instance.
+ *
+ * @param {Coordinator} coordinator the coordinator instance to set
+ * @returns {Coordinator} the coordinator instance
+ */
 export function coordinator(instance) {
   if (instance) {
     _instance = instance;
@@ -16,90 +22,100 @@ export function coordinator(instance) {
 }
 
 export class Coordinator {
-  constructor(db = socketClient(), options = {}) {
+  constructor(db = socketConnector(), options = {}) {
     this.catalog = new Catalog(this);
+    this.manager = options.manager || QueryManager();
     this.logger(options.logger || console);
     this.configure(options);
-    this.databaseClient(db);
+    this.databaseConnector(db);
     this.clear();
   }
 
   logger(logger) {
-    return arguments.length
-      ? (this._logger = logger || voidLogger())
-      : this._logger;
+    if (arguments.length) {
+      this._logger = logger || voidLogger();
+      this.manager.logger(this._logger);
+    }
+    return this._logger;
   }
 
-  configure({ cache = true, indexes = true }) {
-    this.cache = cache ? new QueryCache() : voidCache();
+  configure({ cache = true, consolidate = true, indexes = true }) {
+    this.manager.cache(cache);
+    this.manager.consolidate(consolidate);
     this.indexes = indexes;
   }
 
   clear({ clients = true, cache = true, catalog = false } = {}) {
+    this.manager.clear();
     if (clients) {
       this.clients?.forEach(client => this.disconnect(client));
       this.filterGroups?.forEach(group => group.finalize());
       this.clients = new Set;
       this.filterGroups = new Map;
     }
-    if (cache) this.cache.clear();
+    if (cache) this.manager.cache().clear();
     if (catalog) this.catalog.clear();
   }
 
-  databaseClient(db) {
-    if (arguments.length > 0) {
-      this.db = db;
-    }
-    return this.db;
+  databaseConnector(db) {
+    return this.manager.connector(db);
   }
 
-  async exec(sql) {
-    try {
-      await this.db.query({ type: 'exec', sql });
-    } catch (err) {
-      this._logger.error(err);
-    }
+  // -- Query Management ----
+
+  cancel(requests) {
+    this.manager.cancel(requests);
   }
 
-  query(query, { type = 'arrow', cache = true } = {}) {
-    const sql = String(query);
-    const t0 = performance.now();
-    const cached = this.cache.get(sql);
-    if (cached) {
-      this._logger.debug('Cache');
-      return cached;
-    } else {
-      const request = this.db.query({ type, sql });
-      const result = cache ? this.cache.set(sql, request) : request;
-      result.then(() => this._logger.debug(`Query: ${performance.now() - t0}`));
-      return result;
-    }
+  exec(query, { priority = Priority.Normal } = {}) {
+    return this.manager.request({ type: 'exec', query }, priority);
   }
 
-  async updateClient(client, query) {
-    let result;
-    try {
-      client.queryPending();
-      result = await this.query(query);
-    } catch (err) {
-      this._logger.error(err);
-      client.queryError(err);
-      return;
-    }
-    try {
-      client.queryResult(result).update();
-    } catch (err) {
-      this._logger.error(err);
-    }
+  query(query, {
+    type = 'arrow',
+    cache = true,
+    priority = Priority.Normal,
+    ...options
+  } = {}) {
+    return this.manager.request({ type, query, cache, options }, priority);
+  }
+
+  prefetch(query, options = {}) {
+    return this.query(query, { ...options, cache: true, priority: Priority.Low });
+  }
+
+  createBundle(name, queries, priority = Priority.Low) {
+    const options = { name, queries };
+    return this.manager.request({ type: 'create-bundle', options }, priority);
+  }
+
+  loadBundle(name, priority = Priority.High) {
+    const options = { name };
+    return this.manager.request({ type: 'load-bundle', options }, priority);
+  }
+
+  // -- Client Management ----
+
+  updateClient(client, query, priority = Priority.Normal) {
+    client.queryPending();
+    return this.query(query, { priority }).then(
+      data => client.queryResult(data).update(),
+      err => { client.queryError(err); this._logger.error(err); }
+    );
   }
 
-  async requestQuery(client, query) {
+  requestQuery(client, query) {
     this.filterGroups.get(client.filterBy)?.reset();
     return query
       ? this.updateClient(client, query)
       : client.update();
   }
 
+  /**
+   * Connect a client to the coordinator.
+   *
+   * @param {import('./MosaicClient.js').MosaicClient} client the client to disconnect
+   */
   async connect(client) {
     const { catalog, clients, filterGroups, indexes } = this;
 
@@ -111,7 +127,7 @@ export class Coordinator {
     // retrieve field statistics
     const fields = client.fields();
     if (fields?.length) {
-      client.fieldStats(await catalog.queryFields(fields));
+      client.fieldInfo(await catalog.queryFields(fields));
     }
 
     // connect filters
@@ -128,6 +144,11 @@ export class Coordinator {
     client.requestQuery();
   }
 
+  /**
+   * Disconnect a client from the coordinator.
+   *
+   * @param {import('./MosaicClient.js').MosaicClient} client the client to disconnect
+   */
   disconnect(client) {
     const { clients, filterGroups } = this;
     if (!clients.has(client)) return;

package/src/DataTileIndexer.js

@@ -1,6 +1,5 @@
-import { Query, expr, and, isBetween, asColumn, epoch_ms } from '@uwdata/mosaic-sql';
+import { Query, and, asColumn, epoch_ms, isBetween, sql } from '@uwdata/mosaic-sql';
 import { fnv_hash } from './util/hash.js';
-import { skipClient } from './util/skip-client.js';
 
 const identity = x => x;
 
@@ -30,35 +29,43 @@ export class DataTileIndexer {
     this.activeView = null;
   }
 
+  clear() {
+    if (this.indices) {
+      this.mc.cancel(Array.from(this.indices.values(), index => index.result));
+      this.indices = null;
+    }
+  }
+
   index(clients, active) {
     if (this.clients !== clients) {
       // test client views for compatibility
-      const cols = Array.from(clients).map(getIndexColumns);
+      const cols = Array.from(clients, getIndexColumns);
       const from = cols[0]?.from;
       this.enabled = cols.every(c => c && c.from === from);
       this.clients = clients;
-      this.indices = null;
       this.activeView = null;
+      this.clear();
     }
     if (!this.enabled) return false; // client views are not indexable
 
     active = active || this.selection.active;
     const { source } = active;
+    if (source && source === this.activeView?.source) return true; // we're good!
+
+    this.clear();
     if (!source) return false; // nothing to work with
-    if (source === this.activeView?.source) return true; // we're good!
     const activeView = this.activeView = getActiveView(active);
     if (!activeView) return false; // active selection clause not compatible
 
     this.mc.logger().warn('DATA TILE INDEX CONSTRUCTION');
 
-    // create a selection with the active client removed
-    const sel = this.selection.clone().update({ source });
+    // create a selection with the active source removed
+    const sel = this.selection.remove(source);
 
     // generate data tile indices
     const indices = this.indices = new Map;
-    const promises = [];
     for (const client of clients) {
-      if (sel.cross && skipClient(client, active)) continue;
+      if (sel.skip(client, active)) continue;
       const index = getIndexColumns(client);
 
       // build index construction query
@@ -76,11 +83,9 @@ export class DataTileIndexer {
       const sql = query.toString();
       const id = (fnv_hash(sql) >>> 0).toString(16);
       const table = `tile_index_${id}`;
-      indices.set(client, { table, ...index });
-      promises.push(createIndex(this.mc, table, sql));
+      const result = createIndex(this.mc, table, sql);
+      indices.set(client, { table, result, ...index });
     }
-
-    return promises;
   }
 
   async update() {
@@ -100,12 +105,12 @@ export class DataTileIndexer {
     }
 
     const { table, dims, aggr } = index;
-    return this.mc.updateClient(client, Query
+    const query = Query
       .select(dims, aggr)
       .from(table)
       .groupby(dims)
-      .where(filter)
-    );
+      .where(filter);
+    return this.mc.updateClient(client, query);
   }
 }
 
@@ -113,22 +118,22 @@ function getActiveView(clause) {
   const { source, schema } = clause;
   let columns = clause.predicate?.columns;
   if (!schema || !columns) return null;
-  const { type, scales } = schema;
+  const { type, scales, pixelSize = 1 } = schema;
   let predicate;
 
   if (type === 'interval' && scales) {
-    const bins = scales.map(s => binInterval(s));
+    const bins = scales.map(s => binInterval(s, pixelSize));
     if (bins.some(b => b == null)) return null; // unsupported scale type
 
     if (bins.length === 1) {
-      predicate = p => p ? isBetween('active0', p.value.map(bins[0])) : [];
-      columns = { active0: bins[0](clause.predicate.expr) };
+      predicate = p => p ? isBetween('active0', p.range.map(bins[0])) : [];
+      columns = { active0: bins[0](clause.predicate.field) };
     } else {
       predicate = p => p
-        ? and(p.value.map(({ value }, i) => isBetween(`active${i}`, value.map(bins[i]))))
+        ? and(p.children.map(({ range }, i) => isBetween(`active${i}`, range.map(bins[i]))))
         : [];
       columns = Object.fromEntries(
-        clause.predicate.value.map((p, i) => [`active${i}`, bins[i](p.expr)])
+        clause.predicate.children.map((p, i) => [`active${i}`, bins[i](p.field)])
       );
     }
   } else if (type === 'point') {
@@ -141,53 +146,47 @@ function getActiveView(clause) {
   return { source, columns, predicate };
 }
 
-function binInterval(scale) {
+function binInterval(scale, pixelSize) {
   const { type, domain, range } = scale;
-  let lift, sql;
+  let lift, toSql;
 
   switch (type) {
     case 'linear':
       lift = identity;
-      sql = asColumn;
+      toSql = asColumn;
       break;
     case 'log':
       lift = Math.log;
-      sql = c => `LN(${asColumn(c)})`;
+      toSql = c => sql`LN(${asColumn(c)})`;
       break;
     case 'symlog':
       // TODO: support log constants other than 1?
       lift = x => Math.sign(x) * Math.log1p(Math.abs(x));
-      sql = c => (c = asColumn(c), `SIGN(${c}) * LN(1 + ABS(${c}))`);
+      toSql = c => (c = asColumn(c), sql`SIGN(${c}) * LN(1 + ABS(${c}))`);
       break;
     case 'sqrt':
       lift = Math.sqrt;
-      sql = c => `SQRT(${asColumn(c)})`;
+      toSql = c => sql`SQRT(${asColumn(c)})`;
       break;
     case 'utc':
     case 'time':
       lift = x => +x;
-      sql = c => c instanceof Date ? +c : epoch_ms(asColumn(c));
+      toSql = c => c instanceof Date ? +c : epoch_ms(asColumn(c));
       break;
   }
-  return lift ? binFunction(domain, range, lift, sql) : null;
+  return lift ? binFunction(domain, range, pixelSize, lift, toSql) : null;
 }
 
-function binFunction(domain, range, lift, sql) {
+function binFunction(domain, range, pixelSize, lift, toSql) {
   const lo = lift(Math.min(domain[0], domain[1]));
   const hi = lift(Math.max(domain[0], domain[1]));
-  const a = Math.abs(lift(range[1]) - lift(range[0])) / (hi - lo);
-  return value => expr(
-    `FLOOR(${a}::DOUBLE * (${sql(value)} - ${lo}::DOUBLE))`,
-    asColumn(value).columns
-  );
+  const a = (Math.abs(lift(range[1]) - lift(range[0])) / (hi - lo)) / pixelSize;
+  const s = pixelSize === 1 ? '' : `${pixelSize}::INTEGER * `;
+  return value => sql`${s}FLOOR(${a}::DOUBLE * (${toSql(value)} - ${lo}::DOUBLE))::INTEGER`;
 }
 
-async function createIndex(mc, table, query) {
-  try {
-    await mc.exec(`CREATE TEMP TABLE IF NOT EXISTS ${table} AS ${query}`);
-  } catch (err) {
-    mc.logger().error(err);
-  }
+function createIndex(mc, table, query) {
+  return mc.exec(`CREATE TEMP TABLE IF NOT EXISTS ${table} AS ${query}`);
 }
 
 const NO_INDEX = { from: NaN };
@@ -199,25 +198,25 @@ function getIndexColumns(client) {
   if (!from || !q.groupby) return NO_INDEX;
   const g = new Set(q.groupby().map(c => c.column));
 
-  let aggr = [];
-  let dims = [];
+  const aggr = [];
+  const dims = [];
   let count;
 
   for (const { as, expr: { aggregate } } of q.select()) {
     switch (aggregate?.toUpperCase()) {
       case 'COUNT':
       case 'SUM':
-        aggr.push({ [as]: expr(`SUM("${as}")::DOUBLE`) });
+        aggr.push({ [as]: sql`SUM("${as}")::DOUBLE` });
         break;
       case 'AVG':
         count = '_count_';
-        aggr.push({ [as]: expr(`(SUM("${as}" * ${count}) / SUM(${count}))::DOUBLE`) });
+        aggr.push({ [as]: sql`(SUM("${as}" * ${count}) / SUM(${count}))::DOUBLE` });
         break;
       case 'MAX':
-        aggr.push({ [as]: expr(`MAX("${as}")`) });
+        aggr.push({ [as]: sql`MAX("${as}")` });
         break;
       case 'MIN':
-        aggr.push({ [as]: expr(`MIN("${as}")`) });
+        aggr.push({ [as]: sql`MIN("${as}")` });
         break;
       default:
         if (g.has(as)) dims.push(as);
@@ -228,7 +227,7 @@ function getIndexColumns(client) {
   return {
     aggr,
     dims,
-    count: count ? { [count]: expr('COUNT(*)') } : {},
+    count: count ? { [count]: sql`COUNT(*)` } : {},
     from
   };
 }
@@ -244,7 +243,7 @@ function getBaseTable(query) {
   }
 
   // handle set operations / subqueries
-  let base = getBaseTable(subq[0]);
+  const base = getBaseTable(subq[0]);
   for (let i = 1; i < subq.length; ++i) {
     const from = getBaseTable(subq[i]);
     if (from === undefined) continue;

package/src/FilterGroup.js

@@ -1,5 +1,4 @@
 import { DataTileIndexer } from './DataTileIndexer.js';
-import { throttle } from './util/throttle.js';
 
 export class FilterGroup {
   constructor(coordinator, selection, index = true) {
@@ -9,10 +8,8 @@ export class FilterGroup {
     this.indexer = index ? new DataTileIndexer(this.mc, selection) : null;
 
     const { value, activate } = this.handlers = {
-      value: throttle(() => this.update()),
-      activate: clause => {
-        this.indexer?.index(this.clients, clause);
-      }
+      value: () => this.update(),
+      activate: clause => this.indexer?.index(this.clients, clause)
     };
     selection.addEventListener('value', value);
     selection.addEventListener('activate', activate);
@@ -40,7 +37,7 @@ export class FilterGroup {
     return this;
   }
 
-  async update() {
+  update() {
     const { mc, indexer, clients, selection } = this;
     return indexer?.index(clients)
       ? indexer.update()

package/src/MosaicClient.js

@@ -40,10 +40,10 @@ export class MosaicClient {
   }
 
   /**
-   * Called by the coordinator to set the field statistics for this client.
+   * Called by the coordinator to set the field info for this client.
    * @returns {this}
    */
-  fieldStats() {
+  fieldInfo() {
     return this;
   }
 
@@ -63,6 +63,9 @@ export class MosaicClient {
 
   /**
    * Called by the coordinator to return a query result.
+   * 
+   * @param {*} data the query result
+   * @returns {this}
    */
   queryResult() {
     return this;