@uwdata/mosaic-core 0.2.0 → 0.3.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uwdata/mosaic-core",
-  "version": "0.2.0",
+  "version": "0.3.2",
   "description": "Scalable and extensible linked data views.",
   "keywords": [
     "mosaic",
@@ -28,9 +28,9 @@
     "prepublishOnly": "npm run test && npm run lint && npm run build"
   },
   "dependencies": {
-    "@duckdb/duckdb-wasm": "^1.25.0",
-    "@uwdata/mosaic-sql": "^0.2.0",
-    "apache-arrow": "^12.0.0"
+    "@duckdb/duckdb-wasm": "^1.27.0",
+    "@uwdata/mosaic-sql": "^0.3.2",
+    "apache-arrow": "^11.0.0"
   },
-  "gitHead": "e53cd914c807f99aabe78dcbe618dd9543e2f438"
+  "gitHead": "788bb137cc402b472fc7e4d84844c78151707c82"
 }

package/src/Catalog.js

@@ -1,3 +1,4 @@
+import { asRelation } from '@uwdata/mosaic-sql';
 import { jsType } from './util/js-type.js';
 import { summarize } from './util/summarize.js';
 
@@ -13,30 +14,16 @@ export class Catalog {
     this.tables = object();
   }
 
-  async tableInfo(table) {
+  tableInfo(table) {
     const cache = this.tables;
     if (cache[table]) {
       return cache[table];
     }
 
-    const q = this.mc.query(
-      `DESCRIBE "${table}"`,
-      { type: 'json', cache: false }
-    );
+    const infoPromise = getTableInfo(this.mc, table)
+      .catch(err => { cache[table] = null; throw err; });
 
-    return (cache[table] = q.then(result => {
-      const columns = object();
-      for (const entry of result) {
-        columns[entry.column_name] = {
-          table,
-          column: entry.column_name,
-          sqlType: entry.column_type,
-          type: jsType(entry.column_type),
-          nullable: entry.null === 'YES'
-        };
-      }
-      return columns;
-    }));
+    return (cache[table] = infoPromise);
   }
 
   async fieldInfo({ table, column, stats }) {
@@ -73,6 +60,26 @@ export class Catalog {
   }
 }
 
+async function getTableInfo(mc, table) {
+  const result = await mc.query(
+    `DESCRIBE ${asRelation(table)}`,
+    { type: 'json', cache: false }
+  );
+
+  const columns = object();
+  for (const entry of result) {
+    columns[entry.column_name] = {
+      table,
+      column: entry.column_name,
+      sqlType: entry.column_type,
+      type: jsType(entry.column_type),
+      nullable: entry.null === 'YES'
+    };
+  }
+
+  return columns;
+}
+
 async function resolveFields(catalog, list) {
   return list.length === 1 && list[0].column === '*'
     ? Object.values(await catalog.tableInfo(list[0].table))

package/src/Coordinator.js

@@ -6,6 +6,12 @@ import { voidLogger } from './util/void-logger.js';
 
 let _instance;
 
+/**
+ * Set or retrieve the coordinator instance.
+ *
+ * @param {Coordinator} instance the coordinator instance to set
+ * @returns {Coordinator} the coordinator instance
+ */
 export function coordinator(instance) {
   if (instance) {
     _instance = instance;
@@ -23,7 +29,6 @@ export class Coordinator {
     this.configure(options);
     this.databaseConnector(db);
     this.clear();
-    this._recorders = [];
   }
 
   logger(logger) {
@@ -34,12 +39,14 @@ export class Coordinator {
     return this._logger;
   }
 
-  configure({ cache = true, indexes = true }) {
+  configure({ cache = true, consolidate = true, indexes = true }) {
     this.manager.cache(cache);
+    this.manager.consolidate(consolidate);
     this.indexes = indexes;
   }
 
   clear({ clients = true, cache = true, catalog = false } = {}) {
+    this.manager.clear();
     if (clients) {
       this.clients?.forEach(client => this.disconnect(client));
       this.filterGroups?.forEach(group => group.finalize());
@@ -104,6 +111,11 @@ export class Coordinator {
       : client.update();
   }
 
+  /**
+   * Connect a client to the coordinator.
+   *
+   * @param {import('./MosaicClient.js').MosaicClient} client the client to disconnect
+   */
   async connect(client) {
     const { catalog, clients, filterGroups, indexes } = this;
 
@@ -132,6 +144,11 @@ export class Coordinator {
     client.requestQuery();
   }
 
+  /**
+   * Disconnect a client from the coordinator.
+   *
+   * @param {import('./MosaicClient.js').MosaicClient} client the client to disconnect
+   */
   disconnect(client) {
     const { clients, filterGroups } = this;
     if (!clients.has(client)) return;

package/src/{DataTileIndexer.js → DataCubeIndexer.js}

@@ -1,24 +1,30 @@
-import { Query, and, asColumn, epoch_ms, isBetween, sql } from '@uwdata/mosaic-sql';
+import { Query, and, asColumn, create, epoch_ms, isBetween, sql } from '@uwdata/mosaic-sql';
 import { fnv_hash } from './util/hash.js';
 
 const identity = x => x;
 
 /**
- * Build and query optimized indices ("data tiles") for fast computation of
+ * Build and query optimized indices ("data cubes") for fast computation of
  * groupby aggregate queries over compatible client queries and selections.
- * A data tile contains pre-aggregated data for a Mosaic client, subdivided
- * by possible query values from an active view. Index tiles are realized as
+ * A data cube contains pre-aggregated data for a Mosaic client, subdivided
+ * by possible query values from an active view. Indexes are realized as
  * as temporary database tables that can be queried for rapid updates.
  * Compatible client queries must pull data from the same backing table and
  * must consist of only groupby dimensions and supported aggregates.
  * Compatible selections must contain an active clause that exposes a schema
  * for an interval or point value predicate.
  */
-export class DataTileIndexer {
-
-  constructor(mc, selection) {
+export class DataCubeIndexer {
+  /**
+   *
+   * @param {import('./Coordinator.js').Coordinator} mc a Mosaic coordinator
+   * @param {*} options Options hash to configure the data cube indexes and pass selections to the coordinator.
+   */
+  constructor(mc, { selection, temp = true }) {
+    /** @type import('./Coordinator.js').Coordinator */
     this.mc = mc;
     this.selection = selection;
+    this.temp = temp;
     this.reset();
   }
 
@@ -57,13 +63,14 @@ export class DataTileIndexer {
     const activeView = this.activeView = getActiveView(active);
     if (!activeView) return false; // active selection clause not compatible
 
-    this.mc.logger().warn('DATA TILE INDEX CONSTRUCTION');
+    this.mc.logger().warn('DATA CUBE INDEX CONSTRUCTION');
 
     // create a selection with the active source removed
     const sel = this.selection.remove(source);
 
-    // generate data tile indices
+    // generate data cube indices
     const indices = this.indices = new Map;
+    const { mc, temp } = this;
     for (const client of clients) {
       if (sel.skip(client, active)) continue;
       const index = getIndexColumns(client);
@@ -82,8 +89,8 @@ export class DataTileIndexer {
 
       const sql = query.toString();
       const id = (fnv_hash(sql) >>> 0).toString(16);
-      const table = `tile_index_${id}`;
-      const result = createIndex(this.mc, table, sql);
+      const table = `cube_index_${id}`;
+      const result = mc.exec(create(table, sql, { temp }));
       indices.set(client, { table, result, ...index });
     }
   }
@@ -185,10 +192,6 @@ function binFunction(domain, range, pixelSize, lift, toSql) {
   return value => sql`${s}FLOOR(${a}::DOUBLE * (${toSql(value)} - ${lo}::DOUBLE))::INTEGER`;
 }
 
-function createIndex(mc, table, query) {
-  return mc.exec(`CREATE TEMP TABLE IF NOT EXISTS ${table} AS ${query}`);
-}
-
 const NO_INDEX = { from: NaN };
 
 function getIndexColumns(client) {

package/src/FilterGroup.js

@@ -1,11 +1,20 @@
-import { DataTileIndexer } from './DataTileIndexer.js';
+import { DataCubeIndexer } from './DataCubeIndexer.js';
 
 export class FilterGroup {
+  /**
+   * @param {import('./Coordinator.js').Coordinator} coordinator The Mosaic coordinator.
+   * @param {*} selection The shared filter selection.
+   * @param {*} index Boolean flag or options hash for data cube indexer.
+   *  Falsy values disable indexing.
+   */
   constructor(coordinator, selection, index = true) {
+    /** @type import('./Coordinator.js').Coordinator */
     this.mc = coordinator;
     this.selection = selection;
     this.clients = new Set();
-    this.indexer = index ? new DataTileIndexer(this.mc, selection) : null;
+    this.indexer = index
+      ? new DataCubeIndexer(this.mc, { ...index, selection })
+      : null;
 
     const { value, activate } = this.handlers = {
       value: () => this.update(),

package/src/MosaicClient.js

@@ -40,10 +40,10 @@ export class MosaicClient {
   }
 
   /**
-   * Called by the coordinator to set the field statistics for this client.
+   * Called by the coordinator to set the field info for this client.
    * @returns {this}
    */
-  fieldStats() {
+  fieldInfo() {
     return this;
   }
 
@@ -63,6 +63,9 @@ export class MosaicClient {
 
   /**
    * Called by the coordinator to return a query result.
+   * 
+   * @param {*} data the query result
+   * @returns {this}
    */
   queryResult() {
     return this;

package/src/QueryConsolidator.js

@@ -0,0 +1,238 @@
+import { Query, Ref } from '@uwdata/mosaic-sql';
+import { queryResult } from './util/query-result.js';
+
+/**
+ * Create a consolidator to combine structurally compatible queries.
+ * @param {*} enqueue Query manager enqueue method
+ * @param {*} cache Client-side query cache (sql -> data)
+ * @param {*} record Query recorder function
+ * @returns A consolidator object
+ */
+export function consolidator(enqueue, cache, record) {
+  let pending = [];
+  let id = 0;
+
+  function run() {
+    // group queries into bundles that can be consolidated
+    const groups = entryGroups(pending, cache);
+    pending = [];
+    id = 0;
+
+    // build and issue consolidated queries
+    for (const group of groups) {
+      consolidate(group, enqueue, record);
+      processResults(group, cache);
+    }
+  }
+
+  return {
+    add(entry, priority) {
+      if (entry.request.type === 'arrow') {
+        // wait one frame, gather an ordered list of queries
+        // only Apache Arrow is supported, so we can project efficiently
+        id = id || requestAnimationFrame(() => run());
+        pending.push({ entry, priority, index: pending.length });
+      } else {
+        enqueue(entry, priority);
+      }
+    }
+  }
+}
+
+/**
+ * Segment query requests into consolidation-compatible groups.
+ * @param {*} entries Query request entries ({ request, result } objects)
+ * @returns An array of grouped entry arrays
+ */
+function entryGroups(entries, cache) {
+  const groups = [];
+  const groupMap = new Map;
+
+  for (const query of entries) {
+    const { entry: { request } } = query;
+    const key = consolidationKey(request.query, cache);
+    if (!groupMap.has(key)) {
+      const list = [];
+      groups.push(list);
+      groupMap.set(key, list);
+    }
+    groupMap.get(key).push(query);
+  }
+
+  return groups;
+}
+
+/**
+ * Generate a key string for query consolidation.
+ * Queries with matching keys are conosolidation-compatible.
+ * If a query is found in the cache, it is exempted from consolidation,
+ * which is indicated by returning the precise query SQL as the key.
+ * @param {*} query The input query.
+ * @param {*} cache The query cache (sql -> data).
+ * @returns a key string
+ */
+function consolidationKey(query, cache) {
+  const sql = `${query}`;
+  if (query instanceof Query && !cache.get(sql)) {
+    if (
+      query.orderby().length || query.where().length ||
+      query.qualify().length || query.having().length
+    ) {
+      // do not try to analyze if query includes clauses
+      // that may refer to *derived* columns we can't resolve
+      return sql;
+    }
+
+    // create a derived query stripped of selections
+    const q = query.clone().$select('*');
+
+    // check group by criteria for compatibility
+    // queries may refer to *derived* columns as group by criteria
+    // we resolve these against the true grouping expressions
+    const groupby = query.groupby();
+    if (groupby.length) {
+      const map = {}; // expression map (as -> expr)
+      query.select().forEach(({ as, expr }) => map[as] = expr);
+      q.$groupby(groupby.map(e => (e instanceof Ref && map[e.column]) || e));
+    }
+
+    // key is just the transformed query as SQL
+    return `${q}`;
+  } else {
+    // can not analyze query, simply return as string
+    return sql;
+  }
+}
+
+/**
+ * Issue queries, consolidating where possible.
+ * @param {*} group Array of bundled query entries
+ * @param {*} enqueue Add entry to query queue
+ * @param {*} record Query recorder function
+ */
+function consolidate(group, enqueue, record) {
+  if (shouldConsolidate(group)) {
+    // issue a single consolidated query
+    enqueue({
+      request: {
+        type: 'arrow',
+        cache: false,
+        record: false,
+        query: consolidatedQuery(group, record)
+      },
+      result: (group.result = queryResult())
+    });
+  } else {
+    // issue queries directly
+    for (const { entry, priority } of group) {
+      enqueue(entry, priority);
+    }
+  }
+}
+
+/**
+ * Check if a group contains multiple distinct queries.
+ * @param {*} group Array of bundled query entries
+ * @returns false if group contains a single (possibly repeated) query,
+ *  otherwise true
+ */
+function shouldConsolidate(group) {
+  if (group.length > 1) {
+    const sql = `${group[0].entry.request.query}`;
+    for (let i = 1; i < group.length; ++i) {
+      if (sql !== `${group[i].entry.request.query}`) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+/**
+ * Create a consolidated query for a group.
+ * @param {*} group Array of bundled query entries
+ * @param {*} record Query recorder function
+ * @returns A consolidated Query instance
+ */
+function consolidatedQuery(group, record) {
+  const maps = group.maps = [];
+  const fields = new Map;
+
+  // gather select fields
+  for (const item of group) {
+    const { query } = item.entry.request;
+    const fieldMap = [];
+    maps.push(fieldMap);
+    for (const { as, expr } of query.select()) {
+      const e = `${expr}`;
+      if (!fields.has(e)) {
+        fields.set(e, [`col${fields.size}`, expr]);
+      }
+      const [name] = fields.get(e);
+      fieldMap.push([name, as]);
+    }
+    record(`${query}`);
+  }
+
+  // use a cloned query as a starting point
+  const query = group[0].entry.request.query.clone();
+
+  // update group by statement as needed
+  const groupby = query.groupby();
+  if (groupby.length) {
+    const map = {};
+    group.maps[0].forEach(([name, as]) => map[as] = name);
+    query.$groupby(groupby.map(e => (e instanceof Ref && map[e.column]) || e));
+  }
+
+  // update select statemenet and return
+  return query.$select(Array.from(fields.values()));
+}
+
+/**
+ * Process query results, dispatch results to original requests
+ * @param {*} group Array of query requests
+ * @param {*} cache Client-side query cache (sql -> data)
+ */
+async function processResults(group, cache) {
+  const { maps, result } = group;
+  if (!maps) return; // no consolidation performed
+
+  let data;
+  try {
+    data = await result;
+  } catch (err) {
+    // pass error to consolidated queries
+    for (const { entry } of group) {
+      entry.result.reject(err);
+    }
+    return;
+  }
+
+  group.forEach(({ entry }, index) => {
+    const { request, result } = entry;
+    const projected = projectResult(data, maps[index]);
+    if (request.cache) {
+      cache.set(String(request.query), projected);
+    }
+    result.fulfill(projected);
+  });
+}
+
+/**
+ * Project a consolidated result to a client result
+ * @param {*} data Consolidated query result, as an Apache Arrow Table
+ * @param {*} map Column name map as [source, target] pairs
+ * @returns the projected Apache Arrow table
+ */
+function projectResult(data, map) {
+  if (map) {
+    const cols = {};
+    for (const [name, as] of map) {
+      cols[as] = data.getChild(name);
+    }
+    return new data.constructor(cols);
+  } else {
+    return data;
+  }
+}

package/src/QueryManager.js

@@ -1,5 +1,7 @@
+import { consolidator } from './QueryConsolidator.js';
 import { lruCache, voidCache } from './util/cache.js';
 import { priorityQueue } from './util/priority-queue.js';
+import { queryResult } from './util/query-result.js';
 
 export const Priority = { High: 0, Normal: 1, Low: 2 };
 
@@ -10,6 +12,7 @@ export function QueryManager() {
   let logger;
   let recorders = [];
   let pending = null;
+  let consolidate;
 
   function next() {
     if (pending || queue.isEmpty()) return;
@@ -18,14 +21,25 @@ export function QueryManager() {
     pending.finally(() => { pending = null; next(); });
   }
 
+  function enqueue(entry, priority = Priority.Normal) {
+    queue.insert(entry, priority);
+    next();
+  }
+
+  function recordQuery(sql) {
+    if (recorders.length && sql) {
+      recorders.forEach(rec => rec.add(sql));
+    }
+  }
+
   async function submit(request, result) {
     try {
-      const { query, type, cache = false, options } = request;
-      const sql = query ? String(query) : null;
+      const { query, type, cache = false, record = true, options } = request;
+      const sql = query ? `${query}` : null;
 
       // update recorders
-      if (recorders.length && sql) {
-        recorders.forEach(rec => rec.add(sql));
+      if (record) {
+        recordQuery(sql);
       }
 
       // check query cache
@@ -64,10 +78,22 @@ export function QueryManager() {
       return connector ? (db = connector) : db;
     },
 
+    consolidate(flag) {
+      if (flag && !consolidate) {
+        consolidate = consolidator(enqueue, clientCache, recordQuery);
+      } else if (!flag && consolidate) {
+        consolidate = null;
+      }
+    },
+
     request(request, priority = Priority.Normal) {
       const result = queryResult();
-      queue.insert({ request, result }, priority);
-      next();
+      const entry = { request, result };
+      if (consolidate) {
+        consolidate.add(entry, priority);
+      } else {
+        enqueue(entry, priority);
+      }
       return result;
     },
 
@@ -76,6 +102,13 @@ export function QueryManager() {
       queue.remove(({ result }) => set.has(result));
     },
 
+    clear() {
+      queue.remove(({ result }) => {
+        result.reject('Cleared');
+        return true;
+      });
+    },
+
     record() {
       let state = [];
       const recorder = {
@@ -98,12 +131,3 @@ export function QueryManager() {
     }
   };
 }
-
-function queryResult() {
-  let resolve;
-  let reject;
-  const p = new Promise((r, e) => { resolve = r; reject = e; });
-  p.fulfill = value => (resolve(value), p);
-  p.reject = err => (reject(err), p);
-  return p;
-}

package/src/Selection.js

@@ -120,6 +120,13 @@ export class Selection extends Param {
     return super.value;
   }
 
+  /**
+   * Indicate if this selection has a single resolution strategy.
+   */
+  get single() {
+    return this._resolver.single;
+  }
+
   /**
    * Emit an activate event with the given selection clause.
    * @param {*} clause The clause repesenting the potential activation.

package/src/util/query-result.js

@@ -0,0 +1,8 @@
+export function queryResult() {
+  let resolve;
+  let reject;
+  const p = new Promise((r, e) => { resolve = r; reject = e; });
+  p.fulfill = value => (resolve(value), p);
+  p.reject = err => (reject(err), p);
+  return p;
+}