@uwdata/mosaic-core 0.1.0 → 0.3.0

package/src/Param.js

@@ -1,46 +1,92 @@
+import { AsyncDispatch } from './util/AsyncDispatch.js';
 import { distinct } from './util/distinct.js';
 
+/**
+ * Test if a value is a Param instance.
+ * @param {*} x The value to test.
+ * @returns {boolean} True if the input is a Param, false otherwise.
+ */
 export function isParam(x) {
   return x instanceof Param;
 }
 
-export class Param {
+/**
+ * Represents a dynamic parameter that dispatches updates
+ * upon parameter changes.
+ */
+export class Param extends AsyncDispatch {
+
+  /**
+   * Create a new Param instance.
+   * @param {*} value The initial value of the Param.
+   */
   constructor(value) {
+    super();
     this._value = value;
-    this._listeners = new Map;
   }
 
+  /**
+   * Create a new Param instance with the given initial value.
+   * @param {*} value The initial value of the Param.
+   * @returns {Param} The new Param instance.
+   */
   static value(value) {
     return new Param(value);
   }
 
+  /**
+   * Create a new Param instance over an array of initial values,
+   * which may contain nested Params.
+   * @param {*} values The initial values of the Param.
+   * @returns {Param} The new Param instance.
+   */
+  static array(values) {
+    if (values.some(v => isParam(v))) {
+      const p = new Param();
+      const update = () => p.update(values.map(v => isParam(v) ? v.value : v));
+      update();
+      values.forEach(v => isParam(v) ? v.addEventListener('value', update) : 0);
+      return p;
+    }
+    return new Param(values);
+  }
+
+  /**
+   * The current value of the Param.
+   */
   get value() {
     return this._value;
   }
 
+  /**
+   * Update the Param value
+   * @param {*} value The new value of the Param.
+   * @param {object} [options] The update options.
+   * @param {boolean} [options.force] A boolean flag indicating if the Param
+   *  should emit a 'value' event even if the internal value is unchanged.
+   * @returns {this} This Param instance.
+   */
   update(value, { force } = {}) {
-    const changed = distinct(this._value, value);
-    if (changed) this._value = value;
-    if (changed || force) this.emit('value', this.value);
-    return this;
-  }
-
-  addEventListener(type, callback) {
-    let list = this._listeners.get(type) || [];
-    if (!list.includes(callback)) {
-      list = list.concat(callback);
+    const shouldEmit = distinct(this._value, value) || force;
+    if (shouldEmit) {
+      this.emit('value', value);
+    } else {
+      this.cancel('value');
     }
-    this._listeners.set(type, list);
+    return this;
   }
 
-  removeEventListener(type, callback) {
-    const list = this._listeners.get(type);
-    if (list?.length) {
-      this._listeners.set(type, list.filter(x => x !== callback));
+  /**
+   * Upon value-typed updates, sets the current value to the input value
+   * immediately prior to the event value being emitted to listeners.
+   * @param {string} type The event type.
+   * @param {*} value The input event value.
+   * @returns {*} The input event value.
+   */
+  willEmit(type, value) {
+    if (type === 'value') {
+      this._value = value;
     }
-  }
-
-  emit(type, event) {
-    this._listeners.get(type)?.forEach(l => l(event));
+    return value;
   }
 }

package/src/QueryConsolidator.js

@@ -0,0 +1,238 @@
+import { Query, Ref } from '@uwdata/mosaic-sql';
+import { queryResult } from './util/query-result.js';
+
+/**
+ * Create a consolidator to combine structurally compatible queries.
+ * @param {*} enqueue Query manager enqueue method
+ * @param {*} cache Client-side query cache (sql -> data)
+ * @param {*} record Query recorder function
+ * @returns A consolidator object
+ */
+export function consolidator(enqueue, cache, record) {
+  let pending = [];
+  let id = 0;
+
+  function run() {
+    // group queries into bundles that can be consolidated
+    const groups = entryGroups(pending, cache);
+    pending = [];
+    id = 0;
+
+    // build and issue consolidated queries
+    for (const group of groups) {
+      consolidate(group, enqueue, record);
+      processResults(group, cache);
+    }
+  }
+
+  return {
+    add(entry, priority) {
+      if (entry.request.type === 'arrow') {
+        // wait one frame, gather an ordered list of queries
+        // only Apache Arrow is supported, so we can project efficiently
+        id = id || requestAnimationFrame(() => run());
+        pending.push({ entry, priority, index: pending.length });
+      } else {
+        enqueue(entry, priority);
+      }
+    }
+  }
+}
+
+/**
+ * Segment query requests into consolidation-compatible groups.
+ * @param {*} entries Query request entries ({ request, result } objects)
+ * @returns An array of grouped entry arrays
+ */
+function entryGroups(entries, cache) {
+  const groups = [];
+  const groupMap = new Map;
+
+  for (const query of entries) {
+    const { entry: { request } } = query;
+    const key = consolidationKey(request.query, cache);
+    if (!groupMap.has(key)) {
+      const list = [];
+      groups.push(list);
+      groupMap.set(key, list);
+    }
+    groupMap.get(key).push(query);
+  }
+
+  return groups;
+}
+
+/**
+ * Generate a key string for query consolidation.
+ * Queries with matching keys are conosolidation-compatible.
+ * If a query is found in the cache, it is exempted from consolidation,
+ * which is indicated by returning the precise query SQL as the key.
+ * @param {*} query The input query.
+ * @param {*} cache The query cache (sql -> data).
+ * @returns a key string
+ */
+function consolidationKey(query, cache) {
+  const sql = `${query}`;
+  if (query instanceof Query && !cache.get(sql)) {
+    if (
+      query.orderby().length || query.where().length ||
+      query.qualify().length || query.having().length
+    ) {
+      // do not try to analyze if query includes clauses
+      // that may refer to *derived* columns we can't resolve
+      return sql;
+    }
+
+    // create a derived query stripped of selections
+    const q = query.clone().$select('*');
+
+    // check group by criteria for compatibility
+    // queries may refer to *derived* columns as group by criteria
+    // we resolve these against the true grouping expressions
+    const groupby = query.groupby();
+    if (groupby.length) {
+      const map = {}; // expression map (as -> expr)
+      query.select().forEach(({ as, expr }) => map[as] = expr);
+      q.$groupby(groupby.map(e => (e instanceof Ref && map[e.column]) || e));
+    }
+
+    // key is just the transformed query as SQL
+    return `${q}`;
+  } else {
+    // can not analyze query, simply return as string
+    return sql;
+  }
+}
+
+/**
+ * Issue queries, consolidating where possible.
+ * @param {*} group Array of bundled query entries
+ * @param {*} enqueue Add entry to query queue
+ * @param {*} record Query recorder function
+ */
+function consolidate(group, enqueue, record) {
+  if (shouldConsolidate(group)) {
+    // issue a single consolidated query
+    enqueue({
+      request: {
+        type: 'arrow',
+        cache: false,
+        record: false,
+        query: consolidatedQuery(group, record)
+      },
+      result: (group.result = queryResult())
+    });
+  } else {
+    // issue queries directly
+    for (const { entry, priority } of group) {
+      enqueue(entry, priority);
+    }
+  }
+}
+
+/**
+ * Check if a group contains multiple distinct queries.
+ * @param {*} group Array of bundled query entries
+ * @returns false if group contains a single (possibly repeated) query,
+ *  otherwise true
+ */
+function shouldConsolidate(group) {
+  if (group.length > 1) {
+    const sql = `${group[0].entry.request.query}`;
+    for (let i = 1; i < group.length; ++i) {
+      if (sql !== `${group[i].entry.request.query}`) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+/**
+ * Create a consolidated query for a group.
+ * @param {*} group Array of bundled query entries
+ * @param {*} record Query recorder function
+ * @returns A consolidated Query instance
+ */
+function consolidatedQuery(group, record) {
+  const maps = group.maps = [];
+  const fields = new Map;
+
+  // gather select fields
+  for (const item of group) {
+    const { query } = item.entry.request;
+    const fieldMap = [];
+    maps.push(fieldMap);
+    for (const { as, expr } of query.select()) {
+      const e = `${expr}`;
+      if (!fields.has(e)) {
+        fields.set(e, [`col${fields.size}`, expr]);
+      }
+      const [name] = fields.get(e);
+      fieldMap.push([name, as]);
+    }
+    record(`${query}`);
+  }
+
+  // use a cloned query as a starting point
+  const query = group[0].entry.request.query.clone();
+
+  // update group by statement as needed
+  const groupby = query.groupby();
+  if (groupby.length) {
+    const map = {};
+    group.maps[0].forEach(([name, as]) => map[as] = name);
+    query.$groupby(groupby.map(e => (e instanceof Ref && map[e.column]) || e));
+  }
+
+  // update select statemenet and return
+  return query.$select(Array.from(fields.values()));
+}
+
+/**
+ * Process query results, dispatch results to original requests
+ * @param {*} group Array of query requests
+ * @param {*} cache Client-side query cache (sql -> data)
+ */
+async function processResults(group, cache) {
+  const { maps, result } = group;
+  if (!maps) return; // no consolidation performed
+
+  let data;
+  try {
+    data = await result;
+  } catch (err) {
+    // pass error to consolidated queries
+    for (const { entry } of group) {
+      entry.result.reject(err);
+    }
+    return;
+  }
+
+  group.forEach(({ entry }, index) => {
+    const { request, result } = entry;
+    const projected = projectResult(data, maps[index]);
+    if (request.cache) {
+      cache.set(String(request.query), projected);
+    }
+    result.fulfill(projected);
+  });
+}
+
+/**
+ * Project a consolidated result to a client result
+ * @param {*} data Consolidated query result, as an Apache Arrow Table
+ * @param {*} map Column name map as [source, target] pairs
+ * @returns the projected Apache Arrow table
+ */
+function projectResult(data, map) {
+  if (map) {
+    const cols = {};
+    for (const [name, as] of map) {
+      cols[as] = data.getChild(name);
+    }
+    return new data.constructor(cols);
+  } else {
+    return data;
+  }
+}

package/src/QueryManager.js

@@ -0,0 +1,133 @@
+import { consolidator } from './QueryConsolidator.js';
+import { lruCache, voidCache } from './util/cache.js';
+import { priorityQueue } from './util/priority-queue.js';
+import { queryResult } from './util/query-result.js';
+
+export const Priority = { High: 0, Normal: 1, Low: 2 };
+
+export function QueryManager() {
+  const queue = priorityQueue(3);
+  let db;
+  let clientCache;
+  let logger;
+  let recorders = [];
+  let pending = null;
+  let consolidate;
+
+  function next() {
+    if (pending || queue.isEmpty()) return;
+    const { request, result } = queue.next();
+    pending = submit(request, result);
+    pending.finally(() => { pending = null; next(); });
+  }
+
+  function enqueue(entry, priority = Priority.Normal) {
+    queue.insert(entry, priority);
+    next();
+  }
+
+  function recordQuery(sql) {
+    if (recorders.length && sql) {
+      recorders.forEach(rec => rec.add(sql));
+    }
+  }
+
+  async function submit(request, result) {
+    try {
+      const { query, type, cache = false, record = true, options } = request;
+      const sql = query ? `${query}` : null;
+
+      // update recorders
+      if (record) {
+        recordQuery(sql);
+      }
+
+      // check query cache
+      if (cache) {
+        const cached = clientCache.get(sql);
+        if (cached) {
+          logger.debug('Cache');
+          result.fulfill(cached);
+          return;
+        }
+      }
+
+      // issue query, potentially cache result
+      const t0 = performance.now();
+      const data = await db.query({ type, sql, ...options });
+      if (cache) clientCache.set(sql, data);
+      logger.debug(`Request: ${(performance.now() - t0).toFixed(1)}`);
+      result.fulfill(data);
+    } catch (err) {
+      result.reject(err);
+    }
+  }
+
+  return {
+    cache(value) {
+      return value !== undefined
+        ? (clientCache = value === true ? lruCache() : (value || voidCache()))
+        : clientCache;
+    },
+
+    logger(value) {
+      return value ? (logger = value) : logger;
+    },
+
+    connector(connector) {
+      return connector ? (db = connector) : db;
+    },
+
+    consolidate(flag) {
+      if (flag && !consolidate) {
+        consolidate = consolidator(enqueue, clientCache, recordQuery);
+      } else if (!flag && consolidate) {
+        consolidate = null;
+      }
+    },
+
+    request(request, priority = Priority.Normal) {
+      const result = queryResult();
+      const entry = { request, result };
+      if (consolidate) {
+        consolidate.add(entry, priority);
+      } else {
+        enqueue(entry, priority);
+      }
+      return result;
+    },
+
+    cancel(requests) {
+      const set = new Set(requests);
+      queue.remove(({ result }) => set.has(result));
+    },
+
+    clear() {
+      queue.remove(({ result }) => {
+        result.reject('Cleared');
+        return true;
+      });
+    },
+
+    record() {
+      let state = [];
+      const recorder = {
+        add(query) {
+          state.push(query);
+        },
+        reset() {
+          state = [];
+        },
+        snapshot() {
+          return state.slice();
+        },
+        stop() {
+          recorders = recorders.filter(x => x !== recorder);
+          return state;
+        }
+      };
+      recorders.push(recorder);
+      return recorder;
+    }
+  };
+}