@uwdata/mosaic-core 0.0.1 → 0.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uwdata/mosaic-core",
-  "version": "0.0.1",
+  "version": "0.1.0",
   "description": "Scalable and extensible linked data views.",
   "keywords": [
     "mosaic",
@@ -29,7 +29,8 @@
   },
   "dependencies": {
     "@duckdb/duckdb-wasm": "^1.20.0",
-    "@uwdata/mosaic-sql": "^0.0.1",
+    "@uwdata/mosaic-sql": "^0.1.0",
     "apache-arrow": "^11.0.0"
-  }
+  },
+  "gitHead": "a7967c35349bdf7f00abb113ce1dd9abb233cd62"
 }

package/src/Catalog.js

@@ -1,17 +1,16 @@
-import { Query, count, max, min, isNull } from '@uwdata/mosaic-sql';
 import { jsType } from './util/js-type.js';
+import { summarize } from './util/summarize.js';
 
 const object = () => Object.create(null);
 
 export class Catalog {
-  constructor(mc) {
-    this.mc = mc;
+  constructor(coordinator) {
+    this.mc = coordinator;
     this.clear();
   }
 
   clear() {
     this.tables = object();
-    this.fields = object();
   }
 
   async tableInfo(table) {
@@ -21,62 +20,49 @@ export class Catalog {
     }
 
     const q = this.mc.query(
-      `PRAGMA table_info('${table}')`,
-      { type: 'json' }
+      `DESCRIBE "${table}"`,
+      { type: 'json', cache: false }
     );
 
     return (cache[table] = q.then(result => {
       const columns = object();
       for (const entry of result) {
-        columns[entry.name] = { ...entry, jstype: jsType(entry.type) };
+        columns[entry.column_name] = {
+          table,
+          column: entry.column_name,
+          sqlType: entry.column_type,
+          type: jsType(entry.column_type),
+          nullable: entry.null === 'YES'
+        };
       }
       return columns;
     }));
   }
 
-  async fieldInfo(table, column) {
-    const info = await this.tableInfo(table);
-    const colInfo = info[column];
+  async fieldInfo({ table, column, stats }) {
+    const tableInfo = await this.tableInfo(table);
+    const colInfo = tableInfo[column];
 
     // column does not exist
     if (colInfo == null) return;
 
-    const cache = this.fields;
-    const key = `${table}.${column}`;
-    if (cache[key]) {
-      return cache[key];
-    }
-
-    const promise = this.mc.query(
-      Query.from(table).select({
-        rows: count(),
-        nulls: count().where(isNull(column)),
-        min: min(column),
-        max: max(column)
-      }, { cache: false })
-    ).then(result => {
-      const [ stats ] = Array.from(result);
-      return { table, column, type: colInfo.jstype, ...stats };
-    });
+    // no need for summary statistics
+    if (!stats?.length) return colInfo;
 
-    return (cache[key] = promise);
+    const result = await this.mc.query(summarize(colInfo, stats));
+    const info = { ...colInfo, ...(Array.from(result)[0]) };
+    return info;
   }
 
   async queryFields(fields) {
     const list = await resolveFields(this, fields);
-    const data = await Promise.all(
-      list.map(f => this.fieldInfo(f.table, f.column))
-    )
+    const data = await Promise.all(list.map(f => this.fieldInfo(f)));
     return data.filter(x => x);
   }
 }
 
 async function resolveFields(catalog, list) {
-  if (list.length === 1 && list[0].column === '*') {
-    const table = list[0].table;
-    const info = await catalog.tableInfo(table);
-    return Object.keys(info).map(column => ({ table, column }));
-  } else {
-    return list;
-  }
+  return list.length === 1 && list[0].column === '*'
+    ? Object.values(await catalog.tableInfo(list[0].table))
+    : list;
 }

package/src/Coordinator.js

@@ -1,7 +1,8 @@
 import { socketClient } from './clients/socket.js';
 import { Catalog } from './Catalog.js';
 import { FilterGroup } from './FilterGroup.js';
-import { QueryCache } from './QueryCache.js';
+import { QueryCache, voidCache } from './QueryCache.js';
+import { voidLogger } from './util/void-logger.js';
 
 let _instance;
 
@@ -15,28 +16,30 @@ export function coordinator(instance) {
 }
 
 export class Coordinator {
-  constructor(db = socketClient()) {
-    this.cache = new QueryCache();
+  constructor(db = socketClient(), options = {}) {
     this.catalog = new Catalog(this);
-    this.indexes = true;
+    this.logger(options.logger || console);
+    this.configure(options);
     this.databaseClient(db);
     this.clear();
   }
 
+  logger(logger) {
+    return arguments.length
+      ? (this._logger = logger || voidLogger())
+      : this._logger;
+  }
+
   configure({ cache = true, indexes = true }) {
-    this.cache = cache ? new QueryCache() : {
-      get: () => undefined,
-      set: (key, result) => result,
-      clear: () => {}
-    };
+    this.cache = cache ? new QueryCache() : voidCache();
     this.indexes = indexes;
   }
 
   clear({ clients = true, cache = true, catalog = false } = {}) {
     if (clients) {
-      this.clients?.forEach((_, client) => this.disconnect(client));
+      this.clients?.forEach(client => this.disconnect(client));
       this.filterGroups?.forEach(group => group.finalize());
-      this.clients = new Map;
+      this.clients = new Set;
       this.filterGroups = new Map;
     }
     if (cache) this.cache.clear();
@@ -54,18 +57,22 @@ export class Coordinator {
     try {
       await this.db.query({ type: 'exec', sql });
     } catch (err) {
-      console.error(err);
+      this._logger.error(err);
     }
   }
 
-  async query(query, { type = 'arrow', cache = true } = {}) {
+  query(query, { type = 'arrow', cache = true } = {}) {
     const sql = String(query);
+    const t0 = performance.now();
     const cached = this.cache.get(sql);
     if (cached) {
+      this._logger.debug('Cache');
       return cached;
     } else {
       const request = this.db.query({ type, sql });
-      return cache ? this.cache.set(sql, request) : request;
+      const result = cache ? this.cache.set(sql, request) : request;
+      result.then(() => this._logger.debug(`Query: ${performance.now() - t0}`));
+      return result;
     }
   }
 
@@ -75,24 +82,31 @@ export class Coordinator {
       client.queryPending();
       result = await this.query(query);
     } catch (err) {
-      console.error(err);
+      this._logger.error(err);
       client.queryError(err);
       return;
     }
     try {
       client.queryResult(result).update();
     } catch (err) {
-      console.error(err);
+      this._logger.error(err);
     }
   }
 
+  async requestQuery(client, query) {
+    this.filterGroups.get(client.filterBy)?.reset();
+    return query
+      ? this.updateClient(client, query)
+      : client.update();
+  }
+
   async connect(client) {
     const { catalog, clients, filterGroups, indexes } = this;
 
     if (clients.has(client)) {
       throw new Error('Client already connected.');
     }
-    clients.set(client, null); // mark as connected
+    clients.add(client); // mark as connected
 
     // retrieve field statistics
     const fields = client.fields();
@@ -111,27 +125,13 @@ export class Coordinator {
       }
     }
 
-    // query handler
-    const handler = async (query) => {
-      const q = query || client.query(filter?.predicate(client));
-      filterGroups.get(filter)?.reset();
-      if (q) this.updateClient(client, q);
-    };
-    clients.set(client, handler);
-
-    // register request handler, if defined
-    client.request?.addEventListener('value', handler);
-
-    // TODO analyze / consolidate queries?
-    handler();
+    client.requestQuery();
   }
 
   disconnect(client) {
     const { clients, filterGroups } = this;
     if (!clients.has(client)) return;
-    const handler = clients.get(client);
     clients.delete(client);
     filterGroups.get(client.filterBy)?.remove(client);
-    client.request?.removeEventListener(handler);
   }
 }

package/src/DataTileIndexer.js

@@ -49,13 +49,14 @@ export class DataTileIndexer {
     const activeView = this.activeView = getActiveView(active);
     if (!activeView) return false; // active selection clause not compatible
 
-    console.warn('DATA TILE INDEX CONSTRUCTION');
+    this.mc.logger().warn('DATA TILE INDEX CONSTRUCTION');
 
     // create a selection with the active client removed
     const sel = this.selection.clone().update({ source });
 
     // generate data tile indices
     const indices = this.indices = new Map;
+    const promises = [];
     for (const client of clients) {
       if (sel.cross && skipClient(client, active)) continue;
       const index = getIndexColumns(client);
@@ -76,10 +77,10 @@ export class DataTileIndexer {
       const id = (fnv_hash(sql) >>> 0).toString(16);
       const table = `tile_index_${id}`;
       indices.set(client, { table, ...index });
-      createIndex(this.mc, table, sql);
+      promises.push(createIndex(this.mc, table, sql));
     }
 
-    return true;
+    return promises;
   }
 
   async update() {
@@ -185,16 +186,17 @@ async function createIndex(mc, table, query) {
   try {
     await mc.exec(`CREATE TEMP TABLE IF NOT EXISTS ${table} AS ${query}`);
   } catch (err) {
-    console.error(err);
+    mc.logger().error(err);
   }
 }
 
+const NO_INDEX = { from: NaN };
+
 function getIndexColumns(client) {
+  if (!client.filterIndexable) return NO_INDEX;
   const q = client.query();
   const from = getBaseTable(q);
-  if (!from || !q.groupby || !client.filterIndexable) {
-    return { from: NaN }; // early exit
-  }
+  if (!from || !q.groupby) return NO_INDEX;
   const g = new Set(q.groupby().map(c => c.column));
 
   let aggr = [];

package/src/FilterGroup.js

@@ -2,11 +2,11 @@ import { DataTileIndexer } from './DataTileIndexer.js';
 import { throttle } from './util/throttle.js';
 
 export class FilterGroup {
-  constructor(mc, selection, index = true) {
-    this.mc = mc;
+  constructor(coordinator, selection, index = true) {
+    this.mc = coordinator;
     this.selection = selection;
     this.clients = new Set();
-    this.indexer = index ? new DataTileIndexer(mc, selection) : null;
+    this.indexer = index ? new DataTileIndexer(this.mc, selection) : null;
 
     const { value, activate } = this.handlers = {
       value: throttle(() => this.update()),

package/src/MosaicClient.js

@@ -1,41 +1,105 @@
+import { coordinator } from './Coordinator.js';
+import { throttle } from './util/throttle.js';
+
+/**
+ * Base class for Mosaic clients.
+ */
 export class MosaicClient {
+  /**
+   * Constructor.
+   * @param {*} filterSelection An optional selection to interactively filter
+   *  this client's data. If provided, a coordinator will re-query and update
+   *  the client when the selection updates.
+   */
   constructor(filterSelection) {
     this._filterBy = filterSelection;
+    this._requestUpdate = throttle(() => this.requestQuery(), true);
   }
 
+  /**
+   * Return this client's filter selection.
+   */
   get filterBy() {
     return this._filterBy;
   }
 
+  /**
+   * Return a boolean indicating if the client query can be indexed. Should
+   * return true if changes to the filterBy selection does not change the
+   * groupby domain of the client query.
+   */
   get filterIndexable() {
     return true;
   }
 
+  /**
+   * Return an array of fields queried by this client.
+   */
   fields() {
     return null;
   }
 
+  /**
+   * Called by the coordinator to set the field statistics for this client.
+   * @returns {this}
+   */
   fieldStats() {
     return this;
   }
 
+  /**
+   * Return a query specifying the data needed by this client.
+   */
   query() {
     return null;
   }
 
+  /**
+   * Called by the coordinator to inform the client that a query is pending.
+   */
   queryPending() {
     return this;
   }
 
+  /**
+   * Called by the coordinator to return a query result.
+   */
   queryResult() {
     return this;
   }
 
+  /**
+   * Called by the coordinator to report a query execution error.
+   */
   queryError(error) {
     console.error(error);
     return this;
   }
 
+  /**
+   * Request the coordinator to execute a query for this client.
+   * If an explicit query is not provided, the client query method will
+   * be called, filtered by the current filterBy selection.
+   */
+  requestQuery(query) {
+    const q = query || this.query(this.filterBy?.predicate(this));
+    return coordinator().requestQuery(this, q);
+  }
+
+  /**
+   * Request that the coordinator perform a throttled update of this client
+   * using the default query. Unlike requestQuery, for which every call will
+   * result in an executed query, multiple calls to requestUpdate may be
+   * consolidated into a single update.
+   */
+  requestUpdate() {
+    this._requestUpdate();
+  }
+
+  /**
+   * Requests a client update.
+   * For example to (re-)render an interface component.
+   */
   update() {
     return this;
   }

package/src/{Signal.js → Param.js}

@@ -1,19 +1,25 @@
-export function isSignal(x) {
-  return x instanceof Signal;
+import { distinct } from './util/distinct.js';
+
+export function isParam(x) {
+  return x instanceof Param;
 }
 
-export class Signal {
+export class Param {
   constructor(value) {
     this._value = value;
     this._listeners = new Map;
   }
 
+  static value(value) {
+    return new Param(value);
+  }
+
   get value() {
     return this._value;
   }
 
   update(value, { force } = {}) {
-    const changed = this._value !== value;
+    const changed = distinct(this._value, value);
     if (changed) this._value = value;
     if (changed || force) this.emit('value', this.value);
     return this;
@@ -21,7 +27,7 @@ export class Signal {
 
   addEventListener(type, callback) {
     let list = this._listeners.get(type) || [];
-    if (list.indexOf(callback) < 0) {
+    if (!list.includes(callback)) {
       list = list.concat(callback);
     }
     this._listeners.set(type, list);

package/src/QueryCache.js

@@ -1,3 +1,13 @@
+const requestIdle = typeof requestIdleCallback !== 'undefined'
+  ? requestIdleCallback
+  : setTimeout;
+
+export const voidCache = () => ({
+  get: () => undefined,
+  set: (key, result) => result,
+  clear: () => {}
+});
+
 export class QueryCache {
   constructor({
     max = 1000, // max entries
@@ -22,19 +32,9 @@ export class QueryCache {
 
   set(key, promise) {
     const { cache, max } = this;
-    const now = performance.now();
-
-    const receive = promise.then(result => {
-      console.log(`Query: ${Math.round(performance.now() - now)}`);
-      return result;
-    });
-
-    cache.set(key, { last: now, promise });
-    if (cache.size > max) {
-      setTimeout(() => this.evict());
-    }
-
-    return receive;
+    cache.set(key, { last: performance.now(), promise });
+    if (cache.size > max) requestIdle(() => this.evict());
+    return promise;
   }
 
   evict() {

package/src/Selection.js

@@ -1,12 +1,12 @@
 import { or } from '@uwdata/mosaic-sql';
-import { Signal } from './Signal.js';
+import { Param } from './Param.js';
 import { skipClient } from './util/skip-client.js';
 
 export function isSelection(x) {
   return x instanceof Selection;
 }
 
-export class Selection extends Signal {
+export class Selection extends Param {
 
   static intersect() {
     return new Selection();

package/src/index.js

@@ -1,7 +1,8 @@
 export { MosaicClient } from './MosaicClient.js';
 export { Coordinator, coordinator } from './Coordinator.js';
 export { Selection, isSelection } from './Selection.js';
-export { Signal, isSignal } from './Signal.js';
+export { Param, isParam } from './Param.js';
+export { distinct } from './util/distinct.js';
 export { sqlFrom } from './util/sql-from.js';
 export { throttle } from './util/throttle.js';
 export { restClient } from './clients/rest.js';

package/src/util/distinct.js

@@ -0,0 +1,14 @@
+export function distinct(a, b) {
+  return a === b ? false
+    : a instanceof Date && b instanceof Date ? +a !== +b
+    : Array.isArray(a) && Array.isArray(b) ? distinctArray(a, b)
+    : true;
+}
+
+export function distinctArray(a, b) {
+  if (a.length !== b.length) return true;
+  for (let i = 0; i < a.length; ++i) {
+    if (a[i] !== b[i]) return true;
+  }
+  return false;
+}

package/src/util/summarize.js

@@ -0,0 +1,23 @@
+import { Query, count, isNull, max, min } from '@uwdata/mosaic-sql';
+
+export const Count = 'count';
+export const Nulls = 'nulls';
+export const Max = 'max';
+export const Min = 'min';
+export const Distinct = 'distinct';
+
+export const Stats = { Count, Nulls, Max, Min, Distinct };
+
+export const statMap = {
+  [Count]: count,
+  [Distinct]: column => count(column).distinct(),
+  [Max]: max,
+  [Min]: min,
+  [Nulls]: column => count().where(isNull(column))
+};
+
+export function summarize({ table, column }, stats) {
+  return Query
+    .from(table)
+    .select(stats.map(s => [s, statMap[s](column)]));
+}

package/src/util/throttle.js

@@ -1,11 +1,14 @@
-export function throttle(callback) {
+const NIL = {};
+
+export function throttle(callback, debounce = false) {
   let curr;
   let next;
+  let pending = NIL;
 
   function invoke(event) {
     curr = callback(event).then(() => {
       if (next) {
-        const value = next;
+        const { value } = next;
         next = null;
         invoke(value);
       } else {
@@ -15,8 +18,23 @@ export function throttle(callback) {
   }
 
   function enqueue(event) {
-    next = event;
+    next = { event };
+  }
+
+  function process(event) {
+    curr ? enqueue(event) : invoke(event);
+  }
+
+  function delay(event) {
+    if (pending !== event) {
+      requestAnimationFrame(() => {
+        const e = pending;
+        pending = NIL;
+        process(e);
+      });
+    }
+    pending = event;
   }
 
-  return event => curr ? enqueue(event) : invoke(event);
+  return debounce ? delay : process;
 }

package/src/util/void-logger.js

@@ -0,0 +1,9 @@
+export function voidLogger() {
+  return {
+    debug() {},
+    info() {},
+    log() {},
+    warn() {},
+    error() {}
+  };
+}