@uwdata/mosaic-core 0.0.1 → 0.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uwdata/mosaic-core",
-  "version": "0.0.1",
+  "version": "0.2.0",
   "description": "Scalable and extensible linked data views.",
   "keywords": [
     "mosaic",
@@ -28,8 +28,9 @@
     "prepublishOnly": "npm run test && npm run lint && npm run build"
   },
   "dependencies": {
-    "@duckdb/duckdb-wasm": "^1.20.0",
-    "@uwdata/mosaic-sql": "^0.0.1",
-    "apache-arrow": "^11.0.0"
-  }
+    "@duckdb/duckdb-wasm": "^1.25.0",
+    "@uwdata/mosaic-sql": "^0.2.0",
+    "apache-arrow": "^12.0.0"
+  },
+  "gitHead": "e53cd914c807f99aabe78dcbe618dd9543e2f438"
 }

package/src/Catalog.js

@@ -1,17 +1,16 @@
-import { Query, count, max, min, isNull } from '@uwdata/mosaic-sql';
 import { jsType } from './util/js-type.js';
+import { summarize } from './util/summarize.js';
 
 const object = () => Object.create(null);
 
 export class Catalog {
-  constructor(mc) {
-    this.mc = mc;
+  constructor(coordinator) {
+    this.mc = coordinator;
     this.clear();
   }
 
   clear() {
     this.tables = object();
-    this.fields = object();
   }
 
   async tableInfo(table) {
@@ -21,62 +20,61 @@ export class Catalog {
     }
 
     const q = this.mc.query(
-      `PRAGMA table_info('${table}')`,
-      { type: 'json' }
+      `DESCRIBE "${table}"`,
+      { type: 'json', cache: false }
     );
 
     return (cache[table] = q.then(result => {
       const columns = object();
       for (const entry of result) {
-        columns[entry.name] = { ...entry, jstype: jsType(entry.type) };
+        columns[entry.column_name] = {
+          table,
+          column: entry.column_name,
+          sqlType: entry.column_type,
+          type: jsType(entry.column_type),
+          nullable: entry.null === 'YES'
+        };
       }
       return columns;
     }));
   }
 
-  async fieldInfo(table, column) {
-    const info = await this.tableInfo(table);
-    const colInfo = info[column];
+  async fieldInfo({ table, column, stats }) {
+    const tableInfo = await this.tableInfo(table);
+    const colInfo = tableInfo[column];
 
     // column does not exist
     if (colInfo == null) return;
 
-    const cache = this.fields;
-    const key = `${table}.${column}`;
-    if (cache[key]) {
-      return cache[key];
-    }
+    // no need for summary statistics
+    if (!stats?.length) return colInfo;
+
+    const result = await this.mc.query(
+      summarize(colInfo, stats),
+      { persist: true }
+    );
+    const info = { ...colInfo, ...(Array.from(result)[0]) };
 
-    const promise = this.mc.query(
-      Query.from(table).select({
-        rows: count(),
-        nulls: count().where(isNull(column)),
-        min: min(column),
-        max: max(column)
-      }, { cache: false })
-    ).then(result => {
-      const [ stats ] = Array.from(result);
-      return { table, column, type: colInfo.jstype, ...stats };
-    });
+    // coerce bigint to number
+    for (const key in info) {
+      const value = info[key];
+      if (typeof value === 'bigint') {
+        info[key] = Number(value);
+      }
+    }
 
-    return (cache[key] = promise);
+    return info;
   }
 
   async queryFields(fields) {
     const list = await resolveFields(this, fields);
-    const data = await Promise.all(
-      list.map(f => this.fieldInfo(f.table, f.column))
-    )
+    const data = await Promise.all(list.map(f => this.fieldInfo(f)));
     return data.filter(x => x);
   }
 }
 
 async function resolveFields(catalog, list) {
-  if (list.length === 1 && list[0].column === '*') {
-    const table = list[0].table;
-    const info = await catalog.tableInfo(table);
-    return Object.keys(info).map(column => ({ table, column }));
-  } else {
-    return list;
-  }
+  return list.length === 1 && list[0].column === '*'
+    ? Object.values(await catalog.tableInfo(list[0].table))
+    : list;
 }

package/src/Coordinator.js

@@ -1,7 +1,8 @@
-import { socketClient } from './clients/socket.js';
+import { socketConnector } from './connectors/socket.js';
 import { Catalog } from './Catalog.js';
 import { FilterGroup } from './FilterGroup.js';
-import { QueryCache } from './QueryCache.js';
+import { QueryManager, Priority } from './QueryManager.js';
+import { voidLogger } from './util/void-logger.js';
 
 let _instance;
 
@@ -15,75 +16,92 @@ export function coordinator(instance) {
 }
 
 export class Coordinator {
-  constructor(db = socketClient()) {
-    this.cache = new QueryCache();
+  constructor(db = socketConnector(), options = {}) {
     this.catalog = new Catalog(this);
-    this.indexes = true;
-    this.databaseClient(db);
+    this.manager = options.manager || QueryManager();
+    this.logger(options.logger || console);
+    this.configure(options);
+    this.databaseConnector(db);
     this.clear();
+    this._recorders = [];
+  }
+
+  logger(logger) {
+    if (arguments.length) {
+      this._logger = logger || voidLogger();
+      this.manager.logger(this._logger);
+    }
+    return this._logger;
   }
 
   configure({ cache = true, indexes = true }) {
-    this.cache = cache ? new QueryCache() : {
-      get: () => undefined,
-      set: (key, result) => result,
-      clear: () => {}
-    };
+    this.manager.cache(cache);
     this.indexes = indexes;
   }
 
   clear({ clients = true, cache = true, catalog = false } = {}) {
     if (clients) {
-      this.clients?.forEach((_, client) => this.disconnect(client));
+      this.clients?.forEach(client => this.disconnect(client));
       this.filterGroups?.forEach(group => group.finalize());
-      this.clients = new Map;
+      this.clients = new Set;
       this.filterGroups = new Map;
     }
-    if (cache) this.cache.clear();
+    if (cache) this.manager.cache().clear();
     if (catalog) this.catalog.clear();
   }
 
-  databaseClient(db) {
-    if (arguments.length > 0) {
-      this.db = db;
-    }
-    return this.db;
+  databaseConnector(db) {
+    return this.manager.connector(db);
   }
 
-  async exec(sql) {
-    try {
-      await this.db.query({ type: 'exec', sql });
-    } catch (err) {
-      console.error(err);
-    }
+  // -- Query Management ----
+
+  cancel(requests) {
+    this.manager.cancel(requests);
   }
 
-  async query(query, { type = 'arrow', cache = true } = {}) {
-    const sql = String(query);
-    const cached = this.cache.get(sql);
-    if (cached) {
-      return cached;
-    } else {
-      const request = this.db.query({ type, sql });
-      return cache ? this.cache.set(sql, request) : request;
-    }
+  exec(query, { priority = Priority.Normal } = {}) {
+    return this.manager.request({ type: 'exec', query }, priority);
   }
 
-  async updateClient(client, query) {
-    let result;
-    try {
-      client.queryPending();
-      result = await this.query(query);
-    } catch (err) {
-      console.error(err);
-      client.queryError(err);
-      return;
-    }
-    try {
-      client.queryResult(result).update();
-    } catch (err) {
-      console.error(err);
-    }
+  query(query, {
+    type = 'arrow',
+    cache = true,
+    priority = Priority.Normal,
+    ...options
+  } = {}) {
+    return this.manager.request({ type, query, cache, options }, priority);
+  }
+
+  prefetch(query, options = {}) {
+    return this.query(query, { ...options, cache: true, priority: Priority.Low });
+  }
+
+  createBundle(name, queries, priority = Priority.Low) {
+    const options = { name, queries };
+    return this.manager.request({ type: 'create-bundle', options }, priority);
+  }
+
+  loadBundle(name, priority = Priority.High) {
+    const options = { name };
+    return this.manager.request({ type: 'load-bundle', options }, priority);
+  }
+
+  // -- Client Management ----
+
+  updateClient(client, query, priority = Priority.Normal) {
+    client.queryPending();
+    return this.query(query, { priority }).then(
+      data => client.queryResult(data).update(),
+      err => { client.queryError(err); this._logger.error(err); }
+    );
+  }
+
+  requestQuery(client, query) {
+    this.filterGroups.get(client.filterBy)?.reset();
+    return query
+      ? this.updateClient(client, query)
+      : client.update();
   }
 
   async connect(client) {
@@ -92,12 +110,12 @@ export class Coordinator {
     if (clients.has(client)) {
       throw new Error('Client already connected.');
     }
-    clients.set(client, null); // mark as connected
+    clients.add(client); // mark as connected
 
     // retrieve field statistics
     const fields = client.fields();
     if (fields?.length) {
-      client.fieldStats(await catalog.queryFields(fields));
+      client.fieldInfo(await catalog.queryFields(fields));
     }
 
     // connect filters
@@ -111,27 +129,13 @@ export class Coordinator {
       }
     }
 
-    // query handler
-    const handler = async (query) => {
-      const q = query || client.query(filter?.predicate(client));
-      filterGroups.get(filter)?.reset();
-      if (q) this.updateClient(client, q);
-    };
-    clients.set(client, handler);
-
-    // register request handler, if defined
-    client.request?.addEventListener('value', handler);
-
-    // TODO analyze / consolidate queries?
-    handler();
+    client.requestQuery();
   }
 
   disconnect(client) {
     const { clients, filterGroups } = this;
     if (!clients.has(client)) return;
-    const handler = clients.get(client);
     clients.delete(client);
     filterGroups.get(client.filterBy)?.remove(client);
-    client.request?.removeEventListener(handler);
   }
 }

package/src/DataTileIndexer.js

@@ -1,6 +1,5 @@
-import { Query, expr, and, isBetween, asColumn, epoch_ms } from '@uwdata/mosaic-sql';
+import { Query, and, asColumn, epoch_ms, isBetween, sql } from '@uwdata/mosaic-sql';
 import { fnv_hash } from './util/hash.js';
-import { skipClient } from './util/skip-client.js';
 
 const identity = x => x;
 
@@ -30,34 +29,43 @@ export class DataTileIndexer {
     this.activeView = null;
   }
 
+  clear() {
+    if (this.indices) {
+      this.mc.cancel(Array.from(this.indices.values(), index => index.result));
+      this.indices = null;
+    }
+  }
+
   index(clients, active) {
     if (this.clients !== clients) {
       // test client views for compatibility
-      const cols = Array.from(clients).map(getIndexColumns);
+      const cols = Array.from(clients, getIndexColumns);
       const from = cols[0]?.from;
       this.enabled = cols.every(c => c && c.from === from);
       this.clients = clients;
-      this.indices = null;
       this.activeView = null;
+      this.clear();
     }
     if (!this.enabled) return false; // client views are not indexable
 
     active = active || this.selection.active;
     const { source } = active;
+    if (source && source === this.activeView?.source) return true; // we're good!
+
+    this.clear();
     if (!source) return false; // nothing to work with
-    if (source === this.activeView?.source) return true; // we're good!
     const activeView = this.activeView = getActiveView(active);
     if (!activeView) return false; // active selection clause not compatible
 
-    console.warn('DATA TILE INDEX CONSTRUCTION');
+    this.mc.logger().warn('DATA TILE INDEX CONSTRUCTION');
 
-    // create a selection with the active client removed
-    const sel = this.selection.clone().update({ source });
+    // create a selection with the active source removed
+    const sel = this.selection.remove(source);
 
     // generate data tile indices
     const indices = this.indices = new Map;
     for (const client of clients) {
-      if (sel.cross && skipClient(client, active)) continue;
+      if (sel.skip(client, active)) continue;
       const index = getIndexColumns(client);
 
       // build index construction query
@@ -75,11 +83,9 @@ export class DataTileIndexer {
       const sql = query.toString();
       const id = (fnv_hash(sql) >>> 0).toString(16);
       const table = `tile_index_${id}`;
-      indices.set(client, { table, ...index });
-      createIndex(this.mc, table, sql);
+      const result = createIndex(this.mc, table, sql);
+      indices.set(client, { table, result, ...index });
     }
-
-    return true;
   }
 
   async update() {
@@ -99,12 +105,12 @@ export class DataTileIndexer {
     }
 
     const { table, dims, aggr } = index;
-    return this.mc.updateClient(client, Query
+    const query = Query
       .select(dims, aggr)
       .from(table)
       .groupby(dims)
-      .where(filter)
-    );
+      .where(filter);
+    return this.mc.updateClient(client, query);
   }
 }
 
@@ -112,22 +118,22 @@ function getActiveView(clause) {
   const { source, schema } = clause;
   let columns = clause.predicate?.columns;
   if (!schema || !columns) return null;
-  const { type, scales } = schema;
+  const { type, scales, pixelSize = 1 } = schema;
   let predicate;
 
   if (type === 'interval' && scales) {
-    const bins = scales.map(s => binInterval(s));
+    const bins = scales.map(s => binInterval(s, pixelSize));
     if (bins.some(b => b == null)) return null; // unsupported scale type
 
     if (bins.length === 1) {
-      predicate = p => p ? isBetween('active0', p.value.map(bins[0])) : [];
-      columns = { active0: bins[0](clause.predicate.expr) };
+      predicate = p => p ? isBetween('active0', p.range.map(bins[0])) : [];
+      columns = { active0: bins[0](clause.predicate.field) };
     } else {
       predicate = p => p
-        ? and(p.value.map(({ value }, i) => isBetween(`active${i}`, value.map(bins[i]))))
+        ? and(p.children.map(({ range }, i) => isBetween(`active${i}`, range.map(bins[i]))))
         : [];
       columns = Object.fromEntries(
-        clause.predicate.value.map((p, i) => [`active${i}`, bins[i](p.expr)])
+        clause.predicate.children.map((p, i) => [`active${i}`, bins[i](p.field)])
       );
     }
   } else if (type === 'point') {
@@ -140,82 +146,77 @@ function getActiveView(clause) {
   return { source, columns, predicate };
 }
 
-function binInterval(scale) {
+function binInterval(scale, pixelSize) {
   const { type, domain, range } = scale;
-  let lift, sql;
+  let lift, toSql;
 
   switch (type) {
     case 'linear':
       lift = identity;
-      sql = asColumn;
+      toSql = asColumn;
       break;
     case 'log':
       lift = Math.log;
-      sql = c => `LN(${asColumn(c)})`;
+      toSql = c => sql`LN(${asColumn(c)})`;
       break;
     case 'symlog':
       // TODO: support log constants other than 1?
       lift = x => Math.sign(x) * Math.log1p(Math.abs(x));
-      sql = c => (c = asColumn(c), `SIGN(${c}) * LN(1 + ABS(${c}))`);
+      toSql = c => (c = asColumn(c), sql`SIGN(${c}) * LN(1 + ABS(${c}))`);
       break;
     case 'sqrt':
       lift = Math.sqrt;
-      sql = c => `SQRT(${asColumn(c)})`;
+      toSql = c => sql`SQRT(${asColumn(c)})`;
       break;
     case 'utc':
     case 'time':
       lift = x => +x;
-      sql = c => c instanceof Date ? +c : epoch_ms(asColumn(c));
+      toSql = c => c instanceof Date ? +c : epoch_ms(asColumn(c));
       break;
   }
-  return lift ? binFunction(domain, range, lift, sql) : null;
+  return lift ? binFunction(domain, range, pixelSize, lift, toSql) : null;
 }
 
-function binFunction(domain, range, lift, sql) {
+function binFunction(domain, range, pixelSize, lift, toSql) {
   const lo = lift(Math.min(domain[0], domain[1]));
   const hi = lift(Math.max(domain[0], domain[1]));
-  const a = Math.abs(lift(range[1]) - lift(range[0])) / (hi - lo);
-  return value => expr(
-    `FLOOR(${a}::DOUBLE * (${sql(value)} - ${lo}::DOUBLE))`,
-    asColumn(value).columns
-  );
+  const a = (Math.abs(lift(range[1]) - lift(range[0])) / (hi - lo)) / pixelSize;
+  const s = pixelSize === 1 ? '' : `${pixelSize}::INTEGER * `;
+  return value => sql`${s}FLOOR(${a}::DOUBLE * (${toSql(value)} - ${lo}::DOUBLE))::INTEGER`;
 }
 
-async function createIndex(mc, table, query) {
-  try {
-    await mc.exec(`CREATE TEMP TABLE IF NOT EXISTS ${table} AS ${query}`);
-  } catch (err) {
-    console.error(err);
-  }
+function createIndex(mc, table, query) {
+  return mc.exec(`CREATE TEMP TABLE IF NOT EXISTS ${table} AS ${query}`);
 }
 
+const NO_INDEX = { from: NaN };
+
 function getIndexColumns(client) {
+  if (!client.filterIndexable) return NO_INDEX;
   const q = client.query();
   const from = getBaseTable(q);
-  if (!from || !q.groupby || !client.filterIndexable) {
-    return { from: NaN }; // early exit
-  }
+  if (!from || !q.groupby) return NO_INDEX;
   const g = new Set(q.groupby().map(c => c.column));
 
-  let aggr = [];
-  let dims = [];
+  const aggr = [];
+  const dims = [];
   let count;
 
   for (const { as, expr: { aggregate } } of q.select()) {
     switch (aggregate?.toUpperCase()) {
       case 'COUNT':
       case 'SUM':
-        aggr.push({ [as]: expr(`SUM("${as}")::DOUBLE`) });
+        aggr.push({ [as]: sql`SUM("${as}")::DOUBLE` });
         break;
       case 'AVG':
         count = '_count_';
-        aggr.push({ [as]: expr(`(SUM("${as}" * ${count}) / SUM(${count}))::DOUBLE`) });
+        aggr.push({ [as]: sql`(SUM("${as}" * ${count}) / SUM(${count}))::DOUBLE` });
         break;
       case 'MAX':
-        aggr.push({ [as]: expr(`MAX("${as}")`) });
+        aggr.push({ [as]: sql`MAX("${as}")` });
         break;
       case 'MIN':
-        aggr.push({ [as]: expr(`MIN("${as}")`) });
+        aggr.push({ [as]: sql`MIN("${as}")` });
         break;
       default:
         if (g.has(as)) dims.push(as);
@@ -226,7 +227,7 @@ function getIndexColumns(client) {
   return {
     aggr,
     dims,
-    count: count ? { [count]: expr('COUNT(*)') } : {},
+    count: count ? { [count]: sql`COUNT(*)` } : {},
     from
   };
 }
@@ -242,7 +243,7 @@ function getBaseTable(query) {
   }
 
   // handle set operations / subqueries
-  let base = getBaseTable(subq[0]);
+  const base = getBaseTable(subq[0]);
   for (let i = 1; i < subq.length; ++i) {
     const from = getBaseTable(subq[i]);
     if (from === undefined) continue;

package/src/FilterGroup.js

@@ -1,18 +1,15 @@
 import { DataTileIndexer } from './DataTileIndexer.js';
-import { throttle } from './util/throttle.js';
 
 export class FilterGroup {
-  constructor(mc, selection, index = true) {
-    this.mc = mc;
+  constructor(coordinator, selection, index = true) {
+    this.mc = coordinator;
     this.selection = selection;
     this.clients = new Set();
-    this.indexer = index ? new DataTileIndexer(mc, selection) : null;
+    this.indexer = index ? new DataTileIndexer(this.mc, selection) : null;
 
     const { value, activate } = this.handlers = {
-      value: throttle(() => this.update()),
-      activate: clause => {
-        this.indexer?.index(this.clients, clause);
-      }
+      value: () => this.update(),
+      activate: clause => this.indexer?.index(this.clients, clause)
     };
     selection.addEventListener('value', value);
     selection.addEventListener('activate', activate);
@@ -40,7 +37,7 @@ export class FilterGroup {
     return this;
   }
 
-  async update() {
+  update() {
     const { mc, indexer, clients, selection } = this;
     return indexer?.index(clients)
       ? indexer.update()