@uwdata/mosaic-core 0.6.1 → 0.7.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uwdata/mosaic-core",
-  "version": "0.6.1",
+  "version": "0.7.0",
   "description": "Scalable and extensible linked data views.",
   "keywords": [
     "mosaic",
@@ -28,9 +28,9 @@
     "prepublishOnly": "npm run test && npm run lint && npm run build"
   },
   "dependencies": {
-    "@duckdb/duckdb-wasm": "^1.28.1-dev106.0",
-    "@uwdata/mosaic-sql": "^0.6.0",
+    "@duckdb/duckdb-wasm": "^1.28.1-dev109.0",
+    "@uwdata/mosaic-sql": "^0.7.0",
     "apache-arrow": "^15.0.0"
   },
-  "gitHead": "9e788e6dc5241fa1c54967a25fd9599f97da1a41"
+  "gitHead": "4680b922f15579b7b527f31507ed71a12230ec35"
 }

package/src/Coordinator.js

@@ -1,7 +1,7 @@
 import { socketConnector } from './connectors/socket.js';
-import { Catalog } from './Catalog.js';
 import { FilterGroup } from './FilterGroup.js';
 import { QueryManager, Priority } from './QueryManager.js';
+import { queryFieldInfo } from './util/field-info.js';
 import { voidLogger } from './util/void-logger.js';
 
 let _instance;
@@ -27,7 +27,6 @@ export class Coordinator {
       logger = console,
       manager = QueryManager()
     } = options;
-    this.catalog = new Catalog(this);
     this.manager = manager;
     this.logger(logger);
     this.configure(options);
@@ -49,7 +48,7 @@ export class Coordinator {
     this.indexes = indexes;
   }
 
-  clear({ clients = true, cache = true, catalog = false } = {}) {
+  clear({ clients = true, cache = true } = {}) {
     this.manager.clear();
     if (clients) {
       this.clients?.forEach(client => this.disconnect(client));
@@ -58,7 +57,6 @@ export class Coordinator {
       this.filterGroups = new Map;
     }
     if (cache) this.manager.cache().clear();
-    if (catalog) this.catalog.clear();
   }
 
   databaseConnector(db) {
@@ -122,7 +120,7 @@ export class Coordinator {
    * @param {import('./MosaicClient.js').MosaicClient} client the client to disconnect
    */
   async connect(client) {
-    const { catalog, clients, filterGroups, indexes } = this;
+    const { clients, filterGroups, indexes } = this;
 
     if (clients.has(client)) {
       throw new Error('Client already connected.');
@@ -133,7 +131,7 @@ export class Coordinator {
     // retrieve field statistics
     const fields = client.fields();
     if (fields?.length) {
-      client.fieldInfo(await catalog.queryFields(fields));
+      client.fieldInfo(await queryFieldInfo(this, fields));
     }
 
     // connect filters

package/src/QueryConsolidator.js

@@ -1,4 +1,4 @@
-import { Query, Ref } from '@uwdata/mosaic-sql';
+import { Query, Ref, isDescribeQuery } from '@uwdata/mosaic-sql';
 import { queryResult } from './util/query-result.js';
 
 function wait(callback) {
@@ -125,7 +125,7 @@ function consolidate(group, enqueue, record) {
         type: 'arrow',
         cache: false,
         record: false,
-        query: consolidatedQuery(group, record)
+        query: (group.query = consolidatedQuery(group, record))
       },
       result: (group.result = queryResult())
     });
@@ -192,7 +192,7 @@ function consolidatedQuery(group, record) {
     query.$groupby(groupby.map(e => (e instanceof Ref && map[e.column]) || e));
   }
 
-  // update select statemenet and return
+  // update select statement and return
   return query.$select(Array.from(fields.values()));
 }
 
@@ -202,9 +202,13 @@ function consolidatedQuery(group, record) {
  * @param {*} cache Client-side query cache (sql -> data)
  */
 async function processResults(group, cache) {
-  const { maps, result } = group;
-  if (!maps) return; // no consolidation performed
+  const { maps, query, result } = group;
 
+  // exit early if no consolidation performed
+  // in this case results are passed directly
+  if (!maps) return;
+
+  // await consolidated query result, pass errors if needed
   let data;
   try {
     data = await result;
@@ -216,13 +220,19 @@ async function processResults(group, cache) {
     return;
   }
 
+  // extract result for each query in the consolidation group
+  // update cache and pass extract to original issuer
+  const describe = isDescribeQuery(query);
   group.forEach(({ entry }, index) => {
     const { request, result } = entry;
-    const projected = projectResult(data, maps[index]);
+    const map = maps[index];
+    const extract = describe && map ? filterResult(data, map)
+      : map ? projectResult(data, map)
+      : data;
     if (request.cache) {
-      cache.set(String(request.query), projected);
+      cache.set(String(request.query), extract);
     }
-    result.fulfill(projected);
+    result.fulfill(extract);
   });
 }
 
@@ -233,13 +243,26 @@ async function processResults(group, cache) {
  * @returns the projected Apache Arrow table
  */
 function projectResult(data, map) {
-  if (map) {
-    const cols = {};
-    for (const [name, as] of map) {
-      cols[as] = data.getChild(name);
+  const cols = {};
+  for (const [name, as] of map) {
+    cols[as] = data.getChild(name);
+  }
+  return new data.constructor(cols);
+}
+
+/**
+ * Filter a consolidated describe query result to a client result
+ * @param {*} data Consolidated query result
+ * @param {*} map Column name map as [source, target] pairs
+ * @returns the filtered table data
+ */
+function filterResult(data, map) {
+  const lookup = new Map(map);
+  const result = [];
+  for (const d of data) {
+    if (lookup.has(d.column_name)) {
+      result.push({ ...d, column_name: lookup.get(d.column_name) })
     }
-    return new data.constructor(cols);
-  } else {
-    return data;
   }
+  return result;
 }

package/src/index.js

@@ -8,6 +8,12 @@ export { restConnector } from './connectors/rest.js';
 export { socketConnector } from './connectors/socket.js';
 export { wasmConnector } from './connectors/wasm.js';
 
+export {
+  isArrowTable,
+  convertArrowArrayType,
+  convertArrowValue,
+  convertArrowColumn
+} from './util/convert-arrow.js'
 export { distinct } from './util/distinct.js';
 export { synchronizer } from './util/synchronizer.js';
 export { throttle } from './util/throttle.js';

package/src/util/convert-arrow.js

@@ -0,0 +1,148 @@
+// arrow type ids
+const INTEGER = 2;
+const FLOAT = 3;
+const DECIMAL = 7;
+const TIMESTAMP = 10;
+
+/**
+ * Test if a value is an Apache Arrow table.
+ * As sometimes multiple Arrow versions may be used simultaneously,
+ * we use a "duck typing" approach and check for a getChild function.
+ * @param {*} values The value to test
+ * @returns true if the value duck types as Apache Arrow data
+ */
+export function isArrowTable(values) {
+  return typeof values?.getChild === 'function';
+}
+
+/**
+ * Return a JavaScript array type for an Apache Arrow column type.
+ * @param {*} type an Apache Arrow column type
+ * @returns a JavaScript array constructor
+ */
+export function convertArrowArrayType(type) {
+  switch (type.typeId) {
+    case INTEGER:
+    case FLOAT:
+    case DECIMAL:
+      return Float64Array;
+    default:
+      return Array;
+  }
+}
+
+/**
+ * Return a function that converts Apache Arrow values to JavaScript values.
+ * Timestamps are converted to Date values.
+ * Large integers (BigInt) are converted to Float64 numbers.
+ * Fixed-point decimal values are convert to Float64 numbers.
+ * Otherwise, the default Arrow values are used.
+ * @param {*} type an Apache Arrow column type
+ * @returns a value conversion function
+ */
+export function convertArrowValue(type) {
+  const { typeId } = type;
+
+  // map timestamp numbers to date objects
+  if (typeId === TIMESTAMP) {
+    return v => v == null ? v : new Date(v);
+  }
+
+  // map bigint to number
+  if (typeId === INTEGER && type.bitWidth >= 64) {
+    return v => v == null ? v : Number(v);
+  }
+
+  // map decimal to number
+  if (typeId === DECIMAL) {
+    const scale = 1 / Math.pow(10, type.scale);
+    return v => v == null ? v : decimalToNumber(v, scale);
+  }
+
+  // otherwise use Arrow JS defaults
+  return v => v;
+}
+
+/**
+ * Convert an Apache Arrow column to a JavaScript array.
+ * Timestamps are converted to Date values.
+ * Large integers (BigInt) are converted to Float64 numbers.
+ * Fixed-point decimal values are convert to Float64 numbers.
+ * Otherwise, the default Arrow values are used.
+ * @param {*} column An Apache Arrow column
+ * @returns an array of values
+ */
+export function convertArrowColumn(column) {
+  const { type } = column;
+  const { typeId } = type;
+
+  // map timestamp numbers to date objects
+  if (typeId === TIMESTAMP) {
+    const size = column.length;
+    const array = new Array(size);
+    for (let row = 0; row < size; ++row) {
+      const v = column.get(row);
+      array[row] = v == null ? null : new Date(v);
+    }
+    return array;
+  }
+
+  // map bigint to number
+  if (typeId === INTEGER && type.bitWidth >= 64) {
+    const size = column.length;
+    const array = new Float64Array(size);
+    for (let row = 0; row < size; ++row) {
+      const v = column.get(row);
+      array[row] = v == null ? NaN : Number(v);
+    }
+    return array;
+  }
+
+  // map decimal to number
+  if (typeId === DECIMAL) {
+    const scale = 1 / Math.pow(10, type.scale);
+    const size = column.length;
+    const array = new Float64Array(size);
+    for (let row = 0; row < size; ++row) {
+      const v = column.get(row);
+      array[row] = v == null ? NaN : decimalToNumber(v, scale);
+    }
+    return array;
+  }
+
+  // otherwise use Arrow JS defaults
+  return column.toArray();
+}
+
+// generate base values for big integers
+// represented within a Uint32Array
+const BASE32 = Array.from(
+  { length: 8 },
+  (_, i) => Math.pow(2, i * 32)
+);
+
+/**
+ * Convert a fixed point decimal value to a double precision number.
+ * Note: if the value is sufficiently large the conversion may be lossy!
+ * @param {Uint32Array} v a fixed decimal value
+ * @param {number} scale a scale factor, corresponding to the
+ *  number of fractional decimal digits in the fixed point value
+ * @returns the resulting number
+ */
+function decimalToNumber(v, scale) {
+  const n = v.length;
+  let x = 0;
+
+  if (v.signed && (v[n-1]|0) < 0) {
+    for (let i = 0; i < n; ++i) {
+      x += ~(v[i] | 0) * BASE32[i];
+    }
+    x = -(x + 1);
+  } else {
+    for (let i = 0; i < n; ++i) {
+      x += v[i] * BASE32[i];
+    }
+  }
+
+  return x * scale;
+}

package/src/util/field-info.js

@@ -0,0 +1,80 @@
+import { Query, asRelation, count, isNull, max, min, sql } from '@uwdata/mosaic-sql';
+import { jsType } from './js-type.js';
+import { convertArrowValue } from './convert-arrow.js';
+
+export const Count = 'count';
+export const Nulls = 'nulls';
+export const Max = 'max';
+export const Min = 'min';
+export const Distinct = 'distinct';
+export const Stats = { Count, Nulls, Max, Min, Distinct };
+
+const statMap = {
+  [Count]: count,
+  [Distinct]: column => count(column).distinct(),
+  [Max]: max,
+  [Min]: min,
+  [Nulls]: column => count().where(isNull(column))
+};
+
+function summarize(table, column, stats) {
+  return Query
+    .from(table)
+    .select(Array.from(stats, s => [s, statMap[s](column)]));
+}
+
+export async function queryFieldInfo(mc, fields) {
+  if (fields.length === 1 && `${fields[0].column}` === '*') {
+    return getTableInfo(mc, fields[0].table);
+  } else {
+    return (await Promise
+      .all(fields.map(f => getFieldInfo(mc, f))))
+      .filter(x => x);
+  }
+}
+
+async function getFieldInfo(mc, { table, column, stats }) {
+  // generate and issue a query for field metadata info
+  // use GROUP BY ALL to differentiate & consolidate aggregates
+  const q = Query.from({ source: table })
+    .select({ column })
+    .groupby(column.aggregate ? sql`ALL` : []);
+  const [desc] = Array.from(await mc.query(Query.describe(q)));
+  const info = {
+    table,
+    column: `${column}`,
+    sqlType: desc.column_type,
+    type: jsType(desc.column_type),
+    nullable: desc.null === 'YES'
+  };
+
+  // no need for summary statistics
+  if (!(stats?.length || stats?.size)) return info;
+
+  // query for summary stats
+  const result = await mc.query(
+    summarize(table, column, stats),
+    { persist: true }
+  );
+
+  // extract summary stats, copy to field info
+  for (let i = 0; i < result.numCols; ++i) {
+    const { name } = result.schema.fields[i];
+    const child = result.getChildAt(i);
+    const convert = convertArrowValue(child.type);
+    info[name] = convert(child.get(0));
+  }
+
+  return info;
+}
+
+async function getTableInfo(mc, table) {
+  const result = await mc.query(`DESCRIBE ${asRelation(table)}`);
+  return Array.from(result).map(desc => ({
+    table,
+    column: desc.column_name,
+    sqlType: desc.column_type,
+    type: jsType(desc.column_type),
+    nullable: desc.null === 'YES'
+  }));
+}

package/src/util/js-type.js

@@ -12,7 +12,6 @@ export function jsType(type) {
     case 'DOUBLE':
     case 'FLOAT':
     case 'REAL':
-    case 'DECIMAL':
       return 'number';
     case 'DATE':
     case 'TIMESTAMP':
@@ -26,13 +25,22 @@ export function jsType(type) {
     case 'VARCHAR':
     case 'UUID':
       return 'string';
+    case 'ARRAY':
     case 'LIST':
       return 'array';
     case 'BLOB':
     case 'STRUCT':
     case 'MAP':
+    case 'GEOMETRY':
       return 'object';
     default:
+      if (type.startsWith('DECIMAL')) {
+        return 'number';
+      } else if (type.startsWith('STRUCT') || type.startsWith('MAP')) {
+        return 'object';
+      } else if (type.endsWith(']')) {
+        return 'array';
+      }
       throw new Error(`Unsupported type: ${type}`);
   }
 }

package/src/Catalog.js

@@ -1,88 +0,0 @@
-import { asRelation } from '@uwdata/mosaic-sql';
-import { jsType } from './util/js-type.js';
-import { summarize } from './util/summarize.js';
-
-const object = () => Object.create(null);
-
-export class Catalog {
-  constructor(coordinator) {
-    /** @type {import('@uwdata/mosaic-core').Coordinator} */
-    this.mc = coordinator;
-    this.clear();
-  }
-
-  clear() {
-    this.tables = object();
-  }
-
-  tableInfo(table) {
-    const cache = this.tables;
-    if (cache[table]) {
-      return cache[table];
-    }
-
-    const infoPromise = getTableInfo(this.mc, table)
-      .catch(err => { cache[table] = null; throw err; });
-
-    return (cache[table] = infoPromise);
-  }
-
-  async fieldInfo({ table, column, stats }) {
-    const tableInfo = await this.tableInfo(table);
-    const colInfo = tableInfo[column];
-
-    // column does not exist
-    if (colInfo == null) return;
-
-    // no need for summary statistics
-    if (!stats?.length) return colInfo;
-
-    const result = await this.mc.query(
-      summarize(colInfo, stats),
-      { persist: true }
-    );
-    const info = { ...colInfo, ...(Array.from(result)[0]) };
-
-    // coerce bigint to number
-    for (const key in info) {
-      const value = info[key];
-      if (typeof value === 'bigint') {
-        info[key] = Number(value);
-      }
-    }
-
-    return info;
-  }
-
-  async queryFields(fields) {
-    const list = await resolveFields(this, fields);
-    const data = await Promise.all(list.map(f => this.fieldInfo(f)));
-    return data.filter(x => x);
-  }
-}
-
-async function getTableInfo(mc, table) {
-  const result = await mc.query(
-    `DESCRIBE ${asRelation(table)}`,
-    { type: 'json', cache: false }
-  );
-
-  const columns = object();
-  for (const entry of result) {
-    columns[entry.column_name] = {
-      table,
-      column: entry.column_name,
-      sqlType: entry.column_type,
-      type: jsType(entry.column_type),
-      nullable: entry.null === 'YES'
-    };
-  }
-
-  return columns;
-}
-
-async function resolveFields(catalog, list) {
-  return list.length === 1 && list[0].column === '*'
-    ? Object.values(await catalog.tableInfo(list[0].table))
-    : list;
-}

package/src/util/summarize.js

@@ -1,23 +0,0 @@
-import { Query, count, isNull, max, min } from '@uwdata/mosaic-sql';
-
-export const Count = 'count';
-export const Nulls = 'nulls';
-export const Max = 'max';
-export const Min = 'min';
-export const Distinct = 'distinct';
-
-export const Stats = { Count, Nulls, Max, Min, Distinct };
-
-export const statMap = {
-  [Count]: count,
-  [Distinct]: column => count(column).distinct(),
-  [Max]: max,
-  [Min]: min,
-  [Nulls]: column => count().where(isNull(column))
-};
-
-export function summarize({ table, column }, stats) {
-  return Query
-    .from(table)
-    .select(stats.map(s => [s, statMap[s](column)]));
-}