@uwdata/mosaic-core 0.16.2 → 0.18.0

package/dist/src/util/to-data-columns.js

@@ -0,0 +1,51 @@
+import { isArrowTable } from './is-arrow-table.js';
+/**
+ * Convert input data to a set of column arrays.
+ * @param data The input data.
+ * @returns An object with named column arrays.
+ */
+export function toDataColumns(data) {
+    if (isArrowTable(data)) {
+        return arrowToColumns(data);
+    }
+    else if (Array.isArray(data)) {
+        return arrayToColumns(data);
+    }
+    else {
+        throw new Error('Unrecognized data format.');
+    }
+}
+/**
+ * Convert an Arrow table to a set of column arrays.
+ * @param data An Arrow Table.
+ * @returns An object with named column arrays.
+ */
+function arrowToColumns(data) {
+    const { numRows } = data;
+    return { numRows, columns: data.toColumns() };
+}
+/**
+ * Convert an array of values to a set of column arrays.
+ * If the array values are objects, build out named columns.
+ * We use the keys of the first object as the column names.
+ * Otherwise, use a special "values" array.
+ * @param data An array of data objects.
+ * @returns An object with named column arrays.
+ */
+function arrayToColumns(data) {
+    const numRows = data.length;
+    if (typeof data[0] === 'object') {
+        const names = numRows ? Object.keys(data[0]) : [];
+        const columns = {};
+        if (names.length > 0) {
+            names.forEach(name => {
+                columns[name] = data.map(d => d[name]);
+            });
+        }
+        return { numRows, columns };
+    }
+    else {
+        return { numRows, values: data };
+    }
+}
+//# sourceMappingURL=to-data-columns.js.map

package/dist/src/util/to-data-columns.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"to-data-columns.js","sourceRoot":"","sources":["../../../src/util/to-data-columns.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAiBnD;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,IAAa;IACzC,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;QACvB,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/B,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,cAAc,CAAC,IAAW;IACjC,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IACzB,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,SAAS,EAA8B,EAAE,CAAC;AAC5E,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,cAAc,CAAC,IAA8B;IACpD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;IAC5B,IAAI,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACnD,MAAM,OAAO,GAA8B,EAAE,CAAC;QAC9C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;gBACnB,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YACzC,CAAC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAC9B,CAAC;SAAM,CAAC;QACN,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACnC,CAAC;AACH,CAAC"}

package/dist/src/util/void-logger.d.ts

@@ -0,0 +1,13 @@
+interface Logger {
+    debug(...args: unknown[]): void;
+    info(...args: unknown[]): void;
+    log(...args: unknown[]): void;
+    warn(...args: unknown[]): void;
+    error(...args: unknown[]): void;
+    group(label?: string): void;
+    groupCollapsed(label?: string): void;
+    groupEnd(): void;
+}
+export declare function voidLogger(): Logger;
+export {};
+//# sourceMappingURL=void-logger.d.ts.map

package/dist/src/util/void-logger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"void-logger.d.ts","sourceRoot":"","sources":["../../../src/util/void-logger.ts"],"names":[],"mappings":"AAAA,UAAU,MAAM;IACd,KAAK,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAChC,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAC/B,GAAG,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAC9B,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAC/B,KAAK,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAChC,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,cAAc,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,IAAI,IAAI,CAAC;CAClB;AAED,wBAAgB,UAAU,IAAI,MAAM,CAWnC"}

package/dist/src/util/void-logger.js

@@ -0,0 +1,13 @@
+export function voidLogger() {
+    return {
+        debug() { },
+        info() { },
+        log() { },
+        warn() { },
+        error() { },
+        group() { },
+        groupCollapsed() { },
+        groupEnd() { }
+    };
+}
+//# sourceMappingURL=void-logger.js.map

package/dist/src/util/void-logger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"void-logger.js","sourceRoot":"","sources":["../../../src/util/void-logger.ts"],"names":[],"mappings":"AAWA,MAAM,UAAU,UAAU;IACxB,OAAO;QACL,KAAK,KAAU,CAAC;QAChB,IAAI,KAAU,CAAC;QACf,GAAG,KAAU,CAAC;QACd,IAAI,KAAU,CAAC;QACf,KAAK,KAAU,CAAC;QAChB,KAAK,KAAU,CAAC;QAChB,cAAc,KAAU,CAAC;QACzB,QAAQ,KAAU,CAAC;KACpB,CAAC;AACJ,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uwdata/mosaic-core",
-  "version": "0.16.2",
+  "version": "0.18.0",
   "description": "Scalable and extensible linked data views.",
   "keywords": [
     "mosaic",
@@ -13,29 +13,31 @@
   "author": "Jeffrey Heer (https://idl.uw.edu)",
   "type": "module",
   "exports": {
-    "types": "./dist/types/index-types.d.ts",
-    "default": "./src/index.js"
+    "types": "./dist/src/index.d.ts",
+    "default": "./dist/src/index.js"
   },
+  "files": [
+    "dist",
+    "!dist/tsconfig.tsbuildinfo",
+    "src"
+  ],
   "repository": {
     "type": "git",
     "url": "https://github.com/uwdata/mosaic.git"
   },
   "scripts": {
-    "prebuild": "rimraf dist && mkdir dist",
-    "build": "npm run types",
-    "types": "tsc",
+    "clean": "rimraf dist && mkdir dist",
     "lint": "eslint src test",
-    "test": "vitest run && npm run tsc",
-    "tsc": "tsc -p jsconfig.json",
-    "prepublishOnly": "npm run test && npm run lint && npm run build"
+    "test": "vitest run",
+    "prepublishOnly": "npm run test && npm run lint && tsc --build"
   },
   "dependencies": {
     "@duckdb/duckdb-wasm": "^1.29.0",
-    "@uwdata/flechette": "^2.0.0",
-    "@uwdata/mosaic-sql": "^0.16.2"
+    "@uwdata/flechette": "^2.0.1",
+    "@uwdata/mosaic-sql": "^0.18.0"
   },
   "devDependencies": {
-    "@uwdata/mosaic-duckdb": "^0.16.2"
+    "@uwdata/mosaic-duckdb": "^0.18.0"
   },
-  "gitHead": "26d2719f4bcab471d2831145e1f03f39f3509869"
+  "gitHead": "dfb9ded0b0307754e3185ca34cc49a1384fe8455"
 }

package/src/Coordinator.ts

@@ -0,0 +1,367 @@
+import { SocketConnector } from './connectors/socket.js';
+import { type Connector } from './connectors/Connector.js';
+import { PreAggregator, type PreAggregateOptions } from './preagg/PreAggregator.js';
+import { voidLogger } from './util/void-logger.js';
+import { QueryManager, Priority } from './QueryManager.js';
+import { type Selection } from './Selection.js';
+import { type Logger, type QueryType } from './types.js';
+import { type QueryResult } from './util/query-result.js';
+import { type MosaicClient } from './MosaicClient.js';
+import { type SelectionClause } from './SelectionClause.js';
+import { MaybeArray } from '@uwdata/mosaic-sql';
+
+interface FilterGroupEntry {
+  selection: Selection;
+  clients: Set<MosaicClient>;
+  disconnect(): void;
+}
+
+/**
+ * The singleton Coordinator instance.
+ */
+let _instance: Coordinator;
+
+/**
+ * Set or retrieve the coordinator instance.
+ * @param instance The coordinator instance to set
+ * @returns The coordinator instance
+ */
+export function coordinator(
+  instance?: Coordinator
+): Coordinator {
+  if (instance) {
+    _instance = instance;
+  } else if (_instance == null) {
+    _instance = new Coordinator();
+  }
+  return _instance;
+}
+
+/**
+ * A Mosaic Coordinator manages all database communication for clients and
+ * handles selection updates. The Coordinator also performs optimizations
+ * including query caching, consolidation, and pre-aggregation.
+ */
+export class Coordinator {
+  public manager: QueryManager;
+  public preaggregator: PreAggregator;
+  public clients = new Set<MosaicClient>;
+  public filterGroups = new Map<Selection, FilterGroupEntry>;
+  protected _logger: Logger = voidLogger();
+
+  /**
+   * @param db Database connector. Defaults to a web socket connection.
+   * @param options Coordinator options.
+   * @param options.logger The logger to use, defaults to `console`.
+   * @param options.manager The query manager to use.
+   * @param options.cache Boolean flag to enable/disable query caching.
+   * @param options.consolidate Boolean flag to enable/disable query consolidation.
+   * @param options.preagg Options for the Pre-aggregator.
+   */
+  constructor(
+    db: Connector = new SocketConnector(),
+    options: {
+      logger?: Logger | null;
+      manager?: QueryManager;
+      cache?: boolean;
+      consolidate?: boolean;
+      preagg?: PreAggregateOptions;
+    } = {}
+  ) {
+    const {
+      logger = console,
+      manager = new QueryManager(),
+      cache = true,
+      consolidate = true,
+      preagg = {}
+    } = options;
+    this.manager = manager;
+    this.manager.cache(cache);
+    this.manager.consolidate(consolidate);
+    this.databaseConnector(db);
+    this.logger(logger);
+    this.clear();
+    this.preaggregator = new PreAggregator(this, preagg);
+  }
+
+  /**
+   * Clear the coordinator state.
+   * @param options Options object.
+   * @param options.clients If true, disconnect all clients.
+   * @param options.cache If true, clear the query cache.
+   */
+  clear(options: { clients?: boolean; cache?: boolean } = {}) {
+    const { clients = true, cache = true } = options;
+    this.manager.clear();
+    if (clients) {
+      this.filterGroups?.forEach(group => group.disconnect());
+      this.filterGroups = new Map;
+      this.clients?.forEach(client => this.disconnect(client));
+      this.clients = new Set;
+    }
+    if (cache) this.manager.cache()!.clear();
+  }
+
+  /**
+   * Get or set the database connector.
+   * @param db The database connector to use.
+   * @returns The current database connector.
+   */
+  databaseConnector(): Connector | null;
+  databaseConnector(db: Connector): Connector;
+  databaseConnector(db?: Connector): Connector | null {
+    return db
+      ? this.manager.connector(db)
+      : this.manager.connector();
+  }
+
+  /**
+   * Get or set the logger.
+   * @param logger The logger to use.
+   * @returns The current logger
+   */
+  logger(logger?: Logger | null): Logger {
+    if (arguments.length) {
+      this._logger = logger || voidLogger();
+      this.manager.logger(this._logger);
+    }
+    return this._logger!;
+  }
+
+  // -- Query Management ----
+
+  /**
+   * Cancel previously submitted query requests. These queries will be
+   * canceled if they are queued but have not yet been submitted.
+   * @param requests An array of query result objects, such as those returned by the `query` method.
+   */
+  cancel(requests: QueryResult[]) {
+    this.manager.cancel(requests);
+  }
+
+  /**
+   * Issue a query for which no result (return value) is needed.
+   * @param query The query or an array of queries. Each query should be either a Query builder object or a SQL string.
+   * @param options An options object.
+   * @param options.priority The query priority, defaults to `Priority.Normal`.
+   * @returns A query result promise.
+   */
+  exec(
+    query: MaybeArray<QueryType>,
+    options: { priority?: number } = {}
+  ): QueryResult {
+    const { priority = Priority.Normal } = options;
+    query = Array.isArray(query) ? query.filter(x => x).join(';\n') : query;
+    return this.manager.request({ type: 'exec', query }, priority);
+  }
+
+  /**
+   * Issue a query to the backing database. The submitted query may be
+   * consolidate with other queries and its results may be cached.
+   * @param query The query as either a Query builder object or a SQL string.
+   * @param options An options object.
+   * @param options.type The query result format type.
+   * @param options.cache If true, cache the query result client-side within the QueryManager.
+   * @param options.persist If true, request the database server to persist a cached query server-side.
+   * @param options.priority The query priority, defaults to `Priority.Normal`.
+   * @returns A query result promise.
+   */
+  query(
+    query: QueryType,
+    options: {
+      type?: 'arrow' | 'json';
+      cache?: boolean;
+      persist?: boolean;
+      priority?: number;
+      [key: string]: unknown;
+    } = {}
+  ): QueryResult {
+    const {
+      type = 'arrow',
+      cache = true,
+      priority = Priority.Normal,
+      ...otherOptions
+    } = options;
+    return this.manager.request({ type, query, cache, options: otherOptions }, priority);
+  }
+
+  /**
+   * Issue a query to prefetch data for later use. The query result is cached
+   * for efficient future access.
+   * @param query The query as either a Query builder object or a SQL string.
+   * @param options An options object.
+   * @param options.type The query result format type.
+   * @returns A query result promise.
+   */
+  prefetch(
+    query: QueryType,
+    options: { type?: 'arrow' | 'json'; [key: string]: unknown } = {}
+  ): QueryResult {
+    return this.query(query, { ...options, cache: true, priority: Priority.Low });
+  }
+
+  // -- Client Management ----
+
+  /**
+   * Update client data by submitting the given query and returning the()
+   * data (or error) to the client.
+   * @param client A Mosaic client.
+   * @param query The data query.
+   * @param priority The query priority.
+   * @returns A Promise that resolves upon completion of the update.
+   */
+  updateClient(
+    client: MosaicClient,
+    query: QueryType,
+    priority: number = Priority.Normal
+  ): Promise<unknown> {
+    client.queryPending();
+    return client._pending = this.query(query, { priority })
+      .then(
+        data => client.queryResult(data).update(),
+        err => { this._logger?.error(err); client.queryError(err); }
+      )
+      .catch(err => this._logger?.error(err));
+  }
+
+  /**
+   * Issue a query request for a client. If the query is null or undefined,
+   * the client is simply updated. Otherwise `updateClient` is called. As a
+   * side effect, this method clears the current preaggregator state.
+   * @param client The client to update.
+   * @param query The query to issue.
+   */
+  requestQuery(client: MosaicClient, query?: QueryType | null): Promise<unknown> {
+    this.preaggregator.clear();
+    return query
+      ? this.updateClient(client, query)
+      : Promise.resolve(client.update());
+  }
+
+  /**
+   * Connect a client to the coordinator.
+   * Throws an error if the client is already connected.
+   * @param client The Mosaic client to connect.
+   */
+  connect(client: MosaicClient): void {
+    const { clients } = this;
+
+    if (clients?.has(client)) {
+      throw new Error('Client already connected.');
+    }
+
+    // add client to client set
+    clients?.add(client);
+
+    // register coordinator on client instance
+    client.coordinator = this;
+
+    // initialize client lifecycle
+    client.initialize();
+
+    // connect filter selection
+    connectSelection(this, client.filterBy!, client);
+  }
+
+  /**
+   * Disconnect a client from the coordinator.
+   * This method has no effect if the client is already disconnected.
+   * @param client The Mosaic client to disconnect.
+   */
+  disconnect(client: MosaicClient): void {
+    const { clients, filterGroups } = this;
+    if (!clients?.has(client)) return;
+    clients.delete(client);
+    client.coordinator = null;
+
+    const group = filterGroups?.get(client.filterBy!);
+    if (group) {
+      group.clients.delete(client);
+    }
+  }
+}
+
+/**
+ * Connect a selection-client pair to the coordinator to process updates.
+ * @param mc The Mosaic coordinator.
+ * @param selection A selection.
+ * @param client A Mosaic client that is filtered by the given selection.
+ */
+function connectSelection(
+  mc: Coordinator,
+  selection: Selection,
+  client: MosaicClient
+): void {
+  if (!selection) return;
+  let entry = mc.filterGroups?.get(selection);
+  if (!entry) {
+    const activate = (clause: SelectionClause) => activateSelection(mc, selection, clause);
+    const value = () => updateSelection(mc, selection);
+
+    // @ts-expect-error todo: update selection dispatch types
+    selection.addEventListener('activate', activate);
+    selection.addEventListener('value', value);
+
+    entry = {
+      selection,
+      clients: new Set,
+      disconnect() {
+        // @ts-expect-error todo: update selection dispatch types
+        selection.removeEventListener('activate', activate);
+        selection.removeEventListener('value', value);
+      }
+    };
+    mc.filterGroups?.set(selection, entry);
+  }
+  entry.clients.add(client);
+}
+
+/**
+ * Activate a selection, providing a clause indicative of potential
+ * next updates. Activation provides a preview of likely next events,
+ * enabling potential precomputation to optimize updates.
+ * @param mc The Mosaic coordinator.
+ * @param selection A selection.
+ * @param clause A selection clause for the activation.
+ */
+function activateSelection(
+  mc: Coordinator,
+  selection: Selection,
+  clause: SelectionClause
+): void {
+  const { preaggregator, filterGroups } = mc;
+  const { clients } = filterGroups.get(selection)!;
+  for (const client of clients) {
+    if (client.enabled) {
+      preaggregator.request(client, selection, clause);
+    }
+  }
+}
+
+/**
+ * Process an updated selection value, querying filtered data for any
+ * associated clients.
+ * @param mc The Mosaic coordinator.
+ * @param selection A selection.
+ * @returns A Promise that resolves when the update completes.
+ */
+function updateSelection(
+  mc: Coordinator,
+  selection: Selection
+): Promise<PromiseSettledResult<unknown>[]> {
+  const { preaggregator, filterGroups } = mc;
+  const { clients } = filterGroups!.get(selection)!;
+  const { active } = selection;
+  return Promise.allSettled(Array.from(clients, (client: MosaicClient) => {
+    if (!client.enabled) return client.requestQuery();
+    const info = preaggregator.request(client, selection, active);
+    const filter = info ? null : selection.predicate(client);
+
+    // skip due to cross-filtering
+    if (info?.skip || (!info && !filter)) return;
+
+    // @ts-expect-error FIXME
+    const query = info?.query(active.predicate) ?? client.query(filter);
+    return mc.updateClient(client, query);
+  }));
+}