@uwdata/mosaic-core 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/mosaic-core.js +5951 -15206
- package/dist/mosaic-core.min.js +6 -15
- package/package.json +7 -7
- package/src/Coordinator.js +56 -31
- package/src/DataCubeIndexer.js +82 -24
- package/src/MosaicClient.js +9 -0
- package/src/QueryConsolidator.js +13 -9
- package/src/QueryManager.js +9 -3
- package/src/Selection.js +41 -10
- package/src/connectors/rest.js +3 -3
- package/src/connectors/socket.js +4 -3
- package/src/connectors/wasm.js +20 -4
- package/src/index.js +3 -8
- package/src/util/decode-ipc.js +11 -0
- package/src/util/field-info.js +3 -11
- package/src/util/index-columns.js +69 -72
- package/src/util/is-arrow-table.js +10 -0
- package/src/util/priority-queue.js +75 -76
- package/src/util/throttle.js +11 -1
- package/src/util/to-data-columns.js +4 -15
- package/src/util/convert-arrow.js +0 -145
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@uwdata/mosaic-core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.0",
|
|
4
4
|
"description": "Scalable and extensible linked data views.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"mosaic",
|
|
@@ -24,16 +24,16 @@
|
|
|
24
24
|
"prebuild": "rimraf dist && mkdir dist",
|
|
25
25
|
"build": "node ../../esbuild.js mosaic-core",
|
|
26
26
|
"lint": "eslint src test",
|
|
27
|
-
"test": "
|
|
27
|
+
"test": "vitest run --dangerouslyIgnoreUnhandledErrors",
|
|
28
28
|
"prepublishOnly": "npm run test && npm run lint && npm run build"
|
|
29
29
|
},
|
|
30
30
|
"dependencies": {
|
|
31
|
-
"@duckdb/duckdb-wasm": "^1.28.1-
|
|
32
|
-
"@uwdata/
|
|
33
|
-
"
|
|
31
|
+
"@duckdb/duckdb-wasm": "^1.28.1-dev278.0",
|
|
32
|
+
"@uwdata/flechette": "^1.0.2",
|
|
33
|
+
"@uwdata/mosaic-sql": "^0.11.0"
|
|
34
34
|
},
|
|
35
35
|
"devDependencies": {
|
|
36
|
-
"@uwdata/mosaic-duckdb": "^0.
|
|
36
|
+
"@uwdata/mosaic-duckdb": "^0.11.0"
|
|
37
37
|
},
|
|
38
|
-
"gitHead": "
|
|
38
|
+
"gitHead": "861d616f39926a1d2aee83b59dbdd70b0b3caf12"
|
|
39
39
|
}
|
package/src/Coordinator.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { socketConnector } from './connectors/socket.js';
|
|
2
2
|
import { DataCubeIndexer } from './DataCubeIndexer.js';
|
|
3
|
+
import { MosaicClient } from './MosaicClient.js';
|
|
3
4
|
import { QueryManager, Priority } from './QueryManager.js';
|
|
4
5
|
import { queryFieldInfo } from './util/field-info.js';
|
|
6
|
+
import { QueryResult } from './util/query-result.js';
|
|
5
7
|
import { voidLogger } from './util/void-logger.js';
|
|
6
8
|
|
|
7
9
|
/**
|
|
@@ -24,6 +26,10 @@ export function coordinator(instance) {
|
|
|
24
26
|
return _instance;
|
|
25
27
|
}
|
|
26
28
|
|
|
29
|
+
/**
|
|
30
|
+
* @typedef {import('@uwdata/mosaic-sql').Query | string} QueryType
|
|
31
|
+
*/
|
|
32
|
+
|
|
27
33
|
/**
|
|
28
34
|
* A Mosaic Coordinator manages all database communication for clients and
|
|
29
35
|
* handles selection updates. The Coordinator also performs optimizations
|
|
@@ -34,7 +40,8 @@ export function coordinator(instance) {
|
|
|
34
40
|
* @param {*} [options.manager] The query manager to use.
|
|
35
41
|
* @param {boolean} [options.cache=true] Boolean flag to enable/disable query caching.
|
|
36
42
|
* @param {boolean} [options.consolidate=true] Boolean flag to enable/disable query consolidation.
|
|
37
|
-
* @param {
|
|
43
|
+
* @param {import('./DataCubeIndexer.js').DataCubeIndexerOptions} [options.indexes]
|
|
44
|
+
* Data cube indexer options.
|
|
38
45
|
*/
|
|
39
46
|
export class Coordinator {
|
|
40
47
|
constructor(db = socketConnector(), {
|
|
@@ -44,13 +51,14 @@ export class Coordinator {
|
|
|
44
51
|
consolidate = true,
|
|
45
52
|
indexes = {}
|
|
46
53
|
} = {}) {
|
|
54
|
+
/** @type {QueryManager} */
|
|
47
55
|
this.manager = manager;
|
|
48
56
|
this.manager.cache(cache);
|
|
49
57
|
this.manager.consolidate(consolidate);
|
|
50
|
-
this.dataCubeIndexer = new DataCubeIndexer(this, indexes);
|
|
51
|
-
this.logger(logger);
|
|
52
58
|
this.databaseConnector(db);
|
|
59
|
+
this.logger(logger);
|
|
53
60
|
this.clear();
|
|
61
|
+
this.dataCubeIndexer = new DataCubeIndexer(this, indexes);
|
|
54
62
|
}
|
|
55
63
|
|
|
56
64
|
/**
|
|
@@ -97,7 +105,7 @@ export class Coordinator {
|
|
|
97
105
|
/**
|
|
98
106
|
* Cancel previosuly submitted query requests. These queries will be
|
|
99
107
|
* canceled if they are queued but have not yet been submitted.
|
|
100
|
-
* @param {
|
|
108
|
+
* @param {QueryResult[]} requests An array
|
|
101
109
|
* of query result objects, such as those returned by the `query` method.
|
|
102
110
|
*/
|
|
103
111
|
cancel(requests) {
|
|
@@ -106,29 +114,30 @@ export class Coordinator {
|
|
|
106
114
|
|
|
107
115
|
/**
|
|
108
116
|
* Issue a query for which no result (return value) is needed.
|
|
109
|
-
* @param {
|
|
117
|
+
* @param {QueryType | QueryType[]} query The query or an array of queries.
|
|
118
|
+
* Each query should be either a Query builder object or a SQL string.
|
|
110
119
|
* @param {object} [options] An options object.
|
|
111
120
|
* @param {number} [options.priority] The query priority, defaults to
|
|
112
121
|
* `Priority.Normal`.
|
|
113
|
-
* @returns {
|
|
122
|
+
* @returns {QueryResult} A query result
|
|
114
123
|
* promise.
|
|
115
124
|
*/
|
|
116
125
|
exec(query, { priority = Priority.Normal } = {}) {
|
|
117
|
-
query = Array.isArray(query) ? query.join(';\n') : query;
|
|
126
|
+
query = Array.isArray(query) ? query.filter(x => x).join(';\n') : query;
|
|
118
127
|
return this.manager.request({ type: 'exec', query }, priority);
|
|
119
128
|
}
|
|
120
129
|
|
|
121
130
|
/**
|
|
122
131
|
* Issue a query to the backing database. The submitted query may be
|
|
123
132
|
* consolidate with other queries and its results may be cached.
|
|
124
|
-
* @param {
|
|
133
|
+
* @param {QueryType} query The query as either a Query builder object
|
|
134
|
+
* or a SQL string.
|
|
125
135
|
* @param {object} [options] An options object.
|
|
126
136
|
* @param {'arrow' | 'json'} [options.type] The query result format type.
|
|
127
137
|
* @param {boolean} [options.cache=true] If true, cache the query result.
|
|
128
138
|
* @param {number} [options.priority] The query priority, defaults to
|
|
129
139
|
* `Priority.Normal`.
|
|
130
|
-
* @returns {
|
|
131
|
-
* promise.
|
|
140
|
+
* @returns {QueryResult} A query result promise.
|
|
132
141
|
*/
|
|
133
142
|
query(query, {
|
|
134
143
|
type = 'arrow',
|
|
@@ -142,21 +151,35 @@ export class Coordinator {
|
|
|
142
151
|
/**
|
|
143
152
|
* Issue a query to prefetch data for later use. The query result is cached
|
|
144
153
|
* for efficient future access.
|
|
145
|
-
* @param {
|
|
154
|
+
* @param {QueryType} query The query as either a Query builder object
|
|
155
|
+
* or a SQL string.
|
|
146
156
|
* @param {object} [options] An options object.
|
|
147
157
|
* @param {'arrow' | 'json'} [options.type] The query result format type.
|
|
148
|
-
* @returns {
|
|
149
|
-
* promise.
|
|
158
|
+
* @returns {QueryResult} A query result promise.
|
|
150
159
|
*/
|
|
151
160
|
prefetch(query, options = {}) {
|
|
152
161
|
return this.query(query, { ...options, cache: true, priority: Priority.Low });
|
|
153
162
|
}
|
|
154
163
|
|
|
164
|
+
/**
|
|
165
|
+
* Create a bundle of queries that can be loaded into the cache.
|
|
166
|
+
*
|
|
167
|
+
* @param {string} name The name of the bundle.
|
|
168
|
+
* @param {[string | {sql: string}, {alias: string}]} queries The queries to save into the bundle.
|
|
169
|
+
* @param {number} priority Request priority.
|
|
170
|
+
* @returns {QueryResult} A query result promise.
|
|
171
|
+
*/
|
|
155
172
|
createBundle(name, queries, priority = Priority.Low) {
|
|
156
|
-
const options = { name, queries };
|
|
173
|
+
const options = { name, queries: queries.map(q => typeof q == 'string' ? {sql: q} : q) };
|
|
157
174
|
return this.manager.request({ type: 'create-bundle', options }, priority);
|
|
158
175
|
}
|
|
159
176
|
|
|
177
|
+
/**
|
|
178
|
+
* Load a bundle into the cache.
|
|
179
|
+
* @param {string} name The name of the bundle.
|
|
180
|
+
* @param {number} priority Request priority.
|
|
181
|
+
* @returns {QueryResult} A query result promise.
|
|
182
|
+
*/
|
|
160
183
|
loadBundle(name, priority = Priority.High) {
|
|
161
184
|
const options = { name };
|
|
162
185
|
return this.manager.request({ type: 'load-bundle', options }, priority);
|
|
@@ -167,8 +190,8 @@ export class Coordinator {
|
|
|
167
190
|
/**
|
|
168
191
|
* Update client data by submitting the given query and returning the
|
|
169
192
|
* data (or error) to the client.
|
|
170
|
-
* @param {
|
|
171
|
-
* @param {
|
|
193
|
+
* @param {MosaicClient} client A Mosaic client.
|
|
194
|
+
* @param {QueryType} query The data query.
|
|
172
195
|
* @param {number} [priority] The query priority.
|
|
173
196
|
* @returns {Promise} A Promise that resolves upon completion of the update.
|
|
174
197
|
*/
|
|
@@ -186,22 +209,19 @@ export class Coordinator {
|
|
|
186
209
|
* Issue a query request for a client. If the query is null or undefined,
|
|
187
210
|
* the client is simply updated. Otherwise `updateClient` is called. As a
|
|
188
211
|
* side effect, this method clears the current data cube indexer state.
|
|
189
|
-
* @param {
|
|
190
|
-
*
|
|
191
|
-
* @param {import('@uwdata/mosaic-sql').Query | string | null} [query]
|
|
192
|
-
* The query to issue.
|
|
212
|
+
* @param {MosaicClient} client The client to update.
|
|
213
|
+
* @param {QueryType | null} [query] The query to issue.
|
|
193
214
|
*/
|
|
194
215
|
requestQuery(client, query) {
|
|
195
216
|
this.dataCubeIndexer.clear();
|
|
196
217
|
return query
|
|
197
218
|
? this.updateClient(client, query)
|
|
198
|
-
: client.update();
|
|
219
|
+
: Promise.resolve(client.update());
|
|
199
220
|
}
|
|
200
221
|
|
|
201
222
|
/**
|
|
202
223
|
* Connect a client to the coordinator.
|
|
203
|
-
* @param {
|
|
204
|
-
* client to connect.
|
|
224
|
+
* @param {MosaicClient} client The Mosaic client to connect.
|
|
205
225
|
*/
|
|
206
226
|
async connect(client) {
|
|
207
227
|
const { clients } = this;
|
|
@@ -212,22 +232,27 @@ export class Coordinator {
|
|
|
212
232
|
clients.add(client); // mark as connected
|
|
213
233
|
client.coordinator = this;
|
|
214
234
|
|
|
235
|
+
// initialize client lifecycle
|
|
236
|
+
this.initializeClient(client);
|
|
237
|
+
|
|
238
|
+
// connect filter selection
|
|
239
|
+
connectSelection(this, client.filterBy, client);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
async initializeClient(client) {
|
|
215
243
|
// retrieve field statistics
|
|
216
244
|
const fields = client.fields();
|
|
217
245
|
if (fields?.length) {
|
|
218
246
|
client.fieldInfo(await queryFieldInfo(this, fields));
|
|
219
247
|
}
|
|
220
248
|
|
|
221
|
-
//
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
client.requestQuery();
|
|
249
|
+
// request data query
|
|
250
|
+
return client.requestQuery();
|
|
225
251
|
}
|
|
226
252
|
|
|
227
253
|
/**
|
|
228
254
|
* Disconnect a client from the coordinator.
|
|
229
|
-
* @param {
|
|
230
|
-
* client to disconnect.
|
|
255
|
+
* @param {MosaicClient} client The Mosaic client to disconnect.
|
|
231
256
|
*/
|
|
232
257
|
disconnect(client) {
|
|
233
258
|
const { clients, filterGroups } = this;
|
|
@@ -246,8 +271,8 @@ export class Coordinator {
|
|
|
246
271
|
* Connect a selection-client pair to the coordinator to process updates.
|
|
247
272
|
* @param {Coordinator} mc The Mosaic coordinator.
|
|
248
273
|
* @param {import('./Selection.js').Selection} selection A selection.
|
|
249
|
-
* @param {
|
|
250
|
-
*
|
|
274
|
+
* @param {MosaicClient} client A Mosiac client that is filtered by the
|
|
275
|
+
* given selection.
|
|
251
276
|
*/
|
|
252
277
|
function connectSelection(mc, selection, client) {
|
|
253
278
|
if (!selection) return;
|
package/src/DataCubeIndexer.js
CHANGED
|
@@ -6,59 +6,114 @@ import { fnv_hash } from './util/hash.js';
|
|
|
6
6
|
|
|
7
7
|
const Skip = { skip: true, result: null };
|
|
8
8
|
|
|
9
|
+
/**
|
|
10
|
+
* @typedef {object} DataCubeIndexerOptions
|
|
11
|
+
* @property {string} [schema] Database schema (namespace) in which to write
|
|
12
|
+
* data cube index tables (default 'mosaic').
|
|
13
|
+
* @property {boolean} [options.enabled=true] Flag to enable or disable the
|
|
14
|
+
* indexer. This setting can later be updated via the `enabled` method.
|
|
15
|
+
*/
|
|
16
|
+
|
|
9
17
|
/**
|
|
10
18
|
* Build and query optimized indices ("data cubes") for fast computation of
|
|
11
19
|
* groupby aggregate queries over compatible client queries and selections.
|
|
12
20
|
* A data cube contains pre-aggregated data for a Mosaic client, subdivided
|
|
13
21
|
* by possible query values from an active selection clause. These cubes are
|
|
14
22
|
* realized as as database tables that can be queried for rapid updates.
|
|
23
|
+
*
|
|
15
24
|
* Compatible client queries must consist of only groupby dimensions and
|
|
16
25
|
* supported aggregate functions. Compatible selections must contain an active
|
|
17
26
|
* clause that exposes metadata for an interval or point value predicate.
|
|
27
|
+
*
|
|
28
|
+
* Data cube index tables are written to a dedicated schema (namespace) that
|
|
29
|
+
* can be set using the *schema* constructor option. This schema acts as a
|
|
30
|
+
* persistent cache, and index tables may be used across sessions. The
|
|
31
|
+
* `dropIndexTables` method issues a query to remove *all* tables within
|
|
32
|
+
* this schema. This may be needed if the original tables have updated data,
|
|
33
|
+
* but should be used with care.
|
|
18
34
|
*/
|
|
19
35
|
export class DataCubeIndexer {
|
|
20
36
|
/**
|
|
21
37
|
* Create a new data cube index table manager.
|
|
22
38
|
* @param {import('./Coordinator.js').Coordinator} coordinator A Mosaic coordinator.
|
|
23
|
-
* @param {
|
|
24
|
-
* @param {boolean} [options.enabled=true] Flag to enable/disable indexer.
|
|
25
|
-
* @param {boolean} [options.temp=true] Flag to indicate if generated data
|
|
26
|
-
* cube index tables should be temporary tables.
|
|
39
|
+
* @param {DataCubeIndexerOptions} [options] Data cube indexer options.
|
|
27
40
|
*/
|
|
28
41
|
constructor(coordinator, {
|
|
29
|
-
|
|
30
|
-
|
|
42
|
+
schema = 'mosaic',
|
|
43
|
+
enabled = true
|
|
31
44
|
} = {}) {
|
|
32
45
|
/** @type {Map<import('./MosaicClient.js').MosaicClient, DataCubeInfo | Skip | null>} */
|
|
33
46
|
this.indexes = new Map();
|
|
34
47
|
this.active = null;
|
|
35
|
-
this.temp = temp;
|
|
36
48
|
this.mc = coordinator;
|
|
49
|
+
this._schema = schema;
|
|
37
50
|
this._enabled = enabled;
|
|
38
51
|
}
|
|
39
52
|
|
|
40
53
|
/**
|
|
41
|
-
* Set the enabled state of this indexer. If false, any
|
|
54
|
+
* Set the enabled state of this indexer. If false, any local state is
|
|
42
55
|
* cleared and subsequent index calls will return null until re-enabled.
|
|
43
|
-
*
|
|
56
|
+
* This method has no effect on any index tables already in the database.
|
|
57
|
+
* @param {boolean} [state] The enabled state to set.
|
|
44
58
|
*/
|
|
45
|
-
enabled(state) {
|
|
46
|
-
if (
|
|
47
|
-
return this._enabled;
|
|
48
|
-
} else if (this._enabled !== state) {
|
|
59
|
+
set enabled(state) {
|
|
60
|
+
if (this._enabled !== state) {
|
|
49
61
|
if (!state) this.clear();
|
|
50
62
|
this._enabled = state;
|
|
51
63
|
}
|
|
52
64
|
}
|
|
53
65
|
|
|
66
|
+
/**
|
|
67
|
+
* Get the enabled state of this indexer.
|
|
68
|
+
* @returns {boolean} The current enabled state.
|
|
69
|
+
*/
|
|
70
|
+
get enabled() {
|
|
71
|
+
return this._enabled;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Set the database schema used by this indexer. Upon changes, any local
|
|
76
|
+
* state is cleared. This method does _not_ drop any existing data cube
|
|
77
|
+
* tables, use `dropIndexTables` before changing the schema to also remove
|
|
78
|
+
* existing index tables in the database.
|
|
79
|
+
* @param {string} [schema] The schema name to set.
|
|
80
|
+
*/
|
|
81
|
+
set schema(schema) {
|
|
82
|
+
if (this._schema !== schema) {
|
|
83
|
+
this.clear();
|
|
84
|
+
this._schema = schema;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Get the database schema used by this indexer.
|
|
90
|
+
* @returns {string} The current schema name.
|
|
91
|
+
*/
|
|
92
|
+
get schema() {
|
|
93
|
+
return this._schema;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Issues a query through the coordinator to drop the current index table
|
|
98
|
+
* schema. *All* tables in the schema will be removed and local state is
|
|
99
|
+
* cleared. Call this method if the underlying base tables have been updated,
|
|
100
|
+
* causing derived index tables to become stale and inaccurate. Use this
|
|
101
|
+
* method with care! Once dropped, the schema will be repopulated by future
|
|
102
|
+
* data cube indexer requests.
|
|
103
|
+
* @returns A query result promise.
|
|
104
|
+
*/
|
|
105
|
+
dropIndexTables() {
|
|
106
|
+
this.clear();
|
|
107
|
+
return this.mc.exec(`DROP SCHEMA IF EXISTS "${this.schema}" CASCADE`);
|
|
108
|
+
}
|
|
109
|
+
|
|
54
110
|
/**
|
|
55
111
|
* Clear the cache of data cube index table entries for the current active
|
|
56
|
-
* selection clause. This method
|
|
57
|
-
*
|
|
58
|
-
*
|
|
112
|
+
* selection clause. This method does _not_ drop any existing data cube
|
|
113
|
+
* tables. Use `dropIndexTables` to remove existing index tables from the
|
|
114
|
+
* database.
|
|
59
115
|
*/
|
|
60
116
|
clear() {
|
|
61
|
-
this.mc.cancel(Array.from(this.indexes.values(), info => info?.result));
|
|
62
117
|
this.indexes.clear();
|
|
63
118
|
this.active = null;
|
|
64
119
|
}
|
|
@@ -79,9 +134,9 @@ export class DataCubeIndexer {
|
|
|
79
134
|
*/
|
|
80
135
|
index(client, selection, activeClause) {
|
|
81
136
|
// if not enabled, do nothing
|
|
82
|
-
if (!this.
|
|
137
|
+
if (!this.enabled) return null;
|
|
83
138
|
|
|
84
|
-
const { indexes, mc,
|
|
139
|
+
const { indexes, mc, schema } = this;
|
|
85
140
|
const { source } = activeClause;
|
|
86
141
|
|
|
87
142
|
// if there is no clause source to track, do nothing
|
|
@@ -127,8 +182,11 @@ export class DataCubeIndexer {
|
|
|
127
182
|
} else {
|
|
128
183
|
// generate data cube index table
|
|
129
184
|
const filter = selection.remove(source).predicate(client);
|
|
130
|
-
info = dataCubeInfo(client.query(filter), active, indexCols);
|
|
131
|
-
info.result = mc.exec(
|
|
185
|
+
info = dataCubeInfo(client.query(filter), active, indexCols, schema);
|
|
186
|
+
info.result = mc.exec([
|
|
187
|
+
`CREATE SCHEMA IF NOT EXISTS ${schema}`,
|
|
188
|
+
create(info.table, info.create, { temp: false })
|
|
189
|
+
]);
|
|
132
190
|
info.result.catch(e => mc.logger().error(e));
|
|
133
191
|
}
|
|
134
192
|
|
|
@@ -225,7 +283,7 @@ function binInterval(scale, pixelSize, bin) {
|
|
|
225
283
|
* @param {*} indexCols Data cube index column definitions.
|
|
226
284
|
* @returns {DataCubeInfo}
|
|
227
285
|
*/
|
|
228
|
-
function dataCubeInfo(clientQuery, active, indexCols) {
|
|
286
|
+
function dataCubeInfo(clientQuery, active, indexCols, schema) {
|
|
229
287
|
const { dims, aggr, aux } = indexCols;
|
|
230
288
|
const { columns } = active;
|
|
231
289
|
|
|
@@ -248,7 +306,7 @@ function dataCubeInfo(clientQuery, active, indexCols) {
|
|
|
248
306
|
// generate creation query string and hash id
|
|
249
307
|
const create = query.toString();
|
|
250
308
|
const id = (fnv_hash(create) >>> 0).toString(16);
|
|
251
|
-
const table =
|
|
309
|
+
const table = `${schema}.cube_${id}`;
|
|
252
310
|
|
|
253
311
|
// generate data cube select query
|
|
254
312
|
const select = Query
|
|
@@ -257,7 +315,7 @@ function dataCubeInfo(clientQuery, active, indexCols) {
|
|
|
257
315
|
.groupby(dims)
|
|
258
316
|
.orderby(order);
|
|
259
317
|
|
|
260
|
-
return new DataCubeInfo({ table, create, active, select });
|
|
318
|
+
return new DataCubeInfo({ id, table, create, active, select });
|
|
261
319
|
}
|
|
262
320
|
|
|
263
321
|
/**
|
package/src/MosaicClient.js
CHANGED
|
@@ -103,6 +103,7 @@ export class MosaicClient {
|
|
|
103
103
|
* Request the coordinator to execute a query for this client.
|
|
104
104
|
* If an explicit query is not provided, the client query method will
|
|
105
105
|
* be called, filtered by the current filterBy selection.
|
|
106
|
+
* @returns {Promise}
|
|
106
107
|
*/
|
|
107
108
|
requestQuery(query) {
|
|
108
109
|
const q = query || this.query(this.filterBy?.predicate(this));
|
|
@@ -119,6 +120,14 @@ export class MosaicClient {
|
|
|
119
120
|
this._requestUpdate();
|
|
120
121
|
}
|
|
121
122
|
|
|
123
|
+
/**
|
|
124
|
+
* Reset this client, initiating new field info and query requests.
|
|
125
|
+
* @returns {Promise}
|
|
126
|
+
*/
|
|
127
|
+
initialize() {
|
|
128
|
+
return this._coordinator.initializeClient(this);
|
|
129
|
+
}
|
|
130
|
+
|
|
122
131
|
/**
|
|
123
132
|
* Requests a client update.
|
|
124
133
|
* For example to (re-)render an interface component.
|
package/src/QueryConsolidator.js
CHANGED
|
@@ -108,6 +108,12 @@ function consolidationKey(query, cache) {
|
|
|
108
108
|
// @ts-ignore
|
|
109
109
|
q.$groupby(groupby.map(e => (e instanceof Ref && map[e.column]) || e));
|
|
110
110
|
}
|
|
111
|
+
// @ts-ignore
|
|
112
|
+
else if (query.select().some(({ expr }) => expr.aggregate)) {
|
|
113
|
+
// if query is an ungrouped aggregate, add an explicit groupby to
|
|
114
|
+
// prevent improper consolidation with non-aggregate queries
|
|
115
|
+
q.$groupby('ALL');
|
|
116
|
+
}
|
|
111
117
|
|
|
112
118
|
// key is just the transformed query as SQL
|
|
113
119
|
return `${q}`;
|
|
@@ -244,22 +250,20 @@ async function processResults(group, cache) {
|
|
|
244
250
|
|
|
245
251
|
/**
|
|
246
252
|
* Project a consolidated result to a client result
|
|
247
|
-
* @param {
|
|
248
|
-
*
|
|
253
|
+
* @param {import('@uwdata/flechette').Table} data
|
|
254
|
+
* Consolidated query result, as an Arrow Table
|
|
255
|
+
* @param {[string, string][]} map Column name map as [source, target] pairs
|
|
249
256
|
* @returns the projected Apache Arrow table
|
|
250
257
|
*/
|
|
251
258
|
function projectResult(data, map) {
|
|
252
|
-
|
|
253
|
-
for (const [name, as] of map) {
|
|
254
|
-
cols[as] = data.getChild(name);
|
|
255
|
-
}
|
|
256
|
-
return new data.constructor(cols);
|
|
259
|
+
return data.select(map.map(x => x[0]), map.map(x => x[1]));
|
|
257
260
|
}
|
|
258
261
|
|
|
259
262
|
/**
|
|
260
263
|
* Filter a consolidated describe query result to a client result
|
|
261
|
-
* @param {
|
|
262
|
-
*
|
|
264
|
+
* @param {import('@uwdata/flechette').Table} data
|
|
265
|
+
* Consolidated query result, as an Arrow Table
|
|
266
|
+
* @param {[string, string][]} map Column name map as [source, target] pairs
|
|
263
267
|
* @returns the filtered table data
|
|
264
268
|
*/
|
|
265
269
|
function filterResult(data, map) {
|
package/src/QueryManager.js
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { consolidator } from './QueryConsolidator.js';
|
|
2
2
|
import { lruCache, voidCache } from './util/cache.js';
|
|
3
|
-
import {
|
|
3
|
+
import { PriorityQueue } from './util/priority-queue.js';
|
|
4
4
|
import { QueryResult } from './util/query-result.js';
|
|
5
5
|
|
|
6
6
|
export const Priority = { High: 0, Normal: 1, Low: 2 };
|
|
7
7
|
|
|
8
8
|
export class QueryManager {
|
|
9
9
|
constructor() {
|
|
10
|
-
this.queue =
|
|
10
|
+
this.queue = new PriorityQueue(3);
|
|
11
11
|
this.db = null;
|
|
12
12
|
this.clientCache = null;
|
|
13
13
|
this._logger = null;
|
|
@@ -109,7 +109,13 @@ export class QueryManager {
|
|
|
109
109
|
cancel(requests) {
|
|
110
110
|
const set = new Set(requests);
|
|
111
111
|
if (set.size) {
|
|
112
|
-
this.queue.remove(({ result }) =>
|
|
112
|
+
this.queue.remove(({ result }) => {
|
|
113
|
+
if (set.has(result)) {
|
|
114
|
+
result.reject('Canceled');
|
|
115
|
+
return true;
|
|
116
|
+
}
|
|
117
|
+
return false;
|
|
118
|
+
});
|
|
113
119
|
}
|
|
114
120
|
}
|
|
115
121
|
|
package/src/Selection.js
CHANGED
|
@@ -10,6 +10,13 @@ export function isSelection(x) {
|
|
|
10
10
|
return x instanceof Selection;
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
+
function create(options, include) {
|
|
14
|
+
return new Selection(
|
|
15
|
+
new SelectionResolver(options),
|
|
16
|
+
include ? [include].flat() : include
|
|
17
|
+
);
|
|
18
|
+
}
|
|
19
|
+
|
|
13
20
|
/**
|
|
14
21
|
* Represents a dynamic set of query filter predicates.
|
|
15
22
|
*/
|
|
@@ -25,10 +32,13 @@ export class Selection extends Param {
|
|
|
25
32
|
* @param {boolean} [options.empty=false] Boolean flag indicating if a lack
|
|
26
33
|
* of clauses should correspond to an empty selection with no records. This
|
|
27
34
|
* setting determines the default selection state.
|
|
35
|
+
* @param {Selection|Selection[]} [options.include] Upstream selections whose
|
|
36
|
+
* clauses should be included as part of the new selection. Any clauses
|
|
37
|
+
* published to upstream selections will be relayed to the new selection.
|
|
28
38
|
* @returns {Selection} The new Selection instance.
|
|
29
39
|
*/
|
|
30
|
-
static intersect({ cross = false, empty = false } = {}) {
|
|
31
|
-
return
|
|
40
|
+
static intersect({ cross = false, empty = false, include = [] } = {}) {
|
|
41
|
+
return create({ cross, empty }, include);
|
|
32
42
|
}
|
|
33
43
|
|
|
34
44
|
/**
|
|
@@ -41,10 +51,13 @@ export class Selection extends Param {
|
|
|
41
51
|
* @param {boolean} [options.empty=false] Boolean flag indicating if a lack
|
|
42
52
|
* of clauses should correspond to an empty selection with no records. This
|
|
43
53
|
* setting determines the default selection state.
|
|
54
|
+
* @param {Selection|Selection[]} [options.include] Upstream selections whose
|
|
55
|
+
* clauses should be included as part of the new selection. Any clauses
|
|
56
|
+
* published to upstream selections will be relayed to the new selection.
|
|
44
57
|
* @returns {Selection} The new Selection instance.
|
|
45
58
|
*/
|
|
46
|
-
static union({ cross = false, empty = false } = {}) {
|
|
47
|
-
return
|
|
59
|
+
static union({ cross = false, empty = false, include = [] } = {}) {
|
|
60
|
+
return create({ cross, empty, union: true }, include);
|
|
48
61
|
}
|
|
49
62
|
|
|
50
63
|
/**
|
|
@@ -57,10 +70,13 @@ export class Selection extends Param {
|
|
|
57
70
|
* @param {boolean} [options.empty=false] Boolean flag indicating if a lack
|
|
58
71
|
* of clauses should correspond to an empty selection with no records. This
|
|
59
72
|
* setting determines the default selection state.
|
|
73
|
+
* @param {Selection|Selection[]} [options.include] Upstream selections whose
|
|
74
|
+
* clauses should be included as part of the new selection. Any clauses
|
|
75
|
+
* published to upstream selections will be relayed to the new selection.
|
|
60
76
|
* @returns {Selection} The new Selection instance.
|
|
61
77
|
*/
|
|
62
|
-
static single({ cross = false, empty = false } = {}) {
|
|
63
|
-
return
|
|
78
|
+
static single({ cross = false, empty = false, include = [] } = {}) {
|
|
79
|
+
return create({ cross, empty, single: true }, include);
|
|
64
80
|
}
|
|
65
81
|
|
|
66
82
|
/**
|
|
@@ -70,21 +86,34 @@ export class Selection extends Param {
|
|
|
70
86
|
* @param {boolean} [options.empty=false] Boolean flag indicating if a lack
|
|
71
87
|
* of clauses should correspond to an empty selection with no records. This
|
|
72
88
|
* setting determines the default selection state.
|
|
89
|
+
* @param {Selection|Selection[]} [options.include] Upstream selections whose
|
|
90
|
+
* clauses should be included as part of the new selection. Any clauses
|
|
91
|
+
* published to upstream selections will be relayed to the new selection.
|
|
73
92
|
* @returns {Selection} The new Selection instance.
|
|
74
93
|
*/
|
|
75
|
-
static crossfilter({ empty = false } = {}) {
|
|
76
|
-
return
|
|
94
|
+
static crossfilter({ empty = false, include = [] } = {}) {
|
|
95
|
+
return create({ cross: true, empty }, include);
|
|
77
96
|
}
|
|
78
97
|
|
|
79
98
|
/**
|
|
80
99
|
* Create a new Selection instance.
|
|
81
|
-
* @param {SelectionResolver} resolver The selection resolution
|
|
100
|
+
* @param {SelectionResolver} [resolver] The selection resolution
|
|
82
101
|
* strategy to apply.
|
|
102
|
+
* @param {Selection[]} [include] Upstream selections whose clauses
|
|
103
|
+
* should be included as part of this selection. Any clauses published
|
|
104
|
+
* to these upstream selections will be relayed to this selection.
|
|
83
105
|
*/
|
|
84
|
-
constructor(resolver = new SelectionResolver()) {
|
|
106
|
+
constructor(resolver = new SelectionResolver(), include = []) {
|
|
85
107
|
super([]);
|
|
86
108
|
this._resolved = this._value;
|
|
87
109
|
this._resolver = resolver;
|
|
110
|
+
/** @type {Set<Selection>} */
|
|
111
|
+
this._relay = new Set;
|
|
112
|
+
if (Array.isArray(include)) {
|
|
113
|
+
for (const sel of include) {
|
|
114
|
+
sel._relay.add(this);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
88
117
|
}
|
|
89
118
|
|
|
90
119
|
/**
|
|
@@ -161,6 +190,7 @@ export class Selection extends Param {
|
|
|
161
190
|
*/
|
|
162
191
|
activate(clause) {
|
|
163
192
|
this.emit('activate', clause);
|
|
193
|
+
this._relay.forEach(sel => sel.activate(clause));
|
|
164
194
|
}
|
|
165
195
|
|
|
166
196
|
/**
|
|
@@ -173,6 +203,7 @@ export class Selection extends Param {
|
|
|
173
203
|
// this ensures consistent clause state across unemitted event values
|
|
174
204
|
this._resolved = this._resolver.resolve(this._resolved, clause, true);
|
|
175
205
|
this._resolved.active = clause;
|
|
206
|
+
this._relay.forEach(sel => sel.update(clause));
|
|
176
207
|
return super.update(this._resolved);
|
|
177
208
|
}
|
|
178
209
|
|
package/src/connectors/rest.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { decodeIPC } from '../util/decode-ipc.js';
|
|
2
2
|
|
|
3
3
|
export function restConnector(uri = 'http://localhost:3000/') {
|
|
4
4
|
return {
|
|
5
5
|
/**
|
|
6
6
|
* Query the DuckDB server.
|
|
7
7
|
* @param {object} query
|
|
8
|
-
* @param {'exec' | 'arrow' | 'json'} [query.type] The query type
|
|
8
|
+
* @param {'exec' | 'arrow' | 'json' | 'create-bundle' | 'load-bundle'} [query.type] The query type.
|
|
9
9
|
* @param {string} query.sql A SQL query string.
|
|
10
10
|
* @returns the query result
|
|
11
11
|
*/
|
|
@@ -20,7 +20,7 @@ export function restConnector(uri = 'http://localhost:3000/') {
|
|
|
20
20
|
});
|
|
21
21
|
|
|
22
22
|
return query.type === 'exec' ? req
|
|
23
|
-
: query.type === 'arrow' ?
|
|
23
|
+
: query.type === 'arrow' ? decodeIPC(await (await req).arrayBuffer())
|
|
24
24
|
: (await req).json();
|
|
25
25
|
}
|
|
26
26
|
};
|