@visactor/vquery 0.3.13 → 0.3.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/esm/VQuery.d.ts +21 -0
- package/dist/browser/esm/browser.d.ts +3 -1
- package/dist/browser/esm/browser.js +121 -129
- package/dist/browser/esm/data-source-builder/dataSourceBuilder.d.ts +2 -3
- package/dist/browser/esm/dataset/dataset.d.ts +21 -3
- package/dist/browser/esm/node.d.ts +3 -1
- package/dist/browser/esm/sql-builder/builders/index.d.ts +1 -0
- package/dist/browser/esm/sql-builder/builders/select.d.ts +3 -0
- package/dist/browser/esm/sql-builder/dslToSQL.d.ts +2 -2
- package/dist/browser/esm/types/DataSource.d.ts +1 -1
- package/dist/browser/esm/types/dsl/GroupBy.d.ts +1 -1
- package/dist/browser/esm/types/dsl/Select.d.ts +7 -2
- package/dist/browser/esm/vquery-browser.d.ts +2 -19
- package/dist/browser/esm/vquery-node.d.ts +2 -19
- package/dist/node/cjs/VQuery.d.ts +21 -0
- package/dist/node/cjs/browser.d.ts +3 -1
- package/dist/node/cjs/data-source-builder/dataSourceBuilder.d.ts +2 -3
- package/dist/node/cjs/dataset/dataset.d.ts +21 -3
- package/dist/node/cjs/node.cjs +149 -164
- package/dist/node/cjs/node.d.ts +3 -1
- package/dist/node/cjs/sql-builder/builders/index.d.ts +1 -0
- package/dist/node/cjs/sql-builder/builders/select.d.ts +3 -0
- package/dist/node/cjs/sql-builder/dslToSQL.d.ts +2 -2
- package/dist/node/cjs/types/DataSource.d.ts +1 -1
- package/dist/node/cjs/types/dsl/GroupBy.d.ts +1 -1
- package/dist/node/cjs/types/dsl/Select.d.ts +7 -2
- package/dist/node/cjs/vquery-browser.d.ts +2 -19
- package/dist/node/cjs/vquery-node.d.ts +2 -19
- package/dist/node/esm/VQuery.d.ts +21 -0
- package/dist/node/esm/browser.d.ts +3 -1
- package/dist/node/esm/data-source-builder/dataSourceBuilder.d.ts +2 -3
- package/dist/node/esm/dataset/dataset.d.ts +21 -3
- package/dist/node/esm/node.d.ts +3 -1
- package/dist/node/esm/node.js +138 -162
- package/dist/node/esm/sql-builder/builders/index.d.ts +1 -0
- package/dist/node/esm/sql-builder/builders/select.d.ts +3 -0
- package/dist/node/esm/sql-builder/dslToSQL.d.ts +2 -2
- package/dist/node/esm/types/DataSource.d.ts +1 -1
- package/dist/node/esm/types/dsl/GroupBy.d.ts +1 -1
- package/dist/node/esm/types/dsl/Select.d.ts +7 -2
- package/dist/node/esm/vquery-browser.d.ts +2 -19
- package/dist/node/esm/vquery-node.d.ts +2 -19
- package/package.json +5 -2
- package/dist/browser/esm/types/dsl/demo.d.ts +0 -1
- package/dist/node/cjs/types/dsl/demo.d.ts +0 -1
- package/dist/node/esm/types/dsl/demo.d.ts +0 -1
package/dist/node/cjs/node.cjs
CHANGED
|
@@ -27,12 +27,15 @@ var __webpack_require__ = {};
|
|
|
27
27
|
var __webpack_exports__ = {};
|
|
28
28
|
__webpack_require__.r(__webpack_exports__);
|
|
29
29
|
__webpack_require__.d(__webpack_exports__, {
|
|
30
|
+
isUrl: ()=>isUrl,
|
|
31
|
+
Dataset: ()=>Dataset,
|
|
32
|
+
isBase64Url: ()=>isBase64Url,
|
|
33
|
+
DuckDBNodeQueryAdapter: ()=>DuckDBNodeQueryAdapter,
|
|
30
34
|
DatasetSourceBuilder: ()=>DatasetSourceBuilder,
|
|
31
35
|
isHttpUrl: ()=>isHttpUrl,
|
|
32
36
|
convertDSLToSQL: ()=>convertDSLToSQL,
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
VQuery: ()=>VQuery
|
|
37
|
+
InMemoryAdapter: ()=>InMemoryAdapter,
|
|
38
|
+
VQuery: ()=>vquery_node_VQuery
|
|
36
39
|
});
|
|
37
40
|
const external_kysely_namespaceObject = require("kysely");
|
|
38
41
|
class PostgresDialect {
|
|
@@ -118,12 +121,12 @@ const applyWhere = (where)=>{
|
|
|
118
121
|
case 'between':
|
|
119
122
|
{
|
|
120
123
|
const [a, b] = value;
|
|
121
|
-
return (0, external_kysely_namespaceObject.sql)`${external_kysely_namespaceObject.sql.ref(field)} between
|
|
124
|
+
return (0, external_kysely_namespaceObject.sql)`${external_kysely_namespaceObject.sql.ref(field)} between ${external_kysely_namespaceObject.sql.val(a)} and ${external_kysely_namespaceObject.sql.val(b)}`;
|
|
122
125
|
}
|
|
123
126
|
case 'not between':
|
|
124
127
|
{
|
|
125
128
|
const [a, b] = value;
|
|
126
|
-
return (0, external_kysely_namespaceObject.sql)
|
|
129
|
+
return (0, external_kysely_namespaceObject.sql)`${external_kysely_namespaceObject.sql.ref(field)} not between ${external_kysely_namespaceObject.sql.val(a)} and ${external_kysely_namespaceObject.sql.val(b)}`;
|
|
127
130
|
}
|
|
128
131
|
default:
|
|
129
132
|
return (0, external_kysely_namespaceObject.sql)`${external_kysely_namespaceObject.sql.ref(field)} ${external_kysely_namespaceObject.sql.raw(leaf.op)} ${external_kysely_namespaceObject.sql.val(value)}`;
|
|
@@ -142,33 +145,62 @@ const applyLimit = (qb, limit)=>{
|
|
|
142
145
|
if (limit && 'number' == typeof limit) qb = qb.limit(limit);
|
|
143
146
|
return qb;
|
|
144
147
|
};
|
|
145
|
-
const
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
148
|
+
const DATE_FORMAT_MAP = {
|
|
149
|
+
year: '%Y',
|
|
150
|
+
month: '%Y-%m',
|
|
151
|
+
day: '%Y-%m-%d',
|
|
152
|
+
week: '%Y-W%W',
|
|
153
|
+
hour: '%Y-%m-%d %H',
|
|
154
|
+
minute: '%Y-%m-%d %H:%M',
|
|
155
|
+
second: '%Y-%m-%d %H:%M:%S'
|
|
156
|
+
};
|
|
157
|
+
const applySelect = (qb, select)=>{
|
|
158
|
+
if (select && select.length > 0) return qb.select((eb)=>select.map((item)=>{
|
|
151
159
|
if (isSelectItem(item)) {
|
|
152
160
|
const field = item.field;
|
|
153
|
-
|
|
161
|
+
const expression = eb.ref(field);
|
|
162
|
+
if (item.aggr) {
|
|
163
|
+
const { func } = item.aggr;
|
|
154
164
|
const alias = item.alias ?? field;
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
165
|
+
if ([
|
|
166
|
+
'avg',
|
|
167
|
+
'sum',
|
|
168
|
+
'min',
|
|
169
|
+
'max',
|
|
170
|
+
'variance',
|
|
171
|
+
'variancePop',
|
|
172
|
+
'stddev',
|
|
173
|
+
'median'
|
|
174
|
+
].includes(func)) {
|
|
175
|
+
if ('variance' === func) return (0, external_kysely_namespaceObject.sql)`var_samp(${expression})`.as(alias);
|
|
176
|
+
if ('variancePop' === func) return (0, external_kysely_namespaceObject.sql)`var_pop(${expression})`.as(alias);
|
|
177
|
+
return (0, external_kysely_namespaceObject.sql)`${external_kysely_namespaceObject.sql.raw(func)}(${expression})`.as(alias);
|
|
178
|
+
}
|
|
179
|
+
if ('count' === func) return (0, external_kysely_namespaceObject.sql)`CAST(count(${expression}) AS INTEGER)`.as(alias);
|
|
180
|
+
if ('quantile' === func) {
|
|
181
|
+
const q = item.aggr.quantile ?? 0.5;
|
|
182
|
+
return (0, external_kysely_namespaceObject.sql)`quantile(${expression}, ${q})`.as(alias);
|
|
183
|
+
} else if ('count_distinct' === func) return (0, external_kysely_namespaceObject.sql)`CAST(count(distinct ${expression}) AS INTEGER)`.as(alias);
|
|
184
|
+
else if (func.startsWith('to_')) {
|
|
185
|
+
const dateTrunc = func.replace('to_', '');
|
|
186
|
+
const format = DATE_FORMAT_MAP[dateTrunc];
|
|
187
|
+
if (format) return (0, external_kysely_namespaceObject.sql)`strftime(CAST(${expression} AS TIMESTAMP), ${format})`.as(alias);
|
|
188
|
+
if ('quarter' === dateTrunc) return (0, external_kysely_namespaceObject.sql)`strftime(CAST(${expression} AS TIMESTAMP), '%Y') || '-Q' || date_part('quarter', CAST(${expression} AS TIMESTAMP))`.as(alias);
|
|
166
189
|
}
|
|
167
190
|
}
|
|
168
|
-
|
|
191
|
+
const alias = item.alias ?? field;
|
|
192
|
+
return expression.as(alias);
|
|
169
193
|
}
|
|
170
194
|
return item;
|
|
171
|
-
}))
|
|
195
|
+
}));
|
|
196
|
+
return qb.selectAll();
|
|
197
|
+
};
|
|
198
|
+
const convertDSLToSQL = (dsl, tableName)=>{
|
|
199
|
+
const db = new external_kysely_namespaceObject.Kysely({
|
|
200
|
+
dialect: new PostgresDialect()
|
|
201
|
+
});
|
|
202
|
+
let qb = db.selectFrom(tableName);
|
|
203
|
+
qb = applySelect(qb, dsl.select);
|
|
172
204
|
if (dsl.where) qb = qb.where(applyWhere(dsl.where));
|
|
173
205
|
qb = applyGroupBy(qb, dsl.groupBy);
|
|
174
206
|
if (dsl.orderBy && dsl.orderBy.length > 0) for (const o of dsl.orderBy)qb = qb.orderBy(o.field, o.order ?? 'asc');
|
|
@@ -176,60 +208,60 @@ const convertDSLToSQL = (dsl, tableName)=>{
|
|
|
176
208
|
const compiled = qb.compile();
|
|
177
209
|
return inlineParameters(compiled.sql, compiled.parameters);
|
|
178
210
|
};
|
|
211
|
+
const READ_FUNCTION_MAP = {
|
|
212
|
+
csv: 'read_csv_auto',
|
|
213
|
+
json: 'read_json_auto',
|
|
214
|
+
parquet: 'read_parquet'
|
|
215
|
+
};
|
|
216
|
+
const DATA_TYPE_MAP = {
|
|
217
|
+
number: 'DOUBLE',
|
|
218
|
+
string: 'VARCHAR',
|
|
219
|
+
date: 'DATE',
|
|
220
|
+
datetime: 'TIMESTAMP',
|
|
221
|
+
timestamp: 'TIMESTAMP'
|
|
222
|
+
};
|
|
179
223
|
class Dataset {
|
|
180
224
|
queryAdapter;
|
|
181
225
|
storageAdapter;
|
|
182
226
|
_datasetId;
|
|
183
|
-
constructor(
|
|
184
|
-
this.queryAdapter =
|
|
185
|
-
this.storageAdapter =
|
|
227
|
+
constructor(queryAdapter, storageAdapter, datasetId){
|
|
228
|
+
this.queryAdapter = queryAdapter;
|
|
229
|
+
this.storageAdapter = storageAdapter;
|
|
186
230
|
this._datasetId = datasetId;
|
|
187
231
|
}
|
|
188
232
|
async init(temporaryColumns, temporaryDatasetSource) {
|
|
189
233
|
const datasetInfo = await this.storageAdapter.readDataset(this._datasetId);
|
|
190
234
|
if (!datasetInfo) throw new Error(`Dataset ${this._datasetId} not found`);
|
|
191
|
-
const columns = temporaryColumns
|
|
192
|
-
const datasetSource = temporaryDatasetSource
|
|
235
|
+
const columns = temporaryColumns ?? datasetInfo.datasetSchema.columns;
|
|
236
|
+
const datasetSource = temporaryDatasetSource ?? datasetInfo.datasetSource;
|
|
193
237
|
if (columns.length > 0 && datasetSource) await this.createOrReplaceView(columns, datasetSource);
|
|
194
238
|
}
|
|
195
239
|
async createOrReplaceView(columns, datasetSource) {
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
};
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
datetime: 'TIMESTAMP',
|
|
207
|
-
timestamp: 'TIMESTAMP'
|
|
208
|
-
};
|
|
209
|
-
if (datasetSource) {
|
|
210
|
-
const readFunction = readFunctionMap[datasetSource.type];
|
|
211
|
-
if (!readFunction) throw new Error(`Unsupported dataSource type: ${datasetSource.type}`);
|
|
212
|
-
await this.queryAdapter.writeFile(this._datasetId, datasetSource.blob);
|
|
213
|
-
const columnsStruct = `{${columns.map((c)=>`'${c.name}': '${dataTypeMap[c.type] || 'VARCHAR'}'`).join(', ')}}`;
|
|
214
|
-
const columnNames = columns.map((c)=>`"${c.name}"`).join(', ');
|
|
215
|
-
const createViewSql = `CREATE OR REPLACE VIEW "${this._datasetId}" AS SELECT ${columnNames} FROM ${readFunction}('${this._datasetId}', columns=${columnsStruct})`;
|
|
216
|
-
await this.queryAdapter.query(createViewSql);
|
|
217
|
-
}
|
|
240
|
+
if (!datasetSource) return;
|
|
241
|
+
const readFunction = READ_FUNCTION_MAP[datasetSource.type];
|
|
242
|
+
if (!readFunction) throw new Error(`Unsupported dataSource type: ${datasetSource.type}`);
|
|
243
|
+
await this.queryAdapter.writeFile(this._datasetId, datasetSource.blob);
|
|
244
|
+
const columnsStruct = this.buildColumnsStruct(columns);
|
|
245
|
+
const columnNames = columns.map((c)=>`"${c.name}"`).join(', ');
|
|
246
|
+
let readSql = `${readFunction}('${this._datasetId}')`;
|
|
247
|
+
if ('csv' === datasetSource.type || 'json' === datasetSource.type) readSql = `${readFunction}('${this._datasetId}', columns=${columnsStruct})`;
|
|
248
|
+
const createViewSql = `CREATE OR REPLACE VIEW "${this._datasetId}" AS SELECT ${columnNames} FROM ${readSql}`;
|
|
249
|
+
await this.queryAdapter.query(createViewSql);
|
|
218
250
|
}
|
|
219
251
|
async query(queryDSL) {
|
|
220
252
|
const sql = convertDSLToSQL(queryDSL, this.datasetId);
|
|
221
253
|
return this.queryBySQL(sql);
|
|
222
254
|
}
|
|
223
255
|
async queryBySQL(sql) {
|
|
224
|
-
const start = performance?.now?.()
|
|
256
|
+
const start = performance?.now?.() ?? Date.now();
|
|
225
257
|
const result = await this.queryAdapter.query(sql);
|
|
226
|
-
const end = performance?.now?.()
|
|
258
|
+
const end = performance?.now?.() ?? Date.now();
|
|
227
259
|
return {
|
|
228
260
|
...result,
|
|
229
261
|
performance: {
|
|
230
|
-
startAt: start,
|
|
231
|
-
endAt: end,
|
|
232
|
-
duration:
|
|
262
|
+
startAt: start.toFixed(3),
|
|
263
|
+
endAt: end.toFixed(3),
|
|
264
|
+
duration: end - start
|
|
233
265
|
}
|
|
234
266
|
};
|
|
235
267
|
}
|
|
@@ -239,10 +271,22 @@ class Dataset {
|
|
|
239
271
|
get datasetId() {
|
|
240
272
|
return this._datasetId;
|
|
241
273
|
}
|
|
274
|
+
buildColumnsStruct(columns) {
|
|
275
|
+
const columnDefs = columns.map((c)=>{
|
|
276
|
+
const duckDBType = DATA_TYPE_MAP[c.type] || 'VARCHAR';
|
|
277
|
+
return `'${c.name}': '${duckDBType}'`;
|
|
278
|
+
});
|
|
279
|
+
return `{${columnDefs.join(', ')}}`;
|
|
280
|
+
}
|
|
242
281
|
}
|
|
243
282
|
const isUrl = (url)=>isHttpUrl(url) || isBase64Url(url);
|
|
244
283
|
const isHttpUrl = (url)=>url.startsWith('http://') || url.startsWith('https://');
|
|
245
284
|
const isBase64Url = (url)=>url.startsWith('data:');
|
|
285
|
+
const MIME_TYPES = {
|
|
286
|
+
csv: 'text/csv',
|
|
287
|
+
json: 'application/json',
|
|
288
|
+
parquet: 'application/parquet'
|
|
289
|
+
};
|
|
246
290
|
class DatasetSourceBuilder {
|
|
247
291
|
type;
|
|
248
292
|
value;
|
|
@@ -254,84 +298,30 @@ class DatasetSourceBuilder {
|
|
|
254
298
|
return new DatasetSourceBuilder(raw);
|
|
255
299
|
}
|
|
256
300
|
async build() {
|
|
257
|
-
const blob = await
|
|
301
|
+
const blob = await this.convertToBlob(this.type, this.value);
|
|
258
302
|
return {
|
|
259
303
|
type: this.type,
|
|
260
304
|
blob: blob
|
|
261
305
|
};
|
|
262
306
|
}
|
|
263
|
-
|
|
307
|
+
async convertToBlob(type, value) {
|
|
264
308
|
if (value instanceof Blob) return value;
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
], {
|
|
269
|
-
type: 'text/csv'
|
|
270
|
-
});
|
|
271
|
-
if ('string' == typeof csvSource && isUrl(csvSource)) return DatasetSourceBuilder.fetchBlob(csvSource);
|
|
272
|
-
return new Blob([
|
|
273
|
-
JSON.stringify(csvSource)
|
|
274
|
-
], {
|
|
275
|
-
type: 'text/csv'
|
|
276
|
-
});
|
|
277
|
-
};
|
|
278
|
-
const convertJsonToBlob = (jsonSource)=>{
|
|
279
|
-
if (jsonSource instanceof ArrayBuffer) return new Blob([
|
|
280
|
-
jsonSource
|
|
281
|
-
], {
|
|
282
|
-
type: 'application/json'
|
|
283
|
-
});
|
|
284
|
-
if ('string' == typeof jsonSource && isUrl(jsonSource)) return DatasetSourceBuilder.fetchBlob(jsonSource);
|
|
285
|
-
return new Blob([
|
|
286
|
-
JSON.stringify(jsonSource)
|
|
287
|
-
], {
|
|
288
|
-
type: 'application/json'
|
|
289
|
-
});
|
|
290
|
-
};
|
|
291
|
-
const convertParquetToBlob = (parquetSource)=>{
|
|
292
|
-
if (parquetSource instanceof ArrayBuffer) return new Blob([
|
|
293
|
-
parquetSource
|
|
294
|
-
], {
|
|
295
|
-
type: 'application/parquet'
|
|
296
|
-
});
|
|
297
|
-
if ('string' == typeof parquetSource && isUrl(parquetSource)) return DatasetSourceBuilder.fetchBlob(parquetSource);
|
|
298
|
-
return new Blob([
|
|
299
|
-
parquetSource
|
|
300
|
-
], {
|
|
301
|
-
type: 'application/parquet'
|
|
302
|
-
});
|
|
303
|
-
};
|
|
304
|
-
const convertXlsxToBlob = (xlsxSource)=>{
|
|
305
|
-
if (xlsxSource instanceof ArrayBuffer) return new Blob([
|
|
306
|
-
xlsxSource
|
|
307
|
-
], {
|
|
308
|
-
type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
|
|
309
|
-
});
|
|
310
|
-
if ('string' == typeof xlsxSource && isUrl(xlsxSource)) return DatasetSourceBuilder.fetchBlob(xlsxSource);
|
|
311
|
-
return new Blob([
|
|
312
|
-
xlsxSource
|
|
313
|
-
], {
|
|
314
|
-
type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
|
|
315
|
-
});
|
|
316
|
-
};
|
|
317
|
-
switch(type){
|
|
318
|
-
case 'csv':
|
|
319
|
-
return convertCsvToBlob(value);
|
|
320
|
-
case 'json':
|
|
321
|
-
return convertJsonToBlob(value);
|
|
322
|
-
case 'xlsx':
|
|
323
|
-
return convertXlsxToBlob(value);
|
|
324
|
-
case 'parquet':
|
|
325
|
-
return convertParquetToBlob(value);
|
|
326
|
-
default:
|
|
327
|
-
return new Blob([
|
|
328
|
-
value
|
|
329
|
-
]);
|
|
309
|
+
if ('string' == typeof value && isUrl(value)) {
|
|
310
|
+
const response = await fetch(value);
|
|
311
|
+
return await response.blob();
|
|
330
312
|
}
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
313
|
+
const mimeType = MIME_TYPES[type] || 'text/plain';
|
|
314
|
+
if (value instanceof ArrayBuffer) return new Blob([
|
|
315
|
+
value
|
|
316
|
+
], {
|
|
317
|
+
type: mimeType
|
|
318
|
+
});
|
|
319
|
+
const content = 'object' == typeof value ? JSON.stringify(value) : String(value);
|
|
320
|
+
return new Blob([
|
|
321
|
+
content
|
|
322
|
+
], {
|
|
323
|
+
type: mimeType
|
|
324
|
+
});
|
|
335
325
|
}
|
|
336
326
|
}
|
|
337
327
|
class InMemoryAdapter {
|
|
@@ -340,47 +330,31 @@ class InMemoryAdapter {
|
|
|
340
330
|
constructor(){}
|
|
341
331
|
open = async ()=>{
|
|
342
332
|
this.isOpen = true;
|
|
343
|
-
|
|
333
|
+
this.datasets = new Map();
|
|
344
334
|
};
|
|
345
335
|
close = async ()=>{
|
|
346
336
|
this.isOpen = false;
|
|
347
337
|
this.datasets.clear();
|
|
348
338
|
};
|
|
349
|
-
writeDataset = (datasetId, datasetSchema, datasetSource)=>new Promise((resolve
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
resolve();
|
|
358
|
-
} catch (error) {
|
|
359
|
-
reject(error);
|
|
360
|
-
}
|
|
339
|
+
writeDataset = (datasetId, datasetSchema, datasetSource)=>new Promise((resolve)=>{
|
|
340
|
+
const record = {
|
|
341
|
+
datasetId,
|
|
342
|
+
datasetSchema,
|
|
343
|
+
datasetSource
|
|
344
|
+
};
|
|
345
|
+
this.datasets.set(datasetId, record);
|
|
346
|
+
resolve();
|
|
361
347
|
});
|
|
362
|
-
readDataset = (datasetId)=>new Promise((resolve
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
resolve(record ? record : null);
|
|
366
|
-
} catch (error) {
|
|
367
|
-
reject(error);
|
|
368
|
-
}
|
|
348
|
+
readDataset = (datasetId)=>new Promise((resolve)=>{
|
|
349
|
+
const record = this.datasets.get(datasetId);
|
|
350
|
+
resolve(record ? record : null);
|
|
369
351
|
});
|
|
370
|
-
deleteDataset = (datasetId)=>new Promise((resolve
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
resolve();
|
|
374
|
-
} catch (error) {
|
|
375
|
-
reject(error);
|
|
376
|
-
}
|
|
352
|
+
deleteDataset = (datasetId)=>new Promise((resolve)=>{
|
|
353
|
+
this.datasets.delete(datasetId);
|
|
354
|
+
resolve();
|
|
377
355
|
});
|
|
378
|
-
listDatasets = ()=>new Promise((resolve
|
|
379
|
-
|
|
380
|
-
resolve(Array.from(this.datasets.values()));
|
|
381
|
-
} catch (error) {
|
|
382
|
-
reject(error);
|
|
383
|
-
}
|
|
356
|
+
listDatasets = ()=>new Promise((resolve)=>{
|
|
357
|
+
resolve(Array.from(this.datasets.values()));
|
|
384
358
|
});
|
|
385
359
|
}
|
|
386
360
|
const blocking_namespaceObject = require("@duckdb/duckdb-wasm/blocking");
|
|
@@ -445,9 +419,9 @@ class VQuery {
|
|
|
445
419
|
queryAdapter;
|
|
446
420
|
storageAdapter;
|
|
447
421
|
isInitialized = false;
|
|
448
|
-
constructor(){
|
|
449
|
-
this.queryAdapter =
|
|
450
|
-
this.storageAdapter =
|
|
422
|
+
constructor(queryAdapter, storageAdapter){
|
|
423
|
+
this.queryAdapter = queryAdapter;
|
|
424
|
+
this.storageAdapter = storageAdapter;
|
|
451
425
|
}
|
|
452
426
|
async checkInitialized() {
|
|
453
427
|
if (!this.isInitialized) {
|
|
@@ -509,14 +483,25 @@ class VQuery {
|
|
|
509
483
|
await this.storageAdapter.close();
|
|
510
484
|
}
|
|
511
485
|
}
|
|
486
|
+
class vquery_node_VQuery extends VQuery {
|
|
487
|
+
constructor(){
|
|
488
|
+
super(new DuckDBNodeQueryAdapter(), new InMemoryAdapter());
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
exports.Dataset = __webpack_exports__.Dataset;
|
|
512
492
|
exports.DatasetSourceBuilder = __webpack_exports__.DatasetSourceBuilder;
|
|
493
|
+
exports.DuckDBNodeQueryAdapter = __webpack_exports__.DuckDBNodeQueryAdapter;
|
|
494
|
+
exports.InMemoryAdapter = __webpack_exports__.InMemoryAdapter;
|
|
513
495
|
exports.VQuery = __webpack_exports__.VQuery;
|
|
514
496
|
exports.convertDSLToSQL = __webpack_exports__.convertDSLToSQL;
|
|
515
497
|
exports.isBase64Url = __webpack_exports__.isBase64Url;
|
|
516
498
|
exports.isHttpUrl = __webpack_exports__.isHttpUrl;
|
|
517
499
|
exports.isUrl = __webpack_exports__.isUrl;
|
|
518
500
|
for(var __webpack_i__ in __webpack_exports__)if (-1 === [
|
|
501
|
+
"Dataset",
|
|
519
502
|
"DatasetSourceBuilder",
|
|
503
|
+
"DuckDBNodeQueryAdapter",
|
|
504
|
+
"InMemoryAdapter",
|
|
520
505
|
"VQuery",
|
|
521
506
|
"convertDSLToSQL",
|
|
522
507
|
"isBase64Url",
|
package/dist/node/cjs/node.d.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
export { convertDSLToSQL } from './dataset';
|
|
1
|
+
export { convertDSLToSQL, Dataset } from './dataset';
|
|
2
2
|
export { DatasetSourceBuilder } from './data-source-builder/dataSourceBuilder';
|
|
3
3
|
export * from './utils';
|
|
4
4
|
export * from './types';
|
|
5
5
|
export { VQuery } from './vquery-node';
|
|
6
|
+
export { DuckDBNodeQueryAdapter } from './adapters/query-adapter/duckdbNodeAdapter';
|
|
7
|
+
export { InMemoryAdapter } from './adapters/storage-adapter/inmemoryAdapter';
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import { SelectItem } from '../../types/dsl/Select';
|
|
2
|
+
import type { SelectQueryBuilder } from 'kysely';
|
|
3
|
+
export declare const applySelect: <DB, TB extends keyof DB & string, O, T>(qb: SelectQueryBuilder<DB, TB, O>, select?: Array<keyof T | SelectItem<T>>) => SelectQueryBuilder<DB, TB, O & import("kysely").Selection<DB, TB, any>>;
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { QueryDSL } from '../types';
|
|
2
|
-
export declare const convertDSLToSQL: <T, TableName extends string>(dsl: QueryDSL<T>, tableName: TableName) => string;
|
|
1
|
+
import { QueryDSL, VQueryDSL } from '../types';
|
|
2
|
+
export declare const convertDSLToSQL: <T, TableName extends string>(dsl: QueryDSL<T> | VQueryDSL<T>, tableName: TableName) => string;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export type TidyDatum = Record<string, number | string | null | boolean | undefined>;
|
|
2
|
-
export type DatasetSourceType = 'csv' | 'json' | '
|
|
2
|
+
export type DatasetSourceType = 'csv' | 'json' | 'parquet';
|
|
3
3
|
export type DatasetSourceValue = string | ArrayBuffer | Blob | TidyDatum[];
|
|
4
4
|
export interface DatasetSource {
|
|
5
5
|
type: DatasetSourceType;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export type GroupBy<T> = Array<keyof T>;
|
|
1
|
+
export type GroupBy<T> = Array<keyof T | (string & {})>;
|
|
@@ -1,7 +1,12 @@
|
|
|
1
|
-
export type
|
|
1
|
+
export type BaseAggregateFunction = 'count' | 'count_distinct' | 'sum' | 'avg' | 'min' | 'max' | 'variance' | 'variancePop' | 'stddev' | 'median' | 'quantile';
|
|
2
|
+
export type DateAggregateFunction = 'to_year' | 'to_quarter' | 'to_month' | 'to_week' | 'to_day' | 'to_hour' | 'to_minute' | 'to_second';
|
|
3
|
+
export type AggregateFunction = BaseAggregateFunction | DateAggregateFunction;
|
|
2
4
|
export type SelectItem<T> = {
|
|
3
5
|
field: keyof T;
|
|
4
6
|
alias?: string;
|
|
5
|
-
|
|
7
|
+
aggr?: {
|
|
8
|
+
func: AggregateFunction;
|
|
9
|
+
quantile?: number;
|
|
10
|
+
};
|
|
6
11
|
};
|
|
7
12
|
export type Select<T> = Array<keyof T | SelectItem<T>>;
|
|
@@ -1,21 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
export declare class VQuery {
|
|
4
|
-
private queryAdapter;
|
|
5
|
-
private storageAdapter;
|
|
6
|
-
private isInitialized;
|
|
1
|
+
import { VQuery as VQueryCore } from './VQuery';
|
|
2
|
+
export declare class VQuery extends VQueryCore {
|
|
7
3
|
constructor();
|
|
8
|
-
private checkInitialized;
|
|
9
|
-
private checkDatasetExists;
|
|
10
|
-
createDataset(datasetId: string, columns?: DatasetColumn[], rawDatasetSource?: RawDatasetSource): Promise<void>;
|
|
11
|
-
updateDatasetSource(datasetId: string, columns?: DatasetColumn[], rawDatasetSource?: RawDatasetSource): Promise<void>;
|
|
12
|
-
dropDataset(datasetId: string): Promise<void>;
|
|
13
|
-
connectDataset(datasetId: string, temporaryColumns?: DatasetColumn[], temporaryRawDatasetSource?: RawDatasetSource): Promise<Dataset>;
|
|
14
|
-
hasDataset(datasetId: string): Promise<boolean>;
|
|
15
|
-
listDatasets(): Promise<{
|
|
16
|
-
datasetId: string;
|
|
17
|
-
datasetSchema: import("./types").DatasetSchema;
|
|
18
|
-
datasetSource?: import("./types").DatasetSource;
|
|
19
|
-
}[]>;
|
|
20
|
-
close(): Promise<void>;
|
|
21
4
|
}
|
|
@@ -1,21 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
export declare class VQuery {
|
|
4
|
-
private queryAdapter;
|
|
5
|
-
private storageAdapter;
|
|
6
|
-
private isInitialized;
|
|
1
|
+
import { VQuery as VQueryCore } from './VQuery';
|
|
2
|
+
export declare class VQuery extends VQueryCore {
|
|
7
3
|
constructor();
|
|
8
|
-
private checkInitialized;
|
|
9
|
-
private checkDatasetExists;
|
|
10
|
-
createDataset(datasetId: string, columns?: DatasetColumn[], rawDatasetSource?: RawDatasetSource): Promise<void>;
|
|
11
|
-
updateDatasetSource(datasetId: string, columns?: DatasetColumn[], rawDatasetSource?: RawDatasetSource): Promise<void>;
|
|
12
|
-
dropDataset(datasetId: string): Promise<void>;
|
|
13
|
-
connectDataset(datasetId: string, temporaryColumns?: DatasetColumn[], temporaryRawDatasetSource?: RawDatasetSource): Promise<Dataset>;
|
|
14
|
-
hasDataset(datasetId: string): Promise<boolean>;
|
|
15
|
-
listDatasets(): Promise<{
|
|
16
|
-
datasetId: string;
|
|
17
|
-
datasetSchema: import("./types").DatasetSchema;
|
|
18
|
-
datasetSource?: import("./types").DatasetSource;
|
|
19
|
-
}[]>;
|
|
20
|
-
close(): Promise<void>;
|
|
21
4
|
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { Dataset } from './dataset/dataset';
|
|
2
|
+
import { RawDatasetSource, DatasetColumn, QueryAdapter, StorageAdapter } from './types';
|
|
3
|
+
export declare class VQuery {
|
|
4
|
+
protected queryAdapter: QueryAdapter;
|
|
5
|
+
protected storageAdapter: StorageAdapter;
|
|
6
|
+
private isInitialized;
|
|
7
|
+
constructor(queryAdapter: QueryAdapter, storageAdapter: StorageAdapter);
|
|
8
|
+
private checkInitialized;
|
|
9
|
+
private checkDatasetExists;
|
|
10
|
+
createDataset(datasetId: string, columns?: DatasetColumn[], rawDatasetSource?: RawDatasetSource): Promise<void>;
|
|
11
|
+
updateDatasetSource(datasetId: string, columns?: DatasetColumn[], rawDatasetSource?: RawDatasetSource): Promise<void>;
|
|
12
|
+
dropDataset(datasetId: string): Promise<void>;
|
|
13
|
+
connectDataset(datasetId: string, temporaryColumns?: DatasetColumn[], temporaryRawDatasetSource?: RawDatasetSource): Promise<Dataset>;
|
|
14
|
+
hasDataset(datasetId: string): Promise<boolean>;
|
|
15
|
+
listDatasets(): Promise<{
|
|
16
|
+
datasetId: string;
|
|
17
|
+
datasetSchema: import("./types").DatasetSchema;
|
|
18
|
+
datasetSource?: import("./types").DatasetSource;
|
|
19
|
+
}[]>;
|
|
20
|
+
close(): Promise<void>;
|
|
21
|
+
}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
export { convertDSLToSQL } from './dataset';
|
|
1
|
+
export { convertDSLToSQL, Dataset } from './dataset';
|
|
2
2
|
export { DatasetSourceBuilder } from './data-source-builder/dataSourceBuilder';
|
|
3
3
|
export * from './utils';
|
|
4
4
|
export * from './types';
|
|
5
5
|
export { VQuery } from './vquery-browser';
|
|
6
|
+
export { DuckDBWebQueryAdapter } from './adapters/query-adapter/duckdbWebAdapter';
|
|
7
|
+
export { IndexedDBAdapter } from './adapters/storage-adapter/indexeddbAdapter';
|
|
@@ -1,13 +1,24 @@
|
|
|
1
|
-
import { DatasetColumn, DatasetSource, QueryDSL } from '../types';
|
|
1
|
+
import { DatasetColumn, DatasetSource, QueryDSL, VQueryDSL } from '../types';
|
|
2
2
|
import { QueryAdapter, StorageAdapter } from '../types';
|
|
3
3
|
export declare class Dataset {
|
|
4
4
|
private queryAdapter;
|
|
5
5
|
private storageAdapter;
|
|
6
6
|
private _datasetId;
|
|
7
|
-
constructor(
|
|
7
|
+
constructor(queryAdapter: QueryAdapter, storageAdapter: StorageAdapter, datasetId: string);
|
|
8
|
+
/**
|
|
9
|
+
* Initialize the dataset by loading it into the query engine
|
|
10
|
+
* @param temporaryColumns Optional temporary columns to override storage schema
|
|
11
|
+
* @param temporaryDatasetSource Optional temporary data source to override storage source
|
|
12
|
+
*/
|
|
8
13
|
init(temporaryColumns?: DatasetColumn[], temporaryDatasetSource?: DatasetSource): Promise<void>;
|
|
14
|
+
/**
|
|
15
|
+
* Register file and create a view in DuckDB
|
|
16
|
+
*/
|
|
9
17
|
createOrReplaceView(columns: DatasetColumn[], datasetSource: DatasetSource): Promise<void>;
|
|
10
|
-
|
|
18
|
+
/**
|
|
19
|
+
* Execute query using VQuery DSL
|
|
20
|
+
*/
|
|
21
|
+
query<T extends Record<string, number | string>>(queryDSL: QueryDSL<T> | VQueryDSL<T>): Promise<{
|
|
11
22
|
performance: {
|
|
12
23
|
startAt: string;
|
|
13
24
|
endAt: string;
|
|
@@ -16,6 +27,9 @@ export declare class Dataset {
|
|
|
16
27
|
dataset: any[];
|
|
17
28
|
table: any;
|
|
18
29
|
}>;
|
|
30
|
+
/**
|
|
31
|
+
* Execute raw SQL query
|
|
32
|
+
*/
|
|
19
33
|
queryBySQL(sql: string): Promise<{
|
|
20
34
|
performance: {
|
|
21
35
|
startAt: string;
|
|
@@ -25,6 +39,10 @@ export declare class Dataset {
|
|
|
25
39
|
dataset: any[];
|
|
26
40
|
table: any;
|
|
27
41
|
}>;
|
|
42
|
+
/**
|
|
43
|
+
* Clean up resources
|
|
44
|
+
*/
|
|
28
45
|
disconnect(): Promise<void>;
|
|
29
46
|
get datasetId(): string;
|
|
47
|
+
private buildColumnsStruct;
|
|
30
48
|
}
|
package/dist/node/esm/node.d.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
export { convertDSLToSQL } from './dataset';
|
|
1
|
+
export { convertDSLToSQL, Dataset } from './dataset';
|
|
2
2
|
export { DatasetSourceBuilder } from './data-source-builder/dataSourceBuilder';
|
|
3
3
|
export * from './utils';
|
|
4
4
|
export * from './types';
|
|
5
5
|
export { VQuery } from './vquery-node';
|
|
6
|
+
export { DuckDBNodeQueryAdapter } from './adapters/query-adapter/duckdbNodeAdapter';
|
|
7
|
+
export { InMemoryAdapter } from './adapters/storage-adapter/inmemoryAdapter';
|