@visactor/vquery 0.2.2 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/adapters/index.d.ts +1 -0
- package/dist/cjs/adapters/query-adapter/duckdbNodeAdapter.d.ts +14 -0
- package/dist/cjs/adapters/query-adapter/duckdbWebAdapter.d.ts +14 -0
- package/dist/cjs/adapters/storage-adapter/index.d.ts +2 -0
- package/dist/{db/indexedDb.d.ts → cjs/adapters/storage-adapter/indexeddbAdapter.d.ts} +5 -5
- package/dist/cjs/adapters/storage-adapter/inmemoryAdapter.d.ts +22 -0
- package/dist/cjs/browser.d.ts +5 -0
- package/dist/{dataset → cjs/dataset}/dataset.d.ts +4 -5
- package/dist/{index.cjs → cjs/node.cjs} +134 -181
- package/dist/{index.d.ts → cjs/node.d.ts} +1 -1
- package/dist/{sql-builder → cjs/sql-builder}/utils.d.ts +1 -1
- package/dist/cjs/types/adapters/QueryAdapter.d.ts +11 -0
- package/dist/cjs/types/adapters/StorageAdapter.d.ts +17 -0
- package/dist/cjs/types/adapters/index.d.ts +2 -0
- package/dist/{types/QueryDSL → cjs/types/dsl}/OrderBy.d.ts +1 -1
- package/dist/{types/QueryDSL → cjs/types/dsl}/Select.d.ts +1 -1
- package/dist/{types → cjs/types}/index.d.ts +2 -1
- package/dist/{vquery.d.ts → cjs/vquery-browser.d.ts} +4 -4
- package/dist/cjs/vquery-node.d.ts +21 -0
- package/dist/esm/adapters/index.d.ts +1 -0
- package/dist/esm/adapters/query-adapter/duckdbNodeAdapter.d.ts +14 -0
- package/dist/esm/adapters/query-adapter/duckdbWebAdapter.d.ts +14 -0
- package/dist/esm/adapters/storage-adapter/index.d.ts +2 -0
- package/dist/esm/adapters/storage-adapter/indexeddbAdapter.d.ts +21 -0
- package/dist/esm/adapters/storage-adapter/inmemoryAdapter.d.ts +22 -0
- package/dist/esm/browser.d.ts +5 -0
- package/dist/{index.js → esm/browser.js} +177 -176
- package/dist/esm/data-source-builder/dataSourceBuilder.d.ts +16 -0
- package/dist/esm/data-source-builder/index.d.ts +1 -0
- package/dist/esm/dataset/dataset.d.ts +30 -0
- package/dist/esm/dataset/index.d.ts +2 -0
- package/dist/esm/node.d.ts +5 -0
- package/dist/esm/sql-builder/builders/groupBy.d.ts +2 -0
- package/dist/esm/sql-builder/builders/index.d.ts +3 -0
- package/dist/esm/sql-builder/builders/limit.d.ts +2 -0
- package/dist/esm/sql-builder/builders/where.d.ts +3 -0
- package/dist/esm/sql-builder/compile/index.d.ts +1 -0
- package/dist/esm/sql-builder/compile/inlineParameters.d.ts +1 -0
- package/dist/esm/sql-builder/dialect/index.d.ts +1 -0
- package/dist/esm/sql-builder/dialect/postgresDialect.d.ts +11 -0
- package/dist/esm/sql-builder/dslToSQL.d.ts +2 -0
- package/dist/esm/sql-builder/index.d.ts +1 -0
- package/dist/esm/sql-builder/utils.d.ts +8 -0
- package/dist/esm/types/DataSet.d.ts +11 -0
- package/dist/esm/types/DataSource.d.ts +11 -0
- package/dist/esm/types/QueryResult.d.ts +1 -0
- package/dist/esm/types/adapters/QueryAdapter.d.ts +11 -0
- package/dist/esm/types/adapters/StorageAdapter.d.ts +17 -0
- package/dist/esm/types/adapters/index.d.ts +2 -0
- package/dist/esm/types/dsl/GroupBy.d.ts +1 -0
- package/dist/esm/types/dsl/OrderBy.d.ts +4 -0
- package/dist/esm/types/dsl/QueryDSL.d.ts +12 -0
- package/dist/esm/types/dsl/Select.d.ts +7 -0
- package/dist/esm/types/dsl/Where.d.ts +24 -0
- package/dist/esm/types/dsl/demo.d.ts +1 -0
- package/dist/esm/types/dsl/index.d.ts +5 -0
- package/dist/esm/types/index.d.ts +5 -0
- package/dist/esm/utils/index.d.ts +1 -0
- package/dist/esm/utils/url.d.ts +3 -0
- package/dist/esm/vquery-browser.d.ts +21 -0
- package/dist/esm/vquery-node.d.ts +21 -0
- package/package.json +23 -17
- package/dist/db/duckDb.d.ts +0 -35
- /package/dist/{data-source-builder → cjs/data-source-builder}/dataSourceBuilder.d.ts +0 -0
- /package/dist/{data-source-builder → cjs/data-source-builder}/index.d.ts +0 -0
- /package/dist/{dataset → cjs/dataset}/index.d.ts +0 -0
- /package/dist/{sql-builder → cjs/sql-builder}/builders/groupBy.d.ts +0 -0
- /package/dist/{sql-builder → cjs/sql-builder}/builders/index.d.ts +0 -0
- /package/dist/{sql-builder → cjs/sql-builder}/builders/limit.d.ts +0 -0
- /package/dist/{sql-builder → cjs/sql-builder}/builders/where.d.ts +0 -0
- /package/dist/{sql-builder → cjs/sql-builder}/compile/index.d.ts +0 -0
- /package/dist/{sql-builder → cjs/sql-builder}/compile/inlineParameters.d.ts +0 -0
- /package/dist/{sql-builder → cjs/sql-builder}/dialect/index.d.ts +0 -0
- /package/dist/{sql-builder → cjs/sql-builder}/dialect/postgresDialect.d.ts +0 -0
- /package/dist/{sql-builder → cjs/sql-builder}/dslToSQL.d.ts +0 -0
- /package/dist/{sql-builder → cjs/sql-builder}/index.d.ts +0 -0
- /package/dist/{types → cjs/types}/DataSet.d.ts +0 -0
- /package/dist/{types → cjs/types}/DataSource.d.ts +0 -0
- /package/dist/{types → cjs/types}/QueryResult.d.ts +0 -0
- /package/dist/{types/QueryDSL → cjs/types/dsl}/GroupBy.d.ts +0 -0
- /package/dist/{types/QueryDSL → cjs/types/dsl}/QueryDSL.d.ts +0 -0
- /package/dist/{types/QueryDSL → cjs/types/dsl}/Where.d.ts +0 -0
- /package/dist/{types/QueryDSL → cjs/types/dsl}/demo.d.ts +0 -0
- /package/dist/{types/QueryDSL → cjs/types/dsl}/index.d.ts +0 -0
- /package/dist/{utils → cjs/utils}/index.d.ts +0 -0
- /package/dist/{utils → cjs/utils}/url.d.ts +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { IndexedDBAdapter, InMemoryAdapter } from './storage-adapter';
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { QueryAdapter } from '../../types';
|
|
2
|
+
export declare class DuckDBNodeQueryAdapter implements QueryAdapter {
|
|
3
|
+
private bindings;
|
|
4
|
+
private connection;
|
|
5
|
+
constructor();
|
|
6
|
+
open: () => Promise<void>;
|
|
7
|
+
close: () => Promise<void>;
|
|
8
|
+
writeFile: <T extends Blob>(fileName: string, source: T) => Promise<void>;
|
|
9
|
+
query: (sql: string) => Promise<{
|
|
10
|
+
dataset: any[];
|
|
11
|
+
table: any;
|
|
12
|
+
}>;
|
|
13
|
+
getSchema: (fileName: string) => Promise<any>;
|
|
14
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { QueryAdapter } from '../../types';
|
|
2
|
+
export declare class DuckDBWebQueryAdapter implements QueryAdapter {
|
|
3
|
+
private db;
|
|
4
|
+
private connection;
|
|
5
|
+
constructor();
|
|
6
|
+
open: () => Promise<void>;
|
|
7
|
+
close: () => Promise<void>;
|
|
8
|
+
writeFile: <T extends Blob>(fileName: string, source: T) => Promise<void>;
|
|
9
|
+
query: (sql: string) => Promise<{
|
|
10
|
+
dataset: any[];
|
|
11
|
+
table: any;
|
|
12
|
+
}>;
|
|
13
|
+
getSchema: (fileName: string) => Promise<any>;
|
|
14
|
+
}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { DatasetSource } from '
|
|
2
|
-
import { DatasetSchema } from '
|
|
3
|
-
export declare class
|
|
1
|
+
import { DatasetSource, StorageAdapter } from '../../types';
|
|
2
|
+
import { DatasetSchema } from '../../types/DataSet';
|
|
3
|
+
export declare class IndexedDBAdapter implements StorageAdapter {
|
|
4
4
|
private db;
|
|
5
5
|
private dbName;
|
|
6
6
|
private datasetStoreName;
|
|
7
|
-
constructor(dbName
|
|
7
|
+
constructor(dbName?: string);
|
|
8
8
|
open: () => Promise<void>;
|
|
9
|
-
close: () => void
|
|
9
|
+
close: () => Promise<void>;
|
|
10
10
|
writeDataset: (datasetId: string, datasetSchema: DatasetSchema, datasetSource?: DatasetSource) => Promise<void>;
|
|
11
11
|
readDataset: (datasetId: string) => Promise<{
|
|
12
12
|
datasetSource?: DatasetSource;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { DatasetSource, StorageAdapter } from '../../types';
|
|
2
|
+
import { DatasetSchema } from '../../types/DataSet';
|
|
3
|
+
type StoredDataset = {
|
|
4
|
+
datasetId: string;
|
|
5
|
+
datasetSchema: DatasetSchema;
|
|
6
|
+
datasetSource?: DatasetSource;
|
|
7
|
+
};
|
|
8
|
+
export declare class InMemoryAdapter implements StorageAdapter {
|
|
9
|
+
private datasets;
|
|
10
|
+
private isOpen;
|
|
11
|
+
constructor();
|
|
12
|
+
open: () => Promise<void>;
|
|
13
|
+
close: () => Promise<void>;
|
|
14
|
+
writeDataset: (datasetId: string, datasetSchema: DatasetSchema, datasetSource?: DatasetSource) => Promise<void>;
|
|
15
|
+
readDataset: (datasetId: string) => Promise<{
|
|
16
|
+
datasetSource?: DatasetSource;
|
|
17
|
+
datasetSchema: DatasetSchema;
|
|
18
|
+
} | null>;
|
|
19
|
+
deleteDataset: (datasetId: string) => Promise<void>;
|
|
20
|
+
listDatasets: () => Promise<StoredDataset[]>;
|
|
21
|
+
}
|
|
22
|
+
export {};
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import { DatasetColumn, DatasetSource, QueryDSL } from '../types';
|
|
2
|
-
import {
|
|
3
|
-
import { IndexedDB } from '../db/indexedDb';
|
|
2
|
+
import { QueryAdapter, StorageAdapter } from '../types';
|
|
4
3
|
export declare class Dataset {
|
|
5
|
-
private
|
|
6
|
-
private
|
|
4
|
+
private queryAdapter;
|
|
5
|
+
private storageAdapter;
|
|
7
6
|
private _datasetId;
|
|
8
|
-
constructor(duckDB:
|
|
7
|
+
constructor(duckDB: QueryAdapter, indexedDB: StorageAdapter, datasetId: string);
|
|
9
8
|
init(temporaryColumns?: DatasetColumn[], temporaryDatasetSource?: DatasetSource): Promise<void>;
|
|
10
9
|
createOrReplaceView(columns: DatasetColumn[], datasetSource: DatasetSource): Promise<void>;
|
|
11
10
|
query<T extends Record<string, number | string>>(queryDSL: QueryDSL<T>): Promise<{
|
|
@@ -177,16 +177,16 @@ const convertDSLToSQL = (dsl, tableName)=>{
|
|
|
177
177
|
return inlineParameters(compiled.sql, compiled.parameters);
|
|
178
178
|
};
|
|
179
179
|
class Dataset {
|
|
180
|
-
|
|
181
|
-
|
|
180
|
+
queryAdapter;
|
|
181
|
+
storageAdapter;
|
|
182
182
|
_datasetId;
|
|
183
|
-
constructor(duckDB,
|
|
184
|
-
this.
|
|
185
|
-
this.
|
|
183
|
+
constructor(duckDB, indexedDB, datasetId){
|
|
184
|
+
this.queryAdapter = duckDB;
|
|
185
|
+
this.storageAdapter = indexedDB;
|
|
186
186
|
this._datasetId = datasetId;
|
|
187
187
|
}
|
|
188
188
|
async init(temporaryColumns, temporaryDatasetSource) {
|
|
189
|
-
const datasetInfo = await this.
|
|
189
|
+
const datasetInfo = await this.storageAdapter.readDataset(this._datasetId);
|
|
190
190
|
if (!datasetInfo) throw new Error(`Dataset ${this._datasetId} not found`);
|
|
191
191
|
const columns = temporaryColumns ? temporaryColumns : datasetInfo.datasetSchema.columns;
|
|
192
192
|
const datasetSource = temporaryDatasetSource || datasetInfo.datasetSource;
|
|
@@ -209,11 +209,11 @@ class Dataset {
|
|
|
209
209
|
if (datasetSource) {
|
|
210
210
|
const readFunction = readFunctionMap[datasetSource.type];
|
|
211
211
|
if (!readFunction) throw new Error(`Unsupported dataSource type: ${datasetSource.type}`);
|
|
212
|
-
await this.
|
|
212
|
+
await this.queryAdapter.writeFile(this._datasetId, datasetSource.blob);
|
|
213
213
|
const columnsStruct = `{${columns.map((c)=>`'${c.name}': '${dataTypeMap[c.type] || 'VARCHAR'}'`).join(', ')}}`;
|
|
214
214
|
const columnNames = columns.map((c)=>`"${c.name}"`).join(', ');
|
|
215
215
|
const createViewSql = `CREATE OR REPLACE VIEW "${this._datasetId}" AS SELECT ${columnNames} FROM ${readFunction}('${this._datasetId}', columns=${columnsStruct})`;
|
|
216
|
-
await this.
|
|
216
|
+
await this.queryAdapter.query(createViewSql);
|
|
217
217
|
}
|
|
218
218
|
}
|
|
219
219
|
async query(queryDSL) {
|
|
@@ -222,7 +222,7 @@ class Dataset {
|
|
|
222
222
|
}
|
|
223
223
|
async queryBySQL(sql) {
|
|
224
224
|
const start = performance?.now?.()?.toFixed(3) ?? Date.now().toFixed(3);
|
|
225
|
-
const result = await this.
|
|
225
|
+
const result = await this.queryAdapter.query(sql);
|
|
226
226
|
const end = performance?.now?.()?.toFixed(3) ?? Date.now().toFixed(3);
|
|
227
227
|
return {
|
|
228
228
|
...result,
|
|
@@ -234,167 +234,12 @@ class Dataset {
|
|
|
234
234
|
};
|
|
235
235
|
}
|
|
236
236
|
async disconnect() {
|
|
237
|
-
await this.
|
|
237
|
+
await this.queryAdapter.query(`DROP VIEW IF EXISTS "${this._datasetId}"`);
|
|
238
238
|
}
|
|
239
239
|
get datasetId() {
|
|
240
240
|
return this._datasetId;
|
|
241
241
|
}
|
|
242
242
|
}
|
|
243
|
-
const duckdb_wasm_namespaceObject = require("@duckdb/duckdb-wasm");
|
|
244
|
-
class DuckDB {
|
|
245
|
-
db = null;
|
|
246
|
-
connection = null;
|
|
247
|
-
constructor(){}
|
|
248
|
-
init = async ()=>{
|
|
249
|
-
const MANUAL_BUNDLES = {
|
|
250
|
-
mvp: {
|
|
251
|
-
mainModule: new URL('@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm', __rslib_import_meta_url__).href,
|
|
252
|
-
mainWorker: new URL('@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js', __rslib_import_meta_url__).toString()
|
|
253
|
-
},
|
|
254
|
-
eh: {
|
|
255
|
-
mainModule: new URL('@duckdb/duckdb-wasm/dist/duckdb-eh.wasm', __rslib_import_meta_url__).href,
|
|
256
|
-
mainWorker: new URL('@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js', __rslib_import_meta_url__).toString()
|
|
257
|
-
}
|
|
258
|
-
};
|
|
259
|
-
const bundle = await (0, duckdb_wasm_namespaceObject.selectBundle)(MANUAL_BUNDLES);
|
|
260
|
-
const worker_url = URL.createObjectURL(new Blob([
|
|
261
|
-
`importScripts("${bundle.mainWorker}");`
|
|
262
|
-
], {
|
|
263
|
-
type: "text/javascript"
|
|
264
|
-
}));
|
|
265
|
-
const worker = new Worker(worker_url);
|
|
266
|
-
const logger = new duckdb_wasm_namespaceObject.ConsoleLogger();
|
|
267
|
-
this.db = new duckdb_wasm_namespaceObject.AsyncDuckDB(logger, worker);
|
|
268
|
-
await this.db.instantiate(bundle.mainModule, bundle.pthreadWorker);
|
|
269
|
-
URL.revokeObjectURL(worker_url);
|
|
270
|
-
this.connection = await this.db.connect();
|
|
271
|
-
};
|
|
272
|
-
close = async ()=>{
|
|
273
|
-
if (this.connection) {
|
|
274
|
-
await this.connection.close();
|
|
275
|
-
this.connection = null;
|
|
276
|
-
}
|
|
277
|
-
if (this.db) {
|
|
278
|
-
await this.db.terminate();
|
|
279
|
-
this.db = null;
|
|
280
|
-
}
|
|
281
|
-
};
|
|
282
|
-
writeFile = async (fileName, source)=>{
|
|
283
|
-
if (!this.db) throw new Error('db is null');
|
|
284
|
-
let uint8Array;
|
|
285
|
-
if (source instanceof Blob) {
|
|
286
|
-
const buffer = await source.arrayBuffer();
|
|
287
|
-
uint8Array = new Uint8Array(buffer);
|
|
288
|
-
} else throw new Error('Unsupported source type');
|
|
289
|
-
await this.db.registerFileBuffer(fileName, uint8Array);
|
|
290
|
-
};
|
|
291
|
-
query = async (sql)=>{
|
|
292
|
-
if (!this.connection) throw new Error('connection is null');
|
|
293
|
-
const table = await this.connection.query(sql);
|
|
294
|
-
const dataset = table.toArray().map((row)=>row.toJSON());
|
|
295
|
-
return {
|
|
296
|
-
dataset,
|
|
297
|
-
table
|
|
298
|
-
};
|
|
299
|
-
};
|
|
300
|
-
getSchema = async (fileName)=>{
|
|
301
|
-
if (!this.connection) throw new Error('connection is null');
|
|
302
|
-
const result = await this.connection.query(`PRAGMA table_info('${fileName}')`);
|
|
303
|
-
return result.toArray().map((row)=>row.toJSON());
|
|
304
|
-
};
|
|
305
|
-
}
|
|
306
|
-
class IndexedDB {
|
|
307
|
-
db = null;
|
|
308
|
-
dbName;
|
|
309
|
-
datasetStoreName = 'vqueryDatasets';
|
|
310
|
-
constructor(dbName){
|
|
311
|
-
this.dbName = dbName;
|
|
312
|
-
}
|
|
313
|
-
open = ()=>new Promise((resolve, reject)=>{
|
|
314
|
-
const request = indexedDB.open(this.dbName, 2);
|
|
315
|
-
request.onupgradeneeded = (event)=>{
|
|
316
|
-
const db = event.target.result;
|
|
317
|
-
if (!db.objectStoreNames.contains(this.datasetStoreName)) db.createObjectStore(this.datasetStoreName, {
|
|
318
|
-
keyPath: 'datasetId'
|
|
319
|
-
});
|
|
320
|
-
};
|
|
321
|
-
request.onsuccess = (event)=>{
|
|
322
|
-
this.db = event.target.result;
|
|
323
|
-
resolve();
|
|
324
|
-
};
|
|
325
|
-
request.onerror = (event)=>{
|
|
326
|
-
reject(event.target.error);
|
|
327
|
-
};
|
|
328
|
-
});
|
|
329
|
-
close = ()=>{
|
|
330
|
-
if (this.db) {
|
|
331
|
-
this.db.close();
|
|
332
|
-
this.db = null;
|
|
333
|
-
}
|
|
334
|
-
};
|
|
335
|
-
writeDataset = (datasetId, datasetSchema, datasetSource)=>new Promise((resolve, reject)=>{
|
|
336
|
-
if (!this.db) return reject('DB is not open');
|
|
337
|
-
const transaction = this.db.transaction([
|
|
338
|
-
this.datasetStoreName
|
|
339
|
-
], 'readwrite');
|
|
340
|
-
const store = transaction.objectStore(this.datasetStoreName);
|
|
341
|
-
const request = store.put({
|
|
342
|
-
datasetId,
|
|
343
|
-
datasetSchema,
|
|
344
|
-
datasetSource
|
|
345
|
-
});
|
|
346
|
-
request.onsuccess = ()=>{
|
|
347
|
-
resolve();
|
|
348
|
-
};
|
|
349
|
-
request.onerror = (event)=>{
|
|
350
|
-
reject(event.target.error);
|
|
351
|
-
};
|
|
352
|
-
});
|
|
353
|
-
readDataset = (datasetId)=>new Promise((resolve, reject)=>{
|
|
354
|
-
if (!this.db) return reject('DB is not open');
|
|
355
|
-
const transaction = this.db.transaction([
|
|
356
|
-
this.datasetStoreName
|
|
357
|
-
], 'readonly');
|
|
358
|
-
const store = transaction.objectStore(this.datasetStoreName);
|
|
359
|
-
const request = store.get(datasetId);
|
|
360
|
-
request.onsuccess = (event)=>{
|
|
361
|
-
const result = event.target.result;
|
|
362
|
-
resolve(result || null);
|
|
363
|
-
};
|
|
364
|
-
request.onerror = (event)=>{
|
|
365
|
-
reject(event.target.error);
|
|
366
|
-
};
|
|
367
|
-
});
|
|
368
|
-
deleteDataset = (datasetId)=>new Promise((resolve, reject)=>{
|
|
369
|
-
if (!this.db) return reject('DB is not open');
|
|
370
|
-
const transaction = this.db.transaction([
|
|
371
|
-
this.datasetStoreName
|
|
372
|
-
], 'readwrite');
|
|
373
|
-
const store = transaction.objectStore(this.datasetStoreName);
|
|
374
|
-
const request = store.delete(datasetId);
|
|
375
|
-
request.onsuccess = ()=>{
|
|
376
|
-
resolve();
|
|
377
|
-
};
|
|
378
|
-
request.onerror = (event)=>{
|
|
379
|
-
reject(event.target.error);
|
|
380
|
-
};
|
|
381
|
-
});
|
|
382
|
-
listDatasets = ()=>new Promise((resolve, reject)=>{
|
|
383
|
-
if (!this.db) return reject('DB is not open');
|
|
384
|
-
const transaction = this.db.transaction([
|
|
385
|
-
this.datasetStoreName
|
|
386
|
-
], 'readonly');
|
|
387
|
-
const store = transaction.objectStore(this.datasetStoreName);
|
|
388
|
-
const request = store.getAll();
|
|
389
|
-
request.onsuccess = (event)=>{
|
|
390
|
-
const result = event.target.result;
|
|
391
|
-
resolve(result);
|
|
392
|
-
};
|
|
393
|
-
request.onerror = (event)=>{
|
|
394
|
-
reject(event.target.error);
|
|
395
|
-
};
|
|
396
|
-
});
|
|
397
|
-
}
|
|
398
243
|
const isUrl = (url)=>isHttpUrl(url) || isBase64Url(url);
|
|
399
244
|
const isHttpUrl = (url)=>url.startsWith('http://') || url.startsWith('https://');
|
|
400
245
|
const isBase64Url = (url)=>url.startsWith('data:');
|
|
@@ -489,18 +334,125 @@ class DatasetSourceBuilder {
|
|
|
489
334
|
return await response.blob();
|
|
490
335
|
}
|
|
491
336
|
}
|
|
337
|
+
class InMemoryAdapter {
|
|
338
|
+
datasets = new Map();
|
|
339
|
+
isOpen = false;
|
|
340
|
+
constructor(){}
|
|
341
|
+
open = async ()=>{
|
|
342
|
+
this.isOpen = true;
|
|
343
|
+
if (!this.isOpen) this.datasets = new Map();
|
|
344
|
+
};
|
|
345
|
+
close = async ()=>{
|
|
346
|
+
this.isOpen = false;
|
|
347
|
+
this.datasets.clear();
|
|
348
|
+
};
|
|
349
|
+
writeDataset = (datasetId, datasetSchema, datasetSource)=>new Promise((resolve, reject)=>{
|
|
350
|
+
try {
|
|
351
|
+
const record = {
|
|
352
|
+
datasetId,
|
|
353
|
+
datasetSchema,
|
|
354
|
+
datasetSource
|
|
355
|
+
};
|
|
356
|
+
this.datasets.set(datasetId, record);
|
|
357
|
+
resolve();
|
|
358
|
+
} catch (error) {
|
|
359
|
+
reject(error);
|
|
360
|
+
}
|
|
361
|
+
});
|
|
362
|
+
readDataset = (datasetId)=>new Promise((resolve, reject)=>{
|
|
363
|
+
try {
|
|
364
|
+
const record = this.datasets.get(datasetId);
|
|
365
|
+
resolve(record ? record : null);
|
|
366
|
+
} catch (error) {
|
|
367
|
+
reject(error);
|
|
368
|
+
}
|
|
369
|
+
});
|
|
370
|
+
deleteDataset = (datasetId)=>new Promise((resolve, reject)=>{
|
|
371
|
+
try {
|
|
372
|
+
this.datasets.delete(datasetId);
|
|
373
|
+
resolve();
|
|
374
|
+
} catch (error) {
|
|
375
|
+
reject(error);
|
|
376
|
+
}
|
|
377
|
+
});
|
|
378
|
+
listDatasets = ()=>new Promise((resolve, reject)=>{
|
|
379
|
+
try {
|
|
380
|
+
resolve(Array.from(this.datasets.values()));
|
|
381
|
+
} catch (error) {
|
|
382
|
+
reject(error);
|
|
383
|
+
}
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
const blocking_namespaceObject = require("@duckdb/duckdb-wasm/blocking");
|
|
387
|
+
const external_node_module_namespaceObject = require("node:module");
|
|
388
|
+
class DuckDBNodeQueryAdapter {
|
|
389
|
+
bindings = null;
|
|
390
|
+
connection = null;
|
|
391
|
+
constructor(){}
|
|
392
|
+
open = async ()=>{
|
|
393
|
+
const require1 = (0, external_node_module_namespaceObject.createRequire)(__rslib_import_meta_url__);
|
|
394
|
+
const MANUAL_BUNDLES = {
|
|
395
|
+
mvp: {
|
|
396
|
+
mainModule: require1.resolve('@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm'),
|
|
397
|
+
mainWorker: ''
|
|
398
|
+
},
|
|
399
|
+
eh: {
|
|
400
|
+
mainModule: require1.resolve('@duckdb/duckdb-wasm/dist/duckdb-eh.wasm'),
|
|
401
|
+
mainWorker: ''
|
|
402
|
+
}
|
|
403
|
+
};
|
|
404
|
+
const logger = new blocking_namespaceObject.ConsoleLogger();
|
|
405
|
+
this.bindings = await (0, blocking_namespaceObject.createDuckDB)(MANUAL_BUNDLES, logger, blocking_namespaceObject.NODE_RUNTIME);
|
|
406
|
+
await this.bindings.instantiate(()=>{});
|
|
407
|
+
this.bindings.open({});
|
|
408
|
+
this.connection = this.bindings.connect();
|
|
409
|
+
};
|
|
410
|
+
close = async ()=>{
|
|
411
|
+
if (this.connection) {
|
|
412
|
+
this.connection.close();
|
|
413
|
+
this.connection = null;
|
|
414
|
+
}
|
|
415
|
+
if (this.bindings) {
|
|
416
|
+
this.bindings.reset();
|
|
417
|
+
this.bindings = null;
|
|
418
|
+
}
|
|
419
|
+
};
|
|
420
|
+
writeFile = async (fileName, source)=>{
|
|
421
|
+
if (!this.bindings) throw new Error('bindings is null');
|
|
422
|
+
let uint8Array;
|
|
423
|
+
if (source instanceof Blob) {
|
|
424
|
+
const buffer = await source.arrayBuffer();
|
|
425
|
+
uint8Array = new Uint8Array(buffer);
|
|
426
|
+
} else throw new Error('Unsupported source type');
|
|
427
|
+
await this.bindings.registerFileBuffer(fileName, uint8Array);
|
|
428
|
+
};
|
|
429
|
+
query = async (sql)=>{
|
|
430
|
+
if (!this.connection) throw new Error('connection is null');
|
|
431
|
+
const table = await this.connection.query(sql);
|
|
432
|
+
const dataset = table.toArray().map((row)=>row.toJSON());
|
|
433
|
+
return {
|
|
434
|
+
dataset,
|
|
435
|
+
table
|
|
436
|
+
};
|
|
437
|
+
};
|
|
438
|
+
getSchema = async (fileName)=>{
|
|
439
|
+
if (!this.connection) throw new Error('connection is null');
|
|
440
|
+
const result = await this.connection.query(`PRAGMA table_info('${fileName}')`);
|
|
441
|
+
return result.toArray().map((row)=>row.toJSON());
|
|
442
|
+
};
|
|
443
|
+
}
|
|
492
444
|
class VQuery {
|
|
493
|
-
|
|
494
|
-
|
|
445
|
+
queryAdapter;
|
|
446
|
+
storageAdapter;
|
|
495
447
|
isInitialized = false;
|
|
496
|
-
constructor(
|
|
497
|
-
this.
|
|
498
|
-
this.
|
|
448
|
+
constructor(){
|
|
449
|
+
this.queryAdapter = new DuckDBNodeQueryAdapter();
|
|
450
|
+
this.storageAdapter = new InMemoryAdapter();
|
|
499
451
|
}
|
|
500
452
|
async checkInitialized() {
|
|
501
453
|
if (!this.isInitialized) {
|
|
502
|
-
await this.
|
|
503
|
-
await this.
|
|
454
|
+
await this.queryAdapter.open();
|
|
455
|
+
await this.storageAdapter.open();
|
|
504
456
|
this.isInitialized = true;
|
|
505
457
|
}
|
|
506
458
|
}
|
|
@@ -516,7 +468,7 @@ class VQuery {
|
|
|
516
468
|
datasetAlias: datasetId,
|
|
517
469
|
columns: columns
|
|
518
470
|
};
|
|
519
|
-
await this.
|
|
471
|
+
await this.storageAdapter.writeDataset(datasetId, datasetSchema, datasetSource);
|
|
520
472
|
}
|
|
521
473
|
async updateDatasetSource(datasetId, columns = [], rawDatasetSource) {
|
|
522
474
|
await this.checkInitialized();
|
|
@@ -527,33 +479,34 @@ class VQuery {
|
|
|
527
479
|
datasetAlias: datasetId,
|
|
528
480
|
columns: columns
|
|
529
481
|
};
|
|
530
|
-
await this.
|
|
482
|
+
await this.storageAdapter.writeDataset(datasetId, datasetSchema, datasetSource);
|
|
531
483
|
}
|
|
532
484
|
async dropDataset(datasetId) {
|
|
533
485
|
await this.checkInitialized();
|
|
534
486
|
await this.checkDatasetExists(datasetId);
|
|
535
|
-
await this.
|
|
487
|
+
await this.storageAdapter.deleteDataset(datasetId);
|
|
536
488
|
}
|
|
537
489
|
async connectDataset(datasetId, temporaryColumns, temporaryRawDatasetSource) {
|
|
538
490
|
await this.checkInitialized();
|
|
539
491
|
await this.checkDatasetExists(datasetId);
|
|
540
|
-
const dataset = new Dataset(this.
|
|
492
|
+
const dataset = new Dataset(this.queryAdapter, this.storageAdapter, datasetId);
|
|
541
493
|
const temporaryDatasetSource = temporaryRawDatasetSource ? await DatasetSourceBuilder.from(temporaryRawDatasetSource).build() : void 0;
|
|
542
494
|
await dataset.init(temporaryColumns, temporaryDatasetSource);
|
|
543
495
|
return dataset;
|
|
544
496
|
}
|
|
545
497
|
async hasDataset(datasetId) {
|
|
546
498
|
await this.checkInitialized();
|
|
547
|
-
const datasets = await this.
|
|
499
|
+
const datasets = await this.storageAdapter.listDatasets();
|
|
548
500
|
return datasets.some((item)=>item.datasetId === datasetId);
|
|
549
501
|
}
|
|
550
502
|
async listDatasets() {
|
|
551
503
|
await this.checkInitialized();
|
|
552
|
-
return this.
|
|
504
|
+
return this.storageAdapter.listDatasets();
|
|
553
505
|
}
|
|
554
506
|
async close() {
|
|
555
507
|
await this.checkInitialized();
|
|
556
|
-
await this.
|
|
508
|
+
await this.queryAdapter.close();
|
|
509
|
+
await this.storageAdapter.close();
|
|
557
510
|
}
|
|
558
511
|
}
|
|
559
512
|
exports.DatasetSourceBuilder = __webpack_exports__.DatasetSourceBuilder;
|
|
@@ -562,14 +515,14 @@ exports.convertDSLToSQL = __webpack_exports__.convertDSLToSQL;
|
|
|
562
515
|
exports.isBase64Url = __webpack_exports__.isBase64Url;
|
|
563
516
|
exports.isHttpUrl = __webpack_exports__.isHttpUrl;
|
|
564
517
|
exports.isUrl = __webpack_exports__.isUrl;
|
|
565
|
-
for(var
|
|
518
|
+
for(var __rspack_i in __webpack_exports__)if (-1 === [
|
|
566
519
|
"DatasetSourceBuilder",
|
|
567
520
|
"VQuery",
|
|
568
521
|
"convertDSLToSQL",
|
|
569
522
|
"isBase64Url",
|
|
570
523
|
"isHttpUrl",
|
|
571
524
|
"isUrl"
|
|
572
|
-
].indexOf(
|
|
525
|
+
].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
|
|
573
526
|
Object.defineProperty(exports, '__esModule', {
|
|
574
527
|
value: true
|
|
575
528
|
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Where, WhereClause, WhereGroup, WhereLeaf } from '../types';
|
|
2
|
-
import { SelectItem } from '../types/
|
|
2
|
+
import { SelectItem } from '../types/dsl/Select';
|
|
3
3
|
export declare const isSelectItem: <T>(item: keyof T | SelectItem<T>) => item is SelectItem<T>;
|
|
4
4
|
export declare const isWhereLeaf: <T>(where: Where<T> | WhereClause<T>) => where is WhereLeaf<T>;
|
|
5
5
|
export declare const isWhereGroup: <T>(where: Where<T> | WhereClause<T>) => where is WhereGroup<T>;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { QueryResult } from '../QueryResult';
|
|
2
|
+
export interface QueryAdapter {
|
|
3
|
+
open: () => Promise<void>;
|
|
4
|
+
close: () => Promise<void>;
|
|
5
|
+
writeFile: <T extends Blob>(fileName: string, source: T) => Promise<void>;
|
|
6
|
+
getSchema: (fileName: string) => Promise<QueryResult>;
|
|
7
|
+
query: (sql: string) => Promise<{
|
|
8
|
+
dataset: any[];
|
|
9
|
+
table: any;
|
|
10
|
+
}>;
|
|
11
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { DatasetSchema } from '../DataSet';
|
|
2
|
+
import { DatasetSource } from '../DataSource';
|
|
3
|
+
export interface StorageAdapter {
|
|
4
|
+
open: () => Promise<void>;
|
|
5
|
+
close: () => Promise<void>;
|
|
6
|
+
writeDataset: (datasetId: string, datasetSchema: DatasetSchema, datasetSource?: DatasetSource) => Promise<void>;
|
|
7
|
+
readDataset: (datasetId: string) => Promise<{
|
|
8
|
+
datasetSchema: DatasetSchema;
|
|
9
|
+
datasetSource?: DatasetSource;
|
|
10
|
+
} | null>;
|
|
11
|
+
deleteDataset: (datasetId: string) => Promise<void>;
|
|
12
|
+
listDatasets: () => Promise<{
|
|
13
|
+
datasetId: string;
|
|
14
|
+
datasetSchema: DatasetSchema;
|
|
15
|
+
datasetSource?: DatasetSource;
|
|
16
|
+
}[]>;
|
|
17
|
+
}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { Dataset } from './dataset/dataset';
|
|
2
2
|
import { RawDatasetSource, DatasetColumn } from './types';
|
|
3
3
|
export declare class VQuery {
|
|
4
|
-
private
|
|
5
|
-
private
|
|
4
|
+
private queryAdapter;
|
|
5
|
+
private storageAdapter;
|
|
6
6
|
private isInitialized;
|
|
7
|
-
constructor(
|
|
7
|
+
constructor();
|
|
8
8
|
private checkInitialized;
|
|
9
9
|
private checkDatasetExists;
|
|
10
10
|
createDataset(datasetId: string, columns?: DatasetColumn[], rawDatasetSource?: RawDatasetSource): Promise<void>;
|
|
@@ -14,8 +14,8 @@ export declare class VQuery {
|
|
|
14
14
|
hasDataset(datasetId: string): Promise<boolean>;
|
|
15
15
|
listDatasets(): Promise<{
|
|
16
16
|
datasetId: string;
|
|
17
|
-
dataSource?: import("./types").DatasetSource;
|
|
18
17
|
datasetSchema: import("./types").DatasetSchema;
|
|
18
|
+
datasetSource?: import("./types").DatasetSource;
|
|
19
19
|
}[]>;
|
|
20
20
|
close(): Promise<void>;
|
|
21
21
|
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { Dataset } from './dataset/dataset';
|
|
2
|
+
import { RawDatasetSource, DatasetColumn } from './types';
|
|
3
|
+
export declare class VQuery {
|
|
4
|
+
private queryAdapter;
|
|
5
|
+
private storageAdapter;
|
|
6
|
+
private isInitialized;
|
|
7
|
+
constructor();
|
|
8
|
+
private checkInitialized;
|
|
9
|
+
private checkDatasetExists;
|
|
10
|
+
createDataset(datasetId: string, columns?: DatasetColumn[], rawDatasetSource?: RawDatasetSource): Promise<void>;
|
|
11
|
+
updateDatasetSource(datasetId: string, columns?: DatasetColumn[], rawDatasetSource?: RawDatasetSource): Promise<void>;
|
|
12
|
+
dropDataset(datasetId: string): Promise<void>;
|
|
13
|
+
connectDataset(datasetId: string, temporaryColumns?: DatasetColumn[], temporaryRawDatasetSource?: RawDatasetSource): Promise<Dataset>;
|
|
14
|
+
hasDataset(datasetId: string): Promise<boolean>;
|
|
15
|
+
listDatasets(): Promise<{
|
|
16
|
+
datasetId: string;
|
|
17
|
+
datasetSchema: import("./types").DatasetSchema;
|
|
18
|
+
datasetSource?: import("./types").DatasetSource;
|
|
19
|
+
}[]>;
|
|
20
|
+
close(): Promise<void>;
|
|
21
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { IndexedDBAdapter, InMemoryAdapter } from './storage-adapter';
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { QueryAdapter } from '../../types';
|
|
2
|
+
export declare class DuckDBNodeQueryAdapter implements QueryAdapter {
|
|
3
|
+
private bindings;
|
|
4
|
+
private connection;
|
|
5
|
+
constructor();
|
|
6
|
+
open: () => Promise<void>;
|
|
7
|
+
close: () => Promise<void>;
|
|
8
|
+
writeFile: <T extends Blob>(fileName: string, source: T) => Promise<void>;
|
|
9
|
+
query: (sql: string) => Promise<{
|
|
10
|
+
dataset: any[];
|
|
11
|
+
table: any;
|
|
12
|
+
}>;
|
|
13
|
+
getSchema: (fileName: string) => Promise<any>;
|
|
14
|
+
}
|