@visactor/vquery 0.2.3 → 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/dist/browser/esm/adapters/index.d.ts +1 -0
  2. package/dist/browser/esm/adapters/query-adapter/duckdbNodeAdapter.d.ts +14 -0
  3. package/dist/browser/esm/adapters/query-adapter/duckdbWebAdapter.d.ts +14 -0
  4. package/dist/browser/esm/adapters/storage-adapter/index.d.ts +2 -0
  5. package/dist/{db/indexedDb.d.ts → browser/esm/adapters/storage-adapter/indexeddbAdapter.d.ts} +5 -5
  6. package/dist/browser/esm/adapters/storage-adapter/inmemoryAdapter.d.ts +22 -0
  7. package/dist/browser/esm/browser.d.ts +5 -0
  8. package/dist/{index.js → browser/esm/browser.js} +177 -176
  9. package/dist/{dataset → browser/esm/dataset}/dataset.d.ts +4 -5
  10. package/dist/{index.d.ts → browser/esm/node.d.ts} +1 -1
  11. package/dist/{sql-builder → browser/esm/sql-builder}/utils.d.ts +1 -1
  12. package/dist/browser/esm/types/adapters/QueryAdapter.d.ts +11 -0
  13. package/dist/browser/esm/types/adapters/StorageAdapter.d.ts +17 -0
  14. package/dist/browser/esm/types/adapters/index.d.ts +2 -0
  15. package/dist/{types/QueryDSL → browser/esm/types/dsl}/OrderBy.d.ts +1 -1
  16. package/dist/{types → browser/esm/types}/index.d.ts +2 -1
  17. package/dist/{vquery.d.ts → browser/esm/vquery-browser.d.ts} +4 -4
  18. package/dist/browser/esm/vquery-node.d.ts +21 -0
  19. package/dist/node/cjs/adapters/index.d.ts +1 -0
  20. package/dist/node/cjs/adapters/query-adapter/duckdbNodeAdapter.d.ts +14 -0
  21. package/dist/node/cjs/adapters/query-adapter/duckdbWebAdapter.d.ts +14 -0
  22. package/dist/node/cjs/adapters/storage-adapter/index.d.ts +2 -0
  23. package/dist/node/cjs/adapters/storage-adapter/indexeddbAdapter.d.ts +21 -0
  24. package/dist/node/cjs/adapters/storage-adapter/inmemoryAdapter.d.ts +22 -0
  25. package/dist/node/cjs/browser.d.ts +5 -0
  26. package/dist/node/cjs/data-source-builder/dataSourceBuilder.d.ts +16 -0
  27. package/dist/node/cjs/data-source-builder/index.d.ts +1 -0
  28. package/dist/node/cjs/dataset/dataset.d.ts +30 -0
  29. package/dist/node/cjs/dataset/index.d.ts +2 -0
  30. package/dist/{index.cjs → node/cjs/node.cjs} +134 -181
  31. package/dist/node/cjs/node.d.ts +5 -0
  32. package/dist/node/cjs/sql-builder/builders/groupBy.d.ts +2 -0
  33. package/dist/node/cjs/sql-builder/builders/index.d.ts +3 -0
  34. package/dist/node/cjs/sql-builder/builders/limit.d.ts +2 -0
  35. package/dist/node/cjs/sql-builder/builders/where.d.ts +3 -0
  36. package/dist/node/cjs/sql-builder/compile/index.d.ts +1 -0
  37. package/dist/node/cjs/sql-builder/compile/inlineParameters.d.ts +1 -0
  38. package/dist/node/cjs/sql-builder/dialect/index.d.ts +1 -0
  39. package/dist/node/cjs/sql-builder/dialect/postgresDialect.d.ts +11 -0
  40. package/dist/node/cjs/sql-builder/dslToSQL.d.ts +2 -0
  41. package/dist/node/cjs/sql-builder/index.d.ts +1 -0
  42. package/dist/node/cjs/sql-builder/utils.d.ts +8 -0
  43. package/dist/node/cjs/types/DataSet.d.ts +11 -0
  44. package/dist/node/cjs/types/DataSource.d.ts +11 -0
  45. package/dist/node/cjs/types/QueryResult.d.ts +1 -0
  46. package/dist/node/cjs/types/adapters/QueryAdapter.d.ts +11 -0
  47. package/dist/node/cjs/types/adapters/StorageAdapter.d.ts +17 -0
  48. package/dist/node/cjs/types/adapters/index.d.ts +2 -0
  49. package/dist/node/cjs/types/dsl/GroupBy.d.ts +1 -0
  50. package/dist/node/cjs/types/dsl/OrderBy.d.ts +4 -0
  51. package/dist/node/cjs/types/dsl/QueryDSL.d.ts +12 -0
  52. package/dist/node/cjs/types/dsl/Select.d.ts +7 -0
  53. package/dist/node/cjs/types/dsl/Where.d.ts +24 -0
  54. package/dist/node/cjs/types/dsl/demo.d.ts +1 -0
  55. package/dist/node/cjs/types/dsl/index.d.ts +5 -0
  56. package/dist/node/cjs/types/index.d.ts +5 -0
  57. package/dist/node/cjs/utils/index.d.ts +1 -0
  58. package/dist/node/cjs/utils/url.d.ts +3 -0
  59. package/dist/node/cjs/vquery-browser.d.ts +21 -0
  60. package/dist/node/cjs/vquery-node.d.ts +21 -0
  61. package/dist/node/esm/adapters/index.d.ts +1 -0
  62. package/dist/node/esm/adapters/query-adapter/duckdbNodeAdapter.d.ts +14 -0
  63. package/dist/node/esm/adapters/query-adapter/duckdbWebAdapter.d.ts +14 -0
  64. package/dist/node/esm/adapters/storage-adapter/index.d.ts +2 -0
  65. package/dist/node/esm/adapters/storage-adapter/indexeddbAdapter.d.ts +21 -0
  66. package/dist/node/esm/adapters/storage-adapter/inmemoryAdapter.d.ts +22 -0
  67. package/dist/node/esm/browser.d.ts +5 -0
  68. package/dist/node/esm/data-source-builder/dataSourceBuilder.d.ts +16 -0
  69. package/dist/node/esm/data-source-builder/index.d.ts +1 -0
  70. package/dist/node/esm/dataset/dataset.d.ts +30 -0
  71. package/dist/node/esm/dataset/index.d.ts +2 -0
  72. package/dist/node/esm/node.d.ts +5 -0
  73. package/dist/node/esm/node.js +476 -0
  74. package/dist/node/esm/sql-builder/builders/groupBy.d.ts +2 -0
  75. package/dist/node/esm/sql-builder/builders/index.d.ts +3 -0
  76. package/dist/node/esm/sql-builder/builders/limit.d.ts +2 -0
  77. package/dist/node/esm/sql-builder/builders/where.d.ts +3 -0
  78. package/dist/node/esm/sql-builder/compile/index.d.ts +1 -0
  79. package/dist/node/esm/sql-builder/compile/inlineParameters.d.ts +1 -0
  80. package/dist/node/esm/sql-builder/dialect/index.d.ts +1 -0
  81. package/dist/node/esm/sql-builder/dialect/postgresDialect.d.ts +11 -0
  82. package/dist/node/esm/sql-builder/dslToSQL.d.ts +2 -0
  83. package/dist/node/esm/sql-builder/index.d.ts +1 -0
  84. package/dist/node/esm/sql-builder/utils.d.ts +8 -0
  85. package/dist/node/esm/types/DataSet.d.ts +11 -0
  86. package/dist/node/esm/types/DataSource.d.ts +11 -0
  87. package/dist/node/esm/types/QueryResult.d.ts +1 -0
  88. package/dist/node/esm/types/adapters/QueryAdapter.d.ts +11 -0
  89. package/dist/node/esm/types/adapters/StorageAdapter.d.ts +17 -0
  90. package/dist/node/esm/types/adapters/index.d.ts +2 -0
  91. package/dist/node/esm/types/dsl/GroupBy.d.ts +1 -0
  92. package/dist/node/esm/types/dsl/OrderBy.d.ts +4 -0
  93. package/dist/node/esm/types/dsl/QueryDSL.d.ts +12 -0
  94. package/dist/node/esm/types/dsl/Select.d.ts +7 -0
  95. package/dist/node/esm/types/dsl/Where.d.ts +24 -0
  96. package/dist/node/esm/types/dsl/demo.d.ts +1 -0
  97. package/dist/node/esm/types/dsl/index.d.ts +5 -0
  98. package/dist/node/esm/types/index.d.ts +5 -0
  99. package/dist/node/esm/utils/index.d.ts +1 -0
  100. package/dist/node/esm/utils/url.d.ts +3 -0
  101. package/dist/node/esm/vquery-browser.d.ts +21 -0
  102. package/dist/node/esm/vquery-node.d.ts +21 -0
  103. package/package.json +28 -17
  104. package/dist/db/duckDb.d.ts +0 -35
  105. /package/dist/{data-source-builder → browser/esm/data-source-builder}/dataSourceBuilder.d.ts +0 -0
  106. /package/dist/{data-source-builder → browser/esm/data-source-builder}/index.d.ts +0 -0
  107. /package/dist/{dataset → browser/esm/dataset}/index.d.ts +0 -0
  108. /package/dist/{sql-builder → browser/esm/sql-builder}/builders/groupBy.d.ts +0 -0
  109. /package/dist/{sql-builder → browser/esm/sql-builder}/builders/index.d.ts +0 -0
  110. /package/dist/{sql-builder → browser/esm/sql-builder}/builders/limit.d.ts +0 -0
  111. /package/dist/{sql-builder → browser/esm/sql-builder}/builders/where.d.ts +0 -0
  112. /package/dist/{sql-builder → browser/esm/sql-builder}/compile/index.d.ts +0 -0
  113. /package/dist/{sql-builder → browser/esm/sql-builder}/compile/inlineParameters.d.ts +0 -0
  114. /package/dist/{sql-builder → browser/esm/sql-builder}/dialect/index.d.ts +0 -0
  115. /package/dist/{sql-builder → browser/esm/sql-builder}/dialect/postgresDialect.d.ts +0 -0
  116. /package/dist/{sql-builder → browser/esm/sql-builder}/dslToSQL.d.ts +0 -0
  117. /package/dist/{sql-builder → browser/esm/sql-builder}/index.d.ts +0 -0
  118. /package/dist/{types → browser/esm/types}/DataSet.d.ts +0 -0
  119. /package/dist/{types → browser/esm/types}/DataSource.d.ts +0 -0
  120. /package/dist/{types → browser/esm/types}/QueryResult.d.ts +0 -0
  121. /package/dist/{types/QueryDSL → browser/esm/types/dsl}/GroupBy.d.ts +0 -0
  122. /package/dist/{types/QueryDSL → browser/esm/types/dsl}/QueryDSL.d.ts +0 -0
  123. /package/dist/{types/QueryDSL → browser/esm/types/dsl}/Select.d.ts +0 -0
  124. /package/dist/{types/QueryDSL → browser/esm/types/dsl}/Where.d.ts +0 -0
  125. /package/dist/{types/QueryDSL → browser/esm/types/dsl}/demo.d.ts +0 -0
  126. /package/dist/{types/QueryDSL → browser/esm/types/dsl}/index.d.ts +0 -0
  127. /package/dist/{utils → browser/esm/utils}/index.d.ts +0 -0
  128. /package/dist/{utils → browser/esm/utils}/url.d.ts +0 -0
@@ -0,0 +1 @@
1
+ export { IndexedDBAdapter, InMemoryAdapter } from './storage-adapter';
@@ -0,0 +1,14 @@
1
+ import { QueryAdapter } from '../../types';
2
+ export declare class DuckDBNodeQueryAdapter implements QueryAdapter {
3
+ private bindings;
4
+ private connection;
5
+ constructor();
6
+ open: () => Promise<void>;
7
+ close: () => Promise<void>;
8
+ writeFile: <T extends Blob>(fileName: string, source: T) => Promise<void>;
9
+ query: (sql: string) => Promise<{
10
+ dataset: any[];
11
+ table: any;
12
+ }>;
13
+ getSchema: (fileName: string) => Promise<any>;
14
+ }
@@ -0,0 +1,14 @@
1
+ import { QueryAdapter } from '../../types';
2
+ export declare class DuckDBWebQueryAdapter implements QueryAdapter {
3
+ private db;
4
+ private connection;
5
+ constructor();
6
+ open: () => Promise<void>;
7
+ close: () => Promise<void>;
8
+ writeFile: <T extends Blob>(fileName: string, source: T) => Promise<void>;
9
+ query: (sql: string) => Promise<{
10
+ dataset: any[];
11
+ table: any;
12
+ }>;
13
+ getSchema: (fileName: string) => Promise<any>;
14
+ }
@@ -0,0 +1,2 @@
1
+ export { IndexedDBAdapter } from './indexeddbAdapter';
2
+ export { InMemoryAdapter } from './inmemoryAdapter';
@@ -1,12 +1,12 @@
1
- import { DatasetSource } from '../types';
2
- import { DatasetSchema } from '../types/DataSet';
3
- export declare class IndexedDB {
1
+ import { DatasetSource, StorageAdapter } from '../../types';
2
+ import { DatasetSchema } from '../../types/DataSet';
3
+ export declare class IndexedDBAdapter implements StorageAdapter {
4
4
  private db;
5
5
  private dbName;
6
6
  private datasetStoreName;
7
- constructor(dbName: string);
7
+ constructor(dbName?: string);
8
8
  open: () => Promise<void>;
9
- close: () => void;
9
+ close: () => Promise<void>;
10
10
  writeDataset: (datasetId: string, datasetSchema: DatasetSchema, datasetSource?: DatasetSource) => Promise<void>;
11
11
  readDataset: (datasetId: string) => Promise<{
12
12
  datasetSource?: DatasetSource;
@@ -0,0 +1,22 @@
1
+ import { DatasetSource, StorageAdapter } from '../../types';
2
+ import { DatasetSchema } from '../../types/DataSet';
3
+ type StoredDataset = {
4
+ datasetId: string;
5
+ datasetSchema: DatasetSchema;
6
+ datasetSource?: DatasetSource;
7
+ };
8
+ export declare class InMemoryAdapter implements StorageAdapter {
9
+ private datasets;
10
+ private isOpen;
11
+ constructor();
12
+ open: () => Promise<void>;
13
+ close: () => Promise<void>;
14
+ writeDataset: (datasetId: string, datasetSchema: DatasetSchema, datasetSource?: DatasetSource) => Promise<void>;
15
+ readDataset: (datasetId: string) => Promise<{
16
+ datasetSource?: DatasetSource;
17
+ datasetSchema: DatasetSchema;
18
+ } | null>;
19
+ deleteDataset: (datasetId: string) => Promise<void>;
20
+ listDatasets: () => Promise<StoredDataset[]>;
21
+ }
22
+ export {};
@@ -0,0 +1,5 @@
1
+ export { convertDSLToSQL } from './dataset';
2
+ export { DatasetSourceBuilder } from './data-source-builder/dataSourceBuilder';
3
+ export * from './utils';
4
+ export * from './types';
5
+ export { VQuery } from './vquery-browser';
@@ -142,16 +142,16 @@ const convertDSLToSQL = (dsl, tableName)=>{
142
142
  return inlineParameters(compiled.sql, compiled.parameters);
143
143
  };
144
144
  class Dataset {
145
- duckDB;
146
- indexedDB;
145
+ queryAdapter;
146
+ storageAdapter;
147
147
  _datasetId;
148
148
  constructor(duckDB, indexedDB1, datasetId){
149
- this.duckDB = duckDB;
150
- this.indexedDB = indexedDB1;
149
+ this.queryAdapter = duckDB;
150
+ this.storageAdapter = indexedDB1;
151
151
  this._datasetId = datasetId;
152
152
  }
153
153
  async init(temporaryColumns, temporaryDatasetSource) {
154
- const datasetInfo = await this.indexedDB.readDataset(this._datasetId);
154
+ const datasetInfo = await this.storageAdapter.readDataset(this._datasetId);
155
155
  if (!datasetInfo) throw new Error(`Dataset ${this._datasetId} not found`);
156
156
  const columns = temporaryColumns ? temporaryColumns : datasetInfo.datasetSchema.columns;
157
157
  const datasetSource = temporaryDatasetSource || datasetInfo.datasetSource;
@@ -174,11 +174,11 @@ class Dataset {
174
174
  if (datasetSource) {
175
175
  const readFunction = readFunctionMap[datasetSource.type];
176
176
  if (!readFunction) throw new Error(`Unsupported dataSource type: ${datasetSource.type}`);
177
- await this.duckDB.writeFile(this._datasetId, datasetSource.blob);
177
+ await this.queryAdapter.writeFile(this._datasetId, datasetSource.blob);
178
178
  const columnsStruct = `{${columns.map((c)=>`'${c.name}': '${dataTypeMap[c.type] || 'VARCHAR'}'`).join(', ')}}`;
179
179
  const columnNames = columns.map((c)=>`"${c.name}"`).join(', ');
180
180
  const createViewSql = `CREATE OR REPLACE VIEW "${this._datasetId}" AS SELECT ${columnNames} FROM ${readFunction}('${this._datasetId}', columns=${columnsStruct})`;
181
- await this.duckDB.query(createViewSql);
181
+ await this.queryAdapter.query(createViewSql);
182
182
  }
183
183
  }
184
184
  async query(queryDSL) {
@@ -187,7 +187,7 @@ class Dataset {
187
187
  }
188
188
  async queryBySQL(sql) {
189
189
  const start = performance?.now?.()?.toFixed(3) ?? Date.now().toFixed(3);
190
- const result = await this.duckDB.query(sql);
190
+ const result = await this.queryAdapter.query(sql);
191
191
  const end = performance?.now?.()?.toFixed(3) ?? Date.now().toFixed(3);
192
192
  return {
193
193
  ...result,
@@ -199,79 +199,111 @@ class Dataset {
199
199
  };
200
200
  }
201
201
  async disconnect() {
202
- await this.duckDB.query(`DROP VIEW IF EXISTS "${this._datasetId}"`);
202
+ await this.queryAdapter.query(`DROP VIEW IF EXISTS "${this._datasetId}"`);
203
203
  }
204
204
  get datasetId() {
205
205
  return this._datasetId;
206
206
  }
207
207
  }
208
- class DuckDB {
209
- db = null;
210
- connection = null;
211
- constructor(){}
212
- init = async ()=>{
213
- const MANUAL_BUNDLES = {
214
- mvp: {
215
- mainModule: new URL('@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm', import.meta.url).href,
216
- mainWorker: new URL('@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js', import.meta.url).toString()
217
- },
218
- eh: {
219
- mainModule: new URL('@duckdb/duckdb-wasm/dist/duckdb-eh.wasm', import.meta.url).href,
220
- mainWorker: new URL('@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js', import.meta.url).toString()
221
- }
222
- };
223
- const bundle = await selectBundle(MANUAL_BUNDLES);
224
- const worker_url = URL.createObjectURL(new Blob([
225
- `importScripts("${bundle.mainWorker}");`
226
- ], {
227
- type: "text/javascript"
228
- }));
229
- const worker = new Worker(worker_url);
230
- const logger = new ConsoleLogger();
231
- this.db = new AsyncDuckDB(logger, worker);
232
- await this.db.instantiate(bundle.mainModule, bundle.pthreadWorker);
233
- URL.revokeObjectURL(worker_url);
234
- this.connection = await this.db.connect();
235
- };
236
- close = async ()=>{
237
- if (this.connection) {
238
- await this.connection.close();
239
- this.connection = null;
240
- }
241
- if (this.db) {
242
- await this.db.terminate();
243
- this.db = null;
244
- }
245
- };
246
- writeFile = async (fileName, source)=>{
247
- if (!this.db) throw new Error('db is null');
248
- let uint8Array;
249
- if (source instanceof Blob) {
250
- const buffer = await source.arrayBuffer();
251
- uint8Array = new Uint8Array(buffer);
252
- } else throw new Error('Unsupported source type');
253
- await this.db.registerFileBuffer(fileName, uint8Array);
254
- };
255
- query = async (sql)=>{
256
- if (!this.connection) throw new Error('connection is null');
257
- const table = await this.connection.query(sql);
258
- const dataset = table.toArray().map((row)=>row.toJSON());
208
+ const isUrl = (url)=>isHttpUrl(url) || isBase64Url(url);
209
+ const isHttpUrl = (url)=>url.startsWith('http://') || url.startsWith('https://');
210
+ const isBase64Url = (url)=>url.startsWith('data:');
211
+ class DatasetSourceBuilder {
212
+ type;
213
+ value;
214
+ constructor(raw){
215
+ this.type = raw.type;
216
+ this.value = raw.rawDataset;
217
+ }
218
+ static from(raw) {
219
+ return new DatasetSourceBuilder(raw);
220
+ }
221
+ async build() {
222
+ const blob = await DatasetSourceBuilder.convertToBlob(this.type, this.value);
259
223
  return {
260
- dataset,
261
- table
224
+ type: this.type,
225
+ blob: blob
262
226
  };
263
- };
264
- getSchema = async (fileName)=>{
265
- if (!this.connection) throw new Error('connection is null');
266
- const result = await this.connection.query(`PRAGMA table_info('${fileName}')`);
267
- return result.toArray().map((row)=>row.toJSON());
268
- };
227
+ }
228
+ static async convertToBlob(type, value) {
229
+ if (value instanceof Blob) return value;
230
+ const convertCsvToBlob = (csvSource)=>{
231
+ if (csvSource instanceof ArrayBuffer) return new Blob([
232
+ csvSource
233
+ ], {
234
+ type: 'text/csv'
235
+ });
236
+ if ('string' == typeof csvSource && isUrl(csvSource)) return DatasetSourceBuilder.fetchBlob(csvSource);
237
+ return new Blob([
238
+ JSON.stringify(csvSource)
239
+ ], {
240
+ type: 'text/csv'
241
+ });
242
+ };
243
+ const convertJsonToBlob = (jsonSource)=>{
244
+ if (jsonSource instanceof ArrayBuffer) return new Blob([
245
+ jsonSource
246
+ ], {
247
+ type: 'application/json'
248
+ });
249
+ if ('string' == typeof jsonSource && isUrl(jsonSource)) return DatasetSourceBuilder.fetchBlob(jsonSource);
250
+ return new Blob([
251
+ JSON.stringify(jsonSource)
252
+ ], {
253
+ type: 'application/json'
254
+ });
255
+ };
256
+ const convertParquetToBlob = (parquetSource)=>{
257
+ if (parquetSource instanceof ArrayBuffer) return new Blob([
258
+ parquetSource
259
+ ], {
260
+ type: 'application/parquet'
261
+ });
262
+ if ('string' == typeof parquetSource && isUrl(parquetSource)) return DatasetSourceBuilder.fetchBlob(parquetSource);
263
+ return new Blob([
264
+ parquetSource
265
+ ], {
266
+ type: 'application/parquet'
267
+ });
268
+ };
269
+ const convertXlsxToBlob = (xlsxSource)=>{
270
+ if (xlsxSource instanceof ArrayBuffer) return new Blob([
271
+ xlsxSource
272
+ ], {
273
+ type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
274
+ });
275
+ if ('string' == typeof xlsxSource && isUrl(xlsxSource)) return DatasetSourceBuilder.fetchBlob(xlsxSource);
276
+ return new Blob([
277
+ xlsxSource
278
+ ], {
279
+ type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
280
+ });
281
+ };
282
+ switch(type){
283
+ case 'csv':
284
+ return convertCsvToBlob(value);
285
+ case 'json':
286
+ return convertJsonToBlob(value);
287
+ case 'xlsx':
288
+ return convertXlsxToBlob(value);
289
+ case 'parquet':
290
+ return convertParquetToBlob(value);
291
+ default:
292
+ return new Blob([
293
+ value
294
+ ]);
295
+ }
296
+ }
297
+ static async fetchBlob(url) {
298
+ const response = await fetch(url);
299
+ return await response.blob();
300
+ }
269
301
  }
270
- class IndexedDB {
302
+ class IndexedDBAdapter {
271
303
  db = null;
272
304
  dbName;
273
305
  datasetStoreName = 'vqueryDatasets';
274
- constructor(dbName){
306
+ constructor(dbName = 'vquery'){
275
307
  this.dbName = dbName;
276
308
  }
277
309
  open = ()=>new Promise((resolve, reject)=>{
@@ -290,7 +322,7 @@ class IndexedDB {
290
322
  reject(event.target.error);
291
323
  };
292
324
  });
293
- close = ()=>{
325
+ close = async ()=>{
294
326
  if (this.db) {
295
327
  this.db.close();
296
328
  this.db = null;
@@ -359,112 +391,80 @@ class IndexedDB {
359
391
  };
360
392
  });
361
393
  }
362
- const isUrl = (url)=>isHttpUrl(url) || isBase64Url(url);
363
- const isHttpUrl = (url)=>url.startsWith('http://') || url.startsWith('https://');
364
- const isBase64Url = (url)=>url.startsWith('data:');
365
- class DatasetSourceBuilder {
366
- type;
367
- value;
368
- constructor(raw){
369
- this.type = raw.type;
370
- this.value = raw.rawDataset;
371
- }
372
- static from(raw) {
373
- return new DatasetSourceBuilder(raw);
374
- }
375
- async build() {
376
- const blob = await DatasetSourceBuilder.convertToBlob(this.type, this.value);
377
- return {
378
- type: this.type,
379
- blob: blob
380
- };
381
- }
382
- static async convertToBlob(type, value) {
383
- if (value instanceof Blob) return value;
384
- const convertCsvToBlob = (csvSource)=>{
385
- if (csvSource instanceof ArrayBuffer) return new Blob([
386
- csvSource
387
- ], {
388
- type: 'text/csv'
389
- });
390
- if ('string' == typeof csvSource && isUrl(csvSource)) return DatasetSourceBuilder.fetchBlob(csvSource);
391
- return new Blob([
392
- JSON.stringify(csvSource)
393
- ], {
394
- type: 'text/csv'
395
- });
396
- };
397
- const convertJsonToBlob = (jsonSource)=>{
398
- if (jsonSource instanceof ArrayBuffer) return new Blob([
399
- jsonSource
400
- ], {
401
- type: 'application/json'
402
- });
403
- if ('string' == typeof jsonSource && isUrl(jsonSource)) return DatasetSourceBuilder.fetchBlob(jsonSource);
404
- return new Blob([
405
- JSON.stringify(jsonSource)
406
- ], {
407
- type: 'application/json'
408
- });
409
- };
410
- const convertParquetToBlob = (parquetSource)=>{
411
- if (parquetSource instanceof ArrayBuffer) return new Blob([
412
- parquetSource
413
- ], {
414
- type: 'application/parquet'
415
- });
416
- if ('string' == typeof parquetSource && isUrl(parquetSource)) return DatasetSourceBuilder.fetchBlob(parquetSource);
417
- return new Blob([
418
- parquetSource
419
- ], {
420
- type: 'application/parquet'
421
- });
422
- };
423
- const convertXlsxToBlob = (xlsxSource)=>{
424
- if (xlsxSource instanceof ArrayBuffer) return new Blob([
425
- xlsxSource
426
- ], {
427
- type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
428
- });
429
- if ('string' == typeof xlsxSource && isUrl(xlsxSource)) return DatasetSourceBuilder.fetchBlob(xlsxSource);
430
- return new Blob([
431
- xlsxSource
432
- ], {
433
- type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
434
- });
394
+ class DuckDBWebQueryAdapter {
395
+ db = null;
396
+ connection = null;
397
+ constructor(){}
398
+ open = async ()=>{
399
+ const MANUAL_BUNDLES = {
400
+ mvp: {
401
+ mainModule: new URL('@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm', import.meta.url).href,
402
+ mainWorker: new URL('@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js', import.meta.url).toString()
403
+ },
404
+ eh: {
405
+ mainModule: new URL('@duckdb/duckdb-wasm/dist/duckdb-eh.wasm', import.meta.url).href,
406
+ mainWorker: new URL('@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js', import.meta.url).toString()
407
+ }
435
408
  };
436
- switch(type){
437
- case 'csv':
438
- return convertCsvToBlob(value);
439
- case 'json':
440
- return convertJsonToBlob(value);
441
- case 'xlsx':
442
- return convertXlsxToBlob(value);
443
- case 'parquet':
444
- return convertParquetToBlob(value);
445
- default:
446
- return new Blob([
447
- value
448
- ]);
409
+ const bundle = await selectBundle(MANUAL_BUNDLES);
410
+ const worker_url = URL.createObjectURL(new Blob([
411
+ `importScripts("${bundle.mainWorker}");`
412
+ ], {
413
+ type: "text/javascript"
414
+ }));
415
+ const worker = new Worker(worker_url);
416
+ const logger = new ConsoleLogger();
417
+ this.db = new AsyncDuckDB(logger, worker);
418
+ await this.db.instantiate(bundle.mainModule, bundle.pthreadWorker);
419
+ URL.revokeObjectURL(worker_url);
420
+ this.connection = await this.db.connect();
421
+ };
422
+ close = async ()=>{
423
+ if (this.connection) {
424
+ await this.connection.close();
425
+ this.connection = null;
449
426
  }
450
- }
451
- static async fetchBlob(url) {
452
- const response = await fetch(url);
453
- return await response.blob();
454
- }
427
+ if (this.db) {
428
+ await this.db.terminate();
429
+ this.db = null;
430
+ }
431
+ };
432
+ writeFile = async (fileName, source)=>{
433
+ if (!this.db) throw new Error('db is null');
434
+ let uint8Array;
435
+ if (source instanceof Blob) {
436
+ const buffer = await source.arrayBuffer();
437
+ uint8Array = new Uint8Array(buffer);
438
+ } else throw new Error('Unsupported source type');
439
+ await this.db.registerFileBuffer(fileName, uint8Array);
440
+ };
441
+ query = async (sql)=>{
442
+ if (!this.connection) throw new Error('connection is null');
443
+ const table = await this.connection.query(sql);
444
+ const dataset = table.toArray().map((row)=>row.toJSON());
445
+ return {
446
+ dataset,
447
+ table
448
+ };
449
+ };
450
+ getSchema = async (fileName)=>{
451
+ if (!this.connection) throw new Error('connection is null');
452
+ const result = await this.connection.query(`PRAGMA table_info('${fileName}')`);
453
+ return result.toArray().map((row)=>row.toJSON());
454
+ };
455
455
  }
456
456
  class VQuery {
457
- duckDB;
458
- indexedDB;
457
+ queryAdapter;
458
+ storageAdapter;
459
459
  isInitialized = false;
460
- constructor(dbName = 'vquery'){
461
- this.duckDB = new DuckDB();
462
- this.indexedDB = new IndexedDB(dbName);
460
+ constructor(){
461
+ this.queryAdapter = new DuckDBWebQueryAdapter();
462
+ this.storageAdapter = new IndexedDBAdapter();
463
463
  }
464
464
  async checkInitialized() {
465
465
  if (!this.isInitialized) {
466
- await this.duckDB.init();
467
- await this.indexedDB.open();
466
+ await this.queryAdapter.open();
467
+ await this.storageAdapter.open();
468
468
  this.isInitialized = true;
469
469
  }
470
470
  }
@@ -480,7 +480,7 @@ class VQuery {
480
480
  datasetAlias: datasetId,
481
481
  columns: columns
482
482
  };
483
- await this.indexedDB.writeDataset(datasetId, datasetSchema, datasetSource);
483
+ await this.storageAdapter.writeDataset(datasetId, datasetSchema, datasetSource);
484
484
  }
485
485
  async updateDatasetSource(datasetId, columns = [], rawDatasetSource) {
486
486
  await this.checkInitialized();
@@ -491,33 +491,34 @@ class VQuery {
491
491
  datasetAlias: datasetId,
492
492
  columns: columns
493
493
  };
494
- await this.indexedDB.writeDataset(datasetId, datasetSchema, datasetSource);
494
+ await this.storageAdapter.writeDataset(datasetId, datasetSchema, datasetSource);
495
495
  }
496
496
  async dropDataset(datasetId) {
497
497
  await this.checkInitialized();
498
498
  await this.checkDatasetExists(datasetId);
499
- await this.indexedDB.deleteDataset(datasetId);
499
+ await this.storageAdapter.deleteDataset(datasetId);
500
500
  }
501
501
  async connectDataset(datasetId, temporaryColumns, temporaryRawDatasetSource) {
502
502
  await this.checkInitialized();
503
503
  await this.checkDatasetExists(datasetId);
504
- const dataset = new Dataset(this.duckDB, this.indexedDB, datasetId);
504
+ const dataset = new Dataset(this.queryAdapter, this.storageAdapter, datasetId);
505
505
  const temporaryDatasetSource = temporaryRawDatasetSource ? await DatasetSourceBuilder.from(temporaryRawDatasetSource).build() : void 0;
506
506
  await dataset.init(temporaryColumns, temporaryDatasetSource);
507
507
  return dataset;
508
508
  }
509
509
  async hasDataset(datasetId) {
510
510
  await this.checkInitialized();
511
- const datasets = await this.indexedDB.listDatasets();
511
+ const datasets = await this.storageAdapter.listDatasets();
512
512
  return datasets.some((item)=>item.datasetId === datasetId);
513
513
  }
514
514
  async listDatasets() {
515
515
  await this.checkInitialized();
516
- return this.indexedDB.listDatasets();
516
+ return this.storageAdapter.listDatasets();
517
517
  }
518
518
  async close() {
519
519
  await this.checkInitialized();
520
- await this.duckDB.close();
520
+ await this.queryAdapter.close();
521
+ await this.storageAdapter.close();
521
522
  }
522
523
  }
523
524
  export { DatasetSourceBuilder, VQuery, convertDSLToSQL, isBase64Url, isHttpUrl, isUrl };
@@ -1,11 +1,10 @@
1
1
  import { DatasetColumn, DatasetSource, QueryDSL } from '../types';
2
- import { DuckDB } from '../db/duckDb';
3
- import { IndexedDB } from '../db/indexedDb';
2
+ import { QueryAdapter, StorageAdapter } from '../types';
4
3
  export declare class Dataset {
5
- private duckDB;
6
- private indexedDB;
4
+ private queryAdapter;
5
+ private storageAdapter;
7
6
  private _datasetId;
8
- constructor(duckDB: DuckDB, indexedDB: IndexedDB, datasetId: string);
7
+ constructor(duckDB: QueryAdapter, indexedDB: StorageAdapter, datasetId: string);
9
8
  init(temporaryColumns?: DatasetColumn[], temporaryDatasetSource?: DatasetSource): Promise<void>;
10
9
  createOrReplaceView(columns: DatasetColumn[], datasetSource: DatasetSource): Promise<void>;
11
10
  query<T extends Record<string, number | string>>(queryDSL: QueryDSL<T>): Promise<{
@@ -1,5 +1,5 @@
1
- export { VQuery } from './vquery';
2
1
  export { convertDSLToSQL } from './dataset';
3
2
  export { DatasetSourceBuilder } from './data-source-builder/dataSourceBuilder';
4
3
  export * from './utils';
5
4
  export * from './types';
5
+ export { VQuery } from './vquery-node';
@@ -1,5 +1,5 @@
1
1
  import { Where, WhereClause, WhereGroup, WhereLeaf } from '../types';
2
- import { SelectItem } from '../types/QueryDSL/Select';
2
+ import { SelectItem } from '../types/dsl/Select';
3
3
  export declare const isSelectItem: <T>(item: keyof T | SelectItem<T>) => item is SelectItem<T>;
4
4
  export declare const isWhereLeaf: <T>(where: Where<T> | WhereClause<T>) => where is WhereLeaf<T>;
5
5
  export declare const isWhereGroup: <T>(where: Where<T> | WhereClause<T>) => where is WhereGroup<T>;
@@ -0,0 +1,11 @@
1
+ import { QueryResult } from '../QueryResult';
2
+ export interface QueryAdapter {
3
+ open: () => Promise<void>;
4
+ close: () => Promise<void>;
5
+ writeFile: <T extends Blob>(fileName: string, source: T) => Promise<void>;
6
+ getSchema: (fileName: string) => Promise<QueryResult>;
7
+ query: (sql: string) => Promise<{
8
+ dataset: any[];
9
+ table: any;
10
+ }>;
11
+ }
@@ -0,0 +1,17 @@
1
+ import { DatasetSchema } from '../DataSet';
2
+ import { DatasetSource } from '../DataSource';
3
+ export interface StorageAdapter {
4
+ open: () => Promise<void>;
5
+ close: () => Promise<void>;
6
+ writeDataset: (datasetId: string, datasetSchema: DatasetSchema, datasetSource?: DatasetSource) => Promise<void>;
7
+ readDataset: (datasetId: string) => Promise<{
8
+ datasetSchema: DatasetSchema;
9
+ datasetSource?: DatasetSource;
10
+ } | null>;
11
+ deleteDataset: (datasetId: string) => Promise<void>;
12
+ listDatasets: () => Promise<{
13
+ datasetId: string;
14
+ datasetSchema: DatasetSchema;
15
+ datasetSource?: DatasetSource;
16
+ }[]>;
17
+ }
@@ -0,0 +1,2 @@
1
+ export * from './QueryAdapter';
2
+ export * from './StorageAdapter';
@@ -1,4 +1,4 @@
1
1
  export type OrderBy<T> = Array<{
2
- field: keyof T;
2
+ field: string | keyof T;
3
3
  order?: 'asc' | 'desc';
4
4
  }>;
@@ -1,4 +1,5 @@
1
1
  export * from './DataSet';
2
2
  export * from './DataSource';
3
3
  export * from './QueryResult';
4
- export * from './QueryDSL';
4
+ export * from './dsl';
5
+ export * from './adapters';
@@ -1,10 +1,10 @@
1
1
  import { Dataset } from './dataset/dataset';
2
2
  import { RawDatasetSource, DatasetColumn } from './types';
3
3
  export declare class VQuery {
4
- private duckDB;
5
- private indexedDB;
4
+ private queryAdapter;
5
+ private storageAdapter;
6
6
  private isInitialized;
7
- constructor(dbName?: string);
7
+ constructor();
8
8
  private checkInitialized;
9
9
  private checkDatasetExists;
10
10
  createDataset(datasetId: string, columns?: DatasetColumn[], rawDatasetSource?: RawDatasetSource): Promise<void>;
@@ -14,8 +14,8 @@ export declare class VQuery {
14
14
  hasDataset(datasetId: string): Promise<boolean>;
15
15
  listDatasets(): Promise<{
16
16
  datasetId: string;
17
- dataSource?: import("./types").DatasetSource;
18
17
  datasetSchema: import("./types").DatasetSchema;
18
+ datasetSource?: import("./types").DatasetSource;
19
19
  }[]>;
20
20
  close(): Promise<void>;
21
21
  }