@visactor/vquery 0.2.3 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/esm/adapters/index.d.ts +1 -0
- package/dist/browser/esm/adapters/query-adapter/duckdbNodeAdapter.d.ts +14 -0
- package/dist/browser/esm/adapters/query-adapter/duckdbWebAdapter.d.ts +14 -0
- package/dist/browser/esm/adapters/storage-adapter/index.d.ts +2 -0
- package/dist/{db/indexedDb.d.ts → browser/esm/adapters/storage-adapter/indexeddbAdapter.d.ts} +5 -5
- package/dist/browser/esm/adapters/storage-adapter/inmemoryAdapter.d.ts +22 -0
- package/dist/browser/esm/browser.d.ts +5 -0
- package/dist/{index.js → browser/esm/browser.js} +177 -176
- package/dist/{dataset → browser/esm/dataset}/dataset.d.ts +4 -5
- package/dist/{index.d.ts → browser/esm/node.d.ts} +1 -1
- package/dist/{sql-builder → browser/esm/sql-builder}/utils.d.ts +1 -1
- package/dist/browser/esm/types/adapters/QueryAdapter.d.ts +11 -0
- package/dist/browser/esm/types/adapters/StorageAdapter.d.ts +17 -0
- package/dist/browser/esm/types/adapters/index.d.ts +2 -0
- package/dist/{types/QueryDSL → browser/esm/types/dsl}/OrderBy.d.ts +1 -1
- package/dist/{types → browser/esm/types}/index.d.ts +2 -1
- package/dist/{vquery.d.ts → browser/esm/vquery-browser.d.ts} +4 -4
- package/dist/browser/esm/vquery-node.d.ts +21 -0
- package/dist/node/cjs/adapters/index.d.ts +1 -0
- package/dist/node/cjs/adapters/query-adapter/duckdbNodeAdapter.d.ts +14 -0
- package/dist/node/cjs/adapters/query-adapter/duckdbWebAdapter.d.ts +14 -0
- package/dist/node/cjs/adapters/storage-adapter/index.d.ts +2 -0
- package/dist/node/cjs/adapters/storage-adapter/indexeddbAdapter.d.ts +21 -0
- package/dist/node/cjs/adapters/storage-adapter/inmemoryAdapter.d.ts +22 -0
- package/dist/node/cjs/browser.d.ts +5 -0
- package/dist/node/cjs/data-source-builder/dataSourceBuilder.d.ts +16 -0
- package/dist/node/cjs/data-source-builder/index.d.ts +1 -0
- package/dist/node/cjs/dataset/dataset.d.ts +30 -0
- package/dist/node/cjs/dataset/index.d.ts +2 -0
- package/dist/{index.cjs → node/cjs/node.cjs} +134 -181
- package/dist/node/cjs/node.d.ts +5 -0
- package/dist/node/cjs/sql-builder/builders/groupBy.d.ts +2 -0
- package/dist/node/cjs/sql-builder/builders/index.d.ts +3 -0
- package/dist/node/cjs/sql-builder/builders/limit.d.ts +2 -0
- package/dist/node/cjs/sql-builder/builders/where.d.ts +3 -0
- package/dist/node/cjs/sql-builder/compile/index.d.ts +1 -0
- package/dist/node/cjs/sql-builder/compile/inlineParameters.d.ts +1 -0
- package/dist/node/cjs/sql-builder/dialect/index.d.ts +1 -0
- package/dist/node/cjs/sql-builder/dialect/postgresDialect.d.ts +11 -0
- package/dist/node/cjs/sql-builder/dslToSQL.d.ts +2 -0
- package/dist/node/cjs/sql-builder/index.d.ts +1 -0
- package/dist/node/cjs/sql-builder/utils.d.ts +8 -0
- package/dist/node/cjs/types/DataSet.d.ts +11 -0
- package/dist/node/cjs/types/DataSource.d.ts +11 -0
- package/dist/node/cjs/types/QueryResult.d.ts +1 -0
- package/dist/node/cjs/types/adapters/QueryAdapter.d.ts +11 -0
- package/dist/node/cjs/types/adapters/StorageAdapter.d.ts +17 -0
- package/dist/node/cjs/types/adapters/index.d.ts +2 -0
- package/dist/node/cjs/types/dsl/GroupBy.d.ts +1 -0
- package/dist/node/cjs/types/dsl/OrderBy.d.ts +4 -0
- package/dist/node/cjs/types/dsl/QueryDSL.d.ts +12 -0
- package/dist/node/cjs/types/dsl/Select.d.ts +7 -0
- package/dist/node/cjs/types/dsl/Where.d.ts +24 -0
- package/dist/node/cjs/types/dsl/demo.d.ts +1 -0
- package/dist/node/cjs/types/dsl/index.d.ts +5 -0
- package/dist/node/cjs/types/index.d.ts +5 -0
- package/dist/node/cjs/utils/index.d.ts +1 -0
- package/dist/node/cjs/utils/url.d.ts +3 -0
- package/dist/node/cjs/vquery-browser.d.ts +21 -0
- package/dist/node/cjs/vquery-node.d.ts +21 -0
- package/dist/node/esm/adapters/index.d.ts +1 -0
- package/dist/node/esm/adapters/query-adapter/duckdbNodeAdapter.d.ts +14 -0
- package/dist/node/esm/adapters/query-adapter/duckdbWebAdapter.d.ts +14 -0
- package/dist/node/esm/adapters/storage-adapter/index.d.ts +2 -0
- package/dist/node/esm/adapters/storage-adapter/indexeddbAdapter.d.ts +21 -0
- package/dist/node/esm/adapters/storage-adapter/inmemoryAdapter.d.ts +22 -0
- package/dist/node/esm/browser.d.ts +5 -0
- package/dist/node/esm/data-source-builder/dataSourceBuilder.d.ts +16 -0
- package/dist/node/esm/data-source-builder/index.d.ts +1 -0
- package/dist/node/esm/dataset/dataset.d.ts +30 -0
- package/dist/node/esm/dataset/index.d.ts +2 -0
- package/dist/node/esm/node.d.ts +5 -0
- package/dist/node/esm/node.js +476 -0
- package/dist/node/esm/sql-builder/builders/groupBy.d.ts +2 -0
- package/dist/node/esm/sql-builder/builders/index.d.ts +3 -0
- package/dist/node/esm/sql-builder/builders/limit.d.ts +2 -0
- package/dist/node/esm/sql-builder/builders/where.d.ts +3 -0
- package/dist/node/esm/sql-builder/compile/index.d.ts +1 -0
- package/dist/node/esm/sql-builder/compile/inlineParameters.d.ts +1 -0
- package/dist/node/esm/sql-builder/dialect/index.d.ts +1 -0
- package/dist/node/esm/sql-builder/dialect/postgresDialect.d.ts +11 -0
- package/dist/node/esm/sql-builder/dslToSQL.d.ts +2 -0
- package/dist/node/esm/sql-builder/index.d.ts +1 -0
- package/dist/node/esm/sql-builder/utils.d.ts +8 -0
- package/dist/node/esm/types/DataSet.d.ts +11 -0
- package/dist/node/esm/types/DataSource.d.ts +11 -0
- package/dist/node/esm/types/QueryResult.d.ts +1 -0
- package/dist/node/esm/types/adapters/QueryAdapter.d.ts +11 -0
- package/dist/node/esm/types/adapters/StorageAdapter.d.ts +17 -0
- package/dist/node/esm/types/adapters/index.d.ts +2 -0
- package/dist/node/esm/types/dsl/GroupBy.d.ts +1 -0
- package/dist/node/esm/types/dsl/OrderBy.d.ts +4 -0
- package/dist/node/esm/types/dsl/QueryDSL.d.ts +12 -0
- package/dist/node/esm/types/dsl/Select.d.ts +7 -0
- package/dist/node/esm/types/dsl/Where.d.ts +24 -0
- package/dist/node/esm/types/dsl/demo.d.ts +1 -0
- package/dist/node/esm/types/dsl/index.d.ts +5 -0
- package/dist/node/esm/types/index.d.ts +5 -0
- package/dist/node/esm/utils/index.d.ts +1 -0
- package/dist/node/esm/utils/url.d.ts +3 -0
- package/dist/node/esm/vquery-browser.d.ts +21 -0
- package/dist/node/esm/vquery-node.d.ts +21 -0
- package/package.json +28 -17
- package/dist/db/duckDb.d.ts +0 -35
- /package/dist/{data-source-builder → browser/esm/data-source-builder}/dataSourceBuilder.d.ts +0 -0
- /package/dist/{data-source-builder → browser/esm/data-source-builder}/index.d.ts +0 -0
- /package/dist/{dataset → browser/esm/dataset}/index.d.ts +0 -0
- /package/dist/{sql-builder → browser/esm/sql-builder}/builders/groupBy.d.ts +0 -0
- /package/dist/{sql-builder → browser/esm/sql-builder}/builders/index.d.ts +0 -0
- /package/dist/{sql-builder → browser/esm/sql-builder}/builders/limit.d.ts +0 -0
- /package/dist/{sql-builder → browser/esm/sql-builder}/builders/where.d.ts +0 -0
- /package/dist/{sql-builder → browser/esm/sql-builder}/compile/index.d.ts +0 -0
- /package/dist/{sql-builder → browser/esm/sql-builder}/compile/inlineParameters.d.ts +0 -0
- /package/dist/{sql-builder → browser/esm/sql-builder}/dialect/index.d.ts +0 -0
- /package/dist/{sql-builder → browser/esm/sql-builder}/dialect/postgresDialect.d.ts +0 -0
- /package/dist/{sql-builder → browser/esm/sql-builder}/dslToSQL.d.ts +0 -0
- /package/dist/{sql-builder → browser/esm/sql-builder}/index.d.ts +0 -0
- /package/dist/{types → browser/esm/types}/DataSet.d.ts +0 -0
- /package/dist/{types → browser/esm/types}/DataSource.d.ts +0 -0
- /package/dist/{types → browser/esm/types}/QueryResult.d.ts +0 -0
- /package/dist/{types/QueryDSL → browser/esm/types/dsl}/GroupBy.d.ts +0 -0
- /package/dist/{types/QueryDSL → browser/esm/types/dsl}/QueryDSL.d.ts +0 -0
- /package/dist/{types/QueryDSL → browser/esm/types/dsl}/Select.d.ts +0 -0
- /package/dist/{types/QueryDSL → browser/esm/types/dsl}/Where.d.ts +0 -0
- /package/dist/{types/QueryDSL → browser/esm/types/dsl}/demo.d.ts +0 -0
- /package/dist/{types/QueryDSL → browser/esm/types/dsl}/index.d.ts +0 -0
- /package/dist/{utils → browser/esm/utils}/index.d.ts +0 -0
- /package/dist/{utils → browser/esm/utils}/url.d.ts +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { IndexedDBAdapter, InMemoryAdapter } from './storage-adapter';
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { QueryAdapter } from '../../types';
|
|
2
|
+
export declare class DuckDBNodeQueryAdapter implements QueryAdapter {
|
|
3
|
+
private bindings;
|
|
4
|
+
private connection;
|
|
5
|
+
constructor();
|
|
6
|
+
open: () => Promise<void>;
|
|
7
|
+
close: () => Promise<void>;
|
|
8
|
+
writeFile: <T extends Blob>(fileName: string, source: T) => Promise<void>;
|
|
9
|
+
query: (sql: string) => Promise<{
|
|
10
|
+
dataset: any[];
|
|
11
|
+
table: any;
|
|
12
|
+
}>;
|
|
13
|
+
getSchema: (fileName: string) => Promise<any>;
|
|
14
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { QueryAdapter } from '../../types';
|
|
2
|
+
export declare class DuckDBWebQueryAdapter implements QueryAdapter {
|
|
3
|
+
private db;
|
|
4
|
+
private connection;
|
|
5
|
+
constructor();
|
|
6
|
+
open: () => Promise<void>;
|
|
7
|
+
close: () => Promise<void>;
|
|
8
|
+
writeFile: <T extends Blob>(fileName: string, source: T) => Promise<void>;
|
|
9
|
+
query: (sql: string) => Promise<{
|
|
10
|
+
dataset: any[];
|
|
11
|
+
table: any;
|
|
12
|
+
}>;
|
|
13
|
+
getSchema: (fileName: string) => Promise<any>;
|
|
14
|
+
}
|
package/dist/{db/indexedDb.d.ts → browser/esm/adapters/storage-adapter/indexeddbAdapter.d.ts}
RENAMED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { DatasetSource } from '
|
|
2
|
-
import { DatasetSchema } from '
|
|
3
|
-
export declare class
|
|
1
|
+
import { DatasetSource, StorageAdapter } from '../../types';
|
|
2
|
+
import { DatasetSchema } from '../../types/DataSet';
|
|
3
|
+
export declare class IndexedDBAdapter implements StorageAdapter {
|
|
4
4
|
private db;
|
|
5
5
|
private dbName;
|
|
6
6
|
private datasetStoreName;
|
|
7
|
-
constructor(dbName
|
|
7
|
+
constructor(dbName?: string);
|
|
8
8
|
open: () => Promise<void>;
|
|
9
|
-
close: () => void
|
|
9
|
+
close: () => Promise<void>;
|
|
10
10
|
writeDataset: (datasetId: string, datasetSchema: DatasetSchema, datasetSource?: DatasetSource) => Promise<void>;
|
|
11
11
|
readDataset: (datasetId: string) => Promise<{
|
|
12
12
|
datasetSource?: DatasetSource;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { DatasetSource, StorageAdapter } from '../../types';
|
|
2
|
+
import { DatasetSchema } from '../../types/DataSet';
|
|
3
|
+
type StoredDataset = {
|
|
4
|
+
datasetId: string;
|
|
5
|
+
datasetSchema: DatasetSchema;
|
|
6
|
+
datasetSource?: DatasetSource;
|
|
7
|
+
};
|
|
8
|
+
export declare class InMemoryAdapter implements StorageAdapter {
|
|
9
|
+
private datasets;
|
|
10
|
+
private isOpen;
|
|
11
|
+
constructor();
|
|
12
|
+
open: () => Promise<void>;
|
|
13
|
+
close: () => Promise<void>;
|
|
14
|
+
writeDataset: (datasetId: string, datasetSchema: DatasetSchema, datasetSource?: DatasetSource) => Promise<void>;
|
|
15
|
+
readDataset: (datasetId: string) => Promise<{
|
|
16
|
+
datasetSource?: DatasetSource;
|
|
17
|
+
datasetSchema: DatasetSchema;
|
|
18
|
+
} | null>;
|
|
19
|
+
deleteDataset: (datasetId: string) => Promise<void>;
|
|
20
|
+
listDatasets: () => Promise<StoredDataset[]>;
|
|
21
|
+
}
|
|
22
|
+
export {};
|
|
@@ -142,16 +142,16 @@ const convertDSLToSQL = (dsl, tableName)=>{
|
|
|
142
142
|
return inlineParameters(compiled.sql, compiled.parameters);
|
|
143
143
|
};
|
|
144
144
|
class Dataset {
|
|
145
|
-
|
|
146
|
-
|
|
145
|
+
queryAdapter;
|
|
146
|
+
storageAdapter;
|
|
147
147
|
_datasetId;
|
|
148
148
|
constructor(duckDB, indexedDB1, datasetId){
|
|
149
|
-
this.
|
|
150
|
-
this.
|
|
149
|
+
this.queryAdapter = duckDB;
|
|
150
|
+
this.storageAdapter = indexedDB1;
|
|
151
151
|
this._datasetId = datasetId;
|
|
152
152
|
}
|
|
153
153
|
async init(temporaryColumns, temporaryDatasetSource) {
|
|
154
|
-
const datasetInfo = await this.
|
|
154
|
+
const datasetInfo = await this.storageAdapter.readDataset(this._datasetId);
|
|
155
155
|
if (!datasetInfo) throw new Error(`Dataset ${this._datasetId} not found`);
|
|
156
156
|
const columns = temporaryColumns ? temporaryColumns : datasetInfo.datasetSchema.columns;
|
|
157
157
|
const datasetSource = temporaryDatasetSource || datasetInfo.datasetSource;
|
|
@@ -174,11 +174,11 @@ class Dataset {
|
|
|
174
174
|
if (datasetSource) {
|
|
175
175
|
const readFunction = readFunctionMap[datasetSource.type];
|
|
176
176
|
if (!readFunction) throw new Error(`Unsupported dataSource type: ${datasetSource.type}`);
|
|
177
|
-
await this.
|
|
177
|
+
await this.queryAdapter.writeFile(this._datasetId, datasetSource.blob);
|
|
178
178
|
const columnsStruct = `{${columns.map((c)=>`'${c.name}': '${dataTypeMap[c.type] || 'VARCHAR'}'`).join(', ')}}`;
|
|
179
179
|
const columnNames = columns.map((c)=>`"${c.name}"`).join(', ');
|
|
180
180
|
const createViewSql = `CREATE OR REPLACE VIEW "${this._datasetId}" AS SELECT ${columnNames} FROM ${readFunction}('${this._datasetId}', columns=${columnsStruct})`;
|
|
181
|
-
await this.
|
|
181
|
+
await this.queryAdapter.query(createViewSql);
|
|
182
182
|
}
|
|
183
183
|
}
|
|
184
184
|
async query(queryDSL) {
|
|
@@ -187,7 +187,7 @@ class Dataset {
|
|
|
187
187
|
}
|
|
188
188
|
async queryBySQL(sql) {
|
|
189
189
|
const start = performance?.now?.()?.toFixed(3) ?? Date.now().toFixed(3);
|
|
190
|
-
const result = await this.
|
|
190
|
+
const result = await this.queryAdapter.query(sql);
|
|
191
191
|
const end = performance?.now?.()?.toFixed(3) ?? Date.now().toFixed(3);
|
|
192
192
|
return {
|
|
193
193
|
...result,
|
|
@@ -199,79 +199,111 @@ class Dataset {
|
|
|
199
199
|
};
|
|
200
200
|
}
|
|
201
201
|
async disconnect() {
|
|
202
|
-
await this.
|
|
202
|
+
await this.queryAdapter.query(`DROP VIEW IF EXISTS "${this._datasetId}"`);
|
|
203
203
|
}
|
|
204
204
|
get datasetId() {
|
|
205
205
|
return this._datasetId;
|
|
206
206
|
}
|
|
207
207
|
}
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
const bundle = await selectBundle(MANUAL_BUNDLES);
|
|
224
|
-
const worker_url = URL.createObjectURL(new Blob([
|
|
225
|
-
`importScripts("${bundle.mainWorker}");`
|
|
226
|
-
], {
|
|
227
|
-
type: "text/javascript"
|
|
228
|
-
}));
|
|
229
|
-
const worker = new Worker(worker_url);
|
|
230
|
-
const logger = new ConsoleLogger();
|
|
231
|
-
this.db = new AsyncDuckDB(logger, worker);
|
|
232
|
-
await this.db.instantiate(bundle.mainModule, bundle.pthreadWorker);
|
|
233
|
-
URL.revokeObjectURL(worker_url);
|
|
234
|
-
this.connection = await this.db.connect();
|
|
235
|
-
};
|
|
236
|
-
close = async ()=>{
|
|
237
|
-
if (this.connection) {
|
|
238
|
-
await this.connection.close();
|
|
239
|
-
this.connection = null;
|
|
240
|
-
}
|
|
241
|
-
if (this.db) {
|
|
242
|
-
await this.db.terminate();
|
|
243
|
-
this.db = null;
|
|
244
|
-
}
|
|
245
|
-
};
|
|
246
|
-
writeFile = async (fileName, source)=>{
|
|
247
|
-
if (!this.db) throw new Error('db is null');
|
|
248
|
-
let uint8Array;
|
|
249
|
-
if (source instanceof Blob) {
|
|
250
|
-
const buffer = await source.arrayBuffer();
|
|
251
|
-
uint8Array = new Uint8Array(buffer);
|
|
252
|
-
} else throw new Error('Unsupported source type');
|
|
253
|
-
await this.db.registerFileBuffer(fileName, uint8Array);
|
|
254
|
-
};
|
|
255
|
-
query = async (sql)=>{
|
|
256
|
-
if (!this.connection) throw new Error('connection is null');
|
|
257
|
-
const table = await this.connection.query(sql);
|
|
258
|
-
const dataset = table.toArray().map((row)=>row.toJSON());
|
|
208
|
+
const isUrl = (url)=>isHttpUrl(url) || isBase64Url(url);
|
|
209
|
+
const isHttpUrl = (url)=>url.startsWith('http://') || url.startsWith('https://');
|
|
210
|
+
const isBase64Url = (url)=>url.startsWith('data:');
|
|
211
|
+
class DatasetSourceBuilder {
|
|
212
|
+
type;
|
|
213
|
+
value;
|
|
214
|
+
constructor(raw){
|
|
215
|
+
this.type = raw.type;
|
|
216
|
+
this.value = raw.rawDataset;
|
|
217
|
+
}
|
|
218
|
+
static from(raw) {
|
|
219
|
+
return new DatasetSourceBuilder(raw);
|
|
220
|
+
}
|
|
221
|
+
async build() {
|
|
222
|
+
const blob = await DatasetSourceBuilder.convertToBlob(this.type, this.value);
|
|
259
223
|
return {
|
|
260
|
-
|
|
261
|
-
|
|
224
|
+
type: this.type,
|
|
225
|
+
blob: blob
|
|
262
226
|
};
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
if (
|
|
266
|
-
const
|
|
267
|
-
|
|
268
|
-
|
|
227
|
+
}
|
|
228
|
+
static async convertToBlob(type, value) {
|
|
229
|
+
if (value instanceof Blob) return value;
|
|
230
|
+
const convertCsvToBlob = (csvSource)=>{
|
|
231
|
+
if (csvSource instanceof ArrayBuffer) return new Blob([
|
|
232
|
+
csvSource
|
|
233
|
+
], {
|
|
234
|
+
type: 'text/csv'
|
|
235
|
+
});
|
|
236
|
+
if ('string' == typeof csvSource && isUrl(csvSource)) return DatasetSourceBuilder.fetchBlob(csvSource);
|
|
237
|
+
return new Blob([
|
|
238
|
+
JSON.stringify(csvSource)
|
|
239
|
+
], {
|
|
240
|
+
type: 'text/csv'
|
|
241
|
+
});
|
|
242
|
+
};
|
|
243
|
+
const convertJsonToBlob = (jsonSource)=>{
|
|
244
|
+
if (jsonSource instanceof ArrayBuffer) return new Blob([
|
|
245
|
+
jsonSource
|
|
246
|
+
], {
|
|
247
|
+
type: 'application/json'
|
|
248
|
+
});
|
|
249
|
+
if ('string' == typeof jsonSource && isUrl(jsonSource)) return DatasetSourceBuilder.fetchBlob(jsonSource);
|
|
250
|
+
return new Blob([
|
|
251
|
+
JSON.stringify(jsonSource)
|
|
252
|
+
], {
|
|
253
|
+
type: 'application/json'
|
|
254
|
+
});
|
|
255
|
+
};
|
|
256
|
+
const convertParquetToBlob = (parquetSource)=>{
|
|
257
|
+
if (parquetSource instanceof ArrayBuffer) return new Blob([
|
|
258
|
+
parquetSource
|
|
259
|
+
], {
|
|
260
|
+
type: 'application/parquet'
|
|
261
|
+
});
|
|
262
|
+
if ('string' == typeof parquetSource && isUrl(parquetSource)) return DatasetSourceBuilder.fetchBlob(parquetSource);
|
|
263
|
+
return new Blob([
|
|
264
|
+
parquetSource
|
|
265
|
+
], {
|
|
266
|
+
type: 'application/parquet'
|
|
267
|
+
});
|
|
268
|
+
};
|
|
269
|
+
const convertXlsxToBlob = (xlsxSource)=>{
|
|
270
|
+
if (xlsxSource instanceof ArrayBuffer) return new Blob([
|
|
271
|
+
xlsxSource
|
|
272
|
+
], {
|
|
273
|
+
type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
|
|
274
|
+
});
|
|
275
|
+
if ('string' == typeof xlsxSource && isUrl(xlsxSource)) return DatasetSourceBuilder.fetchBlob(xlsxSource);
|
|
276
|
+
return new Blob([
|
|
277
|
+
xlsxSource
|
|
278
|
+
], {
|
|
279
|
+
type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
|
|
280
|
+
});
|
|
281
|
+
};
|
|
282
|
+
switch(type){
|
|
283
|
+
case 'csv':
|
|
284
|
+
return convertCsvToBlob(value);
|
|
285
|
+
case 'json':
|
|
286
|
+
return convertJsonToBlob(value);
|
|
287
|
+
case 'xlsx':
|
|
288
|
+
return convertXlsxToBlob(value);
|
|
289
|
+
case 'parquet':
|
|
290
|
+
return convertParquetToBlob(value);
|
|
291
|
+
default:
|
|
292
|
+
return new Blob([
|
|
293
|
+
value
|
|
294
|
+
]);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
static async fetchBlob(url) {
|
|
298
|
+
const response = await fetch(url);
|
|
299
|
+
return await response.blob();
|
|
300
|
+
}
|
|
269
301
|
}
|
|
270
|
-
class
|
|
302
|
+
class IndexedDBAdapter {
|
|
271
303
|
db = null;
|
|
272
304
|
dbName;
|
|
273
305
|
datasetStoreName = 'vqueryDatasets';
|
|
274
|
-
constructor(dbName){
|
|
306
|
+
constructor(dbName = 'vquery'){
|
|
275
307
|
this.dbName = dbName;
|
|
276
308
|
}
|
|
277
309
|
open = ()=>new Promise((resolve, reject)=>{
|
|
@@ -290,7 +322,7 @@ class IndexedDB {
|
|
|
290
322
|
reject(event.target.error);
|
|
291
323
|
};
|
|
292
324
|
});
|
|
293
|
-
close = ()=>{
|
|
325
|
+
close = async ()=>{
|
|
294
326
|
if (this.db) {
|
|
295
327
|
this.db.close();
|
|
296
328
|
this.db = null;
|
|
@@ -359,112 +391,80 @@ class IndexedDB {
|
|
|
359
391
|
};
|
|
360
392
|
});
|
|
361
393
|
}
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
const blob = await DatasetSourceBuilder.convertToBlob(this.type, this.value);
|
|
377
|
-
return {
|
|
378
|
-
type: this.type,
|
|
379
|
-
blob: blob
|
|
380
|
-
};
|
|
381
|
-
}
|
|
382
|
-
static async convertToBlob(type, value) {
|
|
383
|
-
if (value instanceof Blob) return value;
|
|
384
|
-
const convertCsvToBlob = (csvSource)=>{
|
|
385
|
-
if (csvSource instanceof ArrayBuffer) return new Blob([
|
|
386
|
-
csvSource
|
|
387
|
-
], {
|
|
388
|
-
type: 'text/csv'
|
|
389
|
-
});
|
|
390
|
-
if ('string' == typeof csvSource && isUrl(csvSource)) return DatasetSourceBuilder.fetchBlob(csvSource);
|
|
391
|
-
return new Blob([
|
|
392
|
-
JSON.stringify(csvSource)
|
|
393
|
-
], {
|
|
394
|
-
type: 'text/csv'
|
|
395
|
-
});
|
|
396
|
-
};
|
|
397
|
-
const convertJsonToBlob = (jsonSource)=>{
|
|
398
|
-
if (jsonSource instanceof ArrayBuffer) return new Blob([
|
|
399
|
-
jsonSource
|
|
400
|
-
], {
|
|
401
|
-
type: 'application/json'
|
|
402
|
-
});
|
|
403
|
-
if ('string' == typeof jsonSource && isUrl(jsonSource)) return DatasetSourceBuilder.fetchBlob(jsonSource);
|
|
404
|
-
return new Blob([
|
|
405
|
-
JSON.stringify(jsonSource)
|
|
406
|
-
], {
|
|
407
|
-
type: 'application/json'
|
|
408
|
-
});
|
|
409
|
-
};
|
|
410
|
-
const convertParquetToBlob = (parquetSource)=>{
|
|
411
|
-
if (parquetSource instanceof ArrayBuffer) return new Blob([
|
|
412
|
-
parquetSource
|
|
413
|
-
], {
|
|
414
|
-
type: 'application/parquet'
|
|
415
|
-
});
|
|
416
|
-
if ('string' == typeof parquetSource && isUrl(parquetSource)) return DatasetSourceBuilder.fetchBlob(parquetSource);
|
|
417
|
-
return new Blob([
|
|
418
|
-
parquetSource
|
|
419
|
-
], {
|
|
420
|
-
type: 'application/parquet'
|
|
421
|
-
});
|
|
422
|
-
};
|
|
423
|
-
const convertXlsxToBlob = (xlsxSource)=>{
|
|
424
|
-
if (xlsxSource instanceof ArrayBuffer) return new Blob([
|
|
425
|
-
xlsxSource
|
|
426
|
-
], {
|
|
427
|
-
type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
|
|
428
|
-
});
|
|
429
|
-
if ('string' == typeof xlsxSource && isUrl(xlsxSource)) return DatasetSourceBuilder.fetchBlob(xlsxSource);
|
|
430
|
-
return new Blob([
|
|
431
|
-
xlsxSource
|
|
432
|
-
], {
|
|
433
|
-
type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
|
|
434
|
-
});
|
|
394
|
+
class DuckDBWebQueryAdapter {
|
|
395
|
+
db = null;
|
|
396
|
+
connection = null;
|
|
397
|
+
constructor(){}
|
|
398
|
+
open = async ()=>{
|
|
399
|
+
const MANUAL_BUNDLES = {
|
|
400
|
+
mvp: {
|
|
401
|
+
mainModule: new URL('@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm', import.meta.url).href,
|
|
402
|
+
mainWorker: new URL('@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js', import.meta.url).toString()
|
|
403
|
+
},
|
|
404
|
+
eh: {
|
|
405
|
+
mainModule: new URL('@duckdb/duckdb-wasm/dist/duckdb-eh.wasm', import.meta.url).href,
|
|
406
|
+
mainWorker: new URL('@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js', import.meta.url).toString()
|
|
407
|
+
}
|
|
435
408
|
};
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
409
|
+
const bundle = await selectBundle(MANUAL_BUNDLES);
|
|
410
|
+
const worker_url = URL.createObjectURL(new Blob([
|
|
411
|
+
`importScripts("${bundle.mainWorker}");`
|
|
412
|
+
], {
|
|
413
|
+
type: "text/javascript"
|
|
414
|
+
}));
|
|
415
|
+
const worker = new Worker(worker_url);
|
|
416
|
+
const logger = new ConsoleLogger();
|
|
417
|
+
this.db = new AsyncDuckDB(logger, worker);
|
|
418
|
+
await this.db.instantiate(bundle.mainModule, bundle.pthreadWorker);
|
|
419
|
+
URL.revokeObjectURL(worker_url);
|
|
420
|
+
this.connection = await this.db.connect();
|
|
421
|
+
};
|
|
422
|
+
close = async ()=>{
|
|
423
|
+
if (this.connection) {
|
|
424
|
+
await this.connection.close();
|
|
425
|
+
this.connection = null;
|
|
449
426
|
}
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
}
|
|
427
|
+
if (this.db) {
|
|
428
|
+
await this.db.terminate();
|
|
429
|
+
this.db = null;
|
|
430
|
+
}
|
|
431
|
+
};
|
|
432
|
+
writeFile = async (fileName, source)=>{
|
|
433
|
+
if (!this.db) throw new Error('db is null');
|
|
434
|
+
let uint8Array;
|
|
435
|
+
if (source instanceof Blob) {
|
|
436
|
+
const buffer = await source.arrayBuffer();
|
|
437
|
+
uint8Array = new Uint8Array(buffer);
|
|
438
|
+
} else throw new Error('Unsupported source type');
|
|
439
|
+
await this.db.registerFileBuffer(fileName, uint8Array);
|
|
440
|
+
};
|
|
441
|
+
query = async (sql)=>{
|
|
442
|
+
if (!this.connection) throw new Error('connection is null');
|
|
443
|
+
const table = await this.connection.query(sql);
|
|
444
|
+
const dataset = table.toArray().map((row)=>row.toJSON());
|
|
445
|
+
return {
|
|
446
|
+
dataset,
|
|
447
|
+
table
|
|
448
|
+
};
|
|
449
|
+
};
|
|
450
|
+
getSchema = async (fileName)=>{
|
|
451
|
+
if (!this.connection) throw new Error('connection is null');
|
|
452
|
+
const result = await this.connection.query(`PRAGMA table_info('${fileName}')`);
|
|
453
|
+
return result.toArray().map((row)=>row.toJSON());
|
|
454
|
+
};
|
|
455
455
|
}
|
|
456
456
|
class VQuery {
|
|
457
|
-
|
|
458
|
-
|
|
457
|
+
queryAdapter;
|
|
458
|
+
storageAdapter;
|
|
459
459
|
isInitialized = false;
|
|
460
|
-
constructor(
|
|
461
|
-
this.
|
|
462
|
-
this.
|
|
460
|
+
constructor(){
|
|
461
|
+
this.queryAdapter = new DuckDBWebQueryAdapter();
|
|
462
|
+
this.storageAdapter = new IndexedDBAdapter();
|
|
463
463
|
}
|
|
464
464
|
async checkInitialized() {
|
|
465
465
|
if (!this.isInitialized) {
|
|
466
|
-
await this.
|
|
467
|
-
await this.
|
|
466
|
+
await this.queryAdapter.open();
|
|
467
|
+
await this.storageAdapter.open();
|
|
468
468
|
this.isInitialized = true;
|
|
469
469
|
}
|
|
470
470
|
}
|
|
@@ -480,7 +480,7 @@ class VQuery {
|
|
|
480
480
|
datasetAlias: datasetId,
|
|
481
481
|
columns: columns
|
|
482
482
|
};
|
|
483
|
-
await this.
|
|
483
|
+
await this.storageAdapter.writeDataset(datasetId, datasetSchema, datasetSource);
|
|
484
484
|
}
|
|
485
485
|
async updateDatasetSource(datasetId, columns = [], rawDatasetSource) {
|
|
486
486
|
await this.checkInitialized();
|
|
@@ -491,33 +491,34 @@ class VQuery {
|
|
|
491
491
|
datasetAlias: datasetId,
|
|
492
492
|
columns: columns
|
|
493
493
|
};
|
|
494
|
-
await this.
|
|
494
|
+
await this.storageAdapter.writeDataset(datasetId, datasetSchema, datasetSource);
|
|
495
495
|
}
|
|
496
496
|
async dropDataset(datasetId) {
|
|
497
497
|
await this.checkInitialized();
|
|
498
498
|
await this.checkDatasetExists(datasetId);
|
|
499
|
-
await this.
|
|
499
|
+
await this.storageAdapter.deleteDataset(datasetId);
|
|
500
500
|
}
|
|
501
501
|
async connectDataset(datasetId, temporaryColumns, temporaryRawDatasetSource) {
|
|
502
502
|
await this.checkInitialized();
|
|
503
503
|
await this.checkDatasetExists(datasetId);
|
|
504
|
-
const dataset = new Dataset(this.
|
|
504
|
+
const dataset = new Dataset(this.queryAdapter, this.storageAdapter, datasetId);
|
|
505
505
|
const temporaryDatasetSource = temporaryRawDatasetSource ? await DatasetSourceBuilder.from(temporaryRawDatasetSource).build() : void 0;
|
|
506
506
|
await dataset.init(temporaryColumns, temporaryDatasetSource);
|
|
507
507
|
return dataset;
|
|
508
508
|
}
|
|
509
509
|
async hasDataset(datasetId) {
|
|
510
510
|
await this.checkInitialized();
|
|
511
|
-
const datasets = await this.
|
|
511
|
+
const datasets = await this.storageAdapter.listDatasets();
|
|
512
512
|
return datasets.some((item)=>item.datasetId === datasetId);
|
|
513
513
|
}
|
|
514
514
|
async listDatasets() {
|
|
515
515
|
await this.checkInitialized();
|
|
516
|
-
return this.
|
|
516
|
+
return this.storageAdapter.listDatasets();
|
|
517
517
|
}
|
|
518
518
|
async close() {
|
|
519
519
|
await this.checkInitialized();
|
|
520
|
-
await this.
|
|
520
|
+
await this.queryAdapter.close();
|
|
521
|
+
await this.storageAdapter.close();
|
|
521
522
|
}
|
|
522
523
|
}
|
|
523
524
|
export { DatasetSourceBuilder, VQuery, convertDSLToSQL, isBase64Url, isHttpUrl, isUrl };
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import { DatasetColumn, DatasetSource, QueryDSL } from '../types';
|
|
2
|
-
import {
|
|
3
|
-
import { IndexedDB } from '../db/indexedDb';
|
|
2
|
+
import { QueryAdapter, StorageAdapter } from '../types';
|
|
4
3
|
export declare class Dataset {
|
|
5
|
-
private
|
|
6
|
-
private
|
|
4
|
+
private queryAdapter;
|
|
5
|
+
private storageAdapter;
|
|
7
6
|
private _datasetId;
|
|
8
|
-
constructor(duckDB:
|
|
7
|
+
constructor(duckDB: QueryAdapter, indexedDB: StorageAdapter, datasetId: string);
|
|
9
8
|
init(temporaryColumns?: DatasetColumn[], temporaryDatasetSource?: DatasetSource): Promise<void>;
|
|
10
9
|
createOrReplaceView(columns: DatasetColumn[], datasetSource: DatasetSource): Promise<void>;
|
|
11
10
|
query<T extends Record<string, number | string>>(queryDSL: QueryDSL<T>): Promise<{
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Where, WhereClause, WhereGroup, WhereLeaf } from '../types';
|
|
2
|
-
import { SelectItem } from '../types/
|
|
2
|
+
import { SelectItem } from '../types/dsl/Select';
|
|
3
3
|
export declare const isSelectItem: <T>(item: keyof T | SelectItem<T>) => item is SelectItem<T>;
|
|
4
4
|
export declare const isWhereLeaf: <T>(where: Where<T> | WhereClause<T>) => where is WhereLeaf<T>;
|
|
5
5
|
export declare const isWhereGroup: <T>(where: Where<T> | WhereClause<T>) => where is WhereGroup<T>;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { QueryResult } from '../QueryResult';
|
|
2
|
+
export interface QueryAdapter {
|
|
3
|
+
open: () => Promise<void>;
|
|
4
|
+
close: () => Promise<void>;
|
|
5
|
+
writeFile: <T extends Blob>(fileName: string, source: T) => Promise<void>;
|
|
6
|
+
getSchema: (fileName: string) => Promise<QueryResult>;
|
|
7
|
+
query: (sql: string) => Promise<{
|
|
8
|
+
dataset: any[];
|
|
9
|
+
table: any;
|
|
10
|
+
}>;
|
|
11
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { DatasetSchema } from '../DataSet';
|
|
2
|
+
import { DatasetSource } from '../DataSource';
|
|
3
|
+
export interface StorageAdapter {
|
|
4
|
+
open: () => Promise<void>;
|
|
5
|
+
close: () => Promise<void>;
|
|
6
|
+
writeDataset: (datasetId: string, datasetSchema: DatasetSchema, datasetSource?: DatasetSource) => Promise<void>;
|
|
7
|
+
readDataset: (datasetId: string) => Promise<{
|
|
8
|
+
datasetSchema: DatasetSchema;
|
|
9
|
+
datasetSource?: DatasetSource;
|
|
10
|
+
} | null>;
|
|
11
|
+
deleteDataset: (datasetId: string) => Promise<void>;
|
|
12
|
+
listDatasets: () => Promise<{
|
|
13
|
+
datasetId: string;
|
|
14
|
+
datasetSchema: DatasetSchema;
|
|
15
|
+
datasetSource?: DatasetSource;
|
|
16
|
+
}[]>;
|
|
17
|
+
}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { Dataset } from './dataset/dataset';
|
|
2
2
|
import { RawDatasetSource, DatasetColumn } from './types';
|
|
3
3
|
export declare class VQuery {
|
|
4
|
-
private
|
|
5
|
-
private
|
|
4
|
+
private queryAdapter;
|
|
5
|
+
private storageAdapter;
|
|
6
6
|
private isInitialized;
|
|
7
|
-
constructor(
|
|
7
|
+
constructor();
|
|
8
8
|
private checkInitialized;
|
|
9
9
|
private checkDatasetExists;
|
|
10
10
|
createDataset(datasetId: string, columns?: DatasetColumn[], rawDatasetSource?: RawDatasetSource): Promise<void>;
|
|
@@ -14,8 +14,8 @@ export declare class VQuery {
|
|
|
14
14
|
hasDataset(datasetId: string): Promise<boolean>;
|
|
15
15
|
listDatasets(): Promise<{
|
|
16
16
|
datasetId: string;
|
|
17
|
-
dataSource?: import("./types").DatasetSource;
|
|
18
17
|
datasetSchema: import("./types").DatasetSchema;
|
|
18
|
+
datasetSource?: import("./types").DatasetSource;
|
|
19
19
|
}[]>;
|
|
20
20
|
close(): Promise<void>;
|
|
21
21
|
}
|