@loaders.gl/sql 5.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +41 -0
- package/README.md +27 -0
- package/dist/duckdb-sql-source.d.ts +84 -0
- package/dist/duckdb-sql-source.d.ts.map +1 -0
- package/dist/duckdb-sql-source.js +298 -0
- package/dist/duckdb-sql-source.js.map +1 -0
- package/dist/index.cjs +759 -0
- package/dist/index.cjs.map +7 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -0
- package/dist/snowflake-sql-source.d.ts +84 -0
- package/dist/snowflake-sql-source.d.ts.map +1 -0
- package/dist/snowflake-sql-source.js +209 -0
- package/dist/snowflake-sql-source.js.map +1 -0
- package/dist/sql-source.d.ts +45 -0
- package/dist/sql-source.d.ts.map +1 -0
- package/dist/sql-source.js +146 -0
- package/dist/sql-source.js.map +1 -0
- package/dist/sql-types.d.ts +107 -0
- package/dist/sql-types.d.ts.map +1 -0
- package/dist/sql-types.js +5 -0
- package/dist/sql-types.js.map +1 -0
- package/dist/sql-utils.d.ts +19 -0
- package/dist/sql-utils.d.ts.map +1 -0
- package/dist/sql-utils.js +108 -0
- package/dist/sql-utils.js.map +1 -0
- package/package.json +57 -0
- package/src/duckdb-sql-source.ts +368 -0
- package/src/index.ts +22 -0
- package/src/snowflake-sql-source.ts +296 -0
- package/src/sql-source.ts +193 -0
- package/src/sql-types.ts +119 -0
- package/src/sql-utils.ts +137 -0
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {Schema} from '@loaders.gl/schema';
|
|
6
|
+
import type {SourceLoader, CoreAPI} from '@loaders.gl/loader-utils';
|
|
7
|
+
import {convertArrowToTable} from '@loaders.gl/schema-utils';
|
|
8
|
+
|
|
9
|
+
import {SQLDataSource, SQL_SOURCE_DEFAULT_OPTIONS} from './sql-source';
|
|
10
|
+
import type {
|
|
11
|
+
SQLAdapter,
|
|
12
|
+
SQLAdapterFactoryContext,
|
|
13
|
+
SQLCatalogInfo,
|
|
14
|
+
SQLColumnInfo,
|
|
15
|
+
SQLQueryOptions,
|
|
16
|
+
SQLSourceOptions,
|
|
17
|
+
SQLTableInfo
|
|
18
|
+
} from './sql-types';
|
|
19
|
+
import {
|
|
20
|
+
convertRowsToArrowTable,
|
|
21
|
+
convertSQLColumnsToSchema,
|
|
22
|
+
escapeSqlString,
|
|
23
|
+
isNodeRuntime
|
|
24
|
+
} from './sql-utils';
|
|
25
|
+
|
|
26
|
+
// __VERSION__ is injected by babel-plugin-version-inline
|
|
27
|
+
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
28
|
+
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
29
|
+
|
|
30
|
+
/** DuckDB-backed SQL source. */
|
|
31
|
+
export const DuckDBSQLSource = {
|
|
32
|
+
dataType: null as unknown as DuckDBSQLDataSource,
|
|
33
|
+
batchType: null as never,
|
|
34
|
+
name: 'DuckDBSQLSource',
|
|
35
|
+
id: 'duckdb-sql',
|
|
36
|
+
module: 'sql',
|
|
37
|
+
version: VERSION,
|
|
38
|
+
extensions: ['duckdb', 'db'],
|
|
39
|
+
mimeTypes: [],
|
|
40
|
+
type: 'duckdb-sql',
|
|
41
|
+
fromUrl: true,
|
|
42
|
+
fromBlob: false,
|
|
43
|
+
options: {
|
|
44
|
+
sql: {
|
|
45
|
+
resultFormat: 'auto',
|
|
46
|
+
adapterOptions: {},
|
|
47
|
+
browserAdapterFactory: undefined,
|
|
48
|
+
nodeAdapterFactory: undefined
|
|
49
|
+
},
|
|
50
|
+
duckdb: {
|
|
51
|
+
accessMode: 'auto',
|
|
52
|
+
bundles: undefined,
|
|
53
|
+
databasePath: undefined,
|
|
54
|
+
remoteUrl: undefined,
|
|
55
|
+
workerUrl: undefined
|
|
56
|
+
},
|
|
57
|
+
snowflake: {
|
|
58
|
+
account: undefined,
|
|
59
|
+
database: undefined,
|
|
60
|
+
schema: undefined,
|
|
61
|
+
token: undefined,
|
|
62
|
+
warehouse: undefined,
|
|
63
|
+
role: undefined,
|
|
64
|
+
authenticator: undefined
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
defaultOptions: SQL_SOURCE_DEFAULT_OPTIONS,
|
|
68
|
+
testURL: (url: string): boolean => /^duckdb:\/\//i.test(url),
|
|
69
|
+
createDataSource: (
|
|
70
|
+
data: string,
|
|
71
|
+
options: SQLSourceOptions,
|
|
72
|
+
coreApi?: CoreAPI
|
|
73
|
+
): DuckDBSQLDataSource => new DuckDBSQLDataSource(data, options, coreApi)
|
|
74
|
+
} as const satisfies SourceLoader<DuckDBSQLDataSource>;
|
|
75
|
+
|
|
76
|
+
/** SQLDataSource specialization for DuckDB. */
|
|
77
|
+
export class DuckDBSQLDataSource extends SQLDataSource {
|
|
78
|
+
constructor(data: string, options: SQLSourceOptions, coreApi?: CoreAPI) {
|
|
79
|
+
super(
|
|
80
|
+
data,
|
|
81
|
+
options,
|
|
82
|
+
DuckDBSQLSource.type,
|
|
83
|
+
SQL_SOURCE_DEFAULT_OPTIONS,
|
|
84
|
+
createDuckDBAdapter,
|
|
85
|
+
coreApi
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
async function createDuckDBAdapter(context: SQLAdapterFactoryContext): Promise<SQLAdapter> {
|
|
91
|
+
if (context.options.duckdb?.remoteUrl) {
|
|
92
|
+
throw new Error('Remote DuckDB adapters are not implemented yet.');
|
|
93
|
+
}
|
|
94
|
+
if (isNodeRuntime()) {
|
|
95
|
+
return await createDuckDBNodeAdapter(context);
|
|
96
|
+
}
|
|
97
|
+
return await createDuckDBBrowserAdapter(context);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
async function createDuckDBNodeAdapter(context: SQLAdapterFactoryContext): Promise<SQLAdapter> {
|
|
101
|
+
const duckdb = await import('@duckdb/node-api');
|
|
102
|
+
|
|
103
|
+
const databasePath = getDuckDBDatabasePath(context.url, context.options);
|
|
104
|
+
const configuration =
|
|
105
|
+
context.options.duckdb?.accessMode && context.options.duckdb.accessMode !== 'auto'
|
|
106
|
+
? {access_mode: context.options.duckdb.accessMode}
|
|
107
|
+
: undefined;
|
|
108
|
+
|
|
109
|
+
let connection: any;
|
|
110
|
+
|
|
111
|
+
const executeRows = async (
|
|
112
|
+
sqlText: string,
|
|
113
|
+
options: SQLQueryOptions = {}
|
|
114
|
+
): Promise<Record<string, unknown>[]> => {
|
|
115
|
+
if (!connection) {
|
|
116
|
+
const instance = await duckdb.DuckDBInstance.create(databasePath, configuration);
|
|
117
|
+
connection = await instance.connect();
|
|
118
|
+
}
|
|
119
|
+
const reader = await connection.runAndReadAll(sqlText, options.parameters);
|
|
120
|
+
return reader.getRowObjectsJson();
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
return {
|
|
124
|
+
capabilities: {
|
|
125
|
+
supportsArrow: true,
|
|
126
|
+
supportsMetadata: true,
|
|
127
|
+
runtime: 'node',
|
|
128
|
+
isDynamic: true
|
|
129
|
+
},
|
|
130
|
+
async connect(): Promise<void> {
|
|
131
|
+
if (!connection) {
|
|
132
|
+
const instance = await duckdb.DuckDBInstance.create(databasePath, configuration);
|
|
133
|
+
connection = await instance.connect();
|
|
134
|
+
}
|
|
135
|
+
},
|
|
136
|
+
async close(): Promise<void> {
|
|
137
|
+
if (connection?.closeSync) {
|
|
138
|
+
connection.closeSync();
|
|
139
|
+
}
|
|
140
|
+
connection = null;
|
|
141
|
+
},
|
|
142
|
+
async listCatalogs(): Promise<SQLCatalogInfo[]> {
|
|
143
|
+
const rows = await executeRows(`
|
|
144
|
+
SELECT DISTINCT catalog_name
|
|
145
|
+
FROM information_schema.schemata
|
|
146
|
+
ORDER BY catalog_name
|
|
147
|
+
`);
|
|
148
|
+
return rows.map(row => ({catalogName: String(row.catalog_name)}));
|
|
149
|
+
},
|
|
150
|
+
async listSchemas(catalogName?: string) {
|
|
151
|
+
const filter = catalogName ? `WHERE catalog_name = '${escapeSqlString(catalogName)}'` : '';
|
|
152
|
+
const rows = await executeRows(`
|
|
153
|
+
SELECT catalog_name, schema_name
|
|
154
|
+
FROM information_schema.schemata
|
|
155
|
+
${filter}
|
|
156
|
+
ORDER BY catalog_name, schema_name
|
|
157
|
+
`);
|
|
158
|
+
return rows.map(row => ({
|
|
159
|
+
catalogName: row.catalog_name ? String(row.catalog_name) : undefined,
|
|
160
|
+
schemaName: String(row.schema_name)
|
|
161
|
+
}));
|
|
162
|
+
},
|
|
163
|
+
async listTables(options?: {
|
|
164
|
+
catalogName?: string;
|
|
165
|
+
schemaName?: string;
|
|
166
|
+
}): Promise<SQLTableInfo[]> {
|
|
167
|
+
const filters = [
|
|
168
|
+
options?.catalogName
|
|
169
|
+
? `table_catalog = '${escapeSqlString(options.catalogName)}'`
|
|
170
|
+
: undefined,
|
|
171
|
+
options?.schemaName ? `table_schema = '${escapeSqlString(options.schemaName)}'` : undefined,
|
|
172
|
+
`table_schema NOT IN ('information_schema', 'pg_catalog')`
|
|
173
|
+
].filter(Boolean);
|
|
174
|
+
const whereClause = filters.length ? `WHERE ${filters.join(' AND ')}` : '';
|
|
175
|
+
const rows = await executeRows(`
|
|
176
|
+
SELECT table_catalog, table_schema, table_name, table_type
|
|
177
|
+
FROM information_schema.tables
|
|
178
|
+
${whereClause}
|
|
179
|
+
ORDER BY table_catalog, table_schema, table_name
|
|
180
|
+
`);
|
|
181
|
+
return rows.map(row => ({
|
|
182
|
+
catalogName: row.table_catalog ? String(row.table_catalog) : undefined,
|
|
183
|
+
schemaName: row.table_schema ? String(row.table_schema) : undefined,
|
|
184
|
+
tableName: String(row.table_name),
|
|
185
|
+
tableType: row.table_type ? String(row.table_type) : undefined
|
|
186
|
+
}));
|
|
187
|
+
},
|
|
188
|
+
async getTableSchema(options: {
|
|
189
|
+
catalogName?: string;
|
|
190
|
+
schemaName?: string;
|
|
191
|
+
tableName: string;
|
|
192
|
+
}): Promise<Schema> {
|
|
193
|
+
const filters = [
|
|
194
|
+
options.catalogName
|
|
195
|
+
? `table_catalog = '${escapeSqlString(options.catalogName)}'`
|
|
196
|
+
: undefined,
|
|
197
|
+
options.schemaName ? `table_schema = '${escapeSqlString(options.schemaName)}'` : undefined,
|
|
198
|
+
`table_name = '${escapeSqlString(options.tableName)}'`
|
|
199
|
+
].filter(Boolean);
|
|
200
|
+
const rows = await executeRows(`
|
|
201
|
+
SELECT table_catalog, table_schema, table_name, column_name, data_type, is_nullable, ordinal_position
|
|
202
|
+
FROM information_schema.columns
|
|
203
|
+
WHERE ${filters.join(' AND ')}
|
|
204
|
+
ORDER BY ordinal_position
|
|
205
|
+
`);
|
|
206
|
+
const columns = rows.map(
|
|
207
|
+
row =>
|
|
208
|
+
({
|
|
209
|
+
catalogName: row.table_catalog ? String(row.table_catalog) : undefined,
|
|
210
|
+
schemaName: row.table_schema ? String(row.table_schema) : undefined,
|
|
211
|
+
tableName: row.table_name ? String(row.table_name) : undefined,
|
|
212
|
+
columnName: String(row.column_name),
|
|
213
|
+
sqlType: String(row.data_type),
|
|
214
|
+
nullable: row.is_nullable === 'YES',
|
|
215
|
+
ordinalPosition: Number(row.ordinal_position)
|
|
216
|
+
}) satisfies SQLColumnInfo
|
|
217
|
+
);
|
|
218
|
+
return convertSQLColumnsToSchema(columns);
|
|
219
|
+
},
|
|
220
|
+
async executeRows(
|
|
221
|
+
sqlText: string,
|
|
222
|
+
options: SQLQueryOptions = {}
|
|
223
|
+
): Promise<Record<string, unknown>[]> {
|
|
224
|
+
return await executeRows(sqlText, options);
|
|
225
|
+
},
|
|
226
|
+
async executeArrow(sqlText: string, options: SQLQueryOptions = {}) {
|
|
227
|
+
const rows = await executeRows(sqlText, options);
|
|
228
|
+
return convertRowsToArrowTable(rows);
|
|
229
|
+
}
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
async function createDuckDBBrowserAdapter(context: SQLAdapterFactoryContext): Promise<SQLAdapter> {
|
|
234
|
+
const duckdb = await import('@duckdb/duckdb-wasm');
|
|
235
|
+
|
|
236
|
+
const logger = new duckdb.ConsoleLogger();
|
|
237
|
+
const selectedBundles = await duckdb.selectBundle(duckdb.getJsDelivrBundles());
|
|
238
|
+
const bundles = {
|
|
239
|
+
...selectedBundles,
|
|
240
|
+
...context.options.duckdb?.bundles
|
|
241
|
+
};
|
|
242
|
+
const workerUrl = context.options.duckdb?.workerUrl || selectedBundles.mainWorker;
|
|
243
|
+
if (!workerUrl) {
|
|
244
|
+
throw new Error('DuckDBSQLSource could not resolve a worker bundle URL.');
|
|
245
|
+
}
|
|
246
|
+
const worker = await duckdb.createWorker(workerUrl);
|
|
247
|
+
const database = new duckdb.AsyncDuckDB(logger, worker);
|
|
248
|
+
await database.instantiate(bundles.mainModule, bundles.pthreadWorker);
|
|
249
|
+
const connection = await database.connect();
|
|
250
|
+
|
|
251
|
+
const executeRows = async (sqlText: string): Promise<Record<string, unknown>[]> => {
|
|
252
|
+
const result = await connection.query(sqlText);
|
|
253
|
+
return convertArrowToTable(result, 'object-row-table').data;
|
|
254
|
+
};
|
|
255
|
+
|
|
256
|
+
return {
|
|
257
|
+
capabilities: {
|
|
258
|
+
supportsArrow: true,
|
|
259
|
+
supportsMetadata: true,
|
|
260
|
+
runtime: 'browser',
|
|
261
|
+
isDynamic: true
|
|
262
|
+
},
|
|
263
|
+
async connect(): Promise<void> {},
|
|
264
|
+
async close(): Promise<void> {
|
|
265
|
+
await connection.close();
|
|
266
|
+
await database.terminate();
|
|
267
|
+
worker.terminate();
|
|
268
|
+
},
|
|
269
|
+
async listCatalogs(): Promise<SQLCatalogInfo[]> {
|
|
270
|
+
const rows = await executeRows(`
|
|
271
|
+
SELECT DISTINCT catalog_name
|
|
272
|
+
FROM information_schema.schemata
|
|
273
|
+
ORDER BY catalog_name
|
|
274
|
+
`);
|
|
275
|
+
return rows.map(row => ({catalogName: String(row.catalog_name)}));
|
|
276
|
+
},
|
|
277
|
+
async listSchemas(catalogName?: string) {
|
|
278
|
+
const filter = catalogName ? `WHERE catalog_name = '${escapeSqlString(catalogName)}'` : '';
|
|
279
|
+
const rows = await executeRows(`
|
|
280
|
+
SELECT catalog_name, schema_name
|
|
281
|
+
FROM information_schema.schemata
|
|
282
|
+
${filter}
|
|
283
|
+
ORDER BY catalog_name, schema_name
|
|
284
|
+
`);
|
|
285
|
+
return rows.map(row => ({
|
|
286
|
+
catalogName: row.catalog_name ? String(row.catalog_name) : undefined,
|
|
287
|
+
schemaName: String(row.schema_name)
|
|
288
|
+
}));
|
|
289
|
+
},
|
|
290
|
+
async listTables(options?: {
|
|
291
|
+
catalogName?: string;
|
|
292
|
+
schemaName?: string;
|
|
293
|
+
}): Promise<SQLTableInfo[]> {
|
|
294
|
+
const filters = [
|
|
295
|
+
options?.catalogName
|
|
296
|
+
? `table_catalog = '${escapeSqlString(options.catalogName)}'`
|
|
297
|
+
: undefined,
|
|
298
|
+
options?.schemaName ? `table_schema = '${escapeSqlString(options.schemaName)}'` : undefined,
|
|
299
|
+
`table_schema NOT IN ('information_schema', 'pg_catalog')`
|
|
300
|
+
].filter(Boolean);
|
|
301
|
+
const whereClause = filters.length ? `WHERE ${filters.join(' AND ')}` : '';
|
|
302
|
+
const rows = await executeRows(`
|
|
303
|
+
SELECT table_catalog, table_schema, table_name, table_type
|
|
304
|
+
FROM information_schema.tables
|
|
305
|
+
${whereClause}
|
|
306
|
+
ORDER BY table_catalog, table_schema, table_name
|
|
307
|
+
`);
|
|
308
|
+
return rows.map(row => ({
|
|
309
|
+
catalogName: row.table_catalog ? String(row.table_catalog) : undefined,
|
|
310
|
+
schemaName: row.table_schema ? String(row.table_schema) : undefined,
|
|
311
|
+
tableName: String(row.table_name),
|
|
312
|
+
tableType: row.table_type ? String(row.table_type) : undefined
|
|
313
|
+
}));
|
|
314
|
+
},
|
|
315
|
+
async getTableSchema(options: {
|
|
316
|
+
catalogName?: string;
|
|
317
|
+
schemaName?: string;
|
|
318
|
+
tableName: string;
|
|
319
|
+
}): Promise<Schema> {
|
|
320
|
+
const filters = [
|
|
321
|
+
options.catalogName
|
|
322
|
+
? `table_catalog = '${escapeSqlString(options.catalogName)}'`
|
|
323
|
+
: undefined,
|
|
324
|
+
options.schemaName ? `table_schema = '${escapeSqlString(options.schemaName)}'` : undefined,
|
|
325
|
+
`table_name = '${escapeSqlString(options.tableName)}'`
|
|
326
|
+
].filter(Boolean);
|
|
327
|
+
const rows = await executeRows(`
|
|
328
|
+
SELECT table_catalog, table_schema, table_name, column_name, data_type, is_nullable, ordinal_position
|
|
329
|
+
FROM information_schema.columns
|
|
330
|
+
WHERE ${filters.join(' AND ')}
|
|
331
|
+
ORDER BY ordinal_position
|
|
332
|
+
`);
|
|
333
|
+
const columns = rows.map(
|
|
334
|
+
row =>
|
|
335
|
+
({
|
|
336
|
+
catalogName: row.table_catalog ? String(row.table_catalog) : undefined,
|
|
337
|
+
schemaName: row.table_schema ? String(row.table_schema) : undefined,
|
|
338
|
+
tableName: row.table_name ? String(row.table_name) : undefined,
|
|
339
|
+
columnName: String(row.column_name),
|
|
340
|
+
sqlType: String(row.data_type),
|
|
341
|
+
nullable: row.is_nullable === 'YES',
|
|
342
|
+
ordinalPosition: Number(row.ordinal_position)
|
|
343
|
+
}) satisfies SQLColumnInfo
|
|
344
|
+
);
|
|
345
|
+
return convertSQLColumnsToSchema(columns);
|
|
346
|
+
},
|
|
347
|
+
async executeRows(sqlText: string): Promise<Record<string, unknown>[]> {
|
|
348
|
+
return await executeRows(sqlText);
|
|
349
|
+
},
|
|
350
|
+
async executeArrow(sqlText: string) {
|
|
351
|
+
const result = await connection.query(sqlText);
|
|
352
|
+
return convertArrowToTable(result, 'arrow-table');
|
|
353
|
+
}
|
|
354
|
+
};
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
function getDuckDBDatabasePath(url: string, options: SQLSourceOptions): string {
|
|
358
|
+
if (options.duckdb?.databasePath) {
|
|
359
|
+
return options.duckdb.databasePath;
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
const parsedUrl = new URL(url);
|
|
363
|
+
const pathname = decodeURIComponent(parsedUrl.pathname || '');
|
|
364
|
+
if (!pathname || pathname === '/') {
|
|
365
|
+
return ':memory:';
|
|
366
|
+
}
|
|
367
|
+
return pathname.slice(1) || ':memory:';
|
|
368
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
export {SQLDataSource, registerSQLAdapter, getSQLAdapterFactory} from './sql-source';
|
|
6
|
+
export {DuckDBSQLSource, DuckDBSQLDataSource} from './duckdb-sql-source';
|
|
7
|
+
export {SnowflakeSQLSource, SnowflakeSQLDataSource} from './snowflake-sql-source';
|
|
8
|
+
|
|
9
|
+
export type {
|
|
10
|
+
SQLAdapter,
|
|
11
|
+
SQLAdapterCapabilities,
|
|
12
|
+
SQLAdapterFactory,
|
|
13
|
+
SQLAdapterFactoryContext,
|
|
14
|
+
SQLCatalogInfo,
|
|
15
|
+
SQLColumnInfo,
|
|
16
|
+
SQLMetadata,
|
|
17
|
+
SQLParameterValues,
|
|
18
|
+
SQLQueryOptions,
|
|
19
|
+
SQLSchemaInfo,
|
|
20
|
+
SQLSourceOptions,
|
|
21
|
+
SQLTableInfo
|
|
22
|
+
} from './sql-types';
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {Schema} from '@loaders.gl/schema';
|
|
6
|
+
import type {SourceLoader, CoreAPI} from '@loaders.gl/loader-utils';
|
|
7
|
+
|
|
8
|
+
import {SQLDataSource, SQL_SOURCE_DEFAULT_OPTIONS} from './sql-source';
|
|
9
|
+
import type {
|
|
10
|
+
SQLAdapter,
|
|
11
|
+
SQLAdapterFactoryContext,
|
|
12
|
+
SQLQueryOptions,
|
|
13
|
+
SQLSourceOptions,
|
|
14
|
+
SQLTableInfo
|
|
15
|
+
} from './sql-types';
|
|
16
|
+
import {
|
|
17
|
+
convertRowsToArrowTable,
|
|
18
|
+
convertSQLColumnsToSchema,
|
|
19
|
+
getQualifiedTableName
|
|
20
|
+
} from './sql-utils';
|
|
21
|
+
|
|
22
|
+
// __VERSION__ is injected by babel-plugin-version-inline
|
|
23
|
+
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
24
|
+
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
25
|
+
|
|
26
|
+
/** Snowflake-backed SQL source. */
|
|
27
|
+
export const SnowflakeSQLSource = {
|
|
28
|
+
dataType: null as unknown as SnowflakeSQLDataSource,
|
|
29
|
+
batchType: null as never,
|
|
30
|
+
name: 'SnowflakeSQLSource',
|
|
31
|
+
id: 'snowflake-sql',
|
|
32
|
+
module: 'sql',
|
|
33
|
+
version: VERSION,
|
|
34
|
+
extensions: [],
|
|
35
|
+
mimeTypes: [],
|
|
36
|
+
type: 'snowflake-sql',
|
|
37
|
+
fromUrl: true,
|
|
38
|
+
fromBlob: false,
|
|
39
|
+
options: {
|
|
40
|
+
sql: {
|
|
41
|
+
resultFormat: 'auto',
|
|
42
|
+
adapterOptions: {},
|
|
43
|
+
browserAdapterFactory: undefined,
|
|
44
|
+
nodeAdapterFactory: undefined
|
|
45
|
+
},
|
|
46
|
+
duckdb: {
|
|
47
|
+
accessMode: 'auto',
|
|
48
|
+
bundles: undefined,
|
|
49
|
+
databasePath: undefined,
|
|
50
|
+
remoteUrl: undefined,
|
|
51
|
+
workerUrl: undefined
|
|
52
|
+
},
|
|
53
|
+
snowflake: {
|
|
54
|
+
account: undefined,
|
|
55
|
+
database: undefined,
|
|
56
|
+
schema: undefined,
|
|
57
|
+
token: undefined,
|
|
58
|
+
warehouse: undefined,
|
|
59
|
+
role: undefined,
|
|
60
|
+
authenticator: undefined
|
|
61
|
+
}
|
|
62
|
+
},
|
|
63
|
+
defaultOptions: SQL_SOURCE_DEFAULT_OPTIONS,
|
|
64
|
+
testURL: (url: string): boolean => /^(sql\+snowflake|snowflake):\/\//i.test(url),
|
|
65
|
+
createDataSource: (
|
|
66
|
+
data: string,
|
|
67
|
+
options: SQLSourceOptions,
|
|
68
|
+
coreApi?: CoreAPI
|
|
69
|
+
): SnowflakeSQLDataSource => new SnowflakeSQLDataSource(data, options, coreApi)
|
|
70
|
+
} as const satisfies SourceLoader<SnowflakeSQLDataSource>;
|
|
71
|
+
|
|
72
|
+
/** SQLDataSource specialization for Snowflake. */
|
|
73
|
+
export class SnowflakeSQLDataSource extends SQLDataSource {
|
|
74
|
+
constructor(data: string, options: SQLSourceOptions, coreApi?: CoreAPI) {
|
|
75
|
+
super(
|
|
76
|
+
data,
|
|
77
|
+
options,
|
|
78
|
+
SnowflakeSQLSource.type,
|
|
79
|
+
SQL_SOURCE_DEFAULT_OPTIONS,
|
|
80
|
+
createSnowflakeAdapter,
|
|
81
|
+
coreApi
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async function createSnowflakeAdapter(context: SQLAdapterFactoryContext): Promise<SQLAdapter> {
|
|
87
|
+
const baseStatementUrl = getSnowflakeStatementsUrl(context.url, context.options);
|
|
88
|
+
const authorization = context.options.snowflake?.token;
|
|
89
|
+
if (!authorization) {
|
|
90
|
+
throw new Error('SnowflakeSQLSource requires snowflake.token.');
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const executeRows = async (
|
|
94
|
+
sqlText: string,
|
|
95
|
+
options: SQLQueryOptions = {}
|
|
96
|
+
): Promise<Record<string, unknown>[]> => {
|
|
97
|
+
const initialResponse = await executeSnowflakeStatement(
|
|
98
|
+
context,
|
|
99
|
+
baseStatementUrl,
|
|
100
|
+
authorization,
|
|
101
|
+
sqlText,
|
|
102
|
+
options
|
|
103
|
+
);
|
|
104
|
+
const responses = [initialResponse];
|
|
105
|
+
|
|
106
|
+
const partitionInfo = initialResponse.resultSetMetaData?.partitionInfo || [];
|
|
107
|
+
for (let partitionIndex = 1; partitionIndex < partitionInfo.length; partitionIndex++) {
|
|
108
|
+
responses.push(
|
|
109
|
+
await fetchSnowflakeJson(
|
|
110
|
+
context,
|
|
111
|
+
`${baseStatementUrl}/${initialResponse.statementHandle}?partition=${partitionIndex}`,
|
|
112
|
+
authorization,
|
|
113
|
+
{method: 'GET', signal: options.signal}
|
|
114
|
+
)
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const rowType = initialResponse.resultSetMetaData?.rowType || [];
|
|
119
|
+
const rowNames = rowType.map((row: {name: string}) => row.name);
|
|
120
|
+
return responses.flatMap((response: {data?: unknown[][]}) =>
|
|
121
|
+
(response.data || []).map((values: unknown[]) => {
|
|
122
|
+
const row: Record<string, unknown> = {};
|
|
123
|
+
rowNames.forEach((name, index) => {
|
|
124
|
+
row[name] = values[index];
|
|
125
|
+
});
|
|
126
|
+
return row;
|
|
127
|
+
})
|
|
128
|
+
);
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
return {
|
|
132
|
+
capabilities: {
|
|
133
|
+
supportsArrow: false,
|
|
134
|
+
supportsMetadata: true,
|
|
135
|
+
runtime: 'both',
|
|
136
|
+
isDynamic: false
|
|
137
|
+
},
|
|
138
|
+
async connect(): Promise<void> {
|
|
139
|
+
if (!authorization) {
|
|
140
|
+
throw new Error('SnowflakeSQLSource requires snowflake.token.');
|
|
141
|
+
}
|
|
142
|
+
},
|
|
143
|
+
async close(): Promise<void> {},
|
|
144
|
+
async listCatalogs() {
|
|
145
|
+
const rows = await executeRows('SHOW DATABASES');
|
|
146
|
+
return rows.map(row => ({
|
|
147
|
+
catalogName: String(row.name || row.NAME)
|
|
148
|
+
}));
|
|
149
|
+
},
|
|
150
|
+
async listSchemas() {
|
|
151
|
+
const rows = await executeRows('SHOW SCHEMAS');
|
|
152
|
+
return rows.map(row => ({
|
|
153
|
+
catalogName: row.database_name ? String(row.database_name) : undefined,
|
|
154
|
+
schemaName: String(row.name || row.NAME)
|
|
155
|
+
}));
|
|
156
|
+
},
|
|
157
|
+
async listTables(): Promise<SQLTableInfo[]> {
|
|
158
|
+
const rows = await executeRows('SHOW TABLES');
|
|
159
|
+
return rows.map(row => ({
|
|
160
|
+
catalogName: row.database_name ? String(row.database_name) : undefined,
|
|
161
|
+
schemaName: row.schema_name ? String(row.schema_name) : undefined,
|
|
162
|
+
tableName: String(row.name || row.NAME),
|
|
163
|
+
tableType: row.kind ? String(row.kind) : undefined
|
|
164
|
+
}));
|
|
165
|
+
},
|
|
166
|
+
async getTableSchema(options: {
|
|
167
|
+
catalogName?: string;
|
|
168
|
+
schemaName?: string;
|
|
169
|
+
tableName: string;
|
|
170
|
+
}): Promise<Schema> {
|
|
171
|
+
const qualifiedTableName = getQualifiedTableName(options);
|
|
172
|
+
const rows = await executeRows(`DESCRIBE TABLE ${qualifiedTableName}`);
|
|
173
|
+
return convertSQLColumnsToSchema(
|
|
174
|
+
rows.map((row, index) => ({
|
|
175
|
+
columnName: String(row.name || row.NAME || `column_${index + 1}`),
|
|
176
|
+
sqlType: String(row.type || row.TYPE || 'VARCHAR'),
|
|
177
|
+
nullable: String(row.null || row.NULL || '').toUpperCase() === 'Y',
|
|
178
|
+
ordinalPosition: index + 1
|
|
179
|
+
}))
|
|
180
|
+
);
|
|
181
|
+
},
|
|
182
|
+
async executeRows(
|
|
183
|
+
sqlText: string,
|
|
184
|
+
options: SQLQueryOptions = {}
|
|
185
|
+
): Promise<Record<string, unknown>[]> {
|
|
186
|
+
return await executeRows(sqlText, options);
|
|
187
|
+
},
|
|
188
|
+
async executeArrow(sqlText: string, options: SQLQueryOptions = {}) {
|
|
189
|
+
const rows = await executeRows(sqlText, options);
|
|
190
|
+
return convertRowsToArrowTable(rows);
|
|
191
|
+
}
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
async function executeSnowflakeStatement(
|
|
196
|
+
context: SQLAdapterFactoryContext,
|
|
197
|
+
baseStatementUrl: string,
|
|
198
|
+
authorization: string,
|
|
199
|
+
sqlText: string,
|
|
200
|
+
options: SQLQueryOptions
|
|
201
|
+
): Promise<any> {
|
|
202
|
+
let response = await fetchSnowflakeJson(context, baseStatementUrl, authorization, {
|
|
203
|
+
method: 'POST',
|
|
204
|
+
headers: {
|
|
205
|
+
'Content-Type': 'application/json'
|
|
206
|
+
},
|
|
207
|
+
body: JSON.stringify({
|
|
208
|
+
statement: sqlText,
|
|
209
|
+
timeout: 60,
|
|
210
|
+
bindings: normalizeSnowflakeBindings(options.parameters),
|
|
211
|
+
warehouse: context.options.snowflake?.warehouse,
|
|
212
|
+
database: context.options.snowflake?.database,
|
|
213
|
+
schema: context.options.snowflake?.schema,
|
|
214
|
+
role: context.options.snowflake?.role
|
|
215
|
+
}),
|
|
216
|
+
signal: options.signal
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
while (response?.statementStatusUrl && !response?.data) {
|
|
220
|
+
const pollUrl = `${getSnowflakeBaseUrl(context.url, context.options)}${response.statementStatusUrl}`;
|
|
221
|
+
response = await fetchSnowflakeJson(context, pollUrl, authorization, {
|
|
222
|
+
method: 'GET',
|
|
223
|
+
signal: options.signal
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
if (response?.code || response?.message) {
|
|
228
|
+
throw new Error(response.message || `Snowflake SQL API error ${response.code}`);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
return response;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
async function fetchSnowflakeJson(
|
|
235
|
+
context: SQLAdapterFactoryContext,
|
|
236
|
+
url: string,
|
|
237
|
+
authorization: string,
|
|
238
|
+
requestInit: RequestInit
|
|
239
|
+
): Promise<any> {
|
|
240
|
+
const customFetch = context.options.core?.loadOptions?.core?.fetch;
|
|
241
|
+
const headers = new Headers(requestInit.headers);
|
|
242
|
+
headers.set(
|
|
243
|
+
'Authorization',
|
|
244
|
+
authorization.startsWith('Bearer ') ? authorization : `Bearer ${authorization}`
|
|
245
|
+
);
|
|
246
|
+
headers.set('Accept', 'application/json');
|
|
247
|
+
|
|
248
|
+
const response =
|
|
249
|
+
typeof customFetch === 'function'
|
|
250
|
+
? await customFetch(url, {
|
|
251
|
+
...requestInit,
|
|
252
|
+
headers
|
|
253
|
+
})
|
|
254
|
+
: await context.coreApi.fetchFile(url, {
|
|
255
|
+
...requestInit,
|
|
256
|
+
headers
|
|
257
|
+
});
|
|
258
|
+
const json = await response.json();
|
|
259
|
+
if (!response.ok) {
|
|
260
|
+
throw new Error(json?.message || `${response.status} ${response.statusText}`);
|
|
261
|
+
}
|
|
262
|
+
return json;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
function normalizeSnowflakeBindings(
|
|
266
|
+
parameters: SQLQueryOptions['parameters']
|
|
267
|
+
): Record<string, {type: string; value: unknown}> | undefined {
|
|
268
|
+
if (!parameters || Array.isArray(parameters)) {
|
|
269
|
+
return undefined;
|
|
270
|
+
}
|
|
271
|
+
const bindings: Record<string, {type: string; value: unknown}> = {};
|
|
272
|
+
for (const [key, value] of Object.entries(parameters)) {
|
|
273
|
+
bindings[key] = {
|
|
274
|
+
type: typeof value === 'number' ? 'FIXED' : 'TEXT',
|
|
275
|
+
value
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
return bindings;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
function getSnowflakeStatementsUrl(url: string, options: SQLSourceOptions): string {
|
|
282
|
+
return `${getSnowflakeBaseUrl(url, options)}/api/v2/statements`;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
function getSnowflakeBaseUrl(url: string, options: SQLSourceOptions): string {
|
|
286
|
+
if (/^https?:\/\//i.test(url)) {
|
|
287
|
+
return url.replace(/\/api\/v2\/statements.*$/, '');
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
const parsedUrl = new URL(url);
|
|
291
|
+
const account = options.snowflake?.account || parsedUrl.hostname;
|
|
292
|
+
if (!account) {
|
|
293
|
+
throw new Error('SnowflakeSQLSource requires a Snowflake account URL or snowflake.account.');
|
|
294
|
+
}
|
|
295
|
+
return `https://${account}.snowflakecomputing.com`;
|
|
296
|
+
}
|