@loaders.gl/sql 5.0.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,193 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import type {ArrowTable, Schema} from '@loaders.gl/schema';
6
+ import {DataSource} from '@loaders.gl/loader-utils';
7
+ import type {CoreAPI, RequiredOptions} from '@loaders.gl/loader-utils';
8
+
9
+ import type {
10
+ SQLAdapter,
11
+ SQLAdapterFactory,
12
+ SQLCatalogInfo,
13
+ SQLMetadata,
14
+ SQLQueryOptions,
15
+ SQLSchemaInfo,
16
+ SQLSourceOptions,
17
+ SQLTableInfo
18
+ } from './sql-types';
19
+ import {convertRowsToArrowTable, isNodeRuntime} from './sql-utils';
20
+
21
+ const SQL_ADAPTER_FACTORIES = new Map<string, SQLAdapterFactory>();
22
+
23
+ /** Registers a runtime adapter for a SQL source type. */
24
+ export function registerSQLAdapter(type: string, factory: SQLAdapterFactory): void {
25
+ SQL_ADAPTER_FACTORIES.set(type, factory);
26
+ }
27
+
28
+ /** Returns the registered adapter factory for a SQL source type. */
29
+ export function getSQLAdapterFactory(type: string): SQLAdapterFactory | undefined {
30
+ return SQL_ADAPTER_FACTORIES.get(type);
31
+ }
32
+
33
+ /** Base options shared by all SQL sources. */
34
+ export const SQL_SOURCE_DEFAULT_OPTIONS: Omit<RequiredOptions<SQLSourceOptions>, 'core'> = {
35
+ sql: {
36
+ resultFormat: 'auto',
37
+ adapterOptions: {},
38
+ browserAdapterFactory: undefined!,
39
+ nodeAdapterFactory: undefined!
40
+ },
41
+ duckdb: {
42
+ accessMode: 'auto',
43
+ bundles: undefined!,
44
+ databasePath: undefined!,
45
+ remoteUrl: undefined!,
46
+ workerUrl: undefined!
47
+ },
48
+ snowflake: {
49
+ account: undefined!,
50
+ database: undefined!,
51
+ schema: undefined!,
52
+ token: undefined!,
53
+ warehouse: undefined!,
54
+ role: undefined!,
55
+ authenticator: undefined!
56
+ }
57
+ };
58
+
59
+ /** Shared SQL-backed data source implementation. */
60
+ export class SQLDataSource extends DataSource<string, SQLSourceOptions> {
61
+ private readonly sourceType: string;
62
+ private readonly adapterFactory?: SQLAdapterFactory;
63
+ private adapterPromise: Promise<SQLAdapter> | null = null;
64
+ private metadataPromise: Promise<SQLMetadata> | null = null;
65
+
66
+ constructor(
67
+ data: string,
68
+ options: SQLSourceOptions,
69
+ sourceType: string,
70
+ defaultOptions: Omit<RequiredOptions<SQLSourceOptions>, 'core'> = SQL_SOURCE_DEFAULT_OPTIONS,
71
+ adapterFactory?: SQLAdapterFactory,
72
+ coreApi?: CoreAPI
73
+ ) {
74
+ super(data, options, defaultOptions, coreApi);
75
+ this.sourceType = sourceType;
76
+ this.adapterFactory = adapterFactory || getSQLAdapterFactory(sourceType);
77
+ }
78
+
79
+ /** Returns cached high-level metadata for the SQL source. */
80
+ async getMetadata(): Promise<SQLMetadata> {
81
+ if (!this.metadataPromise) {
82
+ this.metadataPromise = this.loadMetadata();
83
+ }
84
+ return this.metadataPromise;
85
+ }
86
+
87
+ /** Lists available catalogs from the connected database. */
88
+ async listCatalogs(): Promise<SQLCatalogInfo[]> {
89
+ const adapter = await this.getAdapter();
90
+ return await adapter.listCatalogs();
91
+ }
92
+
93
+ /** Lists available schemas from the connected database. */
94
+ async listSchemas(catalogName?: string): Promise<SQLSchemaInfo[]> {
95
+ const adapter = await this.getAdapter();
96
+ return await adapter.listSchemas(catalogName);
97
+ }
98
+
99
+ /** Lists available tables from the connected database. */
100
+ async listTables(options?: {catalogName?: string; schemaName?: string}): Promise<SQLTableInfo[]> {
101
+ const adapter = await this.getAdapter();
102
+ return await adapter.listTables(options);
103
+ }
104
+
105
+ /** Returns the schema for a specific SQL table. */
106
+ async getTableSchema(options: {
107
+ catalogName?: string;
108
+ schemaName?: string;
109
+ tableName: string;
110
+ }): Promise<Schema> {
111
+ const adapter = await this.getAdapter();
112
+ return await adapter.getTableSchema(options);
113
+ }
114
+
115
+ /** Executes a SQL query and returns object rows. */
116
+ async queryRows(
117
+ sqlText: string,
118
+ options: SQLQueryOptions = {}
119
+ ): Promise<Record<string, unknown>[]> {
120
+ const adapter = await this.getAdapter();
121
+ return await adapter.executeRows(sqlText, options);
122
+ }
123
+
124
+ /** Executes a SQL query and returns a loaders.gl Arrow table. */
125
+ async queryArrow(sqlText: string, options: SQLQueryOptions = {}): Promise<ArrowTable> {
126
+ const adapter = await this.getAdapter();
127
+ if (adapter.executeArrow) {
128
+ return await adapter.executeArrow(sqlText, options);
129
+ }
130
+ const rows = await adapter.executeRows(sqlText, options);
131
+ return convertRowsToArrowTable(rows);
132
+ }
133
+
134
+ /** Closes any underlying adapter connection. */
135
+ async close(): Promise<void> {
136
+ if (!this.adapterPromise) {
137
+ return;
138
+ }
139
+ const adapter = await this.adapterPromise;
140
+ await adapter.close();
141
+ this.adapterPromise = null;
142
+ this.metadataPromise = null;
143
+ }
144
+
145
+ protected async getAdapter(): Promise<SQLAdapter> {
146
+ if (!this.adapterPromise) {
147
+ this.adapterPromise = this.loadAdapter();
148
+ }
149
+ return await this.adapterPromise;
150
+ }
151
+
152
+ private async loadAdapter(): Promise<SQLAdapter> {
153
+ try {
154
+ const runtimeFactory = isNodeRuntime()
155
+ ? this.options.sql?.nodeAdapterFactory
156
+ : this.options.sql?.browserAdapterFactory;
157
+ const factory = runtimeFactory || this.adapterFactory;
158
+ if (!factory) {
159
+ throw new Error(`No SQL adapter is registered for source type "${this.sourceType}".`);
160
+ }
161
+
162
+ const adapter = await factory({
163
+ coreApi: this.coreApi,
164
+ options: this.options,
165
+ sourceType: this.sourceType,
166
+ url: this.url
167
+ });
168
+ await adapter.connect();
169
+ return adapter;
170
+ } catch (error) {
171
+ throw this.reportError(
172
+ error,
173
+ `Failed to create SQL adapter for ${this.sourceType} from ${this.url}`
174
+ );
175
+ }
176
+ }
177
+
178
+ private async loadMetadata(): Promise<SQLMetadata> {
179
+ const adapter = await this.getAdapter();
180
+ const [catalogs, schemas, tables] = await Promise.all([
181
+ adapter.listCatalogs(),
182
+ adapter.listSchemas(),
183
+ adapter.listTables()
184
+ ]);
185
+ return {
186
+ type: this.sourceType,
187
+ capabilities: adapter.capabilities,
188
+ catalogs,
189
+ schemas,
190
+ tables
191
+ };
192
+ }
193
+ }
@@ -0,0 +1,119 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import type {ArrowTable, Schema} from '@loaders.gl/schema';
6
+ import type {CoreAPI, DataSourceOptions} from '@loaders.gl/loader-utils';
7
+
8
+ export type SQLParameterValues = unknown[] | Record<string, unknown>;
9
+
10
+ /** Query-level options for SQL data sources. */
11
+ export type SQLQueryOptions = {
12
+ parameters?: SQLParameterValues;
13
+ signal?: AbortSignal;
14
+ };
15
+
16
+ /** Normalized database catalog metadata. */
17
+ export type SQLCatalogInfo = {
18
+ catalogName: string;
19
+ };
20
+
21
+ /** Normalized database schema metadata. */
22
+ export type SQLSchemaInfo = {
23
+ catalogName?: string;
24
+ schemaName: string;
25
+ };
26
+
27
+ /** Normalized table metadata returned by SQL sources. */
28
+ export type SQLTableInfo = {
29
+ catalogName?: string;
30
+ schemaName?: string;
31
+ tableName: string;
32
+ tableType?: string;
33
+ };
34
+
35
+ /** Normalized column metadata returned by SQL sources. */
36
+ export type SQLColumnInfo = {
37
+ catalogName?: string;
38
+ schemaName?: string;
39
+ tableName?: string;
40
+ columnName: string;
41
+ sqlType: string;
42
+ nullable?: boolean;
43
+ ordinalPosition?: number;
44
+ };
45
+
46
+ /** Top-level metadata returned by `SQLDataSource#getMetadata`. */
47
+ export type SQLMetadata = {
48
+ type: string;
49
+ capabilities: SQLAdapterCapabilities;
50
+ catalogs: SQLCatalogInfo[];
51
+ schemas: SQLSchemaInfo[];
52
+ tables: SQLTableInfo[];
53
+ };
54
+
55
+ /** Adapter capability flags used for runtime selection and reporting. */
56
+ export type SQLAdapterCapabilities = {
57
+ supportsArrow: boolean;
58
+ supportsMetadata: boolean;
59
+ runtime: 'node' | 'browser' | 'both';
60
+ isDynamic?: boolean;
61
+ };
62
+
63
+ /** Runtime adapter interface used by SQL-backed data sources. */
64
+ export interface SQLAdapter {
65
+ readonly capabilities: SQLAdapterCapabilities;
66
+ connect(): Promise<void>;
67
+ close(): Promise<void>;
68
+ listCatalogs(): Promise<SQLCatalogInfo[]>;
69
+ listSchemas(catalogName?: string): Promise<SQLSchemaInfo[]>;
70
+ listTables(options?: {catalogName?: string; schemaName?: string}): Promise<SQLTableInfo[]>;
71
+ getTableSchema(options: {
72
+ catalogName?: string;
73
+ schemaName?: string;
74
+ tableName: string;
75
+ }): Promise<Schema>;
76
+ executeRows(sql: string, options?: SQLQueryOptions): Promise<Record<string, unknown>[]>;
77
+ executeArrow?(sql: string, options?: SQLQueryOptions): Promise<ArrowTable>;
78
+ }
79
+
80
+ /** Context passed to SQL adapter factories during source initialization. */
81
+ export type SQLAdapterFactoryContext = {
82
+ coreApi: CoreAPI;
83
+ options: SQLSourceOptions;
84
+ sourceType: string;
85
+ url: string;
86
+ };
87
+
88
+ /** Factory signature for creating runtime-specific SQL adapters. */
89
+ export type SQLAdapterFactory = (context: SQLAdapterFactoryContext) => Promise<SQLAdapter>;
90
+
91
+ /** Shared options accepted by SQL-backed data sources. */
92
+ export type SQLSourceOptions = DataSourceOptions & {
93
+ sql?: {
94
+ resultFormat?: 'auto' | 'arrow' | 'rows';
95
+ adapterOptions?: Record<string, unknown>;
96
+ browserAdapterFactory?: SQLAdapterFactory;
97
+ nodeAdapterFactory?: SQLAdapterFactory;
98
+ };
99
+ duckdb?: {
100
+ accessMode?: 'auto' | 'read-write' | 'read-only';
101
+ bundles?: {
102
+ mainModule?: string;
103
+ mainWorker?: string;
104
+ pthreadWorker?: string;
105
+ };
106
+ databasePath?: string;
107
+ remoteUrl?: string;
108
+ workerUrl?: string;
109
+ };
110
+ snowflake?: {
111
+ account?: string;
112
+ database?: string;
113
+ schema?: string;
114
+ token?: string;
115
+ warehouse?: string;
116
+ role?: string;
117
+ authenticator?: string;
118
+ };
119
+ };
@@ -0,0 +1,137 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import * as arrow from 'apache-arrow';
6
+ import type {ArrowTable, DataType, Field, ObjectRowTable, Schema} from '@loaders.gl/schema';
7
+ import {makeTableFromData, getDataTypeFromArray} from '@loaders.gl/schema-utils';
8
+
9
+ import type {SQLColumnInfo} from './sql-types';
10
+
11
+ /** Returns true when the current runtime is Node.js. */
12
+ export function isNodeRuntime(): boolean {
13
+ return typeof process !== 'undefined' && Boolean(process.versions?.node);
14
+ }
15
+
16
+ /** Escapes a SQL string literal using single-quote doubling. */
17
+ export function escapeSqlString(value: string): string {
18
+ return value.replace(/'/g, "''");
19
+ }
20
+
21
+ /** Wraps an identifier in double quotes, escaping embedded quotes. */
22
+ export function quoteSqlIdentifier(identifier: string): string {
23
+ return `"${identifier.replace(/"/g, '""')}"`;
24
+ }
25
+
26
+ /** Converts object rows into a loaders.gl Arrow table. */
27
+ export function convertRowsToArrowTable(rows: Record<string, unknown>[]): ArrowTable {
28
+ const table: ObjectRowTable =
29
+ rows.length > 0
30
+ ? makeTableFromData(rows)
31
+ : {shape: 'object-row-table', schema: {fields: [], metadata: {}}, data: rows};
32
+ const schema = deducePrimitiveAwareSchema(table);
33
+
34
+ if (table.data.length === 0 || schema.fields.length === 0) {
35
+ return {
36
+ shape: 'arrow-table',
37
+ schema,
38
+ data: new arrow.Table(new arrow.Schema([]))
39
+ };
40
+ }
41
+
42
+ return {
43
+ shape: 'arrow-table',
44
+ schema,
45
+ data: arrow.tableFromJSON(table.data)
46
+ };
47
+ }
48
+
49
+ /** Converts normalized SQL columns into a loaders.gl schema. */
50
+ export function convertSQLColumnsToSchema(columns: SQLColumnInfo[]): Schema {
51
+ return {
52
+ metadata: {},
53
+ fields: columns
54
+ .sort((left, right) => (left.ordinalPosition || 0) - (right.ordinalPosition || 0))
55
+ .map(column => ({
56
+ name: column.columnName,
57
+ type: mapSQLTypeToDataType(column.sqlType),
58
+ nullable: column.nullable,
59
+ metadata: {
60
+ sqlType: column.sqlType
61
+ }
62
+ }))
63
+ };
64
+ }
65
+
66
+ function mapSQLTypeToDataType(sqlType: string): DataType {
67
+ const normalizedType = sqlType.toUpperCase();
68
+ if (
69
+ normalizedType.includes('BIGINT') ||
70
+ normalizedType === 'HUGEINT' ||
71
+ normalizedType === 'LONG'
72
+ ) {
73
+ return 'int64';
74
+ }
75
+ if (
76
+ normalizedType.includes('INT') ||
77
+ normalizedType === 'INTEGER' ||
78
+ normalizedType === 'SMALLINT' ||
79
+ normalizedType === 'TINYINT'
80
+ ) {
81
+ return 'int32';
82
+ }
83
+ if (
84
+ normalizedType.includes('DOUBLE') ||
85
+ normalizedType.includes('FLOAT8') ||
86
+ normalizedType.includes('REAL')
87
+ ) {
88
+ return 'float64';
89
+ }
90
+ if (
91
+ normalizedType.includes('FLOAT') ||
92
+ normalizedType.includes('DECIMAL') ||
93
+ normalizedType.includes('NUMERIC')
94
+ ) {
95
+ return 'float64';
96
+ }
97
+ if (normalizedType.includes('BOOL')) {
98
+ return 'bool';
99
+ }
100
+ if (normalizedType.includes('DATE')) {
101
+ return 'date-day';
102
+ }
103
+ if (normalizedType.includes('TIME') || normalizedType.includes('TIMESTAMP')) {
104
+ return 'timestamp-millisecond';
105
+ }
106
+ if (normalizedType.includes('BLOB') || normalizedType.includes('BINARY')) {
107
+ return 'binary';
108
+ }
109
+ return 'utf8';
110
+ }
111
+
112
+ function deducePrimitiveAwareSchema(table: ObjectRowTable): Schema {
113
+ return {
114
+ ...table.schema!,
115
+ fields: table.schema!.fields.map((field): Field => {
116
+ if (field.type !== 'null') {
117
+ return field;
118
+ }
119
+
120
+ const columnValues = table.data.map(row => row[field.name]);
121
+ const inferredType = getDataTypeFromArray(columnValues);
122
+ return {...field, type: inferredType.type, nullable: inferredType.nullable};
123
+ })
124
+ };
125
+ }
126
+
127
+ /** Returns a fully qualified SQL object name. */
128
+ export function getQualifiedTableName(options: {
129
+ catalogName?: string;
130
+ schemaName?: string;
131
+ tableName: string;
132
+ }): string {
133
+ const identifiers = [options.catalogName, options.schemaName, options.tableName].filter(
134
+ (value): value is string => Boolean(value)
135
+ );
136
+ return identifiers.map(identifier => quoteSqlIdentifier(identifier)).join('.');
137
+ }