@malloydata/db-snowflake 0.0.195-dev241003204905 → 0.0.195
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { RunSQLOptions, MalloyQueryData, QueryRunStats, Connection, PersistSQLResults, StreamingConnection, PooledConnection, SQLSourceDef, TableSourceDef, QueryDataRow, TestableConnection } from '@malloydata/malloy';
|
|
2
2
|
import { BaseConnection } from '@malloydata/malloy/connection';
|
|
3
3
|
import { ConnectionOptions } from 'snowflake-sdk';
|
|
4
4
|
import { Options as PoolOptions } from 'generic-pool';
|
|
@@ -16,8 +16,6 @@ export declare class SnowflakeConnection extends BaseConnection implements Conne
|
|
|
16
16
|
readonly name: string;
|
|
17
17
|
private readonly dialect;
|
|
18
18
|
private executor;
|
|
19
|
-
private schemaCache;
|
|
20
|
-
private sqlSchemaCache;
|
|
21
19
|
private scratchSpace?;
|
|
22
20
|
private queryOptions;
|
|
23
21
|
constructor(name: string, options?: SnowflakeConnectionOptions);
|
|
@@ -32,22 +30,10 @@ export declare class SnowflakeConnection extends BaseConnection implements Conne
|
|
|
32
30
|
runSQL(sql: string, options?: RunSQLOptions): Promise<MalloyQueryData>;
|
|
33
31
|
runSQLStream(sqlCommand: string, options?: RunSQLOptions): AsyncIterableIterator<QueryDataRow>;
|
|
34
32
|
test(): Promise<void>;
|
|
35
|
-
private variantToMalloyType;
|
|
36
33
|
private addFieldsToStructDef;
|
|
37
34
|
private schemaFromTablePath;
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
schemas: Record<string, StructDef>;
|
|
41
|
-
errors: Record<string, string>;
|
|
42
|
-
}>;
|
|
43
|
-
private getSQLBlockSchema;
|
|
44
|
-
fetchSchemaForSQLBlock(sqlRef: SQLBlock, { refreshTimestamp }: FetchSchemaOptions): Promise<{
|
|
45
|
-
structDef: StructDef;
|
|
46
|
-
error?: undefined;
|
|
47
|
-
} | {
|
|
48
|
-
error: string;
|
|
49
|
-
structDef?: undefined;
|
|
50
|
-
}>;
|
|
35
|
+
fetchTableSchema(tableKey: string, tablePath: string): Promise<TableSourceDef>;
|
|
36
|
+
fetchSelectSchema(sqlRef: SQLSourceDef): Promise<SQLSourceDef>;
|
|
51
37
|
manifestTemporaryTable(sqlCommand: string): Promise<string>;
|
|
52
38
|
}
|
|
53
39
|
export {};
|
|
@@ -70,8 +70,6 @@ class SnowflakeConnection extends connection_1.BaseConnection {
|
|
|
70
70
|
super();
|
|
71
71
|
this.name = name;
|
|
72
72
|
this.dialect = new malloy_1.SnowflakeDialect();
|
|
73
|
-
this.schemaCache = new Map();
|
|
74
|
-
this.sqlSchemaCache = new Map();
|
|
75
73
|
let connOptions = options === null || options === void 0 ? void 0 : options.connOptions;
|
|
76
74
|
if (connOptions === undefined) {
|
|
77
75
|
// try to get connection options from ~/.snowflake/connections.toml
|
|
@@ -128,19 +126,7 @@ class SnowflakeConnection extends connection_1.BaseConnection {
|
|
|
128
126
|
async test() {
|
|
129
127
|
await this.executor.batch('SELECT 1 as one');
|
|
130
128
|
}
|
|
131
|
-
variantToMalloyType(type) {
|
|
132
|
-
if (type === 'integer') {
|
|
133
|
-
return 'number';
|
|
134
|
-
}
|
|
135
|
-
else if (type === 'varchar') {
|
|
136
|
-
return 'string';
|
|
137
|
-
}
|
|
138
|
-
else {
|
|
139
|
-
return 'sql native';
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
129
|
addFieldsToStructDef(structDef, structMap) {
|
|
143
|
-
var _a;
|
|
144
130
|
if (structMap.fieldMap.size === 0)
|
|
145
131
|
return;
|
|
146
132
|
for (const [field, value] of structMap.fieldMap) {
|
|
@@ -148,43 +134,43 @@ class SnowflakeConnection extends connection_1.BaseConnection {
|
|
|
148
134
|
const name = field;
|
|
149
135
|
// check for an array
|
|
150
136
|
if (value.isArray && type !== 'object') {
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
structRelationship: {
|
|
159
|
-
type: 'nested',
|
|
160
|
-
fieldName: name,
|
|
161
|
-
isArray: true,
|
|
162
|
-
},
|
|
163
|
-
fields: [{ type: malloyType, name: 'value' }],
|
|
164
|
-
};
|
|
165
|
-
structDef.fields.push(innerStructDef);
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
else if (type === 'object') {
|
|
137
|
+
// Apparently there can only be arrays of integers, strings, or unknowns?
|
|
138
|
+
// TODO is this true or is this just all that got implemented?
|
|
139
|
+
const malloyType = type === 'integer'
|
|
140
|
+
? { type: 'number', numberType: 'integer' }
|
|
141
|
+
: type === 'varchar'
|
|
142
|
+
? { type: 'string' }
|
|
143
|
+
: { type: 'sql native', rawType: type };
|
|
169
144
|
const innerStructDef = {
|
|
170
|
-
type: '
|
|
145
|
+
type: 'array',
|
|
171
146
|
name,
|
|
172
147
|
dialect: this.dialectName,
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
: { type: 'inline' },
|
|
177
|
-
fields: [],
|
|
148
|
+
join: 'many',
|
|
149
|
+
elementTypeDef: malloyType,
|
|
150
|
+
fields: (0, malloy_1.arrayEachFields)(malloyType),
|
|
178
151
|
};
|
|
152
|
+
structDef.fields.push(innerStructDef);
|
|
153
|
+
}
|
|
154
|
+
else if (type === 'object') {
|
|
155
|
+
const structParts = { name, dialect: this.dialectName, fields: [] };
|
|
156
|
+
const innerStructDef = value.isArray
|
|
157
|
+
? {
|
|
158
|
+
...structParts,
|
|
159
|
+
type: 'array',
|
|
160
|
+
elementTypeDef: { type: 'record_element' },
|
|
161
|
+
join: 'many',
|
|
162
|
+
}
|
|
163
|
+
: {
|
|
164
|
+
...structParts,
|
|
165
|
+
type: 'record',
|
|
166
|
+
join: 'one',
|
|
167
|
+
};
|
|
179
168
|
this.addFieldsToStructDef(innerStructDef, value);
|
|
180
169
|
structDef.fields.push(innerStructDef);
|
|
181
170
|
}
|
|
182
171
|
else {
|
|
183
|
-
const malloyType =
|
|
184
|
-
|
|
185
|
-
rawType: type.toLowerCase(),
|
|
186
|
-
};
|
|
187
|
-
structDef.fields.push({ name, ...malloyType });
|
|
172
|
+
const malloyType = this.dialect.sqlTypeToMalloyType(type);
|
|
173
|
+
structDef.fields.push({ ...malloyType, name });
|
|
188
174
|
}
|
|
189
175
|
}
|
|
190
176
|
}
|
|
@@ -259,93 +245,26 @@ class SnowflakeConnection extends connection_1.BaseConnection {
|
|
|
259
245
|
this.addFieldsToStructDef(structDef, structMap);
|
|
260
246
|
}
|
|
261
247
|
}
|
|
262
|
-
async
|
|
248
|
+
async fetchTableSchema(tableKey, tablePath) {
|
|
263
249
|
const structDef = {
|
|
264
|
-
type: '
|
|
250
|
+
type: 'table',
|
|
265
251
|
dialect: 'snowflake',
|
|
266
252
|
name: tableKey,
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
type: 'basetable',
|
|
270
|
-
connectionName: this.name,
|
|
271
|
-
},
|
|
253
|
+
tablePath,
|
|
254
|
+
connection: this.name,
|
|
272
255
|
fields: [],
|
|
273
256
|
};
|
|
274
257
|
await this.schemaFromTablePath(tablePath, structDef);
|
|
275
258
|
return structDef;
|
|
276
259
|
}
|
|
277
|
-
async
|
|
278
|
-
const
|
|
279
|
-
const errors = {};
|
|
280
|
-
for (const tableKey in missing) {
|
|
281
|
-
let inCache = this.schemaCache.get(tableKey);
|
|
282
|
-
if (!inCache ||
|
|
283
|
-
(refreshTimestamp && refreshTimestamp > inCache.timestamp)) {
|
|
284
|
-
const tablePath = missing[tableKey];
|
|
285
|
-
const timestamp = refreshTimestamp || Date.now();
|
|
286
|
-
try {
|
|
287
|
-
inCache = {
|
|
288
|
-
schema: await this.getTableSchema(tableKey, tablePath),
|
|
289
|
-
timestamp,
|
|
290
|
-
};
|
|
291
|
-
this.schemaCache.set(tableKey, inCache);
|
|
292
|
-
}
|
|
293
|
-
catch (error) {
|
|
294
|
-
inCache = { error: error.message, timestamp };
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
if (inCache.schema !== undefined) {
|
|
298
|
-
schemas[tableKey] = inCache.schema;
|
|
299
|
-
}
|
|
300
|
-
else {
|
|
301
|
-
errors[tableKey] = inCache.error || 'Unknown schema fetch error';
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
|
-
return { schemas, errors };
|
|
305
|
-
}
|
|
306
|
-
async getSQLBlockSchema(sqlRef) {
|
|
307
|
-
const structDef = {
|
|
308
|
-
type: 'struct',
|
|
309
|
-
dialect: 'snowflake',
|
|
310
|
-
name: sqlRef.name,
|
|
311
|
-
structSource: {
|
|
312
|
-
type: 'sql',
|
|
313
|
-
method: 'subquery',
|
|
314
|
-
sqlBlock: sqlRef,
|
|
315
|
-
},
|
|
316
|
-
structRelationship: {
|
|
317
|
-
type: 'basetable',
|
|
318
|
-
connectionName: this.name,
|
|
319
|
-
},
|
|
320
|
-
fields: [],
|
|
321
|
-
};
|
|
260
|
+
async fetchSelectSchema(sqlRef) {
|
|
261
|
+
const structDef = { ...sqlRef, fields: [] };
|
|
322
262
|
// create temp table with same schema as the query
|
|
323
263
|
const tempTableName = this.getTempViewName(sqlRef.selectStr);
|
|
324
|
-
this.runSQL(`
|
|
325
|
-
CREATE OR REPLACE TEMP VIEW ${tempTableName} as ${sqlRef.selectStr};
|
|
326
|
-
`);
|
|
264
|
+
this.runSQL(`CREATE OR REPLACE TEMP VIEW ${tempTableName} AS (${sqlRef.selectStr});`);
|
|
327
265
|
await this.schemaFromTablePath(tempTableName, structDef);
|
|
328
266
|
return structDef;
|
|
329
267
|
}
|
|
330
|
-
async fetchSchemaForSQLBlock(sqlRef, { refreshTimestamp }) {
|
|
331
|
-
const key = sqlRef.name;
|
|
332
|
-
let inCache = this.sqlSchemaCache.get(key);
|
|
333
|
-
if (!inCache ||
|
|
334
|
-
(refreshTimestamp && refreshTimestamp > inCache.timestamp)) {
|
|
335
|
-
const timestamp = refreshTimestamp !== null && refreshTimestamp !== void 0 ? refreshTimestamp : Date.now();
|
|
336
|
-
try {
|
|
337
|
-
inCache = {
|
|
338
|
-
structDef: await this.getSQLBlockSchema(sqlRef),
|
|
339
|
-
timestamp,
|
|
340
|
-
};
|
|
341
|
-
}
|
|
342
|
-
catch (error) {
|
|
343
|
-
inCache = { error: error.message, timestamp };
|
|
344
|
-
}
|
|
345
|
-
this.sqlSchemaCache.set(key, inCache);
|
|
346
|
-
}
|
|
347
|
-
return inCache;
|
|
348
|
-
}
|
|
349
268
|
async manifestTemporaryTable(sqlCommand) {
|
|
350
269
|
const tableName = this.getTempViewName(sqlCommand);
|
|
351
270
|
const cmd = `CREATE OR REPLACE TEMP TABLE ${tableName} AS (${sqlCommand});`;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snowflake_connection.js","sourceRoot":"","sources":["../src/snowflake_connection.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,+CAAiC;AACjC,+CAgB4B;AAC5B,8DAA6D;AAE7D,6DAAuD;AAoBvD,MAAM,SAAS;IAKb,YAAY,IAAY,EAAE,OAAgB;QAJ1C,aAAQ,GAAG,IAAI,GAAG,EAAqB,CAAC;QACxC,SAAI,GAAG,QAAQ,CAAC;QAChB,YAAO,GAAG,KAAK,CAAC;QAGd,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,QAAQ,CAAC,IAAY,EAAE,IAAY;QACjC,MAAM,CAAC,GAAG,IAAI,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACrC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3B,OAAO,CAAC,CAAC;IACX,CAAC;CACF;AAED,MAAa,mBACX,SAAQ,2BAAc;
|
|
1
|
+
{"version":3,"file":"snowflake_connection.js","sourceRoot":"","sources":["../src/snowflake_connection.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,+CAAiC;AACjC,+CAgB4B;AAC5B,8DAA6D;AAE7D,6DAAuD;AAoBvD,MAAM,SAAS;IAKb,YAAY,IAAY,EAAE,OAAgB;QAJ1C,aAAQ,GAAG,IAAI,GAAG,EAAqB,CAAC;QACxC,SAAI,GAAG,QAAQ,CAAC;QAChB,YAAO,GAAG,KAAK,CAAC;QAGd,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,QAAQ,CAAC,IAAY,EAAE,IAAY;QACjC,MAAM,CAAC,GAAG,IAAI,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACrC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3B,OAAO,CAAC,CAAC;IACX,CAAC;CACF;AAED,MAAa,mBACX,SAAQ,2BAAc;IActB,YACkB,IAAY,EAC5B,OAAoC;;QAEpC,KAAK,EAAE,CAAC;QAHQ,SAAI,GAAJ,IAAI,CAAQ;QARb,YAAO,GAAG,IAAI,yBAAgB,EAAE,CAAC;QAYhD,IAAI,WAAW,GAAG,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,WAAW,CAAC;QACvC,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,mEAAmE;YACnE,WAAW,GAAG,sCAAiB,CAAC,4BAA4B,EAAE,CAAC;QACjE,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,IAAI,sCAAiB,CAAC,WAAW,EAAE,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,WAAW,CAAC,CAAC;QACzE,IAAI,CAAC,YAAY,GAAG,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,YAAY,CAAC;QAC1C,IAAI,CAAC,YAAY,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,YAAY,mCAAI,EAAE,CAAC;IAClD,CAAC;IAED,IAAI,WAAW;QACb,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,qCAAqC;IACrC,IAAW,eAAe;QACxB,OAAO,KAAK,CAAC;IACf,CAAC;IAEM,MAAM;QACX,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,UAAU;QACf,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,SAAS;QACd,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,iBAAiB,CAAC,WAAmB;QAChD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC7B,CAAC;IAEO,eAAe,CAAC,UAAkB;QACxC,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,OAAO,KAAK,IAAI,EAAE,CAAC;IACrB,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,GAAW,EACX,OAAuB;;QAEvB,MAAM,QAAQ,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,QAAQ,mCAAI,MAAA,IAAI,CAAC,YAAY,0CAAE,QAAQ,CAAC;QAClE,IAAI,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC1C,IAAI,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YACrD,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,EAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,EAAC,CAAC;IACxC,CAAC;IAEM,KAAK,CAAC,CAAC,YAAY,CACxB,UAAkB,EAClB,UAAyB,EAAE;QAE3B,MAAM,kBAAkB,GAAG;YACzB,GAAG,IAAI,CAAC,YAAY;YACpB,GAAG,OAAO;SACX,CAAC;QAEF,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAChD,UAAU,EACV,kBAAkB,CACnB,EAAE,CAAC;YACF,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;IAC/C,CAAC;IAEO,oBAAoB,CAC1B,SAAoB,EACpB,SAAoB;QAEpB,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO;QAC1C,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;YAChD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;YACxB,MAAM,IAAI,GAAG,KAAK,CAAC;YAEnB,qBAAqB;YACrB,IAAI,KAAK,CAAC,OAAO,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACvC,yEAAyE;gBACzE,8DAA8D;gBAC9D,MAAM,UAAU,GACd,IAAI,KAAK,SAAS;oBAChB,CAAC,CAAC,EAAC,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAC;oBACzC,CAAC,CAAC,IAAI,KAAK,SAAS;wBACpB,CAAC,CAAC,EAAC,IAAI,EAAE,QAAQ,EAAC;wBAClB,CAAC,CAAC,EAAC,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAC,CAAC;gBAC1C,MAAM,cAAc,GAAc;oBAChC,IAAI,EAAE,OAAO;oBACb,IAAI;oBACJ,OAAO,EAAE,IAAI,CAAC,WAAW;oBACzB,IAAI,EAAE,MAAM;oBACZ,cAAc,EAAE,UAAU;oBAC1B,MAAM,EAAE,IAAA,wBAAe,EAAC,UAAU,CAAC;iBACpC,CAAC;gBACF,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACxC,CAAC;iBAAM,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,MAAM,WAAW,GAAG,EAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAC,CAAC;gBAClE,MAAM,cAAc,GAAc,KAAK,CAAC,OAAO;oBAC7C,CAAC,CAAC;wBACE,GAAG,WAAW;wBACd,IAAI,EAAE,OAAO;wBACb,cAAc,EAAE,EAAC,IAAI,EAAE,gBAAgB,EAAC;wBACxC,IAAI,EAAE,MAAM;qBACb;oBACH,CAAC,CAAC;wBACE,GAAG,WAAW;wBACd,IAAI,EAAE,QAAQ;wBACd,IAAI,EAAE,KAAK;qBACZ,CAAC;gBACN,IAAI,CAAC,oBAAoB,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;gBACjD,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACN,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;gBAC1D,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAC,GAAG,UAAU,EAAE,IAAI,EAAC,CAAC,CAAC;YAC/C,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC/B,SAAiB,EACjB,SAAoB;;QAEpB,MAAM,SAAS,GAAG,kBAAkB,SAAS,EAAE,CAAC;QAChD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAClD,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,UAAU,GAAG,IAAI,GAAG,EAAmB,CAAC;QAC9C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,uCAAuC;YACvC,IAAI,iBAAiB,GAAG,GAAG,CAAC,MAAM,CAAW,CAAC;YAC9C,iBAAiB,GAAG,iBAAiB,CAAC,iBAAiB,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACxE,MAAM,CAAC,GAAG,SAAS,CAAC;YACpB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,iBAAiB,CAAC,CAAC;YACvE,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAW,CAAC;YAEnC,IAAI,iBAAiB,KAAK,SAAS,IAAI,iBAAiB,KAAK,OAAO,EAAE,CAAC;gBACrE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpB,SAAS;YACX,CAAC;YAED,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC3B,IAAI,UAAU,EAAE,CAAC;gBACf,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAC,GAAG,UAAU,EAAE,IAAI,EAAC,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,YAAY;oBAClB,OAAO,EAAE,iBAAiB;oBAC1B,IAAI;iBACL,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,uCAAuC;QACvC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,WAAW,GAAG;;mDAEyB,SAAS;;;;OAIrD,CAAC;YACF,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAE7D,0DAA0D;YAE1D,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAEhD,KAAK,MAAM,CAAC,IAAI,aAAa,EAAE,CAAC;gBAC9B,MAAM,UAAU,GAAG,MAAA,CAAC,CAAC,MAAM,CAAC,0CAAE,OAAO,GAAG,QAAQ,EAAE,CAAC;gBACnD,MAAM,SAAS,GAAG,MAAA,CAAC,CAAC,MAAM,CAAC,0CAAE,OAAO,GAAG,QAAQ,EAAE,CAAC;gBAClD,IAAI,UAAU,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS;oBAAE,SAAS;gBAClE,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACnC,IAAI,MAAM,GAAG,SAAS,CAAC;gBAEvB,yCAAyC;gBACzC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC;oBAAE,SAAS;gBAE9D,IAAI,KAAK,GAAG,CAAC,CAAC;gBACd,KAAK,MAAM,OAAO,IAAI,IAAI,EAAE,CAAC;oBAC3B,IAAI,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBAC5C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;wBAC3B,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;oBACjD,CAAC;oBACD,IAAI,SAAS,KAAK,OAAO,EAAE,CAAC;wBAC1B,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC;wBACxB,sBAAsB;oBACxB,CAAC;yBAAM,IAAI,KAAK,KAAK,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACrC,QAAQ,CAAC,IAAI,GAAG,SAAS,CAAC;oBAC5B,CAAC;oBACD,MAAM,GAAG,QAAQ,CAAC;oBAClB,KAAK,IAAI,CAAC,CAAC;gBACb,CAAC;YACH,CAAC;YACD,IAAI,CAAC,oBAAoB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,QAAgB,EAChB,SAAiB;QAEjB,MAAM,SAAS,GAAmB;YAChC,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,WAAW;YACpB,IAAI,EAAE,QAAQ;YACd,SAAS;YACT,UAAU,EAAE,IAAI,CAAC,IAAI;YACrB,MAAM,EAAE,EAAE;SACX,CAAC;QACF,MAAM,IAAI,CAAC,mBAAmB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QACrD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,MAAoB;QAC1C,MAAM,SAAS,GAAG,EAAC,GAAG,MAAM,EAAE,MAAM,EAAE,EAAE,EAAC,CAAC;QAC1C,kDAAkD;QAClD,MAAM,aAAa,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,MAAM,CACT,+BAA+B,aAAa,QAAQ,MAAM,CAAC,SAAS,IAAI,CACzE,CAAC;QAEF,MAAM,IAAI,CAAC,mBAAmB,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QACzD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEM,KAAK,CAAC,sBAAsB,CAAC,UAAkB;QACpD,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QACnD,MAAM,GAAG,GAAG,gCAAgC,SAAS,QAAQ,UAAU,IAAI,CAAC;QAC5E,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACvB,OAAO,SAAS,CAAC;IACnB,CAAC;CACF;AAnQD,kDAmQC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@malloydata/db-snowflake",
|
|
3
|
-
"version": "0.0.195
|
|
3
|
+
"version": "0.0.195",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"prepublishOnly": "npm run build"
|
|
22
22
|
},
|
|
23
23
|
"dependencies": {
|
|
24
|
-
"@malloydata/malloy": "^0.0.195
|
|
24
|
+
"@malloydata/malloy": "^0.0.195",
|
|
25
25
|
"@types/snowflake-sdk": "^1.6.16",
|
|
26
26
|
"generic-pool": "^3.9.0",
|
|
27
27
|
"snowflake-sdk": "1.10.0",
|
|
@@ -23,7 +23,6 @@
|
|
|
23
23
|
|
|
24
24
|
import * as crypto from 'crypto';
|
|
25
25
|
import {
|
|
26
|
-
FetchSchemaOptions,
|
|
27
26
|
RunSQLOptions,
|
|
28
27
|
MalloyQueryData,
|
|
29
28
|
QueryRunStats,
|
|
@@ -31,13 +30,14 @@ import {
|
|
|
31
30
|
PersistSQLResults,
|
|
32
31
|
StreamingConnection,
|
|
33
32
|
PooledConnection,
|
|
34
|
-
|
|
33
|
+
SQLSourceDef,
|
|
34
|
+
TableSourceDef,
|
|
35
35
|
StructDef,
|
|
36
36
|
QueryDataRow,
|
|
37
37
|
SnowflakeDialect,
|
|
38
|
-
NamedStructDefs,
|
|
39
|
-
FieldTypeDef,
|
|
40
38
|
TestableConnection,
|
|
39
|
+
arrayEachFields,
|
|
40
|
+
LeafAtomicDef,
|
|
41
41
|
} from '@malloydata/malloy';
|
|
42
42
|
import {BaseConnection} from '@malloydata/malloy/connection';
|
|
43
43
|
|
|
@@ -88,20 +88,6 @@ export class SnowflakeConnection
|
|
|
88
88
|
{
|
|
89
89
|
private readonly dialect = new SnowflakeDialect();
|
|
90
90
|
private executor: SnowflakeExecutor;
|
|
91
|
-
private schemaCache = new Map<
|
|
92
|
-
string,
|
|
93
|
-
| {schema: StructDef; error?: undefined; timestamp: number}
|
|
94
|
-
| {error: string; schema?: undefined; timestamp: number}
|
|
95
|
-
>();
|
|
96
|
-
private sqlSchemaCache = new Map<
|
|
97
|
-
string,
|
|
98
|
-
| {
|
|
99
|
-
structDef: StructDef;
|
|
100
|
-
error?: undefined;
|
|
101
|
-
timestamp: number;
|
|
102
|
-
}
|
|
103
|
-
| {error: string; structDef?: undefined; timestamp: number}
|
|
104
|
-
>();
|
|
105
91
|
|
|
106
92
|
// the database & schema where we do temporary operations like creating a temp table
|
|
107
93
|
private scratchSpace?: namespace;
|
|
@@ -189,16 +175,6 @@ export class SnowflakeConnection
|
|
|
189
175
|
await this.executor.batch('SELECT 1 as one');
|
|
190
176
|
}
|
|
191
177
|
|
|
192
|
-
private variantToMalloyType(type: string): string {
|
|
193
|
-
if (type === 'integer') {
|
|
194
|
-
return 'number';
|
|
195
|
-
} else if (type === 'varchar') {
|
|
196
|
-
return 'string';
|
|
197
|
-
} else {
|
|
198
|
-
return 'sql native';
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
|
|
202
178
|
private addFieldsToStructDef(
|
|
203
179
|
structDef: StructDef,
|
|
204
180
|
structMap: StructMap
|
|
@@ -210,41 +186,42 @@ export class SnowflakeConnection
|
|
|
210
186
|
|
|
211
187
|
// check for an array
|
|
212
188
|
if (value.isArray && type !== 'object') {
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
type: 'nested',
|
|
222
|
-
fieldName: name,
|
|
223
|
-
isArray: true,
|
|
224
|
-
},
|
|
225
|
-
fields: [{type: malloyType, name: 'value'} as FieldTypeDef],
|
|
226
|
-
};
|
|
227
|
-
structDef.fields.push(innerStructDef);
|
|
228
|
-
}
|
|
229
|
-
} else if (type === 'object') {
|
|
189
|
+
// Apparently there can only be arrays of integers, strings, or unknowns?
|
|
190
|
+
// TODO is this true or is this just all that got implemented?
|
|
191
|
+
const malloyType: LeafAtomicDef =
|
|
192
|
+
type === 'integer'
|
|
193
|
+
? {type: 'number', numberType: 'integer'}
|
|
194
|
+
: type === 'varchar'
|
|
195
|
+
? {type: 'string'}
|
|
196
|
+
: {type: 'sql native', rawType: type};
|
|
230
197
|
const innerStructDef: StructDef = {
|
|
231
|
-
type: '
|
|
198
|
+
type: 'array',
|
|
232
199
|
name,
|
|
233
200
|
dialect: this.dialectName,
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
: {type: 'inline'},
|
|
238
|
-
fields: [],
|
|
201
|
+
join: 'many',
|
|
202
|
+
elementTypeDef: malloyType,
|
|
203
|
+
fields: arrayEachFields(malloyType),
|
|
239
204
|
};
|
|
205
|
+
structDef.fields.push(innerStructDef);
|
|
206
|
+
} else if (type === 'object') {
|
|
207
|
+
const structParts = {name, dialect: this.dialectName, fields: []};
|
|
208
|
+
const innerStructDef: StructDef = value.isArray
|
|
209
|
+
? {
|
|
210
|
+
...structParts,
|
|
211
|
+
type: 'array',
|
|
212
|
+
elementTypeDef: {type: 'record_element'},
|
|
213
|
+
join: 'many',
|
|
214
|
+
}
|
|
215
|
+
: {
|
|
216
|
+
...structParts,
|
|
217
|
+
type: 'record',
|
|
218
|
+
join: 'one',
|
|
219
|
+
};
|
|
240
220
|
this.addFieldsToStructDef(innerStructDef, value);
|
|
241
221
|
structDef.fields.push(innerStructDef);
|
|
242
222
|
} else {
|
|
243
|
-
const malloyType = this.dialect.sqlTypeToMalloyType(type)
|
|
244
|
-
|
|
245
|
-
rawType: type.toLowerCase(),
|
|
246
|
-
};
|
|
247
|
-
structDef.fields.push({name, ...malloyType} as FieldTypeDef);
|
|
223
|
+
const malloyType = this.dialect.sqlTypeToMalloyType(type);
|
|
224
|
+
structDef.fields.push({...malloyType, name});
|
|
248
225
|
}
|
|
249
226
|
}
|
|
250
227
|
}
|
|
@@ -326,118 +303,34 @@ export class SnowflakeConnection
|
|
|
326
303
|
}
|
|
327
304
|
}
|
|
328
305
|
|
|
329
|
-
|
|
306
|
+
async fetchTableSchema(
|
|
330
307
|
tableKey: string,
|
|
331
308
|
tablePath: string
|
|
332
|
-
): Promise<
|
|
333
|
-
const structDef:
|
|
334
|
-
type: '
|
|
309
|
+
): Promise<TableSourceDef> {
|
|
310
|
+
const structDef: TableSourceDef = {
|
|
311
|
+
type: 'table',
|
|
335
312
|
dialect: 'snowflake',
|
|
336
313
|
name: tableKey,
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
type: 'basetable',
|
|
340
|
-
connectionName: this.name,
|
|
341
|
-
},
|
|
314
|
+
tablePath,
|
|
315
|
+
connection: this.name,
|
|
342
316
|
fields: [],
|
|
343
317
|
};
|
|
344
318
|
await this.schemaFromTablePath(tablePath, structDef);
|
|
345
319
|
return structDef;
|
|
346
320
|
}
|
|
347
321
|
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
{refreshTimestamp}: FetchSchemaOptions
|
|
351
|
-
): Promise<{
|
|
352
|
-
schemas: Record<string, StructDef>;
|
|
353
|
-
errors: Record<string, string>;
|
|
354
|
-
}> {
|
|
355
|
-
const schemas: NamedStructDefs = {};
|
|
356
|
-
const errors: {[name: string]: string} = {};
|
|
357
|
-
|
|
358
|
-
for (const tableKey in missing) {
|
|
359
|
-
let inCache = this.schemaCache.get(tableKey);
|
|
360
|
-
if (
|
|
361
|
-
!inCache ||
|
|
362
|
-
(refreshTimestamp && refreshTimestamp > inCache.timestamp)
|
|
363
|
-
) {
|
|
364
|
-
const tablePath = missing[tableKey];
|
|
365
|
-
const timestamp = refreshTimestamp || Date.now();
|
|
366
|
-
try {
|
|
367
|
-
inCache = {
|
|
368
|
-
schema: await this.getTableSchema(tableKey, tablePath),
|
|
369
|
-
timestamp,
|
|
370
|
-
};
|
|
371
|
-
this.schemaCache.set(tableKey, inCache);
|
|
372
|
-
} catch (error) {
|
|
373
|
-
inCache = {error: error.message, timestamp};
|
|
374
|
-
}
|
|
375
|
-
}
|
|
376
|
-
if (inCache.schema !== undefined) {
|
|
377
|
-
schemas[tableKey] = inCache.schema;
|
|
378
|
-
} else {
|
|
379
|
-
errors[tableKey] = inCache.error || 'Unknown schema fetch error';
|
|
380
|
-
}
|
|
381
|
-
}
|
|
382
|
-
return {schemas, errors};
|
|
383
|
-
}
|
|
384
|
-
|
|
385
|
-
private async getSQLBlockSchema(sqlRef: SQLBlock): Promise<StructDef> {
|
|
386
|
-
const structDef: StructDef = {
|
|
387
|
-
type: 'struct',
|
|
388
|
-
dialect: 'snowflake',
|
|
389
|
-
name: sqlRef.name,
|
|
390
|
-
structSource: {
|
|
391
|
-
type: 'sql',
|
|
392
|
-
method: 'subquery',
|
|
393
|
-
sqlBlock: sqlRef,
|
|
394
|
-
},
|
|
395
|
-
structRelationship: {
|
|
396
|
-
type: 'basetable',
|
|
397
|
-
connectionName: this.name,
|
|
398
|
-
},
|
|
399
|
-
fields: [],
|
|
400
|
-
};
|
|
401
|
-
|
|
322
|
+
async fetchSelectSchema(sqlRef: SQLSourceDef): Promise<SQLSourceDef> {
|
|
323
|
+
const structDef = {...sqlRef, fields: []};
|
|
402
324
|
// create temp table with same schema as the query
|
|
403
325
|
const tempTableName = this.getTempViewName(sqlRef.selectStr);
|
|
404
326
|
this.runSQL(
|
|
405
|
-
`
|
|
406
|
-
CREATE OR REPLACE TEMP VIEW ${tempTableName} as ${sqlRef.selectStr};
|
|
407
|
-
`
|
|
327
|
+
`CREATE OR REPLACE TEMP VIEW ${tempTableName} AS (${sqlRef.selectStr});`
|
|
408
328
|
);
|
|
409
329
|
|
|
410
330
|
await this.schemaFromTablePath(tempTableName, structDef);
|
|
411
331
|
return structDef;
|
|
412
332
|
}
|
|
413
333
|
|
|
414
|
-
public async fetchSchemaForSQLBlock(
|
|
415
|
-
sqlRef: SQLBlock,
|
|
416
|
-
{refreshTimestamp}: FetchSchemaOptions
|
|
417
|
-
): Promise<
|
|
418
|
-
| {structDef: StructDef; error?: undefined}
|
|
419
|
-
| {error: string; structDef?: undefined}
|
|
420
|
-
> {
|
|
421
|
-
const key = sqlRef.name;
|
|
422
|
-
let inCache = this.sqlSchemaCache.get(key);
|
|
423
|
-
if (
|
|
424
|
-
!inCache ||
|
|
425
|
-
(refreshTimestamp && refreshTimestamp > inCache.timestamp)
|
|
426
|
-
) {
|
|
427
|
-
const timestamp = refreshTimestamp ?? Date.now();
|
|
428
|
-
try {
|
|
429
|
-
inCache = {
|
|
430
|
-
structDef: await this.getSQLBlockSchema(sqlRef),
|
|
431
|
-
timestamp,
|
|
432
|
-
};
|
|
433
|
-
} catch (error) {
|
|
434
|
-
inCache = {error: error.message, timestamp};
|
|
435
|
-
}
|
|
436
|
-
this.sqlSchemaCache.set(key, inCache);
|
|
437
|
-
}
|
|
438
|
-
return inCache;
|
|
439
|
-
}
|
|
440
|
-
|
|
441
334
|
public async manifestTemporaryTable(sqlCommand: string): Promise<string> {
|
|
442
335
|
const tableName = this.getTempViewName(sqlCommand);
|
|
443
336
|
const cmd = `CREATE OR REPLACE TEMP TABLE ${tableName} AS (${sqlCommand});`;
|