@malloydata/db-snowflake 0.0.127 → 0.0.128-dev240311005120

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/dist/snowflake_connection.d.ts +3 -1
  2. package/dist/snowflake_connection.js +125 -16
  3. package/dist/snowflake_connection.js.map +1 -1
  4. package/dist/snowflake_connection.spec.js +2 -2
  5. package/package.json +2 -2
  6. package/src/snowflake_connection.spec.ts +2 -2
  7. package/src/snowflake_connection.ts +133 -20
package/dist/snowflake_connection.d.ts CHANGED
@@ -32,7 +32,9 @@ export declare class SnowflakeConnection implements Connection, PersistSQLResult
32
32
  runSQL(sql: string, options?: RunSQLOptions): Promise<MalloyQueryData>;
33
33
  runSQLStream(sqlCommand: string, options?: RunSQLOptions): AsyncIterableIterator<QueryDataRow>;
34
34
  test(): Promise<void>;
35
- private schemaFromQuery;
35
+ private variantToMalloyType;
36
+ private addFieldsToStructDef;
37
+ private schemaFromTablePath;
36
38
  private getTableSchema;
37
39
  fetchSchemaForTables(missing: Record<string, string>, { refreshTimestamp }: FetchSchemaOptions): Promise<{
38
40
  schemas: Record<string, StructDef>;
package/dist/snowflake_connection.js CHANGED
@@ -49,6 +49,20 @@ exports.SnowflakeConnection = void 0;
49
49
  const crypto = __importStar(require("crypto"));
50
50
  const malloy_1 = require("@malloydata/malloy");
51
51
  const snowflake_executor_1 = require("./snowflake_executor");
52
+ class StructMap {
53
+ constructor(type, isArray) {
54
+ this.fieldMap = new Map();
55
+ this.type = 'record';
56
+ this.isArray = false;
57
+ this.type = type;
58
+ this.isArray = isArray;
59
+ }
60
+ addChild(name, type) {
61
+ const s = new StructMap(type, false);
62
+ this.fieldMap.set(name, s);
63
+ return s;
64
+ }
65
+ }
52
66
  class SnowflakeConnection {
53
67
  constructor(name, options) {
54
68
  var _a;
@@ -110,10 +124,74 @@ class SnowflakeConnection {
110
124
  }
111
125
  }
112
126
  async test() {
113
- await this.executor.batch('SELECT 1');
127
+ await this.executor.batch('SELECT 1 as one');
128
+ }
129
+ variantToMalloyType(type) {
130
+ if (type === 'integer') {
131
+ return 'number';
132
+ }
133
+ else if (type === 'varchar') {
134
+ return 'string';
135
+ }
136
+ else {
137
+ return 'unknown';
138
+ }
114
139
  }
115
- async schemaFromQuery(infoQuery, structDef) {
140
+ addFieldsToStructDef(structDef, structMap) {
141
+ var _a;
142
+ if (structMap.fieldMap.size === 0)
143
+ return;
144
+ for (const [field, value] of structMap.fieldMap) {
145
+ const type = value.type;
146
+ const name = field;
147
+ // check for an array
148
+ if (value.isArray && type !== 'object') {
149
+ const malloyType = this.variantToMalloyType(type);
150
+ if (malloyType) {
151
+ const innerStructDef = {
152
+ type: 'struct',
153
+ name,
154
+ dialect: this.dialectName,
155
+ structSource: { type: 'nested' },
156
+ structRelationship: {
157
+ type: 'nested',
158
+ fieldName: name,
159
+ isArray: true,
160
+ },
161
+ fields: [{ type: malloyType, name: 'value' }],
162
+ };
163
+ structDef.fields.push(innerStructDef);
164
+ }
165
+ }
166
+ else if (type === 'object') {
167
+ const innerStructDef = {
168
+ type: 'struct',
169
+ name,
170
+ dialect: this.dialectName,
171
+ structSource: value.isArray ? { type: 'nested' } : { type: 'inline' },
172
+ structRelationship: value.isArray
173
+ ? { type: 'nested', fieldName: name, isArray: false }
174
+ : { type: 'inline' },
175
+ fields: [],
176
+ };
177
+ this.addFieldsToStructDef(innerStructDef, value);
178
+ structDef.fields.push(innerStructDef);
179
+ }
180
+ else {
181
+ const malloyType = (_a = this.dialect.sqlTypeToMalloyType(type)) !== null && _a !== void 0 ? _a : {
182
+ type: 'unsupported',
183
+ rawType: type.toLowerCase(),
184
+ };
185
+ structDef.fields.push({ name, ...malloyType });
186
+ }
187
+ }
188
+ }
189
+ async schemaFromTablePath(tablePath, structDef) {
190
+ var _a, _b;
191
+ const infoQuery = `DESCRIBE TABLE ${tablePath}`;
116
192
  const rows = await this.executor.batch(infoQuery);
193
+ const variants = [];
194
+ const notVariant = new Map();
117
195
  for (const row of rows) {
118
196
  // data types look like `VARCHAR(1234)`
119
197
  let snowflakeDataType = row['type'];
@@ -121,6 +199,11 @@ class SnowflakeConnection {
121
199
  const s = structDef;
122
200
  const malloyType = this.dialect.sqlTypeToMalloyType(snowflakeDataType);
123
201
  const name = row['name'];
202
+ if (snowflakeDataType === 'variant' || snowflakeDataType === 'array') {
203
+ variants.push(name);
204
+ continue;
205
+ }
206
+ notVariant.set(name, true);
124
207
  if (malloyType) {
125
208
  s.fields.push({ ...malloyType, name });
126
209
  }
@@ -132,6 +215,44 @@ class SnowflakeConnection {
132
215
  });
133
216
  }
134
217
  }
218
+ // if we have variants, sample the data
219
+ if (variants.length > 0) {
220
+ const sampleQuery = `
221
+ SELECT regexp_replace(PATH, '\\\\[[0-9]*\\\\]', '') as PATH, lower(TYPEOF(value)) as type
222
+ FROM (select object_construct(*) o from ${tablePath} limit 100)
223
+ ,table(flatten(input => o, recursive => true)) as meta
224
+ GROUP BY 1,2
225
+ ORDER BY PATH;
226
+ `;
227
+ const fieldPathRows = await this.executor.batch(sampleQuery);
228
+ // take the schema in list form an convert it into a tree.
229
+ const structMap = new StructMap('object', true);
230
+ for (const f of fieldPathRows) {
231
+ const pathString = (_a = f['PATH']) === null || _a === void 0 ? void 0 : _a.valueOf().toString();
232
+ const fieldType = (_b = f['TYPE']) === null || _b === void 0 ? void 0 : _b.valueOf().toString();
233
+ if (pathString === undefined || fieldType === undefined)
234
+ continue;
235
+ const path = pathString.split('.');
236
+ let parent = structMap;
237
+ // ignore the fields we've already added.
238
+ if (path.length === 1 && notVariant.get(pathString))
239
+ continue;
240
+ for (const segment of path) {
241
+ let thisNode = parent.fieldMap.get(segment);
242
+ if (thisNode === undefined) {
243
+ thisNode = parent.addChild(segment, fieldType);
244
+ }
245
+ if (fieldType === 'array') {
246
+ thisNode.isArray = true;
247
+ }
248
+ else {
249
+ thisNode.type = fieldType;
250
+ }
251
+ parent = thisNode;
252
+ }
253
+ }
254
+ this.addFieldsToStructDef(structDef, structMap);
255
+ }
135
256
  }
136
257
  async getTableSchema(tableKey, tablePath) {
137
258
  const structDef = {
@@ -145,18 +266,7 @@ class SnowflakeConnection {
145
266
  },
146
267
  fields: [],
147
268
  };
148
- // This is how we get variant information
149
- // WITH tbl as (
150
- // SELECT * FROM malloytest.ga_sample
151
- // )
152
- // SELECT regexp_replace(PATH, '\\[.*\\]', '[]') as PATH, lower(TYPEOF(value)) as type
153
- // FROM (select object_construct(*) o from tbl limit 100)
154
- // ,table(flatten(input => o, recursive => true)) as meta
155
- // WHERE lower(TYPEOF(value)) <> 'array'
156
- // GROUP BY 1,2
157
- // ORDER BY PATH
158
- const infoQuery = `DESCRIBE TABLE ${tablePath}`;
159
- await this.schemaFromQuery(infoQuery, structDef);
269
+ await this.schemaFromTablePath(tablePath, structDef);
160
270
  return structDef;
161
271
  }
162
272
  async fetchSchemaForTables(missing, { refreshTimestamp }) {
@@ -209,8 +319,7 @@ class SnowflakeConnection {
209
319
  this.runSQL(`
210
320
  CREATE OR REPLACE TEMP VIEW ${tempTableName} as ${sqlRef.selectStr};
211
321
  `);
212
- const infoQuery = `DESCRIBE TABLE ${tempTableName}`;
213
- await this.schemaFromQuery(infoQuery, structDef);
322
+ await this.schemaFromTablePath(tempTableName, structDef);
214
323
  return structDef;
215
324
  }
216
325
  async fetchSchemaForSQLBlock(sqlRef, { refreshTimestamp }) {
package/dist/snowflake_connection.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"snowflake_connection.js","sourceRoot":"","sources":["../src/snowflake_connection.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,+CAAiC;AACjC,+CAa4B;AAC5B,6DAAuD;AAwBvD,MAAa,mBAAmB;IA4B9B,YACkB,IAAY,EAC5B,OAAoC;;QADpB,SAAI,GAAJ,IAAI,CAAQ;QAtBb,YAAO,GAAG,IAAI,yBAAgB,EAAE,CAAC;QAE1C,gBAAW,GAAG,IAAI,GAAG,EAI1B,CAAC;QACI,mBAAc,GAAG,IAAI,GAAG,EAQ7B,CAAC;QAUF,IAAI,WAAW,GAAG,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,WAAW,CAAC;QACvC,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,mEAAmE;YACnE,WAAW,GAAG,sCAAiB,CAAC,4BAA4B,EAAE,CAAC;QACjE,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,IAAI,sCAAiB,CAAC,WAAW,EAAE,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,WAAW,CAAC,CAAC;QACzE,IAAI,CAAC,YAAY,GAAG,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,YAAY,CAAC;QAC1C,IAAI,CAAC,YAAY,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,YAAY,mCAAI,EAAE,CAAC;IAClD,CAAC;IAED,IAAI,WAAW;QACb,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,qCAAqC;IACrC,IAAW,eAAe;QACxB,OAAO,KAAK,CAAC;IACf,CAAC;IAEM,MAAM;QACX,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,UAAU;QACf,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,SAAS;QACd,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,iBAAiB,CAAC,WAAmB;QAChD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC7B,CAAC;IAEO,eAAe,CAAC,UAAkB;QACxC,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,OAAO,KAAK,IAAI,EAAE,CAAC;IACrB,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,GAAW,EACX,OAAuB;;QAEvB,MAAM,QAAQ,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,QAAQ,mCAAI,MAAA,IAAI,CAAC,YAAY,0CAAE,QAAQ,CAAC;QAClE,IAAI,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC1C,IAAI,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YACrD,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,EAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,EAAC,CAAC;IACxC,CAAC;IAEM,KAAK,CAAC,CAAC,YAAY,CACxB,UAAkB,EAClB,UAAyB,EAAE;QAE3B,MAAM,kBAAkB,GAAG;YACzB,GAAG,IAAI,CAAC,YAAY;YACpB,GAAG,OAAO;SACX,CAAC;QAEF,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAChD,UAAU,EACV,kBAAkB,CACnB,EAAE,CAAC;YACF,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACxC,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,SAAiB,EACjB,SAAoB;QAEpB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAClD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,uCAAuC;YACvC,IAAI,iBAAiB,GAAG,GAAG,CAAC,MAAM,CAAW,CAAC;YAC9C,iBAAiB,GAAG,iBAAiB,CAAC,iBAAiB,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACxE,MAAM,CAAC,GAAG,SAAS,CAAC;YACpB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,iBAAiB,CAAC,CAAC;YACvE,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAW,CAAC;YACnC,IAAI,UAAU,EAAE,CAAC;gBACf,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAC,GAAG,UAAU,EAAE,IAAI,EAAC,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,aAAa;oBACnB,OAAO,EAAE,iBAAiB;oBAC1B,IAAI;iBACL,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,QAAgB,EAChB,SAAiB;QAEjB,MAAM,SAAS,GAAc;YAC3B,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,WAAW;YACpB,IAAI,EAAE,QAAQ;YACd,YAAY,EAAE,EAAC,IAAI,EAAE,OAAO,EAAE,SAAS,EAAC;YACxC,kBAAkB,EAAE;gBAClB,IAAI,EAAE,WAAW;gBACjB,cAAc,EAAE,IAAI,CAAC,IAAI;aAC1B;YACD,MAAM,EAAE,EAAE;SACX,CAAC;QACF,yCAAyC;QAEzC,gBAAgB;QAChB,uCAAuC;QACvC,KAAK;QACL,uFAAuF;QACvF,2DAA2D;QAC3D,8DAA8D;QAC9D,yCAAyC;QACzC,gBAAgB;QAChB,iBAAiB;QAEjB,MAAM,SAAS,GAAG,kBAAkB,SAAS,EAAE,CAAC;QAEhD,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QACjD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEM,KAAK,CAAC,oBAAoB,CAC/B,OAA+B,EAC/B,EAAC,gBAAgB,EAAqB;QAKtC,MAAM,OAAO,GAAoB,EAAE,CAAC;QACpC,MAAM,MAAM,GAA6B,EAAE,CAAC;QAE5C,KAAK,MAAM,QAAQ,IAAI,OAAO,EAAE,CAAC;YAC/B,IAAI,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC7C,IACE,CAAC,OAAO;gBACR,CAAC,gBAAgB,IAAI,gBAAgB,GAAG,OAAO,CAAC,SAAS,CAAC,EAC1D,CAAC;gBACD,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;gBACpC,MAAM,SAAS,GAAG,gBAAgB,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;gBACjD,IAAI,CAAC;oBACH,OAAO,GAAG;wBACR,MAAM,EAAE,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,SAAS,CAAC;wBACtD,SAAS;qBACV,CAAC;oBACF,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAC1C,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,OAAO,GAAG,EAAC,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,EAAC,CAAC;gBAC9C,CAAC;YACH,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBACjC,OAAO,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;YACrC,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC,KAAK,IAAI,4BAA4B,CAAC;YACnE,CAAC;QACH,CAAC;QACD,OAAO,EAAC,OAAO,EAAE,MAAM,EAAC,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,MAAgB;QAC9C,MAAM,SAAS,GAAc;YAC3B,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,WAAW;YACpB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,YAAY,EAAE;gBACZ,IAAI,EAAE,KAAK;gBACX,MAAM,EAAE,UAAU;gBAClB,QAAQ,EAAE,MAAM;aACjB;YACD,kBAAkB,EAAE;gBAClB,IAAI,EAAE,WAAW;gBACjB,cAAc,EAAE,IAAI,CAAC,IAAI;aAC1B;YACD,MAAM,EAAE,EAAE;SACX,CAAC;QAEF,kDAAkD;QAClD,MAAM,aAAa,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,MAAM,CACT;oCAC8B,aAAa,OAAO,MAAM,CAAC,SAAS;OACjE,CACF,CAAC;QAEF,MAAM,SAAS,GAAG,kBAAkB,aAAa,EAAE,CAAC;QACpD,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QACjD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEM,KAAK,CAAC,sBAAsB,CACjC,MAAgB,EAChB,EAAC,gBAAgB,EAAqB;QAKtC,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC;QACxB,IAAI,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC3C,IACE,CAAC,OAAO;YACR,CAAC,gBAAgB,IAAI,gBAAgB,GAAG,OAAO,CAAC,SAAS,CAAC,EAC1D,CAAC;YACD,MAAM,SAAS,GAAG,gBAAgB,aAAhB,gBAAgB,cAAhB,gBAAgB,GAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACjD,IAAI,CAAC;gBACH,OAAO,GAAG;oBACR,SAAS,EAAE,MAAM,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC;oBAC/C,SAAS;iBACV,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,GAAG,EAAC,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,EAAC,CAAC;YAC9C,CAAC;YACD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACxC,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAEM,KAAK,CAAC,sBAAsB,CAAC,UAAkB;QACpD,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QACnD,MAAM,GAAG,GAAG,gCAAgC,SAAS,QAAQ,UAAU,IAAI,CAAC;QAC5E,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACvB,OAAO,SAAS,CAAC;IACnB,CAAC;CACF;AA1QD,kDA0QC"}
1
+ {"version":3,"file":"snowflake_connection.js","sourceRoot":"","sources":["../src/snowflake_connection.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,+CAAiC;AACjC,+CAc4B;AAC5B,6DAAuD;AAwBvD,MAAM,SAAS;IAKb,YAAY,IAAY,EAAE,OAAgB;QAJ1C,aAAQ,GAAG,IAAI,GAAG,EAAqB,CAAC;QACxC,SAAI,GAAG,QAAQ,CAAC;QAChB,YAAO,GAAG,KAAK,CAAC;QAGd,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,QAAQ,CAAC,IAAY,EAAE,IAAY;QACjC,MAAM,CAAC,GAAG,IAAI,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACrC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3B,OAAO,CAAC,CAAC;IACX,CAAC;CACF;AAED,MAAa,mBAAmB;IA4B9B,YACkB,IAAY,EAC5B,OAAoC;;QADpB,SAAI,GAAJ,IAAI,CAAQ;QAtBb,YAAO,GAAG,IAAI,yBAAgB,EAAE,CAAC;QAE1C,gBAAW,GAAG,IAAI,GAAG,EAI1B,CAAC;QACI,mBAAc,GAAG,IAAI,GAAG,EAQ7B,CAAC;QAUF,IAAI,WAAW,GAAG,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,WAAW,CAAC;QACvC,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,mEAAmE;YACnE,WAAW,GAAG,sCAAiB,CAAC,4BAA4B,EAAE,CAAC;QACjE,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,IAAI,sCAAiB,CAAC,WAAW,EAAE,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,WAAW,CAAC,CAAC;QACzE,IAAI,CAAC,YAAY,GAAG,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,YAAY,CAAC;QAC1C,IAAI,CAAC,YAAY,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,YAAY,mCAAI,EAAE,CAAC;IAClD,CAAC;IAED,IAAI,WAAW;QACb,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,qCAAqC;IACrC,IAAW,eAAe;QACxB,OAAO,KAAK,CAAC;IACf,CAAC;IAEM,MAAM;QACX,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,UAAU;QACf,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,SAAS;QACd,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,iBAAiB,CAAC,WAAmB;QAChD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC7B,CAAC;IAEO,eAAe,CAAC,UAAkB;QACxC,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,OAAO,KAAK,IAAI,EAAE,CAAC;IACrB,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,GAAW,EACX,OAAuB;;QAEvB,MAAM,QAAQ,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,QAAQ,mCAAI,MAAA,IAAI,CAAC,YAAY,0CAAE,QAAQ,CAAC;QAClE,IAAI,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC1C,IAAI,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YACrD,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,EAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,EAAC,CAAC;IACxC,CAAC;IAEM,KAAK,CAAC,CAAC,YAAY,CACxB,UAAkB,EAClB,UAAyB,EAAE;QAE3B,MAAM,kBAAkB,GAAG;YACzB,GAAG,IAAI,CAAC,YAAY;YACpB,GAAG,OAAO;SACX,CAAC;QAEF,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAChD,UAAU,EACV,kBAAkB,CACnB,EAAE,CAAC;YACF,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;IAC/C,CAAC;IAEO,mBAAmB,CAAC,IAAY;QACtC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,OAAO,QAAQ,CAAC;QAClB,CAAC;aAAM,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YAC9B,OAAO,QAAQ,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAEO,oBAAoB,CAC1B,SAAoB,EACpB,SAAoB;;QAEpB,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO;QAC1C,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;YAChD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;YACxB,MAAM,IAAI,GAAG,KAAK,CAAC;YAEnB,qBAAqB;YACrB,IAAI,KAAK,CAAC,OAAO,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACvC,MAAM,UAAU,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;gBAClD,IAAI,UAAU,EAAE,CAAC;oBACf,MAAM,cAAc,GAAc;wBAChC,IAAI,EAAE,QAAQ;wBACd,IAAI;wBACJ,OAAO,EAAE,IAAI,CAAC,WAAW;wBACzB,YAAY,EAAE,EAAC,IAAI,EAAE,QAAQ,EAAC;wBAC9B,kBAAkB,EAAE;4BAClB,IAAI,EAAE,QAAQ;4BACd,SAAS,EAAE,IAAI;4BACf,OAAO,EAAE,IAAI;yBACd;wBACD,MAAM,EAAE,CAAC,EAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAiB,CAAC;qBAC5D,CAAC;oBACF,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBACxC,CAAC;YACH,CAAC;iBAAM,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,MAAM,cAAc,GAAc;oBAChC,IAAI,EAAE,QAAQ;oBACd,IAAI;oBACJ,OAAO,EAAE,IAAI,CAAC,WAAW;oBACzB,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAC,IAAI,EAAE,QAAQ,EAAC,CAAC,CAAC,CAAC,EAAC,IAAI,EAAE,QAAQ,EAAC;oBACjE,kBAAkB,EAAE,KAAK,CAAC,OAAO;wBAC/B,CAAC,CAAC,EAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAC;wBACnD,CAAC,CAAC,EAAC,IAAI,EAAE,QAAQ,EAAC;oBACpB,MAAM,EAAE,EAAE;iBACX,CAAC;gBACF,IAAI,CAAC,oBAAoB,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;gBACjD,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACN,MAAM,UAAU,GAAG,MAAA,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,IAAI,CAAC,mCAAI;oBAC3D,IAAI,EAAE,aAAa;oBACnB,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE;iBAC5B,CAAC;gBACF,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAC,IAAI,EAAE,GAAG,UAAU,EAAiB,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC/B,SAAiB,EACjB,SAAoB;;QAEpB,MAAM,SAAS,GAAG,kBAAkB,SAAS,EAAE,CAAC;QAChD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAClD,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,UAAU,GAAG,IAAI,GAAG,EAAmB,CAAC;QAC9C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,uCAAuC;YACvC,IAAI,iBAAiB,GAAG,GAAG,CAAC,MAAM,CAAW,CAAC;YAC9C,iBAAiB,GAAG,iBAAiB,CAAC,iBAAiB,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACxE,MAAM,CAAC,GAAG,SAAS,CAAC;YACpB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,iBAAiB,CAAC,CAAC;YACvE,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAW,CAAC;YAEnC,IAAI,iBAAiB,KAAK,SAAS,IAAI,iBAAiB,KAAK,OAAO,EAAE,CAAC;gBACrE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpB,SAAS;YACX,CAAC;YAED,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC3B,IAAI,UAAU,EAAE,CAAC;gBACf,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAC,GAAG,UAAU,EAAE,IAAI,EAAC,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,aAAa;oBACnB,OAAO,EAAE,iBAAiB;oBAC1B,IAAI;iBACL,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,uCAAuC;QACvC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,WAAW,GAAG;;mDAEyB,SAAS;;;;OAIrD,CAAC;YACF,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAE7D,0DAA0D;YAE1D,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAEhD,KAAK,MAAM,CAAC,IAAI,aAAa,EAAE,CAAC;gBAC9B,MAAM,UAAU,GAAG,MAAA,CAAC,CAAC,MAAM,CAAC,0CAAE,OAAO,GAAG,QAAQ,EAAE,CAAC;gBACnD,MAAM,SAAS,GAAG,MAAA,CAAC,CAAC,MAAM,CAAC,0CAAE,OAAO,GAAG,QAAQ,EAAE,CAAC;gBAClD,IAAI,UAAU,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS;oBAAE,SAAS;gBAClE,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACnC,IAAI,MAAM,GAAG,SAAS,CAAC;gBAEvB,yCAAyC;gBACzC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC;oBAAE,SAAS;gBAE9D,KAAK,MAAM,OAAO,IAAI,IAAI,EAAE,CAAC;oBAC3B,IAAI,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBAC5C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;wBAC3B,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;oBACjD,CAAC;oBACD,IAAI,SAAS,KAAK,OAAO,EAAE,CAAC;wBAC1B,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC;oBAC1B,CAAC;yBAAM,CAAC;wBACN,QAAQ,CAAC,IAAI,GAAG,SAAS,CAAC;oBAC5B,CAAC;oBACD,MAAM,GAAG,QAAQ,CAAC;gBACpB,CAAC;YACH,CAAC;YACD,IAAI,CAAC,oBAAoB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,QAAgB,EAChB,SAAiB;QAEjB,MAAM,SAAS,GAAc;YAC3B,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,WAAW;YACpB,IAAI,EAAE,QAAQ;YACd,YAAY,EAAE,EAAC,IAAI,EAAE,OAAO,EAAE,SAAS,EAAC;YACxC,kBAAkB,EAAE;gBAClB,IAAI,EAAE,WAAW;gBACjB,cAAc,EAAE,IAAI,CAAC,IAAI;aAC1B;YACD,MAAM,EAAE,EAAE;SACX,CAAC;QACF,MAAM,IAAI,CAAC,mBAAmB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QACrD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEM,KAAK,CAAC,oBAAoB,CAC/B,OAA+B,EAC/B,EAAC,gBAAgB,EAAqB;QAKtC,MAAM,OAAO,GAAoB,EAAE,CAAC;QACpC,MAAM,MAAM,GAA6B,EAAE,CAAC;QAE5C,KAAK,MAAM,QAAQ,IAAI,OAAO,EAAE,CAAC;YAC/B,IAAI,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC7C,IACE,CAAC,OAAO;gBACR,CAAC,gBAAgB,IAAI,gBAAgB,GAAG,OAAO,CAAC,SAAS,CAAC,EAC1D,CAAC;gBACD,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;gBACpC,MAAM,SAAS,GAAG,gBAAgB,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;gBACjD,IAAI,CAAC;oBACH,OAAO,GAAG;wBACR,MAAM,EAAE,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,SAAS,CAAC;wBACtD,SAAS;qBACV,CAAC;oBACF,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAC1C,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,OAAO,GAAG,EAAC,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,EAAC,CAAC;gBAC9C,CAAC;YACH,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBACjC,OAAO,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;YACrC,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC,KAAK,IAAI,4BAA4B,CAAC;YACnE,CAAC;QACH,CAAC;QACD,OAAO,EAAC,OAAO,EAAE,MAAM,EAAC,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,MAAgB;QAC9C,MAAM,SAAS,GAAc;YAC3B,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,WAAW;YACpB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,YAAY,EAAE;gBACZ,IAAI,EAAE,KAAK;gBACX,MAAM,EAAE,UAAU;gBAClB,QAAQ,EAAE,MAAM;aACjB;YACD,kBAAkB,EAAE;gBAClB,IAAI,EAAE,WAAW;gBACjB,cAAc,EAAE,IAAI,CAAC,IAAI;aAC1B;YACD,MAAM,EAAE,EAAE;SACX,CAAC;QAEF,kDAAkD;QAClD,MAAM,aAAa,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,MAAM,CACT;oCAC8B,aAAa,OAAO,MAAM,CAAC,SAAS;OACjE,CACF,CAAC;QAEF,MAAM,IAAI,CAAC,mBAAmB,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QACzD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEM,KAAK,CAAC,sBAAsB,CACjC,MAAgB,EAChB,EAAC,gBAAgB,EAAqB;QAKtC,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC;QACxB,IAAI,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC3C,IACE,CAAC,OAAO;YACR,CAAC,gBAAgB,IAAI,gBAAgB,GAAG,OAAO,CAAC,SAAS,CAAC,EAC1D,CAAC;YACD,MAAM,SAAS,GAAG,gBAAgB,aAAhB,gBAAgB,cAAhB,gBAAgB,GAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACjD,IAAI,CAAC;gBACH,OAAO,GAAG;oBACR,SAAS,EAAE,MAAM,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC;oBAC/C,SAAS;iBACV,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,GAAG,EAAC,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,EAAC,CAAC;YAC9C,CAAC;YACD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACxC,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAEM,KAAK,CAAC,sBAAsB,CAAC,UAAkB;QACpD,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QACnD,MAAM,GAAG,GAAG,gCAAgC,SAAS,QAAQ,UAAU,IAAI,CAAC;QAC5E,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACvB,OAAO,SAAS,CAAC;IACnB,CAAC;CACF;AAzWD,kDAyWC"}
package/dist/snowflake_connection.spec.js CHANGED
@@ -80,8 +80,8 @@ describe('db:Snowflake', () => {
80
80
  await conn.close();
81
81
  });
82
82
  it('runs a SQL query', async () => {
83
- const res = await conn.runSQL('SELECT 1 as t');
84
- expect(res.rows[0]['t']).toBe(1);
83
+ const res = await conn.runSQL('SELECT 1 as T');
84
+ expect(res.rows[0]['T']).toBe(1);
85
85
  });
86
86
  it('runs a Malloy query', async () => {
87
87
  var _a;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@malloydata/db-snowflake",
3
- "version": "0.0.127",
3
+ "version": "0.0.128-dev240311005120",
4
4
  "license": "MIT",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
@@ -21,7 +21,7 @@
21
21
  "prepublishOnly": "npm run build"
22
22
  },
23
23
  "dependencies": {
24
- "@malloydata/malloy": "^0.0.127",
24
+ "@malloydata/malloy": "^0.0.128-dev240311005120",
25
25
  "@types/snowflake-sdk": "^1.6.16",
26
26
  "generic-pool": "^3.9.0",
27
27
  "snowflake-sdk": "^1.9.0",
package/src/snowflake_connection.spec.ts CHANGED
@@ -65,8 +65,8 @@ describe('db:Snowflake', () => {
65
65
  });
66
66
 
67
67
  it('runs a SQL query', async () => {
68
- const res = await conn.runSQL('SELECT 1 as t');
69
- expect(res.rows[0]['t']).toBe(1);
68
+ const res = await conn.runSQL('SELECT 1 as T');
69
+ expect(res.rows[0]['T']).toBe(1);
70
70
  });
71
71
 
72
72
  it('runs a Malloy query', async () => {
package/src/snowflake_connection.ts CHANGED
@@ -35,6 +35,7 @@ import {
35
35
  QueryDataRow,
36
36
  SnowflakeDialect,
37
37
  NamedStructDefs,
38
+ FieldTypeDef,
38
39
  } from '@malloydata/malloy';
39
40
  import {SnowflakeExecutor} from './snowflake_executor';
40
41
  import {
@@ -60,6 +61,23 @@ export interface SnowflakeConnectionOptions {
60
61
  queryOptions?: RunSQLOptions;
61
62
  }
62
63
 
64
+ class StructMap {
65
+ fieldMap = new Map<string, StructMap>();
66
+ type = 'record';
67
+ isArray = false;
68
+
69
+ constructor(type: string, isArray: boolean) {
70
+ this.type = type;
71
+ this.isArray = isArray;
72
+ }
73
+
74
+ addChild(name: string, type: string): StructMap {
75
+ const s = new StructMap(type, false);
76
+ this.fieldMap.set(name, s);
77
+ return s;
78
+ }
79
+ }
80
+
63
81
  export class SnowflakeConnection
64
82
  implements
65
83
  Connection,
@@ -166,14 +184,77 @@ export class SnowflakeConnection
166
184
  }
167
185
 
168
186
  public async test(): Promise<void> {
169
- await this.executor.batch('SELECT 1');
187
+ await this.executor.batch('SELECT 1 as one');
188
+ }
189
+
190
+ private variantToMalloyType(type: string): string {
191
+ if (type === 'integer') {
192
+ return 'number';
193
+ } else if (type === 'varchar') {
194
+ return 'string';
195
+ } else {
196
+ return 'unknown';
197
+ }
198
+ }
199
+
200
+ private addFieldsToStructDef(
201
+ structDef: StructDef,
202
+ structMap: StructMap
203
+ ): void {
204
+ if (structMap.fieldMap.size === 0) return;
205
+ for (const [field, value] of structMap.fieldMap) {
206
+ const type = value.type;
207
+ const name = field;
208
+
209
+ // check for an array
210
+ if (value.isArray && type !== 'object') {
211
+ const malloyType = this.variantToMalloyType(type);
212
+ if (malloyType) {
213
+ const innerStructDef: StructDef = {
214
+ type: 'struct',
215
+ name,
216
+ dialect: this.dialectName,
217
+ structSource: {type: 'nested'},
218
+ structRelationship: {
219
+ type: 'nested',
220
+ fieldName: name,
221
+ isArray: true,
222
+ },
223
+ fields: [{type: malloyType, name: 'value'} as FieldTypeDef],
224
+ };
225
+ structDef.fields.push(innerStructDef);
226
+ }
227
+ } else if (type === 'object') {
228
+ const innerStructDef: StructDef = {
229
+ type: 'struct',
230
+ name,
231
+ dialect: this.dialectName,
232
+ structSource: value.isArray ? {type: 'nested'} : {type: 'inline'},
233
+ structRelationship: value.isArray
234
+ ? {type: 'nested', fieldName: name, isArray: false}
235
+ : {type: 'inline'},
236
+ fields: [],
237
+ };
238
+ this.addFieldsToStructDef(innerStructDef, value);
239
+ structDef.fields.push(innerStructDef);
240
+ } else {
241
+ const malloyType = this.dialect.sqlTypeToMalloyType(type) ?? {
242
+ type: 'unsupported',
243
+ rawType: type.toLowerCase(),
244
+ };
245
+ structDef.fields.push({name, ...malloyType} as FieldTypeDef);
246
+ }
247
+ }
170
248
  }
171
249
 
172
- private async schemaFromQuery(
173
- infoQuery: string,
250
+ private async schemaFromTablePath(
251
+ tablePath: string,
174
252
  structDef: StructDef
175
253
  ): Promise<void> {
254
+ const infoQuery = `DESCRIBE TABLE ${tablePath}`;
176
255
  const rows = await this.executor.batch(infoQuery);
256
+ const variants: string[] = [];
257
+ const notVariant = new Map<string, boolean>();
177
258
  for (const row of rows) {
178
259
  // data types look like `VARCHAR(1234)`
179
260
  let snowflakeDataType = row['type'] as string;
@@ -181,6 +262,13 @@ export class SnowflakeConnection
181
262
  const s = structDef;
182
263
  const malloyType = this.dialect.sqlTypeToMalloyType(snowflakeDataType);
183
264
  const name = row['name'] as string;
265
+
266
+ if (snowflakeDataType === 'variant' || snowflakeDataType === 'array') {
267
+ variants.push(name);
268
+ continue;
269
+ }
270
+
271
+ notVariant.set(name, true);
184
272
  if (malloyType) {
185
273
  s.fields.push({...malloyType, name});
186
274
  } else {
@@ -191,6 +279,46 @@ export class SnowflakeConnection
191
279
  });
192
280
  }
193
281
  }
282
+ // if we have variants, sample the data
283
+ if (variants.length > 0) {
284
+ const sampleQuery = `
285
+ SELECT regexp_replace(PATH, '\\\\[[0-9]*\\\\]', '') as PATH, lower(TYPEOF(value)) as type
286
+ FROM (select object_construct(*) o from ${tablePath} limit 100)
287
+ ,table(flatten(input => o, recursive => true)) as meta
288
+ GROUP BY 1,2
289
+ ORDER BY PATH;
290
+ `;
291
+ const fieldPathRows = await this.executor.batch(sampleQuery);
292
+
293
+ // take the schema in list form an convert it into a tree.
294
+
295
+ const structMap = new StructMap('object', true);
296
+
297
+ for (const f of fieldPathRows) {
298
+ const pathString = f['PATH']?.valueOf().toString();
299
+ const fieldType = f['TYPE']?.valueOf().toString();
300
+ if (pathString === undefined || fieldType === undefined) continue;
301
+ const path = pathString.split('.');
302
+ let parent = structMap;
303
+
304
+ // ignore the fields we've already added.
305
+ if (path.length === 1 && notVariant.get(pathString)) continue;
306
+
307
+ for (const segment of path) {
308
+ let thisNode = parent.fieldMap.get(segment);
309
+ if (thisNode === undefined) {
310
+ thisNode = parent.addChild(segment, fieldType);
311
+ }
312
+ if (fieldType === 'array') {
313
+ thisNode.isArray = true;
314
+ } else {
315
+ thisNode.type = fieldType;
316
+ }
317
+ parent = thisNode;
318
+ }
319
+ }
320
+ this.addFieldsToStructDef(structDef, structMap);
321
+ }
194
322
  }
195
323
 
196
324
  private async getTableSchema(
@@ -208,21 +336,7 @@ export class SnowflakeConnection
208
336
  },
209
337
  fields: [],
210
338
  };
211
- // This is how we get variant information
212
-
213
- // WITH tbl as (
214
- // SELECT * FROM malloytest.ga_sample
215
- // )
216
- // SELECT regexp_replace(PATH, '\\[.*\\]', '[]') as PATH, lower(TYPEOF(value)) as type
217
- // FROM (select object_construct(*) o from tbl limit 100)
218
- // ,table(flatten(input => o, recursive => true)) as meta
219
- // WHERE lower(TYPEOF(value)) <> 'array'
220
- // GROUP BY 1,2
221
- // ORDER BY PATH
222
-
223
- const infoQuery = `DESCRIBE TABLE ${tablePath}`;
224
-
225
- await this.schemaFromQuery(infoQuery, structDef);
339
+ await this.schemaFromTablePath(tablePath, structDef);
226
340
  return structDef;
227
341
  }
228
342
 
@@ -288,8 +402,7 @@ export class SnowflakeConnection
288
402
  `
289
403
  );
290
404
 
291
- const infoQuery = `DESCRIBE TABLE ${tempTableName}`;
292
- await this.schemaFromQuery(infoQuery, structDef);
405
+ await this.schemaFromTablePath(tempTableName, structDef);
293
406
  return structDef;
294
407
  }
295
408