@malloydata/db-bigquery 0.0.195-dev241003204905 → 0.0.195

package/dist/bigquery.spec.js

@@ -212,17 +212,17 @@ describe('db:BigQuery', () => {
             expect(getTableFieldSchema).toBeCalledTimes(2);
         });
         it('caches sql schema', async () => {
-            await bq.fetchSchemaForSQLBlock(SQL_BLOCK_1, {});
+            await bq.fetchSchemaForSQLStruct(SQL_BLOCK_1, {});
             expect(getSQLBlockSchema).toBeCalledTimes(1);
             await new Promise(resolve => setTimeout(resolve));
-            await bq.fetchSchemaForSQLBlock(SQL_BLOCK_1, {});
+            await bq.fetchSchemaForSQLStruct(SQL_BLOCK_1, {});
             expect(getSQLBlockSchema).toBeCalledTimes(1);
         });
         it('refreshes sql schema', async () => {
-            await bq.fetchSchemaForSQLBlock(SQL_BLOCK_2, {});
+            await bq.fetchSchemaForSQLStruct(SQL_BLOCK_2, {});
             expect(getSQLBlockSchema).toBeCalledTimes(1);
             await new Promise(resolve => setTimeout(resolve));
-            await bq.fetchSchemaForSQLBlock(SQL_BLOCK_2, {
+            await bq.fetchSchemaForSQLStruct(SQL_BLOCK_2, {
                 refreshTimestamp: Date.now() + 10,
             });
             expect(getSQLBlockSchema).toBeCalledTimes(2);
@@ -230,8 +230,11 @@ describe('db:BigQuery', () => {
     });
 });
 const SQL_BLOCK_1 = {
-    type: 'sqlBlock',
+    type: 'sql_select',
     name: 'block1',
+    dialect: 'standardsql',
+    connection: 'bigquery',
+    fields: [],
     selectStr: `
 SELECT
 created_at,
@@ -247,8 +250,11 @@ FROM "inventory_items.parquet"
 `,
 };
 const SQL_BLOCK_2 = {
-    type: 'sqlBlock',
+    type: 'sql_select',
     name: 'block2',
+    dialect: 'standardsql',
+    connection: 'bigquery',
+    fields: [],
     selectStr: `
 SELECT
 created_at,

package/dist/bigquery_connection.d.ts

@@ -1,7 +1,7 @@
 import { RowMetadata } from '@google-cloud/bigquery';
 import bigquery from '@google-cloud/bigquery/build/src/types';
 import { ResourceStream } from '@google-cloud/paginator';
-import { Connection, ConnectionConfig, FetchSchemaOptions, MalloyQueryData, PersistSQLResults, QueryData, QueryDataRow, QueryOptionsReader, QueryRunStats, RunSQLOptions, SQLBlock, StreamingConnection, StructDef } from '@malloydata/malloy';
+import { Connection, ConnectionConfig, MalloyQueryData, PersistSQLResults, QueryData, QueryDataRow, QueryOptionsReader, QueryRunStats, RunSQLOptions, StreamingConnection, TableSourceDef, SQLSourceDef } from '@malloydata/malloy';
 import { BaseConnection, TableMetadata } from '@malloydata/malloy/connection';
 export interface BigQueryManagerOptions {
     credentials?: {
@@ -54,8 +54,6 @@ export declare class BigQueryConnection extends BaseConnection implements Connec
     private billingProjectId;
     private temporaryTables;
     private projectId;
-    private schemaCache;
-    private sqlSchemaCache;
     private queryOptions?;
     private config;
     private location;
@@ -68,9 +66,9 @@ export declare class BigQueryConnection extends BaseConnection implements Connec
     get supportsNesting(): boolean;
     private _runSQL;
     runSQL(sqlCommand: string, options?: RunSQLOptions, rowIndex?: number): Promise<MalloyQueryData>;
-    runSQLBlockAndFetchResultSchema(sqlBlock: SQLBlock, options?: RunSQLOptions): Promise<{
+    runSQLBlockAndFetchResultSchema(sqlBlock: SQLSourceDef, options?: RunSQLOptions): Promise<{
         data: MalloyQueryData;
-        schema: StructDef;
+        schema: SQLSourceDef;
     }>;
     downloadMalloyQuery(sqlCommand: string): Promise<ResourceStream<RowMetadata>>;
     private dryRunSQLQuery;
@@ -82,20 +80,9 @@ export declare class BigQueryConnection extends BaseConnection implements Connec
     manifestTemporaryTable(sqlCommand: string): Promise<string>;
     manifestPermanentTable(sqlCommand: string, datasetName: string, tableName: string, overwriteExistingTable?: boolean, createDataset?: boolean): Promise<string>;
     private addFieldsToStructDef;
-    private structDefFromTableSchema;
-    private structDefFromSQLSchema;
-    fetchSchemaForTables(missing: Record<string, string>, { refreshTimestamp }: FetchSchemaOptions): Promise<{
-        schemas: Record<string, StructDef>;
-        errors: Record<string, string>;
-    }>;
+    fetchSelectSchema(sqlSource: SQLSourceDef): Promise<SQLSourceDef | string>;
+    fetchTableSchema(tableName: string, tablePath: string): Promise<TableSourceDef | string>;
     private getSQLBlockSchema;
-    fetchSchemaForSQLBlock(sqlRef: SQLBlock, { refreshTimestamp }: FetchSchemaOptions): Promise<{
-        structDef: StructDef;
-        error?: undefined;
-    } | {
-        error: string;
-        structDef?: undefined;
-    }>;
     private createBigQueryJobAndGetResults;
     private createBigQueryJob;
     initiateJobAndGetLinkToConsole(sqlCommand: string, dryRun?: boolean): Promise<string>;

package/dist/bigquery_connection.js

@@ -93,8 +93,6 @@ class BigQueryConnection extends connection_1.BaseConnection {
         super();
         this.dialect = new malloy_1.StandardSQLDialect();
         this.temporaryTables = new Map();
-        this.schemaCache = new Map();
-        this.sqlSchemaCache = new Map();
         if (typeof arg === 'string') {
             this.name = arg;
         }
@@ -188,7 +186,8 @@ class BigQueryConnection extends connection_1.BaseConnection {
         if (schemaRaw === undefined) {
             throw new Error('Schema not present');
         }
-        const schema = this.structDefFromSQLSchema(sqlBlock, schemaRaw);
+        const schema = { ...sqlBlock, fields: [] };
+        this.addFieldsToStructDef(schema, schemaRaw);
         return { data, schema };
     }
     async downloadMalloyQuery(sqlCommand) {
@@ -357,38 +356,37 @@ class BigQueryConnection extends connection_1.BaseConnection {
             // BigQuery SDK sets type & name to optional even though they are required, assume they exist
             const type = field.type;
             const name = field.name;
-            // check for an array
-            if (field.mode === 'REPEATED' && !['STRUCT', 'RECORD'].includes(type)) {
+            const isRecord = ['STRUCT', 'RECORD'].includes(type);
+            const structShared = { name, dialect: this.dialectName, fields: [] };
+            if (field.mode === 'REPEATED' && !isRecord) {
+                // Malloy treats repeated values as an array of scalars.
                 const malloyType = this.dialect.sqlTypeToMalloyType(type);
                 if (malloyType) {
-                    const innerStructDef = {
-                        type: 'struct',
-                        name,
-                        dialect: this.dialectName,
-                        structSource: { type: 'nested' },
-                        structRelationship: {
-                            type: 'nested',
-                            fieldName: name,
-                            isArray: true,
-                        },
-                        fields: [{ ...malloyType, name: 'value' }],
+                    const arrayField = {
+                        ...structShared,
+                        type: 'array',
+                        elementTypeDef: malloyType,
+                        join: 'many',
+                        fields: (0, malloy_1.arrayEachFields)(malloyType),
                     };
-                    structDef.fields.push(innerStructDef);
+                    structDef.fields.push(arrayField);
                 }
             }
-            else if (['STRUCT', 'RECORD'].includes(type)) {
-                const innerStructDef = {
-                    type: 'struct',
-                    name,
-                    dialect: this.dialectName,
-                    structSource: field.mode === 'REPEATED' ? { type: 'nested' } : { type: 'inline' },
-                    structRelationship: field.mode === 'REPEATED'
-                        ? { type: 'nested', fieldName: name, isArray: false }
-                        : { type: 'inline' },
-                    fields: [],
+            else if (isRecord) {
+                const ifRepeatedRecord = {
+                    ...structShared,
+                    type: 'array',
+                    elementTypeDef: { type: 'record_element' },
+                    join: 'many',
                 };
-                this.addFieldsToStructDef(innerStructDef, field);
-                structDef.fields.push(innerStructDef);
+                const elseRecord = {
+                    ...structShared,
+                    type: 'record',
+                    join: 'one',
+                };
+                const recStruct = field.mode === 'REPEATED' ? ifRepeatedRecord : elseRecord;
+                this.addFieldsToStructDef(recStruct, field);
+                structDef.fields.push(recStruct);
             }
             else {
                 const malloyType = (_a = this.dialect.sqlTypeToMalloyType(type)) !== null && _a !== void 0 ? _a : {
@@ -399,90 +397,53 @@ class BigQueryConnection extends connection_1.BaseConnection {
             }
         }
     }
-    structDefFromTableSchema(tableKey, tablePath, schemaInfo) {
-        const structDef = {
-            type: 'struct',
-            name: tableKey,
-            dialect: this.dialectName,
-            structSource: {
-                type: 'table',
-                tablePath,
-            },
-            structRelationship: {
-                type: 'basetable',
-                connectionName: this.name,
-            },
-            fields: [],
-        };
-        this.addFieldsToStructDef(structDef, schemaInfo.schema);
-        if (schemaInfo.needsTableSuffixPseudoColumn) {
-            structDef.fields.push({
-                type: 'string',
-                name: '_TABLE_SUFFIX',
-            });
+    async fetchSelectSchema(sqlSource) {
+        try {
+            const ret = { ...sqlSource };
+            ret.fields = [];
+            this.addFieldsToStructDef(ret, await this.getSQLBlockSchema(sqlSource));
+            return ret;
         }
-        if (schemaInfo.needsPartitionTimePseudoColumn) {
-            structDef.fields.push({
-                type: 'timestamp',
-                name: '_PARTITIONTIME',
-            });
+        catch (error) {
+            return error.message;
         }
-        if (schemaInfo.needsPartitionDatePseudoColumn) {
-            structDef.fields.push({
-                type: 'date',
-                name: '_PARTITIONDATE',
-            });
-        }
-        return structDef;
-    }
-    structDefFromSQLSchema(sqlBlock, tableFieldSchema) {
-        const structDef = {
-            type: 'struct',
-            name: sqlBlock.name,
-            dialect: this.dialectName,
-            structSource: {
-                type: 'sql',
-                method: 'subquery',
-                sqlBlock,
-            },
-            structRelationship: {
-                type: 'basetable',
-                connectionName: this.name,
-            },
-            fields: [],
-        };
-        this.addFieldsToStructDef(structDef, tableFieldSchema);
-        return structDef;
-    }
-    async fetchSchemaForTables(missing, { refreshTimestamp }) {
-        const schemas = {};
-        const errors = {};
-        for (const tableKey in missing) {
-            let inCache = this.schemaCache.get(tableKey);
-            if (!inCache ||
-                (refreshTimestamp && refreshTimestamp > inCache.timestamp)) {
-                const timestamp = refreshTimestamp !== null && refreshTimestamp !== void 0 ? refreshTimestamp : Date.now();
-                const tablePath = this.normalizeTablePath(missing[tableKey]);
-                try {
-                    const tableFieldSchema = await this.getTableFieldSchema(tablePath);
-                    inCache = {
-                        schema: this.structDefFromTableSchema(tableKey, tablePath, tableFieldSchema),
-                        timestamp,
-                    };
-                    this.schemaCache.set(tableKey, inCache);
-                }
-                catch (error) {
-                    inCache = { error: error.message, timestamp };
-                }
+    }
+    async fetchTableSchema(tableName, tablePath) {
+        tablePath = this.normalizeTablePath(tablePath);
+        try {
+            const tableFieldSchema = await this.getTableFieldSchema(tablePath);
+            const tableDef = {
+                type: 'table',
+                name: tableName,
+                dialect: this.dialectName,
+                tablePath,
+                connection: this.name,
+                fields: [],
+            };
+            this.addFieldsToStructDef(tableDef, tableFieldSchema.schema);
+            if (tableFieldSchema.needsTableSuffixPseudoColumn) {
+                tableDef.fields.push({
+                    type: 'string',
+                    name: '_TABLE_SUFFIX',
+                });
             }
-            if (inCache.schema !== undefined) {
-                schemas[tableKey] = inCache.schema;
+            if (tableFieldSchema.needsPartitionTimePseudoColumn) {
+                tableDef.fields.push({
+                    type: 'timestamp',
+                    name: '_PARTITIONTIME',
+                });
             }
-            else {
-                errors[tableKey] = inCache.error || 'Unknown schema fetch error';
+            if (tableFieldSchema.needsPartitionDatePseudoColumn) {
+                tableDef.fields.push({
+                    type: 'date',
+                    name: '_PARTITIONDATE',
+                });
             }
+            return tableDef;
+        }
+        catch (error) {
+            return error.message;
         }
-        return { schemas, errors };
     }
     async getSQLBlockSchema(sqlRef) {
         // We do a simple retry-loop here, as a temporary fix for a transient
@@ -506,26 +467,6 @@ class BigQueryConnection extends connection_1.BaseConnection {
         }
         throw lastFetchError;
     }
-    async fetchSchemaForSQLBlock(sqlRef, { refreshTimestamp }) {
-        const key = sqlRef.name;
-        let inCache = this.sqlSchemaCache.get(key);
-        if (!inCache ||
-            (refreshTimestamp && refreshTimestamp > inCache.timestamp)) {
-            const timestamp = refreshTimestamp !== null && refreshTimestamp !== void 0 ? refreshTimestamp : Date.now();
-            try {
-                const tableFieldSchema = await this.getSQLBlockSchema(sqlRef);
-                inCache = {
-                    structDef: this.structDefFromSQLSchema(sqlRef, tableFieldSchema),
-                    timestamp,
-                };
-            }
-            catch (error) {
-                inCache = { error: error.message, timestamp };
-            }
-            this.sqlSchemaCache.set(key, inCache);
-        }
-        return inCache;
-    }
     // TODO this needs to extend the wait for results using a timeout set by the user,
     // and probably needs to loop to check for results - BQ docs now say that after ~2min of waiting,
     // no matter what you set for timeoutMs, they will probably just return.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/db-bigquery",
-  "version": "0.0.195-dev241003204905",
+  "version": "0.0.195",
   "license": "MIT",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -25,7 +25,7 @@
     "@google-cloud/bigquery": "^7.3.0",
     "@google-cloud/common": "^5.0.1",
     "@google-cloud/paginator": "^5.0.0",
-    "@malloydata/malloy": "^0.0.195-dev241003204905",
+    "@malloydata/malloy": "^0.0.195",
     "gaxios": "^4.2.0"
   }
 }