@malloydata/db-bigquery 0.0.96-dev231025212541 → 0.0.96-dev231025215824

package/dist/bigquery.spec.js

@@ -166,5 +166,94 @@ describe('db:BigQuery', () => {
             expect(results[0]).toStrictEqual({ t: 1 });
         });
     });
+    describe('Caching', () => {
+        let getTableFieldSchema;
+        let getSQLBlockSchema;
+        beforeEach(async () => {
+            getTableFieldSchema = jest
+                .spyOn(bigquery_connection_1.BigQueryConnection.prototype, 'getTableFieldSchema')
+                .mockResolvedValue({
+                schema: {},
+                needsTableSuffixPseudoColumn: false,
+                needsPartitionTimePseudoColumn: false,
+                needsPartitionDatePseudoColumn: false,
+            });
+            getSQLBlockSchema = jest
+                .spyOn(bigquery_connection_1.BigQueryConnection.prototype, 'getSQLBlockSchema')
+                .mockResolvedValue({
+                schema: {},
+                needsTableSuffixPseudoColumn: false,
+                needsPartitionTimePseudoColumn: false,
+                needsPartitionDatePseudoColumn: false,
+            });
+        });
+        afterEach(() => {
+            jest.resetAllMocks();
+        });
+        it('caches table schema', async () => {
+            await bq.fetchSchemaForTables({ 'test1': 'table1' }, {});
+            expect(getTableFieldSchema).toBeCalledTimes(1);
+            await new Promise(resolve => setTimeout(resolve));
+            await bq.fetchSchemaForTables({ 'test1': 'table1' }, {});
+            expect(getTableFieldSchema).toBeCalledTimes(1);
+        });
+        it('refreshes table schema', async () => {
+            await bq.fetchSchemaForTables({ 'test2': 'table2' }, {});
+            expect(getTableFieldSchema).toBeCalledTimes(1);
+            await new Promise(resolve => setTimeout(resolve));
+            await bq.fetchSchemaForTables({ 'test2': 'table2' }, { refreshTimestamp: Date.now() });
+            expect(getTableFieldSchema).toBeCalledTimes(2);
+        });
+        it('caches sql schema', async () => {
+            await bq.fetchSchemaForSQLBlock(SQL_BLOCK_1, {});
+            expect(getSQLBlockSchema).toBeCalledTimes(1);
+            await new Promise(resolve => setTimeout(resolve));
+            await bq.fetchSchemaForSQLBlock(SQL_BLOCK_1, {});
+            expect(getSQLBlockSchema).toBeCalledTimes(1);
+        });
+        it('refreshes sql schema', async () => {
+            await bq.fetchSchemaForSQLBlock(SQL_BLOCK_2, {});
+            expect(getSQLBlockSchema).toBeCalledTimes(1);
+            await new Promise(resolve => setTimeout(resolve));
+            await bq.fetchSchemaForSQLBlock(SQL_BLOCK_2, {
+                refreshTimestamp: Date.now(),
+            });
+            expect(getSQLBlockSchema).toBeCalledTimes(2);
+        });
+    });
 });
+const SQL_BLOCK_1 = {
+    type: 'sqlBlock',
+    name: 'block1',
+    selectStr: `
+SELECT
+created_at,
+sale_price,
+inventory_item_id
+FROM 'order_items.parquet'
+SELECT
+id,
+product_department,
+product_category,
+created_at AS inventory_items_created_at
+FROM "inventory_items.parquet"
+`,
+};
+const SQL_BLOCK_2 = {
+    type: 'sqlBlock',
+    name: 'block2',
+    selectStr: `
+SELECT
+created_at,
+sale_price,
+inventory_item_id
+FROM read_parquet('order_items2.parquet', arg='value')
+SELECT
+id,
+product_department,
+product_category,
+created_at AS inventory_items_created_at
+FROM read_parquet("inventory_items2.parquet")
+`,
+};
 //# sourceMappingURL=bigquery.spec.js.map

package/dist/bigquery_connection.d.ts

@@ -2,6 +2,7 @@ import { RowMetadata } from '@google-cloud/bigquery';
 import bigquery from '@google-cloud/bigquery/build/src/types';
 import { ResourceStream } from '@google-cloud/paginator';
 import { Connection, MalloyQueryData, PersistSQLResults, PooledConnection, QueryData, QueryDataRow, QueryRunStats, SQLBlock, StreamingConnection, StructDef } from '@malloydata/malloy';
+import { FetchSchemaOptions } from '@malloydata/malloy-interfaces';
 export interface BigQueryManagerOptions {
     credentials?: {
         clientId: string;
@@ -77,12 +78,12 @@ export declare class BigQueryConnection implements Connection, PersistSQLResults
     private addFieldsToStructDef;
     private structDefFromTableSchema;
     private structDefFromSQLSchema;
-    fetchSchemaForTables(missing: Record<string, string>): Promise<{
+    fetchSchemaForTables(missing: Record<string, string>, { refreshTimestamp }: FetchSchemaOptions): Promise<{
         schemas: Record<string, StructDef>;
         errors: Record<string, string>;
     }>;
     private getSQLBlockSchema;
-    fetchSchemaForSQLBlock(sqlRef: SQLBlock): Promise<{
+    fetchSchemaForSQLBlock(sqlRef: SQLBlock, { refreshTimestamp }: FetchSchemaOptions): Promise<{
         structDef: StructDef;
         error?: undefined;
     } | {

package/dist/bigquery_connection.js

@@ -442,22 +442,25 @@ class BigQueryConnection {
         this.addFieldsToStructDef(structDef, tableFieldSchema);
         return structDef;
     }
-    async fetchSchemaForTables(missing) {
+    async fetchSchemaForTables(missing, { refreshTimestamp }) {
         const schemas = {};
         const errors = {};
         for (const tableKey in missing) {
             let inCache = this.schemaCache.get(tableKey);
-            if (!inCache) {
+            if (!inCache ||
+                (refreshTimestamp && refreshTimestamp > inCache.timestamp)) {
+                const timestamp = refreshTimestamp !== null && refreshTimestamp !== void 0 ? refreshTimestamp : Date.now();
                 const tablePath = this.normalizeTablePath(missing[tableKey]);
                 try {
                     const tableFieldSchema = await this.getTableFieldSchema(tablePath);
                     inCache = {
                         schema: this.structDefFromTableSchema(tableKey, tablePath, tableFieldSchema),
+                        timestamp,
                     };
                     this.schemaCache.set(tableKey, inCache);
                 }
                 catch (error) {
-                    inCache = { error: error.message };
+                    inCache = { error: error.message, timestamp };
                 }
             }
             if (inCache.schema !== undefined) {
@@ -491,18 +494,21 @@ class BigQueryConnection {
         }
         throw lastFetchError;
     }
-    async fetchSchemaForSQLBlock(sqlRef) {
+    async fetchSchemaForSQLBlock(sqlRef, { refreshTimestamp }) {
         const key = sqlRef.name;
         let inCache = this.sqlSchemaCache.get(key);
-        if (!inCache) {
+        if (!inCache ||
+            (refreshTimestamp && refreshTimestamp > inCache.timestamp)) {
+            const timestamp = refreshTimestamp !== null && refreshTimestamp !== void 0 ? refreshTimestamp : Date.now();
             try {
                 const tableFieldSchema = await this.getSQLBlockSchema(sqlRef);
                 inCache = {
                     structDef: this.structDefFromSQLSchema(sqlRef, tableFieldSchema),
+                    timestamp,
                 };
             }
             catch (error) {
-                inCache = { error: error.message };
+                inCache = { error: error.message, timestamp };
             }
             this.sqlSchemaCache.set(key, inCache);
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/db-bigquery",
-  "version": "0.0.96-dev231025212541",
+  "version": "0.0.96-dev231025215824",
   "license": "MIT",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -25,7 +25,8 @@
     "@google-cloud/bigquery": "^5.5.0",
     "@google-cloud/common": "^3.6.0",
     "@google-cloud/paginator": "^4.0.1",
-    "@malloydata/malloy": "^0.0.96-dev231025212541",
+    "@malloydata/malloy": "^0.0.96-dev231025215824",
+    "@malloydata/malloy-interfaces": "^0.0.96-dev231025215824",
     "gaxios": "^4.2.0"
   }
 }