@malloydata/db-bigquery 0.0.22-dev230207182613 → 0.0.22-dev230207183024

package/dist/bigquery.spec.js

@@ -57,10 +57,10 @@ describe("db:BigQuery", () => {
     beforeAll(() => {
         bq = new bigquery_connection_1.BigQueryConnection("test");
         const files = {
-            readURL: async (url) => {
+            "readURL": async (url) => {
                 const filePath = (0, url_1.fileURLToPath)(url);
                 return await util.promisify(fs.readFile)(filePath, "utf8");
-            },
+            }
         };
         runtime = new malloy.Runtime(files, bq);
     });
@@ -75,11 +75,11 @@ describe("db:BigQuery", () => {
     it("gets table schema", async () => {
         const res = await bq.getTableFieldSchema(`malloy-data.faa.carriers`);
         expect(res.schema).toStrictEqual({
-            fields: [
-                { name: "code", type: "STRING" },
-                { name: "name", type: "STRING" },
-                { name: "nickname", type: "STRING" },
-            ],
+            "fields": [
+                { "name": "code", "type": "STRING" },
+                { "name": "name", "type": "STRING" },
+                { "name": "nickname", "type": "STRING" }
+            ]
         });
     });
     it.todo("gets table structdefs");
@@ -124,7 +124,7 @@ describe("db:BigQuery", () => {
             const dataset = sdk.dataset(datasetName);
             if ((await dataset.exists())[0]) {
                 await dataset.delete({
-                    force: true,
+                    "force": true
                 });
             }
         };
@@ -145,7 +145,7 @@ describe("db:BigQuery", () => {
         it("throws if table exist and overwriteExistingTable=false", async () => {
             const newDatasetResponse = await sdk.createDataset(datasetName);
             const dataset = newDatasetResponse[0];
-            const tableMeta = { name: tableName };
+            const tableMeta = { "name": tableName };
             await dataset.createTable(tableName, tableMeta);
             await expect(async () => {
                 await bq.manifestPermanentTable("SELECT 1 as t", datasetName, tableName, false, true);
@@ -163,7 +163,7 @@ describe("db:BigQuery", () => {
             // query the new table
             const [queryJob] = await sdk.createQueryJob(`SELECT * FROM ${datasetName}.${tableName}`);
             const [results] = await queryJob.getQueryResults();
-            expect(results[0]).toStrictEqual({ t: 1 });
+            expect(results[0]).toStrictEqual({ "t": 1 });
         });
     });
 });

package/dist/bigquery_connection.d.ts

@@ -1,7 +1,7 @@
 import { RowMetadata } from "@google-cloud/bigquery";
 import bigquery from "@google-cloud/bigquery/build/src/types";
 import { ResourceStream } from "@google-cloud/paginator";
-import { PersistSQLResults, PooledConnection, QueryData, StructDef, StreamingConnection, MalloyQueryData, FieldTypeDef, SQLBlock, Connection, QueryDataRow } from "@malloydata/malloy";
+import { Connection, FieldTypeDef, MalloyQueryData, PersistSQLResults, PooledConnection, QueryData, QueryDataRow, SQLBlock, StreamingConnection, StructDef } from "@malloydata/malloy";
 export interface BigQueryManagerOptions {
     credentials?: {
         clientId: string;

package/dist/bigquery_connection.js

@@ -93,18 +93,18 @@ class BigQueryConnection {
         this.schemaCache = new Map();
         this.sqlSchemaCache = new Map();
         this.bqToMalloyTypes = {
-            DATE: { type: "date" },
-            STRING: { type: "string" },
-            INTEGER: { type: "number", numberType: "integer" },
-            INT64: { type: "number", numberType: "integer" },
-            FLOAT: { type: "number", numberType: "float" },
-            FLOAT64: { type: "number", numberType: "float" },
-            NUMERIC: { type: "number", numberType: "float" },
-            BIGNUMERIC: { type: "number", numberType: "float" },
-            TIMESTAMP: { type: "timestamp" },
-            BOOLEAN: { type: "boolean" },
-            BOOL: { type: "boolean" },
-            JSON: { type: "json" },
+            "DATE": { "type": "date" },
+            "STRING": { "type": "string" },
+            "INTEGER": { "type": "number", "numberType": "integer" },
+            "INT64": { "type": "number", "numberType": "integer" },
+            "FLOAT": { "type": "number", "numberType": "float" },
+            "FLOAT64": { "type": "number", "numberType": "float" },
+            "NUMERIC": { "type": "number", "numberType": "float" },
+            "BIGNUMERIC": { "type": "number", "numberType": "float" },
+            "TIMESTAMP": { "type": "timestamp" },
+            "BOOLEAN": { "type": "boolean" },
+            "BOOL": { "type": "boolean" },
+            "JSON": { "type": "json" }
             // TODO (https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#tablefieldschema):
             // BYTES
             // DATETIME
@@ -113,8 +113,8 @@ class BigQueryConnection {
         };
         this.name = name;
         this.bigQuery = new bigquery_1.BigQuery({
-            userAgent: `Malloy/${malloy_1.Malloy.version}`,
-            keyFilename: config.serviceAccountKeyPath,
+            "userAgent": `Malloy/${malloy_1.Malloy.version}`,
+            "keyFilename": config.serviceAccountKeyPath
         });
         // record project ID because for unclear reasons we have to modify the project ID on the SDK when
         // we want to use the tables API
@@ -156,8 +156,8 @@ class BigQueryConnection {
         const pageSize = (_a = options.rowLimit) !== null && _a !== void 0 ? _a : defaultOptions.rowLimit;
         try {
             const queryResultsOptions = {
-                maxResults: pageSize,
-                startIndex: rowIndex.toString(),
+                "maxResults": pageSize,
+                "startIndex": rowIndex.toString()
             };
             const jobResult = await this.createBigQueryJobAndGetResults(sqlCommand, undefined, queryResultsOptions);
             const totalRows = +(((_b = jobResult[2]) === null || _b === void 0 ? void 0 : _b.totalRows)
@@ -168,7 +168,7 @@ class BigQueryConnection {
                 throw new Error("Schema not present");
             }
             // TODO even though we have 10 minute timeout limit, we still should confirm that resulting metadata has "jobComplete: true"
-            const data = { rows: jobResult[0], totalRows };
+            const data = { "rows": jobResult[0], totalRows };
             const schema = (_d = jobResult[2]) === null || _d === void 0 ? void 0 : _d.schema;
             return { data, schema };
         }
@@ -181,22 +181,22 @@ class BigQueryConnection {
         return data;
     }
     async runSQLBlockAndFetchResultSchema(sqlBlock, options) {
-        const { data, schema: schemaRaw } = await this._runSQL(sqlBlock.selectStr, options);
+        const { data, "schema": schemaRaw } = await this._runSQL(sqlBlock.selectStr, options);
         const schema = this.structDefFromSQLSchema(sqlBlock, schemaRaw);
         return { data, schema };
     }
     async downloadMalloyQuery(sqlCommand) {
         const job = await this.createBigQueryJob({
-            query: sqlCommand,
+            "query": sqlCommand
         });
         return job.getQueryResultsStream();
     }
     async dryRunSQLQuery(sqlCommand) {
         try {
             const [result] = await this.bigQuery.createQueryJob({
-                location: this.location,
-                query: sqlCommand,
-                dryRun: true,
+                "location": this.location,
+                "query": sqlCommand,
+                "dryRun": true
             });
             return result;
         }
@@ -215,7 +215,7 @@ class BigQueryConnection {
     }
     async getTableFieldSchema(tableURL) {
         var _a, _b;
-        const { tablePath: tableName } = (0, malloy_1.parseTableURI)(tableURL);
+        const { "tablePath": tableName } = (0, malloy_1.parseTableURI)(tableURL);
         const segments = tableName.split(".");
         // paths can have two or three segments
         // if there are only two segments, assume the dataset is "local" to the current billing project
@@ -241,9 +241,9 @@ class BigQueryConnection {
             this.bigQuery.projectId = this.projectId;
             const [metadata] = await metadataPromise;
             return {
-                schema: metadata.schema,
-                needsPartitionPsuedoColumn: ((_a = metadata.timePartitioning) === null || _a === void 0 ? void 0 : _a.type) !== undefined &&
-                    ((_b = metadata.timePartitioning) === null || _b === void 0 ? void 0 : _b.field) === undefined,
+                "schema": metadata.schema,
+                "needsPartitionPsuedoColumn": ((_a = metadata.timePartitioning) === null || _a === void 0 ? void 0 : _a.type) !== undefined &&
+                    ((_b = metadata.timePartitioning) === null || _b === void 0 ? void 0 : _b.field) === undefined
             };
         }
         catch (e) {
@@ -284,8 +284,8 @@ class BigQueryConnection {
         else {
             try {
                 const [job] = await this.bigQuery.createQueryJob({
-                    location: this.location,
-                    query: sqlCommand,
+                    "location": this.location,
+                    "query": sqlCommand
                 });
                 let [metaData] = await job.getMetadata();
                 // wait for job to complete, because we need the table name
@@ -332,9 +332,9 @@ class BigQueryConnection {
             throw new Error(`Table ${tableName} already exists`);
         }
         const [job] = await this.bigQuery.createQueryJob({
-            query: sqlCommand,
-            location: this.location,
-            destination: table,
+            "query": sqlCommand,
+            "location": this.location,
+            "destination": table
         });
         // if creating this job didn't throw, there's an ID.
         // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
@@ -352,26 +352,32 @@ class BigQueryConnection {
                 const malloyType = this.bqToMalloyTypes[type];
                 if (malloyType) {
                     const innerStructDef = {
-                        type: "struct",
+                        "type": "struct",
                         name,
-                        dialect: this.dialectName,
-                        structSource: { type: "nested" },
-                        structRelationship: { type: "nested", field: name, isArray: true },
-                        fields: [{ ...malloyType, name: "value" }],
+                        "dialect": this.dialectName,
+                        "structSource": { "type": "nested" },
+                        "structRelationship": {
+                            "type": "nested",
+                            "field": name,
+                            "isArray": true
+                        },
+                        "fields": [{ ...malloyType, "name": "value" }]
                     };
                     structDef.fields.push(innerStructDef);
                 }
             }
             else if (["STRUCT", "RECORD"].includes(type)) {
                 const innerStructDef = {
-                    type: "struct",
+                    "type": "struct",
                     name,
-                    dialect: this.dialectName,
-                    structSource: field.mode === "REPEATED" ? { type: "nested" } : { type: "inline" },
-                    structRelationship: field.mode === "REPEATED"
-                        ? { type: "nested", field: name, isArray: false }
-                        : { type: "inline" },
-                    fields: [],
+                    "dialect": this.dialectName,
+                    "structSource": field.mode === "REPEATED"
+                        ? { "type": "nested" }
+                        : { "type": "inline" },
+                    "structRelationship": field.mode === "REPEATED"
+                        ? { "type": "nested", "field": name, "isArray": false }
+                        : { "type": "inline" },
+                    "fields": []
                 };
                 this.addFieldsToStructDef(innerStructDef, field);
                 structDef.fields.push(innerStructDef);
@@ -384,8 +390,8 @@ class BigQueryConnection {
                 else {
                     structDef.fields.push({
                         name,
-                        type: "string",
-                        e: [`'BigQuery type "${type}" not supported by Malloy'`],
+                        "type": "string",
+                        "e": [`'BigQuery type "${type}" not supported by Malloy'`]
                     });
                 }
             }
@@ -402,37 +408,43 @@ class BigQueryConnection {
     }
     structDefFromTableSchema(tableURL, schemaInfo) {
         const structDef = {
-            type: "struct",
-            name: tableURL,
-            dialect: this.dialectName,
-            structSource: {
-                type: "table",
-                tablePath: this.tableURLtoTablePath(tableURL),
+            "type": "struct",
+            "name": tableURL,
+            "dialect": this.dialectName,
+            "structSource": {
+                "type": "table",
+                "tablePath": this.tableURLtoTablePath(tableURL)
             },
-            structRelationship: { type: "basetable", connectionName: this.name },
-            fields: [],
+            "structRelationship": {
+                "type": "basetable",
+                "connectionName": this.name
+            },
+            "fields": []
         };
         this.addFieldsToStructDef(structDef, schemaInfo.schema);
         if (schemaInfo.needsPartitionPsuedoColumn) {
             structDef.fields.push({
-                type: "timestamp",
-                name: "_PARTITIONTIME",
+                "type": "timestamp",
+                "name": "_PARTITIONTIME"
             });
         }
         return structDef;
     }
     structDefFromSQLSchema(sqlBlock, tableFieldSchema) {
         const structDef = {
-            type: "struct",
-            name: sqlBlock.name,
-            dialect: this.dialectName,
-            structSource: {
-                type: "sql",
-                method: "subquery",
-                sqlBlock,
+            "type": "struct",
+            "name": sqlBlock.name,
+            "dialect": this.dialectName,
+            "structSource": {
+                "type": "sql",
+                "method": "subquery",
+                sqlBlock
+            },
+            "structRelationship": {
+                "type": "basetable",
+                "connectionName": this.name
             },
-            structRelationship: { type: "basetable", connectionName: this.name },
-            fields: [],
+            "fields": []
         };
         this.addFieldsToStructDef(structDef, tableFieldSchema);
         return structDef;
@@ -446,12 +458,12 @@ class BigQueryConnection {
                 try {
                     const tableFieldSchema = await this.getTableFieldSchema(tableURL);
                     inCache = {
-                        schema: this.structDefFromTableSchema(tableURL, tableFieldSchema),
+                        "schema": this.structDefFromTableSchema(tableURL, tableFieldSchema)
                     };
                     this.schemaCache.set(tableURL, inCache);
                 }
                 catch (error) {
-                    inCache = { error: error.message };
+                    inCache = { "error": error.message };
                 }
             }
             if (inCache.schema !== undefined) {
@@ -473,9 +485,9 @@ class BigQueryConnection {
         for (let retries = 0; retries < 3; retries++) {
             try {
                 const [job] = await this.bigQuery.createQueryJob({
-                    location: this.location,
-                    query: sqlRef.selectStr,
-                    dryRun: true,
+                    "location": this.location,
+                    "query": sqlRef.selectStr,
+                    "dryRun": true
                 });
                 return job.metadata.statistics.query.schema;
             }
@@ -492,11 +504,11 @@ class BigQueryConnection {
             try {
                 const tableFieldSchema = await this.getSQLBlockSchema(sqlRef);
                 inCache = {
-                    structDef: this.structDefFromSQLSchema(sqlRef, tableFieldSchema),
+                    "structDef": this.structDefFromSQLSchema(sqlRef, tableFieldSchema)
                 };
             }
             catch (error) {
-                inCache = { error: error.message };
+                inCache = { "error": error.message };
             }
             this.sqlSchemaCache.set(key, inCache);
         }
@@ -508,8 +520,8 @@ class BigQueryConnection {
     async createBigQueryJobAndGetResults(sqlCommand, createQueryJobOptions, getQueryResultsOptions) {
         try {
             const job = await this.createBigQueryJob({
-                query: sqlCommand,
-                ...createQueryJobOptions,
+                "query": sqlCommand,
+                ...createQueryJobOptions
             });
             // TODO we should check if this is still required?
             // We do a simple retry-loop here, as a temporary fix for a transient
@@ -521,8 +533,8 @@ class BigQueryConnection {
             for (let retries = 0; retries < 3; retries++) {
                 try {
                     return await job.getQueryResults({
-                        timeoutMs: 1000 * 60 * 2,
-                        ...getQueryResultsOptions,
+                        "timeoutMs": 1000 * 60 * 2,
+                        ...getQueryResultsOptions
                     });
                 }
                 catch (fetchError) {
@@ -537,10 +549,10 @@ class BigQueryConnection {
     }
     async createBigQueryJob(createQueryJobOptions) {
         const [job] = await this.bigQuery.createQueryJob({
-            location: this.location,
-            maximumBytesBilled: this.config.maximumBytesBilled || MAXIMUM_BYTES_BILLED,
-            jobTimeoutMs: Number(this.config.timeoutMs) || TIMEOUT_MS,
-            ...createQueryJobOptions,
+            "location": this.location,
+            "maximumBytesBilled": this.config.maximumBytesBilled || MAXIMUM_BYTES_BILLED,
+            "jobTimeoutMs": Number(this.config.timeoutMs) || TIMEOUT_MS,
+            ...createQueryJobOptions
         });
         return job;
     }
@@ -569,6 +581,6 @@ class BigQueryConnection {
 }
 exports.BigQueryConnection = BigQueryConnection;
 BigQueryConnection.DEFAULT_QUERY_OPTIONS = {
-    rowLimit: 10,
+    "rowLimit": 10
 };
 //# sourceMappingURL=bigquery_connection.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/db-bigquery",
-  "version": "0.0.22-dev230207182613",
+  "version": "0.0.22-dev230207183024",
   "license": "MIT",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -21,7 +21,7 @@
   "dependencies": {
     "@google-cloud/bigquery": "^5.5.0",
     "@google-cloud/common": "^3.6.0",
-    "@malloydata/malloy": "^0.0.22-dev230207182613",
+    "@malloydata/malloy": "^0.0.22-dev230207183024",
     "gaxios": "^4.2.0"
   }
 }