@malloydata/db-bigquery 0.0.1

package/README.md

@@ -0,0 +1,7 @@
+# Malloy
+
+Malloy is an experimental language for describing data relationships and transformations. It is both a semantic modeling language and a querying language that runs queries against a relational database. Malloy currently connects to BigQuery and Postgres, and natively supports DuckDB. We've built a Visual Studio Code extension to facilitate building Malloy data models, querying and transforming data, and creating simple visualizations and dashboards.
+
+## This package
+
+This package facilitates using the `malloydata/malloy` library with BigQuery - see [here](https://github.com/looker-open-source/malloy/blob/main/packages/malloy/README.md) for additional information.

package/dist/bigquery.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/bigquery.spec.js

@@ -0,0 +1,148 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const malloy = __importStar(require("@malloydata/malloy"));
+const bigquery_connection_1 = require("./bigquery_connection");
+const bigquery_1 = require("@google-cloud/bigquery");
+const util = __importStar(require("util"));
+const fs = __importStar(require("fs"));
+const url_1 = require("url");
+describe("db:BigQuery", () => {
+    let bq;
+    let runtime;
+    beforeAll(() => {
+        bq = new bigquery_connection_1.BigQueryConnection("test");
+        const files = {
+            readURL: async (url) => {
+                const filePath = (0, url_1.fileURLToPath)(url);
+                return await util.promisify(fs.readFile)(filePath, "utf8");
+            },
+        };
+        runtime = new malloy.Runtime(files, bq);
+    });
+    it("runs a SQL query", async () => {
+        const res = await bq.runSQL(`SELECT 1 as t`);
+        expect(res.rows[0]["t"]).toBe(1);
+    });
+    it("costs a SQL query", async () => {
+        const res = await bq.costQuery(`SELECT * FROM malloy-data.faa.airports`);
+        expect(res).toBe(3029200);
+    });
+    it("gets table schema", async () => {
+        const res = await bq.getTableFieldSchema(`malloy-data.faa.carriers`);
+        expect(res.schema).toStrictEqual({
+            fields: [
+                { name: "code", type: "STRING" },
+                { name: "name", type: "STRING" },
+                { name: "nickname", type: "STRING" },
+            ],
+        });
+    });
+    it.todo("gets table structdefs");
+    it("runs a Malloy query", async () => {
+        const sql = await runtime
+            .loadModel("explore: carriers is table('malloy-data.faa.carriers') { measure: carrier_count is count() }")
+            .loadQuery("query: carriers -> { aggregate: carrier_count }")
+            .getSQL();
+        const res = await bq.runSQL(sql);
+        expect(res.rows[0]["carrier_count"]).toBe(21);
+    });
+    it("streams a Malloy query for download", async () => {
+        const sql = await runtime
+            .loadModel("explore: carriers is table('malloy-data.faa.carriers') { measure: carrier_count is count() }")
+            .loadQuery("query: carriers -> { group_by: name }")
+            .getSQL();
+        const res = await bq.downloadMalloyQuery(sql);
+        return new Promise((resolve) => {
+            let count = 0;
+            res.on("data", () => (count += 1));
+            res.on("end", () => {
+                expect(count).toBe(21);
+                resolve(true);
+            });
+        });
+    });
+    it("manifests a temporary table", async () => {
+        const fullTempTableName = await bq.manifestTemporaryTable("SELECT 1 as t");
+        const splitTableName = fullTempTableName.split(".");
+        const sdk = new bigquery_1.BigQuery();
+        const dataset = sdk.dataset(splitTableName[1]);
+        const table = dataset.table(splitTableName[2]);
+        const exists = await table.exists();
+        expect(exists).toBeTruthy();
+    });
+    describe.skip("manifests permanent table", () => {
+        const datasetName = "test_malloy_test_dataset";
+        const tableName = "test_malloy_test_table";
+        const sdk = new bigquery_1.BigQuery();
+        // delete entire dataset before each test and once tests are complete
+        const deleteTestDataset = async () => {
+            const dataset = sdk.dataset(datasetName);
+            if ((await dataset.exists())[0]) {
+                await dataset.delete({
+                    force: true,
+                });
+            }
+        };
+        beforeEach(deleteTestDataset);
+        afterAll(deleteTestDataset);
+        it("throws if dataset does not exist and createDataset=false", async () => {
+            await expect(async () => {
+                await bq.manifestPermanentTable("SELECT 1 as t", datasetName, tableName, false, false);
+            }).rejects.toThrowError(`Dataset ${datasetName} does not exist`);
+        });
+        it("creates dataset if createDataset=true", async () => {
+            // note - dataset does not exist b/c of beforeEach()
+            await bq.manifestPermanentTable("SELECT 1 as t", datasetName, tableName, false, true);
+            const dataset = sdk.dataset(datasetName);
+            const [exists] = await dataset.exists();
+            expect(exists).toBeTruthy();
+        });
+        it("throws if table exist and overwriteExistingTable=false", async () => {
+            const newDatasetResponse = await sdk.createDataset(datasetName);
+            const dataset = newDatasetResponse[0];
+            const tableMeta = { name: tableName };
+            await dataset.createTable(tableName, tableMeta);
+            await expect(async () => {
+                await bq.manifestPermanentTable("SELECT 1 as t", datasetName, tableName, false, true);
+            }).rejects.toThrowError(`Table ${tableName} already exists`);
+        });
+        it("manifests a table", async () => {
+            const jobId = await bq.manifestPermanentTable("SELECT 1 as t", datasetName, tableName, false, true);
+            // wait for job to complete
+            const [job] = await sdk.job(jobId).get();
+            let [metaData] = await job.getMetadata();
+            while (metaData.status.state !== "DONE") {
+                await new Promise((resolve) => setTimeout(resolve, 1000));
+                [metaData] = await job.getMetadata();
+            }
+            // query the new table
+            const [queryJob] = await sdk.createQueryJob(`SELECT * FROM ${datasetName}.${tableName}`);
+            const [results] = await queryJob.getQueryResults();
+            expect(results[0]).toStrictEqual({ t: 1 });
+        });
+    });
+});
+//# sourceMappingURL=bigquery.spec.js.map

package/dist/bigquery_connection.d.ts

@@ -0,0 +1,91 @@
+import { RowMetadata } from "@google-cloud/bigquery";
+import bigquery from "@google-cloud/bigquery/build/src/types";
+import { ResourceStream } from "@google-cloud/paginator";
+import { QueryData, StructDef, MalloyQueryData, FieldTypeDef, SQLBlock, Connection, QueryDataRow } from "@malloydata/malloy";
+import { PooledConnection } from "@malloydata/malloy";
+import { FetchSchemaAndRunSimultaneously, FetchSchemaAndRunStreamSimultaneously, PersistSQLResults, StreamingConnection } from "@malloydata/malloy/src/runtime_types";
+export interface BigQueryManagerOptions {
+    credentials?: {
+        clientId: string;
+        clientSecret: string;
+        refreshToken: string | null;
+    };
+    projectId?: string | undefined;
+    userAgent: string;
+}
+export interface BigQueryQueryOptions {
+    rowLimit: number;
+}
+interface BigQueryConnectionConfiguration {
+    defaultProject?: string;
+    serviceAccountKeyPath?: string;
+    location?: string;
+    maximumBytesBilled?: string;
+    timeoutMs?: string;
+}
+interface SchemaInfo {
+    schema: bigquery.ITableFieldSchema;
+    needsPartitionPsuedoColumn: boolean;
+}
+declare type QueryOptionsReader = Partial<BigQueryQueryOptions> | (() => Partial<BigQueryQueryOptions>);
+export declare class BigQueryAuthenticationError extends Error {
+    constructor(message: string);
+}
+export declare class BigQueryConnection implements Connection, PersistSQLResults, StreamingConnection, FetchSchemaAndRunSimultaneously {
+    static DEFAULT_QUERY_OPTIONS: BigQueryQueryOptions;
+    private bigQuery;
+    private projectId;
+    private temporaryTables;
+    private defaultProject;
+    private schemaCache;
+    private sqlSchemaCache;
+    private queryOptions?;
+    private config;
+    private location;
+    readonly name: string;
+    bqToMalloyTypes: {
+        [key: string]: Partial<FieldTypeDef>;
+    };
+    constructor(name: string, queryOptions?: QueryOptionsReader, config?: BigQueryConnectionConfiguration);
+    get dialectName(): string;
+    private readQueryOptions;
+    isPool(): this is PooledConnection;
+    canPersist(): this is PersistSQLResults;
+    canStream(): this is StreamingConnection;
+    canFetchSchemaAndRunSimultaneously(): this is FetchSchemaAndRunSimultaneously;
+    canFetchSchemaAndRunStreamSimultaneously(): this is FetchSchemaAndRunStreamSimultaneously;
+    private _runSQL;
+    runSQL(sqlCommand: string, options?: Partial<BigQueryQueryOptions>, rowIndex?: number): Promise<MalloyQueryData>;
+    runSQLBlockAndFetchResultSchema(sqlBlock: SQLBlock, options?: {
+        rowLimit?: number | undefined;
+    }): Promise<{
+        data: MalloyQueryData;
+        schema: StructDef;
+    }>;
+    downloadMalloyQuery(sqlCommand: string): Promise<ResourceStream<RowMetadata>>;
+    private dryRunSQLQuery;
+    costQuery(sqlCommand: string): Promise<number>;
+    structDefFromSQL(sqlCommand: string): Promise<StructDef>;
+    getTableFieldSchema(tableURL: string): Promise<SchemaInfo>;
+    executeSQLRaw(sqlCommand: string): Promise<QueryData>;
+    test(): Promise<void>;
+    manifestTemporaryTable(sqlCommand: string): Promise<string>;
+    manifestPermanentTable(sqlCommand: string, datasetName: string, tableName: string, overwriteExistingTable?: boolean, createDataset?: boolean): Promise<string>;
+    private addFieldsToStructDef;
+    private tableURLtoTablePath;
+    private structDefFromTableSchema;
+    private structDefFromSQLSchema;
+    fetchSchemaForTables(missing: string[]): Promise<{
+        schemas: Record<string, StructDef>;
+        errors: Record<string, string>;
+    }>;
+    private getSQLBlockSchema;
+    fetchSchemaForSQLBlocks(sqlRefs: SQLBlock[]): Promise<{
+        schemas: Record<string, StructDef>;
+        errors: Record<string, string>;
+    }>;
+    private createBigQueryJobAndGetResults;
+    private createBigQueryJob;
+    runSQLStream(sqlCommand: string, options?: Partial<BigQueryQueryOptions>): AsyncIterableIterator<QueryDataRow>;
+}
+export {};

package/dist/bigquery_connection.js

@@ -0,0 +1,577 @@
+"use strict";
+/*
+ * Copyright 2022 Google LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BigQueryConnection = exports.BigQueryAuthenticationError = void 0;
+const crypto = __importStar(require("crypto"));
+const bigquery_1 = require("@google-cloud/bigquery");
+const googleCommon = __importStar(require("@google-cloud/common"));
+const gaxios_1 = require("gaxios");
+const malloy_1 = require("@malloydata/malloy");
+const malloy_2 = require("@malloydata/malloy");
+// BigQuery SDK apparently throws various authentication errors from Gaxios (https://github.com/googleapis/gaxios) and from
+// @google-cloud/common (https://www.npmjs.com/package/@google-cloud/common)
+// catching and rewriting here a single kind of authentication error we can consistently catch further up the stack
+class BigQueryAuthenticationError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "BigQueryAuthenticationError";
+    }
+}
+exports.BigQueryAuthenticationError = BigQueryAuthenticationError;
+const maybeRewriteError = (e) => {
+    // GaxiosError happens if credentials are revoked (for example, client.revokeCredentials()) or if
+    // the refresh token is invalid
+    // ApiErrors happen if token is revoked (for example, client.revokeToken(creds.access_token!))
+    if (e instanceof Error) {
+        if ((e instanceof gaxios_1.GaxiosError && e.code === "400") ||
+            (e instanceof googleCommon.ApiError && e.code === 401)) {
+            return new BigQueryAuthenticationError(e.message);
+        }
+        else
+            return e;
+    }
+    else {
+        // something throw a non-Error, and we didn't expect that
+        throw e;
+    }
+};
+/**
+ * Default maximumBytesBilled value, 25GiB
+ */
+const MAXIMUM_BYTES_BILLED = String(25 * 1024 * 1024 * 1024);
+/**
+ * Default timeoutMs value, 10 Mins
+ */
+const TIMEOUT_MS = 1000 * 60 * 10;
+// manage access to BQ, control costs, enforce global data/API limits
+class BigQueryConnection {
+    constructor(name, queryOptions, config = {}) {
+        this.temporaryTables = new Map();
+        this.schemaCache = new Map();
+        this.sqlSchemaCache = new Map();
+        this.bqToMalloyTypes = {
+            DATE: { type: "date" },
+            STRING: { type: "string" },
+            INTEGER: { type: "number", numberType: "integer" },
+            INT64: { type: "number", numberType: "integer" },
+            FLOAT: { type: "number", numberType: "float" },
+            FLOAT64: { type: "number", numberType: "float" },
+            NUMERIC: { type: "number", numberType: "float" },
+            BIGNUMERIC: { type: "number", numberType: "float" },
+            TIMESTAMP: { type: "timestamp" },
+            BOOLEAN: { type: "boolean" },
+            BOOL: { type: "boolean" },
+            // TODO (https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#tablefieldschema):
+            // BYTES
+            // DATETIME
+            // TIME
+            // GEOGRAPHY
+        };
+        this.name = name;
+        this.bigQuery = new bigquery_1.BigQuery({
+            userAgent: `Malloy/${malloy_1.Malloy.version}`,
+            keyFilename: config.serviceAccountKeyPath,
+        });
+        // record project ID because for unclear reasons we have to modify the project ID on the SDK when
+        // we want to use the tables API
+        this.projectId = this.bigQuery.projectId;
+        this.defaultProject = config.defaultProject || this.bigQuery.projectId;
+        this.queryOptions = queryOptions;
+        this.config = config;
+        this.location = config.location || "US";
+    }
+    get dialectName() {
+        return "standardsql";
+    }
+    readQueryOptions() {
+        const options = BigQueryConnection.DEFAULT_QUERY_OPTIONS;
+        if (this.queryOptions) {
+            if (this.queryOptions instanceof Function) {
+                return { ...options, ...this.queryOptions() };
+            }
+            else {
+                return { ...options, ...this.queryOptions };
+            }
+        }
+        else {
+            return options;
+        }
+    }
+    isPool() {
+        return false;
+    }
+    canPersist() {
+        return true;
+    }
+    canStream() {
+        return true;
+    }
+    canFetchSchemaAndRunSimultaneously() {
+        return true;
+    }
+    canFetchSchemaAndRunStreamSimultaneously() {
+        return false;
+    }
+    async _runSQL(sqlCommand, options = {}, rowIndex = 0) {
+        var _a, _b, _c, _d;
+        const defaultOptions = this.readQueryOptions();
+        const pageSize = (_a = options.rowLimit) !== null && _a !== void 0 ? _a : defaultOptions.rowLimit;
+        try {
+            const queryResultsOptions = {
+                maxResults: pageSize,
+                startIndex: rowIndex.toString(),
+            };
+            const jobResult = await this.createBigQueryJobAndGetResults(sqlCommand, undefined, queryResultsOptions);
+            const totalRows = +(((_b = jobResult[2]) === null || _b === void 0 ? void 0 : _b.totalRows)
+                ? jobResult[2].totalRows
+                : "0");
+            // TODO need to probably surface the cause of the schema not present error
+            if (((_c = jobResult[2]) === null || _c === void 0 ? void 0 : _c.schema) === undefined) {
+                throw new Error("Schema not present");
+            }
+            // TODO even though we have 10 minute timeout limit, we still should confirm that resulting metadata has "jobComplete: true"
+            const data = { rows: jobResult[0], totalRows };
+            const schema = (_d = jobResult[2]) === null || _d === void 0 ? void 0 : _d.schema;
+            return { data, schema };
+        }
+        catch (e) {
+            throw maybeRewriteError(e);
+        }
+    }
+    async runSQL(sqlCommand, options = {}, rowIndex = 0) {
+        const { data } = await this._runSQL(sqlCommand, options, rowIndex);
+        return data;
+    }
+    async runSQLBlockAndFetchResultSchema(sqlBlock, options) {
+        const { data, schema: schemaRaw } = await this._runSQL(sqlBlock.select, options);
+        const schema = this.structDefFromSQLSchema(sqlBlock, schemaRaw);
+        return { data, schema };
+    }
+    async downloadMalloyQuery(sqlCommand) {
+        const job = await this.createBigQueryJob({
+            query: sqlCommand,
+        });
+        return job.getQueryResultsStream();
+    }
+    async dryRunSQLQuery(sqlCommand) {
+        try {
+            const [result] = await this.bigQuery.createQueryJob({
+                location: this.location,
+                query: sqlCommand,
+                dryRun: true,
+            });
+            return result;
+        }
+        catch (e) {
+            throw maybeRewriteError(e);
+        }
+    }
+    async costQuery(sqlCommand) {
+        const dryRunResults = await this.dryRunSQLQuery(sqlCommand);
+        return Number(dryRunResults.metadata.statistics.totalBytesProcessed);
+    }
+    async structDefFromSQL(sqlCommand) {
+        const dryRunResults = await this.dryRunSQLQuery(sqlCommand);
+        const destinationTable = dryRunResults.metadata.configuration.query.destinationTable;
+        return this.structDefFromTableSchema(`${destinationTable.projectId}.${destinationTable.datasetId}.${destinationTable.tableId}`, dryRunResults.metadata.statistics.query.schema);
+    }
+    async getTableFieldSchema(tableURL) {
+        var _a, _b;
+        const { tablePath: tableName } = (0, malloy_2.parseTableURL)(tableURL);
+        const segments = tableName.split(".");
+        // paths can have two or three segments
+        // if there are only two segments, assume the dataset is "local" to the current billing project
+        let projectId, datasetNamePart, tableNamePart;
+        if (segments.length === 2) {
+            [datasetNamePart, tableNamePart] = segments;
+            projectId = this.defaultProject;
+        }
+        else if (segments.length === 3)
+            [projectId, datasetNamePart, tableNamePart] = segments;
+        else
+            throw new Error(`Improper table path: ${tableName}. A table path requires 2 or 3 segments`);
+        try {
+            // TODO The `dataset` API has no way to set a different `projectId` than the one stored in the BQ
+            //      instance. So we hack it until a better way exists: we set the `this.bigQuery.projectId`
+            //      to the `projectId` for the dataset, then put it back when we're done. Importantly, we
+            //      set it back _before_ we await the promise, thus avoiding a "concurrency" issue. We've decided
+            //      this is better than creating a new BQ instance every time we need to get a table schema.
+            if (projectId)
+                this.bigQuery.projectId = projectId;
+            const table = this.bigQuery.dataset(datasetNamePart).table(tableNamePart);
+            const metadataPromise = table.getMetadata();
+            this.bigQuery.projectId = this.projectId;
+            const [metadata] = await metadataPromise;
+            return {
+                schema: metadata.schema,
+                needsPartitionPsuedoColumn: ((_a = metadata.timePartitioning) === null || _a === void 0 ? void 0 : _a.type) !== undefined &&
+                    ((_b = metadata.timePartitioning) === null || _b === void 0 ? void 0 : _b.field) === undefined,
+            };
+        }
+        catch (e) {
+            throw maybeRewriteError(e);
+        }
+    }
+    async executeSQLRaw(sqlCommand) {
+        const result = await this.createBigQueryJobAndGetResults(sqlCommand);
+        return result[0];
+    }
+    async test() {
+        await this.dryRunSQLQuery("SELECT 1");
+    }
+    /*
+     *   Do we need to care about multiple overlapping calls to this function? No.
+     *
+     *   If two long-running jobs using the exact same SQL are started by the same user and overlap - meaning, there is
+     *   a job running with some SQL, and while it is PENDING or RUNNING, another job with the same SQL is added -
+     *   the ultimate destination table for those jobs is the same table, if the source table(s) haven't changed, and
+     *   SQL didn't use some mutable functions.
+     *
+     *   If second job goes to RUNNING after first job finished, it will just reuse its results (this is the caching
+     *   behavior). But if both of them are RUNNING at the same time, they will both do the work and consume the slots.
+     *   If they finish around same time, the second job discards its results. If second job finishes much later than
+     *   the first one (like an hour or so), it will update this destination table to extend its TTL
+     *
+     *   This means that longer-term, we may want to block multiple calls at the app level (if tons of dashboards were
+     *   hitting this, creating many instances of the same table run, that might start to cost $$) but for now we're
+     *   ok with this simple approach
+     *
+     */
+    async manifestTemporaryTable(sqlCommand) {
+        const hash = crypto.createHash("md5").update(sqlCommand).digest("hex");
+        const tempTableName = this.temporaryTables.get(hash);
+        if (tempTableName !== undefined) {
+            return tempTableName;
+        }
+        else {
+            try {
+                const [job] = await this.bigQuery.createQueryJob({
+                    location: this.location,
+                    query: sqlCommand,
+                });
+                let [metaData] = await job.getMetadata();
+                // wait for job to complete, because we need the table name
+                // TODO just because a job is "DONE" doesn't mean it ended correctly, should probably also confirm
+                // status is successful & that table was created
+                // TODO this needs better error handling and a timeout so that issues dont result in infinite looping
+                while (metaData.status.state !== "DONE") {
+                    await new Promise((resolve) => setTimeout(resolve, 1000));
+                    [metaData] = await job.getMetadata();
+                }
+                // save table name
+                if (metaData.configuration &&
+                    metaData.configuration.query &&
+                    metaData.configuration.query.destinationTable) {
+                    const destinationTable = metaData.configuration.query.destinationTable;
+                    const fullTableName = `${destinationTable.projectId}.${destinationTable.datasetId}.${destinationTable.tableId}`;
+                    this.temporaryTables.set(hash, fullTableName);
+                    return fullTableName;
+                }
+                else {
+                    throw new Error("bigquery.job.getMetadata() - metadata should have configuration but does not");
+                }
+            }
+            catch (e) {
+                throw maybeRewriteError(e);
+            }
+        }
+    }
+    // TODO there is reasonable argument that this has nothing to do with Malloy, and should be implemented
+    // by whatever library is using Malloy.
+    async manifestPermanentTable(sqlCommand, datasetName, tableName, overwriteExistingTable = false, createDataset = false) {
+        let dataset = this.bigQuery.dataset(datasetName);
+        if (!(await dataset.exists())[0]) {
+            if (createDataset) {
+                const newDatasetResponse = await this.bigQuery.createDataset(datasetName);
+                dataset = newDatasetResponse[0];
+            }
+            else {
+                throw new Error(`Dataset ${datasetName} does not exist`);
+            }
+        }
+        const table = dataset.table(tableName);
+        if ((await table.exists())[0] && !overwriteExistingTable) {
+            throw new Error(`Table ${tableName} already exists`);
+        }
+        const [job] = await this.bigQuery.createQueryJob({
+            query: sqlCommand,
+            location: this.location,
+            destination: table,
+        });
+        // if creating this job didn't throw, there's an ID.
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        return job.id;
+    }
+    addFieldsToStructDef(structDef, tableFieldSchema) {
+        if (!tableFieldSchema.fields)
+            return;
+        for (const field of tableFieldSchema.fields) {
+            // BigQuery SDK sets type & name to optional even though they are required, assume they exist
+            const type = field.type;
+            const name = field.name;
+            // check for an array
+            if (field.mode === "REPEATED" && !["STRUCT", "RECORD"].includes(type)) {
+                const malloyType = this.bqToMalloyTypes[type];
+                if (malloyType) {
+                    const innerStructDef = {
+                        type: "struct",
+                        name,
+                        dialect: this.dialectName,
+                        structSource: { type: "nested" },
+                        structRelationship: { type: "nested", field: name, isArray: true },
+                        fields: [{ ...malloyType, name: "value" }],
+                    };
+                    structDef.fields.push(innerStructDef);
+                }
+            }
+            else if (["STRUCT", "RECORD"].includes(type)) {
+                const innerStructDef = {
+                    type: "struct",
+                    name,
+                    dialect: this.dialectName,
+                    structSource: field.mode === "REPEATED" ? { type: "nested" } : { type: "inline" },
+                    structRelationship: field.mode === "REPEATED"
+                        ? { type: "nested", field: name, isArray: false }
+                        : { type: "inline" },
+                    fields: [],
+                };
+                this.addFieldsToStructDef(innerStructDef, field);
+                structDef.fields.push(innerStructDef);
+            }
+            else {
+                const malloyType = this.bqToMalloyTypes[type];
+                if (malloyType) {
+                    structDef.fields.push({ name, ...malloyType });
+                }
+                else {
+                    structDef.fields.push({
+                        name,
+                        type: "string",
+                        e: [`BigQuery type "${type}" not supported by Malloy`],
+                    });
+                }
+            }
+        }
+    }
+    tableURLtoTablePath(tableURL) {
+        const { tablePath } = (0, malloy_2.parseTableURL)(tableURL);
+        if (tablePath.split(".").length === 2) {
+            return `${this.defaultProject}.${tablePath}`;
+        }
+        else {
+            return tablePath;
+        }
+    }
+    structDefFromTableSchema(tableURL, schemaInfo) {
+        const structDef = {
+            type: "struct",
+            name: tableURL,
+            dialect: this.dialectName,
+            structSource: {
+                type: "table",
+                tablePath: this.tableURLtoTablePath(tableURL),
+            },
+            structRelationship: { type: "basetable", connectionName: this.name },
+            fields: [],
+        };
+        this.addFieldsToStructDef(structDef, schemaInfo.schema);
+        if (schemaInfo.needsPartitionPsuedoColumn) {
+            structDef.fields.push({
+                type: "timestamp",
+                name: "_PARTITIONTIME",
+            });
+        }
+        return structDef;
+    }
+    structDefFromSQLSchema(sqlBlock, tableFieldSchema) {
+        const structDef = {
+            type: "struct",
+            name: sqlBlock.name,
+            dialect: this.dialectName,
+            structSource: {
+                type: "sql",
+                method: "subquery",
+                sqlBlock,
+            },
+            structRelationship: { type: "basetable", connectionName: this.name },
+            fields: [],
+        };
+        this.addFieldsToStructDef(structDef, tableFieldSchema);
+        return structDef;
+    }
+    async fetchSchemaForTables(missing) {
+        const schemas = {};
+        const errors = {};
+        for (const tableURL of missing) {
+            let inCache = this.schemaCache.get(tableURL);
+            if (!inCache) {
+                try {
+                    const tableFieldSchema = await this.getTableFieldSchema(tableURL);
+                    inCache = {
+                        schema: this.structDefFromTableSchema(tableURL, tableFieldSchema),
+                    };
+                    this.schemaCache.set(tableURL, inCache);
+                }
+                catch (error) {
+                    inCache = { error: error.message };
+                }
+            }
+            if (inCache.schema !== undefined) {
+                schemas[tableURL] = inCache.schema;
+            }
+            else {
+                errors[tableURL] = inCache.error;
+            }
+        }
+        return { schemas, errors };
+    }
+    async getSQLBlockSchema(sqlRef) {
+        // We do a simple retry-loop here, as a temporary fix for a transient
+        // error in which sometimes requesting results from a job yields an
+        // access denied error. It seems that in these cases, simply trying again
+        // solves the problem. This is being currently investigated by
+        // @christopherswenson and @lloydtabb. Same as below.
+        let lastFetchError;
+        for (let retries = 0; retries < 3; retries++) {
+            try {
+                const [job] = await this.bigQuery.createQueryJob({
+                    location: this.location,
+                    query: sqlRef.select,
+                    dryRun: true,
+                });
+                return job.metadata.statistics.query.schema;
+            }
+            catch (fetchError) {
+                lastFetchError = fetchError;
+            }
+        }
+        throw lastFetchError;
+    }
+    async fetchSchemaForSQLBlocks(sqlRefs) {
+        const schemas = {};
+        const errors = {};
+        for (const sqlRef of sqlRefs) {
+            const key = sqlRef.name;
+            let inCache = this.sqlSchemaCache.get(key);
+            if (!inCache) {
+                try {
+                    const tableFieldSchema = await this.getSQLBlockSchema(sqlRef);
+                    inCache = {
+                        schema: this.structDefFromSQLSchema(sqlRef, tableFieldSchema),
+                    };
+                    this.schemaCache.set(key, inCache);
+                }
+                catch (error) {
+                    inCache = { error: error.message };
+                }
+            }
+            if (inCache.schema !== undefined) {
+                schemas[key] = inCache.schema;
+            }
+            else {
+                errors[key] = inCache.error;
+            }
+        }
+        return { schemas, errors };
+    }
+    // TODO this needs to extend the wait for results using a timeout set by the user,
+    // and probably needs to loop to check for results - BQ docs now say that after ~2min of waiting,
+    // no matter what you set for timeoutMs, they will probably just return.
+    async createBigQueryJobAndGetResults(sqlCommand, createQueryJobOptions, getQueryResultsOptions) {
+        try {
+            const job = await this.createBigQueryJob({
+                query: sqlCommand,
+                ...createQueryJobOptions,
+            });
+            // TODO we should check if this is still required?
+            // We do a simple retry-loop here, as a temporary fix for a transient
+            // error in which sometimes requesting results from a job yields an
+            // access denied error. It seems that in these cases, simply trying again
+            // solves the problem. This is being currently investigated by
+            // @christopherswenson and @lloydtabb.
+            let lastFetchError;
+            for (let retries = 0; retries < 3; retries++) {
+                try {
+                    return await job.getQueryResults({
+                        timeoutMs: 1000 * 60 * 2,
+                        ...getQueryResultsOptions,
+                    });
+                }
+                catch (fetchError) {
+                    lastFetchError = fetchError;
+                }
+            }
+            throw lastFetchError;
+        }
+        catch (e) {
+            throw maybeRewriteError(e);
+        }
+    }
+    async createBigQueryJob(createQueryJobOptions) {
+        const [job] = await this.bigQuery.createQueryJob({
+            location: this.location,
+            maximumBytesBilled: this.config.maximumBytesBilled || MAXIMUM_BYTES_BILLED,
+            jobTimeoutMs: Number(this.config.timeoutMs) || TIMEOUT_MS,
+            ...createQueryJobOptions,
+        });
+        return job;
+    }
+    runSQLStream(sqlCommand, options = {}) {
+        const bigQuery = this.bigQuery;
+        function streamBigQuery(onError, onData, onEnd) {
+            let index = 0;
+            function handleData(rowMetadata) {
+                onData(rowMetadata);
+                index += 1;
+                if (options.rowLimit !== undefined && index >= options.rowLimit) {
+                    this.end();
+                }
+            }
+            bigQuery
+                .createQueryStream(sqlCommand)
+                .on("error", onError)
+                .on("data", handleData)
+                .on("end", onEnd);
+        }
+        return (0, malloy_1.toAsyncGenerator)(streamBigQuery);
+    }
+}
+exports.BigQueryConnection = BigQueryConnection;
+BigQueryConnection.DEFAULT_QUERY_OPTIONS = {
+    rowLimit: 10,
+};
+//# sourceMappingURL=bigquery_connection.js.map

package/dist/index.d.ts

@@ -0,0 +1 @@
+export { BigQueryConnection } from "./bigquery_connection";

package/dist/index.js

@@ -0,0 +1,18 @@
+"use strict";
+/*
+ * Copyright 2021 Google LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BigQueryConnection = void 0;
+var bigquery_connection_1 = require("./bigquery_connection");
+Object.defineProperty(exports, "BigQueryConnection", { enumerable: true, get: function () { return bigquery_connection_1.BigQueryConnection; } });
+//# sourceMappingURL=index.js.map

package/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "@malloydata/db-bigquery",
+  "version": "0.0.1",
+  "license": "GPL-2.0",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "lint": "eslint '**/*.ts{,x}'",
+    "lint-fix": "eslint '**/*.ts{,x}' --fix",
+    "test": "jest --config=../../jest.config.js",
+    "build": "tsc --build",
+    "malloyc": "ts-node ../../scripts/malloy-to-json"
+  },
+  "dependencies": {
+    "@google-cloud/bigquery": "^5.5.0",
+    "@google-cloud/common": "^3.6.0",
+    "@malloydata/malloy": "^0.0.1",
+    "gaxios": "^4.2.0"
+  }
+}