@malloydata/db-bigquery 0.0.430 → 0.0.432

package/dist/bigquery_connection.d.ts

@@ -1,7 +1,7 @@
-import type { Job, PagedResponse, Query, QueryResultsOptions, RowMetadata } from '@google-cloud/bigquery';
+import type { BigQueryOptions, Job, PagedResponse, Query, QueryResultsOptions, RowMetadata } from '@google-cloud/bigquery';
 import type bigquery from '@google-cloud/bigquery/build/src/types';
 import type { ResourceStream } from '@google-cloud/paginator';
-import type { Connection, ConnectionConfig, ConnectionParameterValue, MalloyQueryData, PersistSQLResults, QueryData, QueryRecord, QueryOptionsReader, QueryRunStats, RunSQLOptions, StreamingConnection, TableSourceDef, SQLSourceDef, SQLSourceRequest } from '@malloydata/malloy';
+import type { Connection, ConnectionConfig, ConnectionConfigEntry, ConnectionParameterValue, MalloyQueryData, PersistSQLResults, QueryData, QueryRecord, QueryOptionsReader, QueryRunStats, RunSQLOptions, StreamingConnection, TableSourceDef, SQLSourceDef, SQLSourceRequest } from '@malloydata/malloy';
 import type { TableMetadata } from '@malloydata/malloy/connection';
 import { BaseConnection } from '@malloydata/malloy/connection';
 export interface BigQueryManagerOptions {
@@ -17,6 +17,12 @@ interface CredentialBody {
     client_email?: string;
     private_key?: string;
 }
+/**
+ * A caller-supplied credential. Anything Malloy cannot build from config text
+ * — impersonation, workload identity federation, a proxied or test credential
+ * — arrives as one of these, from a host overlay.
+ */
+type AuthClient = BigQueryOptions['authClient'];
 interface BigQueryConnectionConfiguration {
     /** This ID is used for Bigquery Table Normalization */
     projectId?: string;
@@ -28,6 +34,7 @@ interface BigQueryConnectionConfiguration {
     credentials?: CredentialBody | {
         [key: string]: ConnectionParameterValue;
     };
+    authClient?: AuthClient;
     setupSQL?: string;
 }
 interface BigQueryConnectionOptions extends ConnectionConfig {
@@ -37,6 +44,15 @@ interface BigQueryConnectionOptions extends ConnectionConfig {
     serviceAccountKey?: {
         [key: string]: ConnectionParameterValue;
     };
+    /** The key file's contents, as a JSON string or base64-encoded JSON. */
+    serviceAccountKeyJson?: string;
+    authClient?: AuthClient;
+    /**
+     * The connection's entry as written, before overlay resolution. Set by the
+     * registered factory; needed only so `getDigest` can identify which auth
+     * client this connection got. See `authIdentityOf`.
+     */
+    rawConfigData?: ConnectionConfigEntry;
     location?: string;
     maximumBytesBilled?: string;
     timeoutMs?: string;
@@ -91,6 +107,7 @@ export declare class BigQueryConnection extends BaseConnection implements Connec
     private config;
     private location?;
     private setupSQL;
+    private authIdentity;
     constructor(option: BigQueryConnectionOptions, queryOptions?: QueryOptionsReader);
     constructor(name: string, queryOptions?: QueryOptionsReader, config?: BigQueryConnectionConfiguration);
     get dialectName(): string;

package/dist/bigquery_connection.js

@@ -44,6 +44,115 @@ const googleCommon = __importStar(require("@google-cloud/common"));
 const gaxios_1 = require("gaxios");
 const malloy_1 = require("@malloydata/malloy");
 const connection_1 = require("@malloydata/malloy/connection");
+/**
+ * What to hash for the auth client, given the connection's unresolved config.
+ *
+ * An `AuthClient` is a live object with nothing stable to hash, so the digest
+ * uses the reference the config named it by — `{tenantAuth: "acme"}`. Without
+ * it, two connections against the same project holding different impersonated
+ * identities produce the same digest, and so the same BuildIDs, and one
+ * tenant is served rows persisted for another.
+ *
+ * The digest is therefore only as distinguishing as the reference path: an
+ * overlay whose path doesn't vary by identity leaves them indistinguishable
+ * here too.
+ */
+function authIdentityOf(rawConfigData) {
+    const reference = rawConfigData === null || rawConfigData === void 0 ? void 0 : rawConfigData['authClient'];
+    return reference === undefined ? undefined : JSON.stringify(reference);
+}
+/**
+ * An `authClient` and a service account key are two answers to one question,
+ * and the SDK does not treat them as competing: `GoogleAuth` caches the
+ * `authClient` and never consults `credentials` or `keyFilename` again
+ * (`google-auth-library`, `googleauth.js`: `cachedCredential = opts.authClient`).
+ * A config carrying both therefore runs entirely on the auth client while the
+ * key sits there looking live, and whoever wrote it believes the wrong
+ * identity is executing their queries. Refuse it instead.
+ */
+function rejectCompetingCredentials(name, config) {
+    if (config.authClient === undefined)
+        return;
+    const alsoSet = [
+        config.credentials !== undefined ? 'a service account key' : undefined,
+        config.serviceAccountKeyPath !== undefined
+            ? 'serviceAccountKeyPath'
+            : undefined,
+    ].filter(what => what !== undefined);
+    if (alsoSet.length === 0)
+        return;
+    throw new Error(`Connection "${name}" sets authClient and also ${alsoSet.join(' and ')}. ` +
+        'An authClient replaces the credential entirely — the key would be ' +
+        'ignored — so supply one or the other.');
+}
+function isJsonObject(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+/**
+ * The two credential shapes the SDK's `credentials` option accepts: a service
+ * account key, and an external account (workload identity federation) config,
+ * which carries no key of its own. Anything else — `{}`, a config file, half a
+ * key — is rejected here rather than at the first query, where it arrives as
+ * "the incoming JSON object does not contain a client_email field" and names
+ * nothing that would lead back to this property.
+ */
+function isCredentialObject(value) {
+    return ((typeof value['client_email'] === 'string' &&
+        typeof value['private_key'] === 'string') ||
+        value['type'] === 'external_account');
+}
+function isProbablyBase64(text) {
+    // `{` is not in the base64 alphabet, so JSON can never be mistaken for an
+    // encoding of itself. Everything else is treated as base64 and allowed to
+    // fail the parse below, which keeps the check to the one thing it can be
+    // certain about.
+    return !text.startsWith('{');
+}
+/**
+ * A service account key that arrives as a *string* rather than as structured
+ * config, in either of the two encodings a deployment can produce.
+ *
+ * `serviceAccountKey` is a `json`-typed property, and a json-typed slot takes
+ * its value literally: an `{env: "..."}` reference is never resolved in one,
+ * because reference indirection into structured config is exactly what the
+ * config compiler refuses. A deployment whose credentials live in an
+ * environment variable — the normal shape for a server — therefore cannot
+ * reach that slot at all, and the literal `{env: "..."}` object it does reach
+ * the SDK with fails as "the incoming JSON object does not contain a
+ * client_email field". This is a string slot, so a reference resolves and the
+ * value arrives here to be parsed.
+ *
+ * Base64 is accepted because a JSON object full of quotes and braces survives
+ * a shell, a CI secret editor, and a `.env` file poorly; base64 is one token
+ * that survives all three. Which one arrived is detected rather than declared,
+ * so a deployment that switches encodings doesn't also have to edit config.
+ *
+ * Nothing from `text` reaches the error message. It is a private key, and
+ * JSON.parse's own SyntaxError quotes the input it choked on.
+ */
+function credentialsFromJson(text) {
+    // Buffer's base64 decoder drops characters outside the alphabet rather than
+    // rejecting them, so a mangled value decodes to garbage instead of throwing.
+    // The parse below is what catches that, which is why one message has to
+    // cover both encodings: at this point either could have been intended.
+    const json = isProbablyBase64(text)
+        ? Buffer.from(text, 'base64').toString('utf8')
+        : text;
+    let parsed;
+    try {
+        parsed = JSON.parse(json);
+    }
+    catch {
+        throw new Error('serviceAccountKeyJson is neither JSON nor base64-encoded JSON. It ' +
+            'must hold the entire service account key file.');
+    }
+    if (!isJsonObject(parsed) || !isCredentialObject(parsed)) {
+        throw new Error('serviceAccountKeyJson parsed but is not a service account key: it ' +
+            'has no client_email and private_key. It must hold the entire key ' +
+            'file, not a fragment of one.');
+    }
+    return parsed;
+}
 // BigQuery label grammar: keys and values are lowercase, <=63 chars, and
 // [a-z0-9_-]; keys must start with a lowercase letter. Values are transformed
 // to fit (lowercase, disallowed chars -> '_', truncate); a key that can't be
@@ -294,12 +403,25 @@ class BigQueryConnection extends connection_1.BaseConnection {
             this.name = arg;
         }
         else {
-            const { name, client_email, private_key, serviceAccountKey, ...args } = arg;
+            // Every key-bearing property is destructured out of `args`, so a key
+            // never lands in `this.config` — only the credentials object the SDK
+            // needs does. `rawConfigData` comes out for the same reason: it is
+            // read once, for the digest, and is not connection config.
+            const { name, client_email, private_key, serviceAccountKey, serviceAccountKeyJson, rawConfigData, ...args } = arg;
             this.name = name;
+            this.authIdentity = authIdentityOf(rawConfigData);
             config = args;
+            // Trimmed before it is looked at: a value that came through a here-doc,
+            // a `$(cat key.json)`, or a secret-store copy tends to carry a trailing
+            // newline, and both the encoding sniff and the emptiness check below
+            // would otherwise read it as content.
+            const keyJson = serviceAccountKeyJson === null || serviceAccountKeyJson === void 0 ? void 0 : serviceAccountKeyJson.trim();
             if (serviceAccountKey) {
                 config.credentials = serviceAccountKey;
             }
+            else if (keyJson) {
+                config.credentials = credentialsFromJson(keyJson);
+            }
             else if (client_email || private_key) {
                 config.credentials = {
                     client_email,
@@ -307,10 +429,12 @@ class BigQueryConnection extends connection_1.BaseConnection {
                 };
             }
         }
+        rejectCompetingCredentials(this.name, config);
         this.bigQuery = new bigquery_1.BigQuery({
             userAgent: `Malloy/${malloy_1.Malloy.version}`,
             keyFilename: config.serviceAccountKeyPath,
             credentials: config.credentials,
+            authClient: config.authClient,
             projectId: config.billingProjectId,
         });
         // record project ID because for unclear reasons we have to modify the project ID on the SDK when
@@ -354,7 +478,11 @@ class BigQueryConnection extends connection_1.BaseConnection {
         return true;
     }
     getDigest() {
-        return (0, malloy_1.makeDigest)('bigquery', this.projectId, this.setupSQL);
+        // Appended only when an auth client is in play, so connections that don't
+        // use one keep the digests — and so the persisted tables — they have now.
+        return this.authIdentity === undefined
+            ? (0, malloy_1.makeDigest)('bigquery', this.projectId, this.setupSQL)
+            : (0, malloy_1.makeDigest)('bigquery', this.projectId, this.setupSQL, this.authIdentity);
     }
     get supportsNesting() {
         return true;

package/dist/index.js

@@ -11,8 +11,11 @@ const malloy_1 = require("@malloydata/malloy");
 const bigquery_connection_2 = require("./bigquery_connection");
 (0, malloy_1.registerConnectionType)('bigquery', {
     displayName: 'BigQuery',
-    factory: async (config) => {
-        return new bigquery_connection_2.BigQueryConnection(config);
+    factory: async (config, rawConfigData) => {
+        // rawConfigData rides in the options bag so `getDigest` can tell which
+        // auth client this connection was given; the constructor keeps it out of
+        // the retained config.
+        return new bigquery_connection_2.BigQueryConnection({ ...config, rawConfigData });
     },
     properties: [
         {
@@ -34,6 +37,39 @@ const bigquery_connection_2 = require("./bigquery_connection");
             type: 'json',
             optional: true,
         },
+        // The string-typed twin of `serviceAccountKey`. A `json` slot holds its
+        // value literally — an `{env: "..."}` reference is never resolved in one —
+        // so a deployment holding its key in an environment variable has no way to
+        // use the slot above. This is a `secret` string, where references do
+        // resolve, and the connection parses what arrives.
+        {
+            name: 'serviceAccountKeyJson',
+            displayName: 'Service Account Key (JSON)',
+            type: 'secret',
+            optional: true,
+            description: 'The entire service account key file, as JSON or base64-encoded ' +
+                'JSON (detected automatically), for supplying the key from an ' +
+                'environment variable or secret manager rather than from disk.',
+        },
+        // A google-auth-library AuthClient, for credentials Malloy cannot build
+        // from config text: impersonation, workload identity federation, a proxied
+        // or test credential. The config file names an overlay, the host registers
+        // it, and the value never appears in the file — which is what `opaque` plus
+        // `source: 'overlay'` say. `mustHaveValue` is the safety half: an overlay
+        // that isn't registered would otherwise drop the property and quietly fall
+        // back to ambient credentials.
+        //
+        // When one is supplied, the SDK resolves the project id through it rather
+        // than through ambient credentials, so set billingProjectId alongside.
+        {
+            name: 'authClient',
+            displayName: 'Auth Client',
+            type: 'opaque',
+            source: 'overlay',
+            mustHaveValue: true,
+            optional: true,
+            description: 'Not settable from the UI; supplied by the host.',
+        },
         {
             name: 'location',
             displayName: 'Location',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/db-bigquery",
-  "version": "0.0.430",
+  "version": "0.0.432",
   "license": "MIT",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -26,7 +26,7 @@
     "@google-cloud/bigquery": "7.9.4",
     "@google-cloud/common": "5.0.2",
     "@google-cloud/paginator": "5.0.2",
-    "@malloydata/malloy": "0.0.430",
+    "@malloydata/malloy": "0.0.432",
     "gaxios": "^4.2.0"
   }
 }