@lightdash/warehouses 2.520.0 → 2.522.0

package/dist/index.d.ts

@@ -16,5 +16,4 @@ export * from './warehouseClients/redshiftIamCredentials';
 export * from './warehouseClients/SnowflakeWarehouseClient';
 export * from './warehouseClients/TrinoWarehouseClient';
 export * from './warehouseSqlBuilderFromType';
-export * from './agentAccess/BigqueryAgentAccessProbe';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,SAAS,CAAC;AACxB,cAAc,qCAAqC,CAAC;AACpD,cAAc,kCAAkC,CAAC;AACjD,cAAc,0CAA0C,CAAC;AACzD,cAAc,4CAA4C,CAAC;AAC3D,cAAc,8CAA8C,CAAC;AAC7D,cAAc,0CAA0C,CAAC;AACzD,cAAc,yCAAyC,CAAC;AACxD,OAAO,EAAE,uBAAuB,EAAE,MAAM,4CAA4C,CAAC;AACrF,YAAY,EAAE,oBAAoB,EAAE,MAAM,4CAA4C,CAAC;AACvF,cAAc,2DAA2D,CAAC;AAC1E,cAAc,4CAA4C,CAAC;AAC3D,cAAc,4CAA4C,CAAC;AAC3D,cAAc,2CAA2C,CAAC;AAC1D,cAAc,6CAA6C,CAAC;AAC5D,cAAc,yCAAyC,CAAC;AACxD,cAAc,+BAA+B,CAAC;AAE9C,cAAc,wCAAwC,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,SAAS,CAAC;AACxB,cAAc,qCAAqC,CAAC;AACpD,cAAc,kCAAkC,CAAC;AACjD,cAAc,0CAA0C,CAAC;AACzD,cAAc,4CAA4C,CAAC;AAC3D,cAAc,8CAA8C,CAAC;AAC7D,cAAc,0CAA0C,CAAC;AACzD,cAAc,yCAAyC,CAAC;AACxD,OAAO,EAAE,uBAAuB,EAAE,MAAM,4CAA4C,CAAC;AACrF,YAAY,EAAE,oBAAoB,EAAE,MAAM,4CAA4C,CAAC;AACvF,cAAc,2DAA2D,CAAC;AAC1E,cAAc,4CAA4C,CAAC;AAC3D,cAAc,4CAA4C,CAAC;AAC3D,cAAc,2CAA2C,CAAC;AAC1D,cAAc,6CAA6C,CAAC;AAC5D,cAAc,yCAAyC,CAAC;AACxD,cAAc,+BAA+B,CAAC"}

package/dist/index.js

@@ -33,4 +33,3 @@ __exportStar(require("./warehouseClients/redshiftIamCredentials"), exports);
 __exportStar(require("./warehouseClients/SnowflakeWarehouseClient"), exports);
 __exportStar(require("./warehouseClients/TrinoWarehouseClient"), exports);
 __exportStar(require("./warehouseSqlBuilderFromType"), exports);
-__exportStar(require("./agentAccess/BigqueryAgentAccessProbe"), exports);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lightdash/warehouses",
-  "version": "2.520.0",
+  "version": "2.522.0",
   "description": "Warehouse connectors for Lightdash",
   "license": "MIT",
   "repository": {
@@ -21,7 +21,7 @@
     "@databricks/sql": "1.13.0",
     "@duckdb/node-api": "1.5.2-r.2",
     "@google-cloud/bigquery": "8.3.1",
-    "@lightdash/common": "2.520.0",
+    "@lightdash/common": "2.522.0",
     "node-fetch": "2.7.0",
     "pg": "8.13.1",
     "pg-cursor": "2.10.0",

package/dist/agentAccess/BigqueryAgentAccessProbe.d.ts

@@ -1,31 +0,0 @@
-import { type AgentAccessDataset, type AgentAccessReport, type AgentTableStatus } from '@lightdash/common';
-import { BigqueryWarehouseClient } from '../warehouseClients/BigqueryWarehouseClient';
-export type AgentAccessInventoryTable = AgentAccessDataset & {
-    name: string;
-    location: string;
-};
-type FailureReason = NonNullable<AgentAccessReport['failureReason']>;
-export declare class AgentAccessProbeError extends Error {
-    readonly reason: FailureReason;
-    constructor(reason: FailureReason);
-}
-export declare const classifyBigqueryAccessError: (error: unknown, phase?: 'identity' | 'tables') => {
-    kind: 'fatal';
-    reason: FailureReason;
-} | Exclude<AgentTableStatus, {
-    kind: 'readable';
-} | {
-    kind: 'not_checked';
-}>;
-export declare class BigqueryAgentAccessProbe {
-    private readonly warehouse;
-    private accessToken;
-    constructor(warehouse: BigqueryWarehouseClient);
-    private withDeadline;
-    private request;
-    principal(deadline: number): Promise<string>;
-    listTables(dataset: AgentAccessDataset, deadline: number): Promise<AgentAccessInventoryTable[]>;
-    probe(table: AgentAccessInventoryTable, deadline: number): Promise<AgentTableStatus>;
-}
-export {};
-//# sourceMappingURL=BigqueryAgentAccessProbe.d.ts.map

package/dist/agentAccess/BigqueryAgentAccessProbe.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"BigqueryAgentAccessProbe.d.ts","sourceRoot":"","sources":["../../src/agentAccess/BigqueryAgentAccessProbe.ts"],"names":[],"mappings":"AACA,OAAO,EACH,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACxB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,uBAAuB,EAAE,MAAM,6CAA6C,CAAC;AAEtF,MAAM,MAAM,yBAAyB,GAAG,kBAAkB,GAAG;IACzD,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,KAAK,aAAa,GAAG,WAAW,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC,CAAC;AAErE,qBAAa,qBAAsB,SAAQ,KAAK;aAChB,MAAM,EAAE,aAAa;IAAjD,YAA4B,MAAM,EAAE,aAAa,EAGhD;CACJ;AAOD,eAAO,MAAM,2BAA2B,UAC7B,OAAO,UACP,UAAU,GAAG,QAAQ,KAE1B;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,aAAa,CAAA;CAAE,GACxC,OAAO,CACH,gBAAgB,EAChB;IAAE,IAAI,EAAE,UAAU,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,CA8EvD,CAAC;AAKF,qBAAa,wBAAwB;IAGrB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAFtC,OAAO,CAAC,WAAW,CAAgC;IAEnD,YAA6B,SAAS,EAAE,uBAAuB,EAAI;YAErD,YAAY;YAuBZ,OAAO;IA0Cf,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CA4BjD;IAEK,UAAU,CACZ,OAAO,EAAE,kBAAkB,EAC3B,QAAQ,EAAE,MAAM,GACjB,OAAO,CAAC,yBAAyB,EAAE,CAAC,CAiCtC;IAEK,KAAK,CACP,KAAK,EAAE,yBAAyB,EAChC,QAAQ,EAAE,MAAM,GACjB,OAAO,CAAC,gBAAgB,CAAC,CAyC3B;CACJ"}

package/dist/agentAccess/BigqueryAgentAccessProbe.js

@@ -1,210 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.BigqueryAgentAccessProbe = exports.classifyBigqueryAccessError = exports.AgentAccessProbeError = void 0;
-class AgentAccessProbeError extends Error {
-    constructor(reason) {
-        super(reason);
-        this.reason = reason;
-        this.name = 'AgentAccessProbeError';
-    }
-}
-exports.AgentAccessProbeError = AgentAccessProbeError;
-const record = (value) => typeof value === 'object' && value !== null
-    ? value
-    : {};
-const classifyBigqueryAccessError = (error, phase = 'tables') => {
-    if (error instanceof AgentAccessProbeError)
-        return { kind: 'fatal', reason: error.reason };
-    const root = record(error);
-    const response = record(root.response);
-    const data = record(response.data);
-    const detail = record(data.error);
-    const errors = root.errors ?? detail.errors;
-    const entries = Array.isArray(errors) ? errors.map(record) : [];
-    const reasons = entries.map((entry) => entry.reason);
-    const code = detail.code ?? response.status ?? root.code;
-    const message = [
-        root.message,
-        detail.message,
-        ...entries.map((entry) => entry.message),
-    ]
-        .filter((value) => typeof value === 'string')
-        .join(' ');
-    if (code === 401 ||
-        reasons.some((reason) => ['authError', 'invalidCredentials', 'invalid_grant'].includes(String(reason))) ||
-        data.error === 'invalid_grant')
-        return { kind: 'fatal', reason: 'invalid_credentials' };
-    if (reasons.some((reason) => [
-        'rateLimitExceeded',
-        'jobRateLimitExceeded',
-        'quotaExceeded',
-        'resourcesExceeded',
-    ].includes(String(reason))) ||
-        code === 429)
-        return { kind: 'error', reason: 'quota' };
-    if (message.includes('bigquery.jobs.create'))
-        return { kind: 'fatal', reason: 'job_permission_denied' };
-    if (reasons.includes('accessDenied'))
-        return { kind: 'blocked', reason: 'access_denied' };
-    if (reasons.includes('notFound') || code === 404)
-        return { kind: 'error', reason: 'not_found' };
-    if (['ETIMEDOUT', 'ESOCKETTIMEDOUT', 'ABORT_ERR'].includes(String(code)) ||
-        root.name === 'AbortError' ||
-        root.name === 'TimeoutError')
-        return { kind: 'error', reason: 'timeout' };
-    if (reasons.some((reason) => ['invalidQuery', 'invalid', 'notImplemented'].includes(String(reason))))
-        return { kind: 'error', reason: 'unsupported' };
-    if ((typeof code === 'number' && code >= 500) ||
-        ['ECONNRESET', 'ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN'].includes(String(code)) ||
-        reasons.includes('backendError'))
-        return { kind: 'error', reason: 'unavailable' };
-    if (code === 403)
-        return { kind: 'fatal', reason: 'warehouse_denied' };
-    if (phase === 'identity' &&
-        root.response == null &&
-        typeof code !== 'number' &&
-        entries.length === 0)
-        return { kind: 'fatal', reason: 'invalid_credentials' };
-    return { kind: 'error', reason: 'unknown' };
-};
-exports.classifyBigqueryAccessError = classifyBigqueryAccessError;
-const quoteTable = (table) => `\`${[table.database, table.schema, table.name].map((part) => part.replaceAll('\\', '\\\\').replaceAll('`', '\\`')).join('.')}\``;
-class BigqueryAgentAccessProbe {
-    constructor(warehouse) {
-        this.warehouse = warehouse;
-        this.accessToken = null;
-    }
-    async withDeadline(operation, deadline) {
-        const timeout = Math.min(5000, deadline - Date.now());
-        if (timeout <= 0)
-            throw new AgentAccessProbeError('timeout');
-        const controller = new AbortController();
-        let timer;
-        try {
-            return await Promise.race([
-                operation(controller.signal),
-                new Promise((_, reject) => {
-                    timer = setTimeout(() => {
-                        controller.abort();
-                        reject(new AgentAccessProbeError('timeout'));
-                    }, timeout);
-                }),
-            ]);
-        }
-        finally {
-            clearTimeout(timer);
-        }
-    }
-    async request(path, method, data, deadline) {
-        const requestDeadline = Math.min(Date.now() + 5000, deadline);
-        return this.withDeadline(async (signal) => {
-            this.accessToken ?? (this.accessToken = this.withDeadline(async () => {
-                const token = await this.warehouse.client.authClient.getAccessToken();
-                if (!token)
-                    throw new AgentAccessProbeError('invalid_credentials');
-                return token;
-            }, requestDeadline));
-            const token = await this.accessToken;
-            signal.throwIfAborted();
-            if (Date.now() >= requestDeadline)
-                throw new AgentAccessProbeError('timeout');
-            const response = await fetch(`${this.warehouse.client.baseUrl}/${path}`, {
-                method,
-                headers: {
-                    Authorization: `Bearer ${token}`,
-                    'Content-Type': 'application/json',
-                },
-                ...(data === null ? {} : { body: JSON.stringify(data) }),
-                signal,
-            });
-            if (!response.ok) {
-                const body = await response.json().catch(() => null);
-                throw Object.assign(new Error('The warehouse rejected the access check.'), { response: { status: response.status, data: body } });
-            }
-            return response.json();
-        }, requestDeadline);
-    }
-    async principal(deadline) {
-        const project = this.warehouse.credentials.executionProject ||
-            this.warehouse.credentials.project;
-        const response = await this.request(`projects/${encodeURIComponent(project)}/queries`, 'POST', {
-            query: 'SELECT SESSION_USER() AS principal',
-            useLegacySql: false,
-            useQueryCache: false,
-            labels: { agent: 'true' },
-            location: this.warehouse.credentials.location || undefined,
-            timeoutMs: 4000,
-            jobTimeoutMs: '5000',
-        }, deadline);
-        if (response.errors?.length)
-            throw Object.assign(new Error('The warehouse rejected the identity check.'), { errors: response.errors });
-        if (!response.jobComplete)
-            throw new AgentAccessProbeError('timeout');
-        const principal = response.rows?.[0]?.f?.[0]?.v;
-        if (typeof principal !== 'string' || principal.length === 0)
-            throw new AgentAccessProbeError('unknown');
-        return principal;
-    }
-    async listTables(dataset, deadline) {
-        const path = `projects/${encodeURIComponent(dataset.database)}/datasets/${encodeURIComponent(dataset.schema)}`;
-        const metadata = await this.request(path, 'GET', null, deadline);
-        const { location } = metadata;
-        if (!location)
-            throw new AgentAccessProbeError('baseline_failed');
-        const tables = [];
-        const listPage = async (pageToken) => {
-            const query = new URLSearchParams({ maxResults: '1000' });
-            if (pageToken !== null)
-                query.set('pageToken', pageToken);
-            const page = await this.request(`${path}/tables?${query}`, 'GET', null, deadline);
-            for (const table of page.tables ?? []) {
-                if (!table.tableReference?.tableId)
-                    throw new AgentAccessProbeError('baseline_failed');
-                tables.push({
-                    ...dataset,
-                    name: table.tableReference.tableId,
-                    location,
-                });
-            }
-            if (page.nextPageToken)
-                await listPage(page.nextPageToken);
-        };
-        await listPage(null);
-        return tables;
-    }
-    async probe(table, deadline) {
-        try {
-            const project = this.warehouse.credentials.executionProject ||
-                this.warehouse.credentials.project;
-            const result = await this.request(`projects/${encodeURIComponent(project)}/jobs`, 'POST', {
-                jobReference: {
-                    projectId: project,
-                    location: table.location,
-                },
-                configuration: {
-                    dryRun: true,
-                    labels: { agent: 'true' },
-                    jobTimeoutMs: '5000',
-                    query: {
-                        query: `SELECT * FROM ${quoteTable(table)}`,
-                        useLegacySql: false,
-                        useQueryCache: false,
-                    },
-                },
-            }, deadline);
-            if (result.status?.errorResult)
-                throw Object.assign(new Error('The warehouse rejected the access check.'), { errors: [result.status.errorResult] });
-            return { kind: 'readable', reason: null };
-        }
-        catch (error) {
-            const status = (0, exports.classifyBigqueryAccessError)(error);
-            if (status.kind === 'fatal') {
-                if (status.reason === 'timeout')
-                    return { kind: 'error', reason: 'timeout' };
-                throw new AgentAccessProbeError(status.reason);
-            }
-            return status;
-        }
-    }
-}
-exports.BigqueryAgentAccessProbe = BigqueryAgentAccessProbe;