@lightdash/warehouses 3.135.1 → 3.136.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo.build +1 -1
- package/dist/warehouseClients/PostgresWarehouseClient.d.ts +21 -6
- package/dist/warehouseClients/PostgresWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/PostgresWarehouseClient.js +21 -0
- package/dist/warehouseClients/RedshiftWarehouseClient.d.ts +12 -7
- package/dist/warehouseClients/RedshiftWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/RedshiftWarehouseClient.js +137 -1
- package/dist/warehouseClients/WarehouseBaseClient.d.ts +2 -1
- package/dist/warehouseClients/WarehouseBaseClient.d.ts.map +1 -1
- package/dist/warehouseClients/WarehouseBaseClient.js +2 -1
- package/package.json +2 -2
|
@@ -6,6 +6,11 @@ import WarehouseBaseClient from './WarehouseBaseClient';
|
|
|
6
6
|
import WarehouseBaseSqlBuilder from './WarehouseBaseSqlBuilder';
|
|
7
7
|
export declare const canConnectToPostgresDatabaseName: (database: string) => boolean;
|
|
8
8
|
export declare const unopenablePostgresDatabaseMessage: (database: string) => string;
|
|
9
|
+
declare module 'pg' {
|
|
10
|
+
interface ClientBase {
|
|
11
|
+
processID: number | null;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
9
14
|
export declare enum PostgresTypes {
|
|
10
15
|
INTEGER = "integer",
|
|
11
16
|
INT = "int",
|
|
@@ -58,6 +63,20 @@ export declare class PostgresSqlBuilder extends WarehouseBaseSqlBuilder {
|
|
|
58
63
|
getMetricSql(sql: string, metric: Metric): string;
|
|
59
64
|
concatString(...args: string[]): string;
|
|
60
65
|
}
|
|
66
|
+
export type PostgresStreamQueryOptions = {
|
|
67
|
+
values?: AnyType[];
|
|
68
|
+
tags?: Record<string, string>;
|
|
69
|
+
timezone?: string;
|
|
70
|
+
recoverStalledQuery?: boolean;
|
|
71
|
+
onPhaseTiming?: (phase: WarehouseQueryPhase, durationMs: number) => void;
|
|
72
|
+
};
|
|
73
|
+
export type PostgresQueryAttempt = {
|
|
74
|
+
getBackendPid: () => number | null;
|
|
75
|
+
hasSentQuery: () => boolean;
|
|
76
|
+
hasReceivedRow: () => boolean;
|
|
77
|
+
/** Fails the attempt unless a row was already streamed; returns whether it did. */
|
|
78
|
+
abort: (error: WarehouseQueryError) => boolean;
|
|
79
|
+
};
|
|
61
80
|
type CatalogQueryFilters = {
|
|
62
81
|
databaseFilters: string[];
|
|
63
82
|
schemaFilters: string[];
|
|
@@ -76,12 +95,8 @@ export declare class PostgresClient<T extends CreatePostgresLikeCredentials> ext
|
|
|
76
95
|
private getSQLWithMetadata;
|
|
77
96
|
static convertQueryResultFields(fields: QueryResult<AnyType>['fields']): WarehouseResults['fields'];
|
|
78
97
|
private static getNoticeError;
|
|
79
|
-
streamQuery(sql: string, streamCallback: (data: WarehouseResults) => void | Promise<void>, options:
|
|
80
|
-
|
|
81
|
-
tags?: Record<string, string>;
|
|
82
|
-
timezone?: string;
|
|
83
|
-
onPhaseTiming?: (phase: WarehouseQueryPhase, durationMs: number) => void;
|
|
84
|
-
}): Promise<void>;
|
|
98
|
+
streamQuery(sql: string, streamCallback: (data: WarehouseResults) => void | Promise<void>, options: PostgresStreamQueryOptions): Promise<void>;
|
|
99
|
+
protected streamQueryAttempt(sql: string, streamCallback: (data: WarehouseResults) => void | Promise<void>, options: PostgresStreamQueryOptions, onAttemptStart: ((attempt: PostgresQueryAttempt) => void) | null): Promise<void>;
|
|
85
100
|
getCatalog(requests: {
|
|
86
101
|
database: string;
|
|
87
102
|
schema: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PostgresWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/PostgresWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAGH,OAAO,EACP,yBAAyB,EACzB,6BAA6B,EAI7B,MAAM,EAIN,mBAAmB,EAEnB,gBAAgB,EAChB,wBAAwB,EACxB,uBAAuB,EACvB,mBAAmB,EACnB,gBAAgB,EAChB,eAAe,EACf,cAAc,EAEd,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAG3B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAc,WAAW,EAAS,MAAM,IAAI,CAAC;AAKpD,OAAO,mBAAmB,CAAC;AAE3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAEhE,eAAO,MAAM,gCAAgC,aAAc,MAAM,KAAG,OACZ,CAAC;AAEzD,eAAO,MAAM,iCAAiC,aAAc,MAAM,WAC2B,CAAC;
|
|
1
|
+
{"version":3,"file":"PostgresWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/PostgresWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAGH,OAAO,EACP,yBAAyB,EACzB,6BAA6B,EAI7B,MAAM,EAIN,mBAAmB,EAEnB,gBAAgB,EAChB,wBAAwB,EACxB,uBAAuB,EACvB,mBAAmB,EACnB,gBAAgB,EAChB,eAAe,EACf,cAAc,EAEd,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAG3B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAc,WAAW,EAAS,MAAM,IAAI,CAAC;AAKpD,OAAO,mBAAmB,CAAC;AAE3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAEhE,eAAO,MAAM,gCAAgC,aAAc,MAAM,KAAG,OACZ,CAAC;AAEzD,eAAO,MAAM,iCAAiC,aAAc,MAAM,WAC2B,CAAC;AAE9F,OAAO,QAAQ,IAAI,CAAC;IAChB,UAAU,UAAU;QAEhB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;KAC5B;CACJ;AAKD,oBAAY,aAAa;IACrB,OAAO,YAAY;IACnB,GAAG,QAAQ;IACX,IAAI,SAAS;IACb,IAAI,SAAS;IACb,IAAI,SAAS;IACb,KAAK,UAAU;IACf,WAAW,gBAAgB;IAC3B,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,SAAS,cAAc;IACvB,MAAM,WAAW;IACjB,QAAQ,aAAa;IACrB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,gBAAgB,qBAAqB;IACrC,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,IAAI,SAAS;IACb,KAAK,UAAU;IACf,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,iBAAiB,sBAAsB;IACvC,QAAQ,aAAa;IACrB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,OAAO,WAAW;IAClB,sBAAsB,2BAA2B;IACjD,SAAS,cAAc;IACvB,YAAY,gBAAgB;IAC5B,2BAA2B,gCAAgC;IAC3D,wBAAwB,6BAA6B;CACxD;AA4CD,eAAO,MAAM,0BAA0B,SAC7B,MAAM,KACb,eAAe,GAAG,SAWpB,CAAC;AAoEF,qBAAa,kBAAmB,SAAQ,uBAAuB;IAC3D,IAAI,iBAA2B;IAE/B,cAAc,IAAI,mBAAmB,CAEpC;IAED,wBAAwB,IAAI,MAAM,CAEjC;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAiBlC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAahD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAED,MAAM,MAAM,0BAA0B,GAAG;IACrC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,mBAAmB,EAAE,UAAU,EAAE,MAAM,KAAK,IAAI,CAAC;CAC5E,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,aAAa,EAAE,MAAM,MAAM,GAAG,IAAI,CAAC;IACnC,YAAY,EAAE,MAAM,OAAO,CAAC;IAC5B,cAAc,EAAE,MAAM,OAAO,CAAC;IAC9B,mFAAmF;IACnF,KAAK,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,OAAO,CAAC;CAClD,CAAC;AAEF,KAAK,mBAAmB,GAAG;IACvB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;CAChC,CAAC;AAEF,qBAAa,cAAc,CACvB,CAAC,SAAS,6BAA6B,CACzC,SAAQ,mBAAmB,CAAC,CAAC,CAAC;IACtB,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAYjD;IAED,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC;IAEtB,YACI,WAAW,EAAE,CAAC,EACd,MAAM,EAAE,EAAE,CAAC,UAAU,EACrB,OAAO,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,OAAO,CAAA;KAAE,EAavC;IAED,SAAS,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAE9D;IAED,SAAS,CAAC,wBAAwB,IAAI,MAAM,CAE3C;IAED,SAAS,CAAC,sBAAsB,CAC5B,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,EACtB,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,EACpB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,GACpB,mBAAmB,CAOrB;IAED,OAAO,CAAC,kBAAkB;IAQ1B,MAAM,CAAC,wBAAwB,CAC3B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,GACvC,gBAAgB,CAAC,QAAQ,CAAC,CAgB5B;IAED,OAAO,CAAC,MAAM,CAAC,cAAc;IAavB,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE,0BAA0B,GACpC,OAAO,CAAC,IAAI,CAAC,CAEf;IAED,UAAgB,kBAAkB,CAC9B,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE,0BAA0B,EACnC,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,oBAAoB,KAAK,IAAI,CAAC,GAAG,IAAI,GACjE,OAAO,CAAC,IAAI,CAAC,CA2Pf;IAEK,UAAU,CACZ,QAAQ,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,gCA4GN;IAED,OAAO,CAAC,aAAa,CAA8B;IAEnD,SAAS,CAAC,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAK5C;IAGD,UAAgB,gBAAgB,IAAI,OAAO,CAAC,OAAO,CAAC,CAKnD;IAED,UAAgB,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC,CAE7C;IAGK,YAAY;;;;;SA4BjB;IAGD,SAAS,CAAC,iBAAiB,CACvB,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM;;;;MAQpB;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAyC3B;IAED,SAAS,CAAC,oBAAoB,CAC1B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAChC,gBAAgB,CAMlB;IAED,UAAU,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,EAAE,KAAK,GAAE,MAAW,uBAsDrD;CACJ;AA0CD,qBAAa,uBAAwB,SAAQ,cAAc,CAAC,yBAAyB,CAAC;IAClF,SAAS,CAAC,sBAAsB,CAC5B,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,EACtB,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,EACpB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,GACpB,mBAAmB,CAarB;IAED,YACI,WAAW,EAAE,yBAAyB,EACtC,OAAO,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,OAAO,CAAA;KAAE,EAevC;IAED,OAAO,CAAC,gBAAgB;YASV,qBAAqB;IAoB7B,aAAa,IAAI,OAAO,CAAC,wBAAwB,CAAC,CAkCvD;IAEK,oBAAoB,CACtB,cAAc,EAAE,uBAAuB,GACxC,OAAO,CAAC,eAAe,CAAC,CAS1B;CACJ"}
|
|
@@ -315,10 +315,16 @@ class PostgresClient extends WarehouseBaseClient_1.default {
|
|
|
315
315
|
return new common_1.WarehouseQueryError(`PostgreSQL identifier is too long: ${notice.message}`);
|
|
316
316
|
}
|
|
317
317
|
async streamQuery(sql, streamCallback, options) {
|
|
318
|
+
return this.streamQueryAttempt(sql, streamCallback, options, null);
|
|
319
|
+
}
|
|
320
|
+
async streamQueryAttempt(sql, streamCallback, options, onAttemptStart) {
|
|
318
321
|
let pool;
|
|
319
322
|
let closeClient;
|
|
320
323
|
let activeStream;
|
|
321
324
|
let clientTimeout;
|
|
325
|
+
let backendPid = null;
|
|
326
|
+
let sentQuery = false;
|
|
327
|
+
let receivedRow = false;
|
|
322
328
|
const reportPhase = options.onPhaseTiming;
|
|
323
329
|
// Enforce the ceiling with a server-side `statement_timeout` (set
|
|
324
330
|
// below) plus this client-side wall-clock backstop, which fires shortly
|
|
@@ -335,6 +341,18 @@ class PostgresClient extends WarehouseBaseClient_1.default {
|
|
|
335
341
|
activeStream?.destroy(timeoutError);
|
|
336
342
|
reject(timeoutError);
|
|
337
343
|
}, clientTimeoutMs);
|
|
344
|
+
onAttemptStart?.({
|
|
345
|
+
getBackendPid: () => backendPid,
|
|
346
|
+
hasSentQuery: () => sentQuery,
|
|
347
|
+
hasReceivedRow: () => receivedRow,
|
|
348
|
+
abort: (error) => {
|
|
349
|
+
if (receivedRow)
|
|
350
|
+
return false;
|
|
351
|
+
activeStream?.destroy(error);
|
|
352
|
+
reject(error);
|
|
353
|
+
return true;
|
|
354
|
+
},
|
|
355
|
+
});
|
|
338
356
|
pool = new pg.Pool({
|
|
339
357
|
...this.config,
|
|
340
358
|
connectionTimeoutMillis: 30000,
|
|
@@ -363,6 +381,7 @@ class PostgresClient extends WarehouseBaseClient_1.default {
|
|
|
363
381
|
return;
|
|
364
382
|
}
|
|
365
383
|
reportPhase?.('connect', performance.now() - connectStart);
|
|
384
|
+
backendPid = client.processID;
|
|
366
385
|
client.on('error', (e) => {
|
|
367
386
|
console.error(`Postgres client error ${(0, common_1.getErrorMessage)(e)}`);
|
|
368
387
|
reject(e);
|
|
@@ -378,6 +397,7 @@ class PostgresClient extends WarehouseBaseClient_1.default {
|
|
|
378
397
|
const runQuery = () => {
|
|
379
398
|
const queryStart = performance.now();
|
|
380
399
|
let fetchStart;
|
|
400
|
+
sentQuery = true;
|
|
381
401
|
// CodeQL: This will raise a security warning because user defined raw SQL is being passed into the database module.
|
|
382
402
|
// In this case this is exactly what we want to do. We're hitting the user's warehouse not the application's database.
|
|
383
403
|
activeStream = client.query(new PgQueryStream_1.default(this.getSQLWithMetadata(sql, options?.tags), options?.values));
|
|
@@ -388,6 +408,7 @@ class PostgresClient extends WarehouseBaseClient_1.default {
|
|
|
388
408
|
const writable = new stream_1.Writable({
|
|
389
409
|
objectMode: true,
|
|
390
410
|
async write(chunk, encoding, callback) {
|
|
411
|
+
receivedRow = true;
|
|
391
412
|
try {
|
|
392
413
|
if (cachedFields === null) {
|
|
393
414
|
cachedFields =
|
|
@@ -1,6 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { CreateRedshiftCredentials, SupportedDbtAdapter, WarehouseCatalog, WarehouseResults, WarehouseTypes } from '@lightdash/common';
|
|
2
2
|
import { PoolClient } from 'pg';
|
|
3
|
-
import { PostgresClient, PostgresSqlBuilder } from './PostgresWarehouseClient';
|
|
3
|
+
import { PostgresClient, PostgresSqlBuilder, type PostgresStreamQueryOptions } from './PostgresWarehouseClient';
|
|
4
|
+
export type RedshiftLogger = {
|
|
5
|
+
info?: (message: string, metadata?: Record<string, unknown>) => void;
|
|
6
|
+
warn?: (message: string, metadata?: Record<string, unknown>) => void;
|
|
7
|
+
};
|
|
4
8
|
export declare class RedshiftSqlBuilder extends PostgresSqlBuilder {
|
|
5
9
|
type: WarehouseTypes;
|
|
6
10
|
getAdapterType(): SupportedDbtAdapter;
|
|
@@ -8,19 +12,20 @@ export declare class RedshiftSqlBuilder extends PostgresSqlBuilder {
|
|
|
8
12
|
export declare class RedshiftWarehouseClient extends PostgresClient<CreateRedshiftCredentials> {
|
|
9
13
|
private readonly ssl;
|
|
10
14
|
private readonly agentJobControls;
|
|
15
|
+
private readonly logger;
|
|
11
16
|
private cachedIamPoolConfig;
|
|
12
17
|
constructor(credentials: CreateRedshiftCredentials, options?: {
|
|
13
18
|
agentSession?: boolean;
|
|
14
19
|
agentJobControls?: boolean;
|
|
20
|
+
logger?: RedshiftLogger;
|
|
15
21
|
});
|
|
22
|
+
private log;
|
|
16
23
|
protected prepareSession(client: PoolClient): Promise<void>;
|
|
17
24
|
protected getAgentSessionStatement(): string;
|
|
18
25
|
private static getConnectionString;
|
|
19
|
-
streamQuery(sql: string, streamCallback: (data: WarehouseResults) => void | Promise<void>, options:
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
timezone?: string;
|
|
23
|
-
}): Promise<void>;
|
|
26
|
+
streamQuery(sql: string, streamCallback: (data: WarehouseResults) => void | Promise<void>, options: PostgresStreamQueryOptions): Promise<void>;
|
|
27
|
+
private streamQueryRecoveringStalls;
|
|
28
|
+
private getStallVerdict;
|
|
24
29
|
getAllTables(): Promise<{
|
|
25
30
|
database: any;
|
|
26
31
|
schema: any;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RedshiftWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/RedshiftWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAGH,
|
|
1
|
+
{"version":3,"file":"RedshiftWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/RedshiftWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAGH,yBAAyB,EAIzB,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAChB,cAAc,EACjB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,EAAE,UAAU,EAAc,MAAM,IAAI,CAAC;AAC5C,OAAO,EACH,cAAc,EACd,kBAAkB,EAElB,KAAK,0BAA0B,EAClC,MAAM,2BAA2B,CAAC;AAsEnC,MAAM,MAAM,cAAc,GAAG;IACzB,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACrE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACxE,CAAC;AAEF,qBAAa,kBAAmB,SAAQ,kBAAkB;IACtD,IAAI,iBAA2B;IAE/B,cAAc,IAAI,mBAAmB,CAEpC;CACJ;AAED,qBAAa,uBAAwB,SAAQ,cAAc,CAAC,yBAAyB,CAAC;IAClF,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAoB;IAExC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAU;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA6B;IAEpD,OAAO,CAAC,mBAAmB,CAEX;IAEhB,YACI,WAAW,EAAE,yBAAyB,EACtC,OAAO,CAAC,EAAE;QACN,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,MAAM,CAAC,EAAE,cAAc,CAAC;KAC3B,EAkCJ;IAED,OAAO,CAAC,GAAG;IASX,UAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhE;IAED,SAAS,CAAC,wBAAwB,IAAI,MAAM,CAE3C;IAED,OAAO,CAAC,MAAM,CAAC,mBAAmB;IAgB5B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE,0BAA0B,GACpC,OAAO,CAAC,IAAI,CAAC,CAaf;YAKa,2BAA2B;YAqF3B,eAAe;IAiDvB,YAAY;;;;;SAkBjB;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CA2B3B;YAEa,oBAAoB;CA+BrC"}
|
|
@@ -40,6 +40,7 @@ exports.RedshiftWarehouseClient = exports.RedshiftSqlBuilder = void 0;
|
|
|
40
40
|
const common_1 = require("@lightdash/common");
|
|
41
41
|
const fs = __importStar(require("fs"));
|
|
42
42
|
const path_1 = __importDefault(require("path"));
|
|
43
|
+
const pg = __importStar(require("pg"));
|
|
43
44
|
const PostgresWarehouseClient_1 = require("./PostgresWarehouseClient");
|
|
44
45
|
const redshiftIamCredentials_1 = require("./redshiftIamCredentials");
|
|
45
46
|
// Refresh minted credentials this long before the AWS-reported expiry so an
|
|
@@ -47,6 +48,25 @@ const redshiftIamCredentials_1 = require("./redshiftIamCredentials");
|
|
|
47
48
|
const CREDENTIALS_REFRESH_BUFFER_MS = 60 * 1000;
|
|
48
49
|
// Fallback lifetime when AWS returns no expiry — under the 15-min minimum.
|
|
49
50
|
const CREDENTIALS_FALLBACK_LIFETIME_MS = 14 * 60 * 1000;
|
|
51
|
+
// Tile queries on a healthy workgroup return a first row well within this.
|
|
52
|
+
const STALL_CHECK_DELAY_MS = 30 * 1000;
|
|
53
|
+
const MAX_STALL_RETRIES = 2;
|
|
54
|
+
// The lookup may itself have to wake a paused workgroup (~15s measured, up
|
|
55
|
+
// to 90s reported), so it needs longer than a warm system-view query.
|
|
56
|
+
const STALL_LOOKUP_TIMEOUT_MS = 60 * 1000;
|
|
57
|
+
// The lookup must see its own running query, otherwise an empty result for
|
|
58
|
+
// the stalled session proves nothing and we keep waiting.
|
|
59
|
+
const STALL_LOOKUP_SQL = `
|
|
60
|
+
SELECT
|
|
61
|
+
(SELECT COUNT(*) FROM sys_query_history
|
|
62
|
+
WHERE session_id = pg_backend_pid()
|
|
63
|
+
AND TRIM(status) = 'running') AS own_running,
|
|
64
|
+
(SELECT COUNT(*) FROM sys_query_history
|
|
65
|
+
WHERE session_id = $1
|
|
66
|
+
AND start_time > DATEADD(minute, -15, GETDATE())
|
|
67
|
+
AND (TRIM(status) IN ('planning', 'queued', 'running', 'returning')
|
|
68
|
+
OR TRIM(query_type) NOT IN ('UTILITY', 'COMMAND'))) AS stalled_session_records
|
|
69
|
+
`;
|
|
50
70
|
const AMAZON_CA_BUNDLE = [
|
|
51
71
|
fs.readFileSync(path_1.default.resolve(__dirname, './ca-bundle-aws-redshift.crt')),
|
|
52
72
|
];
|
|
@@ -103,9 +123,14 @@ class RedshiftWarehouseClient extends PostgresWarehouseClient_1.PostgresClient {
|
|
|
103
123
|
}, options);
|
|
104
124
|
this.ssl = ssl;
|
|
105
125
|
this.agentJobControls = options?.agentJobControls ?? false;
|
|
126
|
+
this.logger = options?.logger;
|
|
106
127
|
// Override the sqlBuilder with RedshiftSqlBuilder
|
|
107
128
|
this.sqlBuilder = new RedshiftSqlBuilder(credentials.startOfWeek);
|
|
108
129
|
}
|
|
130
|
+
log(level, message, metadata) {
|
|
131
|
+
const target = this.logger ?? console;
|
|
132
|
+
(target[level] ?? console[level]).call(target, message, metadata);
|
|
133
|
+
}
|
|
109
134
|
async prepareSession(client) {
|
|
110
135
|
if (this.agentJobControls) {
|
|
111
136
|
await client.query('SET enable_result_cache_for_session TO off');
|
|
@@ -128,7 +153,118 @@ class RedshiftWarehouseClient extends PostgresWarehouseClient_1.PostgresClient {
|
|
|
128
153
|
common_1.RedshiftAuthenticationType.IAM_BROWSER) {
|
|
129
154
|
this.config = await this.resolveIamPoolConfig();
|
|
130
155
|
}
|
|
131
|
-
|
|
156
|
+
if (!options.recoverStalledQuery) {
|
|
157
|
+
return super.streamQuery(sql, streamCallback, options);
|
|
158
|
+
}
|
|
159
|
+
return this.streamQueryRecoveringStalls(sql, streamCallback, options);
|
|
160
|
+
}
|
|
161
|
+
// A stalled attempt has streamed no rows, so retrying with the same
|
|
162
|
+
// callback is safe. The last attempt runs unwatched, leaving the
|
|
163
|
+
// statement timeout as the final backstop.
|
|
164
|
+
async streamQueryRecoveringStalls(sql, streamCallback, options) {
|
|
165
|
+
const queryUuid = options.tags?.query_uuid ?? null;
|
|
166
|
+
const startedAt = Date.now();
|
|
167
|
+
for (let retry = 0; retry < MAX_STALL_RETRIES; retry += 1) {
|
|
168
|
+
const stallError = new common_1.WarehouseQueryError('Warehouse query stalled before reaching the warehouse');
|
|
169
|
+
let attempt;
|
|
170
|
+
let settled = false;
|
|
171
|
+
const stallCheck = setTimeout(() => {
|
|
172
|
+
if (!attempt || attempt.hasReceivedRow())
|
|
173
|
+
return;
|
|
174
|
+
const watched = attempt;
|
|
175
|
+
const lookupStart = Date.now();
|
|
176
|
+
void this.getStallVerdict(watched).then((verdict) => {
|
|
177
|
+
if (settled)
|
|
178
|
+
return;
|
|
179
|
+
const metadata = {
|
|
180
|
+
queryUuid,
|
|
181
|
+
pid: watched.getBackendPid(),
|
|
182
|
+
reason: verdict.reason,
|
|
183
|
+
retry,
|
|
184
|
+
maxRetries: MAX_STALL_RETRIES,
|
|
185
|
+
stallCheckDelayMs: STALL_CHECK_DELAY_MS,
|
|
186
|
+
lookupMs: Date.now() - lookupStart,
|
|
187
|
+
elapsedMs: Date.now() - startedAt,
|
|
188
|
+
};
|
|
189
|
+
if (verdict.stalled && watched.abort(stallError)) {
|
|
190
|
+
this.log('warn', 'Redshift query stalled with no first row, retrying on a new connection', metadata);
|
|
191
|
+
}
|
|
192
|
+
else {
|
|
193
|
+
this.log('info', 'Redshift query has no first row yet, still waiting', metadata);
|
|
194
|
+
}
|
|
195
|
+
});
|
|
196
|
+
}, STALL_CHECK_DELAY_MS);
|
|
197
|
+
try {
|
|
198
|
+
// eslint-disable-next-line no-await-in-loop
|
|
199
|
+
await this.streamQueryAttempt(sql, streamCallback, options, (started) => {
|
|
200
|
+
attempt = started;
|
|
201
|
+
});
|
|
202
|
+
if (retry > 0) {
|
|
203
|
+
this.log('info', 'Redshift query recovered after stall retry', {
|
|
204
|
+
queryUuid,
|
|
205
|
+
retry,
|
|
206
|
+
elapsedMs: Date.now() - startedAt,
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
catch (error) {
|
|
212
|
+
if (error !== stallError)
|
|
213
|
+
throw error;
|
|
214
|
+
}
|
|
215
|
+
finally {
|
|
216
|
+
settled = true;
|
|
217
|
+
clearTimeout(stallCheck);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
this.log('warn', 'Redshift query stalled on every retry, running the final attempt unwatched', {
|
|
221
|
+
queryUuid,
|
|
222
|
+
retries: MAX_STALL_RETRIES,
|
|
223
|
+
elapsedMs: Date.now() - startedAt,
|
|
224
|
+
});
|
|
225
|
+
await this.streamQueryAttempt(sql, streamCallback, options, null);
|
|
226
|
+
}
|
|
227
|
+
async getStallVerdict(attempt) {
|
|
228
|
+
const pid = attempt.getBackendPid();
|
|
229
|
+
if (pid === null)
|
|
230
|
+
return { stalled: false, reason: 'not_connected' };
|
|
231
|
+
// Still on the session SETs, so the real query was never sent
|
|
232
|
+
if (!attempt.hasSentQuery()) {
|
|
233
|
+
return { stalled: true, reason: 'session_setup' };
|
|
234
|
+
}
|
|
235
|
+
let client;
|
|
236
|
+
try {
|
|
237
|
+
// Same user as the stalled query: non-superusers only see their own history
|
|
238
|
+
client = new pg.Client({
|
|
239
|
+
...this.config,
|
|
240
|
+
connectionTimeoutMillis: STALL_LOOKUP_TIMEOUT_MS,
|
|
241
|
+
query_timeout: STALL_LOOKUP_TIMEOUT_MS,
|
|
242
|
+
});
|
|
243
|
+
client.on('error', (error) => {
|
|
244
|
+
this.log('warn', 'Redshift stall lookup connection error', {
|
|
245
|
+
pid,
|
|
246
|
+
error: (0, common_1.getErrorMessage)(error),
|
|
247
|
+
});
|
|
248
|
+
});
|
|
249
|
+
await client.connect();
|
|
250
|
+
const { rows } = await client.query(STALL_LOOKUP_SQL, [pid]);
|
|
251
|
+
if (!rows[0] || Number(rows[0].own_running) === 0) {
|
|
252
|
+
return { stalled: false, reason: 'lookup_not_visible' };
|
|
253
|
+
}
|
|
254
|
+
return Number(rows[0].stalled_session_records) === 0
|
|
255
|
+
? { stalled: true, reason: 'no_query_record' }
|
|
256
|
+
: { stalled: false, reason: 'query_recorded' };
|
|
257
|
+
}
|
|
258
|
+
catch (error) {
|
|
259
|
+
this.log('warn', 'Redshift stall lookup failed', {
|
|
260
|
+
pid,
|
|
261
|
+
error: (0, common_1.getErrorMessage)(error),
|
|
262
|
+
});
|
|
263
|
+
return { stalled: false, reason: 'lookup_failed' };
|
|
264
|
+
}
|
|
265
|
+
finally {
|
|
266
|
+
await client?.end().catch(() => undefined);
|
|
267
|
+
}
|
|
132
268
|
}
|
|
133
269
|
// SVV_ALL_* are grant-filtered catalog views; SVV_TABLES and SVV_COLUMNS are
|
|
134
270
|
// visible to all users unless metadata_security is on. Keep SVV_COLUMNS for
|
|
@@ -28,9 +28,10 @@ export default abstract class WarehouseBaseClient<T extends CreateWarehouseCrede
|
|
|
28
28
|
queryParams?: Record<string, AnyType>;
|
|
29
29
|
tags?: Record<string, string>;
|
|
30
30
|
timezone?: string;
|
|
31
|
+
recoverStalledQuery?: boolean;
|
|
31
32
|
onPhaseTiming?: (phase: WarehouseQueryPhase, durationMs: number) => void;
|
|
32
33
|
}): Promise<void>;
|
|
33
|
-
executeAsyncQuery({ sql, values, queryParams, tags, timezone, }: WarehouseExecuteAsyncQueryArgs, resultsStreamCallback?: (rows: WarehouseResults['rows'], fields: WarehouseResults['fields']) => void | Promise<void>): Promise<WarehouseExecuteAsyncQuery>;
|
|
34
|
+
executeAsyncQuery({ sql, values, queryParams, tags, timezone, recoverStalledQuery, }: WarehouseExecuteAsyncQueryArgs, resultsStreamCallback?: (rows: WarehouseResults['rows'], fields: WarehouseResults['fields']) => void | Promise<void>): Promise<WarehouseExecuteAsyncQuery>;
|
|
34
35
|
runQuery(sql: string, tags?: Record<string, string>, timezone?: string, values?: AnyType[], queryParams?: Record<string, AnyType>): Promise<{
|
|
35
36
|
fields: Record<string, import("@lightdash/common").WarehouseResultField>;
|
|
36
37
|
rows: Record<string, any>[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"WarehouseBaseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/WarehouseBaseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,OAAO,EACP,0BAA0B,EAC1B,aAAa,EACb,MAAM,EAEN,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,EAChB,SAAS,EACT,aAAa,EACb,gBAAgB,EAChB,wBAAwB,EAExB,uBAAuB,EACvB,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,OAAO,EACP,KAAK,eAAe,EACpB,KAAK,0BAA0B,EAC/B,KAAK,8BAA8B,EAEnC,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,UAAU,CAAC;AAEhD,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,mBAAmB,CAC7C,CAAC,SAAS,0BAA0B,CACtC,YAAW,eAAe;IACxB,WAAW,EAAE,CAAC,CAAC;IAEf,SAAS,CAAC,UAAU,EAAE,mBAAmB,CAAC;IAE1C,SAAS,CAAC,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC;IAEzC,SAAS,aACL,WAAW,EAAE,CAAC,EACd,UAAU,EAAE,mBAAmB,EAC/B,OAAO,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,OAAO,CAAA;KAAE,EAKvC;IAED,cAAc,IAAI,mBAAmB,CAEpC;IAED,0BAA0B,IAAI,OAAO,CAEpC;IAED,kBAAkB,IAAI,MAAM,CAE3B;IAED,wBAAwB,IAAI,MAAM,CAEjC;IAED,iBAAiB,IAAI,MAAM,CAE1B;IAED,YAAY,CAAC,IAAI,EAAE,aAAa,GAAG,SAAS,GAAG,IAAI,CAElD;IAED,eAAe,IAAI,MAAM,CAExB;IAED,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAEvD;IAED,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE3D;IAED,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAE1D;IAED,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAErC;IAED,QAAQ,CAAC,UAAU,CACf,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAE7B,QAAQ,CAAC,WAAW,CAChB,KAAK,EAAE,MAAM,EACb,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CAAC;IAEX,iBAAiB,CACnB,EACI,GAAG,EACH,MAAM,EACN,WAAW,EACX,IAAI,EACJ,QAAQ,
|
|
1
|
+
{"version":3,"file":"WarehouseBaseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/WarehouseBaseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,OAAO,EACP,0BAA0B,EAC1B,aAAa,EACb,MAAM,EAEN,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,EAChB,SAAS,EACT,aAAa,EACb,gBAAgB,EAChB,wBAAwB,EAExB,uBAAuB,EACvB,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,OAAO,EACP,KAAK,eAAe,EACpB,KAAK,0BAA0B,EAC/B,KAAK,8BAA8B,EAEnC,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,UAAU,CAAC;AAEhD,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,mBAAmB,CAC7C,CAAC,SAAS,0BAA0B,CACtC,YAAW,eAAe;IACxB,WAAW,EAAE,CAAC,CAAC;IAEf,SAAS,CAAC,UAAU,EAAE,mBAAmB,CAAC;IAE1C,SAAS,CAAC,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC;IAEzC,SAAS,aACL,WAAW,EAAE,CAAC,EACd,UAAU,EAAE,mBAAmB,EAC/B,OAAO,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,OAAO,CAAA;KAAE,EAKvC;IAED,cAAc,IAAI,mBAAmB,CAEpC;IAED,0BAA0B,IAAI,OAAO,CAEpC;IAED,kBAAkB,IAAI,MAAM,CAE3B;IAED,wBAAwB,IAAI,MAAM,CAEjC;IAED,iBAAiB,IAAI,MAAM,CAE1B;IAED,YAAY,CAAC,IAAI,EAAE,aAAa,GAAG,SAAS,GAAG,IAAI,CAElD;IAED,eAAe,IAAI,MAAM,CAExB;IAED,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAEvD;IAED,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE3D;IAED,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAE1D;IAED,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAErC;IAED,QAAQ,CAAC,UAAU,CACf,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAE7B,QAAQ,CAAC,WAAW,CAChB,KAAK,EAAE,MAAM,EACb,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,mBAAmB,CAAC,EAAE,OAAO,CAAC;QAC9B,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CAAC;IAEX,iBAAiB,CACnB,EACI,GAAG,EACH,MAAM,EACN,WAAW,EACX,IAAI,EACJ,QAAQ,EACR,mBAAmB,GACtB,EAAE,8BAA8B,EACjC,qBAAqB,CAAC,EAAE,CACpB,IAAI,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC,KACjC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAC1B,OAAO,CAAC,0BAA0B,CAAC,CAgCrC;IAEK,QAAQ,CACV,GAAG,EAAE,MAAM,EACX,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC7B,QAAQ,CAAC,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,OAAO,EAAE,EAClB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;;;OAoBxC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAEhD;IAED,cAAc,IAAI,OAAO,GAAG,IAAI,GAAG,SAAS,CAE3C;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAGK,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAEjD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;IAED,QAAQ,CAAC,YAAY,CACjB,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,eAAe,CAAC,CAAC;IAE5B,QAAQ,CAAC,SAAS,CACd,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAEvB,oBAAoB,CACtB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,GAClB,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAEnC;IAEK,aAAa,CACf,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,wBAAwB,CAAC,CAInC;IAEK,oBAAoB,CACtB,SAAS,EAAE,uBAAuB,EAClC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,eAAe,CAAC,CAI1B;IAED,qBAAqB,CACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAC/B,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,aAAa,EAC7C,kBAAkB,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,eAAe,GAAG,SAAS,GACnE,gBAAgB,CAiClB;IAED,UAAU,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK,CAE9B;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAElC;IAED,eAAe,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAElC;IAED,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAE7B;IAED,oBAAoB,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAEvC;IAED,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAE5D;IAED,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM,CAKR;IAED,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAErC;CACJ"}
|
|
@@ -40,7 +40,7 @@ class WarehouseBaseClient {
|
|
|
40
40
|
buildArray(elements) {
|
|
41
41
|
return this.sqlBuilder.buildArray(elements);
|
|
42
42
|
}
|
|
43
|
-
async executeAsyncQuery({ sql, values, queryParams, tags, timezone, }, resultsStreamCallback) {
|
|
43
|
+
async executeAsyncQuery({ sql, values, queryParams, tags, timezone, recoverStalledQuery, }, resultsStreamCallback) {
|
|
44
44
|
let rowCount = 0;
|
|
45
45
|
const phaseTimings = {};
|
|
46
46
|
const startTime = performance.now();
|
|
@@ -52,6 +52,7 @@ class WarehouseBaseClient {
|
|
|
52
52
|
queryParams,
|
|
53
53
|
tags,
|
|
54
54
|
timezone,
|
|
55
|
+
recoverStalledQuery,
|
|
55
56
|
onPhaseTiming: (phase, durationMs) => {
|
|
56
57
|
phaseTimings[phase] =
|
|
57
58
|
(phaseTimings[phase] ?? 0) + durationMs;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.136.1",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"@databricks/sql": "1.13.0",
|
|
22
22
|
"@duckdb/node-api": "1.5.2-r.2",
|
|
23
23
|
"@google-cloud/bigquery": "8.3.1",
|
|
24
|
-
"@lightdash/common": "3.
|
|
24
|
+
"@lightdash/common": "3.136.1",
|
|
25
25
|
"node-fetch": "2.7.0",
|
|
26
26
|
"pg": "8.13.1",
|
|
27
27
|
"pg-cursor": "2.10.0",
|