@lightdash/warehouses 2.314.2 → 2.316.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo.build +1 -1
- package/dist/warehouseClients/AthenaWarehouseClient.d.ts +4 -1
- package/dist/warehouseClients/AthenaWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/AthenaWarehouseClient.js +77 -0
- package/dist/warehouseClients/PostgresWarehouseClient.d.ts +5 -1
- package/dist/warehouseClients/PostgresWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/PostgresWarehouseClient.js +49 -0
- package/dist/warehouseClients/WarehouseBaseClient.d.ts +3 -1
- package/dist/warehouseClients/WarehouseBaseClient.d.ts.map +1 -1
- package/dist/warehouseClients/WarehouseBaseClient.js +6 -0
- package/package.json +2 -2
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { AthenaClient } from '@aws-sdk/client-athena';
|
|
2
|
-
import { CreateAthenaCredentials, DimensionType, Metric, SupportedDbtAdapter, TimeIntervalUnit, WarehouseResults, WarehouseTables, WarehouseTypes, type ResultNumericKind, type TimestampDomain } from '@lightdash/common';
|
|
2
|
+
import { CreateAthenaCredentials, DimensionType, Metric, SupportedDbtAdapter, TimeIntervalUnit, WarehouseDatabaseListing, WarehouseListedDatabase, WarehouseResults, WarehouseTables, WarehouseTypes, type ResultNumericKind, type TimestampDomain } from '@lightdash/common';
|
|
3
3
|
import { WarehouseCatalog } from '../types';
|
|
4
4
|
import WarehouseBaseClient from './WarehouseBaseClient';
|
|
5
5
|
import WarehouseBaseSqlBuilder from './WarehouseBaseSqlBuilder';
|
|
@@ -58,6 +58,9 @@ export declare class AthenaWarehouseClient extends WarehouseBaseClient<CreateAth
|
|
|
58
58
|
table: string;
|
|
59
59
|
}[]): Promise<WarehouseCatalog>;
|
|
60
60
|
getAllTables(): Promise<WarehouseTables>;
|
|
61
|
+
private toListedDatabase;
|
|
62
|
+
listDatabases(): Promise<WarehouseDatabaseListing>;
|
|
63
|
+
getTablesForDatabase(listedDatabase: WarehouseListedDatabase): Promise<WarehouseTables>;
|
|
61
64
|
getFields(tableName: string, schema?: string, database?: string): Promise<WarehouseCatalog>;
|
|
62
65
|
parseError(error: Error): Error;
|
|
63
66
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AthenaWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/AthenaWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,YAAY,
|
|
1
|
+
{"version":3,"file":"AthenaWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/AthenaWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,YAAY,EAQf,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAGH,uBAAuB,EACvB,aAAa,EAGb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAGhB,wBAAwB,EACxB,uBAAuB,EAEvB,gBAAgB,EAChB,eAAe,EACf,cAAc,EACd,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACvB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAM5C,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAEhE,oBAAY,WAAW;IACnB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,QAAQ,aAAa;IACrB,OAAO,YAAY;IACnB,MAAM,WAAW;IACjB,IAAI,SAAS;IACb,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,IAAI,SAAS;IACb,OAAO,wBAAwB;IAC/B,SAAS,cAAc;IACvB,YAAY,6BAA6B;IACzC,KAAK,UAAU;IACf,GAAG,QAAQ;IACX,GAAG,QAAQ;IACX,SAAS,cAAc;IACvB,IAAI,SAAS;CAChB;AAGD,eAAO,MAAM,wBAAwB,SAC3B,WAAW,GAAG,MAAM,KAC3B,eAAe,GAAG,SASpB,CAAC;AAGF,eAAO,MAAM,oBAAoB,SACvB,WAAW,GAAG,MAAM,SACnB,MAAM,GAAG,SAAS,KAC1B,iBAAiB,GAAG,IAiBtB,CAAC;AAEF,eAAO,MAAM,8BAA8B,SACjC,WAAW,GAAG,MAAM,KAC3B,aAwBF,CAAC;AAiGF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,QAAQ,CAAC,IAAI,yBAAyB;IAEtC,cAAc,IAAI,mBAAmB,CAEpC;IAGD,0BAA0B,IAAI,OAAO,CAEpC;IAED,wBAAwB,IAAI,MAAM,CAEjC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAWhD;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAiBlC;IAED,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAI5D;IAED,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM,CAGR;IAED,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGrC;CACJ;AAKD,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,uBAAuB,CAAC;IACnF,MAAM,EAAE,YAAY,CAAC;IAErB,YAAY,WAAW,EAAE,uBAAuB,EAuD/C;YAEa,sBAAsB;IAiDpC,OAAO,CAAC,UAAU;IA+BZ,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC,CA6Gf;IAEK,UAAU,CACZ,QAAQ,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAChE,OAAO,CAAC,gBAAgB,CAAC,CAwD3B;IAEK,YAAY,IAAI,OAAO,CAAC,eAAe,CAAC,CAwC7C;IAED,OAAO,CAAC,gBAAgB;IASlB,aAAa,IAAI,OAAO,CAAC,wBAAwB,CAAC,CA0CvD;IAEK,oBAAoB,CACtB,cAAc,EAAE,uBAAuB,GACxC,OAAO,CAAC,eAAe,CAAC,CAyC1B;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,gBAAgB,CAAC,CAuC3B;IAED,UAAU,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK,CAE9B;CACJ"}
|
|
@@ -482,6 +482,83 @@ class AthenaWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
482
482
|
}
|
|
483
483
|
return tables;
|
|
484
484
|
}
|
|
485
|
+
toListedDatabase(name) {
|
|
486
|
+
return {
|
|
487
|
+
name,
|
|
488
|
+
database: this.credentials.database,
|
|
489
|
+
schema: name,
|
|
490
|
+
isDefault: name === this.credentials.schema,
|
|
491
|
+
};
|
|
492
|
+
}
|
|
493
|
+
async listDatabases() {
|
|
494
|
+
const databaseNames = new Set([this.credentials.schema]);
|
|
495
|
+
try {
|
|
496
|
+
let nextToken;
|
|
497
|
+
do {
|
|
498
|
+
// eslint-disable-next-line no-await-in-loop
|
|
499
|
+
const response = await this.client.send(new client_athena_1.ListDatabasesCommand({
|
|
500
|
+
CatalogName: this.credentials.database,
|
|
501
|
+
MaxResults: 50,
|
|
502
|
+
NextToken: nextToken,
|
|
503
|
+
}));
|
|
504
|
+
response.DatabaseList?.forEach((database) => {
|
|
505
|
+
if (database.Name) {
|
|
506
|
+
databaseNames.add(database.Name);
|
|
507
|
+
}
|
|
508
|
+
});
|
|
509
|
+
nextToken = response.NextToken;
|
|
510
|
+
} while (nextToken &&
|
|
511
|
+
databaseNames.size <= common_1.WAREHOUSE_LISTED_DATABASES_LIMIT);
|
|
512
|
+
}
|
|
513
|
+
catch (e) {
|
|
514
|
+
throw translateAthenaError(e, {
|
|
515
|
+
contextPrefix: `Failed to list databases in catalog '${this.credentials.database}'.`,
|
|
516
|
+
defaultErrorClass: 'connection',
|
|
517
|
+
});
|
|
518
|
+
}
|
|
519
|
+
const databases = [...databaseNames]
|
|
520
|
+
.slice(0, common_1.WAREHOUSE_LISTED_DATABASES_LIMIT)
|
|
521
|
+
.map((database) => this.toListedDatabase(database));
|
|
522
|
+
return {
|
|
523
|
+
databases,
|
|
524
|
+
truncated: databaseNames.size > common_1.WAREHOUSE_LISTED_DATABASES_LIMIT,
|
|
525
|
+
limit: common_1.WAREHOUSE_LISTED_DATABASES_LIMIT,
|
|
526
|
+
};
|
|
527
|
+
}
|
|
528
|
+
async getTablesForDatabase(listedDatabase) {
|
|
529
|
+
const tables = [];
|
|
530
|
+
const schema = listedDatabase.schema ?? listedDatabase.name;
|
|
531
|
+
try {
|
|
532
|
+
let nextToken;
|
|
533
|
+
do {
|
|
534
|
+
// eslint-disable-next-line no-await-in-loop
|
|
535
|
+
const response = await this.client.send(new client_athena_1.ListTableMetadataCommand({
|
|
536
|
+
CatalogName: listedDatabase.database,
|
|
537
|
+
DatabaseName: schema,
|
|
538
|
+
NextToken: nextToken,
|
|
539
|
+
MaxResults: 50,
|
|
540
|
+
}));
|
|
541
|
+
response.TableMetadataList?.forEach((tableMeta) => {
|
|
542
|
+
if (tableMeta.Name) {
|
|
543
|
+
tables.push({
|
|
544
|
+
database: listedDatabase.database,
|
|
545
|
+
schema,
|
|
546
|
+
table: tableMeta.Name,
|
|
547
|
+
tableType: (0, common_1.getWarehouseTableType)(tableMeta.TableType),
|
|
548
|
+
});
|
|
549
|
+
}
|
|
550
|
+
});
|
|
551
|
+
nextToken = response.NextToken;
|
|
552
|
+
} while (nextToken);
|
|
553
|
+
}
|
|
554
|
+
catch (e) {
|
|
555
|
+
throw translateAthenaError(e, {
|
|
556
|
+
contextPrefix: `Failed to list tables in '${listedDatabase.database}.${schema}'.`,
|
|
557
|
+
defaultErrorClass: 'connection',
|
|
558
|
+
});
|
|
559
|
+
}
|
|
560
|
+
return tables;
|
|
561
|
+
}
|
|
485
562
|
async getFields(tableName, schema, database) {
|
|
486
563
|
const db = database || this.credentials.database;
|
|
487
564
|
const sch = schema || this.credentials.schema;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AnyType, CreatePostgresCredentials, CreatePostgresLikeCredentials, Metric, SupportedDbtAdapter, WarehouseCatalog, WarehouseQueryError, WarehouseResults, WarehouseTypes, type TimestampDomain, type WarehouseQueryPhase } from '@lightdash/common';
|
|
1
|
+
import { AnyType, CreatePostgresCredentials, CreatePostgresLikeCredentials, Metric, SupportedDbtAdapter, WarehouseCatalog, WarehouseDatabaseListing, WarehouseListedDatabase, WarehouseQueryError, WarehouseResults, WarehouseTables, WarehouseTypes, type TimestampDomain, type WarehouseQueryPhase } from '@lightdash/common';
|
|
2
2
|
import * as pg from 'pg';
|
|
3
3
|
import { QueryResult } from 'pg';
|
|
4
4
|
import './pgProtocolGuard';
|
|
@@ -103,6 +103,10 @@ export declare class PostgresClient<T extends CreatePostgresLikeCredentials> ext
|
|
|
103
103
|
export declare class PostgresWarehouseClient extends PostgresClient<CreatePostgresCredentials> {
|
|
104
104
|
protected getCatalogQueryFilters(databases: Set<string>, schemas: Set<string>, tables: Set<string>): CatalogQueryFilters;
|
|
105
105
|
constructor(credentials: CreatePostgresCredentials);
|
|
106
|
+
private toListedDatabase;
|
|
107
|
+
private withClientForDatabase;
|
|
108
|
+
listDatabases(): Promise<WarehouseDatabaseListing>;
|
|
109
|
+
getTablesForDatabase(listedDatabase: WarehouseListedDatabase): Promise<WarehouseTables>;
|
|
106
110
|
}
|
|
107
111
|
export {};
|
|
108
112
|
//# sourceMappingURL=PostgresWarehouseClient.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PostgresWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/PostgresWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,6BAA6B,EAI7B,MAAM,EAIN,mBAAmB,
|
|
1
|
+
{"version":3,"file":"PostgresWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/PostgresWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,6BAA6B,EAI7B,MAAM,EAIN,mBAAmB,EAEnB,gBAAgB,EAChB,wBAAwB,EACxB,uBAAuB,EACvB,mBAAmB,EACnB,gBAAgB,EAChB,eAAe,EACf,cAAc,EAEd,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAG3B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAc,WAAW,EAAS,MAAM,IAAI,CAAC;AAKpD,OAAO,mBAAmB,CAAC;AAE3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAKhE,oBAAY,aAAa;IACrB,OAAO,YAAY;IACnB,GAAG,QAAQ;IACX,IAAI,SAAS;IACb,IAAI,SAAS;IACb,IAAI,SAAS;IACb,KAAK,UAAU;IACf,WAAW,gBAAgB;IAC3B,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,SAAS,cAAc;IACvB,MAAM,WAAW;IACjB,QAAQ,aAAa;IACrB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,gBAAgB,qBAAqB;IACrC,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,IAAI,SAAS;IACb,KAAK,UAAU;IACf,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,iBAAiB,sBAAsB;IACvC,QAAQ,aAAa;IACrB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,OAAO,WAAW;IAClB,sBAAsB,2BAA2B;IACjD,SAAS,cAAc;IACvB,YAAY,gBAAgB;IAC5B,2BAA2B,gCAAgC;IAC3D,wBAAwB,6BAA6B;CACxD;AA4CD,eAAO,MAAM,0BAA0B,SAC7B,MAAM,KACb,eAAe,GAAG,SAWpB,CAAC;AAoEF,qBAAa,kBAAmB,SAAQ,uBAAuB;IAC3D,IAAI,iBAA2B;IAE/B,cAAc,IAAI,mBAAmB,CAEpC;IAED,wBAAwB,IAAI,MAAM,CAEjC;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAiBlC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAahD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAED,KAAK,mBAAmB,GAAG;IACvB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;CAChC,CAAC;AAEF,qBAAa,cAAc,CACvB,CAAC,SAAS,6BAA6B,CACzC,SAAQ,mBAAmB,CAAC,CAAC,CAAC;IACtB,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAYjD;IAED,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC;IAEtB,YAAY,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,UAAU,EAGhD;IAED,SAAS,CAAC,sBAAsB,CAC5B,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,EACtB,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,EACpB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,GACpB,mBAAmB,CAOrB;IAED,OAAO,CAAC,kBAAkB;IAQ1B,MAAM,CAAC,wBAAwB,CAC3B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,GACvC,gBAAgB,CAAC,QAAQ,CAAC,CAgB5B;IAED,OAAO,CAAC,MAAM,CAAC,cAAc;IAavB,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CAiOf;IAEK,UAAU,CACZ,QAAQ,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,gCA4GN;IAED,OAAO,CAAC,aAAa,CAA8B;IAEnD,SAAS,CAAC,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAK5C;IAGD,UAAgB,gBAAgB,IAAI,OAAO,CAAC,OAAO,CAAC,CAKnD;IAED,UAAgB,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC,CAE7C;IAGK,YAAY;;;;;SA4BjB;IAGD,SAAS,CAAC,iBAAiB,CACvB,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM;;;;MAQpB;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAyC3B;IAED,SAAS,CAAC,oBAAoB,CAC1B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAChC,gBAAgB,CAMlB;IAED,UAAU,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,EAAE,KAAK,GAAE,MAAW,uBAsDrD;CACJ;AA0CD,qBAAa,uBAAwB,SAAQ,cAAc,CAAC,yBAAyB,CAAC;IAClF,SAAS,CAAC,sBAAsB,CAC5B,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,EACtB,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,EACpB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,GACpB,mBAAmB,CAarB;IAED,YAAY,WAAW,EAAE,yBAAyB,EAUjD;IAED,OAAO,CAAC,gBAAgB;YASV,qBAAqB;IAe7B,aAAa,IAAI,OAAO,CAAC,wBAAwB,CAAC,CAyBvD;IAEK,oBAAoB,CACtB,cAAc,EAAE,uBAAuB,GACxC,OAAO,CAAC,eAAe,CAAC,CAS1B;CACJ"}
|
|
@@ -736,5 +736,54 @@ class PostgresWarehouseClient extends PostgresClient {
|
|
|
736
736
|
ssl,
|
|
737
737
|
});
|
|
738
738
|
}
|
|
739
|
+
toListedDatabase(name) {
|
|
740
|
+
return {
|
|
741
|
+
name,
|
|
742
|
+
database: name,
|
|
743
|
+
schema: null,
|
|
744
|
+
isDefault: name === this.credentials.dbname,
|
|
745
|
+
};
|
|
746
|
+
}
|
|
747
|
+
async withClientForDatabase(database, operation) {
|
|
748
|
+
if (database === this.credentials.dbname) {
|
|
749
|
+
return operation(this);
|
|
750
|
+
}
|
|
751
|
+
const client = new PostgresWarehouseClient({
|
|
752
|
+
...this.credentials,
|
|
753
|
+
dbname: database,
|
|
754
|
+
});
|
|
755
|
+
return operation(client);
|
|
756
|
+
}
|
|
757
|
+
async listDatabases() {
|
|
758
|
+
const databaseNames = new Set([this.credentials.dbname]);
|
|
759
|
+
const { rows } = await this.runQuery(`
|
|
760
|
+
SELECT datname
|
|
761
|
+
FROM pg_database
|
|
762
|
+
WHERE datallowconn
|
|
763
|
+
AND NOT datistemplate
|
|
764
|
+
AND has_database_privilege(datname, 'CONNECT')
|
|
765
|
+
ORDER BY datname
|
|
766
|
+
LIMIT ${common_1.WAREHOUSE_LISTED_DATABASES_LIMIT + 1}
|
|
767
|
+
`);
|
|
768
|
+
rows.forEach(({ datname }) => {
|
|
769
|
+
if (typeof datname === 'string') {
|
|
770
|
+
databaseNames.add(datname);
|
|
771
|
+
}
|
|
772
|
+
});
|
|
773
|
+
return {
|
|
774
|
+
databases: [...databaseNames]
|
|
775
|
+
.slice(0, common_1.WAREHOUSE_LISTED_DATABASES_LIMIT)
|
|
776
|
+
.map((database) => this.toListedDatabase(database)),
|
|
777
|
+
truncated: databaseNames.size > common_1.WAREHOUSE_LISTED_DATABASES_LIMIT,
|
|
778
|
+
limit: common_1.WAREHOUSE_LISTED_DATABASES_LIMIT,
|
|
779
|
+
};
|
|
780
|
+
}
|
|
781
|
+
async getTablesForDatabase(listedDatabase) {
|
|
782
|
+
const tables = await this.withClientForDatabase(listedDatabase.name, (client) => client.getAllTables());
|
|
783
|
+
return tables.map((table) => ({
|
|
784
|
+
...table,
|
|
785
|
+
database: listedDatabase.name,
|
|
786
|
+
}));
|
|
787
|
+
}
|
|
739
788
|
}
|
|
740
789
|
exports.PostgresWarehouseClient = PostgresWarehouseClient;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AnyType, CreateWarehouseCredentials, DimensionType, Metric, SupportedDbtAdapter, TimeIntervalUnit, UnnestSql, UnnestSqlArgs, WarehouseCatalog, WarehouseResults, WarehouseSqlBuilder, WarehouseTables, WeekDay, type TimestampDomain, type WarehouseExecuteAsyncQuery, type WarehouseExecuteAsyncQueryArgs, type WarehouseQueryPhase } from '@lightdash/common';
|
|
1
|
+
import { AnyType, CreateWarehouseCredentials, DimensionType, Metric, SupportedDbtAdapter, TimeIntervalUnit, UnnestSql, UnnestSqlArgs, WarehouseCatalog, WarehouseDatabaseListing, WarehouseListedDatabase, WarehouseResults, WarehouseSqlBuilder, WarehouseTables, WeekDay, type TimestampDomain, type WarehouseExecuteAsyncQuery, type WarehouseExecuteAsyncQueryArgs, type WarehouseQueryPhase } from '@lightdash/common';
|
|
2
2
|
import { type WarehouseClient } from '../types';
|
|
3
3
|
export default abstract class WarehouseBaseClient<T extends CreateWarehouseCredentials> implements WarehouseClient {
|
|
4
4
|
credentials: T;
|
|
@@ -39,6 +39,8 @@ export default abstract class WarehouseBaseClient<T extends CreateWarehouseCrede
|
|
|
39
39
|
concatString(...args: string[]): string;
|
|
40
40
|
abstract getAllTables(schema?: string, tags?: Record<string, string>): Promise<WarehouseTables>;
|
|
41
41
|
abstract getFields(tableName: string, schema?: string, database?: string, tags?: Record<string, string>): Promise<WarehouseCatalog>;
|
|
42
|
+
listDatabases(_tags?: Record<string, string>): Promise<WarehouseDatabaseListing>;
|
|
43
|
+
getTablesForDatabase(_database: WarehouseListedDatabase, _tags?: Record<string, string>): Promise<WarehouseTables>;
|
|
42
44
|
parseWarehouseCatalog(rows: Record<string, AnyType>[], mapFieldType: (type: string) => DimensionType, mapTimestampDomain?: (type: string) => TimestampDomain | undefined): WarehouseCatalog;
|
|
43
45
|
parseError(error: Error): Error;
|
|
44
46
|
escapeString(value: string): string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"WarehouseBaseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/WarehouseBaseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,OAAO,EACP,0BAA0B,EAC1B,aAAa,EACb,MAAM,EAEN,mBAAmB,EACnB,gBAAgB,EAChB,SAAS,EACT,aAAa,EACb,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,OAAO,EACP,KAAK,eAAe,EACpB,KAAK,0BAA0B,EAC/B,KAAK,8BAA8B,EAEnC,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,UAAU,CAAC;AAEhD,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,mBAAmB,CAC7C,CAAC,SAAS,0BAA0B,CACtC,YAAW,eAAe;IACxB,WAAW,EAAE,CAAC,CAAC;IAEf,SAAS,CAAC,UAAU,EAAE,mBAAmB,CAAC;IAE1C,SAAS,aAAa,WAAW,EAAE,CAAC,EAAE,UAAU,EAAE,mBAAmB,EAGpE;IAED,cAAc,IAAI,mBAAmB,CAEpC;IAED,0BAA0B,IAAI,OAAO,CAEpC;IAED,kBAAkB,IAAI,MAAM,CAE3B;IAED,wBAAwB,IAAI,MAAM,CAEjC;IAED,iBAAiB,IAAI,MAAM,CAE1B;IAED,YAAY,CAAC,IAAI,EAAE,aAAa,GAAG,SAAS,GAAG,IAAI,CAElD;IAED,eAAe,IAAI,MAAM,CAExB;IAED,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAEvD;IAED,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE3D;IAED,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAE1D;IAED,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAErC;IAED,QAAQ,CAAC,UAAU,CACf,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAE7B,QAAQ,CAAC,WAAW,CAChB,KAAK,EAAE,MAAM,EACb,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CAAC;IAEX,iBAAiB,CACnB,EACI,GAAG,EACH,MAAM,EACN,WAAW,EACX,IAAI,EACJ,QAAQ,GACX,EAAE,8BAA8B,EACjC,qBAAqB,CAAC,EAAE,CACpB,IAAI,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC,KACjC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAC1B,OAAO,CAAC,0BAA0B,CAAC,CA+BrC;IAEK,QAAQ,CACV,GAAG,EAAE,MAAM,EACX,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC7B,QAAQ,CAAC,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,OAAO,EAAE,EAClB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;;;OAoBxC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAEhD;IAED,cAAc,IAAI,OAAO,GAAG,IAAI,GAAG,SAAS,CAE3C;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAGK,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAEjD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;IAED,QAAQ,CAAC,YAAY,CACjB,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,eAAe,CAAC,CAAC;IAE5B,QAAQ,CAAC,SAAS,CACd,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"WarehouseBaseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/WarehouseBaseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,OAAO,EACP,0BAA0B,EAC1B,aAAa,EACb,MAAM,EAEN,mBAAmB,EACnB,gBAAgB,EAChB,SAAS,EACT,aAAa,EACb,gBAAgB,EAChB,wBAAwB,EAExB,uBAAuB,EACvB,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,OAAO,EACP,KAAK,eAAe,EACpB,KAAK,0BAA0B,EAC/B,KAAK,8BAA8B,EAEnC,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,UAAU,CAAC;AAEhD,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,mBAAmB,CAC7C,CAAC,SAAS,0BAA0B,CACtC,YAAW,eAAe;IACxB,WAAW,EAAE,CAAC,CAAC;IAEf,SAAS,CAAC,UAAU,EAAE,mBAAmB,CAAC;IAE1C,SAAS,aAAa,WAAW,EAAE,CAAC,EAAE,UAAU,EAAE,mBAAmB,EAGpE;IAED,cAAc,IAAI,mBAAmB,CAEpC;IAED,0BAA0B,IAAI,OAAO,CAEpC;IAED,kBAAkB,IAAI,MAAM,CAE3B;IAED,wBAAwB,IAAI,MAAM,CAEjC;IAED,iBAAiB,IAAI,MAAM,CAE1B;IAED,YAAY,CAAC,IAAI,EAAE,aAAa,GAAG,SAAS,GAAG,IAAI,CAElD;IAED,eAAe,IAAI,MAAM,CAExB;IAED,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAEvD;IAED,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE3D;IAED,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAE1D;IAED,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAErC;IAED,QAAQ,CAAC,UAAU,CACf,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAE7B,QAAQ,CAAC,WAAW,CAChB,KAAK,EAAE,MAAM,EACb,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CAAC;IAEX,iBAAiB,CACnB,EACI,GAAG,EACH,MAAM,EACN,WAAW,EACX,IAAI,EACJ,QAAQ,GACX,EAAE,8BAA8B,EACjC,qBAAqB,CAAC,EAAE,CACpB,IAAI,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC,KACjC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAC1B,OAAO,CAAC,0BAA0B,CAAC,CA+BrC;IAEK,QAAQ,CACV,GAAG,EAAE,MAAM,EACX,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC7B,QAAQ,CAAC,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,OAAO,EAAE,EAClB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;;;OAoBxC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAEhD;IAED,cAAc,IAAI,OAAO,GAAG,IAAI,GAAG,SAAS,CAE3C;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAGK,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAEjD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;IAED,QAAQ,CAAC,YAAY,CACjB,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,eAAe,CAAC,CAAC;IAE5B,QAAQ,CAAC,SAAS,CACd,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAEvB,aAAa,CACf,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,wBAAwB,CAAC,CAInC;IAEK,oBAAoB,CACtB,SAAS,EAAE,uBAAuB,EAClC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,eAAe,CAAC,CAI1B;IAED,qBAAqB,CACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAC/B,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,aAAa,EAC7C,kBAAkB,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,eAAe,GAAG,SAAS,GACnE,gBAAgB,CAiClB;IAED,UAAU,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK,CAE9B;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAElC;IAED,eAAe,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAElC;IAED,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAE7B;IAED,oBAAoB,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAEvC;IAED,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAE5D;IAED,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM,CAKR;IAED,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAErC;CACJ"}
|
|
@@ -95,6 +95,12 @@ class WarehouseBaseClient {
|
|
|
95
95
|
concatString(...args) {
|
|
96
96
|
return this.sqlBuilder.concatString(...args);
|
|
97
97
|
}
|
|
98
|
+
async listDatabases(_tags) {
|
|
99
|
+
throw new common_1.WarehouseDatabaseListingNotSupportedError(this.credentials.type);
|
|
100
|
+
}
|
|
101
|
+
async getTablesForDatabase(_database, _tags) {
|
|
102
|
+
throw new common_1.WarehouseDatabaseListingNotSupportedError(this.credentials.type);
|
|
103
|
+
}
|
|
98
104
|
parseWarehouseCatalog(rows, mapFieldType, mapTimestampDomain) {
|
|
99
105
|
return rows.reduce((acc, { table_catalog, table_schema, table_name, column_name, data_type, }) => {
|
|
100
106
|
acc[table_catalog] = acc[table_catalog] || {};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.316.0",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"@databricks/sql": "1.13.0",
|
|
22
22
|
"@duckdb/node-api": "1.5.2-r.2",
|
|
23
23
|
"@google-cloud/bigquery": "8.3.1",
|
|
24
|
-
"@lightdash/common": "2.
|
|
24
|
+
"@lightdash/common": "2.316.0",
|
|
25
25
|
"node-fetch": "2.7.0",
|
|
26
26
|
"pg": "8.13.1",
|
|
27
27
|
"pg-cursor": "2.10.0",
|