@onivoro/server-aws-redshift 24.10.0 → 24.11.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@onivoro/server-aws-redshift",
-  "version": "24.10.0",
+  "version": "24.11.0",
   "type": "commonjs",
   "main": "./index.js",
   "types": "./index.d.ts",
@@ -9,8 +9,8 @@
     "url": "https://github.com/onivoro/monorepo.git"
   },
   "dependencies": {
-    "@onivoro/server-aws-credential-providers": "24.10.0",
-    "@onivoro/server-common": "24.10.0",
+    "@onivoro/server-aws-credential-providers": "24.11.0",
+    "@onivoro/server-common": "24.11.0",
     "tslib": "^2.3.0"
   },
   "peerDependencies": {
@@ -18,5 +18,11 @@
     "@aws-sdk/client-redshift-data": "~3.782.0",
     "@aws-sdk/client-redshift-serverless": "~3.782.0",
     "@nestjs/common": "~10.4.6"
-  }
+  },
+  "files": [
+    "**/*.js",
+    "**/*.d.ts",
+    "**/*.js.map",
+    "README.md"
+  ]
 }

package/jest.config.ts

@@ -1,11 +0,0 @@
-/* eslint-disable */
-export default {
-  displayName: 'lib-server-aws-redshift',
-  preset: '../../../jest.preset.js',
-  testEnvironment: 'node',
-  transform: {
-    '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '<rootDir>/tsconfig.spec.json' }],
-  },
-  moduleFileExtensions: ['ts', 'js', 'html'],
-  coverageDirectory: '../../../coverage/libs/server/aws-redshift',
-};

package/project.json

@@ -1,33 +0,0 @@
-{
-  "name": "lib-server-aws-redshift",
-  "$schema": "../../../node_modules/nx/schemas/project-schema.json",
-  "sourceRoot": "libs/server/aws-redshift/src",
-  "projectType": "library",
-  "targets": {
-    "build": {
-      "executor": "@nx/js:tsc",
-      "outputs": ["{options.outputPath}"],
-      "options": {
-        "outputPath": "dist/libs/server/aws-redshift",
-        "main": "libs/server/aws-redshift/src/index.ts",
-        "tsConfig": "libs/server/aws-redshift/tsconfig.lib.json",
-        "assets": [
-          "libs/server/aws-redshift/README.md",
-          "libs/server/aws-redshift/package.json"
-        ],
-        "declaration": true,
-        "updateBuildableProjectPackageJsonDependencies": true,
-        "rootDir": "libs/server/aws-redshift/src"
-      }
-    },
-    "publish": {
-      "executor": "@nx/js:npm",
-      "outputs": [],
-      "dependsOn": ["build"],
-      "options": {
-        "packageRoot": "dist/libs/server/aws-redshift"
-      }
-    }
-  },
-  "tags": []
-}

package/src/index.ts

@@ -1,5 +0,0 @@
-export * from './lib/classes/server-aws-redshift-config.class';
-
-export * from './lib/services/redshift.service';
-
-export * from './lib/server-aws-redshift.module';

package/src/lib/classes/server-aws-redshift-config.class.ts

@@ -1,4 +0,0 @@
-export class ServerAwsRedshiftDataConfig {
-    AWS_PROFILE?: string;
-    AWS_REGION: string;
-}

package/src/lib/server-aws-redshift.module.ts

@@ -1,56 +0,0 @@
-import { Module } from '@nestjs/common';
-import { moduleFactory } from '@onivoro/server-common';
-import { RedshiftDataClient, RedshiftDataClientConfig } from '@aws-sdk/client-redshift-data';
-import { RedshiftDataService } from './services/redshift.service';
-import { ServerAwsRedshiftDataConfig } from './classes/server-aws-redshift-config.class';
-import { RedshiftServerlessClient } from '@aws-sdk/client-redshift-serverless';
-import { RedshiftClient } from '@aws-sdk/client-redshift';
-import { AwsCredentials, ServerAwsCredentialProvidersModule } from '@onivoro/server-aws-credential-providers';
-
-let redshiftClient: RedshiftClient | null = null;
-let redshiftDataClient: RedshiftDataClient | null = null;
-let redshiftServerlessClient: RedshiftServerlessClient | null = null;
-
-
-@Module({})
-export class ServerAwsRedshiftDataModule {
-  static configure(config: ServerAwsRedshiftDataConfig) {
-    return moduleFactory({
-      module: ServerAwsRedshiftDataModule,
-      imports: [ServerAwsCredentialProvidersModule.configure(config)],
-      providers: [
-        {
-          provide: RedshiftDataClient,
-          useFactory: (credentials: AwsCredentials) => redshiftDataClient
-            ? redshiftDataClient
-            : redshiftDataClient = new RedshiftDataClient(from(config, credentials)),
-          inject: [AwsCredentials]
-        },
-        {
-          provide: RedshiftServerlessClient,
-          useFactory: (credentials: AwsCredentials) => redshiftServerlessClient
-            ? redshiftServerlessClient
-            : redshiftServerlessClient = new RedshiftServerlessClient(from(config, credentials)),
-          inject: [AwsCredentials]
-        },
-        {
-          provide: RedshiftClient,
-          useFactory: (credentials: AwsCredentials) => redshiftClient
-            ? redshiftClient
-            : redshiftClient = new RedshiftClient(from(config, credentials)),
-          inject: [AwsCredentials]
-        },
-        { provide: ServerAwsRedshiftDataConfig, useValue: config },
-        RedshiftDataService
-      ]
-    })
-  }
-}
-
-function from(config: ServerAwsRedshiftDataConfig, credentials: AwsCredentials): RedshiftDataClientConfig {
-  return {
-    region: config.AWS_REGION,
-    logger: console,
-    credentials
-  };
-}

package/src/lib/services/redshift.service.ts

@@ -1,293 +0,0 @@
-import { Injectable } from '@nestjs/common';
-import { DescribeStatementCommand, ExecuteStatementCommand, GetStatementResultCommand, RedshiftDataClient } from "@aws-sdk/client-redshift-data";
-import { GetWorkgroupCommand, GetWorkgroupCommandInput, RedshiftServerlessClient } from '@aws-sdk/client-redshift-serverless';
-import { randomUUID } from 'crypto';
-
-@Injectable()
-export class RedshiftDataService {
-    constructor(
-        private redshiftDataClient: RedshiftDataClient,
-        private redshiftServerlessClient: RedshiftServerlessClient,
-    ) { }
-
-    async waitForStatement(statementId: string, maxAttempts = 10, delay = 1000) {
-        let attempts = 0;
-
-        while (attempts < maxAttempts) {
-            const describeStatementCommand = new DescribeStatementCommand({
-                Id: statementId
-            });
-
-            const status = await this.redshiftDataClient.send(describeStatementCommand);
-
-            if (status.Status === 'FINISHED') {
-                return status.ResultRows;
-            }
-
-            if (status.Status === 'FAILED') {
-                throw new Error(`SQL statement failed: ${status.Error}`);
-            }
-
-            attempts++;
-            await new Promise(resolve => setTimeout(resolve, delay));
-        }
-
-        throw new Error('Timeout waiting for SQL statement to complete');
-    }
-
-    async verifyEndpointAccess(workgroupName: string) {
-        const workgroupResponse = await this.redshiftServerlessClient.send(
-            new GetWorkgroupCommand({ workgroupName })
-        );
-
-        if (!workgroupResponse?.workgroup?.endpoint?.address) {
-            console.error(`Could not retrieve endpoint for workgroupName "${workgroupName}"`);
-
-            return;
-        }
-
-        return workgroupResponse;
-    }
-
-    async addIamUserToDatabaseGroup(_: { database: string, workgroupName: string, user: string, group: string }) {
-        try {
-            const groupResult = await this.redshiftDataClient.send(new ExecuteStatementCommand({
-                Database: _.database,
-                Sql: `ALTER GROUP ${_.group} ADD USER "${_.user}";`,
-                WorkgroupName: _.workgroupName
-            }));
-
-            await this.waitForStatement(groupResult.Id!);
-
-            console.log(`Added user "${_.user}" to group "${_.group}`);
-        } catch (error: any) {
-            console.warn({ detail: `Warning adding ${_.user} to group:`, error });
-        }
-    }
-
-    async createDatabaseUser(_: { database: string, workgroupName: string, user: string }): Promise<string> {
-        try {
-
-            const checkResult = await this.redshiftDataClient.send(new ExecuteStatementCommand({
-                Database: _.database,
-                Sql: `SELECT u.usename as username FROM pg_user u;`,
-                WorkgroupName: _.workgroupName
-            }));
-
-            await this.waitForStatement(checkResult.Id!);
-
-            const describeCommand = new DescribeStatementCommand({ Id: checkResult.Id });
-            const checkStatus = await this.redshiftDataClient.send(describeCommand);
-
-            if (!checkStatus.ResultRows) {
-                const placeholderPassword = `IAM_${randomUUID().replace(/-/g, '_')}`;
-
-                const createUserCommand = new ExecuteStatementCommand({
-                    Database: _.database,
-                    Sql: `CREATE USER ${_.user} PASSWORD '${placeholderPassword}';`,
-                    WorkgroupName: _.workgroupName
-                });
-
-                const createResult = await this.redshiftDataClient.send(createUserCommand);
-
-                await this.waitForStatement(createResult.Id!);
-
-                console.log(`Created user: ${_.user}`);
-
-                return placeholderPassword;
-            } else {
-                console.log(`User "${_.user} already exists`);
-
-                return '';
-            }
-
-        } catch (error: any) {
-            console.warn({ detail: `Warning creating user ${_.user}`, error });
-            throw error;
-        }
-    }
-
-    async grantUsageOnSchema(_: { database: string, workgroupName: string, schema: string, group: string }) {
-
-        try {
-            const setupStatements = [
-                `GRANT USAGE ON SCHEMA ${_.schema} TO GROUP ${_.group};`,
-                `GRANT SELECT ON ALL TABLES IN SCHEMA ${_.schema} TO GROUP ${_.group};`,
-                `ALTER DEFAULT PRIVILEGES IN SCHEMA ${_.schema} GRANT SELECT ON TABLES TO GROUP ${_.group};`,
-            ];
-
-            for (const Sql of setupStatements) {
-                const executeStatementCommand = new ExecuteStatementCommand({
-                    Database: _.database,
-                    Sql,
-                    WorkgroupName: _.workgroupName
-                });
-
-                try {
-                    const result = await this.redshiftDataClient.send(executeStatementCommand);
-                    await this.waitForStatement(result.Id!);
-                } catch (error: any) {
-                    console.warn({ Sql, error });
-                }
-            }
-
-        } catch (error) {
-            console.error({ error });
-            throw error;
-        }
-    }
-
-    async createDbGroupFromIamGroupIfNotExists(_: { iamGroup: string, database: string, workgroupName: string }): Promise<any> {
-
-        try {
-            const checkResult = await this.redshiftDataClient.send(new ExecuteStatementCommand({
-                Database: _.database,
-                Sql: `SELECT 1 FROM pg_group WHERE groname = '${_.iamGroup}';`,
-                WorkgroupName: _.workgroupName
-            }));
-
-            await this.waitForStatement(checkResult.Id!);
-
-            const describeCommand = new DescribeStatementCommand({ Id: checkResult.Id });
-            const checkStatus = await this.redshiftDataClient.send(describeCommand);
-
-            if (!checkStatus.ResultRows) {
-                const createGroupCommand = new ExecuteStatementCommand({
-                    Database: _.database,
-                    Sql: `CREATE GROUP ${_.iamGroup};`,
-                    WorkgroupName: _.workgroupName
-                });
-
-                const createResult = await this.redshiftDataClient.send(createGroupCommand);
-                await this.waitForStatement(createResult.Id!);
-                console.log(`Created "${_.iamGroup}" group`);
-            } else {
-                console.log(`"${_.iamGroup}" group already exists`);
-            }
-        } catch (error: any) {
-            console.error({ error });
-            throw error;
-        }
-    }
-
-    async queryV1(_: { database: string, workgroupName: string }, Sql: string) {
-        const result = await this.redshiftDataClient.send(new ExecuteStatementCommand({
-            Database: _.database,
-            Sql,
-            WorkgroupName: _.workgroupName
-        }));
-
-        return await this.waitForStatement(result.Id!);
-    }
-
-
-    async query(_: { database: string, workgroupName: string }, Sql: string) {
-        try {
-            const executeCommand = new ExecuteStatementCommand({
-                Database: _.database,
-                Sql,
-                WorkgroupName: _.workgroupName,
-            });
-
-            const { Id } = await this.redshiftDataClient.send(executeCommand);
-
-            if (!Id) {
-                throw new Error("Failed to get statement ID");
-            }
-
-            let status = "";
-
-            do {
-                const describeCommand = new DescribeStatementCommand({ Id });
-                const { Status, Error: err } = await this.redshiftDataClient.send(describeCommand);
-                status = Status || "";
-
-                if (status === "FAILED") {
-                    throw new Error(err);
-                }
-
-                if (status !== "FINISHED") {
-                    await new Promise(resolve => setTimeout(resolve, 1000));
-                }
-            } while (status !== "FINISHED");
-
-            try {
-                const getResultsCommand = new GetStatementResultCommand({ Id });
-                const results = await this.redshiftDataClient.send(getResultsCommand);
-
-                const records = results.Records || [];
-                return records.map(record => {
-                    return record.map(field => field.stringValue || field.longValue || field.doubleValue);
-                });
-            } catch (error) {
-                return [];
-            }
-        } catch (error) {
-            console.error({ detail: "Error executing query:", error });
-            throw error;
-        }
-    }
-
-    async getAssociatedIAmRolesByWorkgroup(_: { database: string, workgroupName: string }) {
-        try {
-            const query = `
-            SELECT
-              g.groname as role_name,
-              u.usename as member_name,
-              g.grosysid as role_id,
-              g.groowner as owner_id,
-              CASE WHEN pg_has_role(u.usename, g.groname, 'MEMBER') THEN true ELSE false END as is_member
-            FROM pg_group g
-            LEFT JOIN pg_user u ON u.usesysid = ANY(g.grolist)
-            WHERE EXISTS (
-              SELECT 1
-              FROM svv_redshift_databases d
-              WHERE d.database_name = current_database()
-              AND d.workgroup_name = $1
-            )
-            ORDER BY g.groname, u.usename;
-          `;
-
-            return await this.query(_, query)
-        } catch (error) {
-            if (error instanceof Error) {
-                console.error(`Error getting workgroup roles: ${error.message}`);
-                throw error;
-            }
-            throw new Error('An unknown error occurred');
-        }
-    }
-
-    private async upsert<TValue>(_: { database: string, workgroupName: string }, value: TValue, tableQualifiers: { schema: string, table: string }) {
-        const { table, schema } = tableQualifiers;
-        // TODO: dynamically build out the select/insert clauses dynamically
-        //         try {
-        //             const upsertQuery = `
-        // WITH ${table}_x AS (SELECT '${value.user_id}' as user_id, '${value.profile_id}' as profile_id, '${value.email}' as email, '${value.date}' as date, '${value.weight}' as weight, '${value.source}' as source, '${value.created_at}' as created_at)
-        // UPDATE ${schema}.${table}
-        // SET
-        //   patient = a.patient,
-        //   data = a.data
-        // FROM ${table}_x AS a
-        // WHERE ${schema}.${table}.id = a.id;
-
-        // INSERT INTO ${schema}.${table} (
-        //   id,
-        //   patient,
-        //   data
-        // )
-        // SELECT
-        //   a.id,
-        //   a.patient,
-        //   a.data
-        // FROM ${table}_x AS a
-        // LEFT JOIN ${schema}.${table} AS b
-        //   ON a.id = b.id
-        // WHERE b.id IS NULL;`;
-
-        //             return await this.query(_, upsertQuery);
-        //         } catch (error: any) {
-        //             console.error({ error, detail: `"${schema}.${table}" upsert failed` });
-        //         }
-    }
-}

package/tsconfig.json

@@ -1,16 +0,0 @@
-{
-  "extends": "../../../tsconfig.server.json",
-  "compilerOptions": {
-    "outDir": "../../../dist/out-tsc"
-  },
-  "files": [],
-  "include": [],
-  "references": [
-    {
-      "path": "./tsconfig.lib.json"
-    },
-    {
-      "path": "./tsconfig.spec.json"
-    }
-  ]
-}

package/tsconfig.lib.json

@@ -1,10 +0,0 @@
-{
-  "extends": "./tsconfig.json",
-  "compilerOptions": {
-    "rootDir": "./src",
-    "outDir": "../../dist/libs/server/aws-redshift",
-    "declaration": true
-  },
-  "exclude": ["jest.config.ts", "**/*.spec.ts", "**/*.test.ts"],
-  "include": ["src/**/*.ts"]
-}

package/tsconfig.spec.json

@@ -1,21 +0,0 @@
-{
-  "extends": "./tsconfig.json",
-  "compilerOptions": {
-    "types": [
-      "jest",
-      "node"
-    ]
-  },
-  "include": [
-    "jest.config.ts",
-    "**/*.test.ts",
-    "**/*.spec.ts",
-    "**/*.test.tsx",
-    "**/*.spec.tsx",
-    "**/*.test.js",
-    "**/*.spec.js",
-    "**/*.test.jsx",
-    "**/*.spec.jsx",
-    "**/*.d.ts"
-  ]
-}