@onivoro/server-aws-redshift 24.13.0 → 24.14.0

package/jest.config.ts

@@ -0,0 +1,11 @@
+/* eslint-disable */
+export default {
+  displayName: 'lib-server-aws-redshift',
+  preset: '../../../jest.preset.js',
+  testEnvironment: 'node',
+  transform: {
+    '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '<rootDir>/tsconfig.spec.json' }],
+  },
+  moduleFileExtensions: ['ts', 'js', 'html'],
+  coverageDirectory: '../../../coverage/libs/server/aws-redshift',
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@onivoro/server-aws-redshift",
-  "version": "24.13.0",
+  "version": "24.14.0",
   "type": "commonjs",
   "main": "./index.js",
   "types": "./index.d.ts",
@@ -9,8 +9,8 @@
     "url": "https://github.com/onivoro/monorepo.git"
   },
   "dependencies": {
-    "@onivoro/server-aws-credential-providers": "24.13.0",
-    "@onivoro/server-common": "24.13.0",
+    "@onivoro/server-aws-credential-providers": "24.14.0",
+    "@onivoro/server-common": "24.14.0",
     "tslib": "^2.3.0"
   },
   "peerDependencies": {
@@ -18,11 +18,5 @@
     "@aws-sdk/client-redshift-data": "~3.782.0",
     "@aws-sdk/client-redshift-serverless": "~3.782.0",
     "@nestjs/common": "~10.4.6"
-  },
-  "files": [
-    "**/*.js",
-    "**/*.d.ts",
-    "**/*.js.map",
-    "README.md"
-  ]
+  }
 }

package/project.json

@@ -0,0 +1,33 @@
+{
+  "name": "lib-server-aws-redshift",
+  "$schema": "../../../node_modules/nx/schemas/project-schema.json",
+  "sourceRoot": "libs/server/aws-redshift/src",
+  "projectType": "library",
+  "targets": {
+    "build": {
+      "executor": "@nx/js:tsc",
+      "outputs": ["{options.outputPath}"],
+      "options": {
+        "outputPath": "dist/libs/server/aws-redshift",
+        "main": "libs/server/aws-redshift/src/index.ts",
+        "tsConfig": "libs/server/aws-redshift/tsconfig.lib.json",
+        "assets": [
+          "libs/server/aws-redshift/README.md",
+          "libs/server/aws-redshift/package.json"
+        ],
+        "declaration": true,
+        "updateBuildableProjectPackageJsonDependencies": true,
+        "rootDir": "libs/server/aws-redshift/src"
+      }
+    },
+    "publish": {
+      "executor": "@nx/js:npm",
+      "outputs": [],
+      "dependsOn": ["build"],
+      "options": {
+        "packageRoot": "dist/libs/server/aws-redshift"
+      }
+    }
+  },
+  "tags": []
+}

package/src/index.ts

@@ -0,0 +1,5 @@
+export * from './lib/classes/server-aws-redshift-config.class';
+
+export * from './lib/services/redshift.service';
+
+export * from './lib/server-aws-redshift.module';

package/src/lib/classes/server-aws-redshift-config.class.ts

@@ -0,0 +1,4 @@
+export class ServerAwsRedshiftDataConfig {
+    AWS_PROFILE?: string;
+    AWS_REGION: string;
+}

package/src/lib/server-aws-redshift.module.ts

@@ -0,0 +1,56 @@
+import { Module } from '@nestjs/common';
+import { moduleFactory } from '@onivoro/server-common';
+import { RedshiftDataClient, RedshiftDataClientConfig } from '@aws-sdk/client-redshift-data';
+import { RedshiftDataService } from './services/redshift.service';
+import { ServerAwsRedshiftDataConfig } from './classes/server-aws-redshift-config.class';
+import { RedshiftServerlessClient } from '@aws-sdk/client-redshift-serverless';
+import { RedshiftClient } from '@aws-sdk/client-redshift';
+import { AwsCredentials, ServerAwsCredentialProvidersModule } from '@onivoro/server-aws-credential-providers';
+
+let redshiftClient: RedshiftClient | null = null;
+let redshiftDataClient: RedshiftDataClient | null = null;
+let redshiftServerlessClient: RedshiftServerlessClient | null = null;
+
+
+@Module({})
+export class ServerAwsRedshiftDataModule {
+  static configure(config: ServerAwsRedshiftDataConfig) {
+    return moduleFactory({
+      module: ServerAwsRedshiftDataModule,
+      imports: [ServerAwsCredentialProvidersModule.configure(config)],
+      providers: [
+        {
+          provide: RedshiftDataClient,
+          useFactory: (credentials: AwsCredentials) => redshiftDataClient
+            ? redshiftDataClient
+            : redshiftDataClient = new RedshiftDataClient(from(config, credentials)),
+          inject: [AwsCredentials]
+        },
+        {
+          provide: RedshiftServerlessClient,
+          useFactory: (credentials: AwsCredentials) => redshiftServerlessClient
+            ? redshiftServerlessClient
+            : redshiftServerlessClient = new RedshiftServerlessClient(from(config, credentials)),
+          inject: [AwsCredentials]
+        },
+        {
+          provide: RedshiftClient,
+          useFactory: (credentials: AwsCredentials) => redshiftClient
+            ? redshiftClient
+            : redshiftClient = new RedshiftClient(from(config, credentials)),
+          inject: [AwsCredentials]
+        },
+        { provide: ServerAwsRedshiftDataConfig, useValue: config },
+        RedshiftDataService
+      ]
+    })
+  }
+}
+
+function from(config: ServerAwsRedshiftDataConfig, credentials: AwsCredentials): RedshiftDataClientConfig {
+  return {
+    region: config.AWS_REGION,
+    logger: console,
+    credentials
+  };
+}

package/src/lib/services/redshift.service.ts

@@ -0,0 +1,293 @@
+import { Injectable } from '@nestjs/common';
+import { DescribeStatementCommand, ExecuteStatementCommand, GetStatementResultCommand, RedshiftDataClient } from "@aws-sdk/client-redshift-data";
+import { GetWorkgroupCommand, GetWorkgroupCommandInput, RedshiftServerlessClient } from '@aws-sdk/client-redshift-serverless';
+import { randomUUID } from 'crypto';
+
+@Injectable()
+export class RedshiftDataService {
+    constructor(
+        private redshiftDataClient: RedshiftDataClient,
+        private redshiftServerlessClient: RedshiftServerlessClient,
+    ) { }
+
+    async waitForStatement(statementId: string, maxAttempts = 10, delay = 1000) {
+        let attempts = 0;
+
+        while (attempts < maxAttempts) {
+            const describeStatementCommand = new DescribeStatementCommand({
+                Id: statementId
+            });
+
+            const status = await this.redshiftDataClient.send(describeStatementCommand);
+
+            if (status.Status === 'FINISHED') {
+                return status.ResultRows;
+            }
+
+            if (status.Status === 'FAILED') {
+                throw new Error(`SQL statement failed: ${status.Error}`);
+            }
+
+            attempts++;
+            await new Promise(resolve => setTimeout(resolve, delay));
+        }
+
+        throw new Error('Timeout waiting for SQL statement to complete');
+    }
+
+    async verifyEndpointAccess(workgroupName: string) {
+        const workgroupResponse = await this.redshiftServerlessClient.send(
+            new GetWorkgroupCommand({ workgroupName })
+        );
+
+        if (!workgroupResponse?.workgroup?.endpoint?.address) {
+            console.error(`Could not retrieve endpoint for workgroupName "${workgroupName}"`);
+
+            return;
+        }
+
+        return workgroupResponse;
+    }
+
+    async addIamUserToDatabaseGroup(_: { database: string, workgroupName: string, user: string, group: string }) {
+        try {
+            const groupResult = await this.redshiftDataClient.send(new ExecuteStatementCommand({
+                Database: _.database,
+                Sql: `ALTER GROUP ${_.group} ADD USER "${_.user}";`,
+                WorkgroupName: _.workgroupName
+            }));
+
+            await this.waitForStatement(groupResult.Id!);
+
+            console.log(`Added user "${_.user}" to group "${_.group}`);
+        } catch (error: any) {
+            console.warn({ detail: `Warning adding ${_.user} to group:`, error });
+        }
+    }
+
+    async createDatabaseUser(_: { database: string, workgroupName: string, user: string }): Promise<string> {
+        try {
+
+            const checkResult = await this.redshiftDataClient.send(new ExecuteStatementCommand({
+                Database: _.database,
+                Sql: `SELECT u.usename as username FROM pg_user u;`,
+                WorkgroupName: _.workgroupName
+            }));
+
+            await this.waitForStatement(checkResult.Id!);
+
+            const describeCommand = new DescribeStatementCommand({ Id: checkResult.Id });
+            const checkStatus = await this.redshiftDataClient.send(describeCommand);
+
+            if (!checkStatus.ResultRows) {
+                const placeholderPassword = `IAM_${randomUUID().replace(/-/g, '_')}`;
+
+                const createUserCommand = new ExecuteStatementCommand({
+                    Database: _.database,
+                    Sql: `CREATE USER ${_.user} PASSWORD '${placeholderPassword}';`,
+                    WorkgroupName: _.workgroupName
+                });
+
+                const createResult = await this.redshiftDataClient.send(createUserCommand);
+
+                await this.waitForStatement(createResult.Id!);
+
+                console.log(`Created user: ${_.user}`);
+
+                return placeholderPassword;
+            } else {
+                console.log(`User "${_.user} already exists`);
+
+                return '';
+            }
+
+        } catch (error: any) {
+            console.warn({ detail: `Warning creating user ${_.user}`, error });
+            throw error;
+        }
+    }
+
+    async grantUsageOnSchema(_: { database: string, workgroupName: string, schema: string, group: string }) {
+
+        try {
+            const setupStatements = [
+                `GRANT USAGE ON SCHEMA ${_.schema} TO GROUP ${_.group};`,
+                `GRANT SELECT ON ALL TABLES IN SCHEMA ${_.schema} TO GROUP ${_.group};`,
+                `ALTER DEFAULT PRIVILEGES IN SCHEMA ${_.schema} GRANT SELECT ON TABLES TO GROUP ${_.group};`,
+            ];
+
+            for (const Sql of setupStatements) {
+                const executeStatementCommand = new ExecuteStatementCommand({
+                    Database: _.database,
+                    Sql,
+                    WorkgroupName: _.workgroupName
+                });
+
+                try {
+                    const result = await this.redshiftDataClient.send(executeStatementCommand);
+                    await this.waitForStatement(result.Id!);
+                } catch (error: any) {
+                    console.warn({ Sql, error });
+                }
+            }
+
+        } catch (error) {
+            console.error({ error });
+            throw error;
+        }
+    }
+
+    async createDbGroupFromIamGroupIfNotExists(_: { iamGroup: string, database: string, workgroupName: string }): Promise<any> {
+
+        try {
+            const checkResult = await this.redshiftDataClient.send(new ExecuteStatementCommand({
+                Database: _.database,
+                Sql: `SELECT 1 FROM pg_group WHERE groname = '${_.iamGroup}';`,
+                WorkgroupName: _.workgroupName
+            }));
+
+            await this.waitForStatement(checkResult.Id!);
+
+            const describeCommand = new DescribeStatementCommand({ Id: checkResult.Id });
+            const checkStatus = await this.redshiftDataClient.send(describeCommand);
+
+            if (!checkStatus.ResultRows) {
+                const createGroupCommand = new ExecuteStatementCommand({
+                    Database: _.database,
+                    Sql: `CREATE GROUP ${_.iamGroup};`,
+                    WorkgroupName: _.workgroupName
+                });
+
+                const createResult = await this.redshiftDataClient.send(createGroupCommand);
+                await this.waitForStatement(createResult.Id!);
+                console.log(`Created "${_.iamGroup}" group`);
+            } else {
+                console.log(`"${_.iamGroup}" group already exists`);
+            }
+        } catch (error: any) {
+            console.error({ error });
+            throw error;
+        }
+    }
+
+    async queryV1(_: { database: string, workgroupName: string }, Sql: string) {
+        const result = await this.redshiftDataClient.send(new ExecuteStatementCommand({
+            Database: _.database,
+            Sql,
+            WorkgroupName: _.workgroupName
+        }));
+
+        return await this.waitForStatement(result.Id!);
+    }
+
+
+    async query(_: { database: string, workgroupName: string }, Sql: string) {
+        try {
+            const executeCommand = new ExecuteStatementCommand({
+                Database: _.database,
+                Sql,
+                WorkgroupName: _.workgroupName,
+            });
+
+            const { Id } = await this.redshiftDataClient.send(executeCommand);
+
+            if (!Id) {
+                throw new Error("Failed to get statement ID");
+            }
+
+            let status = "";
+
+            do {
+                const describeCommand = new DescribeStatementCommand({ Id });
+                const { Status, Error: err } = await this.redshiftDataClient.send(describeCommand);
+                status = Status || "";
+
+                if (status === "FAILED") {
+                    throw new Error(err);
+                }
+
+                if (status !== "FINISHED") {
+                    await new Promise(resolve => setTimeout(resolve, 1000));
+                }
+            } while (status !== "FINISHED");
+
+            try {
+                const getResultsCommand = new GetStatementResultCommand({ Id });
+                const results = await this.redshiftDataClient.send(getResultsCommand);
+
+                const records = results.Records || [];
+                return records.map(record => {
+                    return record.map(field => field.stringValue || field.longValue || field.doubleValue);
+                });
+            } catch (error) {
+                return [];
+            }
+        } catch (error) {
+            console.error({ detail: "Error executing query:", error });
+            throw error;
+        }
+    }
+
+    async getAssociatedIAmRolesByWorkgroup(_: { database: string, workgroupName: string }) {
+        try {
+            const query = `
+            SELECT
+              g.groname as role_name,
+              u.usename as member_name,
+              g.grosysid as role_id,
+              g.groowner as owner_id,
+              CASE WHEN pg_has_role(u.usename, g.groname, 'MEMBER') THEN true ELSE false END as is_member
+            FROM pg_group g
+            LEFT JOIN pg_user u ON u.usesysid = ANY(g.grolist)
+            WHERE EXISTS (
+              SELECT 1
+              FROM svv_redshift_databases d
+              WHERE d.database_name = current_database()
+              AND d.workgroup_name = $1
+            )
+            ORDER BY g.groname, u.usename;
+          `;
+
+            return await this.query(_, query)
+        } catch (error) {
+            if (error instanceof Error) {
+                console.error(`Error getting workgroup roles: ${error.message}`);
+                throw error;
+            }
+            throw new Error('An unknown error occurred');
+        }
+    }
+
+    private async upsert<TValue>(_: { database: string, workgroupName: string }, value: TValue, tableQualifiers: { schema: string, table: string }) {
+        const { table, schema } = tableQualifiers;
+        // TODO: dynamically build out the select/insert clauses dynamically
+        //         try {
+        //             const upsertQuery = `
+        // WITH ${table}_x AS (SELECT '${value.user_id}' as user_id, '${value.profile_id}' as profile_id, '${value.email}' as email, '${value.date}' as date, '${value.weight}' as weight, '${value.source}' as source, '${value.created_at}' as created_at)
+        // UPDATE ${schema}.${table}
+        // SET
+        //   patient = a.patient,
+        //   data = a.data
+        // FROM ${table}_x AS a
+        // WHERE ${schema}.${table}.id = a.id;
+
+        // INSERT INTO ${schema}.${table} (
+        //   id,
+        //   patient,
+        //   data
+        // )
+        // SELECT
+        //   a.id,
+        //   a.patient,
+        //   a.data
+        // FROM ${table}_x AS a
+        // LEFT JOIN ${schema}.${table} AS b
+        //   ON a.id = b.id
+        // WHERE b.id IS NULL;`;
+
+        //             return await this.query(_, upsertQuery);
+        //         } catch (error: any) {
+        //             console.error({ error, detail: `"${schema}.${table}" upsert failed` });
+        //         }
+    }
+}

package/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "extends": "../../../tsconfig.server.json",
+  "compilerOptions": {
+    "outDir": "../../../dist/out-tsc"
+  },
+  "files": [],
+  "include": [],
+  "references": [
+    {
+      "path": "./tsconfig.lib.json"
+    },
+    {
+      "path": "./tsconfig.spec.json"
+    }
+  ]
+}

package/tsconfig.lib.json

@@ -0,0 +1,10 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "../../dist/libs/server/aws-redshift",
+    "declaration": true
+  },
+  "exclude": ["jest.config.ts", "**/*.spec.ts", "**/*.test.ts"],
+  "include": ["src/**/*.ts"]
+}

package/tsconfig.spec.json

@@ -0,0 +1,21 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "jest.config.ts",
+    "**/*.test.ts",
+    "**/*.spec.ts",
+    "**/*.test.tsx",
+    "**/*.spec.tsx",
+    "**/*.test.js",
+    "**/*.spec.js",
+    "**/*.test.jsx",
+    "**/*.spec.jsx",
+    "**/*.d.ts"
+  ]
+}