@dbos-inc/sqs-receive 3.0.10-preview

package/index.ts

@@ -0,0 +1,226 @@
+import { DBOS, Error as DBOSError, DBOSLifecycleCallback } from '@dbos-inc/dbos-sdk';
+
+import {
+  DeleteMessageCommand,
+  GetQueueAttributesCommand,
+  GetQueueAttributesCommandInput,
+  Message,
+  ReceiveMessageCommand,
+  ReceiveMessageCommandOutput,
+  SQSClient,
+} from '@aws-sdk/client-sqs';
+
+interface SQSConfig {
+  client?: SQSClient | (() => SQSClient);
+  queueUrl?: string;
+  getWorkflowKey?: (m: Message) => string;
+  workflowQueueName?: string;
+}
+
+interface SQSRecvClassConfig {
+  config?: SQSConfig;
+}
+
+interface SQSRecvMethodConfig {
+  config?: SQSConfig;
+}
+
+class SQSReceiver extends DBOSLifecycleCallback {
+  config?: SQSConfig;
+
+  constructor(config?: SQSConfig) {
+    super();
+    this.config = config;
+  }
+
+  listeners: Promise<void>[] = [];
+  isShuttingDown = false;
+
+  override async destroy() {
+    this.isShuttingDown = true;
+    try {
+      await Promise.allSettled(this.listeners);
+    } catch (e) {
+      // yawn
+    }
+    this.listeners = [];
+  }
+
+  // async function that uses .then/.catch to handle potentially unreliable library calls
+  static async sendReceiveMessageCommandSafe(
+    sqs: SQSClient,
+    params: ReceiveMessageCommand,
+  ): Promise<ReceiveMessageCommandOutput> {
+    return new Promise((resolve, reject) => {
+      sqs
+        .send(params)
+        .then((response) => resolve(response))
+        .catch((error) => reject(error as Error));
+    });
+  }
+
+  static async validateConnection(client: SQSClient, url: string) {
+    const params: GetQueueAttributesCommandInput = {
+      QueueUrl: url,
+      AttributeNames: ['All'],
+    };
+
+    const _validateSQSConfiguration = await client.send(new GetQueueAttributesCommand(params));
+  }
+
+  override async initialize() {
+    const regops = DBOS.getAssociatedInfo(this);
+
+    for (const regop of regops) {
+      const { methodConfig, classConfig, methodReg } = regop;
+      const sqsclass = classConfig as SQSRecvClassConfig;
+      const sqsmethod = methodConfig as SQSRecvMethodConfig;
+      if (!sqsmethod) continue;
+
+      const url = sqsclass.config?.queueUrl ?? sqsmethod.config?.queueUrl ?? this.config?.queueUrl;
+      if (!url) continue;
+
+      const cname = methodReg.className;
+      const mname = methodReg.name;
+
+      if (!methodReg.workflowConfig) {
+        throw new DBOSError.DBOSError(
+          `Error registering method ${cname}.${mname}: An SQS decorator can only be assigned to a workflow!`,
+        );
+      }
+
+      let sqs: SQSClient | undefined = undefined;
+      if (sqsmethod.config?.client) {
+        sqs = typeof sqsmethod.config.client === 'function' ? sqsmethod.config.client() : sqsmethod.config.client;
+      } else if (sqsclass.config?.client) {
+        sqs = typeof sqsclass.config.client === 'function' ? sqsclass.config.client() : sqsclass.config.client;
+      } else if (this.config?.client) {
+        sqs = typeof this.config.client === 'function' ? this.config.client() : this.config.client;
+      }
+
+      if (!sqs) {
+        throw new DBOSError.DBOSError(`Error initializing SQS method ${cname}.${mname}: No SQSClient provided`);
+      }
+      try {
+        await SQSReceiver.validateConnection(sqs, url);
+        DBOS.logger.info(`Successfully connected to SQS queue ${url} for ${cname}.${mname}`);
+      } catch (e) {
+        const err = e as Error;
+        DBOS.logger.error(err);
+        throw new DBOSError.DBOSError(
+          `SQS Receiver for ${cname}.${mname} was unable to connect to ${url}: ${err.message}`,
+        );
+      }
+
+      // eslint-disable-next-line @typescript-eslint/no-this-alias
+      const sqsr = this;
+      this.listeners.push(
+        (async (client: SQSClient, url: string) => {
+          while (!this.isShuttingDown) {
+            // Get message
+            try {
+              const response = await SQSReceiver.sendReceiveMessageCommandSafe(
+                client,
+                new ReceiveMessageCommand({
+                  QueueUrl: url,
+                  MaxNumberOfMessages: 1,
+                  WaitTimeSeconds: 5,
+                }),
+              );
+
+              if (!response.Messages || response.Messages.length === 0) {
+                DBOS.logger.debug(`No messages for ${url} - `);
+                continue;
+              }
+
+              const message = response.Messages[0];
+
+              // Start workflow
+              let wfid = sqsmethod.config?.getWorkflowKey ? sqsmethod.config.getWorkflowKey(message) : undefined;
+              if (!wfid) {
+                wfid = sqsclass.config?.getWorkflowKey ? sqsclass.config.getWorkflowKey(message) : undefined;
+              }
+              if (!wfid) {
+                wfid = sqsr.config?.getWorkflowKey ? sqsr.config.getWorkflowKey(message) : undefined;
+              }
+              if (!wfid) wfid = message.MessageId;
+              await DBOS.startWorkflow(methodReg.registeredFunction as (...args: unknown[]) => Promise<unknown>, {
+                workflowID: wfid,
+                queueName:
+                  sqsmethod.config?.workflowQueueName ??
+                  sqsclass.config?.workflowQueueName ??
+                  sqsr.config?.workflowQueueName,
+              })(message);
+
+              // Delete the message from the queue after starting workflow (which will take over retries)
+              await client.send(
+                new DeleteMessageCommand({
+                  QueueUrl: url,
+                  ReceiptHandle: message.ReceiptHandle,
+                }),
+              );
+            } catch (e) {
+              DBOS.logger.error(e);
+            }
+          }
+        })(sqs, url),
+      );
+    }
+  }
+
+  override logRegisteredEndpoints() {
+    DBOS.logger.info('SQS receiver endpoints:');
+
+    const eps = DBOS.getAssociatedInfo(this);
+
+    for (const e of eps) {
+      const { methodConfig, classConfig, methodReg } = e;
+      const sqsclass = classConfig as SQSRecvClassConfig;
+      const sqsmethod = methodConfig as SQSRecvMethodConfig;
+      if (sqsmethod) {
+        const url = sqsclass.config?.queueUrl ?? sqsmethod.config?.queueUrl ?? this.config?.queueUrl;
+        if (url) {
+          const cname = methodReg.className;
+          const mname = methodReg.name;
+          DBOS.logger.info(`    ${url} -> ${cname}.${mname}`);
+        }
+      }
+    }
+  }
+
+  // Decorator - class
+  configuration(config: SQSConfig) {
+    // eslint-disable-next-line @typescript-eslint/no-this-alias
+    const er = this;
+
+    function clsdec<T extends { new (...args: unknown[]): object }>(ctor: T) {
+      const erInfo = DBOS.associateClassWithInfo(er, ctor) as SQSRecvClassConfig;
+      erInfo.config = config;
+    }
+    return clsdec;
+  }
+
+  // Decorators - method
+  messageConsumer(config?: SQSConfig) {
+    // eslint-disable-next-line @typescript-eslint/no-this-alias
+    const er = this;
+    function mtddec<This, Args extends [Message], Return>(
+      target: object,
+      propertyKey: string,
+      inDescriptor: TypedPropertyDescriptor<(this: This, ...args: Args) => Promise<Return>>,
+    ) {
+      const { regInfo: receiverInfo } = DBOS.associateFunctionWithInfo(er, inDescriptor.value!, {
+        classOrInst: target,
+        name: propertyKey,
+      });
+
+      const mRegistration = receiverInfo as SQSRecvMethodConfig;
+      mRegistration.config = config;
+
+      return inDescriptor;
+    }
+    return mtddec;
+  }
+}
+
+export { SQSConfig, SQSReceiver };

package/jest.config.js

@@ -0,0 +1,8 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  testRegex: '((\\.|/)(test|spec))\\.ts?$',
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
+  modulePaths: ['./'],
+};

package/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "@dbos-inc/sqs-receive",
+  "version": "3.0.10-preview",
+  "description": "DBOS event receiver for AWS SQS queues",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dbos-inc/dbos-transact-ts",
+    "directory": "packages/sqs-receive"
+  },
+  "homepage": "https://docs.dbos.dev/",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "tsc --project tsconfig.json",
+    "test": "echo 'no tests'",
+    "testsqs": "npm run build && jest --detectOpenHandles"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.12",
+    "@types/supertest": "^6.0.2",
+    "jest": "^29.7.0",
+    "supertest": "^7.0.0",
+    "ts-jest": "^29.1.4",
+    "typescript": "^5.3.3"
+  },
+  "peerDependencies": {
+    "@dbos-inc/dbos-sdk": "*",
+    "@dbos-inc/aws-config": "*"
+  },
+  "dependencies": {
+    "@aws-sdk/client-sqs": "^3.596.0"
+  }
+}

package/sqs-test-dbos-config.yaml

@@ -0,0 +1,10 @@
+# To enable auto-completion and validation for this file in VSCode, install the RedHat YAML extension
+# https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml
+
+# yaml-language-server: $schema=https://raw.githubusercontent.com/dbos-inc/dbos-transact/main/dbos-config.schema.json
+
+database:
+  hostname: 'localhost'
+  port: 5432
+  username: 'postgres'
+  password: ${PGPASSWORD}

package/sqs.test.ts

@@ -0,0 +1,111 @@
+import { Message, SendMessageCommand, SendMessageCommandInput, SQSClient } from '@aws-sdk/client-sqs';
+import { SQSReceiver } from './index';
+import { DBOS } from '@dbos-inc/dbos-sdk';
+
+const sleepms = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+interface ValueObj {
+  val: number;
+}
+
+function createSQS() {
+  return new SQSClient({
+    region: process.env['AWS_REGION'] ?? '',
+    endpoint: process.env['AWS_ENDPOINT_URL_SQS'],
+    credentials: {
+      accessKeyId: process.env['AWS_ACCESS_KEY_ID'] ?? '',
+      secretAccessKey: process.env['AWS_SECRET_ACCESS_KEY'] ?? '',
+    },
+    //logger: console,
+  });
+}
+
+// Create a new type that omits the QueueUrl property
+type MessageWithoutQueueUrl = Omit<SendMessageCommandInput, 'QueueUrl'>;
+
+// Create a new type that allows QueueUrl to be added later
+type MessageWithOptionalQueueUrl = MessageWithoutQueueUrl & { QueueUrl?: string };
+
+async function sendMessageInternal(msg: MessageWithOptionalQueueUrl) {
+  try {
+    const smsg = { ...msg, QueueUrl: msg.QueueUrl || process.env['SQS_QUEUE_URL']! };
+    return await createSQS().send(new SendMessageCommand(smsg));
+  } catch (e) {
+    DBOS.logger.error(e);
+    throw e;
+  }
+}
+
+const sendMessageStep = DBOS.registerStep(sendMessageInternal, { name: 'Send SQS Message' });
+
+const sqsReceiver = new SQSReceiver({
+  client: createSQS,
+});
+
+class SQSRcv {
+  static msgRcvCount: number = 0;
+  static msgValueSum: number = 0;
+  @sqsReceiver.messageConsumer({ queueUrl: process.env['SQS_QUEUE_URL'] })
+  @DBOS.workflow()
+  static async recvMessage(msg: Message) {
+    const ms = msg.Body!;
+    const res = JSON.parse(ms) as ValueObj;
+    SQSRcv.msgRcvCount++;
+    SQSRcv.msgValueSum += res.val;
+    return Promise.resolve();
+  }
+}
+
+describe('sqs-tests', () => {
+  let sqsIsAvailable = true;
+
+  beforeAll(() => {
+    // Check if SES is available and update app config, skip the test if it's not
+    if (!process.env['AWS_REGION'] || !process.env['SQS_QUEUE_URL']) {
+      sqsIsAvailable = false;
+    } else {
+      // This would normally be a global or static or something
+      DBOS.setConfig({ name: 'dbossqstest' });
+    }
+  });
+
+  beforeEach(async () => {
+    if (sqsIsAvailable) {
+      DBOS.registerLifecycleCallback(sqsReceiver);
+      await DBOS.launch();
+    } else {
+      console.log(
+        'SQS Test is not configured.  To run, set AWS_REGION, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, and SQS_QUEUE_URL',
+      );
+    }
+  });
+
+  afterEach(async () => {
+    if (sqsIsAvailable) {
+      await DBOS.shutdown();
+    }
+  }, 10000);
+
+  // This tests receive also; which is already wired up
+  test('sqs-send', async () => {
+    if (!sqsIsAvailable) {
+      console.log('SQS unavailable, skipping SQS tests');
+      return;
+    }
+    const sv: ValueObj = {
+      val: 10,
+    };
+    const ser = await sendMessageStep({
+      MessageBody: JSON.stringify(sv),
+    });
+    expect(ser.MessageId).toBeDefined();
+
+    // Wait for receipt
+    for (let i = 0; i < 100; ++i) {
+      if (SQSRcv.msgRcvCount === 1) break;
+      await sleepms(100);
+    }
+    expect(SQSRcv.msgRcvCount).toBe(1);
+    expect(SQSRcv.msgValueSum).toBe(10);
+  });
+});

package/tsconfig.json

@@ -0,0 +1,9 @@
+/* Visit https://aka.ms/tsconfig to read more about this file */
+{
+  "extends": "../../tsconfig.shared.json",
+  "compilerOptions": {
+    "outDir": "./dist"
+  },
+  "include": [/* Specifies an array of filenames or patterns to include in the program. */ "."],
+  "exclude": ["dist"]
+}