@dbos-inc/aws-s3-workflows 3.0.29-preview

package/index.ts

@@ -0,0 +1,7 @@
+export {
+  FileRecord,
+  S3WorkflowCallbacks,
+  registerS3UploadWorkflow,
+  registerS3PresignedUploadWorkflow,
+  registerS3DeleteWorkflow,
+} from './src/s3_utils';

package/jest.config.js

@@ -0,0 +1,8 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  testRegex: '((\\.|/)(test|spec))\\.ts?$',
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
+  modulePaths: ['./'],
+};

package/knexfile.ts

@@ -0,0 +1,21 @@
+import { Knex } from 'knex';
+import { parseConfigFile } from '@dbos-inc/dbos-sdk';
+import { DBOSConfig } from '@dbos-inc/dbos-sdk';
+
+const [dbosConfig]: [DBOSConfig, unknown] = parseConfigFile();
+
+const config: Knex.Config = {
+  client: 'pg',
+  connection: {
+    host: dbosConfig.poolConfig!.host,
+    user: dbosConfig.poolConfig!.user,
+    password: dbosConfig.poolConfig!.password,
+    database: dbosConfig.poolConfig!.database,
+    ssl: dbosConfig.poolConfig!.ssl,
+  },
+  migrations: {
+    directory: './migrations',
+  },
+};
+
+export default config;

package/migrations/20240410.ts

@@ -0,0 +1,16 @@
+import { Knex } from 'knex';
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.createTable('user_files', (table) => {
+    table.uuid('file_id').primary();
+    table.uuid('user_id'); //.index().references("cusers.user_id");
+    table.string('file_status', 16);
+    table.string('file_type', 16);
+    table.bigint('file_time');
+    table.string('file_name', 128);
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTable('user_files');
+}

package/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@dbos-inc/aws-s3-workflows",
+  "version": "3.0.29-preview",
+  "description": "Component library - DBOS S3 steps and workflows",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dbos-inc/dbos-transact-ts",
+    "directory": "packages/aws-s3-workflows"
+  },
+  "homepage": "https://docs.dbos.dev/",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "tsc --project tsconfig.json",
+    "test": "echo 'no tests'",
+    "tests3": "npm run build && npx dbos rollback && npx dbos migrate && jest --detectOpenHandles"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.12",
+    "@types/supertest": "^6.0.2",
+    "axios": "^1.7.4",
+    "jest": "^29.7.0",
+    "knex": "^3.1.0",
+    "supertest": "^7.0.0",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.3.3"
+  },
+  "peerDependencies": {
+    "@dbos-inc/dbos-sdk": "*"
+  },
+  "dependencies": {
+    "@aws-sdk/s3-presigned-post": "^3.842.0",
+    "@aws-sdk/s3-request-presigner": "^3.842.0",
+    "uuid": "^9.0.1"
+  }
+}

package/src/s3_utils.test.ts

@@ -0,0 +1,350 @@
+import {
+  FileRecord,
+  S3WorkflowCallbacks,
+  registerS3UploadWorkflow,
+  registerS3PresignedUploadWorkflow,
+  registerS3DeleteWorkflow,
+} from './s3_utils';
+import { DBOS } from '@dbos-inc/dbos-sdk';
+
+import { S3Client, DeleteObjectCommand, GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3';
+import { createPresignedPost, PresignedPost } from '@aws-sdk/s3-presigned-post';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+
+import FormData from 'form-data';
+import axios, { AxiosResponse } from 'axios';
+import { Readable } from 'stream';
+import * as fs from 'fs';
+import { randomUUID } from 'node:crypto';
+
+enum FileStatus {
+  PENDING = 'Pending',
+  RECEIVED = 'Received',
+  ACTIVE = 'Active',
+}
+
+interface FileDetails {
+  user_id: string;
+  file_type: string;
+  file_name: string;
+  file_id?: string;
+  file_status?: string;
+  file_time?: number;
+}
+
+interface UserFile extends FileDetails, FileRecord {
+  user_id: string;
+  file_type: string;
+  file_name: string;
+  file_id: string;
+  file_status: string;
+  file_time: number;
+}
+
+class TestUserFileTable {
+  //////////
+  //// Database table
+  //////////
+
+  // Pick a file ID
+  @DBOS.step()
+  static chooseFileRecord(details: FileDetails): Promise<UserFile> {
+    const rec: UserFile = {
+      user_id: details.user_id,
+      file_status: FileStatus.PENDING,
+      file_type: details.file_type,
+      file_id: randomUUID(),
+      file_name: details.file_name,
+      file_time: new Date().getTime(),
+      key: '',
+    };
+    rec.key = TestUserFileTable.createS3Key(rec);
+    return Promise.resolve(rec);
+  }
+
+  static createS3Key(rec: UserFile) {
+    const key = `${rec.file_type}/${rec.user_id}/${rec.file_id}/${rec.file_time}`;
+    return key;
+  }
+
+  static toFileDetails(rec: UserFile) {
+    return {
+      user_id: rec.user_id,
+      file_type: rec.file_type,
+      file_name: rec.file_name,
+      file_id: rec.file_id,
+      file_status: rec.file_status,
+      file_time: rec.file_time,
+    };
+  }
+
+  // File table DML operations
+  // Whole record is known
+  @DBOS.transaction()
+  static async insertFileRecord(rec: UserFile) {
+    await DBOS.knexClient<FileDetails>('user_files').insert(TestUserFileTable.toFileDetails(rec));
+  }
+  @DBOS.transaction()
+  static async updateFileRecord(rec: UserFile) {
+    await DBOS.knexClient<FileDetails>('user_files')
+      .update(TestUserFileTable.toFileDetails(rec))
+      .where({ file_id: rec.file_id });
+  }
+  // Delete when part of record is known
+  @DBOS.transaction()
+  static async deleteFileRecordById(file_id: string) {
+    await DBOS.knexClient<FileDetails>('user_files').delete().where({ file_id });
+  }
+
+  // Queries
+  @DBOS.transaction({ readOnly: true })
+  static async lookUpByFields(fields: FileDetails) {
+    const rv = await DBOS.knexClient<FileDetails>('user_files')
+      .select()
+      .where({ ...fields, file_status: FileStatus.ACTIVE })
+      .orderBy('file_time', 'desc')
+      .first();
+    return rv ? [rv] : [];
+  }
+  @DBOS.transaction({ readOnly: true })
+  static async lookUpByName(user_id: string, file_type: string, file_name: string) {
+    const rv = await DBOS.knexClient<FileDetails>('user_files')
+      .select()
+      .where({ user_id, file_type, file_name, file_status: FileStatus.ACTIVE })
+      .orderBy('file_time', 'desc')
+      .first();
+    return rv ? [rv] : [];
+  }
+  @DBOS.transaction({ readOnly: true })
+  static async lookUpByType(user_id: string, file_type: string) {
+    const rv = await DBOS.knexClient<FileDetails>('user_files')
+      .select()
+      .where({ user_id, file_type, file_status: FileStatus.ACTIVE });
+    return rv;
+  }
+  @DBOS.transaction({ readOnly: true })
+  static async lookUpByUser(user_id: string) {
+    const rv = await DBOS.knexClient<FileDetails>('user_files')
+      .select()
+      .where({ user_id, file_status: FileStatus.ACTIVE });
+    return rv;
+  }
+}
+
+const s3bucket = process.env['S3_BUCKET'];
+const s3region = process.env['AWS_REGION'];
+const s3accessKey = process.env['AWS_ACCESS_KEY_ID'];
+const s3accessSecret = process.env['AWS_SECRET_ACCESS_KEY'];
+
+let s3client: S3Client | undefined = undefined;
+
+interface Opts {
+  contentType?: string;
+}
+
+const s3callback: S3WorkflowCallbacks<UserFile, Opts> = {
+  // Database operations (these should be transactions)
+  newActiveFile: async (rec: UserFile) => {
+    rec.file_status = FileStatus.ACTIVE;
+    return await TestUserFileTable.insertFileRecord(rec);
+  },
+  newPendingFile: async (rec: UserFile) => {
+    rec.file_status = FileStatus.PENDING;
+    return await TestUserFileTable.insertFileRecord(rec);
+  },
+  fileActivated: async (rec: UserFile) => {
+    rec.file_status = FileStatus.ACTIVE;
+    return await TestUserFileTable.updateFileRecord(rec);
+  },
+  fileDeleted: async (rec: UserFile) => {
+    return await TestUserFileTable.deleteFileRecordById(rec.file_id);
+  },
+
+  // S3 interaction options, these will be run as steps
+  putS3Contents: async (rec: UserFile, content: string, options?: Opts) => {
+    return await s3client?.send(
+      new PutObjectCommand({
+        Bucket: s3bucket,
+        Key: rec.key,
+        ContentType: options?.contentType ?? 'text/plain',
+        Body: content,
+      }),
+    );
+  },
+  createPresignedPost: async (rec: UserFile, timeout?: number, opts?: Opts) => {
+    const postPresigned = await createPresignedPost(s3client!, {
+      Conditions: [
+        ['content-length-range', 1, 10000000], // 10MB
+      ],
+      Bucket: s3bucket!,
+      Key: rec.key,
+      Expires: timeout || 60,
+      Fields: {
+        'Content-Type': opts?.contentType || '*',
+      },
+    });
+    return { url: postPresigned.url, fields: postPresigned.fields };
+  },
+  validateS3Upload: undefined,
+  deleteS3Object: async (rec: UserFile) => {
+    return await s3client?.send(
+      new DeleteObjectCommand({
+        Bucket: s3bucket,
+        Key: rec.key,
+      }),
+    );
+  },
+};
+
+export const uploadWF = registerS3UploadWorkflow({ className: 'UserFile', name: 'uploadWF' }, s3callback);
+export const uploadPWF = registerS3PresignedUploadWorkflow({ className: 'UserFile', name: 'uploadPWF' }, s3callback);
+export const deleteWF = registerS3DeleteWorkflow({ className: 'UserFile', name: 'deleteWF' }, s3callback);
+
+describe('ses-tests', () => {
+  let s3IsAvailable = true;
+
+  beforeAll(async () => {
+    // Check if S3 is available and update app config, skip the test if it's not
+    if (!s3region || !s3bucket || !s3accessKey || !s3accessSecret) {
+      s3IsAvailable = false;
+      console.log(
+        'S3 Test is not configured.  To run, set AWS_REGION, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, and S3_BUCKET',
+      );
+    } else {
+      s3client = new S3Client({
+        region: s3region,
+        credentials: {
+          accessKeyId: s3accessKey,
+          secretAccessKey: s3accessSecret,
+        },
+      });
+
+      await DBOS.launch();
+    }
+  });
+
+  afterAll(async () => {
+    if (s3IsAvailable) {
+      await DBOS.shutdown();
+    }
+  }, 10000);
+
+  test('s3-simple-wfs', async () => {
+    if (!s3IsAvailable) {
+      console.log('S3 unavailable, skipping S3 tests');
+      return;
+    }
+
+    const userid = randomUUID();
+
+    // The simple workflows that will be performed are to:
+    //   Put file contents into DBOS (w/ table index)
+    const myFile: FileDetails = { user_id: userid, file_type: 'text', file_name: 'mytextfile.txt' };
+    const myFileRec = await TestUserFileTable.chooseFileRecord(myFile);
+    await uploadWF(myFileRec, 'This is my file');
+
+    // Get the file contents out of DBOS (using the table index)
+    const mytxt = await getS3KeyContents(myFileRec.key);
+    expect(mytxt).toBe('This is my file');
+
+    // Delete the file contents out of DBOS (using the table index)
+    const dfhandle = await deleteWF(myFileRec);
+    expect(dfhandle).toBeDefined();
+  }, 10000);
+
+  test('s3-complex-wfs', async () => {
+    if (!s3IsAvailable) {
+      console.log('S3 unavailable, skipping S3 tests');
+      return;
+    }
+
+    // The complex workflows that will be performed are to:
+    //  Put the file contents into DBOS with a presigned post
+    const userid = randomUUID();
+
+    // The simple workflows that will be performed are to:
+    //   Put file contents into DBOS (w/ table index)
+    const myFile: FileDetails = { user_id: userid, file_type: 'text', file_name: 'mytextfile.txt' };
+    const myFileRec = await TestUserFileTable.chooseFileRecord(myFile);
+    const wfhandle = await DBOS.startWorkflow(uploadPWF)(myFileRec, 60, { contentType: 'text/plain' });
+    //    Get the presigned post
+    const ppost = await DBOS.getEvent<PresignedPost>(wfhandle.workflowID, 'uploadkey');
+    //    Upload to the URL
+    try {
+      const res = await uploadToS3(ppost!, './src/s3_utils.test.ts');
+      expect(res.status.toString()[0]).toBe('2');
+    } catch (e) {
+      // You do NOT want to accidentally serialize an AxiosError - they don't!
+      console.log('Caught something awful!', e);
+      expect(e).toBeUndefined();
+    }
+    //    Notify WF
+    await DBOS.send<boolean>(wfhandle.workflowID, true, 'uploadfinish');
+
+    //    Wait for WF complete
+    const _myFileRecord = await wfhandle.getResult();
+
+    // Get the file out of DBOS (using a signed URL)
+    const myurl = await getS3KeyUrl(myFileRec.key, 60);
+    expect(myurl).not.toBeNull();
+    // Get the file contents out of S3
+    await downloadFromS3(myurl, './deleteme.xxx');
+    expect(fs.existsSync('./deleteme.xxx')).toBeTruthy();
+    fs.rmSync('./deleteme.xxx');
+
+    // Delete the file contents out of DBOS (using the table index)
+    const dfhandle = await deleteWF(myFileRec);
+    expect(dfhandle).toBeDefined();
+  }, 10000);
+
+  async function getS3KeyContents(key: string) {
+    return (
+      await s3client!.send(
+        new GetObjectCommand({
+          Bucket: s3bucket!,
+          Key: key,
+        }),
+      )
+    ).Body?.transformToString();
+  }
+
+  async function getS3KeyUrl(key: string, expirationSecs: number) {
+    const getObjectCommand = new GetObjectCommand({
+      Bucket: s3bucket!,
+      Key: key,
+    });
+
+    const presignedUrl = await getSignedUrl(s3client!, getObjectCommand, { expiresIn: expirationSecs });
+    return presignedUrl;
+  }
+
+  async function uploadToS3(presignedPostData: PresignedPost, filePath: string) {
+    const formData = new FormData();
+
+    // Append all the fields from the presigned post data
+    Object.keys(presignedPostData.fields).forEach((key) => {
+      formData.append(key, presignedPostData.fields[key]);
+    });
+
+    // Append the file you want to upload
+    const fileStream = fs.createReadStream(filePath);
+    formData.append('file', fileStream);
+
+    return await axios.post(presignedPostData.url, formData);
+  }
+
+  async function downloadFromS3(presignedGetUrl: string, outputPath: string) {
+    const response: AxiosResponse<Readable> = await axios.get(presignedGetUrl, {
+      responseType: 'stream', // Important to handle large files
+    });
+
+    // Use a write stream to save the file to the desired path
+    const writer = fs.createWriteStream(outputPath);
+    response.data.pipe(writer);
+
+    return new Promise<void>((resolve, reject) => {
+      writer.on('finish', resolve);
+      writer.on('error', reject);
+    });
+  }
+});

package/src/s3_utils.ts

@@ -0,0 +1,157 @@
+import { type PresignedPost } from '@aws-sdk/s3-presigned-post';
+
+import { DBOS, WorkflowConfig } from '@dbos-inc/dbos-sdk';
+
+export interface FileRecord {
+  key: string;
+}
+
+export interface S3WorkflowCallbacks<R extends FileRecord, Options = unknown> {
+  // Database operations (these should be transactions)
+  /** Called back when a new, active file is created; should write the file to the database as active */
+  newActiveFile: (rec: R) => Promise<unknown>;
+  /** Called back when a file might get uploaded; this function may write the file to the database as 'pending' */
+  newPendingFile: (rec: R) => Promise<unknown>;
+  /** Called back when a pending file becomes active; should write the file to the database as active */
+  fileActivated: (rec: R) => Promise<unknown>;
+  /** Called back when a file is in the process of getting deleted; should remove the file from the database */
+  fileDeleted: (rec: R) => Promise<unknown>;
+
+  // S3 interaction options
+  /** Should execute the S3 operation to write contents to rec.key; this will be run as a step */
+  putS3Contents: (rec: R, content: string, options?: Options) => Promise<unknown>;
+  /** Should execute the S3 operation to create a presigned post for external upload, this will be run as a step */
+  createPresignedPost: (rec: R, timeout?: number, options?: Options) => Promise<PresignedPost>;
+  /** Optional validation to check if a client S3 upload is valid, before activating in the datbase.  Will run as a step */
+  validateS3Upload?: (rec: R) => Promise<void>;
+  /** Should execute the S3 operation to delete rec.key; this will be run as a step */
+  deleteS3Object: (rec: R) => Promise<unknown>;
+}
+
+/**
+ * Create a workflow function for deleting S3 objects and removing the DB entry
+ * @param options - Registration options for the workflow
+ * @param callbacks - S3 operation implementation and database recordkeeping transactions
+ */
+export function registerS3DeleteWorkflow<R extends FileRecord, Options = unknown>(
+  options: {
+    name?: string;
+    ctorOrProto?: object;
+    className?: string;
+    config?: WorkflowConfig;
+  },
+  callbacks: S3WorkflowCallbacks<R, Options>,
+) {
+  return DBOS.registerWorkflow(async (fileDetails: R) => {
+    await callbacks.fileDeleted(fileDetails);
+    return await DBOS.runStep(
+      async () => {
+        return callbacks.deleteS3Object(fileDetails);
+      },
+      { name: 'deleteS3Object' },
+    );
+  }, options);
+}
+
+/**
+ * Create a workflow function for uploading S3 contents from DBOS
+ * @param options - Registration options for the workflow
+ * @param callbacks - S3 operation implementation and database recordkeeping transactions
+ */
+export function registerS3UploadWorkflow<R extends FileRecord, Options = unknown>(
+  options: {
+    name?: string;
+    ctorOrProto?: object;
+    className?: string;
+    config?: WorkflowConfig;
+  },
+  callbacks: S3WorkflowCallbacks<R, Options>,
+) {
+  return DBOS.registerWorkflow(async (fileDetails: R, content: string, objOptions?: Options) => {
+    try {
+      await DBOS.runStep(
+        async () => {
+          await callbacks.putS3Contents(fileDetails, content, objOptions);
+        },
+        { name: 'putS3Contents' },
+      );
+    } catch (e) {
+      try {
+        await DBOS.runStep(
+          async () => {
+            return callbacks.deleteS3Object(fileDetails);
+          },
+          { name: 'deleteS3Object' },
+        );
+      } catch (e2) {
+        DBOS.logger.debug(e2);
+      }
+      throw e;
+    }
+
+    await callbacks.newActiveFile(fileDetails);
+    return fileDetails;
+  }, options);
+}
+
+/**
+ * Create a workflow function for uploading S3 contents externally, via a presigned URL
+ * @param options - Registration options for the workflow
+ * @param callbacks - S3 operation implementation and database recordkeeping transactions
+ */
+export function registerS3PresignedUploadWorkflow<R extends FileRecord, Options = unknown>(
+  options: {
+    name?: string;
+    ctorOrProto?: object;
+    className?: string;
+    config?: WorkflowConfig;
+  },
+  callbacks: S3WorkflowCallbacks<R, Options>,
+) {
+  return DBOS.registerWorkflow(async (fileDetails: R, timeoutSeconds: number, objOptions?: Options) => {
+    await callbacks.newPendingFile(fileDetails);
+
+    const upkey = await DBOS.runStep(
+      async () => {
+        return await callbacks.createPresignedPost(fileDetails, timeoutSeconds, objOptions);
+      },
+      { name: 'createPresignedPost' },
+    );
+    await DBOS.setEvent<PresignedPost>('uploadkey', upkey);
+
+    try {
+      const res = await DBOS.recv<boolean>('uploadfinish', timeoutSeconds + 60);
+
+      if (!res) {
+        throw new Error('S3 operation timed out or canceled');
+      }
+
+      // Validate the file, if we have code for that
+      if (callbacks.validateS3Upload) {
+        await DBOS.runStep(
+          async () => {
+            await callbacks.validateS3Upload!(fileDetails);
+          },
+          { name: 'validateFileUpload' },
+        );
+      }
+
+      await callbacks?.fileActivated(fileDetails);
+    } catch (e) {
+      try {
+        await callbacks.fileDeleted(fileDetails);
+        await DBOS.runStep(
+          async () => {
+            await callbacks.deleteS3Object(fileDetails);
+          },
+          { name: 'deleteS3Object' },
+        );
+      } catch (e2) {
+        DBOS.logger.debug(e2);
+      }
+      throw e;
+    }
+
+    return fileDetails;
+  }, options);
+}

package/tsconfig.json

@@ -0,0 +1,9 @@
+/* Visit https://aka.ms/tsconfig to read more about this file */
+{
+  "extends": "../../tsconfig.shared.json",
+  "compilerOptions": {
+    "outDir": "./dist"
+  },
+  "include": [/* Specifies an array of filenames or patterns to include in the program. */ "."],
+  "exclude": ["dist"]
+}