@dbos-inc/aws-s3-workflows 3.0.29-preview

package/README.md

@@ -0,0 +1,241 @@
+# DBOS AWS Simple Storage Service (S3) Workflows
+
+This is a [DBOS](https://docs.dbos.dev/) workflow library for working with [Amazon Web Services Simple Storage Service (S3)](https://aws.amazon.com/s3/). The primary feature of this library is workflows that keep a database table in sync with the S3 contents, regardless of failures. These workflows can be used as-is, or as an example for your own code.
+
+## Keeping S3 And Database In Sync
+
+AWS applications often store large / unstructured data as S3 objects. S3 is, as the name implies, simple storage, and it doesn't comprehensively track file attributes, permissions, fine-grained ownership, dependencies, etc., and listing the objects can be inefficient and expensive.
+
+Keeping an indexed set of file metadata records, including referential links to their owners, is a "database problem". And, while keeping the database in sync with the contents of S3 sounds like it may be tricky, [DBOS Workflows](https://docs.dbos.dev/typescript/tutorials/workflow-tutorial) provide the perfect tool for accomplishing this, even in the face of client or server failures.
+
+When keeping a database table in sync with S3 contents, several failures have to be anticipated:
+
+1. The S3 object could get added without a database record getting added, perhaps due to a machine failure during processing. This would create an orphaned S3 object.
+2. Likewise, the a database table entry could get removed, without the S3 object getting cleaned up.
+3. An outside process responsible for putting an object into S3 could fail, leaving the S3 object state in an unknown condition.
+
+This library solves all three of these problems via lightweight, durable DBOS workflows. DBOS workflows ensure that execution resumes, finishing the job of adding or removing the database table entries, or performing cleanup if an outside process does not indicate success within a timeout period.
+
+## Getting Started
+
+To use AWS S3:
+
+- Register with AWS, create an S3 bucket, and create access credentials. (See [Getting started with Amazon S3](https://aws.amazon.com/s3/getting-started/) in AWS documentation.)
+- If browser-based clients will read from S3 directly, set up S3 CORS. (See [Using cross-origin resource sharing (CORS)](https://docs.aws.amazon.com/AmazonS3/latest/userguide/cors.html) in AWS documentation.)
+
+## Using AWS S3 From a DBOS Application
+
+First, ensure that the AWS S3 SDK and DBOS S3 workflows are installed into the application:
+
+```
+npm install --save @dbos-inc/aws-s3-workflows aws-sdk/s3-presigned-post @aws-sdk/s3-request-presigner
+```
+
+Second, ensure that workflow registration functions are imported:
+
+```typescript
+import {
+  FileRecord,
+  S3WorkflowCallbacks,
+  registerS3UploadWorkflow,
+  registerS3PresignedUploadWorkflow,
+  registerS3DeleteWorkflow,
+} from '@dbos-inc/aws-s3-workflows';
+```
+
+Third, create your database transactions, and which will be used in the callback for the S3 workflows. The transactions below are just examples, you can use your own table schema, data source, and migrations.
+
+```typescript
+// See the test for schema, migrations, file lookup SQL, etc...
+class TestUserFileTable {
+  // File table DML operations
+  @DBOS.transaction()
+  static async insertFileRecord(rec: UserFile) {
+    await DBOS.knexClient<FileDetails>('user_files').insert(TestUserFileTable.toFileDetails(rec));
+  }
+  @DBOS.transaction()
+  static async updateFileRecord(rec: UserFile) {
+    await DBOS.knexClient<FileDetails>('user_files')
+      .update(TestUserFileTable.toFileDetails(rec))
+      .where({ file_id: rec.file_id });
+  }
+  @DBOS.transaction()
+  static async deleteFileRecordById(file_id: string) {
+    await DBOS.knexClient<FileDetails>('user_files').delete().where({ file_id });
+  }
+  // Lookups, etc. ...
+}
+```
+
+Fourth, create an callback for the workflows to use. The callback will specify the transactions and exact S3 calls to make, as these can be customized based on the method used for obtaining an S3 client.
+
+```typescript
+const s3callback: S3WorkflowCallbacks<UserFile, Opts> = {
+  // Database operations (these should be transactions)
+  newActiveFile: async (rec: UserFile) => {
+    rec.file_status = FileStatus.ACTIVE;
+    return await TestUserFileTable.insertFileRecord(rec);
+  },
+  newPendingFile: async (rec: UserFile) => {
+    rec.file_status = FileStatus.PENDING;
+    return await TestUserFileTable.insertFileRecord(rec);
+  },
+  fileActivated: async (rec: UserFile) => {
+    rec.file_status = FileStatus.ACTIVE;
+    return await TestUserFileTable.updateFileRecord(rec);
+  },
+  fileDeleted: async (rec: UserFile) => {
+    return await TestUserFileTable.deleteFileRecordById(rec.file_id);
+  },
+
+  // S3 interaction options, these will be run as steps
+  putS3Contents: async (rec: UserFile, content: string, options?: Opts) => {
+    return await s3client?.send(
+      new PutObjectCommand({
+        Bucket: s3bucket,
+        Key: rec.key,
+        ContentType: options?.contentType ?? 'text/plain',
+        Body: content,
+      }),
+    );
+  },
+  createPresignedPost: async (rec: UserFile, timeout?: number, opts?: Opts) => {
+    const postPresigned = await createPresignedPost(s3client!, {
+      Conditions: [
+        ['content-length-range', 1, 10000000], // 10MB
+      ],
+      Bucket: s3bucket!,
+      Key: rec.key,
+      Expires: timeout || 60,
+      Fields: {
+        'Content-Type': opts?.contentType || '*',
+      },
+    });
+    return { url: postPresigned.url, fields: postPresigned.fields };
+  },
+  deleteS3Object: async (rec: UserFile) => {
+    return await s3client?.send(
+      new DeleteObjectCommand({
+        Bucket: s3bucket,
+        Key: rec.key,
+      }),
+    );
+  },
+};
+```
+
+Finally, register and use the S3 workflows:
+
+```typescript
+export const uploadWF = registerS3UploadWorkflow({ className: 'UserFile', name: 'uploadWF' }, s3callback);
+...
+await uploadWF(myFileRec, 'This is my file');
+```
+
+### Workflow to Upload a String to S3
+
+The `registerS3UploadWorkflow` function registers a workflow that stores a string to an S3 key, and runs the callback function to update the database. If anything goes wrong during the workflow, S3 will be cleaned up and the database will be unchanged by the workflow.
+
+```typescript
+export const uploadWF = registerS3UploadWorkflow({ className: 'UserFile', name: 'uploadWF' }, s3callback);
+...
+await uploadWF(myFileRec, 'These contents should be saved to S3');
+```
+
+This workflow performs the following actions:
+
+- Puts the string in S3
+- If there is difficulty with S3, ensures that no entry is left there and throws an error
+- Invokes the transaction for a new active file record
+
+### Workflow to Delete a File
+
+The `registerS3DeleteWorkflow` function creates a workflow that removes a file from both S3 and the database.
+
+```typescript
+export const deleteWF = registerS3DeleteWorkflow({ className: 'UserFile', name: 'deleteWF' }, s3callback);
+
+await deleteWF(fileDBRecord);
+```
+
+This workflow performs the following actions:
+
+- Invokes the callback transaction for a deleting the file record
+- Removes the key from S3
+
+### Workflow to Allow Client File Upload via Presigned URLs
+
+It is often not convenient to send or retrieve large S3 object contents through DBOS; the client should exchange data directly with S3. S3 accomodates this use case very well, using a feature called ["presigned URLs"](https://docs.aws.amazon.com/AmazonS3/latest/userguide/PresignedUrlUploadObject.html).
+
+In these cases, the client can place a request to DBOS that produces a presigned POST URL, which the client can use for a limited time and purpose for S3 access. The DBOS workflow takes care of making sure that table entries are cleaned up if the client does not finish writing files.
+
+The workflow interaction generally proceeds as follows:
+
+- The client makes some request to a DBOS API handler.
+- The handler decides that the client will be uploading a file, and starts the upload workflow.
+- The upload workflow sends the handler a presigned post, which the handler returns to the client. The workflow continues in the background, waiting to hear that the client upload is complete.
+- The client (ideally) uploads the file using the presigned post and notifies the application; if not, the workflow times out and cleans up. (The workflow timeout should be set to occur after the presigned post expires.)
+- If successful, the database is updated to reflect the new file, otherwise S3 is cleaned of any partial work.
+
+The workflow can be initiated in the following way:
+
+```typescript
+export const uploadPWF = registerS3PresignedUploadWorkflow({ className: 'UserFile', name: 'uploadPWF' }, s3callback);
+
+// Start the workflow
+const wfHandle = await DBOS.startWorkflow(uploadPWF)(
+  myFileRec,
+  60, // Expiration (seconds)
+  { contentType: 'text/plain' },
+);
+
+// Get the presigned post (to pass to client)
+const ppost = await DBOS.getEvent<PresignedPost>(wfHandle.workflowID, 'uploadkey');
+```
+
+The client will then use the presigned post to upload data to S3 directly:
+
+```typescript
+// Upload to the URL from the presigned post
+const res = await uploadToS3(ppost!, '/path/to/file');
+```
+
+Upon a completion call from the client, the workflow should be notified so that it proceeds:
+
+```typescript
+// Look up wfHandle by the workflow ID
+const wfHandle = DBOS.retrieveWorkflow(wfid);
+
+// Notify workflow - truish means success, any falsy value indicates failure / cancel
+await DBOS.send<boolean>(wfHandle.workflowID(), true, 'uploadfinish');
+
+// Optionally, await completion of the workflow; this ensures that the database record is written,
+//  or will throw an error if anything went wrong in the workflow
+await wfHandle.getResult();
+```
+
+## Notes
+
+_Do not reuse S3 keys._ Assigning unique identifiers to files is a much better idea, if a "name" is to be reused, it can be reused in the lookup database.
+
+Reasons why S3 keys should not be reused:
+
+- S3 caches the key contents. Even a response of "this key doesn't exist" can be cached. If you reuse keys, you may get a stale value.
+- Workflow operations against an old use of a key may still be in process... for example a delete workflow may still be attempting to delete the old object at the same time a new file is being placed under the same key.
+
+## Simple Testing
+
+The `s3_utils.test.ts` file included in the source repository can be used to upload and download files to/from S3 using various approaches. Before running, set the following environment variables:
+
+- `S3_BUCKET`: The S3 bucket for setting / retrieving test objects
+- `AWS_REGION`: AWS region to use
+- `AWS_ACCESS_KEY_ID`: The access key with permission to use the S3 service
+- `AWS_SECRET_ACCESS_KEY`: The secret access key corresponding to `AWS_ACCESS_KEY_ID`
+
+The test illustrates S3 setup, use of the workflows, and some S3 helper functions.
+
+## Next Steps
+
+- To start a DBOS app from a template, visit our [quickstart](https://docs.dbos.dev/quickstart).
+- For DBOS programming tutorials, check out our [programming guide](https://docs.dbos.dev/typescript/programming-guide).
+- To learn more about DBOS, take a look at [our documentation](https://docs.dbos.dev/) or our [source code](https://github.com/dbos-inc/dbos-transact-ts).

package/dbos-config.yaml

@@ -0,0 +1,20 @@
+# To enable auto-completion and validation for this file in VSCode, install the RedHat YAML extension
+# https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml
+
+# yaml-language-server: $schema=https://raw.githubusercontent.com/dbos-inc/dbos-transact/main/dbos-config.schema.json
+
+# NOTE this is all stuff for the s3 test.
+#   While some is a useful guide, not all of it needs to be the same in a real app.
+
+database:
+  hostname: 'localhost'
+  port: 5432
+  username: 'postgres'
+  password: ${PGPASSWORD}
+  app_db_name: 'dbostest'
+  connectionTimeoutMillis: 3000
+  app_db_client: 'knex'
+  migrate: ['npx knex migrate:latest']
+  rollback: ['npx knex migrate:rollback']
+runtimeConfig:
+  port: 3300 # Optional, defaults to 3000

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export { FileRecord, S3WorkflowCallbacks, registerS3UploadWorkflow, registerS3PresignedUploadWorkflow, registerS3DeleteWorkflow, } from './src/s3_utils';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,EACV,mBAAmB,EACnB,wBAAwB,EACxB,iCAAiC,EACjC,wBAAwB,GACzB,MAAM,gBAAgB,CAAC"}

package/dist/index.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerS3DeleteWorkflow = exports.registerS3PresignedUploadWorkflow = exports.registerS3UploadWorkflow = void 0;
+var s3_utils_1 = require("./src/s3_utils");
+Object.defineProperty(exports, "registerS3UploadWorkflow", { enumerable: true, get: function () { return s3_utils_1.registerS3UploadWorkflow; } });
+Object.defineProperty(exports, "registerS3PresignedUploadWorkflow", { enumerable: true, get: function () { return s3_utils_1.registerS3PresignedUploadWorkflow; } });
+Object.defineProperty(exports, "registerS3DeleteWorkflow", { enumerable: true, get: function () { return s3_utils_1.registerS3DeleteWorkflow; } });
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":";;;AAAA,2CAMwB;AAHtB,oHAAA,wBAAwB,OAAA;AACxB,6HAAA,iCAAiC,OAAA;AACjC,oHAAA,wBAAwB,OAAA"}

package/dist/knexfile.d.ts

@@ -0,0 +1,4 @@
+import { Knex } from 'knex';
+declare const config: Knex.Config;
+export default config;
+//# sourceMappingURL=knexfile.d.ts.map

package/dist/knexfile.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"knexfile.d.ts","sourceRoot":"","sources":["../knexfile.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAM5B,QAAA,MAAM,MAAM,EAAE,IAAI,CAAC,MAYlB,CAAC;AAEF,eAAe,MAAM,CAAC"}

package/dist/knexfile.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const dbos_sdk_1 = require("@dbos-inc/dbos-sdk");
+const [dbosConfig] = (0, dbos_sdk_1.parseConfigFile)();
+const config = {
+    client: 'pg',
+    connection: {
+        host: dbosConfig.poolConfig.host,
+        user: dbosConfig.poolConfig.user,
+        password: dbosConfig.poolConfig.password,
+        database: dbosConfig.poolConfig.database,
+        ssl: dbosConfig.poolConfig.ssl,
+    },
+    migrations: {
+        directory: './migrations',
+    },
+};
+exports.default = config;
+//# sourceMappingURL=knexfile.js.map

package/dist/knexfile.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"knexfile.js","sourceRoot":"","sources":["../knexfile.ts"],"names":[],"mappings":";;AACA,iDAAqD;AAGrD,MAAM,CAAC,UAAU,CAAC,GAA0B,IAAA,0BAAe,GAAE,CAAC;AAE9D,MAAM,MAAM,GAAgB;IAC1B,MAAM,EAAE,IAAI;IACZ,UAAU,EAAE;QACV,IAAI,EAAE,UAAU,CAAC,UAAW,CAAC,IAAI;QACjC,IAAI,EAAE,UAAU,CAAC,UAAW,CAAC,IAAI;QACjC,QAAQ,EAAE,UAAU,CAAC,UAAW,CAAC,QAAQ;QACzC,QAAQ,EAAE,UAAU,CAAC,UAAW,CAAC,QAAQ;QACzC,GAAG,EAAE,UAAU,CAAC,UAAW,CAAC,GAAG;KAChC;IACD,UAAU,EAAE;QACV,SAAS,EAAE,cAAc;KAC1B;CACF,CAAC;AAEF,kBAAe,MAAM,CAAC"}

package/dist/migrations/20240410.d.ts

@@ -0,0 +1,4 @@
+import { Knex } from 'knex';
+export declare function up(knex: Knex): Promise<void>;
+export declare function down(knex: Knex): Promise<void>;
+//# sourceMappingURL=20240410.d.ts.map

package/dist/migrations/20240410.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"20240410.d.ts","sourceRoot":"","sources":["../../migrations/20240410.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,wBAAsB,EAAE,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CASlD;AAED,wBAAsB,IAAI,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAEpD"}

package/dist/migrations/20240410.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.down = exports.up = void 0;
+async function up(knex) {
+    await knex.schema.createTable('user_files', (table) => {
+        table.uuid('file_id').primary();
+        table.uuid('user_id'); //.index().references("cusers.user_id");
+        table.string('file_status', 16);
+        table.string('file_type', 16);
+        table.bigint('file_time');
+        table.string('file_name', 128);
+    });
+}
+exports.up = up;
+async function down(knex) {
+    await knex.schema.dropTable('user_files');
+}
+exports.down = down;
+//# sourceMappingURL=20240410.js.map

package/dist/migrations/20240410.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"20240410.js","sourceRoot":"","sources":["../../migrations/20240410.ts"],"names":[],"mappings":";;;AAEO,KAAK,UAAU,EAAE,CAAC,IAAU;IACjC,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,YAAY,EAAE,CAAC,KAAK,EAAE,EAAE;QACpD,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,wCAAwC;QAC/D,KAAK,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;QAChC,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QAC9B,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAC1B,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;AACL,CAAC;AATD,gBASC;AAEM,KAAK,UAAU,IAAI,CAAC,IAAU;IACnC,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;AAC5C,CAAC;AAFD,oBAEC"}

package/dist/src/s3_utils.d.ts

@@ -0,0 +1,57 @@
+import { type PresignedPost } from '@aws-sdk/s3-presigned-post';
+import { WorkflowConfig } from '@dbos-inc/dbos-sdk';
+export interface FileRecord {
+    key: string;
+}
+export interface S3WorkflowCallbacks<R extends FileRecord, Options = unknown> {
+    /** Called back when a new, active file is created; should write the file to the database as active */
+    newActiveFile: (rec: R) => Promise<unknown>;
+    /** Called back when a file might get uploaded; this function may write the file to the database as 'pending' */
+    newPendingFile: (rec: R) => Promise<unknown>;
+    /** Called back when a pending file becomes active; should write the file to the database as active */
+    fileActivated: (rec: R) => Promise<unknown>;
+    /** Called back when a file is in the process of getting deleted; should remove the file from the database */
+    fileDeleted: (rec: R) => Promise<unknown>;
+    /** Should execute the S3 operation to write contents to rec.key; this will be run as a step */
+    putS3Contents: (rec: R, content: string, options?: Options) => Promise<unknown>;
+    /** Should execute the S3 operation to create a presigned post for external upload, this will be run as a step */
+    createPresignedPost: (rec: R, timeout?: number, options?: Options) => Promise<PresignedPost>;
+    /** Optional validation to check if a client S3 upload is valid, before activating in the datbase.  Will run as a step */
+    validateS3Upload?: (rec: R) => Promise<void>;
+    /** Should execute the S3 operation to delete rec.key; this will be run as a step */
+    deleteS3Object: (rec: R) => Promise<unknown>;
+}
+/**
+ * Create a workflow function for deleting S3 objects and removing the DB entry
+ * @param options - Registration options for the workflow
+ * @param callbacks - S3 operation implementation and database recordkeeping transactions
+ */
+export declare function registerS3DeleteWorkflow<R extends FileRecord, Options = unknown>(options: {
+    name?: string;
+    ctorOrProto?: object;
+    className?: string;
+    config?: WorkflowConfig;
+}, callbacks: S3WorkflowCallbacks<R, Options>): (this: unknown, fileDetails: R) => Promise<unknown>;
+/**
+ * Create a workflow function for uploading S3 contents from DBOS
+ * @param options - Registration options for the workflow
+ * @param callbacks - S3 operation implementation and database recordkeeping transactions
+ */
+export declare function registerS3UploadWorkflow<R extends FileRecord, Options = unknown>(options: {
+    name?: string;
+    ctorOrProto?: object;
+    className?: string;
+    config?: WorkflowConfig;
+}, callbacks: S3WorkflowCallbacks<R, Options>): (this: unknown, fileDetails: R, content: string, objOptions?: Options | undefined) => Promise<R>;
+/**
+ * Create a workflow function for uploading S3 contents externally, via a presigned URL
+ * @param options - Registration options for the workflow
+ * @param callbacks - S3 operation implementation and database recordkeeping transactions
+ */
+export declare function registerS3PresignedUploadWorkflow<R extends FileRecord, Options = unknown>(options: {
+    name?: string;
+    ctorOrProto?: object;
+    className?: string;
+    config?: WorkflowConfig;
+}, callbacks: S3WorkflowCallbacks<R, Options>): (this: unknown, fileDetails: R, timeoutSeconds: number, objOptions?: Options | undefined) => Promise<R>;
+//# sourceMappingURL=s3_utils.d.ts.map

package/dist/src/s3_utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"s3_utils.d.ts","sourceRoot":"","sources":["../../src/s3_utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAEhE,OAAO,EAAQ,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAE1D,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,mBAAmB,CAAC,CAAC,SAAS,UAAU,EAAE,OAAO,GAAG,OAAO;IAE1E,sGAAsG;IACtG,aAAa,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,gHAAgH;IAChH,cAAc,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAC7C,sGAAsG;IACtG,aAAa,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,6GAA6G;IAC7G,WAAW,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAG1C,+FAA+F;IAC/F,aAAa,EAAE,CAAC,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAChF,iHAAiH;IACjH,mBAAmB,EAAE,CAAC,GAAG,EAAE,CAAC,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,KAAK,OAAO,CAAC,aAAa,CAAC,CAAC;IAC7F,yHAAyH;IACzH,gBAAgB,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7C,oFAAoF;IACpF,cAAc,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAC9C;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,CAAC,SAAS,UAAU,EAAE,OAAO,GAAG,OAAO,EAC9E,OAAO,EAAE;IACP,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,cAAc,CAAC;CACzB,EACD,SAAS,EAAE,mBAAmB,CAAC,CAAC,EAAE,OAAO,CAAC,uDAW3C;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,CAAC,SAAS,UAAU,EAAE,OAAO,GAAG,OAAO,EAC9E,OAAO,EAAE;IACP,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,cAAc,CAAC;CACzB,EACD,SAAS,EAAE,mBAAmB,CAAC,CAAC,EAAE,OAAO,CAAC,oGA2B3C;AAED;;;;GAIG;AACH,wBAAgB,iCAAiC,CAAC,CAAC,SAAS,UAAU,EAAE,OAAO,GAAG,OAAO,EACvF,OAAO,EAAE;IACP,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,cAAc,CAAC;CACzB,EACD,SAAS,EAAE,mBAAmB,CAAC,CAAC,EAAE,OAAO,CAAC,2GAgD3C"}

package/dist/src/s3_utils.js

@@ -0,0 +1,88 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerS3PresignedUploadWorkflow = exports.registerS3UploadWorkflow = exports.registerS3DeleteWorkflow = void 0;
+const dbos_sdk_1 = require("@dbos-inc/dbos-sdk");
+/**
+ * Create a workflow function for deleting S3 objects and removing the DB entry
+ * @param options - Registration options for the workflow
+ * @param callbacks - S3 operation implementation and database recordkeeping transactions
+ */
+function registerS3DeleteWorkflow(options, callbacks) {
+    return dbos_sdk_1.DBOS.registerWorkflow(async (fileDetails) => {
+        await callbacks.fileDeleted(fileDetails);
+        return await dbos_sdk_1.DBOS.runStep(async () => {
+            return callbacks.deleteS3Object(fileDetails);
+        }, { name: 'deleteS3Object' });
+    }, options);
+}
+exports.registerS3DeleteWorkflow = registerS3DeleteWorkflow;
+/**
+ * Create a workflow function for uploading S3 contents from DBOS
+ * @param options - Registration options for the workflow
+ * @param callbacks - S3 operation implementation and database recordkeeping transactions
+ */
+function registerS3UploadWorkflow(options, callbacks) {
+    return dbos_sdk_1.DBOS.registerWorkflow(async (fileDetails, content, objOptions) => {
+        try {
+            await dbos_sdk_1.DBOS.runStep(async () => {
+                await callbacks.putS3Contents(fileDetails, content, objOptions);
+            }, { name: 'putS3Contents' });
+        }
+        catch (e) {
+            try {
+                await dbos_sdk_1.DBOS.runStep(async () => {
+                    return callbacks.deleteS3Object(fileDetails);
+                }, { name: 'deleteS3Object' });
+            }
+            catch (e2) {
+                dbos_sdk_1.DBOS.logger.debug(e2);
+            }
+            throw e;
+        }
+        await callbacks.newActiveFile(fileDetails);
+        return fileDetails;
+    }, options);
+}
+exports.registerS3UploadWorkflow = registerS3UploadWorkflow;
+/**
+ * Create a workflow function for uploading S3 contents externally, via a presigned URL
+ * @param options - Registration options for the workflow
+ * @param callbacks - S3 operation implementation and database recordkeeping transactions
+ */
+function registerS3PresignedUploadWorkflow(options, callbacks) {
+    return dbos_sdk_1.DBOS.registerWorkflow(async (fileDetails, timeoutSeconds, objOptions) => {
+        await callbacks.newPendingFile(fileDetails);
+        const upkey = await dbos_sdk_1.DBOS.runStep(async () => {
+            return await callbacks.createPresignedPost(fileDetails, timeoutSeconds, objOptions);
+        }, { name: 'createPresignedPost' });
+        await dbos_sdk_1.DBOS.setEvent('uploadkey', upkey);
+        try {
+            const res = await dbos_sdk_1.DBOS.recv('uploadfinish', timeoutSeconds + 60);
+            if (!res) {
+                throw new Error('S3 operation timed out or canceled');
+            }
+            // Validate the file, if we have code for that
+            if (callbacks.validateS3Upload) {
+                await dbos_sdk_1.DBOS.runStep(async () => {
+                    await callbacks.validateS3Upload(fileDetails);
+                }, { name: 'validateFileUpload' });
+            }
+            await callbacks?.fileActivated(fileDetails);
+        }
+        catch (e) {
+            try {
+                await callbacks.fileDeleted(fileDetails);
+                await dbos_sdk_1.DBOS.runStep(async () => {
+                    await callbacks.deleteS3Object(fileDetails);
+                }, { name: 'deleteS3Object' });
+            }
+            catch (e2) {
+                dbos_sdk_1.DBOS.logger.debug(e2);
+            }
+            throw e;
+        }
+        return fileDetails;
+    }, options);
+}
+exports.registerS3PresignedUploadWorkflow = registerS3PresignedUploadWorkflow;
+//# sourceMappingURL=s3_utils.js.map

package/dist/src/s3_utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"s3_utils.js","sourceRoot":"","sources":["../../src/s3_utils.ts"],"names":[],"mappings":";;;AAEA,iDAA0D;AA4B1D;;;;GAIG;AACH,SAAgB,wBAAwB,CACtC,OAKC,EACD,SAA0C;IAE1C,OAAO,eAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,WAAc,EAAE,EAAE;QACpD,MAAM,SAAS,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;QACzC,OAAO,MAAM,eAAI,CAAC,OAAO,CACvB,KAAK,IAAI,EAAE;YACT,OAAO,SAAS,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAC/C,CAAC,EACD,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAC3B,CAAC;IACJ,CAAC,EAAE,OAAO,CAAC,CAAC;AACd,CAAC;AAlBD,4DAkBC;AAED;;;;GAIG;AACH,SAAgB,wBAAwB,CACtC,OAKC,EACD,SAA0C;IAE1C,OAAO,eAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,WAAc,EAAE,OAAe,EAAE,UAAoB,EAAE,EAAE;QAC3F,IAAI,CAAC;YACH,MAAM,eAAI,CAAC,OAAO,CAChB,KAAK,IAAI,EAAE;gBACT,MAAM,SAAS,CAAC,aAAa,CAAC,WAAW,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;YAClE,CAAC,EACD,EAAE,IAAI,EAAE,eAAe,EAAE,CAC1B,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,eAAI,CAAC,OAAO,CAChB,KAAK,IAAI,EAAE;oBACT,OAAO,SAAS,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;gBAC/C,CAAC,EACD,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAC3B,CAAC;YACJ,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,eAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACxB,CAAC;YACD,MAAM,CAAC,CAAC;QACV,CAAC;QAED,MAAM,SAAS,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;QAC3C,OAAO,WAAW,CAAC;IACrB,CAAC,EAAE,OAAO,CAAC,CAAC;AACd,CAAC;AAlCD,4DAkCC;AAED;;;;GAIG;AACH,SAAgB,iCAAiC,CAC/C,OAKC,EACD,SAA0C;IAE1C,OAAO,eAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,WAAc,EAAE,cAAsB,EAAE,UAAoB,EAAE,EAAE;QAClG,MAAM,SAAS,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAE5C,MAAM,KAAK,GAAG,MAAM,eAAI,CAAC,OAAO,CAC9B,KAAK,IAAI,EAAE;YACT,OAAO,MAAM,SAAS,CAAC,mBAAmB,CAAC,WAAW,EAAE,cAAc,EAAE,UAAU,CAAC,CAAC;QACtF,CAAC,EACD,EAAE,IAAI,EAAE,qBAAqB,EAAE,CAChC,CAAC;QACF,MAAM,eAAI,CAAC,QAAQ,CAAgB,WAAW,EAAE,KAAK,CAAC,CAAC;QAEvD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,eAAI,CAAC,IAAI,CAAU,cAAc,EAAE,cAAc,GAAG,EAAE,CAAC,CAAC;YAE1E,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;YACxD,CAAC;YAED,8CAA8C;YAC9C,IAAI,SAAS,CAAC,gBAAgB,EAAE,CAAC;gBAC/B,MAAM,eAAI,CAAC,OAAO,CAChB,KAAK,IAAI,EAAE;oBACT,MAAM,SAAS,CAAC,gBAAiB,CAAC,WAAW,CAAC,CAAC;gBACjD,CAAC,EACD,EAAE,IAAI,EAAE,oBAAoB,EAAE,CAC/B,CAAC;YACJ,CAAC;YAED,MAAM,SAAS,EAAE,aAAa,CAAC,WAAW,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,SAAS,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;gBACzC,MAAM,eAAI,CAAC,OAAO,CAChB,KAAK,IAAI,EAAE;oBACT,MAAM,SAAS,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;gBAC9C,CAAC,EACD,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAC3B,CAAC;YACJ,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,eAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACxB,CAAC;YACD,MAAM,CAAC,CAAC;QACV,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC,EAAE,OAAO,CAAC,CAAC;AACd,CAAC;AAvDD,8EAuDC"}

package/dist/src/s3_utils.test.d.ts

@@ -0,0 +1,25 @@
+import { FileRecord } from './s3_utils';
+interface FileDetails {
+    user_id: string;
+    file_type: string;
+    file_name: string;
+    file_id?: string;
+    file_status?: string;
+    file_time?: number;
+}
+interface UserFile extends FileDetails, FileRecord {
+    user_id: string;
+    file_type: string;
+    file_name: string;
+    file_id: string;
+    file_status: string;
+    file_time: number;
+}
+interface Opts {
+    contentType?: string;
+}
+export declare const uploadWF: (this: unknown, fileDetails: UserFile, content: string, objOptions?: Opts | undefined) => Promise<UserFile>;
+export declare const uploadPWF: (this: unknown, fileDetails: UserFile, timeoutSeconds: number, objOptions?: Opts | undefined) => Promise<UserFile>;
+export declare const deleteWF: (this: unknown, fileDetails: UserFile) => Promise<unknown>;
+export {};
+//# sourceMappingURL=s3_utils.test.d.ts.map

package/dist/src/s3_utils.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"s3_utils.test.d.ts","sourceRoot":"","sources":["../../src/s3_utils.test.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,EAKX,MAAM,YAAY,CAAC;AAmBpB,UAAU,WAAW;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,QAAS,SAAQ,WAAW,EAAE,UAAU;IAChD,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAmGD,UAAU,IAAI;IACZ,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAwDD,eAAO,MAAM,QAAQ,6GAAoF,CAAC;AAC1G,eAAO,MAAM,SAAS,oHAA8F,CAAC;AACrH,eAAO,MAAM,QAAQ,4DAAoF,CAAC"}