@temporalio/external-storage-s3 1.21.0

package/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2021-2025 Temporal Technologies Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

package/README.md

@@ -0,0 +1,159 @@
+# Amazon S3 External Storage Driver for the Temporal TypeScript SDK
+
+> ⚠️ **This package is experimental and may be subject to change.** ⚠️
+
+`@temporalio/external-storage-s3` stores and retrieves Temporal payloads in Amazon S3 via the [External Storage](https://docs.temporal.io/external-storage) feature.
+
+This package has no AWS dependency: it defines the driver and the `S3StorageDriverClient` interface, and you supply the S3 client. Use the companion [`@temporalio/external-storage-s3-aws-sdk`](../external-storage-s3-aws-sdk) package for an [`@aws-sdk/client-s3`](https://www.npmjs.com/package/@aws-sdk/client-s3)-backed client, or implement the interface yourself.
+
+## Using the AWS SDK client
+
+Install the adapter package alongside this one:
+
+    npm install @temporalio/external-storage-s3 @temporalio/external-storage-s3-aws-sdk @aws-sdk/client-s3
+
+```ts
+import { S3Client } from '@aws-sdk/client-s3';
+import { S3StorageDriver } from '@temporalio/external-storage-s3';
+import { AwsSdkS3StorageDriverClient } from '@temporalio/external-storage-s3-aws-sdk';
+
+const s3Client = new S3Client({ region: 'us-east-1' });
+const driver = new S3StorageDriver({
+  client: new AwsSdkS3StorageDriverClient(s3Client),
+  bucket: 'my-temporal-payloads',
+});
+```
+
+Register the resulting driver with the SDK's External Storage configuration so the
+client and worker offload eligible payloads to it.
+
+## Custom S3 client implementations
+
+To use a different S3 library, implement `S3StorageDriverClient`.
+
+```ts
+import type { S3StorageDriverClient } from '@temporalio/external-storage-s3';
+
+const myClient: S3StorageDriverClient = {
+  async putObject(bucket, key, data, options) {
+    /* ... */
+  },
+  async objectExists(bucket, key, options) {
+    /* ... */
+    return false;
+  },
+  async getObject(bucket, key, options) {
+    /* ... */
+    return new Uint8Array();
+  },
+};
+```
+
+## Dynamic bucket selection
+
+Pass a callable as `bucket` to choose the destination per payload:
+
+```ts
+const driver = new S3StorageDriver({
+  client: new AwsSdkS3StorageDriverClient(s3Client),
+  bucket: (_context, payload) => ((payload.data?.length ?? 0) > 10 * 1024 * 1024 ? 'large-payloads' : 'small-payloads'),
+});
+```
+
+## Required IAM permissions
+
+The credentials used by your S3 client must have these S3 permissions on the target bucket and its objects:
+
+```json
+{
+  "Effect": "Allow",
+  "Action": ["s3:PutObject", "s3:GetObject"],
+  "Resource": "arn:aws:s3:::my-temporal-payloads/*"
+}
+```
+
+`s3:PutObject` is required by components that store payloads (typically the client and worker sending workflow/activity inputs); `s3:GetObject` is required by components that retrieve them. Components that only retrieve do not need `s3:PutObject`, and vice versa.
+
+## S3 Storage Key Specification
+
+All Temporal S3 drivers generate S3 keys in a consistent manner.
+
+### Key format
+
+Workflow key:
+
+```text
+v0/ns/{namespace}/wt/{workflow-type}/wi/{workflow-id}/ri/{run-id}/d/{hash-algorithm}/{hex-digest}
+```
+
+Activity key:
+
+```text
+v0/ns/{namespace}/at/{activity-type}/ai/{activity-id}/ri/{run-id}/d/{hash-algorithm}/{hex-digest}
+```
+
+Fallback key (unknown target):
+
+```text
+v0/d/{hash-algorithm}/{hex-digest}
+```
+
+- If no namespace, workflow, or activity information is available, the fallback is used.
+- Dynamic path segments are percent-encoded (rules below).
+- Missing values (including a missing `run-id`) are encoded as `null`.
+- `hex-digest` is lower-case SHA-256 hex (64 characters).
+
+### Percent-encoding rules
+
+The Temporal SDKs escape anything that isn't listed in S3's safe character set: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html
+
+Safe Characters:
+
+```text
+Alphanumeric characters
+  0-9
+  a-z
+  A-Z
+
+Special characters
+  Exclamation point (!)
+  Hyphen (-)
+  Underscore (_)
+  Period (.)
+  Asterisk (*)
+  Single quotation mark (')
+  Opening parenthesis (()
+  Closing parenthesis ())
+```
+
+### Examples
+
+Workflow key example:
+
+```text
+input:
+  namespace=payments prod
+  workflow-type=ChargeWorkflow
+  workflow-id=order+123=abc
+  run-id=3f1d6c7a-8b2e-4f7a-9d0a-87a6f95e4d31
+  hash-algorithm=sha256
+  hex-digest=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08
+
+output:
+  v0/ns/payments%20prod/wt/ChargeWorkflow/wi/order%2B123%3Dabc/ri/3f1d6c7a-8b2e-4f7a-9d0a-87a6f95e4d31/d/sha256/9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08
+```
+
+Activity key example:
+
+```text
+input:
+  namespace=payments prod
+  activity-type=Capture/Charge
+  activity-id=activity id+42
+  run-id=9e1d1fd9-2f8a-4c40-93e2-731f31b9268b
+  hash-algorithm=sha256
+  hex-digest=2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824
+
+output:
+  v0/ns/payments%20prod/at/Capture%2FCharge/ai/activity%20id%2B42/ri/9e1d1fd9-2f8a-4c40-93e2-731f31b9268b/d/sha256/2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824
+```

package/lib/client.d.ts

@@ -0,0 +1,30 @@
+/**
+ * S3 object operations used by {@link S3StorageDriver}.
+ *
+ * @experimental
+ */
+/** Per-request options. */
+export interface S3RequestOptions {
+    /**
+     * Aborts the in-flight request.
+     */
+    abortSignal?: AbortSignal;
+}
+/**
+ * S3 driver client operations
+ *
+ * @experimental
+ */
+export interface S3StorageDriverClient {
+    /** Upload `data` to the given bucket and key. */
+    putObject(bucket: string, key: string, data: Uint8Array, options?: S3RequestOptions): Promise<void>;
+    /** Resolve `true` if an object exists at the given bucket and key. */
+    objectExists(bucket: string, key: string, options?: S3RequestOptions): Promise<boolean>;
+    /** Download and resolve the bytes stored at the given bucket and key. */
+    getObject(bucket: string, key: string, options?: S3RequestOptions): Promise<Uint8Array>;
+    /**
+     * Optional client-specific diagnostic metadata (e.g. region) that the driver
+     * appends to error messages to help diagnose common misconfigurations.
+     */
+    describe?(): Record<string, string>;
+}

package/lib/client.js

@@ -0,0 +1,8 @@
+"use strict";
+/**
+ * S3 object operations used by {@link S3StorageDriver}.
+ *
+ * @experimental
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=client.js.map

package/lib/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":";AAAA;;;;GAIG"}

package/lib/driver.d.ts

@@ -0,0 +1,52 @@
+import type { Payload } from '@temporalio/common';
+import { StorageDriverClaim, type StorageDriver, type StorageDriverStoreContext, type StorageDriverRetrieveContext } from '@temporalio/common/lib/converter/extstore';
+import type { S3StorageDriverClient } from './client';
+/** Picks the destination bucket for a given payload. Enables dynamic per-payload routing. */
+export type BucketSelector = (context: StorageDriverStoreContext, payload: Payload) => string;
+/**
+ * Configuration for an {@link S3StorageDriver}.
+ *
+ * @experimental
+ */
+export interface S3StorageDriverOptions {
+    /**
+     * An {@link S3StorageDriverClient} that performs the underlying requests,
+     * e.g. an `AwsSdkS3StorageDriverClient`.
+     */
+    client: S3StorageDriverClient;
+    /** Static bucket name or a {@link BucketSelector}. */
+    bucket: string | BucketSelector;
+    /**
+     * Per-instance routing name written into the wire format. Defaults to
+     * `"aws.s3driver"`. Override only when registering multiple S3 drivers with
+     * distinct configurations under the same `ExternalStorage.drivers` list.
+     */
+    driverName?: string;
+    /**
+     * Hard upper limit, in bytes, on the serialized size of a single payload.
+     * Defaults to 52428800 (50 MiB). Stores larger than this throw.
+     */
+    maxPayloadSize?: number;
+}
+/**
+ * Stores and retrieves Temporal payloads in Amazon S3.
+ *
+ * Payloads are content-addressed by a SHA-256 hash of their serialized bytes. S3 keys
+ * are created using the namespace, workflow/activity type, other metadata from the store
+ * context. Keys adhere to S3's safe-character set.
+ *
+ * @experimental
+ */
+export declare class S3StorageDriver implements StorageDriver {
+    readonly name: string;
+    readonly type = "aws.s3driver";
+    private readonly client;
+    private readonly bucket;
+    private readonly maxPayloadSize;
+    constructor(options: S3StorageDriverOptions);
+    store(context: StorageDriverStoreContext, payloads: Payload[]): Promise<StorageDriverClaim[]>;
+    retrieve(context: StorageDriverRetrieveContext, claims: StorageDriverClaim[]): Promise<Payload[]>;
+    private storePayload;
+    private uploadIfAbsent;
+    private retrievePayload;
+}

package/lib/driver.js

@@ -0,0 +1,184 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.S3StorageDriver = void 0;
+const node_crypto_1 = require("node:crypto");
+const proto = __importStar(require("@temporalio/proto"));
+const common_1 = require("@temporalio/common");
+const extstore_1 = require("@temporalio/common/lib/converter/extstore");
+const PayloadProto = proto.temporal.api.common.v1.Payload;
+const DRIVER_TYPE = 'aws.s3driver';
+const DEFAULT_MAX_PAYLOAD_SIZE = 50 * 1024 * 1024;
+/** Key segment written when a context value is empty or absent. */
+const NULL_SEGMENT = 'null';
+/**
+ * Percent-encodes a key segment, escaping everything outside S3's safe-character
+ * set (alphanumerics and `! - _ . * ' ( )`). An empty or absent value becomes the
+ * literal `null`.
+ */
+function encodeKeySegment(value) {
+    if (!value)
+        return NULL_SEGMENT;
+    return encodeURIComponent(value).replace(/~/g, '%7E');
+}
+function buildContextSegments(target) {
+    if (target?.kind === 'workflow') {
+        return (`/ns/${encodeKeySegment(target.namespace)}` +
+            `/wt/${encodeKeySegment(target.type)}` +
+            `/wi/${encodeKeySegment(target.id)}` +
+            `/ri/${encodeKeySegment(target.runId)}`);
+    }
+    if (target?.kind === 'activity') {
+        return (`/ns/${encodeKeySegment(target.namespace)}` +
+            `/at/${encodeKeySegment(target.type)}` +
+            `/ai/${encodeKeySegment(target.id)}` +
+            `/ri/${encodeKeySegment(target.runId)}`);
+    }
+    return '';
+}
+/**
+ * Formats `describe()` output as ", k=v, k=v" for error messages. Returns an
+ * empty string when the client reports no metadata or `describe` itself throws.
+ */
+function formatClientContext(client) {
+    let info;
+    try {
+        info = client.describe?.() ?? {};
+    }
+    catch {
+        return '';
+    }
+    const entries = Object.entries(info);
+    return entries.map(([k, v]) => `, ${k}=${v}`).join('');
+}
+/**
+ * Runs the per-payload operations concurrently. On the first failure, aborts the
+ * shared signal so in-flight sibling requests are cancelled, then waits for them
+ * to settle before propagating the original error.
+ */
+async function runAllWithAbortOnError(external, makeTasks) {
+    const controller = new AbortController();
+    const signal = external ? AbortSignal.any([external, controller.signal]) : controller.signal;
+    const tasks = makeTasks(signal);
+    try {
+        return await Promise.all(tasks);
+    }
+    catch (err) {
+        controller.abort();
+        await Promise.allSettled(tasks);
+        throw err;
+    }
+}
+/**
+ * Stores and retrieves Temporal payloads in Amazon S3.
+ *
+ * Payloads are content-addressed by a SHA-256 hash of their serialized bytes. S3 keys
+ * are created using the namespace, workflow/activity type, other metadata from the store
+ * context. Keys adhere to S3's safe-character set.
+ *
+ * @experimental
+ */
+class S3StorageDriver {
+    name;
+    type = DRIVER_TYPE;
+    client;
+    bucket;
+    maxPayloadSize;
+    constructor(options) {
+        const { client, bucket, driverName = DRIVER_TYPE, maxPayloadSize = DEFAULT_MAX_PAYLOAD_SIZE } = options;
+        if (!Number.isFinite(maxPayloadSize) || maxPayloadSize <= 0) {
+            throw new common_1.ValueError(`maxPayloadSize must be a positive finite number, got ${String(maxPayloadSize)}`);
+        }
+        this.client = client;
+        this.bucket = typeof bucket === 'string' ? () => bucket : bucket;
+        this.name = driverName;
+        this.maxPayloadSize = maxPayloadSize;
+    }
+    async store(context, payloads) {
+        const contextSegments = buildContextSegments(context.target);
+        return runAllWithAbortOnError(context.abortSignal, (signal) => {
+            const uploads = new Map();
+            return payloads.map((payload) => this.storePayload(context, payload, contextSegments, signal, uploads));
+        });
+    }
+    async retrieve(context, claims) {
+        return runAllWithAbortOnError(context.abortSignal, (signal) => claims.map((claim) => this.retrievePayload(claim, signal)));
+    }
+    async storePayload(context, payload, contextSegments, abortSignal, uploads) {
+        const bucket = this.bucket(context, payload);
+        const payloadBytes = PayloadProto.encode(payload).finish();
+        if (payloadBytes.length > this.maxPayloadSize) {
+            throw new common_1.ValueError(`Payload size ${payloadBytes.length} bytes exceeds the configured maxPayloadSize of ${this.maxPayloadSize} bytes`);
+        }
+        const hashValue = (0, node_crypto_1.createHash)('sha256').update(payloadBytes).digest('hex');
+        const key = `v0${contextSegments}/d/sha256/${hashValue}`;
+        try {
+            const dedupeKey = `${bucket} ${key}`;
+            let upload = uploads.get(dedupeKey);
+            if (!upload) {
+                upload = this.uploadIfAbsent(bucket, key, payloadBytes, abortSignal);
+                uploads.set(dedupeKey, upload);
+            }
+            await upload;
+        }
+        catch (err) {
+            throw new Error(`S3StorageDriver store failed [bucket=${bucket}, key=${key}${formatClientContext(this.client)}]`, { cause: err });
+        }
+        return new extstore_1.StorageDriverClaim({ bucket, key, hashAlgorithm: 'sha256', hashValue });
+    }
+    async uploadIfAbsent(bucket, key, payloadBytes, abortSignal) {
+        if (!(await this.client.objectExists(bucket, key, { abortSignal }))) {
+            await this.client.putObject(bucket, key, payloadBytes, { abortSignal });
+        }
+    }
+    async retrievePayload(claim, abortSignal) {
+        const { bucket, key, hashAlgorithm, hashValue: expectedHash } = claim.claimData;
+        if (!bucket || !key) {
+            throw new common_1.ValueError(`S3StorageDriver claim is missing required location information: ` + `claimData must contain 'bucket' and 'key'`);
+        }
+        if (!hashAlgorithm || !expectedHash) {
+            throw new common_1.ValueError(`S3StorageDriver claim is missing required content hash information [bucket=${bucket}, key=${key}]: ` +
+                `claimData must contain 'hashAlgorithm' and 'hashValue'`);
+        }
+        if (hashAlgorithm !== 'sha256') {
+            throw new common_1.ValueError(`S3StorageDriver unsupported hash algorithm [bucket=${bucket}, key=${key}]: expected sha256, got ${hashAlgorithm}`);
+        }
+        let payloadBytes;
+        try {
+            payloadBytes = await this.client.getObject(bucket, key, { abortSignal });
+        }
+        catch (err) {
+            throw new Error(`S3StorageDriver retrieve failed [bucket=${bucket}, key=${key}${formatClientContext(this.client)}]`, { cause: err });
+        }
+        const actualHash = (0, node_crypto_1.createHash)('sha256').update(payloadBytes).digest('hex');
+        if (actualHash !== expectedHash) {
+            throw new common_1.ValueError(`S3StorageDriver integrity check failed [bucket=${bucket}, key=${key}]: ` +
+                `expected ${hashAlgorithm}:${expectedHash}, got ${hashAlgorithm}:${actualHash}`);
+        }
+        return PayloadProto.decode(payloadBytes);
+    }
+}
+exports.S3StorageDriver = S3StorageDriver;
+//# sourceMappingURL=driver.js.map

package/lib/driver.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"driver.js","sourceRoot":"","sources":["../src/driver.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,6CAAyC;AACzC,yDAA2C;AAC3C,+CAAgD;AAEhD,wEAMmD;AAGnD,MAAM,YAAY,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC;AAE1D,MAAM,WAAW,GAAG,cAAc,CAAC;AACnC,MAAM,wBAAwB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAClD,mEAAmE;AACnE,MAAM,YAAY,GAAG,MAAM,CAAC;AA+B5B;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,KAAyB;IACjD,IAAI,CAAC,KAAK;QAAE,OAAO,YAAY,CAAC;IAChC,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AACxD,CAAC;AAED,SAAS,oBAAoB,CAAC,MAA2C;IACvE,IAAI,MAAM,EAAE,IAAI,KAAK,UAAU,EAAE,CAAC;QAChC,OAAO,CACL,OAAO,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE;YAC3C,OAAO,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;YACtC,OAAO,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE;YACpC,OAAO,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACxC,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,EAAE,IAAI,KAAK,UAAU,EAAE,CAAC;QAChC,OAAO,CACL,OAAO,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE;YAC3C,OAAO,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;YACtC,OAAO,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE;YACpC,OAAO,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACxC,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;;GAGG;AACH,SAAS,mBAAmB,CAAC,MAA6B;IACxD,IAAI,IAA4B,CAAC;IACjC,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACrC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACzD,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,sBAAsB,CACnC,QAAiC,EACjC,SAAgD;IAEhD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;IAC7F,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IAChC,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,UAAU,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAa,eAAe;IACjB,IAAI,CAAS;IACb,IAAI,GAAG,WAAW,CAAC;IACX,MAAM,CAAwB;IAC9B,MAAM,CAAiB;IACvB,cAAc,CAAS;IAExC,YAAY,OAA+B;QACzC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,GAAG,WAAW,EAAE,cAAc,GAAG,wBAAwB,EAAE,GAAG,OAAO,CAAC;QACxG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,cAAc,IAAI,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,mBAAU,CAAC,wDAAwD,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QACzG,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QACjE,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC;QACvB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,OAAkC,EAAE,QAAmB;QACjE,MAAM,eAAe,GAAG,oBAAoB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC7D,OAAO,sBAAsB,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,MAAM,EAAE,EAAE;YAC5D,MAAM,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;YACjD,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;QAC1G,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAqC,EAAE,MAA4B;QAChF,OAAO,sBAAsB,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,MAAM,EAAE,EAAE,CAC5D,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAC3D,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,OAAkC,EAClC,OAAgB,EAChB,eAAuB,EACvB,WAAwB,EACxB,OAAmC;QAEnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAE7C,MAAM,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;QAC3D,IAAI,YAAY,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YAC9C,MAAM,IAAI,mBAAU,CAClB,gBAAgB,YAAY,CAAC,MAAM,mDAAmD,IAAI,CAAC,cAAc,QAAQ,CAClH,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,IAAA,wBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC1E,MAAM,GAAG,GAAG,KAAK,eAAe,aAAa,SAAS,EAAE,CAAC;QAEzD,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,GAAG,MAAM,IAAI,GAAG,EAAE,CAAC;YACrC,IAAI,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACpC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;gBACrE,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YACjC,CAAC;YACD,MAAM,MAAM,CAAC;QACf,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CACb,wCAAwC,MAAM,SAAS,GAAG,GAAG,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,EAChG,EAAE,KAAK,EAAE,GAAG,EAAE,CACf,CAAC;QACJ,CAAC;QAED,OAAO,IAAI,6BAAkB,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,aAAa,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;IACrF,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,MAAc,EACd,GAAW,EACX,YAAwB,EACxB,WAAwB;QAExB,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC;YACpE,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,GAAG,EAAE,YAAY,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,KAAyB,EAAE,WAAwB;QAC/E,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,aAAa,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC;QAChF,IAAI,CAAC,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;YACpB,MAAM,IAAI,mBAAU,CAClB,kEAAkE,GAAG,2CAA2C,CACjH,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,aAAa,IAAI,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,IAAI,mBAAU,CAClB,8EAA8E,MAAM,SAAS,GAAG,KAAK;gBACnG,wDAAwD,CAC3D,CAAC;QACJ,CAAC;QACD,IAAI,aAAa,KAAK,QAAQ,EAAE,CAAC;YAC/B,MAAM,IAAI,mBAAU,CAClB,sDAAsD,MAAM,SAAS,GAAG,2BAA2B,aAAa,EAAE,CACnH,CAAC;QACJ,CAAC;QAED,IAAI,YAAwB,CAAC;QAC7B,IAAI,CAAC;YACH,YAAY,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;QAC3E,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CACb,2CAA2C,MAAM,SAAS,GAAG,GAAG,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,EACnG,EAAE,KAAK,EAAE,GAAG,EAAE,CACf,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,IAAA,wBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3E,IAAI,UAAU,KAAK,YAAY,EAAE,CAAC;YAChC,MAAM,IAAI,mBAAU,CAClB,kDAAkD,MAAM,SAAS,GAAG,KAAK;gBACvE,YAAY,aAAa,IAAI,YAAY,SAAS,aAAa,IAAI,UAAU,EAAE,CAClF,CAAC;QACJ,CAAC;QAED,OAAO,YAAY,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;CACF;AAvHD,0CAuHC"}

package/lib/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * @experimental The External Storage S3 driver is an experimental feature and may be subject to change.
+ */
+export { S3StorageDriver } from './driver';
+export type { S3StorageDriverOptions, BucketSelector } from './driver';
+export type { S3StorageDriverClient, S3RequestOptions } from './client';

package/lib/index.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.S3StorageDriver = void 0;
+/**
+ * @experimental The External Storage S3 driver is an experimental feature and may be subject to change.
+ */
+var driver_1 = require("./driver");
+Object.defineProperty(exports, "S3StorageDriver", { enumerable: true, get: function () { return driver_1.S3StorageDriver; } });
+//# sourceMappingURL=index.js.map

package/lib/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA;;GAEG;AACH,mCAA2C;AAAlC,yGAAA,eAAe,OAAA"}

package/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@temporalio/external-storage-s3",
+  "version": "1.21.0",
+  "description": "Amazon S3 external storage driver for the Temporal TypeScript SDK",
+  "main": "lib/index.js",
+  "types": "./lib/index.d.ts",
+  "keywords": [
+    "temporal",
+    "workflow",
+    "external storage",
+    "payload",
+    "s3",
+    "aws"
+  ],
+  "author": "Temporal Technologies Inc. <sdk@temporal.io>",
+  "license": "MIT",
+  "ava": {
+    "timeout": "60s"
+  },
+  "dependencies": {
+    "@temporalio/common": "1.21.0",
+    "@temporalio/proto": "1.21.0"
+  },
+  "devDependencies": {
+    "ava": "^5.3.1"
+  },
+  "engines": {
+    "node": ">= 20.3.0"
+  },
+  "bugs": {
+    "url": "https://github.com/temporalio/sdk-typescript/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/temporalio/sdk-typescript.git",
+    "directory": "contrib/external-storage-s3"
+  },
+  "homepage": "https://github.com/temporalio/sdk-typescript/tree/main/contrib/external-storage-s3",
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "src",
+    "lib",
+    "!src/__tests__",
+    "!lib/__tests__"
+  ],
+  "scripts": {
+    "build": "tsc --build",
+    "test": "ava ./lib/__tests__/test-*.js"
+  }
+}

package/src/client.ts

@@ -0,0 +1,32 @@
+/**
+ * S3 object operations used by {@link S3StorageDriver}.
+ *
+ * @experimental
+ */
+
+/** Per-request options. */
+export interface S3RequestOptions {
+  /**
+   * Aborts the in-flight request.
+   */
+  abortSignal?: AbortSignal;
+}
+
+/**
+ * S3 driver client operations
+ *
+ * @experimental
+ */
+export interface S3StorageDriverClient {
+  /** Upload `data` to the given bucket and key. */
+  putObject(bucket: string, key: string, data: Uint8Array, options?: S3RequestOptions): Promise<void>;
+  /** Resolve `true` if an object exists at the given bucket and key. */
+  objectExists(bucket: string, key: string, options?: S3RequestOptions): Promise<boolean>;
+  /** Download and resolve the bytes stored at the given bucket and key. */
+  getObject(bucket: string, key: string, options?: S3RequestOptions): Promise<Uint8Array>;
+  /**
+   * Optional client-specific diagnostic metadata (e.g. region) that the driver
+   * appends to error messages to help diagnose common misconfigurations.
+   */
+  describe?(): Record<string, string>;
+}

package/src/driver.ts

@@ -0,0 +1,244 @@
+import { createHash } from 'node:crypto';
+import * as proto from '@temporalio/proto';
+import { ValueError } from '@temporalio/common';
+import type { Payload } from '@temporalio/common';
+import {
+  StorageDriverClaim,
+  type StorageDriver,
+  type StorageDriverStoreContext,
+  type StorageDriverRetrieveContext,
+  type StorageDriverTargetInfo,
+} from '@temporalio/common/lib/converter/extstore';
+import type { S3StorageDriverClient } from './client';
+
+const PayloadProto = proto.temporal.api.common.v1.Payload;
+
+const DRIVER_TYPE = 'aws.s3driver';
+const DEFAULT_MAX_PAYLOAD_SIZE = 50 * 1024 * 1024;
+/** Key segment written when a context value is empty or absent. */
+const NULL_SEGMENT = 'null';
+
+/** Picks the destination bucket for a given payload. Enables dynamic per-payload routing. */
+export type BucketSelector = (context: StorageDriverStoreContext, payload: Payload) => string;
+
+/**
+ * Configuration for an {@link S3StorageDriver}.
+ *
+ * @experimental
+ */
+export interface S3StorageDriverOptions {
+  /**
+   * An {@link S3StorageDriverClient} that performs the underlying requests,
+   * e.g. an `AwsSdkS3StorageDriverClient`.
+   */
+  client: S3StorageDriverClient;
+  /** Static bucket name or a {@link BucketSelector}. */
+  bucket: string | BucketSelector;
+  /**
+   * Per-instance routing name written into the wire format. Defaults to
+   * `"aws.s3driver"`. Override only when registering multiple S3 drivers with
+   * distinct configurations under the same `ExternalStorage.drivers` list.
+   */
+  driverName?: string;
+  /**
+   * Hard upper limit, in bytes, on the serialized size of a single payload.
+   * Defaults to 52428800 (50 MiB). Stores larger than this throw.
+   */
+  maxPayloadSize?: number;
+}
+
+/**
+ * Percent-encodes a key segment, escaping everything outside S3's safe-character
+ * set (alphanumerics and `! - _ . * ' ( )`). An empty or absent value becomes the
+ * literal `null`.
+ */
+function encodeKeySegment(value: string | undefined): string {
+  if (!value) return NULL_SEGMENT;
+  return encodeURIComponent(value).replace(/~/g, '%7E');
+}
+
+function buildContextSegments(target: StorageDriverTargetInfo | undefined): string {
+  if (target?.kind === 'workflow') {
+    return (
+      `/ns/${encodeKeySegment(target.namespace)}` +
+      `/wt/${encodeKeySegment(target.type)}` +
+      `/wi/${encodeKeySegment(target.id)}` +
+      `/ri/${encodeKeySegment(target.runId)}`
+    );
+  }
+  if (target?.kind === 'activity') {
+    return (
+      `/ns/${encodeKeySegment(target.namespace)}` +
+      `/at/${encodeKeySegment(target.type)}` +
+      `/ai/${encodeKeySegment(target.id)}` +
+      `/ri/${encodeKeySegment(target.runId)}`
+    );
+  }
+  return '';
+}
+
+/**
+ * Formats `describe()` output as ", k=v, k=v" for error messages. Returns an
+ * empty string when the client reports no metadata or `describe` itself throws.
+ */
+function formatClientContext(client: S3StorageDriverClient): string {
+  let info: Record<string, string>;
+  try {
+    info = client.describe?.() ?? {};
+  } catch {
+    return '';
+  }
+  const entries = Object.entries(info);
+  return entries.map(([k, v]) => `, ${k}=${v}`).join('');
+}
+
+/**
+ * Runs the per-payload operations concurrently. On the first failure, aborts the
+ * shared signal so in-flight sibling requests are cancelled, then waits for them
+ * to settle before propagating the original error.
+ */
+async function runAllWithAbortOnError<T>(
+  external: AbortSignal | undefined,
+  makeTasks: (signal: AbortSignal) => Promise<T>[]
+): Promise<T[]> {
+  const controller = new AbortController();
+  const signal = external ? AbortSignal.any([external, controller.signal]) : controller.signal;
+  const tasks = makeTasks(signal);
+  try {
+    return await Promise.all(tasks);
+  } catch (err) {
+    controller.abort();
+    await Promise.allSettled(tasks);
+    throw err;
+  }
+}
+
+/**
+ * Stores and retrieves Temporal payloads in Amazon S3.
+ *
+ * Payloads are content-addressed by a SHA-256 hash of their serialized bytes. S3 keys
+ * are created using the namespace, workflow/activity type, other metadata from the store
+ * context. Keys adhere to S3's safe-character set.
+ *
+ * @experimental
+ */
+export class S3StorageDriver implements StorageDriver {
+  readonly name: string;
+  readonly type = DRIVER_TYPE;
+  private readonly client: S3StorageDriverClient;
+  private readonly bucket: BucketSelector;
+  private readonly maxPayloadSize: number;
+
+  constructor(options: S3StorageDriverOptions) {
+    const { client, bucket, driverName = DRIVER_TYPE, maxPayloadSize = DEFAULT_MAX_PAYLOAD_SIZE } = options;
+    if (!Number.isFinite(maxPayloadSize) || maxPayloadSize <= 0) {
+      throw new ValueError(`maxPayloadSize must be a positive finite number, got ${String(maxPayloadSize)}`);
+    }
+    this.client = client;
+    this.bucket = typeof bucket === 'string' ? () => bucket : bucket;
+    this.name = driverName;
+    this.maxPayloadSize = maxPayloadSize;
+  }
+
+  async store(context: StorageDriverStoreContext, payloads: Payload[]): Promise<StorageDriverClaim[]> {
+    const contextSegments = buildContextSegments(context.target);
+    return runAllWithAbortOnError(context.abortSignal, (signal) => {
+      const uploads = new Map<string, Promise<void>>();
+      return payloads.map((payload) => this.storePayload(context, payload, contextSegments, signal, uploads));
+    });
+  }
+
+  async retrieve(context: StorageDriverRetrieveContext, claims: StorageDriverClaim[]): Promise<Payload[]> {
+    return runAllWithAbortOnError(context.abortSignal, (signal) =>
+      claims.map((claim) => this.retrievePayload(claim, signal))
+    );
+  }
+
+  private async storePayload(
+    context: StorageDriverStoreContext,
+    payload: Payload,
+    contextSegments: string,
+    abortSignal: AbortSignal,
+    uploads: Map<string, Promise<void>>
+  ): Promise<StorageDriverClaim> {
+    const bucket = this.bucket(context, payload);
+
+    const payloadBytes = PayloadProto.encode(payload).finish();
+    if (payloadBytes.length > this.maxPayloadSize) {
+      throw new ValueError(
+        `Payload size ${payloadBytes.length} bytes exceeds the configured maxPayloadSize of ${this.maxPayloadSize} bytes`
+      );
+    }
+
+    const hashValue = createHash('sha256').update(payloadBytes).digest('hex');
+    const key = `v0${contextSegments}/d/sha256/${hashValue}`;
+
+    try {
+      const dedupeKey = `${bucket} ${key}`;
+      let upload = uploads.get(dedupeKey);
+      if (!upload) {
+        upload = this.uploadIfAbsent(bucket, key, payloadBytes, abortSignal);
+        uploads.set(dedupeKey, upload);
+      }
+      await upload;
+    } catch (err) {
+      throw new Error(
+        `S3StorageDriver store failed [bucket=${bucket}, key=${key}${formatClientContext(this.client)}]`,
+        { cause: err }
+      );
+    }
+
+    return new StorageDriverClaim({ bucket, key, hashAlgorithm: 'sha256', hashValue });
+  }
+
+  private async uploadIfAbsent(
+    bucket: string,
+    key: string,
+    payloadBytes: Uint8Array,
+    abortSignal: AbortSignal
+  ): Promise<void> {
+    if (!(await this.client.objectExists(bucket, key, { abortSignal }))) {
+      await this.client.putObject(bucket, key, payloadBytes, { abortSignal });
+    }
+  }
+
+  private async retrievePayload(claim: StorageDriverClaim, abortSignal: AbortSignal): Promise<Payload> {
+    const { bucket, key, hashAlgorithm, hashValue: expectedHash } = claim.claimData;
+    if (!bucket || !key) {
+      throw new ValueError(
+        `S3StorageDriver claim is missing required location information: ` + `claimData must contain 'bucket' and 'key'`
+      );
+    }
+    if (!hashAlgorithm || !expectedHash) {
+      throw new ValueError(
+        `S3StorageDriver claim is missing required content hash information [bucket=${bucket}, key=${key}]: ` +
+          `claimData must contain 'hashAlgorithm' and 'hashValue'`
+      );
+    }
+    if (hashAlgorithm !== 'sha256') {
+      throw new ValueError(
+        `S3StorageDriver unsupported hash algorithm [bucket=${bucket}, key=${key}]: expected sha256, got ${hashAlgorithm}`
+      );
+    }
+
+    let payloadBytes: Uint8Array;
+    try {
+      payloadBytes = await this.client.getObject(bucket, key, { abortSignal });
+    } catch (err) {
+      throw new Error(
+        `S3StorageDriver retrieve failed [bucket=${bucket}, key=${key}${formatClientContext(this.client)}]`,
+        { cause: err }
+      );
+    }
+
+    const actualHash = createHash('sha256').update(payloadBytes).digest('hex');
+    if (actualHash !== expectedHash) {
+      throw new ValueError(
+        `S3StorageDriver integrity check failed [bucket=${bucket}, key=${key}]: ` +
+          `expected ${hashAlgorithm}:${expectedHash}, got ${hashAlgorithm}:${actualHash}`
+      );
+    }
+
+    return PayloadProto.decode(payloadBytes);
+  }
+}

package/src/index.ts

@@ -0,0 +1,6 @@
+/**
+ * @experimental The External Storage S3 driver is an experimental feature and may be subject to change.
+ */
+export { S3StorageDriver } from './driver';
+export type { S3StorageDriverOptions, BucketSelector } from './driver';
+export type { S3StorageDriverClient, S3RequestOptions } from './client';