@tstdl/base 0.93.117 → 0.93.118

package/object-storage/s3/s3.object-storage.js

@@ -9,20 +9,19 @@ var __metadata = (this && this.__metadata) || function (k, v) {
 };
 var S3ObjectStorage_1;
 import { Readable } from 'node:stream';
-import { CopyDestinationOptions, CopySourceOptions } from 'minio';
+import { CopyObjectCommand, CreateBucketCommand, DeleteBucketLifecycleCommand, DeleteObjectCommand, DeleteObjectsCommand, GetBucketLifecycleConfigurationCommand, GetObjectCommand, HeadBucketCommand, HeadObjectCommand, ListObjectsV2Command, PutBucketLifecycleConfigurationCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 import { match, P } from 'ts-pattern';
 import { Singleton } from '../../injector/decorators.js';
 import { registerAfterResolve } from '../../injector/resolution.js';
 import { ObjectStorage } from '../../object-storage/index.js';
-import { toArray } from '../../utils/array/array.js';
-import { mapAsync } from '../../utils/async-iterable-helpers/map.js';
 import { toArrayAsync } from '../../utils/async-iterable-helpers/to-array.js';
 import { now } from '../../utils/date-time.js';
 import { mapObjectKeys } from '../../utils/object/object.js';
 import { readableStreamFromPromise } from '../../utils/stream/index.js';
 import { readBinaryStream } from '../../utils/stream/stream-reader.js';
 import { _throw } from '../../utils/throw.js';
-import { assertDefinedPass, isDefined, isObject, isString, isUint8Array, isUndefined } from '../../utils/type-guards.js';
+import { assertDefinedPass, isBlob, isDate, isDefined, isObject, isReadableStream, isString, isUint8Array, isUndefined } from '../../utils/type-guards.js';
 import { secondsPerDay } from '../../utils/units.js';
 import { S3ObjectStorageProvider } from './s3.object-storage-provider.js';
 import { S3Object } from './s3.object.js';
@@ -42,27 +41,37 @@ let S3ObjectStorage = S3ObjectStorage_1 = class S3ObjectStorage extends ObjectSt
         });
     }
     async ensureBucketExists(region, options) {
-        const exists = await this.client.bucketExists(this.bucket);
-        if (exists) {
+        try {
+            await this.client.send(new HeadBucketCommand({ Bucket: this.bucket }));
             return;
         }
-        await this.client.makeBucket(this.bucket, region ?? '', { ObjectLocking: options?.objectLocking ?? false });
+        catch (error) {
+            // ignore error if bucket already exists (e.g. 403 Forbidden when we have no head permission but it might still exist)
+            if (this.isNotFoundError(error)) {
+                await this.client.send(new CreateBucketCommand({
+                    Bucket: this.bucket,
+                    CreateBucketConfiguration: isDefined(region) ? { LocationConstraint: region } : undefined,
+                    ObjectLockEnabledForBucket: options?.objectLocking,
+                }));
+                return;
+            }
+            throw error;
+        }
     }
     async configureBucket(configuration) {
         if (isUndefined(configuration.lifecycle)) {
             return;
         }
-        let currentLifecycle = null;
+        let currentLifecycleRules;
         try {
-            currentLifecycle = await this.client.getBucketLifecycle(this.bucket);
+            const result = await this.client.send(new GetBucketLifecycleConfigurationCommand({ Bucket: this.bucket }));
+            currentLifecycleRules = result.Rules;
         }
         catch (error) {
-            // ignore error if lifecycle configuration is not set
-            if (!isObject(error) || (error.code != 'NoSuchLifecycleConfiguration')) {
+            if (!this.isError(error, 'NoSuchLifecycleConfiguration')) {
                 throw error;
             }
         }
-        const currentLifecycleRules = isDefined(currentLifecycle?.Rule) ? toArray(currentLifecycle.Rule) : undefined; // https://github.com/minio/minio-js/issues/1407
         const tstdlRule = currentLifecycleRules?.find((rule) => rule.ID == 'TstdlExpireObjects');
         const tstdlRuleExpiration = tstdlRule?.Expiration?.Days;
         const targetExpirationDays = configuration.lifecycle?.expiration?.after;
@@ -73,35 +82,43 @@ let S3ObjectStorage = S3ObjectStorage_1 = class S3ObjectStorage extends ObjectSt
         const nonTstdlRules = currentLifecycleRules?.filter((rule) => rule.ID != 'TstdlExpireObjects') ?? [];
         if (isUndefined(targetExpiration)) {
             if (nonTstdlRules.length == 0) {
-                await this.client.removeBucketLifecycle(this.bucket);
+                await this.client.send(new DeleteBucketLifecycleCommand({
+                    Bucket: this.bucket,
+                }));
             }
             else {
-                await this.client.setBucketLifecycle(this.bucket, { Rule: nonTstdlRules });
+                await this.client.send(new PutBucketLifecycleConfigurationCommand({
+                    Bucket: this.bucket,
+                    LifecycleConfiguration: { Rules: nonTstdlRules },
+                }));
             }
         }
         else {
-            await this.client.setBucketLifecycle(this.bucket, {
-                Rule: [
-                    ...nonTstdlRules,
-                    {
-                        ID: 'TstdlExpireObjects',
-                        Status: 'Enabled',
-                        Expiration: {
-                            Days: targetExpiration,
+            await this.client.send(new PutBucketLifecycleConfigurationCommand({
+                Bucket: this.bucket,
+                LifecycleConfiguration: {
+                    Rules: [
+                        ...nonTstdlRules,
+                        {
+                            ID: 'TstdlExpireObjects',
+                            Status: 'Enabled',
+                            Expiration: {
+                                Days: targetExpiration,
+                            },
                         },
-                    },
-                ],
-            });
+                    ],
+                },
+            }));
         }
     }
     async exists(key) {
         const bucketKey = this.getBucketKey(key);
         try {
-            await this.client.statObject(this.bucket, bucketKey);
+            await this.client.send(new HeadObjectCommand({ Bucket: this.bucket, Key: bucketKey }));
             return true;
         }
         catch (error) {
-            if (isObject(error) && error.code == 'NotFound') {
+            if (this.isNotFoundError(error)) {
                 return false;
             }
             throw error;
@@ -109,21 +126,28 @@ let S3ObjectStorage = S3ObjectStorage_1 = class S3ObjectStorage extends ObjectSt
     }
     async statObject(key) {
         const bucketKey = this.getBucketKey(key);
-        return await this.client.statObject(this.bucket, bucketKey);
+        const result = await this.client.send(new HeadObjectCommand({ Bucket: this.bucket, Key: bucketKey }));
+        return {
+            size: result.ContentLength ?? 0,
+            etag: result.ETag ?? '',
+            lastModified: result.LastModified?.getTime(),
+            metadata: result.Metadata ?? {},
+        };
     }
     async uploadObject(key, content, options) {
         const bucketKey = this.getBucketKey(key);
-        if (isUint8Array(content)) {
-            await this.client.putObject(this.bucket, bucketKey, Buffer.from(content), options?.contentLength, options?.metadata);
-        }
-        else {
-            const readable = Readable.fromWeb(content);
-            const errorPromise = new Promise((_, reject) => readable.on('error', reject));
-            await Promise.race([
-                this.client.putObject(this.bucket, bucketKey, readable, options?.contentLength, options?.metadata),
-                errorPromise,
-            ]);
-        }
+        const body = isUint8Array(content) ? content : Readable.fromWeb(content);
+        const metadata = isDefined(options?.metadata) ? mapObjectKeys(options.metadata, (key) => key.toLowerCase()) : undefined;
+        await this.client.send(new PutObjectCommand({
+            Bucket: this.bucket,
+            Key: bucketKey,
+            Body: body,
+            ContentLength: options?.contentLength,
+            ContentType: options?.contentType,
+            ContentDisposition: options?.contentDisposition,
+            CacheControl: options?.cacheControl,
+            Metadata: metadata,
+        }));
         return await this.getObject(key);
     }
     async copyObject(source, destination, options) {
@@ -137,11 +161,16 @@ let S3ObjectStorage = S3ObjectStorage_1 = class S3ObjectStorage extends ObjectSt
             .with([P.instanceOf(S3ObjectStorage_1), P.string], async ([storage, key]) => await storage.getObject(key))
             .otherwise(() => _throw(new Error('Destination must be a S3Object, string key, or [S3ObjectStorage, string] tuple.')));
         const sourceMetadata = await sourceObject.getMetadata();
-        await this.client.copyObject(new CopySourceOptions({ Bucket: this.bucket, Object: sourceObject.storage.getBucketKey(sourceObject.key) }), new CopyDestinationOptions({
+        const metadata = mapObjectKeys({ ...sourceMetadata, ...options?.metadata }, (key) => key.toLowerCase());
+        await this.client.send(new CopyObjectCommand({
+            CopySource: `${this.bucket}/${sourceObject.storage.getBucketKey(sourceObject.key)}`,
             Bucket: destinationObject.storage.bucket,
-            Object: destinationObject.storage.getBucketKey(destinationObject.key),
+            Key: destinationObject.storage.getBucketKey(destinationObject.key),
             MetadataDirective: 'REPLACE',
-            UserMetadata: { ...sourceMetadata, ...options?.metadata },
+            ContentType: options?.contentType,
+            ContentDisposition: options?.contentDisposition,
+            CacheControl: options?.cacheControl,
+            Metadata: metadata,
         }));
         return destinationObject;
     }
@@ -153,22 +182,45 @@ let S3ObjectStorage = S3ObjectStorage_1 = class S3ObjectStorage extends ObjectSt
     }
     async getContent(key) {
         const bucketKey = this.getBucketKey(key);
-        const result = await this.client.getObject(this.bucket, bucketKey);
-        return await readBinaryStream(result);
+        const result = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: bucketKey }));
+        if (isUint8Array(result.Body)) {
+            return result.Body;
+        }
+        return await readBinaryStream(result.Body);
     }
     getContentStream(key) {
         const bucketKey = this.getBucketKey(key);
         return readableStreamFromPromise(async () => {
-            const readable = await this.client.getObject(this.bucket, bucketKey);
-            return Readable.toWeb(readable);
+            const result = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: bucketKey }));
+            if (isReadableStream(result.Body)) {
+                return result.Body;
+            }
+            if (isBlob(result.Body)) {
+                return result.Body.stream();
+            }
+            return Readable.toWeb(result.Body);
         });
     }
     async getObjects() {
         return await toArrayAsync(this.getObjectsCursor());
     }
-    getObjectsCursor() {
-        const stream = this.client.listObjectsV2(this.bucket, this.prefix, true);
-        return mapAsync(stream, (item) => new S3Object(this.module, this.getKey(item.name), `s3://${this.bucket}/${item.name}`, item.size, this));
+    async *getObjectsCursor() {
+        let continuationToken;
+        do {
+            const result = await this.client.send(new ListObjectsV2Command({
+                Bucket: this.bucket,
+                Prefix: this.prefix,
+                ContinuationToken: continuationToken,
+            }));
+            if (isDefined(result.Contents)) {
+                for (const item of result.Contents) {
+                    if (isDefined(item.Key)) {
+                        yield new S3Object(this.module, this.getKey(item.Key), `s3://${this.bucket}/${item.Key}`, item.Size, this);
+                    }
+                }
+            }
+            continuationToken = result.NextContinuationToken;
+        } while (isDefined(continuationToken));
     }
     // eslint-disable-next-line @typescript-eslint/require-await
     async getObject(key) {
@@ -180,22 +232,42 @@ let S3ObjectStorage = S3ObjectStorage_1 = class S3ObjectStorage extends ObjectSt
     }
     async getDownloadUrl(key, expirationTimestamp, responseHeaders) {
         const bucketKey = this.getBucketKey(key);
-        const { date, expiration } = getDateAndExpiration(expirationTimestamp);
-        return await this.client.presignedGetObject(this.bucket, bucketKey, expiration, responseHeaders ?? {}, date);
+        const expiration = getExpiration(expirationTimestamp);
+        const expiresHeader = responseHeaders?.['Expires'];
+        const mappedExpiresHeader = isDefined(expiresHeader) ? (isDate(expiresHeader) ? expiresHeader : new Date(expiresHeader)) : undefined;
+        return await getSignedUrl(this.client, new GetObjectCommand({
+            Bucket: this.bucket,
+            Key: bucketKey,
+            ResponseContentType: responseHeaders?.['Content-Type'],
+            ResponseContentDisposition: responseHeaders?.['Content-Disposition'],
+            ResponseCacheControl: responseHeaders?.['Cache-Control'],
+            ResponseContentLanguage: responseHeaders?.['Content-Language'],
+            ResponseContentEncoding: responseHeaders?.['Content-Encoding'],
+            ResponseExpires: mappedExpiresHeader,
+        }), { expiresIn: expiration });
     }
     async getUploadUrl(key, expirationTimestamp, options) {
         const bucketKey = this.getBucketKey(key);
-        const { date, expiration } = getDateAndExpiration(expirationTimestamp);
-        const query = mapObjectKeys(options?.metadata ?? {}, (key) => `X-Amz-Meta-${key}`);
-        return await this.client.presignedUrl('PUT', this.bucket, bucketKey, expiration, query, date);
+        const expiration = getExpiration(expirationTimestamp);
+        return await getSignedUrl(this.client, new PutObjectCommand({
+            Bucket: this.bucket,
+            Key: bucketKey,
+            Metadata: options?.metadata,
+            ContentType: options?.contentType,
+            ContentDisposition: options?.contentDisposition,
+            CacheControl: options?.cacheControl,
+        }), { expiresIn: expiration });
     }
     async deleteObject(key) {
         const bucketKey = this.getBucketKey(key);
-        await this.client.removeObject(this.bucket, bucketKey);
+        await this.client.send(new DeleteObjectCommand({ Bucket: this.bucket, Key: bucketKey }));
     }
     async deleteObjects(keys) {
-        const bucketKeys = keys.map((key) => this.getBucketKey(key));
-        await this.client.removeObjects(this.bucket, bucketKeys);
+        const bucketKeys = keys.map((key) => ({ Key: this.getBucketKey(key) }));
+        await this.client.send(new DeleteObjectsCommand({
+            Bucket: this.bucket,
+            Delete: { Objects: bucketKeys },
+        }));
     }
     getResourceUriSync(key) {
         const bucketKey = this.getBucketKey(key);
@@ -207,6 +279,15 @@ let S3ObjectStorage = S3ObjectStorage_1 = class S3ObjectStorage extends ObjectSt
     getKey(bucketKey) {
         return bucketKey.slice(this.prefix.length);
     }
+    isNotFoundError(error) {
+        return this.isError(error, 'NotFound', 'NoSuchKey');
+    }
+    isForbiddenError(error) {
+        return this.isError(error, 'Forbidden', 'AccessDenied', 'InvalidAccessKeyId', 'SignatureDoesNotMatch') || ((isObject(error) && isObject(error.$metadata) && error.$metadata.httpStatusCode == 403));
+    }
+    isError(error, ...names) {
+        return isObject(error) && names.includes(error.name);
+    }
 };
 S3ObjectStorage = S3ObjectStorage_1 = __decorate([
     Singleton({
@@ -218,11 +299,11 @@ S3ObjectStorage = S3ObjectStorage_1 = __decorate([
         },
         argumentIdentityProvider: JSON.stringify,
     }),
-    __metadata("design:paramtypes", [Function, String, String, String])
+    __metadata("design:paramtypes", [S3Client, String, String, String])
 ], S3ObjectStorage);
 export { S3ObjectStorage };
-function getDateAndExpiration(expirationTimestamp) {
+function getExpiration(expirationTimestamp) {
     const date = now();
     const diffSeconds = Math.floor((expirationTimestamp - date.getTime()) / 1000);
-    return { date, expiration: diffSeconds };
+    return diffSeconds;
 }

package/object-storage/s3/s3.object.d.ts

@@ -1,6 +1,12 @@
 import type { ObjectMetadata } from '../../object-storage/object.js';
 import { ObjectStorageObject } from '../../object-storage/object.js';
 import type { S3ObjectStorage } from './s3.object-storage.js';
+export type S3BucketItemStat = {
+    size: number;
+    etag?: string;
+    lastModified?: number;
+    metadata: ObjectMetadata;
+};
 export declare class S3Object extends ObjectStorageObject {
     private readonly resourceUri;
     private contentLength;

package/object-storage/s3/s3.object.js

@@ -24,7 +24,7 @@ export class S3Object extends ObjectStorageObject {
     }
     async getMetadata() {
         const stat = await this.stat();
-        return stat.metaData;
+        return stat.metadata;
     }
     async getContent() {
         return await this.storage.getContent(this.key);

package/object-storage/s3/tests/s3.object-storage.integration.test.d.ts

@@ -0,0 +1 @@
+export {};