@travetto/model-s3 8.0.0-alpha.23 → 8.0.0-alpha.25

package/README.md

@@ -13,11 +13,11 @@ npm install @travetto/model-s3
 yarn add @travetto/model-s3
 ```
 
-This module provides an [s3](https://aws.amazon.com/documentation/s3/)-based implementation for the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.").  This source allows the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") module to read, write and stream against [s3](https://aws.amazon.com/documentation/s3/). 
+This module provides an [s3](https://aws.amazon.com/documentation/s3/)-based implementation for the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations."). This source allows the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") module to read, write and stream against [s3](https://aws.amazon.com/documentation/s3/). 
 
 Supported features:
-   *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L11)
    *  [Blob](https://github.com/travetto/travetto/tree/main/module/model/src/types/blob.ts#L8)
+   *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L10)
    *  [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10)
 
 Out of the box, by installing the module, everything should be wired up by default.If you need to customize any aspect of the source or config, you can override and register it with the [Dependency Injection](https://github.com/travetto/travetto/tree/main/module/di#readme "Dependency registration/management and injection support.") module.
@@ -114,4 +114,4 @@ export class S3ModelConfig {
 
 Additionally, you can see that the class is registered with the [@Config](https://github.com/travetto/travetto/tree/main/module/config/src/decorator.ts#L13) annotation, and so these values can be overridden using the standard [Configuration](https://github.com/travetto/travetto/tree/main/module/config#readme "Configuration support") resolution paths.
 
-**Note**: Do not commit your `accessKeyId` or `secretAccessKey` values to your source repository, especially if it is public facing.  Not only is it a security risk, but Amazon will scan public repos, looking for keys, and if found will react swiftly.
+**Note**: Do not commit your `accessKeyId` or `secretAccessKey` values to your source repository, especially if it is public facing. Not only is it a security risk, but Amazon will scan public repos, looking for keys, and if found will react swiftly.

package/__index__.ts

@@ -1,2 +1,2 @@
 export * from './src/config.ts';
-export * from './src/service.ts';
+export * from './src/service.ts';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-s3",
-  "version": "8.0.0-alpha.23",
+  "version": "8.0.0-alpha.25",
   "type": "module",
   "description": "S3 backing for the travetto model module.",
   "keywords": [
@@ -29,11 +29,11 @@
     "@aws-sdk/client-s3": "^3.1081.0",
     "@aws-sdk/credential-provider-ini": "^3.972.62",
     "@aws-sdk/s3-request-presigner": "^3.1081.0",
-    "@travetto/config": "^8.0.0-alpha.21",
-    "@travetto/model": "^8.0.0-alpha.22"
+    "@travetto/config": "^8.0.0-alpha.23",
+    "@travetto/model": "^8.0.0-alpha.24"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.27"
+    "@travetto/cli": "^8.0.0-alpha.29"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/config.ts

@@ -1,10 +1,10 @@
-import { fromIni } from '@aws-sdk/credential-provider-ini';
 import type s3 from '@aws-sdk/client-s3';
+import { fromIni } from '@aws-sdk/credential-provider-ini';
 
 import { Config, EnvVar } from '@travetto/config';
-import { Required, Url } from '@travetto/schema';
-import { Runtime } from '@travetto/runtime';
 import { PostConstruct } from '@travetto/di';
+import { Runtime } from '@travetto/runtime';
+import { Required, Url } from '@travetto/schema';
 
 /**
  * S3 Support as an Asset Source
@@ -77,4 +77,4 @@ export class S3ModelConfig {
       this.config.forcePathStyle ??= true;
     }
   }
-}
+}

package/src/service.ts

@@ -1,19 +1,42 @@
 import { Agent } from 'node:https';
 
-import { S3, type CompletedPart, type CreateMultipartUploadRequest, GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3';
+import { type CompletedPart, type CreateMultipartUploadRequest, GetObjectCommand, PutObjectCommand, S3 } from '@aws-sdk/client-s3';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 import type { MetadataBearer } from '@aws-sdk/types';
 import { NodeHttpHandler } from '@smithy/node-http-handler';
-import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 
+import { Injectable, PostConstruct } from '@travetto/di';
 import {
-  type ModelCrudSupport, type ModelStorageSupport, type ModelType, ModelRegistryIndex, ExistsError, NotFoundError, type OptionalId,
-  type ModelBlobSupport, type ModelExpirySupport, ModelCrudUtil, ModelExpiryUtil, ModelStorageUtil,
-  type ModelListOptions
+  ExistsError,
+  type ModelBlobSupport,
+  type ModelCrudSupport,
+  ModelCrudUtil,
+  type ModelExpirySupport,
+  ModelExpiryUtil,
+  type ModelListOptions,
+  ModelRegistryIndex,
+  type ModelStorageSupport,
+  ModelStorageUtil,
+  type ModelType,
+  NotFoundError,
+  type OptionalId
 } from '@travetto/model';
-import { Injectable, PostConstruct } from '@travetto/di';
 import {
-  type Class, RuntimeError, castTo, asFull, type BinaryMetadata, type ByteRange, type BinaryType,
-  BinaryUtil, type TimeSpan, TimeUtil, type BinaryArray, CodecUtil, BinaryMetadataUtil, TypedObject, JSONUtil
+  asFull,
+  type BinaryArray,
+  type BinaryMetadata,
+  BinaryMetadataUtil,
+  type BinaryType,
+  BinaryUtil,
+  type ByteRange,
+  type Class,
+  CodecUtil,
+  castTo,
+  JSONUtil,
+  RuntimeError,
+  type TimeSpan,
+  TimeUtil,
+  TypedObject
 } from '@travetto/runtime';
 
 import type { S3ModelConfig } from './config.ts';
@@ -26,7 +49,8 @@ function hasLowerContentType<T>(value: T): value is T & { contenttype?: string }
   return value !== undefined && value !== null && Object.hasOwn(value, 'contenttype');
 }
 
-type S3Metadata = Pick<CreateMultipartUploadRequest,
+type S3Metadata = Pick<
+  CreateMultipartUploadRequest,
   'ContentType' | 'Metadata' | 'ContentEncoding' | 'ContentLanguage' | 'CacheControl' | 'ContentDisposition'
 >;
 
@@ -35,12 +59,13 @@ type S3Metadata = Pick<CreateMultipartUploadRequest,
  */
 @Injectable()
 export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, ModelStorageSupport, ModelExpirySupport {
-
   idSource = ModelCrudUtil.uuidSource();
   client: S3;
   config: S3ModelConfig;
 
-  constructor(config: S3ModelConfig) { this.config = config; }
+  constructor(config: S3ModelConfig) {
+    this.config = config;
+  }
 
   #getMetadata(metadata: BinaryMetadata): S3Metadata {
     return {
@@ -49,8 +74,7 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
       ...(metadata.contentLanguage ? { ContentLanguage: metadata.contentLanguage } : {}),
       ...(metadata.cacheControl ? { CacheControl: metadata.cacheControl } : {}),
       Metadata: TypedObject.fromEntries(
-        TypedObject.entries(metadata)
-          .map(([key, value]) => [key, typeof value === 'string' ? value : JSONUtil.toUTF8(value)] as const)
+        TypedObject.entries(metadata).map(([key, value]) => [key, typeof value === 'string' ? value : JSONUtil.toUTF8(value)] as const)
       )
     };
   }
@@ -74,12 +98,12 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
     return this.#basicKey(key);
   }
 
-  #query<U extends object>(cls: string | Class, id: string, extra: U = asFull({})): (U & { Key: string, Bucket: string }) {
+  #query<U extends object>(cls: string | Class, id: string, extra: U = asFull({})): U & { Key: string; Bucket: string } {
     const key = this.#resolveKey(cls, id);
     return { Key: key, Bucket: this.config.bucket, ...extra };
   }
 
-  #queryBlob<U extends object>(id: string, extra: U = asFull({})): (U & { Key: string, Bucket: string }) {
+  #queryBlob<U extends object>(id: string, extra: U = asFull({})): U & { Key: string; Bucket: string } {
     const key = this.#basicKey(id);
     return { Key: key, Bucket: this.config.bucket, ...extra };
   }
@@ -94,12 +118,12 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
     return {};
   }
 
-  async * #iterateBucket(cls?: string | Class, options?: ModelListOptions): AsyncIterable<{ Key: string, id: string }[]> {
+  async *#iterateBucket(cls?: string | Class, options?: ModelListOptions): AsyncIterable<{ Key: string; id: string }[]> {
     let Marker: string | undefined;
     const batchSize = options?.batchSizeHint ?? 100;
     const maxCount = options?.limit ?? Number.MAX_SAFE_INTEGER;
     let produced = 0;
-    for (; !(options?.abort?.aborted) && produced < maxCount;) {
+    for (; !options?.abort?.aborted && produced < maxCount; ) {
       const items = await this.client.listObjects({
         Bucket: this.config.bucket,
         Prefix: cls ? this.#resolveKey(cls) : this.config.namespace,
@@ -134,12 +158,16 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
     let total = 0;
     let i = 1;
     const flush = async (): Promise<void> => {
-      if (!total) { return; }
-      const part = await this.client.uploadPart(this.#queryBlob(id, {
-        Body: BinaryUtil.binaryArrayToUint8Array(BinaryUtil.combineBinaryArrays(buffers)),
-        PartNumber: i,
-        UploadId
-      }));
+      if (!total) {
+        return;
+      }
+      const part = await this.client.uploadPart(
+        this.#queryBlob(id, {
+          Body: BinaryUtil.binaryArrayToUint8Array(BinaryUtil.combineBinaryArrays(buffers)),
+          PartNumber: i,
+          UploadId
+        })
+      );
       parts.push({ PartNumber: i, ETag: part.ETag });
       i += 1;
       buffers = [];
@@ -156,10 +184,12 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
       }
       await flush();
 
-      await this.client.completeMultipartUpload(this.#queryBlob(id, {
-        UploadId,
-        MultipartUpload: { Parts: parts }
-      }));
+      await this.client.completeMultipartUpload(
+        this.#queryBlob(id, {
+          UploadId,
+          MultipartUpload: { Parts: parts }
+        })
+      );
     } catch (error) {
       await this.client.abortMultipartUpload(this.#queryBlob(id, { UploadId }));
       throw error;
@@ -193,14 +223,18 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
   async initializeClient(): Promise<void> {
     this.client = new S3({
       ...this.config.config,
-      ...('requestHandler' in this.config.config ? {
-        requestHandler: new NodeHttpHandler({
-          ...this.config.config.requestHandler,
-          ...('httpsAgent' in this.config.config.requestHandler! ? {
-            httpsAgent: new Agent({ ...this.config.config.requestHandler?.httpsAgent ?? {} }),
-          } : {})
-        }),
-      } : {})
+      ...('requestHandler' in this.config.config
+        ? {
+            requestHandler: new NodeHttpHandler({
+              ...this.config.config.requestHandler,
+              ...('httpsAgent' in this.config.config.requestHandler!
+                ? {
+                    httpsAgent: new Agent({ ...(this.config.config.requestHandler?.httpsAgent ?? {}) })
+                  }
+                : {})
+            })
+          }
+        : {})
     });
     ModelStorageUtil.storageInitialization(this);
   }
@@ -243,10 +277,8 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
       }
       throw new NotFoundError(cls, id);
     } catch (error) {
-      if (isMetadataBearer(error)) {
-        if (error.$metadata.httpStatusCode === 404) {
-          error = new NotFoundError(cls, id);
-        }
+      if (isMetadataBearer(error) && error.$metadata.httpStatusCode === 404) {
+        throw new NotFoundError(cls, id);
       }
       throw error;
     }
@@ -258,12 +290,14 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
       prepped = await ModelCrudUtil.preStore(cls, item, this);
     }
     const content = JSONUtil.toBinaryArray(prepped);
-    await this.client.putObject(this.#query(cls, prepped.id, {
-      Body: BinaryUtil.binaryArrayToUint8Array(content),
-      ContentType: 'application/json',
-      ContentLength: content.byteLength,
-      ...this.#getExpiryConfig(cls, prepped)
-    }));
+    await this.client.putObject(
+      this.#query(cls, prepped.id, {
+        Body: BinaryUtil.binaryArrayToUint8Array(content),
+        ContentType: 'application/json',
+        ContentLength: content.byteLength,
+        ...this.#getExpiryConfig(cls, prepped)
+      })
+    );
     return prepped;
   }
 
@@ -304,7 +338,7 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
     await this.client.deleteObject(this.#query(cls, id));
   }
 
-  async * list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]> {
+  async *list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]> {
     for await (const batch of this.#iterateBucket(cls, options)) {
       yield ModelCrudUtil.filterOutNotFound(batch.map(item => this.get(cls, item.id)));
     }
@@ -317,7 +351,13 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
 
   // Blob support
   async upsertBlob(location: string, input: BinaryType, metadata?: BinaryMetadata, overwrite = true): Promise<void> {
-    if (!overwrite && await this.getBlobMetadata(location).then(() => true, () => false)) {
+    if (
+      !overwrite &&
+      (await this.getBlobMetadata(location).then(
+        () => true,
+        () => false
+      ))
+    ) {
       return;
     }
 
@@ -325,11 +365,12 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
 
     const length = BinaryMetadataUtil.readLength(resolved);
 
-    if (length && length < this.config.chunkSize) { // If smaller than chunk size
+    if (length && length < this.config.chunkSize) {
+      // If smaller than chunk size
       const blob = this.#queryBlob(location, {
         Body: BinaryUtil.toReadable(input),
         ContentLength: length,
-        ...this.#getMetadata(resolved),
+        ...this.#getMetadata(resolved)
       });
       // Upload to s3
       await this.client.putObject(blob);
@@ -340,18 +381,26 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
 
   async #getObject(location: string, range?: Required<ByteRange>): Promise<BinaryType> {
     // Read from s3
-    const result = await this.client.getObject(this.#queryBlob(location, range ? {
-      Range: `bytes=${range.start}-${range.end}`
-    } : {}));
+    const result = await this.client.getObject(
+      this.#queryBlob(
+        location,
+        range
+          ? {
+              Range: `bytes=${range.start}-${range.end}`
+            }
+          : {}
+      )
+    );
 
     const body: BinaryType | string | undefined = castTo(result.Body);
 
     switch (typeof body) {
-      case 'undefined': throw new RuntimeError('Unable to read type: undefined');
-      case 'string': return body.endsWith('=') ?
-        CodecUtil.fromBase64String(body) :
-        CodecUtil.fromUTF8String(body);
-      default: return body;
+      case 'undefined':
+        throw new RuntimeError('Unable to read type: undefined');
+      case 'string':
+        return body.endsWith('=') ? CodecUtil.fromBase64String(body) : CodecUtil.fromUTF8String(body);
+      default:
+        return body;
     }
   }
 
@@ -361,15 +410,13 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
     return BinaryMetadataUtil.makeBlob(() => this.#getObject(location, final), { ...metadata, range: final });
   }
 
-  async headBlob(location: string): Promise<{ Metadata?: BinaryMetadata, ContentLength?: number, ContentType?: string }> {
+  async headBlob(location: string): Promise<{ Metadata?: BinaryMetadata; ContentLength?: number; ContentType?: string }> {
     const query = this.#queryBlob(location);
     try {
-      return (await this.client.headObject(query));
+      return await this.client.headObject(query);
     } catch (error) {
-      if (isMetadataBearer(error)) {
-        if (error.$metadata.httpStatusCode === 404) {
-          error = new NotFoundError('Blob', location);
-        }
+      if (isMetadataBearer(error) && error.$metadata.httpStatusCode === 404) {
+        throw new NotFoundError('Blob', location);
       }
       throw error;
     }
@@ -382,11 +429,11 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
       const metadata: BinaryMetadata = {
         contentType: blob.ContentType ?? '',
         ...blob.Metadata,
-        size: blob.ContentLength!,
+        size: blob.ContentLength!
       };
       if (hasLowerContentType(metadata)) {
-        metadata['contentType'] = metadata['contenttype']!;
-        delete metadata['contenttype'];
+        metadata.contentType = metadata.contenttype!;
+        delete metadata.contenttype;
       }
       return metadata;
     } else {
@@ -419,11 +466,9 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
         return `${baseUrl}/${key}`;
       }
     }
-    return await getSignedUrl(
-      this.client,
-      new GetObjectCommand(this.#queryBlob(location)),
-      { expiresIn: TimeUtil.duration(expiresIn, 's') }
-    );
+    return await getSignedUrl(this.client, new GetObjectCommand(this.#queryBlob(location)), {
+      expiresIn: TimeUtil.duration(expiresIn, 's')
+    });
   }
 
   async getBlobWriteUrl(location: string, metadata: BinaryMetadata, expiresIn: TimeSpan = '1h'): Promise<string> {
@@ -434,7 +479,7 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
         ...this.#queryBlob(location),
         ...base,
         ...(metadata.size ? { ContentLength: metadata.size } : {}),
-        ...((metadata.hash && metadata.hash !== '-1') ? { ChecksumSHA256: metadata.hash } : {}),
+        ...(metadata.hash && metadata.hash !== '-1' ? { ChecksumSHA256: metadata.hash } : {})
       }),
       {
         expiresIn: TimeUtil.duration(expiresIn, 's'),
@@ -467,4 +512,4 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
       await this.client.deleteBucket({ Bucket: this.config.bucket });
     }
   }
-}
+}

package/support/service.s3.ts

@@ -11,4 +11,4 @@ export const service: ServiceDescriptor = {
   env: {
     COM_ADOBE_TESTING_S3MOCK_STORE_INITIAL_BUCKETS: 'app'
   }
-};
+};