@travetto/model-s3 6.0.0-rc.1 → 6.0.0-rc.3

package/README.md

@@ -16,9 +16,10 @@ yarn add @travetto/model-s3
 This module provides an [s3](https://aws.amazon.com/documentation/s3/)-based implementation for the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.").  This source allows the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") module to read, write and stream against [s3](https://aws.amazon.com/documentation/s3/). 
 
 Supported features:
-   *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/service/crud.ts#L11)
-   *  [Blob](https://github.com/travetto/travetto/tree/main/module/model/src/service/blob.ts#L8)
-   *  [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/service/expiry.ts#L11)
+   *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L11)
+   *  [Blob](https://github.com/travetto/travetto/tree/main/module/model/src/types/blob.ts#L8)
+   *  [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10)
+
 Out of the box, by installing the module, everything should be wired up by default.If you need to customize any aspect of the source or config, you can override and register it with the [Dependency Injection](https://github.com/travetto/travetto/tree/main/module/di#readme "Dependency registration/management and injection support.") module.
 
 **Code: Wiring up a custom Model Source**
@@ -40,16 +41,6 @@ where the [S3ModelConfig](https://github.com/travetto/travetto/tree/main/module/
 
 **Code: Structure of S3ModelConfig**
 ```typescript
-import { fromIni } from '@aws-sdk/credential-provider-ini';
-import type s3 from '@aws-sdk/client-s3';
-
-import { Config, EnvVar } from '@travetto/config';
-import { Field, Required } from '@travetto/schema';
-import { Runtime } from '@travetto/runtime';
-
-/**
- * S3 Support as an Asset Source
- */
 @Config('model.s3')
 export class S3ModelConfig {
   region = 'us-east-1'; // AWS Region

package/__index__.ts

@@ -1,2 +1,2 @@
-export * from './src/config';
-export * from './src/service';
+export * from './src/config.ts';
+export * from './src/service.ts';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-s3",
-  "version": "6.0.0-rc.1",
+  "version": "6.0.0-rc.3",
   "description": "S3 backing for the travetto model module.",
   "keywords": [
     "s3",
@@ -25,12 +25,12 @@
     "directory": "module/model-s3"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.740.0",
-    "@aws-sdk/credential-provider-ini": "^3.734.0",
-    "@aws-sdk/s3-request-presigner": "^3.740.0",
-    "@travetto/cli": "^6.0.0-rc.1",
-    "@travetto/config": "^6.0.0-rc.1",
-    "@travetto/model": "^6.0.0-rc.1"
+    "@aws-sdk/client-s3": "^3.796.0",
+    "@aws-sdk/credential-provider-ini": "^3.796.0",
+    "@aws-sdk/s3-request-presigner": "^3.796.0",
+    "@travetto/cli": "^6.0.0-rc.3",
+    "@travetto/config": "^6.0.0-rc.2",
+    "@travetto/model": "^6.0.0-rc.2"
   },
   "travetto": {
     "displayName": "S3 Model Support"

package/src/service.ts

@@ -9,18 +9,13 @@ import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 
 import {
   ModelCrudSupport, ModelStorageSupport, ModelType, ModelRegistry, ExistsError, NotFoundError, OptionalId,
-  ModelBlobSupport
+  ModelBlobSupport, ModelExpirySupport, ModelBlobUtil, ModelCrudUtil, ModelExpiryUtil, ModelStorageUtil,
+
 } from '@travetto/model';
 import { Injectable } from '@travetto/di';
 import { Class, AppError, castTo, asFull, BlobMeta, ByteRange, BinaryInput, BinaryUtil, TimeSpan, TimeUtil } from '@travetto/runtime';
 
-import { MODEL_BLOB, ModelBlobUtil } from '@travetto/model/src/internal/service/blob';
-import { ModelCrudUtil } from '@travetto/model/src/internal/service/crud';
-import { ModelExpirySupport } from '@travetto/model/src/service/expiry';
-import { ModelExpiryUtil } from '@travetto/model/src/internal/service/expiry';
-import { ModelStorageUtil } from '@travetto/model/src/internal/service/storage';
-
-import { S3ModelConfig } from './config';
+import { S3ModelConfig } from './config.ts';
 
 function isMetadataBearer(o: unknown): o is MetadataBearer {
   return !!o && typeof o === 'object' && '$metadata' in o;
@@ -46,7 +41,7 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
 
   constructor(config: S3ModelConfig) { this.config = config; }
 
-  #getMetaBase({ range, size, ...meta }: BlobMeta): MetaBase {
+  #getMetaBase({ range: _, size, ...meta }: BlobMeta): MetaBase {
     return {
       ContentType: meta.contentType,
       ...(meta.contentEncoding ? { ContentEncoding: meta.contentEncoding } : {}),
@@ -59,17 +54,7 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
     };
   }
 
-  #resolveKey(cls: Class | string, id?: string): string {
-    let key: string;
-    if (cls === MODEL_BLOB) { // If we are streaming, treat as primary use case
-      key = id!; // Store it directly at root
-    } else {
-      key = typeof cls === 'string' ? cls : ModelRegistry.getStore(cls);
-      if (id) {
-        key = `${key}:${id}`;
-      }
-      key = `_data/${key}`; // Separate data
-    }
+  #basicKey(key: string): string {
     if (key?.startsWith('/')) {
       key = key.substring(1);
     }
@@ -79,11 +64,26 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
     return key;
   }
 
+  #resolveKey(cls: Class | string, id?: string): string {
+    let key = typeof cls === 'string' ? cls : ModelRegistry.getStore(cls);
+    if (id) {
+      key = `${key}:${id}`;
+    }
+    key = `_data/${key}`; // Separate data
+    return this.#basicKey(key);
+  }
+
   #q<U extends object>(cls: string | Class, id: string, extra: U = asFull({})): (U & { Key: string, Bucket: string }) {
     const key = this.#resolveKey(cls, id);
     return { Key: key, Bucket: this.config.bucket, ...extra };
   }
 
+  #qBlob<U extends object>(id: string, extra: U = asFull({})): (U & { Key: string, Bucket: string }) {
+    const key = this.#basicKey(id);
+    return { Key: key, Bucket: this.config.bucket, ...extra };
+  }
+
+
   #getExpiryConfig<T extends ModelType>(cls: Class<T>, item: T): { Expires?: Date } {
     if (ModelRegistry.get(cls).expiresAt) {
       const { expiresAt } = ModelExpiryUtil.getExpiryState(cls, item);
@@ -117,7 +117,7 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
    * Write multipart file upload, in chunks
    */
   async #writeMultipart(id: string, input: Readable, meta: BlobMeta): Promise<void> {
-    const { UploadId } = await this.client.createMultipartUpload(this.#q(MODEL_BLOB, id, this.#getMetaBase(meta)));
+    const { UploadId } = await this.client.createMultipartUpload(this.#qBlob(id, this.#getMetaBase(meta)));
 
     const parts: CompletedPart[] = [];
     let buffers: Buffer[] = [];
@@ -125,7 +125,7 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
     let n = 1;
     const flush = async (): Promise<void> => {
       if (!total) { return; }
-      const part = await this.client.uploadPart(this.#q(MODEL_BLOB, id, {
+      const part = await this.client.uploadPart(this.#qBlob(id, {
         Body: Buffer.concat(buffers),
         PartNumber: n,
         UploadId
@@ -146,12 +146,12 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
       }
       await flush();
 
-      await this.client.completeMultipartUpload(this.#q(MODEL_BLOB, id, {
+      await this.client.completeMultipartUpload(this.#qBlob(id, {
         UploadId,
         MultipartUpload: { Parts: parts }
       }));
     } catch (err) {
-      await this.client.abortMultipartUpload(this.#q(MODEL_BLOB, id, { UploadId }));
+      await this.client.abortMultipartUpload(this.#qBlob(id, { UploadId }));
       throw err;
     }
   }
@@ -196,9 +196,9 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
 
   async head<T extends ModelType>(cls: Class<T>, id: string): Promise<boolean> {
     try {
-      const res = await this.client.headObject(this.#q(cls, id));
+      const result = await this.client.headObject(this.#q(cls, id));
       const { expiresAt } = ModelRegistry.get(cls);
-      if (expiresAt && res.Expires && res.Expires.getTime() < Date.now()) {
+      if (expiresAt && result.Expires && result.Expires.getTime() < Date.now()) {
         return false;
       }
       return true;
@@ -308,7 +308,7 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
   }
 
   // Expiry
-  async deleteExpired<T extends ModelType>(cls: Class<T>): Promise<number> {
+  async deleteExpired<T extends ModelType>(_cls: Class<T>): Promise<number> {
     return -1;
   }
 
@@ -322,7 +322,7 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
 
     if (blobMeta.size && blobMeta.size < this.config.chunkSize) { // If smaller than chunk size
       // Upload to s3
-      await this.client.putObject(this.#q(MODEL_BLOB, location, {
+      await this.client.putObject(this.#qBlob(location, {
         Body: await toBuffer(stream),
         ContentLength: blobMeta.size,
         ...this.#getMetaBase(blobMeta),
@@ -334,39 +334,39 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
 
   async #getObject(location: string, range?: Required<ByteRange>): Promise<Readable> {
     // Read from s3
-    const res = await this.client.getObject(this.#q(MODEL_BLOB, location, range ? {
+    const result = await this.client.getObject(this.#qBlob(location, range ? {
       Range: `bytes=${range.start}-${range.end}`
     } : {}));
 
-    if (!res.Body) {
+    if (!result.Body) {
       throw new AppError('Unable to read type: undefined');
     }
 
-    if (typeof res.Body === 'string') { // string
-      return Readable.from(res.Body, { encoding: castTo<string>(res.Body).endsWith('=') ? 'base64' : 'utf8' });
-    } else if (res.Body instanceof Buffer) { // Buffer
-      return Readable.from(res.Body);
-    } else if ('pipe' in res.Body) { // Stream
-      return castTo<Readable>(res.Body);
+    if (typeof result.Body === 'string') { // string
+      return Readable.from(result.Body, { encoding: castTo<string>(result.Body).endsWith('=') ? 'base64' : 'utf8' });
+    } else if (result.Body instanceof Buffer) { // Buffer
+      return Readable.from(result.Body);
+    } else if ('pipe' in result.Body) { // Stream
+      return castTo<Readable>(result.Body);
     }
-    throw new AppError(`Unable to read type: ${typeof res.Body}`);
+    throw new AppError(`Unable to read type: ${typeof result.Body}`);
   }
 
   async getBlob(location: string, range?: ByteRange): Promise<Blob> {
     const meta = await this.getBlobMeta(location);
     const final = range ? ModelBlobUtil.enforceRange(range, meta.size!) : undefined;
-    const res = (): Promise<Readable> => this.#getObject(location, final);
-    return BinaryUtil.readableBlob(res, { ...meta, range: final });
+    const result = (): Promise<Readable> => this.#getObject(location, final);
+    return BinaryUtil.readableBlob(result, { ...meta, range: final });
   }
 
   async headBlob(location: string): Promise<{ Metadata?: BlobMeta, ContentLength?: number }> {
-    const query = this.#q(MODEL_BLOB, location);
+    const query = this.#qBlob(location);
     try {
       return (await this.client.headObject(query));
     } catch (err) {
       if (isMetadataBearer(err)) {
         if (err.$metadata.httpStatusCode === 404) {
-          err = new NotFoundError(MODEL_BLOB, location);
+          err = new NotFoundError('Blob', location);
         }
       }
       throw err;
@@ -388,19 +388,19 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
       }
       return ret;
     } else {
-      throw new NotFoundError(MODEL_BLOB, location);
+      throw new NotFoundError('Blob', location);
     }
   }
 
   async deleteBlob(location: string): Promise<void> {
-    await this.client.deleteObject(this.#q(MODEL_BLOB, location));
+    await this.client.deleteObject(this.#qBlob(location));
   }
 
   async updateBlobMeta(location: string, meta: BlobMeta): Promise<void> {
     await this.client.copyObject({
       Bucket: this.config.bucket,
-      Key: this.#resolveKey(MODEL_BLOB, location),
-      CopySource: `/${this.config.bucket}/${this.#resolveKey(MODEL_BLOB, location)}`,
+      Key: this.#basicKey(location),
+      CopySource: `/${this.config.bucket}/${this.#basicKey(location)}`,
       ...this.#getMetaBase(meta),
       MetadataDirective: 'REPLACE'
     });
@@ -410,7 +410,7 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
   async getBlobReadUrl(location: string, exp: TimeSpan = '1h'): Promise<string> {
     return await getSignedUrl(
       this.client,
-      new GetObjectCommand(this.#q(MODEL_BLOB, location)),
+      new GetObjectCommand(this.#qBlob(location)),
       { expiresIn: TimeUtil.asSeconds(exp) }
     );
   }
@@ -420,7 +420,7 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
     return await getSignedUrl(
       this.client,
       new PutObjectCommand({
-        ...this.#q(MODEL_BLOB, location),
+        ...this.#qBlob(location),
         ...base,
         ...(meta.size ? { ContentLength: meta.size } : {}),
         ...((meta.hash && meta.hash !== '-1') ? { ChecksumSHA256: meta.hash } : {}),