@travetto/model-s3 2.0.2 → 2.1.0

package/README.md

@@ -39,7 +39,7 @@ import * as S3 from '@aws-sdk/client-s3';
 
 import { EnvUtil } from '@travetto/boot';
 import { Config } from '@travetto/config';
-import { Field } from '@travetto/schema';
+import { Field, Required } from '@travetto/schema';
 
 /**
  * S3 Support as an Asset Source
@@ -55,7 +55,8 @@ export class S3ModelConfig {
   secretAccessKey = EnvUtil.get('AWS_SECRET_ACCESS_KEY', '');
 
   @Field(Object)
-  config?: S3.S3ClientConfig; // Additional s3 config
+  @Required(false)
+  config: S3.S3ClientConfig; // Additional s3 config
 
   chunkSize = 5 * 2 ** 20; // Chunk size in bytes
 
@@ -80,6 +81,7 @@ export class S3ModelConfig {
 
     this.config = {
       ...(this.config ?? {}),
+      region: this.region,
       endpoint: this.endpoint,
       credentials: {
         accessKeyId: this.accessKeyId,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@travetto/model-s3",
   "displayName": "S3 Model Support",
-  "version": "2.0.2",
+  "version": "2.1.0",
   "description": "S3 backing for the travetto model module.",
   "keywords": [
     "s3",
@@ -25,10 +25,10 @@
     "directory": "module/model-s3"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.18.0",
-    "@aws-sdk/credential-provider-ini": "^3.18.0",
-    "@travetto/config": "^2.0.2",
-    "@travetto/model": "^2.0.2"
+    "@aws-sdk/client-s3": "^3.49.0",
+    "@aws-sdk/credential-provider-ini": "^3.49.0",
+    "@travetto/config": "^2.1.0",
+    "@travetto/model": "^2.1.0"
   },
   "publishConfig": {
     "access": "public"

package/src/config.ts

@@ -3,7 +3,7 @@ import * as S3 from '@aws-sdk/client-s3';
 
 import { EnvUtil } from '@travetto/boot';
 import { Config } from '@travetto/config';
-import { Field } from '@travetto/schema';
+import { Field, Required } from '@travetto/schema';
 
 /**
  * S3 Support as an Asset Source
@@ -19,7 +19,8 @@ export class S3ModelConfig {
   secretAccessKey = EnvUtil.get('AWS_SECRET_ACCESS_KEY', '');
 
   @Field(Object)
-  config?: S3.S3ClientConfig; // Additional s3 config
+  @Required(false)
+  config: S3.S3ClientConfig; // Additional s3 config
 
   chunkSize = 5 * 2 ** 20; // Chunk size in bytes
 
@@ -44,6 +45,7 @@ export class S3ModelConfig {
 
     this.config = {
       ...(this.config ?? {}),
+      region: this.region,
       endpoint: this.endpoint,
       credentials: {
         accessKeyId: this.accessKeyId,

package/src/service.ts

@@ -4,7 +4,7 @@ import type { MetadataBearer } from '@aws-sdk/types';
 import { StreamUtil } from '@travetto/boot';
 import {
   ModelCrudSupport, ModelStreamSupport, ModelStorageSupport, StreamMeta,
-  ModelType, ModelRegistry, ExistsError, NotFoundError, SubTypeNotSupportedError, OptionalId
+  ModelType, ModelRegistry, ExistsError, NotFoundError, OptionalId
 } from '@travetto/model';
 import { Injectable } from '@travetto/di';
 import { Class, AppError, Util } from '@travetto/base';
@@ -23,6 +23,8 @@ function hasContenttype<T>(o: T): o is T & { contenttype?: string } {
   return o && 'contenttype' in o;
 }
 
+const STREAM_SPACE = '@trv:stream';
+
 /**
  * Asset source backed by S3
  */
@@ -34,9 +36,15 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
   constructor(public readonly config: S3ModelConfig) { }
 
   #resolveKey(cls: Class | string, id?: string) {
-    let key = typeof cls === 'string' ? cls : ModelRegistry.getStore(cls);
-    if (id) {
-      key = `${key}:${id}`;
+    let key: string;
+    if (cls === STREAM_SPACE) { // If we are streaming, treat as primary use case
+      key = id!; // Store it directly at root
+    } else {
+      key = typeof cls === 'string' ? cls : ModelRegistry.getStore(cls);
+      if (id) {
+        key = `${key}:${id}`;
+      }
+      key = `_data/${key}`; // Separate data
     }
     if (this.config.namespace) {
       key = `${this.config.namespace}/${key}`;
@@ -78,7 +86,7 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
    * Write multipart file upload, in chunks
    */
   async #writeMultipart(id: string, input: NodeJS.ReadableStream, meta: StreamMeta): Promise<void> {
-    const { UploadId } = await this.client.createMultipartUpload(this.#q('_stream', id, {
+    const { UploadId } = await this.client.createMultipartUpload(this.#q(STREAM_SPACE, id, {
       ContentType: meta.contentType,
       ContentLength: meta.size,
     }));
@@ -89,7 +97,7 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
     let n = 1;
     const flush = async () => {
       if (!total) { return; }
-      const part = await this.client.uploadPart(this.#q('_stream', id, {
+      const part = await this.client.uploadPart(this.#q(STREAM_SPACE, id, {
         Body: Buffer.concat(buffers),
         PartNumber: n,
         UploadId
@@ -109,12 +117,12 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
       }
       await flush();
 
-      await this.client.completeMultipartUpload(this.#q('_stream', id, {
+      await this.client.completeMultipartUpload(this.#q(STREAM_SPACE, id, {
         UploadId,
         MultipartUpload: { Parts: parts }
       }));
     } catch (e) {
-      await this.client.abortMultipartUpload(this.#q('_stream', id, { UploadId }));
+      await this.client.abortMultipartUpload(this.#q(STREAM_SPACE, id, { UploadId }));
       throw e;
     }
   }
@@ -213,9 +221,7 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
   }
 
   async update<T extends ModelType>(cls: Class<T>, item: T) {
-    if (ModelRegistry.get(cls).subType) {
-      throw new SubTypeNotSupportedError(cls);
-    }
+    ModelCrudUtil.ensureNotSubType(cls);
     if (!(await this.head(cls, item.id))) {
       throw new NotFoundError(cls, item.id);
     }
@@ -223,25 +229,19 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
   }
 
   async upsert<T extends ModelType>(cls: Class<T>, item: OptionalId<T>) {
-    if (ModelRegistry.get(cls).subType) {
-      throw new SubTypeNotSupportedError(cls);
-    }
+    ModelCrudUtil.ensureNotSubType(cls);
     return this.store(cls, item);
   }
 
   async updatePartial<T extends ModelType>(cls: Class<T>, item: Partial<T> & { id: string }, view?: string) {
-    if (ModelRegistry.get(cls).subType) {
-      throw new SubTypeNotSupportedError(cls);
-    }
+    ModelCrudUtil.ensureNotSubType(cls);
     const id = item.id;
     item = await ModelCrudUtil.naivePartialUpdate(cls, item, view, () => this.get(cls, id)) as T;
     return this.store<T>(cls, item as T, false);
   }
 
   async delete<T extends ModelType>(cls: Class<T>, id: string) {
-    if (ModelRegistry.get(cls).subType) {
-      throw new SubTypeNotSupportedError(cls);
-    }
+    ModelCrudUtil.ensureNotSubType(cls);
     if (!(await this.head(cls, id))) {
       throw new NotFoundError(cls, id);
     }
@@ -270,7 +270,7 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
   async upsertStream(location: string, input: NodeJS.ReadableStream, meta: StreamMeta) {
     if (meta.size < this.config.chunkSize) { // If bigger than 5 mb
       // Upload to s3
-      await this.client.putObject(this.#q('_stream', location, {
+      await this.client.putObject(this.#q(STREAM_SPACE, location, {
         Body: await StreamUtil.toBuffer(input),
         ContentType: meta.contentType,
         ContentLength: meta.size,
@@ -286,7 +286,7 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
 
   async getStream(location: string) {
     // Read from s3
-    const res = await this.client.getObject(this.#q('_stream', location));
+    const res = await this.client.getObject(this.#q(STREAM_SPACE, location));
     if (res.Body instanceof Buffer || // Buffer
       typeof res.Body === 'string' || // string
       res.Body && ('pipe' in res.Body) // Stream
@@ -297,13 +297,13 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
   }
 
   async headStream(location: string) {
-    const query = this.#q('_stream', location);
+    const query = this.#q(STREAM_SPACE, location);
     try {
       return await this.client.headObject(query);
     } catch (e) {
       if (isMetadataBearer(e)) {
         if (e.$metadata.httpStatusCode === 404) {
-          e = new NotFoundError('_stream', location);
+          e = new NotFoundError(STREAM_SPACE, location);
         }
       }
       throw e;
@@ -324,7 +324,7 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
       }
       return ret;
     } else {
-      throw new NotFoundError('_stream', location);
+      throw new NotFoundError(STREAM_SPACE, location);
     }
   }
 
@@ -335,7 +335,7 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
   }
 
   async deleteStream(location: string) {
-    await this.client.deleteObject(this.#q('_stream', location));
+    await this.client.deleteObject(this.#q(STREAM_SPACE, location));
   }
 
   async createStorage() {