@travetto/model-s3 5.0.0-rc.9 → 5.0.1

package/README.md

@@ -17,7 +17,7 @@ This module provides an [s3](https://aws.amazon.com/documentation/s3/)-based imp
 
 Supported features:
    *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/service/crud.ts#L11)
-   *  [Streaming](https://github.com/travetto/travetto/tree/main/module/model/src/service/stream.ts#L3)
+   *  [Blob](https://github.com/travetto/travetto/tree/main/module/model/src/service/blob.ts#L8)
    *  [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/service/expiry.ts#L11)
 Out of the box, by installing the module, everything should be wired up by default.If you need to customize any aspect of the source or config, you can override and register it with the [Dependency Injection](https://github.com/travetto/travetto/tree/main/module/di#readme "Dependency registration/management and injection support.") module.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-s3",
-  "version": "5.0.0-rc.9",
+  "version": "5.0.1",
   "description": "S3 backing for the travetto model module.",
   "keywords": [
     "s3",
@@ -25,13 +25,13 @@
     "directory": "module/model-s3"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.609.0",
+    "@aws-sdk/client-s3": "^3.637.0",
     "@aws-sdk/credential-provider-ini": "^3.609.0",
-    "@travetto/config": "^5.0.0-rc.9",
-    "@travetto/model": "^5.0.0-rc.9"
+    "@travetto/config": "^5.0.1",
+    "@travetto/model": "^5.0.1"
   },
   "peerDependencies": {
-    "@travetto/command": "^5.0.0-rc.9"
+    "@travetto/command": "^5.0.1"
   },
   "peerDependenciesMeta": {
     "@travetto/command": {

package/src/service.ts

@@ -1,5 +1,5 @@
 import { Readable } from 'node:stream';
-import { buffer as toBuffer, text as toText } from 'node:stream/consumers';
+import { text as toText } from 'node:stream/consumers';
 import { Agent } from 'node:https';
 
 import { S3, CompletedPart, type CreateMultipartUploadRequest } from '@aws-sdk/client-s3';
@@ -7,20 +7,19 @@ import type { MetadataBearer } from '@aws-sdk/types';
 import { NodeHttpHandler } from '@smithy/node-http-handler';
 
 import {
-  ModelCrudSupport, ModelStreamSupport, ModelStorageSupport, StreamMeta,
-  ModelType, ModelRegistry, ExistsError, NotFoundError, OptionalId,
-  StreamRange
+  ModelCrudSupport, ModelStorageSupport, ModelType, ModelRegistry, ExistsError, NotFoundError, OptionalId,
+  ModelBlobSupport
 } from '@travetto/model';
 import { Injectable } from '@travetto/di';
-import { Class, AppError, castTo, asFull } from '@travetto/runtime';
+import { Class, AppError, castTo, asFull, BlobMeta, ByteRange, BinaryInput, BinaryUtil } from '@travetto/runtime';
 
+import { MODEL_BLOB, ModelBlobUtil } from '@travetto/model/src/internal/service/blob';
 import { ModelCrudUtil } from '@travetto/model/src/internal/service/crud';
 import { ModelExpirySupport } from '@travetto/model/src/service/expiry';
 import { ModelExpiryUtil } from '@travetto/model/src/internal/service/expiry';
 import { ModelStorageUtil } from '@travetto/model/src/internal/service/storage';
 
 import { S3ModelConfig } from './config';
-import { enforceRange } from '@travetto/model/src/internal/service/stream';
 
 function isMetadataBearer(o: unknown): o is MetadataBearer {
   return !!o && typeof o === 'object' && '$metadata' in o;
@@ -30,8 +29,6 @@ function hasContentType<T>(o: T): o is T & { contenttype?: string } {
   return o !== undefined && o !== null && Object.hasOwn(o, 'contenttype');
 }
 
-const STREAM_SPACE = '@travetto/model-s3:stream';
-
 type MetaBase = Pick<CreateMultipartUploadRequest,
   'ContentType' | 'Metadata' | 'ContentEncoding' | 'ContentLanguage' | 'CacheControl' | 'ContentDisposition'
 >;
@@ -40,14 +37,14 @@ type MetaBase = Pick<CreateMultipartUploadRequest,
  * Asset source backed by S3
  */
 @Injectable()
-export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, ModelStorageSupport, ModelExpirySupport {
+export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, ModelStorageSupport, ModelExpirySupport {
 
   idSource = ModelCrudUtil.uuidSource();
   client: S3;
 
   constructor(public readonly config: S3ModelConfig) { }
 
-  #getMetaBase(meta: StreamMeta): MetaBase {
+  #getMetaBase({ range, size, ...meta }: BlobMeta): MetaBase {
     return {
       ContentType: meta.contentType,
       ...(meta.contentEncoding ? { ContentEncoding: meta.contentEncoding } : {}),
@@ -55,14 +52,14 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
       ...(meta.cacheControl ? { CacheControl: meta.cacheControl } : {}),
       Metadata: {
         ...meta,
-        size: `${meta.size}`
+        ...(size ? { size: `${size}` } : {})
       }
     };
   }
 
   #resolveKey(cls: Class | string, id?: string): string {
     let key: string;
-    if (cls === STREAM_SPACE) { // If we are streaming, treat as primary use case
+    if (cls === MODEL_BLOB) { // If we are streaming, treat as primary use case
       key = id!; // Store it directly at root
     } else {
       key = typeof cls === 'string' ? cls : ModelRegistry.getStore(cls);
@@ -95,9 +92,13 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
   async * #iterateBucket(cls?: string | Class): AsyncIterable<{ Key: string, id: string }[]> {
     let Marker: string | undefined;
     for (; ;) {
-      const obs = await this.client.listObjects({ Bucket: this.config.bucket, Prefix: cls ? this.#resolveKey(cls) : undefined, Marker });
-      if (obs.Contents && obs.Contents.length) {
-        yield (obs.Contents ?? []).map(o => ({ Key: o.Key!, id: o.Key!.split(':').pop()! }));
+      const obs = await this.client.listObjects({
+        Bucket: this.config.bucket,
+        Prefix: cls ? this.#resolveKey(cls) : this.config.namespace,
+        Marker
+      });
+      if (obs.Contents?.length) {
+        yield obs.Contents.map(o => ({ Key: o.Key!, id: o.Key!.split(':').pop()! }));
       }
       if (obs.NextMarker) {
         Marker = obs.NextMarker;
@@ -110,8 +111,8 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
   /**
    * Write multipart file upload, in chunks
    */
-  async #writeMultipart(id: string, input: Readable, meta: StreamMeta): Promise<void> {
-    const { UploadId } = await this.client.createMultipartUpload(this.#q(STREAM_SPACE, id, this.#getMetaBase(meta)));
+  async #writeMultipart(id: string, input: Readable, meta: BlobMeta): Promise<void> {
+    const { UploadId } = await this.client.createMultipartUpload(this.#q(MODEL_BLOB, id, this.#getMetaBase(meta)));
 
     const parts: CompletedPart[] = [];
     let buffers: Buffer[] = [];
@@ -119,7 +120,7 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
     let n = 1;
     const flush = async (): Promise<void> => {
       if (!total) { return; }
-      const part = await this.client.uploadPart(this.#q(STREAM_SPACE, id, {
+      const part = await this.client.uploadPart(this.#q(MODEL_BLOB, id, {
         Body: Buffer.concat(buffers),
         PartNumber: n,
         UploadId
@@ -131,38 +132,32 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
     };
     try {
       for await (const chunk of input) {
-        buffers.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
-        total += chunk.length;
+        const chunked = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+        buffers.push(chunked);
+        total += chunked.length;
         if (total > this.config.chunkSize) {
           await flush();
         }
       }
       await flush();
 
-      await this.client.completeMultipartUpload(this.#q(STREAM_SPACE, id, {
+      await this.client.completeMultipartUpload(this.#q(MODEL_BLOB, id, {
         UploadId,
         MultipartUpload: { Parts: parts }
       }));
     } catch (err) {
-      await this.client.abortMultipartUpload(this.#q(STREAM_SPACE, id, { UploadId }));
+      await this.client.abortMultipartUpload(this.#q(MODEL_BLOB, id, { UploadId }));
       throw err;
     }
   }
 
   async #deleteKeys(items: { Key: string }[]): Promise<void> {
-    if (this.config.endpoint.includes('localhost')) {
-      await Promise.all(items.map(item => this.client.deleteObject({
-        Bucket: this.config.bucket,
-        Key: item.Key
-      })));
-    } else {
-      await this.client.deleteObjects({
-        Bucket: this.config.bucket,
-        Delete: {
-          Objects: items
-        }
-      });
-    }
+    await this.client.deleteObjects({
+      Bucket: this.config.bucket,
+      Delete: {
+        Objects: items
+      }
+    });
   }
 
   async postConstruct(): Promise<void> {
@@ -265,7 +260,7 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
   async updatePartial<T extends ModelType>(cls: Class<T>, item: Partial<T> & { id: string }, view?: string): Promise<T> {
     ModelCrudUtil.ensureNotSubType(cls);
     const id = item.id;
-    const prepped = await ModelCrudUtil.naivePartialUpdate(cls, item, view, (): Promise<T> => this.get(cls, id));
+    const prepped = await ModelCrudUtil.naivePartialUpdate(cls, () => this.get(cls, id), item, view);
     return this.store<T>(cls, prepped, false);
   }
 
@@ -296,22 +291,29 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
     return -1;
   }
 
-  async upsertStream(location: string, input: Readable, meta: StreamMeta): Promise<void> {
-    if (meta.size < this.config.chunkSize) { // If smaller than chunk size
+  // Blob support
+  async upsertBlob(location: string, input: BinaryInput, meta?: BlobMeta, overwrite = true): Promise<void> {
+    if (!overwrite && await this.describeBlob(location).then(() => true, () => false)) {
+      return;
+    }
+
+    const [stream, blobMeta] = await ModelBlobUtil.getInput(input, meta);
+
+    if (blobMeta.size && blobMeta.size < this.config.chunkSize) { // If smaller than chunk size
       // Upload to s3
-      await this.client.putObject(this.#q(STREAM_SPACE, location, {
-        Body: await toBuffer(input),
-        ContentLength: meta.size,
-        ...this.#getMetaBase(meta),
+      await this.client.putObject(this.#q(MODEL_BLOB, location, {
+        Body: stream,
+        ContentLength: blobMeta.size,
+        ...this.#getMetaBase(blobMeta),
       }));
     } else {
-      await this.#writeMultipart(location, input, meta);
+      await this.#writeMultipart(location, stream, blobMeta);
     }
   }
 
-  async #getObject(location: string, range: Required<StreamRange>): Promise<Readable> {
+  async #getObject(location: string, range?: Required<ByteRange>): Promise<Readable> {
     // Read from s3
-    const res = await this.client.getObject(this.#q(STREAM_SPACE, location, range ? {
+    const res = await this.client.getObject(this.#q(MODEL_BLOB, location, range ? {
       Range: `bytes=${range.start}-${range.end}`
     } : {}));
 
@@ -329,33 +331,32 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
     throw new AppError(`Unable to read type: ${typeof res.Body}`);
   }
 
-  async getStream(location: string, range?: StreamRange): Promise<Readable> {
-    if (range) {
-      const meta = await this.describeStream(location);
-      range = enforceRange(range, meta.size);
-    }
-    return this.#getObject(location, castTo(range));
+  async getBlob(location: string, range?: ByteRange): Promise<Blob> {
+    const meta = await this.describeBlob(location);
+    const final = range ? ModelBlobUtil.enforceRange(range, meta.size!) : undefined;
+    const res = (): Promise<Readable> => this.#getObject(location, final);
+    return BinaryUtil.readableBlob(res, { ...meta, range: final });
   }
 
-  async headStream(location: string): Promise<{ Metadata?: Partial<StreamMeta>, ContentLength?: number }> {
-    const query = this.#q(STREAM_SPACE, location);
+  async headBlob(location: string): Promise<{ Metadata?: BlobMeta, ContentLength?: number }> {
+    const query = this.#q(MODEL_BLOB, location);
     try {
       return (await this.client.headObject(query));
     } catch (err) {
       if (isMetadataBearer(err)) {
         if (err.$metadata.httpStatusCode === 404) {
-          err = new NotFoundError(STREAM_SPACE, location);
+          err = new NotFoundError(MODEL_BLOB, location);
         }
       }
       throw err;
     }
   }
 
-  async describeStream(location: string): Promise<StreamMeta> {
-    const obj = await this.headStream(location);
+  async describeBlob(location: string): Promise<BlobMeta> {
+    const obj = await this.headBlob(location);
 
     if (obj) {
-      const ret: StreamMeta = {
+      const ret: BlobMeta = {
         contentType: '',
         ...obj.Metadata,
         size: obj.ContentLength!,
@@ -366,20 +367,21 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
       }
       return ret;
     } else {
-      throw new NotFoundError(STREAM_SPACE, location);
+      throw new NotFoundError(MODEL_BLOB, location);
     }
   }
 
+  async deleteBlob(location: string): Promise<void> {
+    await this.client.deleteObject(this.#q(MODEL_BLOB, location));
+  }
+
+  // Storage
   async truncateModel<T extends ModelType>(model: Class<T>): Promise<void> {
     for await (const items of this.#iterateBucket(model)) {
       await this.#deleteKeys(items);
     }
   }
 
-  async deleteStream(location: string): Promise<void> {
-    await this.client.deleteObject(this.#q(STREAM_SPACE, location));
-  }
-
   async createStorage(): Promise<void> {
     try {
       await this.client.headBucket({ Bucket: this.config.bucket });
@@ -390,7 +392,7 @@ export class S3ModelService implements ModelCrudSupport, ModelStreamSupport, Mod
 
   async deleteStorage(): Promise<void> {
     if (this.config.namespace) {
-      for await (const items of this.#iterateBucket('')) {
+      for await (const items of this.#iterateBucket()) {
         await this.#deleteKeys(items);
       }
     } else {