@travetto/model-s3 7.1.4 → 8.0.0-alpha.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-s3",
-  "version": "7.1.4",
+  "version": "8.0.0-alpha.0",
   "type": "module",
   "description": "S3 backing for the travetto model module.",
   "keywords": [
@@ -26,14 +26,14 @@
     "directory": "module/model-s3"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.966.0",
-    "@aws-sdk/credential-provider-ini": "^3.966.0",
-    "@aws-sdk/s3-request-presigner": "^3.966.0",
-    "@travetto/config": "^7.1.4",
-    "@travetto/model": "^7.1.4"
+    "@aws-sdk/client-s3": "^3.1000.0",
+    "@aws-sdk/credential-provider-ini": "^3.972.13",
+    "@aws-sdk/s3-request-presigner": "^3.1000.0",
+    "@travetto/config": "^8.0.0-alpha.0",
+    "@travetto/model": "^8.0.0-alpha.0"
   },
   "peerDependencies": {
-    "@travetto/cli": "^7.1.4"
+    "@travetto/cli": "^8.0.0-alpha.0"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/service.ts

@@ -1,5 +1,3 @@
-import { Readable } from 'node:stream';
-import { text as toText, buffer as toBuffer } from 'node:stream/consumers';
 import { Agent } from 'node:https';
 
 import { S3, type CompletedPart, type CreateMultipartUploadRequest, GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3';
@@ -9,12 +7,12 @@ import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 
 import {
   type ModelCrudSupport, type ModelStorageSupport, type ModelType, ModelRegistryIndex, ExistsError, NotFoundError, type OptionalId,
-  type ModelBlobSupport, type ModelExpirySupport, ModelBlobUtil, ModelCrudUtil, ModelExpiryUtil, ModelStorageUtil
+  type ModelBlobSupport, type ModelExpirySupport, ModelCrudUtil, ModelExpiryUtil, ModelStorageUtil
 } from '@travetto/model';
 import { Injectable } from '@travetto/di';
 import {
-  type Class, AppError, castTo, asFull, type BlobMeta,
-  type ByteRange, type BinaryInput, BinaryUtil, type TimeSpan, TimeUtil
+  type Class, RuntimeError, castTo, asFull, type BinaryMetadata, type ByteRange, type BinaryType,
+  BinaryUtil, type TimeSpan, TimeUtil, type BinaryArray, CodecUtil, BinaryMetadataUtil, TypedObject, JSONUtil
 } from '@travetto/runtime';
 
 import type { S3ModelConfig } from './config.ts';
@@ -27,7 +25,7 @@ function hasContentType<T>(value: T): value is T & { contenttype?: string } {
   return value !== undefined && value !== null && Object.hasOwn(value, 'contenttype');
 }
 
-type MetaBase = Pick<CreateMultipartUploadRequest,
+type S3Metadata = Pick<CreateMultipartUploadRequest,
   'ContentType' | 'Metadata' | 'ContentEncoding' | 'ContentLanguage' | 'CacheControl' | 'ContentDisposition'
 >;
 
@@ -43,16 +41,16 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
 
   constructor(config: S3ModelConfig) { this.config = config; }
 
-  #getMetaBase({ range: _, size, ...meta }: BlobMeta): MetaBase {
+  #getMetadata(metadata: BinaryMetadata): S3Metadata {
     return {
-      ContentType: meta.contentType,
-      ...(meta.contentEncoding ? { ContentEncoding: meta.contentEncoding } : {}),
-      ...(meta.contentLanguage ? { ContentLanguage: meta.contentLanguage } : {}),
-      ...(meta.cacheControl ? { CacheControl: meta.cacheControl } : {}),
-      Metadata: {
-        ...meta,
-        ...(size ? { size: `${size}` } : {})
-      }
+      ContentType: metadata.contentType,
+      ...(metadata.contentEncoding ? { ContentEncoding: metadata.contentEncoding } : {}),
+      ...(metadata.contentLanguage ? { ContentLanguage: metadata.contentLanguage } : {}),
+      ...(metadata.cacheControl ? { CacheControl: metadata.cacheControl } : {}),
+      Metadata: TypedObject.fromEntries(
+        TypedObject.entries(metadata)
+          .map(([key, value]) => [key, typeof value === 'string' ? value : JSONUtil.toUTF8(value)] as const)
+      )
     };
   }
 
@@ -117,17 +115,17 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
   /**
    * Write multipart file upload, in chunks
    */
-  async #writeMultipart(id: string, input: Readable, meta: BlobMeta): Promise<void> {
-    const { UploadId } = await this.client.createMultipartUpload(this.#queryBlob(id, this.#getMetaBase(meta)));
+  async #writeMultipart(id: string, input: BinaryType, metadata: BinaryMetadata): Promise<void> {
+    const { UploadId } = await this.client.createMultipartUpload(this.#queryBlob(id, this.#getMetadata(metadata)));
 
     const parts: CompletedPart[] = [];
-    let buffers: Buffer[] = [];
+    let buffers: BinaryArray[] = [];
     let total = 0;
     let i = 1;
     const flush = async (): Promise<void> => {
       if (!total) { return; }
       const part = await this.client.uploadPart(this.#queryBlob(id, {
-        Body: Buffer.concat(buffers),
+        Body: BinaryUtil.binaryArrayToUint8Array(BinaryUtil.combineBinaryArrays(buffers)),
         PartNumber: i,
         UploadId
       }));
@@ -137,10 +135,10 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
       total = 0;
     };
     try {
-      for await (const chunk of input) {
-        const chunked = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+      for await (const chunk of BinaryUtil.toBinaryStream(input)) {
+        const chunked = CodecUtil.readUtf8Chunk(chunk);
         buffers.push(chunked);
-        total += chunked.length;
+        total += chunked.byteLength;
         if (total > this.config.chunkSize) {
           await flush();
         }
@@ -217,7 +215,7 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
     try {
       const result = await this.client.getObject(this.#query(cls, id));
       if (result.Body) {
-        const body = await toText(castTo(result.Body));
+        const body = await BinaryUtil.toBinaryArray(result.Body);
         const output = await ModelCrudUtil.load(cls, body);
         if (output) {
           const { expiresAt } = ModelRegistryIndex.getConfig(cls);
@@ -247,11 +245,11 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
     if (preStore) {
       prepped = await ModelCrudUtil.preStore(cls, item, this);
     }
-    const content = Buffer.from(JSON.stringify(prepped), 'utf8');
+    const content = JSONUtil.toBinaryArray(prepped);
     await this.client.putObject(this.#query(cls, prepped.id, {
-      Body: content,
+      Body: BinaryUtil.binaryArrayToUint8Array(content),
       ContentType: 'application/json',
-      ContentLength: content.length,
+      ContentLength: content.byteLength,
       ...this.#getExpiryConfig(cls, prepped)
     }));
     return prepped;
@@ -314,53 +312,52 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
   }
 
   // Blob support
-  async upsertBlob(location: string, input: BinaryInput, meta?: BlobMeta, overwrite = true): Promise<void> {
-    if (!overwrite && await this.getBlobMeta(location).then(() => true, () => false)) {
+  async upsertBlob(location: string, input: BinaryType, metadata?: BinaryMetadata, overwrite = true): Promise<void> {
+    if (!overwrite && await this.getBlobMetadata(location).then(() => true, () => false)) {
       return;
     }
 
-    const [stream, blobMeta] = await ModelBlobUtil.getInput(input, meta);
+    const resolved = await BinaryMetadataUtil.compute(input, metadata);
+
+    const length = BinaryMetadataUtil.readLength(resolved);
 
-    if (blobMeta.size && blobMeta.size < this.config.chunkSize) { // If smaller than chunk size
+    if (length && length < this.config.chunkSize) { // If smaller than chunk size
+      const blob = this.#queryBlob(location, {
+        Body: BinaryUtil.toReadable(input),
+        ContentLength: length,
+        ...this.#getMetadata(resolved),
+      });
       // Upload to s3
-      await this.client.putObject(this.#queryBlob(location, {
-        Body: await toBuffer(stream),
-        ContentLength: blobMeta.size,
-        ...this.#getMetaBase(blobMeta),
-      }));
+      await this.client.putObject(blob);
     } else {
-      await this.#writeMultipart(location, stream, blobMeta);
+      await this.#writeMultipart(location, input, resolved);
     }
   }
 
-  async #getObject(location: string, range?: Required<ByteRange>): Promise<Readable> {
+  async #getObject(location: string, range?: Required<ByteRange>): Promise<BinaryType> {
     // Read from s3
     const result = await this.client.getObject(this.#queryBlob(location, range ? {
       Range: `bytes=${range.start}-${range.end}`
     } : {}));
 
-    if (!result.Body) {
-      throw new AppError('Unable to read type: undefined');
-    }
+    const body: BinaryType | string | undefined = castTo(result.Body);
 
-    if (typeof result.Body === 'string') { // string
-      return Readable.from(result.Body, { encoding: castTo<string>(result.Body).endsWith('=') ? 'base64' : 'utf8' });
-    } else if (result.Body instanceof Buffer) { // Buffer
-      return Readable.from(result.Body);
-    } else if ('pipe' in result.Body) { // Stream
-      return castTo<Readable>(result.Body);
+    switch (typeof body) {
+      case 'undefined': throw new RuntimeError('Unable to read type: undefined');
+      case 'string': return body.endsWith('=') ?
+        CodecUtil.fromBase64String(body) :
+        CodecUtil.fromUTF8String(body);
+      default: return body;
     }
-    throw new AppError(`Unable to read type: ${typeof result.Body}`);
   }
 
   async getBlob(location: string, range?: ByteRange): Promise<Blob> {
-    const meta = await this.getBlobMeta(location);
-    const final = range ? ModelBlobUtil.enforceRange(range, meta.size!) : undefined;
-    const result = (): Promise<Readable> => this.#getObject(location, final);
-    return BinaryUtil.readableBlob(result, { ...meta, range: final });
+    const metadata = await this.getBlobMetadata(location);
+    const final = range ? BinaryMetadataUtil.enforceRange(range, metadata) : undefined;
+    return BinaryMetadataUtil.makeBlob(() => this.#getObject(location, final), { ...metadata, range: final });
   }
 
-  async headBlob(location: string): Promise<{ Metadata?: BlobMeta, ContentLength?: number }> {
+  async headBlob(location: string): Promise<{ Metadata?: BinaryMetadata, ContentLength?: number }> {
     const query = this.#queryBlob(location);
     try {
       return (await this.client.headObject(query));
@@ -374,20 +371,20 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
     }
   }
 
-  async getBlobMeta(location: string): Promise<BlobMeta> {
+  async getBlobMetadata(location: string): Promise<BinaryMetadata> {
     const blob = await this.headBlob(location);
 
     if (blob) {
-      const meta: BlobMeta = {
+      const metadata: BinaryMetadata = {
         contentType: '',
         ...blob.Metadata,
         size: blob.ContentLength!,
       };
-      if (hasContentType(meta)) {
-        meta['contentType'] = meta['contenttype']!;
-        delete meta['contenttype'];
+      if (hasContentType(metadata)) {
+        metadata['contentType'] = metadata['contenttype']!;
+        delete metadata['contenttype'];
       }
-      return meta;
+      return metadata;
     } else {
       throw new NotFoundError('Blob', location);
     }
@@ -397,38 +394,38 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
     await this.client.deleteObject(this.#queryBlob(location));
   }
 
-  async updateBlobMeta(location: string, meta: BlobMeta): Promise<void> {
+  async updateBlobMetadata(location: string, metadata: BinaryMetadata): Promise<void> {
     await this.client.copyObject({
       Bucket: this.config.bucket,
       Key: this.#basicKey(location),
       CopySource: `/${this.config.bucket}/${this.#basicKey(location)}`,
-      ...this.#getMetaBase(meta),
+      ...this.#getMetadata(metadata),
       MetadataDirective: 'REPLACE'
     });
   }
 
   // Signed urls
-  async getBlobReadUrl(location: string, exp: TimeSpan = '1h'): Promise<string> {
+  async getBlobReadUrl(location: string, expiresIn: TimeSpan = '1h'): Promise<string> {
     return await getSignedUrl(
       this.client,
       new GetObjectCommand(this.#queryBlob(location)),
-      { expiresIn: TimeUtil.asSeconds(exp) }
+      { expiresIn: TimeUtil.duration(expiresIn, 's') }
     );
   }
 
-  async getBlobWriteUrl(location: string, meta: BlobMeta, exp: TimeSpan = '1h'): Promise<string> {
-    const base = this.#getMetaBase(meta);
+  async getBlobWriteUrl(location: string, metadata: BinaryMetadata, expiresIn: TimeSpan = '1h'): Promise<string> {
+    const base = this.#getMetadata(metadata);
     return await getSignedUrl(
       this.client,
       new PutObjectCommand({
         ...this.#queryBlob(location),
         ...base,
-        ...(meta.size ? { ContentLength: meta.size } : {}),
-        ...((meta.hash && meta.hash !== '-1') ? { ChecksumSHA256: meta.hash } : {}),
+        ...(metadata.size ? { ContentLength: metadata.size } : {}),
+        ...((metadata.hash && metadata.hash !== '-1') ? { ChecksumSHA256: metadata.hash } : {}),
       }),
       {
-        expiresIn: TimeUtil.asSeconds(exp),
-        ...(meta.contentType ? { signableHeaders: new Set(['content-type']) } : {})
+        expiresIn: TimeUtil.duration(expiresIn, 's'),
+        ...(metadata.contentType ? { signableHeaders: new Set(['content-type']) } : {})
       }
     );
   }