@travetto/model-s3 8.0.0-alpha.2 → 8.0.0-alpha.20

package/README.md

@@ -65,11 +65,10 @@ export class S3ModelConfig {
   modifyStorage?: boolean;
 
   /**
-   * Provide host to bucket
+   * Provide base URL for public access
    */
-  get hostName(): string {
-    return `${this.bucket}.s3.amazonaws.com`;
-  }
+  @Required(false)
+  publicBaseUrl: string;
 
   /**
    * Produces the s3 config from the provide details, post construction
@@ -81,6 +80,26 @@ export class S3ModelConfig {
       this.bucket ??= 'app';
     }
 
+    if (!this.publicBaseUrl) {
+      if (this.endpoint) {
+        if (this.endpoint.includes('localhost')) {
+          this.publicBaseUrl = this.endpoint;
+        } else {
+          try {
+            this.publicBaseUrl = new URL(this.endpoint).origin;
+          } catch {
+            this.publicBaseUrl = this.endpoint;
+          }
+        }
+      } else {
+        this.publicBaseUrl = `https://${this.bucket}.s3.amazonaws.com`;
+      }
+    }
+
+    if (this.publicBaseUrl && !this.publicBaseUrl.includes('://')) {
+      this.publicBaseUrl = `https://${this.publicBaseUrl}`;
+    }
+
     if (!this.accessKeyId && !this.secretAccessKey) {
       const creds = await fromIni({ profile: this.profile })();
       this.accessKeyId = creds.accessKeyId;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-s3",
-  "version": "8.0.0-alpha.2",
+  "version": "8.0.0-alpha.20",
   "type": "module",
   "description": "S3 backing for the travetto model module.",
   "keywords": [
@@ -26,14 +26,14 @@
     "directory": "module/model-s3"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.1004.0",
-    "@aws-sdk/credential-provider-ini": "^3.972.17",
-    "@aws-sdk/s3-request-presigner": "^3.1004.0",
-    "@travetto/config": "^8.0.0-alpha.2",
-    "@travetto/model": "^8.0.0-alpha.2"
+    "@aws-sdk/client-s3": "^3.1063.0",
+    "@aws-sdk/credential-provider-ini": "^3.972.50",
+    "@aws-sdk/s3-request-presigner": "^3.1063.0",
+    "@travetto/config": "^8.0.0-alpha.19",
+    "@travetto/model": "^8.0.0-alpha.20"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.3"
+    "@travetto/cli": "^8.0.0-alpha.25"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/config.ts

@@ -2,7 +2,7 @@ import { fromIni } from '@aws-sdk/credential-provider-ini';
 import type s3 from '@aws-sdk/client-s3';
 
 import { Config, EnvVar } from '@travetto/config';
-import { Required } from '@travetto/schema';
+import { Required, Url } from '@travetto/schema';
 import { Runtime } from '@travetto/runtime';
 import { PostConstruct } from '@travetto/di';
 
@@ -33,11 +33,11 @@ export class S3ModelConfig {
   modifyStorage?: boolean;
 
   /**
-   * Provide host to bucket
+   * Provide base URL for public access
    */
-  get hostName(): string {
-    return `${this.bucket}.s3.amazonaws.com`;
-  }
+  @Url()
+  @Required(false)
+  publicBaseUrl: string;
 
   /**
    * Produces the s3 config from the provide details, post construction
@@ -49,6 +49,14 @@ export class S3ModelConfig {
       this.bucket ??= 'app';
     }
 
+    if (!this.publicBaseUrl) {
+      if (this.endpoint?.includes('localhost')) {
+        this.publicBaseUrl = this.endpoint;
+      } else {
+        this.publicBaseUrl = `https://${this.bucket}.s3.amazonaws.com`;
+      }
+    }
+
     if (!this.accessKeyId && !this.secretAccessKey) {
       const creds = await fromIni({ profile: this.profile })();
       this.accessKeyId = creds.accessKeyId;

package/src/service.ts

@@ -7,7 +7,8 @@ import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 
 import {
   type ModelCrudSupport, type ModelStorageSupport, type ModelType, ModelRegistryIndex, ExistsError, NotFoundError, type OptionalId,
-  type ModelBlobSupport, type ModelExpirySupport, ModelCrudUtil, ModelExpiryUtil, ModelStorageUtil
+  type ModelBlobSupport, type ModelExpirySupport, ModelCrudUtil, ModelExpiryUtil, ModelStorageUtil,
+  type ModelListOptions
 } from '@travetto/model';
 import { Injectable, PostConstruct } from '@travetto/di';
 import {
@@ -93,16 +94,26 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
     return {};
   }
 
-  async * #iterateBucket(cls?: string | Class): AsyncIterable<{ Key: string, id: string }[]> {
+  async * #iterateBucket(cls?: string | Class, options?: ModelListOptions): AsyncIterable<{ Key: string, id: string }[]> {
     let Marker: string | undefined;
-    for (; ;) {
+    const batchSize = options?.batchSizeHint ?? 100;
+    const maxCount = options?.limit ?? Number.MAX_SAFE_INTEGER;
+    let produced = 0;
+    for (; !(options?.abort?.aborted) && produced < maxCount;) {
       const items = await this.client.listObjects({
         Bucket: this.config.bucket,
         Prefix: cls ? this.#resolveKey(cls) : this.config.namespace,
-        Marker
+        Marker,
+        MaxKeys: batchSize
       });
-      if (items.Contents?.length) {
-        yield items.Contents.map(item => ({ Key: item.Key!, id: item.Key!.split(':').pop()! }));
+
+      let toSend = items.Contents?.map(item => ({ Key: item.Key!, id: item.Key!.split(':').pop()! }));
+      if (toSend) {
+        produced += toSend.length;
+        if (produced > maxCount) {
+          toSend = toSend.slice(0, maxCount - (produced - toSend.length));
+        }
+        yield toSend;
       }
       if (items.NextMarker) {
         Marker = items.NextMarker;
@@ -293,17 +304,9 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
     await this.client.deleteObject(this.#query(cls, id));
   }
 
-  async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T> {
-    for await (const batch of this.#iterateBucket(cls)) {
-      for (const { id } of batch) {
-        try {
-          yield await this.get(cls, id);
-        } catch (error) {
-          if (!(error instanceof NotFoundError)) {
-            throw error;
-          }
-        }
-      }
+  async * list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]> {
+    for await (const batch of this.#iterateBucket(cls, options)) {
+      yield ModelCrudUtil.filterOutNotFound(batch.map(item => this.get(cls, item.id)));
     }
   }
 
@@ -406,7 +409,16 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
   }
 
   // Signed urls
-  async getBlobReadUrl(location: string, expiresIn: TimeSpan = '1h'): Promise<string> {
+  async getBlobReadUrl(location: string, expiresIn: TimeSpan | false = '1h'): Promise<string> {
+    if (expiresIn === false) {
+      const key = this.#basicKey(location);
+      const baseUrl = this.config.publicBaseUrl;
+      if (this.config.config.forcePathStyle) {
+        return `${baseUrl}/${this.config.bucket}/${key}`;
+      } else {
+        return `${baseUrl}/${key}`;
+      }
+    }
     return await getSignedUrl(
       this.client,
       new GetObjectCommand(this.#queryBlob(location)),

package/support/service.s3.ts

@@ -1,6 +1,6 @@
 import type { ServiceDescriptor } from '@travetto/cli';
 
-const version = '4.11.0';
+const version = '4.12.4';
 
 export const service: ServiceDescriptor = {
   name: 's3',