@travetto/model-s3 8.0.0-alpha.1 → 8.0.0-alpha.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-s3",
-  "version": "8.0.0-alpha.1",
+  "version": "8.0.0-alpha.11",
   "type": "module",
   "description": "S3 backing for the travetto model module.",
   "keywords": [
@@ -26,14 +26,14 @@
     "directory": "module/model-s3"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.1004.0",
-    "@aws-sdk/credential-provider-ini": "^3.972.17",
-    "@aws-sdk/s3-request-presigner": "^3.1004.0",
-    "@travetto/config": "^8.0.0-alpha.1",
-    "@travetto/model": "^8.0.0-alpha.1"
+    "@aws-sdk/client-s3": "^3.1024.0",
+    "@aws-sdk/credential-provider-ini": "^3.972.28",
+    "@aws-sdk/s3-request-presigner": "^3.1024.0",
+    "@travetto/config": "^8.0.0-alpha.11",
+    "@travetto/model": "^8.0.0-alpha.11"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.1"
+    "@travetto/cli": "^8.0.0-alpha.16"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/service.ts

@@ -7,7 +7,8 @@ import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 
 import {
   type ModelCrudSupport, type ModelStorageSupport, type ModelType, ModelRegistryIndex, ExistsError, NotFoundError, type OptionalId,
-  type ModelBlobSupport, type ModelExpirySupport, ModelCrudUtil, ModelExpiryUtil, ModelStorageUtil
+  type ModelBlobSupport, type ModelExpirySupport, ModelCrudUtil, ModelExpiryUtil, ModelStorageUtil,
+  type ModelListOptions
 } from '@travetto/model';
 import { Injectable, PostConstruct } from '@travetto/di';
 import {
@@ -93,16 +94,26 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
     return {};
   }
 
-  async * #iterateBucket(cls?: string | Class): AsyncIterable<{ Key: string, id: string }[]> {
+  async * #iterateBucket(cls?: string | Class, options?: ModelListOptions): AsyncIterable<{ Key: string, id: string }[]> {
     let Marker: string | undefined;
-    for (; ;) {
+    const batchSize = options?.batchSizeHint ?? 100;
+    const maxCount = options?.limit ?? Number.MAX_SAFE_INTEGER;
+    let produced = 0;
+    for (; !(options?.abort?.aborted) && produced < maxCount;) {
       const items = await this.client.listObjects({
         Bucket: this.config.bucket,
         Prefix: cls ? this.#resolveKey(cls) : this.config.namespace,
-        Marker
+        Marker,
+        MaxKeys: batchSize
       });
-      if (items.Contents?.length) {
-        yield items.Contents.map(item => ({ Key: item.Key!, id: item.Key!.split(':').pop()! }));
+
+      let toSend = items.Contents?.map(item => ({ Key: item.Key!, id: item.Key!.split(':').pop()! }));
+      if (toSend) {
+        produced += toSend.length;
+        if (produced > maxCount) {
+          toSend = toSend.slice(0, maxCount - (produced - toSend.length));
+        }
+        yield toSend;
       }
       if (items.NextMarker) {
         Marker = items.NextMarker;
@@ -293,17 +304,9 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
     await this.client.deleteObject(this.#query(cls, id));
   }
 
-  async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T> {
-    for await (const batch of this.#iterateBucket(cls)) {
-      for (const { id } of batch) {
-        try {
-          yield await this.get(cls, id);
-        } catch (error) {
-          if (!(error instanceof NotFoundError)) {
-            throw error;
-          }
-        }
-      }
+  async * list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]> {
+    for await (const batch of this.#iterateBucket(cls, options)) {
+      yield ModelCrudUtil.filterOutNotFound(batch.map(item => this.get(cls, item.id)));
     }
   }