@travetto/model-file 8.0.0-alpha.1 → 8.0.0-alpha.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-file",
-  "version": "8.0.0-alpha.1",
+  "version": "8.0.0-alpha.11",
   "type": "module",
   "description": "File system backing for the travetto model module.",
   "keywords": [
@@ -26,14 +26,14 @@
     "directory": "module/model-file"
   },
   "dependencies": {
-    "@travetto/config": "^8.0.0-alpha.1",
-    "@travetto/di": "^8.0.0-alpha.1",
-    "@travetto/model": "^8.0.0-alpha.1",
-    "@travetto/schema": "^8.0.0-alpha.1"
+    "@travetto/config": "^8.0.0-alpha.11",
+    "@travetto/di": "^8.0.0-alpha.11",
+    "@travetto/model": "^8.0.0-alpha.11",
+    "@travetto/schema": "^8.0.0-alpha.11"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.1",
-    "@travetto/test": "^8.0.0-alpha.1"
+    "@travetto/cli": "^8.0.0-alpha.16",
+    "@travetto/test": "^8.0.0-alpha.11"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/service.ts

@@ -12,7 +12,8 @@ import { Config } from '@travetto/config';
 import { Required } from '@travetto/schema';
 import {
   type ModelCrudSupport, type ModelExpirySupport, type ModelStorageSupport, type ModelType, ModelRegistryIndex,
-  NotFoundError, type OptionalId, ExistsError, type ModelBlobSupport, ModelCrudUtil, ModelExpiryUtil
+  NotFoundError, type OptionalId, ExistsError, type ModelBlobSupport, ModelCrudUtil, ModelExpiryUtil,
+  type ModelListOptions
 } from '@travetto/model';
 
 type Suffix = '.bin' | '.meta' | '.json' | '.expires';
@@ -45,14 +46,34 @@ const exists = (file: string): Promise<boolean> => fs.stat(file).then(() => true
 export class FileModelService implements ModelCrudSupport, ModelBlobSupport, ModelExpirySupport, ModelStorageSupport {
 
   /** @private */
-  static async * scanFolder(folder: string, suffix: string): AsyncGenerator<[id: string, field: string]> {
+  static async * scanFolder(folder: string, suffix: string, options?: ModelListOptions): AsyncGenerator<[id: string, field: string][]> {
+    const found: [id: string, field: string][] = [];
+    const batchSize = options?.batchSizeHint ?? 100;
+    const maxCount = options?.limit ?? Number.MAX_SAFE_INTEGER;
+    let produced = 0;
     for (const sub of await fs.readdir(folder)) {
+      if (options?.abort?.aborted) {
+        break;
+      }
       for (const file of await fs.readdir(path.resolve(folder, sub))) {
+        if (options?.abort?.aborted) {
+          break;
+        }
         if (file.endsWith(suffix)) {
-          yield [file.replace(suffix, ''), path.resolve(folder, sub, file)];
+          found.push([file.replace(suffix, ''), path.resolve(folder, sub, file)]);
+          produced += 1;
+          if (produced >= maxCount) {
+            break;
+          }
+          if (found.length >= batchSize) {
+            yield found.splice(0, found.length);
+          }
         }
       }
     }
+    if (found.length) {
+      yield found;
+    }
   }
 
   idSource = ModelCrudUtil.uuidSource();
@@ -160,15 +181,9 @@ export class FileModelService implements ModelCrudSupport, ModelBlobSupport, Mod
     await fs.unlink(file);
   }
 
-  async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T> {
-    for await (const [id] of FileModelService.scanFolder(await this.#resolveName(cls, '.json'), '.json')) {
-      try {
-        yield await this.get(cls, id);
-      } catch (error) {
-        if (!(error instanceof NotFoundError)) {
-          throw error;
-        }
-      }
+  async * list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]> {
+    for await (const batch of FileModelService.scanFolder(await this.#resolveName(cls, '.json'), '.json', options)) {
+      yield ModelCrudUtil.filterOutNotFound(batch.map(([id]) => this.get(cls, id)));
     }
   }
 
@@ -221,14 +236,16 @@ export class FileModelService implements ModelCrudSupport, ModelBlobSupport, Mod
   // Expiry
   async deleteExpired<T extends ModelType>(cls: Class<T>): Promise<number> {
     let deleted = 0;
-    for await (const [_id, file] of FileModelService.scanFolder(await this.#resolveName(cls, '.json'), '.json')) {
-      try {
-        const item = await ModelCrudUtil.load(cls, await fs.readFile(file));
-        if (ModelExpiryUtil.getExpiryState(cls, item).expired) {
-          await fs.rm(file, { force: true });
-          deleted += 1;
-        }
-      } catch { } // Don't let a single failure stop the process
+    for await (const batch of FileModelService.scanFolder(await this.#resolveName(cls, '.json'), '.json')) {
+      for (const [_id, file] of batch) {
+        try {
+          const item = await ModelCrudUtil.load(cls, await fs.readFile(file));
+          if (ModelExpiryUtil.getExpiryState(cls, item).expired) {
+            await fs.rm(file, { force: true });
+            deleted += 1;
+          }
+        } catch { } // Don't let a single failure stop the process
+      }
     }
     return deleted;
   }