@travetto/model-file 8.0.0-alpha.9 → 8.0.0

package/README.md

@@ -14,6 +14,6 @@ yarn add @travetto/model-file
 ```
 
 This module provides an file-based implementation for the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations."). Supported features:
-   *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L11)
-   *  [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10)
    *  [Blob](https://github.com/travetto/travetto/tree/main/module/model/src/types/blob.ts#L8)
+   *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L10)
+   *  [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10)

package/__index__.ts

@@ -1 +1 @@
-export * from './src/service.ts';
+export * from './src/service.ts';

package/package.json

@@ -1,39 +1,42 @@
 {
   "name": "@travetto/model-file",
-  "version": "8.0.0-alpha.9",
-  "type": "module",
+  "version": "8.0.0",
   "description": "File system backing for the travetto model module.",
   "keywords": [
     "datastore",
     "file",
     "schema",
-    "typescript",
-    "travetto"
+    "travetto",
+    "typescript"
   ],
   "homepage": "https://travetto.io",
   "license": "MIT",
   "author": {
-    "email": "travetto.framework@gmail.com",
-    "name": "Travetto Framework"
+    "name": "Travetto Framework",
+    "email": "travetto.framework@gmail.com"
+  },
+  "repository": {
+    "url": "git+https://github.com/travetto/travetto.git",
+    "directory": "module/model-file"
   },
   "files": [
     "__index__.ts",
     "src"
   ],
+  "type": "module",
   "main": "__index__.ts",
-  "repository": {
-    "url": "git+https://github.com/travetto/travetto.git",
-    "directory": "module/model-file"
+  "publishConfig": {
+    "access": "public"
   },
   "dependencies": {
-    "@travetto/config": "^8.0.0-alpha.9",
-    "@travetto/di": "^8.0.0-alpha.9",
-    "@travetto/model": "^8.0.0-alpha.9",
-    "@travetto/schema": "^8.0.0-alpha.9"
+    "@travetto/config": "^8.0.0",
+    "@travetto/di": "^8.0.0",
+    "@travetto/model": "^8.0.0",
+    "@travetto/schema": "^8.0.0"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.14",
-    "@travetto/test": "^8.0.0-alpha.9"
+    "@travetto/cli": "^8.0.0",
+    "@travetto/test": "^8.0.0"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {
@@ -45,8 +48,5 @@
   },
   "travetto": {
     "displayName": "File Model Support"
-  },
-  "publishConfig": {
-    "access": "public"
   }
 }

package/src/service.ts

@@ -1,19 +1,36 @@
-import fs from 'node:fs/promises';
 import { createReadStream, createWriteStream } from 'node:fs';
+import fs from 'node:fs/promises';
 import os from 'node:os';
 import path from 'node:path';
 
-import {
-  type Class, type TimeSpan, Runtime, type BinaryMetadata, type ByteRange, type BinaryType,
-  BinaryUtil, JSONUtil, BinaryMetadataUtil
-} from '@travetto/runtime';
-import { Injectable, PostConstruct } from '@travetto/di';
 import { Config } from '@travetto/config';
-import { Required } from '@travetto/schema';
+import { Injectable, PostConstruct } from '@travetto/di';
 import {
-  type ModelCrudSupport, type ModelExpirySupport, type ModelStorageSupport, type ModelType, ModelRegistryIndex,
-  NotFoundError, type OptionalId, ExistsError, type ModelBlobSupport, ModelCrudUtil, ModelExpiryUtil
+  ExistsError,
+  type ModelBlobSupport,
+  type ModelCrudSupport,
+  ModelCrudUtil,
+  type ModelExpirySupport,
+  ModelExpiryUtil,
+  type ModelListOptions,
+  ModelRegistryIndex,
+  type ModelStorageSupport,
+  type ModelType,
+  NotFoundError,
+  type OptionalId
 } from '@travetto/model';
+import {
+  type BinaryMetadata,
+  BinaryMetadataUtil,
+  type BinaryType,
+  BinaryUtil,
+  type ByteRange,
+  type Class,
+  JSONUtil,
+  Runtime,
+  type TimeSpan
+} from '@travetto/runtime';
+import { Required } from '@travetto/schema';
 
 type Suffix = '.bin' | '.meta' | '.json' | '.expires';
 
@@ -32,33 +49,58 @@ export class FileModelConfig {
 
   @PostConstruct()
   async finalizeConfig(): Promise<void> {
-    this.folder ??= path.resolve(os.tmpdir(), `trv_file_${Runtime.main.name.replace(/[^a-z]/ig, '_')}`);
+    this.folder ??= path.resolve(os.tmpdir(), `trv_file_${Runtime.main.name.replace(/[^a-z]/gi, '_')}`);
   }
 }
 
-const exists = (file: string): Promise<boolean> => fs.stat(file).then(() => true, () => false);
+const exists = (file: string): Promise<boolean> =>
+  fs.stat(file).then(
+    () => true,
+    () => false
+  );
 
 /**
  * Standard file support
  */
 @Injectable()
 export class FileModelService implements ModelCrudSupport, ModelBlobSupport, ModelExpirySupport, ModelStorageSupport {
-
   /** @private */
-  static async * scanFolder(folder: string, suffix: string): AsyncGenerator<[id: string, field: string]> {
+  static async *scanFolder(folder: string, suffix: string, options?: ModelListOptions): AsyncGenerator<[id: string, field: string][]> {
+    const found: [id: string, field: string][] = [];
+    const batchSize = options?.batchSizeHint ?? 100;
+    const maxCount = options?.limit ?? Number.MAX_SAFE_INTEGER;
+    let produced = 0;
     for (const sub of await fs.readdir(folder)) {
+      if (options?.abort?.aborted) {
+        break;
+      }
       for (const file of await fs.readdir(path.resolve(folder, sub))) {
+        if (options?.abort?.aborted) {
+          break;
+        }
         if (file.endsWith(suffix)) {
-          yield [file.replace(suffix, ''), path.resolve(folder, sub, file)];
+          found.push([file.replace(suffix, ''), path.resolve(folder, sub, file)]);
+          produced += 1;
+          if (produced >= maxCount) {
+            break;
+          }
+          if (found.length >= batchSize) {
+            yield found.splice(0, found.length);
+          }
         }
       }
     }
+    if (found.length) {
+      yield found;
+    }
   }
 
   idSource = ModelCrudUtil.uuidSource();
   config: FileModelConfig;
 
-  constructor(config: FileModelConfig) { this.config = config; }
+  constructor(config: FileModelConfig) {
+    this.config = config;
+  }
 
   /**
    * The root location for all activity
@@ -160,30 +202,28 @@ export class FileModelService implements ModelCrudSupport, ModelBlobSupport, Mod
     await fs.unlink(file);
   }
 
-  async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T> {
-    for await (const [id] of FileModelService.scanFolder(await this.#resolveName(cls, '.json'), '.json')) {
-      try {
-        yield await this.get(cls, id);
-      } catch (error) {
-        if (!(error instanceof NotFoundError)) {
-          throw error;
-        }
-      }
+  async *list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]> {
+    for await (const batch of FileModelService.scanFolder(await this.#resolveName(cls, '.json'), '.json', options)) {
+      yield ModelCrudUtil.filterOutNotFound(batch.map(([id]) => this.get(cls, id)));
     }
   }
 
   // Blob
   async upsertBlob(location: string, input: BinaryType, metadata?: BinaryMetadata, overwrite = true): Promise<void> {
-    if (!overwrite && await this.getBlobMetadata(location).then(() => true, () => false)) {
+    if (
+      !overwrite &&
+      (await this.getBlobMetadata(location).then(
+        () => true,
+        () => false
+      ))
+    ) {
       return;
     }
     const resolved = await BinaryMetadataUtil.compute(input, metadata);
     const file = await this.#resolveName(ModelBlobNamespace, BIN, location);
     await Promise.all([
       BinaryUtil.pipeline(input, createWriteStream(file)),
-      BinaryUtil.pipeline(
-        JSONUtil.toBinaryArray(resolved),
-        createWriteStream(file.replace(BIN, META)))
+      BinaryUtil.pipeline(JSONUtil.toBinaryArray(resolved), createWriteStream(file.replace(BIN, META)))
     ]);
   }
 
@@ -204,10 +244,7 @@ export class FileModelService implements ModelCrudSupport, ModelBlobSupport, Mod
   async deleteBlob(location: string): Promise<void> {
     const file = await this.#resolveName(ModelBlobNamespace, BIN, location);
     if (await exists(file)) {
-      await Promise.all([
-        fs.unlink(file),
-        fs.unlink(file.replace('.bin', META))
-      ]);
+      await Promise.all([fs.unlink(file), fs.unlink(file.replace('.bin', META))]);
     } else {
       throw new NotFoundError(ModelBlobNamespace, location);
     }
@@ -221,14 +258,16 @@ export class FileModelService implements ModelCrudSupport, ModelBlobSupport, Mod
   // Expiry
   async deleteExpired<T extends ModelType>(cls: Class<T>): Promise<number> {
     let deleted = 0;
-    for await (const [_id, file] of FileModelService.scanFolder(await this.#resolveName(cls, '.json'), '.json')) {
-      try {
-        const item = await ModelCrudUtil.load(cls, await fs.readFile(file));
-        if (ModelExpiryUtil.getExpiryState(cls, item).expired) {
-          await fs.rm(file, { force: true });
-          deleted += 1;
-        }
-      } catch { } // Don't let a single failure stop the process
+    for await (const batch of FileModelService.scanFolder(await this.#resolveName(cls, '.json'), '.json')) {
+      for (const [_id, file] of batch) {
+        try {
+          const item = await ModelCrudUtil.load(cls, await fs.readFile(file));
+          if (ModelExpiryUtil.getExpiryState(cls, item).expired) {
+            await fs.rm(file, { force: true });
+            deleted += 1;
+          }
+        } catch {} // Don't let a single failure stop the process
+      }
     }
     return deleted;
   }
@@ -250,4 +289,4 @@ export class FileModelService implements ModelCrudSupport, ModelBlobSupport, Mod
   async truncateBlob(): Promise<void> {
     await fs.rm(await this.#resolveName(ModelBlobNamespace), { recursive: true, force: true });
   }
-}
+}