@travetto/model 5.0.0-rc.10 → 5.0.0-rc.11

package/support/test/blob.ts

@@ -0,0 +1,117 @@
+import assert from 'node:assert';
+
+import { Suite, Test, TestFixtures } from '@travetto/test';
+import { BaseModelSuite } from '@travetto/model/support/test/base';
+import { Util } from '@travetto/runtime';
+
+import { ModelBlobSupport } from '../../src/service/blob';
+import { ModelBlobUtil } from '../../src/util/blob';
+
+@Suite()
+export abstract class ModelBlobSuite extends BaseModelSuite<ModelBlobSupport> {
+
+  fixture = new TestFixtures(['@travetto/model']);
+
+  @Test()
+  async writeBasic(): Promise<void> {
+    const service = await this.service;
+    const buffer = await this.fixture.read('/asset.yml', true);
+
+    const id = Util.uuid();
+
+    await service.upsertBlob(id, buffer);
+    const meta = await service.describeBlob(id);
+    const retrieved = await service.describeBlob(id);
+    assert.deepStrictEqual(meta, retrieved);
+  }
+
+  @Test()
+  async writeStream(): Promise<void> {
+    const service = await this.service;
+    const buffer = await this.fixture.read('/asset.yml', true);
+
+    const id = Util.uuid();
+    await service.upsertBlob(id, buffer);
+    const meta = await service.describeBlob(id);
+
+    const retrieved = await service.getBlob(id);
+    const retrievedMeta = retrieved.meta!;
+    assert(meta.hash === retrievedMeta.hash);
+  }
+
+  @Test()
+  async writeAndDelete(): Promise<void> {
+    const service = await this.service;
+    const buffer = await this.fixture.read('/asset.yml', true);
+
+    const id = Util.uuid();
+    await service.upsertBlob(id, buffer);
+
+    await service.deleteBlob(id);
+
+    await assert.rejects(async () => {
+      await service.getBlob(id);
+    });
+  }
+
+  @Test()
+  async partialStream(): Promise<void> {
+    const service = await this.service;
+    const buffer = await this.fixture.read('/text.txt', true);
+
+    const id = Util.uuid();
+    await service.upsertBlob(id, buffer);
+
+    const retrieved = await service.getBlob(id);
+    const content = await retrieved.text();
+    assert(content.startsWith('abc'));
+    assert(content.endsWith('xyz'));
+
+    const partial = await service.getBlob(id, { start: 10, end: 20 });
+    assert(partial.size === 11);
+    const partialMeta = partial.meta!;
+    const subContent = await partial.text();
+    const range = await ModelBlobUtil.enforceRange({ start: 10, end: 20 }, partialMeta.size!);
+    assert(subContent.length === (range.end - range.start) + 1);
+
+    const og = await this.fixture.read('/text.txt');
+
+    assert(subContent === og.substring(10, 21));
+
+    const partialUnbounded = await service.getBlob(id, { start: 10 });
+    const partialUnboundedMeta = partial.meta!;
+    const subContent2 = await partialUnbounded.text();
+    const range2 = await ModelBlobUtil.enforceRange({ start: 10 }, partialUnboundedMeta.size!);
+    assert(subContent2.length === (range2.end - range2.start) + 1);
+    assert(subContent2.startsWith('klm'));
+    assert(subContent2.endsWith('xyz'));
+
+    const partialSingle = await service.getBlob(id, { start: 10, end: 10 });
+    const subContent3 = await partialSingle.text();
+    assert(subContent3.length === 1);
+    assert(subContent3 === 'k');
+
+    const partialOverBounded = await service.getBlob(id, { start: 20, end: 40 });
+    const subContent4 = await partialOverBounded.text();
+    assert(subContent4.length === 6);
+    assert(subContent4.endsWith('xyz'));
+
+    await assert.rejects(() => service.getBlob(id, { start: -10, end: 10 }));
+    await assert.rejects(() => service.getBlob(id, { start: 30, end: 37 }));
+  }
+
+
+  @Test()
+  async writeAndGet() {
+    const service = await this.service;
+    const buffer = await this.fixture.read('/asset.yml', true);
+    await service.upsertBlob('orange', buffer, { contentType: 'text/yaml', filename: 'asset.yml' });
+    const saved = await service.getBlob('orange');
+    const savedMeta = saved.meta!;
+
+    assert('text/yaml' === savedMeta.contentType);
+    assert(buffer.length === savedMeta.size);
+    assert('asset.yml' === savedMeta.filename);
+    assert(undefined === savedMeta.hash);
+  }
+}

package/support/test/suite.ts

@@ -3,8 +3,8 @@ import { DependencyRegistry } from '@travetto/di';
 import { RootRegistry } from '@travetto/registry';
 import { SuiteRegistry, TestFixtures } from '@travetto/test';
 
-import { isStorageSupported, isStreamSupported } from '../../src/internal/service/common';
-import { StreamModel } from '../../src/internal/service/stream';
+import { isBlobSupported, isStorageSupported } from '../../src/internal/service/common';
+import { MODEL_BLOB } from '../../src/internal/service/blob';
 import { ModelRegistry } from '../../src/registry/model';
 
 const Loaded = Symbol();
@@ -61,11 +61,11 @@ export function ModelSuite<T extends { configClass: Class<{ autoCreate?: boolean
                 }
               }
             }
-            if (isStreamSupported(service)) {
+            if (isBlobSupported(service)) {
               if (service.truncateModel) {
-                await service.truncateModel(StreamModel);
+                await service.truncateModel(MODEL_BLOB);
               } else if (service.deleteModel) {
-                await service.deleteModel(StreamModel);
+                await service.deleteModel(MODEL_BLOB);
               }
             }
           } else {

package/src/internal/service/stream.ts

@@ -1,22 +0,0 @@
-import { AppError, Class } from '@travetto/runtime';
-
-import { ModelType } from '../../types/model';
-import { StreamRange } from '../../service/stream';
-
-class Cls { id: string; }
-export const StreamModel: Class<ModelType> = Cls;
-export const STREAMS = '_streams';
-
-
-/**
- * Enforce byte range for stream stream/file of a certain size
- */
-export function enforceRange({ start, end }: StreamRange, size: number): Required<StreamRange> {
-  end = Math.min(end ?? size - 1, size - 1);
-
-  if (Number.isNaN(start) || Number.isNaN(end) || !Number.isFinite(start) || start >= size || start < 0 || start > end) {
-    throw new AppError('Invalid position, out of range', 'data');
-  }
-
-  return { start, end };
-}

package/src/provider/file.ts

@@ -1,231 +0,0 @@
-import fs from 'node:fs/promises';
-import { createReadStream, createWriteStream } from 'node:fs';
-import os from 'node:os';
-import { Readable } from 'node:stream';
-import { pipeline } from 'node:stream/promises';
-import path from 'node:path';
-
-import { Class, TimeSpan, Runtime, asFull } from '@travetto/runtime';
-import { Injectable } from '@travetto/di';
-import { Config } from '@travetto/config';
-import { Required } from '@travetto/schema';
-
-import { ModelCrudSupport } from '../service/crud';
-import { ModelStreamSupport, StreamMeta, StreamRange } from '../service/stream';
-import { ModelType, OptionalId } from '../types/model';
-import { ModelExpirySupport } from '../service/expiry';
-import { ModelRegistry } from '../registry/model';
-import { ModelStorageSupport } from '../service/storage';
-import { ModelCrudUtil } from '../internal/service/crud';
-import { ModelExpiryUtil } from '../internal/service/expiry';
-import { NotFoundError } from '../error/not-found';
-import { ExistsError } from '../error/exists';
-import { enforceRange, StreamModel, STREAMS } from '../internal/service/stream';
-
-type Suffix = '.bin' | '.meta' | '.json' | '.expires';
-
-const BIN = '.bin';
-const META = '.meta';
-
-@Config('model.file')
-export class FileModelConfig {
-  @Required(false)
-  folder: string;
-  namespace: string = '.';
-  autoCreate?: boolean;
-  cullRate?: number | TimeSpan;
-
-  async postConstruct(): Promise<void> {
-    this.folder ??= path.resolve(os.tmpdir(), `trv_file_${Runtime.main.name.replace(/[^a-z]/ig, '_')}`);
-  }
-}
-
-const exists = (f: string): Promise<boolean> => fs.stat(f).then(() => true, () => false);
-
-/**
- * Standard file support
- */
-@Injectable()
-export class FileModelService implements ModelCrudSupport, ModelStreamSupport, ModelExpirySupport, ModelStorageSupport {
-
-  private static async * scanFolder(folder: string, suffix: string): AsyncGenerator<[id: string, field: string]> {
-    for (const sub of await fs.readdir(folder)) {
-      for (const file of await fs.readdir(path.resolve(folder, sub))) {
-        if (file.endsWith(suffix)) {
-          yield [file.replace(suffix, ''), path.resolve(folder, sub, file)];
-        }
-      }
-    }
-  }
-
-  idSource = ModelCrudUtil.uuidSource();
-
-  get client(): string {
-    return this.config.folder;
-  }
-
-  /**
-   * The root location for all activity
-   */
-  constructor(public readonly config: FileModelConfig) { }
-
-  async #resolveName<T extends ModelType>(cls: Class<T> | string, suffix?: Suffix, id?: string): Promise<string> {
-    const name = typeof cls === 'string' ? cls : ModelRegistry.getStore(cls);
-    let resolved = path.resolve(this.config.folder, this.config.namespace, name);
-    if (id) {
-      resolved = path.resolve(resolved, id.replace(/^[/]/, '').substring(0, 3));
-    }
-    let dir = resolved;
-    if (id) {
-      resolved = path.resolve(resolved, `${id}${suffix}`);
-      dir = path.dirname(resolved);
-    }
-
-    await fs.mkdir(dir, { recursive: true });
-    return resolved;
-  }
-
-  async #find<T extends ModelType>(cls: Class<T> | string, suffix: Suffix, id?: string): Promise<string> {
-    const file = await this.#resolveName(cls, suffix, id);
-    if (id && !(await exists(file))) {
-      throw new NotFoundError(cls, id);
-    }
-    return file;
-  }
-
-  postConstruct(): void {
-    ModelExpiryUtil.registerCull(this);
-  }
-
-  checkExpiry<T extends ModelType>(cls: Class<T>, item: T): T {
-    const { expiresAt } = ModelRegistry.get(cls);
-    if (expiresAt && ModelExpiryUtil.getExpiryState(cls, item).expired) {
-      throw new NotFoundError(cls, item.id);
-    }
-    return item;
-  }
-
-  async get<T extends ModelType>(cls: Class<T>, id: string): Promise<T> {
-    await this.#find(cls, '.json', id);
-
-    const file = await this.#resolveName(cls, '.json', id);
-
-    if (await exists(file)) {
-      const content = await fs.readFile(file);
-      return this.checkExpiry(cls, await ModelCrudUtil.load(cls, content));
-    }
-
-    throw new NotFoundError(cls, id);
-  }
-
-  async create<T extends ModelType>(cls: Class<T>, item: OptionalId<T>): Promise<T> {
-    if (!item.id) {
-      item.id = this.idSource.create();
-    }
-
-    const file = await this.#resolveName(cls, '.json', item.id);
-
-    if (await exists(file)) {
-      throw new ExistsError(cls, item.id!);
-    }
-
-    return await this.upsert(cls, item);
-  }
-
-  async update<T extends ModelType>(cls: Class<T>, item: T): Promise<T> {
-    await this.get(cls, item.id);
-    return await this.upsert(cls, item);
-  }
-
-  async upsert<T extends ModelType>(cls: Class<T>, item: OptionalId<T>): Promise<T> {
-    ModelCrudUtil.ensureNotSubType(cls);
-    const prepped = await ModelCrudUtil.preStore(cls, item, this);
-
-    const file = await this.#resolveName(cls, '.json', item.id);
-    await fs.writeFile(file, JSON.stringify(item), { encoding: 'utf8' });
-
-    return prepped;
-  }
-
-  async updatePartial<T extends ModelType>(cls: Class<T>, item: Partial<T> & { id: string }, view?: string): Promise<T> {
-    ModelCrudUtil.ensureNotSubType(cls);
-    const id = item.id;
-    item = await ModelCrudUtil.naivePartialUpdate(cls, item, view, () => this.get(cls, id));
-    const file = await this.#resolveName(cls, '.json', item.id);
-    await fs.writeFile(file, JSON.stringify(item), { encoding: 'utf8' });
-    return asFull<T>(item);
-  }
-
-  async delete<T extends ModelType>(cls: Class<T>, id: string): Promise<void> {
-    const file = await this.#find(cls, '.json', id);
-    await fs.unlink(file);
-  }
-
-  async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T> {
-    for await (const [id] of FileModelService.scanFolder(await this.#resolveName(cls, '.json'), '.json')) {
-      try {
-        yield await this.get(cls, id);
-      } catch (err) {
-        if (!(err instanceof NotFoundError)) {
-          throw err;
-        }
-      }
-    }
-  }
-
-  // Stream
-  async upsertStream(location: string, input: Readable, meta: StreamMeta): Promise<void> {
-    const file = await this.#resolveName(STREAMS, BIN, location);
-    await Promise.all([
-      await pipeline(input, createWriteStream(file)),
-      fs.writeFile(file.replace(BIN, META), JSON.stringify(meta), 'utf8')
-    ]);
-  }
-
-  async getStream(location: string, range?: StreamRange): Promise<Readable> {
-    const file = await this.#find(STREAMS, BIN, location);
-    if (range) {
-      const meta = await this.describeStream(location);
-      range = enforceRange(range, meta.size);
-    }
-    return createReadStream(file, range);
-  }
-
-  async describeStream(location: string): Promise<StreamMeta> {
-    const file = await this.#find(STREAMS, META, location);
-    const content = await fs.readFile(file);
-    const text: StreamMeta = JSON.parse(content.toString('utf8'));
-    return text;
-  }
-
-  async deleteStream(location: string): Promise<void> {
-    const file = await this.#resolveName(STREAMS, BIN, location);
-    if (await exists(file)) {
-      await Promise.all([
-        fs.unlink(file),
-        fs.unlink(file.replace('.bin', META))
-      ]);
-    } else {
-      throw new NotFoundError('Stream', location);
-    }
-  }
-
-  // Expiry
-  async deleteExpired<T extends ModelType>(cls: Class<T>): Promise<number> {
-    return ModelExpiryUtil.naiveDeleteExpired(this, cls);
-  }
-
-  // Storage management
-  async createStorage(): Promise<void> {
-    const dir = path.resolve(this.config.folder, this.config.namespace);
-    await fs.mkdir(dir, { recursive: true });
-  }
-
-  async deleteStorage(): Promise<void> {
-    await fs.rm(path.resolve(this.config.folder, this.config.namespace), { recursive: true, force: true });
-  }
-
-  async truncateModel(cls: Class<ModelType>): Promise<void> {
-    await fs.rm(await this.#resolveName(cls === StreamModel ? STREAMS : cls), { recursive: true, force: true });
-  }
-}