@travetto/model 5.0.0-rc.8 → 5.0.0

package/src/provider/file.ts

@@ -1,233 +0,0 @@
-import fs from 'node:fs/promises';
-import { createReadStream, createWriteStream } from 'node:fs';
-import os from 'node:os';
-import { Readable } from 'node:stream';
-import { pipeline } from 'node:stream/promises';
-import path from 'node:path';
-
-import { Class, TimeSpan, Runtime } from '@travetto/runtime';
-import { Injectable } from '@travetto/di';
-import { Config } from '@travetto/config';
-import { Required } from '@travetto/schema';
-
-import { ModelCrudSupport } from '../service/crud';
-import { ModelStreamSupport, StreamMeta, StreamRange } from '../service/stream';
-import { ModelType, OptionalId } from '../types/model';
-import { ModelExpirySupport } from '../service/expiry';
-import { ModelRegistry } from '../registry/model';
-import { ModelStorageSupport } from '../service/storage';
-import { ModelCrudUtil } from '../internal/service/crud';
-import { ModelExpiryUtil } from '../internal/service/expiry';
-import { NotFoundError } from '../error/not-found';
-import { ExistsError } from '../error/exists';
-import { enforceRange, StreamModel, STREAMS } from '../internal/service/stream';
-
-type Suffix = '.bin' | '.meta' | '.json' | '.expires';
-
-const BIN = '.bin';
-const META = '.meta';
-
-@Config('model.file')
-export class FileModelConfig {
-  @Required(false)
-  folder: string;
-  namespace: string = '.';
-  autoCreate?: boolean;
-  cullRate?: number | TimeSpan;
-
-  async postConstruct(): Promise<void> {
-    this.folder ??= path.resolve(os.tmpdir(), `trv_file_${Runtime.main.name.replace(/[^a-z]/ig, '_')}`);
-  }
-}
-
-const exists = (f: string): Promise<boolean> => fs.stat(f).then(() => true, () => false);
-
-/**
- * Standard file support
- */
-@Injectable()
-export class FileModelService implements ModelCrudSupport, ModelStreamSupport, ModelExpirySupport, ModelStorageSupport {
-
-  private static async * scanFolder(folder: string, suffix: string): AsyncGenerator<[id: string, field: string]> {
-    for (const sub of await fs.readdir(folder)) {
-      for (const file of await fs.readdir(path.resolve(folder, sub))) {
-        if (file.endsWith(suffix)) {
-          yield [file.replace(suffix, ''), path.resolve(folder, sub, file)];
-        }
-      }
-    }
-  }
-
-  idSource = ModelCrudUtil.uuidSource();
-
-  get client(): string {
-    return this.config.folder;
-  }
-
-  /**
-   * The root location for all activity
-   */
-  constructor(public readonly config: FileModelConfig) { }
-
-  async #resolveName<T extends ModelType>(cls: Class<T> | string, suffix?: Suffix, id?: string): Promise<string> {
-    const name = typeof cls === 'string' ? cls : ModelRegistry.getStore(cls);
-    let resolved = path.resolve(this.config.folder, this.config.namespace, name);
-    if (id) {
-      resolved = path.resolve(resolved, id.replace(/^[/]/, '').substring(0, 3));
-    }
-    let dir = resolved;
-    if (id) {
-      resolved = path.resolve(resolved, `${id}${suffix}`);
-      dir = path.dirname(resolved);
-    }
-
-    await fs.mkdir(dir, { recursive: true });
-    return resolved;
-  }
-
-  async #find<T extends ModelType>(cls: Class<T> | string, suffix: Suffix, id?: string): Promise<string> {
-    const file = await this.#resolveName(cls, suffix, id);
-    if (id && !(await exists(file))) {
-      throw new NotFoundError(cls, id);
-    }
-    return file;
-  }
-
-  postConstruct(): void {
-    ModelExpiryUtil.registerCull(this);
-  }
-
-  checkExpiry<T extends ModelType>(cls: Class<T>, item: T): T {
-    const { expiresAt } = ModelRegistry.get(cls);
-    if (expiresAt && ModelExpiryUtil.getExpiryState(cls, item).expired) {
-      throw new NotFoundError(cls, item.id);
-    }
-    return item;
-  }
-
-  async get<T extends ModelType>(cls: Class<T>, id: string): Promise<T> {
-    await this.#find(cls, '.json', id);
-
-    const file = await this.#resolveName(cls, '.json', id);
-
-    if (await exists(file)) {
-      const content = await fs.readFile(file);
-      return this.checkExpiry(cls, await ModelCrudUtil.load(cls, content));
-    }
-
-    throw new NotFoundError(cls, id);
-  }
-
-  async create<T extends ModelType>(cls: Class<T>, item: OptionalId<T>): Promise<T> {
-    if (!item.id) {
-      item.id = this.idSource.create();
-    }
-
-    const file = await this.#resolveName(cls, '.json', item.id);
-
-    if (await exists(file)) {
-      throw new ExistsError(cls, item.id!);
-    }
-
-    return await this.upsert(cls, item);
-  }
-
-  async update<T extends ModelType>(cls: Class<T>, item: T): Promise<T> {
-    await this.get(cls, item.id);
-    return await this.upsert(cls, item);
-  }
-
-  async upsert<T extends ModelType>(cls: Class<T>, item: OptionalId<T>): Promise<T> {
-    ModelCrudUtil.ensureNotSubType(cls);
-    const prepped = await ModelCrudUtil.preStore(cls, item, this);
-
-    const file = await this.#resolveName(cls, '.json', item.id);
-    await fs.writeFile(file, JSON.stringify(item), { encoding: 'utf8' });
-
-    return prepped;
-  }
-
-  async updatePartial<T extends ModelType>(cls: Class<T>, item: Partial<T> & { id: string }, view?: string): Promise<T> {
-    ModelCrudUtil.ensureNotSubType(cls);
-    const id = item.id;
-    item = await ModelCrudUtil.naivePartialUpdate(cls, item, view, () => this.get(cls, id));
-    const file = await this.#resolveName(cls, '.json', item.id);
-    await fs.writeFile(file, JSON.stringify(item), { encoding: 'utf8' });
-
-    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-    return item as T;
-  }
-
-  async delete<T extends ModelType>(cls: Class<T>, id: string): Promise<void> {
-    const file = await this.#find(cls, '.json', id);
-    await fs.unlink(file);
-  }
-
-  async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T> {
-    for await (const [id] of FileModelService.scanFolder(await this.#resolveName(cls, '.json'), '.json')) {
-      try {
-        yield await this.get(cls, id);
-      } catch (err) {
-        if (!(err instanceof NotFoundError)) {
-          throw err;
-        }
-      }
-    }
-  }
-
-  // Stream
-  async upsertStream(location: string, input: Readable, meta: StreamMeta): Promise<void> {
-    const file = await this.#resolveName(STREAMS, BIN, location);
-    await Promise.all([
-      await pipeline(input, createWriteStream(file)),
-      fs.writeFile(file.replace(BIN, META), JSON.stringify(meta), 'utf8')
-    ]);
-  }
-
-  async getStream(location: string, range?: StreamRange): Promise<Readable> {
-    const file = await this.#find(STREAMS, BIN, location);
-    if (range) {
-      const meta = await this.describeStream(location);
-      range = enforceRange(range, meta.size);
-    }
-    return createReadStream(file, range);
-  }
-
-  async describeStream(location: string): Promise<StreamMeta> {
-    const file = await this.#find(STREAMS, META, location);
-    const content = await fs.readFile(file);
-    const text: StreamMeta = JSON.parse(content.toString('utf8'));
-    return text;
-  }
-
-  async deleteStream(location: string): Promise<void> {
-    const file = await this.#resolveName(STREAMS, BIN, location);
-    if (await exists(file)) {
-      await Promise.all([
-        fs.unlink(file),
-        fs.unlink(file.replace('.bin', META))
-      ]);
-    } else {
-      throw new NotFoundError('Stream', location);
-    }
-  }
-
-  // Expiry
-  async deleteExpired<T extends ModelType>(cls: Class<T>): Promise<number> {
-    return ModelExpiryUtil.naiveDeleteExpired(this, cls);
-  }
-
-  // Storage management
-  async createStorage(): Promise<void> {
-    const dir = path.resolve(this.config.folder, this.config.namespace);
-    await fs.mkdir(dir, { recursive: true });
-  }
-
-  async deleteStorage(): Promise<void> {
-    await fs.rm(path.resolve(this.config.folder, this.config.namespace), { recursive: true, force: true });
-  }
-
-  async truncateModel(cls: Class<ModelType>): Promise<void> {
-    await fs.rm(await this.#resolveName(cls === StreamModel ? STREAMS : cls), { recursive: true, force: true });
-  }
-}

package/src/provider/memory.ts

@@ -1,341 +0,0 @@
-import { Readable } from 'node:stream';
-import { buffer as toBuffer } from 'node:stream/consumers';
-
-import { Class, TimeSpan, DeepPartial } from '@travetto/runtime';
-import { Injectable } from '@travetto/di';
-import { Config } from '@travetto/config';
-
-import { ModelCrudSupport } from '../service/crud';
-import { ModelStreamSupport, StreamMeta, StreamRange } from '../service/stream';
-import { ModelType, OptionalId } from '../types/model';
-import { ModelExpirySupport } from '../service/expiry';
-import { ModelRegistry } from '../registry/model';
-import { ModelStorageSupport } from '../service/storage';
-import { ModelCrudUtil } from '../internal/service/crud';
-import { ModelExpiryUtil } from '../internal/service/expiry';
-import { NotFoundError } from '../error/not-found';
-import { ExistsError } from '../error/exists';
-import { ModelIndexedSupport } from '../service/indexed';
-import { ModelIndexedUtil } from '../internal/service/indexed';
-import { ModelStorageUtil } from '../internal/service/storage';
-import { enforceRange, StreamModel, STREAMS } from '../internal/service/stream';
-import { IndexConfig } from '../registry/types';
-
-const STREAM_META = `${STREAMS}_meta`;
-
-type StoreType = Map<string, Buffer>;
-
-
-@Config('model.memory')
-export class MemoryModelConfig {
-  autoCreate?: boolean = true;
-  namespace?: string;
-  cullRate?: number | TimeSpan;
-}
-
-function indexName<T extends ModelType>(cls: Class<T>, idx: IndexConfig<T> | string, suffix?: string): string {
-  return [cls.Ⲑid, typeof idx === 'string' ? idx : idx.name, suffix].filter(x => !!x).join(':');
-}
-
-function getFirstId(data: Map<string, unknown> | Set<string>, value?: string | number): string | undefined {
-  let id: string | undefined;
-  if (data instanceof Set) {
-    id = data.values().next().value;
-  } else {
-    id = [...data.entries()].find(([k, v]) => value === undefined || v === value)?.[0];
-  }
-  return id;
-}
-
-/**
- * Standard in-memory support
- */
-@Injectable()
-export class MemoryModelService implements ModelCrudSupport, ModelStreamSupport, ModelExpirySupport, ModelStorageSupport, ModelIndexedSupport {
-
-  #store = new Map<string, StoreType>();
-  #indices = {
-    sorted: new Map<string, Map<string, Map<string, number>>>(),
-    unsorted: new Map<string, Map<string, Set<string>>>()
-  };
-
-  idSource = ModelCrudUtil.uuidSource();
-  get client(): Map<string, StoreType> { return this.#store; }
-
-  constructor(public readonly config: MemoryModelConfig) { }
-
-  #getStore<T extends ModelType>(cls: Class<T> | string): StoreType {
-    const key = typeof cls === 'string' ? cls : ModelRegistry.getStore(cls);
-    if (!this.#store.has(key)) {
-      this.#store.set(key, new Map());
-    }
-    return this.#store.get(key)!;
-  }
-
-  #find<T extends ModelType>(cls: Class<T> | string, id?: string, errorState?: 'data' | 'notfound'): StoreType {
-    const store = this.#getStore(cls);
-
-    if (id && errorState && (errorState === 'notfound' ? !store.has(id) : store.has(id))) {
-      throw errorState === 'notfound' ? new NotFoundError(cls, id) : new ExistsError(cls, id);
-    }
-
-    return store;
-  }
-
-  async #removeIndices<T extends ModelType>(cls: Class<T>, id: string): Promise<void> {
-    try {
-      const item = await this.get(cls, id);
-      for (const idx of ModelRegistry.getIndices(cls, ['sorted', 'unsorted'])) {
-        const idxName = indexName(cls, idx);
-        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-        const { key } = ModelIndexedUtil.computeIndexKey(cls, idx, item as DeepPartial<T>);
-        this.#indices[idx.type].get(idxName)?.get(key)?.delete(id);
-      }
-    } catch (err) {
-      if (!(err instanceof NotFoundError)) {
-        throw err;
-      }
-    }
-  }
-
-  async #writeIndices<T extends ModelType>(cls: Class<T>, item: T): Promise<void> {
-    for (const idx of ModelRegistry.getIndices(cls, ['sorted', 'unsorted'])) {
-      const idxName = indexName(cls, idx);
-      // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-      const { key, sort } = ModelIndexedUtil.computeIndexKey(cls, idx, item as DeepPartial<T>);
-      let index = this.#indices[idx.type].get(idxName)?.get(key);
-
-      if (!index) {
-        if (!this.#indices[idx.type].has(idxName)) {
-          this.#indices[idx.type].set(idxName, new Map());
-        }
-        if (idx.type === 'sorted') {
-          this.#indices[idx.type].get(idxName)!.set(key, index = new Map());
-        } else {
-          this.#indices[idx.type].get(idxName)!.set(key, index = new Set());
-        }
-      }
-
-      if (index instanceof Map) {
-        index?.set(item.id, +sort!);
-      } else {
-        index?.add(item.id);
-      }
-    }
-  }
-
-  async #persist<T extends ModelType>(cls: Class<T>, item: T, action: 'remove'): Promise<void>;
-  async #persist<T extends ModelType>(cls: Class<T>, item: T, action: 'write'): Promise<T>;
-  async #persist<T extends ModelType>(cls: Class<T>, item: T, action: 'write' | 'remove'): Promise<T | void> {
-    const store = this.#getStore(cls);
-    await this.#removeIndices(cls, item.id);
-    if (action === 'write') {
-      store.set(item.id, Buffer.from(JSON.stringify(item)));
-      await this.#writeIndices(cls, item);
-      return item;
-    } else {
-      store.delete(item.id);
-    }
-  }
-
-  async #getIdByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<string> {
-    const config = ModelRegistry.getIndex(cls, idx, ['sorted', 'unsorted']);
-    const { key, sort } = ModelIndexedUtil.computeIndexKey(cls, config, body);
-    const index = this.#indices[config.type].get(indexName(cls, idx))?.get(key);
-    let id: string | undefined;
-    if (index) {
-      if (index instanceof Map) {
-        id = getFirstId(index, +sort!); // Grab first id
-      } else {
-        id = getFirstId(index); // Grab first id
-      }
-    }
-    if (id) {
-      return id;
-    }
-    throw new NotFoundError(cls, key);
-  }
-
-  async postConstruct(): Promise<void> {
-    await ModelStorageUtil.registerModelChangeListener(this);
-    ModelExpiryUtil.registerCull(this);
-
-    for (const el of ModelRegistry.getClasses()) {
-      for (const idx of ModelRegistry.get(el).indices ?? []) {
-        switch (idx.type) {
-          case 'unique': {
-            console.error('Unique indices are not supported for', { cls: el.Ⲑid, idx: idx.name });
-            break;
-          }
-        }
-      }
-    }
-  }
-
-  // CRUD Support
-  async get<T extends ModelType>(cls: Class<T>, id: string): Promise<T> {
-    const store = this.#getStore(cls);
-    if (store.has(id)) {
-      const res = await ModelCrudUtil.load(cls, store.get(id)!);
-      if (res) {
-        if (ModelRegistry.get(cls).expiresAt) {
-          if (!ModelExpiryUtil.getExpiryState(cls, res).expired) {
-            return res;
-          }
-        } else {
-          return res;
-        }
-      }
-    }
-    throw new NotFoundError(cls, id);
-  }
-
-  async create<T extends ModelType>(cls: Class<T>, item: OptionalId<T>): Promise<T> {
-    if (!item.id) {
-      item.id = this.idSource.create();
-    }
-    this.#find(cls, item.id, 'data');
-    return await this.upsert(cls, item);
-  }
-
-  async update<T extends ModelType>(cls: Class<T>, item: T): Promise<T> {
-    await this.get(cls, item.id);
-    return await this.upsert(cls, item);
-  }
-
-  async upsert<T extends ModelType>(cls: Class<T>, item: OptionalId<T>): Promise<T> {
-    const store = this.#getStore(cls);
-    if (item.id && store.has(item.id)) {
-      await ModelCrudUtil.load(cls, store.get(item.id)!, 'exists');
-    }
-    const prepped = await ModelCrudUtil.preStore(cls, item, this);
-    return await this.#persist(cls, prepped, 'write');
-  }
-
-  async updatePartial<T extends ModelType>(cls: Class<T>, item: Partial<T> & { id: string }, view?: string): Promise<T> {
-    const id = item.id;
-    const clean = await ModelCrudUtil.naivePartialUpdate(cls, item, view, () => this.get(cls, id));
-    return await this.#persist(cls, clean, 'write');
-  }
-
-  async delete<T extends ModelType>(cls: Class<T>, id: string): Promise<void> {
-    const store = this.#getStore(cls);
-    if (!store.has(id)) {
-      throw new NotFoundError(cls, id);
-    }
-    await ModelCrudUtil.load(cls, store.get(id)!);
-    const where: ModelType = { id };
-    await this.#persist(cls, where, 'remove');
-  }
-
-  async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T> {
-    for (const id of this.#getStore(cls).keys()) {
-      try {
-        yield await this.get(cls, id);
-      } catch (err) {
-        if (!(err instanceof NotFoundError)) {
-          throw err;
-        }
-      }
-    }
-  }
-
-  // Stream Support
-  async upsertStream(location: string, input: Readable, meta: StreamMeta): Promise<void> {
-    const streams = this.#getStore(STREAMS);
-    const metaContent = this.#getStore(STREAM_META);
-    metaContent.set(location, Buffer.from(JSON.stringify(meta)));
-    streams.set(location, await toBuffer(input));
-  }
-
-  async getStream(location: string, range?: StreamRange): Promise<Readable> {
-    const streams = this.#find(STREAMS, location, 'notfound');
-    let buffer = streams.get(location)!;
-    if (range) {
-      range = enforceRange(range, buffer.length);
-      buffer = buffer.subarray(range.start, range.end! + 1);
-    }
-    return Readable.from(buffer);
-  }
-
-  async describeStream(location: string): Promise<StreamMeta> {
-    const metaContent = this.#find(STREAM_META, location, 'notfound');
-    const meta: StreamMeta = JSON.parse(metaContent.get(location)!.toString('utf8'));
-    return meta;
-  }
-
-  async deleteStream(location: string): Promise<void> {
-    const streams = this.#getStore(STREAMS);
-    const metaContent = this.#getStore(STREAM_META);
-    if (streams.has(location)) {
-      streams.delete(location);
-      metaContent.delete(location);
-    } else {
-      throw new NotFoundError('Stream', location);
-    }
-  }
-
-  // Expiry
-  async deleteExpired<T extends ModelType>(cls: Class<T>): Promise<number> {
-    return ModelExpiryUtil.naiveDeleteExpired(this, cls);
-  }
-
-  // Storage Support
-  async createStorage(): Promise<void> {
-  }
-
-  async deleteStorage(): Promise<void> {
-    this.#store.clear();
-    this.#indices.sorted.clear();
-    this.#indices.unsorted.clear();
-  }
-
-
-  async createModel<T extends ModelType>(cls: Class<T>): Promise<void> {
-    for (const idx of ModelRegistry.get(cls).indices ?? []) {
-      if (idx.type === 'sorted' || idx.type === 'unsorted') {
-        this.#indices[idx.type].set(indexName(cls, idx), new Map());
-      }
-    }
-  }
-
-  async truncateModel<T extends ModelType>(cls: Class<T>): Promise<void> {
-    if (cls === StreamModel) {
-      this.#getStore(STREAMS).clear();
-      this.#getStore(STREAM_META).clear();
-    } else {
-      this.#getStore(cls).clear();
-    }
-  }
-
-  // Indexed
-  async getByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<T> {
-    return this.get(cls, await this.#getIdByIndex(cls, idx, body));
-  }
-
-  async deleteByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<void> {
-    await this.delete(cls, await this.#getIdByIndex(cls, idx, body));
-  }
-
-  upsertByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: OptionalId<T>): Promise<T> {
-    return ModelIndexedUtil.naiveUpsert(this, cls, idx, body);
-  }
-
-  async * listByIndex<T extends ModelType>(cls: Class<T>, idx: string, body?: DeepPartial<T>): AsyncIterable<T> {
-    const config = ModelRegistry.getIndex(cls, idx, ['sorted', 'unsorted']);
-    const { key } = ModelIndexedUtil.computeIndexKey(cls, idx, body, { emptySortValue: null });
-    const index = this.#indices[config.type].get(indexName(cls, idx))?.get(key);
-
-    if (index) {
-      if (index instanceof Set) {
-        for (const id of index) {
-          yield this.get(cls, id);
-        }
-      } else {
-        for (const id of [...index.entries()].sort((a, b) => +a[1] - +b[1]).map(([a, b]) => a)) {
-          yield this.get(cls, id);
-        }
-      }
-    }
-  }
-}

package/src/service/stream.ts

@@ -1,72 +0,0 @@
-import { Readable } from 'node:stream';
-
-export interface StreamMeta {
-  /**
-   * File size
-   */
-  size: number;
-  /**
-   * Mime type of the content
-   */
-  contentType: string;
-  /**
-   * Hash of the file contents.  Different files with the same name, will have the same hash
-   */
-  hash: string;
-  /**
-   * The original base filename of the file
-   */
-  filename: string;
-  /**
-   * Filenames title, optional for elements like images, audio, videos
-   */
-  title?: string;
-  /**
-   * Content encoding
-   */
-  contentEncoding?: string;
-  /**
-   * Content language
-   */
-  contentLanguage?: string;
-  /**
-   * Cache control
-   */
-  cacheControl?: string;
-}
-
-export type StreamRange = { start: number, end?: number };
-
-/**
- * Support for Streams CRD.  Stream update is not supported.
- *
- * @concrete ../internal/service/common#ModelStreamSupportTarget
- */
-export interface ModelStreamSupport {
-
-  /**
-   * Upsert stream to storage
-   * @param location The location of the stream
-   * @param input The actual stream to write
-   * @param meta The stream metadata
-   */
-  upsertStream(location: string, input: Readable, meta: StreamMeta): Promise<void>;
-
-  /**
-   * Get stream from asset store
-   * @param location The location of the stream
-   */
-  getStream(location: string, range?: StreamRange): Promise<Readable>;
-
-  /**
-   * Get metadata for stream
-   * @param location The location of the stream
-   */
-  describeStream(location: string): Promise<StreamMeta>;
-
-  /**
-   * Delete stream by location
-   * @param location The location of the stream
-   */
-  deleteStream(location: string): Promise<void>;
-}