@travetto/model-memory 8.0.0-alpha.3 → 8.0.0-alpha.30

package/README.md

@@ -14,7 +14,7 @@ yarn add @travetto/model-memory
 ```
 
 This module provides a memory-based implementation for the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations."). Supported features:
-   *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L11)
-   *  [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10)
-   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model/src/types/indexed.ts#L11)
    *  [Blob](https://github.com/travetto/travetto/tree/main/module/model/src/types/blob.ts#L8)
+   *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L10)
+   *  [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10)
+   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model-indexed/src/types/service.ts#L21)

package/__index__.ts

@@ -1 +1 @@
-export * from './src/service.ts';
+export * from './src/service.ts';

package/package.json

@@ -1,39 +1,43 @@
 {
   "name": "@travetto/model-memory",
-  "version": "8.0.0-alpha.3",
-  "type": "module",
+  "version": "8.0.0-alpha.30",
   "description": "Memory backing for the travetto model module.",
   "keywords": [
     "datastore",
     "memory",
     "schema",
-    "typescript",
-    "travetto"
+    "travetto",
+    "typescript"
   ],
   "homepage": "https://travetto.io",
   "license": "MIT",
   "author": {
-    "email": "travetto.framework@gmail.com",
-    "name": "Travetto Framework"
+    "name": "Travetto Framework",
+    "email": "travetto.framework@gmail.com"
+  },
+  "repository": {
+    "url": "git+https://github.com/travetto/travetto.git",
+    "directory": "module/model-memory"
   },
   "files": [
     "__index__.ts",
     "src"
   ],
+  "type": "module",
   "main": "__index__.ts",
-  "repository": {
-    "url": "git+https://github.com/travetto/travetto.git",
-    "directory": "module/model-memory"
+  "publishConfig": {
+    "access": "public"
   },
   "dependencies": {
-    "@travetto/config": "^8.0.0-alpha.3",
-    "@travetto/di": "^8.0.0-alpha.3",
-    "@travetto/model": "^8.0.0-alpha.3",
-    "@travetto/schema": "^8.0.0-alpha.3"
+    "@travetto/config": "^8.0.0-alpha.27",
+    "@travetto/di": "^8.0.0-alpha.24",
+    "@travetto/model": "^8.0.0-alpha.28",
+    "@travetto/model-indexed": "^8.0.0-alpha.30",
+    "@travetto/schema": "^8.0.0-alpha.27"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.4",
-    "@travetto/test": "^8.0.0-alpha.3"
+    "@travetto/cli": "^8.0.0-alpha.33",
+    "@travetto/test": "^8.0.0-alpha.26"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {
@@ -45,8 +49,5 @@
   },
   "travetto": {
     "displayName": "Memory Model Support"
-  },
-  "publishConfig": {
-    "access": "public"
   }
 }

package/src/service.ts

@@ -1,20 +1,61 @@
-import {
-  type Class, type TimeSpan, type DeepPartial, castTo, type BinaryMetadata,
-  type ByteRange, type BinaryType, BinaryUtil, type BinaryArray, JSONUtil, BinaryMetadataUtil
-} from '@travetto/runtime';
-import { Injectable, PostConstruct } from '@travetto/di';
 import { Config } from '@travetto/config';
+import { Injectable, PostConstruct } from '@travetto/di';
 import {
-  type ModelType, type IndexConfig, type ModelCrudSupport, type ModelExpirySupport, type ModelStorageSupport, type ModelIndexedSupport,
-  ModelRegistryIndex, NotFoundError, ExistsError, type OptionalId, type ModelBlobSupport,
-  ModelCrudUtil, ModelExpiryUtil, ModelIndexedUtil, ModelStorageUtil
+  ExistsError,
+  IndexNotSupported,
+  type ModelBlobSupport,
+  type ModelCrudSupport,
+  ModelCrudUtil,
+  type ModelExpirySupport,
+  ModelExpiryUtil,
+  type ModelListOptions,
+  ModelRegistryIndex,
+  type ModelStorageSupport,
+  ModelStorageUtil,
+  type ModelType,
+  NotFoundError,
+  type OptionalId,
+  UniqueError
 } from '@travetto/model';
+import {
+  type AllIndexes,
+  type FullKeyedIndexBody,
+  type FullKeyedIndexWithPartialBody,
+  isModelIndexedIndex,
+  type KeyedIndexBody,
+  type KeyedIndexSelection,
+  ModelIndexedComputedIndex,
+  type ModelIndexedSearchOptions,
+  type ModelIndexedSupport,
+  ModelIndexedUtil,
+  type ModelPageOptions,
+  type ModelPageResult,
+  type SingleItemIndex,
+  type SortedIndex,
+  type SortedIndexSelection,
+  type SortedIndexSelectionType
+} from '@travetto/model-indexed';
+import {
+  type BinaryArray,
+  type BinaryMetadata,
+  BinaryMetadataUtil,
+  type BinaryType,
+  BinaryUtil,
+  type ByteRange,
+  type Class,
+  castTo,
+  JSONUtil,
+  type TimeSpan
+} from '@travetto/runtime';
 
 const ModelBlobNamespace = '__blobs';
 const ModelBlobMetaNamespace = `${ModelBlobNamespace}_meta`;
 
 type StoreType = Map<string, BinaryArray>;
 
+const sortValue = (a: string | number, b: string | number): number =>
+  typeof a === 'number' && typeof b === 'number' ? a - b : typeof a === 'string' && typeof b === 'string' ? a.localeCompare(b) : 0;
+
 @Config('model.memory')
 export class MemoryModelConfig {
   modifyStorage?: boolean = true;
@@ -22,8 +63,8 @@ export class MemoryModelConfig {
   cullRate?: number | TimeSpan;
 }
 
-function indexName<T extends ModelType>(cls: Class<T>, idx: IndexConfig<T> | string, suffix?: string): string {
-  return [cls.Ⲑid, typeof idx === 'string' ? idx : idx.name, suffix].filter(part => !!part).join(':');
+function indexName<T extends ModelType>(cls: Class<T>, idx: AllIndexes<T>, suffix?: string): string {
+  return [cls.Ⲑid, idx.name, suffix].filter(part => !!part).join(':');
 }
 
 function getFirstId(data: Map<string, unknown> | Set<string>, value?: string | number): string | undefined {
@@ -40,20 +81,25 @@ function getFirstId(data: Map<string, unknown> | Set<string>, value?: string | n
  * Standard in-memory support
  */
 @Injectable()
-export class MemoryModelService implements ModelCrudSupport, ModelBlobSupport, ModelExpirySupport, ModelStorageSupport, ModelIndexedSupport {
-
+export class MemoryModelService
+  implements ModelCrudSupport, ModelBlobSupport, ModelExpirySupport, ModelStorageSupport, ModelIndexedSupport
+{
   #store = new Map<string, StoreType>();
   #indices = {
-    sorted: new Map<string, Map<string, Map<string, number>>>(),
-    unsorted: new Map<string, Map<string, Set<string>>>()
-  };
+    'indexed:sorted': new Map<string, Map<string, Map<string, number | string>>>(),
+    'indexed:keyed': new Map<string, Map<string, Set<string>>>()
+  } as const;
 
   idSource = ModelCrudUtil.uuidSource();
   config: MemoryModelConfig;
 
-  constructor(config: MemoryModelConfig) { this.config = config; }
+  constructor(config: MemoryModelConfig) {
+    this.config = config;
+  }
 
-  get client(): Map<string, StoreType> { return this.#store; }
+  get client(): Map<string, StoreType> {
+    return this.#store;
+  }
 
   #getStore<T extends ModelType>(cls: Class<T> | string): StoreType {
     const key = typeof cls === 'string' ? cls : ModelRegistryIndex.getStoreName(cls);
@@ -73,10 +119,18 @@ export class MemoryModelService implements ModelCrudSupport, ModelBlobSupport, M
   async #removeIndices<T extends ModelType>(cls: Class<T>, id: string): Promise<void> {
     try {
       const item = await this.get(cls, id);
-      for (const idx of ModelRegistryIndex.getIndices(cls, ['sorted', 'unsorted'])) {
+      for (const idx of ModelRegistryIndex.getIndices(cls)) {
+        if (!isModelIndexedIndex(idx)) {
+          continue; // Only support ModelIndexed indices
+        }
         const idxName = indexName(cls, idx);
-        const { key } = ModelIndexedUtil.computeIndexKey(cls, idx, castTo(item));
-        this.#indices[idx.type].get(idxName)?.get(key)?.delete(id);
+        const computed = ModelIndexedComputedIndex.get(idx, item).validate({ sort: true });
+        switch (idx.type) {
+          case 'indexed:sorted':
+          case 'indexed:keyed':
+            this.#indices[idx.type].get(idxName)?.get(computed.getKey())?.delete(id);
+            break;
+        }
       }
     } catch (error) {
       if (!(error instanceof NotFoundError)) {
@@ -86,14 +140,28 @@ export class MemoryModelService implements ModelCrudSupport, ModelBlobSupport, M
   }
 
   async #writeIndices<T extends ModelType>(cls: Class<T>, item: T): Promise<void> {
-    for (const idx of ModelRegistryIndex.getIndices(cls, ['sorted', 'unsorted'])) {
+    for (const idx of ModelRegistryIndex.getIndices(cls)) {
+      if (!isModelIndexedIndex(idx)) {
+        continue; // Only support ModelIndexed indices
+      }
       const idxName = indexName(cls, idx);
-      const { key, sort } = ModelIndexedUtil.computeIndexKey(cls, idx, castTo(item));
-
-      if (idx.type === 'sorted') {
-        this.#indices[idx.type].getOrInsert(idxName, new Map()).getOrInsert(key, new Map()).set(item.id, +sort!);
-      } else {
-        this.#indices[idx.type].getOrInsert(idxName, new Map()).getOrInsert(key, new Set()).add(item.id);
+      const computed = ModelIndexedComputedIndex.get(idx, item).validate({ sort: true });
+      const key = computed.getKey();
+      switch (idx.type) {
+        case 'indexed:keyed': {
+          if (idx.unique) {
+            const existing = this.#indices[idx.type].get(idxName)?.get(key);
+            if (existing && existing.size > 0 && !existing.has(item.id)) {
+              throw new UniqueError(cls, key);
+            }
+          }
+          this.#indices[idx.type].getOrInsert(idxName, new Map()).getOrInsert(key, new Set()).add(item.id);
+          break;
+        }
+        case 'indexed:sorted': {
+          this.#indices[idx.type].getOrInsert(idxName, new Map()).getOrInsert(key, new Map()).set(item.id, computed.getSort());
+          break;
+        }
       }
     }
   }
@@ -112,22 +180,82 @@ export class MemoryModelService implements ModelCrudSupport, ModelBlobSupport, M
     }
   }
 
-  async #getIdByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<string> {
-    const config = ModelRegistryIndex.getIndex(cls, idx, ['sorted', 'unsorted']);
-    const { key, sort } = ModelIndexedUtil.computeIndexKey(cls, config, body);
-    const index = this.#indices[config.type].get(indexName(cls, idx))?.get(key);
+  async #getIdByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: FullKeyedIndexBody<T, K, S>
+  ): Promise<string> {
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate({ sort: true });
+
+    const index = this.#indices[idx.type].get(indexName(cls, idx))?.get(computed.getKey());
     let id: string | undefined;
     if (index) {
-      if (index instanceof Map) {
-        id = getFirstId(index, +sort!); // Grab first id
+      if (computed.idPart) {
+        if (index.has(computed.idPart.value)) {
+          id = computed.idPart.value;
+        } else {
+          throw new NotFoundError(cls, computed.getKey({ sort: true }));
+        }
       } else {
-        id = getFirstId(index); // Grab first id
+        if (index instanceof Map) {
+          id = getFirstId(index, computed.getSort()); // Grab first id
+        } else if (index instanceof Set) {
+          id = getFirstId(index); // Grab first id
+        }
       }
     }
     if (id) {
       return id;
     }
-    throw new NotFoundError(cls, key);
+    throw new NotFoundError(cls, computed.getKey({ sort: true }));
+  }
+
+  async *#iterateIds(ids: string[], options?: ModelListOptions & ModelPageOptions<number>): AsyncIterable<string[]> {
+    let offset = options && 'offset' in options ? (options.offset ?? 0) : 0;
+    const batchSize = options?.batchSizeHint ?? 100;
+    let produced = 0;
+    const maxCount = options?.limit ?? Number.MAX_SAFE_INTEGER;
+    while (offset < ids.length && produced < maxCount) {
+      if (options?.abort?.aborted) {
+        break;
+      }
+      const end = Math.min(offset + batchSize, ids.length, maxCount - produced + offset);
+      const batch = ids.slice(offset, end);
+      if (batch.length) {
+        yield batch;
+      }
+      offset += batchSize;
+      produced += batch.length;
+    }
+  }
+
+  async *#getIndexIds<T extends ModelType>(
+    cls: Class<T>,
+    idx: AllIndexes<T>,
+    body: KeyedIndexBody<T>,
+    options?: ModelListOptions & ModelPageOptions<number>,
+    filterIndex?: (indexValue: string | number) => boolean
+  ): AsyncIterable<string[]> {
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate();
+    if (!isModelIndexedIndex(idx)) {
+      throw new IndexNotSupported(cls, idx, 'Only ModelIndexed indices can be used with MemoryModelService');
+    }
+
+    const base = this.#indices[idx.type].get(indexName(cls, idx));
+    const index = base?.get(computed.getKey());
+    let ids: string[];
+    if (!index) {
+      ids = [];
+    } else if (index instanceof Map) {
+      ids = [...index.entries()]
+        .filter(([, sort]) => (filterIndex ? filterIndex(sort) : true))
+        .sort((a, b) => sortValue(a[1], b[1]))
+        .map(([id]) => id);
+    } else {
+      ids = [...index];
+    }
+
+    yield* this.#iterateIds(ids, options);
   }
 
   @PostConstruct()
@@ -136,12 +264,9 @@ export class MemoryModelService implements ModelCrudSupport, ModelBlobSupport, M
     ModelExpiryUtil.registerCull(this);
 
     for (const cls of ModelRegistryIndex.getClasses()) {
-      for (const idx of ModelRegistryIndex.getConfig(cls).indices ?? []) {
-        switch (idx.type) {
-          case 'unique': {
-            console.error('Unique indices are not supported for', { cls: cls.Ⲑid, idx: idx.name });
-            break;
-          }
+      for (const idx of Object.values(ModelRegistryIndex.getConfig(cls).indices ?? {})) {
+        if (!isModelIndexedIndex(idx)) {
+          console.error(`Indices of type ${idx.type} are not supported for`, { cls: cls.Ⲑid, name: idx.name, type: idx.type });
         }
       }
     }
@@ -203,21 +328,21 @@ export class MemoryModelService implements ModelCrudSupport, ModelBlobSupport, M
     await this.#persist(cls, where, 'remove');
   }
 
-  async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T> {
-    for (const id of this.#getStore(cls).keys()) {
-      try {
-        yield await this.get(cls, id);
-      } catch (error) {
-        if (!(error instanceof NotFoundError)) {
-          throw error;
-        }
-      }
+  async *list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]> {
+    for await (const batch of this.#iterateIds([...this.#getStore(cls).keys()], options)) {
+      yield ModelCrudUtil.filterOutNotFound(batch.map(id => this.get(cls, id)));
     }
   }
 
   // Blob Support
   async upsertBlob(location: string, input: BinaryType, metadata?: BinaryMetadata, overwrite = true): Promise<void> {
-    if (!overwrite && await this.getBlobMetadata(location).then(() => true, () => false)) {
+    if (
+      !overwrite &&
+      (await this.getBlobMetadata(location).then(
+        () => true,
+        () => false
+      ))
+    ) {
       return;
     }
     const resolved = await BinaryMetadataUtil.compute(input, metadata);
@@ -228,7 +353,6 @@ export class MemoryModelService implements ModelCrudSupport, ModelBlobSupport, M
   }
 
   async getBlob(location: string, range?: ByteRange): Promise<Blob> {
-
     const blobs = this.#find(ModelBlobNamespace, location, 'notfound');
 
     let data = blobs.get(location)!;
@@ -274,24 +398,24 @@ export class MemoryModelService implements ModelCrudSupport, ModelBlobSupport, M
           store.delete(key);
           deleted += 1;
         }
-      } catch { } // Do not let a single error stop the process
+      } catch {} // Do not let a single error stop the process
     }
     return deleted;
   }
 
   // Storage Support
-  async createStorage(): Promise<void> {
-  }
+  async createStorage(): Promise<void> {}
 
   async deleteStorage(): Promise<void> {
     this.#store.clear();
-    this.#indices.sorted.clear();
-    this.#indices.unsorted.clear();
+    for (const value of Object.values(this.#indices)) {
+      value.clear();
+    }
   }
 
   async upsertModel<T extends ModelType>(cls: Class<T>): Promise<void> {
-    for (const idx of ModelRegistryIndex.getConfig(cls).indices ?? []) {
-      if (idx.type === 'sorted' || idx.type === 'unsorted') {
+    for (const idx of ModelRegistryIndex.getIndices(cls)) {
+      if (isModelIndexedIndex(idx)) {
         this.#indices[idx.type].set(indexName(cls, idx), new Map());
       }
     }
@@ -307,33 +431,85 @@ export class MemoryModelService implements ModelCrudSupport, ModelBlobSupport, M
   }
 
   // Indexed
-  async getByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<T> {
+  async getByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: FullKeyedIndexBody<T, K, S>
+  ): Promise<T> {
     return this.get(cls, await this.#getIdByIndex(cls, idx, body));
   }
 
-  async deleteByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<void> {
+  async deleteByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: FullKeyedIndexBody<T, K, S>
+  ): Promise<void> {
     await this.delete(cls, await this.#getIdByIndex(cls, idx, body));
   }
 
-  upsertByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: OptionalId<T>): Promise<T> {
+  upsertByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: OptionalId<T>
+  ): Promise<T> {
     return ModelIndexedUtil.naiveUpsert(this, cls, idx, body);
   }
 
-  async * listByIndex<T extends ModelType>(cls: Class<T>, idx: string, body?: DeepPartial<T>): AsyncIterable<T> {
-    const config = ModelRegistryIndex.getIndex(cls, idx, ['sorted', 'unsorted']);
-    const { key } = ModelIndexedUtil.computeIndexKey(cls, idx, body, { emptySortValue: null });
-    const index = this.#indices[config.type].get(indexName(cls, idx))?.get(key);
+  updateByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: T
+  ): Promise<T> {
+    return ModelIndexedUtil.naiveUpdate(this, cls, idx, body);
+  }
 
-    if (index) {
-      if (index instanceof Set) {
-        for (const id of index) {
-          yield this.get(cls, id);
-        }
-      } else {
-        for (const id of [...index.entries()].toSorted((a, b) => +a[1] - +b[1]).map(([a,]) => a)) {
-          yield this.get(cls, id);
-        }
-      }
+  async updatePartialByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: FullKeyedIndexWithPartialBody<T, K, S>
+  ): Promise<T> {
+    const item = await ModelCrudUtil.naivePartialUpdate(cls, () => this.getByIndex(cls, idx, castTo(body)), castTo(body));
+    return this.update(cls, item);
+  }
+
+  async pageByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ModelPageOptions
+  ): Promise<ModelPageResult<T>> {
+    const offset = options?.offset ? JSONUtil.fromBase64<number>(options.offset) : 0;
+    let produced = 0;
+
+    const items: T[] = [];
+    for await (const batch of this.#getIndexIds(cls, idx, body, { limit: 100, ...options, offset })) {
+      produced += batch.length;
+      items.push(...(await ModelCrudUtil.filterOutNotFound(batch.map(id => this.get(cls, id)))));
     }
+    return { items, nextOffset: items.length ? JSONUtil.toBase64(offset + produced) : undefined };
   }
-}
+
+  async *listByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ModelListOptions
+  ): AsyncIterable<T[]> {
+    for await (const batch of this.#getIndexIds(cls, idx, body, options)) {
+      yield ModelCrudUtil.filterOutNotFound(batch.map(id => this.get(cls, id)));
+    }
+  }
+
+  async suggestByIndex<
+    T extends ModelType,
+    S extends SortedIndexSelection<T>,
+    K extends KeyedIndexSelection<T>,
+    B extends SortedIndexSelectionType<T, S> & string
+  >(cls: Class<T>, idx: SortedIndex<T, K, S>, body: KeyedIndexBody<T, K>, prefix: B, options?: ModelIndexedSearchOptions): Promise<T[]> {
+    const items: T[] = [];
+    for await (const batch of this.#getIndexIds(cls, idx, body, options, id => typeof id === 'string' && id.startsWith(prefix))) {
+      items.push(...(await ModelCrudUtil.filterOutNotFound(batch.map(id => this.get(cls, id)))));
+    }
+    return items;
+  }
+}