@travetto/model-memory 8.0.0-alpha.2 → 8.0.0-alpha.20

package/README.md

@@ -16,5 +16,5 @@ yarn add @travetto/model-memory
 This module provides a memory-based implementation for the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations."). Supported features:
    *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L11)
    *  [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10)
-   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model/src/types/indexed.ts#L11)
    *  [Blob](https://github.com/travetto/travetto/tree/main/module/model/src/types/blob.ts#L8)
+   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model-indexed/src/types/service.ts#L16)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-memory",
-  "version": "8.0.0-alpha.2",
+  "version": "8.0.0-alpha.20",
   "type": "module",
   "description": "Memory backing for the travetto model module.",
   "keywords": [
@@ -26,14 +26,15 @@
     "directory": "module/model-memory"
   },
   "dependencies": {
-    "@travetto/config": "^8.0.0-alpha.2",
-    "@travetto/di": "^8.0.0-alpha.2",
-    "@travetto/model": "^8.0.0-alpha.2",
-    "@travetto/schema": "^8.0.0-alpha.2"
+    "@travetto/config": "^8.0.0-alpha.18",
+    "@travetto/di": "^8.0.0-alpha.17",
+    "@travetto/model": "^8.0.0-alpha.18",
+    "@travetto/model-indexed": "^8.0.0-alpha.20",
+    "@travetto/schema": "^8.0.0-alpha.18"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.3",
-    "@travetto/test": "^8.0.0-alpha.2"
+    "@travetto/cli": "^8.0.0-alpha.23",
+    "@travetto/test": "^8.0.0-alpha.17"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/service.ts

@@ -1,20 +1,31 @@
 import {
-  type Class, type TimeSpan, type DeepPartial, castTo, type BinaryMetadata,
-  type ByteRange, type BinaryType, BinaryUtil, type BinaryArray, JSONUtil, BinaryMetadataUtil
+  type Class, type TimeSpan, castTo, type BinaryMetadata,
+  type ByteRange, type BinaryType, BinaryUtil, type BinaryArray, JSONUtil, BinaryMetadataUtil,
 } from '@travetto/runtime';
 import { Injectable, PostConstruct } from '@travetto/di';
 import { Config } from '@travetto/config';
 import {
-  type ModelType, type IndexConfig, type ModelCrudSupport, type ModelExpirySupport, type ModelStorageSupport, type ModelIndexedSupport,
-  ModelRegistryIndex, NotFoundError, ExistsError, type OptionalId, type ModelBlobSupport,
-  ModelCrudUtil, ModelExpiryUtil, ModelIndexedUtil, ModelStorageUtil
+  type ModelType, type ModelCrudSupport, type ModelExpirySupport, type ModelStorageSupport, ModelRegistryIndex,
+  NotFoundError, ExistsError, type OptionalId, type ModelBlobSupport, ModelCrudUtil, ModelExpiryUtil, ModelStorageUtil,
+  IndexNotSupported,
+  type ModelListOptions,
 } from '@travetto/model';
+import {
+  type ModelIndexedSupport, type KeyedIndexSelection, type KeyedIndexBody, type ModelPageOptions, ModelIndexedUtil,
+  type SingleItemIndex, type SortedIndexSelection, type ModelPageResult, type SortedIndex,
+  type AllIndexes, isModelIndexedIndex, type FullKeyedIndexBody, type FullKeyedIndexWithPartialBody, ModelIndexedComputedIndex,
+  type ModelIndexedSearchOptions, type SortedIndexSelectionType,
+} from '@travetto/model-indexed';
 
 const ModelBlobNamespace = '__blobs';
 const ModelBlobMetaNamespace = `${ModelBlobNamespace}_meta`;
 
 type StoreType = Map<string, BinaryArray>;
 
+const sortValue = (a: string | number, b: string | number): number =>
+  (typeof a === 'number' && typeof b === 'number') ?
+    a - b : (typeof a === 'string' && typeof b === 'string') ? a.localeCompare(b) : 0;
+
 @Config('model.memory')
 export class MemoryModelConfig {
   modifyStorage?: boolean = true;
@@ -22,8 +33,8 @@ export class MemoryModelConfig {
   cullRate?: number | TimeSpan;
 }
 
-function indexName<T extends ModelType>(cls: Class<T>, idx: IndexConfig<T> | string, suffix?: string): string {
-  return [cls.Ⲑid, typeof idx === 'string' ? idx : idx.name, suffix].filter(part => !!part).join(':');
+function indexName<T extends ModelType>(cls: Class<T>, idx: AllIndexes<T>, suffix?: string): string {
+  return [cls.Ⲑid, idx.name, suffix].filter(part => !!part).join(':');
 }
 
 function getFirstId(data: Map<string, unknown> | Set<string>, value?: string | number): string | undefined {
@@ -40,13 +51,16 @@ function getFirstId(data: Map<string, unknown> | Set<string>, value?: string | n
  * Standard in-memory support
  */
 @Injectable()
-export class MemoryModelService implements ModelCrudSupport, ModelBlobSupport, ModelExpirySupport, ModelStorageSupport, ModelIndexedSupport {
+export class MemoryModelService implements
+  ModelCrudSupport, ModelBlobSupport,
+  ModelExpirySupport, ModelStorageSupport,
+  ModelIndexedSupport {
 
   #store = new Map<string, StoreType>();
   #indices = {
-    sorted: new Map<string, Map<string, Map<string, number>>>(),
-    unsorted: new Map<string, Map<string, Set<string>>>()
-  };
+    'indexed:sorted': new Map<string, Map<string, Map<string, number | string>>>(),
+    'indexed:keyed': new Map<string, Map<string, Set<string>>>(),
+  } as const;
 
   idSource = ModelCrudUtil.uuidSource();
   config: MemoryModelConfig;
@@ -73,10 +87,16 @@ export class MemoryModelService implements ModelCrudSupport, ModelBlobSupport, M
   async #removeIndices<T extends ModelType>(cls: Class<T>, id: string): Promise<void> {
     try {
       const item = await this.get(cls, id);
-      for (const idx of ModelRegistryIndex.getIndices(cls, ['sorted', 'unsorted'])) {
+      for (const idx of ModelRegistryIndex.getIndices(cls)) {
+        if (!isModelIndexedIndex(idx)) {
+          continue; // Only support ModelIndexed indices
+        }
         const idxName = indexName(cls, idx);
-        const { key } = ModelIndexedUtil.computeIndexKey(cls, idx, castTo(item));
-        this.#indices[idx.type].get(idxName)?.get(key)?.delete(id);
+        const computed = ModelIndexedComputedIndex.get(idx, item).validate({ sort: true });
+        switch (idx.type) {
+          case 'indexed:sorted':
+          case 'indexed:keyed': this.#indices[idx.type].get(idxName)?.get(computed.getKey())?.delete(id); break;
+        }
       }
     } catch (error) {
       if (!(error instanceof NotFoundError)) {
@@ -86,14 +106,28 @@ export class MemoryModelService implements ModelCrudSupport, ModelBlobSupport, M
   }
 
   async #writeIndices<T extends ModelType>(cls: Class<T>, item: T): Promise<void> {
-    for (const idx of ModelRegistryIndex.getIndices(cls, ['sorted', 'unsorted'])) {
+    for (const idx of ModelRegistryIndex.getIndices(cls)) {
+      if (!isModelIndexedIndex(idx)) {
+        continue; // Only support ModelIndexed indices
+      }
       const idxName = indexName(cls, idx);
-      const { key, sort } = ModelIndexedUtil.computeIndexKey(cls, idx, castTo(item));
-
-      if (idx.type === 'sorted') {
-        this.#indices[idx.type].getOrInsert(idxName, new Map()).getOrInsert(key, new Map()).set(item.id, +sort!);
-      } else {
-        this.#indices[idx.type].getOrInsert(idxName, new Map()).getOrInsert(key, new Set()).add(item.id);
+      const computed = ModelIndexedComputedIndex.get(idx, item).validate({ sort: true });
+      const key = computed.getKey();
+      switch (idx.type) {
+        case 'indexed:keyed': {
+          if (idx.unique) {
+            const existing = this.#indices[idx.type].get(idxName)?.get(key);
+            if (existing && existing.size > 0 && !existing.has(item.id)) {
+              throw new ExistsError(cls, key);
+            }
+          }
+          this.#indices[idx.type].getOrInsert(idxName, new Map()).getOrInsert(key, new Set()).add(item.id);
+          break;
+        }
+        case 'indexed:sorted': {
+          this.#indices[idx.type].getOrInsert(idxName, new Map()).getOrInsert(key, new Map()).set(item.id, computed.getSort());
+          break;
+        }
       }
     }
   }
@@ -112,22 +146,87 @@ export class MemoryModelService implements ModelCrudSupport, ModelBlobSupport, M
     }
   }
 
-  async #getIdByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<string> {
-    const config = ModelRegistryIndex.getIndex(cls, idx, ['sorted', 'unsorted']);
-    const { key, sort } = ModelIndexedUtil.computeIndexKey(cls, config, body);
-    const index = this.#indices[config.type].get(indexName(cls, idx))?.get(key);
+  async #getIdByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexBody<T, K, S>): Promise<string> {
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate({ sort: true });
+
+    const index = this.#indices[idx.type].get(indexName(cls, idx))?.get(computed.getKey());
     let id: string | undefined;
     if (index) {
-      if (index instanceof Map) {
-        id = getFirstId(index, +sort!); // Grab first id
+      if (computed.idPart) {
+        if (index.has(computed.idPart.value)) {
+          id = computed.idPart.value;
+        } else {
+          throw new NotFoundError(cls, computed.getKey({ sort: true }));
+        }
       } else {
-        id = getFirstId(index); // Grab first id
+        if (index instanceof Map) {
+          id = getFirstId(index, computed.getSort()); // Grab first id
+        } else if (index instanceof Set) {
+          id = getFirstId(index); // Grab first id
+        }
       }
     }
     if (id) {
       return id;
     }
-    throw new NotFoundError(cls, key);
+    throw new NotFoundError(cls, computed.getKey({ sort: true }));
+  }
+
+  async * #iterateIds(
+    ids: string[],
+    options?: ModelListOptions & ModelPageOptions<number>,
+  ): AsyncIterable<string[]> {
+    let offset = options && 'offset' in options ? options.offset ?? 0 : 0;
+    const batchSize = options?.batchSizeHint ?? 100;
+    let produced = 0;
+    const maxCount = options?.limit ?? Number.MAX_SAFE_INTEGER;
+    while (offset < ids.length && produced < maxCount) {
+      if (options?.abort?.aborted) {
+        break;
+      }
+      const end = Math.min(offset + batchSize, ids.length, maxCount - produced + offset);
+      const batch = ids.slice(offset, end);
+      if (batch.length) {
+        yield batch;
+      }
+      offset += batchSize;
+      produced += batch.length;
+    }
+  }
+
+  async * #getIndexIds<
+    T extends ModelType,
+  >(
+    cls: Class<T>,
+    idx: AllIndexes<T>,
+    body: KeyedIndexBody<T>,
+    options?: ModelListOptions & ModelPageOptions<number>,
+    filterIndex?: (indexValue: string | number) => boolean
+  ): AsyncIterable<string[]> {
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate();
+    if (!isModelIndexedIndex(idx)) {
+      throw new IndexNotSupported(cls, idx, 'Only ModelIndexed indices can be used with MemoryModelService');
+    }
+
+    const base = this.#indices[idx.type].get(indexName(cls, idx));
+    const index = base?.get(computed.getKey());
+    let ids: string[];
+    if (!index) {
+      ids = [];
+    } else if (index instanceof Map) {
+      ids = [...index.entries()]
+        .filter(([, sort]) => filterIndex ? filterIndex(sort) : true)
+        .sort((a, b) => sortValue(a[1], b[1]))
+        .map(([id,]) => id);
+    } else {
+      ids = [...index];
+    }
+
+    yield* this.#iterateIds(ids, options);
   }
 
   @PostConstruct()
@@ -136,12 +235,9 @@ export class MemoryModelService implements ModelCrudSupport, ModelBlobSupport, M
     ModelExpiryUtil.registerCull(this);
 
     for (const cls of ModelRegistryIndex.getClasses()) {
-      for (const idx of ModelRegistryIndex.getConfig(cls).indices ?? []) {
-        switch (idx.type) {
-          case 'unique': {
-            console.error('Unique indices are not supported for', { cls: cls.Ⲑid, idx: idx.name });
-            break;
-          }
+      for (const idx of Object.values(ModelRegistryIndex.getConfig(cls).indices ?? {})) {
+        if (!isModelIndexedIndex(idx)) {
+          console.error(`Indices of type ${idx.type} are not supported for`, { cls: cls.Ⲑid, name: idx.name, type: idx.type });
         }
       }
     }
@@ -203,15 +299,9 @@ export class MemoryModelService implements ModelCrudSupport, ModelBlobSupport, M
     await this.#persist(cls, where, 'remove');
   }
 
-  async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T> {
-    for (const id of this.#getStore(cls).keys()) {
-      try {
-        yield await this.get(cls, id);
-      } catch (error) {
-        if (!(error instanceof NotFoundError)) {
-          throw error;
-        }
-      }
+  async * list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]> {
+    for await (const batch of this.#iterateIds([...this.#getStore(cls).keys()], options)) {
+      yield ModelCrudUtil.filterOutNotFound(batch.map(id => this.get(cls, id)));
     }
   }
 
@@ -285,13 +375,14 @@ export class MemoryModelService implements ModelCrudSupport, ModelBlobSupport, M
 
   async deleteStorage(): Promise<void> {
     this.#store.clear();
-    this.#indices.sorted.clear();
-    this.#indices.unsorted.clear();
+    for (const value of Object.values(this.#indices)) {
+      value.clear();
+    }
   }
 
   async upsertModel<T extends ModelType>(cls: Class<T>): Promise<void> {
-    for (const idx of ModelRegistryIndex.getConfig(cls).indices ?? []) {
-      if (idx.type === 'sorted' || idx.type === 'unsorted') {
+    for (const idx of ModelRegistryIndex.getIndices(cls)) {
+      if (isModelIndexedIndex(idx)) {
         this.#indices[idx.type].set(indexName(cls, idx), new Map());
       }
     }
@@ -307,33 +398,97 @@ export class MemoryModelService implements ModelCrudSupport, ModelBlobSupport, M
   }
 
   // Indexed
-  async getByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<T> {
+  async getByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexBody<T, K, S>): Promise<T> {
     return this.get(cls, await this.#getIdByIndex(cls, idx, body));
+
   }
 
-  async deleteByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<void> {
+  async deleteByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexBody<T, K, S>): Promise<void> {
     await this.delete(cls, await this.#getIdByIndex(cls, idx, body));
   }
 
-  upsertByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: OptionalId<T>): Promise<T> {
+  upsertByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: OptionalId<T>): Promise<T> {
     return ModelIndexedUtil.naiveUpsert(this, cls, idx, body);
   }
 
-  async * listByIndex<T extends ModelType>(cls: Class<T>, idx: string, body?: DeepPartial<T>): AsyncIterable<T> {
-    const config = ModelRegistryIndex.getIndex(cls, idx, ['sorted', 'unsorted']);
-    const { key } = ModelIndexedUtil.computeIndexKey(cls, idx, body, { emptySortValue: null });
-    const index = this.#indices[config.type].get(indexName(cls, idx))?.get(key);
+  updateByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: T): Promise<T> {
+    return ModelIndexedUtil.naiveUpdate(this, cls, idx, body);
+  }
 
-    if (index) {
-      if (index instanceof Set) {
-        for (const id of index) {
-          yield this.get(cls, id);
-        }
-      } else {
-        for (const id of [...index.entries()].toSorted((a, b) => +a[1] - +b[1]).map(([a,]) => a)) {
-          yield this.get(cls, id);
-        }
-      }
+  async updatePartialByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexWithPartialBody<T, K, S>): Promise<T> {
+    const item = await ModelCrudUtil.naivePartialUpdate(cls, () => this.getByIndex(cls, idx, castTo(body)), castTo(body));
+    return this.update(cls, item);
+  }
+
+  async pageByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ModelPageOptions
+  ): Promise<ModelPageResult<T>> {
+    const offset = options?.offset ? JSONUtil.fromBase64<number>(options.offset) : 0;
+    let produced = 0;
+
+    const items: T[] = [];
+    for await (const batch of this.#getIndexIds(cls, idx, body, { limit: 100, ...options, offset })) {
+      produced += batch.length;
+      items.push(...await ModelCrudUtil.filterOutNotFound(batch.map(id => this.get(cls, id))));
+    }
+    return { items, nextOffset: items.length ? JSONUtil.toBase64(offset + produced) : undefined };
+  }
+
+  async  * listByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ModelListOptions
+  ): AsyncIterable<T[]> {
+    for await (const batch of this.#getIndexIds(cls, idx, body, options)) {
+      yield ModelCrudUtil.filterOutNotFound(batch.map(id => this.get(cls, id)));
+    }
+  }
+
+  async suggestByIndex<
+    T extends ModelType,
+    S extends SortedIndexSelection<T>,
+    K extends KeyedIndexSelection<T>,
+    B extends SortedIndexSelectionType<T, S> & string
+  >(cls: Class<T>, idx: SortedIndex<T, K, S>, body: KeyedIndexBody<T, K>, prefix: B, options?: ModelIndexedSearchOptions): Promise<T[]> {
+    const items: T[] = [];
+    for await (const batch of this.#getIndexIds(
+      cls, idx, body, options,
+      id => (typeof id === 'string' && id.startsWith(prefix))
+    )) {
+      items.push(...await ModelCrudUtil.filterOutNotFound(batch.map(id => this.get(cls, id))));
     }
+    return items;
   }
 }