@travetto/model-redis 8.0.0-alpha.2 → 8.0.0-alpha.20

package/README.md

@@ -18,7 +18,7 @@ This module provides an [redis](https://redis.io)-based implementation for the [
 Supported features:
    *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L11)
    *  [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10)
-   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model/src/types/indexed.ts#L11)
+   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model-indexed/src/types/service.ts#L16)
 
 Out of the box, by installing the module, everything should be wired up by default.If you need to customize any aspect of the source or config, you can override and register it with the [Dependency Injection](https://github.com/travetto/travetto/tree/main/module/di#readme "Dependency registration/management and injection support.") module.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-redis",
-  "version": "8.0.0-alpha.2",
+  "version": "8.0.0-alpha.20",
   "type": "module",
   "description": "Redis backing for the travetto model module.",
   "keywords": [
@@ -26,12 +26,13 @@
     "directory": "module/model-redis"
   },
   "dependencies": {
-    "@redis/client": "^5.11.0",
-    "@travetto/config": "^8.0.0-alpha.2",
-    "@travetto/model": "^8.0.0-alpha.2"
+    "@redis/client": "^6.0.0",
+    "@travetto/config": "^8.0.0-alpha.18",
+    "@travetto/model": "^8.0.0-alpha.18",
+    "@travetto/model-indexed": "^8.0.0-alpha.20"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.3"
+    "@travetto/cli": "^8.0.0-alpha.23"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/service.ts

@@ -1,18 +1,29 @@
 import { createClient } from '@redis/client';
 
-import { JSONUtil, ShutdownManager, type Class, type DeepPartial } from '@travetto/runtime';
+import { castTo, JSONUtil, ShutdownManager, type Class } from '@travetto/runtime';
 import {
-  type ModelCrudSupport, type ModelExpirySupport, ModelRegistryIndex, type ModelType, type ModelStorageSupport,
-  NotFoundError, ExistsError, type ModelIndexedSupport, type OptionalId,
-  ModelCrudUtil, ModelExpiryUtil, ModelIndexedUtil, ModelStorageUtil,
+  type ModelCrudSupport, type ModelExpirySupport, ModelRegistryIndex, type ModelType, type ModelStorageSupport, NotFoundError,
+  ExistsError, type OptionalId, ModelCrudUtil, ModelExpiryUtil, ModelStorageUtil,
+  type ModelListOptions,
 } from '@travetto/model';
+import {
+  type ModelIndexedSupport, type KeyedIndexSelection, type KeyedIndexBody, type ModelPageOptions, ModelIndexedUtil,
+  type SingleItemIndex, type SortedIndexSelection, type ModelPageResult, type SortedIndex, isModelIndexedIndex,
+  type FullKeyedIndexWithPartialBody, type FullKeyedIndexBody, ModelIndexedComputedIndex,
+  warnIfIndexedUniqueIndex, warnIfNonIndexedIndex, type ModelIndexedSearchOptions, type SortedIndexSelectionType,
+} from '@travetto/model-indexed';
+
 import { Injectable, PostConstruct } from '@travetto/di';
 
 import type { RedisModelConfig } from './config.ts';
 
-type RedisScan = { key: string } | { match: string };
+const TERMINATOR = '\xff';
+
+type RedisScan = ({ key: string } | { match: string } | { prefix: string }) & { reverse?: boolean };
 type RedisClient = ReturnType<typeof createClient>;
 type RedisMulti = ReturnType<RedisClient['multi']>;
+type ScanState = { cursor?: string, ids: string[] };
+type ScanOp = 'scan' | 'sScan' | 'zRange';
 
 /**
  * A model service backed by redis
@@ -40,55 +51,114 @@ export class RedisModelService implements ModelCrudSupport, ModelExpirySupport,
     return key;
   }
 
-  async * #streamValues(operation: 'scan' | 'sScan' | 'zScan', search: RedisScan, count = 100): AsyncIterable<string[]> {
-    let previousCursor = '0';
-    let done = false;
-
-    const flags = { COUNT: count, ...('match' in search ? { MATCH: search.match } : {}) };
+  async #scan(operation: ScanOp, cursor: string, search: RedisScan, count = 100): Promise<ScanState> {
     const key = 'key' in search ? search.key : '';
+    const flags = { COUNT: count, ...('match' in search ? { MATCH: search.match } : {}) };
 
-    while (!done) {
-      const [cursor, results] = await (
-        operation === 'scan' ?
-          this.client.scan(previousCursor, flags).then(result => [result.cursor, result.keys] as const) :
-          operation === 'sScan' ?
-            this.client.sScan(key, previousCursor, flags).then(result => [result.cursor, result.members] as const) :
-            this.client.zScan(key, previousCursor, flags).then(result => [result.cursor, result.members.map(item => item.value)] as const)
-      );
+    let output: ScanState;
+    switch (operation) {
+      case 'scan': output = await this.client.scan(cursor ?? '0', flags).then(result => ({ cursor: result.cursor, ids: result.keys })); break;
+      case 'sScan': output = await this.client.sScan(key!, cursor ?? '0', flags).then(result => ({ cursor: result.cursor, ids: result.members })); break;
+      case 'zRange': {
+        const offset = cursor ? +cursor : 0;
+        const prefix = 'prefix' in search ? search.prefix : undefined;
 
-      previousCursor = cursor;
+        let bounds: [number | string, number | string];
 
-      yield results;
+        if (prefix !== undefined) {
+          bounds = [prefix ? `[${prefix}` : '-', `(${prefix}${TERMINATOR}`];
+        } else {
+          bounds = [Number.NEGATIVE_INFINITY, Number.POSITIVE_INFINITY];
+        }
+        if (search.reverse) {
+          bounds = [bounds[1], bounds[0]];
+        }
 
-      if (cursor === '0') {
-        done = true;
+        const result = await this.client.zRange(key!, ...bounds, { BY: prefix ? 'LEX' : 'SCORE', REV: search.reverse, LIMIT: { offset, count } });
+        output = { cursor: result.length ? (offset + result.length).toString() : undefined, ids: result };
+        break;
       }
     }
+    return { ...output, cursor: output.cursor === '0' ? undefined : output.cursor };
+  }
+
+  async * #streamValues(
+    operation: ScanOp,
+    search: RedisScan,
+    options?: ModelPageOptions & ModelListOptions
+  ): AsyncIterable<ScanState> {
+    const limit = options?.limit ?? Number.MAX_SAFE_INTEGER;
+    let matched: ScanState = { cursor: options?.offset, ids: [] };
+    let produced = 0;
+    const batchSize = options?.batchSizeHint ?? 100;
+
+    do {
+      const remaining = limit - produced;
+      matched = await this.#scan(operation, matched.cursor!, search, Math.min(remaining, batchSize));
+      if (matched.ids.length) {
+        yield matched;
+        produced += matched.ids.length;
+      }
+    } while (matched.cursor && produced < limit && !(options?.abort?.aborted));
   }
 
-  #iterate(prefix: Class | string): AsyncIterable<string[]> {
-    return this.#streamValues('scan', { match: `${this.#resolveKey(prefix)}*` });
+  #scanIndex<T extends ModelType>(
+    cls: Class<T>,
+    idx: SortedIndex<T>,
+    body: KeyedIndexBody<T>,
+    options?: ModelPageOptions & { prefix?: string }
+  ): AsyncIterable<ScanState> {
+    ModelCrudUtil.ensureNotSubType(cls);
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate();
+    const fullKey = this.#resolveKey(cls, idx.name, computed.getKey());
+    switch (idx.type) {
+      // case 'indexed:keyed': return this.#streamValues('sScan', { key: fullKey }, options);
+      case 'indexed:sorted': {
+        return this.#streamValues('zRange', { key: fullKey, prefix: options?.prefix }, options);
+      }
+    }
+  }
+
+  async #getBodies<T extends ModelType>(cls: Class<T>, ids: string[], transform: (id: string) => string): Promise<T[]> {
+    if (ids.length === 0) {
+      return [];
+    }
+    const bodies = (await this.client.mGet(ids.map(transform)))
+      .filter((result): result is string => !!result);
+    return ModelCrudUtil.filterOutNotFound(bodies.map(body => ModelCrudUtil.load(cls, body)));
   }
 
   #removeIndices<T extends ModelType>(cls: Class, item: T, multi: RedisMulti): void {
-    for (const idx of ModelRegistryIndex.getIndices(cls, ['sorted', 'unsorted'])) {
-      const { key } = ModelIndexedUtil.computeIndexKey(cls, idx, item);
-      const fullKey = this.#resolveKey(cls, idx.name, key);
-      switch (idx.type) {
-        case 'unsorted': multi.sRem(fullKey, item.id); break;
-        case 'sorted': multi.zRem(fullKey, item.id); break;
+    for (const idx of ModelRegistryIndex.getIndices(cls)) {
+      if (isModelIndexedIndex(idx)) {
+        const computed = ModelIndexedComputedIndex.get(idx, item).validate({ sort: true });
+        const resolvedKey = this.#resolveKey(cls, idx.name, computed.getKey());
+        switch (idx.type) {
+          case 'indexed:keyed': multi.sRem(resolvedKey, item.id); break;
+          case 'indexed:sorted': multi.zRem(resolvedKey, item.id); break;
+        }
       }
     }
   }
 
   #addIndices<T extends ModelType>(cls: Class, item: T, multi: RedisMulti): void {
-    for (const idx of ModelRegistryIndex.getIndices(cls, ['sorted', 'unsorted'])) {
-      const { key, sort } = ModelIndexedUtil.computeIndexKey(cls, idx, item);
-      const fullKey = this.#resolveKey(cls, idx.name, key);
+    for (const idx of ModelRegistryIndex.getIndices(cls)) {
+      if (isModelIndexedIndex(idx)) {
+        const computed = ModelIndexedComputedIndex.get(idx, item).validate({ sort: true });
+        const resolvedKey = this.#resolveKey(cls, idx.name, computed.getKey());
 
-      switch (idx.type) {
-        case 'unsorted': multi.sAdd(fullKey, item.id); break;
-        case 'sorted': multi.zAdd(fullKey, { score: +sort!, value: item.id }); break;
+        switch (idx.type) {
+          case 'indexed:keyed': multi.sAdd(resolvedKey, item.id); break;
+          case 'indexed:sorted': {
+            const value = computed.getSort();
+            if (typeof value === 'string') {
+              multi.zAdd(resolvedKey, { score: 0, value: `${value}${TERMINATOR}${item.id}` });
+            } else {
+              multi.zAdd(resolvedKey, { score: computed.getSort(), value: item.id });
+            }
+            break;
+          }
+        }
       }
     }
   }
@@ -96,10 +166,11 @@ export class RedisModelService implements ModelCrudSupport, ModelExpirySupport,
   async #store<T extends ModelType>(cls: Class<T>, item: T, action: 'write' | 'delete'): Promise<void> {
     const key = this.#resolveKey(cls, item.id);
     const config = ModelRegistryIndex.getConfig(cls);
+    const indices = ModelRegistryIndex.getIndices(cls);
     const existing = await this.get(cls, item.id).catch(() => undefined);
 
     // Store with indices
-    if (config.indices?.length) {
+    if (indices.length) {
       const multi = this.client.multi();
       if (existing) {
         this.#removeIndices(cls, existing, multi);
@@ -146,25 +217,47 @@ export class RedisModelService implements ModelCrudSupport, ModelExpirySupport,
     }
   }
 
-  async #getIdByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<string> {
+  async #getIdByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexBody<T, K, S>): Promise<string> {
     ModelCrudUtil.ensureNotSubType(cls);
 
-    const idxConfig = ModelRegistryIndex.getIndex(cls, idx, ['sorted', 'unsorted']);
-    const { key, sort } = ModelIndexedUtil.computeIndexKey(cls, idxConfig, body);
-    const fullKey = this.#resolveKey(cls, idxConfig.name, key);
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate({ sort: true });
+    const resolvedKey = this.#resolveKey(cls, idx.name, computed.getKey());
     let id: string | undefined;
-    if (idxConfig.type === 'unsorted') {
-      id = (await this.client.sRandMember(fullKey))!;
-    } else {
-      const result = await this.client.zRangeByScore(
-        fullKey, +sort!, +sort!
-      );
-      id = result[0];
+    switch (idx.type) {
+      case 'indexed:keyed': {
+        if (computed.idPart) {
+          const all = await this.client.sMembers(resolvedKey);
+          if (!all.find(item => item === computed.idPart!.value)) {
+            throw new NotFoundError(`${cls.name} Index=${idx}`, computed.getKey());
+          }
+          id = computed.idPart!.value;
+        } else {
+          id = await this.client.sRandMember(resolvedKey) ?? undefined;
+        }
+        break;
+      }
+      case 'indexed:sorted': {
+        const sort = computed.getSort();
+        const result = await this.client.zRangeByScore(resolvedKey, sort, sort);
+        if (computed.idPart) {
+          if (!result.find(item => item === computed.idPart!.value)) {
+            throw new NotFoundError(`${cls.name} Index=${idx}`, computed.getKey());
+          }
+          id = computed.idPart!.value;
+        } else {
+          id = result[0];
+        }
+        break;
+      }
     }
     if (id) {
       return id;
     }
-    throw new NotFoundError(`${cls.name}: ${idx}`, key);
+    throw new NotFoundError(`${cls.name}: ${idx}`, computed.getKey({ sort: true }));
   }
 
   @PostConstruct()
@@ -174,14 +267,8 @@ export class RedisModelService implements ModelCrudSupport, ModelExpirySupport,
     await ModelStorageUtil.storageInitialization(this);
     ShutdownManager.signal.addEventListener('abort', () => this.client.close());
     for (const cls of ModelRegistryIndex.getClasses()) {
-      for (const idx of ModelRegistryIndex.getConfig(cls).indices ?? []) {
-        switch (idx.type) {
-          case 'unique': {
-            console.error('Unique indices are not supported in redis for', { cls: cls.Ⲑid, idx: idx.name });
-            break;
-          }
-        }
-      }
+      warnIfIndexedUniqueIndex(this, cls, ModelRegistryIndex.getIndices(cls));
+      warnIfNonIndexedIndex(this, cls, ModelRegistryIndex.getIndices(cls));
     }
   }
 
@@ -241,25 +328,9 @@ export class RedisModelService implements ModelCrudSupport, ModelExpirySupport,
     await this.#store(cls, where, 'delete');
   }
 
-  async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T> {
-    for await (const ids of this.#iterate(cls)) {
-
-      if (!ids.length) {
-        return;
-      }
-
-      const bodies = (await this.client.mGet(ids))
-        .filter((result): result is string => !!result);
-
-      for (const body of bodies) {
-        try {
-          yield await ModelCrudUtil.load(cls, body);
-        } catch (error) {
-          if (!(error instanceof NotFoundError)) {
-            throw error;
-          }
-        }
-      }
+  async * list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]> {
+    for await (const { ids } of this.#streamValues('scan', { match: `${this.#resolveKey(cls)}:*` }, options)) {
+      yield await this.#getBodies(cls, ids, id => id);
     }
   }
 
@@ -271,14 +342,17 @@ export class RedisModelService implements ModelCrudSupport, ModelExpirySupport,
 
   // Storage
   async createStorage(): Promise<void> {
-    // Do nothing
+    for (const cls of ModelRegistryIndex.getClasses()) {
+      warnIfIndexedUniqueIndex(this, cls, ModelRegistryIndex.getIndices(cls));
+      warnIfNonIndexedIndex(this, cls, ModelRegistryIndex.getIndices(cls));
+    }
   }
 
   async deleteStorage(): Promise<void> {
     if (!this.config.namespace) {
       await this.client.flushDb();
     } else {
-      for await (const ids of this.#iterate('')) {
+      for await (const { ids } of this.#streamValues('scan', { match: `${this.#resolveKey('')}*` }, { limit: Number.MAX_SAFE_INTEGER })) {
         if (ids.length) {
           await this.client.del(ids);
         }
@@ -287,7 +361,7 @@ export class RedisModelService implements ModelCrudSupport, ModelExpirySupport,
   }
 
   async truncateModel<T extends ModelType>(model: Class<T>): Promise<void> {
-    for await (const ids of this.#iterate(model)) {
+    for await (const { ids } of this.#streamValues('scan', { match: `${this.#resolveKey(model)}:*` }, { limit: Number.MAX_SAFE_INTEGER })) {
       if (ids.length) {
         await this.client.del(ids);
       }
@@ -295,53 +369,95 @@ export class RedisModelService implements ModelCrudSupport, ModelExpirySupport,
   }
 
   // Indexed
-  async getByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<T> {
+
+  async getByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexBody<T, K, S>): Promise<T> {
     return this.get(cls, await this.#getIdByIndex(cls, idx, body));
   }
 
-  async deleteByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<void> {
+  async deleteByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexBody<T, K, S>): Promise<void> {
     return this.delete(cls, await this.#getIdByIndex(cls, idx, body));
   }
 
-  upsertByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: OptionalId<T>): Promise<T> {
+  upsertByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: OptionalId<T>): Promise<T> {
     return ModelIndexedUtil.naiveUpsert(this, cls, idx, body);
   }
 
-  async * listByIndex<T extends ModelType>(cls: Class<T>, idx: string, body?: DeepPartial<T>): AsyncIterable<T> {
-    ModelCrudUtil.ensureNotSubType(cls);
-
-    const idxConfig = ModelRegistryIndex.getIndex(cls, idx, ['sorted', 'unsorted']);
-
-    let stream: AsyncIterable<string[]>;
+  updateByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: T): Promise<T> {
+    return ModelIndexedUtil.naiveUpdate(this, cls, idx, body);
+  }
 
-    const { key } = ModelIndexedUtil.computeIndexKey(cls, idxConfig, body, { emptySortValue: null });
-    const fullKey = this.#resolveKey(cls, idx, key);
+  async updatePartialByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexWithPartialBody<T, K, S>): Promise<T> {
+    const item = await ModelCrudUtil.naivePartialUpdate(cls, () => this.getByIndex(cls, idx, castTo(body)), castTo(body));
+    return this.update(cls, item);
+  }
 
-    if (idxConfig.type === 'unsorted') {
-      stream = this.#streamValues('sScan', { key: fullKey });
-    } else {
-      stream = this.#streamValues('zScan', { key: fullKey });
+  async pageByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ModelPageOptions
+  ): Promise<ModelPageResult<T>> {
+    const items: T[] = [];
+    let lastCursor: string | undefined;
+    for await (const { ids, cursor } of this.#scanIndex(cls, idx, body, { limit: 100, ...options })) {
+      items.push(...await this.#getBodies(cls, ids, id => this.#resolveKey(cls, id)));
+      lastCursor = cursor;
     }
+    return { items, nextOffset: lastCursor };
+  }
 
-    for await (const ids of stream) {
-      if (!ids.length) {
-        return;
-      }
+  async * listByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ModelListOptions
+  ): AsyncIterable<T[]> {
+    for await (const { ids } of this.#scanIndex(cls, idx, body, options)) {
+      yield await this.#getBodies(cls, ids, id => this.#resolveKey(cls, id));
+    }
+  }
 
-      const bodies = (await this.client.mGet(
-        ids.map(id => this.#resolveKey(cls, id))
-      ))
-        .filter((result): result is string => !!result);
-
-      for (const full of bodies) {
-        try {
-          yield await ModelCrudUtil.load(cls, full);
-        } catch (error) {
-          if (!(error instanceof NotFoundError)) {
-            throw error;
-          }
-        }
-      }
+  async suggestByIndex<
+    T extends ModelType,
+    S extends SortedIndexSelection<T>,
+    K extends KeyedIndexSelection<T>,
+    B extends SortedIndexSelectionType<T, S> & string
+  >(cls: Class<T>, idx: SortedIndex<T, K, S>, body: KeyedIndexBody<T, K>, prefix: B, options?: ModelIndexedSearchOptions): Promise<T[]> {
+
+    const items: T[] = [];
+    for await (const { ids } of this.#scanIndex(cls, idx, body, { limit: 10, ...options, prefix })) {
+      const cleaned = ids.map(id => id.split(TERMINATOR).at(-1)!);
+      items.push(...await this.#getBodies(cls, cleaned, id => this.#resolveKey(cls, id)));
     }
+
+    return items;
   }
 }