@travetto/model-dynamodb 8.0.0-alpha.3 → 8.0.0-alpha.30

package/README.md

@@ -13,20 +13,19 @@ npm install @travetto/model-dynamodb
 yarn add @travetto/model-dynamodb
 ```
 
-This module provides an [DynamoDB](https://aws.amazon.com/dynamodb/)-based implementation for the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.").  This source allows the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") module to read, write and query against [DynamoDB](https://aws.amazon.com/dynamodb/). The entire document is stored as a single value, so nothing is needed to handle schema updates in real time. Indices on the other hand are more complicated, and will not be retroactively computed for new values. 
+This module provides an [DynamoDB](https://aws.amazon.com/dynamodb/)-based implementation for the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations."). This source allows the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") module to read, write and query against [DynamoDB](https://aws.amazon.com/dynamodb/). The entire document is stored as a single value, so nothing is needed to handle schema updates in real time. Indices on the other hand are more complicated, and will not be retroactively computed for new values. 
 
 Supported features:
-   *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L11)
+   *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L10)
    *  [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10)
-   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model/src/types/indexed.ts#L11)
+   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model-indexed/src/types/service.ts#L21)
 
 Out of the box, by installing the module, everything should be wired up by default.If you need to customize any aspect of the source or config, you can override and register it with the [Dependency Injection](https://github.com/travetto/travetto/tree/main/module/di#readme "Dependency registration/management and injection support.") module.
 
 **Code: Wiring up a custom Model Source**
 ```typescript
 import { InjectableFactory } from '@travetto/di';
-
-import { DynamoDBModelService, type DynamoDBModelConfig } from '@travetto/model-dynamodb';
+import { type DynamoDBModelConfig, DynamoDBModelService } from '@travetto/model-dynamodb';
 
 export class Init {
   @InjectableFactory({ primary: true })

package/__index__.ts

@@ -1,3 +1,3 @@
+export * from './src/config.ts';
 export * from './src/service.ts';
 export * from './src/util.ts';
-export * from './src/config.ts';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-dynamodb",
-  "version": "8.0.0-alpha.3",
+  "version": "8.0.0-alpha.30",
   "type": "module",
   "description": "DynamoDB backing for the travetto model module.",
   "keywords": [
@@ -26,12 +26,13 @@
     "directory": "module/model-dynamodb"
   },
   "dependencies": {
-    "@aws-sdk/client-dynamodb": "^3.1006.0",
-    "@travetto/config": "^8.0.0-alpha.3",
-    "@travetto/model": "^8.0.0-alpha.3"
+    "@aws-sdk/client-dynamodb": "^3.1116.0",
+    "@travetto/config": "^8.0.0-alpha.26",
+    "@travetto/model": "^8.0.0-alpha.27",
+    "@travetto/model-indexed": "^8.0.0-alpha.29"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.4"
+    "@travetto/cli": "^8.0.0-alpha.32"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/config.ts

@@ -18,4 +18,4 @@ export class DynamoDBModelConfig {
       this.client.endpoint ??= 'http://localhost:8000'; // From docker
     }
   }
-}
+}

package/src/service.ts

@@ -1,30 +1,67 @@
-import { type AttributeValue, DynamoDB, type PutItemCommandInput, type PutItemCommandOutput } from '@aws-sdk/client-dynamodb';
+import {
+  type AttributeValue,
+  DynamoDB,
+  type PutItemCommandInput,
+  type PutItemCommandOutput,
+  type QueryCommandInput,
+  type QueryCommandOutput
+} from '@aws-sdk/client-dynamodb';
 
-import { JSONUtil, ShutdownManager, TimeUtil, type Class, type DeepPartial } from '@travetto/runtime';
 import { Injectable, PostConstruct } from '@travetto/di';
 import {
-  type ModelCrudSupport, type ModelExpirySupport, ModelRegistryIndex, type ModelStorageSupport,
-  type ModelIndexedSupport, type ModelType, NotFoundError, ExistsError,
-  IndexNotSupported, type OptionalId,
-  ModelCrudUtil, ModelExpiryUtil, ModelIndexedUtil, ModelStorageUtil
+  ExistsError,
+  type ModelCrudSupport,
+  ModelCrudUtil,
+  type ModelExpirySupport,
+  ModelExpiryUtil,
+  type ModelListOptions,
+  ModelRegistryIndex,
+  type ModelStorageSupport,
+  ModelStorageUtil,
+  type ModelType,
+  NotFoundError,
+  type OptionalId
 } from '@travetto/model';
+import {
+  type FullKeyedIndexBody,
+  type FullKeyedIndexWithPartialBody,
+  isModelIndexedIndex,
+  type KeyedIndexBody,
+  type KeyedIndexSelection,
+  ModelIndexedComputedIndex,
+  type ModelIndexedSearchOptions,
+  type ModelIndexedSupport,
+  ModelIndexedUtil,
+  type ModelPageOptions,
+  type ModelPageResult,
+  type SingleItemIndex,
+  type SortedIndex,
+  type SortedIndexSelection,
+  type SortedIndexSelectionType
+} from '@travetto/model-indexed';
+import { type Class, castTo, JSONUtil, ShutdownManager, TimeUtil } from '@travetto/runtime';
 
 import type { DynamoDBModelConfig } from './config.ts';
 import { DynamoDBUtil } from './util.ts';
 
-const EXP_ATTR = 'expires_at__';
+const EXPIRES_ATTRIBUTE = 'expires_at__';
+
+const getKey = <T extends ModelType>(computed: ModelIndexedComputedIndex<T>): AttributeValue =>
+  DynamoDBUtil.toValue(computed.getKey() || 'NULL');
+const getSort = <T extends ModelType>(computed: ModelIndexedComputedIndex<T>): AttributeValue => DynamoDBUtil.toValue(computed.getSort());
 
 /**
  * A model service backed by DynamoDB
  */
 @Injectable()
 export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySupport, ModelStorageSupport, ModelIndexedSupport {
-
   idSource = ModelCrudUtil.uuidSource();
   client: DynamoDB;
   config: DynamoDBModelConfig;
 
-  constructor(config: DynamoDBModelConfig) { this.config = config; }
+  constructor(config: DynamoDBModelConfig) {
+    this.config = config;
+  }
 
   #resolveTable(cls: Class): string {
     let table = ModelRegistryIndex.getStoreName(cls);
@@ -34,7 +71,119 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
     return table;
   }
 
-  async #putItem<T extends ModelType>(cls: Class<T>, id: string, item: T, mode: 'create' | 'update' | 'upsert'): Promise<PutItemCommandOutput> {
+  async *#scanCollection<T extends ModelType>(
+    cls: Class<T>,
+    query: (batchSize: number, lastKey: Record<string, AttributeValue> | undefined) => Promise<QueryCommandOutput>,
+    options?: ModelListOptions & ModelPageOptions<Record<string, AttributeValue>>
+  ): AsyncIterable<{ items: T[]; lastKey?: Record<string, AttributeValue> }> {
+    const batchSize = options?.batchSizeHint ?? 100;
+    const limit = options?.limit ?? Number.MAX_SAFE_INTEGER;
+    let startKey = options?.offset ?? undefined;
+    let produced = 0;
+    do {
+      const remaining = limit - produced;
+      const batch = await query(Math.min(remaining, batchSize), startKey);
+
+      if (batch.Count && batch.Items) {
+        produced += batch.Count;
+
+        const items = produced > limit ? batch.Items.slice(0, remaining) : batch.Items;
+        startKey = batch.LastEvaluatedKey;
+        yield {
+          items: await ModelCrudUtil.filterOutNotFound(items.map(item => DynamoDBUtil.loadAndCheckExpiry(cls, item.body.S!))),
+          lastKey: startKey
+        };
+      } else {
+        startKey = undefined;
+      }
+    } while (startKey && produced < limit && !options?.abort?.aborted);
+  }
+
+  async *#scanIndex<T extends ModelType>(
+    cls: Class<T>,
+    idx: SortedIndex<T>,
+    body: KeyedIndexBody<T>,
+    options?: ModelPageOptions<Record<string, AttributeValue>> & ModelListOptions,
+    transform?: (query: QueryCommandInput) => QueryCommandInput
+  ): AsyncIterable<{ items: T[]; lastKey?: Record<string, AttributeValue> }> {
+    ModelCrudUtil.ensureNotSubType(cls);
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate();
+    const { keyIndexName, keyIndexAttribute } = DynamoDBUtil.indexNames(idx.name);
+    const expression = { [`:${keyIndexName}`]: getKey(computed) };
+
+    const finalTransform = transform ?? ((query): QueryCommandInput => query);
+
+    yield* this.#scanCollection(
+      cls,
+      (batchSize, lastKey) => {
+        const finalized = finalTransform({
+          TableName: this.#resolveTable(cls),
+          IndexName: keyIndexName,
+          ProjectionExpression: 'body',
+          KeyConditionExpression: `${keyIndexAttribute} = :${keyIndexName}`,
+          ExpressionAttributeValues: expression,
+          Limit: batchSize,
+          ExclusiveStartKey: lastKey
+        });
+        return this.client.query(finalized);
+      },
+      options
+    );
+  }
+
+  async #getIdByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: FullKeyedIndexBody<T, K, S>
+  ): Promise<string> {
+    ModelCrudUtil.ensureNotSubType(cls);
+
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate({ sort: true });
+
+    const { keyIndexName, keyIndexAttribute, sortIndexAttribute, sortIndexName } = DynamoDBUtil.indexNames(idx.name);
+    const sorted = idx.type === 'indexed:sorted';
+
+    const query: QueryCommandInput = {
+      TableName: this.#resolveTable(cls),
+      IndexName: keyIndexName,
+      ProjectionExpression: 'id',
+      KeyConditionExpression: [
+        ...(sorted ? [`${sortIndexAttribute} = :${sortIndexName}`] : []),
+        `${keyIndexAttribute} = :${keyIndexName}`
+      ].join(' and '),
+      ...(computed.idPart
+        ? {
+            FilterExpression: 'id = :id'
+          }
+        : {}),
+      ExpressionAttributeValues: {
+        [`:${keyIndexName}`]: getKey(computed),
+        ...(sorted ? { [`:${sortIndexName}`]: getSort(computed) } : {}),
+        ...(computed.idPart ? { ':id': DynamoDBUtil.toValue(computed.idPart.value) } : {})
+      }
+    };
+
+    try {
+      const result = await this.client.query(query);
+
+      if (result.Count && result.Items?.[0]) {
+        return result.Items[0].id.S!;
+      }
+      throw new NotFoundError(`${cls.name} Index=${idx}`, computed.getKey({ sort: true }));
+    } catch (error) {
+      if (error instanceof Error && error.message.includes('The table does not have the specified index')) {
+        throw new NotFoundError(`${cls.name} Index=${idx}`, computed.getKey({ sort: true }));
+      }
+      throw error;
+    }
+  }
+
+  async #putItem<T extends ModelType>(
+    cls: Class<T>,
+    id: string,
+    item: T,
+    mode: 'create' | 'update' | 'upsert'
+  ): Promise<PutItemCommandOutput> {
     const config = ModelRegistryIndex.getConfig(cls);
     let expiry: number | undefined;
 
@@ -48,12 +197,22 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
     try {
       if (mode === 'create') {
         const indices: Record<string, unknown> = {};
-        for (const idx of config.indices ?? []) {
-          const { key, sort } = ModelIndexedUtil.computeIndexKey(cls, idx, item);
-          const property = DynamoDBUtil.simpleName(idx.name);
-          indices[`${property}__`] = DynamoDBUtil.toValue(key);
-          if (sort) {
-            indices[`${property}_sort__`] = DynamoDBUtil.toValue(+sort);
+        for (const idx of ModelRegistryIndex.getIndices(cls)) {
+          if (isModelIndexedIndex(idx)) {
+            const { keyIndexAttribute, sortIndexAttribute } = DynamoDBUtil.indexNames(idx.name);
+            const computed = ModelIndexedComputedIndex.get(idx, item).validate({ sort: true });
+            switch (idx.type) {
+              case 'indexed:keyed':
+                indices[keyIndexAttribute] = getKey(computed);
+                break;
+              case 'indexed:sorted': {
+                indices[keyIndexAttribute] = getKey(computed);
+                indices[sortIndexAttribute] = getSort(computed);
+                break;
+              }
+            }
+          } else {
+            console.warn('Unsupported index type on update', { cls: cls.name, idx });
           }
         }
         const query: PutItemCommandInput = {
@@ -62,7 +221,7 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
           Item: {
             id: DynamoDBUtil.toValue(item.id),
             body: DynamoDBUtil.toValue(JSONUtil.toUTF8(item)),
-            ...(expiry !== undefined ? { [EXP_ATTR]: DynamoDBUtil.toValue(expiry) } : {}),
+            ...(expiry !== undefined ? { [EXPIRES_ATTRIBUTE]: DynamoDBUtil.toValue(expiry) } : {}),
             ...indices
           },
           ReturnValues: 'NONE'
@@ -71,14 +230,27 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
       } else {
         const indices: Record<string, unknown> = {};
         const expr: string[] = [];
-        for (const idx of config.indices ?? []) {
-          const { key, sort } = ModelIndexedUtil.computeIndexKey(cls, idx, item);
-          const property = DynamoDBUtil.simpleName(idx.name);
-          indices[`:${property}`] = DynamoDBUtil.toValue(key);
-          expr.push(`${property}__ = :${property}`);
-          if (sort) {
-            indices[`:${property}_sort`] = DynamoDBUtil.toValue(+sort);
-            expr.push(`${property}_sort__ = :${property}_sort`);
+
+        for (const idx of ModelRegistryIndex.getIndices(cls)) {
+          if (isModelIndexedIndex(idx)) {
+            const { keyIndexAttribute, sortIndexAttribute, keyIndexName, sortIndexName } = DynamoDBUtil.indexNames(idx.name);
+            const computed = ModelIndexedComputedIndex.get(idx, item).validate({ sort: true });
+            switch (idx.type) {
+              case 'indexed:keyed': {
+                indices[`:${keyIndexName}`] = getKey(computed);
+                expr.push(`${keyIndexAttribute} = :${keyIndexName}`);
+                break;
+              }
+              case 'indexed:sorted': {
+                indices[`:${keyIndexName}`] = getKey(computed);
+                indices[`:${sortIndexName}`] = getSort(computed);
+                expr.push(`${keyIndexAttribute} = :${keyIndexName}`);
+                expr.push(`${sortIndexAttribute} = :${sortIndexName}`);
+                break;
+              }
+            }
+          } else {
+            console.warn('Unsupported index type on update', { cls: cls.name, idx });
           }
         }
 
@@ -86,11 +258,9 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
           TableName: this.#resolveTable(cls),
           ConditionExpression: mode === 'update' ? 'attribute_exists(body)' : undefined,
           Key: { id: { S: id } },
-          UpdateExpression: `SET ${[
-            'body=:body',
-            expiry !== undefined ? `${EXP_ATTR}=:expr` : undefined,
-            ...expr
-          ].filter(part => !!part).join(', ')}`,
+          UpdateExpression: `SET ${['body=:body', expiry !== undefined ? `${EXPIRES_ATTRIBUTE}=:expr` : undefined, ...expr]
+            .filter(part => !!part)
+            .join(', ')}`,
           ExpressionAttributeValues: {
             ':body': DynamoDBUtil.toValue(JSONUtil.toUTF8(item)),
             ...(expiry !== undefined ? { ':expr': DynamoDBUtil.toValue(expiry) } : {}),
@@ -139,10 +309,7 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
         TableName: table,
         KeySchema: [{ KeyType: 'HASH', AttributeName: 'id' }],
         BillingMode: 'PAY_PER_REQUEST',
-        AttributeDefinitions: [
-          { AttributeName: 'id', AttributeType: 'S' },
-          ...idx.attributes
-        ],
+        AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }, ...idx.attributes],
         GlobalSecondaryIndexes: idx.indices
       });
     } else {
@@ -154,10 +321,7 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
       if (changedAttributes.length || indexUpdates?.length) {
         await this.client.updateTable({
           TableName: table,
-          AttributeDefinitions: [
-            { AttributeName: 'id', AttributeType: 'S' },
-            ...idx.attributes
-          ],
+          AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }, ...idx.attributes],
           GlobalSecondaryIndexUpdates: indexUpdates
         });
       }
@@ -168,7 +332,7 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
     if (ttlEnabled !== ttlRequired) {
       await this.client.updateTimeToLive({
         TableName: table,
-        TimeToLiveSpecification: { AttributeName: ttlRequired ? EXP_ATTR : undefined, Enabled: ttlRequired }
+        TimeToLiveSpecification: { AttributeName: ttlRequired ? EXPIRES_ATTRIBUTE : undefined, Enabled: ttlRequired }
       });
     }
   }
@@ -179,7 +343,7 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
    */
   async deleteModel(cls: Class<ModelType>): Promise<void> {
     const table = this.#resolveTable(cls);
-    const { Table: verify } = (await this.client.describeTable({ TableName: table }).catch(() => ({ Table: undefined })));
+    const { Table: verify } = await this.client.describeTable({ TableName: table }).catch(() => ({ Table: undefined }));
     if (verify) {
       await this.client.deleteTable({ TableName: table });
     }
@@ -191,9 +355,11 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
 
   async deleteStorage(): Promise<void> {
     for (const model of ModelRegistryIndex.getClasses()) {
-      await this.client.deleteTable({
-        TableName: this.#resolveTable(model)
-      }).catch(() => { });
+      await this.client
+        .deleteTable({
+          TableName: this.#resolveTable(model)
+        })
+        .catch(() => {});
     }
   }
 
@@ -204,7 +370,7 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
       Key: { id: DynamoDBUtil.toValue(id) }
     });
 
-    if (result && result.Item && result.Item.body) {
+    if (result?.Item?.body) {
       return DynamoDBUtil.loadAndCheckExpiry(cls, result.Item.body.S!);
     }
     throw new NotFoundError(cls, id);
@@ -253,32 +419,18 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
     }
   }
 
-  async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T> {
-    let done = false;
-    let token: Record<string, AttributeValue> | undefined;
-    while (!done) {
-      const batch = await this.client.scan({
-        TableName: this.#resolveTable(cls),
-        ExclusiveStartKey: token
-      });
-
-      if (batch.Count && batch.Items) {
-        for (const item of batch.Items) {
-          try {
-            yield await DynamoDBUtil.loadAndCheckExpiry(cls, item.body.S!);
-          } catch (error) {
-            if (!(error instanceof NotFoundError)) {
-              throw error;
-            }
-          }
-        }
-      }
-
-      if (!batch.Count || !batch.LastEvaluatedKey) {
-        done = true;
-      } else {
-        token = batch.LastEvaluatedKey;
-      }
+  async *list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]> {
+    for await (const { items } of this.#scanCollection(
+      cls,
+      (batchSize, lastKey) =>
+        this.client.scan({
+          TableName: this.#resolveTable(cls),
+          ExclusiveStartKey: lastKey,
+          Limit: batchSize
+        }),
+      options
+    )) {
+      yield items;
     }
   }
 
@@ -288,92 +440,106 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
   }
 
   // Indexed
-  async #getIdByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<string> {
-    ModelCrudUtil.ensureNotSubType(cls);
-
-    const idxConfig = ModelRegistryIndex.getIndex(cls, idx, ['sorted', 'unsorted']);
-
-    const { key, sort } = ModelIndexedUtil.computeIndexKey(cls, idxConfig, body);
+  async getByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: FullKeyedIndexBody<T, K, S>
+  ): Promise<T> {
+    return this.get(cls, await this.#getIdByIndex(cls, idx, body));
+  }
 
-    if (idxConfig.type === 'sorted' && sort === undefined) {
-      throw new IndexNotSupported(cls, idxConfig, 'Sorted indices require the sort field');
-    }
+  async deleteByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: FullKeyedIndexBody<T, K, S>
+  ): Promise<void> {
+    return this.delete(cls, await this.#getIdByIndex(cls, idx, body));
+  }
 
-    const idxName = DynamoDBUtil.simpleName(idx);
+  upsertByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: OptionalId<T>
+  ): Promise<T> {
+    return ModelIndexedUtil.naiveUpsert(this, cls, idx, body);
+  }
 
-    const query = {
-      TableName: this.#resolveTable(cls),
-      IndexName: idxName,
-      ProjectionExpression: 'id',
-      KeyConditionExpression: [sort ? `${idxName}_sort__ = :${idxName}_sort` : '', `${idxName}__ = :${idxName}`]
-        .filter(expr => !!expr)
-        .join(' and '),
-      ExpressionAttributeValues: {
-        [`:${idxName}`]: DynamoDBUtil.toValue(key),
-        ...(sort ? { [`:${idxName}_sort`]: DynamoDBUtil.toValue(+sort) } : {})
-      }
-    };
+  async updateByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: T
+  ): Promise<T> {
+    return ModelIndexedUtil.naiveUpdate(this, cls, idx, body);
+  }
 
-    const result = await this.client.query(query);
+  async updatePartialByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: FullKeyedIndexWithPartialBody<T, K, S>
+  ): Promise<T> {
+    const item = await ModelCrudUtil.naivePartialUpdate(cls, () => this.getByIndex(cls, idx, castTo(body)), castTo(body));
+    return this.update(cls, item);
+  }
 
-    if (result.Count && result.Items && result.Items[0]) {
-      return result.Items[0].id.S!;
+  async pageByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ModelPageOptions
+  ): Promise<ModelPageResult<T>> {
+    const output: T[] = [];
+    const offset = options?.offset ? JSONUtil.fromBase64<Record<string, AttributeValue>>(options.offset) : undefined;
+    for await (const { items } of this.#scanIndex(cls, idx, body, { limit: 100, ...options, offset })) {
+      output.push(...items);
     }
-    throw new NotFoundError(`${cls.name} Index=${idx}`, key);
-  }
 
-  // Indexed
-  async getByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<T> {
-    return this.get(cls, await this.#getIdByIndex(cls, idx, body));
-  }
+    let nextOffset: string | undefined;
+    if (output.length) {
+      const last: T = output.at(-1)!;
+      const computed = ModelIndexedComputedIndex.get(idx, last).validate();
+      const { keyIndexAttribute, sortIndexAttribute } = DynamoDBUtil.indexNames(idx.name);
+      nextOffset = JSONUtil.toBase64({
+        [keyIndexAttribute]: getKey(computed),
+        [sortIndexAttribute]: getSort(computed),
+        id: DynamoDBUtil.toValue(last.id)
+      });
+    }
 
-  async deleteByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<void> {
-    return this.delete(cls, await this.#getIdByIndex(cls, idx, body));
+    return { items: output, nextOffset };
   }
 
-  upsertByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: OptionalId<T>): Promise<T> {
-    return ModelIndexedUtil.naiveUpsert(this, cls, idx, body);
+  async *listByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ModelListOptions
+  ): AsyncIterable<T[]> {
+    for await (const { items } of this.#scanIndex(cls, idx, body, options)) {
+      yield items;
+    }
   }
 
-  async * listByIndex<T extends ModelType>(cls: Class<T>, idx: string, body?: DeepPartial<T>): AsyncIterable<T> {
-    ModelCrudUtil.ensureNotSubType(cls);
+  async suggestByIndex<
+    T extends ModelType,
+    S extends SortedIndexSelection<T>,
+    K extends KeyedIndexSelection<T>,
+    B extends SortedIndexSelectionType<T, S> & string
+  >(cls: Class<T>, idx: SortedIndex<T, K, S>, body: KeyedIndexBody<T, K>, prefix: B, options?: ModelIndexedSearchOptions): Promise<T[]> {
+    const results: T[] = [];
 
-    const config = ModelRegistryIndex.getIndex(cls, idx, ['sorted', 'unsorted']);
-    const { key } = ModelIndexedUtil.computeIndexKey(cls, config, body, { emptySortValue: null });
-
-    const idxName = DynamoDBUtil.simpleName(idx);
-
-    let done = false;
-    let token: Record<string, AttributeValue> | undefined;
-    while (!done) {
-      const batch = await this.client.query({
-        TableName: this.#resolveTable(cls),
-        IndexName: idxName,
-        ProjectionExpression: 'body',
-        KeyConditionExpression: `${idxName}__ = :${idxName}`,
-        ExpressionAttributeValues: {
-          [`:${idxName}`]: DynamoDBUtil.toValue(key)
-        },
-        ExclusiveStartKey: token
-      });
+    const { sortIndexAttribute } = DynamoDBUtil.indexNames(idx.name);
 
-      if (batch.Count && batch.Items) {
-        for (const item of batch.Items) {
-          try {
-            yield await DynamoDBUtil.loadAndCheckExpiry(cls, item.body.S!);
-          } catch (error) {
-            if (!(error instanceof NotFoundError)) {
-              throw error;
-            }
-          }
-        }
-      }
-
-      if (!batch.Count || !batch.LastEvaluatedKey) {
-        done = true;
-      } else {
-        token = batch.LastEvaluatedKey;
+    for await (const { items } of this.#scanIndex(cls, idx, body, { limit: 10, ...options }, query => ({
+      ...query,
+      KeyConditionExpression: [query.KeyConditionExpression, `begins_with(${sortIndexAttribute}, :prefix)`].filter(Boolean).join(' AND '),
+      ExpressionAttributeValues: {
+        ...query.ExpressionAttributeValues,
+        ':prefix': { S: prefix }
       }
+    }))) {
+      results.push(...items);
     }
+
+    return results;
   }
-}
+}

package/src/util.ts

@@ -1,10 +1,16 @@
 import type {
-  AttributeDefinition, AttributeValue, GlobalSecondaryIndex, GlobalSecondaryIndexDescription,
-  GlobalSecondaryIndexUpdate, KeySchemaElement
+  AttributeDefinition,
+  AttributeValue,
+  GlobalSecondaryIndex,
+  GlobalSecondaryIndexDescription,
+  GlobalSecondaryIndexUpdate,
+  KeySchemaElement
 } from '@aws-sdk/client-dynamodb';
 
-import type { Class } from '@travetto/runtime';
-import { ModelCrudUtil, ModelExpiryUtil, ModelRegistryIndex, NotFoundError, type ModelType } from '@travetto/model';
+import { ModelCrudUtil, ModelExpiryUtil, ModelRegistryIndex, type ModelType, NotFoundError } from '@travetto/model';
+import { isModelIndexedIndex, warnIfIndexedUniqueIndex, warnIfNonIndexedIndex } from '@travetto/model-indexed';
+import { type Class, castTo } from '@travetto/runtime';
+import { SchemaRegistryIndex } from '@travetto/schema';
 
 /**
  * Configuration for DynamoDB indices
@@ -18,13 +24,17 @@ type DynamoIndexConfig = {
  * Utility class for DynamoDB operations and transformations.
  */
 export class DynamoDBUtil {
-
-  /**
-   * Converts an index name to a simplified format by removing non-alphanumeric characters
-   */
-  static simpleName(idx: string): string {
-    return idx.replace(/[^A-Za-z0-9]/g, '');
-  }
+  static indexNames = (
+    name: string
+  ): { keyIndexName: string; sortIndexName: string; keyIndexAttribute: string; sortIndexAttribute: string } => {
+    const base = name.toLowerCase().replace(/[^A-Za-z0-9]+/g, '_');
+    return {
+      keyIndexName: base,
+      sortIndexName: `${base}_sort`,
+      keyIndexAttribute: `${base}__`,
+      sortIndexAttribute: `${base}_sort__`
+    };
+  };
 
   /**
    * Converts a JavaScript value to a DynamoDB AttributeValue format
@@ -51,29 +61,50 @@ export class DynamoDBUtil {
    * Generates global secondary indices and attribute definitions based on the model's index configuration.
    */
   static computeIndexConfig<T extends ModelType>(cls: Class<T>): DynamoIndexConfig {
-    const config = ModelRegistryIndex.getConfig(cls);
+    const indexes = ModelRegistryIndex.getIndices(cls);
     const attributes: AttributeDefinition[] = [];
-    const indices: GlobalSecondaryIndex[] = [];
-
-    for (const idx of config.indices ?? []) {
-      const idxName = this.simpleName(idx.name);
-      attributes.push({ AttributeName: `${idxName}__`, AttributeType: 'S' });
-
-      const keys: KeySchemaElement[] = [{
-        AttributeName: `${idxName}__`,
-        KeyType: 'HASH'
-      }];
-
-      if (idx.type === 'sorted') {
-        keys.push({
-          AttributeName: `${idxName}_sort__`,
-          KeyType: 'RANGE'
-        });
-        attributes.push({ AttributeName: `${idxName}_sort__`, AttributeType: 'N' });
+    const toCreate: GlobalSecondaryIndex[] = [];
+
+    const filtered = indexes
+      .filter(idx => !warnIfIndexedUniqueIndex(this, cls, [idx]))
+      .filter(idx => !warnIfNonIndexedIndex(this, cls, [idx]))
+      .filter(isModelIndexedIndex);
+
+    for (const idx of filtered) {
+      const keys: KeySchemaElement[] = [];
+
+      const { keyIndexName, keyIndexAttribute, sortIndexAttribute } = this.indexNames(idx.name);
+
+      switch (idx.type) {
+        case 'indexed:sorted': {
+          const path = idx.sortTemplate[0].path;
+          let fieldType = cls;
+          for (const field of path) {
+            if (SchemaRegistryIndex.has(fieldType)) {
+              const schema = SchemaRegistryIndex.getConfig(fieldType);
+              if (field in schema.fields) {
+                fieldType = schema.fields[field].type;
+              }
+            } else {
+              break;
+            }
+          }
+
+          keys.push({ AttributeName: keyIndexAttribute, KeyType: 'HASH' });
+          keys.push({ AttributeName: sortIndexAttribute, KeyType: 'RANGE' });
+          attributes.push({ AttributeName: keyIndexAttribute, AttributeType: 'S' });
+          attributes.push({ AttributeName: sortIndexAttribute, AttributeType: castTo(fieldType) === String ? 'S' : 'N' });
+          break;
+        }
+        case 'indexed:keyed': {
+          keys.push({ AttributeName: keyIndexAttribute, KeyType: 'HASH' });
+          attributes.push({ AttributeName: keyIndexAttribute, AttributeType: 'S' });
+          break;
+        }
       }
 
-      indices.push({
-        IndexName: idxName,
+      toCreate.push({
+        IndexName: keyIndexName,
         // ProvisionedThroughput: '',
         Projection: {
           ProjectionType: 'INCLUDE',
@@ -83,14 +114,17 @@ export class DynamoDBUtil {
       });
     }
 
-    return { indices: indices.length ? indices : undefined, attributes };
+    return { indices: toCreate.length ? toCreate : undefined, attributes };
   }
 
   /**
    * Identifies attribute definitions that have changed between current and requested configurations.
    * Compares attribute names and types to determine additions, removals, or modifications.
    */
-  static findChangedAttributes(current: AttributeDefinition[] | undefined, requested: AttributeDefinition[] | undefined): AttributeDefinition[] {
+  static findChangedAttributes(
+    current: AttributeDefinition[] | undefined,
+    requested: AttributeDefinition[] | undefined
+  ): AttributeDefinition[] {
     const currentMap = Object.fromEntries((current ?? []).map(attr => [attr.AttributeName, attr]));
     const pendingMap = Object.fromEntries((requested ?? []).map(attr => [attr.AttributeName, attr]));
 
@@ -119,7 +153,10 @@ export class DynamoDBUtil {
    * Identifies global secondary indices that have changed between current and requested configurations.
    * Generates update operations for creating new indices or deleting removed indices.
    */
-  static findChangedGlobalIndexes(current: GlobalSecondaryIndexDescription[] | undefined, requested: GlobalSecondaryIndex[] | undefined): GlobalSecondaryIndexUpdate[] {
+  static findChangedGlobalIndexes(
+    current: GlobalSecondaryIndexDescription[] | undefined,
+    requested: GlobalSecondaryIndex[] | undefined
+  ): GlobalSecondaryIndexUpdate[] {
     const existingMap = Object.fromEntries((current ?? []).map(index => [index.IndexName, index]));
     const pendingMap = Object.fromEntries((requested ?? []).map(index => [index.IndexName, index]));
 

package/support/service.dynamodb.ts

@@ -7,4 +7,4 @@ export const service: ServiceDescriptor = {
   version,
   port: 8000,
   image: `amazon/dynamodb-local:${version}`
-};
+};