@travetto/model-dynamodb 8.0.0-alpha.1 → 8.0.0-alpha.10

package/README.md

@@ -18,7 +18,7 @@ This module provides an [DynamoDB](https://aws.amazon.com/dynamodb/)-based imple
 Supported features:
    *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L11)
    *  [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10)
-   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model/src/types/indexed.ts#L11)
+   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model-indexed/src/types/service.ts#L23)
 
 Out of the box, by installing the module, everything should be wired up by default.If you need to customize any aspect of the source or config, you can override and register it with the [Dependency Injection](https://github.com/travetto/travetto/tree/main/module/di#readme "Dependency registration/management and injection support.") module.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-dynamodb",
-  "version": "8.0.0-alpha.1",
+  "version": "8.0.0-alpha.10",
   "type": "module",
   "description": "DynamoDB backing for the travetto model module.",
   "keywords": [
@@ -26,12 +26,13 @@
     "directory": "module/model-dynamodb"
   },
   "dependencies": {
-    "@aws-sdk/client-dynamodb": "^3.1004.0",
-    "@travetto/config": "^8.0.0-alpha.1",
-    "@travetto/model": "^8.0.0-alpha.1"
+    "@aws-sdk/client-dynamodb": "^3.1019.0",
+    "@travetto/config": "^8.0.0-alpha.10",
+    "@travetto/model": "^8.0.0-alpha.10",
+    "@travetto/model-indexed": "^8.0.0-alpha.10"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.1"
+    "@travetto/cli": "^8.0.0-alpha.15"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/service.ts

@@ -1,18 +1,26 @@
-import { type AttributeValue, DynamoDB, type PutItemCommandInput, type PutItemCommandOutput } from '@aws-sdk/client-dynamodb';
+import { type AttributeValue, DynamoDB, type PutItemCommandInput, type PutItemCommandOutput, type QueryCommandOutput } from '@aws-sdk/client-dynamodb';
 
-import { JSONUtil, ShutdownManager, TimeUtil, type Class, type DeepPartial } from '@travetto/runtime';
+import { castTo, JSONUtil, ShutdownManager, TimeUtil, type Class } from '@travetto/runtime';
 import { Injectable, PostConstruct } from '@travetto/di';
 import {
   type ModelCrudSupport, type ModelExpirySupport, ModelRegistryIndex, type ModelStorageSupport,
-  type ModelIndexedSupport, type ModelType, NotFoundError, ExistsError,
-  IndexNotSupported, type OptionalId,
-  ModelCrudUtil, ModelExpiryUtil, ModelIndexedUtil, ModelStorageUtil
+  type ModelType, NotFoundError, ExistsError, type OptionalId, ModelCrudUtil,
+  ModelExpiryUtil, ModelStorageUtil,
 } from '@travetto/model';
+import {
+  isModelIndexedIndex, ModelIndexedUtil, type KeyedIndexBody, type KeyedIndexSelection,
+  type ListPageOptions, type ListPageResult, type ModelIndexedSupport, type SingleItemIndex,
+  type FullKeyedIndexBody, type FullKeyedIndexWithPartialBody, type SortedIndex, type SortedIndexSelection,
+  ModelIndexedComputedIndex
+} from '@travetto/model-indexed';
 
 import type { DynamoDBModelConfig } from './config.ts';
 import { DynamoDBUtil } from './util.ts';
 
-const EXP_ATTR = 'expires_at__';
+const EXPIRES_ATTRIBUTE = 'expires_at__';
+
+const getKey = <T extends ModelType>(computed: ModelIndexedComputedIndex<T>): AttributeValue => DynamoDBUtil.toValue(computed.getKey() || 'NULL');
+const getSort = <T extends ModelType>(computed: ModelIndexedComputedIndex<T>): AttributeValue => DynamoDBUtil.toValue(computed.getSort());
 
 /**
  * A model service backed by DynamoDB
@@ -34,6 +42,93 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
     return table;
   }
 
+  async * #scanIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(
+    cls: Class,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ListPageOptions<Record<string, AttributeValue>>
+  ): AsyncIterable<QueryCommandOutput & { LastEvaluatedOffset?: string }> {
+    ModelCrudUtil.ensureNotSubType(cls);
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate();
+    const safeName = DynamoDBUtil.toSafeName(idx.name);
+    const expression = { [`:${safeName}`]: getKey(computed) };
+    const limit = options?.limit ?? 100;
+
+    let startKey = options?.offset ?? undefined;
+    let produced = 0;
+
+    do {
+      const remaining = limit - produced;
+      const batch = await this.client.query({
+        TableName: this.#resolveTable(cls),
+        IndexName: safeName,
+        ProjectionExpression: 'body',
+        KeyConditionExpression: `${safeName}__ = :${safeName}`,
+        ExpressionAttributeValues: expression,
+        Limit: Math.min(remaining, 100),
+        ExclusiveStartKey: startKey,
+      });
+
+      if (batch.Count && batch.Items) {
+        produced += batch.Count;
+
+        if (produced > limit) {
+          const items = batch.Items.slice(0, remaining);
+          yield { ...batch, Items: items, };
+        } else {
+          yield batch;
+        }
+        startKey = batch.LastEvaluatedKey;
+      } else {
+        startKey = undefined;
+      }
+    } while (startKey && produced < limit);
+  }
+
+  async #getIdByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexBody<T, K, S>): Promise<string> {
+    ModelCrudUtil.ensureNotSubType(cls);
+
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate({ sort: true });
+
+    const safeName = DynamoDBUtil.toSafeName(idx.name);
+    const sorted = idx.type === 'indexed:sorted';
+
+    const query = {
+      TableName: this.#resolveTable(cls),
+      IndexName: safeName,
+      ProjectionExpression: 'id',
+      KeyConditionExpression: [sorted ? `${safeName}_sort__ = :${safeName}_sort` : '', `${safeName}__ = :${safeName}`]
+        .filter(expr => !!expr)
+        .join(' and '),
+      ExpressionAttributeValues: {
+        [`:${safeName}`]: getKey(computed),
+        ...(sorted ? { [`:${safeName}_sort`]: getSort(computed) } : {})
+      }
+    };
+
+    try {
+      const result = await this.client.query(query);
+
+      if (result.Count && result.Items && result.Items[0]) {
+        return result.Items[0].id.S!;
+      }
+      throw new NotFoundError(`${cls.name} Index=${idx}`, computed.getKey({ sort: true }));
+    } catch (error) {
+      if (error instanceof Error && error.message.includes('The table does not have the specified index')) {
+        throw new NotFoundError(`${cls.name} Index=${idx}`, computed.getKey({ sort: true }));
+      }
+      throw error;
+    }
+  }
+
   async #putItem<T extends ModelType>(cls: Class<T>, id: string, item: T, mode: 'create' | 'update' | 'upsert'): Promise<PutItemCommandOutput> {
     const config = ModelRegistryIndex.getConfig(cls);
     let expiry: number | undefined;
@@ -48,12 +143,20 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
     try {
       if (mode === 'create') {
         const indices: Record<string, unknown> = {};
-        for (const idx of config.indices ?? []) {
-          const { key, sort } = ModelIndexedUtil.computeIndexKey(cls, idx, item);
-          const property = DynamoDBUtil.simpleName(idx.name);
-          indices[`${property}__`] = DynamoDBUtil.toValue(key);
-          if (sort) {
-            indices[`${property}_sort__`] = DynamoDBUtil.toValue(+sort);
+        for (const idx of ModelRegistryIndex.getIndices(cls)) {
+          if (isModelIndexedIndex(idx)) {
+            const safeName = DynamoDBUtil.toSafeName(idx.name);
+            const computed = ModelIndexedComputedIndex.get(idx, item).validate({ sort: true });
+            switch (idx.type) {
+              case 'indexed:keyed': indices[`${safeName}__`] = getKey(computed); break;
+              case 'indexed:sorted': {
+                indices[`${safeName}__`] = getKey(computed);
+                indices[`${safeName}_sort__`] = getSort(computed);
+                break;
+              }
+            }
+          } else {
+            console.warn('Unsupported index type on update', { cls: cls.name, idx });
           }
         }
         const query: PutItemCommandInput = {
@@ -62,7 +165,7 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
           Item: {
             id: DynamoDBUtil.toValue(item.id),
             body: DynamoDBUtil.toValue(JSONUtil.toUTF8(item)),
-            ...(expiry !== undefined ? { [EXP_ATTR]: DynamoDBUtil.toValue(expiry) } : {}),
+            ...(expiry !== undefined ? { [EXPIRES_ATTRIBUTE]: DynamoDBUtil.toValue(expiry) } : {}),
             ...indices
           },
           ReturnValues: 'NONE'
@@ -71,14 +174,27 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
       } else {
         const indices: Record<string, unknown> = {};
         const expr: string[] = [];
-        for (const idx of config.indices ?? []) {
-          const { key, sort } = ModelIndexedUtil.computeIndexKey(cls, idx, item);
-          const property = DynamoDBUtil.simpleName(idx.name);
-          indices[`:${property}`] = DynamoDBUtil.toValue(key);
-          expr.push(`${property}__ = :${property}`);
-          if (sort) {
-            indices[`:${property}_sort`] = DynamoDBUtil.toValue(+sort);
-            expr.push(`${property}_sort__ = :${property}_sort`);
+
+        for (const idx of ModelRegistryIndex.getIndices(cls)) {
+          if (isModelIndexedIndex(idx)) {
+            const safeName = DynamoDBUtil.toSafeName(idx.name);
+            const computed = ModelIndexedComputedIndex.get(idx, item).validate({ sort: true });
+            switch (idx.type) {
+              case 'indexed:keyed': {
+                indices[`:${safeName}`] = getKey(computed);
+                expr.push(`${safeName}__ = :${safeName}`);
+                break;
+              }
+              case 'indexed:sorted': {
+                indices[`:${safeName}`] = getKey(computed);
+                indices[`:${safeName}_sort`] = getSort(computed);
+                expr.push(`${safeName}__ = :${safeName}`);
+                expr.push(`${safeName}_sort__ = :${safeName}_sort`);
+                break;
+              }
+            }
+          } else {
+            console.warn('Unsupported index type on update', { cls: cls.name, idx });
           }
         }
 
@@ -88,7 +204,7 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
           Key: { id: { S: id } },
           UpdateExpression: `SET ${[
             'body=:body',
-            expiry !== undefined ? `${EXP_ATTR}=:expr` : undefined,
+            expiry !== undefined ? `${EXPIRES_ATTRIBUTE}=:expr` : undefined,
             ...expr
           ].filter(part => !!part).join(', ')}`,
           ExpressionAttributeValues: {
@@ -168,7 +284,7 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
     if (ttlEnabled !== ttlRequired) {
       await this.client.updateTimeToLive({
         TableName: table,
-        TimeToLiveSpecification: { AttributeName: ttlRequired ? EXP_ATTR : undefined, Enabled: ttlRequired }
+        TimeToLiveSpecification: { AttributeName: ttlRequired ? EXPIRES_ATTRIBUTE : undefined, Enabled: ttlRequired }
       });
     }
   }
@@ -288,92 +404,86 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
   }
 
   // Indexed
-  async #getIdByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<string> {
-    ModelCrudUtil.ensureNotSubType(cls);
-
-    const idxConfig = ModelRegistryIndex.getIndex(cls, idx, ['sorted', 'unsorted']);
-
-    const { key, sort } = ModelIndexedUtil.computeIndexKey(cls, idxConfig, body);
-
-    if (idxConfig.type === 'sorted' && sort === undefined) {
-      throw new IndexNotSupported(cls, idxConfig, 'Sorted indices require the sort field');
-    }
-
-    const idxName = DynamoDBUtil.simpleName(idx);
-
-    const query = {
-      TableName: this.#resolveTable(cls),
-      IndexName: idxName,
-      ProjectionExpression: 'id',
-      KeyConditionExpression: [sort ? `${idxName}_sort__ = :${idxName}_sort` : '', `${idxName}__ = :${idxName}`]
-        .filter(expr => !!expr)
-        .join(' and '),
-      ExpressionAttributeValues: {
-        [`:${idxName}`]: DynamoDBUtil.toValue(key),
-        ...(sort ? { [`:${idxName}_sort`]: DynamoDBUtil.toValue(+sort) } : {})
-      }
-    };
-
-    const result = await this.client.query(query);
-
-    if (result.Count && result.Items && result.Items[0]) {
-      return result.Items[0].id.S!;
-    }
-    throw new NotFoundError(`${cls.name} Index=${idx}`, key);
-  }
-
-  // Indexed
-  async getByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<T> {
+  async getByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexBody<T, K, S>): Promise<T> {
     return this.get(cls, await this.#getIdByIndex(cls, idx, body));
   }
 
-  async deleteByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<void> {
+  async deleteByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexBody<T, K, S>): Promise<void> {
     return this.delete(cls, await this.#getIdByIndex(cls, idx, body));
   }
 
-  upsertByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: OptionalId<T>): Promise<T> {
+  upsertByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: OptionalId<T>): Promise<T> {
     return ModelIndexedUtil.naiveUpsert(this, cls, idx, body);
   }
 
-  async * listByIndex<T extends ModelType>(cls: Class<T>, idx: string, body?: DeepPartial<T>): AsyncIterable<T> {
-    ModelCrudUtil.ensureNotSubType(cls);
-
-    const config = ModelRegistryIndex.getIndex(cls, idx, ['sorted', 'unsorted']);
-    const { key } = ModelIndexedUtil.computeIndexKey(cls, config, body, { emptySortValue: null });
-
-    const idxName = DynamoDBUtil.simpleName(idx);
+  async updateByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: T): Promise<T> {
+    return ModelIndexedUtil.naiveUpdate(this, cls, idx, body);
+  }
 
-    let done = false;
-    let token: Record<string, AttributeValue> | undefined;
-    while (!done) {
-      const batch = await this.client.query({
-        TableName: this.#resolveTable(cls),
-        IndexName: idxName,
-        ProjectionExpression: 'body',
-        KeyConditionExpression: `${idxName}__ = :${idxName}`,
-        ExpressionAttributeValues: {
-          [`:${idxName}`]: DynamoDBUtil.toValue(key)
-        },
-        ExclusiveStartKey: token
-      });
+  async updatePartialByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexWithPartialBody<T, K, S>): Promise<T> {
+    const item = await ModelCrudUtil.naivePartialUpdate(cls, () => this.getByIndex(cls, idx, castTo(body)), castTo(body));
+    return this.update(cls, item);
+  }
 
-      if (batch.Count && batch.Items) {
-        for (const item of batch.Items) {
-          try {
-            yield await DynamoDBUtil.loadAndCheckExpiry(cls, item.body.S!);
-          } catch (error) {
-            if (!(error instanceof NotFoundError)) {
-              throw error;
-            }
+  async listByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ListPageOptions,
+  ): Promise<ListPageResult<T>> {
+    const items: T[] = [];
+    const offset = options?.offset ? JSONUtil.fromBase64<Record<string, AttributeValue>>(options.offset) : undefined;
+    for await (const batch of this.#scanIndex(cls, idx, body, { ...options, offset })) {
+      for (const item of batch.Items ?? []) {
+        try {
+          items.push(await DynamoDBUtil.loadAndCheckExpiry(cls, item.body.S!));
+          if (options?.limit && items.length >= options.limit) {
+            break;
+          }
+        } catch (error) {
+          if (!(error instanceof NotFoundError)) {
+            throw error;
           }
         }
       }
+    }
 
-      if (!batch.Count || !batch.LastEvaluatedKey) {
-        done = true;
-      } else {
-        token = batch.LastEvaluatedKey;
-      }
+    let nextOffset;
+    if (items.length) {
+      const last: T = items.at(-1)!;
+      const computed = ModelIndexedComputedIndex.get(idx, last).validate();
+      const safeName = DynamoDBUtil.toSafeName(idx.name);
+      nextOffset = JSONUtil.toBase64({
+        [`${safeName}__`]: getKey(computed),
+        [`${safeName}_sort__`]: getSort(computed),
+        id: DynamoDBUtil.toValue(last.id)
+      });
     }
+
+    return { items, nextOffset };
   }
 }

package/src/util.ts

@@ -5,6 +5,7 @@ import type {
 
 import type { Class } from '@travetto/runtime';
 import { ModelCrudUtil, ModelExpiryUtil, ModelRegistryIndex, NotFoundError, type ModelType } from '@travetto/model';
+import { isModelIndexedIndex } from '@travetto/model-indexed';
 
 /**
  * Configuration for DynamoDB indices
@@ -19,12 +20,7 @@ type DynamoIndexConfig = {
  */
 export class DynamoDBUtil {
 
-  /**
-   * Converts an index name to a simplified format by removing non-alphanumeric characters
-   */
-  static simpleName(idx: string): string {
-    return idx.replace(/[^A-Za-z0-9]/g, '');
-  }
+  static toSafeName = (name: string): string => name.toLowerCase().replace(/[^A-Za-z0-9]+/g, '_');
 
   /**
    * Converts a JavaScript value to a DynamoDB AttributeValue format
@@ -51,29 +47,36 @@ export class DynamoDBUtil {
    * Generates global secondary indices and attribute definitions based on the model's index configuration.
    */
   static computeIndexConfig<T extends ModelType>(cls: Class<T>): DynamoIndexConfig {
-    const config = ModelRegistryIndex.getConfig(cls);
+    const indexes = ModelRegistryIndex.getIndices(cls);
     const attributes: AttributeDefinition[] = [];
-    const indices: GlobalSecondaryIndex[] = [];
-
-    for (const idx of config.indices ?? []) {
-      const idxName = this.simpleName(idx.name);
-      attributes.push({ AttributeName: `${idxName}__`, AttributeType: 'S' });
-
-      const keys: KeySchemaElement[] = [{
-        AttributeName: `${idxName}__`,
-        KeyType: 'HASH'
-      }];
-
-      if (idx.type === 'sorted') {
-        keys.push({
-          AttributeName: `${idxName}_sort__`,
-          KeyType: 'RANGE'
-        });
-        attributes.push({ AttributeName: `${idxName}_sort__`, AttributeType: 'N' });
+    const toCreate: GlobalSecondaryIndex[] = [];
+
+    for (const idx of indexes) {
+      if (!isModelIndexedIndex(idx) || ('unique' in idx && idx.unique)) {
+        console.warn('Non-indexed indices are not supported in DynamoDB for', { cls: cls.Ⲑid, idx: idx.name });
+        continue;
+      }
+
+      const keys: KeySchemaElement[] = [];
+
+      const safeName = this.toSafeName(idx.name);
+
+      switch (idx.type) {
+        case 'indexed:sorted':
+          keys.push({ AttributeName: `${safeName}__`, KeyType: 'HASH' });
+          keys.push({ AttributeName: `${safeName}_sort__`, KeyType: 'RANGE', });
+          attributes.push({ AttributeName: `${safeName}__`, AttributeType: 'S' });
+          attributes.push({ AttributeName: `${safeName}_sort__`, AttributeType: 'N' });
+          break;
+        case 'indexed:keyed': {
+          keys.push({ AttributeName: `${safeName}__`, KeyType: 'HASH' });
+          attributes.push({ AttributeName: `${safeName}__`, AttributeType: 'S' });
+          break;
+        }
       }
 
-      indices.push({
-        IndexName: idxName,
+      toCreate.push({
+        IndexName: safeName,
         // ProvisionedThroughput: '',
         Projection: {
           ProjectionType: 'INCLUDE',
@@ -83,7 +86,7 @@ export class DynamoDBUtil {
       });
     }
 
-    return { indices: indices.length ? indices : undefined, attributes };
+    return { indices: toCreate.length ? toCreate : undefined, attributes };
   }
 
   /**