@travetto/model-dynamodb 8.0.0-alpha.10 → 8.0.0-alpha.12

package/README.md

@@ -18,7 +18,7 @@ This module provides an [DynamoDB](https://aws.amazon.com/dynamodb/)-based imple
 Supported features:
    *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L11)
    *  [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10)
-   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model-indexed/src/types/service.ts#L23)
+   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model-indexed/src/types/service.ts#L15)
 
 Out of the box, by installing the module, everything should be wired up by default.If you need to customize any aspect of the source or config, you can override and register it with the [Dependency Injection](https://github.com/travetto/travetto/tree/main/module/di#readme "Dependency registration/management and injection support.") module.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-dynamodb",
-  "version": "8.0.0-alpha.10",
+  "version": "8.0.0-alpha.12",
   "type": "module",
   "description": "DynamoDB backing for the travetto model module.",
   "keywords": [
@@ -26,13 +26,13 @@
     "directory": "module/model-dynamodb"
   },
   "dependencies": {
-    "@aws-sdk/client-dynamodb": "^3.1019.0",
-    "@travetto/config": "^8.0.0-alpha.10",
-    "@travetto/model": "^8.0.0-alpha.10",
-    "@travetto/model-indexed": "^8.0.0-alpha.10"
+    "@aws-sdk/client-dynamodb": "^3.1024.0",
+    "@travetto/config": "^8.0.0-alpha.11",
+    "@travetto/model": "^8.0.0-alpha.11",
+    "@travetto/model-indexed": "^8.0.0-alpha.12"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.15"
+    "@travetto/cli": "^8.0.0-alpha.16"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/service.ts

@@ -1,4 +1,4 @@
-import { type AttributeValue, DynamoDB, type PutItemCommandInput, type PutItemCommandOutput, type QueryCommandOutput } from '@aws-sdk/client-dynamodb';
+import { type AttributeValue, DynamoDB, type PutItemCommandInput, type PutItemCommandOutput, type QueryCommandInput, type QueryCommandOutput } from '@aws-sdk/client-dynamodb';
 
 import { castTo, JSONUtil, ShutdownManager, TimeUtil, type Class } from '@travetto/runtime';
 import { Injectable, PostConstruct } from '@travetto/di';
@@ -6,10 +6,11 @@ import {
   type ModelCrudSupport, type ModelExpirySupport, ModelRegistryIndex, type ModelStorageSupport,
   type ModelType, NotFoundError, ExistsError, type OptionalId, ModelCrudUtil,
   ModelExpiryUtil, ModelStorageUtil,
+  type ModelListOptions,
 } from '@travetto/model';
 import {
   isModelIndexedIndex, ModelIndexedUtil, type KeyedIndexBody, type KeyedIndexSelection,
-  type ListPageOptions, type ListPageResult, type ModelIndexedSupport, type SingleItemIndex,
+  type ModelPageOptions, type ModelPageResult, type ModelIndexedSupport, type SingleItemIndex,
   type FullKeyedIndexBody, type FullKeyedIndexWithPartialBody, type SortedIndex, type SortedIndexSelection,
   ModelIndexedComputedIndex
 } from '@travetto/model-indexed';
@@ -42,51 +43,59 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
     return table;
   }
 
-  async * #scanIndex<
-    T extends ModelType,
-    K extends KeyedIndexSelection<T>,
-    S extends SortedIndexSelection<T>
-  >(
-    cls: Class,
-    idx: SortedIndex<T, K, S>,
-    body: KeyedIndexBody<T, K>,
-    options?: ListPageOptions<Record<string, AttributeValue>>
-  ): AsyncIterable<QueryCommandOutput & { LastEvaluatedOffset?: string }> {
-    ModelCrudUtil.ensureNotSubType(cls);
-    const computed = ModelIndexedComputedIndex.get(idx, body).validate();
-    const safeName = DynamoDBUtil.toSafeName(idx.name);
-    const expression = { [`:${safeName}`]: getKey(computed) };
-    const limit = options?.limit ?? 100;
-
+  async * #scanCollection<T extends ModelType>(
+    cls: Class<T>,
+    query: (batchSize: number, lastKey: Record<string, AttributeValue> | undefined) => Promise<QueryCommandOutput>,
+    options?: ModelListOptions & ModelPageOptions<Record<string, AttributeValue>>,
+  ): AsyncIterable<{ items: T[], lastKey?: Record<string, AttributeValue> }> {
+    const batchSize = options?.batchSizeHint ?? 100;
+    const limit = options?.limit ?? Number.MAX_SAFE_INTEGER;
     let startKey = options?.offset ?? undefined;
     let produced = 0;
-
     do {
       const remaining = limit - produced;
-      const batch = await this.client.query({
-        TableName: this.#resolveTable(cls),
-        IndexName: safeName,
-        ProjectionExpression: 'body',
-        KeyConditionExpression: `${safeName}__ = :${safeName}`,
-        ExpressionAttributeValues: expression,
-        Limit: Math.min(remaining, 100),
-        ExclusiveStartKey: startKey,
-      });
+      const batch = await query(Math.min(remaining, batchSize), startKey);
 
       if (batch.Count && batch.Items) {
         produced += batch.Count;
 
-        if (produced > limit) {
-          const items = batch.Items.slice(0, remaining);
-          yield { ...batch, Items: items, };
-        } else {
-          yield batch;
-        }
+        const items = (produced > limit) ? batch.Items.slice(0, remaining) : batch.Items;
         startKey = batch.LastEvaluatedKey;
+        yield {
+          items: await ModelCrudUtil.filterOutNotFound(
+            items.map(item => DynamoDBUtil.loadAndCheckExpiry(cls, item.body.S!))),
+          lastKey: startKey
+        };
       } else {
         startKey = undefined;
       }
-    } while (startKey && produced < limit);
+    } while (startKey && produced < limit && !(options?.abort?.aborted));
+  }
+
+  async * #scanIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ModelPageOptions<Record<string, AttributeValue>> & ModelListOptions
+  ): AsyncIterable<{ items: T[], lastKey?: Record<string, AttributeValue> }> {
+    ModelCrudUtil.ensureNotSubType(cls);
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate();
+    const safeName = DynamoDBUtil.toSafeName(idx.name);
+    const expression = { [`:${safeName}`]: getKey(computed) };
+
+    yield* this.#scanCollection(cls, (batchSize, lastKey) => this.client.query({
+      TableName: this.#resolveTable(cls),
+      IndexName: safeName,
+      ProjectionExpression: 'body',
+      KeyConditionExpression: `${safeName}__ = :${safeName}`,
+      ExpressionAttributeValues: expression,
+      Limit: batchSize,
+      ExclusiveStartKey: lastKey,
+    }), options);
   }
 
   async #getIdByIndex<
@@ -101,16 +110,22 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
     const safeName = DynamoDBUtil.toSafeName(idx.name);
     const sorted = idx.type === 'indexed:sorted';
 
-    const query = {
+    const query: QueryCommandInput = {
       TableName: this.#resolveTable(cls),
       IndexName: safeName,
       ProjectionExpression: 'id',
-      KeyConditionExpression: [sorted ? `${safeName}_sort__ = :${safeName}_sort` : '', `${safeName}__ = :${safeName}`]
-        .filter(expr => !!expr)
+      KeyConditionExpression: [
+        ...(sorted ? [`${safeName}_sort__ = :${safeName}_sort`] : []),
+        `${safeName}__ = :${safeName}`
+      ]
         .join(' and '),
+      ...(computed.idPart ? {
+        FilterExpression: 'id = :id'
+      } : {}),
       ExpressionAttributeValues: {
         [`:${safeName}`]: getKey(computed),
-        ...(sorted ? { [`:${safeName}_sort`]: getSort(computed) } : {})
+        ...(sorted ? { [`:${safeName}_sort`]: getSort(computed) } : {}),
+        ...(computed.idPart ? { ':id': DynamoDBUtil.toValue(computed.idPart.value) } : {})
       }
     };
 
@@ -369,32 +384,13 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
     }
   }
 
-  async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T> {
-    let done = false;
-    let token: Record<string, AttributeValue> | undefined;
-    while (!done) {
-      const batch = await this.client.scan({
-        TableName: this.#resolveTable(cls),
-        ExclusiveStartKey: token
-      });
-
-      if (batch.Count && batch.Items) {
-        for (const item of batch.Items) {
-          try {
-            yield await DynamoDBUtil.loadAndCheckExpiry(cls, item.body.S!);
-          } catch (error) {
-            if (!(error instanceof NotFoundError)) {
-              throw error;
-            }
-          }
-        }
-      }
-
-      if (!batch.Count || !batch.LastEvaluatedKey) {
-        done = true;
-      } else {
-        token = batch.LastEvaluatedKey;
-      }
+  async * list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]> {
+    for await (const { items } of this.#scanCollection(cls, (batchSize, lastKey) => this.client.scan({
+      TableName: this.#resolveTable(cls),
+      ExclusiveStartKey: lastKey,
+      Limit: batchSize
+    }), options)) {
+      yield items;
     }
   }
 
@@ -445,7 +441,7 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
     return this.update(cls, item);
   }
 
-  async listByIndex<
+  async pageByIndex<
     T extends ModelType,
     K extends KeyedIndexSelection<T>,
     S extends SortedIndexSelection<T>
@@ -453,28 +449,17 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
     cls: Class<T>,
     idx: SortedIndex<T, K, S>,
     body: KeyedIndexBody<T, K>,
-    options?: ListPageOptions,
-  ): Promise<ListPageResult<T>> {
-    const items: T[] = [];
+    options?: ModelPageOptions,
+  ): Promise<ModelPageResult<T>> {
+    const output: T[] = [];
     const offset = options?.offset ? JSONUtil.fromBase64<Record<string, AttributeValue>>(options.offset) : undefined;
-    for await (const batch of this.#scanIndex(cls, idx, body, { ...options, offset })) {
-      for (const item of batch.Items ?? []) {
-        try {
-          items.push(await DynamoDBUtil.loadAndCheckExpiry(cls, item.body.S!));
-          if (options?.limit && items.length >= options.limit) {
-            break;
-          }
-        } catch (error) {
-          if (!(error instanceof NotFoundError)) {
-            throw error;
-          }
-        }
-      }
+    for await (const { items } of this.#scanIndex(cls, idx, body, { limit: 100, ...options, offset })) {
+      output.push(...items);
     }
 
     let nextOffset;
-    if (items.length) {
-      const last: T = items.at(-1)!;
+    if (output.length) {
+      const last: T = output.at(-1)!;
       const computed = ModelIndexedComputedIndex.get(idx, last).validate();
       const safeName = DynamoDBUtil.toSafeName(idx.name);
       nextOffset = JSONUtil.toBase64({
@@ -484,6 +469,21 @@ export class DynamoDBModelService implements ModelCrudSupport, ModelExpirySuppor
       });
     }
 
-    return { items, nextOffset };
+    return { items: output, nextOffset };
+  }
+
+  async * listByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ModelListOptions
+  ): AsyncIterable<T[]> {
+    for await (const { items } of this.#scanIndex(cls, idx, body, options)) {
+      yield items;
+    }
   }
 }