@ftschopp/dynatable-core 1.2.5 → 1.4.0

package/src/builders/scan/create-scan-builder.test.ts

@@ -1,7 +1,11 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
+import { ScanCommand } from '@aws-sdk/lib-dynamodb';
+import { mockClient } from 'aws-sdk-client-mock';
 import { createScanBuilder } from './create-scan-builder';
 
+const ddbMock = mockClient(DynamoDBClient);
+
 describe('ScanBuilder', () => {
   const client = new DynamoDBClient({});
   const tableName = 'TestTable';
@@ -257,4 +261,121 @@ describe('ScanBuilder', () => {
       expect(Object.values(params.ExpressionAttributeValues || {})).toContain(65);
     });
   });
+
+  describe('executeWithPagination', () => {
+    beforeEach(() => {
+      ddbMock.reset();
+    });
+
+    test('returns items, lastEvaluatedKey and counts', async () => {
+      const items = [
+        { pk: 'USER#alice', sk: 'USER#alice', name: 'Alice' },
+        { pk: 'USER#bob', sk: 'USER#bob', name: 'Bob' },
+      ];
+      const lastKey = { pk: 'USER#bob', sk: 'USER#bob' };
+
+      ddbMock.on(ScanCommand).resolves({
+        Items: items,
+        LastEvaluatedKey: lastKey,
+        Count: 2,
+        ScannedCount: 5,
+      });
+
+      const result = await createScanBuilder<TestModel>(tableName, client).executeWithPagination();
+
+      expect(result.items).toEqual(items);
+      expect(result.lastEvaluatedKey).toEqual(lastKey);
+      expect(result.count).toBe(2);
+      expect(result.scannedCount).toBe(5);
+    });
+
+    test('returns undefined lastEvaluatedKey on the final page', async () => {
+      ddbMock.on(ScanCommand).resolves({ Items: [], Count: 0, ScannedCount: 0 });
+
+      const result = await createScanBuilder<TestModel>(tableName, client).executeWithPagination();
+
+      expect(result.items).toEqual([]);
+      expect(result.lastEvaluatedKey).toBeUndefined();
+    });
+  });
+
+  describe('iterate', () => {
+    beforeEach(() => {
+      ddbMock.reset();
+    });
+
+    test('walks every page transparently and yields items in order', async () => {
+      const page1 = [
+        { pk: 'USER#1', sk: 'USER#1', name: 'User 1' },
+        { pk: 'USER#2', sk: 'USER#2', name: 'User 2' },
+      ];
+      const page2 = [{ pk: 'USER#3', sk: 'USER#3', name: 'User 3' }];
+
+      ddbMock
+        .on(ScanCommand)
+        .resolvesOnce({ Items: page1, LastEvaluatedKey: { pk: 'USER#2', sk: 'USER#2' } })
+        .resolvesOnce({ Items: page2 });
+
+      const collected: TestModel[] = [];
+      for await (const item of createScanBuilder<TestModel>(tableName, client).iterate()) {
+        collected.push(item);
+      }
+
+      expect(collected.map((u) => u.name)).toEqual(['User 1', 'User 2', 'User 3']);
+      expect(ddbMock.calls()).toHaveLength(2);
+    });
+
+    test('forwards LastEvaluatedKey from each response into the next call', async () => {
+      const cursor1 = { pk: 'USER#2', sk: 'USER#2' };
+
+      ddbMock
+        .on(ScanCommand)
+        .resolvesOnce({ Items: [{ pk: 'USER#1', sk: 'USER#1' }], LastEvaluatedKey: cursor1 })
+        .resolvesOnce({ Items: [{ pk: 'USER#3', sk: 'USER#3' }] });
+
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      for await (const _ of createScanBuilder<TestModel>(tableName, client).iterate()) {
+        // drain
+      }
+
+      const calls = ddbMock.calls();
+      expect((calls[0]!.args[0].input as any).ExclusiveStartKey).toBeUndefined();
+      expect((calls[1]!.args[0].input as any).ExclusiveStartKey).toEqual(cursor1);
+    });
+
+    test('starts from the user-provided cursor on the first call', async () => {
+      const startKey = { pk: 'USER#10', sk: 'USER#10' };
+
+      ddbMock.on(ScanCommand).resolves({ Items: [{ pk: 'USER#11', sk: 'USER#11' }] });
+
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      for await (const _ of createScanBuilder<TestModel>(tableName, client)
+        .startFrom(startKey)
+        .iterate()) {
+        // drain
+      }
+
+      const calls = ddbMock.calls();
+      expect((calls[0]!.args[0].input as any).ExclusiveStartKey).toEqual(startKey);
+    });
+
+    test('break out of the loop stops further DynamoDB calls', async () => {
+      ddbMock.on(ScanCommand).resolvesOnce({
+        Items: [
+          { pk: 'USER#1', sk: 'USER#1' },
+          { pk: 'USER#2', sk: 'USER#2' },
+        ],
+        LastEvaluatedKey: { pk: 'USER#2', sk: 'USER#2' },
+      });
+
+      const collected: TestModel[] = [];
+      for await (const item of createScanBuilder<TestModel>(tableName, client).iterate()) {
+        collected.push(item);
+        if (collected.length >= 1) break;
+      }
+
+      expect(collected).toHaveLength(1);
+      expect(ddbMock.calls()).toHaveLength(1);
+    });
+  });
 });

package/src/builders/scan/create-scan-builder.ts

@@ -2,7 +2,7 @@
 import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
 import { ScanCommand } from '@aws-sdk/lib-dynamodb';
 import { buildExpression, AttrBuilder, Condition, createOpBuilder } from '../shared';
-import { ScanBuilder } from './types';
+import { ScanBuilder, ScanResult } from './types';
 import { DynamoDBLogger } from '../../utils/dynamodb-logger';
 
 /**
@@ -205,12 +205,46 @@ export function createScanBuilder<Model>(
       return params;
     },
 
+    /**
+     * ⚠️ Returns only the FIRST page of results. DynamoDB caps each Scan
+     * response at ~1MB (or `Limit` if set). If the matching set is larger,
+     * the remaining items are silently dropped.
+     *
+     * Use {@link executeWithPagination} when you need to drive pagination
+     * yourself, or {@link iterate} to walk every matching item lazily.
+     */
     async execute() {
       const params = build().dbParams();
       const response = await client.send(new ScanCommand(params));
       logger?.log('ScanCommand', params, response);
       return (response.Items || []) as Model[];
     },
+
+    async executeWithPagination(): Promise<ScanResult<Model>> {
+      const params = build().dbParams();
+      const response = await client.send(new ScanCommand(params));
+      logger?.log('ScanCommand', params, response);
+      return {
+        items: (response.Items ?? []) as Model[],
+        lastEvaluatedKey: response.LastEvaluatedKey,
+        count: response.Count,
+        scannedCount: response.ScannedCount,
+      };
+    },
+
+    async *iterate(): AsyncIterableIterator<Model> {
+      const baseParams = build().dbParams();
+      let cursor: Record<string, any> | undefined = baseParams.ExclusiveStartKey;
+      do {
+        const params = { ...baseParams, ExclusiveStartKey: cursor };
+        const response = await client.send(new ScanCommand(params));
+        logger?.log('ScanCommand', params, response);
+        for (const item of (response.Items ?? []) as Model[]) {
+          yield item;
+        }
+        cursor = response.LastEvaluatedKey;
+      } while (cursor);
+    },
   });
 
   return build();

package/src/builders/scan/types.ts

@@ -1,5 +1,20 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
 import { ExecutableBuilder, AttrBuilder, OpBuilder, Condition } from '../shared';
 
+/**
+ * A single page of scan results plus metadata for manual pagination.
+ */
+export interface ScanResult<Model> {
+  /** Items returned in this page. */
+  items: Model[];
+  /** Cursor for the next page; `undefined` when there are no more results. */
+  lastEvaluatedKey?: Record<string, any>;
+  /** Number of items returned after applying the filter. */
+  count?: number;
+  /** Number of items examined before applying the filter. */
+  scannedCount?: number;
+}
+
 /**
  * Builder for DynamoDB Scan operations.
  * Scans the entire table or index without requiring key conditions.
@@ -46,4 +61,28 @@ export interface ScanBuilder<Model> extends ExecutableBuilder<Model[]> {
    * Returns a new immutable builder.
    */
   segment(segmentNumber: number, totalSegments: number): ScanBuilder<Model>;
+
+  /**
+   * Executes a single Scan request and returns the page along with the
+   * `lastEvaluatedKey` cursor and counts. Use this when you want to drive
+   * pagination yourself (e.g. expose a cursor to a client).
+   */
+  executeWithPagination(): Promise<ScanResult<Model>>;
+
+  /**
+   * Returns an async iterator that paginates internally and yields one item at
+   * a time. Memory stays at one page; you can `break` out of the loop early.
+   *
+   * `Limit` set via `.limit()` is forwarded to DynamoDB as a *per-request*
+   * cap, not a total cap. To cap the total, count yourself inside the loop:
+   *
+   * ```ts
+   * let n = 0;
+   * for await (const item of builder.iterate()) {
+   *   if (n++ >= 1000) break;
+   *   process(item);
+   * }
+   * ```
+   */
+  iterate(): AsyncIterableIterator<Model>;
 }

package/src/builders/shared/operators.ts

@@ -21,6 +21,7 @@ export function createOpBuilder(): OpBuilder {
         expression: `#${attr.name} = :${valueName}`,
         names: { [`#${attr.name}`]: attr.name },
         values: { [`:${valueName}`]: value },
+        keyOperator: 'eq',
       };
     },
     ne: (attr, value) => {
@@ -29,6 +30,7 @@ export function createOpBuilder(): OpBuilder {
         expression: `#${attr.name} <> :${valueName}`,
         names: { [`#${attr.name}`]: attr.name },
         values: { [`:${valueName}`]: value },
+        keyOperator: 'ne',
       };
     },
     lt: (attr, value) => {
@@ -37,6 +39,7 @@ export function createOpBuilder(): OpBuilder {
         expression: `#${attr.name} < :${valueName}`,
         names: { [`#${attr.name}`]: attr.name },
         values: { [`:${valueName}`]: value },
+        keyOperator: 'lt',
       };
     },
     lte: (attr, value) => {
@@ -45,6 +48,7 @@ export function createOpBuilder(): OpBuilder {
         expression: `#${attr.name} <= :${valueName}`,
         names: { [`#${attr.name}`]: attr.name },
         values: { [`:${valueName}`]: value },
+        keyOperator: 'lte',
       };
     },
     gt: (attr, value) => {
@@ -53,6 +57,7 @@ export function createOpBuilder(): OpBuilder {
         expression: `#${attr.name} > :${valueName}`,
         names: { [`#${attr.name}`]: attr.name },
         values: { [`:${valueName}`]: value },
+        keyOperator: 'gt',
       };
     },
     gte: (attr, value) => {
@@ -61,6 +66,7 @@ export function createOpBuilder(): OpBuilder {
         expression: `#${attr.name} >= :${valueName}`,
         names: { [`#${attr.name}`]: attr.name },
         values: { [`:${valueName}`]: value },
+        keyOperator: 'gte',
       };
     },
     between: (attr, low, high) => {
@@ -73,6 +79,7 @@ export function createOpBuilder(): OpBuilder {
           [`:${lowName}`]: low,
           [`:${highName}`]: high,
         },
+        keyOperator: 'between',
       };
     },
     beginsWith: (attr, value) => {
@@ -81,6 +88,7 @@ export function createOpBuilder(): OpBuilder {
         expression: `begins_with(#${attr.name}, :${valueName})`,
         names: { [`#${attr.name}`]: attr.name },
         values: { [`:${valueName}`]: value },
+        keyOperator: 'beginsWith',
       };
     },
     contains: (attr, value) => {

package/src/builders/shared/types.ts

@@ -11,6 +11,7 @@ export type Condition = {
   names?: Record<string, string>;
   values?: Record<string, any>;
   operator?: 'AND' | 'OR';
+  keyOperator?: 'eq' | 'ne' | 'lt' | 'lte' | 'gt' | 'gte' | 'between' | 'beginsWith';
   isNegated?: boolean;
   children?: Condition[];
 };

package/src/builders/update/create-update-builder.test.ts

@@ -76,6 +76,30 @@ describe('UpdateBuilder', () => {
       });
     });
 
+    test('should build params with single-property object whose key is "name"', () => {
+      const key: Partial<TestModel> = { pk: 'USER#1', sk: 'USER#1' };
+      const params = createUpdateBuilder<TestModel>(tableName, key, client)
+        .set({ name: 'Ministro Pistarini' })
+        .dbParams();
+
+      expect(params.UpdateExpression).toBe('SET #name = :name_0');
+      expect(params.ExpressionAttributeNames).toEqual({ '#name': 'name' });
+      expect(params.ExpressionAttributeValues).toEqual({
+        ':name_0': 'Ministro Pistarini',
+      });
+    });
+
+    test('should build params with single-property object whose key is not "name"', () => {
+      const key: Partial<TestModel> = { pk: 'USER#1', sk: 'USER#1' };
+      const params = createUpdateBuilder<TestModel>(tableName, key, client)
+        .set({ age: 30 })
+        .dbParams();
+
+      expect(params.UpdateExpression).toBe('SET #age = :age_0');
+      expect(params.ExpressionAttributeNames).toEqual({ '#age': 'age' });
+      expect(params.ExpressionAttributeValues).toEqual({ ':age_0': 30 });
+    });
+
     test('should combine single and multiple SET operations', () => {
       const key: Partial<TestModel> = { pk: 'USER#1', sk: 'USER#1' };
       const params = createUpdateBuilder<TestModel>(tableName, key, client)
@@ -344,6 +368,179 @@ describe('UpdateBuilder', () => {
     });
   });
 
+  describe('Secondary-index recomputation (indexContext)', () => {
+    interface PersonnelModel {
+      id: string;
+      airportId: string;
+      firstName: string;
+      lastName: string;
+      role: string;
+    }
+
+    const personnelModel = {
+      key: {
+        PK: { type: String, value: 'PERSON#${id}' },
+        SK: { type: String, value: 'PROFILE' },
+      },
+      index: {
+        GSI1PK: { type: String, value: 'AIRPORT#${airportId}' },
+        GSI1SK: { type: String, value: 'PERSON#${lastName}#${firstName}' },
+      },
+      attributes: {
+        id: { type: String, required: true },
+        airportId: { type: String, required: true },
+        firstName: { type: String, required: true },
+        lastName: { type: String, required: true },
+        role: { type: String },
+      },
+    } as const;
+
+    test('does nothing when no .set() field appears in any index template', () => {
+      const params = createUpdateBuilder<PersonnelModel>(
+        tableName,
+        { id: '1' } as Partial<PersonnelModel>,
+        client,
+        [],
+        { set: [], remove: [], add: [], delete: [] },
+        'NONE',
+        0,
+        false,
+        undefined,
+        { model: personnelModel as any, keyVars: { id: '1' } }
+      )
+        .set('role', 'pilot')
+        .dbParams();
+
+      expect(params.UpdateExpression).toBe('SET #role = :role_0');
+      expect(params.ExpressionAttributeNames).toEqual({ '#role': 'role' });
+    });
+
+    test('recomputes affected index when all template vars are present in updates', () => {
+      const params = createUpdateBuilder<PersonnelModel>(
+        tableName,
+        { id: '1' } as Partial<PersonnelModel>,
+        client,
+        [],
+        { set: [], remove: [], add: [], delete: [] },
+        'NONE',
+        0,
+        false,
+        undefined,
+        { model: personnelModel as any, keyVars: { id: '1' } }
+      )
+        .set({ firstName: 'Ada', lastName: 'Lovelace' })
+        .dbParams();
+
+      // GSI1SK depends on lastName + firstName — both supplied → recomputed.
+      // GSI1PK depends on airportId — untouched, no recompute.
+      expect(params.UpdateExpression).toBe(
+        'SET #firstName = :firstName_0, #lastName = :lastName_1, #GSI1SK = :GSI1SK_2'
+      );
+      expect(params.ExpressionAttributeValues).toMatchObject({
+        ':firstName_0': 'Ada',
+        ':lastName_1': 'Lovelace',
+        ':GSI1SK_2': 'PERSON#Lovelace#Ada',
+      });
+    });
+
+    test('throws when an affected index template references a field not in updates or key', () => {
+      const builder = createUpdateBuilder<PersonnelModel>(
+        tableName,
+        { id: '1' } as Partial<PersonnelModel>,
+        client,
+        [],
+        { set: [], remove: [], add: [], delete: [] },
+        'NONE',
+        0,
+        false,
+        undefined,
+        { model: personnelModel as any, keyVars: { id: '1' } }
+      ).set({ lastName: 'Lovelace' });
+
+      // GSI1SK template = "PERSON#${lastName}#${firstName}" — firstName is missing.
+      expect(() => builder.dbParams()).toThrow(/firstName/);
+      expect(() => builder.dbParams()).toThrow(/GSI1SK/);
+    });
+
+    test('uses primary-key template vars when resolving index templates', () => {
+      const userModel = {
+        key: {
+          PK: { type: String, value: 'USER#${username}' },
+          SK: { type: String, value: 'USER#${username}' },
+        },
+        index: {
+          GSI1PK: { type: String, value: 'USER#${username}#STATUS#${status}' },
+        },
+        attributes: {
+          username: { type: String, required: true },
+          status: { type: String, required: true },
+        },
+      } as const;
+
+      const params = createUpdateBuilder<{ username: string; status: string }>(
+        tableName,
+        { username: 'jane' } as any,
+        client,
+        [],
+        { set: [], remove: [], add: [], delete: [] },
+        'NONE',
+        0,
+        false,
+        undefined,
+        { model: userModel as any, keyVars: { username: 'jane' } }
+      )
+        .set('status', 'active')
+        .dbParams();
+
+      expect(params.UpdateExpression).toBe(
+        'SET #status = :status_0, #GSI1PK = :GSI1PK_1'
+      );
+      expect(params.ExpressionAttributeValues).toMatchObject({
+        ':status_0': 'active',
+        ':GSI1PK_1': 'USER#jane#STATUS#active',
+      });
+    });
+
+    test('accumulates set inputs across chained calls before resolving', () => {
+      const params = createUpdateBuilder<PersonnelModel>(
+        tableName,
+        { id: '1' } as Partial<PersonnelModel>,
+        client,
+        [],
+        { set: [], remove: [], add: [], delete: [] },
+        'NONE',
+        0,
+        false,
+        undefined,
+        { model: personnelModel as any, keyVars: { id: '1' } }
+      )
+        .set('lastName', 'Lovelace')
+        .set('firstName', 'Ada')
+        .dbParams();
+
+      // Each individual .set() was missing one var, but the accumulator has both
+      // by the time dbParams() runs.
+      expect(params.UpdateExpression).toBe(
+        'SET #lastName = :lastName_0, #firstName = :firstName_1, #GSI1SK = :GSI1SK_2'
+      );
+      expect(params.ExpressionAttributeValues).toMatchObject({
+        ':GSI1SK_2': 'PERSON#Lovelace#Ada',
+      });
+    });
+
+    test('does nothing when indexContext is omitted (backwards-compatible default)', () => {
+      const params = createUpdateBuilder<PersonnelModel>(
+        tableName,
+        { id: '1' } as Partial<PersonnelModel>,
+        client
+      )
+        .set({ lastName: 'Lovelace' })
+        .dbParams();
+
+      expect(params.UpdateExpression).toBe('SET #lastName = :lastName_0');
+    });
+  });
+
   describe('Complex scenarios', () => {
     test('should handle complex update with all features', () => {
       const key: Partial<TestModel> = { pk: 'USER#1', sk: 'USER#1' };