@model-ts/dynamodb 2.0.0 → 3.0.0

package/src/__test__/client.test.ts

@@ -134,6 +134,10 @@ beforeEach(async () => {
   sandbox = await createSandbox(client)
 })
 
+afterEach(async () => {
+  await sandbox.destroy()
+})
+
 describe("put", () => {
   describe("via instance", () => {
     test("it inserts a simple model", async () => {
@@ -2135,6 +2139,56 @@ describe("paginate", () => {
       expect(page1.edges[0].node.c).toBe("0")
       expect(page1.edges[49].node.c).toBe("49")
     })
+
+    test("it respects custom pagination default", async () => {
+      client.paginationOptions = {
+        default: 40,
+      }
+
+      const items = Array.from({ length: 50 }).map(
+        (_, i) =>
+          new C({ pk: "PK", sk: String(i).padStart(3, "0"), c: String(i) })
+      )
+
+      await sandbox.seed(...items)
+
+      const page = await client.paginate(
+        C,
+        {},
+        {
+          KeyConditionExpression: "PK = :pk",
+          ExpressionAttributeValues: { ":pk": "PK" },
+        }
+      )
+      expect(page.edges.length).toBe(40)
+
+      delete client.paginationOptions
+    })
+
+    test("it respects custom pagination limit", async () => {
+      client.paginationOptions = {
+        limit: 100,
+      }
+
+      const items = Array.from({ length: 120 }).map(
+        (_, i) =>
+          new C({ pk: "PK", sk: String(i).padStart(3, "0"), c: String(i) })
+      )
+
+      await sandbox.seed(...items)
+
+      const page = await client.paginate(
+        C,
+        { first: 110 },
+        {
+          KeyConditionExpression: "PK = :pk",
+          ExpressionAttributeValues: { ":pk": "PK" },
+        }
+      )
+      expect(page.edges.length).toBe(100)
+
+      delete client.paginationOptions
+    })
   })
 
   describe("model", () => {
@@ -2301,6 +2355,54 @@ describe("paginate", () => {
       expect(page1.edges[0].node.c).toBe("0")
       expect(page1.edges[49].node.c).toBe("49")
     })
+
+    test("it respects custom pagination default", async () => {
+      client.paginationOptions = {
+        default: 40,
+      }
+
+      const items = Array.from({ length: 50 }).map(
+        (_, i) =>
+          new C({ pk: "PK", sk: String(i).padStart(3, "0"), c: String(i) })
+      )
+
+      await sandbox.seed(...items)
+
+      const page = await C.paginate(
+        {},
+        {
+          KeyConditionExpression: "PK = :pk",
+          ExpressionAttributeValues: { ":pk": "PK" },
+        }
+      )
+      expect(page.edges.length).toBe(40)
+
+      delete client.paginationOptions
+    })
+
+    test("it respects custom pagination limit", async () => {
+      client.paginationOptions = {
+        limit: 100,
+      }
+
+      const items = Array.from({ length: 120 }).map(
+        (_, i) =>
+          new C({ pk: "PK", sk: String(i).padStart(3, "0"), c: String(i) })
+      )
+
+      await sandbox.seed(...items)
+
+      const page = await C.paginate(
+        { first: 110 },
+        {
+          KeyConditionExpression: "PK = :pk",
+          ExpressionAttributeValues: { ":pk": "PK" },
+        }
+      )
+      expect(page.edges.length).toBe(100)
+
+      delete client.paginationOptions
+    })
   })
 
   describe("union", () => {
@@ -2470,5 +2572,71 @@ describe("paginate", () => {
       expect(page1.edges[0].node.SK).toBe("000")
       expect(page1.edges[49].node.SK).toBe("049")
     })
+
+    test("it respects custom pagination default", async () => {
+      client.paginationOptions = {
+        default: 40,
+      }
+
+      const items = Array.from({ length: 50 }).map((_, i) =>
+        i > 30
+          ? new C({
+              pk: "PK",
+              sk: String(i).padStart(3, "0"),
+              c: String(i),
+            })
+          : new D({
+              pk: "PK",
+              sk: String(i).padStart(3, "0"),
+              d: String(i),
+            })
+      )
+
+      await sandbox.seed(...items)
+
+      const page = await Union.paginate(
+        {},
+        {
+          KeyConditionExpression: "PK = :pk",
+          ExpressionAttributeValues: { ":pk": "PK" },
+        }
+      )
+      expect(page.edges.length).toBe(40)
+
+      delete client.paginationOptions
+    })
+
+    test("it respects custom pagination limit", async () => {
+      client.paginationOptions = {
+        limit: 100,
+      }
+
+      const items = Array.from({ length: 110 }).map((_, i) =>
+        i > 30
+          ? new C({
+              pk: "PK",
+              sk: String(i).padStart(3, "0"),
+              c: String(i),
+            })
+          : new D({
+              pk: "PK",
+              sk: String(i).padStart(3, "0"),
+              d: String(i),
+            })
+      )
+
+      await sandbox.seed(...items)
+
+      const page = await Union.paginate(
+        { first: 110 },
+        {
+          KeyConditionExpression: "PK = :pk",
+          ExpressionAttributeValues: { ":pk": "PK" },
+        }
+      )
+      expect(page.edges.length).toBe(100)
+
+      delete client.paginationOptions
+    })
   })
 })

package/src/client.ts

@@ -40,8 +40,10 @@ import {
   encodeDDBCursor,
   PaginationDirection,
   PaginationInput,
+  PaginationOptions,
   PaginationResult,
 } from "./pagination"
+import { GSI, GSI_NAMES, GSIPK, GSISK } from "./gsi"
 
 export type QueryParams = Omit<
   DocumentClient.QueryInput,
@@ -85,6 +87,11 @@ export interface ClientProps
    * Must be a 32 character string, 256 bits.
    */
   cursorEncryptionKey?: Buffer
+
+  /**
+   * Defaults for pagination.
+   */
+  paginationOptions?: PaginationOptions
 }
 
 export interface Key {
@@ -97,10 +104,12 @@ export class Client {
   documentClient: DocumentClient
   dataLoader: DataLoader<GetOperation<Decodable>, DynamoDBModelInstance, string>
   cursorEncryptionKey?: Buffer
+  paginationOptions?: PaginationOptions
 
   constructor(props: ClientProps) {
     this.tableName = props?.tableName
     this.cursorEncryptionKey = props?.cursorEncryptionKey
+    this.paginationOptions = props?.paginationOptions
     this.documentClient = new DocumentClient(props)
     this.dataLoader = new DataLoader<
       GetOperation<Decodable>,
@@ -525,7 +534,10 @@ export class Client {
     args: PaginationInput,
     params: PaginationParams
   ): Promise<PaginationResult<DecodableInstance<M>>> {
-    const { cursor, limit, direction } = decodePagination(args)
+    const { cursor, limit, direction } = decodePagination(
+      args,
+      this.paginationOptions
+    )
 
     const { results } = await this.query(
       {
@@ -537,7 +549,7 @@ export class Client {
               cursor,
               // GSI1 is the inverse index and uses PK and SK (switched around)
               params.IndexName && params.IndexName !== "GSI1"
-                ? (params.IndexName as "GSI2" | "GSI3" | "GSI4" | "GSI5")
+                ? (params.IndexName as GSI)
                 : undefined,
               this.cursorEncryptionKey
             )
@@ -888,15 +900,8 @@ export class Client {
     T extends {
       PK: string
       SK: string
-      GSI2PK?: string
-      GSI2SK?: string
-      GSI3PK?: string
-      GSI3SK?: string
-      GSI4PK?: string
-      GSI4SK?: string
-      GSI5PK?: string
-      GSI5SK?: string
-    }
+    } & { [key in GSIPK]?: string } &
+      { [key in GSISK]?: string }
   >(item: T): T {
     const prefix = "$$DELETED$$"
 
@@ -908,14 +913,10 @@ export class Client {
       ...item,
       PK: maybeWithPrefix(item.PK),
       SK: maybeWithPrefix(item.SK),
-      GSI2PK: maybeWithPrefix(item.GSI2PK),
-      GSI2SK: maybeWithPrefix(item.GSI2SK),
-      GSI3PK: maybeWithPrefix(item.GSI3PK),
-      GSI3SK: maybeWithPrefix(item.GSI3SK),
-      GSI4PK: maybeWithPrefix(item.GSI4PK),
-      GSI4SK: maybeWithPrefix(item.GSI4SK),
-      GSI5PK: maybeWithPrefix(item.GSI5PK),
-      GSI5SK: maybeWithPrefix(item.GSI5SK),
+      ...GSI_NAMES.map((GSI) => ({
+        [`${GSI}PK`]: maybeWithPrefix(item[`${GSI}PK` as const]),
+        [`${GSI}SK`]: maybeWithPrefix(item[`${GSI}SK` as const]),
+      })).reduce((acc, cur) => Object.assign(acc, cur), {}),
     }
   }
 }

package/src/dynamodb-model.ts

@@ -1,4 +1,5 @@
 import { ModelInstance, ModelConstructor, Union } from "@model-ts/core"
+import { GSIPK, GSISK } from "./gsi"
 
 export interface DynamoDBModelInstance extends ModelInstance<string, any> {
   /**
@@ -7,15 +8,8 @@ export interface DynamoDBModelInstance extends ModelInstance<string, any> {
   keys(): {
     PK: string
     SK: string
-    GSI2PK?: string
-    GSI2SK?: string
-    GSI3PK?: string
-    GSI3SK?: string
-    GSI4PK?: string
-    GSI4SK?: string
-    GSI5PK?: string
-    GSI5SK?: string
-  }
+  } & { [key in GSIPK]?: string } &
+    { [key in GSISK]?: string }
 
   PK: string
   SK: string
@@ -27,6 +21,34 @@ export interface DynamoDBModelInstance extends ModelInstance<string, any> {
   GSI4SK?: string
   GSI5PK?: string
   GSI5SK?: string
+  GSI6PK?: string
+  GSI6SK?: string
+  GSI7PK?: string
+  GSI7SK?: string
+  GSI8PK?: string
+  GSI8SK?: string
+  GSI9PK?: string
+  GSI9SK?: string
+  GSI10PK?: string
+  GSI10SK?: string
+  GSI11PK?: string
+  GSI11SK?: string
+  GSI12PK?: string
+  GSI12SK?: string
+  GSI13PK?: string
+  GSI13SK?: string
+  GSI14PK?: string
+  GSI14SK?: string
+  GSI15PK?: string
+  GSI15SK?: string
+  GSI16PK?: string
+  GSI16SK?: string
+  GSI17PK?: string
+  GSI17SK?: string
+  GSI18PK?: string
+  GSI18SK?: string
+  GSI19PK?: string
+  GSI19SK?: string
 }
 
 // export interface DynamoDBModel {}

package/src/gsi.ts

@@ -0,0 +1,25 @@
+export const GSI_NAMES = [
+  "GSI2",
+  "GSI3",
+  "GSI4",
+  "GSI5",
+  "GSI6",
+  "GSI7",
+  "GSI8",
+  "GSI9",
+  "GSI10",
+  "GSI11",
+  "GSI12",
+  "GSI13",
+  "GSI14",
+  "GSI15",
+  "GSI16",
+  "GSI17",
+  "GSI18",
+  "GSI19",
+] as const
+
+export type GSI = typeof GSI_NAMES[number]
+
+export type GSIPK = `${GSI}PK`
+export type GSISK = `${GSI}SK`

package/src/operations.ts

@@ -6,6 +6,7 @@ import {
   Decodable,
 } from "./dynamodb-model"
 import { Key } from "./client"
+import { GSIPK, GSISK } from "./gsi"
 
 export type Operation<
   T extends DynamoDBModelInstance,
@@ -57,16 +58,9 @@ export interface UpdateRawOperation<M extends DynamoDBModelConstructor<any>>
   _operation: "updateRaw"
   _model: M
   key: Key
-  attributes: Partial<TypeOf<M>> & {
-    GSI2PK?: string | null
-    GSI2SK?: string | null
-    GSI3PK?: string | null
-    GSI3SK?: string | null
-    GSI4PK?: string | null
-    GSI4SK?: string | null
-    GSI5PK?: string | null
-    GSI5SK?: string | null
-  }
+  attributes: Partial<TypeOf<M>> &
+    { [key in GSIPK]?: string | null } &
+    { [key in GSISK]?: string | null }
 }
 
 // -------------------------------------------------------------------------------------

package/src/pagination.ts

@@ -1,5 +1,6 @@
 import crypto from "crypto"
 import { PaginationError } from "./errors"
+import { GSI, GSI_NAMES, GSIPK, GSISK } from "./gsi"
 
 const SIV = "Q05yyCR+0tyWl6glrZhlNw=="
 const ENCRYPTION_ALG = "aes-256-ctr"
@@ -33,13 +34,27 @@ export interface PaginationInput {
   after?: string | null
 }
 
-// TODO: make configurable
+export interface PaginationOptions {
+  /**
+   * Maximum number of items to return.
+   */
+  limit?: number
+
+  /**
+   * Default number of items to return if no limit is provided.
+   */
+  default?: number
+}
+
 const DEFAULT_OPTIONS = {
   limit: 50,
   default: 20,
 }
 
-export function decodePagination(pagination: PaginationInput): {
+export function decodePagination(
+  pagination: PaginationInput,
+  paginationOptions: PaginationOptions = DEFAULT_OPTIONS
+): {
   cursor?: string
   limit: number
   direction: PaginationDirection
@@ -65,8 +80,8 @@ export function decodePagination(pagination: PaginationInput): {
   return {
     cursor: before ?? after ?? undefined,
     limit: Math.min(
-      first ?? last ?? DEFAULT_OPTIONS.default,
-      DEFAULT_OPTIONS.limit
+      first ?? last ?? paginationOptions.default ?? DEFAULT_OPTIONS.default,
+      paginationOptions.limit ?? DEFAULT_OPTIONS.limit
     ),
     direction:
       before || last
@@ -120,40 +135,22 @@ export const encodeDDBCursor = (
   {
     PK,
     SK,
-    GSI2PK,
-    GSI2SK,
-    GSI3PK,
-    GSI3SK,
-    GSI4PK,
-    GSI4SK,
-    GSI5PK,
-    GSI5SK,
+    ...values
   }: {
     PK: string
     SK: string
-    GSI2PK?: string
-    GSI2SK?: string
-    GSI3PK?: string
-    GSI3SK?: string
-    GSI4PK?: string
-    GSI4SK?: string
-    GSI5PK?: string
-    GSI5SK?: string
-  },
+  } & { [key in GSIPK]?: string } &
+    { [key in GSISK]?: string },
   encryptionKey?: Buffer
 ) => {
   const cursor = Buffer.from(
     JSON.stringify({
       PK,
       SK,
-      GSI2PK,
-      GSI2SK,
-      GSI3PK,
-      GSI3SK,
-      GSI4PK,
-      GSI4SK,
-      GSI5PK,
-      GSI5SK,
+      ...GSI_NAMES.map((GSI) => ({
+        [`${GSI}PK`]: values[`${GSI}PK` as const],
+        [`${GSI}SK`]: values[`${GSI}SK` as const],
+      })).reduce((acc, cur) => Object.assign(acc, cur), {}),
     })
   ).toString("base64")
 
@@ -164,7 +161,7 @@ export const encodeDDBCursor = (
 
 export const decodeDDBCursor = (
   encoded: string,
-  index?: "GSI2" | "GSI3" | "GSI4" | "GSI5",
+  index?: GSI,
   encryptionKey?: Buffer
 ) => {
   try {
@@ -173,26 +170,20 @@ export const decodeDDBCursor = (
 
     if (!json) throw new Error("Couldn't decrypt cursor")
 
-    const {
-      PK,
-      SK,
-      GSI2PK,
-      GSI2SK,
-      GSI3PK,
-      GSI3SK,
-      GSI4PK,
-      GSI4SK,
-      GSI5PK,
-      GSI5SK,
-    } = JSON.parse(Buffer.from(json, "base64").toString())
+    const { PK, SK, ...values } = JSON.parse(
+      Buffer.from(json, "base64").toString()
+    )
 
     if (typeof PK !== "string" || typeof SK !== "string") throw new Error()
 
     if (!index) return { PK, SK }
-    if (index === "GSI2") return { PK, SK, GSI2PK, GSI2SK }
-    if (index === "GSI3") return { PK, SK, GSI3PK, GSI3SK }
-    if (index === "GSI4") return { PK, SK, GSI4PK, GSI4SK }
-    if (index === "GSI5") return { PK, SK, GSI5PK, GSI5SK }
+
+    return {
+      PK,
+      SK,
+      [`${index}PK`]: values[`${index}PK`],
+      [`${index}SK`]: values[`${index}SK`],
+    }
   } catch (error) {
     throw new PaginationError("Couldn't decode cursor")
   }

package/src/provider.ts

@@ -18,6 +18,7 @@ import { OutputOf, TypeOf, ModelOf } from "@model-ts/core"
 import { RaceConditionError } from "./errors"
 import { absurd } from "fp-ts/lib/function"
 import { encodeDDBCursor, PaginationInput } from "./pagination"
+import { GSI_NAMES, GSIPK, GSISK } from "./gsi"
 
 export interface DynamoDBInternals<M extends Decodable> {
   __dynamoDBDecode(
@@ -512,18 +513,17 @@ export const getProvider = (client: Client) => {
     },
     instanceProps: {
       dynamodb: client,
-      keys<T extends DynamoDBModelInstance>(this: T) {
+      keys<T extends DynamoDBModelInstance>(
+        this: T
+      ): { PK: string; SK: string } & { [key in GSIPK]?: string } &
+        { [key in GSISK]?: string } {
         return {
           PK: this.PK,
           SK: this.SK,
-          GSI2PK: this.GSI2PK,
-          GSI2SK: this.GSI2SK,
-          GSI3PK: this.GSI3PK,
-          GSI3SK: this.GSI3SK,
-          GSI4PK: this.GSI4PK,
-          GSI4SK: this.GSI4SK,
-          GSI5PK: this.GSI5PK,
-          GSI5SK: this.GSI5SK,
+          ...GSI_NAMES.map((GSI) => ({
+            [`${GSI}PK`]: this[`${GSI}PK`],
+            [`${GSI}SK`]: this[`${GSI}SK`],
+          })).reduce((acc, cur) => Object.assign(acc, cur), {}),
         }
       },
       cursor<T extends DynamoDBModelInstance>(this: T) {

package/src/sandbox.ts

@@ -3,6 +3,7 @@ import { chunksOf } from "fp-ts/lib/Array"
 import DynamoDB from "aws-sdk/clients/dynamodb"
 import diff from "snapshot-diff"
 import { Client } from "./client"
+import { GSI_NAMES } from "./gsi"
 
 const ddb = new DynamoDB({
   accessKeyId: "xxx",
@@ -27,14 +28,10 @@ export const createTable = async () => {
       AttributeDefinitions: [
         { AttributeName: "PK", AttributeType: "S" },
         { AttributeName: "SK", AttributeType: "S" },
-        { AttributeName: "GSI2PK", AttributeType: "S" },
-        { AttributeName: "GSI2SK", AttributeType: "S" },
-        { AttributeName: "GSI3PK", AttributeType: "S" },
-        { AttributeName: "GSI3SK", AttributeType: "S" },
-        { AttributeName: "GSI4PK", AttributeType: "S" },
-        { AttributeName: "GSI4SK", AttributeType: "S" },
-        { AttributeName: "GSI5PK", AttributeType: "S" },
-        { AttributeName: "GSI5SK", AttributeType: "S" },
+        ...GSI_NAMES.flatMap((GSI) => [
+          { AttributeName: `${GSI}PK`, AttributeType: "S" },
+          { AttributeName: `${GSI}SK`, AttributeType: "S" },
+        ]),
       ],
       KeySchema: [
         { AttributeName: "PK", KeyType: "HASH" },
@@ -51,46 +48,16 @@ export const createTable = async () => {
             ProjectionType: "ALL",
           },
         },
-        {
-          IndexName: "GSI2",
-          KeySchema: [
-            { AttributeName: "GSI2PK", KeyType: "HASH" },
-            { AttributeName: "GSI2SK", KeyType: "RANGE" },
-          ],
-          Projection: {
-            ProjectionType: "ALL",
-          },
-        },
-        {
-          IndexName: "GSI3",
+        ...GSI_NAMES.map((GSI) => ({
+          IndexName: GSI,
           KeySchema: [
-            { AttributeName: "GSI3PK", KeyType: "HASH" },
-            { AttributeName: "GSI3SK", KeyType: "RANGE" },
+            { AttributeName: `${GSI}PK`, KeyType: "HASH" },
+            { AttributeName: `${GSI}SK`, KeyType: "RANGE" },
           ],
           Projection: {
             ProjectionType: "ALL",
           },
-        },
-        {
-          IndexName: "GSI4",
-          KeySchema: [
-            { AttributeName: "GSI4PK", KeyType: "HASH" },
-            { AttributeName: "GSI4SK", KeyType: "RANGE" },
-          ],
-          Projection: {
-            ProjectionType: "ALL",
-          },
-        },
-        {
-          IndexName: "GSI5",
-          KeySchema: [
-            { AttributeName: "GSI5PK", KeyType: "HASH" },
-            { AttributeName: "GSI5SK", KeyType: "RANGE" },
-          ],
-          Projection: {
-            ProjectionType: "ALL",
-          },
-        },
+        })),
       ],
       BillingMode: "PAY_PER_REQUEST",
     })

package/tsconfig.esm.json

@@ -5,9 +5,19 @@
     "module": "ES2015",
     "outDir": "./dist/esm",
     "paths": {
-      "@model-ts/core": ["../core"]
+      "@model-ts/core": [
+        "../core"
+      ]
     }
   },
-  "references": [{ "path": "../core/tsconfig.esm.json" }],
-  "exclude": ["__test__", "test-utils"]
-}
+  "references": [
+    {
+      "path": "../core/tsconfig.esm.json"
+    }
+  ],
+  "exclude": [
+    "dist",
+    "__test__",
+    "test-utils"
+  ]
+}

package/tsconfig.json

@@ -4,9 +4,19 @@
     "rootDir": "./src",
     "outDir": "./dist/cjs",
     "paths": {
-      "@model-ts/core": ["../core"]
+      "@model-ts/core": [
+        "../core"
+      ]
     }
   },
-  "references": [{ "path": "../core" }],
-  "exclude": ["__test__", "test-utils"]
-}
+  "references": [
+    {
+      "path": "../core"
+    }
+  ],
+  "exclude": [
+    "dist",
+    "__test__",
+    "test-utils"
+  ]
+}