@superblocksteam/sdk-api 0.0.12-body-types → 0.0.13

package/src/integrations/dynamodb/README.md

@@ -12,6 +12,7 @@ Execute operations against Amazon DynamoDB with full type safety and runtime val
 | `updateItem(table, key, expr, values, names?, metadata?)`       | Update an existing item      |
 | `deleteItem(table, key, metadata?)`                             | Delete an item by key        |
 | `scan(table, schema, filter?, values?, names?, metadata?)`      | Scan a table                 |
+| `scan(table, schema, options, metadata?)`                       | Scan with pagination options |
 | `queryTable(table, keyExpr, values, schema, names?, metadata?)` | Query by key condition       |
 | `batchWriteItem(requestItems, metadata?)`                       | Batch write multiple items   |
 | `listTables(schema, metadata?)`                                 | List all tables              |
@@ -133,21 +134,41 @@ await ctx.integrations.dynamodb.deleteItem("users", { id: { S: "user-123" } });
 ### Scan Table
 
 ```typescript
-// Full scan
-const allUsers = await ctx.integrations.dynamodb.scan(
-  "users",
-  z.array(z.object({ id: z.string(), name: z.string() })),
-);
+// DynamoDB Scan returns raw AttributeValue maps.
+// import type { DynamoDBAttributeValue } from "@superblocksteam/sdk-api";
+const AttributeValueSchema = z.custom<DynamoDBAttributeValue>();
+const ItemSchema = z.record(AttributeValueSchema);
+const ScanPageSchema = z.object({
+  Items: z.array(ItemSchema).optional(),
+  LastEvaluatedKey: z.record(AttributeValueSchema).optional(),
+});
 
-// With filter expression
-const activeUsers = await ctx.integrations.dynamodb.scan(
-  "users",
-  z.array(z.object({ id: z.string(), name: z.string() })),
-  "status = :s",
-  { ":s": { S: "active" } },
-);
+// Paginate past DynamoDB's 1 MB per-Scan limit. Bound the number of
+// requests so throttling or an unexpectedly large table cannot consume
+// the entire API execution timeout.
+const MAX_PAGES = 100;
+let exclusiveStartKey: Record<string, DynamoDBAttributeValue> | undefined;
+const allItems: Array<Record<string, DynamoDBAttributeValue>> = [];
+for (let pageNumber = 0; pageNumber < MAX_PAGES; pageNumber += 1) {
+  const page = await ctx.integrations.dynamodb.scan("users", ScanPageSchema, {
+    exclusiveStartKey,
+    filterExpression: "status = :s",
+    expressionAttributeValues: { ":s": { S: "active" } },
+  });
+  allItems.push(...(page.Items ?? []));
+  exclusiveStartKey = page.LastEvaluatedKey;
+  if (!exclusiveStartKey) {
+    break;
+  }
+  if (pageNumber === MAX_PAGES - 1) {
+    throw new Error(`Scan exceeded the ${MAX_PAGES}-page safety limit`);
+  }
+}
 ```
 
+Choose a page cap appropriate for the API timeout and provisioned read
+capacity. For very large tables, process and persist pages incrementally.
+
 ### Query Table by Key Condition
 
 ```typescript

package/src/integrations/dynamodb/client.test.ts

@@ -0,0 +1,254 @@
+/**
+ * Unit tests for DynamoDB client request bodies.
+ *
+ * The orchestrator forwards `action` + JSON `body` to the AWS SDK, so these
+ * tests lock the wire params - especially ExclusiveStartKey pagination.
+ */
+
+import { describe, it, expect, vi } from "vitest";
+import { z } from "zod";
+
+import type { IntegrationConfig } from "../types.js";
+import { DynamoDBClientImpl } from "./client.js";
+
+const TEST_CONFIG: IntegrationConfig = {
+  id: "dynamodb-test-id",
+  name: "Test DynamoDB",
+  pluginId: "dynamodb",
+  configuration: {},
+};
+
+const ItemsSchema = z.object({ Items: z.array(z.unknown()).optional() });
+
+function createClient(mockResult: unknown = { Items: [] }) {
+  const executeQuery = vi.fn().mockResolvedValue(mockResult);
+  const client = new DynamoDBClientImpl(TEST_CONFIG, executeQuery);
+  return { client, executeQuery };
+}
+
+function parsedBody(executeQuery: ReturnType<typeof vi.fn>, callIndex = 0) {
+  const request = z
+    .object({ action: z.string(), body: z.string() })
+    .parse(executeQuery.mock.calls[callIndex]?.[0]);
+  return { action: request.action, body: JSON.parse(request.body) };
+}
+
+describe("DynamoDBClientImpl", () => {
+  describe("scan", () => {
+    it("omits ExclusiveStartKey on a positional scan", async () => {
+      const { client, executeQuery } = createClient();
+
+      await client.scan("retailers", ItemsSchema, "status = :s", {
+        ":s": { S: "active" },
+      });
+
+      expect(parsedBody(executeQuery)).toEqual({
+        action: "scan",
+        body: {
+          TableName: "retailers",
+          FilterExpression: "status = :s",
+          ExpressionAttributeValues: { ":s": { S: "active" } },
+        },
+      });
+    });
+
+    it("forwards ExclusiveStartKey from the options object", async () => {
+      const { client, executeQuery } = createClient();
+      const exclusiveStartKey = { id: { S: "retailer-18" } };
+
+      await client.scan("retailers", ItemsSchema, { exclusiveStartKey });
+
+      expect(parsedBody(executeQuery)).toEqual({
+        action: "scan",
+        body: {
+          TableName: "retailers",
+          ExclusiveStartKey: exclusiveStartKey,
+        },
+      });
+    });
+
+    it("treats an empty object as empty scan options", async () => {
+      const { client, executeQuery } = createClient();
+
+      await client.scan("retailers", ItemsSchema, {});
+
+      expect(parsedBody(executeQuery)).toEqual({
+        action: "scan",
+        body: { TableName: "retailers" },
+      });
+    });
+
+    it("rejects objects with unknown scan option keys", async () => {
+      const { client, executeQuery } = createClient();
+
+      await expect(
+        Reflect.apply(client.scan, client, [
+          "retailers",
+          ItemsSchema,
+          { label: "not scan options" },
+        ]),
+      ).rejects.toThrow("Invalid DynamoDB scan options: label");
+      expect(executeQuery).not.toHaveBeenCalled();
+    });
+
+    it("forwards pagination and parallel-scan options", async () => {
+      const { client, executeQuery } = createClient();
+
+      await client.scan(
+        "retailers",
+        ItemsSchema,
+        {
+          exclusiveStartKey: { id: { S: "page-2" } },
+          limit: 25,
+          projectionExpression: "id, name",
+          indexName: "gsi1",
+          segment: 0,
+          totalSegments: 4,
+          filterExpression: "#s = :s",
+          expressionAttributeNames: { "#s": "status" },
+          expressionAttributeValues: { ":s": { S: "active" } },
+        },
+        { label: "page retailers" },
+      );
+
+      expect(parsedBody(executeQuery)).toEqual({
+        action: "scan",
+        body: {
+          TableName: "retailers",
+          ExclusiveStartKey: { id: { S: "page-2" } },
+          Limit: 25,
+          ProjectionExpression: "id, name",
+          IndexName: "gsi1",
+          Segment: 0,
+          TotalSegments: 4,
+          FilterExpression: "#s = :s",
+          ExpressionAttributeNames: { "#s": "status" },
+          ExpressionAttributeValues: { ":s": { S: "active" } },
+        },
+      });
+      expect(executeQuery).toHaveBeenCalledWith(expect.anything(), undefined, {
+        label: "page retailers",
+      });
+    });
+
+    it("keeps positional names and metadata working together", async () => {
+      const { client, executeQuery } = createClient();
+
+      await client.scan(
+        "retailers",
+        ItemsSchema,
+        "#s = :s",
+        { ":s": { S: "active" } },
+        { "#s": "status" },
+        { label: "filtered scan" },
+      );
+
+      expect(parsedBody(executeQuery).body).toEqual({
+        TableName: "retailers",
+        FilterExpression: "#s = :s",
+        ExpressionAttributeValues: { ":s": { S: "active" } },
+        ExpressionAttributeNames: { "#s": "status" },
+      });
+      expect(executeQuery.mock.calls[0][2]).toEqual({ label: "filtered scan" });
+    });
+
+    it("keeps schemas that strip pagination cursors backward compatible", async () => {
+      const lastEvaluatedKey = {
+        id: { S: "retailer-18" },
+        version: { N: "2" },
+      };
+      const { client } = createClient({
+        Items: [{ id: { S: "retailer-1" } }],
+        LastEvaluatedKey: lastEvaluatedKey,
+      });
+
+      await expect(
+        client.scan("retailers", ItemsSchema, { limit: 25 }),
+      ).resolves.toEqual({
+        Items: [{ id: { S: "retailer-1" } }],
+      });
+    });
+
+    it("preserves mixed AttributeValue pagination cursors", async () => {
+      const lastEvaluatedKey = {
+        id: { S: "retailer-18" },
+        version: { N: "2" },
+      };
+      const PageSchema = z.object({
+        Items: z.array(z.unknown()).optional(),
+        LastEvaluatedKey: z.record(z.unknown()).optional(),
+      });
+      const { client } = createClient({
+        Items: [],
+        LastEvaluatedKey: lastEvaluatedKey,
+      });
+
+      const page = await client.scan("retailers", PageSchema, {});
+
+      expect(page.LastEvaluatedKey).toEqual(lastEvaluatedKey);
+    });
+
+    it("threads each returned pagination cursor into the next request", async () => {
+      const LastEvaluatedKeySchema = z.object({
+        id: z.object({ S: z.string() }),
+      });
+      const PageSchema = z.object({
+        Items: z.array(z.string()),
+        LastEvaluatedKey: LastEvaluatedKeySchema.optional(),
+      });
+      const lastEvaluatedKey = { id: { S: "retailer-1" } };
+      const executeQuery = vi
+        .fn()
+        .mockResolvedValueOnce({
+          Items: ["retailer-1"],
+          LastEvaluatedKey: lastEvaluatedKey,
+        })
+        .mockResolvedValueOnce({ Items: ["retailer-2"] });
+      const client = new DynamoDBClientImpl(TEST_CONFIG, executeQuery);
+
+      const items: string[] = [];
+      let exclusiveStartKey: z.infer<typeof LastEvaluatedKeySchema> | undefined;
+      do {
+        const page = await client.scan("retailers", PageSchema, {
+          exclusiveStartKey,
+        });
+        items.push(...page.Items);
+        exclusiveStartKey = page.LastEvaluatedKey;
+      } while (exclusiveStartKey);
+
+      expect(items).toEqual(["retailer-1", "retailer-2"]);
+      expect(executeQuery).toHaveBeenCalledTimes(2);
+      expect(parsedBody(executeQuery, 0).body).toEqual({
+        TableName: "retailers",
+      });
+      expect(parsedBody(executeQuery, 1).body).toEqual({
+        TableName: "retailers",
+        ExclusiveStartKey: lastEvaluatedKey,
+      });
+    });
+  });
+
+  describe("queryTable", () => {
+    it("treats a plain names map as ExpressionAttributeNames", async () => {
+      const { client, executeQuery } = createClient();
+
+      await client.queryTable(
+        "orders",
+        "#uid = :uid",
+        { ":uid": { S: "user-123" } },
+        ItemsSchema,
+        { "#uid": "userId" },
+      );
+
+      expect(parsedBody(executeQuery)).toEqual({
+        action: "query",
+        body: {
+          TableName: "orders",
+          KeyConditionExpression: "#uid = :uid",
+          ExpressionAttributeValues: { ":uid": { S: "user-123" } },
+          ExpressionAttributeNames: { "#uid": "userId" },
+        },
+      });
+    });
+  });
+});

package/src/integrations/dynamodb/client.ts

@@ -8,11 +8,30 @@
 
 import type { z } from "zod";
 
-import { RestApiValidationError } from "../../errors.js";
+import {
+  RestApiValidationError,
+  restApiValidationErrorFromZodError,
+} from "../../errors.js";
 import { IntegrationError } from "../../runtime/errors.js";
 import type { QueryExecutor, TraceMetadata } from "../registry.js";
 import type { IntegrationConfig, IntegrationClientImpl } from "../types.js";
-import type { DynamoDBClient, DynamoDBAttributeValue } from "./types.js";
+import type {
+  DynamoDBClient,
+  DynamoDBAttributeValue,
+  DynamoDBScanOptions,
+} from "./types.js";
+
+const DYNAMODB_SCAN_OPTION_KEYS = new Set<string>([
+  "filterExpression",
+  "expressionAttributeValues",
+  "expressionAttributeNames",
+  "exclusiveStartKey",
+  "limit",
+  "projectionExpression",
+  "indexName",
+  "segment",
+  "totalSegments",
+]);
 
 /**
  * Internal implementation of DynamoDBClient.
@@ -69,12 +88,10 @@ export class DynamoDBClientImpl
     const parseResult = schema.safeParse(result);
 
     if (!parseResult.success) {
-      throw new RestApiValidationError(
-        `Result validation failed: ${parseResult.error.message}`,
-        {
-          zodError: parseResult.error,
-          data: result,
-        },
+      throw restApiValidationErrorFromZodError(
+        "Result validation failed",
+        parseResult.error,
+        result,
       );
     }
 
@@ -189,33 +206,120 @@ export class DynamoDBClientImpl
     return this.executeWithErrorHandling(request, "deleteItem", metadata);
   }
 
+  /** Copy optional AWS Scan fields onto the request body. */
+  private applyScanParams(
+    params: Record<string, unknown>,
+    options: DynamoDBScanOptions,
+  ): void {
+    if (options.filterExpression) {
+      params.FilterExpression = options.filterExpression;
+    }
+    if (
+      options.expressionAttributeValues &&
+      Object.keys(options.expressionAttributeValues).length > 0
+    ) {
+      params.ExpressionAttributeValues = options.expressionAttributeValues;
+    }
+    if (
+      options.expressionAttributeNames &&
+      Object.keys(options.expressionAttributeNames).length > 0
+    ) {
+      params.ExpressionAttributeNames = options.expressionAttributeNames;
+    }
+    if (options.exclusiveStartKey) {
+      params.ExclusiveStartKey = options.exclusiveStartKey;
+    }
+    if (options.limit !== undefined) {
+      params.Limit = options.limit;
+    }
+    if (options.projectionExpression) {
+      params.ProjectionExpression = options.projectionExpression;
+    }
+    if (options.indexName) {
+      params.IndexName = options.indexName;
+    }
+    if (options.segment !== undefined) {
+      params.Segment = options.segment;
+    }
+    if (options.totalSegments !== undefined) {
+      params.TotalSegments = options.totalSegments;
+    }
+  }
+
+  private isScanOptions(value: unknown): value is DynamoDBScanOptions {
+    return (
+      typeof value === "object" &&
+      value !== null &&
+      Object.keys(value).every((key) => DYNAMODB_SCAN_OPTION_KEYS.has(key))
+    );
+  }
+
+  private isTraceMetadata(value: unknown): value is TraceMetadata {
+    return (
+      typeof value === "object" &&
+      value !== null &&
+      Object.keys(value).every(
+        (key) => key === "label" || key === "description",
+      ) &&
+      Object.values(value).every(
+        (entry) => entry === undefined || typeof entry === "string",
+      )
+    );
+  }
+
   async scan<T>(
     table: string,
     schema: z.ZodSchema<T>,
-    filterExpression?: string,
-    expressionAttributeValues?: Record<string, DynamoDBAttributeValue>,
+    filterExpressionOrOptions?: string | DynamoDBScanOptions,
+    expressionAttributeValuesOrMetadata?:
+      | Record<string, DynamoDBAttributeValue>
+      | TraceMetadata,
     expressionAttributeNames?: Record<string, string>,
     metadata?: TraceMetadata,
   ): Promise<T> {
     const params: Record<string, unknown> = { TableName: table };
+    let resolvedMetadata = metadata;
+
+    if (
+      typeof filterExpressionOrOptions === "object" &&
+      filterExpressionOrOptions !== null
+    ) {
+      if (!this.isScanOptions(filterExpressionOrOptions)) {
+        throw new Error(
+          `Invalid DynamoDB scan options: ${Object.keys(filterExpressionOrOptions).join(", ")}`,
+        );
+      }
+      this.applyScanParams(params, filterExpressionOrOptions);
 
-    if (filterExpression) {
-      params.FilterExpression = filterExpression;
-    }
-
-    if (expressionAttributeValues) {
-      params.ExpressionAttributeValues = expressionAttributeValues;
-    }
-
-    if (expressionAttributeNames) {
-      params.ExpressionAttributeNames = expressionAttributeNames;
+      if (expressionAttributeValuesOrMetadata !== undefined) {
+        if (!this.isTraceMetadata(expressionAttributeValuesOrMetadata)) {
+          throw new Error("Invalid DynamoDB scan trace metadata");
+        }
+        resolvedMetadata = expressionAttributeValuesOrMetadata;
+      }
+    } else {
+      if (filterExpressionOrOptions) {
+        params.FilterExpression = filterExpressionOrOptions;
+      }
+      if (
+        expressionAttributeValuesOrMetadata &&
+        Object.keys(expressionAttributeValuesOrMetadata).length > 0
+      ) {
+        params.ExpressionAttributeValues = expressionAttributeValuesOrMetadata;
+      }
+      if (
+        expressionAttributeNames &&
+        Object.keys(expressionAttributeNames).length > 0
+      ) {
+        params.ExpressionAttributeNames = expressionAttributeNames;
+      }
     }
 
     const request = this.buildRequest("scan", params);
     const result = await this.executeWithErrorHandling(
       request,
       "scan",
-      metadata,
+      resolvedMetadata,
     );
     return this.validateResult(result, schema, "scan");
   }
@@ -233,8 +337,10 @@ export class DynamoDBClientImpl
       KeyConditionExpression: keyConditionExpression,
       ExpressionAttributeValues: expressionAttributeValues,
     };
-
-    if (expressionAttributeNames) {
+    if (
+      expressionAttributeNames &&
+      Object.keys(expressionAttributeNames).length > 0
+    ) {
       params.ExpressionAttributeNames = expressionAttributeNames;
     }
 

package/src/integrations/dynamodb/index.ts

@@ -4,5 +4,9 @@
  * @module
  */
 
-export type { DynamoDBClient, DynamoDBAttributeValue } from "./types.js";
+export type {
+  DynamoDBClient,
+  DynamoDBAttributeValue,
+  DynamoDBScanOptions,
+} from "./types.js";
 export { DynamoDBClientImpl } from "./client.js";

package/src/integrations/dynamodb/types.ts

@@ -37,6 +37,24 @@ export type DynamoDBAttributeValue =
   | { NS: string[] }
   | { BS: string[] };
 
+/**
+ * Optional Scan parameters forwarded to the AWS SDK.
+ *
+ * Use `exclusiveStartKey` with `LastEvaluatedKey` from a previous page to
+ * continue past DynamoDB's 1 MB per-Scan limit.
+ */
+export interface DynamoDBScanOptions {
+  filterExpression?: string;
+  expressionAttributeValues?: Record<string, DynamoDBAttributeValue>;
+  expressionAttributeNames?: Record<string, string>;
+  exclusiveStartKey?: Record<string, DynamoDBAttributeValue>;
+  limit?: number;
+  projectionExpression?: string;
+  indexName?: string;
+  segment?: number;
+  totalSegments?: number;
+}
+
 /**
  * DynamoDB client for database operations.
  *
@@ -84,6 +102,11 @@ export type DynamoDBAttributeValue =
  *   'status = :s',
  *   { ':s': { S: 'active' } }
  * );
+ *
+ * // Paginate past the 1 MB Scan limit
+ * const page = await ctx.integrations.db.scan('users', UsersSchema, {
+ *   exclusiveStartKey: lastEvaluatedKey,
+ * });
  * ```
  */
 export interface DynamoDBClient extends BaseIntegrationClient {
@@ -122,7 +145,10 @@ export interface DynamoDBClient extends BaseIntegrationClient {
   ): Promise<T>;
 
   /**
-   * Scan a DynamoDB table with optional filter.
+   * Scan a DynamoDB table.
+   *
+   * Prefer the options-object form when paginating (`exclusiveStartKey`) or
+   * when you need `limit`, `indexName`, or parallel scan segments.
    *
    * @param table - The table name
    * @param schema - Zod schema for validating the result
@@ -133,6 +159,12 @@ export interface DynamoDBClient extends BaseIntegrationClient {
    * @param metadata - Optional trace metadata for diagnostics
    * @returns The validated result
    */
+  scan<T>(
+    table: string,
+    schema: z.ZodSchema<T>,
+    options: DynamoDBScanOptions,
+    metadata?: TraceMetadata,
+  ): Promise<T>;
   scan<T>(
     table: string,
     schema: z.ZodSchema<T>,

package/src/integrations/elasticsearch/README.md

@@ -226,70 +226,6 @@ const result = await ctx.integrations.es.apiRequest(
 );
 ```
 
-### Bulk Operations
-
-```typescript
-const BulkResponseSchema = z.object({
-  took: z.number(),
-  errors: z.boolean(),
-  items: z.array(
-    z.object({
-      index: z
-        .object({
-          _index: z.string(),
-          _id: z.string(),
-          status: z.number(),
-          result: z.string().optional(),
-          error: z
-            .object({
-              type: z.string(),
-              reason: z.string(),
-            })
-            .optional(),
-        })
-        .optional(),
-      delete: z
-        .object({
-          _index: z.string(),
-          _id: z.string(),
-          status: z.number(),
-          result: z.string().optional(),
-        })
-        .optional(),
-    }),
-  ),
-});
-
-// Bulk index
-const result = await ctx.integrations.es.apiRequest(
-  {
-    method: "POST",
-    path: "/_bulk",
-    headers: {
-      "Content-Type": "application/x-ndjson",
-    },
-    body:
-      [
-        { index: { _index: "products", _id: "1" } },
-        { name: "Product 1", price: 10 },
-        { index: { _index: "products", _id: "2" } },
-        { name: "Product 2", price: 20 },
-      ]
-        .map(JSON.stringify)
-        .join("\n") + "\n",
-  },
-  { response: BulkResponseSchema },
-);
-
-if (result.errors) {
-  result.items.forEach((item) => {
-    if (item.index?.error) {
-      console.error(`Failed to index: ${item.index.error.reason}`);
-    }
-  });
-}
-```
-
 ### Aggregations
 
 ```typescript
@@ -493,22 +429,6 @@ const body = {
 };
 ```
 
-### Bulk Format
-
-Bulk operations require NDJSON format:
-
-```typescript
-// WRONG - Regular JSON array
-const body = [{ index: { _index: "products" } }, { name: "Product" }];
-
-// CORRECT - Newline-delimited JSON
-const body = '{"index":{"_index":"products"}}\n{"name":"Product"}\n';
-
-// Or build programmatically
-const operations = [{ index: { _index: "products" } }, { name: "Product" }];
-const body = operations.map(JSON.stringify).join("\n") + "\n";
-```
-
 ### Total Hits Accuracy
 
 For large result sets, total is approximate by default:

package/src/integrations/elasticsearch/docs.manifest.json

@@ -1,5 +1,14 @@
 {
   "pluginId": "elasticsearch",
   "base": "README.md",
-  "overlays": []
+  "overlays": [
+    {
+      "file": "overlays/bulk.md",
+      "sdkVersionRange": ">=0.0.7"
+    },
+    {
+      "file": "overlays/bulk-unsupported.md",
+      "sdkVersionRange": "<0.0.7"
+    }
+  ]
 }