@xube/kit-aws 0.2.14 → 0.2.16

package/dist/resources/kinesis/put.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"put.d.ts","sourceRoot":"","sources":["../../../src/resources/kinesis/put.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,aAAa,EAEb,sBAAsB,EACvB,MAAM,yBAAyB,CAAC;AAEjC,eAAO,MAAM,UAAU,eACT,MAAM,WACT,sBAAsB,EAAE,WACxB,aAAa,kBAQvB,CAAC"}
+{"version":3,"file":"put.d.ts","sourceRoot":"","sources":["../../../src/resources/kinesis/put.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,aAAa,EAEb,sBAAsB,EACvB,MAAM,yBAAyB,CAAC;AAQjC,eAAO,MAAM,UAAU,eACT,MAAM,WACT,sBAAsB,EAAE,WACxB,aAAa,kBA0BvB,CAAC"}

package/dist/resources/kinesis/put.js

@@ -2,11 +2,28 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.putRecords = void 0;
 const client_kinesis_1 = require("@aws-sdk/client-kinesis");
+const MAX_RECORDS_PER_CALL = 500;
+const MAX_BATCH_PAYLOAD_BYTES = 4_500_000;
+const recordSize = (r) => (r.Data?.byteLength ?? 0) + (r.PartitionKey?.length ?? 0);
 const putRecords = async (streamName, records, kinesis) => {
-    const command = new client_kinesis_1.PutRecordsCommand({
-        StreamName: streamName,
-        Records: records,
-    });
-    await kinesis.send(command);
+    let batch = [];
+    let batchBytes = 0;
+    const flush = async () => {
+        if (batch.length === 0)
+            return;
+        await kinesis.send(new client_kinesis_1.PutRecordsCommand({ StreamName: streamName, Records: batch }));
+        batch = [];
+        batchBytes = 0;
+    };
+    for (const record of records) {
+        const size = recordSize(record);
+        if (batch.length >= MAX_RECORDS_PER_CALL ||
+            (batch.length > 0 && batchBytes + size > MAX_BATCH_PAYLOAD_BYTES)) {
+            await flush();
+        }
+        batch.push(record);
+        batchBytes += size;
+    }
+    await flush();
 };
 exports.putRecords = putRecords;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xube/kit-aws",
-  "version": "0.2.14",
+  "version": "0.2.16",
   "description": "",
   "main": "dist/index.js",
   "scripts": {
@@ -18,7 +18,7 @@
   "homepage": "https://github.com/XubeLtd/dev-kit#readme",
   "devDependencies": {
     "@types/aws-lambda": "^8.10.119",
-    "@xube/kit-build": "^0.2.14"
+    "@xube/kit-build": "^0.2.16"
   },
   "dependencies": {
     "@aws-sdk/client-athena": "^3.656.0",
@@ -29,10 +29,10 @@
     "@aws-sdk/client-ssm": "^3.654.0",
     "@aws-sdk/lib-dynamodb": "^3.656.0",
     "@aws-sdk/util-dynamodb": "^3.656.0",
-    "@xube/kit-aws-schema": "^0.2.14",
-    "@xube/kit-log": "^0.2.14",
-    "@xube/kit-request": "^0.2.14",
-    "@xube/kit-schema": "^0.2.14",
+    "@xube/kit-aws-schema": "^0.2.16",
+    "@xube/kit-log": "^0.2.16",
+    "@xube/kit-request": "^0.2.16",
+    "@xube/kit-schema": "^0.2.16",
     "zod": "^4.4.3"
   }
 }

package/src/resources/kinesis/put.test.ts

@@ -0,0 +1,75 @@
+import {
+  KinesisClient,
+  PutRecordsCommand,
+  PutRecordsRequestEntry,
+} from "@aws-sdk/client-kinesis";
+import { putRecords } from "./put";
+
+const makeRecord = (size: number, key = "p"): PutRecordsRequestEntry => ({
+  Data: Buffer.alloc(size),
+  PartitionKey: key,
+});
+
+const captureCalls = () => {
+  const calls: PutRecordsRequestEntry[][] = [];
+  const send = jest.fn(async (cmd: PutRecordsCommand) => {
+    calls.push(cmd.input.Records ?? []);
+    return {};
+  });
+  const client = { send } as unknown as KinesisClient;
+  return { calls, send, client };
+};
+
+describe("putRecords (batched)", () => {
+  it("sends a single call when records fit in one batch", async () => {
+    const { calls, client } = captureCalls();
+    const records = Array.from({ length: 100 }, () => makeRecord(100));
+    await putRecords("stream", records, client);
+    expect(calls).toHaveLength(1);
+    expect(calls[0]).toHaveLength(100);
+  });
+
+  it("splits when record count exceeds 500", async () => {
+    const { calls, client } = captureCalls();
+    const records = Array.from({ length: 1200 }, () => makeRecord(100));
+    await putRecords("stream", records, client);
+    expect(calls.map((c) => c.length)).toEqual([500, 500, 200]);
+  });
+
+  it("splits exactly at the 500 boundary", async () => {
+    const { calls, client } = captureCalls();
+    const records = Array.from({ length: 500 }, () => makeRecord(100));
+    await putRecords("stream", records, client);
+    expect(calls).toHaveLength(1);
+    expect(calls[0]).toHaveLength(500);
+  });
+
+  it("splits when cumulative payload approaches 4.5 MB", async () => {
+    const { calls, client } = captureCalls();
+    const records = Array.from({ length: 100 }, () => makeRecord(100_000));
+    await putRecords("stream", records, client);
+    expect(calls.length).toBeGreaterThan(1);
+    for (const c of calls) {
+      const bytes = c.reduce(
+        (sum, r) => sum + (r.Data?.byteLength ?? 0) + (r.PartitionKey?.length ?? 0),
+        0,
+      );
+      expect(bytes).toBeLessThanOrEqual(4_500_000);
+    }
+  });
+
+  it("does nothing when records array is empty", async () => {
+    const { send, client } = captureCalls();
+    await putRecords("stream", [], client);
+    expect(send).not.toHaveBeenCalled();
+  });
+
+  it("does not orphan oversized first record into an empty flush", async () => {
+    const { calls, client } = captureCalls();
+    const records = [makeRecord(4_400_000), makeRecord(100), makeRecord(100)];
+    await putRecords("stream", records, client);
+    expect(calls.length).toBeGreaterThan(0);
+    const allRecords = calls.flat();
+    expect(allRecords).toHaveLength(3);
+  });
+});

package/src/resources/kinesis/put.ts

@@ -4,15 +4,39 @@ import {
   PutRecordsRequestEntry,
 } from "@aws-sdk/client-kinesis";
 
+const MAX_RECORDS_PER_CALL = 500;
+const MAX_BATCH_PAYLOAD_BYTES = 4_500_000;
+
+const recordSize = (r: PutRecordsRequestEntry): number =>
+  (r.Data?.byteLength ?? 0) + (r.PartitionKey?.length ?? 0);
+
 export const putRecords = async (
   streamName: string,
   records: PutRecordsRequestEntry[],
   kinesis: KinesisClient
 ) => {
-  const command = new PutRecordsCommand({
-    StreamName: streamName,
-    Records: records,
-  });
+  let batch: PutRecordsRequestEntry[] = [];
+  let batchBytes = 0;
+
+  const flush = async () => {
+    if (batch.length === 0) return;
+    await kinesis.send(
+      new PutRecordsCommand({ StreamName: streamName, Records: batch })
+    );
+    batch = [];
+    batchBytes = 0;
+  };
 
-  await kinesis.send(command);
+  for (const record of records) {
+    const size = recordSize(record);
+    if (
+      batch.length >= MAX_RECORDS_PER_CALL ||
+      (batch.length > 0 && batchBytes + size > MAX_BATCH_PAYLOAD_BYTES)
+    ) {
+      await flush();
+    }
+    batch.push(record);
+    batchBytes += size;
+  }
+  await flush();
 };