@dshahi468/nd-dataset-split 1.0.0

package/README.md

@@ -0,0 +1,157 @@
+# nd-dataset-split
+
+大きなデータセットを安全に分割し、非同期処理をチャンク単位で実行するための軽量ユーティリティです。  
+「1リクエストあたり最大20件」などの制約があるAPI連携（例: Bedrock 呼び出し）を共通化できます。
+
+## 特徴
+
+- データを最大20件単位でチャンク化
+- `Promise.all` ベースの並列チャンク処理
+- 並列数（`maxConcurrency`）の制御
+- リトライ + バックオフ（指数バックオフ対応）
+- エラーごとの再試行可否を `shouldRetry` で制御
+- 結果のマージ処理を `mergeResults` で自由に定義
+- TypeScript 型定義同梱（`index.d.ts`）
+
+## インストール
+
+### npm レジストリから
+
+```bash
+npm install nd-dataset-split
+```
+
+## 使い方
+
+### 1. シンプルに使う（`processInChunks`）
+
+```ts
+import { processInChunks } from "nd-dataset-split";
+
+const data = Array.from({ length: 45 }, (_, i) => i + 1);
+
+const result = await processInChunks(
+  data,
+  async (chunk) => chunk.map((n) => n * 2),
+  { chunkSize: 20 }, // 1〜20
+);
+
+console.log(result.length); // 45
+```
+
+### 2. 実運用向け（`processDatasetInChunks`）
+
+```ts
+import { processDatasetInChunks } from "nd-dataset-split";
+
+const output = await processDatasetInChunks(
+  events,
+  async (chunk, chunkIndex, totalChunks) => {
+    // or you can use your logic here
+    const res = await fetch("your-api-name", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        events: chunk,
+      }),
+    });
+
+    const payload = await res.json().catch(() => ({}));
+    if (!res.ok) {
+      throw new Error(
+        `Chunk ${chunkIndex + 1}/${totalChunks}: ${payload?.error || "Failed"}`,
+      );
+    }
+    return payload; // チャンク単位の結果
+  },
+  {
+    chunkSize: 20,
+    maxConcurrency: 3,
+    retries: 2,
+    retryDelayMs: 600,
+    backoffMultiplier: 1.5,
+    shouldRetry: (error) => {
+      const message = String((error as Error)?.message || "");
+      return message.includes("throttl") || message.includes("timeout");
+    },
+    mergeResults: (chunkResults) => chunkResults, // 必要に応じて独自マージ
+  },
+);
+```
+
+## API
+
+## `chunkArray(data, chunkSize = 20)`
+
+配列をチャンクに分割します。
+
+- `data`: 対象配列
+- `chunkSize`: 1〜20 の整数
+- 戻り値: `T[][]`
+
+## `wait(ms)`
+
+指定ミリ秒待機するヘルパーです。
+
+- `ms`: 待機時間（ミリ秒）
+- 戻り値: `Promise<void>`
+
+## `processInChunks(data, processChunk, options?)`
+
+シンプルなチャンク処理用APIです。  
+内部で全チャンクを `Promise.all` で一括実行します。
+
+- `data`: 対象配列
+- `processChunk`: `(chunk, chunkIndex, totalChunks) => Promise<R[] | R> | (R[] | R)`
+- `options.chunkSize`:
+  - デフォルト `20`
+  - 1〜20 の整数
+- `options.flatten`:
+  - デフォルト `true`
+  - `true`: 各チャンク結果を1つの配列にフラット化
+  - `false`: チャンクごとの結果配列をそのまま返却
+
+## `processDatasetInChunks(data, processChunk, options?)`
+
+実運用向けの拡張APIです。  
+並列数制御、リトライ、バックオフ、カスタムマージに対応します。
+
+- `data`: 対象配列
+- `processChunk`: `(chunk, chunkIndex, totalChunks) => Promise<R> | R`
+- `options.chunkSize`:
+  - デフォルト `20`
+  - 1〜20 の整数
+- `options.maxConcurrency`:
+  - デフォルト `Infinity`
+  - 同時実行するチャンク数
+- `options.retries`:
+  - デフォルト `0`
+  - 失敗時の再試行回数
+- `options.retryDelayMs`:
+  - デフォルト `0`
+  - リトライ間隔の基準ミリ秒
+- `options.backoffMultiplier`:
+  - デフォルト `1`
+  - 実遅延: `retryDelayMs * backoffMultiplier^attempt`
+- `options.shouldRetry`:
+  - デフォルト `() => true`
+  - `false` を返すと即時失敗
+- `options.mergeResults`:
+  - 指定時は `chunkResults` を任意の最終形式へ変換して返却
+  - 未指定時は `R[]`（チャンク結果配列）を返却
+
+## エラーハンドリングの考え方
+
+- API制限（429 / throttling）や一時的なタイムアウトは `shouldRetry` で再試行対象にする
+- 認可エラーや入力不正など恒久エラーは `shouldRetry` で `false` を返して即失敗
+- エラーメッセージに `chunkIndex + 1` と `totalChunks` を入れると追跡しやすい
+
+## 注意点
+
+- `chunkSize` は 20 を超えられません（仕様）
+- `maxConcurrency` を上げすぎると、相手API側のレート制限にかかりやすくなります
+- `mergeResults` で重複排除・集計・並び替えなどを実装すると、呼び出し側コードを薄くできます
+
+## ライセンス
+
+ISC

package/index.d.ts

@@ -0,0 +1,42 @@
+export type ProcessChunk<T, R> = (
+  chunk: T[],
+  chunkIndex: number,
+  totalChunks: number
+) => Promise<R> | R;
+
+export type ShouldRetry = (
+  error: unknown,
+  attempt: number,
+  chunkIndex: number,
+  totalChunks: number
+) => boolean;
+
+export type ProcessInChunksOptions = {
+  chunkSize?: number;
+  flatten?: boolean;
+};
+
+export type ProcessDatasetInChunksOptions<R, M> = {
+  chunkSize?: number;
+  maxConcurrency?: number;
+  retries?: number;
+  retryDelayMs?: number;
+  backoffMultiplier?: number;
+  shouldRetry?: ShouldRetry;
+  mergeResults?: (chunkResults: R[]) => M;
+};
+
+export function chunkArray<T>(data: T[], chunkSize?: number): T[][];
+export function wait(ms: number): Promise<void>;
+
+export function processInChunks<T, R>(
+  data: T[],
+  processChunk: ProcessChunk<T, R[] | R>,
+  options?: ProcessInChunksOptions
+): Promise<R[] | (R[] | R)[]>;
+
+export function processDatasetInChunks<T, R, M = R[]>(
+  data: T[],
+  processChunk: ProcessChunk<T, R>,
+  options?: ProcessDatasetInChunksOptions<R, M>
+): Promise<M | R[]>;

package/index.js

@@ -0,0 +1,190 @@
+/**
+ * Split an array into chunks.
+ * @template T
+ * @param {T[]} data
+ * @param {number} [chunkSize=20]
+ * @returns {T[][]}
+ */
+export function chunkArray(data, chunkSize = 20) {
+  if (!Array.isArray(data)) {
+    throw new TypeError("data must be an array");
+  }
+
+  if (!Number.isInteger(chunkSize) || chunkSize <= 0 || chunkSize > 20) {
+    throw new RangeError("chunkSize must be an integer between 1 and 20");
+  }
+
+  const chunks = [];
+  for (let i = 0; i < data.length; i += chunkSize) {
+    chunks.push(data.slice(i, i + chunkSize));
+  }
+  return chunks;
+}
+
+/**
+ * Sleep helper for retry backoff.
+ * @param {number} ms
+ * @returns {Promise<void>}
+ */
+export function wait(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Process a large dataset in chunks (max 20 items per chunk) using Promise.all.
+ * @template T
+ * @template R
+ * @param {T[]} data
+ * @param {(chunk: T[], chunkIndex: number, totalChunks: number) => Promise<R[] | R> | (R[] | R)} processChunk
+ * @param {{ chunkSize?: number, flatten?: boolean }} [options]
+ * @returns {Promise<R[] | (R[] | R)[]>}
+ */
+export async function processInChunks(data, processChunk, options = {}) {
+  if (typeof processChunk !== "function") {
+    throw new TypeError("processChunk must be a function");
+  }
+
+  const { chunkSize = 20, flatten = true } = options;
+  const chunks = chunkArray(data, chunkSize);
+
+  const results = await Promise.all(
+    chunks.map((chunk, chunkIndex) => processChunk(chunk, chunkIndex, chunks.length))
+  );
+
+  if (!flatten) return results;
+
+  const merged = [];
+  for (const result of results) {
+    if (Array.isArray(result)) {
+      merged.push(...result);
+    } else {
+      merged.push(result);
+    }
+  }
+  return merged;
+}
+
+/**
+ * Retry one chunk operation with configurable retry logic.
+ * @template T
+ * @template R
+ * @param {T[]} chunk
+ * @param {number} chunkIndex
+ * @param {number} totalChunks
+ * @param {(chunk: T[], chunkIndex: number, totalChunks: number) => Promise<R>} processChunk
+ * @param {{
+ *  retries: number,
+ *  getRetryDelay: (attempt: number, error: unknown) => number,
+ *  shouldRetry: (error: unknown, attempt: number, chunkIndex: number, totalChunks: number) => boolean
+ * }} config
+ * @returns {Promise<R>}
+ */
+async function runChunkWithRetry(chunk, chunkIndex, totalChunks, processChunk, config) {
+  let attempt = 0;
+
+  while (attempt <= config.retries) {
+    try {
+      return await processChunk(chunk, chunkIndex, totalChunks);
+    } catch (error) {
+      if (attempt >= config.retries) {
+        throw error;
+      }
+
+      if (!config.shouldRetry(error, attempt, chunkIndex, totalChunks)) {
+        throw error;
+      }
+
+      const delay = config.getRetryDelay(attempt, error);
+      if (delay > 0) {
+        await wait(delay);
+      }
+      attempt += 1;
+    }
+  }
+
+  throw new Error(`Chunk ${chunkIndex + 1}/${totalChunks} failed after retries`);
+}
+
+/**
+ * Process any dataset in chunks with controlled concurrency, retry, and custom merge.
+ * @template T
+ * @template R
+ * @template M
+ * @param {T[]} data
+ * @param {(chunk: T[], chunkIndex: number, totalChunks: number) => Promise<R> | R} processChunk
+ * @param {{
+ *  chunkSize?: number,
+ *  maxConcurrency?: number,
+ *  retries?: number,
+ *  retryDelayMs?: number,
+ *  backoffMultiplier?: number,
+ *  shouldRetry?: (error: unknown, attempt: number, chunkIndex: number, totalChunks: number) => boolean,
+ *  mergeResults?: (chunkResults: R[]) => M
+ * }} [options]
+ * @returns {Promise<M | R[]>}
+ */
+export async function processDatasetInChunks(data, processChunk, options = {}) {
+  if (typeof processChunk !== "function") {
+    throw new TypeError("processChunk must be a function");
+  }
+
+  const {
+    chunkSize = 20,
+    maxConcurrency = Number.POSITIVE_INFINITY,
+    retries = 0,
+    retryDelayMs = 0,
+    backoffMultiplier = 1,
+    shouldRetry = () => true,
+    mergeResults,
+  } = options;
+
+  if (!Number.isInteger(maxConcurrency) && maxConcurrency !== Number.POSITIVE_INFINITY) {
+    throw new TypeError("maxConcurrency must be a positive integer or Infinity");
+  }
+
+  if (
+    maxConcurrency !== Number.POSITIVE_INFINITY &&
+    (!Number.isInteger(maxConcurrency) || maxConcurrency <= 0)
+  ) {
+    throw new RangeError("maxConcurrency must be greater than 0");
+  }
+
+  if (!Number.isInteger(retries) || retries < 0) {
+    throw new RangeError("retries must be a non-negative integer");
+  }
+
+  if (typeof shouldRetry !== "function") {
+    throw new TypeError("shouldRetry must be a function");
+  }
+
+  if (mergeResults && typeof mergeResults !== "function") {
+    throw new TypeError("mergeResults must be a function");
+  }
+
+  const chunks = chunkArray(data, chunkSize);
+  /** @type {R[]} */
+  const chunkResults = [];
+
+  const getRetryDelay = (attempt) => retryDelayMs * Math.pow(backoffMultiplier, attempt);
+
+  for (let i = 0; i < chunks.length; i += maxConcurrency) {
+    const batch = chunks.slice(i, i + maxConcurrency);
+    const batchResults = await Promise.all(
+      batch.map((chunk, batchOffset) => {
+        const chunkIndex = i + batchOffset;
+        return runChunkWithRetry(chunk, chunkIndex, chunks.length, processChunk, {
+          retries,
+          getRetryDelay,
+          shouldRetry,
+        });
+      })
+    );
+    chunkResults.push(...batchResults);
+  }
+
+  if (mergeResults) {
+    return mergeResults(chunkResults);
+  }
+
+  return chunkResults;
+}

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@dshahi468/nd-dataset-split",
+  "version": "1.0.0",
+  "description": "Split large dataset and resolve timeout issues",
+  "keywords": [
+    "chunk",
+    "batch",
+    "dataset",
+    "promise-all",
+    "retry",
+    "concurrency",
+    "bedrock"
+  ],
+  "license": "ISC",
+  "author": "Dilendra Vikram Shahi",
+  "type": "module",
+  "main": "index.js",
+  "types": "index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./index.d.ts",
+      "import": "./index.js"
+    }
+  },
+  "files": [
+    "index.js",
+    "index.d.ts",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  }
+}