@yutaura/csv-batch-reader 1.1.0 → 1.2.1

package/dist/csv-batch-read.d.ts

@@ -1,2 +1,3 @@
-export declare const csvBatchRead: <T extends Record<string, string> = Record<string, string>>(filePath: string, batchSize: number, handleHeader: (header: (keyof T)[]) => unknown, handleBatch: (rows: T[], batchCount: number, isLastChunk: boolean, header: (keyof T)[]) => unknown) => Promise<void>;
+import { PassThrough } from "node:stream";
+export declare const csvBatchRead: <T extends Record<string, string> = Record<string, string>>(filePath: string, batchSize: number, handleHeader: (header: (keyof T)[]) => unknown, handleBatch: (rows: PassThrough, batchCount: number, isLastChunk: boolean, header: (keyof T)[]) => unknown) => Promise<void>;
 //# sourceMappingURL=csv-batch-read.d.ts.map

package/dist/csv-batch-read.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csv-batch-read.d.ts","sourceRoot":"","sources":["../src/csv-batch-read.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,YAAY,GACvB,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,qCAEtB,MAAM,aACL,MAAM,gBACH,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,OAAO,eACjC,CACX,IAAI,EAAE,CAAC,EAAE,EACT,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,OAAO,EACpB,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,KAChB,OAAO,kBA+Db,CAAC"}
+{"version":3,"file":"csv-batch-read.d.ts","sourceRoot":"","sources":["../src/csv-batch-read.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAK1C,eAAO,MAAM,YAAY,GACvB,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,qCAEtB,MAAM,aACL,MAAM,gBACH,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,OAAO,eACjC,CACX,IAAI,EAAE,WAAW,EACjB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,OAAO,EACpB,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,KAChB,OAAO,kBAgEb,CAAC"}

package/dist/index.js

@@ -36,6 +36,8 @@ module.exports = __toCommonJS(src_exports);
 
 // src/csv-batch-read.ts
 var import_node_fs = require("fs");
+var import_node_stream = require("stream");
+var import_promises = require("stream/promises");
 var import_papaparse = __toESM(require("papaparse"));
 
 // src/promise-with-resolvers.ts
@@ -59,7 +61,8 @@ var csvBatchRead = async (filePath, batchSize, handleHeader, handleBatch) => {
     const stream = (0, import_node_fs.createReadStream)(filePath).pipe(
       import_papaparse.default.parse(import_papaparse.default.NODE_STREAM_INPUT, { header: true })
     );
-    let buf = [];
+    let currentStream = new import_node_stream.PassThrough({ objectMode: true });
+    let currentStreamCount = 0;
     let batchCount = 0;
     let isFirstChunk = true;
     const { promise: headerPromise, resolve: resolveHeaders } = promiseWithResolvers();
@@ -73,27 +76,27 @@ var csvBatchRead = async (filePath, batchSize, handleHeader, handleBatch) => {
         resolveHeaders(Object.keys(chunk));
         isFirstChunk = false;
       }
-      if (buf.length === batchSize) {
+      if (currentStreamCount === batchSize) {
+        shouldResolve.push((0, import_promises.finished)(currentStream.end()));
+        currentStream = new import_node_stream.PassThrough({ objectMode: true });
+        currentStreamCount = 0;
+        batchCount++;
+      }
+      if (currentStreamCount === 0) {
+        const stream2 = currentStream;
         const currentBatchCount = batchCount;
-        const currentBuf = buf.slice();
         shouldResolve.push(
           headerResolved.then(
-            (headers) => handleBatch(currentBuf, currentBatchCount, false, headers)
+            (headers) => handleBatch(stream2, currentBatchCount, false, headers)
           )
         );
-        buf = [];
-        batchCount++;
       }
-      buf.push(chunk);
+      currentStream.push(chunk);
+      currentStreamCount++;
     });
     stream.on("end", () => {
-      const currentBatchCount = batchCount;
-      const currentBuf = buf.slice();
-      shouldResolve.push(
-        headerResolved.then(
-          (headers) => handleBatch(currentBuf, currentBatchCount, true, headers)
-        )
-      );
+      console.log("stream end");
+      currentStream.end();
       Promise.all(shouldResolve).then(() => {
         stream.destroy();
         resolve();

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/index.ts","../src/csv-batch-read.ts","../src/promise-with-resolvers.ts"],"sourcesContent":["export { csvBatchRead } from \"./csv-batch-read.js\";\n","import { createReadStream } from \"node:fs\";\nimport Papa from \"papaparse\";\nimport { promiseWithResolvers } from \"./promise-with-resolvers.js\";\n\nexport const csvBatchRead = async <\n  T extends Record<string, string> = Record<string, string>,\n>(\n  filePath: string,\n  batchSize: number,\n  handleHeader: (header: (keyof T)[]) => unknown,\n  handleBatch: (\n    rows: T[],\n    batchCount: number,\n    isLastChunk: boolean,\n    header: (keyof T)[],\n  ) => unknown,\n) => {\n  return await new Promise<void>((resolve, reject) => {\n    const stream = createReadStream(filePath).pipe(\n      Papa.parse(Papa.NODE_STREAM_INPUT, { header: true }),\n    );\n\n    let buf: T[] = [];\n    let batchCount = 0;\n    let isFirstChunk = true;\n    const { promise: headerPromise, resolve: resolveHeaders } =\n      promiseWithResolvers<(keyof T)[]>();\n    const headerResolved = headerPromise.then(async (headers) => {\n      await handleHeader(headers);\n      return headers;\n    });\n    const shouldResolve: unknown[] = [headerResolved];\n\n    stream.on(\"data\", (chunk: T) => {\n      if (isFirstChunk) {\n        resolveHeaders(Object.keys(chunk));\n        isFirstChunk = false;\n      }\n      if (buf.length === batchSize) {\n        const currentBatchCount = batchCount;\n        const currentBuf = buf.slice();\n        shouldResolve.push(\n          headerResolved.then((headers) =>\n            handleBatch(currentBuf, currentBatchCount, false, headers),\n          ),\n        );\n        buf = [];\n        batchCount++;\n      }\n      buf.push(chunk);\n    });\n\n    stream.on(\"end\", () => {\n      const currentBatchCount = batchCount;\n      const currentBuf = buf.slice();\n      shouldResolve.push(\n        headerResolved.then((headers) =>\n          handleBatch(currentBuf, currentBatchCount, true, headers),\n        ),\n      );\n\n      Promise.all(shouldResolve)\n        .then(() => {\n          stream.destroy();\n          resolve();\n        })\n        .catch((error) => {\n          stream.destroy();\n          reject(error);\n        });\n    });\n\n    stream.on(\"error\", (error) => {\n      console.error(\"error\", error);\n      stream.destroy();\n      reject(error);\n    });\n  });\n};\n","export const promiseWithResolvers = <T = void>() => {\n  // new Promise のコンストラクタ自体は非同期処理を行わないため、\n  // resolve, reject への代入は promise の生成が終わったタイミングで完了している\n  let resolve: (value: T | PromiseLike<T>) => void = () => undefined;\n  let reject: (reason?: unknown) => void = () => undefined;\n  const promise = new Promise<T>((res, rej) => {\n    resolve = res;\n    reject = rej;\n  });\n  return {\n    promise,\n    resolve,\n    reject,\n  };\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,qBAAiC;AACjC,uBAAiB;;;ACDV,IAAM,uBAAuB,MAAgB;AAGlD,MAAI,UAA+C,MAAM;AACzD,MAAI,SAAqC,MAAM;AAC/C,QAAM,UAAU,IAAI,QAAW,CAAC,KAAK,QAAQ;AAC3C,cAAU;AACV,aAAS;AAAA,EACX,CAAC;AACD,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;;;ADVO,IAAM,eAAe,OAG1B,UACA,WACA,cACA,gBAMG;AACH,SAAO,MAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAClD,UAAM,aAAS,iCAAiB,QAAQ,EAAE;AAAA,MACxC,iBAAAA,QAAK,MAAM,iBAAAA,QAAK,mBAAmB,EAAE,QAAQ,KAAK,CAAC;AAAA,IACrD;AAEA,QAAI,MAAW,CAAC;AAChB,QAAI,aAAa;AACjB,QAAI,eAAe;AACnB,UAAM,EAAE,SAAS,eAAe,SAAS,eAAe,IACtD,qBAAkC;AACpC,UAAM,iBAAiB,cAAc,KAAK,OAAO,YAAY;AAC3D,YAAM,aAAa,OAAO;AAC1B,aAAO;AAAA,IACT,CAAC;AACD,UAAM,gBAA2B,CAAC,cAAc;AAEhD,WAAO,GAAG,QAAQ,CAAC,UAAa;AAC9B,UAAI,cAAc;AAChB,uBAAe,OAAO,KAAK,KAAK,CAAC;AACjC,uBAAe;AAAA,MACjB;AACA,UAAI,IAAI,WAAW,WAAW;AAC5B,cAAM,oBAAoB;AAC1B,cAAM,aAAa,IAAI,MAAM;AAC7B,sBAAc;AAAA,UACZ,eAAe;AAAA,YAAK,CAAC,YACnB,YAAY,YAAY,mBAAmB,OAAO,OAAO;AAAA,UAC3D;AAAA,QACF;AACA,cAAM,CAAC;AACP;AAAA,MACF;AACA,UAAI,KAAK,KAAK;AAAA,IAChB,CAAC;AAED,WAAO,GAAG,OAAO,MAAM;AACrB,YAAM,oBAAoB;AAC1B,YAAM,aAAa,IAAI,MAAM;AAC7B,oBAAc;AAAA,QACZ,eAAe;AAAA,UAAK,CAAC,YACnB,YAAY,YAAY,mBAAmB,MAAM,OAAO;AAAA,QAC1D;AAAA,MACF;AAEA,cAAQ,IAAI,aAAa,EACtB,KAAK,MAAM;AACV,eAAO,QAAQ;AACf,gBAAQ;AAAA,MACV,CAAC,EACA,MAAM,CAAC,UAAU;AAChB,eAAO,QAAQ;AACf,eAAO,KAAK;AAAA,MACd,CAAC;AAAA,IACL,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,UAAU;AAC5B,cAAQ,MAAM,SAAS,KAAK;AAC5B,aAAO,QAAQ;AACf,aAAO,KAAK;AAAA,IACd,CAAC;AAAA,EACH,CAAC;AACH;","names":["Papa"]}
+{"version":3,"sources":["../src/index.ts","../src/csv-batch-read.ts","../src/promise-with-resolvers.ts"],"sourcesContent":["export { csvBatchRead } from \"./csv-batch-read.js\";\n","import { createReadStream } from \"node:fs\";\nimport { PassThrough } from \"node:stream\";\nimport { finished } from \"node:stream/promises\";\nimport Papa from \"papaparse\";\nimport { promiseWithResolvers } from \"./promise-with-resolvers.js\";\n\nexport const csvBatchRead = async <\n  T extends Record<string, string> = Record<string, string>,\n>(\n  filePath: string,\n  batchSize: number,\n  handleHeader: (header: (keyof T)[]) => unknown,\n  handleBatch: (\n    rows: PassThrough,\n    batchCount: number,\n    isLastChunk: boolean,\n    header: (keyof T)[],\n  ) => unknown,\n) => {\n  return await new Promise<void>((resolve, reject) => {\n    const stream = createReadStream(filePath).pipe(\n      Papa.parse(Papa.NODE_STREAM_INPUT, { header: true }),\n    );\n\n    let currentStream = new PassThrough({ objectMode: true });\n    let currentStreamCount = 0;\n    let batchCount = 0;\n    let isFirstChunk = true;\n    const { promise: headerPromise, resolve: resolveHeaders } =\n      promiseWithResolvers<(keyof T)[]>();\n    const headerResolved = headerPromise.then(async (headers) => {\n      await handleHeader(headers);\n      return headers;\n    });\n    const shouldResolve: unknown[] = [headerResolved];\n\n    stream.on(\"data\", (chunk: T) => {\n      if (isFirstChunk) {\n        resolveHeaders(Object.keys(chunk));\n        isFirstChunk = false;\n      }\n      if (currentStreamCount === batchSize) {\n        shouldResolve.push(finished(currentStream.end()));\n        currentStream = new PassThrough({ objectMode: true });\n        currentStreamCount = 0;\n        batchCount++;\n      }\n      if (currentStreamCount === 0) {\n        const stream = currentStream;\n        const currentBatchCount = batchCount;\n        shouldResolve.push(\n          headerResolved.then((headers) =>\n            handleBatch(stream, currentBatchCount, false, headers),\n          ),\n        );\n      }\n      currentStream.push(chunk);\n      currentStreamCount++;\n    });\n\n    stream.on(\"end\", () => {\n      console.log(\"stream end\");\n      currentStream.end();\n\n      Promise.all(shouldResolve)\n        .then(() => {\n          stream.destroy();\n          resolve();\n        })\n        .catch((error) => {\n          stream.destroy();\n          reject(error);\n        });\n    });\n\n    stream.on(\"error\", (error) => {\n      console.error(\"error\", error);\n      stream.destroy();\n      reject(error);\n    });\n  });\n};\n","export const promiseWithResolvers = <T = void>() => {\n  // new Promise のコンストラクタ自体は非同期処理を行わないため、\n  // resolve, reject への代入は promise の生成が終わったタイミングで完了している\n  let resolve: (value: T | PromiseLike<T>) => void = () => undefined;\n  let reject: (reason?: unknown) => void = () => undefined;\n  const promise = new Promise<T>((res, rej) => {\n    resolve = res;\n    reject = rej;\n  });\n  return {\n    promise,\n    resolve,\n    reject,\n  };\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,qBAAiC;AACjC,yBAA4B;AAC5B,sBAAyB;AACzB,uBAAiB;;;ACHV,IAAM,uBAAuB,MAAgB;AAGlD,MAAI,UAA+C,MAAM;AACzD,MAAI,SAAqC,MAAM;AAC/C,QAAM,UAAU,IAAI,QAAW,CAAC,KAAK,QAAQ;AAC3C,cAAU;AACV,aAAS;AAAA,EACX,CAAC;AACD,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;;;ADRO,IAAM,eAAe,OAG1B,UACA,WACA,cACA,gBAMG;AACH,SAAO,MAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAClD,UAAM,aAAS,iCAAiB,QAAQ,EAAE;AAAA,MACxC,iBAAAA,QAAK,MAAM,iBAAAA,QAAK,mBAAmB,EAAE,QAAQ,KAAK,CAAC;AAAA,IACrD;AAEA,QAAI,gBAAgB,IAAI,+BAAY,EAAE,YAAY,KAAK,CAAC;AACxD,QAAI,qBAAqB;AACzB,QAAI,aAAa;AACjB,QAAI,eAAe;AACnB,UAAM,EAAE,SAAS,eAAe,SAAS,eAAe,IACtD,qBAAkC;AACpC,UAAM,iBAAiB,cAAc,KAAK,OAAO,YAAY;AAC3D,YAAM,aAAa,OAAO;AAC1B,aAAO;AAAA,IACT,CAAC;AACD,UAAM,gBAA2B,CAAC,cAAc;AAEhD,WAAO,GAAG,QAAQ,CAAC,UAAa;AAC9B,UAAI,cAAc;AAChB,uBAAe,OAAO,KAAK,KAAK,CAAC;AACjC,uBAAe;AAAA,MACjB;AACA,UAAI,uBAAuB,WAAW;AACpC,sBAAc,SAAK,0BAAS,cAAc,IAAI,CAAC,CAAC;AAChD,wBAAgB,IAAI,+BAAY,EAAE,YAAY,KAAK,CAAC;AACpD,6BAAqB;AACrB;AAAA,MACF;AACA,UAAI,uBAAuB,GAAG;AAC5B,cAAMC,UAAS;AACf,cAAM,oBAAoB;AAC1B,sBAAc;AAAA,UACZ,eAAe;AAAA,YAAK,CAAC,YACnB,YAAYA,SAAQ,mBAAmB,OAAO,OAAO;AAAA,UACvD;AAAA,QACF;AAAA,MACF;AACA,oBAAc,KAAK,KAAK;AACxB;AAAA,IACF,CAAC;AAED,WAAO,GAAG,OAAO,MAAM;AACrB,cAAQ,IAAI,YAAY;AACxB,oBAAc,IAAI;AAElB,cAAQ,IAAI,aAAa,EACtB,KAAK,MAAM;AACV,eAAO,QAAQ;AACf,gBAAQ;AAAA,MACV,CAAC,EACA,MAAM,CAAC,UAAU;AAChB,eAAO,QAAQ;AACf,eAAO,KAAK;AAAA,MACd,CAAC;AAAA,IACL,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,UAAU;AAC5B,cAAQ,MAAM,SAAS,KAAK;AAC5B,aAAO,QAAQ;AACf,aAAO,KAAK;AAAA,IACd,CAAC;AAAA,EACH,CAAC;AACH;","names":["Papa","stream"]}

package/dist/index.mjs

@@ -1,5 +1,7 @@
 // src/csv-batch-read.ts
 import { createReadStream } from "node:fs";
+import { PassThrough } from "node:stream";
+import { finished } from "node:stream/promises";
 import Papa from "papaparse";
 
 // src/promise-with-resolvers.ts
@@ -23,7 +25,8 @@ var csvBatchRead = async (filePath, batchSize, handleHeader, handleBatch) => {
     const stream = createReadStream(filePath).pipe(
       Papa.parse(Papa.NODE_STREAM_INPUT, { header: true })
     );
-    let buf = [];
+    let currentStream = new PassThrough({ objectMode: true });
+    let currentStreamCount = 0;
     let batchCount = 0;
     let isFirstChunk = true;
     const { promise: headerPromise, resolve: resolveHeaders } = promiseWithResolvers();
@@ -37,27 +40,27 @@ var csvBatchRead = async (filePath, batchSize, handleHeader, handleBatch) => {
         resolveHeaders(Object.keys(chunk));
         isFirstChunk = false;
       }
-      if (buf.length === batchSize) {
+      if (currentStreamCount === batchSize) {
+        shouldResolve.push(finished(currentStream.end()));
+        currentStream = new PassThrough({ objectMode: true });
+        currentStreamCount = 0;
+        batchCount++;
+      }
+      if (currentStreamCount === 0) {
+        const stream2 = currentStream;
         const currentBatchCount = batchCount;
-        const currentBuf = buf.slice();
         shouldResolve.push(
           headerResolved.then(
-            (headers) => handleBatch(currentBuf, currentBatchCount, false, headers)
+            (headers) => handleBatch(stream2, currentBatchCount, false, headers)
           )
         );
-        buf = [];
-        batchCount++;
       }
-      buf.push(chunk);
+      currentStream.push(chunk);
+      currentStreamCount++;
     });
     stream.on("end", () => {
-      const currentBatchCount = batchCount;
-      const currentBuf = buf.slice();
-      shouldResolve.push(
-        headerResolved.then(
-          (headers) => handleBatch(currentBuf, currentBatchCount, true, headers)
-        )
-      );
+      console.log("stream end");
+      currentStream.end();
       Promise.all(shouldResolve).then(() => {
         stream.destroy();
         resolve();

package/dist/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/csv-batch-read.ts","../src/promise-with-resolvers.ts"],"sourcesContent":["import { createReadStream } from \"node:fs\";\nimport Papa from \"papaparse\";\nimport { promiseWithResolvers } from \"./promise-with-resolvers.js\";\n\nexport const csvBatchRead = async <\n  T extends Record<string, string> = Record<string, string>,\n>(\n  filePath: string,\n  batchSize: number,\n  handleHeader: (header: (keyof T)[]) => unknown,\n  handleBatch: (\n    rows: T[],\n    batchCount: number,\n    isLastChunk: boolean,\n    header: (keyof T)[],\n  ) => unknown,\n) => {\n  return await new Promise<void>((resolve, reject) => {\n    const stream = createReadStream(filePath).pipe(\n      Papa.parse(Papa.NODE_STREAM_INPUT, { header: true }),\n    );\n\n    let buf: T[] = [];\n    let batchCount = 0;\n    let isFirstChunk = true;\n    const { promise: headerPromise, resolve: resolveHeaders } =\n      promiseWithResolvers<(keyof T)[]>();\n    const headerResolved = headerPromise.then(async (headers) => {\n      await handleHeader(headers);\n      return headers;\n    });\n    const shouldResolve: unknown[] = [headerResolved];\n\n    stream.on(\"data\", (chunk: T) => {\n      if (isFirstChunk) {\n        resolveHeaders(Object.keys(chunk));\n        isFirstChunk = false;\n      }\n      if (buf.length === batchSize) {\n        const currentBatchCount = batchCount;\n        const currentBuf = buf.slice();\n        shouldResolve.push(\n          headerResolved.then((headers) =>\n            handleBatch(currentBuf, currentBatchCount, false, headers),\n          ),\n        );\n        buf = [];\n        batchCount++;\n      }\n      buf.push(chunk);\n    });\n\n    stream.on(\"end\", () => {\n      const currentBatchCount = batchCount;\n      const currentBuf = buf.slice();\n      shouldResolve.push(\n        headerResolved.then((headers) =>\n          handleBatch(currentBuf, currentBatchCount, true, headers),\n        ),\n      );\n\n      Promise.all(shouldResolve)\n        .then(() => {\n          stream.destroy();\n          resolve();\n        })\n        .catch((error) => {\n          stream.destroy();\n          reject(error);\n        });\n    });\n\n    stream.on(\"error\", (error) => {\n      console.error(\"error\", error);\n      stream.destroy();\n      reject(error);\n    });\n  });\n};\n","export const promiseWithResolvers = <T = void>() => {\n  // new Promise のコンストラクタ自体は非同期処理を行わないため、\n  // resolve, reject への代入は promise の生成が終わったタイミングで完了している\n  let resolve: (value: T | PromiseLike<T>) => void = () => undefined;\n  let reject: (reason?: unknown) => void = () => undefined;\n  const promise = new Promise<T>((res, rej) => {\n    resolve = res;\n    reject = rej;\n  });\n  return {\n    promise,\n    resolve,\n    reject,\n  };\n};\n"],"mappings":";AAAA,SAAS,wBAAwB;AACjC,OAAO,UAAU;;;ACDV,IAAM,uBAAuB,MAAgB;AAGlD,MAAI,UAA+C,MAAM;AACzD,MAAI,SAAqC,MAAM;AAC/C,QAAM,UAAU,IAAI,QAAW,CAAC,KAAK,QAAQ;AAC3C,cAAU;AACV,aAAS;AAAA,EACX,CAAC;AACD,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;;;ADVO,IAAM,eAAe,OAG1B,UACA,WACA,cACA,gBAMG;AACH,SAAO,MAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAClD,UAAM,SAAS,iBAAiB,QAAQ,EAAE;AAAA,MACxC,KAAK,MAAM,KAAK,mBAAmB,EAAE,QAAQ,KAAK,CAAC;AAAA,IACrD;AAEA,QAAI,MAAW,CAAC;AAChB,QAAI,aAAa;AACjB,QAAI,eAAe;AACnB,UAAM,EAAE,SAAS,eAAe,SAAS,eAAe,IACtD,qBAAkC;AACpC,UAAM,iBAAiB,cAAc,KAAK,OAAO,YAAY;AAC3D,YAAM,aAAa,OAAO;AAC1B,aAAO;AAAA,IACT,CAAC;AACD,UAAM,gBAA2B,CAAC,cAAc;AAEhD,WAAO,GAAG,QAAQ,CAAC,UAAa;AAC9B,UAAI,cAAc;AAChB,uBAAe,OAAO,KAAK,KAAK,CAAC;AACjC,uBAAe;AAAA,MACjB;AACA,UAAI,IAAI,WAAW,WAAW;AAC5B,cAAM,oBAAoB;AAC1B,cAAM,aAAa,IAAI,MAAM;AAC7B,sBAAc;AAAA,UACZ,eAAe;AAAA,YAAK,CAAC,YACnB,YAAY,YAAY,mBAAmB,OAAO,OAAO;AAAA,UAC3D;AAAA,QACF;AACA,cAAM,CAAC;AACP;AAAA,MACF;AACA,UAAI,KAAK,KAAK;AAAA,IAChB,CAAC;AAED,WAAO,GAAG,OAAO,MAAM;AACrB,YAAM,oBAAoB;AAC1B,YAAM,aAAa,IAAI,MAAM;AAC7B,oBAAc;AAAA,QACZ,eAAe;AAAA,UAAK,CAAC,YACnB,YAAY,YAAY,mBAAmB,MAAM,OAAO;AAAA,QAC1D;AAAA,MACF;AAEA,cAAQ,IAAI,aAAa,EACtB,KAAK,MAAM;AACV,eAAO,QAAQ;AACf,gBAAQ;AAAA,MACV,CAAC,EACA,MAAM,CAAC,UAAU;AAChB,eAAO,QAAQ;AACf,eAAO,KAAK;AAAA,MACd,CAAC;AAAA,IACL,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,UAAU;AAC5B,cAAQ,MAAM,SAAS,KAAK;AAC5B,aAAO,QAAQ;AACf,aAAO,KAAK;AAAA,IACd,CAAC;AAAA,EACH,CAAC;AACH;","names":[]}
+{"version":3,"sources":["../src/csv-batch-read.ts","../src/promise-with-resolvers.ts"],"sourcesContent":["import { createReadStream } from \"node:fs\";\nimport { PassThrough } from \"node:stream\";\nimport { finished } from \"node:stream/promises\";\nimport Papa from \"papaparse\";\nimport { promiseWithResolvers } from \"./promise-with-resolvers.js\";\n\nexport const csvBatchRead = async <\n  T extends Record<string, string> = Record<string, string>,\n>(\n  filePath: string,\n  batchSize: number,\n  handleHeader: (header: (keyof T)[]) => unknown,\n  handleBatch: (\n    rows: PassThrough,\n    batchCount: number,\n    isLastChunk: boolean,\n    header: (keyof T)[],\n  ) => unknown,\n) => {\n  return await new Promise<void>((resolve, reject) => {\n    const stream = createReadStream(filePath).pipe(\n      Papa.parse(Papa.NODE_STREAM_INPUT, { header: true }),\n    );\n\n    let currentStream = new PassThrough({ objectMode: true });\n    let currentStreamCount = 0;\n    let batchCount = 0;\n    let isFirstChunk = true;\n    const { promise: headerPromise, resolve: resolveHeaders } =\n      promiseWithResolvers<(keyof T)[]>();\n    const headerResolved = headerPromise.then(async (headers) => {\n      await handleHeader(headers);\n      return headers;\n    });\n    const shouldResolve: unknown[] = [headerResolved];\n\n    stream.on(\"data\", (chunk: T) => {\n      if (isFirstChunk) {\n        resolveHeaders(Object.keys(chunk));\n        isFirstChunk = false;\n      }\n      if (currentStreamCount === batchSize) {\n        shouldResolve.push(finished(currentStream.end()));\n        currentStream = new PassThrough({ objectMode: true });\n        currentStreamCount = 0;\n        batchCount++;\n      }\n      if (currentStreamCount === 0) {\n        const stream = currentStream;\n        const currentBatchCount = batchCount;\n        shouldResolve.push(\n          headerResolved.then((headers) =>\n            handleBatch(stream, currentBatchCount, false, headers),\n          ),\n        );\n      }\n      currentStream.push(chunk);\n      currentStreamCount++;\n    });\n\n    stream.on(\"end\", () => {\n      console.log(\"stream end\");\n      currentStream.end();\n\n      Promise.all(shouldResolve)\n        .then(() => {\n          stream.destroy();\n          resolve();\n        })\n        .catch((error) => {\n          stream.destroy();\n          reject(error);\n        });\n    });\n\n    stream.on(\"error\", (error) => {\n      console.error(\"error\", error);\n      stream.destroy();\n      reject(error);\n    });\n  });\n};\n","export const promiseWithResolvers = <T = void>() => {\n  // new Promise のコンストラクタ自体は非同期処理を行わないため、\n  // resolve, reject への代入は promise の生成が終わったタイミングで完了している\n  let resolve: (value: T | PromiseLike<T>) => void = () => undefined;\n  let reject: (reason?: unknown) => void = () => undefined;\n  const promise = new Promise<T>((res, rej) => {\n    resolve = res;\n    reject = rej;\n  });\n  return {\n    promise,\n    resolve,\n    reject,\n  };\n};\n"],"mappings":";AAAA,SAAS,wBAAwB;AACjC,SAAS,mBAAmB;AAC5B,SAAS,gBAAgB;AACzB,OAAO,UAAU;;;ACHV,IAAM,uBAAuB,MAAgB;AAGlD,MAAI,UAA+C,MAAM;AACzD,MAAI,SAAqC,MAAM;AAC/C,QAAM,UAAU,IAAI,QAAW,CAAC,KAAK,QAAQ;AAC3C,cAAU;AACV,aAAS;AAAA,EACX,CAAC;AACD,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;;;ADRO,IAAM,eAAe,OAG1B,UACA,WACA,cACA,gBAMG;AACH,SAAO,MAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAClD,UAAM,SAAS,iBAAiB,QAAQ,EAAE;AAAA,MACxC,KAAK,MAAM,KAAK,mBAAmB,EAAE,QAAQ,KAAK,CAAC;AAAA,IACrD;AAEA,QAAI,gBAAgB,IAAI,YAAY,EAAE,YAAY,KAAK,CAAC;AACxD,QAAI,qBAAqB;AACzB,QAAI,aAAa;AACjB,QAAI,eAAe;AACnB,UAAM,EAAE,SAAS,eAAe,SAAS,eAAe,IACtD,qBAAkC;AACpC,UAAM,iBAAiB,cAAc,KAAK,OAAO,YAAY;AAC3D,YAAM,aAAa,OAAO;AAC1B,aAAO;AAAA,IACT,CAAC;AACD,UAAM,gBAA2B,CAAC,cAAc;AAEhD,WAAO,GAAG,QAAQ,CAAC,UAAa;AAC9B,UAAI,cAAc;AAChB,uBAAe,OAAO,KAAK,KAAK,CAAC;AACjC,uBAAe;AAAA,MACjB;AACA,UAAI,uBAAuB,WAAW;AACpC,sBAAc,KAAK,SAAS,cAAc,IAAI,CAAC,CAAC;AAChD,wBAAgB,IAAI,YAAY,EAAE,YAAY,KAAK,CAAC;AACpD,6BAAqB;AACrB;AAAA,MACF;AACA,UAAI,uBAAuB,GAAG;AAC5B,cAAMA,UAAS;AACf,cAAM,oBAAoB;AAC1B,sBAAc;AAAA,UACZ,eAAe;AAAA,YAAK,CAAC,YACnB,YAAYA,SAAQ,mBAAmB,OAAO,OAAO;AAAA,UACvD;AAAA,QACF;AAAA,MACF;AACA,oBAAc,KAAK,KAAK;AACxB;AAAA,IACF,CAAC;AAED,WAAO,GAAG,OAAO,MAAM;AACrB,cAAQ,IAAI,YAAY;AACxB,oBAAc,IAAI;AAElB,cAAQ,IAAI,aAAa,EACtB,KAAK,MAAM;AACV,eAAO,QAAQ;AACf,gBAAQ;AAAA,MACV,CAAC,EACA,MAAM,CAAC,UAAU;AAChB,eAAO,QAAQ;AACf,eAAO,KAAK;AAAA,MACd,CAAC;AAAA,IACL,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,UAAU;AAC5B,cAAQ,MAAM,SAAS,KAAK;AAC5B,aAAO,QAAQ;AACf,aAAO,KAAK;AAAA,IACd,CAAC;AAAA,EACH,CAAC;AACH;","names":["stream"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yutaura/csv-batch-reader",
-  "version": "1.1.0",
+  "version": "1.2.1",
   "publishConfig": {
     "access": "public"
   },

package/src/csv-batch-read.test.ts

@@ -57,8 +57,8 @@ describe("csvBatchRead", () => {
       TEST_CSV_FILE,
       3,
       () => {},
-      (r) => {
-        rows.push(r);
+      async (stream) => {
+        rows.push(await stream.toArray());
       },
     );
     // then
@@ -89,8 +89,8 @@ describe("csvBatchRead", () => {
       TEST_CSV_FILE,
       1,
       () => {},
-      (r) => {
-        rows.push(r);
+      async (r) => {
+        rows.push(await r.toArray());
       },
     );
     // then
@@ -115,8 +115,8 @@ describe("csvBatchRead", () => {
       TEST_CSV_FILE,
       9,
       () => {},
-      (r) => {
-        rows.push(r);
+      async (r) => {
+        rows.push(await r.toArray());
       },
     );
     // then
@@ -143,8 +143,8 @@ describe("csvBatchRead", () => {
       TEST_CSV_FILE,
       1000,
       () => {},
-      (r) => {
-        rows.push(r);
+      async (r) => {
+        rows.push(await r.toArray());
       },
     );
     // then
@@ -175,7 +175,8 @@ describe("csvBatchRead", () => {
         await setTimeout(500);
         headerResolvedAt = performance.now();
       },
-      () => {
+      async (stream) => {
+        await stream.toArray();
         rowsResolvedAt.push(performance.now());
       },
     );
@@ -193,7 +194,8 @@ describe("csvBatchRead", () => {
       TEST_CSV_FILE,
       3,
       () => {},
-      async () => {
+      async (stream) => {
+        await stream.toArray();
         await setTimeout(500);
         rowsResolvedAt.push(performance.now());
       },

package/src/csv-batch-read.ts

@@ -1,4 +1,6 @@
 import { createReadStream } from "node:fs";
+import { PassThrough } from "node:stream";
+import { finished } from "node:stream/promises";
 import Papa from "papaparse";
 import { promiseWithResolvers } from "./promise-with-resolvers.js";
 
@@ -9,7 +11,7 @@ export const csvBatchRead = async <
   batchSize: number,
   handleHeader: (header: (keyof T)[]) => unknown,
   handleBatch: (
-    rows: T[],
+    rows: PassThrough,
     batchCount: number,
     isLastChunk: boolean,
     header: (keyof T)[],
@@ -20,7 +22,8 @@ export const csvBatchRead = async <
       Papa.parse(Papa.NODE_STREAM_INPUT, { header: true }),
     );
 
-    let buf: T[] = [];
+    let currentStream = new PassThrough({ objectMode: true });
+    let currentStreamCount = 0;
     let batchCount = 0;
     let isFirstChunk = true;
     const { promise: headerPromise, resolve: resolveHeaders } =
@@ -36,28 +39,28 @@ export const csvBatchRead = async <
         resolveHeaders(Object.keys(chunk));
         isFirstChunk = false;
       }
-      if (buf.length === batchSize) {
+      if (currentStreamCount === batchSize) {
+        shouldResolve.push(finished(currentStream.end()));
+        currentStream = new PassThrough({ objectMode: true });
+        currentStreamCount = 0;
+        batchCount++;
+      }
+      if (currentStreamCount === 0) {
+        const stream = currentStream;
         const currentBatchCount = batchCount;
-        const currentBuf = buf.slice();
         shouldResolve.push(
           headerResolved.then((headers) =>
-            handleBatch(currentBuf, currentBatchCount, false, headers),
+            handleBatch(stream, currentBatchCount, false, headers),
           ),
         );
-        buf = [];
-        batchCount++;
       }
-      buf.push(chunk);
+      currentStream.push(chunk);
+      currentStreamCount++;
     });
 
     stream.on("end", () => {
-      const currentBatchCount = batchCount;
-      const currentBuf = buf.slice();
-      shouldResolve.push(
-        headerResolved.then((headers) =>
-          handleBatch(currentBuf, currentBatchCount, true, headers),
-        ),
-      );
+      console.log("stream end");
+      currentStream.end();
 
       Promise.all(shouldResolve)
         .then(() => {