@yutaura/csv-batch-reader 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/csv-batch-read.d.ts +1 -1
- package/dist/csv-batch-read.d.ts.map +1 -1
- package/dist/index.js +6 -5
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +6 -5
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
- package/src/csv-batch-read.ts +14 -8
package/dist/csv-batch-read.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export declare const csvBatchRead: <T extends Record<string, string> = Record<string, string>>(filePath: string, batchSize: number, handleHeader: (header: (keyof T)[]) => unknown, handleBatch: (rows: T[], batchCount: number, isLastChunk: boolean) => unknown) => Promise<void>;
|
|
1
|
+
export declare const csvBatchRead: <T extends Record<string, string> = Record<string, string>>(filePath: string, batchSize: number, handleHeader: (header: (keyof T)[]) => unknown, handleBatch: (rows: T[], batchCount: number, isLastChunk: boolean, header: (keyof T)[]) => unknown) => Promise<void>;
|
|
2
2
|
//# sourceMappingURL=csv-batch-read.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"csv-batch-read.d.ts","sourceRoot":"","sources":["../src/csv-batch-read.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,YAAY,GACvB,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,qCAEtB,MAAM,aACL,MAAM,gBACH,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,OAAO,eACjC,
|
|
1
|
+
{"version":3,"file":"csv-batch-read.d.ts","sourceRoot":"","sources":["../src/csv-batch-read.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,YAAY,GACvB,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,qCAEtB,MAAM,aACL,MAAM,gBACH,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,OAAO,eACjC,CACX,IAAI,EAAE,CAAC,EAAE,EACT,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,OAAO,EACpB,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,KAChB,OAAO,kBA+Db,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -63,9 +63,10 @@ var csvBatchRead = async (filePath, batchSize, handleHeader, handleBatch) => {
|
|
|
63
63
|
let batchCount = 0;
|
|
64
64
|
let isFirstChunk = true;
|
|
65
65
|
const { promise: headerPromise, resolve: resolveHeaders } = promiseWithResolvers();
|
|
66
|
-
const headerResolved = headerPromise.then(
|
|
67
|
-
|
|
68
|
-
|
|
66
|
+
const headerResolved = headerPromise.then(async (headers) => {
|
|
67
|
+
await handleHeader(headers);
|
|
68
|
+
return headers;
|
|
69
|
+
});
|
|
69
70
|
const shouldResolve = [headerResolved];
|
|
70
71
|
stream.on("data", (chunk) => {
|
|
71
72
|
if (isFirstChunk) {
|
|
@@ -77,7 +78,7 @@ var csvBatchRead = async (filePath, batchSize, handleHeader, handleBatch) => {
|
|
|
77
78
|
const currentBuf = buf.slice();
|
|
78
79
|
shouldResolve.push(
|
|
79
80
|
headerResolved.then(
|
|
80
|
-
() => handleBatch(currentBuf, currentBatchCount, false)
|
|
81
|
+
(headers) => handleBatch(currentBuf, currentBatchCount, false, headers)
|
|
81
82
|
)
|
|
82
83
|
);
|
|
83
84
|
buf = [];
|
|
@@ -90,7 +91,7 @@ var csvBatchRead = async (filePath, batchSize, handleHeader, handleBatch) => {
|
|
|
90
91
|
const currentBuf = buf.slice();
|
|
91
92
|
shouldResolve.push(
|
|
92
93
|
headerResolved.then(
|
|
93
|
-
() => handleBatch(currentBuf, currentBatchCount, true)
|
|
94
|
+
(headers) => handleBatch(currentBuf, currentBatchCount, true, headers)
|
|
94
95
|
)
|
|
95
96
|
);
|
|
96
97
|
Promise.all(shouldResolve).then(() => {
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/index.ts","../src/csv-batch-read.ts","../src/promise-with-resolvers.ts"],"sourcesContent":["export { csvBatchRead } from \"./csv-batch-read.js\";\n","import { createReadStream } from \"node:fs\";\nimport Papa from \"papaparse\";\nimport { promiseWithResolvers } from \"./promise-with-resolvers.js\";\n\nexport const csvBatchRead = async <\n T extends Record<string, string> = Record<string, string>,\n>(\n filePath: string,\n batchSize: number,\n handleHeader: (header: (keyof T)[]) => unknown,\n handleBatch: (rows: T[]
|
|
1
|
+
{"version":3,"sources":["../src/index.ts","../src/csv-batch-read.ts","../src/promise-with-resolvers.ts"],"sourcesContent":["export { csvBatchRead } from \"./csv-batch-read.js\";\n","import { createReadStream } from \"node:fs\";\nimport Papa from \"papaparse\";\nimport { promiseWithResolvers } from \"./promise-with-resolvers.js\";\n\nexport const csvBatchRead = async <\n T extends Record<string, string> = Record<string, string>,\n>(\n filePath: string,\n batchSize: number,\n handleHeader: (header: (keyof T)[]) => unknown,\n handleBatch: (\n rows: T[],\n batchCount: number,\n isLastChunk: boolean,\n header: (keyof T)[],\n ) => unknown,\n) => {\n return await new Promise<void>((resolve, reject) => {\n const stream = createReadStream(filePath).pipe(\n Papa.parse(Papa.NODE_STREAM_INPUT, { header: true }),\n );\n\n let buf: T[] = [];\n let batchCount = 0;\n let isFirstChunk = true;\n const { promise: headerPromise, resolve: resolveHeaders } =\n promiseWithResolvers<(keyof T)[]>();\n const headerResolved = headerPromise.then(async (headers) => {\n await handleHeader(headers);\n return headers;\n });\n const shouldResolve: unknown[] = [headerResolved];\n\n stream.on(\"data\", (chunk: T) => {\n if (isFirstChunk) {\n resolveHeaders(Object.keys(chunk));\n isFirstChunk = false;\n }\n if (buf.length === batchSize) {\n const currentBatchCount = batchCount;\n const currentBuf = buf.slice();\n shouldResolve.push(\n headerResolved.then((headers) =>\n handleBatch(currentBuf, currentBatchCount, false, headers),\n ),\n );\n buf = [];\n batchCount++;\n }\n buf.push(chunk);\n });\n\n stream.on(\"end\", () => {\n const currentBatchCount = batchCount;\n const currentBuf = buf.slice();\n shouldResolve.push(\n headerResolved.then((headers) =>\n handleBatch(currentBuf, currentBatchCount, true, headers),\n ),\n );\n\n Promise.all(shouldResolve)\n .then(() => {\n stream.destroy();\n resolve();\n })\n .catch((error) => {\n stream.destroy();\n reject(error);\n });\n });\n\n stream.on(\"error\", (error) => {\n console.error(\"error\", error);\n stream.destroy();\n reject(error);\n });\n });\n};\n","export const promiseWithResolvers = <T = void>() => {\n // new Promise のコンストラクタ自体は非同期処理を行わないため、\n // resolve, reject への代入は promise の生成が終わったタイミングで完了している\n let resolve: (value: T | PromiseLike<T>) => void = () => undefined;\n let reject: (reason?: unknown) => void = () => undefined;\n const promise = new Promise<T>((res, rej) => {\n resolve = res;\n reject = rej;\n });\n return {\n promise,\n resolve,\n reject,\n };\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,qBAAiC;AACjC,uBAAiB;;;ACDV,IAAM,uBAAuB,MAAgB;AAGlD,MAAI,UAA+C,MAAM;AACzD,MAAI,SAAqC,MAAM;AAC/C,QAAM,UAAU,IAAI,QAAW,CAAC,KAAK,QAAQ;AAC3C,cAAU;AACV,aAAS;AAAA,EACX,CAAC;AACD,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;;;ADVO,IAAM,eAAe,OAG1B,UACA,WACA,cACA,gBAMG;AACH,SAAO,MAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAClD,UAAM,aAAS,iCAAiB,QAAQ,EAAE;AAAA,MACxC,iBAAAA,QAAK,MAAM,iBAAAA,QAAK,mBAAmB,EAAE,QAAQ,KAAK,CAAC;AAAA,IACrD;AAEA,QAAI,MAAW,CAAC;AAChB,QAAI,aAAa;AACjB,QAAI,eAAe;AACnB,UAAM,EAAE,SAAS,eAAe,SAAS,eAAe,IACtD,qBAAkC;AACpC,UAAM,iBAAiB,cAAc,KAAK,OAAO,YAAY;AAC3D,YAAM,aAAa,OAAO;AAC1B,aAAO;AAAA,IACT,CAAC;AACD,UAAM,gBAA2B,CAAC,cAAc;AAEhD,WAAO,GAAG,QAAQ,CAAC,UAAa;AAC9B,UAAI,cAAc;AAChB,uBAAe,OAAO,KAAK,KAAK,CAAC;AACjC,uBAAe;AAAA,MACjB;AACA,UAAI,IAAI,WAAW,WAAW;AAC5B,cAAM,oBAAoB;AAC1B,cAAM,aAAa,IAAI,MAAM;AAC7B,sBAAc;AAAA,UACZ,eAAe;AAAA,YAAK,CAAC,YACnB,YAAY,YAAY,mBAAmB,OAAO,OAAO;AAAA,UAC3D;AAAA,QACF;AACA,cAAM,CAAC;AACP;AAAA,MACF;AACA,UAAI,KAAK,KAAK;AAAA,IAChB,CAAC;AAED,WAAO,GAAG,OAAO,MAAM;AACrB,YAAM,oBAAoB;AAC1B,YAAM,aAAa,IAAI,MAAM;AAC7B,oBAAc;AAAA,QACZ,eAAe;AAAA,UAAK,CAAC,YACnB,YAAY,YAAY,mBAAmB,MAAM,OAAO;AAAA,QAC1D;AAAA,MACF;AAEA,cAAQ,IAAI,aAAa,EACtB,KAAK,MAAM;AACV,eAAO,QAAQ;AACf,gBAAQ;AAAA,MACV,CAAC,EACA,MAAM,CAAC,UAAU;AAChB,eAAO,QAAQ;AACf,eAAO,KAAK;AAAA,MACd,CAAC;AAAA,IACL,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,UAAU;AAC5B,cAAQ,MAAM,SAAS,KAAK;AAC5B,aAAO,QAAQ;AACf,aAAO,KAAK;AAAA,IACd,CAAC;AAAA,EACH,CAAC;AACH;","names":["Papa"]}
|
package/dist/index.mjs
CHANGED
|
@@ -27,9 +27,10 @@ var csvBatchRead = async (filePath, batchSize, handleHeader, handleBatch) => {
|
|
|
27
27
|
let batchCount = 0;
|
|
28
28
|
let isFirstChunk = true;
|
|
29
29
|
const { promise: headerPromise, resolve: resolveHeaders } = promiseWithResolvers();
|
|
30
|
-
const headerResolved = headerPromise.then(
|
|
31
|
-
|
|
32
|
-
|
|
30
|
+
const headerResolved = headerPromise.then(async (headers) => {
|
|
31
|
+
await handleHeader(headers);
|
|
32
|
+
return headers;
|
|
33
|
+
});
|
|
33
34
|
const shouldResolve = [headerResolved];
|
|
34
35
|
stream.on("data", (chunk) => {
|
|
35
36
|
if (isFirstChunk) {
|
|
@@ -41,7 +42,7 @@ var csvBatchRead = async (filePath, batchSize, handleHeader, handleBatch) => {
|
|
|
41
42
|
const currentBuf = buf.slice();
|
|
42
43
|
shouldResolve.push(
|
|
43
44
|
headerResolved.then(
|
|
44
|
-
() => handleBatch(currentBuf, currentBatchCount, false)
|
|
45
|
+
(headers) => handleBatch(currentBuf, currentBatchCount, false, headers)
|
|
45
46
|
)
|
|
46
47
|
);
|
|
47
48
|
buf = [];
|
|
@@ -54,7 +55,7 @@ var csvBatchRead = async (filePath, batchSize, handleHeader, handleBatch) => {
|
|
|
54
55
|
const currentBuf = buf.slice();
|
|
55
56
|
shouldResolve.push(
|
|
56
57
|
headerResolved.then(
|
|
57
|
-
() => handleBatch(currentBuf, currentBatchCount, true)
|
|
58
|
+
(headers) => handleBatch(currentBuf, currentBatchCount, true, headers)
|
|
58
59
|
)
|
|
59
60
|
);
|
|
60
61
|
Promise.all(shouldResolve).then(() => {
|
package/dist/index.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/csv-batch-read.ts","../src/promise-with-resolvers.ts"],"sourcesContent":["import { createReadStream } from \"node:fs\";\nimport Papa from \"papaparse\";\nimport { promiseWithResolvers } from \"./promise-with-resolvers.js\";\n\nexport const csvBatchRead = async <\n T extends Record<string, string> = Record<string, string>,\n>(\n filePath: string,\n batchSize: number,\n handleHeader: (header: (keyof T)[]) => unknown,\n handleBatch: (rows: T[]
|
|
1
|
+
{"version":3,"sources":["../src/csv-batch-read.ts","../src/promise-with-resolvers.ts"],"sourcesContent":["import { createReadStream } from \"node:fs\";\nimport Papa from \"papaparse\";\nimport { promiseWithResolvers } from \"./promise-with-resolvers.js\";\n\nexport const csvBatchRead = async <\n T extends Record<string, string> = Record<string, string>,\n>(\n filePath: string,\n batchSize: number,\n handleHeader: (header: (keyof T)[]) => unknown,\n handleBatch: (\n rows: T[],\n batchCount: number,\n isLastChunk: boolean,\n header: (keyof T)[],\n ) => unknown,\n) => {\n return await new Promise<void>((resolve, reject) => {\n const stream = createReadStream(filePath).pipe(\n Papa.parse(Papa.NODE_STREAM_INPUT, { header: true }),\n );\n\n let buf: T[] = [];\n let batchCount = 0;\n let isFirstChunk = true;\n const { promise: headerPromise, resolve: resolveHeaders } =\n promiseWithResolvers<(keyof T)[]>();\n const headerResolved = headerPromise.then(async (headers) => {\n await handleHeader(headers);\n return headers;\n });\n const shouldResolve: unknown[] = [headerResolved];\n\n stream.on(\"data\", (chunk: T) => {\n if (isFirstChunk) {\n resolveHeaders(Object.keys(chunk));\n isFirstChunk = false;\n }\n if (buf.length === batchSize) {\n const currentBatchCount = batchCount;\n const currentBuf = buf.slice();\n shouldResolve.push(\n headerResolved.then((headers) =>\n handleBatch(currentBuf, currentBatchCount, false, headers),\n ),\n );\n buf = [];\n batchCount++;\n }\n buf.push(chunk);\n });\n\n stream.on(\"end\", () => {\n const currentBatchCount = batchCount;\n const currentBuf = buf.slice();\n shouldResolve.push(\n headerResolved.then((headers) =>\n handleBatch(currentBuf, currentBatchCount, true, headers),\n ),\n );\n\n Promise.all(shouldResolve)\n .then(() => {\n stream.destroy();\n resolve();\n })\n .catch((error) => {\n stream.destroy();\n reject(error);\n });\n });\n\n stream.on(\"error\", (error) => {\n console.error(\"error\", error);\n stream.destroy();\n reject(error);\n });\n });\n};\n","export const promiseWithResolvers = <T = void>() => {\n // new Promise のコンストラクタ自体は非同期処理を行わないため、\n // resolve, reject への代入は promise の生成が終わったタイミングで完了している\n let resolve: (value: T | PromiseLike<T>) => void = () => undefined;\n let reject: (reason?: unknown) => void = () => undefined;\n const promise = new Promise<T>((res, rej) => {\n resolve = res;\n reject = rej;\n });\n return {\n promise,\n resolve,\n reject,\n };\n};\n"],"mappings":";AAAA,SAAS,wBAAwB;AACjC,OAAO,UAAU;;;ACDV,IAAM,uBAAuB,MAAgB;AAGlD,MAAI,UAA+C,MAAM;AACzD,MAAI,SAAqC,MAAM;AAC/C,QAAM,UAAU,IAAI,QAAW,CAAC,KAAK,QAAQ;AAC3C,cAAU;AACV,aAAS;AAAA,EACX,CAAC;AACD,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;;;ADVO,IAAM,eAAe,OAG1B,UACA,WACA,cACA,gBAMG;AACH,SAAO,MAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAClD,UAAM,SAAS,iBAAiB,QAAQ,EAAE;AAAA,MACxC,KAAK,MAAM,KAAK,mBAAmB,EAAE,QAAQ,KAAK,CAAC;AAAA,IACrD;AAEA,QAAI,MAAW,CAAC;AAChB,QAAI,aAAa;AACjB,QAAI,eAAe;AACnB,UAAM,EAAE,SAAS,eAAe,SAAS,eAAe,IACtD,qBAAkC;AACpC,UAAM,iBAAiB,cAAc,KAAK,OAAO,YAAY;AAC3D,YAAM,aAAa,OAAO;AAC1B,aAAO;AAAA,IACT,CAAC;AACD,UAAM,gBAA2B,CAAC,cAAc;AAEhD,WAAO,GAAG,QAAQ,CAAC,UAAa;AAC9B,UAAI,cAAc;AAChB,uBAAe,OAAO,KAAK,KAAK,CAAC;AACjC,uBAAe;AAAA,MACjB;AACA,UAAI,IAAI,WAAW,WAAW;AAC5B,cAAM,oBAAoB;AAC1B,cAAM,aAAa,IAAI,MAAM;AAC7B,sBAAc;AAAA,UACZ,eAAe;AAAA,YAAK,CAAC,YACnB,YAAY,YAAY,mBAAmB,OAAO,OAAO;AAAA,UAC3D;AAAA,QACF;AACA,cAAM,CAAC;AACP;AAAA,MACF;AACA,UAAI,KAAK,KAAK;AAAA,IAChB,CAAC;AAED,WAAO,GAAG,OAAO,MAAM;AACrB,YAAM,oBAAoB;AAC1B,YAAM,aAAa,IAAI,MAAM;AAC7B,oBAAc;AAAA,QACZ,eAAe;AAAA,UAAK,CAAC,YACnB,YAAY,YAAY,mBAAmB,MAAM,OAAO;AAAA,QAC1D;AAAA,MACF;AAEA,cAAQ,IAAI,aAAa,EACtB,KAAK,MAAM;AACV,eAAO,QAAQ;AACf,gBAAQ;AAAA,MACV,CAAC,EACA,MAAM,CAAC,UAAU;AAChB,eAAO,QAAQ;AACf,eAAO,KAAK;AAAA,MACd,CAAC;AAAA,IACL,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,UAAU;AAC5B,cAAQ,MAAM,SAAS,KAAK;AAC5B,aAAO,QAAQ;AACf,aAAO,KAAK;AAAA,IACd,CAAC;AAAA,EACH,CAAC;AACH;","names":[]}
|
package/package.json
CHANGED
package/src/csv-batch-read.ts
CHANGED
|
@@ -8,7 +8,12 @@ export const csvBatchRead = async <
|
|
|
8
8
|
filePath: string,
|
|
9
9
|
batchSize: number,
|
|
10
10
|
handleHeader: (header: (keyof T)[]) => unknown,
|
|
11
|
-
handleBatch: (
|
|
11
|
+
handleBatch: (
|
|
12
|
+
rows: T[],
|
|
13
|
+
batchCount: number,
|
|
14
|
+
isLastChunk: boolean,
|
|
15
|
+
header: (keyof T)[],
|
|
16
|
+
) => unknown,
|
|
12
17
|
) => {
|
|
13
18
|
return await new Promise<void>((resolve, reject) => {
|
|
14
19
|
const stream = createReadStream(filePath).pipe(
|
|
@@ -20,9 +25,10 @@ export const csvBatchRead = async <
|
|
|
20
25
|
let isFirstChunk = true;
|
|
21
26
|
const { promise: headerPromise, resolve: resolveHeaders } =
|
|
22
27
|
promiseWithResolvers<(keyof T)[]>();
|
|
23
|
-
const headerResolved = headerPromise.then((headers) =>
|
|
24
|
-
handleHeader(headers)
|
|
25
|
-
|
|
28
|
+
const headerResolved = headerPromise.then(async (headers) => {
|
|
29
|
+
await handleHeader(headers);
|
|
30
|
+
return headers;
|
|
31
|
+
});
|
|
26
32
|
const shouldResolve: unknown[] = [headerResolved];
|
|
27
33
|
|
|
28
34
|
stream.on("data", (chunk: T) => {
|
|
@@ -34,8 +40,8 @@ export const csvBatchRead = async <
|
|
|
34
40
|
const currentBatchCount = batchCount;
|
|
35
41
|
const currentBuf = buf.slice();
|
|
36
42
|
shouldResolve.push(
|
|
37
|
-
headerResolved.then(() =>
|
|
38
|
-
handleBatch(currentBuf, currentBatchCount, false),
|
|
43
|
+
headerResolved.then((headers) =>
|
|
44
|
+
handleBatch(currentBuf, currentBatchCount, false, headers),
|
|
39
45
|
),
|
|
40
46
|
);
|
|
41
47
|
buf = [];
|
|
@@ -48,8 +54,8 @@ export const csvBatchRead = async <
|
|
|
48
54
|
const currentBatchCount = batchCount;
|
|
49
55
|
const currentBuf = buf.slice();
|
|
50
56
|
shouldResolve.push(
|
|
51
|
-
headerResolved.then(() =>
|
|
52
|
-
handleBatch(currentBuf, currentBatchCount, true),
|
|
57
|
+
headerResolved.then((headers) =>
|
|
58
|
+
handleBatch(currentBuf, currentBatchCount, true, headers),
|
|
53
59
|
),
|
|
54
60
|
);
|
|
55
61
|
|