@0xtorch/csv 0.0.41 → 0.0.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -2,11 +2,9 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.streamCsvToActions = void 0;
|
|
4
4
|
const papaparse_1 = require("papaparse");
|
|
5
|
-
const zod_1 = require("zod");
|
|
6
5
|
const formatCsvRows_1 = require("./formatCsvRows");
|
|
7
6
|
const parseRowsToActions_1 = require("./parseRowsToActions");
|
|
8
7
|
const key_1 = require("./parseRowsToActions/key");
|
|
9
|
-
const recordSchema = zod_1.z.record(zod_1.z.string());
|
|
10
8
|
const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, save, chunkSize = 100, }) => {
|
|
11
9
|
let success = true;
|
|
12
10
|
let error;
|
|
@@ -15,18 +13,43 @@ const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, sav
|
|
|
15
13
|
let currentKeyValue;
|
|
16
14
|
let currentRows = [];
|
|
17
15
|
let actions = [];
|
|
16
|
+
let formatIndexRecord;
|
|
18
17
|
await new Promise((resolve) => {
|
|
19
18
|
(0, papaparse_1.parse)(file, {
|
|
20
|
-
header:
|
|
19
|
+
header: false,
|
|
21
20
|
skipEmptyLines: true,
|
|
22
21
|
encoding,
|
|
23
22
|
complete: () => {
|
|
24
23
|
resolve();
|
|
25
24
|
},
|
|
26
25
|
step: async (result, parser) => {
|
|
27
|
-
|
|
26
|
+
if (!Array.isArray(result.data)) {
|
|
27
|
+
console.log('unexpected data:', result.data);
|
|
28
|
+
parser.abort();
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
if (formatIndexRecord === undefined) {
|
|
32
|
+
const formatIndexRecordTemp = {};
|
|
33
|
+
for (const key of Object.keys(format.formatter)) {
|
|
34
|
+
const index = result.data.indexOf(key);
|
|
35
|
+
if (index === -1) {
|
|
36
|
+
console.log('key not found:', key);
|
|
37
|
+
parser.abort();
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
formatIndexRecordTemp[key] = index;
|
|
41
|
+
}
|
|
42
|
+
formatIndexRecord = formatIndexRecordTemp;
|
|
43
|
+
console.log('formatIndexRecord:');
|
|
44
|
+
console.log(formatIndexRecord);
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
28
47
|
rowCount += 1;
|
|
29
|
-
const row =
|
|
48
|
+
const row = {};
|
|
49
|
+
for (const key of Object.keys(format.formatter)) {
|
|
50
|
+
const index = formatIndexRecord[key];
|
|
51
|
+
row[key] = result.data[index];
|
|
52
|
+
}
|
|
30
53
|
const formattedRow = formatRow(row, format.formatter);
|
|
31
54
|
if (formattedRow === undefined) {
|
|
32
55
|
console.log('ignore:', row);
|
|
@@ -75,6 +98,7 @@ const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, sav
|
|
|
75
98
|
currentRows.push(formattedRow);
|
|
76
99
|
}
|
|
77
100
|
if (actions.length >= chunkSize + 1) {
|
|
101
|
+
parser.pause();
|
|
78
102
|
const lastActionSource = actions[actions.length - 1].source;
|
|
79
103
|
const savingActions = [];
|
|
80
104
|
const remainingActions = [];
|
|
@@ -95,8 +119,8 @@ const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, sav
|
|
|
95
119
|
error = e;
|
|
96
120
|
parser.abort();
|
|
97
121
|
}
|
|
122
|
+
parser.resume();
|
|
98
123
|
}
|
|
99
|
-
parser.resume();
|
|
100
124
|
},
|
|
101
125
|
});
|
|
102
126
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamCsvToActions.js","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":";;;AACA,yCAAiC;AACjC,
|
|
1
|
+
{"version":3,"file":"streamCsvToActions.js","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":";;;AACA,yCAAiC;AACjC,mDAA8C;AAC9C,6DAA+D;AAC/D,kDAAyD;AAoBlD,MAAM,kBAAkB,GAAG,KAAK,EAAE,EACvC,IAAI,EACJ,QAAQ,EACR,MAAM,EACN,QAAQ,EACR,KAAK,EACL,IAAI,EACJ,SAAS,GAAG,GAAG,GACc,EAA0C,EAAE;IAEzE,IAAI,OAAO,GAAG,IAAI,CAAA;IAClB,IAAI,KAAc,CAAA;IAClB,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,eAAmC,CAAA;IACvC,IAAI,WAAW,GAAmB,EAAE,CAAA;IACpC,IAAI,OAAO,GAAa,EAAE,CAAA;IAE1B,IAAI,iBAAqD,CAAA;IACzD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,IAAA,iBAAK,EAAC,IAAI,EAAE;YACV,MAAM,EAAE,KAAK;YACb,cAAc,EAAE,IAAI;YACpB,QAAQ;YACR,QAAQ,EAAE,GAAG,EAAE;gBACb,OAAO,EAAE,CAAA;YACX,CAAC;YACD,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;gBAC7B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChC,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;oBAC5C,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,OAAM;gBACR,CAAC;gBACD,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACpC,MAAM,qBAAqB,GAA2B,EAAE,CAAA;oBACxD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;wBAChD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;wBACtC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;4BACjB,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAA;4BAClC,MAAM,CAAC,KAAK,EAAE,CAAA;4BACd,OAAM;wBACR,CAAC;wBACD,qBAAqB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;oBACpC,CAAC;oBACD,iBAAiB,GAAG,qBAAqB,CAAA;oBACzC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAA;oBACjC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAA;oBAC9B,OAAM;gBACR,CAAC;gBAED,QAAQ,IAAI,CAAC,CAAA;gBAGb,MAAM,GAAG,GAA2B,EAAE,CAAA;gBACtC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;oBAChD,MAAM,KAAK,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAA;oBACpC,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;gBAC/B,CAAC;gBACD,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,SAAS,CAAC,CAAA;gBACrD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;oBAC/B,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;oBAC3B,cAAc,IAAI,CAAC,CAAA;oBACnB,OAAM;gBACR,CAAC;gBAGD,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;oBACpC,WAAW,GAAG,EAAE,CAAA;oBAChB,MAAM,UAAU,GAAG,IAAA,6CAAwB,EAAC;wBAC1C,IAAI,EAAE,CAAC,YAAY,CAAC;wBACpB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;wBAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;wBACrC,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ;wBACR,KAAK;qBACN,CAAC,CAAA;oBACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBAC5B,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;wBAC3B,cAAc,IAAI,CAAC,CAAA;oBACrB,CAAC;oBACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;gBAC7B,CAAC;qBAAM,CAAC;oBACN,MAAM,QAAQ,GAAG,IAAA,oBAAc,EAAC;wBAC9B,GAAG,EAAE,YAAY;wBACjB,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG;qBACvB,CAAC,CAAA;oBACF,IACE,eAAe,KAAK,SAAS;wBAC7B,eAAe,KAAK,QAAQ;wBAC5B,WAAW,CAAC,MAAM,GAAG,CAAC,EACtB,CAAC;wBACD,MAAM,UAAU,GAAG,IAAA,6CAAwB,EAAC;4BAC1C,IAAI,EAAE,WAAW;4BACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;4BAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;4BACrC,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,QAAQ;4BACR,KAAK;yBACN,CAAC,CAAA;wBACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BAC5B,cAAc,IAAI,WAAW,CAAC,MAAM,CAAA;wBACtC,CAAC;wBACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;wBAC3B,WAAW,GAAG,EAAE,CAAA;oBAClB,CAAC;oBACD,eAAe,GAAG,QAAQ,CAAA;oBAC1B,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAChC,CAAC;gBAGD,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;oBACpC,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,MAAM,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAA;oBAC3D,MAAM,aAAa,GAAa,EAAE,CAAA;oBAClC,MAAM,gBAAgB,GAAa,EAAE,CAAA;oBACrC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;wBAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,gBAAgB,EAAE,CAAC;4BACvC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;wBAC/B,CAAC;6BAAM,CAAC;4BACN,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;wBAC5B,CAAC;oBACH,CAAC;oBACD,OAAO,GAAG,gBAAgB,CAAA;oBAC1B,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAA;oBACrC,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,OAAO,GAAG,KAAK,CAAA;wBACf,KAAK,GAAG,CAAC,CAAA;wBACT,MAAM,CAAC,KAAK,EAAE,CAAA;oBAChB,CAAC;oBACD,MAAM,CAAC,MAAM,EAAE,CAAA;gBACjB,CAAC;YACH,CAAC;SACF,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK;YACL,QAAQ;YACR,cAAc;SACf,CAAA;IACH,CAAC;IAGD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,IAAA,6CAAwB,EAAC;YAC1C,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;YAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,QAAQ;YACR,KAAK;SACN,CAAC,CAAA;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,cAAc,IAAI,WAAW,CAAC,MAAM,CAAA;QACtC,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;IAC7B,CAAC;IAGD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAC/B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,GAAG,KAAK,CAAA;YACf,KAAK,GAAG,CAAC,CAAA;QACX,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO;QACP,KAAK;QACL,QAAQ;QACR,cAAc;KACf,CAAA;AACH,CAAC,CAAA;AAjLY,QAAA,kBAAkB,sBAiL9B;AAED,MAAM,SAAS,GAAG,CAChB,GAA2B,EAC3B,SAAiC,EACP,EAAE;IAC5B,IAAI,CAAC;QACH,OAAO,IAAA,4BAAY,EAAC,GAAG,EAAE,SAAS,CAAC,CAAA;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,SAAS,CAAA;IAClB,CAAC;AACH,CAAC,CAAA"}
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
import { parse } from 'papaparse';
|
|
2
|
-
import { z } from 'zod';
|
|
3
2
|
import { formatCsvRow } from './formatCsvRows';
|
|
4
3
|
import { parseTargetRowsToActions } from './parseRowsToActions';
|
|
5
4
|
import { createKeyValue } from './parseRowsToActions/key';
|
|
6
|
-
const recordSchema = z.record(z.string());
|
|
7
5
|
export const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, save, chunkSize = 100, }) => {
|
|
8
6
|
// stream で処理
|
|
9
7
|
let success = true;
|
|
@@ -13,19 +11,44 @@ export const streamCsvToActions = async ({ file, encoding, format, cryptoes, fia
|
|
|
13
11
|
let currentKeyValue;
|
|
14
12
|
let currentRows = [];
|
|
15
13
|
let actions = [];
|
|
14
|
+
let formatIndexRecord;
|
|
16
15
|
await new Promise((resolve) => {
|
|
17
16
|
parse(file, {
|
|
18
|
-
header:
|
|
17
|
+
header: false,
|
|
19
18
|
skipEmptyLines: true,
|
|
20
19
|
encoding,
|
|
21
20
|
complete: () => {
|
|
22
21
|
resolve();
|
|
23
22
|
},
|
|
24
23
|
step: async (result, parser) => {
|
|
25
|
-
|
|
24
|
+
if (!Array.isArray(result.data)) {
|
|
25
|
+
console.log('unexpected data:', result.data);
|
|
26
|
+
parser.abort();
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
if (formatIndexRecord === undefined) {
|
|
30
|
+
const formatIndexRecordTemp = {};
|
|
31
|
+
for (const key of Object.keys(format.formatter)) {
|
|
32
|
+
const index = result.data.indexOf(key);
|
|
33
|
+
if (index === -1) {
|
|
34
|
+
console.log('key not found:', key);
|
|
35
|
+
parser.abort();
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
formatIndexRecordTemp[key] = index;
|
|
39
|
+
}
|
|
40
|
+
formatIndexRecord = formatIndexRecordTemp;
|
|
41
|
+
console.log('formatIndexRecord:');
|
|
42
|
+
console.log(formatIndexRecord);
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
26
45
|
rowCount += 1;
|
|
27
46
|
// format
|
|
28
|
-
const row =
|
|
47
|
+
const row = {};
|
|
48
|
+
for (const key of Object.keys(format.formatter)) {
|
|
49
|
+
const index = formatIndexRecord[key];
|
|
50
|
+
row[key] = result.data[index];
|
|
51
|
+
}
|
|
29
52
|
const formattedRow = formatRow(row, format.formatter);
|
|
30
53
|
if (formattedRow === undefined) {
|
|
31
54
|
console.log('ignore:', row);
|
|
@@ -76,6 +99,7 @@ export const streamCsvToActions = async ({ file, encoding, format, cryptoes, fia
|
|
|
76
99
|
}
|
|
77
100
|
// actions が chunkSize に達したら最後の action source と同一 source を除いて actions を保存
|
|
78
101
|
if (actions.length >= chunkSize + 1) {
|
|
102
|
+
parser.pause();
|
|
79
103
|
const lastActionSource = actions[actions.length - 1].source;
|
|
80
104
|
const savingActions = [];
|
|
81
105
|
const remainingActions = [];
|
|
@@ -96,8 +120,8 @@ export const streamCsvToActions = async ({ file, encoding, format, cryptoes, fia
|
|
|
96
120
|
error = e;
|
|
97
121
|
parser.abort();
|
|
98
122
|
}
|
|
123
|
+
parser.resume();
|
|
99
124
|
}
|
|
100
|
-
parser.resume();
|
|
101
125
|
},
|
|
102
126
|
});
|
|
103
127
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamCsvToActions.js","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AACjC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"streamCsvToActions.js","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAA;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAoBzD,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EAAE,EACvC,IAAI,EACJ,QAAQ,EACR,MAAM,EACN,QAAQ,EACR,KAAK,EACL,IAAI,EACJ,SAAS,GAAG,GAAG,GACc,EAA0C,EAAE;IACzE,aAAa;IACb,IAAI,OAAO,GAAG,IAAI,CAAA;IAClB,IAAI,KAAc,CAAA;IAClB,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,eAAmC,CAAA;IACvC,IAAI,WAAW,GAAmB,EAAE,CAAA;IACpC,IAAI,OAAO,GAAa,EAAE,CAAA;IAE1B,IAAI,iBAAqD,CAAA;IACzD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,KAAK,CAAC,IAAI,EAAE;YACV,MAAM,EAAE,KAAK;YACb,cAAc,EAAE,IAAI;YACpB,QAAQ;YACR,QAAQ,EAAE,GAAG,EAAE;gBACb,OAAO,EAAE,CAAA;YACX,CAAC;YACD,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;gBAC7B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChC,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;oBAC5C,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,OAAM;gBACR,CAAC;gBACD,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACpC,MAAM,qBAAqB,GAA2B,EAAE,CAAA;oBACxD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;wBAChD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;wBACtC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;4BACjB,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAA;4BAClC,MAAM,CAAC,KAAK,EAAE,CAAA;4BACd,OAAM;wBACR,CAAC;wBACD,qBAAqB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;oBACpC,CAAC;oBACD,iBAAiB,GAAG,qBAAqB,CAAA;oBACzC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAA;oBACjC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAA;oBAC9B,OAAM;gBACR,CAAC;gBAED,QAAQ,IAAI,CAAC,CAAA;gBAEb,SAAS;gBACT,MAAM,GAAG,GAA2B,EAAE,CAAA;gBACtC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;oBAChD,MAAM,KAAK,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAA;oBACpC,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;gBAC/B,CAAC;gBACD,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,SAAS,CAAC,CAAA;gBACrD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;oBAC/B,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;oBAC3B,cAAc,IAAI,CAAC,CAAA;oBACnB,OAAM;gBACR,CAAC;gBAED,QAAQ;gBACR,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;oBACpC,WAAW,GAAG,EAAE,CAAA;oBAChB,MAAM,UAAU,GAAG,wBAAwB,CAAC;wBAC1C,IAAI,EAAE,CAAC,YAAY,CAAC;wBACpB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;wBAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;wBACrC,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ;wBACR,KAAK;qBACN,CAAC,CAAA;oBACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBAC5B,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;wBAC3B,cAAc,IAAI,CAAC,CAAA;oBACrB,CAAC;oBACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;gBAC7B,CAAC;qBAAM,CAAC;oBACN,MAAM,QAAQ,GAAG,cAAc,CAAC;wBAC9B,GAAG,EAAE,YAAY;wBACjB,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG;qBACvB,CAAC,CAAA;oBACF,IACE,eAAe,KAAK,SAAS;wBAC7B,eAAe,KAAK,QAAQ;wBAC5B,WAAW,CAAC,MAAM,GAAG,CAAC,EACtB,CAAC;wBACD,MAAM,UAAU,GAAG,wBAAwB,CAAC;4BAC1C,IAAI,EAAE,WAAW;4BACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;4BAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;4BACrC,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,QAAQ;4BACR,KAAK;yBACN,CAAC,CAAA;wBACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BAC5B,cAAc,IAAI,WAAW,CAAC,MAAM,CAAA;wBACtC,CAAC;wBACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;wBAC3B,WAAW,GAAG,EAAE,CAAA;oBAClB,CAAC;oBACD,eAAe,GAAG,QAAQ,CAAA;oBAC1B,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAChC,CAAC;gBAED,yEAAyE;gBACzE,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;oBACpC,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,MAAM,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAA;oBAC3D,MAAM,aAAa,GAAa,EAAE,CAAA;oBAClC,MAAM,gBAAgB,GAAa,EAAE,CAAA;oBACrC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;wBAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,gBAAgB,EAAE,CAAC;4BACvC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;wBAC/B,CAAC;6BAAM,CAAC;4BACN,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;wBAC5B,CAAC;oBACH,CAAC;oBACD,OAAO,GAAG,gBAAgB,CAAA;oBAC1B,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAA;oBACrC,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,OAAO,GAAG,KAAK,CAAA;wBACf,KAAK,GAAG,CAAC,CAAA;wBACT,MAAM,CAAC,KAAK,EAAE,CAAA;oBAChB,CAAC;oBACD,MAAM,CAAC,MAAM,EAAE,CAAA;gBACjB,CAAC;YACH,CAAC;SACF,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK;YACL,QAAQ;YACR,cAAc;SACf,CAAA;IACH,CAAC;IAED,kCAAkC;IAClC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,wBAAwB,CAAC;YAC1C,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;YAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,QAAQ;YACR,KAAK;SACN,CAAC,CAAA;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,cAAc,IAAI,WAAW,CAAC,MAAM,CAAA;QACtC,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;IAC7B,CAAC;IAED,+BAA+B;IAC/B,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAC/B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,GAAG,KAAK,CAAA;YACf,KAAK,GAAG,CAAC,CAAA;QACX,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO;QACP,KAAK;QACL,QAAQ;QACR,cAAc;KACf,CAAA;AACH,CAAC,CAAA;AAED,MAAM,SAAS,GAAG,CAChB,GAA2B,EAC3B,SAAiC,EACP,EAAE;IAC5B,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,GAAG,EAAE,SAAS,CAAC,CAAA;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,SAAS,CAAA;IAClB,CAAC;AACH,CAAC,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamCsvToActions.d.ts","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"streamCsvToActions.d.ts","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAKzE,OAAO,KAAK,EAAE,SAAS,EAAgB,MAAM,SAAS,CAAA;AAEtD,KAAK,4BAA4B,GAAG;IAClC,IAAI,EAAE,IAAI,GAAG,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,SAAS,CAAA;IACjB,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAA;IACnC,KAAK,EAAE,SAAS,YAAY,EAAE,CAAA;IAC9B,IAAI,EAAE,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IACrE,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,CAAA;AAED,KAAK,6BAA6B,GAAG;IACnC,OAAO,EAAE,OAAO,CAAA;IAChB,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;IAChB,cAAc,EAAE,MAAM,CAAA;CACvB,CAAA;AAED,eAAO,MAAM,kBAAkB,kEAQ5B,4BAA4B,KAAG,OAAO,CAAC,6BAA6B,CAyKtE,CAAA"}
|
package/package.json
CHANGED
package/streamCsvToActions.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import type { Action, CryptoCurrency, FiatCurrency } from '@0xtorch/core'
|
|
2
2
|
import { parse } from 'papaparse'
|
|
3
|
-
import { z } from 'zod'
|
|
4
3
|
import { formatCsvRow } from './formatCsvRows'
|
|
5
4
|
import { parseTargetRowsToActions } from './parseRowsToActions'
|
|
6
5
|
import { createKeyValue } from './parseRowsToActions/key'
|
|
@@ -16,8 +15,6 @@ type StreamCsvToActionsParameters = {
|
|
|
16
15
|
chunkSize?: number
|
|
17
16
|
}
|
|
18
17
|
|
|
19
|
-
const recordSchema = z.record(z.string())
|
|
20
|
-
|
|
21
18
|
type StreamCsvToActionsReturnTypes = {
|
|
22
19
|
success: boolean
|
|
23
20
|
error?: unknown
|
|
@@ -43,20 +40,46 @@ export const streamCsvToActions = async ({
|
|
|
43
40
|
let currentRows: FormattedRow[] = []
|
|
44
41
|
let actions: Action[] = []
|
|
45
42
|
|
|
43
|
+
let formatIndexRecord: Record<string, number> | undefined
|
|
46
44
|
await new Promise<void>((resolve) => {
|
|
47
45
|
parse(file, {
|
|
48
|
-
header:
|
|
46
|
+
header: false,
|
|
49
47
|
skipEmptyLines: true,
|
|
50
48
|
encoding,
|
|
51
49
|
complete: () => {
|
|
52
50
|
resolve()
|
|
53
51
|
},
|
|
54
52
|
step: async (result, parser) => {
|
|
55
|
-
|
|
53
|
+
if (!Array.isArray(result.data)) {
|
|
54
|
+
console.log('unexpected data:', result.data)
|
|
55
|
+
parser.abort()
|
|
56
|
+
return
|
|
57
|
+
}
|
|
58
|
+
if (formatIndexRecord === undefined) {
|
|
59
|
+
const formatIndexRecordTemp: Record<string, number> = {}
|
|
60
|
+
for (const key of Object.keys(format.formatter)) {
|
|
61
|
+
const index = result.data.indexOf(key)
|
|
62
|
+
if (index === -1) {
|
|
63
|
+
console.log('key not found:', key)
|
|
64
|
+
parser.abort()
|
|
65
|
+
return
|
|
66
|
+
}
|
|
67
|
+
formatIndexRecordTemp[key] = index
|
|
68
|
+
}
|
|
69
|
+
formatIndexRecord = formatIndexRecordTemp
|
|
70
|
+
console.log('formatIndexRecord:')
|
|
71
|
+
console.log(formatIndexRecord)
|
|
72
|
+
return
|
|
73
|
+
}
|
|
74
|
+
|
|
56
75
|
rowCount += 1
|
|
57
76
|
|
|
58
77
|
// format
|
|
59
|
-
const row =
|
|
78
|
+
const row: Record<string, string> = {}
|
|
79
|
+
for (const key of Object.keys(format.formatter)) {
|
|
80
|
+
const index = formatIndexRecord[key]
|
|
81
|
+
row[key] = result.data[index]
|
|
82
|
+
}
|
|
60
83
|
const formattedRow = formatRow(row, format.formatter)
|
|
61
84
|
if (formattedRow === undefined) {
|
|
62
85
|
console.log('ignore:', row)
|
|
@@ -110,6 +133,7 @@ export const streamCsvToActions = async ({
|
|
|
110
133
|
|
|
111
134
|
// actions が chunkSize に達したら最後の action source と同一 source を除いて actions を保存
|
|
112
135
|
if (actions.length >= chunkSize + 1) {
|
|
136
|
+
parser.pause()
|
|
113
137
|
const lastActionSource = actions[actions.length - 1].source
|
|
114
138
|
const savingActions: Action[] = []
|
|
115
139
|
const remainingActions: Action[] = []
|
|
@@ -128,8 +152,8 @@ export const streamCsvToActions = async ({
|
|
|
128
152
|
error = e
|
|
129
153
|
parser.abort()
|
|
130
154
|
}
|
|
155
|
+
parser.resume()
|
|
131
156
|
}
|
|
132
|
-
parser.resume()
|
|
133
157
|
},
|
|
134
158
|
})
|
|
135
159
|
})
|