@0xtorch/csv 0.0.37 → 0.0.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/_cjs/hasValidHeaderRow.js +20 -19
- package/_cjs/hasValidHeaderRow.js.map +1 -1
- package/_cjs/streamCsvToActions.js +68 -63
- package/_cjs/streamCsvToActions.js.map +1 -1
- package/_esm/hasValidHeaderRow.js +20 -19
- package/_esm/hasValidHeaderRow.js.map +1 -1
- package/_esm/streamCsvToActions.js +71 -66
- package/_esm/streamCsvToActions.js.map +1 -1
- package/_types/hasValidHeaderRow.d.ts.map +1 -1
- package/_types/streamCsvToActions.d.ts.map +1 -1
- package/hasValidHeaderRow.ts +22 -19
- package/package.json +1 -1
- package/streamCsvToActions.ts +72 -66
|
@@ -4,29 +4,30 @@ exports.hasValidHeaderRow = void 0;
|
|
|
4
4
|
const papaparse_1 = require("papaparse");
|
|
5
5
|
const hasValidHeaderRow = async ({ file, encoding, format, }) => {
|
|
6
6
|
let isValid = false;
|
|
7
|
-
(
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
}
|
|
18
|
-
for (const key of Object.keys(format.formatter)) {
|
|
19
|
-
if (!result.data.includes(key)) {
|
|
20
|
-
console.log(`Missing key: ${key}`);
|
|
7
|
+
await new Promise((resolve) => {
|
|
8
|
+
(0, papaparse_1.parse)(file, {
|
|
9
|
+
header: false,
|
|
10
|
+
skipEmptyLines: true,
|
|
11
|
+
encoding,
|
|
12
|
+
complete: () => {
|
|
13
|
+
resolve();
|
|
14
|
+
},
|
|
15
|
+
step: (result, parser) => {
|
|
16
|
+
if (!Array.isArray(result.data)) {
|
|
21
17
|
parser.abort();
|
|
22
18
|
return;
|
|
23
19
|
}
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
20
|
+
for (const key of Object.keys(format.formatter)) {
|
|
21
|
+
if (!result.data.includes(key)) {
|
|
22
|
+
parser.abort();
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
isValid = true;
|
|
27
|
+
parser.abort();
|
|
28
|
+
},
|
|
29
|
+
});
|
|
28
30
|
});
|
|
29
|
-
console.log('isValid', isValid);
|
|
30
31
|
return isValid;
|
|
31
32
|
};
|
|
32
33
|
exports.hasValidHeaderRow = hasValidHeaderRow;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hasValidHeaderRow.js","sourceRoot":"","sources":["../hasValidHeaderRow.ts"],"names":[],"mappings":";;;AAAA,yCAAiC;AAS1B,MAAM,iBAAiB,GAAG,KAAK,EAAE,EACtC,IAAI,EACJ,QAAQ,EACR,MAAM,GACsB,EAAE,EAAE;IAChC,IAAI,OAAO,GAAG,KAAK,CAAA;
|
|
1
|
+
{"version":3,"file":"hasValidHeaderRow.js","sourceRoot":"","sources":["../hasValidHeaderRow.ts"],"names":[],"mappings":";;;AAAA,yCAAiC;AAS1B,MAAM,iBAAiB,GAAG,KAAK,EAAE,EACtC,IAAI,EACJ,QAAQ,EACR,MAAM,GACsB,EAAE,EAAE;IAChC,IAAI,OAAO,GAAG,KAAK,CAAA;IAEnB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,IAAA,iBAAK,EAAC,IAAI,EAAE;YACV,MAAM,EAAE,KAAK;YACb,cAAc,EAAE,IAAI;YACpB,QAAQ;YACR,QAAQ,EAAE,GAAG,EAAE;gBACb,OAAO,EAAE,CAAA;YACX,CAAC;YACD,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE;gBACvB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChC,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,OAAM;gBACR,CAAC;gBACD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;oBAChD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC/B,MAAM,CAAC,KAAK,EAAE,CAAA;wBACd,OAAM;oBACR,CAAC;gBACH,CAAC;gBACD,OAAO,GAAG,IAAI,CAAA;gBACd,MAAM,CAAC,KAAK,EAAE,CAAA;YAChB,CAAC;SACF,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,OAAO,OAAO,CAAA;AAChB,CAAC,CAAA;AAjCY,QAAA,iBAAiB,qBAiC7B"}
|
|
@@ -15,83 +15,88 @@ const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, sav
|
|
|
15
15
|
let currentKeyValue;
|
|
16
16
|
let currentRows = [];
|
|
17
17
|
let actions = [];
|
|
18
|
-
(
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
if (format.parser.key === undefined) {
|
|
32
|
-
currentRows = [];
|
|
33
|
-
const rowActions = (0, parseRowsToActions_1.parseTargetRowsToActions)({
|
|
34
|
-
rows: [formattedRow],
|
|
35
|
-
parsers: format.parser.parsers,
|
|
36
|
-
symbolAssetMap: format.symbolAssetMap,
|
|
37
|
-
service: format.service,
|
|
38
|
-
cryptoes,
|
|
39
|
-
fiats,
|
|
40
|
-
});
|
|
41
|
-
if (rowActions.length === 0) {
|
|
18
|
+
await new Promise((resolve) => {
|
|
19
|
+
(0, papaparse_1.parse)(file, {
|
|
20
|
+
header: true,
|
|
21
|
+
skipEmptyLines: true,
|
|
22
|
+
encoding,
|
|
23
|
+
complete: () => {
|
|
24
|
+
resolve();
|
|
25
|
+
},
|
|
26
|
+
step: async (result, parsers) => {
|
|
27
|
+
rowCount += 1;
|
|
28
|
+
const row = recordSchema.parse(result.data);
|
|
29
|
+
const formattedRow = formatRow(row, format.formatter);
|
|
30
|
+
if (formattedRow === undefined) {
|
|
42
31
|
console.log('ignore:', row);
|
|
43
32
|
ignoreRowCount += 1;
|
|
33
|
+
return;
|
|
44
34
|
}
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
row: formattedRow,
|
|
50
|
-
key: format.parser.key,
|
|
51
|
-
});
|
|
52
|
-
if (currentKeyValue !== undefined &&
|
|
53
|
-
currentKeyValue !== keyValue &&
|
|
54
|
-
currentRows.length > 0) {
|
|
55
|
-
const keyActions = (0, parseRowsToActions_1.parseTargetRowsToActions)({
|
|
56
|
-
rows: currentRows,
|
|
35
|
+
if (format.parser.key === undefined) {
|
|
36
|
+
currentRows = [];
|
|
37
|
+
const rowActions = (0, parseRowsToActions_1.parseTargetRowsToActions)({
|
|
38
|
+
rows: [formattedRow],
|
|
57
39
|
parsers: format.parser.parsers,
|
|
58
40
|
symbolAssetMap: format.symbolAssetMap,
|
|
59
41
|
service: format.service,
|
|
60
42
|
cryptoes,
|
|
61
43
|
fiats,
|
|
62
44
|
});
|
|
63
|
-
if (
|
|
64
|
-
|
|
45
|
+
if (rowActions.length === 0) {
|
|
46
|
+
console.log('ignore:', row);
|
|
47
|
+
ignoreRowCount += 1;
|
|
65
48
|
}
|
|
66
|
-
actions.push(...
|
|
67
|
-
currentRows = [];
|
|
49
|
+
actions.push(...rowActions);
|
|
68
50
|
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
51
|
+
else {
|
|
52
|
+
const keyValue = (0, key_1.createKeyValue)({
|
|
53
|
+
row: formattedRow,
|
|
54
|
+
key: format.parser.key,
|
|
55
|
+
});
|
|
56
|
+
if (currentKeyValue !== undefined &&
|
|
57
|
+
currentKeyValue !== keyValue &&
|
|
58
|
+
currentRows.length > 0) {
|
|
59
|
+
const keyActions = (0, parseRowsToActions_1.parseTargetRowsToActions)({
|
|
60
|
+
rows: currentRows,
|
|
61
|
+
parsers: format.parser.parsers,
|
|
62
|
+
symbolAssetMap: format.symbolAssetMap,
|
|
63
|
+
service: format.service,
|
|
64
|
+
cryptoes,
|
|
65
|
+
fiats,
|
|
66
|
+
});
|
|
67
|
+
if (keyActions.length === 0) {
|
|
68
|
+
ignoreRowCount += currentRows.length;
|
|
69
|
+
}
|
|
70
|
+
actions.push(...keyActions);
|
|
71
|
+
currentRows = [];
|
|
82
72
|
}
|
|
73
|
+
currentKeyValue = keyValue;
|
|
74
|
+
currentRows.push(formattedRow);
|
|
83
75
|
}
|
|
84
|
-
actions
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
76
|
+
if (actions.length >= chunkSize + 1) {
|
|
77
|
+
const lastActionSource = actions[actions.length - 1].source;
|
|
78
|
+
const savingActions = [];
|
|
79
|
+
const remainingActions = [];
|
|
80
|
+
for (const action of actions) {
|
|
81
|
+
if (action.source === lastActionSource) {
|
|
82
|
+
remainingActions.push(action);
|
|
83
|
+
}
|
|
84
|
+
else {
|
|
85
|
+
savingActions.push(action);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
actions = remainingActions;
|
|
89
|
+
try {
|
|
90
|
+
await save(savingActions);
|
|
91
|
+
}
|
|
92
|
+
catch (e) {
|
|
93
|
+
success = false;
|
|
94
|
+
error = e;
|
|
95
|
+
parsers.abort();
|
|
96
|
+
}
|
|
92
97
|
}
|
|
93
|
-
}
|
|
94
|
-
}
|
|
98
|
+
},
|
|
99
|
+
});
|
|
95
100
|
});
|
|
96
101
|
if (!success) {
|
|
97
102
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamCsvToActions.js","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":";;;AACA,yCAAiC;AACjC,6BAAuB;AACvB,mDAA8C;AAC9C,6DAA+D;AAC/D,kDAAyD;AAazD,MAAM,YAAY,GAAG,OAAC,CAAC,MAAM,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAA;AASlC,MAAM,kBAAkB,GAAG,KAAK,EAAE,EACvC,IAAI,EACJ,QAAQ,EACR,MAAM,EACN,QAAQ,EACR,KAAK,EACL,IAAI,EACJ,SAAS,GAAG,GAAG,GACc,EAA0C,EAAE;IAEzE,IAAI,OAAO,GAAG,IAAI,CAAA;IAClB,IAAI,KAAc,CAAA;IAClB,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,eAAmC,CAAA;IACvC,IAAI,WAAW,GAAmB,EAAE,CAAA;IACpC,IAAI,OAAO,GAAa,EAAE,CAAA;
|
|
1
|
+
{"version":3,"file":"streamCsvToActions.js","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":";;;AACA,yCAAiC;AACjC,6BAAuB;AACvB,mDAA8C;AAC9C,6DAA+D;AAC/D,kDAAyD;AAazD,MAAM,YAAY,GAAG,OAAC,CAAC,MAAM,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAA;AASlC,MAAM,kBAAkB,GAAG,KAAK,EAAE,EACvC,IAAI,EACJ,QAAQ,EACR,MAAM,EACN,QAAQ,EACR,KAAK,EACL,IAAI,EACJ,SAAS,GAAG,GAAG,GACc,EAA0C,EAAE;IAEzE,IAAI,OAAO,GAAG,IAAI,CAAA;IAClB,IAAI,KAAc,CAAA;IAClB,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,eAAmC,CAAA;IACvC,IAAI,WAAW,GAAmB,EAAE,CAAA;IACpC,IAAI,OAAO,GAAa,EAAE,CAAA;IAE1B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,IAAA,iBAAK,EAAC,IAAI,EAAE;YACV,MAAM,EAAE,IAAI;YACZ,cAAc,EAAE,IAAI;YACpB,QAAQ;YACR,QAAQ,EAAE,GAAG,EAAE;gBACb,OAAO,EAAE,CAAA;YACX,CAAC;YACD,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE;gBAC9B,QAAQ,IAAI,CAAC,CAAA;gBAGb,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBAC3C,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,SAAS,CAAC,CAAA;gBACrD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;oBAC/B,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;oBAC3B,cAAc,IAAI,CAAC,CAAA;oBACnB,OAAM;gBACR,CAAC;gBAGD,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;oBACpC,WAAW,GAAG,EAAE,CAAA;oBAChB,MAAM,UAAU,GAAG,IAAA,6CAAwB,EAAC;wBAC1C,IAAI,EAAE,CAAC,YAAY,CAAC;wBACpB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;wBAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;wBACrC,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ;wBACR,KAAK;qBACN,CAAC,CAAA;oBACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBAC5B,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;wBAC3B,cAAc,IAAI,CAAC,CAAA;oBACrB,CAAC;oBACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;gBAC7B,CAAC;qBAAM,CAAC;oBACN,MAAM,QAAQ,GAAG,IAAA,oBAAc,EAAC;wBAC9B,GAAG,EAAE,YAAY;wBACjB,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG;qBACvB,CAAC,CAAA;oBACF,IACE,eAAe,KAAK,SAAS;wBAC7B,eAAe,KAAK,QAAQ;wBAC5B,WAAW,CAAC,MAAM,GAAG,CAAC,EACtB,CAAC;wBACD,MAAM,UAAU,GAAG,IAAA,6CAAwB,EAAC;4BAC1C,IAAI,EAAE,WAAW;4BACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;4BAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;4BACrC,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,QAAQ;4BACR,KAAK;yBACN,CAAC,CAAA;wBACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BAC5B,cAAc,IAAI,WAAW,CAAC,MAAM,CAAA;wBACtC,CAAC;wBACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;wBAC3B,WAAW,GAAG,EAAE,CAAA;oBAClB,CAAC;oBACD,eAAe,GAAG,QAAQ,CAAA;oBAC1B,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAChC,CAAC;gBAGD,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;oBACpC,MAAM,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAA;oBAC3D,MAAM,aAAa,GAAa,EAAE,CAAA;oBAClC,MAAM,gBAAgB,GAAa,EAAE,CAAA;oBACrC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;wBAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,gBAAgB,EAAE,CAAC;4BACvC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;wBAC/B,CAAC;6BAAM,CAAC;4BACN,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;wBAC5B,CAAC;oBACH,CAAC;oBACD,OAAO,GAAG,gBAAgB,CAAA;oBAC1B,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,aAAa,CAAC,CAAA;oBAC3B,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,OAAO,GAAG,KAAK,CAAA;wBACf,KAAK,GAAG,CAAC,CAAA;wBACT,OAAO,CAAC,KAAK,EAAE,CAAA;oBACjB,CAAC;gBACH,CAAC;YACH,CAAC;SACF,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK;YACL,QAAQ;YACR,cAAc;SACf,CAAA;IACH,CAAC;IAGD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,IAAA,6CAAwB,EAAC;YAC1C,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;YAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,QAAQ;YACR,KAAK;SACN,CAAC,CAAA;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,cAAc,IAAI,WAAW,CAAC,MAAM,CAAA;QACtC,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;IAC7B,CAAC;IAGD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,CAAC,CAAA;QACrB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,GAAG,KAAK,CAAA;YACf,KAAK,GAAG,CAAC,CAAA;QACX,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO;QACP,KAAK;QACL,QAAQ;QACR,cAAc;KACf,CAAA;AACH,CAAC,CAAA;AApJY,QAAA,kBAAkB,sBAoJ9B;AAED,MAAM,SAAS,GAAG,CAChB,GAA2B,EAC3B,SAAiC,EACP,EAAE;IAC5B,IAAI,CAAC;QACH,OAAO,IAAA,4BAAY,EAAC,GAAG,EAAE,SAAS,CAAC,CAAA;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,SAAS,CAAA;IAClB,CAAC;AACH,CAAC,CAAA"}
|
|
@@ -1,29 +1,30 @@
|
|
|
1
1
|
import { parse } from 'papaparse';
|
|
2
2
|
export const hasValidHeaderRow = async ({ file, encoding, format, }) => {
|
|
3
3
|
let isValid = false;
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
}
|
|
15
|
-
for (const key of Object.keys(format.formatter)) {
|
|
16
|
-
if (!result.data.includes(key)) {
|
|
17
|
-
console.log(`Missing key: ${key}`);
|
|
4
|
+
await new Promise((resolve) => {
|
|
5
|
+
parse(file, {
|
|
6
|
+
header: false,
|
|
7
|
+
skipEmptyLines: true,
|
|
8
|
+
encoding,
|
|
9
|
+
complete: () => {
|
|
10
|
+
resolve();
|
|
11
|
+
},
|
|
12
|
+
step: (result, parser) => {
|
|
13
|
+
if (!Array.isArray(result.data)) {
|
|
18
14
|
parser.abort();
|
|
19
15
|
return;
|
|
20
16
|
}
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
17
|
+
for (const key of Object.keys(format.formatter)) {
|
|
18
|
+
if (!result.data.includes(key)) {
|
|
19
|
+
parser.abort();
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
isValid = true;
|
|
24
|
+
parser.abort();
|
|
25
|
+
},
|
|
26
|
+
});
|
|
25
27
|
});
|
|
26
|
-
console.log('isValid', isValid);
|
|
27
28
|
return isValid;
|
|
28
29
|
};
|
|
29
30
|
//# sourceMappingURL=hasValidHeaderRow.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hasValidHeaderRow.js","sourceRoot":"","sources":["../hasValidHeaderRow.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AASjC,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,EAAE,EACtC,IAAI,EACJ,QAAQ,EACR,MAAM,GACsB,EAAE,EAAE;IAChC,IAAI,OAAO,GAAG,KAAK,CAAA;
|
|
1
|
+
{"version":3,"file":"hasValidHeaderRow.js","sourceRoot":"","sources":["../hasValidHeaderRow.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AASjC,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,EAAE,EACtC,IAAI,EACJ,QAAQ,EACR,MAAM,GACsB,EAAE,EAAE;IAChC,IAAI,OAAO,GAAG,KAAK,CAAA;IAEnB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,KAAK,CAAC,IAAI,EAAE;YACV,MAAM,EAAE,KAAK;YACb,cAAc,EAAE,IAAI;YACpB,QAAQ;YACR,QAAQ,EAAE,GAAG,EAAE;gBACb,OAAO,EAAE,CAAA;YACX,CAAC;YACD,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE;gBACvB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChC,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,OAAM;gBACR,CAAC;gBACD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;oBAChD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC/B,MAAM,CAAC,KAAK,EAAE,CAAA;wBACd,OAAM;oBACR,CAAC;gBACH,CAAC;gBACD,OAAO,GAAG,IAAI,CAAA;gBACd,MAAM,CAAC,KAAK,EAAE,CAAA;YAChB,CAAC;SACF,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,OAAO,OAAO,CAAA;AAChB,CAAC,CAAA"}
|
|
@@ -13,86 +13,91 @@ export const streamCsvToActions = async ({ file, encoding, format, cryptoes, fia
|
|
|
13
13
|
let currentKeyValue;
|
|
14
14
|
let currentRows = [];
|
|
15
15
|
let actions = [];
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
// parse
|
|
31
|
-
if (format.parser.key === undefined) {
|
|
32
|
-
currentRows = [];
|
|
33
|
-
const rowActions = parseTargetRowsToActions({
|
|
34
|
-
rows: [formattedRow],
|
|
35
|
-
parsers: format.parser.parsers,
|
|
36
|
-
symbolAssetMap: format.symbolAssetMap,
|
|
37
|
-
service: format.service,
|
|
38
|
-
cryptoes,
|
|
39
|
-
fiats,
|
|
40
|
-
});
|
|
41
|
-
if (rowActions.length === 0) {
|
|
16
|
+
await new Promise((resolve) => {
|
|
17
|
+
parse(file, {
|
|
18
|
+
header: true,
|
|
19
|
+
skipEmptyLines: true,
|
|
20
|
+
encoding,
|
|
21
|
+
complete: () => {
|
|
22
|
+
resolve();
|
|
23
|
+
},
|
|
24
|
+
step: async (result, parsers) => {
|
|
25
|
+
rowCount += 1;
|
|
26
|
+
// format
|
|
27
|
+
const row = recordSchema.parse(result.data);
|
|
28
|
+
const formattedRow = formatRow(row, format.formatter);
|
|
29
|
+
if (formattedRow === undefined) {
|
|
42
30
|
console.log('ignore:', row);
|
|
43
31
|
ignoreRowCount += 1;
|
|
32
|
+
return;
|
|
44
33
|
}
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
key: format.parser.key,
|
|
51
|
-
});
|
|
52
|
-
if (currentKeyValue !== undefined &&
|
|
53
|
-
currentKeyValue !== keyValue &&
|
|
54
|
-
currentRows.length > 0) {
|
|
55
|
-
const keyActions = parseTargetRowsToActions({
|
|
56
|
-
rows: currentRows,
|
|
34
|
+
// parse
|
|
35
|
+
if (format.parser.key === undefined) {
|
|
36
|
+
currentRows = [];
|
|
37
|
+
const rowActions = parseTargetRowsToActions({
|
|
38
|
+
rows: [formattedRow],
|
|
57
39
|
parsers: format.parser.parsers,
|
|
58
40
|
symbolAssetMap: format.symbolAssetMap,
|
|
59
41
|
service: format.service,
|
|
60
42
|
cryptoes,
|
|
61
43
|
fiats,
|
|
62
44
|
});
|
|
63
|
-
if (
|
|
64
|
-
|
|
45
|
+
if (rowActions.length === 0) {
|
|
46
|
+
console.log('ignore:', row);
|
|
47
|
+
ignoreRowCount += 1;
|
|
65
48
|
}
|
|
66
|
-
actions.push(...
|
|
67
|
-
currentRows = [];
|
|
49
|
+
actions.push(...rowActions);
|
|
68
50
|
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
51
|
+
else {
|
|
52
|
+
const keyValue = createKeyValue({
|
|
53
|
+
row: formattedRow,
|
|
54
|
+
key: format.parser.key,
|
|
55
|
+
});
|
|
56
|
+
if (currentKeyValue !== undefined &&
|
|
57
|
+
currentKeyValue !== keyValue &&
|
|
58
|
+
currentRows.length > 0) {
|
|
59
|
+
const keyActions = parseTargetRowsToActions({
|
|
60
|
+
rows: currentRows,
|
|
61
|
+
parsers: format.parser.parsers,
|
|
62
|
+
symbolAssetMap: format.symbolAssetMap,
|
|
63
|
+
service: format.service,
|
|
64
|
+
cryptoes,
|
|
65
|
+
fiats,
|
|
66
|
+
});
|
|
67
|
+
if (keyActions.length === 0) {
|
|
68
|
+
ignoreRowCount += currentRows.length;
|
|
69
|
+
}
|
|
70
|
+
actions.push(...keyActions);
|
|
71
|
+
currentRows = [];
|
|
83
72
|
}
|
|
73
|
+
currentKeyValue = keyValue;
|
|
74
|
+
currentRows.push(formattedRow);
|
|
84
75
|
}
|
|
85
|
-
actions
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
76
|
+
// actions が chunkSize に達したら最後の action source と同一 source を除いて actions を保存
|
|
77
|
+
if (actions.length >= chunkSize + 1) {
|
|
78
|
+
const lastActionSource = actions[actions.length - 1].source;
|
|
79
|
+
const savingActions = [];
|
|
80
|
+
const remainingActions = [];
|
|
81
|
+
for (const action of actions) {
|
|
82
|
+
if (action.source === lastActionSource) {
|
|
83
|
+
remainingActions.push(action);
|
|
84
|
+
}
|
|
85
|
+
else {
|
|
86
|
+
savingActions.push(action);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
actions = remainingActions;
|
|
90
|
+
try {
|
|
91
|
+
await save(savingActions);
|
|
92
|
+
}
|
|
93
|
+
catch (e) {
|
|
94
|
+
success = false;
|
|
95
|
+
error = e;
|
|
96
|
+
parsers.abort();
|
|
97
|
+
}
|
|
93
98
|
}
|
|
94
|
-
}
|
|
95
|
-
}
|
|
99
|
+
},
|
|
100
|
+
});
|
|
96
101
|
});
|
|
97
102
|
if (!success) {
|
|
98
103
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamCsvToActions.js","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AACjC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAA;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAazD,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;AASzC,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EAAE,EACvC,IAAI,EACJ,QAAQ,EACR,MAAM,EACN,QAAQ,EACR,KAAK,EACL,IAAI,EACJ,SAAS,GAAG,GAAG,GACc,EAA0C,EAAE;IACzE,aAAa;IACb,IAAI,OAAO,GAAG,IAAI,CAAA;IAClB,IAAI,KAAc,CAAA;IAClB,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,eAAmC,CAAA;IACvC,IAAI,WAAW,GAAmB,EAAE,CAAA;IACpC,IAAI,OAAO,GAAa,EAAE,CAAA;
|
|
1
|
+
{"version":3,"file":"streamCsvToActions.js","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AACjC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAA;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAazD,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;AASzC,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EAAE,EACvC,IAAI,EACJ,QAAQ,EACR,MAAM,EACN,QAAQ,EACR,KAAK,EACL,IAAI,EACJ,SAAS,GAAG,GAAG,GACc,EAA0C,EAAE;IACzE,aAAa;IACb,IAAI,OAAO,GAAG,IAAI,CAAA;IAClB,IAAI,KAAc,CAAA;IAClB,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,eAAmC,CAAA;IACvC,IAAI,WAAW,GAAmB,EAAE,CAAA;IACpC,IAAI,OAAO,GAAa,EAAE,CAAA;IAE1B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,KAAK,CAAC,IAAI,EAAE;YACV,MAAM,EAAE,IAAI;YACZ,cAAc,EAAE,IAAI;YACpB,QAAQ;YACR,QAAQ,EAAE,GAAG,EAAE;gBACb,OAAO,EAAE,CAAA;YACX,CAAC;YACD,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE;gBAC9B,QAAQ,IAAI,CAAC,CAAA;gBAEb,SAAS;gBACT,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBAC3C,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,SAAS,CAAC,CAAA;gBACrD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;oBAC/B,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;oBAC3B,cAAc,IAAI,CAAC,CAAA;oBACnB,OAAM;gBACR,CAAC;gBAED,QAAQ;gBACR,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;oBACpC,WAAW,GAAG,EAAE,CAAA;oBAChB,MAAM,UAAU,GAAG,wBAAwB,CAAC;wBAC1C,IAAI,EAAE,CAAC,YAAY,CAAC;wBACpB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;wBAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;wBACrC,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ;wBACR,KAAK;qBACN,CAAC,CAAA;oBACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBAC5B,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;wBAC3B,cAAc,IAAI,CAAC,CAAA;oBACrB,CAAC;oBACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;gBAC7B,CAAC;qBAAM,CAAC;oBACN,MAAM,QAAQ,GAAG,cAAc,CAAC;wBAC9B,GAAG,EAAE,YAAY;wBACjB,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG;qBACvB,CAAC,CAAA;oBACF,IACE,eAAe,KAAK,SAAS;wBAC7B,eAAe,KAAK,QAAQ;wBAC5B,WAAW,CAAC,MAAM,GAAG,CAAC,EACtB,CAAC;wBACD,MAAM,UAAU,GAAG,wBAAwB,CAAC;4BAC1C,IAAI,EAAE,WAAW;4BACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;4BAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;4BACrC,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,QAAQ;4BACR,KAAK;yBACN,CAAC,CAAA;wBACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BAC5B,cAAc,IAAI,WAAW,CAAC,MAAM,CAAA;wBACtC,CAAC;wBACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;wBAC3B,WAAW,GAAG,EAAE,CAAA;oBAClB,CAAC;oBACD,eAAe,GAAG,QAAQ,CAAA;oBAC1B,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAChC,CAAC;gBAED,yEAAyE;gBACzE,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;oBACpC,MAAM,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAA;oBAC3D,MAAM,aAAa,GAAa,EAAE,CAAA;oBAClC,MAAM,gBAAgB,GAAa,EAAE,CAAA;oBACrC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;wBAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,gBAAgB,EAAE,CAAC;4BACvC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;wBAC/B,CAAC;6BAAM,CAAC;4BACN,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;wBAC5B,CAAC;oBACH,CAAC;oBACD,OAAO,GAAG,gBAAgB,CAAA;oBAC1B,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,aAAa,CAAC,CAAA;oBAC3B,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,OAAO,GAAG,KAAK,CAAA;wBACf,KAAK,GAAG,CAAC,CAAA;wBACT,OAAO,CAAC,KAAK,EAAE,CAAA;oBACjB,CAAC;gBACH,CAAC;YACH,CAAC;SACF,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK;YACL,QAAQ;YACR,cAAc;SACf,CAAA;IACH,CAAC;IAED,kCAAkC;IAClC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,wBAAwB,CAAC;YAC1C,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;YAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,QAAQ;YACR,KAAK;SACN,CAAC,CAAA;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,cAAc,IAAI,WAAW,CAAC,MAAM,CAAA;QACtC,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;IAC7B,CAAC;IAED,+BAA+B;IAC/B,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,CAAC,CAAA;QACrB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,GAAG,KAAK,CAAA;YACf,KAAK,GAAG,CAAC,CAAA;QACX,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO;QACP,KAAK;QACL,QAAQ;QACR,cAAc;KACf,CAAA;AACH,CAAC,CAAA;AAED,MAAM,SAAS,GAAG,CAChB,GAA2B,EAC3B,SAAiC,EACP,EAAE;IAC5B,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,GAAG,EAAE,SAAS,CAAC,CAAA;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,SAAS,CAAA;IAClB,CAAC;AACH,CAAC,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hasValidHeaderRow.d.ts","sourceRoot":"","sources":["../hasValidHeaderRow.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AAExC,KAAK,2BAA2B,GAAG;IACjC,IAAI,EAAE,IAAI,GAAG,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,SAAS,CAAA;CAClB,CAAA;AAED,eAAO,MAAM,iBAAiB,gCAI3B,2BAA2B,
|
|
1
|
+
{"version":3,"file":"hasValidHeaderRow.d.ts","sourceRoot":"","sources":["../hasValidHeaderRow.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AAExC,KAAK,2BAA2B,GAAG;IACjC,IAAI,EAAE,IAAI,GAAG,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,SAAS,CAAA;CAClB,CAAA;AAED,eAAO,MAAM,iBAAiB,gCAI3B,2BAA2B,qBA6B7B,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamCsvToActions.d.ts","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAMzE,OAAO,KAAK,EAAE,SAAS,EAAgB,MAAM,SAAS,CAAA;AAEtD,KAAK,4BAA4B,GAAG;IAClC,IAAI,EAAE,IAAI,GAAG,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,SAAS,CAAA;IACjB,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAA;IACnC,KAAK,EAAE,SAAS,YAAY,EAAE,CAAA;IAC9B,IAAI,EAAE,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IACnD,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,CAAA;AAID,KAAK,6BAA6B,GAAG;IACnC,OAAO,EAAE,OAAO,CAAA;IAChB,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;IAChB,cAAc,EAAE,MAAM,CAAA;CACvB,CAAA;AAED,eAAO,MAAM,kBAAkB,kEAQ5B,4BAA4B,KAAG,OAAO,CAAC,6BAA6B,
|
|
1
|
+
{"version":3,"file":"streamCsvToActions.d.ts","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAMzE,OAAO,KAAK,EAAE,SAAS,EAAgB,MAAM,SAAS,CAAA;AAEtD,KAAK,4BAA4B,GAAG;IAClC,IAAI,EAAE,IAAI,GAAG,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,SAAS,CAAA;IACjB,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAA;IACnC,KAAK,EAAE,SAAS,YAAY,EAAE,CAAA;IAC9B,IAAI,EAAE,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IACnD,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,CAAA;AAID,KAAK,6BAA6B,GAAG;IACnC,OAAO,EAAE,OAAO,CAAA;IAChB,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;IAChB,cAAc,EAAE,MAAM,CAAA;CACvB,CAAA;AAED,eAAO,MAAM,kBAAkB,kEAQ5B,4BAA4B,KAAG,OAAO,CAAC,6BAA6B,CA4ItE,CAAA"}
|
package/hasValidHeaderRow.ts
CHANGED
|
@@ -13,28 +13,31 @@ export const hasValidHeaderRow = async ({
|
|
|
13
13
|
format,
|
|
14
14
|
}: HasValidHeaderRowParameters) => {
|
|
15
15
|
let isValid = false
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
for (const key of Object.keys(format.formatter)) {
|
|
28
|
-
if (!result.data.includes(key)) {
|
|
29
|
-
console.log(`Missing key: ${key}`)
|
|
16
|
+
|
|
17
|
+
await new Promise<void>((resolve) => {
|
|
18
|
+
parse(file, {
|
|
19
|
+
header: false,
|
|
20
|
+
skipEmptyLines: true,
|
|
21
|
+
encoding,
|
|
22
|
+
complete: () => {
|
|
23
|
+
resolve()
|
|
24
|
+
},
|
|
25
|
+
step: (result, parser) => {
|
|
26
|
+
if (!Array.isArray(result.data)) {
|
|
30
27
|
parser.abort()
|
|
31
28
|
return
|
|
32
29
|
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
30
|
+
for (const key of Object.keys(format.formatter)) {
|
|
31
|
+
if (!result.data.includes(key)) {
|
|
32
|
+
parser.abort()
|
|
33
|
+
return
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
isValid = true
|
|
37
|
+
parser.abort()
|
|
38
|
+
},
|
|
39
|
+
})
|
|
37
40
|
})
|
|
38
|
-
|
|
41
|
+
|
|
39
42
|
return isValid
|
|
40
43
|
}
|
package/package.json
CHANGED
package/streamCsvToActions.ts
CHANGED
|
@@ -42,88 +42,94 @@ export const streamCsvToActions = async ({
|
|
|
42
42
|
let currentKeyValue: string | undefined
|
|
43
43
|
let currentRows: FormattedRow[] = []
|
|
44
44
|
let actions: Action[] = []
|
|
45
|
-
parse(file, {
|
|
46
|
-
header: true,
|
|
47
|
-
skipEmptyLines: true,
|
|
48
|
-
encoding,
|
|
49
|
-
step: async (result, parsers) => {
|
|
50
|
-
rowCount += 1
|
|
51
45
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
}
|
|
46
|
+
await new Promise<void>((resolve) => {
|
|
47
|
+
parse(file, {
|
|
48
|
+
header: true,
|
|
49
|
+
skipEmptyLines: true,
|
|
50
|
+
encoding,
|
|
51
|
+
complete: () => {
|
|
52
|
+
resolve()
|
|
53
|
+
},
|
|
54
|
+
step: async (result, parsers) => {
|
|
55
|
+
rowCount += 1
|
|
60
56
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
rows: [formattedRow],
|
|
66
|
-
parsers: format.parser.parsers,
|
|
67
|
-
symbolAssetMap: format.symbolAssetMap,
|
|
68
|
-
service: format.service,
|
|
69
|
-
cryptoes,
|
|
70
|
-
fiats,
|
|
71
|
-
})
|
|
72
|
-
if (rowActions.length === 0) {
|
|
57
|
+
// format
|
|
58
|
+
const row = recordSchema.parse(result.data)
|
|
59
|
+
const formattedRow = formatRow(row, format.formatter)
|
|
60
|
+
if (formattedRow === undefined) {
|
|
73
61
|
console.log('ignore:', row)
|
|
74
62
|
ignoreRowCount += 1
|
|
63
|
+
return
|
|
75
64
|
}
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
if (
|
|
83
|
-
currentKeyValue !== undefined &&
|
|
84
|
-
currentKeyValue !== keyValue &&
|
|
85
|
-
currentRows.length > 0
|
|
86
|
-
) {
|
|
87
|
-
const keyActions = parseTargetRowsToActions({
|
|
88
|
-
rows: currentRows,
|
|
65
|
+
|
|
66
|
+
// parse
|
|
67
|
+
if (format.parser.key === undefined) {
|
|
68
|
+
currentRows = []
|
|
69
|
+
const rowActions = parseTargetRowsToActions({
|
|
70
|
+
rows: [formattedRow],
|
|
89
71
|
parsers: format.parser.parsers,
|
|
90
72
|
symbolAssetMap: format.symbolAssetMap,
|
|
91
73
|
service: format.service,
|
|
92
74
|
cryptoes,
|
|
93
75
|
fiats,
|
|
94
76
|
})
|
|
95
|
-
if (
|
|
96
|
-
|
|
77
|
+
if (rowActions.length === 0) {
|
|
78
|
+
console.log('ignore:', row)
|
|
79
|
+
ignoreRowCount += 1
|
|
97
80
|
}
|
|
98
|
-
actions.push(...
|
|
99
|
-
|
|
81
|
+
actions.push(...rowActions)
|
|
82
|
+
} else {
|
|
83
|
+
const keyValue = createKeyValue({
|
|
84
|
+
row: formattedRow,
|
|
85
|
+
key: format.parser.key,
|
|
86
|
+
})
|
|
87
|
+
if (
|
|
88
|
+
currentKeyValue !== undefined &&
|
|
89
|
+
currentKeyValue !== keyValue &&
|
|
90
|
+
currentRows.length > 0
|
|
91
|
+
) {
|
|
92
|
+
const keyActions = parseTargetRowsToActions({
|
|
93
|
+
rows: currentRows,
|
|
94
|
+
parsers: format.parser.parsers,
|
|
95
|
+
symbolAssetMap: format.symbolAssetMap,
|
|
96
|
+
service: format.service,
|
|
97
|
+
cryptoes,
|
|
98
|
+
fiats,
|
|
99
|
+
})
|
|
100
|
+
if (keyActions.length === 0) {
|
|
101
|
+
ignoreRowCount += currentRows.length
|
|
102
|
+
}
|
|
103
|
+
actions.push(...keyActions)
|
|
104
|
+
currentRows = []
|
|
105
|
+
}
|
|
106
|
+
currentKeyValue = keyValue
|
|
107
|
+
currentRows.push(formattedRow)
|
|
100
108
|
}
|
|
101
|
-
currentKeyValue = keyValue
|
|
102
|
-
currentRows.push(formattedRow)
|
|
103
|
-
}
|
|
104
109
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
110
|
+
// actions が chunkSize に達したら最後の action source と同一 source を除いて actions を保存
|
|
111
|
+
if (actions.length >= chunkSize + 1) {
|
|
112
|
+
const lastActionSource = actions[actions.length - 1].source
|
|
113
|
+
const savingActions: Action[] = []
|
|
114
|
+
const remainingActions: Action[] = []
|
|
115
|
+
for (const action of actions) {
|
|
116
|
+
if (action.source === lastActionSource) {
|
|
117
|
+
remainingActions.push(action)
|
|
118
|
+
} else {
|
|
119
|
+
savingActions.push(action)
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
actions = remainingActions
|
|
123
|
+
try {
|
|
124
|
+
await save(savingActions)
|
|
125
|
+
} catch (e) {
|
|
126
|
+
success = false
|
|
127
|
+
error = e
|
|
128
|
+
parsers.abort()
|
|
115
129
|
}
|
|
116
130
|
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
await save(savingActions)
|
|
120
|
-
} catch (e) {
|
|
121
|
-
success = false
|
|
122
|
-
error = e
|
|
123
|
-
parsers.abort()
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
},
|
|
131
|
+
},
|
|
132
|
+
})
|
|
127
133
|
})
|
|
128
134
|
|
|
129
135
|
if (!success) {
|