@0xtorch/csv 0.0.42 → 0.0.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -2,11 +2,9 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.streamCsvToActions = void 0;
|
|
4
4
|
const papaparse_1 = require("papaparse");
|
|
5
|
-
const zod_1 = require("zod");
|
|
6
5
|
const formatCsvRows_1 = require("./formatCsvRows");
|
|
7
6
|
const parseRowsToActions_1 = require("./parseRowsToActions");
|
|
8
7
|
const key_1 = require("./parseRowsToActions/key");
|
|
9
|
-
const recordSchema = zod_1.z.record(zod_1.z.string());
|
|
10
8
|
const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, save, chunkSize = 100, }) => {
|
|
11
9
|
let success = true;
|
|
12
10
|
let error;
|
|
@@ -15,24 +13,49 @@ const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, sav
|
|
|
15
13
|
let currentKeyValue;
|
|
16
14
|
let currentRows = [];
|
|
17
15
|
let actions = [];
|
|
16
|
+
let formatIndexRecord;
|
|
18
17
|
await new Promise((resolve) => {
|
|
19
18
|
(0, papaparse_1.parse)(file, {
|
|
20
|
-
header:
|
|
19
|
+
header: false,
|
|
21
20
|
skipEmptyLines: true,
|
|
22
21
|
encoding,
|
|
23
22
|
complete: () => {
|
|
24
23
|
resolve();
|
|
25
24
|
},
|
|
26
25
|
step: async (result, parser) => {
|
|
26
|
+
if (!Array.isArray(result.data)) {
|
|
27
|
+
console.log('unexpected data:', result.data);
|
|
28
|
+
parser.abort();
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
if (formatIndexRecord === undefined) {
|
|
32
|
+
const formatIndexRecordTemp = {};
|
|
33
|
+
for (const key of Object.keys(format.formatter)) {
|
|
34
|
+
const index = result.data.indexOf(key);
|
|
35
|
+
if (index === -1) {
|
|
36
|
+
console.log('key not found:', key);
|
|
37
|
+
parser.abort();
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
formatIndexRecordTemp[key] = index;
|
|
41
|
+
}
|
|
42
|
+
formatIndexRecord = formatIndexRecordTemp;
|
|
43
|
+
console.log('formatIndexRecord:');
|
|
44
|
+
console.log(formatIndexRecord);
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
27
47
|
rowCount += 1;
|
|
28
|
-
const row =
|
|
48
|
+
const row = {};
|
|
49
|
+
for (const key of Object.keys(format.formatter)) {
|
|
50
|
+
const index = formatIndexRecord[key];
|
|
51
|
+
row[key] = result.data[index];
|
|
52
|
+
}
|
|
29
53
|
const formattedRow = formatRow(row, format.formatter);
|
|
30
54
|
if (formattedRow === undefined) {
|
|
31
55
|
console.log('ignore:', row);
|
|
32
56
|
ignoreRowCount += 1;
|
|
33
57
|
return;
|
|
34
58
|
}
|
|
35
|
-
parser.pause();
|
|
36
59
|
if (format.parser.key === undefined) {
|
|
37
60
|
currentRows = [];
|
|
38
61
|
const rowActions = (0, parseRowsToActions_1.parseTargetRowsToActions)({
|
|
@@ -75,6 +98,7 @@ const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, sav
|
|
|
75
98
|
currentRows.push(formattedRow);
|
|
76
99
|
}
|
|
77
100
|
if (actions.length >= chunkSize + 1) {
|
|
101
|
+
parser.pause();
|
|
78
102
|
const lastActionSource = actions[actions.length - 1].source;
|
|
79
103
|
const savingActions = [];
|
|
80
104
|
const remainingActions = [];
|
|
@@ -95,8 +119,8 @@ const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, sav
|
|
|
95
119
|
error = e;
|
|
96
120
|
parser.abort();
|
|
97
121
|
}
|
|
122
|
+
parser.resume();
|
|
98
123
|
}
|
|
99
|
-
parser.resume();
|
|
100
124
|
},
|
|
101
125
|
});
|
|
102
126
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamCsvToActions.js","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":";;;AACA,yCAAiC;AACjC,
|
|
1
|
+
{"version":3,"file":"streamCsvToActions.js","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":";;;AACA,yCAAiC;AACjC,mDAA8C;AAC9C,6DAA+D;AAC/D,kDAAyD;AAoBlD,MAAM,kBAAkB,GAAG,KAAK,EAAE,EACvC,IAAI,EACJ,QAAQ,EACR,MAAM,EACN,QAAQ,EACR,KAAK,EACL,IAAI,EACJ,SAAS,GAAG,GAAG,GACc,EAA0C,EAAE;IAEzE,IAAI,OAAO,GAAG,IAAI,CAAA;IAClB,IAAI,KAAc,CAAA;IAClB,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,eAAmC,CAAA;IACvC,IAAI,WAAW,GAAmB,EAAE,CAAA;IACpC,IAAI,OAAO,GAAa,EAAE,CAAA;IAE1B,IAAI,iBAAqD,CAAA;IACzD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,IAAA,iBAAK,EAAC,IAAI,EAAE;YACV,MAAM,EAAE,KAAK;YACb,cAAc,EAAE,IAAI;YACpB,QAAQ;YACR,QAAQ,EAAE,GAAG,EAAE;gBACb,OAAO,EAAE,CAAA;YACX,CAAC;YACD,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;gBAC7B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChC,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;oBAC5C,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,OAAM;gBACR,CAAC;gBACD,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACpC,MAAM,qBAAqB,GAA2B,EAAE,CAAA;oBACxD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;wBAChD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;wBACtC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;4BACjB,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAA;4BAClC,MAAM,CAAC,KAAK,EAAE,CAAA;4BACd,OAAM;wBACR,CAAC;wBACD,qBAAqB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;oBACpC,CAAC;oBACD,iBAAiB,GAAG,qBAAqB,CAAA;oBACzC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAA;oBACjC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAA;oBAC9B,OAAM;gBACR,CAAC;gBAED,QAAQ,IAAI,CAAC,CAAA;gBAGb,MAAM,GAAG,GAA2B,EAAE,CAAA;gBACtC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;oBAChD,MAAM,KAAK,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAA;oBACpC,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;gBAC/B,CAAC;gBACD,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,SAAS,CAAC,CAAA;gBACrD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;oBAC/B,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;oBAC3B,cAAc,IAAI,CAAC,CAAA;oBACnB,OAAM;gBACR,CAAC;gBAGD,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;oBACpC,WAAW,GAAG,EAAE,CAAA;oBAChB,MAAM,UAAU,GAAG,IAAA,6CAAwB,EAAC;wBAC1C,IAAI,EAAE,CAAC,YAAY,CAAC;wBACpB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;wBAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;wBACrC,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ;wBACR,KAAK;qBACN,CAAC,CAAA;oBACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBAC5B,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;wBAC3B,cAAc,IAAI,CAAC,CAAA;oBACrB,CAAC;oBACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;gBAC7B,CAAC;qBAAM,CAAC;oBACN,MAAM,QAAQ,GAAG,IAAA,oBAAc,EAAC;wBAC9B,GAAG,EAAE,YAAY;wBACjB,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG;qBACvB,CAAC,CAAA;oBACF,IACE,eAAe,KAAK,SAAS;wBAC7B,eAAe,KAAK,QAAQ;wBAC5B,WAAW,CAAC,MAAM,GAAG,CAAC,EACtB,CAAC;wBACD,MAAM,UAAU,GAAG,IAAA,6CAAwB,EAAC;4BAC1C,IAAI,EAAE,WAAW;4BACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;4BAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;4BACrC,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,QAAQ;4BACR,KAAK;yBACN,CAAC,CAAA;wBACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BAC5B,cAAc,IAAI,WAAW,CAAC,MAAM,CAAA;wBACtC,CAAC;wBACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;wBAC3B,WAAW,GAAG,EAAE,CAAA;oBAClB,CAAC;oBACD,eAAe,GAAG,QAAQ,CAAA;oBAC1B,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAChC,CAAC;gBAGD,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;oBACpC,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,MAAM,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAA;oBAC3D,MAAM,aAAa,GAAa,EAAE,CAAA;oBAClC,MAAM,gBAAgB,GAAa,EAAE,CAAA;oBACrC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;wBAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,gBAAgB,EAAE,CAAC;4BACvC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;wBAC/B,CAAC;6BAAM,CAAC;4BACN,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;wBAC5B,CAAC;oBACH,CAAC;oBACD,OAAO,GAAG,gBAAgB,CAAA;oBAC1B,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAA;oBACrC,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,OAAO,GAAG,KAAK,CAAA;wBACf,KAAK,GAAG,CAAC,CAAA;wBACT,MAAM,CAAC,KAAK,EAAE,CAAA;oBAChB,CAAC;oBACD,MAAM,CAAC,MAAM,EAAE,CAAA;gBACjB,CAAC;YACH,CAAC;SACF,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK;YACL,QAAQ;YACR,cAAc;SACf,CAAA;IACH,CAAC;IAGD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,IAAA,6CAAwB,EAAC;YAC1C,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;YAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,QAAQ;YACR,KAAK;SACN,CAAC,CAAA;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,cAAc,IAAI,WAAW,CAAC,MAAM,CAAA;QACtC,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;IAC7B,CAAC;IAGD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAC/B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,GAAG,KAAK,CAAA;YACf,KAAK,GAAG,CAAC,CAAA;QACX,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO;QACP,KAAK;QACL,QAAQ;QACR,cAAc;KACf,CAAA;AACH,CAAC,CAAA;AAjLY,QAAA,kBAAkB,sBAiL9B;AAED,MAAM,SAAS,GAAG,CAChB,GAA2B,EAC3B,SAAiC,EACP,EAAE;IAC5B,IAAI,CAAC;QACH,OAAO,IAAA,4BAAY,EAAC,GAAG,EAAE,SAAS,CAAC,CAAA;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,SAAS,CAAA;IAClB,CAAC;AACH,CAAC,CAAA"}
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
import { parse } from 'papaparse';
|
|
2
|
-
import { z } from 'zod';
|
|
3
2
|
import { formatCsvRow } from './formatCsvRows';
|
|
4
3
|
import { parseTargetRowsToActions } from './parseRowsToActions';
|
|
5
4
|
import { createKeyValue } from './parseRowsToActions/key';
|
|
6
|
-
const recordSchema = z.record(z.string());
|
|
7
5
|
export const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, save, chunkSize = 100, }) => {
|
|
8
6
|
// stream で処理
|
|
9
7
|
let success = true;
|
|
@@ -13,25 +11,50 @@ export const streamCsvToActions = async ({ file, encoding, format, cryptoes, fia
|
|
|
13
11
|
let currentKeyValue;
|
|
14
12
|
let currentRows = [];
|
|
15
13
|
let actions = [];
|
|
14
|
+
let formatIndexRecord;
|
|
16
15
|
await new Promise((resolve) => {
|
|
17
16
|
parse(file, {
|
|
18
|
-
header:
|
|
17
|
+
header: false,
|
|
19
18
|
skipEmptyLines: true,
|
|
20
19
|
encoding,
|
|
21
20
|
complete: () => {
|
|
22
21
|
resolve();
|
|
23
22
|
},
|
|
24
23
|
step: async (result, parser) => {
|
|
24
|
+
if (!Array.isArray(result.data)) {
|
|
25
|
+
console.log('unexpected data:', result.data);
|
|
26
|
+
parser.abort();
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
if (formatIndexRecord === undefined) {
|
|
30
|
+
const formatIndexRecordTemp = {};
|
|
31
|
+
for (const key of Object.keys(format.formatter)) {
|
|
32
|
+
const index = result.data.indexOf(key);
|
|
33
|
+
if (index === -1) {
|
|
34
|
+
console.log('key not found:', key);
|
|
35
|
+
parser.abort();
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
formatIndexRecordTemp[key] = index;
|
|
39
|
+
}
|
|
40
|
+
formatIndexRecord = formatIndexRecordTemp;
|
|
41
|
+
console.log('formatIndexRecord:');
|
|
42
|
+
console.log(formatIndexRecord);
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
25
45
|
rowCount += 1;
|
|
26
46
|
// format
|
|
27
|
-
const row =
|
|
47
|
+
const row = {};
|
|
48
|
+
for (const key of Object.keys(format.formatter)) {
|
|
49
|
+
const index = formatIndexRecord[key];
|
|
50
|
+
row[key] = result.data[index];
|
|
51
|
+
}
|
|
28
52
|
const formattedRow = formatRow(row, format.formatter);
|
|
29
53
|
if (formattedRow === undefined) {
|
|
30
54
|
console.log('ignore:', row);
|
|
31
55
|
ignoreRowCount += 1;
|
|
32
56
|
return;
|
|
33
57
|
}
|
|
34
|
-
parser.pause();
|
|
35
58
|
// parse
|
|
36
59
|
if (format.parser.key === undefined) {
|
|
37
60
|
currentRows = [];
|
|
@@ -76,6 +99,7 @@ export const streamCsvToActions = async ({ file, encoding, format, cryptoes, fia
|
|
|
76
99
|
}
|
|
77
100
|
// actions が chunkSize に達したら最後の action source と同一 source を除いて actions を保存
|
|
78
101
|
if (actions.length >= chunkSize + 1) {
|
|
102
|
+
parser.pause();
|
|
79
103
|
const lastActionSource = actions[actions.length - 1].source;
|
|
80
104
|
const savingActions = [];
|
|
81
105
|
const remainingActions = [];
|
|
@@ -96,8 +120,8 @@ export const streamCsvToActions = async ({ file, encoding, format, cryptoes, fia
|
|
|
96
120
|
error = e;
|
|
97
121
|
parser.abort();
|
|
98
122
|
}
|
|
123
|
+
parser.resume();
|
|
99
124
|
}
|
|
100
|
-
parser.resume();
|
|
101
125
|
},
|
|
102
126
|
});
|
|
103
127
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamCsvToActions.js","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AACjC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"streamCsvToActions.js","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAA;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAoBzD,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EAAE,EACvC,IAAI,EACJ,QAAQ,EACR,MAAM,EACN,QAAQ,EACR,KAAK,EACL,IAAI,EACJ,SAAS,GAAG,GAAG,GACc,EAA0C,EAAE;IACzE,aAAa;IACb,IAAI,OAAO,GAAG,IAAI,CAAA;IAClB,IAAI,KAAc,CAAA;IAClB,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,eAAmC,CAAA;IACvC,IAAI,WAAW,GAAmB,EAAE,CAAA;IACpC,IAAI,OAAO,GAAa,EAAE,CAAA;IAE1B,IAAI,iBAAqD,CAAA;IACzD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,KAAK,CAAC,IAAI,EAAE;YACV,MAAM,EAAE,KAAK;YACb,cAAc,EAAE,IAAI;YACpB,QAAQ;YACR,QAAQ,EAAE,GAAG,EAAE;gBACb,OAAO,EAAE,CAAA;YACX,CAAC;YACD,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;gBAC7B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChC,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;oBAC5C,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,OAAM;gBACR,CAAC;gBACD,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACpC,MAAM,qBAAqB,GAA2B,EAAE,CAAA;oBACxD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;wBAChD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;wBACtC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;4BACjB,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAA;4BAClC,MAAM,CAAC,KAAK,EAAE,CAAA;4BACd,OAAM;wBACR,CAAC;wBACD,qBAAqB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;oBACpC,CAAC;oBACD,iBAAiB,GAAG,qBAAqB,CAAA;oBACzC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAA;oBACjC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAA;oBAC9B,OAAM;gBACR,CAAC;gBAED,QAAQ,IAAI,CAAC,CAAA;gBAEb,SAAS;gBACT,MAAM,GAAG,GAA2B,EAAE,CAAA;gBACtC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;oBAChD,MAAM,KAAK,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAA;oBACpC,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;gBAC/B,CAAC;gBACD,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,SAAS,CAAC,CAAA;gBACrD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;oBAC/B,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;oBAC3B,cAAc,IAAI,CAAC,CAAA;oBACnB,OAAM;gBACR,CAAC;gBAED,QAAQ;gBACR,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;oBACpC,WAAW,GAAG,EAAE,CAAA;oBAChB,MAAM,UAAU,GAAG,wBAAwB,CAAC;wBAC1C,IAAI,EAAE,CAAC,YAAY,CAAC;wBACpB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;wBAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;wBACrC,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ;wBACR,KAAK;qBACN,CAAC,CAAA;oBACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBAC5B,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;wBAC3B,cAAc,IAAI,CAAC,CAAA;oBACrB,CAAC;oBACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;gBAC7B,CAAC;qBAAM,CAAC;oBACN,MAAM,QAAQ,GAAG,cAAc,CAAC;wBAC9B,GAAG,EAAE,YAAY;wBACjB,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG;qBACvB,CAAC,CAAA;oBACF,IACE,eAAe,KAAK,SAAS;wBAC7B,eAAe,KAAK,QAAQ;wBAC5B,WAAW,CAAC,MAAM,GAAG,CAAC,EACtB,CAAC;wBACD,MAAM,UAAU,GAAG,wBAAwB,CAAC;4BAC1C,IAAI,EAAE,WAAW;4BACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;4BAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;4BACrC,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,QAAQ;4BACR,KAAK;yBACN,CAAC,CAAA;wBACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BAC5B,cAAc,IAAI,WAAW,CAAC,MAAM,CAAA;wBACtC,CAAC;wBACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;wBAC3B,WAAW,GAAG,EAAE,CAAA;oBAClB,CAAC;oBACD,eAAe,GAAG,QAAQ,CAAA;oBAC1B,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAChC,CAAC;gBAED,yEAAyE;gBACzE,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;oBACpC,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,MAAM,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAA;oBAC3D,MAAM,aAAa,GAAa,EAAE,CAAA;oBAClC,MAAM,gBAAgB,GAAa,EAAE,CAAA;oBACrC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;wBAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,gBAAgB,EAAE,CAAC;4BACvC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;wBAC/B,CAAC;6BAAM,CAAC;4BACN,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;wBAC5B,CAAC;oBACH,CAAC;oBACD,OAAO,GAAG,gBAAgB,CAAA;oBAC1B,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAA;oBACrC,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,OAAO,GAAG,KAAK,CAAA;wBACf,KAAK,GAAG,CAAC,CAAA;wBACT,MAAM,CAAC,KAAK,EAAE,CAAA;oBAChB,CAAC;oBACD,MAAM,CAAC,MAAM,EAAE,CAAA;gBACjB,CAAC;YACH,CAAC;SACF,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK;YACL,QAAQ;YACR,cAAc;SACf,CAAA;IACH,CAAC;IAED,kCAAkC;IAClC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,wBAAwB,CAAC;YAC1C,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;YAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,QAAQ;YACR,KAAK;SACN,CAAC,CAAA;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,cAAc,IAAI,WAAW,CAAC,MAAM,CAAA;QACtC,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;IAC7B,CAAC;IAED,+BAA+B;IAC/B,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAC/B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,GAAG,KAAK,CAAA;YACf,KAAK,GAAG,CAAC,CAAA;QACX,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO;QACP,KAAK;QACL,QAAQ;QACR,cAAc;KACf,CAAA;AACH,CAAC,CAAA;AAED,MAAM,SAAS,GAAG,CAChB,GAA2B,EAC3B,SAAiC,EACP,EAAE;IAC5B,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,GAAG,EAAE,SAAS,CAAC,CAAA;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,SAAS,CAAA;IAClB,CAAC;AACH,CAAC,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamCsvToActions.d.ts","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"streamCsvToActions.d.ts","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAKzE,OAAO,KAAK,EAAE,SAAS,EAAgB,MAAM,SAAS,CAAA;AAEtD,KAAK,4BAA4B,GAAG;IAClC,IAAI,EAAE,IAAI,GAAG,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,SAAS,CAAA;IACjB,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAA;IACnC,KAAK,EAAE,SAAS,YAAY,EAAE,CAAA;IAC9B,IAAI,EAAE,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IACrE,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,CAAA;AAED,KAAK,6BAA6B,GAAG;IACnC,OAAO,EAAE,OAAO,CAAA;IAChB,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;IAChB,cAAc,EAAE,MAAM,CAAA;CACvB,CAAA;AAED,eAAO,MAAM,kBAAkB,kEAQ5B,4BAA4B,KAAG,OAAO,CAAC,6BAA6B,CAyKtE,CAAA"}
|
package/package.json
CHANGED
package/streamCsvToActions.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import type { Action, CryptoCurrency, FiatCurrency } from '@0xtorch/core'
|
|
2
2
|
import { parse } from 'papaparse'
|
|
3
|
-
import { z } from 'zod'
|
|
4
3
|
import { formatCsvRow } from './formatCsvRows'
|
|
5
4
|
import { parseTargetRowsToActions } from './parseRowsToActions'
|
|
6
5
|
import { createKeyValue } from './parseRowsToActions/key'
|
|
@@ -16,8 +15,6 @@ type StreamCsvToActionsParameters = {
|
|
|
16
15
|
chunkSize?: number
|
|
17
16
|
}
|
|
18
17
|
|
|
19
|
-
const recordSchema = z.record(z.string())
|
|
20
|
-
|
|
21
18
|
type StreamCsvToActionsReturnTypes = {
|
|
22
19
|
success: boolean
|
|
23
20
|
error?: unknown
|
|
@@ -43,19 +40,46 @@ export const streamCsvToActions = async ({
|
|
|
43
40
|
let currentRows: FormattedRow[] = []
|
|
44
41
|
let actions: Action[] = []
|
|
45
42
|
|
|
43
|
+
let formatIndexRecord: Record<string, number> | undefined
|
|
46
44
|
await new Promise<void>((resolve) => {
|
|
47
45
|
parse(file, {
|
|
48
|
-
header:
|
|
46
|
+
header: false,
|
|
49
47
|
skipEmptyLines: true,
|
|
50
48
|
encoding,
|
|
51
49
|
complete: () => {
|
|
52
50
|
resolve()
|
|
53
51
|
},
|
|
54
52
|
step: async (result, parser) => {
|
|
53
|
+
if (!Array.isArray(result.data)) {
|
|
54
|
+
console.log('unexpected data:', result.data)
|
|
55
|
+
parser.abort()
|
|
56
|
+
return
|
|
57
|
+
}
|
|
58
|
+
if (formatIndexRecord === undefined) {
|
|
59
|
+
const formatIndexRecordTemp: Record<string, number> = {}
|
|
60
|
+
for (const key of Object.keys(format.formatter)) {
|
|
61
|
+
const index = result.data.indexOf(key)
|
|
62
|
+
if (index === -1) {
|
|
63
|
+
console.log('key not found:', key)
|
|
64
|
+
parser.abort()
|
|
65
|
+
return
|
|
66
|
+
}
|
|
67
|
+
formatIndexRecordTemp[key] = index
|
|
68
|
+
}
|
|
69
|
+
formatIndexRecord = formatIndexRecordTemp
|
|
70
|
+
console.log('formatIndexRecord:')
|
|
71
|
+
console.log(formatIndexRecord)
|
|
72
|
+
return
|
|
73
|
+
}
|
|
74
|
+
|
|
55
75
|
rowCount += 1
|
|
56
76
|
|
|
57
77
|
// format
|
|
58
|
-
const row =
|
|
78
|
+
const row: Record<string, string> = {}
|
|
79
|
+
for (const key of Object.keys(format.formatter)) {
|
|
80
|
+
const index = formatIndexRecord[key]
|
|
81
|
+
row[key] = result.data[index]
|
|
82
|
+
}
|
|
59
83
|
const formattedRow = formatRow(row, format.formatter)
|
|
60
84
|
if (formattedRow === undefined) {
|
|
61
85
|
console.log('ignore:', row)
|
|
@@ -63,8 +87,6 @@ export const streamCsvToActions = async ({
|
|
|
63
87
|
return
|
|
64
88
|
}
|
|
65
89
|
|
|
66
|
-
parser.pause()
|
|
67
|
-
|
|
68
90
|
// parse
|
|
69
91
|
if (format.parser.key === undefined) {
|
|
70
92
|
currentRows = []
|
|
@@ -111,6 +133,7 @@ export const streamCsvToActions = async ({
|
|
|
111
133
|
|
|
112
134
|
// actions が chunkSize に達したら最後の action source と同一 source を除いて actions を保存
|
|
113
135
|
if (actions.length >= chunkSize + 1) {
|
|
136
|
+
parser.pause()
|
|
114
137
|
const lastActionSource = actions[actions.length - 1].source
|
|
115
138
|
const savingActions: Action[] = []
|
|
116
139
|
const remainingActions: Action[] = []
|
|
@@ -129,8 +152,8 @@ export const streamCsvToActions = async ({
|
|
|
129
152
|
error = e
|
|
130
153
|
parser.abort()
|
|
131
154
|
}
|
|
155
|
+
parser.resume()
|
|
132
156
|
}
|
|
133
|
-
parser.resume()
|
|
134
157
|
},
|
|
135
158
|
})
|
|
136
159
|
})
|