@0xtorch/csv 0.0.42 → 0.0.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -2,11 +2,9 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.streamCsvToActions = void 0;
|
|
4
4
|
const papaparse_1 = require("papaparse");
|
|
5
|
-
const zod_1 = require("zod");
|
|
6
5
|
const formatCsvRows_1 = require("./formatCsvRows");
|
|
7
6
|
const parseRowsToActions_1 = require("./parseRowsToActions");
|
|
8
7
|
const key_1 = require("./parseRowsToActions/key");
|
|
9
|
-
const recordSchema = zod_1.z.record(zod_1.z.string());
|
|
10
8
|
const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, save, chunkSize = 100, }) => {
|
|
11
9
|
let success = true;
|
|
12
10
|
let error;
|
|
@@ -15,24 +13,46 @@ const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, sav
|
|
|
15
13
|
let currentKeyValue;
|
|
16
14
|
let currentRows = [];
|
|
17
15
|
let actions = [];
|
|
16
|
+
let formatIndexRecord;
|
|
18
17
|
await new Promise((resolve) => {
|
|
19
18
|
(0, papaparse_1.parse)(file, {
|
|
20
|
-
header:
|
|
19
|
+
header: false,
|
|
21
20
|
skipEmptyLines: true,
|
|
22
21
|
encoding,
|
|
23
22
|
complete: () => {
|
|
24
23
|
resolve();
|
|
25
24
|
},
|
|
26
25
|
step: async (result, parser) => {
|
|
26
|
+
if (!Array.isArray(result.data)) {
|
|
27
|
+
console.warn('unexpected data:', result.data);
|
|
28
|
+
parser.abort();
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
if (formatIndexRecord === undefined) {
|
|
32
|
+
const formatIndexRecordTemp = {};
|
|
33
|
+
for (const key of Object.keys(format.formatter)) {
|
|
34
|
+
const index = result.data.indexOf(key);
|
|
35
|
+
if (index === -1) {
|
|
36
|
+
console.warn('key not found:', key);
|
|
37
|
+
parser.abort();
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
formatIndexRecordTemp[key] = index;
|
|
41
|
+
}
|
|
42
|
+
formatIndexRecord = formatIndexRecordTemp;
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
27
45
|
rowCount += 1;
|
|
28
|
-
const row =
|
|
46
|
+
const row = {};
|
|
47
|
+
for (const key of Object.keys(format.formatter)) {
|
|
48
|
+
const index = formatIndexRecord[key];
|
|
49
|
+
row[key] = result.data[index];
|
|
50
|
+
}
|
|
29
51
|
const formattedRow = formatRow(row, format.formatter);
|
|
30
52
|
if (formattedRow === undefined) {
|
|
31
|
-
console.log('ignore:', row);
|
|
32
53
|
ignoreRowCount += 1;
|
|
33
54
|
return;
|
|
34
55
|
}
|
|
35
|
-
parser.pause();
|
|
36
56
|
if (format.parser.key === undefined) {
|
|
37
57
|
currentRows = [];
|
|
38
58
|
const rowActions = (0, parseRowsToActions_1.parseTargetRowsToActions)({
|
|
@@ -44,7 +64,6 @@ const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, sav
|
|
|
44
64
|
fiats,
|
|
45
65
|
});
|
|
46
66
|
if (rowActions.length === 0) {
|
|
47
|
-
console.log('ignore:', row);
|
|
48
67
|
ignoreRowCount += 1;
|
|
49
68
|
}
|
|
50
69
|
actions.push(...rowActions);
|
|
@@ -75,6 +94,7 @@ const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, sav
|
|
|
75
94
|
currentRows.push(formattedRow);
|
|
76
95
|
}
|
|
77
96
|
if (actions.length >= chunkSize + 1) {
|
|
97
|
+
parser.pause();
|
|
78
98
|
const lastActionSource = actions[actions.length - 1].source;
|
|
79
99
|
const savingActions = [];
|
|
80
100
|
const remainingActions = [];
|
|
@@ -95,8 +115,8 @@ const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, sav
|
|
|
95
115
|
error = e;
|
|
96
116
|
parser.abort();
|
|
97
117
|
}
|
|
118
|
+
parser.resume();
|
|
98
119
|
}
|
|
99
|
-
parser.resume();
|
|
100
120
|
},
|
|
101
121
|
});
|
|
102
122
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamCsvToActions.js","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":";;;AACA,yCAAiC;AACjC,
|
|
1
|
+
{"version":3,"file":"streamCsvToActions.js","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":";;;AACA,yCAAiC;AACjC,mDAA8C;AAC9C,6DAA+D;AAC/D,kDAAyD;AAoBlD,MAAM,kBAAkB,GAAG,KAAK,EAAE,EACvC,IAAI,EACJ,QAAQ,EACR,MAAM,EACN,QAAQ,EACR,KAAK,EACL,IAAI,EACJ,SAAS,GAAG,GAAG,GACc,EAA0C,EAAE;IAEzE,IAAI,OAAO,GAAG,IAAI,CAAA;IAClB,IAAI,KAAc,CAAA;IAClB,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,eAAmC,CAAA;IACvC,IAAI,WAAW,GAAmB,EAAE,CAAA;IACpC,IAAI,OAAO,GAAa,EAAE,CAAA;IAE1B,IAAI,iBAAqD,CAAA;IACzD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,IAAA,iBAAK,EAAC,IAAI,EAAE;YACV,MAAM,EAAE,KAAK;YACb,cAAc,EAAE,IAAI;YACpB,QAAQ;YACR,QAAQ,EAAE,GAAG,EAAE;gBACb,OAAO,EAAE,CAAA;YACX,CAAC;YACD,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;gBAC7B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChC,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;oBAC7C,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,OAAM;gBACR,CAAC;gBACD,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACpC,MAAM,qBAAqB,GAA2B,EAAE,CAAA;oBACxD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;wBAChD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;wBACtC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;4BACjB,OAAO,CAAC,IAAI,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAA;4BACnC,MAAM,CAAC,KAAK,EAAE,CAAA;4BACd,OAAM;wBACR,CAAC;wBACD,qBAAqB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;oBACpC,CAAC;oBACD,iBAAiB,GAAG,qBAAqB,CAAA;oBACzC,OAAM;gBACR,CAAC;gBAED,QAAQ,IAAI,CAAC,CAAA;gBAGb,MAAM,GAAG,GAA2B,EAAE,CAAA;gBACtC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;oBAChD,MAAM,KAAK,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAA;oBACpC,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;gBAC/B,CAAC;gBACD,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,SAAS,CAAC,CAAA;gBACrD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;oBAC/B,cAAc,IAAI,CAAC,CAAA;oBACnB,OAAM;gBACR,CAAC;gBAGD,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;oBACpC,WAAW,GAAG,EAAE,CAAA;oBAChB,MAAM,UAAU,GAAG,IAAA,6CAAwB,EAAC;wBAC1C,IAAI,EAAE,CAAC,YAAY,CAAC;wBACpB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;wBAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;wBACrC,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ;wBACR,KAAK;qBACN,CAAC,CAAA;oBACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBAC5B,cAAc,IAAI,CAAC,CAAA;oBACrB,CAAC;oBACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;gBAC7B,CAAC;qBAAM,CAAC;oBACN,MAAM,QAAQ,GAAG,IAAA,oBAAc,EAAC;wBAC9B,GAAG,EAAE,YAAY;wBACjB,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG;qBACvB,CAAC,CAAA;oBACF,IACE,eAAe,KAAK,SAAS;wBAC7B,eAAe,KAAK,QAAQ;wBAC5B,WAAW,CAAC,MAAM,GAAG,CAAC,EACtB,CAAC;wBACD,MAAM,UAAU,GAAG,IAAA,6CAAwB,EAAC;4BAC1C,IAAI,EAAE,WAAW;4BACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;4BAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;4BACrC,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,QAAQ;4BACR,KAAK;yBACN,CAAC,CAAA;wBACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BAC5B,cAAc,IAAI,WAAW,CAAC,MAAM,CAAA;wBACtC,CAAC;wBACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;wBAC3B,WAAW,GAAG,EAAE,CAAA;oBAClB,CAAC;oBACD,eAAe,GAAG,QAAQ,CAAA;oBAC1B,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAChC,CAAC;gBAGD,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;oBACpC,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,MAAM,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAA;oBAC3D,MAAM,aAAa,GAAa,EAAE,CAAA;oBAClC,MAAM,gBAAgB,GAAa,EAAE,CAAA;oBACrC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;wBAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,gBAAgB,EAAE,CAAC;4BACvC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;wBAC/B,CAAC;6BAAM,CAAC;4BACN,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;wBAC5B,CAAC;oBACH,CAAC;oBACD,OAAO,GAAG,gBAAgB,CAAA;oBAC1B,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAA;oBACrC,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,OAAO,GAAG,KAAK,CAAA;wBACf,KAAK,GAAG,CAAC,CAAA;wBACT,MAAM,CAAC,KAAK,EAAE,CAAA;oBAChB,CAAC;oBACD,MAAM,CAAC,MAAM,EAAE,CAAA;gBACjB,CAAC;YACH,CAAC;SACF,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK;YACL,QAAQ;YACR,cAAc;SACf,CAAA;IACH,CAAC;IAGD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,IAAA,6CAAwB,EAAC;YAC1C,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;YAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,QAAQ;YACR,KAAK;SACN,CAAC,CAAA;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,cAAc,IAAI,WAAW,CAAC,MAAM,CAAA;QACtC,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;IAC7B,CAAC;IAGD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAC/B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,GAAG,KAAK,CAAA;YACf,KAAK,GAAG,CAAC,CAAA;QACX,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO;QACP,KAAK;QACL,QAAQ;QACR,cAAc;KACf,CAAA;AACH,CAAC,CAAA;AA7KY,QAAA,kBAAkB,sBA6K9B;AAED,MAAM,SAAS,GAAG,CAChB,GAA2B,EAC3B,SAAiC,EACP,EAAE;IAC5B,IAAI,CAAC;QACH,OAAO,IAAA,4BAAY,EAAC,GAAG,EAAE,SAAS,CAAC,CAAA;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,SAAS,CAAA;IAClB,CAAC;AACH,CAAC,CAAA"}
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
import { parse } from 'papaparse';
|
|
2
|
-
import { z } from 'zod';
|
|
3
2
|
import { formatCsvRow } from './formatCsvRows';
|
|
4
3
|
import { parseTargetRowsToActions } from './parseRowsToActions';
|
|
5
4
|
import { createKeyValue } from './parseRowsToActions/key';
|
|
6
|
-
const recordSchema = z.record(z.string());
|
|
7
5
|
export const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, save, chunkSize = 100, }) => {
|
|
8
6
|
// stream で処理
|
|
9
7
|
let success = true;
|
|
@@ -13,25 +11,47 @@ export const streamCsvToActions = async ({ file, encoding, format, cryptoes, fia
|
|
|
13
11
|
let currentKeyValue;
|
|
14
12
|
let currentRows = [];
|
|
15
13
|
let actions = [];
|
|
14
|
+
let formatIndexRecord;
|
|
16
15
|
await new Promise((resolve) => {
|
|
17
16
|
parse(file, {
|
|
18
|
-
header:
|
|
17
|
+
header: false,
|
|
19
18
|
skipEmptyLines: true,
|
|
20
19
|
encoding,
|
|
21
20
|
complete: () => {
|
|
22
21
|
resolve();
|
|
23
22
|
},
|
|
24
23
|
step: async (result, parser) => {
|
|
24
|
+
if (!Array.isArray(result.data)) {
|
|
25
|
+
console.warn('unexpected data:', result.data);
|
|
26
|
+
parser.abort();
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
if (formatIndexRecord === undefined) {
|
|
30
|
+
const formatIndexRecordTemp = {};
|
|
31
|
+
for (const key of Object.keys(format.formatter)) {
|
|
32
|
+
const index = result.data.indexOf(key);
|
|
33
|
+
if (index === -1) {
|
|
34
|
+
console.warn('key not found:', key);
|
|
35
|
+
parser.abort();
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
formatIndexRecordTemp[key] = index;
|
|
39
|
+
}
|
|
40
|
+
formatIndexRecord = formatIndexRecordTemp;
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
25
43
|
rowCount += 1;
|
|
26
44
|
// format
|
|
27
|
-
const row =
|
|
45
|
+
const row = {};
|
|
46
|
+
for (const key of Object.keys(format.formatter)) {
|
|
47
|
+
const index = formatIndexRecord[key];
|
|
48
|
+
row[key] = result.data[index];
|
|
49
|
+
}
|
|
28
50
|
const formattedRow = formatRow(row, format.formatter);
|
|
29
51
|
if (formattedRow === undefined) {
|
|
30
|
-
console.log('ignore:', row);
|
|
31
52
|
ignoreRowCount += 1;
|
|
32
53
|
return;
|
|
33
54
|
}
|
|
34
|
-
parser.pause();
|
|
35
55
|
// parse
|
|
36
56
|
if (format.parser.key === undefined) {
|
|
37
57
|
currentRows = [];
|
|
@@ -44,7 +64,6 @@ export const streamCsvToActions = async ({ file, encoding, format, cryptoes, fia
|
|
|
44
64
|
fiats,
|
|
45
65
|
});
|
|
46
66
|
if (rowActions.length === 0) {
|
|
47
|
-
console.log('ignore:', row);
|
|
48
67
|
ignoreRowCount += 1;
|
|
49
68
|
}
|
|
50
69
|
actions.push(...rowActions);
|
|
@@ -76,6 +95,7 @@ export const streamCsvToActions = async ({ file, encoding, format, cryptoes, fia
|
|
|
76
95
|
}
|
|
77
96
|
// actions が chunkSize に達したら最後の action source と同一 source を除いて actions を保存
|
|
78
97
|
if (actions.length >= chunkSize + 1) {
|
|
98
|
+
parser.pause();
|
|
79
99
|
const lastActionSource = actions[actions.length - 1].source;
|
|
80
100
|
const savingActions = [];
|
|
81
101
|
const remainingActions = [];
|
|
@@ -96,8 +116,8 @@ export const streamCsvToActions = async ({ file, encoding, format, cryptoes, fia
|
|
|
96
116
|
error = e;
|
|
97
117
|
parser.abort();
|
|
98
118
|
}
|
|
119
|
+
parser.resume();
|
|
99
120
|
}
|
|
100
|
-
parser.resume();
|
|
101
121
|
},
|
|
102
122
|
});
|
|
103
123
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamCsvToActions.js","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AACjC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"streamCsvToActions.js","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAA;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAoBzD,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EAAE,EACvC,IAAI,EACJ,QAAQ,EACR,MAAM,EACN,QAAQ,EACR,KAAK,EACL,IAAI,EACJ,SAAS,GAAG,GAAG,GACc,EAA0C,EAAE;IACzE,aAAa;IACb,IAAI,OAAO,GAAG,IAAI,CAAA;IAClB,IAAI,KAAc,CAAA;IAClB,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,eAAmC,CAAA;IACvC,IAAI,WAAW,GAAmB,EAAE,CAAA;IACpC,IAAI,OAAO,GAAa,EAAE,CAAA;IAE1B,IAAI,iBAAqD,CAAA;IACzD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,KAAK,CAAC,IAAI,EAAE;YACV,MAAM,EAAE,KAAK;YACb,cAAc,EAAE,IAAI;YACpB,QAAQ;YACR,QAAQ,EAAE,GAAG,EAAE;gBACb,OAAO,EAAE,CAAA;YACX,CAAC;YACD,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;gBAC7B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChC,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;oBAC7C,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,OAAM;gBACR,CAAC;gBACD,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACpC,MAAM,qBAAqB,GAA2B,EAAE,CAAA;oBACxD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;wBAChD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;wBACtC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;4BACjB,OAAO,CAAC,IAAI,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAA;4BACnC,MAAM,CAAC,KAAK,EAAE,CAAA;4BACd,OAAM;wBACR,CAAC;wBACD,qBAAqB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;oBACpC,CAAC;oBACD,iBAAiB,GAAG,qBAAqB,CAAA;oBACzC,OAAM;gBACR,CAAC;gBAED,QAAQ,IAAI,CAAC,CAAA;gBAEb,SAAS;gBACT,MAAM,GAAG,GAA2B,EAAE,CAAA;gBACtC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;oBAChD,MAAM,KAAK,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAA;oBACpC,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;gBAC/B,CAAC;gBACD,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,SAAS,CAAC,CAAA;gBACrD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;oBAC/B,cAAc,IAAI,CAAC,CAAA;oBACnB,OAAM;gBACR,CAAC;gBAED,QAAQ;gBACR,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;oBACpC,WAAW,GAAG,EAAE,CAAA;oBAChB,MAAM,UAAU,GAAG,wBAAwB,CAAC;wBAC1C,IAAI,EAAE,CAAC,YAAY,CAAC;wBACpB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;wBAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;wBACrC,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ;wBACR,KAAK;qBACN,CAAC,CAAA;oBACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBAC5B,cAAc,IAAI,CAAC,CAAA;oBACrB,CAAC;oBACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;gBAC7B,CAAC;qBAAM,CAAC;oBACN,MAAM,QAAQ,GAAG,cAAc,CAAC;wBAC9B,GAAG,EAAE,YAAY;wBACjB,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG;qBACvB,CAAC,CAAA;oBACF,IACE,eAAe,KAAK,SAAS;wBAC7B,eAAe,KAAK,QAAQ;wBAC5B,WAAW,CAAC,MAAM,GAAG,CAAC,EACtB,CAAC;wBACD,MAAM,UAAU,GAAG,wBAAwB,CAAC;4BAC1C,IAAI,EAAE,WAAW;4BACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;4BAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;4BACrC,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,QAAQ;4BACR,KAAK;yBACN,CAAC,CAAA;wBACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BAC5B,cAAc,IAAI,WAAW,CAAC,MAAM,CAAA;wBACtC,CAAC;wBACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;wBAC3B,WAAW,GAAG,EAAE,CAAA;oBAClB,CAAC;oBACD,eAAe,GAAG,QAAQ,CAAA;oBAC1B,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAChC,CAAC;gBAED,yEAAyE;gBACzE,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;oBACpC,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,MAAM,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAA;oBAC3D,MAAM,aAAa,GAAa,EAAE,CAAA;oBAClC,MAAM,gBAAgB,GAAa,EAAE,CAAA;oBACrC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;wBAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,gBAAgB,EAAE,CAAC;4BACvC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;wBAC/B,CAAC;6BAAM,CAAC;4BACN,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;wBAC5B,CAAC;oBACH,CAAC;oBACD,OAAO,GAAG,gBAAgB,CAAA;oBAC1B,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAA;oBACrC,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,OAAO,GAAG,KAAK,CAAA;wBACf,KAAK,GAAG,CAAC,CAAA;wBACT,MAAM,CAAC,KAAK,EAAE,CAAA;oBAChB,CAAC;oBACD,MAAM,CAAC,MAAM,EAAE,CAAA;gBACjB,CAAC;YACH,CAAC;SACF,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK;YACL,QAAQ;YACR,cAAc;SACf,CAAA;IACH,CAAC;IAED,kCAAkC;IAClC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,wBAAwB,CAAC;YAC1C,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;YAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,QAAQ;YACR,KAAK;SACN,CAAC,CAAA;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,cAAc,IAAI,WAAW,CAAC,MAAM,CAAA;QACtC,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;IAC7B,CAAC;IAED,+BAA+B;IAC/B,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAC/B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,GAAG,KAAK,CAAA;YACf,KAAK,GAAG,CAAC,CAAA;QACX,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO;QACP,KAAK;QACL,QAAQ;QACR,cAAc;KACf,CAAA;AACH,CAAC,CAAA;AAED,MAAM,SAAS,GAAG,CAChB,GAA2B,EAC3B,SAAiC,EACP,EAAE;IAC5B,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,GAAG,EAAE,SAAS,CAAC,CAAA;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,SAAS,CAAA;IAClB,CAAC;AACH,CAAC,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamCsvToActions.d.ts","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"streamCsvToActions.d.ts","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAKzE,OAAO,KAAK,EAAE,SAAS,EAAgB,MAAM,SAAS,CAAA;AAEtD,KAAK,4BAA4B,GAAG;IAClC,IAAI,EAAE,IAAI,GAAG,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,SAAS,CAAA;IACjB,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAA;IACnC,KAAK,EAAE,SAAS,YAAY,EAAE,CAAA;IAC9B,IAAI,EAAE,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IACrE,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,CAAA;AAED,KAAK,6BAA6B,GAAG;IACnC,OAAO,EAAE,OAAO,CAAA;IAChB,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;IAChB,cAAc,EAAE,MAAM,CAAA;CACvB,CAAA;AAED,eAAO,MAAM,kBAAkB,kEAQ5B,4BAA4B,KAAG,OAAO,CAAC,6BAA6B,CAqKtE,CAAA"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@0xtorch/csv",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.44",
|
|
4
4
|
"description": "Cryptorch CSV extension",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"cryptorch",
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
],
|
|
36
36
|
"dependencies": {
|
|
37
37
|
"@0xtorch/big-decimal": "^0.0.12",
|
|
38
|
-
"@0xtorch/core": "^0.0.
|
|
38
|
+
"@0xtorch/core": "^0.0.56",
|
|
39
39
|
"encoding-japanese": "^2.1.0",
|
|
40
40
|
"papaparse": "^5.4.1",
|
|
41
41
|
"zod": "^3.23.8"
|
package/streamCsvToActions.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import type { Action, CryptoCurrency, FiatCurrency } from '@0xtorch/core'
|
|
2
2
|
import { parse } from 'papaparse'
|
|
3
|
-
import { z } from 'zod'
|
|
4
3
|
import { formatCsvRow } from './formatCsvRows'
|
|
5
4
|
import { parseTargetRowsToActions } from './parseRowsToActions'
|
|
6
5
|
import { createKeyValue } from './parseRowsToActions/key'
|
|
@@ -16,8 +15,6 @@ type StreamCsvToActionsParameters = {
|
|
|
16
15
|
chunkSize?: number
|
|
17
16
|
}
|
|
18
17
|
|
|
19
|
-
const recordSchema = z.record(z.string())
|
|
20
|
-
|
|
21
18
|
type StreamCsvToActionsReturnTypes = {
|
|
22
19
|
success: boolean
|
|
23
20
|
error?: unknown
|
|
@@ -43,28 +40,50 @@ export const streamCsvToActions = async ({
|
|
|
43
40
|
let currentRows: FormattedRow[] = []
|
|
44
41
|
let actions: Action[] = []
|
|
45
42
|
|
|
43
|
+
let formatIndexRecord: Record<string, number> | undefined
|
|
46
44
|
await new Promise<void>((resolve) => {
|
|
47
45
|
parse(file, {
|
|
48
|
-
header:
|
|
46
|
+
header: false,
|
|
49
47
|
skipEmptyLines: true,
|
|
50
48
|
encoding,
|
|
51
49
|
complete: () => {
|
|
52
50
|
resolve()
|
|
53
51
|
},
|
|
54
52
|
step: async (result, parser) => {
|
|
53
|
+
if (!Array.isArray(result.data)) {
|
|
54
|
+
console.warn('unexpected data:', result.data)
|
|
55
|
+
parser.abort()
|
|
56
|
+
return
|
|
57
|
+
}
|
|
58
|
+
if (formatIndexRecord === undefined) {
|
|
59
|
+
const formatIndexRecordTemp: Record<string, number> = {}
|
|
60
|
+
for (const key of Object.keys(format.formatter)) {
|
|
61
|
+
const index = result.data.indexOf(key)
|
|
62
|
+
if (index === -1) {
|
|
63
|
+
console.warn('key not found:', key)
|
|
64
|
+
parser.abort()
|
|
65
|
+
return
|
|
66
|
+
}
|
|
67
|
+
formatIndexRecordTemp[key] = index
|
|
68
|
+
}
|
|
69
|
+
formatIndexRecord = formatIndexRecordTemp
|
|
70
|
+
return
|
|
71
|
+
}
|
|
72
|
+
|
|
55
73
|
rowCount += 1
|
|
56
74
|
|
|
57
75
|
// format
|
|
58
|
-
const row =
|
|
76
|
+
const row: Record<string, string> = {}
|
|
77
|
+
for (const key of Object.keys(format.formatter)) {
|
|
78
|
+
const index = formatIndexRecord[key]
|
|
79
|
+
row[key] = result.data[index]
|
|
80
|
+
}
|
|
59
81
|
const formattedRow = formatRow(row, format.formatter)
|
|
60
82
|
if (formattedRow === undefined) {
|
|
61
|
-
console.log('ignore:', row)
|
|
62
83
|
ignoreRowCount += 1
|
|
63
84
|
return
|
|
64
85
|
}
|
|
65
86
|
|
|
66
|
-
parser.pause()
|
|
67
|
-
|
|
68
87
|
// parse
|
|
69
88
|
if (format.parser.key === undefined) {
|
|
70
89
|
currentRows = []
|
|
@@ -77,7 +96,6 @@ export const streamCsvToActions = async ({
|
|
|
77
96
|
fiats,
|
|
78
97
|
})
|
|
79
98
|
if (rowActions.length === 0) {
|
|
80
|
-
console.log('ignore:', row)
|
|
81
99
|
ignoreRowCount += 1
|
|
82
100
|
}
|
|
83
101
|
actions.push(...rowActions)
|
|
@@ -111,6 +129,7 @@ export const streamCsvToActions = async ({
|
|
|
111
129
|
|
|
112
130
|
// actions が chunkSize に達したら最後の action source と同一 source を除いて actions を保存
|
|
113
131
|
if (actions.length >= chunkSize + 1) {
|
|
132
|
+
parser.pause()
|
|
114
133
|
const lastActionSource = actions[actions.length - 1].source
|
|
115
134
|
const savingActions: Action[] = []
|
|
116
135
|
const remainingActions: Action[] = []
|
|
@@ -129,8 +148,8 @@ export const streamCsvToActions = async ({
|
|
|
129
148
|
error = e
|
|
130
149
|
parser.abort()
|
|
131
150
|
}
|
|
151
|
+
parser.resume()
|
|
132
152
|
}
|
|
133
|
-
parser.resume()
|
|
134
153
|
},
|
|
135
154
|
})
|
|
136
155
|
})
|