@0xtorch/csv 0.0.72 → 0.0.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/_cjs/hasValidHeaderRow.js +10 -8
- package/_cjs/hasValidHeaderRow.js.map +1 -1
- package/_cjs/streamCsvToActions.js +13 -10
- package/_cjs/streamCsvToActions.js.map +1 -1
- package/_esm/hasValidHeaderRow.js +10 -8
- package/_esm/hasValidHeaderRow.js.map +1 -1
- package/_esm/streamCsvToActions.js +13 -10
- package/_esm/streamCsvToActions.js.map +1 -1
- package/_types/streamCsvToActions.d.ts.map +1 -1
- package/hasValidHeaderRow.ts +10 -10
- package/package.json +1 -1
- package/streamCsvToActions.ts +15 -10
|
@@ -3,7 +3,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.hasValidHeaderRow = void 0;
|
|
4
4
|
const papaparse_1 = require("papaparse");
|
|
5
5
|
const hasValidHeaderRow = async ({ file, encoding, format, }) => {
|
|
6
|
-
console.warn('start hasValidHeaderRow');
|
|
7
6
|
let isValid = false;
|
|
8
7
|
let rowCount = 0;
|
|
9
8
|
const headerStartColumnRegExps = format.headerStartColumns === undefined
|
|
@@ -12,15 +11,12 @@ const hasValidHeaderRow = async ({ file, encoding, format, }) => {
|
|
|
12
11
|
await new Promise((resolve) => {
|
|
13
12
|
(0, papaparse_1.parse)(file, {
|
|
14
13
|
header: false,
|
|
15
|
-
skipEmptyLines:
|
|
14
|
+
skipEmptyLines: true,
|
|
16
15
|
encoding,
|
|
17
16
|
complete: () => {
|
|
18
|
-
console.warn('complete');
|
|
19
17
|
resolve();
|
|
20
18
|
},
|
|
21
19
|
step: (result, parser) => {
|
|
22
|
-
console.warn('row:');
|
|
23
|
-
console.warn(result);
|
|
24
20
|
rowCount += 1;
|
|
25
21
|
if (format.headerCheckRowSize !== undefined &&
|
|
26
22
|
rowCount > format.headerCheckRowSize) {
|
|
@@ -32,10 +28,16 @@ const hasValidHeaderRow = async ({ file, encoding, format, }) => {
|
|
|
32
28
|
return;
|
|
33
29
|
}
|
|
34
30
|
if (headerStartColumnRegExps !== undefined) {
|
|
35
|
-
console.warn('check row is header:');
|
|
36
|
-
console.warn(result.data);
|
|
37
31
|
for (const [index, column] of headerStartColumnRegExps.entries()) {
|
|
38
|
-
|
|
32
|
+
const cell = result.data[index];
|
|
33
|
+
if (typeof cell !== 'string') {
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
const value = cell
|
|
37
|
+
.replaceAll('\r', '')
|
|
38
|
+
.replaceAll('\u{65279}', '')
|
|
39
|
+
.trim();
|
|
40
|
+
if (!column.test(value)) {
|
|
39
41
|
return;
|
|
40
42
|
}
|
|
41
43
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hasValidHeaderRow.js","sourceRoot":"","sources":["../hasValidHeaderRow.ts"],"names":[],"mappings":";;;AAAA,yCAAiC;AAS1B,MAAM,iBAAiB,GAAG,KAAK,EAAE,EACtC,IAAI,EACJ,QAAQ,EACR,MAAM,GACsB,EAAE,EAAE;IAChC,
|
|
1
|
+
{"version":3,"file":"hasValidHeaderRow.js","sourceRoot":"","sources":["../hasValidHeaderRow.ts"],"names":[],"mappings":";;;AAAA,yCAAiC;AAS1B,MAAM,iBAAiB,GAAG,KAAK,EAAE,EACtC,IAAI,EACJ,QAAQ,EACR,MAAM,GACsB,EAAE,EAAE;IAChC,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAEhB,MAAM,wBAAwB,GAC5B,MAAM,CAAC,kBAAkB,KAAK,SAAS;QACrC,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,MAAM,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;IAEzD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,IAAA,iBAAK,EAAC,IAAI,EAAE;YACV,MAAM,EAAE,KAAK;YACb,cAAc,EAAE,IAAI;YACpB,QAAQ;YACR,QAAQ,EAAE,GAAG,EAAE;gBACb,OAAO,EAAE,CAAA;YACX,CAAC;YACD,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE;gBACvB,QAAQ,IAAI,CAAC,CAAA;gBACb,IACE,MAAM,CAAC,kBAAkB,KAAK,SAAS;oBACvC,QAAQ,GAAG,MAAM,CAAC,kBAAkB,EACpC,CAAC;oBACD,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,OAAM;gBACR,CAAC;gBACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChC,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,OAAM;gBACR,CAAC;gBAED,IAAI,wBAAwB,KAAK,SAAS,EAAE,CAAC;oBAC3C,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,wBAAwB,CAAC,OAAO,EAAE,EAAE,CAAC;wBACjE,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;wBAC/B,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;4BAC7B,OAAM;wBACR,CAAC;wBACD,MAAM,KAAK,GAAG,IAAI;6BACf,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC;6BACpB,UAAU,CAAC,WAAW,EAAE,EAAE,CAAC;6BAC3B,IAAI,EAAE,CAAA;wBACT,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;4BACxB,OAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;oBAChD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC/B,MAAM,CAAC,KAAK,EAAE,CAAA;wBACd,OAAM;oBACR,CAAC;gBACH,CAAC;gBACD,OAAO,GAAG,IAAI,CAAA;gBACd,MAAM,CAAC,KAAK,EAAE,CAAA;YAChB,CAAC;SACF,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,OAAO,OAAO,CAAA;AAChB,CAAC,CAAA;AAhEY,QAAA,iBAAiB,qBAgE7B"}
|
|
@@ -34,17 +34,20 @@ const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, sav
|
|
|
34
34
|
parser.abort();
|
|
35
35
|
return;
|
|
36
36
|
}
|
|
37
|
+
const cells = result.data.map((v) => typeof v === 'string'
|
|
38
|
+
? v.replaceAll('\r', '').replaceAll('\u{65279}', '').trim()
|
|
39
|
+
: v);
|
|
37
40
|
if (formatIndexRecord === undefined) {
|
|
38
41
|
if (headerStartColumnRegExps !== undefined) {
|
|
39
42
|
for (const [index, column] of headerStartColumnRegExps.entries()) {
|
|
40
|
-
if (!column.test(
|
|
43
|
+
if (!column.test(cells[index])) {
|
|
41
44
|
return;
|
|
42
45
|
}
|
|
43
46
|
}
|
|
44
47
|
}
|
|
45
48
|
const formatIndexRecordTemp = {};
|
|
46
49
|
for (const key of Object.keys(format.formatter)) {
|
|
47
|
-
const index =
|
|
50
|
+
const index = cells.indexOf(key);
|
|
48
51
|
if (index === -1) {
|
|
49
52
|
console.warn('key not found:', key);
|
|
50
53
|
parser.abort();
|
|
@@ -53,16 +56,16 @@ const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, sav
|
|
|
53
56
|
formatIndexRecordTemp[key] = index;
|
|
54
57
|
}
|
|
55
58
|
formatIndexRecord = formatIndexRecordTemp;
|
|
56
|
-
headerRow =
|
|
59
|
+
headerRow = cells;
|
|
57
60
|
return;
|
|
58
61
|
}
|
|
59
|
-
if (
|
|
62
|
+
if (cells.every((v) => v === '')) {
|
|
60
63
|
return;
|
|
61
64
|
}
|
|
62
65
|
if (format.footerStartColumns !== undefined) {
|
|
63
66
|
let matched = true;
|
|
64
67
|
for (const [index, column] of format.footerStartColumns.entries()) {
|
|
65
|
-
if (
|
|
68
|
+
if (cells[index] !== column) {
|
|
66
69
|
matched = false;
|
|
67
70
|
break;
|
|
68
71
|
}
|
|
@@ -77,7 +80,7 @@ const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, sav
|
|
|
77
80
|
for (const columnPatterns of format.ignoreStartColumns) {
|
|
78
81
|
let matched = true;
|
|
79
82
|
for (const [index, pattern] of columnPatterns.entries()) {
|
|
80
|
-
if (!new RegExp(pattern).test(
|
|
83
|
+
if (!new RegExp(pattern).test(cells[index])) {
|
|
81
84
|
matched = false;
|
|
82
85
|
break;
|
|
83
86
|
}
|
|
@@ -90,11 +93,11 @@ const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, sav
|
|
|
90
93
|
const row = {};
|
|
91
94
|
for (const key of Object.keys(format.formatter)) {
|
|
92
95
|
const index = formatIndexRecord[key];
|
|
93
|
-
row[key] =
|
|
96
|
+
row[key] = cells[index];
|
|
94
97
|
}
|
|
95
98
|
const formattedRow = formatRow(row, format.formatter);
|
|
96
99
|
if (formattedRow === undefined) {
|
|
97
|
-
ignoreRows.push(
|
|
100
|
+
ignoreRows.push(cells);
|
|
98
101
|
ignoreRowCount += 1;
|
|
99
102
|
return;
|
|
100
103
|
}
|
|
@@ -110,7 +113,7 @@ const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, sav
|
|
|
110
113
|
fiats,
|
|
111
114
|
});
|
|
112
115
|
if (rowActions.length === 0) {
|
|
113
|
-
ignoreRows.push(
|
|
116
|
+
ignoreRows.push(cells);
|
|
114
117
|
ignoreRowCount += 1;
|
|
115
118
|
}
|
|
116
119
|
actions.push(...rowActions);
|
|
@@ -140,7 +143,7 @@ const streamCsvToActions = async ({ file, encoding, format, cryptoes, fiats, sav
|
|
|
140
143
|
currentRows = [];
|
|
141
144
|
}
|
|
142
145
|
currentKeyValue = keyValue;
|
|
143
|
-
currentRawRows.push(
|
|
146
|
+
currentRawRows.push(cells);
|
|
144
147
|
currentRows.push(formattedRow);
|
|
145
148
|
}
|
|
146
149
|
if (actions.length >= chunkSize + 1) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamCsvToActions.js","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":";;;AACA,yCAAiC;AACjC,mDAA8C;AAC9C,6DAA+D;AAC/D,kDAAyD;AAsBlD,MAAM,kBAAkB,GAAG,KAAK,EAAE,EACvC,IAAI,EACJ,QAAQ,EACR,MAAM,EACN,QAAQ,EACR,KAAK,EACL,IAAI,EACJ,SAAS,GAAG,GAAG,GACc,EAA0C,EAAE;IAEzE,IAAI,OAAO,GAAG,IAAI,CAAA;IAClB,IAAI,KAAc,CAAA;IAClB,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,SAA+B,CAAA;IACnC,MAAM,UAAU,GAAe,EAAE,CAAA;IACjC,IAAI,eAAmC,CAAA;IACvC,IAAI,WAAW,GAAmB,EAAE,CAAA;IACpC,IAAI,cAAc,GAAe,EAAE,CAAA;IACnC,IAAI,OAAO,GAAa,EAAE,CAAA;IAE1B,MAAM,wBAAwB,GAC5B,MAAM,CAAC,kBAAkB,KAAK,SAAS;QACrC,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,MAAM,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;IAEzD,IAAI,iBAAqD,CAAA;IACzD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,IAAA,iBAAK,EAAC,IAAI,EAAE;YACV,MAAM,EAAE,KAAK;YACb,cAAc,EAAE,IAAI;YACpB,QAAQ;YACR,QAAQ,EAAE,GAAG,EAAE;gBACb,OAAO,EAAE,CAAA;YACX,CAAC;YACD,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;gBAC7B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChC,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;oBAC7C,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,OAAM;gBACR,CAAC;gBACD,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACpC,IAAI,wBAAwB,KAAK,SAAS,EAAE,CAAC;wBAC3C,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,wBAAwB,CAAC,OAAO,EAAE,EAAE,CAAC;4BACjE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"streamCsvToActions.js","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":";;;AACA,yCAAiC;AACjC,mDAA8C;AAC9C,6DAA+D;AAC/D,kDAAyD;AAsBlD,MAAM,kBAAkB,GAAG,KAAK,EAAE,EACvC,IAAI,EACJ,QAAQ,EACR,MAAM,EACN,QAAQ,EACR,KAAK,EACL,IAAI,EACJ,SAAS,GAAG,GAAG,GACc,EAA0C,EAAE;IAEzE,IAAI,OAAO,GAAG,IAAI,CAAA;IAClB,IAAI,KAAc,CAAA;IAClB,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,SAA+B,CAAA;IACnC,MAAM,UAAU,GAAe,EAAE,CAAA;IACjC,IAAI,eAAmC,CAAA;IACvC,IAAI,WAAW,GAAmB,EAAE,CAAA;IACpC,IAAI,cAAc,GAAe,EAAE,CAAA;IACnC,IAAI,OAAO,GAAa,EAAE,CAAA;IAE1B,MAAM,wBAAwB,GAC5B,MAAM,CAAC,kBAAkB,KAAK,SAAS;QACrC,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,MAAM,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;IAEzD,IAAI,iBAAqD,CAAA;IACzD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,IAAA,iBAAK,EAAC,IAAI,EAAE;YACV,MAAM,EAAE,KAAK;YACb,cAAc,EAAE,IAAI;YACpB,QAAQ;YACR,QAAQ,EAAE,GAAG,EAAE;gBACb,OAAO,EAAE,CAAA;YACX,CAAC;YACD,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;gBAC7B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChC,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;oBAC7C,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,OAAM;gBACR,CAAC;gBACD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAClC,OAAO,CAAC,KAAK,QAAQ;oBACnB,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE;oBAC3D,CAAC,CAAC,CAAC,CACN,CAAA;gBACD,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACpC,IAAI,wBAAwB,KAAK,SAAS,EAAE,CAAC;wBAC3C,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,wBAAwB,CAAC,OAAO,EAAE,EAAE,CAAC;4BACjE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gCAC/B,OAAM;4BACR,CAAC;wBACH,CAAC;oBACH,CAAC;oBAED,MAAM,qBAAqB,GAA2B,EAAE,CAAA;oBACxD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;wBAChD,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;wBAChC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;4BACjB,OAAO,CAAC,IAAI,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAA;4BACnC,MAAM,CAAC,KAAK,EAAE,CAAA;4BACd,OAAM;wBACR,CAAC;wBACD,qBAAqB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;oBACpC,CAAC;oBACD,iBAAiB,GAAG,qBAAqB,CAAA;oBACzC,SAAS,GAAG,KAAK,CAAA;oBACjB,OAAM;gBACR,CAAC;gBAED,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC;oBACjC,OAAM;gBACR,CAAC;gBAGD,IAAI,MAAM,CAAC,kBAAkB,KAAK,SAAS,EAAE,CAAC;oBAC5C,IAAI,OAAO,GAAG,IAAI,CAAA;oBAClB,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,kBAAkB,CAAC,OAAO,EAAE,EAAE,CAAC;wBAClE,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,MAAM,EAAE,CAAC;4BAC5B,OAAO,GAAG,KAAK,CAAA;4BACf,MAAK;wBACP,CAAC;oBACH,CAAC;oBACD,IAAI,OAAO,EAAE,CAAC;wBACZ,MAAM,CAAC,KAAK,EAAE,CAAA;wBACd,OAAM;oBACR,CAAC;gBACH,CAAC;gBAED,QAAQ,IAAI,CAAC,CAAA;gBAGb,IAAI,MAAM,CAAC,kBAAkB,KAAK,SAAS,EAAE,CAAC;oBAC5C,KAAK,MAAM,cAAc,IAAI,MAAM,CAAC,kBAAkB,EAAE,CAAC;wBACvD,IAAI,OAAO,GAAG,IAAI,CAAA;wBAClB,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,cAAc,CAAC,OAAO,EAAE,EAAE,CAAC;4BACxD,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gCAC5C,OAAO,GAAG,KAAK,CAAA;gCACf,MAAK;4BACP,CAAC;wBACH,CAAC;wBACD,IAAI,OAAO,EAAE,CAAC;4BACZ,OAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;gBAGD,MAAM,GAAG,GAA2B,EAAE,CAAA;gBACtC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;oBAChD,MAAM,KAAK,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAA;oBACpC,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAA;gBACzB,CAAC;gBACD,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,SAAS,CAAC,CAAA;gBACrD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;oBAC/B,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBACtB,cAAc,IAAI,CAAC,CAAA;oBACnB,OAAM;gBACR,CAAC;gBAGD,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;oBACpC,cAAc,GAAG,EAAE,CAAA;oBACnB,WAAW,GAAG,EAAE,CAAA;oBAChB,MAAM,UAAU,GAAG,IAAA,6CAAwB,EAAC;wBAC1C,IAAI,EAAE,CAAC,YAAY,CAAC;wBACpB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;wBAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;wBACrC,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ;wBACR,KAAK;qBACN,CAAC,CAAA;oBACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBAC5B,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;wBACtB,cAAc,IAAI,CAAC,CAAA;oBACrB,CAAC;oBACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;gBAC7B,CAAC;qBAAM,CAAC;oBACN,MAAM,QAAQ,GAAG,IAAA,oBAAc,EAAC;wBAC9B,GAAG,EAAE,YAAY;wBACjB,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG;qBACvB,CAAC,CAAA;oBACF,IACE,eAAe,KAAK,SAAS;wBAC7B,eAAe,KAAK,QAAQ;wBAC5B,WAAW,CAAC,MAAM,GAAG,CAAC,EACtB,CAAC;wBACD,MAAM,UAAU,GAAG,IAAA,6CAAwB,EAAC;4BAC1C,IAAI,EAAE,WAAW;4BACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;4BAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;4BACrC,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,QAAQ;4BACR,KAAK;yBACN,CAAC,CAAA;wBACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BAC5B,UAAU,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,CAAA;4BAClC,cAAc,IAAI,WAAW,CAAC,MAAM,CAAA;wBACtC,CAAC;wBACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;wBAC3B,cAAc,GAAG,EAAE,CAAA;wBACnB,WAAW,GAAG,EAAE,CAAA;oBAClB,CAAC;oBACD,eAAe,GAAG,QAAQ,CAAA;oBAC1B,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBAC1B,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAChC,CAAC;gBAGD,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;oBACpC,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,MAAM,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAA;oBAC3D,MAAM,aAAa,GAAa,EAAE,CAAA;oBAClC,MAAM,gBAAgB,GAAa,EAAE,CAAA;oBACrC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;wBAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,gBAAgB,EAAE,CAAC;4BACvC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;wBAC/B,CAAC;6BAAM,CAAC;4BACN,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;wBAC5B,CAAC;oBACH,CAAC;oBACD,OAAO,GAAG,gBAAgB,CAAA;oBAC1B,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAA;oBACrC,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,OAAO,GAAG,KAAK,CAAA;wBACf,KAAK,GAAG,CAAC,CAAA;wBACT,MAAM,CAAC,KAAK,EAAE,CAAA;oBAChB,CAAC;oBACD,MAAM,CAAC,MAAM,EAAE,CAAA;gBACjB,CAAC;YACH,CAAC;SACF,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK;YACL,QAAQ;YACR,cAAc;YACd,SAAS;YACT,UAAU;SACX,CAAA;IACH,CAAC;IAGD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,IAAA,6CAAwB,EAAC;YAC1C,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;YAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,QAAQ;YACR,KAAK;SACN,CAAC,CAAA;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,UAAU,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,CAAA;YAClC,cAAc,IAAI,WAAW,CAAC,MAAM,CAAA;QACtC,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;IAC7B,CAAC;IAGD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAC/B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,GAAG,KAAK,CAAA;YACf,KAAK,GAAG,CAAC,CAAA;QACX,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO;QACP,KAAK;QACL,QAAQ;QACR,cAAc;QACd,SAAS;QACT,UAAU;KACX,CAAA;AACH,CAAC,CAAA;AAjPY,QAAA,kBAAkB,sBAiP9B;AAED,MAAM,SAAS,GAAG,CAChB,GAA2B,EAC3B,SAAiC,EACP,EAAE;IAC5B,IAAI,CAAC;QACH,OAAO,IAAA,4BAAY,EAAC,GAAG,EAAE,SAAS,CAAC,CAAA;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,SAAS,CAAA;IAClB,CAAC;AACH,CAAC,CAAA"}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { parse } from 'papaparse';
|
|
2
2
|
export const hasValidHeaderRow = async ({ file, encoding, format, }) => {
|
|
3
|
-
console.warn('start hasValidHeaderRow');
|
|
4
3
|
let isValid = false;
|
|
5
4
|
let rowCount = 0;
|
|
6
5
|
const headerStartColumnRegExps = format.headerStartColumns === undefined
|
|
@@ -9,15 +8,12 @@ export const hasValidHeaderRow = async ({ file, encoding, format, }) => {
|
|
|
9
8
|
await new Promise((resolve) => {
|
|
10
9
|
parse(file, {
|
|
11
10
|
header: false,
|
|
12
|
-
skipEmptyLines:
|
|
11
|
+
skipEmptyLines: true,
|
|
13
12
|
encoding,
|
|
14
13
|
complete: () => {
|
|
15
|
-
console.warn('complete');
|
|
16
14
|
resolve();
|
|
17
15
|
},
|
|
18
16
|
step: (result, parser) => {
|
|
19
|
-
console.warn('row:');
|
|
20
|
-
console.warn(result);
|
|
21
17
|
rowCount += 1;
|
|
22
18
|
if (format.headerCheckRowSize !== undefined &&
|
|
23
19
|
rowCount > format.headerCheckRowSize) {
|
|
@@ -29,10 +25,16 @@ export const hasValidHeaderRow = async ({ file, encoding, format, }) => {
|
|
|
29
25
|
return;
|
|
30
26
|
}
|
|
31
27
|
if (headerStartColumnRegExps !== undefined) {
|
|
32
|
-
console.warn('check row is header:');
|
|
33
|
-
console.warn(result.data);
|
|
34
28
|
for (const [index, column] of headerStartColumnRegExps.entries()) {
|
|
35
|
-
|
|
29
|
+
const cell = result.data[index];
|
|
30
|
+
if (typeof cell !== 'string') {
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
const value = cell
|
|
34
|
+
.replaceAll('\r', '')
|
|
35
|
+
.replaceAll('\u{65279}', '')
|
|
36
|
+
.trim();
|
|
37
|
+
if (!column.test(value)) {
|
|
36
38
|
return;
|
|
37
39
|
}
|
|
38
40
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hasValidHeaderRow.js","sourceRoot":"","sources":["../hasValidHeaderRow.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AASjC,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,EAAE,EACtC,IAAI,EACJ,QAAQ,EACR,MAAM,GACsB,EAAE,EAAE;IAChC,
|
|
1
|
+
{"version":3,"file":"hasValidHeaderRow.js","sourceRoot":"","sources":["../hasValidHeaderRow.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AASjC,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,EAAE,EACtC,IAAI,EACJ,QAAQ,EACR,MAAM,GACsB,EAAE,EAAE;IAChC,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAEhB,MAAM,wBAAwB,GAC5B,MAAM,CAAC,kBAAkB,KAAK,SAAS;QACrC,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,MAAM,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;IAEzD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,KAAK,CAAC,IAAI,EAAE;YACV,MAAM,EAAE,KAAK;YACb,cAAc,EAAE,IAAI;YACpB,QAAQ;YACR,QAAQ,EAAE,GAAG,EAAE;gBACb,OAAO,EAAE,CAAA;YACX,CAAC;YACD,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE;gBACvB,QAAQ,IAAI,CAAC,CAAA;gBACb,IACE,MAAM,CAAC,kBAAkB,KAAK,SAAS;oBACvC,QAAQ,GAAG,MAAM,CAAC,kBAAkB,EACpC,CAAC;oBACD,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,OAAM;gBACR,CAAC;gBACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChC,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,OAAM;gBACR,CAAC;gBAED,IAAI,wBAAwB,KAAK,SAAS,EAAE,CAAC;oBAC3C,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,wBAAwB,CAAC,OAAO,EAAE,EAAE,CAAC;wBACjE,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;wBAC/B,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;4BAC7B,OAAM;wBACR,CAAC;wBACD,MAAM,KAAK,GAAG,IAAI;6BACf,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC;6BACpB,UAAU,CAAC,WAAW,EAAE,EAAE,CAAC;6BAC3B,IAAI,EAAE,CAAA;wBACT,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;4BACxB,OAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;oBAChD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC/B,MAAM,CAAC,KAAK,EAAE,CAAA;wBACd,OAAM;oBACR,CAAC;gBACH,CAAC;gBACD,OAAO,GAAG,IAAI,CAAA;gBACd,MAAM,CAAC,KAAK,EAAE,CAAA;YAChB,CAAC;SACF,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,OAAO,OAAO,CAAA;AAChB,CAAC,CAAA"}
|
|
@@ -32,17 +32,20 @@ export const streamCsvToActions = async ({ file, encoding, format, cryptoes, fia
|
|
|
32
32
|
parser.abort();
|
|
33
33
|
return;
|
|
34
34
|
}
|
|
35
|
+
const cells = result.data.map((v) => typeof v === 'string'
|
|
36
|
+
? v.replaceAll('\r', '').replaceAll('\u{65279}', '').trim()
|
|
37
|
+
: v);
|
|
35
38
|
if (formatIndexRecord === undefined) {
|
|
36
39
|
if (headerStartColumnRegExps !== undefined) {
|
|
37
40
|
for (const [index, column] of headerStartColumnRegExps.entries()) {
|
|
38
|
-
if (!column.test(
|
|
41
|
+
if (!column.test(cells[index])) {
|
|
39
42
|
return;
|
|
40
43
|
}
|
|
41
44
|
}
|
|
42
45
|
}
|
|
43
46
|
const formatIndexRecordTemp = {};
|
|
44
47
|
for (const key of Object.keys(format.formatter)) {
|
|
45
|
-
const index =
|
|
48
|
+
const index = cells.indexOf(key);
|
|
46
49
|
if (index === -1) {
|
|
47
50
|
console.warn('key not found:', key);
|
|
48
51
|
parser.abort();
|
|
@@ -51,17 +54,17 @@ export const streamCsvToActions = async ({ file, encoding, format, cryptoes, fia
|
|
|
51
54
|
formatIndexRecordTemp[key] = index;
|
|
52
55
|
}
|
|
53
56
|
formatIndexRecord = formatIndexRecordTemp;
|
|
54
|
-
headerRow =
|
|
57
|
+
headerRow = cells;
|
|
55
58
|
return;
|
|
56
59
|
}
|
|
57
|
-
if (
|
|
60
|
+
if (cells.every((v) => v === '')) {
|
|
58
61
|
return;
|
|
59
62
|
}
|
|
60
63
|
// end analysis if footerStartColumns is matched
|
|
61
64
|
if (format.footerStartColumns !== undefined) {
|
|
62
65
|
let matched = true;
|
|
63
66
|
for (const [index, column] of format.footerStartColumns.entries()) {
|
|
64
|
-
if (
|
|
67
|
+
if (cells[index] !== column) {
|
|
65
68
|
matched = false;
|
|
66
69
|
break;
|
|
67
70
|
}
|
|
@@ -77,7 +80,7 @@ export const streamCsvToActions = async ({ file, encoding, format, cryptoes, fia
|
|
|
77
80
|
for (const columnPatterns of format.ignoreStartColumns) {
|
|
78
81
|
let matched = true;
|
|
79
82
|
for (const [index, pattern] of columnPatterns.entries()) {
|
|
80
|
-
if (!new RegExp(pattern).test(
|
|
83
|
+
if (!new RegExp(pattern).test(cells[index])) {
|
|
81
84
|
matched = false;
|
|
82
85
|
break;
|
|
83
86
|
}
|
|
@@ -91,11 +94,11 @@ export const streamCsvToActions = async ({ file, encoding, format, cryptoes, fia
|
|
|
91
94
|
const row = {};
|
|
92
95
|
for (const key of Object.keys(format.formatter)) {
|
|
93
96
|
const index = formatIndexRecord[key];
|
|
94
|
-
row[key] =
|
|
97
|
+
row[key] = cells[index];
|
|
95
98
|
}
|
|
96
99
|
const formattedRow = formatRow(row, format.formatter);
|
|
97
100
|
if (formattedRow === undefined) {
|
|
98
|
-
ignoreRows.push(
|
|
101
|
+
ignoreRows.push(cells);
|
|
99
102
|
ignoreRowCount += 1;
|
|
100
103
|
return;
|
|
101
104
|
}
|
|
@@ -112,7 +115,7 @@ export const streamCsvToActions = async ({ file, encoding, format, cryptoes, fia
|
|
|
112
115
|
fiats,
|
|
113
116
|
});
|
|
114
117
|
if (rowActions.length === 0) {
|
|
115
|
-
ignoreRows.push(
|
|
118
|
+
ignoreRows.push(cells);
|
|
116
119
|
ignoreRowCount += 1;
|
|
117
120
|
}
|
|
118
121
|
actions.push(...rowActions);
|
|
@@ -142,7 +145,7 @@ export const streamCsvToActions = async ({ file, encoding, format, cryptoes, fia
|
|
|
142
145
|
currentRows = [];
|
|
143
146
|
}
|
|
144
147
|
currentKeyValue = keyValue;
|
|
145
|
-
currentRawRows.push(
|
|
148
|
+
currentRawRows.push(cells);
|
|
146
149
|
currentRows.push(formattedRow);
|
|
147
150
|
}
|
|
148
151
|
// actions が chunkSize に達したら最後の action source と同一 source を除いて actions を保存
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamCsvToActions.js","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAA;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAsBzD,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EAAE,EACvC,IAAI,EACJ,QAAQ,EACR,MAAM,EACN,QAAQ,EACR,KAAK,EACL,IAAI,EACJ,SAAS,GAAG,GAAG,GACc,EAA0C,EAAE;IACzE,aAAa;IACb,IAAI,OAAO,GAAG,IAAI,CAAA;IAClB,IAAI,KAAc,CAAA;IAClB,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,SAA+B,CAAA;IACnC,MAAM,UAAU,GAAe,EAAE,CAAA;IACjC,IAAI,eAAmC,CAAA;IACvC,IAAI,WAAW,GAAmB,EAAE,CAAA;IACpC,IAAI,cAAc,GAAe,EAAE,CAAA;IACnC,IAAI,OAAO,GAAa,EAAE,CAAA;IAE1B,MAAM,wBAAwB,GAC5B,MAAM,CAAC,kBAAkB,KAAK,SAAS;QACrC,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,MAAM,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;IAEzD,IAAI,iBAAqD,CAAA;IACzD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,KAAK,CAAC,IAAI,EAAE;YACV,MAAM,EAAE,KAAK;YACb,cAAc,EAAE,IAAI;YACpB,QAAQ;YACR,QAAQ,EAAE,GAAG,EAAE;gBACb,OAAO,EAAE,CAAA;YACX,CAAC;YACD,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;gBAC7B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChC,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;oBAC7C,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,OAAM;gBACR,CAAC;gBACD,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACpC,IAAI,wBAAwB,KAAK,SAAS,EAAE,CAAC;wBAC3C,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,wBAAwB,CAAC,OAAO,EAAE,EAAE,CAAC;4BACjE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"streamCsvToActions.js","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAA;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAsBzD,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EAAE,EACvC,IAAI,EACJ,QAAQ,EACR,MAAM,EACN,QAAQ,EACR,KAAK,EACL,IAAI,EACJ,SAAS,GAAG,GAAG,GACc,EAA0C,EAAE;IACzE,aAAa;IACb,IAAI,OAAO,GAAG,IAAI,CAAA;IAClB,IAAI,KAAc,CAAA;IAClB,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,SAA+B,CAAA;IACnC,MAAM,UAAU,GAAe,EAAE,CAAA;IACjC,IAAI,eAAmC,CAAA;IACvC,IAAI,WAAW,GAAmB,EAAE,CAAA;IACpC,IAAI,cAAc,GAAe,EAAE,CAAA;IACnC,IAAI,OAAO,GAAa,EAAE,CAAA;IAE1B,MAAM,wBAAwB,GAC5B,MAAM,CAAC,kBAAkB,KAAK,SAAS;QACrC,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,MAAM,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;IAEzD,IAAI,iBAAqD,CAAA;IACzD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,KAAK,CAAC,IAAI,EAAE;YACV,MAAM,EAAE,KAAK;YACb,cAAc,EAAE,IAAI;YACpB,QAAQ;YACR,QAAQ,EAAE,GAAG,EAAE;gBACb,OAAO,EAAE,CAAA;YACX,CAAC;YACD,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;gBAC7B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChC,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;oBAC7C,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,OAAM;gBACR,CAAC;gBACD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAClC,OAAO,CAAC,KAAK,QAAQ;oBACnB,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE;oBAC3D,CAAC,CAAC,CAAC,CACN,CAAA;gBACD,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACpC,IAAI,wBAAwB,KAAK,SAAS,EAAE,CAAC;wBAC3C,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,wBAAwB,CAAC,OAAO,EAAE,EAAE,CAAC;4BACjE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gCAC/B,OAAM;4BACR,CAAC;wBACH,CAAC;oBACH,CAAC;oBAED,MAAM,qBAAqB,GAA2B,EAAE,CAAA;oBACxD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;wBAChD,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;wBAChC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;4BACjB,OAAO,CAAC,IAAI,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAA;4BACnC,MAAM,CAAC,KAAK,EAAE,CAAA;4BACd,OAAM;wBACR,CAAC;wBACD,qBAAqB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;oBACpC,CAAC;oBACD,iBAAiB,GAAG,qBAAqB,CAAA;oBACzC,SAAS,GAAG,KAAK,CAAA;oBACjB,OAAM;gBACR,CAAC;gBAED,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC;oBACjC,OAAM;gBACR,CAAC;gBAED,gDAAgD;gBAChD,IAAI,MAAM,CAAC,kBAAkB,KAAK,SAAS,EAAE,CAAC;oBAC5C,IAAI,OAAO,GAAG,IAAI,CAAA;oBAClB,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,kBAAkB,CAAC,OAAO,EAAE,EAAE,CAAC;wBAClE,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,MAAM,EAAE,CAAC;4BAC5B,OAAO,GAAG,KAAK,CAAA;4BACf,MAAK;wBACP,CAAC;oBACH,CAAC;oBACD,IAAI,OAAO,EAAE,CAAC;wBACZ,MAAM,CAAC,KAAK,EAAE,CAAA;wBACd,OAAM;oBACR,CAAC;gBACH,CAAC;gBAED,QAAQ,IAAI,CAAC,CAAA;gBAEb,8CAA8C;gBAC9C,IAAI,MAAM,CAAC,kBAAkB,KAAK,SAAS,EAAE,CAAC;oBAC5C,KAAK,MAAM,cAAc,IAAI,MAAM,CAAC,kBAAkB,EAAE,CAAC;wBACvD,IAAI,OAAO,GAAG,IAAI,CAAA;wBAClB,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,cAAc,CAAC,OAAO,EAAE,EAAE,CAAC;4BACxD,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gCAC5C,OAAO,GAAG,KAAK,CAAA;gCACf,MAAK;4BACP,CAAC;wBACH,CAAC;wBACD,IAAI,OAAO,EAAE,CAAC;4BACZ,OAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,SAAS;gBACT,MAAM,GAAG,GAA2B,EAAE,CAAA;gBACtC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;oBAChD,MAAM,KAAK,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAA;oBACpC,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAA;gBACzB,CAAC;gBACD,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,SAAS,CAAC,CAAA;gBACrD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;oBAC/B,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBACtB,cAAc,IAAI,CAAC,CAAA;oBACnB,OAAM;gBACR,CAAC;gBAED,QAAQ;gBACR,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;oBACpC,cAAc,GAAG,EAAE,CAAA;oBACnB,WAAW,GAAG,EAAE,CAAA;oBAChB,MAAM,UAAU,GAAG,wBAAwB,CAAC;wBAC1C,IAAI,EAAE,CAAC,YAAY,CAAC;wBACpB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;wBAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;wBACrC,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ;wBACR,KAAK;qBACN,CAAC,CAAA;oBACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBAC5B,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;wBACtB,cAAc,IAAI,CAAC,CAAA;oBACrB,CAAC;oBACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;gBAC7B,CAAC;qBAAM,CAAC;oBACN,MAAM,QAAQ,GAAG,cAAc,CAAC;wBAC9B,GAAG,EAAE,YAAY;wBACjB,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG;qBACvB,CAAC,CAAA;oBACF,IACE,eAAe,KAAK,SAAS;wBAC7B,eAAe,KAAK,QAAQ;wBAC5B,WAAW,CAAC,MAAM,GAAG,CAAC,EACtB,CAAC;wBACD,MAAM,UAAU,GAAG,wBAAwB,CAAC;4BAC1C,IAAI,EAAE,WAAW;4BACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;4BAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;4BACrC,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,QAAQ;4BACR,KAAK;yBACN,CAAC,CAAA;wBACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BAC5B,UAAU,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,CAAA;4BAClC,cAAc,IAAI,WAAW,CAAC,MAAM,CAAA;wBACtC,CAAC;wBACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;wBAC3B,cAAc,GAAG,EAAE,CAAA;wBACnB,WAAW,GAAG,EAAE,CAAA;oBAClB,CAAC;oBACD,eAAe,GAAG,QAAQ,CAAA;oBAC1B,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBAC1B,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAChC,CAAC;gBAED,yEAAyE;gBACzE,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;oBACpC,MAAM,CAAC,KAAK,EAAE,CAAA;oBACd,MAAM,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAA;oBAC3D,MAAM,aAAa,GAAa,EAAE,CAAA;oBAClC,MAAM,gBAAgB,GAAa,EAAE,CAAA;oBACrC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;wBAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,gBAAgB,EAAE,CAAC;4BACvC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;wBAC/B,CAAC;6BAAM,CAAC;4BACN,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;wBAC5B,CAAC;oBACH,CAAC;oBACD,OAAO,GAAG,gBAAgB,CAAA;oBAC1B,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAA;oBACrC,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,OAAO,GAAG,KAAK,CAAA;wBACf,KAAK,GAAG,CAAC,CAAA;wBACT,MAAM,CAAC,KAAK,EAAE,CAAA;oBAChB,CAAC;oBACD,MAAM,CAAC,MAAM,EAAE,CAAA;gBACjB,CAAC;YACH,CAAC;SACF,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK;YACL,QAAQ;YACR,cAAc;YACd,SAAS;YACT,UAAU;SACX,CAAA;IACH,CAAC;IAED,kCAAkC;IAClC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,wBAAwB,CAAC;YAC1C,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;YAC9B,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,QAAQ;YACR,KAAK;SACN,CAAC,CAAA;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,UAAU,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,CAAA;YAClC,cAAc,IAAI,WAAW,CAAC,MAAM,CAAA;QACtC,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;IAC7B,CAAC;IAED,+BAA+B;IAC/B,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAC/B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,GAAG,KAAK,CAAA;YACf,KAAK,GAAG,CAAC,CAAA;QACX,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO;QACP,KAAK;QACL,QAAQ;QACR,cAAc;QACd,SAAS;QACT,UAAU;KACX,CAAA;AACH,CAAC,CAAA;AAED,MAAM,SAAS,GAAG,CAChB,GAA2B,EAC3B,SAAiC,EACP,EAAE;IAC5B,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,GAAG,EAAE,SAAS,CAAC,CAAA;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,SAAS,CAAA;IAClB,CAAC;AACH,CAAC,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamCsvToActions.d.ts","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAKzE,OAAO,KAAK,EAAE,SAAS,EAAgB,MAAM,SAAS,CAAA;AAEtD,KAAK,4BAA4B,GAAG;IAClC,IAAI,EAAE,IAAI,GAAG,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,SAAS,CAAA;IACjB,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAA;IACnC,KAAK,EAAE,SAAS,YAAY,EAAE,CAAA;IAC9B,IAAI,EAAE,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IACrE,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,CAAA;AAED,KAAK,6BAA6B,GAAG;IACnC,OAAO,EAAE,OAAO,CAAA;IAChB,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;IAChB,cAAc,EAAE,MAAM,CAAA;IACtB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAA;IACpB,UAAU,EAAE,MAAM,EAAE,EAAE,CAAA;CACvB,CAAA;AAED,eAAO,MAAM,kBAAkB,kEAQ5B,4BAA4B,KAAG,OAAO,CAAC,6BAA6B,
|
|
1
|
+
{"version":3,"file":"streamCsvToActions.d.ts","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAKzE,OAAO,KAAK,EAAE,SAAS,EAAgB,MAAM,SAAS,CAAA;AAEtD,KAAK,4BAA4B,GAAG;IAClC,IAAI,EAAE,IAAI,GAAG,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,SAAS,CAAA;IACjB,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAA;IACnC,KAAK,EAAE,SAAS,YAAY,EAAE,CAAA;IAC9B,IAAI,EAAE,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IACrE,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,CAAA;AAED,KAAK,6BAA6B,GAAG;IACnC,OAAO,EAAE,OAAO,CAAA;IAChB,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;IAChB,cAAc,EAAE,MAAM,CAAA;IACtB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAA;IACpB,UAAU,EAAE,MAAM,EAAE,EAAE,CAAA;CACvB,CAAA;AAED,eAAO,MAAM,kBAAkB,kEAQ5B,4BAA4B,KAAG,OAAO,CAAC,6BAA6B,CAyOtE,CAAA"}
|
package/hasValidHeaderRow.ts
CHANGED
|
@@ -12,8 +12,6 @@ export const hasValidHeaderRow = async ({
|
|
|
12
12
|
encoding,
|
|
13
13
|
format,
|
|
14
14
|
}: HasValidHeaderRowParameters) => {
|
|
15
|
-
console.warn('start hasValidHeaderRow')
|
|
16
|
-
|
|
17
15
|
let isValid = false
|
|
18
16
|
let rowCount = 0
|
|
19
17
|
|
|
@@ -25,16 +23,12 @@ export const hasValidHeaderRow = async ({
|
|
|
25
23
|
await new Promise<void>((resolve) => {
|
|
26
24
|
parse(file, {
|
|
27
25
|
header: false,
|
|
28
|
-
skipEmptyLines:
|
|
26
|
+
skipEmptyLines: true,
|
|
29
27
|
encoding,
|
|
30
28
|
complete: () => {
|
|
31
|
-
console.warn('complete')
|
|
32
29
|
resolve()
|
|
33
30
|
},
|
|
34
31
|
step: (result, parser) => {
|
|
35
|
-
console.warn('row:')
|
|
36
|
-
console.warn(result)
|
|
37
|
-
|
|
38
32
|
rowCount += 1
|
|
39
33
|
if (
|
|
40
34
|
format.headerCheckRowSize !== undefined &&
|
|
@@ -49,10 +43,16 @@ export const hasValidHeaderRow = async ({
|
|
|
49
43
|
}
|
|
50
44
|
|
|
51
45
|
if (headerStartColumnRegExps !== undefined) {
|
|
52
|
-
console.warn('check row is header:')
|
|
53
|
-
console.warn(result.data)
|
|
54
46
|
for (const [index, column] of headerStartColumnRegExps.entries()) {
|
|
55
|
-
|
|
47
|
+
const cell = result.data[index]
|
|
48
|
+
if (typeof cell !== 'string') {
|
|
49
|
+
return
|
|
50
|
+
}
|
|
51
|
+
const value = cell
|
|
52
|
+
.replaceAll('\r', '')
|
|
53
|
+
.replaceAll('\u{65279}', '')
|
|
54
|
+
.trim()
|
|
55
|
+
if (!column.test(value)) {
|
|
56
56
|
return
|
|
57
57
|
}
|
|
58
58
|
}
|
package/package.json
CHANGED
package/streamCsvToActions.ts
CHANGED
|
@@ -65,10 +65,15 @@ export const streamCsvToActions = async ({
|
|
|
65
65
|
parser.abort()
|
|
66
66
|
return
|
|
67
67
|
}
|
|
68
|
+
const cells = result.data.map((v) =>
|
|
69
|
+
typeof v === 'string'
|
|
70
|
+
? v.replaceAll('\r', '').replaceAll('\u{65279}', '').trim()
|
|
71
|
+
: v,
|
|
72
|
+
)
|
|
68
73
|
if (formatIndexRecord === undefined) {
|
|
69
74
|
if (headerStartColumnRegExps !== undefined) {
|
|
70
75
|
for (const [index, column] of headerStartColumnRegExps.entries()) {
|
|
71
|
-
if (!column.test(
|
|
76
|
+
if (!column.test(cells[index])) {
|
|
72
77
|
return
|
|
73
78
|
}
|
|
74
79
|
}
|
|
@@ -76,7 +81,7 @@ export const streamCsvToActions = async ({
|
|
|
76
81
|
|
|
77
82
|
const formatIndexRecordTemp: Record<string, number> = {}
|
|
78
83
|
for (const key of Object.keys(format.formatter)) {
|
|
79
|
-
const index =
|
|
84
|
+
const index = cells.indexOf(key)
|
|
80
85
|
if (index === -1) {
|
|
81
86
|
console.warn('key not found:', key)
|
|
82
87
|
parser.abort()
|
|
@@ -85,11 +90,11 @@ export const streamCsvToActions = async ({
|
|
|
85
90
|
formatIndexRecordTemp[key] = index
|
|
86
91
|
}
|
|
87
92
|
formatIndexRecord = formatIndexRecordTemp
|
|
88
|
-
headerRow =
|
|
93
|
+
headerRow = cells
|
|
89
94
|
return
|
|
90
95
|
}
|
|
91
96
|
|
|
92
|
-
if (
|
|
97
|
+
if (cells.every((v) => v === '')) {
|
|
93
98
|
return
|
|
94
99
|
}
|
|
95
100
|
|
|
@@ -97,7 +102,7 @@ export const streamCsvToActions = async ({
|
|
|
97
102
|
if (format.footerStartColumns !== undefined) {
|
|
98
103
|
let matched = true
|
|
99
104
|
for (const [index, column] of format.footerStartColumns.entries()) {
|
|
100
|
-
if (
|
|
105
|
+
if (cells[index] !== column) {
|
|
101
106
|
matched = false
|
|
102
107
|
break
|
|
103
108
|
}
|
|
@@ -115,7 +120,7 @@ export const streamCsvToActions = async ({
|
|
|
115
120
|
for (const columnPatterns of format.ignoreStartColumns) {
|
|
116
121
|
let matched = true
|
|
117
122
|
for (const [index, pattern] of columnPatterns.entries()) {
|
|
118
|
-
if (!new RegExp(pattern).test(
|
|
123
|
+
if (!new RegExp(pattern).test(cells[index])) {
|
|
119
124
|
matched = false
|
|
120
125
|
break
|
|
121
126
|
}
|
|
@@ -130,11 +135,11 @@ export const streamCsvToActions = async ({
|
|
|
130
135
|
const row: Record<string, string> = {}
|
|
131
136
|
for (const key of Object.keys(format.formatter)) {
|
|
132
137
|
const index = formatIndexRecord[key]
|
|
133
|
-
row[key] =
|
|
138
|
+
row[key] = cells[index]
|
|
134
139
|
}
|
|
135
140
|
const formattedRow = formatRow(row, format.formatter)
|
|
136
141
|
if (formattedRow === undefined) {
|
|
137
|
-
ignoreRows.push(
|
|
142
|
+
ignoreRows.push(cells)
|
|
138
143
|
ignoreRowCount += 1
|
|
139
144
|
return
|
|
140
145
|
}
|
|
@@ -152,7 +157,7 @@ export const streamCsvToActions = async ({
|
|
|
152
157
|
fiats,
|
|
153
158
|
})
|
|
154
159
|
if (rowActions.length === 0) {
|
|
155
|
-
ignoreRows.push(
|
|
160
|
+
ignoreRows.push(cells)
|
|
156
161
|
ignoreRowCount += 1
|
|
157
162
|
}
|
|
158
163
|
actions.push(...rowActions)
|
|
@@ -183,7 +188,7 @@ export const streamCsvToActions = async ({
|
|
|
183
188
|
currentRows = []
|
|
184
189
|
}
|
|
185
190
|
currentKeyValue = keyValue
|
|
186
|
-
currentRawRows.push(
|
|
191
|
+
currentRawRows.push(cells)
|
|
187
192
|
currentRows.push(formattedRow)
|
|
188
193
|
}
|
|
189
194
|
|