@node-red/nodes 3.1.7 → 4.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. package/core/common/21-debug.js +2 -1
  2. package/core/common/lib/debug/debug-utils.js +2 -1
  3. package/core/function/15-change.js +4 -2
  4. package/core/network/22-websocket.html +206 -2
  5. package/core/network/22-websocket.js +37 -0
  6. package/core/network/31-tcpin.js +52 -16
  7. package/core/parsers/70-CSV.html +39 -4
  8. package/core/parsers/70-CSV.js +612 -260
  9. package/core/parsers/70-HTML.html +14 -1
  10. package/core/parsers/70-HTML.js +11 -0
  11. package/core/parsers/lib/csv/index.js +324 -0
  12. package/core/sequence/17-split.html +13 -5
  13. package/core/sequence/17-split.js +54 -34
  14. package/locales/de/messages.json +2 -1
  15. package/locales/en-US/messages.json +15 -5
  16. package/locales/en-US/network/31-tcpin.html +4 -0
  17. package/locales/en-US/parsers/70-CSV.html +4 -1
  18. package/locales/fr/messages.json +2 -1
  19. package/locales/ja/messages.json +2 -1
  20. package/locales/ko/messages.json +2 -1
  21. package/locales/pt-BR/messages.json +2 -1
  22. package/locales/ru/messages.json +2 -1
  23. package/locales/zh-CN/messages.json +2 -1
  24. package/locales/zh-TW/messages.json +2 -1
  25. package/package.json +1 -1
package/core/parsers/70-CSV.js CHANGED
@@ -15,322 +15,674 @@
15
15
  **/
16
16
 
17
17
  module.exports = function(RED) {
18
+ const csv = require('./lib/csv')
19
+
18
20
  "use strict";
19
21
  function CSVNode(n) {
20
- RED.nodes.createNode(this,n);
21
- this.template = (n.temp || "");
22
- this.sep = (n.sep || ',').replace(/\\t/g,"\t").replace(/\\n/g,"\n").replace(/\\r/g,"\r");
23
- this.quo = '"';
24
- this.ret = (n.ret || "\n").replace(/\\n/g,"\n").replace(/\\r/g,"\r");
25
- this.winflag = (this.ret === "\r\n");
26
- this.lineend = "\n";
27
- this.multi = n.multi || "one";
28
- this.hdrin = n.hdrin || false;
29
- this.hdrout = n.hdrout || "none";
30
- this.goodtmpl = true;
31
- this.skip = parseInt(n.skip || 0);
32
- this.store = [];
33
- this.parsestrings = n.strings;
34
- this.include_empty_strings = n.include_empty_strings || false;
35
- this.include_null_values = n.include_null_values || false;
36
- if (this.parsestrings === undefined) { this.parsestrings = true; }
37
- if (this.hdrout === false) { this.hdrout = "none"; }
38
- if (this.hdrout === true) { this.hdrout = "all"; }
39
- var tmpwarn = true;
40
- var node = this;
41
- var re = new RegExp(node.sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') + '(?=(?:(?:[^"]*"){2})*[^"]*$)','g');
22
+ RED.nodes.createNode(this,n)
23
+ const node = this
24
+ const RFC4180Mode = n.spec === 'rfc'
25
+ const legacyMode = !RFC4180Mode
42
26
 
43
- // pass in an array of column names to be trimmed, de-quoted and retrimmed
44
- var clean = function(col,sep) {
45
- if (sep) { re = new RegExp(sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') +'(?=(?:(?:[^"]*"){2})*[^"]*$)','g'); }
46
- col = col.trim().split(re) || [""];
47
- col = col.map(x => x.replace(/"/g,'').trim());
48
- if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; }
49
- else { node.goodtmpl = true; }
50
- return col;
51
- }
52
- var template = clean(node.template,',');
53
- var notemplate = template.length === 1 && template[0] === '';
54
- node.hdrSent = false;
27
+ node.status({}) // clear status
55
28
 
56
- this.on("input", function(msg, send, done) {
57
- if (msg.hasOwnProperty("reset")) {
58
- node.hdrSent = false;
29
+ if (legacyMode) {
30
+ this.template = (n.temp || "");
31
+ this.sep = (n.sep || ',').replace(/\\t/g,"\t").replace(/\\n/g,"\n").replace(/\\r/g,"\r");
32
+ this.quo = '"';
33
+ this.ret = (n.ret || "\n").replace(/\\n/g,"\n").replace(/\\r/g,"\r");
34
+ this.winflag = (this.ret === "\r\n");
35
+ this.lineend = "\n";
36
+ this.multi = n.multi || "one";
37
+ this.hdrin = n.hdrin || false;
38
+ this.hdrout = n.hdrout || "none";
39
+ this.goodtmpl = true;
40
+ this.skip = parseInt(n.skip || 0);
41
+ this.store = [];
42
+ this.parsestrings = n.strings;
43
+ this.include_empty_strings = n.include_empty_strings || false;
44
+ this.include_null_values = n.include_null_values || false;
45
+ if (this.parsestrings === undefined) { this.parsestrings = true; }
46
+ if (this.hdrout === false) { this.hdrout = "none"; }
47
+ if (this.hdrout === true) { this.hdrout = "all"; }
48
+ var tmpwarn = true;
49
+ // var node = this;
50
+ var re = new RegExp(node.sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') + '(?=(?:(?:[^"]*"){2})*[^"]*$)','g');
51
+
52
+ // pass in an array of column names to be trimmed, de-quoted and retrimmed
53
+ var clean = function(col,sep) {
54
+ if (sep) { re = new RegExp(sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') +'(?=(?:(?:[^"]*"){2})*[^"]*$)','g'); }
55
+ col = col.trim().split(re) || [""];
56
+ col = col.map(x => x.replace(/"/g,'').trim());
57
+ if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; }
58
+ else { node.goodtmpl = true; }
59
+ return col;
59
60
  }
60
- if (msg.hasOwnProperty("payload")) {
61
- if (typeof msg.payload == "object") { // convert object to CSV string
62
- try {
63
- if (!(notemplate && (msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("index") && msg.parts.index > 0))) {
64
- template = clean(node.template);
65
- }
66
- const ou = [];
67
- if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; }
68
- if (node.hdrout !== "none" && node.hdrSent === false) {
69
- if ((template.length === 1) && (template[0] === '')) {
70
- if (msg.hasOwnProperty("columns")) {
71
- template = clean(msg.columns || "",",");
72
- }
73
- else {
74
- template = Object.keys(msg.payload[0]);
75
- }
61
+ var template = clean(node.template,',');
62
+ var notemplate = template.length === 1 && template[0] === '';
63
+ node.hdrSent = false;
64
+
65
+ this.on("input", function(msg, send, done) {
66
+ if (msg.hasOwnProperty("reset")) {
67
+ node.hdrSent = false;
68
+ }
69
+ if (msg.hasOwnProperty("payload")) {
70
+ if (typeof msg.payload == "object") { // convert object to CSV string
71
+ try {
72
+ if (!(notemplate && (msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("index") && msg.parts.index > 0))) {
73
+ template = clean(node.template);
76
74
  }
77
- ou.push(template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).join(node.sep));
78
- if (node.hdrout === "once") { node.hdrSent = true; }
79
- }
80
- for (var s = 0; s < msg.payload.length; s++) {
81
- if ((Array.isArray(msg.payload[s])) || (typeof msg.payload[s] !== "object")) {
82
- if (typeof msg.payload[s] !== "object") { msg.payload = [ msg.payload ]; }
83
- for (var t = 0; t < msg.payload[s].length; t++) {
84
- if (msg.payload[s][t] === undefined) { msg.payload[s][t] = ""; }
85
- if (msg.payload[s][t].toString().indexOf(node.quo) !== -1) { // add double quotes if any quotes
86
- msg.payload[s][t] = msg.payload[s][t].toString().replace(/"/g, '""');
87
- msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
88
- }
89
- else if (msg.payload[s][t].toString().indexOf(node.sep) !== -1) { // add quotes if any "commas"
90
- msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
75
+ const ou = [];
76
+ if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; }
77
+ if (node.hdrout !== "none" && node.hdrSent === false) {
78
+ if ((template.length === 1) && (template[0] === '')) {
79
+ if (msg.hasOwnProperty("columns")) {
80
+ template = clean(msg.columns || "",",");
91
81
  }
92
- else if (msg.payload[s][t].toString().indexOf("\n") !== -1) { // add quotes if any "\n"
93
- msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
82
+ else {
83
+ template = Object.keys(msg.payload[0]);
94
84
  }
95
85
  }
96
- ou.push(msg.payload[s].join(node.sep));
86
+ ou.push(template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).join(node.sep));
87
+ if (node.hdrout === "once") { node.hdrSent = true; }
97
88
  }
98
- else {
99
- if ((template.length === 1) && (template[0] === '') && (msg.hasOwnProperty("columns"))) {
100
- template = clean(msg.columns || "",",");
89
+ for (var s = 0; s < msg.payload.length; s++) {
90
+ if ((Array.isArray(msg.payload[s])) || (typeof msg.payload[s] !== "object")) {
91
+ if (typeof msg.payload[s] !== "object") { msg.payload = [ msg.payload ]; }
92
+ for (var t = 0; t < msg.payload[s].length; t++) {
93
+ if (msg.payload[s][t] === undefined) { msg.payload[s][t] = ""; }
94
+ if (msg.payload[s][t].toString().indexOf(node.quo) !== -1) { // add double quotes if any quotes
95
+ msg.payload[s][t] = msg.payload[s][t].toString().replace(/"/g, '""');
96
+ msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
97
+ }
98
+ else if (msg.payload[s][t].toString().indexOf(node.sep) !== -1) { // add quotes if any "commas"
99
+ msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
100
+ }
101
+ else if (msg.payload[s][t].toString().indexOf("\n") !== -1) { // add quotes if any "\n"
102
+ msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
103
+ }
104
+ }
105
+ ou.push(msg.payload[s].join(node.sep));
101
106
  }
102
- if ((template.length === 1) && (template[0] === '')) {
103
- /* istanbul ignore else */
104
- if (tmpwarn === true) { // just warn about missing template once
105
- node.warn(RED._("csv.errors.obj_csv"));
106
- tmpwarn = false;
107
+ else {
108
+ if ((template.length === 1) && (template[0] === '') && (msg.hasOwnProperty("columns"))) {
109
+ template = clean(msg.columns || "",",");
107
110
  }
108
- const row = [];
109
- for (var p in msg.payload[0]) {
111
+ if ((template.length === 1) && (template[0] === '')) {
110
112
  /* istanbul ignore else */
111
- if (msg.payload[s].hasOwnProperty(p)) {
113
+ if (tmpwarn === true) { // just warn about missing template once
114
+ node.warn(RED._("csv.errors.obj_csv"));
115
+ tmpwarn = false;
116
+ }
117
+ const row = [];
118
+ for (var p in msg.payload[0]) {
112
119
  /* istanbul ignore else */
113
- if (typeof msg.payload[s][p] !== "object") {
114
- // Fix to honour include null values flag
115
- //if (typeof msg.payload[s][p] !== "object" || (node.include_null_values === true && msg.payload[s][p] === null)) {
116
- var q = "";
117
- if (msg.payload[s][p] !== undefined) {
118
- q += msg.payload[s][p];
120
+ if (msg.payload[s].hasOwnProperty(p)) {
121
+ /* istanbul ignore else */
122
+ if (typeof msg.payload[s][p] !== "object") {
123
+ // Fix to honour include null values flag
124
+ //if (typeof msg.payload[s][p] !== "object" || (node.include_null_values === true && msg.payload[s][p] === null)) {
125
+ var q = "";
126
+ if (msg.payload[s][p] !== undefined) {
127
+ q += msg.payload[s][p];
128
+ }
129
+ if (q.indexOf(node.quo) !== -1) { // add double quotes if any quotes
130
+ q = q.replace(/"/g, '""');
131
+ row.push(node.quo + q + node.quo);
132
+ }
133
+ else if (q.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n"
134
+ row.push(node.quo + q + node.quo);
135
+ }
136
+ else { row.push(q); } // otherwise just add
119
137
  }
120
- if (q.indexOf(node.quo) !== -1) { // add double quotes if any quotes
121
- q = q.replace(/"/g, '""');
122
- row.push(node.quo + q + node.quo);
138
+ }
139
+ }
140
+ ou.push(row.join(node.sep)); // add separator
141
+ }
142
+ else {
143
+ const row = [];
144
+ for (var t=0; t < template.length; t++) {
145
+ if (template[t] === '') {
146
+ row.push('');
147
+ }
148
+ else {
149
+ var tt = template[t];
150
+ if (template[t].indexOf('"') >=0 ) { tt = "'"+tt+"'"; }
151
+ else { tt = '"'+tt+'"'; }
152
+ var p = RED.util.getMessageProperty(msg,'payload["'+s+'"]['+tt+']');
153
+ /* istanbul ignore else */
154
+ if (p === undefined) { p = ""; }
155
+ // fix to honour include null values flag
156
+ //if (p === null && node.include_null_values !== true) { p = "";}
157
+ p = RED.util.ensureString(p);
158
+ if (p.indexOf(node.quo) !== -1) { // add double quotes if any quotes
159
+ p = p.replace(/"/g, '""');
160
+ row.push(node.quo + p + node.quo);
123
161
  }
124
- else if (q.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n"
125
- row.push(node.quo + q + node.quo);
162
+ else if (p.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n"
163
+ row.push(node.quo + p + node.quo);
126
164
  }
127
- else { row.push(q); } // otherwise just add
165
+ else { row.push(p); } // otherwise just add
128
166
  }
129
167
  }
168
+ ou.push(row.join(node.sep)); // add separator
169
+ }
170
+ }
171
+ }
172
+ // join lines, don't forget to add the last new line
173
+ msg.payload = ou.join(node.ret) + node.ret;
174
+ msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).join(',');
175
+ if (msg.payload !== '') {
176
+ send(msg);
177
+ }
178
+ done();
179
+ }
180
+ catch(e) { done(e); }
181
+ }
182
+ else if (typeof msg.payload == "string") { // convert CSV string to object
183
+ try {
184
+ var f = true; // flag to indicate if inside or outside a pair of quotes true = outside.
185
+ var j = 0; // pointer into array of template items
186
+ var k = [""]; // array of data for each of the template items
187
+ var o = {}; // output object to build up
188
+ var a = []; // output array is needed for multiline option
189
+ var first = true; // is this the first line
190
+ var last = false;
191
+ var line = msg.payload;
192
+ var linecount = 0;
193
+ var tmp = "";
194
+ var has_parts = msg.hasOwnProperty("parts");
195
+ var reg = /^[-]?(?!E)(?!0\d)\d*\.?\d*(E-?\+?)?\d+$/i;
196
+ if (msg.hasOwnProperty("parts")) {
197
+ linecount = msg.parts.index;
198
+ if (msg.parts.index > node.skip) { first = false; }
199
+ if (msg.parts.hasOwnProperty("count") && (msg.parts.index+1 >= msg.parts.count)) { last = true; }
200
+ }
201
+
202
+ // For now we are just going to assume that any \r or \n means an end of line...
203
+ // got to be a weird csv that has singleton \r \n in it for another reason...
204
+
205
+ // Now process the whole file/line
206
+ var nocr = (line.match(/[\r\n]/g)||[]).length;
207
+ if (has_parts && node.multi === "mult" && nocr > 1) { tmp = ""; first = true; }
208
+ for (var i = 0; i < line.length; i++) {
209
+ if (first && (linecount < node.skip)) {
210
+ if (line[i] === "\n") { linecount += 1; }
211
+ continue;
212
+ }
213
+ if ((node.hdrin === true) && first) { // if the template is in the first line
214
+ if ((line[i] === "\n")||(line[i] === "\r")||(line.length - i === 1)) { // look for first line break
215
+ if (line.length - i === 1) { tmp += line[i]; }
216
+ template = clean(tmp,node.sep);
217
+ first = false;
130
218
  }
131
- ou.push(row.join(node.sep)); // add separator
219
+ else { tmp += line[i]; }
132
220
  }
133
221
  else {
134
- const row = [];
135
- for (var t=0; t < template.length; t++) {
136
- if (template[t] === '') {
137
- row.push('');
222
+ if (line[i] === node.quo) { // if it's a quote toggle inside or outside
223
+ f = !f;
224
+ if (line[i-1] === node.quo) {
225
+ if (f === false) { k[j] += '\"'; }
226
+ } // if it's a quotequote then it's actually a quote
227
+ //if ((line[i-1] !== node.sep) && (line[i+1] !== node.sep)) { k[j] += line[i]; }
228
+ }
229
+ else if ((line[i] === node.sep) && f) { // if it is the end of the line then finish
230
+ if (!node.goodtmpl) { template[j] = "col"+(j+1); }
231
+ if ( template[j] && (template[j] !== "") ) {
232
+ // if no value between separators ('1,,"3"...') or if the line beings with separator (',1,"2"...') treat value as null
233
+ if (line[i-1] === node.sep || line[i-1].includes('\n','\r')) k[j] = null;
234
+ if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); }
235
+ if (node.include_null_values && k[j] === null) o[template[j]] = k[j];
236
+ if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
237
+ if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
138
238
  }
139
- else {
140
- var tt = template[t];
141
- if (template[t].indexOf('"') >=0 ) { tt = "'"+tt+"'"; }
142
- else { tt = '"'+tt+'"'; }
143
- var p = RED.util.getMessageProperty(msg,'payload["'+s+'"]['+tt+']');
144
- /* istanbul ignore else */
145
- if (p === undefined) { p = ""; }
146
- // fix to honour include null values flag
147
- //if (p === null && node.include_null_values !== true) { p = "";}
148
- p = RED.util.ensureString(p);
149
- if (p.indexOf(node.quo) !== -1) { // add double quotes if any quotes
150
- p = p.replace(/"/g, '""');
151
- row.push(node.quo + p + node.quo);
152
- }
153
- else if (p.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n"
154
- row.push(node.quo + p + node.quo);
155
- }
156
- else { row.push(p); } // otherwise just add
239
+ j += 1;
240
+ // if separator is last char in processing string line (without end of line), add null value at the end - example: '1,2,3\n3,"3",'
241
+ k[j] = line.length - 1 === i ? null : "";
242
+ }
243
+ else if (((line[i] === "\n") || (line[i] === "\r")) && f) { // handle multiple lines
244
+ //console.log(j,k,o,k[j]);
245
+ if (!node.goodtmpl) { template[j] = "col"+(j+1); }
246
+ if ( template[j] && (template[j] !== "") ) {
247
+ // if separator before end of line, set null value ie. '1,2,"3"\n1,2,\n1,2,3'
248
+ if (line[i-1] === node.sep) k[j] = null;
249
+ if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); }
250
+ else { if (k[j] !== null) k[j].replace(/\r$/,''); }
251
+ if (node.include_null_values && k[j] === null) o[template[j]] = k[j];
252
+ if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
253
+ if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
254
+ }
255
+ if (JSON.stringify(o) !== "{}") { // don't send empty objects
256
+ a.push(o); // add to the array
157
257
  }
258
+ j = 0;
259
+ k = [""];
260
+ o = {};
261
+ f = true; // reset in/out flag ready for next line.
262
+ }
263
+ else { // just add to the part of the message
264
+ k[j] += line[i];
158
265
  }
159
- ou.push(row.join(node.sep)); // add separator
160
266
  }
161
267
  }
162
- }
163
- // join lines, don't forget to add the last new line
164
- msg.payload = ou.join(node.ret) + node.ret;
165
- msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).join(',');
166
- if (msg.payload !== '') { send(msg); }
167
- done();
168
- }
169
- catch(e) { done(e); }
170
- }
171
- else if (typeof msg.payload == "string") { // convert CSV string to object
172
- try {
173
- var f = true; // flag to indicate if inside or outside a pair of quotes true = outside.
174
- var j = 0; // pointer into array of template items
175
- var k = [""]; // array of data for each of the template items
176
- var o = {}; // output object to build up
177
- var a = []; // output array is needed for multiline option
178
- var first = true; // is this the first line
179
- var last = false;
180
- var line = msg.payload;
181
- var linecount = 0;
182
- var tmp = "";
183
- var has_parts = msg.hasOwnProperty("parts");
184
- var reg = /^[-]?(?!E)(?!0\d)\d*\.?\d*(E-?\+?)?\d+$/i;
185
- if (msg.hasOwnProperty("parts")) {
186
- linecount = msg.parts.index;
187
- if (msg.parts.index > node.skip) { first = false; }
188
- if (msg.parts.hasOwnProperty("count") && (msg.parts.index+1 >= msg.parts.count)) { last = true; }
189
- }
268
+ // Finished so finalize and send anything left
269
+ if (f === false) { node.warn(RED._("csv.errors.bad_csv")); }
270
+ if (!node.goodtmpl) { template[j] = "col"+(j+1); }
190
271
 
191
- // For now we are just going to assume that any \r or \n means an end of line...
192
- // got to be a weird csv that has singleton \r \n in it for another reason...
272
+ if ( template[j] && (template[j] !== "") ) {
273
+ if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); }
274
+ else { if (k[j] !== null) k[j].replace(/\r$/,''); }
275
+ if (node.include_null_values && k[j] === null) o[template[j]] = k[j];
276
+ if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
277
+ if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
278
+ }
193
279
 
194
- // Now process the whole file/line
195
- var nocr = (line.match(/[\r\n]/g)||[]).length;
196
- if (has_parts && node.multi === "mult" && nocr > 1) { tmp = ""; first = true; }
197
- for (var i = 0; i < line.length; i++) {
198
- if (first && (linecount < node.skip)) {
199
- if (line[i] === "\n") { linecount += 1; }
200
- continue;
280
+ if (JSON.stringify(o) !== "{}") { // don't send empty objects
281
+ a.push(o); // add to the array
201
282
  }
202
- if ((node.hdrin === true) && first) { // if the template is in the first line
203
- if ((line[i] === "\n")||(line[i] === "\r")||(line.length - i === 1)) { // look for first line break
204
- if (line.length - i === 1) { tmp += line[i]; }
205
- template = clean(tmp,node.sep);
206
- first = false;
283
+
284
+ if (node.multi !== "one") {
285
+ msg.payload = a;
286
+ if (has_parts && nocr <= 1) {
287
+ if (JSON.stringify(o) !== "{}") {
288
+ node.store.push(o);
289
+ }
290
+ if (msg.parts.index + 1 === msg.parts.count) {
291
+ msg.payload = node.store;
292
+ msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(',');
293
+ delete msg.parts;
294
+ send(msg);
295
+ node.store = [];
296
+ }
297
+ }
298
+ else {
299
+ msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(',');
300
+ send(msg); // finally send the array
207
301
  }
208
- else { tmp += line[i]; }
209
302
  }
210
303
  else {
211
- if (line[i] === node.quo) { // if it's a quote toggle inside or outside
212
- f = !f;
213
- if (line[i-1] === node.quo) {
214
- if (f === false) { k[j] += '\"'; }
215
- } // if it's a quotequote then it's actually a quote
216
- //if ((line[i-1] !== node.sep) && (line[i+1] !== node.sep)) { k[j] += line[i]; }
217
- }
218
- else if ((line[i] === node.sep) && f) { // if it is the end of the line then finish
219
- if (!node.goodtmpl) { template[j] = "col"+(j+1); }
220
- if ( template[j] && (template[j] !== "") ) {
221
- // if no value between separators ('1,,"3"...') or if the line beings with separator (',1,"2"...') treat value as null
222
- if (line[i-1] === node.sep || line[i-1].includes('\n','\r')) k[j] = null;
223
- if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); }
224
- if (node.include_null_values && k[j] === null) o[template[j]] = k[j];
225
- if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
226
- if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
304
+ var len = a.length;
305
+ for (var i = 0; i < len; i++) {
306
+ var newMessage = RED.util.cloneMessage(msg);
307
+ newMessage.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(',');
308
+ newMessage.payload = a[i];
309
+ if (!has_parts) {
310
+ newMessage.parts = {
311
+ id: msg._msgid,
312
+ index: i,
313
+ count: len
314
+ };
315
+ }
316
+ else {
317
+ newMessage.parts.index -= node.skip;
318
+ newMessage.parts.count -= node.skip;
319
+ if (node.hdrin) { // if we removed the header line then shift the counts by 1
320
+ newMessage.parts.index -= 1;
321
+ newMessage.parts.count -= 1;
322
+ }
227
323
  }
228
- j += 1;
229
- // if separator is last char in processing string line (without end of line), add null value at the end - example: '1,2,3\n3,"3",'
230
- k[j] = line.length - 1 === i ? null : "";
324
+ if (last) { newMessage.complete = true; }
325
+ send(newMessage);
326
+ }
327
+ if (has_parts && last && len === 0) {
328
+ send({complete:true});
231
329
  }
232
- else if (((line[i] === "\n") || (line[i] === "\r")) && f) { // handle multiple lines
233
- //console.log(j,k,o,k[j]);
234
- if (!node.goodtmpl) { template[j] = "col"+(j+1); }
235
- if ( template[j] && (template[j] !== "") ) {
236
- // if separator before end of line, set null value ie. '1,2,"3"\n1,2,\n1,2,3'
237
- if (line[i-1] === node.sep) k[j] = null;
238
- if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); }
239
- else { if (k[j] !== null) k[j].replace(/\r$/,''); }
240
- if (node.include_null_values && k[j] === null) o[template[j]] = k[j];
241
- if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
242
- if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
330
+ }
331
+ node.linecount = 0;
332
+ done();
333
+ }
334
+ catch(e) { done(e); }
335
+ }
336
+ else { node.warn(RED._("csv.errors.csv_js")); done(); }
337
+ }
338
+ else {
339
+ if (!msg.hasOwnProperty("reset")) {
340
+ node.send(msg); // If no payload and not reset - just pass it on.
341
+ }
342
+ done();
343
+ }
344
+ });
345
+ }
346
+
347
+ if(RFC4180Mode) {
348
+ node.template = (n.temp || "")
349
+ node.sep = (n.sep || ',').replace(/\\t/g, "\t").replace(/\\n/g, "\n").replace(/\\r/g, "\r")
350
+ node.quo = '"'
351
+ // default to CRLF (RFC4180 Sec 2.1: "Each record is located on a separate line, delimited by a line break (CRLF)")
352
+ node.ret = (n.ret || "\r\n").replace(/\\n/g, "\n").replace(/\\r/g, "\r")
353
+ node.multi = n.multi || "one"
354
+ node.hdrin = n.hdrin || false
355
+ node.hdrout = n.hdrout || "none"
356
+ node.goodtmpl = true
357
+ node.skip = parseInt(n.skip || 0)
358
+ node.store = []
359
+ node.parsestrings = n.strings
360
+ node.include_empty_strings = n.include_empty_strings || false
361
+ node.include_null_values = n.include_null_values || false
362
+ if (node.parsestrings === undefined) { node.parsestrings = true }
363
+ if (node.hdrout === false) { node.hdrout = "none" }
364
+ if (node.hdrout === true) { node.hdrout = "all" }
365
+ const dontSendHeaders = node.hdrout === "none"
366
+ const sendHeadersOnce = node.hdrout === "once"
367
+ const sendHeadersAlways = node.hdrout === "all"
368
+ const sendHeaders = !dontSendHeaders && (sendHeadersOnce || sendHeadersAlways)
369
+ const quoteables = [node.sep, node.quo, "\n", "\r"]
370
+ const templateQuoteables = [',', '"', "\n", "\r"]
371
+ let badTemplateWarnOnce = true
372
+
373
+ const columnStringToTemplateArray = function (col, sep) {
374
+ // NOTE: enforce strict column template parsing in RFC4180 mode
375
+ const parsed = csv.parse(col, { separator: sep, quote: node.quo, outputStyle: 'array', strict: true })
376
+ if (parsed.headers.length > 0) { node.goodtmpl = true } else { node.goodtmpl = false }
377
+ return parsed.headers.length ? parsed.headers : null
378
+ }
379
+ const templateArrayToColumnString = function (template, keepEmptyColumns) {
380
+ // NOTE: enforce strict column template parsing in RFC4180 mode
381
+ const parsed = csv.parse('', {headers: template, headersOnly:true, separator: ',', quote: node.quo, outputStyle: 'array', strict: true })
382
+ return keepEmptyColumns
383
+ ? parsed.headers.map(e => addQuotes(e || '', { separator: ',', quoteables: templateQuoteables}))
384
+ : parsed.header // exclues empty columns
385
+ // TODO: resolve inconsistency between CSV->JSON and JSON->CSV
386
+ // CSV->JSON: empty columns are excluded
387
+ // JSON->CSV: empty columns are kept in some cases
388
+ }
389
+ function addQuotes(cell, options) {
390
+ options = options || {}
391
+ return csv.quoteCell(cell, {
392
+ quote: options.quote || node.quo || '"',
393
+ separator: options.separator || node.sep || ',',
394
+ quoteables: options.quoteables || quoteables
395
+ })
396
+ }
397
+ const hasTemplate = (t) => t?.length > 0 && !(t.length === 1 && t[0] === '')
398
+ let template
399
+ try {
400
+ template = columnStringToTemplateArray(node.template, ',') || ['']
401
+ } catch (e) {
402
+ node.warn(RED._("csv.errors.bad_template")) // is warning really necessary now we have status?
403
+ node.status({ fill: "red", shape: "dot", text: RED._("csv.errors.bad_template") })
404
+ return // dont hook up the node
405
+ }
406
+ const noTemplate = hasTemplate(template) === false
407
+ node.hdrSent = false
408
+
409
+ node.on("input", function (msg, send, done) {
410
+ node.status({}) // clear status
411
+ if (msg.hasOwnProperty("reset")) {
412
+ node.hdrSent = false
413
+ }
414
+ if (msg.hasOwnProperty("payload")) {
415
+ let inputData = msg.payload
416
+ if (typeof inputData == "object") { // convert object to CSV string
417
+ try {
418
+ // first determine the payload kind. Array or objects? Array of primitives? Array of arrays? Just an object?
419
+ // then, if necessary, convert to an array of objects/arrays
420
+ let isObject = !Array.isArray(inputData) && typeof inputData === 'object'
421
+ let isArrayOfObjects = Array.isArray(inputData) && inputData.length > 0 && typeof inputData[0] === 'object'
422
+ let isArrayOfArrays = Array.isArray(inputData) && inputData.length > 0 && Array.isArray(inputData[0])
423
+ let isArrayOfPrimitives = Array.isArray(inputData) && inputData.length > 0 && typeof inputData[0] !== 'object'
424
+
425
+ if (isObject) {
426
+ inputData = [inputData]
427
+ isArrayOfObjects = true
428
+ isObject = false
429
+ } else if (isArrayOfPrimitives) {
430
+ inputData = [inputData]
431
+ isArrayOfArrays = true
432
+ isArrayOfPrimitives = false
433
+ }
434
+
435
+ const stringBuilder = []
436
+ if (!(noTemplate && (msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("index") && msg.parts.index > 0))) {
437
+ template = columnStringToTemplateArray(node.template) || ['']
438
+ }
439
+
440
+ // build header line
441
+ if (sendHeaders && node.hdrSent === false) {
442
+ if (hasTemplate(template) === false) {
443
+ if (msg.hasOwnProperty("columns")) {
444
+ template = columnStringToTemplateArray(msg.columns || "", ",") || ['']
243
445
  }
244
- if (JSON.stringify(o) !== "{}") { // don't send empty objects
245
- a.push(o); // add to the array
446
+ else {
447
+ template = Object.keys(inputData[0]) || ['']
246
448
  }
247
- j = 0;
248
- k = [""];
249
- o = {};
250
- f = true; // reset in/out flag ready for next line.
251
449
  }
252
- else { // just add to the part of the message
253
- k[j] += line[i];
450
+ stringBuilder.push(templateArrayToColumnString(template, true))
451
+ if (sendHeadersOnce) { node.hdrSent = true }
452
+ }
453
+
454
+ // build csv lines
455
+ for (let s = 0; s < inputData.length; s++) {
456
+ let row = inputData[s]
457
+ if (isArrayOfArrays) {
458
+ /*** row is an array of arrays ***/
459
+ const _hasTemplate = hasTemplate(template)
460
+ const len = _hasTemplate ? template.length : row.length
461
+ const result = []
462
+ for (let t = 0; t < len; t++) {
463
+ let cell = row[t]
464
+ if (cell === undefined) { cell = "" }
465
+ if(_hasTemplate) {
466
+ const header = template[t]
467
+ if (header) {
468
+ result[t] = addQuotes(RED.util.ensureString(cell))
469
+ }
470
+ } else {
471
+ result[t] = addQuotes(RED.util.ensureString(cell))
472
+ }
473
+ }
474
+ stringBuilder.push(result.join(node.sep))
475
+ } else {
476
+ /*** row is an object ***/
477
+ if (hasTemplate(template) === false && (msg.hasOwnProperty("columns"))) {
478
+ template = columnStringToTemplateArray(msg.columns || "", ",")
479
+ }
480
+ if (hasTemplate(template) === false) {
481
+ /*** row is an object but we still don't have a template ***/
482
+ if (badTemplateWarnOnce === true) {
483
+ node.warn(RED._("csv.errors.obj_csv"))
484
+ badTemplateWarnOnce = false
485
+ }
486
+ const rowData = []
487
+ for (let header in inputData[0]) {
488
+ if (row.hasOwnProperty(header)) {
489
+ const cell = row[header]
490
+ if (typeof cell !== "object") {
491
+ let cellValue = ""
492
+ if (cell !== undefined) {
493
+ cellValue += cell
494
+ }
495
+ rowData.push(addQuotes(cellValue))
496
+ }
497
+ }
498
+ }
499
+ stringBuilder.push(rowData.join(node.sep))
500
+ } else {
501
+ /*** row is an object and we have a template ***/
502
+ const rowData = []
503
+ for (let t = 0; t < template.length; t++) {
504
+ if (!template[t]) {
505
+ rowData.push('')
506
+ }
507
+ else {
508
+ let cellValue = inputData[s][template[t]]
509
+ if (cellValue === undefined) { cellValue = "" }
510
+ cellValue = RED.util.ensureString(cellValue)
511
+ rowData.push(addQuotes(cellValue))
512
+ }
513
+ }
514
+ stringBuilder.push(rowData.join(node.sep)); // add separator
515
+ }
254
516
  }
255
517
  }
256
- }
257
- // Finished so finalize and send anything left
258
- if (f === false) { node.warn(RED._("csv.errors.bad_csv")); }
259
- if (!node.goodtmpl) { template[j] = "col"+(j+1); }
260
518
 
261
- if ( template[j] && (template[j] !== "") ) {
262
- if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); }
263
- else { if (k[j] !== null) k[j].replace(/\r$/,''); }
264
- if (node.include_null_values && k[j] === null) o[template[j]] = k[j];
265
- if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
266
- if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
519
+ // join lines, don't forget to add the last new line
520
+ msg.payload = stringBuilder.join(node.ret) + node.ret
521
+ msg.columns = templateArrayToColumnString(template)
522
+ if (msg.payload !== '') { send(msg) }
523
+ done()
267
524
  }
268
-
269
- if (JSON.stringify(o) !== "{}") { // don't send empty objects
270
- a.push(o); // add to the array
525
+ catch (e) {
526
+ done(e)
271
527
  }
528
+ }
529
+ else if (typeof inputData == "string") { // convert CSV string to object
530
+ try {
531
+ let firstLine = true; // is this the first line
532
+ let last = false
533
+ let linecount = 0
534
+ const has_parts = msg.hasOwnProperty("parts")
535
+
536
+ // determine if this is a multi part message and if so what part we are processing
537
+ if (msg.hasOwnProperty("parts")) {
538
+ linecount = msg.parts.index
539
+ if (msg.parts.index > node.skip) { firstLine = false }
540
+ if (msg.parts.hasOwnProperty("count") && (msg.parts.index + 1 >= msg.parts.count)) { last = true }
541
+ }
272
542
 
273
- if (node.multi !== "one") {
274
- msg.payload = a;
275
- if (has_parts && nocr <= 1) {
276
- if (JSON.stringify(o) !== "{}") {
277
- node.store.push(o);
543
+ // If skip is set, compute the cursor position to start parsing from
544
+ let _cursor = 0
545
+ if (node.skip > 0 && linecount < node.skip) {
546
+ for (; _cursor < inputData.length; _cursor++) {
547
+ if (firstLine && (linecount < node.skip)) {
548
+ if (inputData[_cursor] === "\r" || inputData[_cursor] === "\n") {
549
+ linecount += 1
550
+ }
551
+ continue
552
+ }
553
+ break
278
554
  }
279
- if (msg.parts.index + 1 === msg.parts.count) {
280
- msg.payload = node.store;
281
- msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(',');
282
- delete msg.parts;
283
- send(msg);
284
- node.store = [];
555
+ if (_cursor >= inputData.length) {
556
+ return // skip this line
285
557
  }
286
558
  }
287
- else {
288
- msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(',');
289
- send(msg); // finally send the array
559
+
560
+ // count the number of line breaks in the string
561
+ const noofCR = ((_cursor ? inputData.slice(_cursor) : inputData).match(/[\r\n]/g) || []).length
562
+
563
+ // if we have `parts` and we are outputting multiple objects and we have more than one line
564
+ // then we need to set firstLine to true so that we process the header line
565
+ if (has_parts && node.multi === "mult" && noofCR > 1) {
566
+ firstLine = true
290
567
  }
291
- }
292
- else {
293
- var len = a.length;
294
- for (var i = 0; i < len; i++) {
295
- var newMessage = RED.util.cloneMessage(msg);
296
- newMessage.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(',');
297
- newMessage.payload = a[i];
298
- if (!has_parts) {
299
- newMessage.parts = {
300
- id: msg._msgid,
301
- index: i,
302
- count: len
303
- };
568
+
569
+ // if we are processing the first line and the node has been set to extract the header line
570
+ // update the template with the header line
571
+ if (firstLine && node.hdrin === true) {
572
+ /** @type {import('./lib/csv/index.js').CSVParseOptions} */
573
+ const csvOptionsForHeaderRow = {
574
+ cursor: _cursor,
575
+ separator: node.sep,
576
+ quote: node.quo,
577
+ dataHasHeaderRow: true,
578
+ headersOnly: true,
579
+ outputStyle: 'array',
580
+ strict: true // enforce strict parsing of the header row
304
581
  }
305
- else {
306
- newMessage.parts.index -= node.skip;
307
- newMessage.parts.count -= node.skip;
308
- if (node.hdrin) { // if we removed the header line then shift the counts by 1
309
- newMessage.parts.index -= 1;
310
- newMessage.parts.count -= 1;
582
+ try {
583
+ const csvHeader = csv.parse(inputData, csvOptionsForHeaderRow)
584
+ template = csvHeader.headers
585
+ _cursor = csvHeader.cursor
586
+ } catch (e) {
587
+ // node.warn(RED._("csv.errors.bad_template")) // add warning?
588
+ node.status({ fill: "red", shape: "dot", text: RED._("csv.errors.bad_template") })
589
+ throw e
590
+ }
591
+ }
592
+
593
+ // now we process the data lines
594
+ /** @type {import('./lib/csv/index.js').CSVParseOptions} */
595
+ const csvOptions = {
596
+ cursor: _cursor,
597
+ separator: node.sep,
598
+ quote: node.quo,
599
+ dataHasHeaderRow: false,
600
+ headers: hasTemplate(template) ? template : null,
601
+ outputStyle: 'object',
602
+ includeNullValues: node.include_null_values,
603
+ includeEmptyStrings: node.include_empty_strings,
604
+ parseNumeric: node.parsestrings,
605
+ strict: false // relax the strictness of the parser for data rows
606
+ }
607
+ const csvParseResult = csv.parse(inputData, csvOptions)
608
+ const data = csvParseResult.data
609
+
610
+ // output results
611
+ if (node.multi !== "one") {
612
+ if (has_parts && noofCR <= 1) {
613
+ if (data.length > 0) {
614
+ node.store.push(...data)
615
+ }
616
+ if (msg.parts.index + 1 === msg.parts.count) {
617
+ msg.payload = node.store
618
+ msg.columns = csvParseResult.header
619
+ // msg._mode = 'RFC4180 mode'
620
+ delete msg.parts
621
+ send(msg)
622
+ node.store = []
311
623
  }
312
624
  }
313
- if (last) { newMessage.complete = true; }
314
- send(newMessage);
625
+ else {
626
+ msg.columns = csvParseResult.header
627
+ // msg._mode = 'RFC4180 mode'
628
+ msg.payload = data
629
+ send(msg); // finally send the array
630
+ }
315
631
  }
316
- if (has_parts && last && len === 0) {
317
- send({complete:true});
632
+ else {
633
+ const len = data.length
634
+ for (let row = 0; row < len; row++) {
635
+ const newMessage = RED.util.cloneMessage(msg)
636
+ newMessage.columns = csvParseResult.header
637
+ newMessage.payload = data[row]
638
+ if (!has_parts) {
639
+ newMessage.parts = {
640
+ id: msg._msgid,
641
+ index: row,
642
+ count: len
643
+ }
644
+ }
645
+ else {
646
+ newMessage.parts.index -= node.skip
647
+ newMessage.parts.count -= node.skip
648
+ if (node.hdrin) { // if we removed the header line then shift the counts by 1
649
+ newMessage.parts.index -= 1
650
+ newMessage.parts.count -= 1
651
+ }
652
+ }
653
+ if (last) { newMessage.complete = true }
654
+ // newMessage._mode = 'RFC4180 mode'
655
+ send(newMessage)
656
+ }
657
+ if (has_parts && last && len === 0) {
658
+ // send({complete:true, _mode: 'RFC4180 mode'})
659
+ send({ complete: true })
660
+ }
318
661
  }
662
+
663
+ node.linecount = 0
664
+ done()
319
665
  }
320
- node.linecount = 0;
321
- done();
666
+ catch (e) {
667
+ done(e)
668
+ }
669
+ }
670
+ else {
671
+ // RFC-vs-legacy mode difference: In RFC mode, we throw catchable errors and provide a status message
672
+ const err = new Error(RED._("csv.errors.csv_js"))
673
+ node.status({ fill: "red", shape: "dot", text: err.message })
674
+ done(err)
322
675
  }
323
- catch(e) { done(e); }
324
676
  }
325
- else { node.warn(RED._("csv.errors.csv_js")); done(); }
326
- }
327
- else {
328
- if (!msg.hasOwnProperty("reset")) {
329
- node.send(msg); // If no payload and not reset - just pass it on.
677
+ else {
678
+ if (!msg.hasOwnProperty("reset")) {
679
+ node.send(msg); // If no payload and not reset - just pass it on.
680
+ }
681
+ done()
330
682
  }
331
- done();
332
- }
333
- });
683
+ })
684
+ }
334
685
  }
335
- RED.nodes.registerType("csv",CSVNode);
686
+
687
+ RED.nodes.registerType("csv",CSVNode)
336
688
  }