@n8n/backend-module-data-table 0.3.2 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,14 @@
1
+ import { Transform, type TransformCallback } from 'stream';
2
+ export declare const CSV_MAX_COLUMNS = 1000;
3
+ export declare class CsvColumnLimit extends Transform {
4
+ private readonly maxColumns;
5
+ private columnCountInLine;
6
+ private insideQuotes;
7
+ private decoder;
8
+ private head;
9
+ constructor(maxColumns?: number);
10
+ _transform(chunk: Buffer, _encoding: BufferEncoding, done: TransformCallback): void;
11
+ _flush(done: TransformCallback): void;
12
+ private passText;
13
+ private tooManyColumnsError;
14
+ }
@@ -0,0 +1,68 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.CsvColumnLimit = exports.CSV_MAX_COLUMNS = void 0;
4
+ const stream_1 = require("stream");
5
+ const string_decoder_1 = require("string_decoder");
6
+ const data_table_file_upload_error_1 = require("./errors/data-table-file-upload.error");
7
+ exports.CSV_MAX_COLUMNS = 1000;
8
+ const COMMA = 0x2c;
9
+ const QUOTE = 0x22;
10
+ const LINE_FEED = 0x0a;
11
+ const CARRIAGE_RETURN = 0x0d;
12
+ const UTF16LE_BOM = Buffer.from([0xff, 0xfe]);
13
+ function createDecoder(head) {
14
+ return new string_decoder_1.StringDecoder(head.subarray(0, UTF16LE_BOM.length).equals(UTF16LE_BOM) ? 'utf16le' : 'utf8');
15
+ }
16
+ class CsvColumnLimit extends stream_1.Transform {
17
+ constructor(maxColumns = exports.CSV_MAX_COLUMNS) {
18
+ super();
19
+ this.maxColumns = maxColumns;
20
+ this.columnCountInLine = 1;
21
+ this.insideQuotes = false;
22
+ this.head = Buffer.alloc(0);
23
+ }
24
+ _transform(chunk, _encoding, done) {
25
+ const bytes = this.decoder ? chunk : Buffer.concat([this.head, chunk]);
26
+ if (!this.decoder) {
27
+ if (bytes.length < UTF16LE_BOM.length) {
28
+ this.head = bytes;
29
+ done();
30
+ return;
31
+ }
32
+ this.decoder = createDecoder(bytes);
33
+ this.head = Buffer.alloc(0);
34
+ }
35
+ this.passText(this.decoder.write(bytes), done);
36
+ }
37
+ _flush(done) {
38
+ this.decoder ??= createDecoder(this.head);
39
+ this.passText(this.decoder.end(this.head), done);
40
+ }
41
+ passText(text, done) {
42
+ for (let index = 0; index < text.length; index++) {
43
+ switch (text.charCodeAt(index)) {
44
+ case QUOTE:
45
+ this.insideQuotes = !this.insideQuotes;
46
+ break;
47
+ case COMMA:
48
+ if (!this.insideQuotes && ++this.columnCountInLine > this.maxColumns) {
49
+ done(this.tooManyColumnsError());
50
+ return;
51
+ }
52
+ break;
53
+ case LINE_FEED:
54
+ case CARRIAGE_RETURN:
55
+ if (!this.insideQuotes) {
56
+ this.columnCountInLine = 1;
57
+ }
58
+ break;
59
+ }
60
+ }
61
+ done(null, Buffer.from(text, 'utf8'));
62
+ }
63
+ tooManyColumnsError() {
64
+ return new data_table_file_upload_error_1.FileUploadError(`The file has more than ${this.maxColumns} columns. Remove columns and try again.`);
65
+ }
66
+ }
67
+ exports.CsvColumnLimit = CsvColumnLimit;
68
+ //# sourceMappingURL=csv-column-limit.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"csv-column-limit.js","sourceRoot":"","sources":["../src/csv-column-limit.ts"],"names":[],"mappings":";;;AAAA,mCAA2D;AAC3D,mDAA+C;AAE/C,wFAAwE;AAE3D,QAAA,eAAe,GAAG,IAAI,CAAC;AAEpC,MAAM,KAAK,GAAG,IAAI,CAAC;AACnB,MAAM,KAAK,GAAG,IAAI,CAAC;AACnB,MAAM,SAAS,GAAG,IAAI,CAAC;AACvB,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AAK9C,SAAS,aAAa,CAAC,IAAY;IAClC,OAAO,IAAI,8BAAa,CACvB,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAC7E,CAAC;AACH,CAAC;AAOD,oBAA4B,SAAQ,kBAAS;IAS5C,YAA6B,UAAU,GAAW,QAAA,eAAe;QAChE,KAAK,EAAE,CAAC;QADoB,eAAU,GAAV,UAAU,CAA0B;QARzD,sBAAiB,GAAG,CAAC,CAAC;QAEtB,iBAAY,GAAG,KAAK,CAAC;QAIrB,SAAI,GAAW,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAIvC,CAAC;IAED,UAAU,CAAC,KAAa,EAAE,SAAyB,EAAE,IAAuB;QAC3E,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;QAEvE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACnB,IAAI,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC;gBACvC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC;gBAClB,IAAI,EAAE,CAAC;gBACP,OAAO;YACR,CAAC;YAED,IAAI,CAAC,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC;IAChD,CAAC;IAED,MAAM,CAAC,IAAuB;QAC7B,IAAI,CAAC,OAAO,KAAK,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;IAClD,CAAC;IAEO,QAAQ,CAAC,IAAY,EAAE,IAAuB;QACrD,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YAClD,QAAQ,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,KAAK,KAAK;oBACT,IAAI,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC;oBACvC,MAAM;gBACP,KAAK,KAAK;oBACT,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;wBACtE,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC,CAAC;wBACjC,OAAO;oBACR,CAAC;oBACD,MAAM;gBACP,KAAK,SAAS,CAAC;gBACf,KAAK,eAAe;oBACnB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;wBACxB,IAAI,CAAC,iBAAiB,GAAG,CAAC,CAAC;oBAC5B,CAAC;oBACD,MAAM;YACR,CAAC;QACF,CAAC;QAGD,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IACvC,CAAC;IAEO,mBAAmB;QAC1B,OAAO,IAAI,8CAAe,CACzB,0BAA0B,IAAI,CAAC,UAAU,yCAAyC,CAClF,CAAC;IACH,CAAC;CACD"}
@@ -1,4 +1,5 @@
1
1
  import { GlobalConfig } from '@n8n/config';
2
+ export declare const CSV_ROW_BATCH_SIZE = 1000;
2
3
  export interface CsvColumnMetadata {
3
4
  name: string;
4
5
  type: 'string' | 'number' | 'boolean' | 'date';
@@ -23,11 +24,9 @@ export declare class CsvParserService {
23
24
  private buildColumnMetadata;
24
25
  private parseCsvFile;
25
26
  parseFile(fileId: string, hasHeaders?: boolean): Promise<CsvMetadata>;
26
- parseFileData(fileId: string, hasHeaders?: boolean): Promise<Array<Record<string, string>>>;
27
- parseFileWithData(fileId: string, hasHeaders?: boolean): Promise<{
28
- metadata: CsvMetadata;
29
- rows: Array<Record<string, string>>;
30
- }>;
27
+ parseColumnNames(fileId: string): Promise<string[]>;
28
+ parseFileInBatches(fileId: string, hasHeaders?: boolean, batchSize?: number): AsyncGenerator<Array<Record<string, string>>>;
29
+ private readRows;
31
30
  private getCompatibleTypes;
32
31
  private inferColumnType;
33
32
  private isDate;
@@ -9,12 +9,15 @@ var __metadata = (this && this.__metadata) || function (k, v) {
9
9
  if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
10
10
  };
11
11
  Object.defineProperty(exports, "__esModule", { value: true });
12
- exports.CsvParserService = void 0;
12
+ exports.CsvParserService = exports.CSV_ROW_BATCH_SIZE = void 0;
13
13
  const backend_common_1 = require("@n8n/backend-common");
14
14
  const config_1 = require("@n8n/config");
15
15
  const di_1 = require("@n8n/di");
16
16
  const csv_parse_1 = require("csv-parse");
17
17
  const fs_1 = require("fs");
18
+ const promises_1 = require("stream/promises");
19
+ const csv_column_limit_1 = require("./csv-column-limit");
20
+ exports.CSV_ROW_BATCH_SIZE = 1000;
18
21
  let CsvParserService = class CsvParserService {
19
22
  constructor(globalConfig) {
20
23
  this.globalConfig = globalConfig;
@@ -37,6 +40,7 @@ let CsvParserService = class CsvParserService {
37
40
  columns: hasHeaders ? true : false,
38
41
  skip_empty_lines: true,
39
42
  bom: true,
43
+ record_delimiter: ['\r\n', '\n', '\r'],
40
44
  };
41
45
  }
42
46
  trimColumnNames(columns) {
@@ -75,30 +79,26 @@ let CsvParserService = class CsvParserService {
75
79
  const filePath = (0, backend_common_1.safeJoinPath)(this.uploadDir, fileId);
76
80
  let columnNames = [];
77
81
  let rowCount = 0;
78
- return await new Promise((resolve, reject) => {
79
- const parser = (0, csv_parse_1.parse)({
80
- ...this.createParserOptions(hasHeaders),
81
- ...(hasHeaders && {
82
- columns: (header) => {
83
- columnNames = this.trimColumnNames(header);
84
- return columnNames;
85
- },
86
- }),
87
- })
88
- .on('data', (row) => {
89
- rowCount++;
90
- if (!hasHeaders && Array.isArray(row) && columnNames.length === 0) {
91
- columnNames = this.generateColumnNames(row.length);
92
- }
93
- const rowObject = this.normalizeRow(row, hasHeaders, columnNames);
94
- if (!rowObject)
95
- return;
82
+ const parser = (0, csv_parse_1.parse)({
83
+ ...this.createParserOptions(hasHeaders),
84
+ ...(hasHeaders && {
85
+ columns: (header) => {
86
+ columnNames = this.trimColumnNames(header);
87
+ return columnNames;
88
+ },
89
+ }),
90
+ }).on('data', (row) => {
91
+ rowCount++;
92
+ if (!hasHeaders && Array.isArray(row) && columnNames.length === 0) {
93
+ columnNames = this.generateColumnNames(row.length);
94
+ }
95
+ const rowObject = this.normalizeRow(row, hasHeaders, columnNames);
96
+ if (rowObject) {
96
97
  onRow(rowObject, columnNames, rowCount);
97
- })
98
- .on('end', () => resolve(onEnd(columnNames, rowCount)))
99
- .on('error', reject);
100
- (0, fs_1.createReadStream)(filePath).on('error', reject).pipe(parser);
98
+ }
101
99
  });
100
+ await (0, promises_1.pipeline)((0, fs_1.createReadStream)(filePath), new csv_column_limit_1.CsvColumnLimit(), parser);
101
+ return onEnd(columnNames, rowCount);
102
102
  }
103
103
  async parseFile(fileId, hasHeaders = true) {
104
104
  const firstNonEmptyValues = new Map();
@@ -111,19 +111,63 @@ let CsvParserService = class CsvParserService {
111
111
  return { rowCount: totalRows, columnCount: columns.length, columns };
112
112
  });
113
113
  }
114
- async parseFileData(fileId, hasHeaders = true) {
115
- const rows = [];
116
- return await this.parseCsvFile(fileId, hasHeaders, (rowObject) => rows.push(rowObject), () => rows);
114
+ async parseColumnNames(fileId) {
115
+ let columnNames = [];
116
+ await this.readRows(fileId, true, {
117
+ toLine: 1,
118
+ onColumnNames: (names) => {
119
+ columnNames = names;
120
+ },
121
+ }).next();
122
+ return columnNames;
117
123
  }
118
- async parseFileWithData(fileId, hasHeaders = true) {
119
- const rows = [];
120
- return await this.parseCsvFile(fileId, hasHeaders, (rowObject) => rows.push(rowObject), (colNames) => {
121
- const columns = colNames.map((name) => ({ name, type: 'string' }));
122
- return {
123
- metadata: { rowCount: rows.length, columnCount: columns.length, columns },
124
- rows,
125
- };
124
+ async *parseFileInBatches(fileId, hasHeaders = true, batchSize = exports.CSV_ROW_BATCH_SIZE) {
125
+ let batch = [];
126
+ for await (const rowObject of this.readRows(fileId, hasHeaders)) {
127
+ batch.push(rowObject);
128
+ if (batch.length >= batchSize) {
129
+ yield batch;
130
+ batch = [];
131
+ }
132
+ }
133
+ if (batch.length > 0) {
134
+ yield batch;
135
+ }
136
+ }
137
+ async *readRows(fileId, hasHeaders, { toLine, onColumnNames, } = {}) {
138
+ const filePath = (0, backend_common_1.safeJoinPath)(this.uploadDir, fileId);
139
+ let columnNames = [];
140
+ const parser = (0, csv_parse_1.parse)({
141
+ ...this.createParserOptions(hasHeaders),
142
+ to_line: toLine,
143
+ ...(hasHeaders && {
144
+ columns: (header) => {
145
+ columnNames = this.trimColumnNames(header);
146
+ onColumnNames?.(columnNames);
147
+ return columnNames;
148
+ },
149
+ }),
126
150
  });
151
+ const input = (0, fs_1.createReadStream)(filePath);
152
+ (0, promises_1.pipeline)(input, new csv_column_limit_1.CsvColumnLimit(), parser).catch(() => { });
153
+ const records = parser;
154
+ try {
155
+ for await (const row of records) {
156
+ const needsGeneratedColumnNames = !hasHeaders && columnNames.length === 0;
157
+ if (needsGeneratedColumnNames && Array.isArray(row)) {
158
+ columnNames = this.generateColumnNames(row.length);
159
+ onColumnNames?.(columnNames);
160
+ }
161
+ const rowObject = this.normalizeRow(row, hasHeaders, columnNames);
162
+ if (rowObject) {
163
+ yield rowObject;
164
+ }
165
+ }
166
+ }
167
+ finally {
168
+ input.destroy();
169
+ parser.destroy();
170
+ }
127
171
  }
128
172
  getCompatibleTypes(detectedType) {
129
173
  switch (detectedType) {
@@ -1 +1 @@
1
- {"version":3,"file":"csv-parser.service.js","sourceRoot":"","sources":["../src/csv-parser.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,wDAAmD;AACnD,wCAA2C;AAC3C,gCAAkC;AAClC,yCAAkC;AAClC,2BAAsC;AAc/B,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;IAK5B,YAA6B,YAA0B;QAA1B,iBAAY,GAAZ,YAAY,CAAc;QAFtC,0BAAqB,GAAG,SAAS,CAAC;QAkBlC,+BAA0B,GAAG,GAAG,CAAC;QAfjD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC;IACxD,CAAC;IAEO,mBAAmB,CAAC,WAAmB;QAC9C,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,qBAAqB,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/F,CAAC;IAEO,kBAAkB,CAAC,GAAa,EAAE,WAAqB;QAC9D,MAAM,SAAS,GAA2B,EAAE,CAAC;QAC7C,GAAG,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YAC5B,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC;QACvC,CAAC,CAAC,CAAC;QACH,OAAO,SAAS,CAAC;IAClB,CAAC;IAIO,mBAAmB,CAAC,UAAmB;QAC9C,OAAO;YACN,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,KAAe;YAC7C,gBAAgB,EAAE,IAAI;YACtB,GAAG,EAAE,IAAI;SACT,CAAC;IACH,CAAC;IAEO,eAAe,CAAC,OAAiB;QACxC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACrC,CAAC;IAEO,YAAY,CACnB,GAAsC,EACtC,UAAmB,EACnB,WAAqB;QAErB,IAAI,CAAC,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YACvC,OAAO,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QAClD,CAAC;aAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAChC,OAAO,GAAG,CAAC;QACZ,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;IAEO,kBAAkB,CACzB,SAAiC,EACjC,WAAqB,EACrB,mBAAwC;QAExC,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;YACnC,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvC,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;gBACjC,IAAI,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC;oBACnB,mBAAmB,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACzC,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAEO,mBAAmB,CAAC,WAAqB,EAAE,mBAAwC;QAC1F,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YACrC,MAAM,YAAY,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;YAC/E,OAAO;gBACN,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,YAAY;gBAClB,eAAe,EAAE,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC;aACtD,CAAC;QACH,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,YAAY,CACzB,MAAc,EACd,UAAmB,EACnB,KAA4F,EAC5F,KAAsD;QAEtD,MAAM,QAAQ,GAAG,IAAA,6BAAY,EAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACtD,IAAI,WAAW,GAAa,EAAE,CAAC;QAC/B,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,OAAO,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC5C,MAAM,MAAM,GAAG,IAAA,iBAAK,EAAC;gBACpB,GAAG,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC;gBACvC,GAAG,CAAC,UAAU,IAAI;oBACjB,OAAO,EAAE,CAAC,MAAgB,EAAE,EAAE;wBAC7B,WAAW,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;wBAC3C,OAAO,WAAW,CAAC;oBACpB,CAAC;iBACD,CAAC;aACF,CAAC;iBACA,EAAE,CAAC,MAAM,EAAE,CAAC,GAAsC,EAAE,EAAE;gBACtD,QAAQ,EAAE,CAAC;gBACX,IAAI,CAAC,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACnE,WAAW,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACpD,CAAC;gBACD,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;gBAClE,IAAI,CAAC,SAAS;oBAAE,OAAO;gBACvB,KAAK,CAAC,SAAS,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC;YACzC,CAAC,CAAC;iBACD,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;iBACtD,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAEtB,IAAA,qBAAgB,EAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC;IACJ,CAAC;IAMD,KAAK,CAAC,SAAS,CAAC,MAAc,EAAE,UAAU,GAAY,IAAI;QACzD,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAkB,CAAC;QAEtD,OAAO,MAAM,IAAI,CAAC,YAAY,CAC7B,MAAM,EACN,UAAU,EACV,CAAC,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE;YAClC,IAAI,SAAS,IAAI,IAAI,CAAC,0BAA0B,EAAE,CAAC;gBAClD,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,mBAAmB,CAAC,CAAC;YACnE,CAAC;QACF,CAAC,EACD,CAAC,QAAQ,EAAE,SAAS,EAAE,EAAE;YACvB,MAAM,OAAO,GAAG,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAC;YACxE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;QACtE,CAAC,CACD,CAAC;IACH,CAAC;IAKD,KAAK,CAAC,aAAa,CAClB,MAAc,EACd,UAAU,GAAY,IAAI;QAE1B,MAAM,IAAI,GAAkC,EAAE,CAAC;QAE/C,OAAO,MAAM,IAAI,CAAC,YAAY,CAC7B,MAAM,EACN,UAAU,EACV,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EACnC,GAAG,EAAE,CAAC,IAAI,CACV,CAAC;IACH,CAAC;IAMD,KAAK,CAAC,iBAAiB,CACtB,MAAc,EACd,UAAU,GAAY,IAAI;QAE1B,MAAM,IAAI,GAAkC,EAAE,CAAC;QAE/C,OAAO,MAAM,IAAI,CAAC,YAAY,CAC7B,MAAM,EACN,UAAU,EACV,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EACnC,CAAC,QAAQ,EAAE,EAAE;YACZ,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,QAAiB,EAAE,CAAC,CAAC,CAAC;YAC5E,OAAO;gBACN,QAAQ,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE;gBACzE,IAAI;aACJ,CAAC;QACH,CAAC,CACD,CAAC;IACH,CAAC;IAMO,kBAAkB,CACzB,YAAsD;QAEtD,QAAQ,YAAY,EAAE,CAAC;YACtB,KAAK,MAAM;gBACV,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAC3B,KAAK,QAAQ;gBACZ,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAC7B,KAAK,SAAS;gBACb,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YAC9B,KAAK,QAAQ;gBACZ,OAAO,CAAC,QAAQ,CAAC,CAAC;YACnB;gBACC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACpB,CAAC;IACF,CAAC;IAMO,eAAe,CAAC,KAAyB;QAChD,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC;YACpB,OAAO,QAAQ,CAAC;QACjB,CAAC;QAED,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAClC,MAAM,UAAU,GAAG,YAAY,CAAC,WAAW,EAAE,CAAC;QAE9C,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;YACrD,OAAO,SAAS,CAAC;QAClB,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC;YACzC,OAAO,QAAQ,CAAC;QACjB,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;YAC/B,OAAO,MAAM,CAAC;QACf,CAAC;QAED,OAAO,QAAQ,CAAC;IACjB,CAAC;IAKO,MAAM,CAAC,KAAa;QAE3B,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC;QAG7B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;YAGnC,MAAM,YAAY,GAAG;gBACpB,oBAAoB;gBACpB,sBAAsB;gBACtB,sBAAsB;gBACtB,oBAAoB;gBACpB,gCAAgC;aAChC,CAAC;YAEF,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAC5D,CAAC;QAED,OAAO,KAAK,CAAC;IACd,CAAC;CACD,CAAA;;2BArPY,gBAAgB;IAD5B,IAAA,YAAO,GAAE;qCAMkC,qBAAY;GAL3C,gBAAgB,CAqP5B"}
1
+ {"version":3,"file":"csv-parser.service.js","sourceRoot":"","sources":["../src/csv-parser.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,wDAAmD;AACnD,wCAA2C;AAC3C,gCAAkC;AAClC,yCAAkC;AAClC,2BAAsC;AACtC,8CAA2C;AAE3C,yDAAoD;AAEvC,QAAA,kBAAkB,GAAG,IAAI,CAAC;AAchC,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;IAK5B,YAA6B,YAA0B;QAA1B,iBAAY,GAAZ,YAAY,CAAc;QAFtC,0BAAqB,GAAG,SAAS,CAAC;QAkBlC,+BAA0B,GAAG,GAAG,CAAC;QAfjD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC;IACxD,CAAC;IAEO,mBAAmB,CAAC,WAAmB;QAC9C,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,qBAAqB,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/F,CAAC;IAEO,kBAAkB,CAAC,GAAa,EAAE,WAAqB;QAC9D,MAAM,SAAS,GAA2B,EAAE,CAAC;QAC7C,GAAG,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YAC5B,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC;QACvC,CAAC,CAAC,CAAC;QACH,OAAO,SAAS,CAAC;IAClB,CAAC;IAIO,mBAAmB,CAAC,UAAmB;QAC9C,OAAO;YACN,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,KAAe;YAC7C,gBAAgB,EAAE,IAAI;YACtB,GAAG,EAAE,IAAI;YAGT,gBAAgB,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC;SACtC,CAAC;IACH,CAAC;IAEO,eAAe,CAAC,OAAiB;QACxC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACrC,CAAC;IAEO,YAAY,CACnB,GAAsC,EACtC,UAAmB,EACnB,WAAqB;QAErB,IAAI,CAAC,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YACvC,OAAO,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QAClD,CAAC;aAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAChC,OAAO,GAAG,CAAC;QACZ,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;IAEO,kBAAkB,CACzB,SAAiC,EACjC,WAAqB,EACrB,mBAAwC;QAExC,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;YACnC,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvC,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;gBACjC,IAAI,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC;oBACnB,mBAAmB,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACzC,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAEO,mBAAmB,CAAC,WAAqB,EAAE,mBAAwC;QAC1F,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YACrC,MAAM,YAAY,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;YAC/E,OAAO;gBACN,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,YAAY;gBAClB,eAAe,EAAE,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC;aACtD,CAAC;QACH,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,YAAY,CACzB,MAAc,EACd,UAAmB,EACnB,KAA4F,EAC5F,KAAsD;QAEtD,MAAM,QAAQ,GAAG,IAAA,6BAAY,EAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACtD,IAAI,WAAW,GAAa,EAAE,CAAC;QAC/B,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,MAAM,MAAM,GAAG,IAAA,iBAAK,EAAC;YACpB,GAAG,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC;YACvC,GAAG,CAAC,UAAU,IAAI;gBACjB,OAAO,EAAE,CAAC,MAAgB,EAAE,EAAE;oBAC7B,WAAW,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;oBAC3C,OAAO,WAAW,CAAC;gBACpB,CAAC;aACD,CAAC;SACF,CAAC,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,GAAsC,EAAE,EAAE;YACxD,QAAQ,EAAE,CAAC;YACX,IAAI,CAAC,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACnE,WAAW,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACpD,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;YAClE,IAAI,SAAS,EAAE,CAAC;gBACf,KAAK,CAAC,SAAS,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC;YACzC,CAAC;QACF,CAAC,CAAC,CAAC;QAEH,MAAM,IAAA,mBAAQ,EAAC,IAAA,qBAAgB,EAAC,QAAQ,CAAC,EAAE,IAAI,iCAAc,EAAE,EAAE,MAAM,CAAC,CAAC;QAEzE,OAAO,KAAK,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IACrC,CAAC;IAMD,KAAK,CAAC,SAAS,CAAC,MAAc,EAAE,UAAU,GAAY,IAAI;QACzD,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAkB,CAAC;QAEtD,OAAO,MAAM,IAAI,CAAC,YAAY,CAC7B,MAAM,EACN,UAAU,EACV,CAAC,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE;YAClC,IAAI,SAAS,IAAI,IAAI,CAAC,0BAA0B,EAAE,CAAC;gBAClD,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,mBAAmB,CAAC,CAAC;YACnE,CAAC;QACF,CAAC,EACD,CAAC,QAAQ,EAAE,SAAS,EAAE,EAAE;YACvB,MAAM,OAAO,GAAG,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAC;YACxE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;QACtE,CAAC,CACD,CAAC;IACH,CAAC;IAKD,KAAK,CAAC,gBAAgB,CAAC,MAAc;QACpC,IAAI,WAAW,GAAa,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE;YACjC,MAAM,EAAE,CAAC;YACT,aAAa,EAAE,CAAC,KAAK,EAAE,EAAE;gBACxB,WAAW,GAAG,KAAK,CAAC;YACrB,CAAC;SACD,CAAC,CAAC,IAAI,EAAE,CAAC;QAEV,OAAO,WAAW,CAAC;IACpB,CAAC;IAMD,KAAK,CAAC,CAAC,kBAAkB,CACxB,MAAc,EACd,UAAU,GAAY,IAAI,EAC1B,SAAS,GAAW,QAAA,kBAAkB;QAEtC,IAAI,KAAK,GAAkC,EAAE,CAAC;QAE9C,IAAI,KAAK,EAAE,MAAM,SAAS,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,CAAC;YACjE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtB,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;gBAC/B,MAAM,KAAK,CAAC;gBACZ,KAAK,GAAG,EAAE,CAAC;YACZ,CAAC;QACF,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,KAAK,CAAC;QACb,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,CAAC,QAAQ,CACtB,MAAc,EACd,UAAmB,EACnB,EACC,MAAM,EACN,aAAa,GACb,GAAyE,EAAE;QAE5E,MAAM,QAAQ,GAAG,IAAA,6BAAY,EAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACtD,IAAI,WAAW,GAAa,EAAE,CAAC;QAE/B,MAAM,MAAM,GAAG,IAAA,iBAAK,EAAC;YACpB,GAAG,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC;YACvC,OAAO,EAAE,MAAM;YACf,GAAG,CAAC,UAAU,IAAI;gBACjB,OAAO,EAAE,CAAC,MAAgB,EAAE,EAAE;oBAC7B,WAAW,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;oBAC3C,aAAa,EAAE,CAAC,WAAW,CAAC,CAAC;oBAC7B,OAAO,WAAW,CAAC;gBACpB,CAAC;aACD,CAAC;SACF,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,IAAA,qBAAgB,EAAC,QAAQ,CAAC,CAAC;QAGzC,IAAA,mBAAQ,EAAC,KAAK,EAAE,IAAI,iCAAc,EAAE,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAE9D,MAAM,OAAO,GAAqD,MAAM,CAAC;QAEzE,IAAI,CAAC;YACJ,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;gBACjC,MAAM,yBAAyB,GAAG,CAAC,UAAU,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC;gBAC1E,IAAI,yBAAyB,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;oBACrD,WAAW,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACnD,aAAa,EAAE,CAAC,WAAW,CAAC,CAAC;gBAC9B,CAAC;gBAED,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;gBAClE,IAAI,SAAS,EAAE,CAAC;oBACf,MAAM,SAAS,CAAC;gBACjB,CAAC;YACF,CAAC;QACF,CAAC;gBAAS,CAAC;YACV,KAAK,CAAC,OAAO,EAAE,CAAC;YAChB,MAAM,CAAC,OAAO,EAAE,CAAC;QAClB,CAAC;IACF,CAAC;IAMO,kBAAkB,CACzB,YAAsD;QAEtD,QAAQ,YAAY,EAAE,CAAC;YACtB,KAAK,MAAM;gBACV,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAC3B,KAAK,QAAQ;gBACZ,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAC7B,KAAK,SAAS;gBACb,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YAC9B,KAAK,QAAQ;gBACZ,OAAO,CAAC,QAAQ,CAAC,CAAC;YACnB;gBACC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACpB,CAAC;IACF,CAAC;IAMO,eAAe,CAAC,KAAyB;QAChD,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC;YACpB,OAAO,QAAQ,CAAC;QACjB,CAAC;QAED,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAClC,MAAM,UAAU,GAAG,YAAY,CAAC,WAAW,EAAE,CAAC;QAE9C,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;YACrD,OAAO,SAAS,CAAC;QAClB,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC;YACzC,OAAO,QAAQ,CAAC;QACjB,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;YAC/B,OAAO,MAAM,CAAC;QACf,CAAC;QAED,OAAO,QAAQ,CAAC;IACjB,CAAC;IAKO,MAAM,CAAC,KAAa;QAE3B,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC;QAG7B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;YAGnC,MAAM,YAAY,GAAG;gBACpB,oBAAoB;gBACpB,sBAAsB;gBACtB,sBAAsB;gBACtB,oBAAoB;gBACpB,gCAAgC;aAChC,CAAC;YAEF,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAC5D,CAAC;QAED,OAAO,KAAK,CAAC;IACd,CAAC;CACD,CAAA;;2BArSY,gBAAgB;IAD5B,IAAA,YAAO,GAAE;qCAMkC,qBAAY;GAL3C,gBAAgB,CAqS5B"}
@@ -9,12 +9,15 @@ export declare class DataTableCsvImportService {
9
9
  private readonly fileCleanupService;
10
10
  private readonly logger;
11
11
  constructor(csvParserService: CsvParserService, fileCleanupService: DataTableFileCleanupService, logger: Logger);
12
- buildRowsForNewTable(fileId: string, hasHeaders: boolean, tableColumns: DataTableColumn[], dtoColumns?: CreateDataTableDto['columns']): Promise<DataTableRow[]>;
13
- validateAndBuildRowsForExistingTable(fileId: string, tableColumns: DataTableColumn[]): Promise<{
14
- rows: DataTableRow[];
12
+ buildRowBatchesForNewTable(fileId: string, hasHeaders: boolean, tableColumns: DataTableColumn[], dtoColumns?: CreateDataTableDto['columns']): Promise<AsyncGenerator<DataTableRow[]>>;
13
+ validateAndBuildRowBatchesForExistingTable(fileId: string, tableColumns: DataTableColumn[]): Promise<{
14
+ rowBatches: AsyncGenerator<DataTableRow[]>;
15
15
  systemColumnsIgnored: string[];
16
16
  }>;
17
+ claimFile(fileId: string): Promise<string>;
17
18
  cleanupFile(fileId: string): Promise<void>;
19
+ private transformRowBatches;
20
+ private toFileUploadError;
18
21
  private matchColumns;
19
22
  private transformRows;
20
23
  private buildColumnMappingForNewTable;
@@ -23,51 +23,76 @@ let DataTableCsvImportService = class DataTableCsvImportService {
23
23
  this.fileCleanupService = fileCleanupService;
24
24
  this.logger = logger.scoped('data-table-csv-import');
25
25
  }
26
- async buildRowsForNewTable(fileId, hasHeaders, tableColumns, dtoColumns) {
26
+ async buildRowBatchesForNewTable(fileId, hasHeaders, tableColumns, dtoColumns) {
27
+ const failure = {
28
+ logMessage: 'Failed to import data from CSV file',
29
+ fallbackMessage: 'Failed to read CSV file',
30
+ };
31
+ let columnMapping;
27
32
  try {
28
- const columnMapping = await this.buildColumnMappingForNewTable(fileId, hasHeaders, tableColumns, dtoColumns);
29
- const csvRows = await this.csvParserService.parseFileData(fileId, hasHeaders);
30
- return this.transformRows(csvRows, columnMapping);
33
+ columnMapping = await this.buildColumnMappingForNewTable(fileId, hasHeaders, tableColumns, dtoColumns);
31
34
  }
32
35
  catch (error) {
33
- this.logger.error('Failed to import data from CSV file', { error, fileId });
34
- throw new data_table_file_upload_error_1.FileUploadError(error instanceof Error ? error.message : 'Failed to read CSV file');
36
+ throw this.toFileUploadError(error, fileId, failure);
35
37
  }
38
+ return this.transformRowBatches(fileId, hasHeaders, columnMapping, failure);
36
39
  }
37
- async validateAndBuildRowsForExistingTable(fileId, tableColumns) {
40
+ async validateAndBuildRowBatchesForExistingTable(fileId, tableColumns) {
41
+ const failure = {
42
+ logMessage: 'Failed to import CSV to existing table',
43
+ fallbackMessage: 'Failed to import CSV file',
44
+ };
45
+ let csvColumnNames;
38
46
  try {
39
- const tableColumnNames = new Set(tableColumns.map((col) => col.name));
40
- const { metadata: csvMetadata, rows: csvRows } = await this.csvParserService.parseFileWithData(fileId);
41
- const { matchedColumns, systemColumnsIgnored } = this.matchColumns(csvMetadata.columns, tableColumnNames);
42
- const mapping = new Map(matchedColumns.map((col) => [col, col]));
43
- return {
44
- rows: this.transformRows(csvRows, mapping),
45
- systemColumnsIgnored,
46
- };
47
+ csvColumnNames = await this.csvParserService.parseColumnNames(fileId);
47
48
  }
48
49
  catch (error) {
49
- if (error instanceof data_table_validation_error_1.DataTableValidationError)
50
- throw error;
51
- this.logger.error('Failed to import CSV to existing table', { error, fileId });
52
- throw new data_table_file_upload_error_1.FileUploadError(error instanceof Error ? error.message : 'Failed to import CSV file');
50
+ throw this.toFileUploadError(error, fileId, failure);
53
51
  }
52
+ const tableColumnNames = new Set(tableColumns.map((col) => col.name));
53
+ const { matchedColumns, systemColumnsIgnored } = this.matchColumns(csvColumnNames, tableColumnNames);
54
+ const mapping = new Map(matchedColumns.map((col) => [col, col]));
55
+ return {
56
+ rowBatches: this.transformRowBatches(fileId, true, mapping, failure),
57
+ systemColumnsIgnored,
58
+ };
59
+ }
60
+ async claimFile(fileId) {
61
+ return await this.fileCleanupService.claimFile(fileId);
54
62
  }
55
63
  async cleanupFile(fileId) {
56
64
  await this.fileCleanupService.deleteFile(fileId);
57
65
  }
58
- matchColumns(csvColumns, tableColumnNames) {
66
+ async *transformRowBatches(fileId, hasHeaders, columnMapping, failure) {
67
+ try {
68
+ for await (const csvRows of this.csvParserService.parseFileInBatches(fileId, hasHeaders)) {
69
+ yield this.transformRows(csvRows, columnMapping);
70
+ }
71
+ }
72
+ catch (error) {
73
+ throw this.toFileUploadError(error, fileId, failure);
74
+ }
75
+ }
76
+ toFileUploadError(error, fileId, failure) {
77
+ this.logger.error(failure.logMessage, { error, fileId });
78
+ if (error instanceof data_table_file_upload_error_1.FileUploadError) {
79
+ return error;
80
+ }
81
+ return new data_table_file_upload_error_1.FileUploadError(error instanceof Error ? error.message : failure.fallbackMessage);
82
+ }
83
+ matchColumns(csvColumnNames, tableColumnNames) {
59
84
  const systemColumnsIgnored = [];
60
85
  const unrecognizedColumns = [];
61
86
  const matchedColumns = [];
62
- for (const csvCol of csvColumns) {
63
- if (n8n_workflow_1.DATA_TABLE_SYSTEM_COLUMNS.includes(csvCol.name)) {
64
- systemColumnsIgnored.push(csvCol.name);
87
+ for (const csvColName of csvColumnNames) {
88
+ if (n8n_workflow_1.DATA_TABLE_SYSTEM_COLUMNS.includes(csvColName)) {
89
+ systemColumnsIgnored.push(csvColName);
65
90
  }
66
- else if (tableColumnNames.has(csvCol.name)) {
67
- matchedColumns.push(csvCol.name);
91
+ else if (tableColumnNames.has(csvColName)) {
92
+ matchedColumns.push(csvColName);
68
93
  }
69
94
  else {
70
- unrecognizedColumns.push(csvCol.name);
95
+ unrecognizedColumns.push(csvColName);
71
96
  }
72
97
  }
73
98
  if (unrecognizedColumns.length > 0) {
@@ -1 +1 @@
1
- {"version":3,"file":"data-table-csv-import.service.js","sourceRoot":"","sources":["../src/data-table-csv-import.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AACA,wDAA6C;AAC7C,gCAAkC;AAElC,+CAAyD;AAGzD,6DAAwD;AAExD,uFAAgF;AAChF,wFAAwE;AACxE,sFAAgF;AAGzE,IAAM,yBAAyB,GAA/B,MAAM,yBAAyB;IAGrC,YACkB,gBAAkC,EAClC,kBAA+C,EAChE,MAAc;gCAFG,gBAAgB;kCAChB,kBAAkB;QAGnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,uBAAuB,CAAC,CAAC;IACtD,CAAC;IAOD,KAAK,CAAC,oBAAoB,CACzB,MAAc,EACd,UAAmB,EACnB,YAA+B,EAC/B,UAA0C;QAE1C,IAAI,CAAC;YACJ,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAC7D,MAAM,EACN,UAAU,EACV,YAAY,EACZ,UAAU,CACV,CAAC;YAEF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;YAE9E,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YAC5E,MAAM,IAAI,8CAAe,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC;QAC/F,CAAC;IACF,CAAC;IAOD,KAAK,CAAC,oCAAoC,CACzC,MAAc,EACd,YAA+B;QAE/B,IAAI,CAAC;YACJ,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;YAEtE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,GAC7C,MAAM,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;YAEvD,MAAM,EAAE,cAAc,EAAE,oBAAoB,EAAE,GAAG,IAAI,CAAC,YAAY,CACjE,WAAW,CAAC,OAAO,EACnB,gBAAgB,CAChB,CAAC;YAEF,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;YACjE,OAAO;gBACN,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC;gBAC1C,oBAAoB;aACpB,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,KAAK,YAAY,sDAAwB;gBAAE,MAAM,KAAK,CAAC;YAC3D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wCAAwC,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YAC/E,MAAM,IAAI,8CAAe,CACxB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,2BAA2B,CACpE,CAAC;QACH,CAAC;IACF,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,MAAc;QAC/B,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC;IAEO,YAAY,CACnB,UAA+B,EAC/B,gBAA6B;QAE7B,MAAM,oBAAoB,GAAa,EAAE,CAAC;QAC1C,MAAM,mBAAmB,GAAa,EAAE,CAAC;QACzC,MAAM,cAAc,GAAa,EAAE,CAAC;QAEpC,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;YACjC,IAAI,wCAAyB,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACrD,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACxC,CAAC;iBAAM,IAAI,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9C,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACP,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACvC,CAAC;QACF,CAAC;QAED,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,IAAI,sDAAwB,CACjC,qDAAqD,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,8BAA8B,CACjH,CAAC;QACH,CAAC;QAED,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,sDAAwB,CACjC,0GAA0G,CAC1G,CAAC;QACH,CAAC;QAED,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,CAAC;IACjD,CAAC;IAEO,aAAa,CACpB,OAAsC,EACtC,aAAkC;QAElC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAC7B,MAAM,cAAc,GAAiB,EAAE,CAAC;YACxC,KAAK,MAAM,CAAC,UAAU,EAAE,YAAY,CAAC,IAAI,aAAa,EAAE,CAAC;gBACxD,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;gBACjC,cAAc,CAAC,YAAY,CAAC,GAAG,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;YACnF,CAAC;YACD,OAAO,cAAc,CAAC;QACvB,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAC1C,MAAc,EACd,UAAmB,EACnB,YAA+B,EAC/B,UAA0C;QAE1C,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;QAEhD,MAAM,iBAAiB,GAAG,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC;QACnE,IAAI,iBAAiB,EAAE,CAAC;YACvB,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7E,KAAK,MAAM,MAAM,IAAI,UAAW,EAAE,CAAC;gBAClC,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;oBAC1B,MAAM,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBAClD,IAAI,SAAS,EAAE,CAAC;wBACf,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;oBACpD,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;aAAM,CAAC;YACP,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;YAC9E,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE;gBAChD,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;oBACzB,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;gBAC7D,CAAC;YACF,CAAC,CAAC,CAAC;QACJ,CAAC;QAED,OAAO,aAAa,CAAC;IACtB,CAAC;CACD,CAAA;;oCA1JY,yBAAyB;IADrC,IAAA,YAAO,GAAE;qCAK2B,qCAAgB,EACd,6DAA2B,EACxD,uBAAM;GANH,yBAAyB,CA0JrC"}
1
+ {"version":3,"file":"data-table-csv-import.service.js","sourceRoot":"","sources":["../src/data-table-csv-import.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AACA,wDAA6C;AAC7C,gCAAkC;AAElC,+CAAyD;AAEzD,6DAAwD;AAExD,uFAAgF;AAChF,wFAAwE;AACxE,sFAAgF;AAGzE,IAAM,yBAAyB,GAA/B,MAAM,yBAAyB;IAGrC,YACkB,gBAAkC,EAClC,kBAA+C,EAChE,MAAc;gCAFG,gBAAgB;kCAChB,kBAAkB;QAGnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,uBAAuB,CAAC,CAAC;IACtD,CAAC;IAOD,KAAK,CAAC,0BAA0B,CAC/B,MAAc,EACd,UAAmB,EACnB,YAA+B,EAC/B,UAA0C;QAE1C,MAAM,OAAO,GAAG;YACf,UAAU,EAAE,qCAAqC;YACjD,eAAe,EAAE,yBAAyB;SAC1C,CAAC;QAEF,IAAI,aAAkC,CAAC;QACvC,IAAI,CAAC;YACJ,aAAa,GAAG,MAAM,IAAI,CAAC,6BAA6B,CACvD,MAAM,EACN,UAAU,EACV,YAAY,EACZ,UAAU,CACV,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QACtD,CAAC;QAED,OAAO,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,UAAU,EAAE,aAAa,EAAE,OAAO,CAAC,CAAC;IAC7E,CAAC;IAOD,KAAK,CAAC,0CAA0C,CAC/C,MAAc,EACd,YAA+B;QAE/B,MAAM,OAAO,GAAG;YACf,UAAU,EAAE,wCAAwC;YACpD,eAAe,EAAE,2BAA2B;SAC5C,CAAC;QAEF,IAAI,cAAwB,CAAC;QAC7B,IAAI,CAAC;YACJ,cAAc,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACvE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QACtD,CAAC;QAED,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QACtE,MAAM,EAAE,cAAc,EAAE,oBAAoB,EAAE,GAAG,IAAI,CAAC,YAAY,CACjE,cAAc,EACd,gBAAgB,CAChB,CAAC;QAEF,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QACjE,OAAO;YACN,UAAU,EAAE,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC;YACpE,oBAAoB;SACpB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAAc;QAC7B,OAAO,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,MAAc;QAC/B,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC;IAEO,KAAK,CAAC,CAAC,mBAAmB,CACjC,MAAc,EACd,UAAmB,EACnB,aAAkC,EAClC,OAAwD;QAExD,IAAI,CAAC;YACJ,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,CAAC;gBAC1F,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;YAClD,CAAC;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QACtD,CAAC;IACF,CAAC;IAEO,iBAAiB,CACxB,KAAc,EACd,MAAc,EACd,OAAwD;QAExD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QACzD,IAAI,KAAK,YAAY,8CAAe,EAAE,CAAC;YACtC,OAAO,KAAK,CAAC;QACd,CAAC;QAED,OAAO,IAAI,8CAAe,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;IAC9F,CAAC;IAEO,YAAY,CACnB,cAAwB,EACxB,gBAA6B;QAE7B,MAAM,oBAAoB,GAAa,EAAE,CAAC;QAC1C,MAAM,mBAAmB,GAAa,EAAE,CAAC;QACzC,MAAM,cAAc,GAAa,EAAE,CAAC;QAEpC,KAAK,MAAM,UAAU,IAAI,cAAc,EAAE,CAAC;YACzC,IAAI,wCAAyB,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBACpD,oBAAoB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACvC,CAAC;iBAAM,IAAI,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC7C,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACjC,CAAC;iBAAM,CAAC;gBACP,mBAAmB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACtC,CAAC;QACF,CAAC;QAED,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,IAAI,sDAAwB,CACjC,qDAAqD,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,8BAA8B,CACjH,CAAC;QACH,CAAC;QAED,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,sDAAwB,CACjC,0GAA0G,CAC1G,CAAC;QACH,CAAC;QAED,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,CAAC;IACjD,CAAC;IAEO,aAAa,CACpB,OAAsC,EACtC,aAAkC;QAElC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAC7B,MAAM,cAAc,GAAiB,EAAE,CAAC;YACxC,KAAK,MAAM,CAAC,UAAU,EAAE,YAAY,CAAC,IAAI,aAAa,EAAE,CAAC;gBACxD,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;gBACjC,cAAc,CAAC,YAAY,CAAC,GAAG,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;YACnF,CAAC;YACD,OAAO,cAAc,CAAC;QACvB,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAC1C,MAAc,EACd,UAAmB,EACnB,YAA+B,EAC/B,UAA0C;QAE1C,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;QAEhD,MAAM,iBAAiB,GAAG,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC;QACnE,IAAI,iBAAiB,EAAE,CAAC;YACvB,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7E,KAAK,MAAM,MAAM,IAAI,UAAW,EAAE,CAAC;gBAClC,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;oBAC1B,MAAM,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBAClD,IAAI,SAAS,EAAE,CAAC;wBACf,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;oBACpD,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;aAAM,CAAC;YACP,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;YAC9E,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE;gBAChD,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;oBACzB,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;gBAC7D,CAAC;YACF,CAAC,CAAC,CAAC;QACJ,CAAC;QAED,OAAO,aAAa,CAAC;IACtB,CAAC;CACD,CAAA;;oCA7LY,yBAAyB;IADrC,IAAA,YAAO,GAAE;qCAK2B,qCAAgB,EACd,6DAA2B,EACxD,uBAAM;GANH,yBAAyB,CA6LrC"}
@@ -4,9 +4,11 @@ export declare class DataTableFileCleanupService {
4
4
  private readonly globalConfig;
5
5
  private readonly logger;
6
6
  private readonly uploadDir;
7
+ private readonly activeClaims;
7
8
  constructor(globalConfig: GlobalConfig, logger: Logger);
8
9
  private isErrnoException;
9
10
  private isNotFound;
10
11
  cleanupOrphanedFiles(signal: AbortSignal): Promise<void>;
12
+ claimFile(fileId: string): Promise<string>;
11
13
  deleteFile(fileId: string): Promise<void>;
12
14
  }
@@ -15,10 +15,15 @@ const config_1 = require("@n8n/config");
15
15
  const di_1 = require("@n8n/di");
16
16
  const ensure_error_1 = require("@n8n/utils/errors/ensure-error");
17
17
  const fs_1 = require("fs");
18
+ const data_table_file_upload_error_1 = require("./errors/data-table-file-upload.error");
19
+ const UPLOADED_FILE_ID_PATTERN = /^[\w-]+$/;
20
+ const CLAIMED_FILE_SUFFIX = '.importing';
21
+ const FILE_NOT_AVAILABLE_MESSAGE = 'The file was not found or was already imported. Upload the file again.';
18
22
  let DataTableFileCleanupService = class DataTableFileCleanupService {
19
23
  constructor(globalConfig, logger) {
20
24
  this.globalConfig = globalConfig;
21
25
  this.logger = logger;
26
+ this.activeClaims = new Set();
22
27
  this.uploadDir = this.globalConfig.dataTable.uploadDir;
23
28
  this.logger = this.logger.scoped('data-table');
24
29
  }
@@ -46,6 +51,9 @@ let DataTableFileCleanupService = class DataTableFileCleanupService {
46
51
  for (const file of files) {
47
52
  if (signal.aborted)
48
53
  break;
54
+ if (this.activeClaims.has(file)) {
55
+ continue;
56
+ }
49
57
  const filePath = (0, backend_common_1.safeJoinPath)(this.uploadDir, file);
50
58
  try {
51
59
  const stats = await fs_1.promises.stat(filePath);
@@ -64,7 +72,25 @@ let DataTableFileCleanupService = class DataTableFileCleanupService {
64
72
  }
65
73
  }
66
74
  }
75
+ async claimFile(fileId) {
76
+ if (!UPLOADED_FILE_ID_PATTERN.test(fileId)) {
77
+ throw new data_table_file_upload_error_1.FileUploadError(FILE_NOT_AVAILABLE_MESSAGE);
78
+ }
79
+ const claimedFileId = `${fileId}${CLAIMED_FILE_SUFFIX}`;
80
+ try {
81
+ await fs_1.promises.rename((0, backend_common_1.safeJoinPath)(this.uploadDir, fileId), (0, backend_common_1.safeJoinPath)(this.uploadDir, claimedFileId));
82
+ }
83
+ catch (error) {
84
+ if (this.isNotFound(error)) {
85
+ throw new data_table_file_upload_error_1.FileUploadError(FILE_NOT_AVAILABLE_MESSAGE);
86
+ }
87
+ throw error;
88
+ }
89
+ this.activeClaims.add(claimedFileId);
90
+ return claimedFileId;
91
+ }
67
92
  async deleteFile(fileId) {
93
+ this.activeClaims.delete(fileId);
68
94
  const filePath = (0, backend_common_1.safeJoinPath)(this.uploadDir, fileId);
69
95
  try {
70
96
  await fs_1.promises.unlink(filePath);
@@ -1 +1 @@
1
- {"version":3,"file":"data-table-file-cleanup.service.js","sourceRoot":"","sources":["../src/data-table-file-cleanup.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,wDAA2D;AAC3D,wCAA2C;AAC3C,gCAAkC;AAClC,iEAA6D;AAC7D,2BAAoC;AAG7B,IAAM,2BAA2B,GAAjC,MAAM,2BAA2B;IAGvC,YACkB,YAA0B,EAC1B,MAAc;4BADd,YAAY;sBACZ,MAAM;QAEvB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC;QACvD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IAChD,CAAC;IAEO,gBAAgB,CAAC,KAAc;QACtC,OAAO,CACN,OAAO,KAAK,KAAK,QAAQ;YACzB,KAAK,KAAK,IAAI;YACd,MAAM,IAAI,KAAK;YACf,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAC9B,CAAC;IACH,CAAC;IAEO,UAAU,CAAC,KAAc;QAChC,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC;IAChE,CAAC;IAMD,KAAK,CAAC,oBAAoB,CAAC,MAAmB;QAC7C,IAAI,KAAe,CAAC;QACpB,IAAI,CAAC;YACJ,KAAK,GAAG,MAAM,aAAE,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC1C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO;YACnC,MAAM,KAAK,CAAC;QACb,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,YAAY,CAAC;QAExD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,MAAM,CAAC,OAAO;gBAAE,MAAM;YAE1B,MAAM,QAAQ,GAAG,IAAA,6BAAY,EAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YACpD,IAAI,CAAC;gBACJ,MAAM,KAAK,GAAG,MAAM,aAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtC,MAAM,OAAO,GAAG,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC;gBAEpC,IAAI,OAAO,GAAG,MAAM,EAAE,CAAC;oBACtB,MAAM,aAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAC3B,CAAC;YACF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAEhB,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;oBAAE,SAAS;gBACrC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qDAAqD,EAAE;oBACvE,IAAI;oBACJ,KAAK,EAAE,IAAA,0BAAW,EAAC,KAAK,CAAC,CAAC,OAAO;iBACjC,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;IACF,CAAC;IAKD,KAAK,CAAC,UAAU,CAAC,MAAc;QAC9B,MAAM,QAAQ,GAAG,IAAA,6BAAY,EAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACtD,IAAI,CAAC;YACJ,MAAM,aAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBAAE,MAAM,KAAK,CAAC;QAC1C,CAAC;IACF,CAAC;CACD,CAAA;;sCAzEY,2BAA2B;IADvC,IAAA,YAAO,GAAE;qCAKuB,qBAAY,EAClB,uBAAM;GALpB,2BAA2B,CAyEvC"}
1
+ {"version":3,"file":"data-table-file-cleanup.service.js","sourceRoot":"","sources":["../src/data-table-file-cleanup.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,wDAA2D;AAC3D,wCAA2C;AAC3C,gCAAkC;AAClC,iEAA6D;AAC7D,2BAAoC;AAEpC,wFAAwE;AAExE,MAAM,wBAAwB,GAAG,UAAU,CAAC;AAE5C,MAAM,mBAAmB,GAAG,YAAY,CAAC;AAEzC,MAAM,0BAA0B,GAC/B,wEAAwE,CAAC;AAGnE,IAAM,2BAA2B,GAAjC,MAAM,2BAA2B;IAOvC,YACkB,YAA0B,EAC1B,MAAc;QADd,iBAAY,GAAZ,YAAY,CAAc;QAC1B,WAAM,GAAN,MAAM,CAAQ;QAJf,iBAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QAMjD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC;QACvD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IAChD,CAAC;IAEO,gBAAgB,CAAC,KAAc;QACtC,OAAO,CACN,OAAO,KAAK,KAAK,QAAQ;YACzB,KAAK,KAAK,IAAI;YACd,MAAM,IAAI,KAAK;YACf,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAC9B,CAAC;IACH,CAAC;IAEO,UAAU,CAAC,KAAc;QAChC,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC;IAChE,CAAC;IAMD,KAAK,CAAC,oBAAoB,CAAC,MAAmB;QAC7C,IAAI,KAAe,CAAC;QACpB,IAAI,CAAC;YACJ,KAAK,GAAG,MAAM,aAAE,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC1C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO;YACnC,MAAM,KAAK,CAAC;QACb,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,YAAY,CAAC;QAExD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,MAAM,CAAC,OAAO;gBAAE,MAAM;YAE1B,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjC,SAAS;YACV,CAAC;YAED,MAAM,QAAQ,GAAG,IAAA,6BAAY,EAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YACpD,IAAI,CAAC;gBACJ,MAAM,KAAK,GAAG,MAAM,aAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtC,MAAM,OAAO,GAAG,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC;gBAEpC,IAAI,OAAO,GAAG,MAAM,EAAE,CAAC;oBACtB,MAAM,aAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAC3B,CAAC;YACF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAEhB,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;oBAAE,SAAS;gBACrC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qDAAqD,EAAE;oBACvE,IAAI;oBACJ,KAAK,EAAE,IAAA,0BAAW,EAAC,KAAK,CAAC,CAAC,OAAO;iBACjC,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;IACF,CAAC;IAOD,KAAK,CAAC,SAAS,CAAC,MAAc;QAE7B,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5C,MAAM,IAAI,8CAAe,CAAC,0BAA0B,CAAC,CAAC;QACvD,CAAC;QAED,MAAM,aAAa,GAAG,GAAG,MAAM,GAAG,mBAAmB,EAAE,CAAC;QACxD,IAAI,CAAC;YACJ,MAAM,aAAE,CAAC,MAAM,CACd,IAAA,6BAAY,EAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,EACpC,IAAA,6BAAY,EAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,CAC3C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC5B,MAAM,IAAI,8CAAe,CAAC,0BAA0B,CAAC,CAAC;YACvD,CAAC;YAED,MAAM,KAAK,CAAC;QACb,CAAC;QAED,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QACrC,OAAO,aAAa,CAAC;IACtB,CAAC;IAKD,KAAK,CAAC,UAAU,CAAC,MAAc;QAC9B,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,QAAQ,GAAG,IAAA,6BAAY,EAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACtD,IAAI,CAAC;YACJ,MAAM,aAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBAAE,MAAM,KAAK,CAAC;QAC1C,CAAC;IACF,CAAC;CACD,CAAA;;sCA/GY,2BAA2B;IADvC,IAAA,YAAO,GAAE;qCASuB,qBAAY,EAClB,uBAAM;GATpB,2BAA2B,CA+GvC"}
@@ -13,6 +13,7 @@ export declare class DataTableRowsRepository {
13
13
  readonly insertedRows: number;
14
14
  }>;
15
15
  insertRows<T extends DataTableInsertRowsReturnType>(dataTableId: string, rows: DataTableRows, columns: DataTableColumn[], returnType: T, trx?: EntityManager): Promise<DataTableInsertRowsResult<T>>;
16
+ insertRowsInContext<T extends DataTableInsertRowsReturnType>(dataTableId: string, rows: DataTableRows, columns: DataTableColumn[], returnType: T, ctx: OperationContext): Promise<DataTableInsertRowsResult<T>>;
16
17
  updateRows<T extends boolean | undefined>(dataTableId: string, data: Record<string, DataTableColumnJsType | null>, filter: DataTableFilter, columns: DataTableColumn[], returnData?: T, trx?: EntityManager): Promise<T extends true ? DataTableRowReturn[] : true>;
17
18
  dryRunUpdateRows(dataTableId: string, data: Record<string, DataTableColumnJsType | null>, filter: DataTableFilter, columns: DataTableColumn[], trx?: EntityManager): Promise<DataTableRowReturnWithState[]>;
18
19
  dryRunUpsertRow(dataTableId: string, data: Record<string, DataTableColumnJsType | null>, filter: DataTableFilter, columns: DataTableColumn[], trx?: EntityManager): Promise<DataTableRowReturnWithState[]>;
@@ -174,6 +174,9 @@ let DataTableRowsRepository = class DataTableRowsRepository {
174
174
  return inserted;
175
175
  });
176
176
  }
177
+ async insertRowsInContext(dataTableId, rows, columns, returnType, ctx) {
178
+ return await this.insertRows(dataTableId, rows, columns, returnType, this.table.managerFor(ctx));
179
+ }
177
180
  async updateRows(dataTableId, data, filter, columns, returnData = false, trx) {
178
181
  return await (0, db_1.withTransaction)(this.dataSource.manager, trx, async (em) => {
179
182
  const dbType = this.dataSource.options.type;