@steedos/data-import 2.1.37 → 2.1.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,598 +3,698 @@ const fs = require('fs');
3
3
  const path = require('path');
4
4
  const _ = require('underscore');
5
5
  const objectql = require('@steedos/objectql');
6
-
7
- const xlsx = require('node-xlsx');
6
+ const Fiber = require("fibers");
7
+ const xlsx = require("node-xlsx");
8
8
  declare var Creator: any;
9
9
 
10
10
  type ImportOptions = {
11
- objectName: string,
12
- externalIdName: string | null,
13
- lookupFieldMap: any,
14
- fireWorkflows: boolean,
15
- operation: string,
16
- userSession: any,
17
- mappings: any[],
18
- keyIndexes: number,
19
- queueImportId: string | null,
20
- }
21
-
11
+ objectName: string;
12
+ externalIdName: string | null;
13
+ lookupFieldMap: any;
14
+ fireWorkflows: boolean;
15
+ operation: string;
16
+ userSession: any;
17
+ mappings: any[];
18
+ keyIndexes: number;
19
+ queueImportId: string | null;
20
+ };
22
21
 
23
22
  function converterString(field_name, dataCell, jsonObj) {
24
- jsonObj[field_name] = dataCell + '';
25
- return '';
26
- };
23
+ jsonObj[field_name] = dataCell + "";
24
+ return "";
25
+ }
27
26
 
28
27
  function converterDate(field_name, dataCell, jsonObj) {
29
- var date, date_error;
30
- date_error = "";
31
- date = new Date(dataCell);
32
- if (date.getFullYear() && Object.prototype.toString.call(date) === '[object Date]') {
33
- jsonObj[field_name] = date;
34
- } else {
35
- date_error = `${dataCell}不是日期类型数据`;
36
- }
37
- return date_error;
38
- };
28
+ var date, date_error;
29
+ date_error = "";
30
+ date = new Date(dataCell);
31
+ if (
32
+ date.getFullYear() &&
33
+ Object.prototype.toString.call(date) === "[object Date]"
34
+ ) {
35
+ jsonObj[field_name] = date;
36
+ } else {
37
+ date_error = `${dataCell}不是日期类型数据`;
38
+ }
39
+ return date_error;
40
+ }
39
41
 
40
42
  function converteNum(field_name, dataCell, jsonObj) {
41
- var number, number_error;
42
- number_error = "";
43
- number = parseFloat(dataCell);
44
- if (!isNaN(number)) {
45
- jsonObj[field_name] = number;
46
- } else {
47
- number_error = `${dataCell}不是数值类型数据`;
48
- }
49
- return number_error;
50
- };
43
+ var number, number_error;
44
+ number_error = "";
45
+ number = parseFloat(dataCell);
46
+ if (!isNaN(number)) {
47
+ jsonObj[field_name] = number;
48
+ } else {
49
+ number_error = `${dataCell}不是数值类型数据`;
50
+ }
51
+ return number_error;
52
+ }
51
53
 
52
54
  async function converterSelect(objectName, field_name, dataCell, jsonObj) {
53
- var allowedValues, allowedLabels, fields, ref, select_error;
54
- select_error = "";
55
- let objectConfig = await objectql.getObject(objectName).toConfig();
56
- fields = objectConfig.fields;
57
- let field = fields[field_name]
58
- allowedValues = _.pluck(field.options, 'value');
59
- allowedLabels = _.pluck(field.options, 'label');
60
- let optionsMap = _.object(allowedLabels, allowedValues);
61
-
62
- let cellContents: any = [];
63
- let noResult = true;
64
- if (field.multiple) {
65
- jsonObj[field_name] = [];
66
- if (dataCell) {
67
- cellContents = dataCell.split(';');
68
- }
55
+ var allowedValues, allowedLabels, fields, ref, select_error;
56
+ select_error = "";
57
+ let objectConfig = await objectql.getObject(objectName).toConfig();
58
+ fields = objectConfig.fields;
59
+ let field = fields[field_name];
60
+ allowedValues = _.pluck(field.options, "value");
61
+ allowedLabels = _.pluck(field.options, "label");
62
+ let optionsMap = _.object(allowedLabels, allowedValues);
63
+
64
+ let cellContents: any = [];
65
+ let noResult = true;
66
+ if (field.multiple) {
67
+ jsonObj[field_name] = [];
68
+ if (dataCell) {
69
+ cellContents = dataCell.split(";");
70
+ }
71
+ } else {
72
+ jsonObj[field_name] = null;
73
+ cellContents.push(dataCell);
74
+ }
75
+
76
+ for (let cellContent of cellContents) {
77
+ if (!cellContent) {
78
+ continue;
79
+ }
80
+
81
+ if (allowedLabels.indexOf(cellContent) >= 0) {
82
+ noResult = false;
83
+ if (field.multiple) {
84
+ jsonObj[field_name].push(optionsMap[cellContent]);
85
+ } else {
86
+ jsonObj[field_name] = optionsMap[cellContent];
87
+ }
69
88
  } else {
70
- jsonObj[field_name] = null;
71
- cellContents.push(dataCell);
89
+ select_error = `${cellContent}不属于${field_name}的可选范围`;
72
90
  }
91
+ }
73
92
 
74
- for (let cellContent of cellContents) {
75
-
76
- if (!cellContent) {
77
- continue;
78
- }
79
-
80
- if (allowedLabels.indexOf(cellContent) >= 0) {
81
- noResult = false;
82
- if (field.multiple) {
83
- jsonObj[field_name].push(optionsMap[cellContent]);
84
- } else {
85
- jsonObj[field_name] = optionsMap[cellContent];
86
- }
87
- } else {
88
- select_error = `${cellContent}不属于${field_name}的可选范围`;
89
- }
90
- }
91
-
92
- if (noResult && field.required) { //
93
- select_error += `${field_name}字段为必填项`;
94
- }
95
- return select_error;
96
- };
93
+ if (noResult && field.required) {
94
+ //
95
+ select_error += `${field_name}字段为必填项`;
96
+ }
97
+ return select_error;
98
+ }
97
99
 
98
100
  function getNameFieldKey(fields) {
99
- let NAME_FIELD_KEY = 'name'
100
- for (let fieldName in fields) {
101
- let field = fields[fieldName];
102
- if (field.is_name) {
103
- NAME_FIELD_KEY = fieldName
104
- break;
105
- }
106
- }
107
- return NAME_FIELD_KEY;
101
+ let NAME_FIELD_KEY = "name";
102
+ for (let fieldName in fields) {
103
+ let field = fields[fieldName];
104
+ if (field.is_name) {
105
+ NAME_FIELD_KEY = fieldName;
106
+ break;
107
+ }
108
+ }
109
+ return NAME_FIELD_KEY;
108
110
  }
109
111
 
110
- async function converterLookup(objectName, field_name, dataCell, jsonObj, fieldMap, options) {
111
- var fields, lookups, lookup_error, field, reference_to_object, reference_to_field, selectfield;
112
- lookup_error = "";
113
- let objectConfig = await objectql.getObject(objectName).toConfig();
114
- fields = objectConfig.fields;
115
- field = fields[field_name]
116
- reference_to_object = field.reference_to;
117
-
118
- reference_to_field = field.reference_to_field;
119
- let noResult = true; // 判断是否所有数据都找不到结果
120
-
121
- if (!reference_to_field) {
122
- reference_to_field = "_id"
123
- }
124
- if (fieldMap[field_name].matched_by) {
125
- selectfield = fieldMap[field_name].matched_by;
126
- } else {
127
- selectfield = getNameFieldKey(fields);
128
- }
129
-
130
- // 这一条加在了permission_set.object.yml里面
131
- // if(field_name == 'profile'){
132
- // reference_to_object = 'permission_set'
133
- // }
134
-
135
- let lookupCollection = await objectql.getObject(reference_to_object)
136
-
137
- let cellContents: any = [];
138
- if (field.multiple) {
139
- jsonObj[field_name] = []
140
- if (dataCell) {
141
- cellContents = dataCell.split(';');
112
+ async function converterLookup(
113
+ objectName,
114
+ field_name,
115
+ dataCell,
116
+ jsonObj,
117
+ fieldMap,
118
+ options
119
+ ) {
120
+ var fields,
121
+ lookups,
122
+ lookup_error,
123
+ field,
124
+ reference_to_object,
125
+ reference_to_field,
126
+ selectfield;
127
+ lookup_error = "";
128
+ let objectConfig = await objectql.getObject(objectName).toConfig();
129
+ fields = objectConfig.fields;
130
+ field = fields[field_name];
131
+ reference_to_object = field.reference_to;
132
+
133
+ reference_to_field = field.reference_to_field;
134
+ let noResult = true; // 判断是否所有数据都找不到结果
135
+
136
+ if (!reference_to_field) {
137
+ reference_to_field = "_id";
138
+ }
139
+ if (fieldMap[field_name].matched_by) {
140
+ selectfield = fieldMap[field_name].matched_by;
141
+ } else {
142
+ selectfield = getNameFieldKey(fields);
143
+ }
144
+
145
+ // 这一条加在了permission_set.object.yml里面
146
+ // if(field_name == 'profile'){
147
+ // reference_to_object = 'permission_set'
148
+ // }
149
+
150
+ let lookupCollection = await objectql.getObject(reference_to_object);
151
+
152
+ let cellContents: any = [];
153
+ if (field.multiple) {
154
+ jsonObj[field_name] = [];
155
+ if (dataCell) {
156
+ cellContents = dataCell.split(";");
157
+ }
158
+ } else {
159
+ jsonObj[field_name] = null;
160
+ cellContents.push(dataCell);
161
+ }
162
+
163
+ for (let cellContent of cellContents) {
164
+ if (!cellContent) {
165
+ continue;
166
+ }
167
+ let cellFilter = [selectfield, "=", cellContent];
168
+ let spaceFilter = ["space", "=", options.userSession.spaceId];
169
+ let filters = [cellFilter, spaceFilter];
170
+ lookups = await lookupCollection.find({ filters: filters });
171
+
172
+ if (lookups.length == 0) {
173
+ //找不到记录可能是对象上没有space属性
174
+ lookups = await lookupCollection.find({ filters: cellFilter });
175
+ let hasSpace = false;
176
+ for (let lookup of lookups) {
177
+ if (lookup.space) {
178
+ hasSpace = true;
179
+ break;
142
180
  }
143
- } else {
144
- jsonObj[field_name] = null;
145
- cellContents.push(dataCell);
146
- }
147
-
148
- for (let cellContent of cellContents) {
149
-
150
- if (!cellContent) {
151
- continue;
152
- }
153
- let cellFilter = [selectfield, "=", cellContent]
154
- let spaceFilter = ['space', "=", options.userSession.spaceId]
155
- let filters = [cellFilter, spaceFilter]
156
- lookups = await lookupCollection.find({ filters: filters });
157
-
158
- if (lookups.length == 0) { //找不到记录可能是对象上没有space属性
159
- lookups = await lookupCollection.find({ filters: cellFilter });
160
- let hasSpace = false;
161
- for (let lookup of lookups) {
162
- if (lookup.space) {
163
- hasSpace = true;
164
- break;
165
- }
166
- }
167
- if (hasSpace) {
168
- lookup_error += `所查找的${reference_to_object}不属于当前space`;
169
- continue;
170
- }
181
+ }
182
+ if (hasSpace) {
183
+ lookup_error += `所查找的${reference_to_object}不属于当前space`;
184
+ continue;
185
+ }
186
+ }
187
+
188
+ // [[selectfield, '=', dataCell], ['space', '=', options.userSession.spaceId]]
189
+
190
+ let allRecordCount = lookups.length;
191
+ let dbRecordCount = lookups.length;
192
+ for (let lookup of lookups) {
193
+ if (lookup.is_system) {
194
+ dbRecordCount--;
195
+ }
196
+ }
197
+ if (dbRecordCount == 1 || allRecordCount == 1) {
198
+ noResult = false;
199
+ if (field.multiple) {
200
+ jsonObj[field_name].push(lookups[0][reference_to_field]);
201
+ } else {
202
+ jsonObj[field_name] = lookups[0][reference_to_field];
203
+ }
204
+ } else if (dbRecordCount.length == 0) {
205
+ if (!dataCell) {
206
+ // 单元格没有填写
207
+
208
+ if (!field.multiple) {
209
+ jsonObj[field_name] = null;
171
210
  }
172
-
173
- // [[selectfield, '=', dataCell], ['space', '=', options.userSession.spaceId]]
174
-
175
- let allRecordCount = lookups.length;
176
- let dbRecordCount = lookups.length;
177
- for (let lookup of lookups) {
178
- if (lookup.is_system) {
179
- dbRecordCount--;
180
- }
181
- }
182
- if (dbRecordCount == 1 || allRecordCount == 1) {
183
- noResult = false;
184
- if (field.multiple) {
185
- jsonObj[field_name].push(lookups[0][reference_to_field]);
186
-
187
- } else {
188
- jsonObj[field_name] = lookups[0][reference_to_field];
189
- }
190
- } else if (dbRecordCount.length == 0) {
191
-
192
- if (!dataCell) {// 单元格没有填写
193
-
194
- if (!field.multiple) {
195
- jsonObj[field_name] = null;
196
- }
197
-
198
- } else { // 单元格有值却找不到记录
199
- if (fieldMap[field_name].save_key_while_fail) {
200
- jsonObj[field_name] = cellContent
201
-
202
- } else {
203
- lookup_error += `${dataCell}不是${field_name}类型数据的key`;
204
- }
205
- }
211
+ } else {
212
+ // 单元格有值却找不到记录
213
+ if (fieldMap[field_name].save_key_while_fail) {
214
+ jsonObj[field_name] = cellContent;
206
215
  } else {
207
- noResult = false;
208
- lookup_error += `无法根据${selectfield}: ${dataCell}找到唯一的${reference_to_object}`;
216
+ lookup_error += `${dataCell}不是${field_name}类型数据的key`;
209
217
  }
210
- }
211
-
212
- if (noResult && field.multiple) { //
213
- jsonObj[field_name] = null;
214
- }
215
- if (noResult && field.required) { //
216
- lookup_error += `${field_name}字段为必填项`;
217
- }
218
- return lookup_error;
219
- };
218
+ }
219
+ } else {
220
+ noResult = false;
221
+ lookup_error += `无法根据${selectfield}: ${dataCell}找到唯一的${reference_to_object}`;
222
+ }
223
+ }
224
+
225
+ if (noResult && field.multiple) {
226
+ //
227
+ jsonObj[field_name] = null;
228
+ }
229
+ if (noResult && field.required) {
230
+ //
231
+ lookup_error += `${field_name}字段为必填项`;
232
+ }
233
+ return lookup_error;
234
+ }
220
235
 
221
236
  function converterBool(field_name, dataCell, jsonObj) {
222
- var bool_error, flag;
223
- bool_error = "";
224
- flag = dataCell.toString().toLowerCase();
225
- if (flag === "是" || flag === "1" || flag === "yes" || flag === "true") {
226
- jsonObj[field_name] = true;
227
- } else if (flag === "否" || flag === "0" || flag === "no" || flag === "false") {
228
- jsonObj[field_name] = false;
229
- } else {
230
- bool_error = `${dataCell}不是bool类型数据`;
231
- }
232
- return bool_error;
233
- };
237
+ var bool_error, flag;
238
+ bool_error = "";
239
+ flag = dataCell.toString().toLowerCase();
240
+ if (flag === "是" || flag === "1" || flag === "yes" || flag === "true") {
241
+ jsonObj[field_name] = true;
242
+ } else if (
243
+ flag === "否" ||
244
+ flag === "0" ||
245
+ flag === "no" ||
246
+ flag === "false"
247
+ ) {
248
+ jsonObj[field_name] = false;
249
+ } else {
250
+ bool_error = `${dataCell}不是bool类型数据`;
251
+ }
252
+ return bool_error;
253
+ }
234
254
 
235
255
  async function insertRow(dataRow, objectName, options: ImportOptions) {
236
- var errorInfo, insertInfo, jsonObj, objFields, ref, lookupFieldMap;
237
- jsonObj = {};
238
- insertInfo = {};
239
- insertInfo["create"] = false;
240
- insertInfo["update"] = false;
241
- errorInfo = "";
242
- lookupFieldMap = options.lookupFieldMap;
243
-
244
- let mappings = options.mappings;
245
- let space = options.userSession.spaceId;
246
-
247
- // 对象的fields
248
- ref = await objectql.getObject(objectName).toConfig()
249
- objFields = ref.fields;
250
- objFields = Object.assign({ _id: { name: "_id", type: "text" } }, objFields);
251
-
252
- let dataLength = dataRow.length > mappings.length ? dataRow.length : mappings.length;
253
- for (let i = 0; i < dataLength; i++) {
254
- let dataCell = dataRow[i];
255
- if (!dataCell) {
256
- dataCell = null;
257
- }
258
-
259
- var error, mapping, noField;
260
- mapping = mappings[i];
261
- if (!mapping) {
262
- continue;
263
- }
264
-
265
- // 找到需要插入的数据
266
- for (let apiName of mapping) {
267
- error = null;
268
- noField = true;
269
- for (let field_name in objFields) {
270
- let field = objFields[field_name]
271
- if (field_name == apiName) {
272
- noField = false;
273
- try{
274
-
275
- switch (field != null ? field.type : void 0) {
276
- case "date":
277
- case "datetime":
278
- error = converterDate(field_name, dataCell, jsonObj);
279
- break;
280
- case "number":
281
- error = converteNum(field_name, dataCell, jsonObj);
282
- break;
283
- case "boolean":
284
- error = converterBool(field_name, dataCell, jsonObj);
285
- break;
286
- case "select":
287
- error = await converterSelect(objectName, field_name, dataCell, jsonObj);
288
- break;
289
- case "lookup":
290
- error = await converterLookup(objectName, field_name, dataCell, jsonObj, lookupFieldMap, options);
291
- break;
292
- case "text":
293
- error = converterString(field_name, dataCell, jsonObj);
294
- break;
295
- case "textarea":
296
- error = converterString(field_name, dataCell, jsonObj);
297
- break;
298
- case "master_detail":
299
- error = await converterLookup(objectName, field_name, dataCell, jsonObj, lookupFieldMap, options);
300
- break;
301
- case "email":
302
- error = converterString(field_name, dataCell, jsonObj);
303
- break;
304
- case "toggle":
305
- error = converterBool(field_name, dataCell, jsonObj);
306
- break;
307
- case "url":
308
- error = converterString(field_name, dataCell, jsonObj);
309
- break;
310
- case "currency":
311
- error = converteNum(field_name, dataCell, jsonObj);
312
- break;
313
- case "percent":
314
- error = converteNum(field_name, dataCell, jsonObj);
315
- break;
316
- default:
317
- throw new Error(`Unsupported data type: ${field.type}`);
318
- }
319
- }catch(err){
320
- console.error(error)
321
- error = err.message;
322
- }
323
- }
324
- }
325
- if (noField) {
326
- if (objectName == 'space_users' && apiName == 'password') {
327
- jsonObj[apiName] = dataCell
328
- } else {
329
- error = `${apiName}不是对象${objectName}的属性`;
330
- }
331
- }
332
- if (error) {
333
- errorInfo = errorInfo + "," + error;
256
+ var errorInfo, insertInfo, jsonObj, objFields, ref, lookupFieldMap;
257
+ jsonObj = {};
258
+ insertInfo = {};
259
+ insertInfo["create"] = false;
260
+ insertInfo["update"] = false;
261
+ errorInfo = "";
262
+ lookupFieldMap = options.lookupFieldMap;
263
+
264
+ let mappings = options.mappings;
265
+ let space = options.userSession.spaceId;
266
+
267
+ // 对象的fields
268
+ ref = await objectql.getObject(objectName).toConfig();
269
+ objFields = ref.fields;
270
+ objFields = Object.assign({ _id: { name: "_id", type: "text" } }, objFields);
271
+
272
+ let dataLength =
273
+ dataRow.length > mappings.length ? dataRow.length : mappings.length;
274
+ for (let i = 0; i < dataLength; i++) {
275
+ let dataCell = dataRow[i];
276
+ if (!dataCell) {
277
+ dataCell = null;
278
+ }
279
+
280
+ var error, mapping, noField;
281
+ mapping = mappings[i];
282
+ if (!mapping) {
283
+ continue;
284
+ }
285
+
286
+ // 找到需要插入的数据
287
+ for (let apiName of mapping) {
288
+ error = null;
289
+ noField = true;
290
+ for (let field_name in objFields) {
291
+ let field = objFields[field_name];
292
+ if (field_name == apiName) {
293
+ noField = false;
294
+ try {
295
+ switch (field != null ? field.type : void 0) {
296
+ case "date":
297
+ case "datetime":
298
+ error = converterDate(field_name, dataCell, jsonObj);
299
+ break;
300
+ case "number":
301
+ error = converteNum(field_name, dataCell, jsonObj);
302
+ break;
303
+ case "boolean":
304
+ error = converterBool(field_name, dataCell, jsonObj);
305
+ break;
306
+ case "select":
307
+ error = await converterSelect(
308
+ objectName,
309
+ field_name,
310
+ dataCell,
311
+ jsonObj
312
+ );
313
+ break;
314
+ case "lookup":
315
+ error = await converterLookup(
316
+ objectName,
317
+ field_name,
318
+ dataCell,
319
+ jsonObj,
320
+ lookupFieldMap,
321
+ options
322
+ );
323
+ break;
324
+ case "text":
325
+ error = converterString(field_name, dataCell, jsonObj);
326
+ break;
327
+ case "textarea":
328
+ error = converterString(field_name, dataCell, jsonObj);
329
+ break;
330
+ case "master_detail":
331
+ error = await converterLookup(
332
+ objectName,
333
+ field_name,
334
+ dataCell,
335
+ jsonObj,
336
+ lookupFieldMap,
337
+ options
338
+ );
339
+ break;
340
+ case "email":
341
+ error = converterString(field_name, dataCell, jsonObj);
342
+ break;
343
+ case "toggle":
344
+ error = converterBool(field_name, dataCell, jsonObj);
345
+ break;
346
+ case "url":
347
+ error = converterString(field_name, dataCell, jsonObj);
348
+ break;
349
+ case "currency":
350
+ error = converteNum(field_name, dataCell, jsonObj);
351
+ break;
352
+ case "percent":
353
+ error = converteNum(field_name, dataCell, jsonObj);
354
+ break;
355
+ default:
356
+ throw new Error(`Unsupported data type: ${field.type}`);
334
357
  }
358
+ } catch (err) {
359
+ console.error(error);
360
+ error = err.message;
361
+ }
335
362
  }
336
-
337
- }
338
-
339
- insertInfo["insertState"] = true;
340
- let selectObj = {};
341
- let recordExists = false;
342
- let objectCollection = await objectql.getObject(objectName);
343
-
344
- if (jsonObj && !errorInfo) {
345
- if (objectCollection.datasource.name == 'default' || objectCollection.datasource.name == 'meteor') {
346
- jsonObj.space = space;
363
+ }
364
+ if (noField) {
365
+ if (objectName == "space_users" && apiName == "password") {
366
+ jsonObj[apiName] = dataCell;
367
+ } else {
368
+ error = `${apiName}不是对象${objectName}的属性`;
347
369
  }
348
- // 不存在则新建,存在则更新
349
- let external_id_name = options.externalIdName
350
- if (external_id_name) {
351
- let allUndefined = true;
352
- for(let _external_id_name of external_id_name){
353
- selectObj[_external_id_name] = jsonObj[_external_id_name];
354
- if (selectObj[_external_id_name] != undefined) {
355
- allUndefined = false;
356
- }
357
- }
358
- if(!allUndefined){
359
- let filters = selectObjectToFilters(selectObj);
360
- let records = await objectCollection.find({ filters });
361
- if (records.length == 1) {
362
-
363
- selectObj['space'] = space
364
- filters = selectObjectToFilters(selectObj);
365
- records = await objectCollection.find({ filters });
366
-
367
- if (records.length == 1) {
368
- recordExists = true;
369
- } else if (records.length > 1) {
370
- errorInfo = `无法根据${external_id_name}: ${selectObj[external_id_name]}找到唯一的${objectName}记录`
371
- } else {
372
- errorInfo = `所查找的记录不属于当前space`
373
- }
374
- } else if (records.length > 1) {
375
- errorInfo = `无法根据${external_id_name}: ${selectObj[external_id_name]}找到唯一的${objectName}记录`
376
- }
377
- }
370
+ }
371
+ if (error) {
372
+ errorInfo = errorInfo + "," + error;
373
+ }
374
+ }
375
+ }
376
+
377
+ insertInfo["insertState"] = true;
378
+ let selectObj = {};
379
+ let recordExists = false;
380
+ let objectCollection = await objectql.getObject(objectName);
381
+
382
+ if (jsonObj && !errorInfo) {
383
+ if (
384
+ objectCollection.datasource.name == "default" ||
385
+ objectCollection.datasource.name == "meteor"
386
+ ) {
387
+ jsonObj.space = space;
388
+ }
389
+ // 不存在则新建,存在则更新
390
+ let external_id_name = options.externalIdName;
391
+ if (external_id_name) {
392
+ let allUndefined = true;
393
+ for (let _external_id_name of external_id_name) {
394
+ selectObj[_external_id_name] = jsonObj[_external_id_name];
395
+ if (selectObj[_external_id_name] != undefined) {
396
+ allUndefined = false;
378
397
  }
379
- }
380
- if (!errorInfo) {
381
- let operation = options.operation // insert update upsert
398
+ }
399
+ if (!allUndefined) {
382
400
  let filters = selectObjectToFilters(selectObj);
383
- delete jsonObj._id;
384
- if (recordExists && (operation == 'update' || operation == "upsert")) {
385
- try {
386
- // if(options.userSession){
387
- // jsonObj['modified_by'] = options.userSession.userId;
388
- // }
389
- await objectCollection.update({ filters }, jsonObj);
390
- insertInfo["create"] = false;
391
- insertInfo["update"] = true;
392
- } catch (error) {
393
- console.error(error)
394
- errorInfo = error.message;
395
- insertInfo["update"] = false;
396
- insertInfo["insertState"] = false;
397
- }
398
- } else if (!recordExists && (operation == 'insert' || operation == "upsert")) {
399
- try {
400
- if (objectCollection.datasource.name == 'default' || objectCollection.datasource.name == 'meteor') {
401
-
402
- if (!jsonObj.owner && options.userSession) {
403
- let userId = options.userSession.userId;
404
- jsonObj['owner'] = userId;
405
- }
406
- }
407
-
408
- // if (jsonObj.owner) {
409
- // let space_user_collection = await objectql.getObject('space_users')
410
- // let space_users = await space_user_collection.find(['user', '=', jsonObj.owner]);
411
- // let space_user = space_users [0]
412
- // jsonObj['created_by'] = space_user.user;
413
- // jsonObj['modified_by'] = space_user.user;
414
- // // jsonObj['company_id'] = space_user.company_id;
415
- // // jsonObj['company_ids'] = space_user.company_ids
416
-
417
- // }
418
- await objectCollection.insert(jsonObj, options.userSession);
419
- insertInfo["create"] = true;
420
- insertInfo["update"] = false;
421
- } catch (error) {
422
- console.error(error)
423
- errorInfo = error.message;
424
- insertInfo["create"] = false;
425
- insertInfo["insertState"] = false;
426
-
427
- }
428
- } else {
429
- insertInfo["ignored"] = true;
401
+ let records = await objectCollection.find({ filters });
402
+ if (records.length == 1) {
403
+ selectObj["space"] = space;
404
+ filters = selectObjectToFilters(selectObj);
405
+ records = await objectCollection.find({ filters });
406
+
407
+ if (records.length == 1) {
408
+ recordExists = true;
409
+ } else if (records.length > 1) {
410
+ errorInfo = `无法根据${external_id_name}: ${selectObj[external_id_name]}找到唯一的${objectName}记录`;
411
+ } else {
412
+ errorInfo = `所查找的记录不属于当前space`;
413
+ }
414
+ } else if (records.length > 1) {
415
+ errorInfo = `无法根据${external_id_name}: ${selectObj[external_id_name]}找到唯一的${objectName}记录`;
430
416
  }
431
- } else {
417
+ }
418
+ }
419
+ }
420
+ if (!errorInfo) {
421
+ let operation = options.operation; // insert update upsert
422
+ let filters = selectObjectToFilters(selectObj);
423
+ delete jsonObj._id;
424
+ if (recordExists && (operation == "update" || operation == "upsert")) {
425
+ try {
426
+ // if(options.userSession){
427
+ // jsonObj['modified_by'] = options.userSession.userId;
428
+ // }
429
+ await objectCollection.update({ filters }, jsonObj);
430
+ insertInfo["create"] = false;
431
+ insertInfo["update"] = true;
432
+ } catch (error) {
433
+ console.error(error);
434
+ errorInfo = error.message;
435
+ insertInfo["update"] = false;
436
+ insertInfo["insertState"] = false;
437
+ }
438
+ } else if (
439
+ !recordExists &&
440
+ (operation == "insert" || operation == "upsert")
441
+ ) {
442
+ try {
443
+ if (
444
+ objectCollection.datasource.name == "default" ||
445
+ objectCollection.datasource.name == "meteor"
446
+ ) {
447
+ if (!jsonObj.owner && options.userSession) {
448
+ let userId = options.userSession.userId;
449
+ jsonObj["owner"] = userId;
450
+ }
451
+ }
452
+
453
+ // if (jsonObj.owner) {
454
+ // let space_user_collection = await objectql.getObject('space_users')
455
+ // let space_users = await space_user_collection.find(['user', '=', jsonObj.owner]);
456
+ // let space_user = space_users [0]
457
+ // jsonObj['created_by'] = space_user.user;
458
+ // jsonObj['modified_by'] = space_user.user;
459
+ // // jsonObj['company_id'] = space_user.company_id;
460
+ // // jsonObj['company_ids'] = space_user.company_ids
461
+
462
+ // }
463
+ await objectCollection.insert(jsonObj, options.userSession);
464
+ insertInfo["create"] = true;
465
+ insertInfo["update"] = false;
466
+ } catch (error) {
467
+ console.error(error);
468
+ errorInfo = error.message;
469
+ insertInfo["create"] = false;
432
470
  insertInfo["insertState"] = false;
471
+ }
472
+ } else {
473
+ insertInfo["ignored"] = true;
433
474
  }
434
- insertInfo["errorInfo"] = errorInfo;
435
- return insertInfo;
436
- };
475
+ } else {
476
+ insertInfo["insertState"] = false;
477
+ }
478
+ insertInfo["errorInfo"] = errorInfo;
479
+ return insertInfo;
480
+ }
437
481
 
438
482
  function selectObjectToFilters(selectObj) {
439
-
440
- let filters: any = [];
441
- for (let k in selectObj) {
442
- let filter: any = [];
443
- filter.push(k);
444
- filter.push('=');
445
- filter.push(selectObj[k]);
446
- filters.push(filter);
447
- }
448
- return filters;
483
+ let filters: any = [];
484
+ for (let k in selectObj) {
485
+ let filter: any = [];
486
+ filter.push(k);
487
+ filter.push("=");
488
+ filter.push(selectObj[k]);
489
+ filters.push(filter);
490
+ }
491
+ return filters;
449
492
  }
450
493
 
451
494
  function loadWorkbook(file) {
452
- var stream, chunks;
453
- stream = file.createReadStream('files');
454
- chunks = [];
455
-
456
- let loadStream = new Promise(function (resolve, reject) {
457
- stream.on('data', function (chunk) {
458
- return chunks.push(chunk);
459
- });
460
- stream.on('end', function () {
461
- let workbook = xlsx.parse(Buffer.concat(chunks), {
462
- cellDates: true
463
- });
464
- resolve(workbook);
465
- });
466
- })
467
-
468
- return loadStream;
495
+ var stream, chunks;
496
+ stream = file.createReadStream("files");
497
+ chunks = [];
498
+
499
+ let loadStream = new Promise(function(resolve, reject) {
500
+ stream.on("data", function(chunk) {
501
+ return chunks.push(chunk);
502
+ });
503
+ stream.on("end", function() {
504
+ let workbook = xlsx.parse(Buffer.concat(chunks), {
505
+ cellDates: true,
506
+ });
507
+ resolve(workbook);
508
+ });
509
+ });
510
+
511
+ return loadStream;
469
512
  }
470
513
 
471
- export async function importWithCmsFile(importObjId, userSession) {
472
-
473
-
474
-
475
- var file, files;
476
-
477
- let queueImport = await objectql.getObject("queue_import").findOne(importObjId);
478
-
479
- if (!queueImport) {
480
- throw new Error(`can not find queue_import record with given id "${importObjId}"`);
481
- }
482
- let options: any = {
483
- userSession,
484
- objectName: queueImport.object_name,
485
- operation: queueImport.operation,
486
- externalIdName: queueImport.external_id_name,
487
- fieldMappings: queueImport.field_mappings,
488
- queueImportId: importObjId
489
- }
490
-
491
- if(options.operation != "insert" && (!options.externalIdName || _.isEmpty(options.externalIdName))){
492
- throw new Error(`external_id_name is required with operation: update or upsert`);
493
- }
494
-
495
- let fileCollection = await objectql.getObject('cfs_files_filerecord')
496
- files = await fileCollection.find({ filters: [['_id', '=', queueImport.file]] });
497
-
498
- if (files && files.length == 0) {
499
- throw new Error(`Upload excel file, please.`)
500
- } else if (files.length > 1) {
501
- throw new Error(`Just need one file.`)
502
- }
503
- file = files[0];
504
-
505
- await importWithExcelFile(file, options);
506
-
507
- //先返回数据导入对象的id
508
- return {
509
- status: "success",
510
- msg: ``,
511
- queueImportId: queueImport._id
512
- };
513
-
514
+ /**
515
+ *
516
+ * @param importObjId
517
+ * @param userSession
518
+ * @param fileId 如果指定了fileId,则使用此id; 否则使用importObject的file
519
+ * @returns
520
+ */
521
+ export async function importWithCmsFile(
522
+ importObjId,
523
+ userSession,
524
+ importObjectHistoryId,
525
+ fileId
526
+ ) {
527
+ var file, files;
528
+
529
+ let queueImport = await objectql
530
+ .getObject("queue_import")
531
+ .findOne(importObjId);
532
+
533
+ if (!queueImport) {
534
+ throw new Error(
535
+ `can not find queue_import record with given id "${importObjId}"`
536
+ );
537
+ }
538
+ let options: any = {
539
+ userSession,
540
+ objectName: queueImport.object_name,
541
+ operation: queueImport.operation,
542
+ externalIdName: queueImport.external_id_name,
543
+ fieldMappings: queueImport.field_mappings,
544
+ queueImportId: importObjId,
545
+ importObjectHistoryId: importObjectHistoryId,
546
+ };
547
+
548
+ if (
549
+ options.operation != "insert" &&
550
+ (!options.externalIdName || _.isEmpty(options.externalIdName))
551
+ ) {
552
+ throw new Error(
553
+ `external_id_name is required with operation: update or upsert`
554
+ );
555
+ }
556
+
557
+ let fileCollection = await objectql.getObject("cfs_files_filerecord");
558
+ files = await fileCollection.find({
559
+ filters: [["_id", "=", fileId || queueImport.file]],
560
+ });
561
+
562
+ if (files && files.length == 0) {
563
+ throw new Error(`Upload excel file, please.`);
564
+ } else if (files.length > 1) {
565
+ throw new Error(`Just need one file.`);
566
+ }
567
+ file = files[0];
568
+
569
+ await importWithExcelFile(file, options);
570
+
571
+ //先返回数据导入对象的id
572
+ return {
573
+ status: "success",
574
+ msg: ``,
575
+ queueImportId: queueImport._id,
576
+ importObjectHistoryId: importObjectHistoryId,
577
+ };
514
578
  }
515
579
 
516
580
  export async function importWithExcelFile(file, options) {
517
-
518
- const start_time = new Date();
519
-
520
- const allowedOperation = ['insert', 'update', 'upsert'];
521
- if (!options.operation || !_.contains(allowedOperation, options.operation)) {
522
- throw new Error(`unsupported operation "${options.operation}"`);
523
- }
524
- let fieldMappings = options.fieldMappings
525
- if (!fieldMappings) {
526
- throw new Error(`fieldMapping is required`)
527
- }
528
- let objectConfig = await objectql.getObject(options.objectName).toConfig();
529
- if (!objectConfig) {
530
- throw new Error(`can not find object "${options.objectName}"`)
531
- }
532
- if (!options.externalIdName) {
533
- throw new Error(`missing option: externalIdName`)
534
- }
535
-
536
- let mappedFieldNames: any = []
537
- for (let i = 0; i < fieldMappings.length; i++) {
538
- let mapping = fieldMappings[i];
539
-
540
- if (!mapping || !mapping.api_name) {
541
- continue;
542
- }
543
-
544
- if (_.contains(mappedFieldNames, mapping.api_name)) {
545
- throw new Error(`field "${mapping.api_name}" should be mapped only once`)
546
- }
547
- mappedFieldNames.push(mapping.api_name);
548
- }
549
-
550
- for(let _externalIdName of options.externalIdName){
551
- if (!_.contains(mappedFieldNames, _externalIdName)) {
552
- throw new Error(`externalIdName "${_externalIdName}" is not mapped in fieldMapping`)
581
+ const start_time = new Date();
582
+
583
+ const allowedOperation = ["insert", "update", "upsert"];
584
+ if (!options.operation || !_.contains(allowedOperation, options.operation)) {
585
+ throw new Error(`unsupported operation "${options.operation}"`);
586
+ }
587
+ let fieldMappings = options.fieldMappings;
588
+ if (!fieldMappings) {
589
+ throw new Error(`fieldMapping is required`);
590
+ }
591
+ let objectConfig = await objectql.getObject(options.objectName).toConfig();
592
+ if (!objectConfig) {
593
+ throw new Error(`can not find object "${options.objectName}"`);
594
+ }
595
+ if (options.operation != "insert" && !options.externalIdName) {
596
+ throw new Error(`missing option: externalIdName`);
597
+ }
598
+
599
+ let mappedFieldNames: any = [];
600
+ for (let i = 0; i < fieldMappings.length; i++) {
601
+ let mapping = fieldMappings[i];
602
+
603
+ if (!mapping || !mapping.api_name) {
604
+ continue;
605
+ }
606
+
607
+ if (_.contains(mappedFieldNames, mapping.api_name)) {
608
+ throw new Error(`field "${mapping.api_name}" should be mapped only once`);
609
+ }
610
+ mappedFieldNames.push(mapping.api_name);
611
+ }
612
+
613
+ if (options.operation != "insert") {
614
+ for (let _externalIdName of options.externalIdName) {
615
+ if (!_.contains(mappedFieldNames, _externalIdName)) {
616
+ throw new Error(
617
+ `externalIdName "${_externalIdName}" is not mapped in fieldMapping`
618
+ );
619
+ }
620
+ if (!_.contains(_.keys(objectConfig.fields), _externalIdName)) {
621
+ if (_externalIdName != "_id") {
622
+ throw new Error(
623
+ `externalIdName "${_externalIdName}" should be a field of object "${options.objectName}"`
624
+ );
553
625
  }
554
- if (!_.contains(_.keys(objectConfig.fields), _externalIdName)) {
555
- if (_externalIdName != "_id") {
556
- throw new Error(`externalIdName "${_externalIdName}" should be a field of object "${options.objectName}"`)
557
- }
626
+ }
627
+ }
628
+ }
629
+
630
+ for (let fieldName in objectConfig.fields) {
631
+ let field = objectConfig.fields[fieldName];
632
+ // update时必填字段需要被映射
633
+ if (
634
+ options.operation == "update" &&
635
+ field.required &&
636
+ !_.contains(mappedFieldNames, fieldName)
637
+ ) {
638
+ throw new Error(
639
+ `field "${fieldName}" is required but not mapped in fieldMapping`
640
+ );
641
+ }
642
+ }
643
+ let recordDatas = await filetoRecords(file, options);
644
+
645
+ async function importDataAsync(recordDatas) {
646
+ //在回调函数里面异步处理导入
647
+
648
+ let importResult: any = await importWithRecords(recordDatas, options);
649
+
650
+ //最后更新导入结果
651
+ const end_time = new Date();
652
+
653
+ if (options.queueImportId) {
654
+ await objectql.getObject("queue_import_history").directUpdate(
655
+ {
656
+ filters: ["_id", "=", options.importObjectHistoryId],
657
+ },
658
+ {
659
+ modified_by: options.userSession.userId,
660
+ error: importResult.errorList,
661
+ total_count: importResult.total_count,
662
+ success_count: importResult.success_count,
663
+ failure_count: importResult.failure_count,
664
+ state: "finished",
665
+ start_time,
666
+ end_time,
558
667
  }
559
- }
560
-
561
- for (let fieldName in objectConfig.fields) {
562
- let field = objectConfig.fields[fieldName];
563
- // update时必填字段需要被映射
564
- if (options.operation == 'update' && field.required && !_.contains(mappedFieldNames, fieldName)) {
565
- throw new Error(`field "${fieldName}" is required but not mapped in fieldMapping`)
566
- }
567
- }
568
- let recordDatas = await filetoRecords(file, options);
569
-
570
- async function importDataAsync(recordDatas) {
571
- //在回调函数里面异步处理导入
572
-
573
- let importResult:any = await importWithRecords(recordDatas, options);
574
-
575
- //最后更新导入结果
576
- const end_time = new Date();
577
-
578
- if(options.queueImportId){
579
- await objectql.getObject("queue_import").directUpdate({
580
- filters: ['_id', '=', options.queueImportId]
581
- }, {
582
- "object_name": options.objectName,
583
- "external_id_name": options.externalIdName,
584
- "modified_by": options.userSession.userId,
585
- error: importResult.errorList,
586
- total_count: importResult.total_count,
587
- success_count: importResult.success_count,
588
- failure_count: importResult.failure_count,
589
- state: "finished",
590
- start_time,
591
- end_time
592
- });
593
- }
594
-
595
- }
596
-
597
- importDataAsync(recordDatas);
668
+ );
669
+ }
670
+ let notificationBody = `总共导入${importResult.total_count}条记录;\n成功: ${importResult.success_count}条;\n失败: ${importResult.failure_count};`;
671
+ if (importResult.errorList && importResult.errorList.length > 0) {
672
+ notificationBody = `${notificationBody}\n错误信息: ${importResult.errorList.join(
673
+ "\n "
674
+ )}`;
675
+ }
676
+
677
+ //发送通知
678
+ return Fiber(function() {
679
+ Creator.addNotifications(
680
+ {
681
+ name: `导入完成: ${file.original.name}`,
682
+ body: notificationBody,
683
+ related_to: {
684
+ o: "queue_import_history",
685
+ ids: [options.importObjectHistoryId],
686
+ },
687
+ related_name: file.original.name,
688
+ from: options.userSession.userId,
689
+ space: options.userSession.spaceId,
690
+ },
691
+ options.userSession.userId,
692
+ options.userSession.userId
693
+ );
694
+ }).run();
695
+ }
696
+
697
+ importDataAsync(recordDatas);
598
698
  }
599
699
 
600
700
  export async function filetoRecords (file, options) {