@solidstarters/solid-core 1.2.115 → 1.2.117
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/iam.config.d.ts +2 -0
- package/dist/config/iam.config.d.ts.map +1 -1
- package/dist/config/iam.config.js +1 -0
- package/dist/config/iam.config.js.map +1 -1
- package/dist/controllers/import-transaction.controller.d.ts +1 -0
- package/dist/controllers/import-transaction.controller.d.ts.map +1 -1
- package/dist/controllers/import-transaction.controller.js +20 -0
- package/dist/controllers/import-transaction.controller.js.map +1 -1
- package/dist/seeders/seed-data/solid-core-metadata.json +7 -0
- package/dist/services/authentication.service.d.ts +1 -0
- package/dist/services/authentication.service.d.ts.map +1 -1
- package/dist/services/authentication.service.js +10 -3
- package/dist/services/authentication.service.js.map +1 -1
- package/dist/services/export-transaction.service.js +1 -1
- package/dist/services/export-transaction.service.js.map +1 -1
- package/dist/services/import-transaction.service.d.ts +15 -3
- package/dist/services/import-transaction.service.d.ts.map +1 -1
- package/dist/services/import-transaction.service.js +139 -30
- package/dist/services/import-transaction.service.js.map +1 -1
- package/dist/services/setting.service.d.ts.map +1 -1
- package/dist/services/setting.service.js +2 -1
- package/dist/services/setting.service.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/config/iam.config.ts +1 -1
- package/src/controllers/import-transaction.controller.ts +15 -0
- package/src/seeders/seed-data/solid-core-metadata.json +7 -0
- package/src/services/authentication.service.ts +13 -7
- package/src/services/export-transaction.service.ts +1 -1
- package/src/services/import-transaction.service.ts +188 -58
- package/src/services/setting.service.ts +3 -2
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@solidstarters/solid-core",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.117",
|
|
4
4
|
"description": "This module is a NestJS module containing all the required core providers required by a Solid application",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
package/src/config/iam.config.ts
CHANGED
|
@@ -12,7 +12,7 @@ export const iamConfig = registerAs('iam', () => {
|
|
|
12
12
|
defaultRole: process.env.IAM_DEFAULT_ROLE ?? 'Public',
|
|
13
13
|
dummyOtp: process.env.IAM_OTP_DUMMY,
|
|
14
14
|
forgotPasswordSendVerificationTokenOn: process.env.IAM_FORGOT_PASSWORD_SEND_VERIFICATION_TOKEN_ON ?? 'email',
|
|
15
|
-
|
|
15
|
+
forceChangePasswordOnFirstLogin:true,
|
|
16
16
|
googleOauth: {
|
|
17
17
|
clientID: process.env.IAM_GOOGLE_OAUTH_CLIENT_ID,
|
|
18
18
|
clientSecret: process.env.IAM_GOOGLE_OAUTH_CLIENT_SECRET,
|
|
@@ -107,6 +107,21 @@ export class ImportTransactionController {
|
|
|
107
107
|
return this.service.getImportMappingInfo(+id);
|
|
108
108
|
}
|
|
109
109
|
|
|
110
|
+
@ApiBearerAuth("jwt")
|
|
111
|
+
@Get(':id/export-failed-import-records')
|
|
112
|
+
async exportFailedImportedImports(@Param('id') id: string, @Res() res: Response) {
|
|
113
|
+
const {stream, fileName, mimeType} = await this.service.exportFailedImportedImports(+id);
|
|
114
|
+
if (stream === null) {
|
|
115
|
+
throw new InternalServerErrorException("Failed records stream is null");
|
|
116
|
+
}
|
|
117
|
+
// ✅ Set response headers for streaming
|
|
118
|
+
res.setHeader('Content-Disposition', `attachment; filename="${fileName}"`);
|
|
119
|
+
res.setHeader('Content-Type', mimeType);
|
|
120
|
+
res.setHeader('Access-Control-Expose-Headers', 'Content-Disposition, Content-Type');
|
|
121
|
+
// Pipe the strea to the response as an excel file
|
|
122
|
+
stream.pipe(res);
|
|
123
|
+
}
|
|
124
|
+
|
|
110
125
|
@ApiBearerAuth("jwt")
|
|
111
126
|
@Post(':id/start-import/sync')
|
|
112
127
|
async startImportSync(@Param('id') id: string) {
|
|
@@ -8811,6 +8811,13 @@
|
|
|
8811
8811
|
"label": "Iam Allow Public Registration"
|
|
8812
8812
|
}
|
|
8813
8813
|
},
|
|
8814
|
+
{
|
|
8815
|
+
"type": "field",
|
|
8816
|
+
"attrs": {
|
|
8817
|
+
"name": "forceChangePasswordOnFirstLogin",
|
|
8818
|
+
"label": "Force Password Change On First Login"
|
|
8819
|
+
}
|
|
8820
|
+
},
|
|
8814
8821
|
{
|
|
8815
8822
|
"type": "field",
|
|
8816
8823
|
"attrs": {
|
|
@@ -82,6 +82,10 @@ export class AuthenticationService {
|
|
|
82
82
|
|
|
83
83
|
) { }
|
|
84
84
|
|
|
85
|
+
private async getConfig( key: string): Promise<any> {
|
|
86
|
+
return this.settingService.getConfigValue(key);
|
|
87
|
+
}
|
|
88
|
+
|
|
85
89
|
private async getCompanyLogo(): Promise<string> {
|
|
86
90
|
return await this.settingService.getConfigValue('companylogo');
|
|
87
91
|
}
|
|
@@ -118,7 +122,7 @@ export class AuthenticationService {
|
|
|
118
122
|
|
|
119
123
|
return user;
|
|
120
124
|
}
|
|
121
|
-
|
|
125
|
+
|
|
122
126
|
async signUp(signUpDto: SignUpDto, activeUser: ActiveUserData = null): Promise<User> {
|
|
123
127
|
// If public registrations are disabled and no activeUser is present when invoking signUp then we throw an exception.
|
|
124
128
|
if (!(await this.settingService.getConfigValue('allowPublicRegistration')) && !activeUser) {
|
|
@@ -126,9 +130,9 @@ export class AuthenticationService {
|
|
|
126
130
|
}
|
|
127
131
|
|
|
128
132
|
try {
|
|
129
|
-
|
|
133
|
+
const onForcePasswordChange = await this.getConfig('forceChangePasswordOnFirstLogin');
|
|
134
|
+
var { user, pwd, autoGeneratedPwd } = await this.populateForSignup(new User(), signUpDto, this.iamConfiguration.activateUserOnRegistration,onForcePasswordChange);
|
|
130
135
|
const savedUser = await this.userRepository.save(user);
|
|
131
|
-
|
|
132
136
|
// Also assign a default role to the newly created user.
|
|
133
137
|
const userRoles = signUpDto.roles ?? [];
|
|
134
138
|
if (this.iamConfiguration.defaultRole) {
|
|
@@ -141,7 +145,7 @@ export class AuthenticationService {
|
|
|
141
145
|
return savedUser;
|
|
142
146
|
} catch (err) {
|
|
143
147
|
const pgUniqueViolationErrorCode = '23505';
|
|
144
|
-
|
|
148
|
+
if (err.code === pgUniqueViolationErrorCode) {
|
|
145
149
|
throw new ConflictException();
|
|
146
150
|
}
|
|
147
151
|
throw err;
|
|
@@ -150,10 +154,11 @@ export class AuthenticationService {
|
|
|
150
154
|
|
|
151
155
|
async signupForExtensionUser<T extends User, U extends CreateUserDto>(signUpDto: SignUpDto, extensionUserDto: U, extensionUserRepo: Repository<T>): Promise<T> {
|
|
152
156
|
try {
|
|
157
|
+
const onForcePasswordChange = await this.getConfig('forceChangePasswordOnFirstLogin');
|
|
153
158
|
// Merge the extended signUpDto attributes into the user entity
|
|
154
159
|
//@ts-ignore
|
|
155
160
|
const extensionUser = extensionUserRepo.merge(extensionUserRepo.create() as T, extensionUserDto);
|
|
156
|
-
var { user, pwd, autoGeneratedPwd } = await this.populateForSignup<T>(extensionUser, signUpDto);
|
|
161
|
+
var { user, pwd, autoGeneratedPwd } = await this.populateForSignup<T>(extensionUser, signUpDto,onForcePasswordChange);
|
|
157
162
|
const savedUser = await extensionUserRepo.save(user);
|
|
158
163
|
|
|
159
164
|
await this.handlePostSignup(savedUser, signUpDto.roles, pwd, autoGeneratedPwd);
|
|
@@ -170,7 +175,7 @@ export class AuthenticationService {
|
|
|
170
175
|
}
|
|
171
176
|
|
|
172
177
|
|
|
173
|
-
private async populateForSignup<T extends User>(user: T, signUpDto: SignUpDto, isUserActive: boolean = true) {
|
|
178
|
+
private async populateForSignup<T extends User>(user: T, signUpDto: SignUpDto, isUserActive: boolean = true,onForcePasswordChange?:boolean) {
|
|
174
179
|
// const user = new User();
|
|
175
180
|
|
|
176
181
|
if (signUpDto.roles && signUpDto.roles.length > 0) {
|
|
@@ -182,10 +187,11 @@ export class AuthenticationService {
|
|
|
182
187
|
user.username = signUpDto.username;
|
|
183
188
|
user.email = signUpDto.email;
|
|
184
189
|
user.fullName = signUpDto.fullName;
|
|
190
|
+
user.forcePasswordChange = onForcePasswordChange;
|
|
185
191
|
if (signUpDto.mobile) {
|
|
186
192
|
user.mobile = signUpDto.mobile;
|
|
187
193
|
}
|
|
188
|
-
|
|
194
|
+
this.logger.debug("user",user);
|
|
189
195
|
// If password has been specified by the user, then we simply create & activate the user based on the configuration parameter "activateUserOnRegistration".
|
|
190
196
|
let pwd = '';
|
|
191
197
|
let autoGeneratedPwd = '';
|
|
@@ -167,7 +167,7 @@ export class ExportTransactionService extends CRUDService<ExportTransaction> {
|
|
|
167
167
|
exportStream = await this.excelService.createExcelStream(dataRecordsFunc, EXPORT_CHUNK_SIZE);
|
|
168
168
|
break;
|
|
169
169
|
case ExportFormat.CSV:
|
|
170
|
-
exportStream = await
|
|
170
|
+
exportStream = await this.csvService.createCsvStream(dataRecordsFunc, EXPORT_CHUNK_SIZE);
|
|
171
171
|
break;
|
|
172
172
|
default:
|
|
173
173
|
throw new Error('Invalid export format');
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Injectable, Logger } from '@nestjs/common';
|
|
1
|
+
import { BadRequestException, Injectable, Logger } from '@nestjs/common';
|
|
2
2
|
import { DiscoveryService, ModuleRef } from "@nestjs/core";
|
|
3
3
|
import { InjectEntityManager, InjectRepository } from '@nestjs/typeorm';
|
|
4
4
|
import { EntityManager, Repository } from 'typeorm';
|
|
@@ -16,9 +16,11 @@ import { HttpService } from '@nestjs/axios';
|
|
|
16
16
|
import { RelationFieldsCommand, RelationType, SolidFieldType } from 'src/dtos/create-field-metadata.dto';
|
|
17
17
|
import { ImportInstructionsResponseDto, StandardImportInstructionsResponseDto } from 'src/dtos/import-instructions.dto';
|
|
18
18
|
import { FieldMetadata } from 'src/entities/field-metadata.entity';
|
|
19
|
+
import { ImportTransactionErrorLog } from 'src/entities/import-transaction-error-log.entity';
|
|
19
20
|
import { ModelMetadata } from 'src/entities/model-metadata.entity';
|
|
20
21
|
import { MediaWithFullUrl } from 'src/interfaces';
|
|
21
22
|
import { Readable } from 'stream';
|
|
23
|
+
import { v4 as uuidv4 } from 'uuid';
|
|
22
24
|
import { ImportTransaction } from '../entities/import-transaction.entity';
|
|
23
25
|
import { CsvService } from './csv.service';
|
|
24
26
|
import { ExcelService } from './excel.service';
|
|
@@ -34,6 +36,11 @@ export enum ImportFormat {
|
|
|
34
36
|
CSV = 'csv',
|
|
35
37
|
EXCEL = 'excel',
|
|
36
38
|
}
|
|
39
|
+
|
|
40
|
+
export enum ImportMimeTypes {
|
|
41
|
+
CSV = 'text/csv',
|
|
42
|
+
EXCEL = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
|
43
|
+
}
|
|
37
44
|
export interface ImportMappingInfo {
|
|
38
45
|
sampleImportedRecordInfo: SampleImportedRecordInfo[];
|
|
39
46
|
importableFields: ImportableFieldInfo[];
|
|
@@ -63,6 +70,11 @@ export interface ImportSyncResult {
|
|
|
63
70
|
importedIds: Array<number>; // The IDs of the records created during the import
|
|
64
71
|
}
|
|
65
72
|
|
|
73
|
+
interface ImportRecordsResult {
|
|
74
|
+
ids: Array<number>; // The IDs of the records created during the import
|
|
75
|
+
errorLogIds: Array<number>; // The IDs of the error log entries created during the import
|
|
76
|
+
}
|
|
77
|
+
|
|
66
78
|
@Injectable()
|
|
67
79
|
export class ImportTransactionService extends CRUDService<ImportTransaction> {
|
|
68
80
|
constructor(
|
|
@@ -242,25 +254,106 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
|
|
|
242
254
|
// Get the import file stream for the import transaction
|
|
243
255
|
const importFileStream = await this.getImportFileStream(importFileMediaObject);
|
|
244
256
|
|
|
245
|
-
const ids = await this.
|
|
257
|
+
const { ids, errorLogIds } = await this.importFromFileToDB(
|
|
258
|
+
importTransaction,
|
|
246
259
|
importFileStream,
|
|
247
260
|
importFileMediaObject.mimeType,
|
|
248
|
-
JSON.parse(importTransaction.mapping) as ImportMapping[], // Parse the mapping from the import transaction
|
|
249
|
-
importTransaction.modelMetadata,
|
|
250
261
|
);
|
|
251
262
|
|
|
252
263
|
// Update the import transaction status to 'completed'
|
|
253
|
-
importTransaction.status = 'import_succeeded';
|
|
264
|
+
(errorLogIds.length > 0) ? importTransaction.status = 'import_failed' : importTransaction.status = 'import_succeeded'; //FIXME: We can probably have import_partially_failed status to differentiate
|
|
254
265
|
// Save the import transaction
|
|
255
266
|
await this.repo.save(importTransaction);
|
|
256
267
|
|
|
257
|
-
return {status: importTransaction.status, importedIds: ids}; // Return the IDs of the created records
|
|
268
|
+
return { status: importTransaction.status, importedIds: ids }; // Return the IDs of the created records
|
|
258
269
|
}
|
|
259
270
|
|
|
260
271
|
startImportAsync(importTransactionId: number): Promise<void> {
|
|
261
272
|
throw new Error('Method not implemented.');
|
|
262
273
|
}
|
|
263
274
|
|
|
275
|
+
async exportFailedImportedImports(importTransactionId: number) {
|
|
276
|
+
// Get the 1st error log entry to determine the headers for the export file
|
|
277
|
+
const firstErrorLogEntry = await this.entityManager.getRepository(ImportTransactionErrorLog).findOne({
|
|
278
|
+
where: {
|
|
279
|
+
importTransaction: { id: importTransactionId },
|
|
280
|
+
},
|
|
281
|
+
});
|
|
282
|
+
|
|
283
|
+
if (!firstErrorLogEntry) {
|
|
284
|
+
throw new BadRequestException(`No error log entries found for import transaction ID ${importTransactionId}.`);
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// Create the headers for the export file
|
|
288
|
+
const headers = [
|
|
289
|
+
'rowNumber', // Row number in the import file
|
|
290
|
+
'errorMessage', // Error message for the failed record
|
|
291
|
+
'errorTrace', // Error trace for debugging
|
|
292
|
+
...Object.keys(firstErrorLogEntry.rowData ? JSON.parse(firstErrorLogEntry.rowData) : {}), // Include all keys from the rowData JSON
|
|
293
|
+
];
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
// Depending upon the format of the import tranaction file, create a readable stream of the error log entries
|
|
297
|
+
const importTransaction = await this.loadImportTransaction(importTransactionId);
|
|
298
|
+
const importFileMediaObject = this.getImportFileObject(importTransaction);
|
|
299
|
+
const mimeType = importFileMediaObject.mimeType;
|
|
300
|
+
const templateFormat = mimeType === ImportMimeTypes.CSV ? "csv" : "excel";
|
|
301
|
+
const dataRecordsFunc = async (chunkIndex: number, chunkSize: number): Promise<any[]> => {
|
|
302
|
+
// Get the error log entries for the import transaction
|
|
303
|
+
const errorLogEntries = await this.entityManager.getRepository(ImportTransactionErrorLog).find({
|
|
304
|
+
where: {
|
|
305
|
+
importTransaction: { id: importTransactionId },
|
|
306
|
+
},
|
|
307
|
+
skip: chunkIndex * chunkSize,
|
|
308
|
+
take: chunkSize,
|
|
309
|
+
});
|
|
310
|
+
|
|
311
|
+
if (!errorLogEntries || errorLogEntries.length === 0) {
|
|
312
|
+
return []; // Return an empty array if no error log entries found
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
// Read the row data json from the error log entry, parse it and write it as a record to the stream
|
|
316
|
+
return errorLogEntries.map(entry => {
|
|
317
|
+
const rowData = entry.rowData ? JSON.parse(entry.rowData) : {};
|
|
318
|
+
return {
|
|
319
|
+
rowNumber: entry.rowNumber,
|
|
320
|
+
errorMessage: entry.errorMessage,
|
|
321
|
+
errorTrace: entry.errorTrace,
|
|
322
|
+
...rowData, // Spread the row data into the record
|
|
323
|
+
};
|
|
324
|
+
});
|
|
325
|
+
};
|
|
326
|
+
|
|
327
|
+
// Get the export stream for the failed records
|
|
328
|
+
const exportStream = await this.getFailedRecordsStream(templateFormat, headers, dataRecordsFunc);
|
|
329
|
+
if (!exportStream) {
|
|
330
|
+
throw new BadRequestException(`Failed to create export stream for import transaction ID ${importTransactionId}.`);
|
|
331
|
+
}
|
|
332
|
+
// Return the export stream
|
|
333
|
+
return {
|
|
334
|
+
stream: exportStream,
|
|
335
|
+
fileName: `${importTransaction.modelMetadata.singularName}-failed-imports.${templateFormat}`,
|
|
336
|
+
mimeType: templateFormat === "excel" ? ImportMimeTypes.EXCEL : ImportMimeTypes.CSV,
|
|
337
|
+
};
|
|
338
|
+
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
private async getFailedRecordsStream(templateFormat: string, headers: string[], dataRecordsFunc: (chunkIndex: number, chunkSize: number) => Promise<any[]>) {
|
|
342
|
+
let exportStream = null;
|
|
343
|
+
switch (templateFormat) {
|
|
344
|
+
case "excel":
|
|
345
|
+
exportStream = await this.excelService.createExcelStream(dataRecordsFunc, 100, headers);
|
|
346
|
+
break;
|
|
347
|
+
case "csv":
|
|
348
|
+
exportStream = await this.csvService.createCsvStream(dataRecordsFunc, 100, headers);
|
|
349
|
+
break;
|
|
350
|
+
default:
|
|
351
|
+
throw new Error('Invalid export format');
|
|
352
|
+
}
|
|
353
|
+
return exportStream;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
|
|
264
357
|
private async loadImportTransaction(importTransactionId: number) {
|
|
265
358
|
const importTransaction = await this.findOne(importTransactionId, {
|
|
266
359
|
populate: ['modelMetadata', 'modelMetadata.fields'],
|
|
@@ -329,48 +422,96 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
|
|
|
329
422
|
return fileUrlResponse.data;
|
|
330
423
|
}
|
|
331
424
|
|
|
332
|
-
private async
|
|
425
|
+
private async importFromFileToDB(
|
|
426
|
+
importTransaction: ImportTransaction,
|
|
333
427
|
importFileStream: Readable,
|
|
334
428
|
mimeType: string,
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
429
|
+
): Promise<ImportRecordsResult> {
|
|
430
|
+
if (!importTransaction.modelMetadata) {
|
|
431
|
+
throw new Error(`Model metadata for import transaction ID ${importTransaction.id} not found.`);
|
|
432
|
+
}
|
|
433
|
+
|
|
340
434
|
const createdRecordIds = [];
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
//
|
|
344
|
-
|
|
345
|
-
|
|
435
|
+
const createdErrorLogIds = [];
|
|
436
|
+
|
|
437
|
+
// Get the model service for the model metadata name
|
|
438
|
+
const modelService = this.getModelService(importTransaction.modelMetadata.singularName);
|
|
439
|
+
|
|
440
|
+
// Depending upon the mime type of the file, read the file in pages and insert the records into the database
|
|
441
|
+
if (mimeType === ImportMimeTypes.CSV) {
|
|
346
442
|
for await (const page of this.csvService.readCsvInPagesFromStream(importFileStream)) {
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
const createdRecords = await modelService.insertMany(dtos, [], {});
|
|
351
|
-
// Set the solidRequestContext to null, as this is a background job;
|
|
352
|
-
// Return the IDs of the created records
|
|
353
|
-
const newIds = createdRecords.map(record => record.id);
|
|
354
|
-
// Add the new IDs to the createdRecordIds array
|
|
355
|
-
createdRecordIds.push(...newIds);
|
|
443
|
+
const { ids, errorLogIds } = await this.importRecords(page, importTransaction, modelService);
|
|
444
|
+
createdRecordIds.push(...ids);
|
|
445
|
+
createdErrorLogIds.push(...errorLogIds);
|
|
356
446
|
}
|
|
357
447
|
}
|
|
358
|
-
else if (mimeType ===
|
|
359
|
-
// Read the excel file in pages
|
|
448
|
+
else if (mimeType === ImportMimeTypes.EXCEL) {
|
|
360
449
|
for await (const page of this.excelService.readExcelInPagesFromStream(importFileStream)) {
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
const createdRecords = await modelService.insertMany(dtos, [], {});
|
|
365
|
-
// Set the solidRequestContext to null, as this is a background job;
|
|
366
|
-
// Return the IDs of the created records
|
|
367
|
-
const newIds = createdRecords.map(record => record.id);
|
|
368
|
-
createdRecordIds.push(...newIds);
|
|
450
|
+
const { ids, errorLogIds } = await this.importRecords(page, importTransaction, modelService);
|
|
451
|
+
createdRecordIds.push(...ids);
|
|
452
|
+
createdErrorLogIds.push(...errorLogIds);
|
|
369
453
|
}
|
|
370
454
|
} else { // If the file is neither CSV nor Excel, throw an error
|
|
371
455
|
throw new Error(`Unsupported file type: ${mimeType}`);
|
|
372
456
|
}
|
|
373
|
-
|
|
457
|
+
|
|
458
|
+
return {
|
|
459
|
+
ids: createdRecordIds, // Return the IDs of the created records
|
|
460
|
+
errorLogIds: createdErrorLogIds, // Return the IDs of the error log entries created during the import
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
private async importRecords(page: ImportPaginatedReadResult, importTransaction: ImportTransaction, modelService: CRUDService<any>): Promise<ImportRecordsResult> {
|
|
465
|
+
if (!importTransaction.modelMetadata || !importTransaction.modelMetadata.fields) {
|
|
466
|
+
throw new Error(`Model metadata with fields for import transaction ID ${importTransaction.id} not found.`);
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
const ids: Array<number> = [];
|
|
470
|
+
const errorLogIds: Array<number> = [];
|
|
471
|
+
for (const record of page.data) {
|
|
472
|
+
try {
|
|
473
|
+
const createdRecord = await this.insertRecord(record, JSON.parse(importTransaction.mapping) as ImportMapping[], importTransaction.modelMetadata, modelService);
|
|
474
|
+
ids.push(createdRecord.id); // Add the ID of the created record to the ids array
|
|
475
|
+
}
|
|
476
|
+
catch (error) {
|
|
477
|
+
this.logger.debug(`Error inserting record: ${JSON.stringify(record)}. Error: ${error.message}`);
|
|
478
|
+
// Get the Import transaction error log repo
|
|
479
|
+
const errorLog = await this.createErrorLogEntry(importTransaction, record, error);
|
|
480
|
+
errorLogIds.push(errorLog.id); // Add the ID of the error log entry to the errorLogIds array
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
return {
|
|
484
|
+
ids: ids, // Return the IDs of the created records
|
|
485
|
+
errorLogIds: errorLogIds, // Return the IDs of the error log entries created during the import
|
|
486
|
+
};
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
private async createErrorLogEntry(importTransaction: ImportTransaction, record: Record<string, any>, error: any) {
|
|
490
|
+
const importTransactionRepo = this.entityManager.getRepository(ImportTransactionErrorLog);
|
|
491
|
+
// Create a new ImportTransactionErrorLog entry
|
|
492
|
+
const rowNumber = uuidv4(); // Generate a unique row number or use page.rowNumber if available
|
|
493
|
+
|
|
494
|
+
const errorLogEntry = {
|
|
495
|
+
importTransactionErrorLogId: `${importTransaction.id}-${rowNumber}`, // FIXME pending to retrieve the row number from the page
|
|
496
|
+
rowNumber: 1, // FIXME pending to retrieve the row number from the page
|
|
497
|
+
rowData: JSON.stringify(record), // Store the row data
|
|
498
|
+
importTransaction: importTransaction, // Link to the import transaction
|
|
499
|
+
errorMessage: error.message, // Store the error message
|
|
500
|
+
errorTrace: error.stack || '', // Store the error stack trace if available
|
|
501
|
+
} as ImportTransactionErrorLog;
|
|
502
|
+
|
|
503
|
+
// Save the error log entry to the database
|
|
504
|
+
const savedEntry = await importTransactionRepo.save(errorLogEntry);
|
|
505
|
+
return savedEntry; // Return the ID of the saved error log entry
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
//FIXME Currently below method fails if any field in the record is not valid or if the record is not valid. It does not collect the errors for all fields in a record
|
|
509
|
+
private async insertRecord(record: Record<string, any>, mapping: ImportMapping[], modelMetadataWithFields: ModelMetadata, modelService: CRUDService<any>): Promise<any> {
|
|
510
|
+
// Convert the imported record to a DTO
|
|
511
|
+
const dto = await this.convertImportedRecordToDto(record, mapping, modelMetadataWithFields);
|
|
512
|
+
// Use the model service to create the record in the database
|
|
513
|
+
const createdRecord = await modelService.create(dto, [], {}); //FIXME: Need to handle this part alongwith the refactoring of the CRUDService for permissions
|
|
514
|
+
return createdRecord; // Return the created record
|
|
374
515
|
}
|
|
375
516
|
|
|
376
517
|
private getModelService(modelSingularName: string): CRUDService<any> {
|
|
@@ -383,22 +524,11 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
|
|
|
383
524
|
return modelService;
|
|
384
525
|
}
|
|
385
526
|
|
|
386
|
-
// This method will
|
|
387
|
-
private async convertPaginatedResultToDtos(importPaginatedResult: ImportPaginatedReadResult, modelMetadataWithFields: ModelMetadata, mapping: ImportMapping[]) {
|
|
388
|
-
const dtos = [];
|
|
389
|
-
// Iterate through the data records in the importPaginatedResult
|
|
390
|
-
for (const record of importPaginatedResult.data) {
|
|
391
|
-
// For every key in the record, get the corresponding field from the mapping, if the field is not found in mapping, skip the field
|
|
392
|
-
const dto = await this.convertImportedRecordToDto(record, mapping, modelMetadataWithFields);
|
|
393
|
-
dtos.push(dto);
|
|
394
|
-
}
|
|
395
|
-
return dtos;
|
|
396
|
-
}
|
|
397
|
-
|
|
398
527
|
private async convertImportedRecordToDto(record: Record<string, any>, mapping: ImportMapping[], modelMetadataWithFields: ModelMetadata) {
|
|
399
528
|
// Create a new record object
|
|
400
529
|
const dtoRecord: Record<string, any> = {};
|
|
401
530
|
|
|
531
|
+
// Iterate through every cell in the record
|
|
402
532
|
// Using the saved mapping, populate the dtoRecord w.r.t the record and fields
|
|
403
533
|
for (const key in record) {
|
|
404
534
|
const mappedField = mapping.find(m => m.header === key);
|
|
@@ -408,7 +538,7 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
|
|
|
408
538
|
// const userKeyField = modelMetadataWithFields.fields.find(f => f.isUserKey === true); // Assuming userKey is a field in the model metadata
|
|
409
539
|
if (fieldMetadata) {
|
|
410
540
|
// If the field is found in the model metadata, set the value in the dtoRecord
|
|
411
|
-
await this.
|
|
541
|
+
await this.populateDtoForACell(dtoRecord, fieldMetadata, record, key);
|
|
412
542
|
} else {
|
|
413
543
|
this.logger.warn(`Field ${mappedField.fieldName} not found in model metadata ${modelMetadataWithFields.singularName}`);
|
|
414
544
|
}
|
|
@@ -417,7 +547,7 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
|
|
|
417
547
|
return dtoRecord;
|
|
418
548
|
}
|
|
419
549
|
|
|
420
|
-
private async
|
|
550
|
+
private async populateDtoForACell(dtoRecord: Record<string, any>, fieldMetadata: FieldMetadata, record: Record<string, any>, key: string): Promise<Record<string, any>> {
|
|
421
551
|
const fieldType = fieldMetadata.type;
|
|
422
552
|
// const userKeyFieldName = userKeyField?.name || 'id'; // Default to 'id' if not found
|
|
423
553
|
|
|
@@ -426,14 +556,14 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
|
|
|
426
556
|
case SolidFieldType.relation: {
|
|
427
557
|
return await this.populateDtoForRelations(fieldMetadata, record, key, dtoRecord);
|
|
428
558
|
}
|
|
429
|
-
case SolidFieldType.date:
|
|
559
|
+
case SolidFieldType.date:
|
|
430
560
|
case SolidFieldType.datetime: return this.populateDtoForDate(record, key, fieldMetadata, dtoRecord);
|
|
431
561
|
case SolidFieldType.int:
|
|
432
562
|
case SolidFieldType.bigint:
|
|
433
563
|
case SolidFieldType.decimal:
|
|
434
|
-
return this.populateDtoForNumber(dtoRecord, fieldMetadata, record, key);
|
|
564
|
+
return this.populateDtoForNumber(dtoRecord, fieldMetadata, record, key);
|
|
435
565
|
case SolidFieldType.boolean:
|
|
436
|
-
return this.populateDtoForBoolean(dtoRecord, fieldMetadata, record, key);
|
|
566
|
+
return this.populateDtoForBoolean(dtoRecord, fieldMetadata, record, key);
|
|
437
567
|
default:
|
|
438
568
|
dtoRecord[fieldMetadata.name] = record[key];
|
|
439
569
|
return dtoRecord;
|
|
@@ -443,7 +573,7 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
|
|
|
443
573
|
private populateDtoForBoolean(dtoRecord: Record<string, any>, fieldMetadata: FieldMetadata, record: Record<string, any>, key: string) {
|
|
444
574
|
const booleanValue = Boolean(record[key]);
|
|
445
575
|
if (typeof booleanValue !== 'boolean') {
|
|
446
|
-
throw new Error(`Invalid boolean value for
|
|
576
|
+
throw new Error(`Invalid boolean value for cell ${key} with value ${record[key]}`);
|
|
447
577
|
}
|
|
448
578
|
dtoRecord[fieldMetadata.name] = booleanValue;
|
|
449
579
|
return dtoRecord;
|
|
@@ -452,7 +582,7 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
|
|
|
452
582
|
private populateDtoForNumber(dtoRecord: Record<string, any>, fieldMetadata: FieldMetadata, record: Record<string, any>, key: string) {
|
|
453
583
|
const numberValue = Number(record[key]);
|
|
454
584
|
if (isNaN(numberValue)) {
|
|
455
|
-
throw new Error(`Invalid number value for
|
|
585
|
+
throw new Error(`Invalid number value for cell ${key} with value ${record[key]}`);
|
|
456
586
|
}
|
|
457
587
|
dtoRecord[fieldMetadata.name] = numberValue;
|
|
458
588
|
return dtoRecord;
|
|
@@ -462,7 +592,7 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
|
|
|
462
592
|
{
|
|
463
593
|
const dateValue = new Date(record[key]);
|
|
464
594
|
if (isNaN(dateValue.getTime())) {
|
|
465
|
-
throw new Error(`Invalid date value for
|
|
595
|
+
throw new Error(`Invalid date value for cell ${key} with value ${record[key]}`);
|
|
466
596
|
}
|
|
467
597
|
dtoRecord[fieldMetadata.name] = dateValue;
|
|
468
598
|
return dtoRecord;
|
|
@@ -511,7 +641,7 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
|
|
|
511
641
|
// From the userKeys, we will get the IDs of the related records using the userKeyFieldName and throw an error if any of the userKeys is not found
|
|
512
642
|
const relatedRecordsResult = await coModelService.find(relationFilterDto);
|
|
513
643
|
if (!relatedRecordsResult || !relatedRecordsResult.records || relatedRecordsResult.records.length === 0 || relatedRecordsResult.records.length !== relationUserKeys.length) {
|
|
514
|
-
throw new Error(`
|
|
644
|
+
throw new Error(`Invalid related records userKey values found for cell ${key} with value ${record[key]}`);
|
|
515
645
|
}
|
|
516
646
|
const relatedRecordsIds = relatedRecordsResult.records.map(record => record.id);
|
|
517
647
|
return relatedRecordsIds;
|
|
@@ -146,7 +146,8 @@ export class SettingService extends CRUDService<Setting> {
|
|
|
146
146
|
shouldQueueEmails: this.commonConfiguration.shouldQueueEmails,
|
|
147
147
|
shouldQueueSms: this.commonConfiguration.shouldQueueSms,
|
|
148
148
|
enableDarkMode: true,
|
|
149
|
-
copyright : ""
|
|
149
|
+
copyright : "",
|
|
150
|
+
forceChangePasswordOnFirstLogin:true
|
|
150
151
|
};
|
|
151
152
|
}
|
|
152
153
|
|
|
@@ -154,7 +155,7 @@ export class SettingService extends CRUDService<Setting> {
|
|
|
154
155
|
try {
|
|
155
156
|
const settingsArray: Setting[] = await this.repo.find();
|
|
156
157
|
const settingEntry = settingsArray.find(setting => setting.key === settingKey);
|
|
157
|
-
|
|
158
|
+
|
|
158
159
|
if (settingEntry && settingEntry.value !== null && settingEntry.value !== undefined) {
|
|
159
160
|
const value = settingEntry.value;
|
|
160
161
|
|