@solidstarters/solid-core 1.2.73 → 1.2.76
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/controllers/export-template.controller.d.ts +36 -0
- package/dist/controllers/export-template.controller.d.ts.map +1 -0
- package/dist/controllers/export-template.controller.js +182 -0
- package/dist/controllers/export-template.controller.js.map +1 -0
- package/dist/controllers/export-transaction.controller.d.ts +33 -0
- package/dist/controllers/export-transaction.controller.d.ts.map +1 -0
- package/dist/controllers/export-transaction.controller.js +150 -0
- package/dist/controllers/export-transaction.controller.js.map +1 -0
- package/dist/dtos/create-export-template.dto.d.ts +9 -0
- package/dist/dtos/create-export-template.dto.d.ts.map +1 -0
- package/dist/dtos/create-export-template.dto.js +55 -0
- package/dist/dtos/create-export-template.dto.js.map +1 -0
- package/dist/dtos/create-export-transaction.dto.d.ts +9 -0
- package/dist/dtos/create-export-transaction.dto.d.ts.map +1 -0
- package/dist/dtos/create-export-transaction.dto.js +50 -0
- package/dist/dtos/create-export-transaction.dto.js.map +1 -0
- package/dist/dtos/update-export-template.dto.d.ts +10 -0
- package/dist/dtos/update-export-template.dto.d.ts.map +1 -0
- package/dist/dtos/update-export-template.dto.js +62 -0
- package/dist/dtos/update-export-template.dto.js.map +1 -0
- package/dist/dtos/update-export-transaction.dto.d.ts +9 -0
- package/dist/dtos/update-export-transaction.dto.d.ts.map +1 -0
- package/dist/dtos/update-export-transaction.dto.js +53 -0
- package/dist/dtos/update-export-transaction.dto.js.map +1 -0
- package/dist/entities/export-template.entity.d.ts +10 -0
- package/dist/entities/export-template.entity.d.ts.map +1 -0
- package/dist/entities/export-template.entity.js +53 -0
- package/dist/entities/export-template.entity.js.map +1 -0
- package/dist/entities/export-transaction.entity.d.ts +10 -0
- package/dist/entities/export-transaction.entity.d.ts.map +1 -0
- package/dist/entities/export-transaction.entity.js +51 -0
- package/dist/entities/export-transaction.entity.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/interfaces.d.ts +2 -0
- package/dist/interfaces.d.ts.map +1 -1
- package/dist/interfaces.js.map +1 -1
- package/dist/seeders/module-metadata-seeder.service.js +1 -1
- package/dist/seeders/module-metadata-seeder.service.js.map +1 -1
- package/dist/seeders/seed-data/solid-core-metadata.json +183 -1
- package/dist/services/csv.service.d.ts +6 -0
- package/dist/services/csv.service.d.ts.map +1 -0
- package/dist/services/csv.service.js +48 -0
- package/dist/services/csv.service.js.map +1 -0
- package/dist/services/excel.service.d.ts +6 -0
- package/dist/services/excel.service.d.ts.map +1 -0
- package/dist/services/excel.service.js +90 -0
- package/dist/services/excel.service.js.map +1 -0
- package/dist/services/export-template.service.d.ts +27 -0
- package/dist/services/export-template.service.d.ts.map +1 -0
- package/dist/services/export-template.service.js +79 -0
- package/dist/services/export-template.service.js.map +1 -0
- package/dist/services/export-transaction.service.d.ts +50 -0
- package/dist/services/export-transaction.service.d.ts.map +1 -0
- package/dist/services/export-transaction.service.js +191 -0
- package/dist/services/export-transaction.service.js.map +1 -0
- package/dist/services/file.service.d.ts +3 -0
- package/dist/services/file.service.d.ts.map +1 -1
- package/dist/services/file.service.js +18 -2
- package/dist/services/file.service.js.map +1 -1
- package/dist/services/mediaStorageProviders/file-s3-storage-provider.d.ts +4 -2
- package/dist/services/mediaStorageProviders/file-s3-storage-provider.d.ts.map +1 -1
- package/dist/services/mediaStorageProviders/file-s3-storage-provider.js +4 -1
- package/dist/services/mediaStorageProviders/file-s3-storage-provider.js.map +1 -1
- package/dist/services/mediaStorageProviders/file-storage-provider.d.ts +5 -3
- package/dist/services/mediaStorageProviders/file-storage-provider.d.ts.map +1 -1
- package/dist/services/mediaStorageProviders/file-storage-provider.js +23 -3
- package/dist/services/mediaStorageProviders/file-storage-provider.js.map +1 -1
- package/dist/services/solid-introspect.service.d.ts.map +1 -1
- package/dist/services/solid-introspect.service.js.map +1 -1
- package/dist/solid-core.module.d.ts.map +1 -1
- package/dist/solid-core.module.js +17 -1
- package/dist/solid-core.module.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +3 -1
- package/src/controllers/export-template.controller.ts +98 -0
- package/src/controllers/export-transaction.controller.ts +76 -0
- package/src/dtos/create-export-template.dto.ts +27 -0
- package/src/dtos/create-export-transaction.dto.ts +26 -0
- package/src/dtos/update-export-template.dto.ts +33 -0
- package/src/dtos/update-export-transaction.dto.ts +28 -0
- package/src/entities/export-template.entity.ts +20 -0
- package/src/entities/export-transaction.entity.ts +22 -0
- package/src/index.ts +2 -0
- package/src/interfaces.ts +2 -0
- package/src/seeders/module-metadata-seeder.service.ts +1 -1
- package/src/seeders/seed-data/solid-core-metadata.json +183 -1
- package/src/services/csv.service.ts +41 -0
- package/src/services/excel.service.ts +105 -0
- package/src/services/export-template.service.ts +71 -0
- package/src/services/export-transaction.service.ts +208 -0
- package/src/services/export_issues.txt +6 -0
- package/src/services/file.service.ts +26 -1
- package/src/services/mediaStorageProviders/file-s3-storage-provider.ts +9 -6
- package/src/services/mediaStorageProviders/file-storage-provider.ts +29 -7
- package/src/services/solid-introspect.service.ts +1 -0
- package/src/solid-core.module.ts +18 -1
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { Injectable, Logger } from '@nestjs/common';
|
|
2
|
+
import * as ExcelJS from 'exceljs';
|
|
3
|
+
import { PassThrough, Readable } from 'stream';
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@Injectable()
|
|
7
|
+
export class ExcelService {
|
|
8
|
+
private logger = new Logger(ExcelService.name);
|
|
9
|
+
// Sample JSON data
|
|
10
|
+
// const jsonData = [
|
|
11
|
+
// { id: 1, name: 'John Doe', age: 25, email: 'john@example.com' },
|
|
12
|
+
// { id: 2, name: 'Jane Doe', age: 28, email: 'jane@example.com' },
|
|
13
|
+
// { id: 3, name: 'Alice Smith', age: 30, email: 'alice@example.com' }
|
|
14
|
+
// ];
|
|
15
|
+
|
|
16
|
+
// public async createExcelFromJson(data: any[], fileName: string) {
|
|
17
|
+
// const workbook = new ExcelJS.Workbook();
|
|
18
|
+
// const worksheet = workbook.addWorksheet('Data');
|
|
19
|
+
|
|
20
|
+
// // Define Columns (Header)
|
|
21
|
+
// worksheet.columns = Object.keys(data[0]).map((key) => ({
|
|
22
|
+
// header: key.toUpperCase(), // Convert header names to uppercase
|
|
23
|
+
// key: key,
|
|
24
|
+
// width: 20, // Set column width
|
|
25
|
+
// }));
|
|
26
|
+
|
|
27
|
+
// // Add Data Rows
|
|
28
|
+
// data.forEach((item) => {
|
|
29
|
+
// worksheet.addRow(item);
|
|
30
|
+
// });
|
|
31
|
+
|
|
32
|
+
// // Apply basic formatting
|
|
33
|
+
// worksheet.getRow(1).font = { bold: true }; // Make headers bold
|
|
34
|
+
|
|
35
|
+
// // Save file
|
|
36
|
+
// await workbook.xlsx.writeFile(fileName);
|
|
37
|
+
// this.logger.log(`✅ Excel file "${fileName}" created successfully!`);
|
|
38
|
+
// // console.log(`✅ Excel file "${fileName}" created successfully!`);
|
|
39
|
+
// }
|
|
40
|
+
|
|
41
|
+
// public async createExcelStreamFromJson(data: any[]): Promise<Readable> {
|
|
42
|
+
// const passThrough = new PassThrough(); // Stream to pipe data
|
|
43
|
+
// const workbook = new ExcelJS.stream.xlsx.WorkbookWriter({ stream: passThrough });
|
|
44
|
+
// const worksheet = workbook.addWorksheet('Data');
|
|
45
|
+
// worksheet.columns = Object.keys(data[0]).map((key) => ({
|
|
46
|
+
// header: key.toUpperCase(),
|
|
47
|
+
// key: key,
|
|
48
|
+
// width: 20,
|
|
49
|
+
// }));
|
|
50
|
+
|
|
51
|
+
// data.forEach((item) => {
|
|
52
|
+
// worksheet.addRow(item);
|
|
53
|
+
// });
|
|
54
|
+
|
|
55
|
+
// worksheet.getRow(1).font = { bold: true };
|
|
56
|
+
|
|
57
|
+
// await workbook.commit();
|
|
58
|
+
// return passThrough;
|
|
59
|
+
// }
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
public async createExcelStream(
|
|
63
|
+
getDataRecords: (chunkIndex: number, chunkSize: number) => Promise<any[]>,
|
|
64
|
+
chunkSize: number
|
|
65
|
+
): Promise<Readable> {
|
|
66
|
+
const passThrough = new PassThrough(); // Create streaming pipe
|
|
67
|
+
try {
|
|
68
|
+
const workbook = new ExcelJS.stream.xlsx.WorkbookWriter({ stream: passThrough });
|
|
69
|
+
const worksheet = workbook.addWorksheet('Data');
|
|
70
|
+
|
|
71
|
+
let chunkIndex = 0;
|
|
72
|
+
let isHeaderWritten = false;
|
|
73
|
+
|
|
74
|
+
while (true) {
|
|
75
|
+
const records = await getDataRecords(chunkIndex, chunkSize); // Fetch chunked data
|
|
76
|
+
if (records.length === 0) break; // Stop if no more records
|
|
77
|
+
|
|
78
|
+
if (!isHeaderWritten) {
|
|
79
|
+
worksheet.columns = Object.keys(records[0]).map((key) => ({
|
|
80
|
+
header: key.toUpperCase(),
|
|
81
|
+
key: key,
|
|
82
|
+
width: 20,
|
|
83
|
+
}));
|
|
84
|
+
isHeaderWritten = true;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
records.forEach((item) => {
|
|
88
|
+
worksheet.addRow(item).commit(); // Commit each row immediately
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
chunkIndex++; // Fetch next chunk
|
|
92
|
+
this.logger.debug(`✅ Chunk ${chunkIndex} written to Excel`);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
await workbook.commit();
|
|
96
|
+
// passThrough.end(); // ✅ Properly close the stream
|
|
97
|
+
} catch (error) {
|
|
98
|
+
this.logger.error(`❌ Error writing Excel: ${error.message}`);
|
|
99
|
+
passThrough.destroy(error); // Destroy stream
|
|
100
|
+
throw error;
|
|
101
|
+
}
|
|
102
|
+
return passThrough; // Return streaming response
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { Injectable } from '@nestjs/common';
|
|
2
|
+
import { DiscoveryService, ModuleRef } from "@nestjs/core";
|
|
3
|
+
import { InjectEntityManager, InjectRepository } from '@nestjs/typeorm';
|
|
4
|
+
import { EntityManager, Repository } from 'typeorm';
|
|
5
|
+
|
|
6
|
+
import { ConfigService } from '@nestjs/config';
|
|
7
|
+
import { CrudHelperService } from 'src/services/crud-helper.service';
|
|
8
|
+
import { CRUDService } from 'src/services/crud.service';
|
|
9
|
+
import { FileService } from 'src/services/file.service';
|
|
10
|
+
import { MediaStorageProviderMetadataService } from 'src/services/media-storage-provider-metadata.service';
|
|
11
|
+
import { MediaService } from 'src/services/media.service';
|
|
12
|
+
import { ModelMetadataService } from 'src/services/model-metadata.service';
|
|
13
|
+
import { ModuleMetadataService } from 'src/services/module-metadata.service';
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
import { CreateExportTransactionDto } from 'src/dtos/create-export-transaction.dto';
|
|
17
|
+
import { ExportTransaction } from 'src/entities/export-transaction.entity';
|
|
18
|
+
import { Readable } from 'stream';
|
|
19
|
+
import { ExportTemplate } from '../entities/export-template.entity';
|
|
20
|
+
import { ExportTransactionFileInfo, ExportTransactionService } from './export-transaction.service';
|
|
21
|
+
|
|
22
|
+
@Injectable()
|
|
23
|
+
export class ExportTemplateService extends CRUDService<ExportTemplate>{
|
|
24
|
+
async startExportSync(id: number): Promise<ExportTransactionFileInfo> {
|
|
25
|
+
// Create the export transaction entry, with status 'started'
|
|
26
|
+
const exportTransaction: CreateExportTransactionDto = await this.exportTransactionService.toDto({
|
|
27
|
+
datetime: new Date(),
|
|
28
|
+
status: 'started',
|
|
29
|
+
exportTemplateId: id,
|
|
30
|
+
});
|
|
31
|
+
const exportTransactionEntity = await this.exportTransactionService.create(exportTransaction);
|
|
32
|
+
|
|
33
|
+
// Trigger the export process
|
|
34
|
+
const exportFileInfo = await this.exportTransactionService.triggerExportSync(exportTransactionEntity.id);
|
|
35
|
+
// It should return the export transaction id
|
|
36
|
+
return exportFileInfo;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async startExportAsync(id: number): Promise<ExportTransaction>{
|
|
40
|
+
// Create the export transaction entry, with status 'started'
|
|
41
|
+
const exportTransaction: CreateExportTransactionDto = await this.exportTransactionService.toDto({
|
|
42
|
+
datetime: new Date(),
|
|
43
|
+
status: 'started',
|
|
44
|
+
exportTemplateId: id,
|
|
45
|
+
});
|
|
46
|
+
const exportTransactionEntity = await this.exportTransactionService.create(exportTransaction);
|
|
47
|
+
|
|
48
|
+
// Trigger the export process
|
|
49
|
+
this.exportTransactionService.triggerExportAsync(exportTransactionEntity.id);
|
|
50
|
+
|
|
51
|
+
// It should return the export transaction id, so client can use this to check the status
|
|
52
|
+
return exportTransactionEntity;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
constructor(
|
|
56
|
+
readonly modelMetadataService: ModelMetadataService,
|
|
57
|
+
readonly moduleMetadataService: ModuleMetadataService,
|
|
58
|
+
readonly configService: ConfigService,
|
|
59
|
+
readonly fileService: FileService,
|
|
60
|
+
readonly discoveryService: DiscoveryService,
|
|
61
|
+
readonly crudHelperService: CrudHelperService,
|
|
62
|
+
@InjectEntityManager()
|
|
63
|
+
readonly entityManager: EntityManager,
|
|
64
|
+
@InjectRepository(ExportTemplate, 'default')
|
|
65
|
+
readonly repo: Repository<ExportTemplate>,
|
|
66
|
+
readonly exportTransactionService: ExportTransactionService,
|
|
67
|
+
readonly moduleRef: ModuleRef
|
|
68
|
+
) {
|
|
69
|
+
super(modelMetadataService, moduleMetadataService, configService, fileService, discoveryService, crudHelperService,entityManager, repo, 'exportTemplate', 'solid-core',moduleRef);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
import { Injectable, Logger } from '@nestjs/common';
|
|
2
|
+
import { DiscoveryService, ModuleRef } from "@nestjs/core";
|
|
3
|
+
import { InjectEntityManager, InjectRepository } from '@nestjs/typeorm';
|
|
4
|
+
import { EntityManager, Repository } from 'typeorm';
|
|
5
|
+
|
|
6
|
+
import { ConfigService } from '@nestjs/config';
|
|
7
|
+
import { CrudHelperService } from 'src/services/crud-helper.service';
|
|
8
|
+
import { CRUDService } from 'src/services/crud.service';
|
|
9
|
+
import { FileService } from 'src/services/file.service';
|
|
10
|
+
import { MediaStorageProviderMetadataService } from 'src/services/media-storage-provider-metadata.service';
|
|
11
|
+
import { MediaService } from 'src/services/media.service';
|
|
12
|
+
import { ModelMetadataService } from 'src/services/model-metadata.service';
|
|
13
|
+
import { ModuleMetadataService } from 'src/services/module-metadata.service';
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
import { classify, dasherize } from '@angular-devkit/core/src/utils/strings';
|
|
17
|
+
import { InstanceWrapper } from '@nestjs/core/injector/instance-wrapper';
|
|
18
|
+
import { validate } from 'class-validator';
|
|
19
|
+
import { BasicFilterDto } from 'src/dtos/basic-filters.dto';
|
|
20
|
+
import { CreateExportTransactionDto } from 'src/dtos/create-export-transaction.dto';
|
|
21
|
+
import { MediaStorageProviderType } from 'src/dtos/create-media-storage-provider-metadata.dto';
|
|
22
|
+
import { FieldMetadata } from 'src/entities/field-metadata.entity';
|
|
23
|
+
import { Readable } from 'stream';
|
|
24
|
+
import { ExportTransaction } from '../entities/export-transaction.entity';
|
|
25
|
+
import { CsvService } from './csv.service';
|
|
26
|
+
import { ExcelService } from './excel.service';
|
|
27
|
+
import { getMediaStorageProvider } from './mediaStorageProviders';
|
|
28
|
+
import { SolidIntrospectService } from './solid-introspect.service';
|
|
29
|
+
|
|
30
|
+
const EXPORT_CHUNK_SIZE = 100;
|
|
31
|
+
enum ExportStatus {
|
|
32
|
+
STARTED = 'started',
|
|
33
|
+
COMPLETED = 'completed',
|
|
34
|
+
FAILED = 'failed',
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
enum ExportFormat {
|
|
38
|
+
CSV = 'csv',
|
|
39
|
+
EXCEL = 'excel',
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export interface ExportTransactionFileInfo {
|
|
43
|
+
exportStream: Readable;
|
|
44
|
+
fileName: string;
|
|
45
|
+
mimeType: string;
|
|
46
|
+
exportTransaction: ExportTransaction;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
@Injectable()
|
|
50
|
+
export class ExportTransactionService extends CRUDService<ExportTransaction> {
|
|
51
|
+
private logger = new Logger(ExportTransactionService.name);
|
|
52
|
+
|
|
53
|
+
constructor(
|
|
54
|
+
readonly modelMetadataService: ModelMetadataService,
|
|
55
|
+
readonly moduleMetadataService: ModuleMetadataService,
|
|
56
|
+
readonly configService: ConfigService,
|
|
57
|
+
readonly fileService: FileService,
|
|
58
|
+
readonly discoveryService: DiscoveryService,
|
|
59
|
+
readonly crudHelperService: CrudHelperService,
|
|
60
|
+
@InjectEntityManager()
|
|
61
|
+
readonly entityManager: EntityManager,
|
|
62
|
+
@InjectRepository(ExportTransaction, 'default')
|
|
63
|
+
readonly repo: Repository<ExportTransaction>,
|
|
64
|
+
readonly introspectService: SolidIntrospectService,
|
|
65
|
+
readonly excelService: ExcelService,
|
|
66
|
+
readonly csvService: CsvService,
|
|
67
|
+
// readonly fieldMetadataService: FieldMetadataService,
|
|
68
|
+
@InjectRepository(FieldMetadata, 'default')
|
|
69
|
+
readonly fieldRepo: Repository<FieldMetadata>,
|
|
70
|
+
readonly moduleRef: ModuleRef
|
|
71
|
+
) {
|
|
72
|
+
super(modelMetadataService, moduleMetadataService, configService, fileService, discoveryService, crudHelperService, entityManager, repo, 'exportTransaction', 'solid-core',moduleRef);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Return the export stream
|
|
76
|
+
async triggerExportSync(id: number): Promise<ExportTransactionFileInfo> {
|
|
77
|
+
try {
|
|
78
|
+
const loadedExportTransaction = await this.loadExportTransaction(id);
|
|
79
|
+
const { exportStream, templateName, uuid, exportTransaction } = await this.getExportStreamDetails(loadedExportTransaction);
|
|
80
|
+
this.updateExportTransaction(id, ExportStatus.COMPLETED);
|
|
81
|
+
const fileName = this.getFileName(templateName, uuid, loadedExportTransaction.exportTemplate.templateFormat);
|
|
82
|
+
const mimeType = this.getMimeType(loadedExportTransaction.exportTemplate.templateFormat);
|
|
83
|
+
return { exportStream, fileName, mimeType, exportTransaction };
|
|
84
|
+
} catch (error) {
|
|
85
|
+
this.updateExportTransaction(id, ExportStatus.FAILED, error.message);
|
|
86
|
+
throw error;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// Store the export stream using the appropriate storage provider
|
|
91
|
+
async triggerExportAsync(id: number): Promise<void> {
|
|
92
|
+
try {
|
|
93
|
+
const loadedExportTransaction = await this.loadExportTransaction(id)
|
|
94
|
+
const { exportStream, templateName, uuid, exportTransaction } = await this.getExportStreamDetails(loadedExportTransaction);
|
|
95
|
+
const fileFormat = loadedExportTransaction.exportTemplate.templateFormat;
|
|
96
|
+
// Store the file using the appropriate storage provider
|
|
97
|
+
await this.storeExportStream(exportStream, exportTransaction, this.getFileName(templateName, uuid, fileFormat));
|
|
98
|
+
this.updateExportTransaction(id, ExportStatus.COMPLETED);
|
|
99
|
+
} catch (error) {
|
|
100
|
+
this.updateExportTransaction(id, ExportStatus.FAILED, error.message);
|
|
101
|
+
throw error;
|
|
102
|
+
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
private async loadExportTransaction(id: number) {
|
|
107
|
+
return await this.repo.findOne({
|
|
108
|
+
where: { id: id },
|
|
109
|
+
relations: { exportTemplate: { modelMetadata: true } },
|
|
110
|
+
}
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
private async updateExportTransaction(id: number, status: string, error?: string) {
|
|
115
|
+
await this.repo.update(id, { status, error });
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
private async getExportStreamDetails(exportTransaction: ExportTransaction) {
|
|
119
|
+
// Get the columns which need to be exported & the model id
|
|
120
|
+
const fields = JSON.parse(exportTransaction.exportTemplate.fields);
|
|
121
|
+
|
|
122
|
+
// Get the appropriate service for the model by trying to fetch a model service matching a particular name
|
|
123
|
+
const modelName = exportTransaction.exportTemplate.modelMetadata.singularName;
|
|
124
|
+
const modelService = this.introspectService.getProvider(`${classify(modelName)}Service`);
|
|
125
|
+
const templateName = exportTransaction.exportTemplate.templateName;
|
|
126
|
+
const uuid = exportTransaction.exportTransactionId; //TODO can be renamed to exportTransactionUUID
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
// Get the data records function
|
|
130
|
+
const dataRecordsFunc = await this.getDataRecordsFunc(fields, modelService);
|
|
131
|
+
|
|
132
|
+
// Get the export passthru stream (since it is a passthru stream, nothing is stored in memory & it is streamed directly when the stream is read)
|
|
133
|
+
let exportStream = await this.getExportStream(exportTransaction.exportTemplate.templateFormat, dataRecordsFunc);
|
|
134
|
+
return { exportStream, templateName, uuid, exportTransaction };
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
private async getExportStream(templateFormat: string, dataRecordsFunc: (chunkIndex: number, chunkSize: number) => Promise<any[]>) {
|
|
138
|
+
let exportStream = null;
|
|
139
|
+
switch (templateFormat) {
|
|
140
|
+
case ExportFormat.EXCEL:
|
|
141
|
+
exportStream = await this.excelService.createExcelStream(dataRecordsFunc, EXPORT_CHUNK_SIZE);
|
|
142
|
+
break;
|
|
143
|
+
case ExportFormat.CSV:
|
|
144
|
+
exportStream = await await this.csvService.createCsvStream(dataRecordsFunc, EXPORT_CHUNK_SIZE);
|
|
145
|
+
break;
|
|
146
|
+
default:
|
|
147
|
+
throw new Error('Invalid export format');
|
|
148
|
+
}
|
|
149
|
+
return exportStream;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
private async storeExportStream(exportStream: Readable, exportTransaction: ExportTransaction, fileName: string) {
|
|
153
|
+
const exportedFileMediaField = await this.fieldRepo.findOne({
|
|
154
|
+
where: {
|
|
155
|
+
name: 'exportedFile',
|
|
156
|
+
model: {
|
|
157
|
+
singularName: 'exportTransaction'
|
|
158
|
+
},
|
|
159
|
+
},
|
|
160
|
+
relations: ['model', 'mediaStorageProvider'],
|
|
161
|
+
});
|
|
162
|
+
// const storageProvider = new FileStorageProvider<ExportTransaction>(this.configService, this.fileService, this.mediaService);
|
|
163
|
+
const storageProviderMetadata = exportedFileMediaField.mediaStorageProvider;
|
|
164
|
+
|
|
165
|
+
// // Use the storage provider metadata to get the appropriate storage provider implementation
|
|
166
|
+
const storageProviderType = storageProviderMetadata.type as MediaStorageProviderType;
|
|
167
|
+
|
|
168
|
+
// // Get the storage provider implementation
|
|
169
|
+
const storageProvider = await getMediaStorageProvider(this.moduleRef, storageProviderType);
|
|
170
|
+
|
|
171
|
+
//Commented the below code since we will be direclty images from server on call from ui
|
|
172
|
+
await storageProvider.storeStreams([[exportStream, fileName]], exportTransaction, exportedFileMediaField)
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
private getFileName(templateName: string, exportTransactionUUID: string, fileFormat: string): string {
|
|
176
|
+
const extension = (fileFormat === ExportFormat.EXCEL) ? 'xlsx' : 'csv';
|
|
177
|
+
return `${dasherize(templateName)}-${exportTransactionUUID}.${extension}`;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
private getMimeType(fileFormat: string): string {
|
|
181
|
+
return (fileFormat === ExportFormat.EXCEL) ? 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' : 'text/csv';
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
private async getDataRecordsFunc(fields: any, modelService: InstanceWrapper<any>): Promise<(chunkIndex: number, chunkSize: number) => Promise<any[]>> {
|
|
185
|
+
// Return a function which will take the chunkIndex & chunkSize and return the data
|
|
186
|
+
return async (chunkIndex: number, chunkSize: number) => {
|
|
187
|
+
const offset = chunkIndex * chunkSize;
|
|
188
|
+
const recordFilterDto: BasicFilterDto = {
|
|
189
|
+
fields,
|
|
190
|
+
limit: chunkSize,
|
|
191
|
+
offset,
|
|
192
|
+
};
|
|
193
|
+
const data = await modelService.instance.find(recordFilterDto);
|
|
194
|
+
const records = data.records ?? [];
|
|
195
|
+
return records;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
async toDto(data: Partial<CreateExportTransactionDto>): Promise<CreateExportTransactionDto> {
|
|
200
|
+
const dto = new CreateExportTransactionDto(data);
|
|
201
|
+
const errors = await validate(dto);
|
|
202
|
+
if (errors.length > 0) {
|
|
203
|
+
console.error("Validation failed:", errors);
|
|
204
|
+
return null;
|
|
205
|
+
}
|
|
206
|
+
return dto;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
1. Do i need to create a storeStreams method for aws service too?
|
|
2
|
+
- Handle later
|
|
3
|
+
2. queues handling -> if queues is enabled by default, i.e triggerExport(exportTransactionEntity.id).
|
|
4
|
+
- startExport should either return the data or return the transaction id
|
|
5
|
+
3. How to handle scenarios wherein, nested related exist.(do i need to only get the userkey)
|
|
6
|
+
- show the userKey
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { Inject, Injectable } from '@nestjs/common';
|
|
1
|
+
import { Inject, Injectable, Logger } from '@nestjs/common';
|
|
2
2
|
import * as fs from 'fs';
|
|
3
3
|
// import * as AWS from 'aws-sdk';
|
|
4
4
|
import { S3Client, PutObjectCommand, DeleteObjectCommand, ObjectCannedACL } from '@aws-sdk/client-s3';
|
|
5
5
|
import { ConfigType } from '@nestjs/config';
|
|
6
6
|
import commonConfig, { AwsS3Config } from '../config/common.config';
|
|
7
7
|
import path from 'path';
|
|
8
|
+
import { Readable } from 'stream';
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
|
|
@@ -13,6 +14,7 @@ import path from 'path';
|
|
|
13
14
|
export class FileService {
|
|
14
15
|
|
|
15
16
|
private readonly s3Client: S3Client;
|
|
17
|
+
private readonly logger = new Logger(FileService.name);
|
|
16
18
|
|
|
17
19
|
constructor(
|
|
18
20
|
@Inject(commonConfig.KEY)
|
|
@@ -164,4 +166,27 @@ export class FileService {
|
|
|
164
166
|
private isValidS3Config(config: AwsS3Config): boolean {
|
|
165
167
|
return !!config.S3_AWS_ACCESS_KEY && !!config.S3_AWS_SECRET_KEY && !!config.S3_AWS_REGION_NAME;
|
|
166
168
|
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Save a stream to a file
|
|
172
|
+
* @param stream - Readable stream
|
|
173
|
+
* @param filePath - Destination file path
|
|
174
|
+
* @returns Promise<void>
|
|
175
|
+
*/
|
|
176
|
+
public async writeStreamToFile(stream: Readable, filePath: string): Promise<void> {
|
|
177
|
+
return new Promise((resolve, reject) => {
|
|
178
|
+
const writeStream = fs.createWriteStream(filePath);
|
|
179
|
+
stream.pipe(writeStream);
|
|
180
|
+
|
|
181
|
+
writeStream.on('finish', () => {
|
|
182
|
+
this.logger.debug(`✅ File saved: ${filePath}`);
|
|
183
|
+
resolve();
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
writeStream.on('error', (err) => {
|
|
187
|
+
this.logger.debug(`❌ Error saving file: ${filePath}`, err);
|
|
188
|
+
reject(err);
|
|
189
|
+
});
|
|
190
|
+
});
|
|
191
|
+
}
|
|
167
192
|
}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { ConfigService
|
|
3
|
-
import { MediaStorageProvider } from "src/interfaces";
|
|
4
|
-
import { FileService } from "src/services/file.service";
|
|
5
|
-
import { Media } from "src/entities/media.entity";
|
|
1
|
+
import { Injectable, Logger } from "@nestjs/common";
|
|
2
|
+
import { ConfigService } from "@nestjs/config";
|
|
6
3
|
import { CommonEntity } from "src/entities/common.entity";
|
|
7
4
|
import { FieldMetadata } from "src/entities/field-metadata.entity";
|
|
8
|
-
import
|
|
5
|
+
import { Media } from "src/entities/media.entity";
|
|
6
|
+
import { MediaStorageProvider } from "src/interfaces";
|
|
7
|
+
import { FileService } from "src/services/file.service";
|
|
8
|
+
import { Readable } from "stream";
|
|
9
9
|
import { MediaRepository } from "src/repository/media.repository";
|
|
10
10
|
|
|
11
11
|
@Injectable()
|
|
@@ -19,6 +19,9 @@ export class FileS3StorageProvider<T> implements MediaStorageProvider<T> {
|
|
|
19
19
|
readonly fileService: FileService,
|
|
20
20
|
readonly mediaRepository: MediaRepository
|
|
21
21
|
) { }
|
|
22
|
+
storeStreams(streamPairs: [Readable, string][], entity: T, mediaFieldMetadata: FieldMetadata): Promise<Media[]> {
|
|
23
|
+
throw new Error("Method not implemented.");
|
|
24
|
+
}
|
|
22
25
|
async retrieve(entity: T, mediaFieldMetadata: FieldMetadata): Promise<Media[]> {
|
|
23
26
|
if (!(entity instanceof CommonEntity)) {
|
|
24
27
|
throw new Error("Entity must be an instance of CommonEntity"); //FIXME This needs to be handled through generics. e.g T extends CommonEntity
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { ConfigService
|
|
3
|
-
import { MediaStorageProvider } from "src/interfaces";
|
|
4
|
-
import { FileService } from "src/services/file.service";
|
|
5
|
-
import { Media } from "src/entities/media.entity";
|
|
1
|
+
import { Injectable, Logger } from "@nestjs/common";
|
|
2
|
+
import { ConfigService } from "@nestjs/config";
|
|
6
3
|
import { CommonEntity } from "src/entities/common.entity";
|
|
7
4
|
import { FieldMetadata } from "src/entities/field-metadata.entity";
|
|
5
|
+
import { Media } from "src/entities/media.entity";
|
|
6
|
+
import { MediaStorageProvider } from "src/interfaces";
|
|
8
7
|
import { MediaRepository } from "src/repository/media.repository";
|
|
8
|
+
import { FileService } from "src/services/file.service";
|
|
9
|
+
import { Readable } from "stream";
|
|
9
10
|
|
|
10
11
|
@Injectable()
|
|
11
12
|
export class FileStorageProvider<T> implements MediaStorageProvider<T> {
|
|
@@ -40,8 +41,8 @@ export class FileStorageProvider<T> implements MediaStorageProvider<T> {
|
|
|
40
41
|
files.forEach(async (file) => {
|
|
41
42
|
// Store the file in the configured file storage directory
|
|
42
43
|
const fileStoragePath = this.getFullFilePath(this.getFileName(file));
|
|
43
|
-
this.fileService.copyFile(file.path, fileStoragePath);
|
|
44
|
-
this.fileService.deleteFile(file.path);
|
|
44
|
+
await this.fileService.copyFile(file.path, fileStoragePath);
|
|
45
|
+
await this.fileService.deleteFile(file.path);
|
|
45
46
|
|
|
46
47
|
// Create an entry in the media table
|
|
47
48
|
const mediaEntity = await this.mediaRepository.createMedia({
|
|
@@ -60,6 +61,27 @@ export class FileStorageProvider<T> implements MediaStorageProvider<T> {
|
|
|
60
61
|
return result;
|
|
61
62
|
}
|
|
62
63
|
|
|
64
|
+
async storeStreams(streamPairs: [Readable, string][], entity: T, mediaFieldMetadata: FieldMetadata): Promise<Media[]> {
|
|
65
|
+
if (!(entity instanceof CommonEntity)) {
|
|
66
|
+
throw new Error("Entity must be an instance of CommonEntity"); //FIXME This needs to be handled through generics. e.g T extends CommonEntity
|
|
67
|
+
}
|
|
68
|
+
const result: Media[] = [];
|
|
69
|
+
streamPairs.forEach(async (pair) => {
|
|
70
|
+
const stream = pair[0];
|
|
71
|
+
const fileName = pair[1];
|
|
72
|
+
this.fileService.writeStreamToFile(stream, fileName);
|
|
73
|
+
const mediaEntity = await this.mediaRepository.createMedia({
|
|
74
|
+
entityId: entity.id,
|
|
75
|
+
modelMetadataId: mediaFieldMetadata.model.id,
|
|
76
|
+
relativeUri: fileName,
|
|
77
|
+
mediaStorageProviderMetadataId: mediaFieldMetadata.mediaStorageProvider.id,
|
|
78
|
+
fieldMetadataId: mediaFieldMetadata.id
|
|
79
|
+
}) as unknown as Media;
|
|
80
|
+
this.logger.debug(`Stored media with`, mediaEntity);
|
|
81
|
+
});
|
|
82
|
+
return result;
|
|
83
|
+
}
|
|
84
|
+
|
|
63
85
|
async delete(entity: T, mediaFieldMetadata: FieldMetadata): Promise<void> {
|
|
64
86
|
if (!(entity instanceof CommonEntity)) {
|
|
65
87
|
throw new Error("Entity must be an instance of CommonEntity"); //FIXME This needs to be handled through generics. e.g T extends CommonEntity
|
|
@@ -80,6 +80,7 @@ export class SolidIntrospectService implements OnApplicationBootstrap {
|
|
|
80
80
|
.getProviders()
|
|
81
81
|
.filter((provider) => this.isModule(provider));
|
|
82
82
|
this.solidRegistry.registerModules(allModules);
|
|
83
|
+
|
|
83
84
|
}
|
|
84
85
|
|
|
85
86
|
// This method identifies a provider as a seeder if it has a seed method i.e duck typing
|
package/src/solid-core.module.ts
CHANGED
|
@@ -161,8 +161,17 @@ import { ChatterMessageDetails } from './entities/chatter-message-details.entity
|
|
|
161
161
|
import { ChatterMessageDetailsService } from './services/chatter-message-details.service';
|
|
162
162
|
import { ChatterMessageDetailsController } from './controllers/chatter-message-details.controller';
|
|
163
163
|
import { AuditSubscriber } from './subscribers/audit.subscriber';
|
|
164
|
+
import { ExportTemplate } from './entities/export-template.entity';
|
|
165
|
+
import { ExportTemplateService } from './services/export-template.service';
|
|
166
|
+
import { ExportTemplateController } from './controllers/export-template.controller';
|
|
167
|
+
import { ExportTransaction } from './entities/export-transaction.entity';
|
|
168
|
+
import { ExportTransactionService } from './services/export-transaction.service';
|
|
169
|
+
import { ExportTransactionController } from './controllers/export-transaction.controller';
|
|
170
|
+
import { ExcelService } from './services/excel.service';
|
|
171
|
+
import { CsvService } from './services/csv.service';
|
|
164
172
|
import { ClsModule } from 'nestjs-cls';
|
|
165
173
|
|
|
174
|
+
|
|
166
175
|
@Global()
|
|
167
176
|
@Module({
|
|
168
177
|
imports: [
|
|
@@ -212,6 +221,8 @@ import { ClsModule } from 'nestjs-cls';
|
|
|
212
221
|
TypeOrmModule.forFeature([ListOfValues]),
|
|
213
222
|
TypeOrmModule.forFeature([ChatterMessage]),
|
|
214
223
|
TypeOrmModule.forFeature([ChatterMessageDetails]),
|
|
224
|
+
TypeOrmModule.forFeature([ExportTemplate]),
|
|
225
|
+
TypeOrmModule.forFeature([ExportTransaction]),
|
|
215
226
|
// TypeOrmModule.forFeature([User]),
|
|
216
227
|
ClsModule.forRoot({
|
|
217
228
|
middleware: {
|
|
@@ -247,7 +258,9 @@ import { ClsModule } from 'nestjs-cls';
|
|
|
247
258
|
SavedFiltersController,
|
|
248
259
|
ListOfValuesController,
|
|
249
260
|
ChatterMessageController,
|
|
250
|
-
ChatterMessageDetailsController
|
|
261
|
+
ChatterMessageDetailsController,
|
|
262
|
+
ExportTemplateController,
|
|
263
|
+
ExportTransactionController,
|
|
251
264
|
],
|
|
252
265
|
providers: [
|
|
253
266
|
{
|
|
@@ -352,6 +365,10 @@ import { ClsModule } from 'nestjs-cls';
|
|
|
352
365
|
ChatterMessageService,
|
|
353
366
|
ChatterMessageDetailsService,
|
|
354
367
|
AuditSubscriber,
|
|
368
|
+
ExportTemplateService,
|
|
369
|
+
ExportTransactionService,
|
|
370
|
+
ExcelService,
|
|
371
|
+
CsvService,
|
|
355
372
|
],
|
|
356
373
|
exports: [
|
|
357
374
|
ModuleMetadataService,
|