@drax/crud-back 3.9.0 → 3.11.0

package/dist/builders/CrudSchemaBuilder.js

@@ -377,6 +377,9 @@ export class CrudSchemaBuilder {
             if (options.export !== false) {
                 fastify.get(`${basePath}/export`, (req, rep) => controller.export(req, rep));
             }
+            if (options.create !== false) {
+                fastify.post(`${basePath}/import`, (req, rep) => controller.import(req, rep));
+            }
             // Search entities
             if (options.search !== false) {
                 fastify.get(`${basePath}/search`, (req, rep) => controller.search(req, rep));

package/dist/controllers/AbstractFastifyController.js

@@ -2,6 +2,8 @@ import { CommonConfig, DraxConfig, LimitError, NotFoundError, BadRequestError, C
 import { join } from "path";
 import QueryFilterRegex from "../regexs/QueryFilterRegex.js";
 import CrudEventEmitter from "../events/CrudEventEmitter.js";
+import ImportCsv from "../imports/ImportCsv.js";
+import ImportCsvReport from "../imports/ImportCsvReport.js";
 class AbstractFastifyController extends CommonController {
     constructor(service, permission, entityName) {
         super();
@@ -42,9 +44,9 @@ class AbstractFastifyController extends CommonController {
             const filterArray = stringFilters.split("|");
             const filters = [];
             filterArray.forEach((filter) => {
-                const [field, operator, value] = filter.split(";");
+                const [field, operator, value, orGroup] = filter.split(";");
                 if (field && operator && (operator === 'empty' || (value !== undefined && value !== ''))) {
-                    filters.push({ field, operator, value });
+                    filters.push({ field, operator, value, orGroup: orGroup || undefined });
                 }
             });
             return filters;
@@ -192,12 +194,20 @@ class AbstractFastifyController extends CommonController {
     async postCreate(request, item) {
         return item;
     }
+    async prepareCreatePayload(request, payload) {
+        this.applyUserAndTenantSetters(payload, request.rbac);
+        return await this.preCreate(request, payload);
+    }
+    getErrorMessage(error) {
+        if (error?.message) {
+            return error.message;
+        }
+        return 'Unknown import error';
+    }
     async create(request, reply) {
         try {
             request.rbac.assertPermission(this.permission.Create);
-            const payload = request.body;
-            this.applyUserAndTenantSetters(payload, request.rbac);
-            await this.preCreate(request, payload);
+            const payload = await this.prepareCreatePayload(request, request.body);
             let item = await this.service.create(payload);
             this.onCreated(request, item);
             item = await this.postCreate(request, item);
@@ -569,6 +579,101 @@ class AbstractFastifyController extends CommonController {
             this.handleError(e, reply);
         }
     }
+    async import(request, reply) {
+        try {
+            request.rbac.assertPermission(this.permission.Create);
+            const data = await request.file();
+            if (!data) {
+                throw new BadRequestError('Import file is required');
+            }
+            const format = (request.query.format || data.filename.split('.').pop() || 'json').toUpperCase();
+            if (!['CSV', 'JSON'].includes(format)) {
+                throw new BadRequestError(`Unsupported import format: ${format}`);
+            }
+            const separator = request.query.separator || ';';
+            data.file.setEncoding('utf8');
+            let rawContent = '';
+            for await (const chunk of data.file) {
+                rawContent += chunk;
+            }
+            const start = Date.now();
+            let rowCount = 0;
+            let successCount = 0;
+            let errorCount = 0;
+            let reportFileName = undefined;
+            let reportUrl = undefined;
+            if (format === 'CSV') {
+                const importer = new ImportCsv({ content: rawContent, separator });
+                const parsedImport = importer.processDetailed();
+                rowCount = parsedImport.rows.length;
+                const reportRows = [];
+                for (const row of parsedImport.rows) {
+                    try {
+                        const payload = await this.prepareCreatePayload(request, row.item);
+                        let item = await this.service.create(payload);
+                        this.onCreated(request, item);
+                        await this.postCreate(request, item);
+                        successCount++;
+                        reportRows.push({ rawValues: row.rawValues, status: 'success', error: '' });
+                    }
+                    catch (e) {
+                        errorCount++;
+                        reportRows.push({
+                            rawValues: row.rawValues,
+                            status: 'error',
+                            error: this.getErrorMessage(e)
+                        });
+                    }
+                }
+                const year = (new Date().getFullYear()).toString();
+                const month = (new Date().getMonth() + 1).toString().padStart(2, '0');
+                const importPath = 'imports';
+                const destinationPath = join(this.baseFileDir, importPath, year, month);
+                const report = await new ImportCsvReport({
+                    destinationPath,
+                    fileName: `${this.entityName.toLowerCase()}_import_report`,
+                    headers: parsedImport.headers,
+                    separator,
+                    rows: reportRows,
+                }).process();
+                reportFileName = report.fileName;
+                reportUrl = `${this.baseURL}/api/file/${importPath}/${year}/${month}/${report.fileName}`;
+            }
+            else {
+                const items = this.service.parseImport({
+                    format,
+                    content: rawContent,
+                    separator,
+                });
+                rowCount = items.length;
+                for (const rawItem of items) {
+                    try {
+                        const payload = await this.prepareCreatePayload(request, rawItem);
+                        let item = await this.service.create(payload);
+                        this.onCreated(request, item);
+                        await this.postCreate(request, item);
+                        successCount++;
+                    }
+                    catch (e) {
+                        errorCount++;
+                    }
+                }
+            }
+            const response = {
+                rowCount,
+                successCount,
+                errorCount,
+                time: Date.now() - start,
+                message: errorCount > 0 ? 'Import completed with errors' : 'Import successful',
+                fileName: reportFileName,
+                url: reportUrl,
+            };
+            return response;
+        }
+        catch (e) {
+            this.handleError(e, reply);
+        }
+    }
     async groupBy(request, reply) {
         try {
             request.rbac.assertPermission(this.permission.View);

package/dist/imports/AbstractImport.js

@@ -0,0 +1,44 @@
+import { setNestedValue } from "@drax/common-back";
+class AbstractImport {
+    constructor(options) {
+        this.content = options.content;
+        this.separator = options.separator || ';';
+    }
+    parseValue(value) {
+        const trimmedValue = value.trim();
+        if (trimmedValue === '') {
+            return '';
+        }
+        if (trimmedValue === 'null') {
+            return null;
+        }
+        if (trimmedValue === 'true') {
+            return true;
+        }
+        if (trimmedValue === 'false') {
+            return false;
+        }
+        if (/^-?\d+(\.\d+)?$/.test(trimmedValue)) {
+            return Number(trimmedValue);
+        }
+        if ((trimmedValue.startsWith('{') && trimmedValue.endsWith('}')) ||
+            (trimmedValue.startsWith('[') && trimmedValue.endsWith(']'))) {
+            try {
+                return JSON.parse(trimmedValue);
+            }
+            catch {
+                return value;
+            }
+        }
+        return value;
+    }
+    assignNestedValue(record, key, value) {
+        if (key.includes('.')) {
+            setNestedValue(record, key, value);
+            return;
+        }
+        record[key] = value;
+    }
+}
+export { AbstractImport };
+export default AbstractImport;

package/dist/imports/ImportCsv.js

@@ -0,0 +1,79 @@
+import AbstractImport from "./AbstractImport.js";
+class ImportCsv extends AbstractImport {
+    constructor(options) {
+        super(options);
+    }
+    process() {
+        return this.processDetailed().rows.map(row => row.item);
+    }
+    processDetailed() {
+        const rows = this.parseRows(this.content);
+        if (rows.length === 0) {
+            return { headers: [], rows: [] };
+        }
+        const [headers, ...dataRows] = rows;
+        return {
+            headers,
+            rows: dataRows
+                .filter(row => row.some(value => value.trim() !== ''))
+                .map((row) => {
+                const item = {};
+                headers.forEach((header, index) => {
+                    const normalizedHeader = header.trim();
+                    if (!normalizedHeader) {
+                        return;
+                    }
+                    this.assignNestedValue(item, normalizedHeader, this.parseValue(row[index] ?? ''));
+                });
+                return {
+                    rawValues: row,
+                    item
+                };
+            })
+        };
+    }
+    parseRows(content) {
+        const rows = [];
+        let currentRow = [];
+        let currentValue = '';
+        let inQuotes = false;
+        for (let index = 0; index < content.length; index++) {
+            const char = content[index];
+            const nextChar = content[index + 1];
+            if (char === '"') {
+                if (inQuotes && nextChar === '"') {
+                    currentValue += '"';
+                    index++;
+                }
+                else {
+                    inQuotes = !inQuotes;
+                }
+                continue;
+            }
+            if (char === this.separator && !inQuotes) {
+                currentRow.push(currentValue);
+                currentValue = '';
+                continue;
+            }
+            if ((char === '\n' || char === '\r') && !inQuotes) {
+                if (char === '\r' && nextChar === '\n') {
+                    index++;
+                }
+                currentRow.push(currentValue);
+                if (currentRow.some(value => value !== '')) {
+                    rows.push(currentRow);
+                }
+                currentRow = [];
+                currentValue = '';
+                continue;
+            }
+            currentValue += char;
+        }
+        currentRow.push(currentValue);
+        if (currentRow.some(value => value !== '')) {
+            rows.push(currentRow);
+        }
+        return rows;
+    }
+}
+export default ImportCsv;

package/dist/imports/ImportCsvReport.js

@@ -0,0 +1,54 @@
+import * as fs from 'fs';
+import AbstractExport from "../exports/AbstractExport.js";
+class ImportCsvReport extends AbstractExport {
+    constructor(options) {
+        super({
+            cursor: [],
+            destinationPath: options.destinationPath,
+            headers: options.headers,
+            fileName: options.fileName || 'import_report',
+        });
+        this.separator = options.separator || ';';
+        this.rows = options.rows;
+    }
+    process() {
+        return new Promise((resolve, reject) => {
+            try {
+                this.generateFilePath('csv');
+                const start = Date.now();
+                const writableStream = fs.createWriteStream(this.relativeFilePath);
+                writableStream.on('error', reject);
+                writableStream.on('finish', () => resolve({
+                    status: 'success',
+                    destinationPath: this.destinationPath,
+                    fileName: this.fileName,
+                    filePath: this.destinationPath + '/' + this.fileName,
+                    relativeFilePath: this.relativeFilePath,
+                    rowCount: this.rows.length,
+                    time: Date.now() - start,
+                    message: 'Import report generated',
+                }));
+                writableStream.write([...this.headers, 'import_status', 'error'].join(this.separator) + '\n');
+                for (const row of this.rows) {
+                    const values = [...row.rawValues, row.status, row.error || ''].map(value => this.escapeCsvValue(value));
+                    writableStream.write(values.join(this.separator) + '\n');
+                }
+                writableStream.end();
+            }
+            catch (e) {
+                reject(e);
+            }
+        });
+    }
+    escapeCsvValue(value) {
+        let formattedValue = value === null || value === undefined ? '' : String(value);
+        if (formattedValue.includes(this.separator) ||
+            formattedValue.includes('"') ||
+            formattedValue.includes('\n') ||
+            formattedValue.includes('\r')) {
+            formattedValue = '"' + formattedValue.replace(/"/g, '""') + '"';
+        }
+        return formattedValue;
+    }
+}
+export default ImportCsvReport;

package/dist/imports/ImportJson.js

@@ -0,0 +1,14 @@
+import AbstractImport from "./AbstractImport.js";
+class ImportJson extends AbstractImport {
+    constructor(options) {
+        super(options);
+    }
+    process() {
+        const parsedContent = JSON.parse(this.content);
+        if (!Array.isArray(parsedContent)) {
+            throw new Error('Invalid JSON import format. Expected an array of objects');
+        }
+        return parsedContent;
+    }
+}
+export default ImportJson;

package/dist/regexs/QueryFilterRegex.js

@@ -1,3 +1,3 @@
-const QueryFilterRegex = /^(?:[a-zA-Z0-9_.\-]+;(?:eq|like|ne|in|nin|gt|gte|lt|lte|empty);[a-zA-Z0-9_.\-:\., áéíóúÁÉÍÓÚ]*)(?:\|[a-zA-Z0-9_.\-]+;(?:eq|like|ne|in|nin|gt|gte|lt|lte|empty);[a-zA-Z0-9_.\-:\., áéíóúÁÉÍÓÚ]*)*$/;
+const QueryFilterRegex = /^(?:[a-zA-Z0-9_.\-]+;(?:eq|like|ne|in|nin|gt|gte|lt|lte|empty);[a-zA-Z0-9_.\-:\., áéíóúÁÉÍÓÚ]*(?:;[a-zA-Z0-9_.\-]+)?)(?:\|[a-zA-Z0-9_.\-]+;(?:eq|like|ne|in|nin|gt|gte|lt|lte|empty);[a-zA-Z0-9_.\-:\., áéíóúÁÉÍÓÚ]*(?:;[a-zA-Z0-9_.\-]+)?)*$/;
 export default QueryFilterRegex;
 export { QueryFilterRegex };

package/dist/schemas/FindSchema.js

@@ -4,6 +4,6 @@ const FindQuerySchema = z.object({
     orderBy: z.string().optional(),
     order: z.enum(["asc", "desc"]).optional(),
     search: z.string().optional(),
-    filters: z.string().regex(QueryFilterRegex).optional().describe("Format: field;operator;value|field;operator;value|..."),
+    filters: z.string().regex(QueryFilterRegex).optional().describe("Format: field;operator;value[;orGroup]|field;operator;value[;orGroup]|..."),
 });
 export { FindQuerySchema };

package/dist/schemas/GroupBySchema.js

@@ -2,6 +2,6 @@ import z from "zod";
 import QueryFilterRegex from "../regexs/QueryFilterRegex.js";
 const GroupByQuerySchema = z.object({
     fields: z.array(z.string()).min(1).max(10),
-    filters: z.string().regex(QueryFilterRegex).optional().describe("Format: field;operator;value|field;operator;value|..."),
+    filters: z.string().regex(QueryFilterRegex).optional().describe("Format: field;operator;value[;orGroup]|field;operator;value[;orGroup]|..."),
 });
 export { GroupByQuerySchema };

package/dist/schemas/PaginateSchema.js

@@ -6,7 +6,7 @@ const PaginateQuerySchema = z.object({
     orderBy: z.string().optional(),
     order: z.enum(["asc", "desc"]).optional(),
     search: z.string().optional(),
-    filters: z.string().regex(QueryFilterRegex).optional().describe("Format: field;operator;value|field;operator;value|..."),
+    filters: z.string().regex(QueryFilterRegex).optional().describe("Format: field;operator;value[;orGroup]|field;operator;value[;orGroup]|..."),
 });
 const PaginateBodyResponseSchema = z.object({
     page: z.number(),

package/dist/services/AbstractService.js

@@ -2,6 +2,8 @@ import { ZodErrorToValidationError } from "@drax/common-back";
 import { ZodError } from "zod";
 import ExportCsv from "../exports/ExportCsv.js";
 import ExportJson from "../exports/ExportJson.js";
+import ImportCsv from "../imports/ImportCsv.js";
+import ImportJson from "../imports/ImportJson.js";
 class AbstractService {
     constructor(repository, baseSchema, fullSchema) {
         this._validateOutput = true;
@@ -354,6 +356,43 @@ class AbstractService {
             throw e;
         }
     }
+    parseImport({ format = 'JSON', content, separator = ';' }) {
+        let importer;
+        switch (format) {
+            case 'JSON':
+                importer = new ImportJson({ content, separator });
+                break;
+            case 'CSV':
+                importer = new ImportCsv({ content, separator });
+                break;
+            default:
+                throw new Error(`Unsupported import format: ${format}`);
+        }
+        return importer.process();
+    }
+    async import({ format = 'JSON', content, separator = ';' }) {
+        try {
+            const start = Date.now();
+            const items = this.parseImport({ format, content, separator });
+            for (const item of items) {
+                await this.create(item);
+            }
+            return {
+                status: 'success',
+                rowCount: items.length,
+                time: Date.now() - start,
+                message: 'Import successful',
+            };
+        }
+        catch (e) {
+            console.error("Error import", {
+                name: e?.name,
+                message: e?.message,
+                stack: e?.stack,
+            });
+            throw e;
+        }
+    }
 }
 export default AbstractService;
 export { AbstractService };

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "3.9.0",
+  "version": "3.11.0",
   "description": "Crud utils across modules",
   "main": "dist/index.js",
   "types": "types/index.d.ts",
@@ -22,10 +22,10 @@
   "author": "Cristian Incarnato & Drax Team",
   "license": "ISC",
   "dependencies": {
-    "@drax/common-back": "^3.0.0",
+    "@drax/common-back": "^3.10.0",
     "@drax/common-share": "^3.0.0",
     "@drax/identity-share": "^3.0.0",
-    "@drax/media-back": "^3.9.0",
+    "@drax/media-back": "^3.11.0",
     "@graphql-tools/load-files": "^7.0.0",
     "@graphql-tools/merge": "^9.0.4",
     "mongoose": "^8.23.0",
@@ -47,5 +47,5 @@
     "typescript": "^5.9.3",
     "vitest": "^3.2.4"
   },
-  "gitHead": "b1af2464a5d229ac728d216fe18e680322cfe158"
+  "gitHead": "9f9de6fe51c4f11d885385bd31a12edec799823e"
 }

package/src/builders/CrudSchemaBuilder.ts

@@ -472,6 +472,10 @@ export class CrudSchemaBuilder<
         fastify.get(`${basePath}/export`, (req, rep) => controller.export(req, rep));
       }
 
+      if (options.create !== false) {
+        fastify.post(`${basePath}/import`, (req, rep) => controller.import(req, rep));
+      }
+
       // Search entities
       if (options.search !== false) {
         fastify.get(`${basePath}/search`, (req, rep) => controller.search(req, rep));

package/src/controllers/AbstractFastifyController.ts

@@ -15,11 +15,14 @@ import {
     IDraxFieldFilter,
     IDraxExportResponse,
     IDraxCrudEvent,
-    IDraxPaginateResult
+    IDraxPaginateResult,
+    IDraxImportResponse
 } from "@drax/crud-share";
 import {join} from "path";
 import QueryFilterRegex from "../regexs/QueryFilterRegex.js";
 import CrudEventEmitter from "../events/CrudEventEmitter.js";
+import ImportCsv from "../imports/ImportCsv.js";
+import ImportCsvReport from "../imports/ImportCsvReport.js";
 
 declare module 'fastify' {
     interface FastifyRequest {
@@ -112,10 +115,10 @@ class AbstractFastifyController<T, C, U> extends CommonController {
             const filterArray = stringFilters.split("|")
             const filters: IDraxFieldFilter[] = []
             filterArray.forEach((filter) => {
-                const [field, operator, value] = filter.split(";")
+                const [field, operator, value, orGroup] = filter.split(";")
 
                 if (field && operator && (operator === 'empty' || (value !== undefined && value !== ''))) {
-                    filters.push({field, operator, value})
+                    filters.push({field, operator, value, orGroup: orGroup || undefined})
                 }
 
             })
@@ -285,12 +288,22 @@ class AbstractFastifyController<T, C, U> extends CommonController {
         return item
     }
 
+    protected async prepareCreatePayload(request: CustomRequest, payload: any): Promise<C> {
+        this.applyUserAndTenantSetters(payload, request.rbac)
+        return await this.preCreate(request, payload as C)
+    }
+
+    protected getErrorMessage(error: any): string {
+        if (error?.message) {
+            return error.message
+        }
+        return 'Unknown import error'
+    }
+
     async create(request: CustomRequest, reply: FastifyReply) {
         try {
             request.rbac.assertPermission(this.permission.Create)
-            const payload = request.body
-            this.applyUserAndTenantSetters(payload, request.rbac)
-            await this.preCreate(request, payload as C)
+            const payload = await this.prepareCreatePayload(request, request.body)
             let item = await this.service.create(payload as C)
             this.onCreated(request, item)
             item = await this.postCreate(request, item as T)
@@ -758,6 +771,112 @@ class AbstractFastifyController<T, C, U> extends CommonController {
         }
     }
 
+    async import(request: CustomRequest, reply: FastifyReply) {
+        try {
+            request.rbac.assertPermission(this.permission.Create)
+
+            const data = await (request as any).file()
+            if (!data) {
+                throw new BadRequestError('Import file is required')
+            }
+
+            const format = ((request.query.format || data.filename.split('.').pop() || 'json') as string).toUpperCase() as 'CSV' | 'JSON'
+            if (!['CSV', 'JSON'].includes(format)) {
+                throw new BadRequestError(`Unsupported import format: ${format}`)
+            }
+            const separator = request.query.separator || ';'
+            data.file.setEncoding('utf8')
+
+            let rawContent = ''
+            for await (const chunk of data.file) {
+                rawContent += chunk
+            }
+
+            const start = Date.now()
+            let rowCount = 0
+            let successCount = 0
+            let errorCount = 0
+            let reportFileName: string | undefined = undefined
+            let reportUrl: string | undefined = undefined
+
+            if (format === 'CSV') {
+                const importer = new ImportCsv({content: rawContent, separator})
+                const parsedImport = importer.processDetailed()
+                rowCount = parsedImport.rows.length
+
+                const reportRows: Array<{rawValues: string[], status: 'success' | 'error', error?: string}> = []
+
+                for (const row of parsedImport.rows) {
+                    try {
+                        const payload = await this.prepareCreatePayload(request, row.item)
+                        let item = await this.service.create(payload as C)
+                        this.onCreated(request, item)
+                        await this.postCreate(request, item as T)
+                        successCount++
+                        reportRows.push({rawValues: row.rawValues, status: 'success', error: ''})
+                    } catch (e) {
+                        errorCount++
+                        reportRows.push({
+                            rawValues: row.rawValues,
+                            status: 'error',
+                            error: this.getErrorMessage(e)
+                        })
+                    }
+                }
+
+                const year = (new Date().getFullYear()).toString()
+                const month = (new Date().getMonth() + 1).toString().padStart(2, '0')
+                const importPath = 'imports'
+                const destinationPath = join(this.baseFileDir, importPath, year, month)
+
+                const report = await new ImportCsvReport({
+                    destinationPath,
+                    fileName: `${this.entityName.toLowerCase()}_import_report`,
+                    headers: parsedImport.headers,
+                    separator,
+                    rows: reportRows,
+                }).process()
+
+                reportFileName = report.fileName
+                reportUrl = `${this.baseURL}/api/file/${importPath}/${year}/${month}/${report.fileName}`
+            } else {
+                const items = this.service.parseImport({
+                    format,
+                    content: rawContent,
+                    separator,
+                })
+
+                rowCount = items.length
+
+                for (const rawItem of items) {
+                    try {
+                        const payload = await this.prepareCreatePayload(request, rawItem)
+                        let item = await this.service.create(payload as C)
+                        this.onCreated(request, item)
+                        await this.postCreate(request, item as T)
+                        successCount++
+                    } catch (e) {
+                        errorCount++
+                    }
+                }
+            }
+
+            const response: IDraxImportResponse = {
+                rowCount,
+                successCount,
+                errorCount,
+                time: Date.now() - start,
+                message: errorCount > 0 ? 'Import completed with errors' : 'Import successful',
+                fileName: reportFileName,
+                url: reportUrl,
+            }
+
+            return response
+        } catch (e) {
+            this.handleError(e, reply)
+        }
+    }
+
     async groupBy(request: CustomRequest, reply: FastifyReply) {
         try {
             request.rbac.assertPermission(this.permission.View)