@drax/crud-back 3.9.0 → 3.11.0

package/src/imports/AbstractImport.ts

@@ -0,0 +1,73 @@
+import {setNestedValue} from "@drax/common-back";
+
+interface ImportOptions {
+    content: string
+    separator?: string
+}
+
+interface ParsedImportRow<T = any> {
+    rawValues: string[]
+    item: T
+}
+
+class AbstractImport {
+
+    protected content: string
+    protected separator: string
+
+    constructor(options: ImportOptions) {
+        this.content = options.content;
+        this.separator = options.separator || ';';
+    }
+
+    parseValue(value: string): any {
+        const trimmedValue = value.trim();
+
+        if (trimmedValue === '') {
+            return '';
+        }
+
+        if (trimmedValue === 'null') {
+            return null;
+        }
+
+        if (trimmedValue === 'true') {
+            return true;
+        }
+
+        if (trimmedValue === 'false') {
+            return false;
+        }
+
+        if (/^-?\d+(\.\d+)?$/.test(trimmedValue)) {
+            return Number(trimmedValue);
+        }
+
+        if (
+            (trimmedValue.startsWith('{') && trimmedValue.endsWith('}')) ||
+            (trimmedValue.startsWith('[') && trimmedValue.endsWith(']'))
+        ) {
+            try {
+                return JSON.parse(trimmedValue);
+            } catch {
+                return value;
+            }
+        }
+
+        return value;
+    }
+
+    assignNestedValue(record: Record<string, any>, key: string, value: any) {
+        if (key.includes('.')) {
+            setNestedValue(record, key, value);
+            return;
+        }
+
+        record[key] = value;
+    }
+}
+
+export type {ImportOptions}
+export type {ParsedImportRow}
+export {AbstractImport}
+export default AbstractImport;

package/src/imports/ImportCsv.ts

@@ -0,0 +1,102 @@
+import AbstractImport, {ImportOptions, ParsedImportRow} from "./AbstractImport.js";
+
+interface ParsedCsvImport<T = any> {
+    headers: string[]
+    rows: ParsedImportRow<T>[]
+}
+
+class ImportCsv extends AbstractImport {
+
+    constructor(options: ImportOptions) {
+        super(options);
+    }
+
+    process(): any[] {
+        return this.processDetailed().rows.map(row => row.item);
+    }
+
+    processDetailed(): ParsedCsvImport {
+        const rows = this.parseRows(this.content);
+        if (rows.length === 0) {
+            return {headers: [], rows: []};
+        }
+
+        const [headers, ...dataRows] = rows;
+
+        return {
+            headers,
+            rows: dataRows
+            .filter(row => row.some(value => value.trim() !== ''))
+            .map((row): ParsedImportRow => {
+                const item: Record<string, any> = {};
+
+                headers.forEach((header, index) => {
+                    const normalizedHeader = header.trim();
+                    if (!normalizedHeader) {
+                        return;
+                    }
+
+                    this.assignNestedValue(item, normalizedHeader, this.parseValue(row[index] ?? ''));
+                });
+
+                return {
+                    rawValues: row,
+                    item
+                };
+            })
+        };
+    }
+
+    protected parseRows(content: string): string[][] {
+        const rows: string[][] = [];
+        let currentRow: string[] = [];
+        let currentValue = '';
+        let inQuotes = false;
+
+        for (let index = 0; index < content.length; index++) {
+            const char = content[index];
+            const nextChar = content[index + 1];
+
+            if (char === '"') {
+                if (inQuotes && nextChar === '"') {
+                    currentValue += '"';
+                    index++;
+                } else {
+                    inQuotes = !inQuotes;
+                }
+                continue;
+            }
+
+            if (char === this.separator && !inQuotes) {
+                currentRow.push(currentValue);
+                currentValue = '';
+                continue;
+            }
+
+            if ((char === '\n' || char === '\r') && !inQuotes) {
+                if (char === '\r' && nextChar === '\n') {
+                    index++;
+                }
+
+                currentRow.push(currentValue);
+                if (currentRow.some(value => value !== '')) {
+                    rows.push(currentRow);
+                }
+                currentRow = [];
+                currentValue = '';
+                continue;
+            }
+
+            currentValue += char;
+        }
+
+        currentRow.push(currentValue);
+        if (currentRow.some(value => value !== '')) {
+            rows.push(currentRow);
+        }
+
+        return rows;
+    }
+}
+
+export default ImportCsv;

package/src/imports/ImportCsvReport.ts

@@ -0,0 +1,83 @@
+import * as fs from 'fs';
+import AbstractExport from "../exports/AbstractExport.js";
+
+interface ImportCsvReportRow {
+    rawValues: string[]
+    status: 'success' | 'error'
+    error?: string
+}
+
+interface ImportCsvReportOptions {
+    destinationPath: string
+    fileName?: string
+    headers: string[]
+    separator?: string
+    rows: ImportCsvReportRow[]
+}
+
+class ImportCsvReport extends AbstractExport {
+
+    protected separator: string
+    protected rows: ImportCsvReportRow[]
+
+    constructor(options: ImportCsvReportOptions) {
+        super({
+            cursor: [],
+            destinationPath: options.destinationPath,
+            headers: options.headers,
+            fileName: options.fileName || 'import_report',
+        });
+        this.separator = options.separator || ';';
+        this.rows = options.rows;
+    }
+
+    process(): Promise<any> {
+        return new Promise((resolve, reject) => {
+            try {
+                this.generateFilePath('csv')
+                const start = Date.now();
+                const writableStream = fs.createWriteStream(this.relativeFilePath);
+
+                writableStream.on('error', reject);
+                writableStream.on('finish', () => resolve({
+                    status: 'success',
+                    destinationPath: this.destinationPath,
+                    fileName: this.fileName,
+                    filePath: this.destinationPath + '/' + this.fileName,
+                    relativeFilePath: this.relativeFilePath,
+                    rowCount: this.rows.length,
+                    time: Date.now() - start,
+                    message: 'Import report generated',
+                }))
+
+                writableStream.write([...this.headers, 'import_status', 'error'].join(this.separator) + '\n');
+
+                for (const row of this.rows) {
+                    const values = [...row.rawValues, row.status, row.error || ''].map(value => this.escapeCsvValue(value));
+                    writableStream.write(values.join(this.separator) + '\n');
+                }
+
+                writableStream.end();
+            } catch (e) {
+                reject(e);
+            }
+        })
+    }
+
+    protected escapeCsvValue(value: any): string {
+        let formattedValue = value === null || value === undefined ? '' : String(value);
+
+        if (
+            formattedValue.includes(this.separator) ||
+            formattedValue.includes('"') ||
+            formattedValue.includes('\n') ||
+            formattedValue.includes('\r')
+        ) {
+            formattedValue = '"' + formattedValue.replace(/"/g, '""') + '"';
+        }
+
+        return formattedValue;
+    }
+}
+
+export default ImportCsvReport;

package/src/imports/ImportJson.ts

@@ -0,0 +1,20 @@
+import AbstractImport, {ImportOptions} from "./AbstractImport.js";
+
+class ImportJson extends AbstractImport {
+
+    constructor(options: ImportOptions) {
+        super(options);
+    }
+
+    process(): any[] {
+        const parsedContent = JSON.parse(this.content);
+
+        if (!Array.isArray(parsedContent)) {
+            throw new Error('Invalid JSON import format. Expected an array of objects');
+        }
+
+        return parsedContent;
+    }
+}
+
+export default ImportJson;

package/src/regexs/QueryFilterRegex.ts

@@ -1,4 +1,4 @@
-const QueryFilterRegex = /^(?:[a-zA-Z0-9_.\-]+;(?:eq|like|ne|in|nin|gt|gte|lt|lte|empty);[a-zA-Z0-9_.\-:\., áéíóúÁÉÍÓÚ]*)(?:\|[a-zA-Z0-9_.\-]+;(?:eq|like|ne|in|nin|gt|gte|lt|lte|empty);[a-zA-Z0-9_.\-:\., áéíóúÁÉÍÓÚ]*)*$/
+const QueryFilterRegex = /^(?:[a-zA-Z0-9_.\-]+;(?:eq|like|ne|in|nin|gt|gte|lt|lte|empty);[a-zA-Z0-9_.\-:\., áéíóúÁÉÍÓÚ]*(?:;[a-zA-Z0-9_.\-]+)?)(?:\|[a-zA-Z0-9_.\-]+;(?:eq|like|ne|in|nin|gt|gte|lt|lte|empty);[a-zA-Z0-9_.\-:\., áéíóúÁÉÍÓÚ]*(?:;[a-zA-Z0-9_.\-]+)?)*$/
 
 export default QueryFilterRegex
 export {QueryFilterRegex}

package/src/schemas/FindSchema.ts

@@ -6,7 +6,7 @@ const FindQuerySchema = z.object({
     orderBy: z.string().optional(),
     order: z.enum(["asc", "desc"]).optional(),
     search: z.string().optional(),
-    filters:  z.string().regex(QueryFilterRegex).optional().describe("Format: field;operator;value|field;operator;value|..."),
+    filters:  z.string().regex(QueryFilterRegex).optional().describe("Format: field;operator;value[;orGroup]|field;operator;value[;orGroup]|..."),
 });
 
 

package/src/schemas/GroupBySchema.ts

@@ -3,7 +3,7 @@ import QueryFilterRegex from "../regexs/QueryFilterRegex.js";
 
 const GroupByQuerySchema = z.object({
     fields: z.array(z.string()).min(1).max(10),
-    filters:  z.string().regex(QueryFilterRegex).optional().describe("Format: field;operator;value|field;operator;value|..."),
+    filters:  z.string().regex(QueryFilterRegex).optional().describe("Format: field;operator;value[;orGroup]|field;operator;value[;orGroup]|..."),
 });
 
 

package/src/schemas/PaginateSchema.ts

@@ -7,7 +7,7 @@ const PaginateQuerySchema = z.object({
     orderBy: z.string().optional(),
     order: z.enum(["asc", "desc"]).optional(),
     search: z.string().optional(),
-    filters:  z.string().regex(QueryFilterRegex).optional().describe("Format: field;operator;value|field;operator;value|..."),
+    filters:  z.string().regex(QueryFilterRegex).optional().describe("Format: field;operator;value[;orGroup]|field;operator;value[;orGroup]|..."),
 });
 
 

package/src/services/AbstractService.ts

@@ -6,13 +6,15 @@ import type {
     IDraxPaginateResult,
     IDraxFindOptions,
     IDraxExportOptions,
-    IDraxCrudRepository, IDraxFieldFilter, IDraxGroupByOptions
+    IDraxCrudRepository, IDraxFieldFilter, IDraxGroupByOptions, IDraxImportOptions, IDraxImportResult
 } from "@drax/crud-share";
 import {IDraxCrudService} from "@drax/crud-share";
 import ExportCsv from "../exports/ExportCsv.js";
 import ExportJson from "../exports/ExportJson.js";
 import {IDraxExportResult} from "@drax/crud-share";
 import {IDraxFindOneOptions} from "@drax/crud-share/types/interfaces/IDraxFindOneOptions";
+import ImportCsv from "../imports/ImportCsv.js";
+import ImportJson from "../imports/ImportJson.js";
 
 abstract class AbstractService<T, C, U> implements IDraxCrudService<T, C, U> {
 
@@ -476,6 +478,56 @@ abstract class AbstractService<T, C, U> implements IDraxCrudService<T, C, U> {
 
     }
 
+    parseImport({
+                    format = 'JSON',
+                    content,
+                    separator = ';'
+                }: IDraxImportOptions): C[] {
+        let importer: ImportCsv | ImportJson;
+
+        switch (format) {
+            case 'JSON':
+                importer = new ImportJson({content, separator});
+                break;
+            case 'CSV':
+                importer = new ImportCsv({content, separator});
+                break;
+            default:
+                throw new Error(`Unsupported import format: ${format}`);
+        }
+
+        return importer.process() as C[];
+    }
+
+    async import({
+                     format = 'JSON',
+                     content,
+                     separator = ';'
+                 }: IDraxImportOptions): Promise<IDraxImportResult> {
+        try {
+            const start = Date.now();
+            const items = this.parseImport({format, content, separator});
+
+            for (const item of items) {
+                await this.create(item as C);
+            }
+
+            return {
+                status: 'success',
+                rowCount: items.length,
+                time: Date.now() - start,
+                message: 'Import successful',
+            };
+        } catch (e) {
+            console.error("Error import", {
+                name: e?.name,
+                message: e?.message,
+                stack: e?.stack,
+            });
+            throw e;
+        }
+    }
+
 }
 
 export default AbstractService

package/test/controllers/PersonController.test.ts

@@ -466,6 +466,70 @@ describe("Person Controller Test", function () {
         expect(findByResult[0].fullname).toBe("Active Person")
     })
 
+    it("should create and find people with filters grouped by orGroup", async () => {
+        const { accessToken } = await testSetup.rootUserLogin()
+        await testSetup.dropCollection('Person')
+
+        const entityData = [
+            { fullname: "Hero Person", race: "human", live: true, address: defaultAddress },
+            { fullname: "Mage Person", race: "elf", live: true, address: defaultAddress },
+            { fullname: "Hidden Hero", race: "orc", live: false, address: defaultAddress },
+            { fullname: "Hidden Rogue", race: "human", live: false, address: defaultAddress }
+        ]
+
+        for (const data of entityData) {
+            await testSetup.fastifyInstance.inject({
+                method: 'POST',
+                url: '/api/person',
+                payload: data,
+                headers: { Authorization: `Bearer ${accessToken}` }
+            })
+        }
+
+        const findByResp = await testSetup.fastifyInstance.inject({
+            method: 'GET',
+            url: '/api/person/find?filters=fullname;like;Hero;group1|race;eq;elf;group1|live;eq;true',
+            headers: { Authorization: `Bearer ${accessToken}` }
+        })
+
+        const findByResult = await findByResp.json()
+        expect(findByResp.statusCode).toBe(200)
+        expect(findByResult.length).toBe(2)
+        expect(findByResult.map((item: any) => item.fullname).sort()).toEqual(["Hero Person", "Mage Person"])
+    })
+
+    it("should combine search with orGroup filters without overriding either condition", async () => {
+        const { accessToken } = await testSetup.rootUserLogin()
+        await testSetup.dropCollection('Person')
+
+        const entityData = [
+            { fullname: "Searchable Hero", race: "human", live: true, address: defaultAddress },
+            { fullname: "Searchable Elf", race: "elf", live: true, address: defaultAddress },
+            { fullname: "Hidden Elf", race: "elf", live: true, address: defaultAddress },
+            { fullname: "Searchable Rogue", race: "human", live: false, address: defaultAddress }
+        ]
+
+        for (const data of entityData) {
+            await testSetup.fastifyInstance.inject({
+                method: 'POST',
+                url: '/api/person',
+                payload: data,
+                headers: { Authorization: `Bearer ${accessToken}` }
+            })
+        }
+
+        const paginateResp = await testSetup.fastifyInstance.inject({
+            method: 'GET',
+            url: '/api/person?search=Searchable&filters=fullname;like;Hero;group1|race;eq;elf;group1|live;eq;true',
+            headers: { Authorization: `Bearer ${accessToken}` }
+        })
+
+        const paginateResult = await paginateResp.json()
+        expect(paginateResp.statusCode).toBe(200)
+        expect(paginateResult.total).toBe(2)
+        expect(paginateResult.items.map((item: any) => item.fullname).sort()).toEqual(["Searchable Elf", "Searchable Hero"])
+    })
+
     // 8. Create and Group By
     it("should create and groupBy for people", async () => {
         const { accessToken } = await testSetup.rootUserLogin()

package/test/services/AbstractService.test.ts

@@ -1,12 +1,6 @@
 import { test, assert } from 'vitest';
 import MockRepository from "../_mocks/MockRepository.js";
 import {AbstractService} from "../../src/services/AbstractService.js";
-import {fileURLToPath} from "url";
-import * as path from "path";
-
-//@ts-ignore
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
 
 const mockRepository = new MockRepository();
 
@@ -21,11 +15,29 @@ test('create', async () => {
     assert.deepStrictEqual(item.name, 'John Doe');
 })
 
+test('import json', async () => {
+    const result: any = await service.import({
+        format: 'JSON',
+        content: JSON.stringify([
+            {name: 'John Doe'},
+            {name: 'Jane Doe'}
+        ])
+    })
 
-test('export', async () => {
-    const file = 'test.csv'
-    const outputPath = path.resolve(__dirname, file);
+    assert.deepStrictEqual(result.rowCount, 2);
+})
+
+test('import csv', async () => {
+    const result: any = await service.import({
+        format: 'CSV',
+        separator: ';',
+        content: '_id;name;profile.age\n1;John Doe;32\n2;Jane Doe;28'
+    })
 
+    assert.deepStrictEqual(result.rowCount, 2);
+})
+
+test('export', async () => {
     const result:any = await service.export(
         {
             format: 'CSV',
@@ -39,6 +51,5 @@ test('export', async () => {
 
     console.log("result",result)
 
-    assert.deepStrictEqual(outputPath, result.outputPath);
     assert.deepStrictEqual(2, result.rowCount);
 })