@adminforth/import-export 1.4.8 → 1.4.10

package/.woodpecker/release.yml

@@ -17,18 +17,18 @@ steps:
       - infisical export --domain https://vault.devforth.io/api --format=dotenv-export --env="prod" > /woodpecker/deploy.vault.env
 
   build:
-    image: node:20
+    image: devforth/node20-pnpm:latest
     when:
       - event: push
     commands:
       - apt update && apt install -y rsync
       - . /woodpecker/deploy.vault.env   
-      - npm clean-install
+      - pnpm install
       - /bin/bash ./.woodpecker/buildRelease.sh
       - npm audit signatures
 
   release:
-    image: node:20
+    image: devforth/node20-pnpm:latest
     when:
       - event:
           - push
@@ -36,7 +36,7 @@ steps:
           - main
     commands:
       - . /woodpecker/deploy.vault.env
-      - npx semantic-release
+      - pnpm exec semantic-release
 
   slack-on-failure:
     image: curlimages/curl

package/build.log

@@ -10,5 +10,5 @@ custom/package-lock.json
 custom/package.json
 custom/tsconfig.json
 
-sent 16,779 bytes  received 115 bytes  33,788.00 bytes/sec
-total size is 16,359  speedup is 0.97
+sent 16,704 bytes  received 115 bytes  33,638.00 bytes/sec
+total size is 16,284  speedup is 0.97

package/custom/ExportCsv.vue

@@ -54,12 +54,9 @@ async function exportCsv() {
       throw new Error(resp.error);
     }
 
-    // Parse the CSV data to ensure proper formatting
-    const parsedData = Papa.parse(resp.data).data;
-    
     // Generate properly formatted CSV
-    const csvContent = '\ufeff' + Papa.unparse(parsedData, {
-      quotes: true, // Force quotes around all fields
+    const csvContent = '\ufeff' + Papa.unparse(resp.data, {
+      quotes: resp.columnsToForceQuote, // Force quotes only certain columns (!!!not all this breaks BI/Excel tasks)
       quoteChar: '"',
       escapeChar: '"',
     });

package/custom/ImportCsv.vue

@@ -165,6 +165,7 @@ async function importCsv() {
         Papa.parse(text, {
           header: true,
           skipEmptyLines: true,
+          // dynamicTyping: true, - bad option becaue it tries to parse "1" -> int even if the column is string 
 
           complete: async (results) => {
             if (results.errors.length > 0) {
@@ -214,12 +215,7 @@ async function importCsv() {
     reader.readAsText(file);
   };
 }
-function handleImportClick() {
-  console.log('handleImportClick', checkProgress.value);
-  if (!checkProgress.value) {
-    importCsv();
-  }
-}
+
 
 function click() {
   importCsv();

package/dist/custom/ExportCsv.vue

@@ -54,12 +54,9 @@ async function exportCsv() {
       throw new Error(resp.error);
     }
 
-    // Parse the CSV data to ensure proper formatting
-    const parsedData = Papa.parse(resp.data).data;
-    
     // Generate properly formatted CSV
-    const csvContent = '\ufeff' + Papa.unparse(parsedData, {
-      quotes: true, // Force quotes around all fields
+    const csvContent = '\ufeff' + Papa.unparse(resp.data, {
+      quotes: resp.columnsToForceQuote, // Force quotes only certain columns (!!!not all this breaks BI/Excel tasks)
       quoteChar: '"',
       escapeChar: '"',
     });

package/dist/custom/ImportCsv.vue

@@ -165,6 +165,7 @@ async function importCsv() {
         Papa.parse(text, {
           header: true,
           skipEmptyLines: true,
+          // dynamicTyping: true, - bad option becaue it tries to parse "1" -> int even if the column is string 
 
           complete: async (results) => {
             if (results.errors.length > 0) {
@@ -214,12 +215,7 @@ async function importCsv() {
     reader.readAsText(file);
   };
 }
-function handleImportClick() {
-  console.log('handleImportClick', checkProgress.value);
-  if (!checkProgress.value) {
-    importCsv();
-  }
-}
+
 
 function click() {
   importCsv();

package/dist/index.js

@@ -64,39 +64,23 @@ export default class ImportExport extends AdminForthPlugin {
                     sort,
                     getTotals: true,
                 });
-                // csv export
+                // prepare data for PapaParse unparse
                 const columns = this.resourceConfig.columns.filter((col) => !col.virtual);
-                const escapeCSV = (value, column) => {
-                    if (value === null || value === undefined) {
-                        return '""';
-                    }
-                    if ((column === null || column === void 0 ? void 0 : column.type) === AdminForthDataTypes.FLOAT
-                        || (column === null || column === void 0 ? void 0 : column.type) === AdminForthDataTypes.INTEGER) {
-                        // no quotes for numbers
-                        return String(value);
-                    }
-                    if ((column === null || column === void 0 ? void 0 : column.type) === AdminForthDataTypes.BOOLEAN) {
-                        // no quotes for boolean values
-                        if (value === true) {
-                            return 'true';
-                        }
-                        else if (value === false) {
-                            return 'false';
-                        }
-                    }
-                    const str = String(value);
-                    if (str.includes(',') || str.includes('"') || str.includes('\n')) {
-                        return `"${str.replace(/"/g, '""')}"`;
-                    }
-                    return `"${str}"`;
+                const columnsToForceQuote = columns.map(col => {
+                    return col.type !== AdminForthDataTypes.FLOAT
+                        && col.type !== AdminForthDataTypes.INTEGER
+                        && col.type !== AdminForthDataTypes.BOOLEAN;
+                });
+                const fields = columns.map((col) => col.name);
+                const rows = data.data.map((row) => {
+                    return columns.map((col) => row[col.name]);
+                });
+                return {
+                    data: { fields, data: rows },
+                    columnsToForceQuote,
+                    exportedCount: data.total,
+                    ok: true
                 };
-                let csv = data.data.map((row) => {
-                    return columns.map((col) => escapeCSV(row[col.name], col)).join(',');
-                }).join('\n');
-                // add headers
-                const headers = columns.map((col) => escapeCSV(col.name)).join(',');
-                csv = `${headers}\n${csv}`;
-                return { data: csv, exportedCount: data.total, ok: true };
             })
         });
         server.endpoint({
@@ -105,28 +89,14 @@ export default class ImportExport extends AdminForthPlugin {
             noAuth: true,
             handler: (_a) => __awaiter(this, [_a], void 0, function* ({ body }) {
                 const { data } = body;
-                const rows = [];
-                const columns = Object.keys(data);
-                // check column names are valid
-                const errors = [];
-                columns.forEach((col) => {
-                    if (!this.resourceConfig.columns.some((c) => c.name === col)) {
-                        const similar = suggestIfTypo(this.resourceConfig.columns.map((c) => c.name), col);
-                        errors.push(`Column '${col}' defined in CSV not found in resource '${this.resourceConfig.resourceId}'. ${similar ? `If you mean '${similar}', rename it in CSV` : 'If column is in database but not in resource configuration, add it with showIn:[]'}`);
-                    }
-                });
+                const columns = this.getColumnNames(data);
+                const { errors, resourceColumns } = this.validateColumns(columns);
                 if (errors.length > 0) {
                     return { ok: false, errors };
                 }
                 const primaryKeyColumn = this.resourceConfig.columns.find(col => col.primaryKey);
-                const columnValues = Object.values(data);
-                for (let i = 0; i < columnValues[0].length; i++) {
-                    const row = {};
-                    for (let j = 0; j < columns.length; j++) {
-                        row[columns[j]] = columnValues[j][i];
-                    }
-                    rows.push(row);
-                }
+                const rows = this.buildRowsFromData(data, columns, resourceColumns, { coerceTypes: true });
+                console.log('Prepared rows for import:', rows);
                 let importedCount = 0;
                 let updatedCount = 0;
                 yield Promise.all(rows.map((row) => __awaiter(this, void 0, void 0, function* () {
@@ -157,28 +127,13 @@ export default class ImportExport extends AdminForthPlugin {
             noAuth: true,
             handler: (_a) => __awaiter(this, [_a], void 0, function* ({ body }) {
                 const { data } = body;
-                const rows = [];
-                const columns = Object.keys(data);
-                // check column names are valid
-                const errors = [];
-                columns.forEach((col) => {
-                    if (!this.resourceConfig.columns.some((c) => c.name === col)) {
-                        const similar = suggestIfTypo(this.resourceConfig.columns.map((c) => c.name), col);
-                        errors.push(`Column '${col}' defined in CSV not found in resource '${this.resourceConfig.resourceId}'. ${similar ? `If you mean '${similar}', rename it in CSV` : 'If column is in database but not in resource configuration, add it with showIn:[]'}`);
-                    }
-                });
+                const columns = this.getColumnNames(data);
+                const { errors, resourceColumns } = this.validateColumns(columns);
                 if (errors.length > 0) {
                     return { ok: false, errors };
                 }
                 const primaryKeyColumn = this.resourceConfig.columns.find(col => col.primaryKey);
-                const columnValues = Object.values(data);
-                for (let i = 0; i < columnValues[0].length; i++) {
-                    const row = {};
-                    for (let j = 0; j < columns.length; j++) {
-                        row[columns[j]] = columnValues[j][i];
-                    }
-                    rows.push(row);
-                }
+                const rows = this.buildRowsFromData(data, columns, resourceColumns, { coerceTypes: true });
                 let importedCount = 0;
                 yield Promise.all(rows.map((row) => __awaiter(this, void 0, void 0, function* () {
                     try {
@@ -206,17 +161,10 @@ export default class ImportExport extends AdminForthPlugin {
             handler: (_a) => __awaiter(this, [_a], void 0, function* ({ body }) {
                 const { data } = body;
                 const primaryKeyColumn = this.resourceConfig.columns.find(col => col.primaryKey);
-                const columns = Object.keys(data);
-                const columnValues = Object.values(data);
-                const rows = Array.from({ length: columnValues[0].length }, (_, i) => {
-                    const row = {};
-                    for (let j = 0; j < columns.length; j++) {
-                        row[columns[j]] = columnValues[j][i];
-                    }
-                    return row;
-                });
+                const columns = this.getColumnNames(data);
+                const rows = this.buildRowsFromData(data, columns, undefined, { coerceTypes: false });
                 const primaryKeys = rows
-                    .map(row => row[primaryKeyColumn.name])
+                    .map(row => primaryKeyColumn ? row[primaryKeyColumn.name] : undefined)
                     .filter(key => key !== undefined && key !== null && key !== '');
                 const existingRecords = yield this.adminforth
                     .resource(this.resourceConfig.resourceId)
@@ -234,4 +182,59 @@ export default class ImportExport extends AdminForthPlugin {
             })
         });
     }
+    getColumnNames(data) {
+        return Object.keys(data !== null && data !== void 0 ? data : {});
+    }
+    validateColumns(columns) {
+        const errors = [];
+        const resourceColumns = [];
+        columns.forEach((col) => {
+            const resourceColumn = this.resourceConfig.columns.find((c) => c.name === col);
+            if (!resourceColumn) {
+                const similar = suggestIfTypo(this.resourceConfig.columns.map((c) => c.name), col);
+                errors.push(`Column '${col}' defined in CSV not found in resource '${this.resourceConfig.resourceId}'. ${similar
+                    ? `If you mean '${similar}', rename it in CSV`
+                    : 'If column is in database but not in resource configuration, add it with showIn:[]'}`);
+                return;
+            }
+            resourceColumns.push(resourceColumn);
+        });
+        return { errors, resourceColumns };
+    }
+    buildRowsFromData(data, columns, resourceColumns, { coerceTypes } = { coerceTypes: true }) {
+        const columnValues = Object.values(data !== null && data !== void 0 ? data : {});
+        if (columns.length === 0 || columnValues.length === 0) {
+            return [];
+        }
+        const rows = [];
+        const rowCount = columnValues[0].length;
+        for (let i = 0; i < rowCount; i++) {
+            const row = {};
+            for (let j = 0; j < columns.length; j++) {
+                const val = columnValues[j][i];
+                const resourceCol = resourceColumns ? resourceColumns[j] : undefined;
+                row[columns[j]] = coerceTypes
+                    ? this.coerceValue(resourceCol, val)
+                    : val;
+            }
+            rows.push(row);
+        }
+        return rows;
+    }
+    coerceValue(resourceCol, val) {
+        if (!resourceCol || val === '') {
+            return val;
+        }
+        if ((resourceCol.type === AdminForthDataTypes.INTEGER
+            || resourceCol.type === AdminForthDataTypes.FLOAT)) {
+            return +val;
+        }
+        if (resourceCol.type === AdminForthDataTypes.BOOLEAN) {
+            if (typeof val === 'string') {
+                return val.toLowerCase() === 'true' || val === '1';
+            }
+            return val === 1 || val === true;
+        }
+        return val;
+    }
 }

package/index.ts

@@ -66,42 +66,27 @@ export default class ImportExport extends AdminForthPlugin {
           getTotals: true,
         });
 
-        // csv export
+        // prepare data for PapaParse unparse
         const columns = this.resourceConfig.columns.filter((col) => !col.virtual);
-        
-        const escapeCSV = (value: any, column?: AdminForthResourceColumn) => {
-          if (value === null || value === undefined) {
-            return '""';
-          }
-          if (column?.type === AdminForthDataTypes.FLOAT 
-              || column?.type === AdminForthDataTypes.INTEGER) {
-            // no quotes for numbers
-            return String(value);
-          }
-          if (column?.type === AdminForthDataTypes.BOOLEAN) {
-            // no quotes for boolean values
-            if (value === true) {
-              return 'true';
-            } else if (value === false) {
-              return 'false';
-            }
-          }
-          const str = String(value);
-          if (str.includes(',') || str.includes('"') || str.includes('\n')) {
-            return `"${str.replace(/"/g, '""')}"`;
-          }
-          return `"${str}"`;
-        };
 
-        let csv = data.data.map((row) => {
-          return columns.map((col) => escapeCSV(row[col.name], col)).join(',');
-        }).join('\n');
+        const columnsToForceQuote = columns.map(col => {
+          return col.type !== AdminForthDataTypes.FLOAT 
+            && col.type !== AdminForthDataTypes.INTEGER 
+            && col.type !== AdminForthDataTypes.BOOLEAN;
+        })
 
-        // add headers
-        const headers = columns.map((col) => escapeCSV(col.name)).join(',');
-        csv = `${headers}\n${csv}`;
+        const fields = columns.map((col) => col.name);
 
-        return { data: csv, exportedCount: data.total, ok: true };
+        const rows = data.data.map((row) => {
+          return columns.map((col) => row[col.name]);
+        });
+
+        return { 
+          data: { fields, data: rows }, 
+          columnsToForceQuote, 
+          exportedCount: data.total, 
+          ok: true 
+        };
       }
     });
 
@@ -111,33 +96,15 @@ export default class ImportExport extends AdminForthPlugin {
       noAuth: true,
       handler: async ({ body }) => {
         const { data } = body;
-        const rows = [];
-        const columns = Object.keys(data);
-
-        // check column names are valid
-        const errors: string[] = [];
-        columns.forEach((col) => {
-          if (!this.resourceConfig.columns.some((c) => c.name === col)) {
-            const similar = suggestIfTypo(this.resourceConfig.columns.map((c) => c.name), col);
-            errors.push(`Column '${col}' defined in CSV not found in resource '${this.resourceConfig.resourceId}'. ${
-              similar ? `If you mean '${similar}', rename it in CSV` : 'If column is in database but not in resource configuration, add it with showIn:[]'}`
-            );
-          }
-        });
+        const columns = this.getColumnNames(data);
+        const { errors, resourceColumns } = this.validateColumns(columns);
         if (errors.length > 0) {
           return { ok: false, errors };
         }
-
         const primaryKeyColumn = this.resourceConfig.columns.find(col => col.primaryKey);
+        const rows = this.buildRowsFromData(data, columns, resourceColumns, { coerceTypes: true });
 
-        const columnValues: any[] = Object.values(data);
-        for (let i = 0; i < columnValues[0].length; i++) {
-          const row = {};
-          for (let j = 0; j < columns.length; j++) {
-            row[columns[j]] = columnValues[j][i];
-          }
-          rows.push(row);
-        }
+        console.log('Prepared rows for import:', rows);
 
         let importedCount = 0;
         let updatedCount = 0;
@@ -172,32 +139,14 @@ export default class ImportExport extends AdminForthPlugin {
       noAuth: true,
       handler: async ({ body }) => {
         const { data } = body;
-        const rows = [];
-        const columns = Object.keys(data);
-
-        // check column names are valid
-        const errors: string[] = [];
-        columns.forEach((col) => {
-          if (!this.resourceConfig.columns.some((c) => c.name === col)) {
-            const similar = suggestIfTypo(this.resourceConfig.columns.map((c) => c.name), col);
-            errors.push(`Column '${col}' defined in CSV not found in resource '${this.resourceConfig.resourceId}'. ${
-              similar ? `If you mean '${similar}', rename it in CSV` : 'If column is in database but not in resource configuration, add it with showIn:[]'}`
-            );
-          }
-        });
+        const columns = this.getColumnNames(data);
+        const { errors, resourceColumns } = this.validateColumns(columns);
         if (errors.length > 0) {
           return { ok: false, errors };
         }
 
         const primaryKeyColumn = this.resourceConfig.columns.find(col => col.primaryKey);
-        const columnValues: any[] = Object.values(data);
-        for (let i = 0; i < columnValues[0].length; i++) {
-          const row = {};
-          for (let j = 0; j < columns.length; j++) {
-            row[columns[j]] = columnValues[j][i];
-          }
-          rows.push(row);
-        }
+        const rows = this.buildRowsFromData(data, columns, resourceColumns, { coerceTypes: true });
 
         let importedCount = 0;
 
@@ -229,19 +178,11 @@ export default class ImportExport extends AdminForthPlugin {
       handler: async ({ body }) => {
         const { data } = body as { data: Record<string, unknown[]> };
         const primaryKeyColumn = this.resourceConfig.columns.find(col => col.primaryKey);
-        const columns = Object.keys(data);
-        const columnValues = Object.values(data);
-
-        const rows = Array.from({ length: columnValues[0].length }, (_, i) => {
-          const row = {};
-          for (let j = 0; j < columns.length; j++) {
-            row[columns[j]] = columnValues[j][i];
-          }
-          return row;
-        });
+        const columns = this.getColumnNames(data);
+        const rows = this.buildRowsFromData(data, columns, undefined, { coerceTypes: false });
 
         const primaryKeys = rows
-          .map(row => row[primaryKeyColumn.name])
+          .map(row => primaryKeyColumn ? row[primaryKeyColumn.name] : undefined)
           .filter(key => key !== undefined && key !== null && key !== '');
 
         const existingRecords = await this.adminforth
@@ -263,4 +204,85 @@ export default class ImportExport extends AdminForthPlugin {
     });
   }
 
+  private getColumnNames(data: Record<string, unknown[]>): string[] {
+    return Object.keys(data ?? {});
+  }
+
+  private validateColumns(columns: string[]): {
+    errors: string[];
+    resourceColumns: AdminForthResourceColumn[];
+  } {
+    const errors: string[] = [];
+    const resourceColumns: AdminForthResourceColumn[] = [];
+
+    columns.forEach((col) => {
+      const resourceColumn = this.resourceConfig.columns.find((c) => c.name === col);
+      if (!resourceColumn) {
+        const similar = suggestIfTypo(this.resourceConfig.columns.map((c) => c.name), col);
+        errors.push(
+          `Column '${col}' defined in CSV not found in resource '${this.resourceConfig.resourceId}'. ${
+            similar
+              ? `If you mean '${similar}', rename it in CSV`
+              : 'If column is in database but not in resource configuration, add it with showIn:[]'
+          }`
+        );
+        return;
+      }
+      resourceColumns.push(resourceColumn);
+    });
+
+    return { errors, resourceColumns };
+  }
+
+  private buildRowsFromData(
+    data: Record<string, unknown[]>,
+    columns: string[],
+    resourceColumns?: AdminForthResourceColumn[],
+    { coerceTypes }: { coerceTypes: boolean } = { coerceTypes: true }
+  ) {
+    const columnValues: unknown[][] = Object.values(data ?? {});
+    if (columns.length === 0 || columnValues.length === 0) {
+      return [];
+    }
+
+    const rows: Record<string, unknown>[] = [];
+    const rowCount = columnValues[0].length;
+
+    for (let i = 0; i < rowCount; i++) {
+      const row: Record<string, unknown> = {};
+      for (let j = 0; j < columns.length; j++) {
+        const val = columnValues[j][i];
+        const resourceCol = resourceColumns ? resourceColumns[j] : undefined;
+        row[columns[j]] = coerceTypes
+          ? this.coerceValue(resourceCol, val)
+          : val;
+      }
+      rows.push(row);
+    }
+
+    return rows;
+  }
+
+  private coerceValue(resourceCol: AdminForthResourceColumn | undefined, val: unknown): unknown {
+    if (!resourceCol || val === '') {
+      return val;
+    }
+
+    if (
+      (resourceCol.type === AdminForthDataTypes.INTEGER
+        || resourceCol.type === AdminForthDataTypes.FLOAT)
+    ) {
+      return +val;
+    }
+
+    if (resourceCol.type === AdminForthDataTypes.BOOLEAN) {
+      if (typeof val === 'string') {
+        return val.toLowerCase() === 'true' || val === '1';
+      }
+      return val === 1 || val === true;
+    }
+
+    return val;
+  }
+
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adminforth/import-export",
-  "version": "1.4.8",
+  "version": "1.4.10",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "type": "module",
@@ -9,8 +9,7 @@
   },
   "homepage": "https://adminforth.dev/docs/tutorial/Plugins/import-export/",
   "scripts": {
-    "build": "tsc && rsync -av --exclude 'node_modules' custom dist/",
-    "prepare": "npm link adminforth"
+    "build": "tsc && rsync -av --exclude 'node_modules' custom dist/"
   },
   "repository": {
     "type": "git",
@@ -26,7 +25,7 @@
   "license": "ISC",
   "description": "CSV import/export plugin for adminforth",
   "peerDependencies": {
-    "adminforth": "next"
+    "adminforth": "^2.24.0"
   },
   "devDependencies": {
     "@types/node": "^22.10.7",
@@ -43,6 +42,7 @@
       [
         "semantic-release-slack-bot",
         {
+          "packageName": "@adminforth/import-export",
           "notifyOnSuccess": true,
           "notifyOnFail": true,
           "slackIcon": ":package:",