@nkhang1902/strapi-plugin-export-import-clsx 1.0.4 → 1.1.0

package/server/services/import-service.js

@@ -1,79 +1,59 @@
-const XLSX = require('xlsx');
-const fs = require('fs');
+const XLSX = require("xlsx");
+const fs = require("fs");
 
 function toCamel(str) {
   return str.replace(/_([a-z])/g, (_, c) => c.toUpperCase());
 }
 
-const SYSTEM_KEYS = [
-  'documentId',
-  'locale',
-  'createdAt',
-  'updatedAt',
-  'publishedAt',
-  'createdBy',
-  'updatedBy',
-  'localizations',
-  'status'
-];
-
 const SHORTCUT_FIELDS = [
-  'email','businessEmail','name','title','tickerCode',
-]
-async function importData(file) {
-  let result;
-  try {
-    let importData;
-    // Check file extension
-    const fileName = file.name || file.originalFilename || 'unknown.json';
-    const fileExtension = fileName.split('.').pop().toLowerCase();
-    const filePath = file.path || file.filepath;
-    if (!filePath) {
-      throw new Error('File path not found');
-    }
+  "email",
+  "businessEmail",
+  "name",
+  "title",
+  "tickerCode",
+];
+module.exports = ({ strapi }) => ({
+  async importData(file, targetContentType = null) {
+    let result;
+    try {
+      let importData;
+      // Check file extension
+      const fileName = file.name || file.originalFilename || "unknown.json";
+      const fileExtension = fileName.split(".").pop().toLowerCase();
+      const filePath = file.path || file.filepath;
+      if (!filePath) {
+        throw new Error("File path not found");
+      }
 
-    if (fileExtension === 'json') {
-      const fileContent = fs.readFileSync(filePath, 'utf8');
-      importData = JSON.parse(fileContent);
-      strapi.log.info('Parsed JSON data:', Object.keys(importData));
-    } else if (fileExtension === 'xlsx' || fileExtension === 'xls') {
-      importData = transformExcelData(filePath);
-    }
-    result = await bulkInsertData(importData);
-    return result;
-  } catch (error) {
-    // Clean up uploaded file on error
-    const filePath = file && (file.path || file.filepath);
-    if (filePath && fs.existsSync(filePath)) {
-      fs.unlinkSync(filePath);
+      if (fileExtension === "json") {
+        const fileContent = fs.readFileSync(filePath, "utf8");
+        importData = JSON.parse(fileContent);
+        strapi.log.info("Parsed JSON data:", Object.keys(importData));
+      } else if (fileExtension === "xlsx" || fileExtension === "xls") {
+        importData = this.transformExcelData(filePath, targetContentType);
+      }
+      result = await this.bulkInsertData(importData);
+      return result;
+    } catch (error) {
+      // Clean up uploaded file on error
+      const filePath = file && (file.path || file.filepath);
+      if (filePath && fs.existsSync(filePath)) {
+        fs.unlinkSync(filePath);
+      }
+      throw error;
     }
-    throw error;
-  }
-}
-
+  },
 
-function transformExcelData(filePath) {
+  transformExcelData(filePath, targetContentType = null) {
     const workbook = XLSX.readFile(filePath);
     const importData = {};
 
-    const parseJsonIfNeeded = (value) => {
-        if (typeof value !== 'string') return value;
-        const trimmed = value.trim();
-        if (!trimmed.startsWith('[') && !trimmed.startsWith('{')) return value;
-
-        try {
-        return JSON.parse(trimmed);
-        } catch {
-        return value; // keep as string if invalid JSON
-        }
-    };
-
     const isComponentField = (key) => {
-        const parts = key.split('_');
-        return parts.length === 2; // exactly one underscore
+      const parts = key.split("_");
+      return parts.length === 2; // exactly one underscore
     };
 
-    function unflattenRow(rows, targetContentType) {
+    const unflattenRow = (rows, targetContentType) => {
       const result = [];
       const attr = strapi.contentTypes[targetContentType].attributes;
       for (const row of rows) {
@@ -83,13 +63,13 @@ function transformExcelData(filePath) {
             if (value === null || value === undefined || value === '') {
               rowData[key] = null
             } else if (attr[key] && attr[key].customField && attr[key].type === 'json' && attr[key].default === '[]') {
-              rowData[key] = parseJsonIfNeeded(value).split('|');
+              rowData[key] = value.split('|');
             } else if (isComponentField(key)) {
               const [comp, field] = key.split('_');
               if (!rowData[comp]) rowData[comp] = {};
-              rowData[comp][field] = parseJsonIfNeeded(value);
+              rowData[comp][field] = value;
             } else {
-              rowData[key] = parseJsonIfNeeded(value);
+              rowData[key] = value;
             }
         }
         result.push(rowData);
@@ -99,280 +79,341 @@ function transformExcelData(filePath) {
     };
 
     const mapSheetNameToContentType = (sheetName) => {
-        return "api::" + sheetName + "." + sheetName;
+      // If targetContentType is provided, use it instead of guessing from sheet name
+      if (targetContentType) {
+        return targetContentType;
+      }
+      return "api::" + sheetName + "." + sheetName;
     };
 
-    workbook.SheetNames.forEach(sheetName => {
-        const worksheet = workbook.Sheets[sheetName];
-        const rows = XLSX.utils.sheet_to_json(worksheet);
+    workbook.SheetNames.forEach((sheetName) => {
+      const worksheet = workbook.Sheets[sheetName];
+      const rows = XLSX.utils.sheet_to_json(worksheet);
 
-        if (!rows.length) return;
+      if (!rows.length) return;
 
-        const contentTypeName = mapSheetNameToContentType(sheetName);
+      const contentTypeName = mapSheetNameToContentType(sheetName);
 
-        strapi.log.info(`Reading sheet "${sheetName}" -> ${rows.length} rows`);
-        strapi.log.info(`Mapped sheet to content-type: ${contentTypeName}`);
+      strapi.log.info(`Reading sheet "${sheetName}" -> ${rows.length} rows`);
+      strapi.log.info(`Mapped sheet to content-type: ${contentTypeName}`);
 
-        if (contentTypeName.startsWith('api::')) {
-          importData[contentTypeName] = unflattenRow(rows, contentTypeName);
-        } else {
-          strapi.log.error(`Unknown content-type: ${contentTypeName}`);
+      if (contentTypeName.startsWith("api::")) {
+        // Validate that the content type exists
+        if (!strapi.contentTypes[contentTypeName]) {
+          strapi.log.error(
+            `Content type ${contentTypeName} not found. Available types:`,
+            Object.keys(strapi.contentTypes)
+          );
           return;
         }
+        importData[contentTypeName] = unflattenRow(rows, contentTypeName);
+      } else {
+        strapi.log.error(`Unknown content-type: ${contentTypeName}`);
+        return;
+      }
     });
 
-    strapi.log.info('Final import data keys:', Object.keys(importData));
+    strapi.log.info("Final import data keys:", Object.keys(importData));
     return importData;
-}
+  },
 
-function getRelationFields(contentType) {
-  const schema = strapi.contentTypes[contentType];
-
-  if (!schema) {
-    strapi.log.warn(`Content type ${contentType} not found`);
-    return [];
-  }
-
-  return Object.entries(schema.attributes)
-    .filter(([_, attr]) => attr.type === "relation")
-    .map(([fieldName, attr]) => ({
-      field: toCamel(fieldName),
-      target: attr.target, // e.g. "api::category.category"
-      relation: attr.relation,
-    }));
-}
+  getRelationFields(contentType) {
+    const schema = strapi.contentTypes[contentType];
 
-function getComponentFields(contentType) {
-  const schema = strapi.contentTypes[contentType];
+    if (!schema) {
+      strapi.log.warn(`Content type ${contentType} not found`);
+      return [];
+    }
 
-  if (!schema) {
-    strapi.log.warn(`Content type ${contentType} not found`);
-    return [];
-  }
+    return Object.entries(schema.attributes)
+      .filter(([_, attr]) => attr.type === "relation")
+      .map(([fieldName, attr]) => ({
+        field: toCamel(fieldName),
+        target: attr.target, // e.g. "api::category.category"
+        relation: attr.relation,
+      }));
+  },
+
+  getComponentFields(contentType) {
+    const schema = strapi.contentTypes[contentType];
+
+    if (!schema) {
+      strapi.log.warn(`Content type ${contentType} not found`);
+      return [];
+    }
 
-  return Object.entries(schema.attributes)
-    .filter(([_, attr]) => attr.type === "component")
-    .map(([fieldName, attr]) => toCamel(fieldName));
-}
+    return Object.entries(schema.attributes)
+      .filter(([_, attr]) => attr.type === "component")
+      .map(([fieldName, attr]) => toCamel(fieldName));
+  },
+
+  async handleRelations(entry, contentType) {
+    const resolveRelationValue = async (field, value, target) => {
+      const targetAttr = strapi.contentTypes[target].attributes;
+      for (const field of SHORTCUT_FIELDS) {
+        if (!targetAttr[field]) continue;
+        const existing = await strapi.documents(target).findFirst({
+          filters: { [field]: { $eq: value } },
+        });
+        if (existing) return { id: existing.id };
+        throw new Error(`Data with ${field} ${value} not found`);
+      }
+      return null;
+    };
 
-async function handleRelations(entry, contentType) {
-  async function resolveRelationValue(field, value, target) {
-    const targetAttr = strapi.contentTypes[target].attributes;
-    for (const field of SHORTCUT_FIELDS) {
-      if (!targetAttr[field]) continue;
-      const existing = await strapi.documents(target).findFirst({
-        filters: { [field]: { $eq: value } },
-      });
-      if (existing) return {id: existing.id};
-      throw new Error(`Data with ${field} ${value} not found`);
-    }
-    return null;
-  }
+    const relationFields = this.getRelationFields(contentType);
+    if (relationFields.length === 0) return entry;
 
-  const relationFields = getRelationFields(contentType);
-  if (relationFields.length === 0) return entry;
+    const updatedEntry = { ...entry };
 
-  const updatedEntry = { ...entry };
+    for (const rel of relationFields) {
+      const { field, target, relation } = rel;
 
-  for (const rel of relationFields) {
-    const { field, target, relation } = rel;
+      let value = entry[field];
+      if (!value || value === "") {
+        if (relation === "manyToMany" || relation === "oneToMany") {
+          updatedEntry[field] = [];
+        } else {
+          updatedEntry[field] = null;
+        }
+        continue;
+      }
 
-    let value = entry[field];
-    if (!value || value === "") {
-      if (relation === "manyToMany" || relation === "oneToMany") {
-        updatedEntry[field] = [];
-      } else {
-        updatedEntry[field] = null;
+      // Convert CSV to array
+      if (
+        typeof value === "string" &&
+        (relation === "manyToMany" || relation === "oneToMany")
+      ) {
+        value = value.split("|");
+      } else if (typeof value === "string" && value.includes("|")) {
+        throw new Error(
+          `Invalid value for field ${field}: ${value}, ${field} is not an array`
+        );
       }
-      continue;
-    };
 
-    // Convert CSV to array
-    if (typeof value === "string" && (relation === "manyToMany" || relation === "oneToMany")) {
-      value = value.split("|");
-    } else if (typeof value === "string" && value.includes("|")) {
-      throw new Error(`Invalid value for field ${field}: ${value}, ${field} is not an array`);
-    }
+      const values = Array.isArray(value) ? value : [value];
+      try {
+        const processed = [];
 
-    const values = Array.isArray(value) ? value : [value];
-    try {
-      const processed = [];
+        for (const v of values) {
+          if (!v || v === "") continue;
+          const resolved = await resolveRelationValue(field, v, target);
+          if (resolved) processed.push(resolved);
+        }
 
-      for (const v of values) {
-        if (!v || v === "") continue;
-        const resolved = await resolveRelationValue(field, v, target);
-        if (resolved) processed.push(resolved);
+        updatedEntry[field] = Array.isArray(value) ? processed : processed[0];
+      } catch (err) {
+        throw new Error(
+          `Failed processing field ${field} with value ${JSON.stringify(value)}: ${err.message}`
+        );
       }
-
-      updatedEntry[field] = Array.isArray(value) ? processed : processed[0];
-    } catch (err) {
-      throw new Error(
-        `Failed processing field ${field} with value ${JSON.stringify(value)}: ${err.message}`
-      );
     }
-  }
-
-  return updatedEntry;
-}
 
-function handleComponents(data, existing, contentType) {
-  // Get the component fields for this content type
-  const compFields = getComponentFields(contentType);
+    return updatedEntry;
+  },
 
-  for (const field of compFields) {
-    const newValue = data[field];
-    const oldValue = existing?.[field];
+  handleComponents(data, existing, contentType) {
+    // Get the component fields for this content type
+    const compFields = this.getComponentFields(contentType);
 
-    if (!newValue || !oldValue) continue;
+    for (const field of compFields) {
+      const newValue = data[field];
+      const oldValue = existing?.[field];
 
-    //single component
-    if (!Array.isArray(newValue)) {
-      if (oldValue?.id) {
-        data[field].id = oldValue.id;
-      }
-      for (const key of Object.keys(data[field])) {
-        if (Array.isArray(oldValue[key])) {
-          data[field][key] = data[field][key].split("|");
-        }
-      }
-      continue;
-    }
+      if (!newValue || !oldValue) continue;
 
-    //multiple components
-    if (Array.isArray(newValue) && Array.isArray(oldValue)) {
-      data[field] = newValue.map((block, i) => {
-        const oldBlock = oldValue[i];
-        if (oldBlock?.id) {
-          return { id: oldBlock.id, ...block };
+      //single component
+      if (!Array.isArray(newValue)) {
+        if (oldValue?.id) {
+          data[field].id = oldValue.id;
         }
-        for (const key of Object.keys(block)) {
-          if (Array.isArray(oldBlock[key])) {
-            block[key] = block[key].split("|");
+        for (const key of Object.keys(data[field])) {
+          if (Array.isArray(oldValue[key])) {
+            data[field][key] = data[field][key].split("|");
           }
         }
-        return block;
-      });
-    }
-  }
-
-  return data;
-}
+        continue;
+      }
 
-function hasChanges(existing, incoming) {
-  if (!incoming || typeof incoming !== "object") return false;
-  if (!existing || typeof existing !== "object") return true;
-  for (const key of Object.keys(incoming)) {
-    // Skip system keys
-    if (SYSTEM_KEYS.includes(key)) continue;
-    const newVal = incoming[key];
-    const oldVal = existing[key];
-
-    // If incoming defines a field but existing doesn't → change
-    if (oldVal === undefined || newVal === undefined) {
-      continue;
+      //multiple components
+      if (Array.isArray(newValue) && Array.isArray(oldValue)) {
+        data[field] = newValue.map((block, i) => {
+          const oldBlock = oldValue[i];
+          if (oldBlock?.id) {
+            return { id: oldBlock.id, ...block };
+          }
+          for (const key of Object.keys(block)) {
+            if (Array.isArray(oldBlock[key])) {
+              block[key] = block[key].split("|");
+            }
+          }
+          return block;
+        });
+      }
     }
 
-    // Primitive comparison
-    if (newVal === null || typeof newVal !== "object") {
-      if (oldVal !== newVal) {
-        return true;
+    return data;
+  },
+
+  sanitizeComponent(data, uid, rowIndex, errors, path) {
+    const schema = strapi.components[uid];
+    if (!schema) return data;
+  
+    const cleaned = {};
+  
+    for (const [key, attr] of Object.entries(schema.attributes)) {
+      const value = data?.[key];
+      const fieldPath = `${path}.${key}`;
+  
+      if (attr.type === 'component') {
+        cleaned[key] = attr.repeatable
+          ? (value || []).map((v, i) =>
+              sanitizeComponent(v, attr.component, rowIndex, errors, `${fieldPath}[${i}]`)
+            )
+          : sanitizeComponent(value, attr.component, rowIndex, errors, fieldPath);
+      } else {
+        cleaned[key] = sanitizePrimitive(value, attr, rowIndex, errors, fieldPath);
       }
-      continue;
     }
-
-    // ARRAY comparison
-    if (Array.isArray(newVal)) {
-      if (!Array.isArray(oldVal)) return true;
-      if (newVal.length !== oldVal.length) return true;
-      // Compare values shallowly
-      for (let i = 0; i < newVal.length; i++) {
-        if (typeof newVal[i] === "object" && typeof oldVal[i] === "object" && hasChanges(oldVal[i], newVal[i])) {
-          return true;
-        } else if (typeof newVal[i] !== "object" && typeof oldVal[i] !== "object" && newVal[i] !== oldVal[i]) {
-          return true;
+  
+    return cleaned;
+  },
+
+  sanitizeEntryBeforeWrite(data, uid, rowIndex, errors, path = '') {
+    const schema = strapi.contentTypes[uid];
+    const cleaned = {};
+  
+    for (const [key, attr] of Object.entries(schema.attributes)) {
+      const value = data[key];
+      const fieldPath = path ? `${path}.${key}` : key;
+  
+      if (value === undefined) continue;
+  
+      if (attr.type === 'component') {
+        if (!value) {
+          cleaned[key] = attr.repeatable ? [] : null;
+          continue;
         }
+  
+        cleaned[key] = attr.repeatable
+          ? value.map((v, i) =>
+              sanitizeComponent(v, attr.component, rowIndex, errors, `${fieldPath}[${i}]`)
+            )
+          : sanitizeComponent(value, attr.component, rowIndex, errors, fieldPath);
+        continue;
       }
-      continue;
+  
+      cleaned[key] = sanitizePrimitive(value, attr, rowIndex, errors, fieldPath);
     }
-
-    // OBJECT comparison (recursive, but ONLY fields in incoming object)
-    if (typeof newVal === "object" && typeof oldVal === "object") {
-      if (hasChanges(oldVal, newVal)) {
-        return true;
+  
+    return cleaned;
+  },
+
+  sanitizePrimitive(value, attr) {
+    if (value === null || value === undefined || value === '') return null;
+    switch (attr.type) {
+      case 'string':
+      case 'text':
+      case 'richtext':
+      case 'email':
+        return String(value).trim();
+      case 'boolean':
+        if ([true, 'true', 1, '1', 'yes', 'y'].includes(value)) return true;
+        if ([false, 'false', 0, '0', 'no', 'n'].includes(value)) return false;
+        return false; // fallback
+      case 'integer':
+      case 'biginteger': {
+        const i = parseInt(value, 10);
+        return Number.isNaN(i) ? 0 : i;
+      }
+      case 'float':
+      case 'decimal': {
+        const f = parseFloat(value);
+        return Number.isNaN(f) ? 0 : f;
+      }
+      case 'date':
+      case 'datetime': {
+        const d = new Date(value);
+        return isNaN(d.getTime()) ? null : d.toISOString();
       }
-      continue;
+      default:
+        return value;
     }
-  }
-
-  return false;
-}
+  },
 
+  async bulkInsertData(importData) {
+    const results = {
+      created: 0,
+      updated: 0,
+      errors: [],
+    };
 
-async function bulkInsertData(importData) {
-  const results = {
-    created: 0,
-    updated: 0,
-    errors: [],
-  };
+    for (const [contentType, entries] of Object.entries(importData)) {
+      // Validate entries
+      if (!strapi.contentTypes[contentType]) {
+        results.errors.push(`Content type ${contentType} not found`);
+        continue;
+      }
+      if (!Array.isArray(entries)) {
+        results.errors.push(`Invalid data format for ${contentType}`);
+        continue;
+      }
 
-  for (const [contentType, entries] of Object.entries(importData)) {
-    // Validate entries
-    if (!strapi.contentTypes[contentType]) {
-      results.errors.push(`Content type ${contentType} not found`);
-      continue;
-    }
-    if (!Array.isArray(entries)) {
-      results.errors.push(`Invalid data format for ${contentType}`);
-      continue;
+      try {
+        const { created, updated, errors } = await this.importEntries(
+          entries,
+          contentType
+        );
+        results.created += created;
+        results.updated += updated;
+        results.errors = results.errors.concat(errors);
+      } catch (err) {
+        results.errors.push(err.message);
+      }
     }
 
-    try {
-      const { created, updated, errors } = await importEntries(entries, contentType);
-      results.created += created;
-      results.updated += updated;
-      results.errors = results.errors.concat(errors);
-    } catch (err) {
-      results.errors.push(err.message);
-    }
-  }
+    return results;
+  },
 
-  return results;
-}
+  async importEntries(entries, contentType) {
+    const results = { created: 0, updated: 0, errors: [] };
 
-async function importEntries(entries, contentType) {
-  const results = { created: 0, updated: 0, errors: [] };
+    await strapi.db.transaction(async ({ trx, rollback, onRollback }) => {
+      onRollback(() => {
+        strapi.log.error("Transaction rolled back due to an error!");
+        strapi.log.error(results.errors);
+      });
 
-  await strapi.db.transaction(async ({ trx, rollback, onRollback }) => {
-    onRollback(() => {
-      strapi.log.error("Transaction rolled back due to an error!");
-      strapi.log.error(results.errors);
-    });
+      for (let i = 0; i < entries.length; i++) {
+        const entry = entries[i];
+        let existing = null;
 
-    for (let i = 0; i < entries.length; i++) {
-      const entry = entries[i];
-      let existing = null;
+        try {
+          let { id, ...data } = entry;
 
-      try {
-        let { id, ...data } = entry;
-
-        // Check if document exists
-        if (id && id !== "null" && id !== "undefined") {
-          existing = await strapi.documents(contentType).findFirst(
-            {
-              filters: { id },
-              populate: "*",
-            },
-            { transaction: trx }
-          );
-        }
+          // Check if document exists
+          if (id && id !== "null" && id !== "undefined") {
+            existing = await strapi.documents(contentType).findFirst(
+              {
+                filters: { id },
+                populate: "*",
+              },
+              { transaction: trx }
+            );
+          }
 
-        // Handle relations & components
-        data = await handleRelations(data, contentType, trx);
-        data = await handleComponents(data, existing, contentType);
+          // Handle relations & components
+          data = await this.handleRelations(data, contentType, trx);
+          data = await this.handleComponents(data, existing, contentType);
+          const sanitizeErrors = [];
+          data = sanitizeEntryBeforeWrite(data, contentType, '', sanitizeErrors);
 
-        // Update
-        if (existing) {
-          if (hasChanges(existing, data)) {
+          if (sanitizeErrors.length) {
+            throw new Error(`Data validation failed:\n${sanitizeErrors.join('\n')}`);
+          }
+
+          // Update
+          if (existing) {
             await strapi.documents(contentType).update(
               {
                 documentId: existing.documentId,
@@ -382,35 +423,29 @@ async function importEntries(entries, contentType) {
             );
             results.updated++;
           }
-        }
 
-        // Create
-        else {
-          await strapi.documents(contentType).create(
-            { data },
-            { transaction: trx }
+          // Create
+          else {
+            await strapi
+              .documents(contentType)
+              .create({ data }, { transaction: trx });
+            results.created++;
+          }
+        } catch (err) {
+          results.errors.push(
+            `Failed ${existing ? "updating" : "creating"} on row ${
+              i + 2
+            }: ${err.message}`
           );
-          results.created++;
-        }
-      } catch (err) {
-        results.errors.push(
-          `Failed ${existing ? "updating" : "creating"} on row ${
-            i + 2
-          }: ${err.message}`
-        );
-        results.created = 0;
-        results.updated = 0;
+          results.created = 0;
+          results.updated = 0;
 
-        // IMPORTANT: force rollback
-        throw err;
+          // IMPORTANT: force rollback
+          throw err;
+        }
       }
-    }
-  });
-
-  return results;
-}
-
+    });
 
-module.exports = {
-  importData,
-};
+    return results;
+  },
+});