@tunghtml/strapi-plugin-export-import-clsx 1.0.1 → 1.1.0

package/server/services/import-service.js

@@ -1,306 +1,447 @@
-const fs = require('fs');
-const XLSX = require('xlsx');
+const XLSX = require("xlsx");
+const fs = require("fs");
 
+function toCamel(str) {
+  return str.replace(/_([a-z])/g, (_, c) => c.toUpperCase());
+}
+
+const SYSTEM_KEYS = [
+  "documentId",
+  "locale",
+  "createdAt",
+  "updatedAt",
+  "publishedAt",
+  "createdBy",
+  "updatedBy",
+  "localizations",
+  "status",
+];
+
+const SHORTCUT_FIELDS = [
+  "email",
+  "businessEmail",
+  "name",
+  "title",
+  "tickerCode",
+];
 module.exports = ({ strapi }) => ({
   async importData(file, targetContentType = null) {
+    let result;
     try {
       let importData;
-      
       // Check file extension
-      const fileName = file.name || file.originalFilename || 'unknown.json';
-      const fileExtension = fileName.split('.').pop().toLowerCase();
-      
+      const fileName = file.name || file.originalFilename || "unknown.json";
+      const fileExtension = fileName.split(".").pop().toLowerCase();
       const filePath = file.path || file.filepath;
-      
       if (!filePath) {
-        throw new Error('File path not found');
+        throw new Error("File path not found");
       }
 
-      if (fileExtension === 'json') {
-        const fileContent = fs.readFileSync(filePath, 'utf8');
+      if (fileExtension === "json") {
+        const fileContent = fs.readFileSync(filePath, "utf8");
         importData = JSON.parse(fileContent);
-        strapi.log.info('Parsed JSON data:', Object.keys(importData));
-      } else if (fileExtension === 'xlsx' || fileExtension === 'xls') {
-        const workbook = XLSX.readFile(filePath);
-        importData = {};
-        
-        strapi.log.info('Excel sheet names:', workbook.SheetNames);
-        
-        // Convert each sheet to data
-        workbook.SheetNames.forEach(sheetName => {
-          const worksheet = workbook.Sheets[sheetName];
-          const jsonData = XLSX.utils.sheet_to_json(worksheet);
-          strapi.log.info(`Sheet ${sheetName} has ${jsonData.length} rows`);
-          if (jsonData.length > 0) {
-            // Map sheet name back to content type
-            let contentTypeName = sheetName;
-            if (sheetName.startsWith('api__')) {
-              // Convert api__corporate_corporate to api::corporate.corporate
-              contentTypeName = sheetName.replace(/^api__/, 'api::').replace(/_/g, '.');
-            }
-            
-            // Unflatten Excel data back to nested objects
-            const unflattened = jsonData.map((row, index) => {
-              const result = {};
-              
-              // Known component prefixes that should be unflattened
-              const componentPrefixes = ['corporateInfo', 'meetingRequirements', 'internalInfo'];
-              
-              for (const [key, value] of Object.entries(row)) {
-                // Skip completely empty values but keep 0, false, etc.
-                if (value === null || value === undefined || value === '') {
-                  continue;
-                }
-                
-                // Check if this key should be unflattened
-                const shouldUnflatten = key.includes('_') && 
-                  !['createdAt', 'updatedAt', 'publishedAt'].includes(key) &&
-                  componentPrefixes.some(prefix => key.startsWith(prefix + '_'));
-                
-                if (shouldUnflatten) {
-                  // Handle nested objects like corporateInfo_companyName
-                  const parts = key.split('_');
-                  let current = result;
-                  
-                  for (let i = 0; i < parts.length - 1; i++) {
-                    if (!current[parts[i]]) {
-                      current[parts[i]] = {};
-                    }
-                    current = current[parts[i]];
-                  }
-                  
-                  current[parts[parts.length - 1]] = value;
-                } else {
-                  // Handle arrays/JSON strings
-                  if (typeof value === 'string' && (value.startsWith('[') || value.startsWith('{'))) {
-                    try {
-                      result[key] = JSON.parse(value);
-                    } catch (error) {
-                      result[key] = value; // Keep as string if parsing fails
-                    }
-                  } else {
-                    result[key] = value;
-                  }
-                }
-              }
-              
-              // Debug info removed for cleaner logs
-              
-              return result;
-            });
-            
-            importData[contentTypeName] = unflattened;
-            strapi.log.info(`Mapped sheet ${sheetName} to ${contentTypeName}`);
+        strapi.log.info("Parsed JSON data:", Object.keys(importData));
+      } else if (fileExtension === "xlsx" || fileExtension === "xls") {
+        importData = this.transformExcelData(filePath, targetContentType);
+      }
+      result = await this.bulkInsertData(importData);
+      return result;
+    } catch (error) {
+      // Clean up uploaded file on error
+      const filePath = file && (file.path || file.filepath);
+      if (filePath && fs.existsSync(filePath)) {
+        fs.unlinkSync(filePath);
+      }
+      throw error;
+    }
+  },
+
+  transformExcelData(filePath, targetContentType = null) {
+    const workbook = XLSX.readFile(filePath);
+    const importData = {};
+
+    const parseJsonIfNeeded = (value) => {
+      if (typeof value !== "string") return value;
+      const trimmed = value.trim();
+      if (!trimmed.startsWith("[") && !trimmed.startsWith("{")) return value;
+
+      try {
+        return JSON.parse(trimmed);
+      } catch {
+        return value; // keep as string if invalid JSON
+      }
+    };
+
+    const isComponentField = (key) => {
+      const parts = key.split("_");
+      return parts.length === 2; // exactly one underscore
+    };
+
+    const unflattenRow = (rows, targetContentType) => {
+      const result = [];
+      const attr = strapi.contentTypes[targetContentType].attributes;
+      for (const row of rows) {
+        const rowData = {};
+
+        for (const [key, value] of Object.entries(row)) {
+          if (value === null || value === undefined || value === "") {
+            rowData[key] = null;
+          } else if (
+            attr[key] &&
+            attr[key].customField &&
+            attr[key].type === "json" &&
+            attr[key].default === "[]"
+          ) {
+            rowData[key] = parseJsonIfNeeded(value).split("|");
+          } else if (isComponentField(key)) {
+            const [comp, field] = key.split("_");
+            if (!rowData[comp]) rowData[comp] = {};
+            rowData[comp][field] = parseJsonIfNeeded(value);
+          } else {
+            rowData[key] = parseJsonIfNeeded(value);
           }
-        });
-        
-        strapi.log.info('Final import data keys:', Object.keys(importData));
+        }
+        result.push(rowData);
+      }
+
+      return result;
+    };
+
+    const mapSheetNameToContentType = (sheetName) => {
+      // If targetContentType is provided, use it instead of guessing from sheet name
+      if (targetContentType) {
+        return targetContentType;
+      }
+      return "api::" + sheetName + "." + sheetName;
+    };
+
+    workbook.SheetNames.forEach((sheetName) => {
+      const worksheet = workbook.Sheets[sheetName];
+      const rows = XLSX.utils.sheet_to_json(worksheet);
+
+      if (!rows.length) return;
+
+      const contentTypeName = mapSheetNameToContentType(sheetName);
+
+      strapi.log.info(`Reading sheet "${sheetName}" -> ${rows.length} rows`);
+      strapi.log.info(`Mapped sheet to content-type: ${contentTypeName}`);
+
+      if (contentTypeName.startsWith("api::")) {
+        // Validate that the content type exists
+        if (!strapi.contentTypes[contentTypeName]) {
+          strapi.log.error(
+            `Content type ${contentTypeName} not found. Available types:`,
+            Object.keys(strapi.contentTypes)
+          );
+          return;
+        }
+        importData[contentTypeName] = unflattenRow(rows, contentTypeName);
       } else {
-        throw new Error('Unsupported file format. Please use JSON or Excel files.');
+        strapi.log.error(`Unknown content-type: ${contentTypeName}`);
+        return;
       }
+    });
 
-      const results = {
-        created: 0,
-        updated: 0,
-        errors: [],
-      };
+    strapi.log.info("Final import data keys:", Object.keys(importData));
+    return importData;
+  },
 
-      // Handle different data structures
-      const dataToProcess = importData.data || importData;
+  getRelationFields(contentType) {
+    const schema = strapi.contentTypes[contentType];
 
-      for (const [contentType, entries] of Object.entries(dataToProcess)) {
-        try {
-          // If targetContentType is specified, only process that content type
-          if (targetContentType && contentType !== targetContentType) {
-            continue;
-          }
+    if (!schema) {
+      strapi.log.warn(`Content type ${contentType} not found`);
+      return [];
+    }
 
-          // Skip if not an API content type
-          if (!contentType.startsWith('api::')) {
-            continue;
-          }
+    return Object.entries(schema.attributes)
+      .filter(([_, attr]) => attr.type === "relation")
+      .map(([fieldName, attr]) => ({
+        field: toCamel(fieldName),
+        target: attr.target, // e.g. "api::category.category"
+        relation: attr.relation,
+      }));
+  },
 
-          // Check if content type exists
-          if (!strapi.contentTypes[contentType]) {
-            results.errors.push(`Content type ${contentType} not found`);
-            continue;
-          }
+  getComponentFields(contentType) {
+    const schema = strapi.contentTypes[contentType];
 
-          if (!Array.isArray(entries)) {
-            results.errors.push(`Invalid data format for ${contentType}`);
-            continue;
-          }
+    if (!schema) {
+      strapi.log.warn(`Content type ${contentType} not found`);
+      return [];
+    }
 
-          for (const entry of entries) {
-            try {
-              // Extract system fields for update/create logic
-              const { 
-                id, 
-                documentId, 
-                status,
-                createdAt, 
-                updatedAt, 
-                publishedAt, 
-                createdBy, 
-                updatedBy, 
-                localizations,
-                locale,
-                ...cleanEntry 
-              } = entry;
-
-              // Skip empty entries
-              if (!cleanEntry || Object.keys(cleanEntry).length === 0) {
-                strapi.log.warn('Skipping empty entry');
-                continue;
-              }
-              
-              // Clean up empty string values and convert to null
-              for (const [key, value] of Object.entries(cleanEntry)) {
-                if (value === '' || value === 'null' || value === 'undefined') {
-                  cleanEntry[key] = null;
-                }
-              }
-              
-              let existingEntry = null;
-              let updateMode = false;
-              
-              // Only try to find existing entry if documentId is provided and valid
-              if (documentId && documentId !== '' && documentId !== 'null' && documentId !== 'undefined') {
-                try {
-                  if (strapi.documents) {
-                    existingEntry = await strapi.documents(contentType).findOne({
-                      documentId: documentId,
-                    });
-                    if (existingEntry) {
-                      updateMode = true;
-                      strapi.log.info(`Found existing entry for update: ${documentId}`);
-                    }
-                  }
-                } catch (error) {
-                  // Entry not found, will create new one
-                  strapi.log.info(`DocumentId ${documentId} not found, will create new entry`);
-                }
-              }
-              
-              // If no documentId provided or not found, this will be a new entry
-              if (!existingEntry) {
-                strapi.log.info(`Creating new entry for: ${cleanEntry.shortName || cleanEntry.name || cleanEntry.title || 'Unknown'}`);
-              }
-              
-              // Skip entries without basic required fields
-              if (!cleanEntry.shortName && !cleanEntry.name && !cleanEntry.title) {
-                continue;
-              }
-              
-              // Ensure required components exist for corporate
-              if (contentType === 'api::corporate.corporate') {
-                if (!cleanEntry.corporateInfo) cleanEntry.corporateInfo = {};
-                if (!cleanEntry.meetingRequirements) cleanEntry.meetingRequirements = {};
-                if (!cleanEntry.internalInfo) cleanEntry.internalInfo = {};
-              }
-              
-              if (existingEntry) {
-                // Check if there are actual changes before updating
-                const hasChanges = this.hasDataChanges(existingEntry, cleanEntry);
-                const statusChanged = (status === 'published') !== !!existingEntry.publishedAt;
-                
-                if (hasChanges || statusChanged) {
-                  try {
-                    if (strapi.documents) {
-                      const updateData = {
-                        documentId: existingEntry.documentId,
-                        data: cleanEntry,
-                      };
-                      
-                      // Handle status change
-                      if (statusChanged) {
-                        updateData.status = status === 'published' ? 'published' : 'draft';
-                      }
-                      
-                      await strapi.documents(contentType).update(updateData);
-                      results.updated++;
-                    } else {
-                      await strapi.entityService.update(contentType, existingEntry.id, {
-                        data: cleanEntry,
-                      });
-                      results.updated++;
-                    }
-                  } catch (updateError) {
-                    results.errors.push(`Failed to update entry: ${updateError.message}`);
-                  }
-                } else {
-                  // No changes, skip update
-                  strapi.log.info(`No changes detected for entry: ${cleanEntry.shortName || 'Unknown'}`);
-                }
-              } else {
-                // Create new entry
-                try {
-                  if (strapi.documents) {
-                    await strapi.documents(contentType).create({
-                      data: cleanEntry,
-                      status: status === 'published' ? 'published' : 'draft',
-                    });
-                    results.created++;
-                  } else {
-                    await strapi.entityService.create(contentType, {
-                      data: cleanEntry,
-                    });
-                    results.created++;
-                  }
-                } catch (createError) {
-                  results.errors.push(`Failed to create entry: ${createError.message}`);
-                }
-              }
-            } catch (error) {
-              results.errors.push(`Failed to import entry in ${contentType}: ${error.message}`);
-              strapi.log.error('Import entry error:', error);
-            }
-          }
-        } catch (error) {
-          results.errors.push(`Failed to process ${contentType}: ${error.message}`);
-          strapi.log.error('Import process error:', error);
+    return Object.entries(schema.attributes)
+      .filter(([_, attr]) => attr.type === "component")
+      .map(([fieldName, attr]) => toCamel(fieldName));
+  },
+
+  async handleRelations(entry, contentType) {
+    const resolveRelationValue = async (field, value, target) => {
+      const targetAttr = strapi.contentTypes[target].attributes;
+      for (const field of SHORTCUT_FIELDS) {
+        if (!targetAttr[field]) continue;
+        const existing = await strapi.documents(target).findFirst({
+          filters: { [field]: { $eq: value } },
+        });
+        if (existing) return { id: existing.id };
+        throw new Error(`Data with ${field} ${value} not found`);
+      }
+      return null;
+    };
+
+    const relationFields = this.getRelationFields(contentType);
+    if (relationFields.length === 0) return entry;
+
+    const updatedEntry = { ...entry };
+
+    for (const rel of relationFields) {
+      const { field, target, relation } = rel;
+
+      let value = entry[field];
+      if (!value || value === "") {
+        if (relation === "manyToMany" || relation === "oneToMany") {
+          updatedEntry[field] = [];
+        } else {
+          updatedEntry[field] = null;
         }
+        continue;
       }
 
-      // Clean up uploaded file
-      if (filePath && fs.existsSync(filePath)) {
-        fs.unlinkSync(filePath);
+      // Convert CSV to array
+      if (
+        typeof value === "string" &&
+        (relation === "manyToMany" || relation === "oneToMany")
+      ) {
+        value = value.split("|");
+      } else if (typeof value === "string" && value.includes("|")) {
+        throw new Error(
+          `Invalid value for field ${field}: ${value}, ${field} is not an array`
+        );
       }
 
-      return results;
-    } catch (error) {
-      // Clean up uploaded file on error
-      const filePath = file && (file.path || file.filepath);
-      if (filePath && fs.existsSync(filePath)) {
-        fs.unlinkSync(filePath);
+      const values = Array.isArray(value) ? value : [value];
+      try {
+        const processed = [];
+
+        for (const v of values) {
+          if (!v || v === "") continue;
+          const resolved = await resolveRelationValue(field, v, target);
+          if (resolved) processed.push(resolved);
+        }
+
+        updatedEntry[field] = Array.isArray(value) ? processed : processed[0];
+      } catch (err) {
+        throw new Error(
+          `Failed processing field ${field} with value ${JSON.stringify(value)}: ${err.message}`
+        );
       }
-      throw error;
     }
+
+    return updatedEntry;
   },
 
-  hasDataChanges(existingEntry, newData) {
-    // Compare key fields to detect changes
-    const fieldsToCompare = ['shortName', 'name', 'title'];
-    
-    for (const field of fieldsToCompare) {
-      if (existingEntry[field] !== newData[field]) {
-        return true;
+  handleComponents(data, existing, contentType) {
+    // Get the component fields for this content type
+    const compFields = this.getComponentFields(contentType);
+
+    for (const field of compFields) {
+      const newValue = data[field];
+      const oldValue = existing?.[field];
+
+      if (!newValue || !oldValue) continue;
+
+      //single component
+      if (!Array.isArray(newValue)) {
+        if (oldValue?.id) {
+          data[field].id = oldValue.id;
+        }
+        for (const key of Object.keys(data[field])) {
+          if (Array.isArray(oldValue[key])) {
+            data[field][key] = data[field][key].split("|");
+          }
+        }
+        continue;
+      }
+
+      //multiple components
+      if (Array.isArray(newValue) && Array.isArray(oldValue)) {
+        data[field] = newValue.map((block, i) => {
+          const oldBlock = oldValue[i];
+          if (oldBlock?.id) {
+            return { id: oldBlock.id, ...block };
+          }
+          for (const key of Object.keys(block)) {
+            if (Array.isArray(oldBlock[key])) {
+              block[key] = block[key].split("|");
+            }
+          }
+          return block;
+        });
       }
     }
-    
-    // Compare nested objects (components)
-    const componentsToCompare = ['corporateInfo', 'meetingRequirements', 'internalInfo'];
-    
-    for (const component of componentsToCompare) {
-      if (existingEntry[component] && newData[component]) {
-        const existingStr = JSON.stringify(existingEntry[component]);
-        const newStr = JSON.stringify(newData[component]);
-        if (existingStr !== newStr) {
+
+    return data;
+  },
+
+  hasChanges(existing, incoming) {
+    if (!incoming || typeof incoming !== "object") return false;
+    if (!existing || typeof existing !== "object") return true;
+    for (const key of Object.keys(incoming)) {
+      // Skip system keys
+      if (SYSTEM_KEYS.includes(key)) continue;
+      const newVal = incoming[key];
+      const oldVal = existing[key];
+
+      // If incoming defines a field but existing doesn't → change
+      if (oldVal === undefined || newVal === undefined) {
+        continue;
+      }
+
+      // Primitive comparison
+      if (newVal === null || typeof newVal !== "object") {
+        if (oldVal !== newVal) {
+          return true;
+        }
+        continue;
+      }
+
+      // ARRAY comparison
+      if (Array.isArray(newVal)) {
+        if (!Array.isArray(oldVal)) return true;
+        if (newVal.length !== oldVal.length) return true;
+        // Compare values shallowly
+        for (let i = 0; i < newVal.length; i++) {
+          if (
+            typeof newVal[i] === "object" &&
+            typeof oldVal[i] === "object" &&
+            this.hasChanges(oldVal[i], newVal[i])
+          ) {
+            return true;
+          } else if (
+            typeof newVal[i] !== "object" &&
+            typeof oldVal[i] !== "object" &&
+            newVal[i] !== oldVal[i]
+          ) {
+            return true;
+          }
+        }
+        continue;
+      }
+
+      // OBJECT comparison (recursive, but ONLY fields in incoming object)
+      if (typeof newVal === "object" && typeof oldVal === "object") {
+        if (this.hasChanges(oldVal, newVal)) {
           return true;
         }
+        continue;
       }
     }
-    
+
     return false;
   },
-});
+
+  async bulkInsertData(importData) {
+    const results = {
+      created: 0,
+      updated: 0,
+      errors: [],
+    };
+
+    for (const [contentType, entries] of Object.entries(importData)) {
+      // Validate entries
+      if (!strapi.contentTypes[contentType]) {
+        results.errors.push(`Content type ${contentType} not found`);
+        continue;
+      }
+      if (!Array.isArray(entries)) {
+        results.errors.push(`Invalid data format for ${contentType}`);
+        continue;
+      }
+
+      try {
+        const { created, updated, errors } = await this.importEntries(
+          entries,
+          contentType
+        );
+        results.created += created;
+        results.updated += updated;
+        results.errors = results.errors.concat(errors);
+      } catch (err) {
+        results.errors.push(err.message);
+      }
+    }
+
+    return results;
+  },
+
+  async importEntries(entries, contentType) {
+    const results = { created: 0, updated: 0, errors: [] };
+
+    await strapi.db.transaction(async ({ trx, rollback, onRollback }) => {
+      onRollback(() => {
+        strapi.log.error("Transaction rolled back due to an error!");
+        strapi.log.error(results.errors);
+      });
+
+      for (let i = 0; i < entries.length; i++) {
+        const entry = entries[i];
+        let existing = null;
+
+        try {
+          let { id, ...data } = entry;
+
+          // Check if document exists
+          if (id && id !== "null" && id !== "undefined") {
+            existing = await strapi.documents(contentType).findFirst(
+              {
+                filters: { id },
+                populate: "*",
+              },
+              { transaction: trx }
+            );
+          }
+
+          // Handle relations & components
+          data = await this.handleRelations(data, contentType, trx);
+          data = await this.handleComponents(data, existing, contentType);
+
+          // Update
+          if (existing) {
+            if (this.hasChanges(existing, data)) {
+              await strapi.documents(contentType).update(
+                {
+                  documentId: existing.documentId,
+                  data,
+                },
+                { transaction: trx }
+              );
+              results.updated++;
+            }
+          }
+
+          // Create
+          else {
+            await strapi
+              .documents(contentType)
+              .create({ data }, { transaction: trx });
+            results.created++;
+          }
+        } catch (err) {
+          results.errors.push(
+            `Failed ${existing ? "updating" : "creating"} on row ${
+              i + 2
+            }: ${err.message}`
+          );
+          results.created = 0;
+          results.updated = 0;
+
+          // IMPORTANT: force rollback
+          throw err;
+        }
+      }
+    });
+
+    return results;
+  },
+});