@tunghtml/strapi-plugin-export-import-clsx 1.0.2 → 1.1.0

package/server/services/import-service.js

@@ -1,102 +1,106 @@
-const XLSX = require('xlsx');
-const fs = require('fs');
+const XLSX = require("xlsx");
+const fs = require("fs");
 
 function toCamel(str) {
   return str.replace(/_([a-z])/g, (_, c) => c.toUpperCase());
 }
+
 const SYSTEM_KEYS = [
-  'documentId', 
-  'locale', 
-  'createdAt', 
-  'updatedAt', 
-  'publishedAt', 
-  'createdBy', 
-  'updatedBy', 
-  'localizations', 
-  'status'
+  "documentId",
+  "locale",
+  "createdAt",
+  "updatedAt",
+  "publishedAt",
+  "createdBy",
+  "updatedBy",
+  "localizations",
+  "status",
 ];
 
-async function importData(file) {
-  let result;
-  try {
-    let importData;
-    // Check file extension
-    const fileName = file.name || file.originalFilename || 'unknown.json';
-    const fileExtension = fileName.split('.').pop().toLowerCase();
-    const filePath = file.path || file.filepath;
-    if (!filePath) {
-      throw new Error('File path not found');
-    }
+const SHORTCUT_FIELDS = [
+  "email",
+  "businessEmail",
+  "name",
+  "title",
+  "tickerCode",
+];
+module.exports = ({ strapi }) => ({
+  async importData(file, targetContentType = null) {
+    let result;
+    try {
+      let importData;
+      // Check file extension
+      const fileName = file.name || file.originalFilename || "unknown.json";
+      const fileExtension = fileName.split(".").pop().toLowerCase();
+      const filePath = file.path || file.filepath;
+      if (!filePath) {
+        throw new Error("File path not found");
+      }
 
-    if (fileExtension === 'json') {
-      const fileContent = fs.readFileSync(filePath, 'utf8');
-      importData = JSON.parse(fileContent);
-      strapi.log.info('Parsed JSON data:', Object.keys(importData));
-    } else if (fileExtension === 'xlsx' || fileExtension === 'xls') {
-      importData = transformExcelData(filePath);
-    }
-    result = await bulkInsertData(importData);
-    return result;
-  } catch (error) {
-    // Clean up uploaded file on error
-    const filePath = file && (file.path || file.filepath);
-    if (filePath && fs.existsSync(filePath)) {
-      fs.unlinkSync(filePath);
+      if (fileExtension === "json") {
+        const fileContent = fs.readFileSync(filePath, "utf8");
+        importData = JSON.parse(fileContent);
+        strapi.log.info("Parsed JSON data:", Object.keys(importData));
+      } else if (fileExtension === "xlsx" || fileExtension === "xls") {
+        importData = this.transformExcelData(filePath, targetContentType);
+      }
+      result = await this.bulkInsertData(importData);
+      return result;
+    } catch (error) {
+      // Clean up uploaded file on error
+      const filePath = file && (file.path || file.filepath);
+      if (filePath && fs.existsSync(filePath)) {
+        fs.unlinkSync(filePath);
+      }
+      throw error;
     }
-    throw error;
-  }
-}
+  },
 
-
-function transformExcelData(filePath) {
+  transformExcelData(filePath, targetContentType = null) {
     const workbook = XLSX.readFile(filePath);
     const importData = {};
 
     const parseJsonIfNeeded = (value) => {
-        if (typeof value !== 'string') return value;
-        const trimmed = value.trim();
-        if (!trimmed.startsWith('[') && !trimmed.startsWith('{')) return value;
+      if (typeof value !== "string") return value;
+      const trimmed = value.trim();
+      if (!trimmed.startsWith("[") && !trimmed.startsWith("{")) return value;
 
-        try {
+      try {
         return JSON.parse(trimmed);
-        } catch {
+      } catch {
         return value; // keep as string if invalid JSON
-        }
+      }
     };
 
     const isComponentField = (key) => {
-        const parts = key.split('_');
-        return parts.length === 2; // exactly one underscore
+      const parts = key.split("_");
+      return parts.length === 2; // exactly one underscore
     };
 
-    function unflattenRow(rows, targetContentType) {
+    const unflattenRow = (rows, targetContentType) => {
       const result = [];
+      const attr = strapi.contentTypes[targetContentType].attributes;
       for (const row of rows) {
         const rowData = {};
 
         for (const [key, value] of Object.entries(row)) {
-            if (value === null || value === undefined || value === '') {
-              rowData[key] = null
-            }
-
-            if (isComponentField(key)) {
-                const [comp, field] = key.split('_');
-
-                if (!rowData[comp]) rowData[comp] = {};
-                rowData[comp][field] = parseJsonIfNeeded(value);
-
-            } else {
-              rowData[key] = parseJsonIfNeeded(value);
-            }
-        }
-
-        existedComponents = getComponentFields(targetContentType);
-        for (const comp of existedComponents) {
-          if (!rowData[comp]) {
-            rowData[comp] = {};
+          if (value === null || value === undefined || value === "") {
+            rowData[key] = null;
+          } else if (
+            attr[key] &&
+            attr[key].customField &&
+            attr[key].type === "json" &&
+            attr[key].default === "[]"
+          ) {
+            rowData[key] = parseJsonIfNeeded(value).split("|");
+          } else if (isComponentField(key)) {
+            const [comp, field] = key.split("_");
+            if (!rowData[comp]) rowData[comp] = {};
+            rowData[comp][field] = parseJsonIfNeeded(value);
+          } else {
+            rowData[key] = parseJsonIfNeeded(value);
           }
         }
-
         result.push(rowData);
       }
 
@@ -104,255 +108,340 @@ function transformExcelData(filePath) {
     };
 
     const mapSheetNameToContentType = (sheetName) => {
-        if (!sheetName.startsWith('api__')) return sheetName;
-        return sheetName.replace(/^api__/, 'api::').replace(/_/g, '.');
+      // If targetContentType is provided, use it instead of guessing from sheet name
+      if (targetContentType) {
+        return targetContentType;
+      }
+      return "api::" + sheetName + "." + sheetName;
     };
 
-    workbook.SheetNames.forEach(sheetName => {
-        const worksheet = workbook.Sheets[sheetName];
-        const rows = XLSX.utils.sheet_to_json(worksheet);
+    workbook.SheetNames.forEach((sheetName) => {
+      const worksheet = workbook.Sheets[sheetName];
+      const rows = XLSX.utils.sheet_to_json(worksheet);
 
-        if (!rows.length) return;
+      if (!rows.length) return;
 
-        const contentTypeName = mapSheetNameToContentType(sheetName);
+      const contentTypeName = mapSheetNameToContentType(sheetName);
 
-        strapi.log.info(`Reading sheet "${sheetName}" -> ${rows.length} rows`);
-        strapi.log.info(`Mapped sheet to content-type: ${contentTypeName}`);
+      strapi.log.info(`Reading sheet "${sheetName}" -> ${rows.length} rows`);
+      strapi.log.info(`Mapped sheet to content-type: ${contentTypeName}`);
 
-        if (contentTypeName.startsWith('api::')) {
-          importData[contentTypeName] = unflattenRow(rows, contentTypeName);
-        } else {
-          strapi.log.error(`Unknown content-type: ${contentTypeName}`);
+      if (contentTypeName.startsWith("api::")) {
+        // Validate that the content type exists
+        if (!strapi.contentTypes[contentTypeName]) {
+          strapi.log.error(
+            `Content type ${contentTypeName} not found. Available types:`,
+            Object.keys(strapi.contentTypes)
+          );
           return;
         }
+        importData[contentTypeName] = unflattenRow(rows, contentTypeName);
+      } else {
+        strapi.log.error(`Unknown content-type: ${contentTypeName}`);
+        return;
+      }
     });
 
-    strapi.log.info('Final import data keys:', Object.keys(importData));
+    strapi.log.info("Final import data keys:", Object.keys(importData));
     return importData;
-}
+  },
 
-function getRelationFields(contentType) {
-  const schema = strapi.contentTypes[contentType];
-
-  if (!schema) {
-    strapi.log.warn(`Content type ${contentType} not found`);
-    return [];
-  }
-
-  return Object.entries(schema.attributes)
-    .filter(([_, attr]) => attr.type === "relation")
-    .map(([fieldName, attr]) => ({
-      field: toCamel(fieldName),
-      target: attr.target, // e.g. "api::category.category"
-      relation: attr.relation,
-    }));
-}
+  getRelationFields(contentType) {
+    const schema = strapi.contentTypes[contentType];
 
-function getRelationFieldsStrArr(contentType) {
-  const schema = strapi.contentTypes[contentType];
-
-  if (!schema) {
-    strapi.log.warn(`Content type ${contentType} not found`);
-    return [];
-  }
+    if (!schema) {
+      strapi.log.warn(`Content type ${contentType} not found`);
+      return [];
+    }
 
-  return Object.entries(schema.attributes)
-    .filter(([_, attr]) => attr.type === "relation")
-    .map(([fieldName, attr]) => toCamel(fieldName));
-}
+    return Object.entries(schema.attributes)
+      .filter(([_, attr]) => attr.type === "relation")
+      .map(([fieldName, attr]) => ({
+        field: toCamel(fieldName),
+        target: attr.target, // e.g. "api::category.category"
+        relation: attr.relation,
+      }));
+  },
+
+  getComponentFields(contentType) {
+    const schema = strapi.contentTypes[contentType];
+
+    if (!schema) {
+      strapi.log.warn(`Content type ${contentType} not found`);
+      return [];
+    }
 
-function getComponentFields(contentType) {
-  const schema = strapi.contentTypes[contentType];
+    return Object.entries(schema.attributes)
+      .filter(([_, attr]) => attr.type === "component")
+      .map(([fieldName, attr]) => toCamel(fieldName));
+  },
+
+  async handleRelations(entry, contentType) {
+    const resolveRelationValue = async (field, value, target) => {
+      const targetAttr = strapi.contentTypes[target].attributes;
+      for (const field of SHORTCUT_FIELDS) {
+        if (!targetAttr[field]) continue;
+        const existing = await strapi.documents(target).findFirst({
+          filters: { [field]: { $eq: value } },
+        });
+        if (existing) return { id: existing.id };
+        throw new Error(`Data with ${field} ${value} not found`);
+      }
+      return null;
+    };
 
-  if (!schema) {
-    strapi.log.warn(`Content type ${contentType} not found`);
-    return [];
-  }
+    const relationFields = this.getRelationFields(contentType);
+    if (relationFields.length === 0) return entry;
 
-  return Object.entries(schema.attributes)
-    .filter(([_, attr]) => attr.type === "component")
-    .map(([fieldName, attr]) => toCamel(fieldName));
-}
+    const updatedEntry = { ...entry };
 
-async function handleRelations(entry, contentType) {
-  let relationFields = getRelationFields(contentType);
-  if (relationFields.length === 0) {
-    return entry;
-  }
+    for (const rel of relationFields) {
+      const { field, target, relation } = rel;
 
-  let existing = null;
-  const newEntry = { ...entry };
-  let isUpdated = false;
+      let value = entry[field];
+      if (!value || value === "") {
+        if (relation === "manyToMany" || relation === "oneToMany") {
+          updatedEntry[field] = [];
+        } else {
+          updatedEntry[field] = null;
+        }
+        continue;
+      }
 
-  for (const rel of relationFields) {
-    const { field, target } = rel;
-    const relValue = entry[field];
-    try {
-      if (!relValue) continue;
+      // Convert CSV to array
+      if (
+        typeof value === "string" &&
+        (relation === "manyToMany" || relation === "oneToMany")
+      ) {
+        value = value.split("|");
+      } else if (typeof value === "string" && value.includes("|")) {
+        throw new Error(
+          `Invalid value for field ${field}: ${value}, ${field} is not an array`
+        );
+      }
 
-      if (Array.isArray(relValue)) {
+      const values = Array.isArray(value) ? value : [value];
+      try {
         const processed = [];
 
-        for (const item of relValue) {
-          if (item.id) {
-            existing = await strapi.documents(target).findFirst({
-              filters: {
-                id: { $eq: item.id }
-              },
-             });
-            if (existing && hasChanges(existing, item, getRelationFieldsStrArr(target))) {
-              await strapi.documents(target).update({ documentId: existing.documentId, data: item });
-              isUpdated = true;
-            }
-            processed.push({ id: item.id });
-          } else {
-            const created = await strapi.documents(target).create({ data: item });
-            processed.push({ id: created.id });
-          }
+        for (const v of values) {
+          if (!v || v === "") continue;
+          const resolved = await resolveRelationValue(field, v, target);
+          if (resolved) processed.push(resolved);
         }
-        newEntry[field] = processed;
-        continue;
-      }
 
-      if (!relValue.id) {
-        const created = await strapi.documents(target).create({ data: relValue });
-        newEntry[field] = { id: created.id };
-      } else {
-        existing = await strapi.documents(target).findFirst({
-          filters: {
-            id: { $eq: relValue.id }
-          },
-         });
-        if (hasChanges(existing, relValue, getRelationFieldsStrArr(target))) {
-          await strapi.documents(target).update({ documentId: existing.documentId, data: relValue });
-          isUpdated = true;
-        }
-        newEntry[field] = { id: relValue.id };
+        updatedEntry[field] = Array.isArray(value) ? processed : processed[0];
+      } catch (err) {
+        throw new Error(
+          `Failed processing field ${field} with value ${JSON.stringify(value)}: ${err.message}`
+        );
       }
-    } catch (err) {
-      throw new Error(`Field: ${field}, data: ${JSON.stringify(relValue)}, error: ${err.message}`);
     }
-  }
 
-  return [newEntry, isUpdated];
-}
+    return updatedEntry;
+  },
 
-function hasChanges(existing, incoming, relationFieldStrArr = []) {
-  if (!incoming || typeof incoming !== "object") return false;
+  handleComponents(data, existing, contentType) {
+    // Get the component fields for this content type
+    const compFields = this.getComponentFields(contentType);
 
-  for (const key of Object.keys(incoming)) {
-    // Skip system keys
-    if (SYSTEM_KEYS.includes(key)) continue;
+    for (const field of compFields) {
+      const newValue = data[field];
+      const oldValue = existing?.[field];
 
-    // Skip relation fields entirely
-    if (relationFieldStrArr.includes(key)) continue;
+      if (!newValue || !oldValue) continue;
 
-    const newVal = incoming[key];
-    const oldVal = existing[key];
+      //single component
+      if (!Array.isArray(newValue)) {
+        if (oldValue?.id) {
+          data[field].id = oldValue.id;
+        }
+        for (const key of Object.keys(data[field])) {
+          if (Array.isArray(oldValue[key])) {
+            data[field][key] = data[field][key].split("|");
+          }
+        }
+        continue;
+      }
 
-    // If incoming defines a field but existing doesn't → change
-    if (oldVal === undefined || newVal === undefined) {
-      continue;
+      //multiple components
+      if (Array.isArray(newValue) && Array.isArray(oldValue)) {
+        data[field] = newValue.map((block, i) => {
+          const oldBlock = oldValue[i];
+          if (oldBlock?.id) {
+            return { id: oldBlock.id, ...block };
+          }
+          for (const key of Object.keys(block)) {
+            if (Array.isArray(oldBlock[key])) {
+              block[key] = block[key].split("|");
+            }
+          }
+          return block;
+        });
+      }
     }
 
-    // Primitive comparison
-    if (newVal === null || typeof newVal !== "object") {
-      if (oldVal !== newVal) {
-        return true;
+    return data;
+  },
+
+  hasChanges(existing, incoming) {
+    if (!incoming || typeof incoming !== "object") return false;
+    if (!existing || typeof existing !== "object") return true;
+    for (const key of Object.keys(incoming)) {
+      // Skip system keys
+      if (SYSTEM_KEYS.includes(key)) continue;
+      const newVal = incoming[key];
+      const oldVal = existing[key];
+
+      // If incoming defines a field but existing doesn't → change
+      if (oldVal === undefined || newVal === undefined) {
+        continue;
       }
-      continue;
-    }
 
-    // ARRAY comparison
-    if (Array.isArray(newVal)) {
-      if (!Array.isArray(oldVal)) return true;
-      if (newVal.length !== oldVal.length) return true;
-      // Compare values shallowly
-      for (let i = 0; i < newVal.length; i++) {
-        if (typeof newVal[i] === "object" && typeof oldVal[i] === "object" && hasChanges(oldVal[i], newVal[i])) {
+      // Primitive comparison
+      if (newVal === null || typeof newVal !== "object") {
+        if (oldVal !== newVal) {
           return true;
-        } else if (typeof newVal[i] !== "object" && typeof oldVal[i] !== "object" && newVal[i] !== oldVal[i]) {
+        }
+        continue;
+      }
+
+      // ARRAY comparison
+      if (Array.isArray(newVal)) {
+        if (!Array.isArray(oldVal)) return true;
+        if (newVal.length !== oldVal.length) return true;
+        // Compare values shallowly
+        for (let i = 0; i < newVal.length; i++) {
+          if (
+            typeof newVal[i] === "object" &&
+            typeof oldVal[i] === "object" &&
+            this.hasChanges(oldVal[i], newVal[i])
+          ) {
+            return true;
+          } else if (
+            typeof newVal[i] !== "object" &&
+            typeof oldVal[i] !== "object" &&
+            newVal[i] !== oldVal[i]
+          ) {
+            return true;
+          }
+        }
+        continue;
+      }
+
+      // OBJECT comparison (recursive, but ONLY fields in incoming object)
+      if (typeof newVal === "object" && typeof oldVal === "object") {
+        if (this.hasChanges(oldVal, newVal)) {
           return true;
         }
+        continue;
       }
-      continue;
     }
 
-    // OBJECT comparison (recursive, but ONLY fields in incoming object)
-    if (typeof newVal === "object" && typeof oldVal === "object") {
-      if (hasChanges(oldVal, newVal)) {
-        return true;
+    return false;
+  },
+
+  async bulkInsertData(importData) {
+    const results = {
+      created: 0,
+      updated: 0,
+      errors: [],
+    };
+
+    for (const [contentType, entries] of Object.entries(importData)) {
+      // Validate entries
+      if (!strapi.contentTypes[contentType]) {
+        results.errors.push(`Content type ${contentType} not found`);
+        continue;
+      }
+      if (!Array.isArray(entries)) {
+        results.errors.push(`Invalid data format for ${contentType}`);
+        continue;
+      }
+
+      try {
+        const { created, updated, errors } = await this.importEntries(
+          entries,
+          contentType
+        );
+        results.created += created;
+        results.updated += updated;
+        results.errors = results.errors.concat(errors);
+      } catch (err) {
+        results.errors.push(err.message);
       }
-      continue;
     }
-  }
 
-  return false;
-}
+    return results;
+  },
 
+  async importEntries(entries, contentType) {
+    const results = { created: 0, updated: 0, errors: [] };
 
-async function bulkInsertData(importData) {
-  const results = {
-    created: 0,
-    updated: 0,
-    errors: [],
-  };
+    await strapi.db.transaction(async ({ trx, rollback, onRollback }) => {
+      onRollback(() => {
+        strapi.log.error("Transaction rolled back due to an error!");
+        strapi.log.error(results.errors);
+      });
 
-  for (const [contentType, entries] of Object.entries(importData)) {
-    // Validate entries
-    if (!strapi.contentTypes[contentType]) {
-      results.errors.push(`Content type ${contentType} not found`);
-      continue;
-    }
-    if (!Array.isArray(entries)) {
-      results.errors.push(`Invalid data format for ${contentType}`);
-      continue;
-    }
+      for (let i = 0; i < entries.length; i++) {
+        const entry = entries[i];
+        let existing = null;
 
-    for (i = 0; i < entries.length; i++) {
-      const entry = entries[i];
-      let existing = null;
-      try {
-        let { id, ...data } = entry; // keep id out, keep everything else
-        let isUpdated = false;
-        let isCreated = false;
-        if (id && id !== 'null' && id !== 'undefined') {
-          existing = await strapi.documents(contentType).findFirst({
-            filters: {
-              id: { $eq: id }
-            },
-            populate: '*'
-          });
-        }
+        try {
+          let { id, ...data } = entry;
+
+          // Check if document exists
+          if (id && id !== "null" && id !== "undefined") {
+            existing = await strapi.documents(contentType).findFirst(
+              {
+                filters: { id },
+                populate: "*",
+              },
+              { transaction: trx }
+            );
+          }
 
-        [data, isUpdated] = await handleRelations(data, contentType);
+          // Handle relations & components
+          data = await this.handleRelations(data, contentType, trx);
+          data = await this.handleComponents(data, existing, contentType);
+
+          // Update
+          if (existing) {
+            if (this.hasChanges(existing, data)) {
+              await strapi.documents(contentType).update(
+                {
+                  documentId: existing.documentId,
+                  data,
+                },
+                { transaction: trx }
+              );
+              results.updated++;
+            }
+          }
 
-        if (existing) {
-          if (hasChanges(existing, data)) {
-            await strapi.documents(contentType).update({
-              documentId: existing.documentId,
-              data,
-            });
-            isUpdated = true;
+          // Create
+          else {
+            await strapi
+              .documents(contentType)
+              .create({ data }, { transaction: trx });
+            results.created++;
           }
-        } else {
-          await strapi.documents(contentType).create({ data });
-          isCreated = true;
+        } catch (err) {
+          results.errors.push(
+            `Failed ${existing ? "updating" : "creating"} on row ${
+              i + 2
+            }: ${err.message}`
+          );
+          results.created = 0;
+          results.updated = 0;
+
+          // IMPORTANT: force rollback
+          throw err;
         }
-        if (isUpdated) {
-          results.updated++;
-        } else if (isCreated) {
-          results.created++;
-        }
-      } catch (err) {
-        results.errors.push(`Failed ${existing ? 'updating' : 'creating'} on row ${i+2}: ${err.message}`);
       }
-    }
-  }
-
-  return results;
-}
+    });
 
-module.exports = {
-  importData,
-};
+    return results;
+  },
+});