lula2 0.5.0 → 0.5.1-nightly.0

package/dist/cli/server/spreadsheetRoutes.js

@@ -90,58 +90,81 @@ async function scanControlSets() {
   }).filter((cs) => cs !== null);
   return { controlSets };
 }
+function processImportParameters(reqBody) {
+  const {
+    controlIdField = "Control ID",
+    startRow = "1",
+    controlSetName = "Imported Control Set",
+    controlSetDescription = "Imported from spreadsheet"
+  } = reqBody;
+  let justificationFields = [];
+  if (reqBody.justificationFields) {
+    try {
+      justificationFields = JSON.parse(reqBody.justificationFields);
+      debug("Justification fields received:", justificationFields);
+    } catch (e) {
+      console.error("Failed to parse justification fields:", e);
+    }
+  }
+  let frontendFieldSchema = null;
+  if (reqBody.fieldSchema) {
+    try {
+      frontendFieldSchema = JSON.parse(reqBody.fieldSchema);
+    } catch (e) {
+      console.error("Failed to parse fieldSchema:", e);
+    }
+  }
+  debug("Import parameters received:", {
+    controlIdField,
+    startRow,
+    controlSetName,
+    controlSetDescription
+  });
+  return {
+    controlIdField,
+    startRow,
+    controlSetName,
+    controlSetDescription,
+    justificationFields,
+    namingConvention: "kebab-case",
+    skipEmpty: true,
+    skipEmptyRows: true,
+    frontendFieldSchema
+  };
+}
+async function parseUploadedFile(file) {
+  const fileName = file.originalname || "";
+  const isCSV = fileName.toLowerCase().endsWith(".csv");
+  let rawData = [];
+  if (isCSV) {
+    const csvContent = file.buffer.toString("utf-8");
+    rawData = parseCSV(csvContent);
+  } else {
+    const workbook = new ExcelJS.Workbook();
+    const buffer = Buffer.from(file.buffer);
+    await workbook.xlsx.load(buffer);
+    const worksheet = workbook.worksheets[0];
+    if (!worksheet) {
+      throw new Error("No worksheet found in file");
+    }
+    worksheet.eachRow({ includeEmpty: true }, (row, rowNumber) => {
+      const rowData = [];
+      row.eachCell({ includeEmpty: true }, (cell, colNumber) => {
+        rowData[colNumber - 1] = cell.value;
+      });
+      rawData[rowNumber - 1] = rowData;
+    });
+  }
+  return rawData;
+}
 router.post("/import-spreadsheet", upload.single("file"), async (req, res) => {
   try {
     if (!req.file) {
       return res.status(400).json({ error: "No file uploaded" });
     }
-    const {
-      controlIdField = "Control ID",
-      startRow = "1",
-      controlSetName = "Imported Control Set",
-      controlSetDescription = "Imported from spreadsheet"
-    } = req.body;
-    let justificationFields = [];
-    if (req.body.justificationFields) {
-      try {
-        justificationFields = JSON.parse(req.body.justificationFields);
-        debug("Justification fields received:", justificationFields);
-      } catch (e) {
-        console.error("Failed to parse justification fields:", e);
-      }
-    }
-    debug("Import parameters received:", {
-      controlIdField,
-      startRow,
-      controlSetName,
-      controlSetDescription
-    });
-    const namingConvention = "kebab-case";
-    const skipEmpty = true;
-    const skipEmptyRows = true;
-    const fileName = req.file.originalname || "";
-    const isCSV = fileName.toLowerCase().endsWith(".csv");
-    let rawData = [];
-    if (isCSV) {
-      const csvContent = req.file.buffer.toString("utf-8");
-      rawData = parseCSV(csvContent);
-    } else {
-      const workbook = new ExcelJS.Workbook();
-      const buffer = Buffer.from(req.file.buffer);
-      await workbook.xlsx.load(buffer);
-      const worksheet = workbook.worksheets[0];
-      if (!worksheet) {
-        return res.status(400).json({ error: "No worksheet found in file" });
-      }
-      worksheet.eachRow({ includeEmpty: true }, (row, rowNumber) => {
-        const rowData = [];
-        row.eachCell({ includeEmpty: true }, (cell, colNumber) => {
-          rowData[colNumber - 1] = cell.value;
-        });
-        rawData[rowNumber - 1] = rowData;
-      });
-    }
-    const startRowIndex = parseInt(startRow) - 1;
+    const params = processImportParameters(req.body);
+    const rawData = await parseUploadedFile(req.file);
+    const startRowIndex = parseInt(params.startRow) - 1;
     if (rawData.length <= startRowIndex) {
       return res.status(400).json({ error: "Start row exceeds sheet data" });
     }
@@ -152,275 +175,288 @@ router.post("/import-spreadsheet", upload.single("file"), async (req, res) => {
     debug("Headers found:", headers);
     debug(
       "After conversion, looking for control ID field:",
-      applyNamingConvention(controlIdField, namingConvention)
+      applyNamingConvention(params.controlIdField, params.namingConvention)
     );
-    const controls = [];
-    const families = /* @__PURE__ */ new Map();
-    const fieldMetadata = /* @__PURE__ */ new Map();
-    headers.forEach((header) => {
-      if (header) {
-        const cleanName = applyNamingConvention(header, namingConvention);
-        fieldMetadata.set(cleanName, {
-          originalName: header,
-          cleanName,
-          type: "string",
-          maxLength: 0,
-          hasMultipleLines: false,
-          uniqueValues: /* @__PURE__ */ new Set(),
-          emptyCount: 0,
-          totalCount: 0,
-          examples: []
-        });
-      }
+    const processedData = processSpreadsheetData(rawData, headers, startRowIndex, params);
+    const fieldSchema = buildFieldSchema(
+      processedData.fieldMetadata,
+      processedData.controls,
+      params,
+      processedData.families
+    );
+    const result = await createOutputStructure(processedData, fieldSchema, params);
+    res.json({
+      success: true,
+      controlCount: processedData.controls.length,
+      families: Array.from(processedData.families.keys()),
+      outputDir: result.folderName
     });
-    for (let i = startRowIndex + 1; i < rawData.length; i++) {
-      const row = rawData[i];
-      if (!row || row.length === 0) continue;
-      const control = {};
-      let hasData = false;
-      headers.forEach((header, index) => {
-        if (header && row[index] !== void 0 && row[index] !== null) {
-          const value = typeof row[index] === "string" ? row[index].trim() : row[index];
-          const fieldName = applyNamingConvention(header, namingConvention);
-          const metadata = fieldMetadata.get(fieldName);
-          metadata.totalCount++;
-          if (value === "" || value === null || value === void 0) {
-            metadata.emptyCount++;
-            if (skipEmpty) return;
-          } else {
-            const normalizedValue = typeof value === "string" ? value.trim() : value;
-            if (normalizedValue !== "") {
-              metadata.uniqueValues.add(normalizedValue);
-            }
-            const valueType = detectValueType(value);
-            if (metadata.type === "string" || metadata.totalCount === 1) {
-              metadata.type = valueType;
-            } else if (metadata.type !== valueType) {
-              metadata.type = "mixed";
-            }
-            if (typeof value === "string") {
-              const length = value.length;
-              if (length > metadata.maxLength) {
-                metadata.maxLength = length;
-              }
-              if (value.includes("\n") || length > 100) {
-                metadata.hasMultipleLines = true;
-              }
+  } catch (error) {
+    console.error("Error processing spreadsheet:", error);
+    res.status(500).json({ error: "Failed to process spreadsheet" });
+  }
+});
+function processSpreadsheetData(rawData, headers, startRowIndex, params) {
+  const controls = [];
+  const families = /* @__PURE__ */ new Map();
+  const fieldMetadata = /* @__PURE__ */ new Map();
+  headers.forEach((header) => {
+    if (header) {
+      const cleanName = applyNamingConvention(header, params.namingConvention);
+      fieldMetadata.set(cleanName, {
+        originalName: header,
+        cleanName,
+        type: "string",
+        maxLength: 0,
+        hasMultipleLines: false,
+        uniqueValues: /* @__PURE__ */ new Set(),
+        emptyCount: 0,
+        totalCount: 0,
+        examples: []
+      });
+    }
+  });
+  for (let i = startRowIndex + 1; i < rawData.length; i++) {
+    const row = rawData[i];
+    if (!row || row.length === 0) continue;
+    const control = {};
+    let hasData = false;
+    headers.forEach((header, index) => {
+      if (header && row[index] !== void 0 && row[index] !== null) {
+        const value = typeof row[index] === "string" ? row[index].trim() : row[index];
+        const fieldName = applyNamingConvention(header, params.namingConvention);
+        const metadata = fieldMetadata.get(fieldName);
+        metadata.totalCount++;
+        if (value === "" || value === null || value === void 0) {
+          metadata.emptyCount++;
+          if (params.skipEmpty) return;
+        } else {
+          const normalizedValue = typeof value === "string" ? value.trim() : value;
+          if (normalizedValue !== "") {
+            metadata.uniqueValues.add(normalizedValue);
+          }
+          const valueType = detectValueType(value);
+          if (metadata.type === "string" || metadata.totalCount === 1) {
+            metadata.type = valueType;
+          } else if (metadata.type !== valueType) {
+            metadata.type = "mixed";
+          }
+          if (typeof value === "string") {
+            const length = value.length;
+            if (length > metadata.maxLength) {
+              metadata.maxLength = length;
             }
-            if (metadata.examples.length < 3 && normalizedValue !== "") {
-              metadata.examples.push(normalizedValue);
+            if (value.includes("\n") || length > 100) {
+              metadata.hasMultipleLines = true;
             }
           }
-          control[fieldName] = value;
-          hasData = true;
-        }
-      });
-      if (hasData && (!skipEmptyRows || Object.keys(control).length > 0)) {
-        const controlIdFieldName = applyNamingConvention(controlIdField, namingConvention);
-        const controlId = control[controlIdFieldName];
-        if (!controlId) {
-          continue;
-        }
-        const family = extractFamilyFromControlId(controlId);
-        control.family = family;
-        controls.push(control);
-        if (!families.has(family)) {
-          families.set(family, []);
+          if (metadata.examples.length < 3 && normalizedValue !== "") {
+            metadata.examples.push(normalizedValue);
+          }
         }
-        families.get(family).push(control);
+        control[fieldName] = value;
+        hasData = true;
       }
+    });
+    if (hasData && (!params.skipEmptyRows || Object.keys(control).length > 0)) {
+      const controlIdFieldName = applyNamingConvention(
+        params.controlIdField,
+        params.namingConvention
+      );
+      const controlId = control[controlIdFieldName];
+      if (!controlId) {
+        continue;
+      }
+      const family = extractFamilyFromControlId(controlId);
+      control.family = family;
+      controls.push(control);
+      if (!families.has(family)) {
+        families.set(family, []);
+      }
+      families.get(family).push(control);
     }
-    const state = getServerState();
-    const folderName = toKebabCase(controlSetName || "imported-controls");
-    const baseDir = join2(state.CONTROL_SET_DIR || process.cwd(), folderName);
-    if (!existsSync(baseDir)) {
-      mkdirSync(baseDir, { recursive: true });
+  }
+  return { controls, families, fieldMetadata };
+}
+function buildFieldSchema(fieldMetadata, controls, params, families) {
+  const fields = {};
+  let displayOrder = 1;
+  const controlIdFieldNameClean = applyNamingConvention(
+    params.controlIdField,
+    params.namingConvention
+  );
+  const familyOptions = Array.from(families.keys()).filter((f) => f && f !== "UNKNOWN").sort();
+  fields["family"] = {
+    type: "string",
+    ui_type: familyOptions.length <= 50 ? "select" : "short_text",
+    is_array: false,
+    max_length: 10,
+    usage_count: controls.length,
+    usage_percentage: 100,
+    required: true,
+    visible: true,
+    show_in_table: true,
+    editable: false,
+    display_order: displayOrder++,
+    category: "core",
+    tab: "overview"
+  };
+  if (familyOptions.length <= 50) {
+    fields["family"].options = familyOptions;
+  }
+  fieldMetadata.forEach((metadata, fieldName) => {
+    if (fieldName === "family" || fieldName === "id" && controlIdFieldNameClean !== "id") {
+      return;
+    }
+    const frontendConfig = params.frontendFieldSchema?.find((f) => f.fieldName === fieldName);
+    if (params.frontendFieldSchema && !frontendConfig) {
+      return;
+    }
+    const usageCount = metadata.totalCount - metadata.emptyCount;
+    const usagePercentage = metadata.totalCount > 0 ? Math.round(usageCount / metadata.totalCount * 100) : 0;
+    let uiType = "short_text";
+    const nonEmptyCount = metadata.totalCount - metadata.emptyCount;
+    const isDropdownCandidate = metadata.uniqueValues.size > 0 && metadata.uniqueValues.size <= 20 && // Max 20 unique values for dropdown
+    nonEmptyCount >= 10 && // At least 10 non-empty values to be meaningful
+    metadata.maxLength <= 100 && // Reasonably short values only
+    metadata.uniqueValues.size / nonEmptyCount <= 0.3;
+    if (metadata.hasMultipleLines || metadata.maxLength > 500) {
+      uiType = "textarea";
+    } else if (isDropdownCandidate) {
+      uiType = "select";
+    } else if (metadata.type === "boolean") {
+      uiType = "checkbox";
+    } else if (metadata.type === "number") {
+      uiType = "number";
+    } else if (metadata.type === "date") {
+      uiType = "date";
+    } else if (metadata.maxLength <= 50) {
+      uiType = "short_text";
+    } else if (metadata.maxLength <= 200) {
+      uiType = "medium_text";
+    } else {
+      uiType = "long_text";
     }
-    const uniqueFamilies = Array.from(families.keys()).filter((f) => f && f !== "UNKNOWN");
-    let frontendFieldSchema = null;
-    if (req.body.fieldSchema) {
-      try {
-        frontendFieldSchema = JSON.parse(req.body.fieldSchema);
-      } catch {
+    let category = frontendConfig?.category || "custom";
+    if (!frontendConfig) {
+      if (fieldName.includes("status") || fieldName.includes("state")) {
+        category = "compliance";
+      } else if (fieldName.includes("title") || fieldName.includes("name") || fieldName.includes("description")) {
+        category = "core";
+      } else if (fieldName.includes("note") || fieldName.includes("comment")) {
+        category = "notes";
       }
     }
-    const fields = {};
-    let displayOrder = 1;
-    const controlIdFieldNameClean = applyNamingConvention(controlIdField, namingConvention);
-    const familyOptions = Array.from(families.keys()).filter((f) => f && f !== "UNKNOWN").sort();
-    fields["family"] = {
-      type: "string",
-      ui_type: familyOptions.length <= 50 ? "select" : "short_text",
-      // Make select if reasonable number of families
+    const isControlIdField = fieldName === controlIdFieldNameClean;
+    const fieldDef = {
+      type: metadata.type,
+      ui_type: uiType,
       is_array: false,
-      max_length: 10,
-      usage_count: controls.length,
-      usage_percentage: 100,
-      required: true,
-      visible: true,
-      show_in_table: true,
-      editable: false,
-      display_order: displayOrder++,
-      category: "core",
-      tab: "overview"
+      max_length: metadata.maxLength,
+      usage_count: usageCount,
+      usage_percentage: usagePercentage,
+      required: isControlIdField ? true : frontendConfig?.required ?? usagePercentage > 95,
+      visible: frontendConfig?.tab !== "hidden",
+      show_in_table: isControlIdField ? true : metadata.maxLength <= 100 && usagePercentage > 30,
+      editable: isControlIdField ? false : true,
+      display_order: isControlIdField ? 1 : frontendConfig?.displayOrder ?? displayOrder++,
+      category: isControlIdField ? "core" : category,
+      tab: isControlIdField ? "overview" : frontendConfig?.tab || void 0
     };
-    if (familyOptions.length <= 50) {
-      fields["family"].options = familyOptions;
+    if (uiType === "select") {
+      fieldDef.options = Array.from(metadata.uniqueValues).sort();
     }
-    fieldMetadata.forEach((metadata, fieldName) => {
-      if (fieldName === "family" || fieldName === "id" && controlIdFieldNameClean !== "id") {
-        return;
-      }
-      const frontendConfig = frontendFieldSchema?.find((f) => f.fieldName === fieldName);
-      if (frontendFieldSchema && !frontendConfig) {
+    if (frontendConfig?.originalName || metadata.originalName) {
+      fieldDef.original_name = frontendConfig?.originalName || metadata.originalName;
+    }
+    fields[fieldName] = fieldDef;
+  });
+  return {
+    fields,
+    total_controls: controls.length,
+    analyzed_at: (/* @__PURE__ */ new Date()).toISOString()
+  };
+}
+async function createOutputStructure(processedData, fieldSchema, params) {
+  const { controls, families } = processedData;
+  const state = getServerState();
+  const folderName = toKebabCase(params.controlSetName || "imported-controls");
+  const baseDir = join2(state.CONTROL_SET_DIR || process.cwd(), folderName);
+  if (!existsSync(baseDir)) {
+    mkdirSync(baseDir, { recursive: true });
+  }
+  const uniqueFamilies = Array.from(families.keys()).filter((f) => f && f !== "UNKNOWN");
+  const controlIdFieldNameClean = applyNamingConvention(
+    params.controlIdField,
+    params.namingConvention
+  );
+  const controlSetData = {
+    name: params.controlSetName,
+    description: params.controlSetDescription,
+    version: "1.0.0",
+    control_id_field: controlIdFieldNameClean,
+    controlCount: controls.length,
+    families: uniqueFamilies,
+    fieldSchema
+  };
+  writeFileSync(join2(baseDir, "lula.yaml"), yaml4.dump(controlSetData));
+  const controlsDir = join2(baseDir, "controls");
+  const mappingsDir = join2(baseDir, "mappings");
+  families.forEach((familyControls, family) => {
+    const familyDir = join2(controlsDir, family);
+    const familyMappingsDir = join2(mappingsDir, family);
+    if (!existsSync(familyDir)) {
+      mkdirSync(familyDir, { recursive: true });
+    }
+    if (!existsSync(familyMappingsDir)) {
+      mkdirSync(familyMappingsDir, { recursive: true });
+    }
+    familyControls.forEach((control) => {
+      const controlId = control[controlIdFieldNameClean];
+      if (!controlId) {
+        console.error("Missing control ID for control:", control);
         return;
       }
-      const usageCount = metadata.totalCount - metadata.emptyCount;
-      const usagePercentage = metadata.totalCount > 0 ? Math.round(usageCount / metadata.totalCount * 100) : 0;
-      let uiType = "short_text";
-      const nonEmptyCount = metadata.totalCount - metadata.emptyCount;
-      const isDropdownCandidate = metadata.uniqueValues.size > 0 && metadata.uniqueValues.size <= 20 && // Max 20 unique values for dropdown
-      nonEmptyCount >= 10 && // At least 10 non-empty values to be meaningful
-      metadata.maxLength <= 100 && // Reasonably short values only
-      metadata.uniqueValues.size / nonEmptyCount <= 0.3;
-      if (metadata.hasMultipleLines || metadata.maxLength > 500) {
-        uiType = "textarea";
-      } else if (isDropdownCandidate) {
-        uiType = "select";
-      } else if (metadata.type === "boolean") {
-        uiType = "checkbox";
-      } else if (metadata.type === "number") {
-        uiType = "number";
-      } else if (metadata.type === "date") {
-        uiType = "date";
-      } else if (metadata.maxLength <= 50) {
-        uiType = "short_text";
-      } else if (metadata.maxLength <= 200) {
-        uiType = "medium_text";
-      } else {
-        uiType = "long_text";
-      }
-      let category = frontendConfig?.category || "custom";
-      if (!frontendConfig) {
-        if (fieldName.includes("status") || fieldName.includes("state")) {
-          category = "compliance";
-        } else if (fieldName.includes("title") || fieldName.includes("name") || fieldName.includes("description")) {
-          category = "core";
-        } else if (fieldName.includes("note") || fieldName.includes("comment")) {
-          category = "notes";
-        }
-      }
-      const isControlIdField = fieldName === controlIdFieldNameClean;
-      const fieldDef = {
-        type: metadata.type,
-        ui_type: uiType,
-        is_array: false,
-        max_length: metadata.maxLength,
-        usage_count: usageCount,
-        usage_percentage: usagePercentage,
-        required: isControlIdField ? true : frontendConfig?.required ?? usagePercentage > 95,
-        // Control ID is always required
-        visible: frontendConfig?.tab !== "hidden",
-        show_in_table: isControlIdField ? true : metadata.maxLength <= 100 && usagePercentage > 30,
-        // Always show control ID in table
-        editable: isControlIdField ? false : true,
-        // Control ID is not editable
-        display_order: isControlIdField ? 1 : frontendConfig?.displayOrder ?? displayOrder++,
-        // Control ID is always first
-        category: isControlIdField ? "core" : category,
-        // Control ID is always core
-        tab: isControlIdField ? "overview" : frontendConfig?.tab || void 0
-        // Use frontend config or default
+      const controlIdStr = String(controlId).slice(0, 50);
+      const fileName = `${controlIdStr.replace(/[^a-zA-Z0-9-]/g, "_")}.yaml`;
+      const filePath = join2(familyDir, fileName);
+      const mappingFileName = `${controlIdStr.replace(/[^a-zA-Z0-9-]/g, "_")}-mappings.yaml`;
+      const mappingFilePath = join2(familyMappingsDir, mappingFileName);
+      const filteredControl = {};
+      const mappingData = {
+        control_id: controlIdStr,
+        justification: "",
+        uuid: crypto.randomUUID()
       };
-      if (uiType === "select") {
-        fieldDef.options = Array.from(metadata.uniqueValues).sort();
-      }
-      if (frontendConfig?.originalName || metadata.originalName) {
-        fieldDef.original_name = frontendConfig?.originalName || metadata.originalName;
-      }
-      fields[fieldName] = fieldDef;
-    });
-    const fieldSchema = {
-      fields,
-      total_controls: controls.length,
-      analyzed_at: (/* @__PURE__ */ new Date()).toISOString()
-    };
-    const controlSetData = {
-      name: controlSetName,
-      description: controlSetDescription,
-      version: "1.0.0",
-      control_id_field: controlIdFieldNameClean,
-      // Add this to indicate which field is the control ID
-      controlCount: controls.length,
-      families: uniqueFamilies,
-      fieldSchema
-    };
-    writeFileSync(join2(baseDir, "lula.yaml"), yaml4.dump(controlSetData));
-    const controlsDir = join2(baseDir, "controls");
-    const mappingsDir = join2(baseDir, "mappings");
-    families.forEach((familyControls, family) => {
-      const familyDir = join2(controlsDir, family);
-      const familyMappingsDir = join2(mappingsDir, family);
-      if (!existsSync(familyDir)) {
-        mkdirSync(familyDir, { recursive: true });
-      }
-      if (!existsSync(familyMappingsDir)) {
-        mkdirSync(familyMappingsDir, { recursive: true });
+      const justificationContents = [];
+      if (control.family !== void 0) {
+        filteredControl.family = control.family;
       }
-      familyControls.forEach((control) => {
-        const controlId = control[controlIdFieldNameClean];
-        if (!controlId) {
-          console.error("Missing control ID for control:", control);
-          return;
-        }
-        const controlIdStr = String(controlId).slice(0, 50);
-        const fileName2 = `${controlIdStr.replace(/[^a-zA-Z0-9-]/g, "_")}.yaml`;
-        const filePath = join2(familyDir, fileName2);
-        const mappingFileName = `${controlIdStr.replace(/[^a-zA-Z0-9-]/g, "_")}-mappings.yaml`;
-        const mappingFilePath = join2(familyMappingsDir, mappingFileName);
-        const filteredControl = {};
-        const mappingData = {
-          control_id: controlIdStr,
-          justification: "",
-          uuid: crypto.randomUUID()
-        };
-        const justificationContents = [];
-        if (control.family !== void 0) {
-          filteredControl.family = control.family;
-        }
-        Object.keys(control).forEach((fieldName) => {
-          if (fieldName === "family") return;
-          if (justificationFields.includes(fieldName) && control[fieldName] !== void 0 && control[fieldName] !== null) {
-            justificationContents.push(control[fieldName]);
-          }
-          const isInFrontendSchema = frontendFieldSchema?.some((f) => f.fieldName === fieldName);
-          const isInFieldsMetadata = fields.hasOwnProperty(fieldName);
-          if (isInFrontendSchema || isInFieldsMetadata) {
-            filteredControl[fieldName] = control[fieldName];
-          }
-        });
-        writeFileSync(filePath, yaml4.dump(filteredControl));
-        if (justificationContents.length > 0) {
-          mappingData.justification = justificationContents.join("\n\n");
+      Object.keys(control).forEach((fieldName) => {
+        if (fieldName === "family") return;
+        if (params.justificationFields.includes(fieldName) && control[fieldName] !== void 0 && control[fieldName] !== null) {
+          justificationContents.push(control[fieldName]);
         }
-        if (mappingData.justification && mappingData.justification.trim() !== "") {
-          const mappingArray = [mappingData];
-          writeFileSync(mappingFilePath, yaml4.dump(mappingArray));
+        const isInFrontendSchema = params.frontendFieldSchema?.some(
+          (f) => f.fieldName === fieldName
+        );
+        const isInFieldsMetadata = fieldSchema.fields.hasOwnProperty(fieldName);
+        if (isInFrontendSchema || isInFieldsMetadata) {
+          filteredControl[fieldName] = control[fieldName];
         }
       });
+      writeFileSync(filePath, yaml4.dump(filteredControl));
+      if (justificationContents.length > 0) {
+        mappingData.justification = justificationContents.join("\n\n");
+      }
+      if (mappingData.justification && mappingData.justification.trim() !== "") {
+        const mappingArray = [mappingData];
+        writeFileSync(mappingFilePath, yaml4.dump(mappingArray));
+      }
     });
-    res.json({
-      success: true,
-      controlCount: controls.length,
-      families: Array.from(families.keys()),
-      outputDir: folderName
-      // Return just the folder name, not full path
-    });
-  } catch (error) {
-    console.error("Error processing spreadsheet:", error);
-    res.status(500).json({ error: "Failed to process spreadsheet" });
-  }
-});
+  });
+  return { folderName };
+}
 function applyNamingConvention(fieldName, convention) {
   if (!fieldName) return fieldName;
   const cleanedName = fieldName.trim();
@@ -1039,6 +1075,18 @@ router.post("/parse-excel-sheet", upload.single("file"), async (req, res) => {
 });
 var spreadsheetRoutes_default = router;
 export {
+  applyNamingConvention,
+  buildFieldSchema,
+  createOutputStructure,
   spreadsheetRoutes_default as default,
-  scanControlSets
+  detectValueType,
+  extractFamilyFromControlId,
+  parseCSV,
+  parseUploadedFile,
+  processImportParameters,
+  processSpreadsheetData,
+  scanControlSets,
+  toCamelCase,
+  toKebabCase,
+  toSnakeCase
 };