lula2 0.5.0 → 0.5.1-nightly.1

package/dist/index.js

@@ -2845,8 +2845,20 @@ var init_serverState = __esm({
 // cli/server/spreadsheetRoutes.ts
 var spreadsheetRoutes_exports = {};
 __export(spreadsheetRoutes_exports, {
+  applyNamingConvention: () => applyNamingConvention,
+  buildFieldSchema: () => buildFieldSchema,
+  createOutputStructure: () => createOutputStructure,
   default: () => spreadsheetRoutes_default,
-  scanControlSets: () => scanControlSets
+  detectValueType: () => detectValueType,
+  extractFamilyFromControlId: () => extractFamilyFromControlId,
+  parseCSV: () => parseCSV,
+  parseUploadedFile: () => parseUploadedFile,
+  processImportParameters: () => processImportParameters,
+  processSpreadsheetData: () => processSpreadsheetData,
+  scanControlSets: () => scanControlSets,
+  toCamelCase: () => toCamelCase,
+  toKebabCase: () => toKebabCase,
+  toSnakeCase: () => toSnakeCase
 });
 import crypto from "crypto";
 import { parse as parseCSVSync } from "csv-parse/sync";
@@ -2895,6 +2907,334 @@ async function scanControlSets() {
   }).filter((cs) => cs !== null);
   return { controlSets };
 }
+function processImportParameters(reqBody) {
+  const {
+    controlIdField = "Control ID",
+    startRow = "1",
+    controlSetName = "Imported Control Set",
+    controlSetDescription = "Imported from spreadsheet"
+  } = reqBody;
+  let justificationFields = [];
+  if (reqBody.justificationFields) {
+    try {
+      justificationFields = JSON.parse(reqBody.justificationFields);
+      debug("Justification fields received:", justificationFields);
+    } catch (e) {
+      console.error("Failed to parse justification fields:", e);
+    }
+  }
+  let frontendFieldSchema = null;
+  if (reqBody.fieldSchema) {
+    try {
+      frontendFieldSchema = JSON.parse(reqBody.fieldSchema);
+    } catch (e) {
+      console.error("Failed to parse fieldSchema:", e);
+    }
+  }
+  debug("Import parameters received:", {
+    controlIdField,
+    startRow,
+    controlSetName,
+    controlSetDescription
+  });
+  return {
+    controlIdField,
+    startRow,
+    controlSetName,
+    controlSetDescription,
+    justificationFields,
+    namingConvention: "kebab-case",
+    skipEmpty: true,
+    skipEmptyRows: true,
+    frontendFieldSchema
+  };
+}
+async function parseUploadedFile(file) {
+  const fileName = file.originalname || "";
+  const isCSV = fileName.toLowerCase().endsWith(".csv");
+  let rawData = [];
+  if (isCSV) {
+    const csvContent = file.buffer.toString("utf-8");
+    rawData = parseCSV(csvContent);
+  } else {
+    const workbook = new ExcelJS.Workbook();
+    const buffer = Buffer.from(file.buffer);
+    await workbook.xlsx.load(buffer);
+    const worksheet = workbook.worksheets[0];
+    if (!worksheet) {
+      throw new Error("No worksheet found in file");
+    }
+    worksheet.eachRow({ includeEmpty: true }, (row, rowNumber) => {
+      const rowData = [];
+      row.eachCell({ includeEmpty: true }, (cell, colNumber) => {
+        rowData[colNumber - 1] = cell.value;
+      });
+      rawData[rowNumber - 1] = rowData;
+    });
+  }
+  return rawData;
+}
+function processSpreadsheetData(rawData, headers, startRowIndex, params) {
+  const controls = [];
+  const families = /* @__PURE__ */ new Map();
+  const fieldMetadata = /* @__PURE__ */ new Map();
+  headers.forEach((header) => {
+    if (header) {
+      const cleanName = applyNamingConvention(header, params.namingConvention);
+      fieldMetadata.set(cleanName, {
+        originalName: header,
+        cleanName,
+        type: "string",
+        maxLength: 0,
+        hasMultipleLines: false,
+        uniqueValues: /* @__PURE__ */ new Set(),
+        emptyCount: 0,
+        totalCount: 0,
+        examples: []
+      });
+    }
+  });
+  for (let i = startRowIndex + 1; i < rawData.length; i++) {
+    const row = rawData[i];
+    if (!row || row.length === 0) continue;
+    const control = {};
+    let hasData = false;
+    headers.forEach((header, index) => {
+      if (header && row[index] !== void 0 && row[index] !== null) {
+        const value = typeof row[index] === "string" ? row[index].trim() : row[index];
+        const fieldName = applyNamingConvention(header, params.namingConvention);
+        const metadata = fieldMetadata.get(fieldName);
+        metadata.totalCount++;
+        if (value === "" || value === null || value === void 0) {
+          metadata.emptyCount++;
+          if (params.skipEmpty) return;
+        } else {
+          const normalizedValue = typeof value === "string" ? value.trim() : value;
+          if (normalizedValue !== "") {
+            metadata.uniqueValues.add(normalizedValue);
+          }
+          const valueType = detectValueType(value);
+          if (metadata.type === "string" || metadata.totalCount === 1) {
+            metadata.type = valueType;
+          } else if (metadata.type !== valueType) {
+            metadata.type = "mixed";
+          }
+          if (typeof value === "string") {
+            const length = value.length;
+            if (length > metadata.maxLength) {
+              metadata.maxLength = length;
+            }
+            if (value.includes("\n") || length > 100) {
+              metadata.hasMultipleLines = true;
+            }
+          }
+          if (metadata.examples.length < 3 && normalizedValue !== "") {
+            metadata.examples.push(normalizedValue);
+          }
+        }
+        control[fieldName] = value;
+        hasData = true;
+      }
+    });
+    if (hasData && (!params.skipEmptyRows || Object.keys(control).length > 0)) {
+      const controlIdFieldName = applyNamingConvention(
+        params.controlIdField,
+        params.namingConvention
+      );
+      const controlId = control[controlIdFieldName];
+      if (!controlId) {
+        continue;
+      }
+      const family = extractFamilyFromControlId(controlId);
+      control.family = family;
+      controls.push(control);
+      if (!families.has(family)) {
+        families.set(family, []);
+      }
+      families.get(family).push(control);
+    }
+  }
+  return { controls, families, fieldMetadata };
+}
+function buildFieldSchema(fieldMetadata, controls, params, families) {
+  const fields = {};
+  let displayOrder = 1;
+  const controlIdFieldNameClean = applyNamingConvention(
+    params.controlIdField,
+    params.namingConvention
+  );
+  const familyOptions = Array.from(families.keys()).filter((f) => f && f !== "UNKNOWN").sort();
+  fields["family"] = {
+    type: "string",
+    ui_type: familyOptions.length <= 50 ? "select" : "short_text",
+    is_array: false,
+    max_length: 10,
+    usage_count: controls.length,
+    usage_percentage: 100,
+    required: true,
+    visible: true,
+    show_in_table: true,
+    editable: false,
+    display_order: displayOrder++,
+    category: "core",
+    tab: "overview"
+  };
+  if (familyOptions.length <= 50) {
+    fields["family"].options = familyOptions;
+  }
+  fieldMetadata.forEach((metadata, fieldName) => {
+    if (fieldName === "family" || fieldName === "id" && controlIdFieldNameClean !== "id") {
+      return;
+    }
+    const frontendConfig = params.frontendFieldSchema?.find((f) => f.fieldName === fieldName);
+    if (params.frontendFieldSchema && !frontendConfig) {
+      return;
+    }
+    const usageCount = metadata.totalCount - metadata.emptyCount;
+    const usagePercentage = metadata.totalCount > 0 ? Math.round(usageCount / metadata.totalCount * 100) : 0;
+    let uiType = "short_text";
+    const nonEmptyCount = metadata.totalCount - metadata.emptyCount;
+    const isDropdownCandidate = metadata.uniqueValues.size > 0 && metadata.uniqueValues.size <= 20 && // Max 20 unique values for dropdown
+    nonEmptyCount >= 10 && // At least 10 non-empty values to be meaningful
+    metadata.maxLength <= 100 && // Reasonably short values only
+    metadata.uniqueValues.size / nonEmptyCount <= 0.3;
+    if (metadata.hasMultipleLines || metadata.maxLength > 500) {
+      uiType = "textarea";
+    } else if (isDropdownCandidate) {
+      uiType = "select";
+    } else if (metadata.type === "boolean") {
+      uiType = "checkbox";
+    } else if (metadata.type === "number") {
+      uiType = "number";
+    } else if (metadata.type === "date") {
+      uiType = "date";
+    } else if (metadata.maxLength <= 50) {
+      uiType = "short_text";
+    } else if (metadata.maxLength <= 200) {
+      uiType = "medium_text";
+    } else {
+      uiType = "long_text";
+    }
+    let category = frontendConfig?.category || "custom";
+    if (!frontendConfig) {
+      if (fieldName.includes("status") || fieldName.includes("state")) {
+        category = "compliance";
+      } else if (fieldName.includes("title") || fieldName.includes("name") || fieldName.includes("description")) {
+        category = "core";
+      } else if (fieldName.includes("note") || fieldName.includes("comment")) {
+        category = "notes";
+      }
+    }
+    const isControlIdField = fieldName === controlIdFieldNameClean;
+    const fieldDef = {
+      type: metadata.type,
+      ui_type: uiType,
+      is_array: false,
+      max_length: metadata.maxLength,
+      usage_count: usageCount,
+      usage_percentage: usagePercentage,
+      required: isControlIdField ? true : frontendConfig?.required ?? usagePercentage > 95,
+      visible: frontendConfig?.tab !== "hidden",
+      show_in_table: isControlIdField ? true : metadata.maxLength <= 100 && usagePercentage > 30,
+      editable: isControlIdField ? false : true,
+      display_order: isControlIdField ? 1 : frontendConfig?.displayOrder ?? displayOrder++,
+      category: isControlIdField ? "core" : category,
+      tab: isControlIdField ? "overview" : frontendConfig?.tab || void 0
+    };
+    if (uiType === "select") {
+      fieldDef.options = Array.from(metadata.uniqueValues).sort();
+    }
+    if (frontendConfig?.originalName || metadata.originalName) {
+      fieldDef.original_name = frontendConfig?.originalName || metadata.originalName;
+    }
+    fields[fieldName] = fieldDef;
+  });
+  return {
+    fields,
+    total_controls: controls.length,
+    analyzed_at: (/* @__PURE__ */ new Date()).toISOString()
+  };
+}
+async function createOutputStructure(processedData, fieldSchema, params) {
+  const { controls, families } = processedData;
+  const state = getServerState();
+  const folderName = toKebabCase(params.controlSetName || "imported-controls");
+  const baseDir = join4(state.CONTROL_SET_DIR || process.cwd(), folderName);
+  if (!existsSync3(baseDir)) {
+    mkdirSync2(baseDir, { recursive: true });
+  }
+  const uniqueFamilies = Array.from(families.keys()).filter((f) => f && f !== "UNKNOWN");
+  const controlIdFieldNameClean = applyNamingConvention(
+    params.controlIdField,
+    params.namingConvention
+  );
+  const controlSetData = {
+    name: params.controlSetName,
+    description: params.controlSetDescription,
+    version: "1.0.0",
+    control_id_field: controlIdFieldNameClean,
+    controlCount: controls.length,
+    families: uniqueFamilies,
+    fieldSchema
+  };
+  writeFileSync2(join4(baseDir, "lula.yaml"), yaml4.dump(controlSetData));
+  const controlsDir = join4(baseDir, "controls");
+  const mappingsDir = join4(baseDir, "mappings");
+  families.forEach((familyControls, family) => {
+    const familyDir = join4(controlsDir, family);
+    const familyMappingsDir = join4(mappingsDir, family);
+    if (!existsSync3(familyDir)) {
+      mkdirSync2(familyDir, { recursive: true });
+    }
+    if (!existsSync3(familyMappingsDir)) {
+      mkdirSync2(familyMappingsDir, { recursive: true });
+    }
+    familyControls.forEach((control) => {
+      const controlId = control[controlIdFieldNameClean];
+      if (!controlId) {
+        console.error("Missing control ID for control:", control);
+        return;
+      }
+      const controlIdStr = String(controlId).slice(0, 50);
+      const fileName = `${controlIdStr.replace(/[^a-zA-Z0-9-]/g, "_")}.yaml`;
+      const filePath = join4(familyDir, fileName);
+      const mappingFileName = `${controlIdStr.replace(/[^a-zA-Z0-9-]/g, "_")}-mappings.yaml`;
+      const mappingFilePath = join4(familyMappingsDir, mappingFileName);
+      const filteredControl = {};
+      const mappingData = {
+        control_id: controlIdStr,
+        justification: "",
+        uuid: crypto.randomUUID()
+      };
+      const justificationContents = [];
+      if (control.family !== void 0) {
+        filteredControl.family = control.family;
+      }
+      Object.keys(control).forEach((fieldName) => {
+        if (fieldName === "family") return;
+        if (params.justificationFields.includes(fieldName) && control[fieldName] !== void 0 && control[fieldName] !== null) {
+          justificationContents.push(control[fieldName]);
+        }
+        const isInFrontendSchema = params.frontendFieldSchema?.some(
+          (f) => f.fieldName === fieldName
+        );
+        const isInFieldsMetadata = fieldSchema.fields.hasOwnProperty(fieldName);
+        if (isInFrontendSchema || isInFieldsMetadata) {
+          filteredControl[fieldName] = control[fieldName];
+        }
+      });
+      writeFileSync2(filePath, yaml4.dump(filteredControl));
+      if (justificationContents.length > 0) {
+        mappingData.justification = justificationContents.join("\n\n");
+      }
+      if (mappingData.justification && mappingData.justification.trim() !== "") {
+        const mappingArray = [mappingData];
+        writeFileSync2(mappingFilePath, yaml4.dump(mappingArray));
+      }
+    });
+  });
+  return { folderName };
+}
 function applyNamingConvention(fieldName, convention) {
   if (!fieldName) return fieldName;
   const cleanedName = fieldName.trim();
@@ -3275,53 +3615,9 @@ var init_spreadsheetRoutes = __esm({
         if (!req.file) {
           return res.status(400).json({ error: "No file uploaded" });
         }
-        const {
-          controlIdField = "Control ID",
-          startRow = "1",
-          controlSetName = "Imported Control Set",
-          controlSetDescription = "Imported from spreadsheet"
-        } = req.body;
-        let justificationFields = [];
-        if (req.body.justificationFields) {
-          try {
-            justificationFields = JSON.parse(req.body.justificationFields);
-            debug("Justification fields received:", justificationFields);
-          } catch (e) {
-            console.error("Failed to parse justification fields:", e);
-          }
-        }
-        debug("Import parameters received:", {
-          controlIdField,
-          startRow,
-          controlSetName,
-          controlSetDescription
-        });
-        const namingConvention = "kebab-case";
-        const skipEmpty = true;
-        const skipEmptyRows = true;
-        const fileName = req.file.originalname || "";
-        const isCSV = fileName.toLowerCase().endsWith(".csv");
-        let rawData = [];
-        if (isCSV) {
-          const csvContent = req.file.buffer.toString("utf-8");
-          rawData = parseCSV(csvContent);
-        } else {
-          const workbook = new ExcelJS.Workbook();
-          const buffer = Buffer.from(req.file.buffer);
-          await workbook.xlsx.load(buffer);
-          const worksheet = workbook.worksheets[0];
-          if (!worksheet) {
-            return res.status(400).json({ error: "No worksheet found in file" });
-          }
-          worksheet.eachRow({ includeEmpty: true }, (row, rowNumber) => {
-            const rowData = [];
-            row.eachCell({ includeEmpty: true }, (cell, colNumber) => {
-              rowData[colNumber - 1] = cell.value;
-            });
-            rawData[rowNumber - 1] = rowData;
-          });
-        }
-        const startRowIndex = parseInt(startRow) - 1;
+        const params = processImportParameters(req.body);
+        const rawData = await parseUploadedFile(req.file);
+        const startRowIndex = parseInt(params.startRow) - 1;
         if (rawData.length <= startRowIndex) {
           return res.status(400).json({ error: "Start row exceeds sheet data" });
         }
@@ -3332,269 +3628,21 @@ var init_spreadsheetRoutes = __esm({
         debug("Headers found:", headers);
         debug(
           "After conversion, looking for control ID field:",
-          applyNamingConvention(controlIdField, namingConvention)
+          applyNamingConvention(params.controlIdField, params.namingConvention)
         );
-        const controls = [];
-        const families = /* @__PURE__ */ new Map();
-        const fieldMetadata = /* @__PURE__ */ new Map();
-        headers.forEach((header) => {
-          if (header) {
-            const cleanName = applyNamingConvention(header, namingConvention);
-            fieldMetadata.set(cleanName, {
-              originalName: header,
-              cleanName,
-              type: "string",
-              maxLength: 0,
-              hasMultipleLines: false,
-              uniqueValues: /* @__PURE__ */ new Set(),
-              emptyCount: 0,
-              totalCount: 0,
-              examples: []
-            });
-          }
-        });
-        for (let i = startRowIndex + 1; i < rawData.length; i++) {
-          const row = rawData[i];
-          if (!row || row.length === 0) continue;
-          const control = {};
-          let hasData = false;
-          headers.forEach((header, index) => {
-            if (header && row[index] !== void 0 && row[index] !== null) {
-              const value = typeof row[index] === "string" ? row[index].trim() : row[index];
-              const fieldName = applyNamingConvention(header, namingConvention);
-              const metadata = fieldMetadata.get(fieldName);
-              metadata.totalCount++;
-              if (value === "" || value === null || value === void 0) {
-                metadata.emptyCount++;
-                if (skipEmpty) return;
-              } else {
-                const normalizedValue = typeof value === "string" ? value.trim() : value;
-                if (normalizedValue !== "") {
-                  metadata.uniqueValues.add(normalizedValue);
-                }
-                const valueType = detectValueType(value);
-                if (metadata.type === "string" || metadata.totalCount === 1) {
-                  metadata.type = valueType;
-                } else if (metadata.type !== valueType) {
-                  metadata.type = "mixed";
-                }
-                if (typeof value === "string") {
-                  const length = value.length;
-                  if (length > metadata.maxLength) {
-                    metadata.maxLength = length;
-                  }
-                  if (value.includes("\n") || length > 100) {
-                    metadata.hasMultipleLines = true;
-                  }
-                }
-                if (metadata.examples.length < 3 && normalizedValue !== "") {
-                  metadata.examples.push(normalizedValue);
-                }
-              }
-              control[fieldName] = value;
-              hasData = true;
-            }
-          });
-          if (hasData && (!skipEmptyRows || Object.keys(control).length > 0)) {
-            const controlIdFieldName = applyNamingConvention(controlIdField, namingConvention);
-            const controlId = control[controlIdFieldName];
-            if (!controlId) {
-              continue;
-            }
-            const family = extractFamilyFromControlId(controlId);
-            control.family = family;
-            controls.push(control);
-            if (!families.has(family)) {
-              families.set(family, []);
-            }
-            families.get(family).push(control);
-          }
-        }
-        const state = getServerState();
-        const folderName = toKebabCase(controlSetName || "imported-controls");
-        const baseDir = join4(state.CONTROL_SET_DIR || process.cwd(), folderName);
-        if (!existsSync3(baseDir)) {
-          mkdirSync2(baseDir, { recursive: true });
-        }
-        const uniqueFamilies = Array.from(families.keys()).filter((f) => f && f !== "UNKNOWN");
-        let frontendFieldSchema = null;
-        if (req.body.fieldSchema) {
-          try {
-            frontendFieldSchema = JSON.parse(req.body.fieldSchema);
-          } catch {
-          }
-        }
-        const fields = {};
-        let displayOrder = 1;
-        const controlIdFieldNameClean = applyNamingConvention(controlIdField, namingConvention);
-        const familyOptions = Array.from(families.keys()).filter((f) => f && f !== "UNKNOWN").sort();
-        fields["family"] = {
-          type: "string",
-          ui_type: familyOptions.length <= 50 ? "select" : "short_text",
-          // Make select if reasonable number of families
-          is_array: false,
-          max_length: 10,
-          usage_count: controls.length,
-          usage_percentage: 100,
-          required: true,
-          visible: true,
-          show_in_table: true,
-          editable: false,
-          display_order: displayOrder++,
-          category: "core",
-          tab: "overview"
-        };
-        if (familyOptions.length <= 50) {
-          fields["family"].options = familyOptions;
-        }
-        fieldMetadata.forEach((metadata, fieldName) => {
-          if (fieldName === "family" || fieldName === "id" && controlIdFieldNameClean !== "id") {
-            return;
-          }
-          const frontendConfig = frontendFieldSchema?.find((f) => f.fieldName === fieldName);
-          if (frontendFieldSchema && !frontendConfig) {
-            return;
-          }
-          const usageCount = metadata.totalCount - metadata.emptyCount;
-          const usagePercentage = metadata.totalCount > 0 ? Math.round(usageCount / metadata.totalCount * 100) : 0;
-          let uiType = "short_text";
-          const nonEmptyCount = metadata.totalCount - metadata.emptyCount;
-          const isDropdownCandidate = metadata.uniqueValues.size > 0 && metadata.uniqueValues.size <= 20 && // Max 20 unique values for dropdown
-          nonEmptyCount >= 10 && // At least 10 non-empty values to be meaningful
-          metadata.maxLength <= 100 && // Reasonably short values only
-          metadata.uniqueValues.size / nonEmptyCount <= 0.3;
-          if (metadata.hasMultipleLines || metadata.maxLength > 500) {
-            uiType = "textarea";
-          } else if (isDropdownCandidate) {
-            uiType = "select";
-          } else if (metadata.type === "boolean") {
-            uiType = "checkbox";
-          } else if (metadata.type === "number") {
-            uiType = "number";
-          } else if (metadata.type === "date") {
-            uiType = "date";
-          } else if (metadata.maxLength <= 50) {
-            uiType = "short_text";
-          } else if (metadata.maxLength <= 200) {
-            uiType = "medium_text";
-          } else {
-            uiType = "long_text";
-          }
-          let category = frontendConfig?.category || "custom";
-          if (!frontendConfig) {
-            if (fieldName.includes("status") || fieldName.includes("state")) {
-              category = "compliance";
-            } else if (fieldName.includes("title") || fieldName.includes("name") || fieldName.includes("description")) {
-              category = "core";
-            } else if (fieldName.includes("note") || fieldName.includes("comment")) {
-              category = "notes";
-            }
-          }
-          const isControlIdField = fieldName === controlIdFieldNameClean;
-          const fieldDef = {
-            type: metadata.type,
-            ui_type: uiType,
-            is_array: false,
-            max_length: metadata.maxLength,
-            usage_count: usageCount,
-            usage_percentage: usagePercentage,
-            required: isControlIdField ? true : frontendConfig?.required ?? usagePercentage > 95,
-            // Control ID is always required
-            visible: frontendConfig?.tab !== "hidden",
-            show_in_table: isControlIdField ? true : metadata.maxLength <= 100 && usagePercentage > 30,
-            // Always show control ID in table
-            editable: isControlIdField ? false : true,
-            // Control ID is not editable
-            display_order: isControlIdField ? 1 : frontendConfig?.displayOrder ?? displayOrder++,
-            // Control ID is always first
-            category: isControlIdField ? "core" : category,
-            // Control ID is always core
-            tab: isControlIdField ? "overview" : frontendConfig?.tab || void 0
-            // Use frontend config or default
-          };
-          if (uiType === "select") {
-            fieldDef.options = Array.from(metadata.uniqueValues).sort();
-          }
-          if (frontendConfig?.originalName || metadata.originalName) {
-            fieldDef.original_name = frontendConfig?.originalName || metadata.originalName;
-          }
-          fields[fieldName] = fieldDef;
-        });
-        const fieldSchema = {
-          fields,
-          total_controls: controls.length,
-          analyzed_at: (/* @__PURE__ */ new Date()).toISOString()
-        };
-        const controlSetData = {
-          name: controlSetName,
-          description: controlSetDescription,
-          version: "1.0.0",
-          control_id_field: controlIdFieldNameClean,
-          // Add this to indicate which field is the control ID
-          controlCount: controls.length,
-          families: uniqueFamilies,
-          fieldSchema
-        };
-        writeFileSync2(join4(baseDir, "lula.yaml"), yaml4.dump(controlSetData));
-        const controlsDir = join4(baseDir, "controls");
-        const mappingsDir = join4(baseDir, "mappings");
-        families.forEach((familyControls, family) => {
-          const familyDir = join4(controlsDir, family);
-          const familyMappingsDir = join4(mappingsDir, family);
-          if (!existsSync3(familyDir)) {
-            mkdirSync2(familyDir, { recursive: true });
-          }
-          if (!existsSync3(familyMappingsDir)) {
-            mkdirSync2(familyMappingsDir, { recursive: true });
-          }
-          familyControls.forEach((control) => {
-            const controlId = control[controlIdFieldNameClean];
-            if (!controlId) {
-              console.error("Missing control ID for control:", control);
-              return;
-            }
-            const controlIdStr = String(controlId).slice(0, 50);
-            const fileName2 = `${controlIdStr.replace(/[^a-zA-Z0-9-]/g, "_")}.yaml`;
-            const filePath = join4(familyDir, fileName2);
-            const mappingFileName = `${controlIdStr.replace(/[^a-zA-Z0-9-]/g, "_")}-mappings.yaml`;
-            const mappingFilePath = join4(familyMappingsDir, mappingFileName);
-            const filteredControl = {};
-            const mappingData = {
-              control_id: controlIdStr,
-              justification: "",
-              uuid: crypto.randomUUID()
-            };
-            const justificationContents = [];
-            if (control.family !== void 0) {
-              filteredControl.family = control.family;
-            }
-            Object.keys(control).forEach((fieldName) => {
-              if (fieldName === "family") return;
-              if (justificationFields.includes(fieldName) && control[fieldName] !== void 0 && control[fieldName] !== null) {
-                justificationContents.push(control[fieldName]);
-              }
-              const isInFrontendSchema = frontendFieldSchema?.some((f) => f.fieldName === fieldName);
-              const isInFieldsMetadata = fields.hasOwnProperty(fieldName);
-              if (isInFrontendSchema || isInFieldsMetadata) {
-                filteredControl[fieldName] = control[fieldName];
-              }
-            });
-            writeFileSync2(filePath, yaml4.dump(filteredControl));
-            if (justificationContents.length > 0) {
-              mappingData.justification = justificationContents.join("\n\n");
-            }
-            if (mappingData.justification && mappingData.justification.trim() !== "") {
-              const mappingArray = [mappingData];
-              writeFileSync2(mappingFilePath, yaml4.dump(mappingArray));
-            }
-          });
-        });
+        const processedData = processSpreadsheetData(rawData, headers, startRowIndex, params);
+        const fieldSchema = buildFieldSchema(
+          processedData.fieldMetadata,
+          processedData.controls,
+          params,
+          processedData.families
+        );
+        const result = await createOutputStructure(processedData, fieldSchema, params);
         res.json({
           success: true,
-          controlCount: controls.length,
-          families: Array.from(families.keys()),
-          outputDir: folderName
-          // Return just the folder name, not full path
+          controlCount: processedData.controls.length,
+          families: Array.from(processedData.families.keys()),
+          outputDir: result.folderName
         });
       } catch (error) {
         console.error("Error processing spreadsheet:", error);
@@ -5580,6 +5628,10 @@ function getChangedBlocks(oldText, newText) {
   }
   return changed;
 }
+function containsLulaAnnotations(text) {
+  const lines = text.split("\n");
+  return lines.some((line) => line.includes("@lulaStart") || line.includes("@lulaEnd"));
+}
 function crawlCommand() {
   return new Command().command("crawl").description("Detect compliance-related changes between @lulaStart and @lulaEnd in PR files").addOption(
     new Option("--post-mode <mode>", "How to post findings").choices(["review", "comment"]).default("review")
@@ -5601,8 +5653,55 @@ Please review the changes to ensure they meet compliance standards.
 Lula reviewed ${files.length} files changed that affect compliance.
 
 `;
+    const deletedFilesWithAnnotations = [];
+    for (const file of files) {
+      if (file.status === "removed") {
+        try {
+          const oldText = await fetchRawFileViaAPI({
+            octokit,
+            owner,
+            repo,
+            path: file.filename,
+            ref: "main"
+          });
+          if (containsLulaAnnotations(oldText)) {
+            deletedFilesWithAnnotations.push(file.filename);
+          }
+        } catch (err) {
+          console.error(`Error checking deleted file ${file.filename}: ${err}`);
+        }
+      }
+    }
+    if (deletedFilesWithAnnotations.length > 0) {
+      leavePost = true;
+      commentBody += `
+
+**Compliance Warning: Files with Lula annotations were deleted**
+
+`;
+      commentBody += `The following files contained compliance annotations (\`@lulaStart\`/\`@lulaEnd\`) and were deleted in this PR. This may affect compliance coverage:
+
+`;
+      for (const filename of deletedFilesWithAnnotations) {
+        commentBody += `- \`${filename}\`
+`;
+      }
+      commentBody += `
+Please review whether:
+`;
+      commentBody += `- The compliance coverage provided by these files is still needed
+`;
+      commentBody += `- Alternative compliance measures have been implemented
+`;
+      commentBody += `- The deletion is intentional and compliance-approved
+
+`;
+      commentBody += `---
+
+`;
+    }
     for (const file of files) {
-      if (file.status === "added") continue;
+      if (file.status === "added" || file.status === "removed") continue;
       try {
         const [oldText, newText] = await Promise.all([
           fetchRawFileViaAPI({ octokit, owner, repo, path: file.filename, ref: "main" }),