@josephyan/qingflow-cli 0.2.0-beta.62 → 0.2.0-beta.64

package/src/qingflow_mcp/tools/import_tools.py

@@ -361,12 +361,32 @@ class ImportTools(ToolBase):
             )
             effective_path = path
             effective_local_check = local_check
-            auto_normalization = self._maybe_auto_normalize_file(
-                source_path=path,
-                expected_columns=expected_columns,
-                template_header_profile=template_header_profile,
-                local_check=local_check,
-            )
+            auto_normalization = None
+            try:
+                auto_normalization = self._maybe_auto_normalize_file(
+                    source_path=path,
+                    expected_columns=expected_columns,
+                    template_header_profile=template_header_profile,
+                    local_check=local_check,
+                )
+            except Exception as exc:
+                effective_local_check = deepcopy(local_check)
+                effective_local_check["issues"].append(
+                    _issue(
+                        "IMPORT_AUTO_NORMALIZATION_FAILED",
+                        f"Workbook compatibility normalization failed before backend verification: {exc}",
+                        severity="error",
+                    )
+                )
+                effective_local_check["warnings"].append(
+                    {
+                        "code": "IMPORT_AUTO_NORMALIZATION_FAILED",
+                        "message": "Workbook compatibility normalization failed during local precheck; returning a structured verification failure instead of crashing.",
+                    }
+                )
+                effective_local_check["local_precheck_passed"] = False
+                effective_local_check["can_import"] = False
+                effective_local_check["error_code"] = "IMPORT_VERIFICATION_FAILED"
             if auto_normalization is not None:
                 effective_path = Path(str(auto_normalization["verified_file_path"]))
                 effective_local_check = self._local_verify(
@@ -456,7 +476,7 @@ class ImportTools(ToolBase):
             return {
                 "ok": True,
                 "status": "success" if can_import else "failed",
-                "error_code": None if can_import else (local_check.get("error_code") or "IMPORT_VERIFICATION_FAILED"),
+                "error_code": None if can_import else (effective_local_check.get("error_code") or local_check.get("error_code") or "IMPORT_VERIFICATION_FAILED"),
                 "can_import": can_import,
                 "verification_id": verification_id,
                 "file_path": str(path.resolve()),
@@ -475,14 +495,14 @@ class ImportTools(ToolBase):
                     "import_auth_prechecked": precheck_known,
                     "import_auth_precheck_passed": True if precheck_known else None,
                     "import_auth_source": import_capability.get("auth_source"),
-                    "local_precheck_passed": bool(local_check["local_precheck_passed"]),
+                    "local_precheck_passed": bool(effective_local_check["local_precheck_passed"]),
                     "backend_verification_passed": isinstance(backend_verification, dict)
                     and backend_verification.get("beingValidated", True) is not False,
                     "schema_fingerprint": schema_fingerprint,
                     "file_sha256": local_check["file_sha256"],
                     "verified_file_sha256": effective_local_check["file_sha256"] if effective_path != path else None,
                     "file_format": local_check["extension"],
-                    "local_precheck_limited": bool(local_check["local_precheck_limited"]),
+                    "local_precheck_limited": bool(effective_local_check["local_precheck_limited"]),
                     "auto_normalized": effective_path != path,
                 },
             }
@@ -911,44 +931,59 @@ class ImportTools(ToolBase):
             base_result["can_import"] = False
             base_result["error_code"] = "IMPORT_VERIFICATION_FAILED"
             return base_result
-        sheet = workbook[workbook.sheetnames[0]]
-        header_row = [cell.value for cell in next(sheet.iter_rows(min_row=1, max_row=1), [])]
-        header_analysis = _analyze_headers(
-            header_row,
-            expected_columns,
-            allowed_titles=allowed_header_titles,
-        )
-        base_result["issues"].extend(header_analysis["issues"])
-        base_result["repair_suggestions"].extend(header_analysis["repair_suggestions"])
-        if not any(issue.get("severity") == "error" for issue in base_result["issues"]):
-            semantic_issues, semantic_warnings = self._inspect_semantic_cells(
-                profile=profile,
-                context=context,
-                sheet=sheet,
-                expected_columns=expected_columns,
-                field_index=field_index,
+        try:
+            sheet = workbook[workbook.sheetnames[0]]
+            header_row = [cell.value for cell in next(sheet.iter_rows(min_row=1, max_row=1), [])]
+            header_analysis = _analyze_headers(
+                header_row,
+                expected_columns,
+                allowed_titles=allowed_header_titles,
             )
-            base_result["issues"].extend(semantic_issues)
-            base_result["warnings"].extend(semantic_warnings)
-        trailing_blank_rows = _count_trailing_blank_rows(sheet)
-        if trailing_blank_rows > 0:
-            base_result["warnings"].append(
-                {
-                    "code": "TRAILING_BLANK_ROWS",
-                    "message": f"Workbook contains {trailing_blank_rows} trailing blank rows that can be safely removed.",
-                }
+            base_result["issues"].extend(header_analysis["issues"])
+            base_result["repair_suggestions"].extend(header_analysis["repair_suggestions"])
+            if not any(issue.get("severity") == "error" for issue in base_result["issues"]):
+                semantic_issues, semantic_warnings = self._inspect_semantic_cells(
+                    profile=profile,
+                    context=context,
+                    sheet=sheet,
+                    expected_columns=expected_columns,
+                    field_index=field_index,
+                )
+                base_result["issues"].extend(semantic_issues)
+                base_result["warnings"].extend(semantic_warnings)
+            trailing_blank_rows = _count_trailing_blank_rows(sheet)
+            if trailing_blank_rows > 0:
+                base_result["warnings"].append(
+                    {
+                        "code": "TRAILING_BLANK_ROWS",
+                        "message": f"Workbook contains {trailing_blank_rows} trailing blank rows that can be safely removed.",
+                    }
+                )
+                base_result["repair_suggestions"].append("trim_trailing_blank_rows")
+            enum_suggestions = _find_enum_repairs(sheet, expected_columns)
+            if enum_suggestions:
+                base_result["warnings"].append(
+                    {
+                        "code": "ENUM_VALUE_NORMALIZATION_AVAILABLE",
+                        "message": "Some enum-like cells can be normalized to exact template values without changing meaning.",
+                    }
+                )
+                base_result["repair_suggestions"].append("normalize_enum_values")
+            base_result["repair_suggestions"] = sorted(set(base_result["repair_suggestions"]))
+        except Exception as exc:
+            base_result["issues"].append(
+                _issue(
+                    "IMPORT_LOCAL_PRECHECK_FAILED",
+                    f"Workbook content could not be fully inspected during local precheck: {exc}",
+                    severity="error",
+                )
             )
-            base_result["repair_suggestions"].append("trim_trailing_blank_rows")
-        enum_suggestions = _find_enum_repairs(sheet, expected_columns)
-        if enum_suggestions:
             base_result["warnings"].append(
                 {
-                    "code": "ENUM_VALUE_NORMALIZATION_AVAILABLE",
-                    "message": "Some enum-like cells can be normalized to exact template values without changing meaning.",
+                    "code": "IMPORT_LOCAL_PRECHECK_FAILED",
+                    "message": "Workbook local precheck encountered an unexpected compatibility problem; returning a structured verification failure instead of crashing.",
                 }
             )
-            base_result["repair_suggestions"].append("normalize_enum_values")
-        base_result["repair_suggestions"] = sorted(set(base_result["repair_suggestions"]))
         if any(issue.get("severity") == "error" for issue in base_result["issues"]):
             base_result["local_precheck_passed"] = False
             base_result["can_import"] = False
@@ -1199,58 +1234,49 @@ class ImportTools(ToolBase):
     ) -> dict[str, Any] | None:
         if source_path.suffix.lower() != ".xlsx":
             return None
-        workbook = load_workbook(source_path, read_only=False, data_only=False)
-        if not workbook.sheetnames:
-            return None
-        sheet = workbook[workbook.sheetnames[0]]
-        header_depth = _infer_header_depth(sheet)
-        trailing_blank_rows = _count_trailing_blank_rows(sheet)
-        if header_depth <= 1 and trailing_blank_rows <= 0:
-            return None
-        extracted_headers = _extract_leaf_header_titles(sheet, header_depth)
-        target_headers = _overlay_header_titles(
-            extracted_headers,
-            template_header_profile.get("leaf_titles"),
-        )
-        verified_path = _resolve_verified_output_path(source_path)
-        normalized_workbook = Workbook()
-        normalized_sheet = normalized_workbook.active
-        normalized_sheet.title = sheet.title
-        normalized_sheet.append(target_headers)
-        last_nonblank_row = max(header_depth, sheet.max_row - trailing_blank_rows)
-        for row_index in range(header_depth + 1, last_nonblank_row + 1):
-            normalized_sheet.append(
-                [sheet.cell(row=row_index, column=column_index).value for column_index in range(1, sheet.max_column + 1)]
+        try:
+            workbook = load_workbook(source_path, read_only=False, data_only=False)
+            if not workbook.sheetnames:
+                return None
+            sheet = workbook[workbook.sheetnames[0]]
+            rows = [list(row) for row in sheet.iter_rows(values_only=True)]
+            header_depth = _infer_header_depth(sheet)
+            return _build_auto_normalized_file(
+                source_path=source_path,
+                sheet_title=sheet.title,
+                rows=rows,
+                header_depth=header_depth,
+                template_leaf_titles=template_header_profile.get("leaf_titles"),
+                local_check=local_check,
             )
-        verified_path.parent.mkdir(parents=True, exist_ok=True)
-        normalized_workbook.save(verified_path)
-        warnings: list[JSONObject] = []
-        applied_repairs: list[str] = []
-        if header_depth > 1:
-            applied_repairs.append("normalize_headers")
-            warnings.append(
-                {
-                    "code": "IMPORT_HEADERS_AUTO_NORMALIZED",
-                    "message": f"Workbook used {header_depth} header rows; record_import_verify normalized it to a single leaf-header row automatically.",
-                }
+        except Exception as exc:
+            workbook = load_workbook(source_path, read_only=True, data_only=False)
+            if not workbook.sheetnames:
+                return None
+            sheet = workbook[workbook.sheetnames[0]]
+            rows = [list(row) for row in sheet.iter_rows(values_only=True)]
+            header_depth = _infer_header_depth_from_rows(
+                rows,
+                template_header_profile=template_header_profile,
+                local_check=local_check,
             )
-        if trailing_blank_rows > 0:
-            applied_repairs.append("trim_trailing_blank_rows")
-            warnings.append(
-                {
-                    "code": "TRAILING_BLANK_ROWS_AUTO_TRIMMED",
-                    "message": f"Removed {trailing_blank_rows} trailing blank rows before backend verification.",
-                }
+            normalized = _build_auto_normalized_file(
+                source_path=source_path,
+                sheet_title=sheet.title,
+                rows=rows,
+                header_depth=header_depth,
+                template_leaf_titles=template_header_profile.get("leaf_titles"),
+                local_check=local_check,
             )
-        return {
-            "verified_file_path": str(verified_path.resolve()),
-            "header_titles": target_headers,
-            "warnings": warnings,
-            "applied_repairs": applied_repairs,
-            "header_depth": header_depth,
-            "trailing_blank_rows": trailing_blank_rows,
-            "source_local_check": local_check,
-        }
+            if normalized is not None:
+                normalized["warnings"].insert(
+                    0,
+                    {
+                        "code": "IMPORT_AUTO_NORMALIZATION_COMPATIBILITY_FALLBACK",
+                        "message": f"Workbook compatibility normalization retried in compatibility mode after a workbook parsing error: {exc}",
+                    },
+                )
+            return normalized
 
     def _fetch_import_capability(self, context, app_key: str) -> tuple[JSONObject, list[JSONObject]]:  # type: ignore[no-untyped-def]
         try:
@@ -1748,6 +1774,107 @@ def _overlay_header_titles(actual_titles: list[str], template_leaf_titles: Any)
     return normalized
 
 
+def _infer_header_depth_from_rows(
+    rows: list[list[Any]],
+    *,
+    template_header_profile: dict[str, Any],
+    local_check: dict[str, Any],
+) -> int:
+    template_depth = max(1, int(template_header_profile.get("header_depth") or 1))
+    header_depth = min(template_depth, max(1, len(rows)))
+    if header_depth > 1:
+        return header_depth
+    if "normalize_headers" in (local_check.get("repair_suggestions") or []) and len(rows) >= 2:
+        if any(_normalize_optional_text(value) for value in rows[1]):
+            return 2
+    return 1
+
+
+def _extract_leaf_header_titles_from_rows(rows: list[list[Any]], header_depth: int) -> list[str]:
+    titles: list[str] = []
+    max_column = max((len(row) for row in rows[: max(1, header_depth)]), default=0)
+    depth = max(1, min(header_depth, len(rows)))
+    for column_index in range(max_column):
+        selected = ""
+        for row_index in range(depth - 1, -1, -1):
+            value = rows[row_index][column_index] if column_index < len(rows[row_index]) else None
+            text = _normalize_optional_text(value)
+            if text:
+                selected = text
+                break
+        titles.append(selected)
+    return titles
+
+
+def _count_trailing_blank_rows_from_rows(rows: list[list[Any]], *, min_data_index: int = 1) -> int:
+    count = 0
+    for row in reversed(rows[min_data_index:]):
+        if any(value not in (None, "") for value in row):
+            break
+        count += 1
+    return count
+
+
+def _build_auto_normalized_file(
+    *,
+    source_path: Path,
+    sheet_title: str,
+    rows: list[list[Any]],
+    header_depth: int,
+    template_leaf_titles: Any,
+    local_check: dict[str, Any],
+) -> dict[str, Any] | None:
+    if not rows:
+        return None
+    normalized_header_depth = max(1, min(header_depth, len(rows)))
+    trailing_blank_rows = _count_trailing_blank_rows_from_rows(rows, min_data_index=normalized_header_depth)
+    if normalized_header_depth <= 1 and trailing_blank_rows <= 0:
+        return None
+    extracted_headers = _extract_leaf_header_titles_from_rows(rows, normalized_header_depth)
+    target_headers = _overlay_header_titles(extracted_headers, template_leaf_titles)
+    row_width = max(len(target_headers), max((len(row) for row in rows), default=0))
+    if row_width <= 0:
+        return None
+    padded_headers = list(target_headers) + [""] * max(0, row_width - len(target_headers))
+    verified_path = _resolve_verified_output_path(source_path)
+    normalized_workbook = Workbook()
+    normalized_sheet = normalized_workbook.active
+    normalized_sheet.title = sheet_title
+    normalized_sheet.append(padded_headers)
+    last_nonblank_row = max(normalized_header_depth, len(rows) - trailing_blank_rows)
+    for row in rows[normalized_header_depth:last_nonblank_row]:
+        normalized_sheet.append(list(row) + [None] * max(0, row_width - len(row)))
+    verified_path.parent.mkdir(parents=True, exist_ok=True)
+    normalized_workbook.save(verified_path)
+    warnings: list[JSONObject] = []
+    applied_repairs: list[str] = []
+    if normalized_header_depth > 1:
+        applied_repairs.append("normalize_headers")
+        warnings.append(
+            {
+                "code": "IMPORT_HEADERS_AUTO_NORMALIZED",
+                "message": f"Workbook used {normalized_header_depth} header rows; record_import_verify normalized it to a single leaf-header row automatically.",
+            }
+        )
+    if trailing_blank_rows > 0:
+        applied_repairs.append("trim_trailing_blank_rows")
+        warnings.append(
+            {
+                "code": "TRAILING_BLANK_ROWS_AUTO_TRIMMED",
+                "message": f"Removed {trailing_blank_rows} trailing blank rows before backend verification.",
+            }
+        )
+    return {
+        "verified_file_path": str(verified_path.resolve()),
+        "header_titles": target_headers or padded_headers,
+        "warnings": warnings,
+        "applied_repairs": applied_repairs,
+        "header_depth": normalized_header_depth,
+        "trailing_blank_rows": trailing_blank_rows,
+        "source_local_check": local_check,
+    }
+
+
 def _count_trailing_blank_rows(sheet) -> int:  # type: ignore[no-untyped-def]
     count = 0
     for row_index in range(sheet.max_row, 1, -1):