@preapexis/piexis 0.1.2

package/skills/xlsx/scripts/office/validators/redlining.py

@@ -0,0 +1,299 @@
+"""
+Validator for tracked changes in Word documents.
+
+Detects untracked edits in word/document.xml: text that differs from the
+original without a <w:ins>/<w:del> wrapper recording it. The tracked changes
+that are new relative to the original are undone, and the result is compared
+against the original; whatever text still differs was edited without being
+tracked.
+
+Only the document body is compared. Headers, footers, footnotes and endnotes
+are separate parts and are not checked.
+"""
+
+import subprocess
+import tempfile
+import zipfile
+from pathlib import Path
+
+import defusedxml.ElementTree as ET
+from defusedxml.common import DefusedXmlException
+
+from helpers import rendered_text, safe_extract
+
+
+class RedliningValidator:
+
+    def __init__(self, unpacked_dir, original_docx, verbose=False):
+        self.unpacked_dir = Path(unpacked_dir)
+        self.original_docx = Path(original_docx)
+        self.verbose = verbose
+        self.namespaces = {
+            "w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
+        }
+
+    def repair(self) -> int:
+        return 0
+
+    def validate(self):
+        modified_file = self.unpacked_dir / "word" / "document.xml"
+        if not modified_file.exists():
+            print(f"FAILED - Modified document.xml not found at {modified_file}")
+            return False
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            temp_path = Path(temp_dir)
+
+            try:
+                with zipfile.ZipFile(self.original_docx, "r") as zip_ref:
+                    safe_extract(zip_ref, temp_path)
+            except Exception as e:
+                print(f"FAILED - Error unpacking original docx: {e}")
+                return False
+
+            original_file = temp_path / "word" / "document.xml"
+            if not original_file.exists():
+                print(
+                    f"FAILED - Original document.xml not found in {self.original_docx}"
+                )
+                return False
+
+            try:
+                modified_tree = ET.parse(modified_file)
+                modified_root = modified_tree.getroot()
+                original_tree = ET.parse(original_file)
+                original_root = original_tree.getroot()
+            except (ET.ParseError, DefusedXmlException) as e:
+                print(f"FAILED - Error parsing XML files: {e}")
+                return False
+
+            new_changes = self._new_tracked_changes(original_root, modified_root)
+            self._remove_tracked_changes(modified_root, new_changes)
+
+            modified_text = self._extract_text_content(modified_root)
+            original_text = self._extract_text_content(original_root)
+
+            if modified_text != original_text:
+                error_message = self._generate_detailed_diff(
+                    original_text, modified_text
+                )
+                print(error_message)
+                return False
+
+            if self.verbose:
+                print(
+                    f"PASSED - All {len(new_changes)} change(s) against the original "
+                    "are properly tracked"
+                )
+            return True
+
+    def _tracked_change_elements(self, root):
+        ins_tag = f"{{{self.namespaces['w']}}}ins"
+        del_tag = f"{{{self.namespaces['w']}}}del"
+        return [elem for elem in root.iter() if elem.tag in (ins_tag, del_tag)]
+
+    def _rendered_text(self, elem):
+        preserve = elem.get("{http://www.w3.org/XML/1998/namespace}space") == "preserve"
+        return rendered_text(elem.text or "", preserve)
+
+    def _text_elements(self, elem):
+        w = self.namespaces["w"]
+        return [
+            node
+            for node in elem.iter()
+            if node.tag in (f"{{{w}}}t", f"{{{w}}}delText")
+        ]
+
+    def _tracked_change_key(self, elem):
+        w = self.namespaces["w"]
+        text = "".join(self._rendered_text(node) for node in self._text_elements(elem))
+        return (elem.tag, elem.get(f"{{{w}}}author"), elem.get(f"{{{w}}}date"), text)
+
+    def _new_tracked_changes(self, original_root, modified_root):
+        original = self._tracked_change_elements(original_root)
+        modified = self._tracked_change_elements(modified_root)
+
+        pool = {}
+        for elem in original:
+            pool.setdefault(self._tracked_change_key(elem), []).append(elem)
+
+        matched, leftover = set(), []
+        for elem in modified:
+            bucket = pool.get(self._tracked_change_key(elem))
+            if bucket:
+                matched.add(bucket.pop())
+            else:
+                leftover.append(elem)
+
+        def group(elem):
+            return self._tracked_change_key(elem)[:3]
+
+        def text_of(elems):
+            return "".join(self._tracked_change_key(e)[3] for e in elems)
+
+        unmatched_original = {}
+        for elem in original:
+            if elem not in matched:
+                unmatched_original.setdefault(group(elem), []).append(elem)
+
+        by_group = {}
+        for elem in leftover:
+            by_group.setdefault(group(elem), []).append(elem)
+
+        new = set()
+        for key, elems in by_group.items():
+            rebuilt = text_of(elems)
+            if rebuilt and rebuilt == text_of(unmatched_original.get(key, [])):
+                continue  
+            new.update(elems)
+        return new
+
+    def _generate_detailed_diff(self, original_text, modified_text):
+        error_parts = [
+            "FAILED - Document text doesn't match after removing the tracked changes",
+            "",
+            "Likely causes:",
+            "  1. Modified text inside another author's <w:ins> or <w:del> tags",
+            "  2. Made edits without proper tracked changes",
+            "  3. Didn't nest <w:del> inside <w:ins> when deleting another's insertion",
+            "  4. Rewrote another author's <w:ins>/<w:del> and changed its text on",
+            "     the way. A tracked change from the original is recognised by its",
+            "     author, date and text; anything that doesn't reproduce one exactly",
+            "     reads as new, and the text it carried is reported missing.",
+            "",
+            "For pre-redlined documents, use correct patterns:",
+            "  - To reject another's INSERTION: Nest <w:del> inside their <w:ins>",
+            "  - To reject PART of one: nest <w:del> around only the runs you reject.",
+            "    Their <w:ins> may be split around it, so long as the pieces keep",
+            "    their author and date and still spell out the same text.",
+            "  - To restore another's DELETION: Add new <w:ins> AFTER their <w:del>",
+            "",
+        ]
+
+        git_diff = self._get_git_word_diff(original_text, modified_text)
+        if git_diff:
+            error_parts.extend(["Differences:", "============", git_diff])
+        else:
+            error_parts.append("Unable to generate word diff (git not available)")
+
+        return "\n".join(error_parts)
+
+    def _get_git_word_diff(self, original_text, modified_text):
+        try:
+            with tempfile.TemporaryDirectory() as temp_dir:
+                temp_path = Path(temp_dir)
+
+                original_file = temp_path / "original.txt"
+                modified_file = temp_path / "modified.txt"
+
+                original_file.write_text(original_text, encoding="utf-8")
+                modified_file.write_text(modified_text, encoding="utf-8")
+
+                result = subprocess.run(
+                    [
+                        "git",
+                        "diff",
+                        "--word-diff=plain",
+                        "--word-diff-regex=.",  
+                        "-U0",  
+                        "--no-index",
+                        str(original_file),
+                        str(modified_file),
+                    ],
+                    capture_output=True,
+                    text=True,
+                )
+
+                if result.stdout.strip():
+                    lines = result.stdout.split("\n")
+                    content_lines = []
+                    in_content = False
+                    for line in lines:
+                        if line.startswith("@@"):
+                            in_content = True
+                            continue
+                        if in_content and line.strip():
+                            content_lines.append(line)
+
+                    if content_lines:
+                        return "\n".join(content_lines)
+
+                result = subprocess.run(
+                    [
+                        "git",
+                        "diff",
+                        "--word-diff=plain",
+                        "-U0",  
+                        "--no-index",
+                        str(original_file),
+                        str(modified_file),
+                    ],
+                    capture_output=True,
+                    text=True,
+                )
+
+                if result.stdout.strip():
+                    lines = result.stdout.split("\n")
+                    content_lines = []
+                    in_content = False
+                    for line in lines:
+                        if line.startswith("@@"):
+                            in_content = True
+                            continue
+                        if in_content and line.strip():
+                            content_lines.append(line)
+                    return "\n".join(content_lines)
+
+        except (subprocess.CalledProcessError, FileNotFoundError, Exception):
+            pass
+
+        return None
+
+    def _remove_tracked_changes(self, root, targets):
+        ins_tag = f"{{{self.namespaces['w']}}}ins"
+        del_tag = f"{{{self.namespaces['w']}}}del"
+
+        for parent in root.iter():
+            to_remove = []
+            for child in parent:
+                if child.tag == ins_tag and child in targets:
+                    to_remove.append(child)
+            for elem in to_remove:
+                parent.remove(elem)
+
+        deltext_tag = f"{{{self.namespaces['w']}}}delText"
+        t_tag = f"{{{self.namespaces['w']}}}t"
+
+        for parent in root.iter():
+            to_process = []
+            for child in parent:
+                if child.tag == del_tag and child in targets:
+                    to_process.append((child, list(parent).index(child)))
+
+            for del_elem, del_index in reversed(to_process):
+                for elem in del_elem.iter():
+                    if elem.tag == deltext_tag:
+                        elem.tag = t_tag
+
+                for child in reversed(list(del_elem)):
+                    parent.insert(del_index, child)
+                parent.remove(del_elem)
+
+    def _extract_text_content(self, root):
+        p_tag = f"{{{self.namespaces['w']}}}p"
+        t_tag = f"{{{self.namespaces['w']}}}t"
+
+        paragraphs = []
+        for p_elem in root.findall(f".//{p_tag}"):
+            text_parts = []
+            for t_elem in p_elem.findall(f".//{t_tag}"):
+                text_parts.append(self._rendered_text(t_elem))
+            paragraph_text = "".join(text_parts)
+            if paragraph_text:
+                paragraphs.append(paragraph_text)
+
+        return "\n".join(paragraphs)
+
+
+if __name__ == "__main__":
+    raise RuntimeError("This module should not be run directly.")

package/skills/xlsx/scripts/recalc.py

@@ -0,0 +1,308 @@
+"""
+Excel Formula Recalculation Script
+Recalculates all formulas in an Excel file using LibreOffice
+"""
+
+import contextlib
+import json
+import os
+import platform
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+import zipfile
+from pathlib import Path
+
+from office.soffice import get_soffice_env, run_soffice
+
+from openpyxl import load_workbook
+
+MACRO_FILENAME = "Module1.xba"
+SOFFICE_MISSING = "soffice not found on PATH; LibreOffice is required to recalculate"
+
+MAX_LOCATIONS = 100
+
+EXTERNAL_REF_RE = re.compile(r"""(?<![\w"\[])'?\[\d+\][^!"\[\]]*'?!""")
+
+RECALCULATE_MACRO = """<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE script:module PUBLIC "-//OpenOffice.org//DTD OfficeDocument 1.0//EN" "module.dtd">
+<script:module xmlns:script="http://openoffice.org/2000/script" script:name="Module1" script:language="StarBasic">
+    Sub RecalculateAndSave()
+      ThisComponent.calculateAll()
+      ThisComponent.store()
+      ThisComponent.close(True)
+    End Sub
+</script:module>"""
+
+
+def has_gtimeout():
+    try:
+        subprocess.run(
+            ["gtimeout", "--version"], capture_output=True, timeout=1, check=False
+        )
+        return True
+    except (FileNotFoundError, subprocess.TimeoutExpired):
+        return False
+
+
+def _stamp(path):
+    st = os.stat(path)
+    return st.st_mtime_ns, st.st_size
+
+
+def setup_libreoffice_macro(profile_dir: Path, timeout=30):
+    url = profile_dir.as_uri()
+    try:
+        run_soffice(
+            ["--headless", "--terminate_after_init", f"-env:UserInstallation={url}"],
+            capture_output=True,
+            timeout=timeout,
+        )
+    except FileNotFoundError:
+        return None, SOFFICE_MISSING
+    except subprocess.TimeoutExpired:
+        return None, "LibreOffice timed out creating its profile; formulas were NOT recalculated"
+
+    macro_dir = profile_dir / "user" / "basic" / "Standard"
+    if not macro_dir.exists():
+        return None, "LibreOffice did not create a usable profile; formulas were NOT recalculated"
+
+    try:
+        (macro_dir / MACRO_FILENAME).write_text(RECALCULATE_MACRO)
+    except OSError as e:
+        return None, f"Could not install the recalculation macro: {e}"
+
+    return url, None
+
+
+def external_links_at_risk(filename):
+    try:
+        with zipfile.ZipFile(filename) as archive:
+            names = archive.namelist()
+    except (zipfile.BadZipFile, OSError):
+        return []
+    if not any(n.startswith("xl/externalLinks/") for n in names):
+        return []
+
+    with contextlib.ExitStack() as stack:
+        formulas = load_workbook(filename, data_only=False)
+        stack.callback(formulas.close)
+        values = load_workbook(filename, data_only=True)
+        stack.callback(values.close)
+
+        external_names = [
+            name
+            for name, dn in formulas.defined_names.items()
+            if isinstance(getattr(dn, "value", None), str) and EXTERNAL_REF_RE.search(dn.value)
+        ]
+        name_re = (
+            re.compile(r"\b(" + "|".join(re.escape(n) for n in external_names) + r")\b")
+            if external_names
+            else None
+        )
+
+        at_risk = []
+        for sheet in formulas.sheetnames:
+            ws = formulas[sheet]
+            if not hasattr(ws, "iter_rows"):  
+                continue
+            cached = values[sheet]
+            for row in ws.iter_rows():
+                for cell in row:
+                    v = cell.value
+                    if not (isinstance(v, str) and v.startswith("=")):
+                        continue
+                    reaches_out = EXTERNAL_REF_RE.search(v) or (name_re and name_re.search(v))
+                    if reaches_out and cached[cell.coordinate].value is None:
+                        at_risk.append(f"{sheet}!{cell.coordinate}")
+        return at_risk
+
+
+def recalc(filename, timeout=30, force=False):
+    if not Path(filename).exists():
+        return {"error": f"File {filename} does not exist"}
+
+    abs_path = str(Path(filename).absolute())
+
+    if not os.access(abs_path, os.W_OK):
+        return {"error": f"{filename} is not writable; recalculation rewrites the file in place"}
+
+    try:
+        get_soffice_env()
+    except Exception as e:  
+        return {"error": f"Could not prepare the LibreOffice environment: {e}"}
+
+    if not force:
+        try:
+            at_risk = external_links_at_risk(filename)
+        except Exception as e:  
+            return {"error": f"Could not inspect {filename} for external links: {e}"}
+        if at_risk:
+            shown = at_risk[:MAX_LOCATIONS]
+            return {
+                "error": (
+                    "Refusing to recalculate: this workbook links to another workbook, and "
+                    f"{len(at_risk)} linked cell(s) have lost their cached value (openpyxl strips "
+                    "these on save). Recalculating would resolve them to #NAME? and delete the "
+                    "external links for good. Copy those cells' values from the original file "
+                    "before saving, or pass --force to accept the loss. Charts and conditional "
+                    "formats can hold external references too, so this list may not be exhaustive."
+                ),
+                "external_link_cells": shown,
+                "external_link_cells_truncated": max(0, len(at_risk) - len(shown)),
+            }
+
+    with tempfile.TemporaryDirectory(
+        prefix="recalc-lo-profile-", ignore_cleanup_errors=True
+    ) as profile_dir:
+        return _recalc_with_profile(filename, abs_path, timeout, Path(profile_dir))
+
+
+def _recalc_with_profile(filename, abs_path, timeout, profile_dir: Path):
+    started = time.monotonic()
+    profile_url, err = setup_libreoffice_macro(profile_dir, timeout=timeout)
+    if err:
+        return {"error": err}
+
+    timeout = max(5, int(timeout - (time.monotonic() - started)))
+
+    before = _stamp(abs_path)
+
+    cmd = [
+        "soffice",
+        "--headless",
+        "--norestore",
+        f"-env:UserInstallation={profile_url}",
+        "vnd.sun.star.script:Standard.Module1.RecalculateAndSave?language=Basic&location=application",
+        abs_path,
+    ]
+
+    if platform.system() == "Linux" and shutil.which("timeout"):
+        cmd = ["timeout", str(timeout)] + cmd
+    elif platform.system() == "Darwin" and has_gtimeout():
+        cmd = ["gtimeout", str(timeout)] + cmd
+
+    timed_out = f"LibreOffice timed out after {timeout}s; formulas were NOT recalculated. Re-run with a longer timeout."
+
+    try:
+        result = subprocess.run(
+            cmd, capture_output=True, text=True, env=get_soffice_env(), timeout=timeout + 15
+        )
+    except subprocess.TimeoutExpired:
+        return {"error": timed_out}
+    except FileNotFoundError:
+        return {"error": SOFFICE_MISSING}
+
+    if result.returncode == 124:
+        return {"error": timed_out}
+
+    if result.returncode != 0:
+        detail = (result.stderr or "").strip() or f"soffice exited {result.returncode}"
+        return {"error": f"LibreOffice failed to recalculate: {detail}"}
+
+    if _stamp(abs_path) == before:
+        return {
+            "error": (
+                "LibreOffice exited cleanly but never rewrote the file, so nothing was "
+                "recalculated. Check that no other LibreOffice instance is running, then retry."
+            )
+        }
+
+    try:
+        wb = load_workbook(filename, data_only=True)
+
+        excel_errors = [
+            "#VALUE!",
+            "#DIV/0!",
+            "#REF!",
+            "#NAME?",
+            "#NULL!",
+            "#NUM!",
+            "#N/A",
+        ]
+        error_details = {err: [] for err in excel_errors}
+        total_errors = 0
+
+        for sheet_name in wb.sheetnames:
+            ws = wb[sheet_name]
+            if not hasattr(ws, "iter_rows"):  
+                continue
+            for row in ws.iter_rows():
+                for cell in row:
+                    if cell.value is not None and isinstance(cell.value, str):
+                        for err in excel_errors:
+                            if err in cell.value:
+                                location = f"{sheet_name}!{cell.coordinate}"
+                                error_details[err].append(location)
+                                total_errors += 1
+                                break
+
+        result = {
+            "status": "success" if total_errors == 0 else "errors_found",
+            "total_errors": total_errors,
+            "error_summary": {},
+        }
+
+        for err_type, locations in error_details.items():
+            if locations:
+                entry = {"count": len(locations), "locations": locations[:MAX_LOCATIONS]}
+                if len(locations) > MAX_LOCATIONS:
+                    entry["locations_truncated"] = len(locations) - MAX_LOCATIONS
+                result["error_summary"][err_type] = entry
+
+        wb.close()
+
+        wb_formulas = load_workbook(filename, data_only=False)
+        formula_count = 0
+        for sheet_name in wb_formulas.sheetnames:
+            ws = wb_formulas[sheet_name]
+            if not hasattr(ws, "iter_rows"):  
+                continue
+            for row in ws.iter_rows():
+                for cell in row:
+                    if (
+                        cell.value
+                        and isinstance(cell.value, str)
+                        and cell.value.startswith("=")
+                    ):
+                        formula_count += 1
+        wb_formulas.close()
+
+        result["total_formulas"] = formula_count
+
+        return result
+
+    except Exception as e:
+        return {"error": str(e)}
+
+
+def main():
+    args = [a for a in sys.argv[1:] if a != "--force"]
+    force = "--force" in sys.argv[1:]
+
+    if not args:
+        print("Usage: python recalc.py <excel_file> [timeout_seconds] [--force]")
+        print("\nRecalculates all formulas in an Excel file using LibreOffice")
+        print("\nReturns JSON with error details:")
+        print("  - status: 'success' or 'errors_found'")
+        print("  - total_errors: Total number of Excel errors found")
+        print("  - total_formulas: Number of formulas in the file")
+        print("  - error_summary: Breakdown by error type with locations")
+        print("    - #VALUE!, #DIV/0!, #REF!, #NAME?, #NULL!, #NUM!, #N/A")
+        print("\nOn any failure the JSON has an 'error' key and no 'status'.")
+        print("--force recalculates even when it would destroy external links.")
+        sys.exit(1)
+
+    filename = args[0]
+    timeout = int(args[1]) if len(args) > 1 else 30
+
+    result = recalc(filename, timeout, force=force)
+    print(json.dumps(result, indent=2))
+    sys.exit(1 if "error" in result else 0)
+
+
+if __name__ == "__main__":
+    main()