anymous 1.3.0 → 1.3.2

package/skills/xlsx/scripts/office/validators/redlining.py

@@ -1,299 +0,0 @@
-"""
-Validator for tracked changes in Word documents.
-
-Detects untracked edits in word/document.xml: text that differs from the
-original without a <w:ins>/<w:del> wrapper recording it. The tracked changes
-that are new relative to the original are undone, and the result is compared
-against the original; whatever text still differs was edited without being
-tracked.
-
-Only the document body is compared. Headers, footers, footnotes and endnotes
-are separate parts and are not checked.
-"""
-
-import subprocess
-import tempfile
-import zipfile
-from pathlib import Path
-
-import defusedxml.ElementTree as ET
-from defusedxml.common import DefusedXmlException
-
-from helpers import rendered_text, safe_extract
-
-
-class RedliningValidator:
-
-    def __init__(self, unpacked_dir, original_docx, verbose=False):
-        self.unpacked_dir = Path(unpacked_dir)
-        self.original_docx = Path(original_docx)
-        self.verbose = verbose
-        self.namespaces = {
-            "w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
-        }
-
-    def repair(self) -> int:
-        return 0
-
-    def validate(self):
-        modified_file = self.unpacked_dir / "word" / "document.xml"
-        if not modified_file.exists():
-            print(f"FAILED - Modified document.xml not found at {modified_file}")
-            return False
-
-        with tempfile.TemporaryDirectory() as temp_dir:
-            temp_path = Path(temp_dir)
-
-            try:
-                with zipfile.ZipFile(self.original_docx, "r") as zip_ref:
-                    safe_extract(zip_ref, temp_path)
-            except Exception as e:
-                print(f"FAILED - Error unpacking original docx: {e}")
-                return False
-
-            original_file = temp_path / "word" / "document.xml"
-            if not original_file.exists():
-                print(
-                    f"FAILED - Original document.xml not found in {self.original_docx}"
-                )
-                return False
-
-            try:
-                modified_tree = ET.parse(modified_file)
-                modified_root = modified_tree.getroot()
-                original_tree = ET.parse(original_file)
-                original_root = original_tree.getroot()
-            except (ET.ParseError, DefusedXmlException) as e:
-                print(f"FAILED - Error parsing XML files: {e}")
-                return False
-
-            new_changes = self._new_tracked_changes(original_root, modified_root)
-            self._remove_tracked_changes(modified_root, new_changes)
-
-            modified_text = self._extract_text_content(modified_root)
-            original_text = self._extract_text_content(original_root)
-
-            if modified_text != original_text:
-                error_message = self._generate_detailed_diff(
-                    original_text, modified_text
-                )
-                print(error_message)
-                return False
-
-            if self.verbose:
-                print(
-                    f"PASSED - All {len(new_changes)} change(s) against the original "
-                    "are properly tracked"
-                )
-            return True
-
-    def _tracked_change_elements(self, root):
-        ins_tag = f"{{{self.namespaces['w']}}}ins"
-        del_tag = f"{{{self.namespaces['w']}}}del"
-        return [elem for elem in root.iter() if elem.tag in (ins_tag, del_tag)]
-
-    def _rendered_text(self, elem):
-        preserve = elem.get("{http://www.w3.org/XML/1998/namespace}space") == "preserve"
-        return rendered_text(elem.text or "", preserve)
-
-    def _text_elements(self, elem):
-        w = self.namespaces["w"]
-        return [
-            node
-            for node in elem.iter()
-            if node.tag in (f"{{{w}}}t", f"{{{w}}}delText")
-        ]
-
-    def _tracked_change_key(self, elem):
-        w = self.namespaces["w"]
-        text = "".join(self._rendered_text(node) for node in self._text_elements(elem))
-        return (elem.tag, elem.get(f"{{{w}}}author"), elem.get(f"{{{w}}}date"), text)
-
-    def _new_tracked_changes(self, original_root, modified_root):
-        original = self._tracked_change_elements(original_root)
-        modified = self._tracked_change_elements(modified_root)
-
-        pool = {}
-        for elem in original:
-            pool.setdefault(self._tracked_change_key(elem), []).append(elem)
-
-        matched, leftover = set(), []
-        for elem in modified:
-            bucket = pool.get(self._tracked_change_key(elem))
-            if bucket:
-                matched.add(bucket.pop())
-            else:
-                leftover.append(elem)
-
-        def group(elem):
-            return self._tracked_change_key(elem)[:3]
-
-        def text_of(elems):
-            return "".join(self._tracked_change_key(e)[3] for e in elems)
-
-        unmatched_original = {}
-        for elem in original:
-            if elem not in matched:
-                unmatched_original.setdefault(group(elem), []).append(elem)
-
-        by_group = {}
-        for elem in leftover:
-            by_group.setdefault(group(elem), []).append(elem)
-
-        new = set()
-        for key, elems in by_group.items():
-            rebuilt = text_of(elems)
-            if rebuilt and rebuilt == text_of(unmatched_original.get(key, [])):
-                continue  
-            new.update(elems)
-        return new
-
-    def _generate_detailed_diff(self, original_text, modified_text):
-        error_parts = [
-            "FAILED - Document text doesn't match after removing the tracked changes",
-            "",
-            "Likely causes:",
-            "  1. Modified text inside another author's <w:ins> or <w:del> tags",
-            "  2. Made edits without proper tracked changes",
-            "  3. Didn't nest <w:del> inside <w:ins> when deleting another's insertion",
-            "  4. Rewrote another author's <w:ins>/<w:del> and changed its text on",
-            "     the way. A tracked change from the original is recognised by its",
-            "     author, date and text; anything that doesn't reproduce one exactly",
-            "     reads as new, and the text it carried is reported missing.",
-            "",
-            "For pre-redlined documents, use correct patterns:",
-            "  - To reject another's INSERTION: Nest <w:del> inside their <w:ins>",
-            "  - To reject PART of one: nest <w:del> around only the runs you reject.",
-            "    Their <w:ins> may be split around it, so long as the pieces keep",
-            "    their author and date and still spell out the same text.",
-            "  - To restore another's DELETION: Add new <w:ins> AFTER their <w:del>",
-            "",
-        ]
-
-        git_diff = self._get_git_word_diff(original_text, modified_text)
-        if git_diff:
-            error_parts.extend(["Differences:", "============", git_diff])
-        else:
-            error_parts.append("Unable to generate word diff (git not available)")
-
-        return "\n".join(error_parts)
-
-    def _get_git_word_diff(self, original_text, modified_text):
-        try:
-            with tempfile.TemporaryDirectory() as temp_dir:
-                temp_path = Path(temp_dir)
-
-                original_file = temp_path / "original.txt"
-                modified_file = temp_path / "modified.txt"
-
-                original_file.write_text(original_text, encoding="utf-8")
-                modified_file.write_text(modified_text, encoding="utf-8")
-
-                result = subprocess.run(
-                    [
-                        "git",
-                        "diff",
-                        "--word-diff=plain",
-                        "--word-diff-regex=.",  
-                        "-U0",  
-                        "--no-index",
-                        str(original_file),
-                        str(modified_file),
-                    ],
-                    capture_output=True,
-                    text=True,
-                )
-
-                if result.stdout.strip():
-                    lines = result.stdout.split("\n")
-                    content_lines = []
-                    in_content = False
-                    for line in lines:
-                        if line.startswith("@@"):
-                            in_content = True
-                            continue
-                        if in_content and line.strip():
-                            content_lines.append(line)
-
-                    if content_lines:
-                        return "\n".join(content_lines)
-
-                result = subprocess.run(
-                    [
-                        "git",
-                        "diff",
-                        "--word-diff=plain",
-                        "-U0",  
-                        "--no-index",
-                        str(original_file),
-                        str(modified_file),
-                    ],
-                    capture_output=True,
-                    text=True,
-                )
-
-                if result.stdout.strip():
-                    lines = result.stdout.split("\n")
-                    content_lines = []
-                    in_content = False
-                    for line in lines:
-                        if line.startswith("@@"):
-                            in_content = True
-                            continue
-                        if in_content and line.strip():
-                            content_lines.append(line)
-                    return "\n".join(content_lines)
-
-        except (subprocess.CalledProcessError, FileNotFoundError, Exception):
-            pass
-
-        return None
-
-    def _remove_tracked_changes(self, root, targets):
-        ins_tag = f"{{{self.namespaces['w']}}}ins"
-        del_tag = f"{{{self.namespaces['w']}}}del"
-
-        for parent in root.iter():
-            to_remove = []
-            for child in parent:
-                if child.tag == ins_tag and child in targets:
-                    to_remove.append(child)
-            for elem in to_remove:
-                parent.remove(elem)
-
-        deltext_tag = f"{{{self.namespaces['w']}}}delText"
-        t_tag = f"{{{self.namespaces['w']}}}t"
-
-        for parent in root.iter():
-            to_process = []
-            for child in parent:
-                if child.tag == del_tag and child in targets:
-                    to_process.append((child, list(parent).index(child)))
-
-            for del_elem, del_index in reversed(to_process):
-                for elem in del_elem.iter():
-                    if elem.tag == deltext_tag:
-                        elem.tag = t_tag
-
-                for child in reversed(list(del_elem)):
-                    parent.insert(del_index, child)
-                parent.remove(del_elem)
-
-    def _extract_text_content(self, root):
-        p_tag = f"{{{self.namespaces['w']}}}p"
-        t_tag = f"{{{self.namespaces['w']}}}t"
-
-        paragraphs = []
-        for p_elem in root.findall(f".//{p_tag}"):
-            text_parts = []
-            for t_elem in p_elem.findall(f".//{t_tag}"):
-                text_parts.append(self._rendered_text(t_elem))
-            paragraph_text = "".join(text_parts)
-            if paragraph_text:
-                paragraphs.append(paragraph_text)
-
-        return "\n".join(paragraphs)
-
-
-if __name__ == "__main__":
-    raise RuntimeError("This module should not be run directly.")

package/skills/xlsx/scripts/recalc.py

@@ -1,308 +0,0 @@
-"""
-Excel Formula Recalculation Script
-Recalculates all formulas in an Excel file using LibreOffice
-"""
-
-import contextlib
-import json
-import os
-import platform
-import re
-import shutil
-import subprocess
-import sys
-import tempfile
-import time
-import zipfile
-from pathlib import Path
-
-from office.soffice import get_soffice_env, run_soffice
-
-from openpyxl import load_workbook
-
-MACRO_FILENAME = "Module1.xba"
-SOFFICE_MISSING = "soffice not found on PATH; LibreOffice is required to recalculate"
-
-MAX_LOCATIONS = 100
-
-EXTERNAL_REF_RE = re.compile(r"""(?<![\w"\[])'?\[\d+\][^!"\[\]]*'?!""")
-
-RECALCULATE_MACRO = """<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE script:module PUBLIC "-//OpenOffice.org//DTD OfficeDocument 1.0//EN" "module.dtd">
-<script:module xmlns:script="http://openoffice.org/2000/script" script:name="Module1" script:language="StarBasic">
-    Sub RecalculateAndSave()
-      ThisComponent.calculateAll()
-      ThisComponent.store()
-      ThisComponent.close(True)
-    End Sub
-</script:module>"""
-
-
-def has_gtimeout():
-    try:
-        subprocess.run(
-            ["gtimeout", "--version"], capture_output=True, timeout=1, check=False
-        )
-        return True
-    except (FileNotFoundError, subprocess.TimeoutExpired):
-        return False
-
-
-def _stamp(path):
-    st = os.stat(path)
-    return st.st_mtime_ns, st.st_size
-
-
-def setup_libreoffice_macro(profile_dir: Path, timeout=30):
-    url = profile_dir.as_uri()
-    try:
-        run_soffice(
-            ["--headless", "--terminate_after_init", f"-env:UserInstallation={url}"],
-            capture_output=True,
-            timeout=timeout,
-        )
-    except FileNotFoundError:
-        return None, SOFFICE_MISSING
-    except subprocess.TimeoutExpired:
-        return None, "LibreOffice timed out creating its profile; formulas were NOT recalculated"
-
-    macro_dir = profile_dir / "user" / "basic" / "Standard"
-    if not macro_dir.exists():
-        return None, "LibreOffice did not create a usable profile; formulas were NOT recalculated"
-
-    try:
-        (macro_dir / MACRO_FILENAME).write_text(RECALCULATE_MACRO)
-    except OSError as e:
-        return None, f"Could not install the recalculation macro: {e}"
-
-    return url, None
-
-
-def external_links_at_risk(filename):
-    try:
-        with zipfile.ZipFile(filename) as archive:
-            names = archive.namelist()
-    except (zipfile.BadZipFile, OSError):
-        return []
-    if not any(n.startswith("xl/externalLinks/") for n in names):
-        return []
-
-    with contextlib.ExitStack() as stack:
-        formulas = load_workbook(filename, data_only=False)
-        stack.callback(formulas.close)
-        values = load_workbook(filename, data_only=True)
-        stack.callback(values.close)
-
-        external_names = [
-            name
-            for name, dn in formulas.defined_names.items()
-            if isinstance(getattr(dn, "value", None), str) and EXTERNAL_REF_RE.search(dn.value)
-        ]
-        name_re = (
-            re.compile(r"\b(" + "|".join(re.escape(n) for n in external_names) + r")\b")
-            if external_names
-            else None
-        )
-
-        at_risk = []
-        for sheet in formulas.sheetnames:
-            ws = formulas[sheet]
-            if not hasattr(ws, "iter_rows"):  
-                continue
-            cached = values[sheet]
-            for row in ws.iter_rows():
-                for cell in row:
-                    v = cell.value
-                    if not (isinstance(v, str) and v.startswith("=")):
-                        continue
-                    reaches_out = EXTERNAL_REF_RE.search(v) or (name_re and name_re.search(v))
-                    if reaches_out and cached[cell.coordinate].value is None:
-                        at_risk.append(f"{sheet}!{cell.coordinate}")
-        return at_risk
-
-
-def recalc(filename, timeout=30, force=False):
-    if not Path(filename).exists():
-        return {"error": f"File {filename} does not exist"}
-
-    abs_path = str(Path(filename).absolute())
-
-    if not os.access(abs_path, os.W_OK):
-        return {"error": f"{filename} is not writable; recalculation rewrites the file in place"}
-
-    try:
-        get_soffice_env()
-    except Exception as e:  
-        return {"error": f"Could not prepare the LibreOffice environment: {e}"}
-
-    if not force:
-        try:
-            at_risk = external_links_at_risk(filename)
-        except Exception as e:  
-            return {"error": f"Could not inspect {filename} for external links: {e}"}
-        if at_risk:
-            shown = at_risk[:MAX_LOCATIONS]
-            return {
-                "error": (
-                    "Refusing to recalculate: this workbook links to another workbook, and "
-                    f"{len(at_risk)} linked cell(s) have lost their cached value (openpyxl strips "
-                    "these on save). Recalculating would resolve them to #NAME? and delete the "
-                    "external links for good. Copy those cells' values from the original file "
-                    "before saving, or pass --force to accept the loss. Charts and conditional "
-                    "formats can hold external references too, so this list may not be exhaustive."
-                ),
-                "external_link_cells": shown,
-                "external_link_cells_truncated": max(0, len(at_risk) - len(shown)),
-            }
-
-    with tempfile.TemporaryDirectory(
-        prefix="recalc-lo-profile-", ignore_cleanup_errors=True
-    ) as profile_dir:
-        return _recalc_with_profile(filename, abs_path, timeout, Path(profile_dir))
-
-
-def _recalc_with_profile(filename, abs_path, timeout, profile_dir: Path):
-    started = time.monotonic()
-    profile_url, err = setup_libreoffice_macro(profile_dir, timeout=timeout)
-    if err:
-        return {"error": err}
-
-    timeout = max(5, int(timeout - (time.monotonic() - started)))
-
-    before = _stamp(abs_path)
-
-    cmd = [
-        "soffice",
-        "--headless",
-        "--norestore",
-        f"-env:UserInstallation={profile_url}",
-        "vnd.sun.star.script:Standard.Module1.RecalculateAndSave?language=Basic&location=application",
-        abs_path,
-    ]
-
-    if platform.system() == "Linux" and shutil.which("timeout"):
-        cmd = ["timeout", str(timeout)] + cmd
-    elif platform.system() == "Darwin" and has_gtimeout():
-        cmd = ["gtimeout", str(timeout)] + cmd
-
-    timed_out = f"LibreOffice timed out after {timeout}s; formulas were NOT recalculated. Re-run with a longer timeout."
-
-    try:
-        result = subprocess.run(
-            cmd, capture_output=True, text=True, env=get_soffice_env(), timeout=timeout + 15
-        )
-    except subprocess.TimeoutExpired:
-        return {"error": timed_out}
-    except FileNotFoundError:
-        return {"error": SOFFICE_MISSING}
-
-    if result.returncode == 124:
-        return {"error": timed_out}
-
-    if result.returncode != 0:
-        detail = (result.stderr or "").strip() or f"soffice exited {result.returncode}"
-        return {"error": f"LibreOffice failed to recalculate: {detail}"}
-
-    if _stamp(abs_path) == before:
-        return {
-            "error": (
-                "LibreOffice exited cleanly but never rewrote the file, so nothing was "
-                "recalculated. Check that no other LibreOffice instance is running, then retry."
-            )
-        }
-
-    try:
-        wb = load_workbook(filename, data_only=True)
-
-        excel_errors = [
-            "#VALUE!",
-            "#DIV/0!",
-            "#REF!",
-            "#NAME?",
-            "#NULL!",
-            "#NUM!",
-            "#N/A",
-        ]
-        error_details = {err: [] for err in excel_errors}
-        total_errors = 0
-
-        for sheet_name in wb.sheetnames:
-            ws = wb[sheet_name]
-            if not hasattr(ws, "iter_rows"):  
-                continue
-            for row in ws.iter_rows():
-                for cell in row:
-                    if cell.value is not None and isinstance(cell.value, str):
-                        for err in excel_errors:
-                            if err in cell.value:
-                                location = f"{sheet_name}!{cell.coordinate}"
-                                error_details[err].append(location)
-                                total_errors += 1
-                                break
-
-        result = {
-            "status": "success" if total_errors == 0 else "errors_found",
-            "total_errors": total_errors,
-            "error_summary": {},
-        }
-
-        for err_type, locations in error_details.items():
-            if locations:
-                entry = {"count": len(locations), "locations": locations[:MAX_LOCATIONS]}
-                if len(locations) > MAX_LOCATIONS:
-                    entry["locations_truncated"] = len(locations) - MAX_LOCATIONS
-                result["error_summary"][err_type] = entry
-
-        wb.close()
-
-        wb_formulas = load_workbook(filename, data_only=False)
-        formula_count = 0
-        for sheet_name in wb_formulas.sheetnames:
-            ws = wb_formulas[sheet_name]
-            if not hasattr(ws, "iter_rows"):  
-                continue
-            for row in ws.iter_rows():
-                for cell in row:
-                    if (
-                        cell.value
-                        and isinstance(cell.value, str)
-                        and cell.value.startswith("=")
-                    ):
-                        formula_count += 1
-        wb_formulas.close()
-
-        result["total_formulas"] = formula_count
-
-        return result
-
-    except Exception as e:
-        return {"error": str(e)}
-
-
-def main():
-    args = [a for a in sys.argv[1:] if a != "--force"]
-    force = "--force" in sys.argv[1:]
-
-    if not args:
-        print("Usage: python recalc.py <excel_file> [timeout_seconds] [--force]")
-        print("\nRecalculates all formulas in an Excel file using LibreOffice")
-        print("\nReturns JSON with error details:")
-        print("  - status: 'success' or 'errors_found'")
-        print("  - total_errors: Total number of Excel errors found")
-        print("  - total_formulas: Number of formulas in the file")
-        print("  - error_summary: Breakdown by error type with locations")
-        print("    - #VALUE!, #DIV/0!, #REF!, #NAME?, #NULL!, #NUM!, #N/A")
-        print("\nOn any failure the JSON has an 'error' key and no 'status'.")
-        print("--force recalculates even when it would destroy external links.")
-        sys.exit(1)
-
-    filename = args[0]
-    timeout = int(args[1]) if len(args) > 1 else 30
-
-    result = recalc(filename, timeout, force=force)
-    print(json.dumps(result, indent=2))
-    sys.exit(1 if "error" in result else 0)
-
-
-if __name__ == "__main__":
-    main()