@aslomon/effectum 0.3.3 → 0.4.0

package/system/skills/docx/scripts/office/schemas/microsoft/wml-cex-2018.xsd

@@ -0,0 +1,20 @@
+ <xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main" xmlns:s="http://schemas.openxmlformats.org/officeDocument/2006/sharedTypes" xmlns:w16="http://schemas.microsoft.com/office/word/2018/wordml" elementFormDefault="qualified" attributeFormDefault="qualified" blockDefault="#all" xmlns="http://schemas.microsoft.com/office/word/2018/wordml/cex" targetNamespace="http://schemas.microsoft.com/office/word/2018/wordml/cex">
+   <xsd:import id="w16" namespace="http://schemas.microsoft.com/office/word/2018/wordml" schemaLocation="wml-2018.xsd"/>
+   <xsd:import id="w" namespace="http://schemas.openxmlformats.org/wordprocessingml/2006/main" schemaLocation="../ISO-IEC29500-4_2016/wml.xsd"/>
+   <xsd:import id="s" namespace="http://schemas.openxmlformats.org/officeDocument/2006/sharedTypes" schemaLocation="../ISO-IEC29500-4_2016/shared-commonSimpleTypes.xsd"/>
+   <xsd:complexType name="CT_CommentsExtensible">
+     <xsd:sequence>
+       <xsd:element name="commentExtensible" type="CT_CommentExtensible" minOccurs="0" maxOccurs="unbounded"/>
+       <xsd:element name="extLst" type="w16:CT_ExtensionList" minOccurs="0" maxOccurs="1"/>
+     </xsd:sequence>
+   </xsd:complexType>
+   <xsd:complexType name="CT_CommentExtensible">
+     <xsd:sequence>
+       <xsd:element name="extLst" type="w16:CT_ExtensionList" minOccurs="0" maxOccurs="1"/>
+     </xsd:sequence>
+     <xsd:attribute name="durableId" type="w:ST_LongHexNumber" use="required"/>
+     <xsd:attribute name="dateUtc" type="w:ST_DateTime" use="optional"/>
+     <xsd:attribute name="intelligentPlaceholder" type="s:ST_OnOff" use="optional"/>
+   </xsd:complexType>
+   <xsd:element name="commentsExtensible" type="CT_CommentsExtensible"/>
+ </xsd:schema>

package/system/skills/docx/scripts/office/schemas/microsoft/wml-cid-2016.xsd

@@ -0,0 +1,13 @@
+ <xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:w12="http://schemas.openxmlformats.org/wordprocessingml/2006/main" elementFormDefault="qualified" attributeFormDefault="qualified" blockDefault="#all" xmlns="http://schemas.microsoft.com/office/word/2016/wordml/cid" targetNamespace="http://schemas.microsoft.com/office/word/2016/wordml/cid">
+   <xsd:import id="w12" namespace="http://schemas.openxmlformats.org/wordprocessingml/2006/main" schemaLocation="../ISO-IEC29500-4_2016/wml.xsd"/>
+   <xsd:complexType name="CT_CommentsIds">
+     <xsd:sequence>
+       <xsd:element name="commentId" type="CT_CommentId" minOccurs="0" maxOccurs="unbounded"/>
+     </xsd:sequence>
+   </xsd:complexType>
+   <xsd:complexType name="CT_CommentId">
+     <xsd:attribute name="paraId" type="w12:ST_LongHexNumber" use="required"/>
+     <xsd:attribute name="durableId" type="w12:ST_LongHexNumber" use="required"/>
+   </xsd:complexType>
+   <xsd:element name="commentsIds" type="CT_CommentsIds"/>
+ </xsd:schema>

package/system/skills/docx/scripts/office/schemas/microsoft/wml-sdtdatahash-2020.xsd

@@ -0,0 +1,4 @@
+ <xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:w12="http://schemas.openxmlformats.org/wordprocessingml/2006/main" elementFormDefault="qualified" attributeFormDefault="qualified" blockDefault="#all" xmlns="http://schemas.microsoft.com/office/word/2020/wordml/sdtdatahash" targetNamespace="http://schemas.microsoft.com/office/word/2020/wordml/sdtdatahash">
+   <xsd:import id="w12" namespace="http://schemas.openxmlformats.org/wordprocessingml/2006/main" schemaLocation="../ISO-IEC29500-4_2016/wml.xsd"/>
+   <xsd:attribute name="storeItemChecksum" type="w12:ST_String"/>
+ </xsd:schema>

package/system/skills/docx/scripts/office/schemas/microsoft/wml-symex-2015.xsd

@@ -0,0 +1,8 @@
+ <xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:w12="http://schemas.openxmlformats.org/wordprocessingml/2006/main" elementFormDefault="qualified" attributeFormDefault="qualified" blockDefault="#all" xmlns="http://schemas.microsoft.com/office/word/2015/wordml/symex" targetNamespace="http://schemas.microsoft.com/office/word/2015/wordml/symex">
+   <xsd:import id="w12" namespace="http://schemas.openxmlformats.org/wordprocessingml/2006/main" schemaLocation="../ISO-IEC29500-4_2016/wml.xsd"/>
+   <xsd:complexType name="CT_SymEx">
+     <xsd:attribute name="font" type="w12:ST_String"/>
+     <xsd:attribute name="char" type="w12:ST_LongHexNumber"/>
+   </xsd:complexType>
+   <xsd:element name="symEx" type="CT_SymEx"/>
+ </xsd:schema>

package/system/skills/docx/scripts/office/soffice.py

@@ -0,0 +1,183 @@
+"""
+Helper for running LibreOffice (soffice) in environments where AF_UNIX
+sockets may be blocked (e.g., sandboxed VMs).  Detects the restriction
+at runtime and applies an LD_PRELOAD shim if needed.
+
+Usage:
+    from office.soffice import run_soffice, get_soffice_env
+
+    # Option 1 – run soffice directly
+    result = run_soffice(["--headless", "--convert-to", "pdf", "input.docx"])
+
+    # Option 2 – get env dict for your own subprocess calls
+    env = get_soffice_env()
+    subprocess.run(["soffice", ...], env=env)
+"""
+
+import os
+import socket
+import subprocess
+import tempfile
+from pathlib import Path
+
+
+def get_soffice_env() -> dict:
+    env = os.environ.copy()
+    env["SAL_USE_VCLPLUGIN"] = "svp"
+
+    if _needs_shim():
+        shim = _ensure_shim()
+        env["LD_PRELOAD"] = str(shim)
+
+    return env
+
+
+def run_soffice(args: list[str], **kwargs) -> subprocess.CompletedProcess:
+    env = get_soffice_env()
+    return subprocess.run(["soffice"] + args, env=env, **kwargs)
+
+
+
+_SHIM_SO = Path(tempfile.gettempdir()) / "lo_socket_shim.so"
+
+
+def _needs_shim() -> bool:
+    try:
+        s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        s.close()
+        return False
+    except OSError:
+        return True
+
+
+def _ensure_shim() -> Path:
+    if _SHIM_SO.exists():
+        return _SHIM_SO
+
+    src = Path(tempfile.gettempdir()) / "lo_socket_shim.c"
+    src.write_text(_SHIM_SOURCE)
+    subprocess.run(
+        ["gcc", "-shared", "-fPIC", "-o", str(_SHIM_SO), str(src), "-ldl"],
+        check=True,
+        capture_output=True,
+    )
+    src.unlink()
+    return _SHIM_SO
+
+
+
+_SHIM_SOURCE = r"""
+#define _GNU_SOURCE
+#include <dlfcn.h>
+#include <errno.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+static int (*real_socket)(int, int, int);
+static int (*real_socketpair)(int, int, int, int[2]);
+static int (*real_listen)(int, int);
+static int (*real_accept)(int, struct sockaddr *, socklen_t *);
+static int (*real_close)(int);
+static int (*real_read)(int, void *, size_t);
+
+/* Per-FD bookkeeping (FDs >= 1024 are passed through unshimmed). */
+static int is_shimmed[1024];
+static int peer_of[1024];
+static int wake_r[1024];            /* accept() blocks reading this */
+static int wake_w[1024];            /* close()  writes to this      */
+static int listener_fd = -1;        /* FD that received listen()    */
+
+__attribute__((constructor))
+static void init(void) {
+    real_socket     = dlsym(RTLD_NEXT, "socket");
+    real_socketpair = dlsym(RTLD_NEXT, "socketpair");
+    real_listen     = dlsym(RTLD_NEXT, "listen");
+    real_accept     = dlsym(RTLD_NEXT, "accept");
+    real_close      = dlsym(RTLD_NEXT, "close");
+    real_read       = dlsym(RTLD_NEXT, "read");
+    for (int i = 0; i < 1024; i++) {
+        peer_of[i] = -1;
+        wake_r[i]  = -1;
+        wake_w[i]  = -1;
+    }
+}
+
+/* ---- socket ---------------------------------------------------------- */
+int socket(int domain, int type, int protocol) {
+    if (domain == AF_UNIX) {
+        int fd = real_socket(domain, type, protocol);
+        if (fd >= 0) return fd;
+        /* socket(AF_UNIX) blocked – fall back to socketpair(). */
+        int sv[2];
+        if (real_socketpair(domain, type, protocol, sv) == 0) {
+            if (sv[0] >= 0 && sv[0] < 1024) {
+                is_shimmed[sv[0]] = 1;
+                peer_of[sv[0]]    = sv[1];
+                int wp[2];
+                if (pipe(wp) == 0) {
+                    wake_r[sv[0]] = wp[0];
+                    wake_w[sv[0]] = wp[1];
+                }
+            }
+            return sv[0];
+        }
+        errno = EPERM;
+        return -1;
+    }
+    return real_socket(domain, type, protocol);
+}
+
+/* ---- listen ---------------------------------------------------------- */
+int listen(int sockfd, int backlog) {
+    if (sockfd >= 0 && sockfd < 1024 && is_shimmed[sockfd]) {
+        listener_fd = sockfd;
+        return 0;
+    }
+    return real_listen(sockfd, backlog);
+}
+
+/* ---- accept ---------------------------------------------------------- */
+int accept(int sockfd, struct sockaddr *addr, socklen_t *addrlen) {
+    if (sockfd >= 0 && sockfd < 1024 && is_shimmed[sockfd]) {
+        /* Block until close() writes to the wake pipe. */
+        if (wake_r[sockfd] >= 0) {
+            char buf;
+            real_read(wake_r[sockfd], &buf, 1);
+        }
+        errno = ECONNABORTED;
+        return -1;
+    }
+    return real_accept(sockfd, addr, addrlen);
+}
+
+/* ---- close ----------------------------------------------------------- */
+int close(int fd) {
+    if (fd >= 0 && fd < 1024 && is_shimmed[fd]) {
+        int was_listener = (fd == listener_fd);
+        is_shimmed[fd] = 0;
+
+        if (wake_w[fd] >= 0) {              /* unblock accept() */
+            char c = 0;
+            write(wake_w[fd], &c, 1);
+            real_close(wake_w[fd]);
+            wake_w[fd] = -1;
+        }
+        if (wake_r[fd] >= 0) { real_close(wake_r[fd]); wake_r[fd]  = -1; }
+        if (peer_of[fd] >= 0) { real_close(peer_of[fd]); peer_of[fd] = -1; }
+
+        if (was_listener)
+            _exit(0);                        /* conversion done – exit */
+    }
+    return real_close(fd);
+}
+"""
+
+
+
+if __name__ == "__main__":
+    import sys
+    result = run_soffice(sys.argv[1:])
+    sys.exit(result.returncode)

package/system/skills/docx/scripts/office/unpack.py

@@ -0,0 +1,132 @@
+"""Unpack Office files (DOCX, PPTX, XLSX) for editing.
+
+Extracts the ZIP archive, pretty-prints XML files, and optionally:
+- Merges adjacent runs with identical formatting (DOCX only)
+- Simplifies adjacent tracked changes from same author (DOCX only)
+
+Usage:
+    python unpack.py <office_file> <output_dir> [options]
+
+Examples:
+    python unpack.py document.docx unpacked/
+    python unpack.py presentation.pptx unpacked/
+    python unpack.py document.docx unpacked/ --merge-runs false
+"""
+
+import argparse
+import sys
+import zipfile
+from pathlib import Path
+
+import defusedxml.minidom
+
+from helpers.merge_runs import merge_runs as do_merge_runs
+from helpers.simplify_redlines import simplify_redlines as do_simplify_redlines
+
+SMART_QUOTE_REPLACEMENTS = {
+    "\u201c": "&#x201C;",  
+    "\u201d": "&#x201D;",  
+    "\u2018": "&#x2018;",  
+    "\u2019": "&#x2019;",  
+}
+
+
+def unpack(
+    input_file: str,
+    output_directory: str,
+    merge_runs: bool = True,
+    simplify_redlines: bool = True,
+) -> tuple[None, str]:
+    input_path = Path(input_file)
+    output_path = Path(output_directory)
+    suffix = input_path.suffix.lower()
+
+    if not input_path.exists():
+        return None, f"Error: {input_file} does not exist"
+
+    if suffix not in {".docx", ".pptx", ".xlsx"}:
+        return None, f"Error: {input_file} must be a .docx, .pptx, or .xlsx file"
+
+    try:
+        output_path.mkdir(parents=True, exist_ok=True)
+
+        with zipfile.ZipFile(input_path, "r") as zf:
+            zf.extractall(output_path)
+
+        xml_files = list(output_path.rglob("*.xml")) + list(output_path.rglob("*.rels"))
+        for xml_file in xml_files:
+            _pretty_print_xml(xml_file)
+
+        message = f"Unpacked {input_file} ({len(xml_files)} XML files)"
+
+        if suffix == ".docx":
+            if simplify_redlines:
+                simplify_count, _ = do_simplify_redlines(str(output_path))
+                message += f", simplified {simplify_count} tracked changes"
+
+            if merge_runs:
+                merge_count, _ = do_merge_runs(str(output_path))
+                message += f", merged {merge_count} runs"
+
+        for xml_file in xml_files:
+            _escape_smart_quotes(xml_file)
+
+        return None, message
+
+    except zipfile.BadZipFile:
+        return None, f"Error: {input_file} is not a valid Office file"
+    except Exception as e:
+        return None, f"Error unpacking: {e}"
+
+
+def _pretty_print_xml(xml_file: Path) -> None:
+    try:
+        content = xml_file.read_text(encoding="utf-8")
+        dom = defusedxml.minidom.parseString(content)
+        xml_file.write_bytes(dom.toprettyxml(indent="  ", encoding="utf-8"))
+    except Exception:
+        pass  
+
+
+def _escape_smart_quotes(xml_file: Path) -> None:
+    try:
+        content = xml_file.read_text(encoding="utf-8")
+        for char, entity in SMART_QUOTE_REPLACEMENTS.items():
+            content = content.replace(char, entity)
+        xml_file.write_text(content, encoding="utf-8")
+    except Exception:
+        pass
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Unpack an Office file (DOCX, PPTX, XLSX) for editing"
+    )
+    parser.add_argument("input_file", help="Office file to unpack")
+    parser.add_argument("output_directory", help="Output directory")
+    parser.add_argument(
+        "--merge-runs",
+        type=lambda x: x.lower() == "true",
+        default=True,
+        metavar="true|false",
+        help="Merge adjacent runs with identical formatting (DOCX only, default: true)",
+    )
+    parser.add_argument(
+        "--simplify-redlines",
+        type=lambda x: x.lower() == "true",
+        default=True,
+        metavar="true|false",
+        help="Merge adjacent tracked changes from same author (DOCX only, default: true)",
+    )
+    args = parser.parse_args()
+
+    _, message = unpack(
+        args.input_file,
+        args.output_directory,
+        merge_runs=args.merge_runs,
+        simplify_redlines=args.simplify_redlines,
+    )
+    print(message)
+
+    if "Error" in message:
+        sys.exit(1)

package/system/skills/docx/scripts/office/validate.py

@@ -0,0 +1,111 @@
+"""
+Command line tool to validate Office document XML files against XSD schemas and tracked changes.
+
+Usage:
+    python validate.py <path> [--original <original_file>] [--auto-repair] [--author NAME]
+
+The first argument can be either:
+- An unpacked directory containing the Office document XML files
+- A packed Office file (.docx/.pptx/.xlsx) which will be unpacked to a temp directory
+
+Auto-repair fixes:
+- paraId/durableId values that exceed OOXML limits
+- Missing xml:space="preserve" on w:t elements with whitespace
+"""
+
+import argparse
+import sys
+import tempfile
+import zipfile
+from pathlib import Path
+
+from validators import DOCXSchemaValidator, PPTXSchemaValidator, RedliningValidator
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Validate Office document XML files")
+    parser.add_argument(
+        "path",
+        help="Path to unpacked directory or packed Office file (.docx/.pptx/.xlsx)",
+    )
+    parser.add_argument(
+        "--original",
+        required=False,
+        default=None,
+        help="Path to original file (.docx/.pptx/.xlsx). If omitted, all XSD errors are reported and redlining validation is skipped.",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="Enable verbose output",
+    )
+    parser.add_argument(
+        "--auto-repair",
+        action="store_true",
+        help="Automatically repair common issues (hex IDs, whitespace preservation)",
+    )
+    parser.add_argument(
+        "--author",
+        default="Claude",
+        help="Author name for redlining validation (default: Claude)",
+    )
+    args = parser.parse_args()
+
+    path = Path(args.path)
+    assert path.exists(), f"Error: {path} does not exist"
+
+    original_file = None
+    if args.original:
+        original_file = Path(args.original)
+        assert original_file.is_file(), f"Error: {original_file} is not a file"
+        assert original_file.suffix.lower() in [".docx", ".pptx", ".xlsx"], (
+            f"Error: {original_file} must be a .docx, .pptx, or .xlsx file"
+        )
+
+    file_extension = (original_file or path).suffix.lower()
+    assert file_extension in [".docx", ".pptx", ".xlsx"], (
+        f"Error: Cannot determine file type from {path}. Use --original or provide a .docx/.pptx/.xlsx file."
+    )
+
+    if path.is_file() and path.suffix.lower() in [".docx", ".pptx", ".xlsx"]:
+        temp_dir = tempfile.mkdtemp()
+        with zipfile.ZipFile(path, "r") as zf:
+            zf.extractall(temp_dir)
+        unpacked_dir = Path(temp_dir)
+    else:
+        assert path.is_dir(), f"Error: {path} is not a directory or Office file"
+        unpacked_dir = path
+
+    match file_extension:
+        case ".docx":
+            validators = [
+                DOCXSchemaValidator(unpacked_dir, original_file, verbose=args.verbose),
+            ]
+            if original_file:
+                validators.append(
+                    RedliningValidator(unpacked_dir, original_file, verbose=args.verbose, author=args.author)  
+                )
+        case ".pptx":
+            validators = [
+                PPTXSchemaValidator(unpacked_dir, original_file, verbose=args.verbose),
+            ]
+        case _:
+            print(f"Error: Validation not supported for file type {file_extension}")
+            sys.exit(1)
+
+    if args.auto_repair:
+        total_repairs = sum(v.repair() for v in validators)
+        if total_repairs:
+            print(f"Auto-repaired {total_repairs} issue(s)")
+
+    success = all(v.validate() for v in validators)
+
+    if success:
+        print("All validations PASSED!")
+
+    sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()

package/system/skills/docx/scripts/office/validators/__init__.py

@@ -0,0 +1,15 @@
+"""
+Validation modules for Word document processing.
+"""
+
+from .base import BaseSchemaValidator
+from .docx import DOCXSchemaValidator
+from .pptx import PPTXSchemaValidator
+from .redlining import RedliningValidator
+
+__all__ = [
+    "BaseSchemaValidator",
+    "DOCXSchemaValidator",
+    "PPTXSchemaValidator",
+    "RedliningValidator",
+]