@panda-agent/panda-cli 0.1.29 → 0.1.30

package/skills/xlsx/scripts/office/validators/docx.py

@@ -0,0 +1,383 @@
+#!/usr/bin/env python3
+# ──────────────────────────────────────────────────────────────────
+# DOCX-specific schema and structural validator.
+#
+# Extends BaseSchemaValidator with checks for:
+#   • whitespace preservation on <w:t>
+#   • invalid <w:t> inside <w:del>
+#   • invalid <w:delText> inside <w:ins>
+#   • paraId / durableId numeric constraints
+#   • comment marker pairing
+#   • paragraph count comparison
+# ──────────────────────────────────────────────────────────────────
+
+import random
+import re
+import tempfile
+import zipfile
+
+import defusedxml.minidom
+import lxml.etree
+
+from .base import BaseSchemaValidator
+
+_PREVIEW_LEN = 50
+
+
+class DOCXSchemaValidator(BaseSchemaValidator):
+    """Validator tailored to Word (.docx) documents."""
+
+    WORD_2006_NAMESPACE = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
+    W14_NAMESPACE = "http://schemas.microsoft.com/office/word/2010/wordml"
+    W16CID_NAMESPACE = "http://schemas.microsoft.com/office/word/2016/wordml/cid"
+
+    ELEMENT_RELATIONSHIP_TYPES = {}
+
+    # ── main orchestrator ──
+
+    def validate(self):
+        if not self.validate_xml():
+            return False
+
+        checks = [
+            self.validate_namespaces,
+            self.validate_unique_ids,
+            self.validate_file_references,
+            self.validate_content_types,
+            self.validate_against_xsd,
+            self.validate_whitespace_preservation,
+            self.validate_deletions,
+            self.validate_insertions,
+            self.validate_all_relationship_ids,
+            self.validate_id_constraints,
+            self.validate_comment_markers,
+        ]
+        ok = True
+        for chk in checks:
+            if not chk():
+                ok = False
+
+        self.compare_paragraph_counts()
+        return ok
+
+    # ──────────────────────────────────────────────────────────────
+    # Whitespace
+    # ──────────────────────────────────────────────────────────────
+
+    def validate_whitespace_preservation(self):
+        errs = []
+        wns = self.WORD_2006_NAMESPACE
+        xns = self.XML_NAMESPACE
+
+        for fp in self.xml_files:
+            if fp.name != "document.xml":
+                continue
+            try:
+                root = lxml.etree.parse(str(fp)).getroot()
+                for t_el in root.iter("{{{}}}t".format(wns)):
+                    txt = t_el.text
+                    if not txt:
+                        continue
+                    if re.search(r"^[ \t\n\r]", txt) or re.search(r"[ \t\n\r]$", txt):
+                        space_attr = "{{{}}}space".format(xns)
+                        if t_el.attrib.get(space_attr) != "preserve":
+                            preview = repr(txt)[:_PREVIEW_LEN] + "..." if len(repr(txt)) > _PREVIEW_LEN else repr(txt)
+                            errs.append(
+                                "  {}: Line {}: w:t element with whitespace missing "
+                                "xml:space='preserve': {}".format(
+                                    fp.relative_to(self.unpacked_dir), t_el.sourceline, preview
+                                )
+                            )
+            except (lxml.etree.XMLSyntaxError, Exception) as exc:
+                errs.append("  {}: Error: {}".format(fp.relative_to(self.unpacked_dir), exc))
+
+        if errs:
+            print("FAILED - Found {} whitespace preservation violations:".format(len(errs)))
+            for ln in errs:
+                print(ln)
+            return False
+        if self.verbose:
+            print("PASSED - All whitespace is properly preserved")
+        return True
+
+    # ──────────────────────────────────────────────────────────────
+    # Deletion integrity
+    # ──────────────────────────────────────────────────────────────
+
+    def validate_deletions(self):
+        errs = []
+        wns = self.WORD_2006_NAMESPACE
+        ns_map = {"w": wns}
+
+        for fp in self.xml_files:
+            if fp.name != "document.xml":
+                continue
+            try:
+                root = lxml.etree.parse(str(fp)).getroot()
+
+                for t in root.xpath(".//w:del//w:t", namespaces=ns_map):
+                    if t.text:
+                        preview = repr(t.text)[:_PREVIEW_LEN] + "..." if len(repr(t.text)) > _PREVIEW_LEN else repr(t.text)
+                        errs.append(
+                            "  {}: Line {}: <w:t> found within <w:del>: {}".format(
+                                fp.relative_to(self.unpacked_dir), t.sourceline, preview
+                            )
+                        )
+
+                for instr in root.xpath(".//w:del//w:instrText", namespaces=ns_map):
+                    preview = repr(instr.text or "")[:_PREVIEW_LEN] + "..." if len(repr(instr.text or "")) > _PREVIEW_LEN else repr(instr.text or "")
+                    errs.append(
+                        "  {}: Line {}: <w:instrText> found within <w:del> "
+                        "(use <w:delInstrText>): {}".format(
+                            fp.relative_to(self.unpacked_dir), instr.sourceline, preview
+                        )
+                    )
+
+            except (lxml.etree.XMLSyntaxError, Exception) as exc:
+                errs.append("  {}: Error: {}".format(fp.relative_to(self.unpacked_dir), exc))
+
+        if errs:
+            print("FAILED - Found {} deletion validation violations:".format(len(errs)))
+            for ln in errs:
+                print(ln)
+            return False
+        if self.verbose:
+            print("PASSED - No w:t elements found within w:del elements")
+        return True
+
+    # ──────────────────────────────────────────────────────────────
+    # Insertion integrity
+    # ──────────────────────────────────────────────────────────────
+
+    def validate_insertions(self):
+        errs = []
+        ns_map = {"w": self.WORD_2006_NAMESPACE}
+
+        for fp in self.xml_files:
+            if fp.name != "document.xml":
+                continue
+            try:
+                root = lxml.etree.parse(str(fp)).getroot()
+                bad = root.xpath(".//w:ins//w:delText[not(ancestor::w:del)]", namespaces=ns_map)
+                for nd in bad:
+                    preview = repr(nd.text or "")[:_PREVIEW_LEN] + "..." if len(repr(nd.text or "")) > _PREVIEW_LEN else repr(nd.text or "")
+                    errs.append(
+                        "  {}: Line {}: <w:delText> within <w:ins>: {}".format(
+                            fp.relative_to(self.unpacked_dir), nd.sourceline, preview
+                        )
+                    )
+            except (lxml.etree.XMLSyntaxError, Exception) as exc:
+                errs.append("  {}: Error: {}".format(fp.relative_to(self.unpacked_dir), exc))
+
+        if errs:
+            print("FAILED - Found {} insertion validation violations:".format(len(errs)))
+            for ln in errs:
+                print(ln)
+            return False
+        if self.verbose:
+            print("PASSED - No w:delText elements within w:ins elements")
+        return True
+
+    # ──────────────────────────────────────────────────────────────
+    # Paragraph counts
+    # ──────────────────────────────────────────────────────────────
+
+    def count_paragraphs_in_unpacked(self):
+        total = 0
+        for fp in self.xml_files:
+            if fp.name != "document.xml":
+                continue
+            try:
+                root = lxml.etree.parse(str(fp)).getroot()
+                total = len(root.findall(".//{{{}}}p".format(self.WORD_2006_NAMESPACE)))
+            except Exception as exc:
+                print("Error counting paragraphs in unpacked document: {}".format(exc))
+        return total
+
+    def count_paragraphs_in_original(self):
+        if self.original_file is None:
+            return 0
+        n = 0
+        try:
+            with tempfile.TemporaryDirectory() as td:
+                with zipfile.ZipFile(self.original_file, "r") as zf:
+                    zf.extractall(td)
+                root = lxml.etree.parse(td + "/word/document.xml").getroot()
+                n = len(root.findall(".//{{{}}}p".format(self.WORD_2006_NAMESPACE)))
+        except Exception as exc:
+            print("Error counting paragraphs in original document: {}".format(exc))
+        return n
+
+    def compare_paragraph_counts(self):
+        orig = self.count_paragraphs_in_original()
+        cur = self.count_paragraphs_in_unpacked()
+        delta = cur - orig
+        sign = "+{}".format(delta) if delta > 0 else str(delta)
+        print("\nParagraphs: {} \u2192 {} ({})".format(orig, cur, sign))
+
+    # ──────────────────────────────────────────────────────────────
+    # ID numeric constraints (paraId, durableId)
+    # ──────────────────────────────────────────────────────────────
+
+    def _parse_id_value(self, raw: str, base: int = 16) -> int:
+        return int(raw, base)
+
+    def validate_id_constraints(self):
+        errs = []
+        pid_attr = "{{{}}}paraId".format(self.W14_NAMESPACE)
+        did_attr = "{{{}}}durableId".format(self.W16CID_NAMESPACE)
+
+        _HEX_CEILING = 0x80000000
+        _DUR_CEILING = 0x7FFFFFFF
+
+        for fp in self.xml_files:
+            try:
+                for nd in lxml.etree.parse(str(fp)).iter():
+                    pv = nd.get(pid_attr)
+                    if pv is not None and self._parse_id_value(pv, 16) >= _HEX_CEILING:
+                        errs.append("  {}:{}: paraId={} >= 0x80000000".format(
+                            fp.name, nd.sourceline, pv
+                        ))
+
+                    dv = nd.get(did_attr)
+                    if dv is not None:
+                        if fp.name == "numbering.xml":
+                            try:
+                                if self._parse_id_value(dv, 10) >= _DUR_CEILING:
+                                    errs.append("  {}:{}: durableId={} >= 0x7FFFFFFF".format(
+                                        fp.name, nd.sourceline, dv
+                                    ))
+                            except ValueError:
+                                errs.append("  {}:{}: durableId={} must be decimal in numbering.xml".format(
+                                    fp.name, nd.sourceline, dv
+                                ))
+                        else:
+                            if self._parse_id_value(dv, 16) >= _DUR_CEILING:
+                                errs.append("  {}:{}: durableId={} >= 0x7FFFFFFF".format(
+                                    fp.name, nd.sourceline, dv
+                                ))
+            except Exception:
+                pass
+
+        if errs:
+            print("FAILED - {} ID constraint violations:".format(len(errs)))
+            for ln in errs:
+                print(ln)
+        elif self.verbose:
+            print("PASSED - All paraId/durableId values within constraints")
+        return not bool(errs)
+
+    # ──────────────────────────────────────────────────────────────
+    # Comment marker pairing
+    # ──────────────────────────────────────────────────────────────
+
+    def validate_comment_markers(self):
+        errs = []
+
+        doc_xml = None
+        cmt_xml = None
+        for fp in self.xml_files:
+            if fp.name == "document.xml" and "word" in str(fp):
+                doc_xml = fp
+            elif fp.name == "comments.xml":
+                cmt_xml = fp
+
+        if doc_xml is None:
+            if self.verbose:
+                print("PASSED - No document.xml found (skipping comment validation)")
+            return True
+
+        try:
+            dr = lxml.etree.parse(str(doc_xml)).getroot()
+            ns = {"w": self.WORD_2006_NAMESPACE}
+            wid = "{{{}}}id".format(self.WORD_2006_NAMESPACE)
+
+            starts = {el.get(wid) for el in dr.xpath(".//w:commentRangeStart", namespaces=ns)}
+            ends = {el.get(wid) for el in dr.xpath(".//w:commentRangeEnd", namespaces=ns)}
+            refs = {el.get(wid) for el in dr.xpath(".//w:commentReference", namespaces=ns)}
+
+            _sort_key = lambda x: int(x) if x and x.isdigit() else 0
+
+            for cid in sorted(ends - starts, key=_sort_key):
+                errs.append('  document.xml: commentRangeEnd id="{}" has no matching commentRangeStart'.format(cid))
+
+            for cid in sorted(starts - ends, key=_sort_key):
+                errs.append('  document.xml: commentRangeStart id="{}" has no matching commentRangeEnd'.format(cid))
+
+            if cmt_xml and cmt_xml.exists():
+                cr = lxml.etree.parse(str(cmt_xml)).getroot()
+                defined = {el.get(wid) for el in cr.xpath(".//w:comment", namespaces=ns)}
+
+                for cid in sorted((starts | ends | refs) - defined, key=_sort_key):
+                    if cid:
+                        errs.append('  document.xml: marker id="{}" references non-existent comment'.format(cid))
+
+        except (lxml.etree.XMLSyntaxError, Exception) as exc:
+            errs.append("  Error parsing XML: {}".format(exc))
+
+        if errs:
+            print("FAILED - {} comment marker violations:".format(len(errs)))
+            for ln in errs:
+                print(ln)
+            return False
+        if self.verbose:
+            print("PASSED - All comment markers properly paired")
+        return True
+
+    # ──────────────────────────────────────────────────────────────
+    # Repair: durableId overflow
+    # ──────────────────────────────────────────────────────────────
+
+    def repair(self) -> int:
+        n = super().repair()
+        n += self._fix_durable_ids()
+        return n
+
+    def _fix_durable_ids(self) -> int:
+        n_fixed = 0
+        _LIMIT = 0x7FFFFFFF
+
+        for fp in self.xml_files:
+            try:
+                raw = fp.read_text(encoding="utf-8")
+                dom = defusedxml.minidom.parseString(raw)
+                changed = False
+
+                for el in dom.getElementsByTagName("*"):
+                    if not el.hasAttribute("w16cid:durableId"):
+                        continue
+                    old_val = el.getAttribute("w16cid:durableId")
+                    bad = False
+
+                    if fp.name == "numbering.xml":
+                        try:
+                            bad = self._parse_id_value(old_val, 10) >= _LIMIT
+                        except ValueError:
+                            bad = True
+                    else:
+                        try:
+                            bad = self._parse_id_value(old_val, 16) >= _LIMIT
+                        except ValueError:
+                            bad = True
+
+                    if bad:
+                        rv = random.randint(1, _LIMIT - 1)
+                        new_val = str(rv) if fp.name == "numbering.xml" else "{:08X}".format(rv)
+                        el.setAttribute("w16cid:durableId", new_val)
+                        print("  Repaired: {}: durableId {} \u2192 {}".format(fp.name, old_val, new_val))
+                        n_fixed += 1
+                        changed = True
+
+                if changed:
+                    fp.write_bytes(dom.toxml(encoding="UTF-8"))
+            except Exception:
+                pass
+
+        return n_fixed
+
+    # keep legacy name
+    repair_durableId = _fix_durable_ids
+
+
+if __name__ == "__main__":
+    raise RuntimeError("This module should not be run directly.")

package/skills/xlsx/scripts/office/validators/pptx.py

@@ -0,0 +1,250 @@
+#!/usr/bin/env python3
+# ──────────────────────────────────────────────────────────────────
+# PPTX-specific schema and structural validator.
+#
+# Extends BaseSchemaValidator with checks for:
+#   • UUID format validation on ID attributes
+#   • Slide-layout ↔ slide-master relationship integrity
+#   • Notes-slide reference uniqueness
+#   • Duplicate slideLayout relationship detection
+# ──────────────────────────────────────────────────────────────────
+
+import re
+
+from .base import BaseSchemaValidator
+
+_UUID_RE = re.compile(
+    r"^[\{\(]?[0-9A-Fa-f]{8}-?[0-9A-Fa-f]{4}-?[0-9A-Fa-f]{4}"
+    r"-?[0-9A-Fa-f]{4}-?[0-9A-Fa-f]{12}[\}\)]?$"
+)
+
+
+class PPTXSchemaValidator(BaseSchemaValidator):
+    """Validator tailored to PowerPoint (.pptx) presentations."""
+
+    PRESENTATIONML_NAMESPACE = (
+        "http://schemas.openxmlformats.org/presentationml/2006/main"
+    )
+
+    ELEMENT_RELATIONSHIP_TYPES = {
+        "sldid": "slide",
+        "sldmasterid": "slidemaster",
+        "notesmasterid": "notesmaster",
+        "sldlayoutid": "slidelayout",
+        "themeid": "theme",
+        "tablestyleid": "tablestyles",
+    }
+
+    # ── orchestrator ──
+
+    def validate(self):
+        if not self.validate_xml():
+            return False
+
+        ok = True
+        for chk in (
+            self.validate_namespaces,
+            self.validate_unique_ids,
+            self.validate_uuid_ids,
+            self.validate_file_references,
+            self.validate_slide_layout_ids,
+            self.validate_content_types,
+            self.validate_against_xsd,
+            self.validate_notes_slide_references,
+            self.validate_all_relationship_ids,
+            self.validate_no_duplicate_slide_layouts,
+        ):
+            if not chk():
+                ok = False
+        return ok
+
+    # ──────────────────────────────────────────────────────────────
+    # UUID-format IDs
+    # ──────────────────────────────────────────────────────────────
+
+    def validate_uuid_ids(self):
+        import lxml.etree
+
+        errs = []
+        for fp in self.xml_files:
+            try:
+                root = lxml.etree.parse(str(fp)).getroot()
+                for nd in root.iter():
+                    for attr_key, attr_val in nd.attrib.items():
+                        aname = attr_key.split("}")[-1].lower()
+                        if aname != "id" and not aname.endswith("id"):
+                            continue
+                        if self._resembles_uuid(attr_val) and not _UUID_RE.match(attr_val):
+                            errs.append(
+                                "  {}: Line {}: ID '{}' appears to be a UUID "
+                                "but contains invalid hex characters".format(
+                                    fp.relative_to(self.unpacked_dir), nd.sourceline, attr_val
+                                )
+                            )
+            except (lxml.etree.XMLSyntaxError, Exception) as exc:
+                errs.append("  {}: Error: {}".format(fp.relative_to(self.unpacked_dir), exc))
+
+        if errs:
+            print("FAILED - Found {} UUID ID validation errors:".format(len(errs)))
+            for ln in errs:
+                print(ln)
+            return False
+        if self.verbose:
+            print("PASSED - All UUID-like IDs contain valid hex values")
+        return True
+
+    @staticmethod
+    def _resembles_uuid(val):
+        stripped = val.strip("{}()").replace("-", "")
+        return len(stripped) == 32 and stripped.isalnum()
+
+    # ──────────────────────────────────────────────────────────────
+    # Slide-layout IDs ↔ relationships
+    # ──────────────────────────────────────────────────────────────
+
+    def validate_slide_layout_ids(self):
+        import lxml.etree
+
+        errs = []
+        masters = list(self.unpacked_dir.glob("ppt/slideMasters/*.xml"))
+
+        if not masters:
+            if self.verbose:
+                print("PASSED - No slide masters found")
+            return True
+
+        for sm in masters:
+            try:
+                root = lxml.etree.parse(str(sm)).getroot()
+                rels_path = sm.parent / "_rels" / "{}.rels".format(sm.name)
+
+                if not rels_path.exists():
+                    errs.append("  {}: Missing relationships file: {}".format(
+                        sm.relative_to(self.unpacked_dir),
+                        rels_path.relative_to(self.unpacked_dir),
+                    ))
+                    continue
+
+                rroot = lxml.etree.parse(str(rels_path)).getroot()
+                valid_rids = {
+                    r.get("Id")
+                    for r in rroot.findall(
+                        ".//{{{}}}Relationship".format(self.PACKAGE_RELATIONSHIPS_NAMESPACE)
+                    )
+                    if "slideLayout" in r.get("Type", "")
+                }
+
+                pml = self.PRESENTATIONML_NAMESPACE
+                for lid in root.findall(".//{{{}}}sldLayoutId".format(pml)):
+                    rid = lid.get("{{{}}}id".format(self.OFFICE_RELATIONSHIPS_NAMESPACE))
+                    layout_id = lid.get("id")
+                    if rid and rid not in valid_rids:
+                        errs.append(
+                            "  {}: Line {}: sldLayoutId with id='{}' "
+                            "references r:id='{}' which is not found in slide layout relationships".format(
+                                sm.relative_to(self.unpacked_dir), lid.sourceline, layout_id, rid
+                            )
+                        )
+            except (lxml.etree.XMLSyntaxError, Exception) as exc:
+                errs.append("  {}: Error: {}".format(sm.relative_to(self.unpacked_dir), exc))
+
+        if errs:
+            print("FAILED - Found {} slide layout ID validation errors:".format(len(errs)))
+            for ln in errs:
+                print(ln)
+            print("Remove invalid references or add missing slide layouts to the relationships file.")
+            return False
+        if self.verbose:
+            print("PASSED - All slide layout IDs reference valid slide layouts")
+        return True
+
+    # ──────────────────────────────────────────────────────────────
+    # Duplicate slideLayout refs per slide
+    # ──────────────────────────────────────────────────────────────
+
+    def validate_no_duplicate_slide_layouts(self):
+        import lxml.etree
+
+        errs = []
+        for rf in self.unpacked_dir.glob("ppt/slides/_rels/*.xml.rels"):
+            try:
+                root = lxml.etree.parse(str(rf)).getroot()
+                layouts = [
+                    r for r in root.findall(
+                        ".//{{{}}}Relationship".format(self.PACKAGE_RELATIONSHIPS_NAMESPACE)
+                    )
+                    if "slideLayout" in r.get("Type", "")
+                ]
+                if len(layouts) > 1:
+                    errs.append("  {}: has {} slideLayout references".format(
+                        rf.relative_to(self.unpacked_dir), len(layouts)
+                    ))
+            except Exception as exc:
+                errs.append("  {}: Error: {}".format(rf.relative_to(self.unpacked_dir), exc))
+
+        if errs:
+            print("FAILED - Found slides with duplicate slideLayout references:")
+            for ln in errs:
+                print(ln)
+            return False
+        if self.verbose:
+            print("PASSED - All slides have exactly one slideLayout reference")
+        return True
+
+    # ──────────────────────────────────────────────────────────────
+    # Notes-slide reference uniqueness
+    # ──────────────────────────────────────────────────────────────
+
+    def validate_notes_slide_references(self):
+        import lxml.etree
+
+        errs = []
+        target_map = {}
+
+        slide_rels = list(self.unpacked_dir.glob("ppt/slides/_rels/*.xml.rels"))
+        if not slide_rels:
+            if self.verbose:
+                print("PASSED - No slide relationship files found")
+            return True
+
+        for rf in slide_rels:
+            try:
+                root = lxml.etree.parse(str(rf)).getroot()
+                for rel in root.findall(
+                    ".//{{{}}}Relationship".format(self.PACKAGE_RELATIONSHIPS_NAMESPACE)
+                ):
+                    if "notesSlide" not in rel.get("Type", ""):
+                        continue
+                    tgt = rel.get("Target", "")
+                    if not tgt:
+                        continue
+                    norm = tgt.replace("../", "")
+                    slide = rf.stem.replace(".xml", "")
+                    target_map.setdefault(norm, []).append((slide, rf))
+
+            except (lxml.etree.XMLSyntaxError, Exception) as exc:
+                errs.append("  {}: Error: {}".format(rf.relative_to(self.unpacked_dir), exc))
+
+        for tgt, refs in target_map.items():
+            if len(refs) > 1:
+                names = [r[0] for r in refs]
+                errs.append("  Notes slide '{}' is referenced by multiple slides: {}".format(
+                    tgt, ", ".join(names)
+                ))
+                for name, rfile in refs:
+                    errs.append("    - {}".format(rfile.relative_to(self.unpacked_dir)))
+
+        if errs:
+            top_errs = [e for e in errs if not e.startswith("    ")]
+            print("FAILED - Found {} notes slide reference validation errors:".format(len(top_errs)))
+            for ln in errs:
+                print(ln)
+            print("Each slide may optionally have its own slide file.")
+            return False
+        if self.verbose:
+            print("PASSED - All notes slide references are unique")
+        return True
+
+
+if __name__ == "__main__":
+    raise RuntimeError("This module should not be run directly.")