@prismer/runtime 2.0.8 → 2.2.63

package/built-in-skills/office-artifacts/references/docx/tests/test_docx_skill.py

@@ -0,0 +1,525 @@
+# MIT License. End-to-end tests for the docx skill.
+"""Pytest suite proving create / read / edit / template round-trips.
+
+Runs the scripts as subprocesses (argparse CLIs) and also verifies the
+outputs with python-docx directly. Stdlib + python-docx only; all
+fixtures are generated on the fly; no network.
+"""
+from __future__ import annotations
+
+import json
+import os
+import struct
+import subprocess
+import sys
+import zlib
+from pathlib import Path
+
+import pytest
+from docx import Document
+
+SKILL = Path(__file__).resolve().parent.parent
+SCRIPTS = SKILL / "scripts"
+
+NON_ASCII = "Фамилия — ‘test’"
+
+
+def make_png(path: Path) -> None:
+    """Write a tiny valid 2x2 red PNG using only stdlib."""
+    def chunk(tag: bytes, data: bytes) -> bytes:
+        return (struct.pack(">I", len(data)) + tag + data
+                + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF))
+
+    ihdr = struct.pack(">IIBBBBB", 2, 2, 8, 2, 0, 0, 0)
+    raw = b"".join(b"\x00" + b"\xff\x00\x00" * 2 for _ in range(2))
+    png = (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr)
+           + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b""))
+    path.write_bytes(png)
+
+
+def run(script: str, *args: str) -> dict:
+    env = dict(os.environ)
+    env["LC_ALL"] = "C"  # prove no locale-default text reads
+    env["PYTHONIOENCODING"] = "utf-8"
+    proc = subprocess.run(
+        [sys.executable, str(SCRIPTS / script), *map(str, args)],
+        capture_output=True, env=env)
+    assert proc.returncode == 0, proc.stderr.decode("utf-8", "replace")
+    return json.loads(proc.stdout.decode("utf-8"))
+
+
+@pytest.fixture(scope="module")
+def workdir(tmp_path_factory) -> Path:
+    return tmp_path_factory.mktemp("docxskill")
+
+
+@pytest.fixture(scope="module")
+def created(workdir: Path) -> Path:
+    """Create a document exercising every create feature."""
+    png = workdir / "pic.png"
+    make_png(png)
+    spec = {
+        "page": {"width_mm": 210, "height_mm": 297,
+                 "margins_mm": {"top": 25, "bottom": 25,
+                                "left": 20, "right": 20}},
+        "header": "Report header",
+        "footer": "Page footer",
+        "styles": [{"name": "FancyNote", "base": "Normal", "font": "Arial",
+                    "size_pt": 11, "italic": True, "color": "1F4E79"}],
+        "blocks": [
+            {"type": "heading", "text": "Main Title", "level": 1},
+            {"type": "heading", "text": "Section One", "level": 2},
+            {"type": "paragraph", "runs": [
+                {"text": "plain "},
+                {"text": "boldbit", "bold": True},
+                {"text": " italicbit", "italic": True},
+                {"text": " underbit", "underline": True}]},
+            {"type": "paragraph", "text": "Styled note.",
+             "style": "FancyNote"},
+            {"type": "bullet_list", "items": ["alpha", "beta"]},
+            {"type": "numbered_list", "items": ["first", "second"]},
+            {"type": "table", "header": ["Name", "Qty"],
+             "rows": [["Widget", "3"], ["Gadget", "5"]],
+             "style": "Table Grid", "header_bold": True},
+            {"type": "image", "path": str(png), "width_mm": 30},
+            {"type": "page_break"},
+            {"type": "paragraph", "text": "After the break."},
+        ],
+    }
+    spec_path = workdir / "spec.json"
+    spec_path.write_text(json.dumps(spec), encoding="utf-8")
+    out = workdir / "created.docx"
+    res = run("docx_create.py", spec_path, out)
+    assert res["ok"] and out.exists()
+    return out
+
+
+class TestCreateAndRead:
+    def test_text_roundtrip(self, created: Path):
+        text = run("docx_read.py", created, "--text")
+        body = "\n".join(text["body"])
+        for expected in ("Main Title", "plain boldbit italicbit underbit",
+                         "Styled note.", "alpha", "second",
+                         "After the break."):
+            assert expected in body
+        assert text["tables"] == [[["Name", "Qty"], ["Widget", "3"],
+                                   ["Gadget", "5"]]]
+        assert "Report header" in text["headers"]
+        assert "Page footer" in text["footers"]
+
+    def test_structure(self, created: Path):
+        st = run("docx_read.py", created, "--structure")
+        outline = [(h["level"], h["text"]) for h in st["outline"]]
+        assert (1, "Main Title") in outline
+        assert (2, "Section One") in outline
+        assert st["table_count"] == 1
+        assert st["tables"][0] == {"rows": 3, "cols": 2}
+
+    def test_styles_used(self, created: Path):
+        styles = run("docx_read.py", created, "--styles")["styles"]
+        for s in ("Heading 1", "FancyNote", "List Bullet", "List Number",
+                  "Table Grid"):
+            assert s in styles
+
+    def test_images_extracted(self, created: Path, workdir: Path):
+        outdir = workdir / "media"
+        res = run("docx_read.py", created, "--images", outdir)
+        assert len(res["images"]) == 1
+        img = Path(res["images"][0])
+        assert img.read_bytes().startswith(b"\x89PNG")
+
+    def test_run_formatting_persisted(self, created: Path):
+        doc = Document(str(created))
+        para = next(p for p in doc.paragraphs if "boldbit" in p.text)
+        flags = {r.text.strip(): (r.bold, r.italic, r.underline)
+                 for r in para.runs if r.text.strip()}
+        assert flags["boldbit"][0] is True
+        assert flags["italicbit"][1] is True
+        assert flags["underbit"][2] is True
+
+    def test_page_setup(self, created: Path):
+        sec = Document(str(created)).sections[0]
+        assert round(sec.page_width.mm) == 210
+        assert round(sec.top_margin.mm) == 25
+
+    def test_revisions_detection(self, created: Path):
+        rev = run("docx_read.py", created, "--revisions")
+        assert rev["has_tracked_changes"] is False
+        assert rev["comments"] is False
+
+
+class TestEdit:
+    def test_replace_preserves_formatting(self, created: Path, workdir: Path):
+        out = workdir / "edited.docx"
+        res = run("docx_edit.py", "replace", created, "--find", "boldbit",
+                  "--replace", "REPLACED", "-o", out)
+        assert res["replacements"] == 1
+        doc = Document(str(out))
+        para = next(p for p in doc.paragraphs if "REPLACED" in p.text)
+        run_ = next(r for r in para.runs if "REPLACED" in r.text)
+        assert run_.bold is True  # formatting survived
+
+    def test_set_cell(self, created: Path, workdir: Path):
+        out = workdir / "cell.docx"
+        run("docx_edit.py", "set-cell", created, "--table", "0", "--row",
+            "1", "--col", "1", "--text", "99", "-o", out)
+        assert Document(str(out)).tables[0].cell(1, 1).text == "99"
+
+    def test_insert_and_delete(self, created: Path, workdir: Path):
+        out = workdir / "ins.docx"
+        run("docx_edit.py", "insert", created, "--index", "0", "--text",
+            "Inserted first", "-o", out)
+        doc = Document(str(out))
+        assert doc.paragraphs[0].text == "Inserted first"
+        out2 = workdir / "del.docx"
+        run("docx_edit.py", "delete", out, "--index", "0", "-o", out2)
+        assert Document(str(out2)).paragraphs[0].text != "Inserted first"
+
+    def test_apply_style(self, created: Path, workdir: Path):
+        out = workdir / "styled.docx"
+        doc = Document(str(created))
+        idx = next(i for i, p in enumerate(doc.paragraphs)
+                   if p.text == "After the break.")
+        run("docx_edit.py", "style", created, "--index", str(idx),
+            "--style", "Heading 2", "-o", out)
+        doc2 = Document(str(out))
+        assert doc2.paragraphs[idx].style.name == "Heading 2"
+
+
+class TestTemplate:
+    def test_fill_everywhere_non_ascii(self, workdir: Path):
+        # Build a template: tokens in body, split runs, table, header, footer.
+        tpl = workdir / "tpl.docx"
+        doc = Document()
+        doc.sections[0].header.paragraphs[0].text = "H: {{name}}"
+        doc.sections[0].footer.paragraphs[0].text = "F: {{date}}"
+        p = doc.add_paragraph()
+        p.add_run("Dear {{na")          # token split across runs
+        p.add_run("me}}, hello.")
+        t = doc.add_table(rows=1, cols=2)
+        t.cell(0, 0).text = "{{name}}"
+        t.cell(0, 1).text = "{{ date }}"   # spaced variant
+        doc.add_paragraph("Unfilled: {{missing}}")
+        doc.save(str(tpl))
+
+        values = workdir / "values.json"
+        values.write_text(
+            json.dumps({"name": NON_ASCII, "date": "2026-08-08"},
+                       ensure_ascii=False), encoding="utf-8")
+        out = workdir / "filled.docx"
+        res = run("docx_template.py", tpl, values, out)
+        assert res["ok"] is True
+        assert res["unfilled_tokens"] == ["missing"]
+
+        text = run("docx_read.py", out, "--text")
+        assert f"Dear {NON_ASCII}, hello." in text["body"]
+        assert text["tables"][0][0] == [NON_ASCII, "2026-08-08"]
+        assert f"H: {NON_ASCII}" in text["headers"]
+        assert "F: 2026-08-08" in text["footers"]
+
+    def test_strict_fails_on_unfilled(self, workdir: Path):
+        tpl = workdir / "tpl2.docx"
+        doc = Document()
+        doc.add_paragraph("{{gone}}")
+        doc.save(str(tpl))
+        values = workdir / "empty.json"
+        values.write_text("{}", encoding="utf-8")
+        env = dict(os.environ, LC_ALL="C", PYTHONIOENCODING="utf-8")
+        proc = subprocess.run(
+            [sys.executable, str(SCRIPTS / "docx_template.py"), str(tpl),
+             str(values), str(workdir / "out2.docx"), "--strict"],
+            capture_output=True, env=env)
+        assert proc.returncode == 1
+        payload = json.loads(proc.stdout.decode("utf-8"))
+        assert payload["unfilled_tokens"] == ["gone"]
+
+
+# --------------------------------------------------------------- new parity
+
+W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
+
+
+def q(tag: str) -> str:
+    return f"{{{W}}}{tag}"
+
+
+def run_raw(script: str, *args: str):
+    env = dict(os.environ, LC_ALL="C", PYTHONIOENCODING="utf-8")
+    return subprocess.run(
+        [sys.executable, str(SCRIPTS / script), *map(str, args)],
+        capture_output=True, env=env)
+
+
+def _add_ins(para, rev_id: int, text: str, author="Editor"):
+    from lxml import etree
+    ins = etree.SubElement(para._p, q("ins"))
+    ins.set(q("id"), str(rev_id))
+    ins.set(q("author"), author)
+    ins.set(q("date"), "2026-01-02T03:04:05Z")
+    r = etree.SubElement(ins, q("r"))
+    t = etree.SubElement(r, q("t"))
+    t.text = text
+    t.set("{http://www.w3.org/XML/1998/namespace}space", "preserve")
+
+
+def _add_del(para, rev_id: int, text: str, author="Editor"):
+    from lxml import etree
+    dele = etree.SubElement(para._p, q("del"))
+    dele.set(q("id"), str(rev_id))
+    dele.set(q("author"), author)
+    dele.set(q("date"), "2026-01-02T03:04:05Z")
+    r = etree.SubElement(dele, q("r"))
+    t = etree.SubElement(r, q("delText"))
+    t.text = text
+    t.set("{http://www.w3.org/XML/1998/namespace}space", "preserve")
+
+
+@pytest.fixture()
+def tracked(tmp_path: Path) -> Path:
+    """Doc with a tracked insertion + deletion in body AND in a table."""
+    doc = Document()
+    p = doc.add_paragraph("Base ")
+    _add_ins(p, 1, "ADDED")
+    _add_del(p, 2, "REMOVED")
+    table = doc.add_table(rows=1, cols=1)
+    cp = table.cell(0, 0).paragraphs[0]
+    cp.add_run("Cell ")
+    _add_ins(cp, 3, "CELLADD")
+    _add_del(cp, 4, "CELLGONE")
+    path = tmp_path / "tracked.docx"
+    doc.save(str(path))
+    return path
+
+
+class TestRevisions:
+    def test_list(self, tracked: Path):
+        res = run("docx_revisions.py", "list", tracked)
+        revs = {r["id"]: r for r in res["revisions"]}
+        assert len(revs) == 4
+        assert revs["1"] == {"id": "1", "author": "Editor",
+                             "date": "2026-01-02T03:04:05Z",
+                             "type": "insertion", "text": "ADDED"}
+        assert revs["2"]["type"] == "deletion"
+        assert revs["2"]["text"] == "REMOVED"
+        assert revs["3"]["text"] == "CELLADD"  # inside table
+        assert revs["4"]["type"] == "deletion"
+
+    def test_accept_all(self, tracked: Path, tmp_path: Path):
+        out = tmp_path / "acc.docx"
+        res = run("docx_revisions.py", "accept-all", tracked, "-o", out)
+        assert res["resolved"] == 4
+        doc = Document(str(out))
+        assert doc.paragraphs[0].text == "Base ADDED"
+        assert doc.tables[0].cell(0, 0).text == "Cell CELLADD"
+        assert run("docx_revisions.py", "list", out)["revisions"] == []
+
+    def test_reject_all(self, tracked: Path, tmp_path: Path):
+        out = tmp_path / "rej.docx"
+        run("docx_revisions.py", "reject-all", tracked, "-o", out)
+        doc = Document(str(out))
+        assert doc.paragraphs[0].text == "Base REMOVED"
+        assert doc.tables[0].cell(0, 0).text == "Cell CELLGONE"
+
+    def test_accept_single_by_id(self, tracked: Path, tmp_path: Path):
+        out = tmp_path / "one.docx"
+        res = run("docx_revisions.py", "accept", tracked, "--id", "1",
+                  "-o", out)
+        assert res["resolved"] == 1
+        doc = Document(str(out))
+        assert doc.paragraphs[0].text == "Base ADDED"  # del 2 unresolved
+        remaining = run("docx_revisions.py", "list", out)["revisions"]
+        assert sorted(r["id"] for r in remaining) == ["2", "3", "4"]
+
+    def test_reject_single_by_id(self, tracked: Path, tmp_path: Path):
+        out = tmp_path / "rone.docx"
+        run("docx_revisions.py", "reject", tracked, "--id", "2", "-o", out)
+        doc = Document(str(out))
+        assert doc.paragraphs[0].text == "Base REMOVED"  # ins 1 unresolved
+
+    def test_unknown_id_fails(self, tracked: Path, tmp_path: Path):
+        proc = run_raw("docx_revisions.py", "accept", tracked, "--id",
+                       "999", "-o", tmp_path / "x.docx")
+        assert proc.returncode == 1
+
+
+class TestComments:
+    @pytest.fixture()
+    def base(self, tmp_path: Path) -> Path:
+        doc = Document()
+        doc.add_paragraph("The quarterly revenue rose sharply.")
+        doc.add_paragraph("Second paragraph.")
+        path = tmp_path / "base.docx"
+        doc.save(str(path))
+        return path
+
+    def test_add_list_delete(self, base: Path, tmp_path: Path):
+        out = tmp_path / "com.docx"
+        res = run("docx_comments.py", "add", base, "--target",
+                  "quarterly revenue", "--text", "Needs a source",
+                  "--author", "Reviewer", "--initials", "R", "-o", out)
+        assert res["ok"] is True
+        cid = res["comment_id"]
+
+        listed = run("docx_comments.py", "list", out)["comments"]
+        assert len(listed) == 1
+        c = listed[0]
+        assert c["id"] == cid
+        assert c["author"] == "Reviewer"
+        assert c["text"] == "Needs a source"
+        assert c["anchored_text"] == "quarterly revenue"
+        assert c["date"]
+
+        # document text unchanged by anchoring
+        text = run("docx_read.py", out, "--text")
+        assert "The quarterly revenue rose sharply." in text["body"]
+
+        out2 = tmp_path / "nocom.docx"
+        run("docx_comments.py", "delete", out, "--id", cid, "-o", out2)
+        assert run("docx_comments.py", "list", out2)["comments"] == []
+        text2 = run("docx_read.py", out2, "--text")
+        assert "The quarterly revenue rose sharply." in text2["body"]
+
+    def test_xml_fallback_path(self, base: Path, tmp_path: Path):
+        out = tmp_path / "xmlcom.docx"
+        res = run("docx_comments.py", "add", base, "--target",
+                  "Second paragraph", "--text", "fallback note",
+                  "--author", "Bot", "--xml", "-o", out)
+        assert res["native_api"] is False
+        listed = run("docx_comments.py", "list", out)["comments"]
+        assert listed[0]["text"] == "fallback note"
+        assert listed[0]["anchored_text"] == "Second paragraph"
+        # file still opens cleanly
+        assert Document(str(out)).paragraphs[1].text == "Second paragraph."
+
+    def test_missing_target_fails(self, base: Path, tmp_path: Path):
+        proc = run_raw("docx_comments.py", "add", base, "--target",
+                       "not present", "--text", "x", "-o",
+                       tmp_path / "y.docx")
+        assert proc.returncode == 1
+
+
+class TestValidate:
+    def test_healthy_file_passes(self, created: Path):
+        res = run("docx_validate.py", created)
+        assert res["ok"] is True
+        assert all(i["severity"] != "error" for i in res["issues"])
+
+    def test_not_a_zip(self, tmp_path: Path):
+        bad = tmp_path / "bad.docx"
+        bad.write_bytes(b"this is not a zip file")
+        proc = run_raw("docx_validate.py", bad)
+        assert proc.returncode == 1
+        rep = json.loads(proc.stdout.decode("utf-8"))
+        assert rep["issues"][0]["code"] == "not-a-zip"
+
+    def test_dangling_rel_and_empty_image(self, created: Path,
+                                          tmp_path: Path):
+        import shutil
+        import zipfile
+        broken = tmp_path / "broken.docx"
+        shutil.copy(created, broken)
+        # rebuild the zip: drop the image part, zero out nothing else
+        src = zipfile.ZipFile(str(created))
+        with zipfile.ZipFile(str(broken), "w") as dst:
+            for item in src.infolist():
+                if item.filename.startswith("word/media/"):
+                    dst.writestr(item.filename, b"")  # empty image
+                else:
+                    dst.writestr(item, src.read(item.filename))
+        proc = run_raw("docx_validate.py", broken)
+        assert proc.returncode == 1
+        rep = json.loads(proc.stdout.decode("utf-8"))
+        codes = {i["code"] for i in rep["issues"]}
+        assert "empty-image" in codes
+
+    def test_missing_style(self, tmp_path: Path):
+        import zipfile
+        doc = Document()
+        doc.add_paragraph("styled", style="Heading 1")
+        path = tmp_path / "styles.docx"
+        doc.save(str(path))
+        # rewrite document.xml to reference a style id that doesn't exist
+        src = zipfile.ZipFile(str(path))
+        broken = tmp_path / "badstyle.docx"
+        with zipfile.ZipFile(str(broken), "w") as dst:
+            for item in src.infolist():
+                data = src.read(item.filename)
+                if item.filename == "word/document.xml":
+                    data = data.replace(b'w:val="Heading1"',
+                                        b'w:val="GhostStyle"')
+                dst.writestr(item, data)
+        proc = run_raw("docx_validate.py", broken)
+        assert proc.returncode == 1
+        rep = json.loads(proc.stdout.decode("utf-8"))
+        assert any(i["code"] == "missing-style" and "GhostStyle"
+                   in i["detail"] for i in rep["issues"])
+
+
+class TestNormalize:
+    def test_merges_split_runs(self, tmp_path: Path):
+        doc = Document()
+        p = doc.add_paragraph()
+        p.add_run("Hel")            # identical (no) formatting, split
+        p.add_run("lo wo")
+        p.add_run("rld")
+        b = p.add_run("BOLD1")
+        b.bold = True
+        b2 = p.add_run("BOLD2")
+        b2.bold = True
+        i = p.add_run("ital")
+        i.italic = True
+        path = tmp_path / "split.docx"
+        doc.save(str(path))
+
+        out = tmp_path / "norm.docx"
+        res = run("docx_edit.py", "normalize", path, "-o", out)
+        assert res["runs_merged"] == 3  # 2 plain merges + 1 bold merge
+
+        doc2 = Document(str(out))
+        para = doc2.paragraphs[0]
+        assert para.text == "Hello worldBOLD1BOLD2ital"
+        assert [r.text for r in para.runs] == \
+            ["Hello world", "BOLD1BOLD2", "ital"]
+        assert para.runs[1].bold is True
+        assert para.runs[2].italic is True
+
+
+class TestFields:
+    def test_toc_and_page_numbers_via_edit(self, created: Path,
+                                           tmp_path: Path):
+        out = tmp_path / "fields.docx"
+        run("docx_edit.py", "toc", created, "--index", "0", "-o", out)
+        run("docx_edit.py", "page-numbers", out)
+
+        import zipfile
+        doc_xml = zipfile.ZipFile(str(out)).read(
+            "word/document.xml").decode("utf-8")
+        assert "TOC \\o" in doc_xml
+        assert "fldChar" in doc_xml
+        footer_names = [n for n in zipfile.ZipFile(str(out)).namelist()
+                        if n.startswith("word/footer")]
+        footers = "".join(zipfile.ZipFile(str(out)).read(n).decode("utf-8")
+                          for n in footer_names)
+        assert "PAGE" in footers and "NUMPAGES" in footers
+        # still a valid document
+        assert run("docx_validate.py", out)["ok"] is True
+
+    def test_toc_and_footer_in_create_spec(self, tmp_path: Path):
+        spec = {
+            "footer_page_numbers": True,
+            "blocks": [
+                {"type": "toc"},
+                {"type": "heading", "text": "Chapter", "level": 1},
+            ],
+        }
+        spec_path = tmp_path / "fspec.json"
+        spec_path.write_text(json.dumps(spec), encoding="utf-8")
+        out = tmp_path / "fcreate.docx"
+        run("docx_create.py", spec_path, out)
+
+        import zipfile
+        z = zipfile.ZipFile(str(out))
+        assert "TOC \\o" in z.read("word/document.xml").decode("utf-8")
+        footers = "".join(z.read(n).decode("utf-8") for n in z.namelist()
+                          if n.startswith("word/footer"))
+        assert "NUMPAGES" in footers

package/built-in-skills/office-artifacts/references/pdf/GUIDE.md

@@ -0,0 +1,125 @@
+---
+name: pdf
+description: "PDF files: create, read, merge, fill, OCR, edit text."
+version: 1.1.0
+author: Nous Research
+license: MIT
+platforms: [linux, macos, windows]
+metadata:
+  hermes:
+    tags: [pdf, documents, forms, ocr, text-extraction, reportlab, pypdf, pdfplumber, pymupdf, marker]
+    category: productivity
+    related_skills: [docx, xlsx, powerpoint]
+---
+
+# PDF Skill
+
+Create PDFs from structured specs, build and fill AcroForm forms (with layout linting and visual overlays), extract text/tables/metadata, merge/split/rotate/watermark/stamp pages, export page images, manage metadata and attachments, and encrypt/decrypt — using pypdf, reportlab, and pdfplumber. Two absorbed capabilities live in references/ (read the matching file before those tasks):
+
+- **Scanned/image-only PDFs and OCR** (pymupdf fast path, marker-pdf quality path, scripts/extract_pymupdf.py + scripts/extract_marker.py): `references/ocr-extraction.md`
+- **Editing text inside an existing PDF via natural-language prompts** (nano-pdf CLI): `references/nano-pdf-editing.md`
+
+## When to Use
+
+- Generate a report, invoice, or multi-page document as PDF.
+- Build a fillable AcroForm (text/checkbox/radio/dropdown) from a JSON spec, linting the layout first.
+- Pull text, tables (JSON/CSV), metadata, or form-field values out of a PDF.
+- Merge, split, rotate, extract page subsets, watermark, stamp text/images at coordinates, bookmark, or compress PDFs.
+- Export pages as PNGs for visual review or for OCR hand-off; set/clear document metadata; add/extract file attachments.
+- Fill or flatten AcroForm forms; encrypt or decrypt with passwords.
+- NOT for scanned/image-only PDFs (use `references/ocr-extraction.md`) and NOT for pixel-perfect HTML-to-PDF rendering (use a headless browser).
+
+## Prerequisites
+
+- Python 3.10+ with `pypdf`, `reportlab`, `pdfplumber`:
+  `python -m pip install pypdf reportlab pdfplumber`
+- Optional, for page rasterization (`pdf_page_image.py`, overlay rendering): `python -m pip install pypdfium2`, or poppler's `pdftoppm` on PATH. Scripts fall back pypdfium2 → pdftoppm and report `{"rendered": false, "missing": [...]}` (exit 0) when neither exists.
+- Each helper script checks imports lazily and prints an install hint if a dependency is missing.
+
+## How to Run
+
+All helpers live in `scripts/` and are argparse CLIs — run them with the `terminal` tool; every one supports `--help`. They read/write JSON strictly as UTF-8, print JSON results to stdout, and exit non-zero on failure.
+
+```bash
+python scripts/pdf_create.py spec.json -o out.pdf         # build PDF from JSON spec
+python scripts/pdf_make_form.py formspec.json -o form.pdf # build fillable AcroForm from JSON spec
+python scripts/pdf_form_layout.py formspec.json           # lint form layout BEFORE building
+python scripts/pdf_form_layout.py formspec.json --render-overlay boxes.png [--pdf form.pdf]
+python scripts/pdf_read.py doc.pdf --text                 # per-page text (JSON)
+python scripts/pdf_read.py doc.pdf --tables --csv-dir t/  # tables to JSON + CSV files
+python scripts/pdf_read.py doc.pdf --meta                 # metadata, page sizes, encrypted/scanned flags
+python scripts/pdf_read.py form.pdf --fields              # form fields: name, type, value
+python scripts/pdf_merge.py a.pdf b.pdf -o merged.pdf [--bookmarks]
+python scripts/pdf_split.py doc.pdf --pages 1-3,7 -o part.pdf [--rotate 90]
+python scripts/pdf_fill_form.py form.pdf --fields-json values.json -o filled.pdf [--flatten]
+python scripts/pdf_secure.py doc.pdf --encrypt -o enc.pdf --user-password your-password
+python scripts/pdf_secure.py enc.pdf --decrypt -o dec.pdf --password your-password
+python scripts/pdf_watermark.py doc.pdf --stamp mark.pdf -o stamped.pdf [--under]
+python scripts/pdf_stamp.py doc.pdf -o out.pdf --text "DRAFT" --x 150 --y 400 \
+    --font-size 60 --rotation 45 --opacity 0.3 --color "#cc0000" [--pages 1-3]
+python scripts/pdf_stamp.py doc.pdf -o out.pdf --image sig.png --x 400 --y 60 --width 120
+python scripts/pdf_page_image.py doc.pdf --pages 1-3 --dpi 150 --out-dir imgs/
+python scripts/pdf_meta.py doc.pdf --set-meta --title "T" --author "A" -o out.pdf
+python scripts/pdf_meta.py doc.pdf --attach data.csv -o out.pdf
+python scripts/pdf_meta.py doc.pdf --list-attachments | --extract-attachments dir/
+```
+
+## Quick Reference
+
+| Task | Tool | Command / API |
+|---|---|---|
+| Create doc (headings, tables, images) | reportlab platypus | `pdf_create.py spec.json -o out.pdf` |
+| Build fillable form | reportlab acroForm | `pdf_make_form.py formspec.json -o form.pdf` |
+| Lint form layout / overlay image | pure python + PIL | `pdf_form_layout.py formspec.json [--render-overlay o.png]` |
+| Per-page text | pdfplumber | `pdf_read.py f.pdf --text` |
+| Tables → JSON/CSV | pdfplumber | `pdf_read.py f.pdf --tables` |
+| Metadata / sizes / encrypted / scanned | pypdf + pdfplumber | `pdf_read.py f.pdf --meta` |
+| Merge (+ outline) | pypdf | `pdf_merge.py a.pdf b.pdf -o m.pdf` |
+| Split / extract / rotate | pypdf | `pdf_split.py f.pdf --pages 2-5 --rotate 90` |
+| List / fill / flatten form | pypdf | `pdf_read.py --fields`, `pdf_fill_form.py` |
+| Encrypt / decrypt (AES-256) | pypdf | `pdf_secure.py --encrypt/--decrypt` |
+| Watermark / stamp PDF page | pypdf | `pdf_watermark.py f.pdf --stamp w.pdf` |
+| Stamp text/image at coordinates | reportlab + pypdf | `pdf_stamp.py f.pdf --text "Sign here" --x 400 --y 60` |
+| Pages → PNG (review / OCR hand-off) | pypdfium2 or pdftoppm | `pdf_page_image.py f.pdf --pages 1-3 --out-dir imgs/` |
+| Set/clear metadata, attachments | pypdf | `pdf_meta.py --set-meta / --attach / --extract-attachments` |
+| Compress content streams | pypdf | `pdf_split.py f.pdf --pages 1-N --compress` |
+
+## Procedure
+
+1. **Inspect first.** Run `pdf_read.py file.pdf --meta`. Check `encrypted` (if true, decrypt first with `pdf_secure.py --decrypt`) and `likely_scanned_pages`. If pages are image-only, export them with `pdf_page_image.py --pages <scanned> --dpi 300 --out-dir imgs/` and hand the PNGs to the `references/ocr-extraction.md` skill — do not report empty text as "no content".
+2. **Create.** Write a JSON spec with `write_file` (elements: `heading`, `paragraph`, `table`, `image`, `pagebreak`; optional `title`/`author` metadata; page numbers are added automatically), then run `pdf_create.py`. Verify visually with `vision_analyze` on a rendered page image if layout matters.
+3. **Extract.** `--text` gives a JSON list of per-page strings; `--tables` gives row arrays per page and can also emit CSV files. Read results with `read_file`; never eyeball a binary PDF directly.
+4. **Manipulate.** `pdf_merge.py` concatenates and can add one bookmark per source file; `pdf_split.py` handles page ranges (1-based, e.g. `1-3,5,9-`), rotation in 90° steps, and `--compress`. Watermark by preparing a single-page stamp PDF (e.g. via `pdf_create.py`) and overlaying it with `pdf_watermark.py`; for one-liner stamps ("sign here", diagonal DRAFT, corner labels) use `pdf_stamp.py` with text or an image at explicit coordinates.
+5. **Build forms.** Write one form-spec JSON (fields with `label_box`/`entry_box` in PDF points — see `references/forms.md`), lint it with `pdf_form_layout.py` and fix every reported problem, optionally review the `--render-overlay` PNG with `vision_analyze`, then build with `pdf_make_form.py` and confirm with `pdf_read.py --fields`.
+6. **Fill forms.** List fields (`--fields`) to learn exact names and types, write a UTF-8 JSON of `{"FieldName": "value"}` with `write_file` (checkboxes accept `true`/`false`; radio/choice values must match the field's export options), then `pdf_fill_form.py`. Re-read with `--fields` to confirm values landed.
+7. **Metadata & attachments.** `pdf_meta.py --set-meta` writes Title/Author/Subject/Keywords (DocInfo); `--clear-meta` drops them; `--attach`/`--list-attachments`/`--extract-attachments` round-trip embedded files.
+8. **Secure.** Encrypt with distinct user/owner passwords and AES-256. To remove a password you know, `--decrypt` writes an unencrypted copy.
+9. **Verify** (see below) before reporting success.
+
+## Pitfalls
+
+- **Scanned PDFs**: empty `extract_text()` plus page images means there is no text layer. Route to `references/ocr-extraction.md`; do not fabricate text.
+- **Flattening limits**: `pdf_fill_form.py --flatten` uses pypdf's flatten support, which converts widget appearances into page content. It is reliable for plain text fields and checkboxes but can drop or misrender exotic widgets (rich text, custom appearance streams, some radio groups). Verify the flattened output visually with `vision_analyze`; for bulletproof flattening use an external renderer (e.g. Ghostscript or `pdftoppm`+reassembly) as a fallback.
+- **NeedAppearances**: after filling, viewers only render values if appearance streams exist. The fill script sets the AcroForm `NeedAppearances` flag so conforming viewers regenerate them; some minimal viewers ignore it — flatten if display fidelity matters.
+- **Non-Latin form values**: values are stored correctly (UTF-16), but the field's default font may lack glyphs, so a viewer can show blanks even though the data round-trips. Verify with `--fields`, not just visually.
+- **Compression expectations**: `--compress` only deflates content streams. Typical savings are 0–20%; it does nothing for PDFs dominated by images or already-compressed streams. It is not a substitute for image downsampling (Ghostscript territory).
+- **Permission flags don't enforce**: owner-password permission bits (no-print, no-copy) are polite requests that viewers may honor; any library (including pypdf) can read and strip them. Only the user password actually gates content via encryption. Never present permission flags as security.
+- **Table extraction is heuristic**: pdfplumber detects tables from ruling lines/word alignment; borderless or merged-cell tables may need `table_settings` tuning or manual cleanup.
+- **Page indexing**: helper CLIs take 1-based pages; pypdf APIs are 0-based. The scripts convert — don't double-convert.
+- **Rotated stamp text extraction**: pdfplumber's line grouping scrambles rotated glyphs (a 45° "DRAFT" extracts as stray letters); verify rotated stamps with `pypdf`'s `extract_text()` or a rendered image instead.
+- **Radio groups**: reportlab needs ≥2 `radio()` widgets per group, fills need the slashed export value (`"/red"`), and flatten fidelity is worst for radios — see `references/forms.md`.
+- **Metadata scope**: `pdf_meta.py` writes the classic DocInfo dictionary only; embedded XMP metadata (if any) is left untouched and may show different values in some viewers.
+- **PDF/A is out of scope**: pypdf/reportlab cannot produce or validate conformant PDF/A. If archival conformance is required, run Ghostscript via the `terminal` tool (e.g. `gs -dPDFA=2 -dPDFACompatibilityPolicy=1 -sColorConversionStrategy=UseDeviceIndependentColor -sDEVICE=pdfwrite -o out.pdf in.pdf` with a suitable ICC profile) and validate with veraPDF — both are external installs, and the result still needs validation, not assumption.
+- Rotation must be a multiple of 90; encrypted inputs must be decrypted before any other operation.
+
+## Verification
+
+- After create/merge/split: `pdf_read.py out.pdf --meta` — confirm `page_count`, and per-page `rotation` when you rotated.
+- After extraction: check the JSON is non-empty and spot-check a known string or cell.
+- Form design loop: `pdf_form_layout.py spec.json` must exit 0; then `--render-overlay boxes.png --pdf form.pdf` and review the PNG with `vision_analyze` (red = entry boxes with field names, blue = label boxes) asking about overlaps, misalignment, and labels detached from their fields. Iterate spec → lint → overlay until clean.
+- After building a form: `pdf_read.py form.pdf --fields` lists every spec field with the right type and options.
+- After form fill: `pdf_read.py filled.pdf --fields` and compare values (exact match, including non-ASCII).
+- After stamping: re-extract text (pypdf for rotated stamps) or render the page with `pdf_page_image.py` and inspect with `vision_analyze`.
+- After metadata/attachment edits: `pdf_read.py --meta` / `pdf_meta.py --list-attachments`, and re-extract an attachment to byte-compare.
+- After encrypt: `--meta` shows `"encrypted": true` and opening without a password fails; after decrypt, text extraction matches the original.
+- For anything visual (watermarks, flattened forms), render and inspect with `vision_analyze`.