dsh-office-kit 0.4.4 → 0.4.5

package/assets/skills/office-pdf/scripts/forms.py

@@ -15,6 +15,7 @@ Exit codes: 0 success, 1 bad arguments, 2 unreadable input or write failure,
 from __future__ import annotations
 
 import argparse
+import io
 import json
 import sys
 from pathlib import Path
@@ -31,6 +32,8 @@ except ImportError:  # pragma: no cover - pypdf is a required dependency
     IndirectObject = None  # type: ignore[assignment]
     NameObject = None  # type: ignore[assignment]
 
+from pdf_fonts import find_cjk_font, has_non_latin, register_cjk_font
+
 
 def emit(value, out=None):
     text = json.dumps(value, ensure_ascii=False, indent=2, default=str)
@@ -49,6 +52,114 @@ def fail(message, code=1, detail=None):
     raise SystemExit(code)
 
 
+def rebuild_cjk_appearances(path, values, font_file):
+    """Replace the appearance of text widgets whose value is non-Latin.
+
+    pypdf can only generate appearance streams with the base-14 fonts, so a
+    Chinese value would paint as question marks even though /V is correct.
+    Each such widget gets a fresh form XObject drawn with a real CJK font,
+    with the field border and background kept.
+    """
+    from pypdf import PdfReader, PdfWriter
+    from pypdf.generic import ArrayObject, DictionaryObject, FloatObject, NameObject, NumberObject, StreamObject
+    from reportlab.lib.colors import Color
+    from reportlab.pdfbase import pdfmetrics
+    from reportlab.pdfgen import canvas as rl_canvas
+
+    reader = PdfReader(str(path))
+    report = inspect_document(reader)
+    targets = []
+    for widget in report["widgets"]:
+        name = widget["name"]
+        value = values.get(name)
+        if not has_non_latin(value) or widget["field_type"] != "/Tx":
+            continue
+        rect = widget["rect"]
+        if len(rect) != 4:
+            continue
+        width = float(rect[2]) - float(rect[0])
+        height = float(rect[3]) - float(rect[1])
+        if width <= 1 or height <= 1:
+            continue
+        targets.append({"page": widget["page"], "name": name, "value": value, "width": width, "height": height})
+    if not targets:
+        return []
+
+    try:
+        font_path = find_cjk_font(font_file)
+    except FileNotFoundError as exc:
+        fail(str(exc), code=2)
+    if font_path is None:
+        fail("values contain non-Latin text but no CJK font was found; pass --font-file", code=2)
+    try:
+        font_name = register_cjk_font(font_path, "FormCJK")
+    except Exception as exc:
+        fail(f"cannot use font file {font_path}: {exc}", code=2)
+
+    overlay = io.BytesIO()
+    page = rl_canvas.Canvas(overlay)
+    for target in targets:
+        width, height = target["width"], target["height"]
+        page.setPageSize((width, height))
+        page.setFillColor(Color(1, 1, 1))
+        page.rect(0, 0, width, height, stroke=0, fill=1)
+        page.setStrokeColor(Color(0.55, 0.55, 0.55))
+        page.setLineWidth(0.5)
+        page.rect(0.25, 0.25, width - 0.5, height - 0.5, stroke=1, fill=0)
+        size = min(10.0, max(height - 3, 4.0))
+        while size > 4 and pdfmetrics.stringWidth(target["value"], font_name, size) > width - 4:
+            size -= 0.5
+        page.setFillColor(Color(0, 0, 0))
+        page.setFont(font_name, size)
+        baseline = (height - size) / 2 + size * 0.22
+        page.drawString(2, baseline, target["value"])
+        page.showPage()
+    page.save()
+    overlay.seek(0)
+    overlay_reader = PdfReader(overlay)
+
+    writer = PdfWriter()
+    writer.clone_document_from_reader(reader)
+    original_pages = len(writer.pages)
+    writer.append(overlay_reader)
+    appended = writer.pages[original_pages:]
+
+    for index, target in enumerate(targets):
+        overlay_page = appended[index]
+        contents = overlay_page.get_contents()
+        data = contents.get_data() if contents is not None else b""
+        resources = overlay_page.get("/Resources")
+        stream = StreamObject()
+        stream.set_data(data)
+        stream[NameObject("/Type")] = NameObject("/XObject")
+        stream[NameObject("/Subtype")] = NameObject("/Form")
+        stream[NameObject("/FormType")] = NumberObject(1)
+        stream[NameObject("/BBox")] = ArrayObject(
+            [FloatObject(0), FloatObject(0), FloatObject(target["width"]), FloatObject(target["height"])]
+        )
+        if resources is not None:
+            stream[NameObject("/Resources")] = resources
+        reference = writer._add_object(stream)
+        page = writer.pages[target["page"] - 1]
+        for annotation in page.get("/Annots") or []:
+            obj = annotation.get_object()
+            if obj.get("/Subtype") != "/Widget":
+                continue
+            parent = obj.get("/Parent")
+            source = parent.get_object() if parent is not None else obj
+            tag = obj.get("/T") or source.get("/T")
+            if str(tag) != target["name"]:
+                continue
+            obj[NameObject("/AP")] = DictionaryObject({NameObject("/N"): reference})
+            break
+
+    while len(writer.pages) > original_pages:
+        writer.remove_page(len(writer.pages) - 1)
+    with open(path, "wb") as handle:
+        writer.write(handle)
+    return [target["name"] for target in targets]
+
+
 def ref_id(node):
     if isinstance(node, IndirectObject):
         return node.idnum
@@ -285,6 +396,85 @@ def values_for_field(field, value):
     return str(value)
 
 
+def flatten_widgets(path):
+    """Paint every widget appearance into page content and drop the form."""
+    from pypdf import PdfReader, PdfWriter
+    from pypdf.generic import ArrayObject, DictionaryObject, FloatObject, NameObject, StreamObject
+
+    reader = PdfReader(str(path))
+    writer = PdfWriter()
+    writer.clone_document_from_reader(reader)
+
+    for page in writer.pages:
+        annotations = page.get("/Annots")
+        if not annotations:
+            continue
+        resources = page.get("/Resources")
+        if resources is None:
+            resources = DictionaryObject()
+            page[NameObject("/Resources")] = resources
+        resources = resources.get_object()
+        xobjects = resources.get("/XObject")
+        if xobjects is None:
+            xobjects = DictionaryObject()
+            resources[NameObject("/XObject")] = xobjects
+        xobjects = xobjects.get_object()
+
+        calls = []
+        for index, annotation in enumerate(annotations):
+            widget = annotation.get_object()
+            if widget.get("/Subtype") != "/Widget":
+                continue
+            normal = widget.get("/AP")
+            if normal is None:
+                continue
+            normal = normal.get_object().get("/N")
+            if normal is None:
+                continue
+            normal = normal.get_object()
+            if not hasattr(normal, "get_data"):
+                if not hasattr(normal, "keys"):
+                    continue
+                state = widget.get("/AS")
+                if state is None or normal.get(state) is None:
+                    continue
+                normal = normal[state].get_object()
+            if not hasattr(normal, "get_data"):
+                continue
+            name = NameObject(f"/DSCJK{index}")
+            xobjects[name] = normal.indirect_reference or normal
+            rect = [float(value) for value in widget.get("/Rect") or []]
+            bbox = [float(value) for value in normal.get("/BBox") or []]
+            if len(rect) != 4 or len(bbox) != 4:
+                continue
+            x, y, x2, y2 = rect[0], rect[1], rect[2], rect[3]
+            width = x2 - x
+            height = y2 - y
+            box_width = bbox[2] - bbox[0] or width
+            box_height = bbox[3] - bbox[1] or height
+            calls.append(
+                f"q {width / box_width:.6f} 0 0 {height / box_height:.6f} {x:.4f} {y:.4f} cm {name} Do Q"
+            )
+        if not calls:
+            continue
+        stream = StreamObject()
+        stream.set_data("\n".join(calls).encode("latin-1"))
+        reference = writer._add_object(stream)
+        existing = page.get("/Contents")
+        if existing is None:
+            page[NameObject("/Contents")] = reference
+        elif hasattr(existing, "get_data"):
+            page[NameObject("/Contents")] = ArrayObject([existing.indirect_reference or existing, reference])
+        else:
+            page[NameObject("/Contents")] = ArrayObject([*existing, reference])
+
+    for page in writer.pages:
+        page[NameObject("/Annots")] = ArrayObject()
+    writer.root_object.pop(NameObject("/AcroForm"), None)
+    with open(path, "wb") as handle:
+        writer.write(handle)
+
+
 def write_fill(args, values):
     from pypdf import PdfReader, PdfWriter
 
@@ -332,16 +522,28 @@ def write_fill(args, values):
         payload = current
     payload = {name: values_for_field(fields[name], value) for name, value in payload.items()}
 
-    writer.update_page_form_field_values(None, payload, auto_regenerate=False, flatten=args.flatten)
-    sync_button_states(writer)
-    if args.flatten:
-        writer.remove_annotations(subtypes="/Widget")
-        writer.root_object.pop(NameObject("/AcroForm"), None)
+    cjk_values = {name: value for name, value in expected.items() if has_non_latin(value)}
+    flatten = args.flatten
+    if flatten and cjk_values:
+        # pypdf can only flatten with base-14 fonts; keep the widgets, give them
+        # real CJK appearances, then paint every appearance into the page.
+        writer.update_page_form_field_values(None, payload, auto_regenerate=False, flatten=False)
+        sync_button_states(writer)
+    else:
+        writer.update_page_form_field_values(None, payload, auto_regenerate=False, flatten=flatten)
+        sync_button_states(writer)
+        if flatten:
+            writer.remove_annotations(subtypes="/Widget")
+            writer.root_object.pop(NameObject("/AcroForm"), None)
 
     args.out.parent.mkdir(parents=True, exist_ok=True)
     with open(args.out, "wb") as handle:
         writer.write(handle)
 
+    cjk_rendered = rebuild_cjk_appearances(args.out, expected, args.font_file)
+    if flatten and cjk_values:
+        flatten_widgets(args.out)
+
     checks = verify(args.out, expected, flatten=args.flatten)
     rendered = render(args.out, args.render_dir, args.dpi)
 
@@ -351,6 +553,7 @@ def write_fill(args, values):
         "output": str(args.out),
         "flatten": args.flatten,
         "reattached_fields": reattached,
+        "cjk_appearance_fields": cjk_rendered,
         "written": expected,
         "checks": checks,
         "rendered_pages": rendered,
@@ -517,6 +720,7 @@ def main():
         help='JSON object of {field: value}; accepts a file path, inline JSON, or "-" for stdin',
     )
     fill_parser.add_argument("--flatten", action="store_true", help="Paint values and remove every widget")
+    fill_parser.add_argument("--font-file", help="TTF/OTF/TTC file used to paint non-Latin values")
     fill_parser.add_argument("--no-reattach", action="store_true", help="Do not repair widgets missing from /AcroForm/Fields")
     fill_parser.add_argument("--allow-ambiguous", action="store_true", help="Fill even when widget and field views disagree")
     fill_parser.add_argument("--render-dir", help="Rasterise the result here for a visual check")

package/assets/skills/office-pdf/scripts/pdf_fonts.py

@@ -0,0 +1,67 @@
+"""Font handling shared by the PDF scripts.
+
+reportlab's base-14 fonts cover Latin-1 only, so a Chinese, Japanese, or Korean
+stamp or form value needs a real font file. These helpers find one, or report
+that the caller must pass --font-file.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+CJK_FONT_CANDIDATES = {
+    "win32": [
+        "C:/Windows/Fonts/msyh.ttc",
+        "C:/Windows/Fonts/msyhbd.ttc",
+        "C:/Windows/Fonts/simhei.ttf",
+        "C:/Windows/Fonts/simsun.ttc",
+    ],
+    "linux": [
+        "/usr/share/fonts/truetype/wqy/wqy-zenhei.ttc",
+        "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc",
+        "/usr/share/fonts/truetype/noto/NotoSansCJK-Regular.ttc",
+        "/usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc",
+    ],
+    "darwin": [
+        "/System/Library/Fonts/PingFang.ttc",
+        "/System/Library/Fonts/STHeiti Medium.ttc",
+        "/System/Library/Fonts/Hiragino Sans GB.ttc",
+    ],
+}
+
+
+def has_non_latin(text) -> bool:
+    """True when text needs a font beyond the base-14 Latin sets."""
+    return isinstance(text, str) and any(ord(character) > 0xFF for character in text)
+
+
+def find_cjk_font(explicit=None) -> Path | None:
+    """Return a usable CJK font path, honouring an explicit path first."""
+    if explicit:
+        path = Path(explicit).expanduser()
+        if not path.is_file():
+            raise FileNotFoundError(f"font file not found: {path}")
+        return path
+    override = os.environ.get("DSH_OFFICE_CJK_FONT")
+    if override and Path(override).expanduser().is_file():
+        return Path(override).expanduser()
+    platform_key = "win32" if sys.platform.startswith("win") else "darwin" if sys.platform == "darwin" else "linux"
+    for candidate in CJK_FONT_CANDIDATES[platform_key]:
+        path = Path(candidate)
+        if path.is_file():
+            return path
+    return None
+
+
+def register_cjk_font(path: Path, name: str) -> str:
+    """Register a TTF/OTF/TTC file with reportlab and return the resource name."""
+    from reportlab.pdfbase import pdfmetrics
+    from reportlab.pdfbase.ttfonts import TTFont
+
+    kwargs = {}
+    if path.suffix.lower() in {".ttc", ".otc", ".dfont"}:
+        kwargs["subfontIndex"] = 0
+    pdfmetrics.registerFont(TTFont(name, str(path), **kwargs))
+    return name

package/assets/skills/office-pdf/scripts/stamp_overlay.py

@@ -28,6 +28,8 @@ from pypdf import PdfReader, PdfWriter
 from reportlab.lib.colors import HexColor
 from reportlab.pdfgen import canvas
 
+from pdf_fonts import find_cjk_font, has_non_latin, register_cjk_font
+
 try:
     import pymupdf
 except ImportError:  # pragma: no cover - PyMuPDF is a required dependency
@@ -90,7 +92,8 @@ def main():
         default="bottom-center",
         help="Anchor of the stamp (default: bottom-center)",
     )
-    parser.add_argument("--font", default="Helvetica", help="reportlab base-14 font name (default: Helvetica)")
+    parser.add_argument("--font", default="Helvetica", help="reportlab font name (default: Helvetica)")
+    parser.add_argument("--font-file", help="TTF/OTF/TTC file used for the stamp; required for CJK unless a system font is found")
     parser.add_argument("--font-size", type=float, default=9.0, help="Font size in points (default: 9)")
     parser.add_argument("--color", default="#666666", help="Hex text colour (default: #666666)")
     parser.add_argument("--margin", type=float, default=28.0, help="Page-edge margin in points (default: 28)")
@@ -127,6 +130,22 @@ def main():
     page_count = len(reader.pages)
     selected = parse_pages(args.pages, page_count)
 
+    sample = args.text.format(page=1, pages=page_count)
+    font_name = args.font
+    font_file = None
+    if args.font_file or has_non_latin(sample):
+        try:
+            path = find_cjk_font(args.font_file)
+        except FileNotFoundError as exc:
+            fail(str(exc), code=2)
+        if path is None:
+            fail("stamp text needs a font beyond the base-14 Latin sets; pass --font-file with a CJK-capable TTF/TTC", code=2)
+        try:
+            font_name = register_cjk_font(path, "StampCJK")
+        except Exception as exc:
+            fail(f"cannot use font file {path}: {exc}", code=2)
+        font_file = str(path)
+
     writer = PdfWriter()
     stamped = []
     for index, page in enumerate(reader.pages, start=1):
@@ -138,9 +157,9 @@ def main():
             buffer = io.BytesIO()
             stamp = canvas.Canvas(buffer, pagesize=(width, height))
             stamp.setFillColor(colour)
-            stamp.setFont(args.font, args.font_size)
+            stamp.setFont(font_name, args.font_size)
             stamp.setFillAlpha(args.opacity)
-            text_width = stamp.stringWidth(label, args.font, args.font_size)
+            text_width = stamp.stringWidth(label, font_name, args.font_size)
             anchor = POSITIONS[args.position]
             if anchor == "left":
                 x = args.margin
@@ -185,7 +204,8 @@ def main():
         "pages": page_count,
         "stamped_pages": stamped,
         "position": args.position,
-        "font": args.font,
+        "font": font_name,
+        "font_file": font_file,
         "font_size": args.font_size,
         "color": args.color,
         "rendered_pages": rendered,

package/assets/skills/office-pdf/tasks/forms.md

@@ -55,6 +55,15 @@ Radio groups fail most: pypdf updates the group `/V` but leaves the old button's
 
 Flatten when the file must stay uneditable: fill with `flatten=True` (the value is painted into page content), remove every widget annotation (`subtypes="/Widget"`), and drop `/AcroForm` from the catalog. Then confirm no `/Widget` and no `/AcroForm` remain; otherwise readers still treat the file as editable.
 
+## Non-Latin values
+
+pypdf's appearance writer encodes WinAnsi, so a Chinese, Japanese, or Korean value lands in `/V` but the widget paints `?` or falls back to Helvetica. The script detects non-Latin values, registers a system CJK font (YaHei, WenQuanYi, PingFang, Noto) or the `--font-file` you pass, draws the value with reportlab, and replaces `/AP /N` with that form; the field names it repainted are listed as `cjk_appearance_fields`. `--flatten` paints those streams into the page content before stripping the widgets. Verify `/V` plus the render: the subset font has no ToUnicode map, so a text dump of the appearance is not reliable evidence.
+
+```bash
+"<python>" "<skill>/scripts/forms.py" fill --input "<form.pdf>" --out "<filled.pdf>" \
+  --values '{"full_name": "张伟"}' --font-file "C:/Windows/Fonts/msyh.ttc" --render-dir "<render>"
+```
+
 ## What cannot be filled, and the fix
 
 - **No appearance stream and a broken `/DA`**: the reader may not show a new value. Render after filling; if invisible, rebuild that widget's appearance with reportlab and merge it over the page.

package/assets/skills/office-pdf/tasks/stamp.md

@@ -52,7 +52,8 @@ Use each page's own `mediabox` when sizes may mix; a stamp drawn for A4 lands in
 - **Existing footers**: check the bottom margin first; the render is the proof, not the coordinate math. If a page number is already there, skip the stamp or use the opposite side.
 - **Rotated pages**: a `/Rotate 90` page renders sideways, so text drawn unrotated appears rotated. Rotate the stamp (`c.rotate(90)`, adjusting the origin), or rotate the page back first.
 - **Repeated stamping**: merging the same overlay twice doubles the opacity at the same position; re-run from the original input when iterating.
-- **Fonts**: Helvetica handles Latin; a CJK stamp needs a registered TTF, or the text renders as blank boxes. Check the render.
+- **Fonts**: Helvetica handles Latin; for CJK the script registers a system CJK font (YaHei, WenQuanYi, PingFang, Noto) and reports it as `font_file`. Pass `--font-file` with a specific TTF/OTF/TTC when the host has none; without one it exits 2 instead of drawing blank boxes. Check the render.
+- **Font files**: a `.ttc` collection needs its first face; the script handles that. Keep `--font` at its default when passing `--font-file` — the file overrides the base-14 name.
 - **Opacity**: `setFillAlpha` works only in viewers that honour transparency groups; a stamp that must show on any viewer is safer at full opacity.
 
 ## Verify

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dsh-office-kit",
   "type": "module",
-  "version": "0.4.4",
+  "version": "0.4.5",
   "description": "Office documents, PDF, presentations, and spreadsheets for DeepSeek Harness",
   "author": "Hairy",
   "license": "MIT",

package/python/office_cli.py

@@ -314,6 +314,94 @@ def xlsx_render(input_path: Path, out_dir: Path, dpi: int) -> dict[str, Any]:
     }
 
 
+def docx_used_fonts(path: Path) -> set[str]:
+    """Font names the document actually applies.
+
+    The kit reports every font in the package, including latent styles inherited
+    from python-docx's template (e.g. Courier in the unused MacroText style), so
+    a document with no Courier text still warns about it. Keep only fonts that
+    appear on a run, in a referenced style, in the defaults, or in the theme.
+    """
+    import zipfile
+
+    from lxml import etree
+
+    w = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
+    fonts: set[str] = set()
+    styles: dict[str, tuple[set[str], str | None]] = {}
+    referenced: set[str] = set()
+    theme_fonts: set[str] = set()
+
+    def rfonts(element) -> set[str]:
+        found = set()
+        for node in element.iter(f"{w}rFonts"):
+            for attribute in ("ascii", "hAnsi", "eastAsia", "cs"):
+                value = node.get(f"{w}{attribute}")
+                if value:
+                    found.add(value)
+        return found
+
+    with zipfile.ZipFile(path) as archive:
+        names = set(archive.namelist())
+
+        def load(name: str):
+            return etree.fromstring(archive.read(name))
+
+        if "word/styles.xml" in names:
+            root = load("word/styles.xml")
+            defaults = root.find(f"{w}docDefaults")
+            if defaults is not None:
+                fonts.update(rfonts(defaults))
+            for style in root.iter(f"{w}style"):
+                style_id = style.get(f"{w}styleId")
+                based_on = style.find(f"{w}basedOn")
+                styles[style_id] = (rfonts(style), based_on.get(f"{w}val") if based_on is not None else None)
+
+        for name in names:
+            if not name.startswith("word/") or not name.endswith(".xml"):
+                continue
+            if name.startswith(("word/styles", "word/theme/")):
+                continue
+            root = load(name)
+            fonts.update(rfonts(root))
+            for tag in ("pStyle", "rStyle", "tblStyle"):
+                for element in root.iter(f"{w}{tag}"):
+                    value = element.get(f"{w}val")
+                    if value:
+                        referenced.add(value)
+
+        if "word/theme/theme1.xml" in names:
+            theme = load("word/theme/theme1.xml")
+            for tag in ("majorFont", "minorFont"):
+                for element in theme.iter(tag):
+                    latin = element.find("latin")
+                    if latin is not None and latin.get("typeface"):
+                        theme_fonts.add(latin.get("typeface"))
+
+    pending = list(referenced)
+    seen = set()
+    while pending:
+        style_id = pending.pop()
+        if style_id in seen:
+            continue
+        seen.add(style_id)
+        entry = styles.get(style_id)
+        if entry is None:
+            continue
+        style_fonts, based_on = entry
+        fonts.update(style_fonts)
+        if based_on:
+            pending.append(based_on)
+    return fonts | theme_fonts
+
+
+def filter_missing_fonts(input_path: Path, kind: str, missing: list[str]) -> list[str]:
+    if kind != "docx" or not missing:
+        return missing
+    used = {name.lower() for name in docx_used_fonts(input_path)}
+    return [name for name in missing if name.lower() in used]
+
+
 def render_office(input_path: Path, kind: str, out_dir: Path, dpi: int, emit_pdf: bool = False) -> dict[str, Any]:
     """Rasterise an Office file page by page through the kit.
 
@@ -346,7 +434,7 @@ def render_office(input_path: Path, kind: str, out_dir: Path, dpi: int, emit_pdf
     shutil.rmtree(staging, ignore_errors=True)
     if not images:
         fail(f"the LibreOffice kit rendered no pages for {input_path.name}")
-    missing = [str(name) for name in report.get("missingFonts") or []]
+    missing = filter_missing_fonts(input_path, kind, [str(name) for name in report.get("missingFonts") or []])
     warnings = glyph_warnings(missing)
     report_warnings(warnings)
     result: dict[str, Any] = {
@@ -365,10 +453,11 @@ def render_office(input_path: Path, kind: str, out_dir: Path, dpi: int, emit_pdf
     if emit_pdf:
         converted = export_pdf(input_path, out_dir, dpi)
         result["pdf"] = converted["pdf"]
-        if not missing and converted["missing_fonts"]:
-            result["missing_fonts"] = converted["missing_fonts"]
+        converted_missing = filter_missing_fonts(input_path, kind, converted["missing_fonts"])
+        if not missing and converted_missing:
+            result["missing_fonts"] = converted_missing
             result["glyph_fallback"] = True
-            result["warnings"] = glyph_warnings(converted["missing_fonts"])
+            result["warnings"] = glyph_warnings(converted_missing)
             report_warnings(result["warnings"])
     return result