@cteno/sdk-linux-x64 0.0.0-stage → 2.10.14

package/skills/pdf/scripts/_raster.py

@@ -0,0 +1,76 @@
+"""Shared page rasterizer with a fallback chain: pypdfium2 -> pdftoppm.
+
+Returns PIL Images so callers can annotate/save. Not a CLI.
+"""
+from __future__ import annotations
+
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+
+
+def available_backends() -> list[str]:
+    """Names of usable rasterizer backends, in preference order."""
+    backends = []
+    try:
+        import pypdfium2
+        backends.append("pypdfium2")
+    except ImportError:
+        pass
+    if shutil.which("pdftoppm"):
+        backends.append("pdftoppm")
+    return backends
+
+
+def missing_hints() -> list[str]:
+    """Install hints for when no backend is available."""
+    return [
+        "python3 -m pip install pypdfium2",
+        "poppler-utils (provides pdftoppm), e.g. apt-get install poppler-utils",
+    ]
+
+
+def rasterize_page(pdf_path: str, page: int, dpi: int = 150, password: str | None = None):
+    """Render one 1-based page to a PIL Image, or None if no backend works.
+
+    Raises ValueError for an out-of-range page when a backend is present.
+    """
+    for backend in available_backends():
+        if backend == "pypdfium2":
+            return _via_pdfium(pdf_path, page, dpi, password)
+        if backend == "pdftoppm":
+            img = _via_pdftoppm(pdf_path, page, dpi, password)
+            if img is not None:
+                return img
+    return None
+
+
+def _via_pdfium(pdf_path: str, page: int, dpi: int, password: str | None):
+    import pypdfium2 as pdfium
+    doc = pdfium.PdfDocument(pdf_path, password=password)
+    try:
+        if not 1 <= page <= len(doc):
+            raise ValueError(f"page {page} out of range 1-{len(doc)}")
+        bitmap = doc[page - 1].render(scale=dpi / 72.0)
+        return bitmap.to_pil().convert("RGB")
+    finally:
+        doc.close()
+
+
+def _via_pdftoppm(pdf_path: str, page: int, dpi: int, password: str | None):
+    from PIL import Image
+    with tempfile.TemporaryDirectory() as tmp:
+        prefix = str(Path(tmp) / "page")
+        cmd = ["pdftoppm", "-png", "-r", str(dpi), "-f", str(page), "-l", str(page)]
+        if password:
+            cmd += ["-upw", password]
+        cmd += [pdf_path, prefix]
+        proc = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", check=False)
+        if proc.returncode != 0:
+            raise ValueError(f"pdftoppm failed: {proc.stderr.strip()}")
+        produced = sorted(Path(tmp).glob("page*.png"))
+        if not produced:
+            raise ValueError(f"page {page} out of range (pdftoppm produced no image)")
+        with Image.open(produced[0]) as img:
+            return img.convert("RGB")

package/skills/pdf/scripts/extract_marker.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""Extract text from documents using marker-pdf. High-quality OCR + layout analysis.
+
+Requires ~3-5GB disk (PyTorch + models downloaded on first use).
+Supports: PDF, DOCX, PPTX, XLSX, HTML, EPUB, images.
+
+Usage:
+    python extract_marker.py document.pdf
+    python extract_marker.py document.pdf --output_dir ./output
+    python extract_marker.py presentation.pptx
+    python extract_marker.py spreadsheet.xlsx
+    python extract_marker.py scanned_doc.pdf           # OCR works here
+    python extract_marker.py document.pdf --json        # Structured output
+    python extract_marker.py document.pdf --use_llm     # LLM-boosted accuracy
+"""
+import sys
+import os
+
+def convert(path, output_dir=None, output_format="markdown", use_llm=False):
+    from marker.converters.pdf import PdfConverter
+    from marker.models import create_model_dict
+    from marker.config.parser import ConfigParser
+
+    config_dict = {}
+    if use_llm:
+        config_dict["use_llm"] = True
+
+    config_parser = ConfigParser(config_dict)
+    models = create_model_dict()
+    converter = PdfConverter(config=config_parser.generate_config_dict(), artifact_dict=models)
+    rendered = converter(path)
+
+    if output_format == "json":
+        import json
+        print(json.dumps({
+            "markdown": rendered.markdown,
+            "metadata": rendered.metadata if hasattr(rendered, "metadata") else {},
+        }, indent=2, ensure_ascii=False))
+    else:
+        print(rendered.markdown)
+
+    # Save images if output_dir specified
+    if output_dir and hasattr(rendered, "images") and rendered.images:
+        from pathlib import Path
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+        for name, img_data in rendered.images.items():
+            img_path = os.path.join(output_dir, name)
+            with open(img_path, "wb") as f:
+                f.write(img_data)
+        print(f"\nSaved {len(rendered.images)} image(s) to {output_dir}/", file=sys.stderr)
+
+
+def check_requirements():
+    """Check disk space before installing."""
+    import shutil
+    free_gb = shutil.disk_usage("/").free / (1024**3)
+    if free_gb < 5:
+        print(f"⚠️  Only {free_gb:.1f}GB free. marker-pdf needs ~5GB for PyTorch + models.")
+        print("Use pymupdf instead (scripts/extract_pymupdf.py) or free up disk space.")
+        sys.exit(1)
+    print(f"✓ {free_gb:.1f}GB free — sufficient for marker-pdf")
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Extract text from documents using marker-pdf (high-quality OCR + layout analysis)."
+    )
+    parser.add_argument("path", nargs="?", help="Document to convert (PDF, DOCX, PPTX, XLSX, HTML, EPUB, image)")
+    parser.add_argument("--output_dir", help="Directory to save extracted images")
+    parser.add_argument("--json", action="store_true", help="Structured JSON output instead of markdown")
+    parser.add_argument("--use_llm", action="store_true", help="LLM-boosted accuracy")
+    parser.add_argument("--check", action="store_true", help="Check disk space requirements and exit")
+    args = parser.parse_args()
+
+    if args.check:
+        check_requirements()
+        sys.exit(0)
+    if not args.path:
+        parser.error("path is required unless --check is given")
+
+    convert(
+        args.path,
+        output_dir=args.output_dir,
+        output_format="json" if args.json else "markdown",
+        use_llm=args.use_llm,
+    )

package/skills/pdf/scripts/extract_pymupdf.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""Extract text from documents using pymupdf. Lightweight (~25MB), no models.
+
+Usage:
+    python extract_pymupdf.py document.pdf
+    python extract_pymupdf.py document.pdf --markdown
+    python extract_pymupdf.py document.pdf --pages 0-4
+    python extract_pymupdf.py document.pdf --images output_dir/
+    python extract_pymupdf.py document.pdf --tables
+    python extract_pymupdf.py document.pdf --metadata
+"""
+import sys
+import json
+
+def extract_text(path, pages=None):
+    import pymupdf
+    doc = pymupdf.open(path)
+    page_range = range(len(doc)) if pages is None else pages
+    for i in page_range:
+        if i < len(doc):
+            print(f"\n--- Page {i+1}/{len(doc)} ---\n")
+            print(doc[i].get_text())
+
+def extract_markdown(path, pages=None):
+    import pymupdf4llm
+    md = pymupdf4llm.to_markdown(path, pages=pages)
+    print(md)
+
+def extract_tables(path):
+    import pymupdf
+    doc = pymupdf.open(path)
+    for i, page in enumerate(doc):
+        tables = page.find_tables()
+        for j, table in enumerate(tables.tables):
+            print(f"\n--- Page {i+1}, Table {j+1} ---\n")
+            df = table.to_pandas()
+            print(df.to_markdown(index=False))
+
+def extract_images(path, output_dir):
+    import pymupdf
+    from pathlib import Path
+    Path(output_dir).mkdir(parents=True, exist_ok=True)
+    doc = pymupdf.open(path)
+    count = 0
+    for i, page in enumerate(doc):
+        for img_idx, img in enumerate(page.get_images(full=True)):
+            xref = img[0]
+            pix = pymupdf.Pixmap(doc, xref)
+            if pix.n >= 5:
+                pix = pymupdf.Pixmap(pymupdf.csRGB, pix)
+            out_path = f"{output_dir}/page{i+1}_img{img_idx+1}.png"
+            pix.save(out_path)
+            count += 1
+    print(f"Extracted {count} images to {output_dir}/")
+
+def show_metadata(path):
+    import pymupdf
+    doc = pymupdf.open(path)
+    print(json.dumps({
+        "pages": len(doc),
+        "title": doc.metadata.get("title", ""),
+        "author": doc.metadata.get("author", ""),
+        "subject": doc.metadata.get("subject", ""),
+        "creator": doc.metadata.get("creator", ""),
+        "producer": doc.metadata.get("producer", ""),
+        "format": doc.metadata.get("format", ""),
+    }, indent=2))
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Extract text/tables/images/metadata from documents using pymupdf (lightweight, no models)."
+    )
+    parser.add_argument("path", help="Document to read")
+    parser.add_argument("--pages", help="Page selection: N or START-END (0-indexed)")
+    parser.add_argument("--markdown", action="store_true", help="Markdown output via pymupdf4llm")
+    parser.add_argument("--tables", action="store_true", help="Extract tables as markdown")
+    parser.add_argument("--images", nargs="?", const="./images", metavar="OUTPUT_DIR",
+                        help="Extract embedded images to OUTPUT_DIR (default ./images)")
+    parser.add_argument("--metadata", action="store_true", help="Show document metadata as JSON")
+    args = parser.parse_args()
+
+    pages = None
+    if args.pages:
+        if "-" in args.pages:
+            start, end = args.pages.split("-")
+            pages = list(range(int(start), int(end) + 1))
+        else:
+            pages = [int(args.pages)]
+
+    if args.metadata:
+        show_metadata(args.path)
+    elif args.tables:
+        extract_tables(args.path)
+    elif args.images is not None:
+        extract_images(args.path, args.images)
+    elif args.markdown:
+        extract_markdown(args.path, pages=pages)
+    else:
+        extract_text(args.path, pages=pages)

package/skills/pdf/scripts/pdf_create.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+"""Create a PDF from a JSON spec using reportlab platypus.
+
+Spec format (UTF-8 JSON):
+{
+  "title": "Example Report",
+  "author": "example-author",
+  "page_size": "A4",            // or "letter" (default: A4)
+  "page_numbers": true,          // default true
+  "font": "auto",                // auto (default): CJK text gets a CJK font automatically;
+                                 // or a path to a .ttf/.otf/.ttc, or "STSong-Light" (built-in CID font)
+  "elements": [
+    {"type": "heading", "text": "Section 1", "level": 1},
+    {"type": "paragraph", "text": "Body text..."},
+    {"type": "table", "rows": [["H1", "H2"], ["a", "b"]], "header": true},
+    {"type": "image", "path": "chart.png", "width": 400},
+    {"type": "pagebreak"}
+  ]
+}
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+
+def _reconfigure_stdio() -> None:
+    for stream in (sys.stdout, sys.stderr):
+        try:
+            stream.reconfigure(encoding="utf-8")
+        except Exception:
+            pass
+
+
+def _has_cjk(spec: dict) -> bool:
+    import re
+    pat = re.compile(r"[\u3000-\u303f\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\uff00-\uffef]")
+    return bool(pat.search(json.dumps(spec, ensure_ascii=False)))
+
+
+SYSTEM_CJK_FONTS = [
+    # (path, subfont index) - first one that exists wins. Fonts are embedded (subset) in the PDF.
+    # reportlab needs TrueType outlines (glyf): Hiragino (CFF) cannot be embedded
+    ("/System/Library/Fonts/STHeiti Medium.ttc", 1),     # Heiti SC Medium
+    ("/System/Library/Fonts/Supplemental/Songti.ttc", 6),  # Songti SC Regular
+    ("C:/Windows/Fonts/msyh.ttc", 0),
+    ("C:/Windows/Fonts/simhei.ttf", 0),
+    ("C:/Windows/Fonts/simsun.ttc", 0),
+    ("/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", 2),
+    ("/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc", 2),
+    ("/usr/share/fonts/truetype/wqy/wqy-microhei.ttc", 0),
+]
+
+
+def register_font(spec: dict):
+    """Return (regular, bold) font names to use; registers a CJK font when needed."""
+    from reportlab.pdfbase import pdfmetrics
+    choice = spec.get("font", "auto")
+    if choice == "auto" and not _has_cjk(spec):
+        return "Helvetica", "Helvetica-Bold"
+    if choice in ("auto", None):
+        from reportlab.pdfbase.ttfonts import TTFont
+        import os
+        for path, idx in SYSTEM_CJK_FONTS:
+            if os.path.exists(path):
+                try:
+                    pdfmetrics.registerFont(TTFont("CJK", path, subfontIndex=idx))
+                    return "CJK", "CJK"
+                except Exception:
+                    continue
+        choice = "STSong-Light"  # built-in Adobe CID font: always works, not embedded
+    if choice in ("STSong-Light", "MSung-Light", "HeiseiMin-W3", "HYSMyeongJo-Medium"):
+        from reportlab.pdfbase.cidfonts import UnicodeCIDFont
+        pdfmetrics.registerFont(UnicodeCIDFont(choice))
+        return choice, choice
+    if str(choice).lower().endswith((".ttf", ".otf", ".ttc")):
+        from reportlab.pdfbase.ttfonts import TTFont
+        pdfmetrics.registerFont(TTFont("UserFont", choice))
+        return "UserFont", "UserFont"
+    return choice, choice
+
+
+def build_pdf(spec: dict, out_path: str) -> int:
+    try:
+        from reportlab.lib import colors
+        from reportlab.lib.pagesizes import A4, letter
+        from reportlab.lib.styles import getSampleStyleSheet
+        from reportlab.lib.units import inch
+        from reportlab.platypus import (
+            Image,
+            PageBreak,
+            Paragraph,
+            SimpleDocTemplate,
+            Spacer,
+            Table,
+            TableStyle,
+        )
+    except ImportError:
+        print("Missing dependency: install with 'python3 -m pip install reportlab'", file=sys.stderr)
+        return 2
+
+    page_size = letter if str(spec.get("page_size", "A4")).lower() == "letter" else A4
+    styles = getSampleStyleSheet()
+    regular, bold = register_font(spec)
+    if regular != "Helvetica":
+        from reportlab.lib.styles import ParagraphStyle
+        for name in ("BodyText", "Heading1", "Heading2", "Heading3", "Title"):
+            st = styles[name]
+            st.fontName = bold if name != "BodyText" else regular
+            st.wordWrap = "CJK"  # break lines between CJK characters
+        styles["BodyText"].leading = styles["BodyText"].fontSize * 1.6
+    story = []
+    for el in spec.get("elements", []):
+        etype = el.get("type")
+        if etype == "heading":
+            level = min(max(int(el.get("level", 1)), 1), 3)
+            story.append(Paragraph(el.get("text", ""), styles[f"Heading{level}"]))
+        elif etype == "paragraph":
+            story.append(Paragraph(el.get("text", ""), styles["BodyText"]))
+            story.append(Spacer(1, 6))
+        elif etype == "table":
+            rows = el.get("rows", [])
+            if not rows:
+                continue
+            body_style = styles["BodyText"]
+            rows = [[Paragraph(str(c), body_style) for c in r] for r in rows]  # cells wrap instead of overflowing
+            table = Table(rows, repeatRows=1 if el.get("header", True) else 0)
+            style = [
+                ("FONTNAME", (0, 0), (-1, -1), regular),
+                ("GRID", (0, 0), (-1, -1), 0.5, colors.grey),
+                ("VALIGN", (0, 0), (-1, -1), "TOP"),
+            ]
+            if el.get("header", True):
+                style += [
+                    ("BACKGROUND", (0, 0), (-1, 0), colors.lightgrey),
+                    ("FONTNAME", (0, 0), (-1, 0), bold),
+                ]
+            table.setStyle(TableStyle(style))
+            story.append(table)
+            story.append(Spacer(1, 10))
+        elif etype == "image":
+            kwargs = {}
+            if el.get("width"):
+                kwargs["width"] = float(el["width"])
+            if el.get("height"):
+                kwargs["height"] = float(el["height"])
+            img = Image(el["path"], **kwargs)
+            if "width" in kwargs and "height" not in kwargs:
+                # keep aspect ratio
+                ratio = img.imageHeight / img.imageWidth
+                img.drawWidth = kwargs["width"]
+                img.drawHeight = kwargs["width"] * ratio
+            story.append(img)
+            story.append(Spacer(1, 10))
+        elif etype == "pagebreak":
+            story.append(PageBreak())
+        else:
+            print(f"Warning: unknown element type {etype!r}, skipped", file=sys.stderr)
+
+    def draw_page_number(canvas, doc):
+        if spec.get("page_numbers", True):
+            canvas.saveState()
+            canvas.setFont(regular, 9)
+            canvas.drawCentredString(page_size[0] / 2.0, 0.5 * inch, f"Page {doc.page}")
+            canvas.restoreState()
+
+    doc = SimpleDocTemplate(
+        out_path,
+        pagesize=page_size,
+        title=spec.get("title", ""),
+        author=spec.get("author", ""),
+    )
+    doc.build(story, onFirstPage=draw_page_number, onLaterPages=draw_page_number)
+    print(json.dumps({"output": out_path, "elements": len(spec.get("elements", []))}))
+    return 0
+
+
+def main() -> int:
+    _reconfigure_stdio()
+    parser = argparse.ArgumentParser(description="Create a PDF from a JSON spec (reportlab).")
+    parser.add_argument("spec", help="Path to UTF-8 JSON spec file")
+    parser.add_argument("-o", "--output", required=True, help="Output PDF path")
+    args = parser.parse_args()
+    with open(args.spec, encoding="utf-8") as fh:
+        spec = json.load(fh)
+    return build_pdf(spec, args.output)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/skills/pdf/scripts/pdf_fill_form.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""Fill AcroForm fields from a UTF-8 JSON file; optionally flatten.
+
+The JSON is a flat object: {"FieldName": "value", "Agree": true, ...}
+- text fields: strings
+- checkboxes: true/false (or an explicit on-state name like "/Yes")
+- radio / dropdown: the export value as a string (see pdf_read.py --fields "options")
+
+Sets NeedAppearances so conforming viewers regenerate field appearances.
+Flattening uses pypdf appearance merging; verify visually for exotic widgets.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+
+def main() -> int:
+    for stream in (sys.stdout, sys.stderr):
+        try:
+            stream.reconfigure(encoding="utf-8")
+        except Exception:
+            pass
+    parser = argparse.ArgumentParser(description="Fill PDF AcroForm fields from JSON (pypdf).")
+    parser.add_argument("pdf", help="Input form PDF")
+    parser.add_argument("--fields-json", required=True, help="UTF-8 JSON file of field values")
+    parser.add_argument("-o", "--output", required=True, help="Output PDF path")
+    parser.add_argument("--flatten", action="store_true",
+                        help="Make fields read-only and burn appearances into the page")
+    parser.add_argument("--password", help="Password if the input is encrypted")
+    args = parser.parse_args()
+
+    try:
+        from pypdf import PdfReader, PdfWriter
+        from pypdf.generic import BooleanObject, NameObject
+    except ImportError:
+        print("Missing dependency: install with 'python3 -m pip install pypdf'", file=sys.stderr)
+        return 2
+
+    with open(args.fields_json, encoding="utf-8") as fh:
+        values = json.load(fh)
+
+    reader = PdfReader(args.pdf)
+    if reader.is_encrypted:
+        if args.password is None or not reader.decrypt(args.password):
+            print("Error: input is encrypted; pass --password", file=sys.stderr)
+            return 3
+    available = set((reader.get_fields() or {}).keys())
+    missing = [name for name in values if name not in available]
+    if missing:
+        print(f"Warning: fields not found in form, skipped: {missing}", file=sys.stderr)
+
+    writer = PdfWriter()
+    writer.append(reader)
+
+    # Normalize checkbox booleans to the field's actual on-state name
+    # (e.g. "/Yes"): pypdf does not reliably map bare True to the on-state.
+    field_info = reader.get_fields() or {}
+    fill = {}
+    for name, value in values.items():
+        if name not in available:
+            continue
+        if isinstance(value, bool):
+            states = [str(s) for s in (field_info[name].get("/_States_") or [])]
+            on_state = next((s for s in states if s != "/Off"), "/Yes")
+            value = on_state if value else "/Off"
+        fill[name] = value
+    for page in writer.pages:
+        writer.update_page_form_field_values(page, fill, auto_regenerate=False)
+
+    # Set NeedAppearances so viewers render values even without appearance streams.
+    root = writer._root_object
+    if "/AcroForm" in root:
+        root["/AcroForm"][NameObject("/NeedAppearances")] = BooleanObject(True)
+
+    flattened = False
+    if args.flatten:
+        try:
+            # pypdf >= 5: flatten via update with flags making fields read-only,
+            # then remove interactivity by merging appearances.
+            for page in writer.pages:
+                writer.update_page_form_field_values(page, fill, flags=1)  # 1 = ReadOnly
+            flattened = True
+        except Exception as exc:
+            print(f"Warning: flatten step failed ({exc}); output keeps interactive fields",
+                  file=sys.stderr)
+
+    with open(args.output, "wb") as fh:
+        writer.write(fh)
+    print(json.dumps({"output": args.output, "filled": sorted(fill), "skipped": missing,
+                      "flattened": flattened}, ensure_ascii=False))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())