@litfamily/litopencode 1.0.7 → 1.0.9

package/skills/lit-docx/scripts/convert_md_to_pdf.py

@@ -0,0 +1,281 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+from convert_md_to_docx import (
+    detect_locale,
+    fix_dashes,
+    fix_ellipsis,
+    fix_quotes,
+    fix_unit_spacing,
+    load_publisher,
+    normalize_double_spaces,
+    parse_frontmatter,
+)
+
+
+def command_path(name: str) -> str | None:
+    return shutil.which(name)
+
+
+def run_checked(command: list[str], workdir: Path | None = None) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        command,
+        cwd=str(workdir) if workdir else None,
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+
+
+def ensure_pandoc() -> None:
+    if not command_path("pandoc"):
+        raise SystemExit("Error: pandoc not found. Install pandoc to use convert_md_to_pdf.py")
+
+
+def preprocess_markdown(md_path: Path, publisher: dict, locale: str) -> tuple[dict, str, str]:
+    md_text = md_path.read_text(encoding="utf-8")
+    frontmatter, body = parse_frontmatter(md_text)
+    detected_locale = detect_locale(body) if locale == "auto" else locale
+    design = publisher["design"]
+    dash_rules = design.get("micro_typography", {}).get("dashes")
+    body = fix_dashes(body, dash_rules)
+    body = fix_quotes(body)
+    body = fix_unit_spacing(body)
+    body = fix_ellipsis(body)
+    body = normalize_double_spaces(body)
+    return frontmatter, body, detected_locale
+
+
+def format_author_meta(frontmatter: dict) -> str:
+    authors = frontmatter.get("authors") or []
+    if not isinstance(authors, list):
+        return str(authors)
+    items: list[str] = []
+    for author in authors:
+        if isinstance(author, dict):
+            name = str(author.get("name") or "").strip()
+            aff = author.get("affiliation")
+            if isinstance(aff, list):
+                suffix = ",".join(str(item) for item in aff)
+            elif aff is None:
+                suffix = ""
+            else:
+                suffix = str(aff)
+            marker = "*" if author.get("corresponding") else ""
+            items.append(f"{name}$^{{{suffix}{marker}}}$" if suffix or marker else name)
+        else:
+            items.append(str(author))
+    return ", ".join(item for item in items if item)
+
+
+def format_affiliation_meta(frontmatter: dict) -> str:
+    affiliations = frontmatter.get("affiliations") or {}
+    if not isinstance(affiliations, dict):
+        return str(affiliations)
+    lines = []
+    for key, value in affiliations.items():
+        lines.append(f"$^{{{key}}}$ {value}")
+    return r" \\ ".join(lines)
+
+
+def format_keywords(frontmatter: dict) -> str:
+    keywords = frontmatter.get("keywords") or []
+    if isinstance(keywords, list):
+        return "; ".join(str(item) for item in keywords)
+    return str(keywords)
+
+
+def short_author_list(frontmatter: dict) -> str:
+    authors = frontmatter.get("authors") or []
+    if not isinstance(authors, list) or not authors:
+        return ""
+    first = authors[0]
+    if isinstance(first, dict):
+        first_name = str(first.get("name") or "")
+    else:
+        first_name = str(first)
+    if len(authors) == 1:
+        return first_name
+    return f"{first_name} et al."
+
+
+def template_variables(frontmatter: dict, publisher_name: str, publisher: dict, detected_locale: str, mode: str) -> dict[str, str]:
+    latex_cfg = publisher["latex"]
+    cjk_cfg = latex_cfg.get("cjk") or {}
+    body_font = ((publisher.get("docx") or {}).get("font") or {}).get("body") or {}
+    title = str(frontmatter.get("title") or "")
+    return {
+        "title": title,
+        "author-meta": format_author_meta(frontmatter),
+        "affiliation-meta": format_affiliation_meta(frontmatter),
+        "abstract": str(frontmatter.get("abstract") or ""),
+        "keywords-meta": format_keywords(frontmatter),
+        "corresponding-email": str(frontmatter.get("corresponding_email") or ""),
+        "short-title": title[:80],
+        "author-list-short": short_author_list(frontmatter),
+        "documentclass": str(latex_cfg.get("documentclass") or "article"),
+        "classoptions": ",".join(str(item) for item in latex_cfg.get("classoptions") or []),
+        "mainfont": str(body_font.get("family") or "Times New Roman"),
+        "cjk-main-font": str(cjk_cfg.get("main_font") or "Pretendard"),
+        "publisher-name": publisher_name,
+        "detected-locale": detected_locale,
+        "mode": mode,
+        "has-cjk": "true" if detected_locale in {"ko", "mixed"} else "false",
+    }
+
+
+def build_pandoc_command(md_path: Path, out_path: Path, template_path: Path, variables: dict[str, str]) -> list[str]:
+    command = [
+        "pandoc",
+        str(md_path),
+        "--standalone",
+        "--from",
+        "markdown",
+        "--to",
+        "latex",
+        "--template",
+        str(template_path),
+        "--output",
+        str(out_path),
+    ]
+    for key, value in variables.items():
+        if value:
+            command.extend(["-V", f"{key}={value}"])
+    return command
+
+
+def render_markdown_to_latex(
+    md_path: Path,
+    tex_path: Path,
+    publisher_name: str,
+    registry_path: Path,
+    locale: str = "auto",
+    mode: str = "submission",
+) -> str:
+    ensure_pandoc()
+    publisher = load_publisher(registry_path, publisher_name)
+    frontmatter, body, detected_locale = preprocess_markdown(md_path, publisher, locale)
+    script_dir = Path(__file__).resolve().parent
+    template_path = script_dir.parent / publisher["latex"]["pandoc_template"]
+    if not template_path.exists():
+        raise SystemExit(f"Error: LaTeX template not found: {template_path}")
+    tex_path.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tmp_md = Path(tmpdir) / "input.md"
+        tmp_md.write_text(body, encoding="utf-8")
+        result = run_checked(
+            build_pandoc_command(tmp_md, tex_path, template_path, template_variables(frontmatter, publisher_name, publisher, detected_locale, mode))
+        )
+        if result.returncode != 0:
+            raise SystemExit(result.stdout + result.stderr)
+    return detected_locale
+
+
+def kpsewhich_has(target: str) -> bool:
+    result = run_checked(["kpsewhich", target])
+    return result.returncode == 0 and bool(result.stdout.strip())
+
+
+def preflight_pdf_compile(publisher_name: str, publisher: dict, detected_locale: str) -> None:
+    ensure_pandoc()
+    if not command_path("xelatex"):
+        raise SystemExit(
+            "Error: xelatex not found. TeX Live is required for PDF output. "
+            "Install TeX Live with elsarticle, achemso, IEEEtran, sn-jnl, oblivoir, xeCJK, and latexmk."
+        )
+    if not command_path("kpsewhich"):
+        raise SystemExit(
+            "Error: kpsewhich not found. Install TeX Live so the script can verify required class files before compilation."
+        )
+    documentclass = str(publisher["latex"].get("documentclass") or "")
+    class_file = f"{documentclass}.cls"
+    if not kpsewhich_has(class_file):
+        raise SystemExit(
+            f"Error: required LaTeX class not found: {class_file}. Install the matching TeX Live package for publisher '{publisher_name}'."
+        )
+    if detected_locale in {"ko", "mixed"} and not kpsewhich_has("xeCJK.sty"):
+        raise SystemExit(
+            "Error: xeCJK.sty not found. Korean/CJK PDF output requires xeCJK from TeX Live."
+        )
+
+
+def compile_latex_to_pdf(tex_path: Path, pdf_path: Path) -> str:
+    result = run_checked(
+        ["xelatex", "-interaction=nonstopmode", "-halt-on-error", tex_path.name],
+        workdir=tex_path.parent,
+    )
+    built_pdf = tex_path.with_suffix(".pdf")
+    if result.returncode != 0 or not built_pdf.exists():
+        raise SystemExit(result.stdout + result.stderr)
+    pdf_path.parent.mkdir(parents=True, exist_ok=True)
+    pdf_path.write_bytes(built_pdf.read_bytes())
+    return result.stdout + result.stderr
+
+
+def convert_md_to_pdf(
+    md_path: Path,
+    pdf_path: Path,
+    publisher_name: str,
+    registry_path: Path,
+    locale: str = "auto",
+    mode: str = "submission",
+) -> str:
+    publisher = load_publisher(registry_path, publisher_name)
+    _, _, detected_locale = preprocess_markdown(md_path, publisher, locale)
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tmp_tex = Path(tmpdir) / f"{publisher_name}.tex"
+        render_markdown_to_latex(md_path, tmp_tex, publisher_name, registry_path, locale, mode)
+        if mode == "draft":
+            draft_tex = pdf_path.with_suffix(".tex")
+            draft_tex.parent.mkdir(parents=True, exist_ok=True)
+            draft_tex.write_text(tmp_tex.read_text(encoding="utf-8"), encoding="utf-8")
+        preflight_pdf_compile(publisher_name, publisher, detected_locale)
+        log = compile_latex_to_pdf(tmp_tex, pdf_path)
+    return log
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Convert Markdown to PDF via pandoc and XeLaTeX")
+    parser.add_argument("input", help="Input Markdown file")
+    parser.add_argument("output", help="Output PDF file")
+    parser.add_argument("--publisher", required=True, help="Publisher profile from templates/registry.yaml")
+    parser.add_argument("--registry", default=None, help="Path to templates/registry.yaml")
+    parser.add_argument("--locale", choices=("auto", "en", "ko", "mixed"), default="auto")
+    parser.add_argument("--submission", action="store_true")
+    parser.add_argument("--draft", action="store_true")
+    args = parser.parse_args()
+    if args.submission and args.draft:
+        parser.error("--submission and --draft are mutually exclusive")
+    return args
+
+
+def main() -> int:
+    args = parse_args()
+    script_dir = Path(__file__).resolve().parent
+    md_path = Path(args.input).expanduser().resolve()
+    pdf_path = Path(args.output).expanduser().resolve()
+    registry_path = Path(args.registry).expanduser().resolve() if args.registry else script_dir.parent / "templates" / "registry.yaml"
+    if not md_path.exists():
+        print(f"Error: Input file not found: {md_path}")
+        return 1
+    mode = "draft" if args.draft else "submission"
+    try:
+        log = convert_md_to_pdf(md_path, pdf_path, args.publisher, registry_path, args.locale, mode)
+    except SystemExit as exc:
+        print(str(exc), file=sys.stderr)
+        return 1
+    print(f"Successfully converted {md_path} to {pdf_path} [publisher={args.publisher}, mode={mode}]")
+    if log.strip():
+        print(log)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/skills/lit-docx/scripts/convert_pdf.py

@@ -0,0 +1,130 @@
+import sys
+import fitz  # PyMuPDF
+import re
+
+def clean_text(text):
+    """
+    Cleans up common PDF extraction artifacts.
+    """
+    # Replace non-breaking spaces with regular spaces
+    text = text.replace('\xa0', ' ')
+
+    # Remove excessive whitespace
+    text = re.sub(r'[ \t]+', ' ', text)
+
+    # Remove common journal header/footer noise (simple heuristics)
+    # Example: "Journal Pre-proof", PAGE numbers alone, DOIs repeated at edges
+    lines = text.split('\n')
+    cleaned_lines = []
+
+    for line in lines:
+        stripped = line.strip()
+        # Skip empty lines
+        if not stripped:
+            continue
+
+        # Skip page numbers (just digits)
+        if re.match(r'^\d+$', stripped):
+            continue
+
+        # Example heuristic: Skip lines that look like standard journal headers
+        if stripped.lower() in ["journal pre-proof", "accepted manuscript"]:
+            continue
+
+        cleaned_lines.append(line)
+
+    return "\n".join(cleaned_lines)
+
+def merge_paragraphs(text_blocks):
+    """
+    Merges text blocks into coherent paragraphs.
+    PDFs often break lines; we need to reassemble them.
+    text_blocks is a list of strings, each representing a block of text.
+    """
+    full_text = ""
+    for block in text_blocks:
+        # Clean the block first
+        block = block.strip()
+        if not block:
+            continue
+
+        # Split block into lines to process internal line breaks
+        lines = block.split('\n')
+        merged_block = ""
+
+        for i, line in enumerate(lines):
+            line = line.strip()
+            if not line:
+                continue
+
+            if not merged_block:
+                merged_block = line
+            else:
+                # specific check for hyphenation
+                if merged_block.endswith('-'):
+                    # Remove hyphen and join (e.g. "for- \n mation" -> "formation")
+                    merged_block = merged_block[:-1] + line
+                else:
+                    # If the previous line ends with a sentence ender, usually keep separate or add space?
+                    # In standard paragraphs, just add space.
+                    merged_block += " " + line
+
+        full_text += merged_block + "\n\n"
+
+    return full_text
+
+def convert_pdf_to_md(pdf_path, md_path):
+    doc = fitz.open(pdf_path)
+    output_text = []
+
+    for page_num, page in enumerate(doc):
+        # get_text("blocks") returns a list of items like:
+        # (x0, y0, x1, y1, "text", block_no, block_type)
+        # sort=True sorts by top-left to bottom-right, handling columns effectively.
+        blocks = page.get_text("blocks", sort=True)
+
+        page_content = []
+
+        # Exclude header/footer area by coordinate
+        page_height = page.rect.height
+        margin_top = page_height * 0.05  # Top 5%
+        margin_bottom = page_height * 0.95 # Bottom 5% checks
+
+        for b in blocks:
+            # Block structure: (x0, y0, x1, y1, text, block_no, block_type)
+            if b[6] == 0:  # text block
+                y0 = b[1]
+                y1 = b[3]
+                text = b[4]
+
+                # Simple filter for obvious header/footers based on position
+                if y1 < margin_top or y0 > margin_bottom:
+                     continue
+
+                page_content.append(text)
+
+        # Process the page's blocks
+        merged_page_text = merge_paragraphs(page_content)
+        cleaned_page_text = clean_text(merged_page_text)
+
+        output_text.append(cleaned_page_text)
+
+        # Add a visual separator between pages (optional, but helpful for debugging context)
+        # output_text.append(f"\n<!-- Page {page_num + 1} -->\n")
+
+    full_text = "\n".join(output_text)
+
+    with open(md_path, "w", encoding="utf-8") as f:
+        f.write(full_text)
+
+    print(f"Successfully converted {pdf_path} to {md_path}")
+
+if __name__ == "__main__":
+    if len(sys.argv) < 3:
+        print("Usage: python convert_pdf.py <input.pdf> <output.md>")
+        sys.exit(1)
+
+    pdf_file = sys.argv[1]
+    md_file = sys.argv[2]
+
+    convert_pdf_to_md(pdf_file, md_file)

package/skills/lit-docx/scripts/craft-thresholds.json

@@ -0,0 +1,11 @@
+{
+  "latinMeasureMax": 90,
+  "cjkMeasureMax": 38,
+  "cjkMajority": 0.5,
+  "latinGlyphFactor": 0.56,
+  "cjkGlyphFactor": 0.98,
+  "fallbackFontPt": 11,
+  "fallbackPageWidthTwips": 12240,
+  "fallbackMarginTwips": 1440,
+  "numericMajority": 0.7
+}

package/skills/lit-docx/scripts/craft_docx.py

@@ -0,0 +1,83 @@
+"""Source-only DOCX craft checks with explicit measurement tiers."""
+
+from __future__ import annotations
+
+import json
+import re
+import zipfile
+from pathlib import Path
+from xml.etree import ElementTree as ET
+
+T = json.loads(Path(__file__).with_name("craft-thresholds.json").read_text())
+W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
+HANGUL = re.compile(r"[가-힣]")
+
+
+def _text(element):
+    return "".join(node.text or "" for node in element.iter(W + "t"))
+
+
+def _number(text):
+    remaining = re.sub(r"[\s,.%+\-±()$€£¥₩▲▼]", "", text.strip())
+    return bool(remaining) and remaining.isdigit()
+
+
+def _attribute(element, name, fallback):
+    value = element.get(W + name) if element is not None else None
+    return float(value) if value is not None else fallback
+
+
+def analyse(path):
+    with zipfile.ZipFile(path) as archive:
+        document = ET.fromstring(archive.read("word/document.xml"))
+        styles = ET.fromstring(archive.read("word/styles.xml"))
+    body = document.find(W + "body")
+    section = body.find(W + "sectPr") if body is not None else None
+    page = section.find(W + "pgSz") if section is not None else None
+    margins = section.find(W + "pgMar") if section is not None else None
+    width = _attribute(page, "w", T["fallbackPageWidthTwips"])
+    left = _attribute(margins, "left", T["fallbackMarginTwips"])
+    right = _attribute(margins, "right", T["fallbackMarginTwips"])
+    usable_inches = max(0, width - left - right) / 1440
+    normal = next((style for style in styles.findall(W + "style") if style.get(W + "styleId") == "Normal"), None)
+    normal_size = normal.find(".//" + W + "sz") if normal is not None else None
+    font_pt = _attribute(normal_size, "val", T["fallbackFontPt"] * 2) / 2
+    findings = []
+
+    if body is None:
+        return findings
+    for paragraph_no, paragraph in enumerate(body.findall(W + "p"), 1):
+        text = _text(paragraph).strip()
+        if len(text) < 40:
+            continue
+        letters = [char for char in text if not char.isspace()]
+        cjk = bool(letters) and len(HANGUL.findall(text)) / len(letters) >= T["cjkMajority"]
+        factor = T["cjkGlyphFactor"] if cjk else T["latinGlyphFactor"]
+        ceiling = T["cjkMeasureMax"] if cjk else T["latinMeasureMax"]
+        run_sizes = [float(node.get(W + "val")) / 2 for node in paragraph.findall(".//" + W + "sz") if node.get(W + "val")]
+        size = max(run_sizes, default=font_pt)
+        measure = usable_inches * 72 / max(1, size * factor)
+        if measure > ceiling:
+            findings.append({"rule": "OF-301", "severity": "MEDIUM", "tier": "derived", "where": f"paragraph {paragraph_no}", "value": round(measure, 1), "threshold": ceiling})
+
+    for table_no, table in enumerate(body.iter(W + "tbl"), 1):
+        rows = table.findall(W + "tr")
+        if len(rows) < 3:
+            continue
+        data = [[cell for cell in row.findall(W + "tc")] for row in rows[1:]]
+        columns = max(map(len, data), default=0)
+        for column in range(columns):
+            cells = [row[column] for row in data if len(row) > column]
+            if len(cells) < 2 or sum(_number(_text(cell)) for cell in cells) / len(cells) < T["numericMajority"]:
+                continue
+            for row_no, cell in enumerate(cells, 2):
+                if not _number(_text(cell)):
+                    continue
+                for paragraph in cell.findall(W + "p"):
+                    if not _text(paragraph).strip():
+                        continue
+                    alignment = paragraph.find("./" + W + "pPr/" + W + "jc")
+                    value = alignment.get(W + "val") if alignment is not None else None
+                    if value != "right":
+                        findings.append({"rule": "OF-302", "severity": "HIGH" if value else "MEDIUM", "tier": "measured" if value else "derived", "where": f"table {table_no} row {row_no} col {column + 1}", "value": value, "threshold": "right"})
+    return findings