create-ccc-tutor 0.1.0 → 0.3.0

package/template/.harness/scripts/pdf_rag.py

@@ -0,0 +1,485 @@
+#!/usr/bin/env python3
+"""pdf_rag.py — 课件 PDF 的检索/建档/渲染引擎（pdf-vision 功能）。
+
+模型无关：Claude 与 Codex 两侧的 /slide、/exam 都调它。输出 JSON 供技能消费。
+
+子命令：
+  index   --subject <s> [--dpi N]          建/更新该科目课件档（幂等）
+  query   --subject <s> -q "<问题>" [-k K]  先 reconcile 再检索，输出命中页 JSON
+  render  --pdf <file>  --page N [--dpi N]  按需渲染单页 PNG，输出路径
+
+设计见 docs/features/pdf-vision-implementation.md。混合检索：建档只嵌入页文字
+（文字过少且有 OCR 则 OCR），看图发生在回答阶段（技能读 PNG）。缓存按
+「内容哈希 + 处理指纹」失效；每次 query 前对课件文件夹做轻量 reconcile（增/删/改）。
+嵌入工具不可用时降级为关键词检索（仅服务文字类问题）。
+"""
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import shutil
+import sys
+import tempfile
+
+# ── 常量（改动任一会进入处理指纹，旧档自动失效，见 spec §3.5）────────────────
+PIPELINE_SCHEMA_VERSION = "1"
+EMBED_MODEL_ID = "intfloat/multilingual-e5-small"   # 多语言：中文问英文课件
+EMBED_DIM = 384
+DEFAULT_DPI = 150
+VISION_PROMPT_VERSION = "1"            # 答题阶段看图提示版本（记入指纹，便于将来回填缓存失效）
+TEXT_MIN_CHARS = 40                    # 低于此视为疑似扫描/纯图页
+SIM_THRESHOLD = 0.74                   # 余弦相似度命中阈值（e5 偏高；保守，宁 miss 不勉强答）
+KEYWORD_MIN_RATIO = 0.34               # 关键词降级：命中内容词占比下限（防常见词冒充命中）
+
+# 关键词降级用停用词（en + 常见中文功能字/单字），过滤后才算「内容词」
+_EN_STOP = {"the", "a", "an", "is", "are", "was", "were", "be", "of", "to", "in",
+            "on", "at", "for", "and", "or", "not", "what", "which", "how", "why",
+            "when", "who", "this", "that", "these", "those", "it", "its", "as",
+            "by", "with", "from", "do", "does", "did", "i", "you", "he", "she",
+            "they", "we", "can", "could", "would", "should", "will", "about"}
+_ZH_STOP = set("的了是在和与或不也我你他她它这那个之把被就都要吗呢啊吧么呀有没和对")
+CACHE_ROOT = ".cache/pdf-vision"
+COURSE_ROOT = "course"
+
+_SLUG_RE = re.compile(r"[^0-9A-Za-z]+")
+_TOKEN_RE = re.compile(r"[0-9A-Za-z_]+|[一-鿿]")
+
+
+def slugify(name):
+    return _SLUG_RE.sub("-", name).strip("-") or "pdf"
+
+
+def file_slug(fn):
+    # 碰撞安全：不同文件名（如 "Lecture 1.pdf" vs "Lecture_1.pdf"）slug 可能相同，
+    # 加文件名哈希后缀确保各文件有独立缓存目录，防 rmtree 误删 / 出处张冠李戴（spec §3.7）。
+    return slugify(fn) + "-" + hashlib.sha256(fn.encode("utf-8")).hexdigest()[:8]
+
+
+def eprint(*a):
+    print(*a, file=sys.stderr)
+
+
+# ── 文件签名 / 指纹 ───────────────────────────────────────────────────────────
+def quick_sig(path):
+    st = os.stat(path)
+    return {"size": st.st_size, "mtime_ns": st.st_mtime_ns}  # 纳秒精度，避免同秒覆盖漏检
+
+
+def file_sha256(path):
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for blk in iter(lambda: f.read(65536), b""):
+            h.update(blk)
+    return h.hexdigest()
+
+
+def processing_fingerprint(dpi, ocr_engine):
+    # 任一字段变化 → 旧档过期（spec §3.5 红线：内容 + 生成方式都要匹配）
+    return {
+        "pipeline_schema_version": PIPELINE_SCHEMA_VERSION,
+        "embed_model_id": _embed_state["model_id"],
+        "embed_model_version": _embed_state["lib_version"],
+        "render_dpi": dpi,
+        "vision_prompt_version": VISION_PROMPT_VERSION,
+        "ocr_engine": ocr_engine,
+    }
+
+
+# ── 嵌入器（fastembed 多语言；不可用则降级关键词）─────────────────────────────
+_embed_state = {"tried": False, "model": None, "model_id": EMBED_MODEL_ID,
+                "lib_version": "none", "mode": "keyword"}
+
+
+def get_embedder():
+    """返回 fastembed 模型或 None（None => 关键词降级）。结果缓存在 _embed_state。"""
+    if _embed_state["tried"]:
+        return _embed_state["model"]
+    _embed_state["tried"] = True
+    if os.environ.get("PDF_RAG_FORCE_KEYWORD") == "1":
+        _embed_state["mode"] = "keyword"
+        return None
+    try:
+        import fastembed  # noqa
+        from fastembed import TextEmbedding
+        model = TextEmbedding(model_name=EMBED_MODEL_ID)
+        _embed_state["model"] = model
+        _embed_state["lib_version"] = getattr(fastembed, "__version__", "unknown")
+        _embed_state["mode"] = "semantic"
+        return model
+    except Exception as e:  # 下载失败/未安装/离线首次 → 降级
+        eprint(f"[pdf_rag] embedding unavailable ({e}); degrading to keyword search")
+        _embed_state["mode"] = "keyword"
+        return None
+
+
+def embed_texts(texts, kind):
+    """kind: 'passage' | 'query'（e5 前缀约定，否则召回质量下降）。返回 list[list[float]]。"""
+    model = get_embedder()
+    if model is None:
+        return None
+    prefix = "query: " if kind == "query" else "passage: "
+    import numpy as np
+    vecs = list(model.embed([prefix + t for t in texts]))
+    return [np.asarray(v, dtype="float32") for v in vecs]
+
+
+# ── PDF 解析（pymupdf）────────────────────────────────────────────────────────
+def _open_pdf(path):
+    try:
+        import pymupdf  # 1.24+
+        return pymupdf.open(path)
+    except ImportError:
+        import fitz       # 兼容旧别名
+        return fitz.open(path)
+
+
+def _ocr_available():
+    """探测 PyMuPDF 是否能 OCR（需系统 Tesseract + tessdata）。"""
+    if os.environ.get("PDF_RAG_DISABLE_OCR") == "1":
+        return False
+    try:
+        doc = _open_pdf  # noqa  仅确认 import 可用；真正能力在抽取时 try
+        return os.environ.get("TESSDATA_PREFIX") is not None or _which("tesseract")
+    except Exception:
+        return False
+
+
+def _which(prog):
+    from shutil import which
+    return which(prog) is not None
+
+
+def extract_pages(pdf_path, pdf_dir, dpi, ocr_engine):
+    """返回 [{page, text, visual_flag, visual_only, png_path}]。坏页 text='' 且 unreadable=True。"""
+    pages = []
+    doc = _open_pdf(pdf_path)
+    use_ocr = ocr_engine != "none"
+    for i in range(len(doc)):
+        page_no = i + 1
+        rec = {"page": page_no, "text": "", "visual_flag": False,
+               "visual_only": False, "unreadable": False, "png_path": None}
+        try:
+            pg = doc[i]
+            text = pg.get_text() or ""
+            try:
+                rec["visual_flag"] = len(pg.get_images(full=True)) > 0
+            except Exception:
+                rec["visual_flag"] = False
+            if len(text.strip()) < TEXT_MIN_CHARS:
+                # 疑似扫描/纯图页：有 OCR 则补文字，否则标 visual_only
+                if use_ocr:
+                    try:
+                        tp = pg.get_textpage_ocr(full=True)
+                        text = pg.get_text(textpage=tp) or text
+                    except Exception as oe:
+                        eprint(f"[pdf_rag] OCR failed p{page_no} of {pdf_path}: {oe}")
+                if len(text.strip()) < TEXT_MIN_CHARS:
+                    rec["visual_only"] = True
+            rec["text"] = text
+        except Exception as pe:
+            rec["unreadable"] = True
+            eprint(f"[pdf_rag] page {page_no} unreadable in {pdf_path}: {pe}")
+        pages.append(rec)
+    doc.close()
+    return pages
+
+
+def render_page(pdf_path, page_no, dpi, out_dir):
+    """渲染单页为 PNG，已存在则跳过。返回路径或 None。"""
+    os.makedirs(out_dir, exist_ok=True)
+    out = os.path.join(out_dir, f"page-{page_no:03d}.png")
+    if os.path.exists(out):
+        return out
+    try:
+        doc = _open_pdf(pdf_path)
+        pix = doc[page_no - 1].get_pixmap(dpi=dpi)
+        pix.save(out)
+        doc.close()
+        return out
+    except Exception as e:
+        eprint(f"[pdf_rag] render failed {pdf_path} p{page_no}: {e}")
+        return None
+
+
+# ── 缓存读写（原子）───────────────────────────────────────────────────────────
+def subject_paths(subject):
+    base = os.path.join(CACHE_ROOT, subject)
+    return {
+        "base": base,
+        "manifest": os.path.join(base, "manifest.json"),
+        "chunks": os.path.join(base, "chunks.json"),
+        "vectors": os.path.join(base, "vectors.npy"),
+    }
+
+
+def _atomic_write_json(path, obj):
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    fd, tmp = tempfile.mkstemp(dir=os.path.dirname(path), suffix=".tmp")
+    with os.fdopen(fd, "w", encoding="utf-8") as f:
+        json.dump(obj, f, ensure_ascii=False, indent=1)
+    os.replace(tmp, path)
+
+
+def _atomic_write_npy(path, arr):
+    import numpy as np
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    fd, tmp = tempfile.mkstemp(dir=os.path.dirname(path), suffix=".tmp")
+    os.close(fd)
+    np.save(tmp, arr)             # numpy 会补 .npy 后缀
+    src = tmp + ".npy" if os.path.exists(tmp + ".npy") else tmp
+    os.replace(src, path)
+
+
+def load_manifest(subject):
+    p = subject_paths(subject)["manifest"]
+    if os.path.exists(p):
+        with open(p, encoding="utf-8") as f:
+            return json.load(f)
+    return {"fingerprint": None, "files": {}}
+
+
+# ── 建档 / reconcile ──────────────────────────────────────────────────────────
+def slide_dir(subject):
+    return os.path.join(COURSE_ROOT, subject, "slide")
+
+
+def list_pdfs(subject):
+    d = slide_dir(subject)
+    if not os.path.isdir(d):
+        return []
+    return sorted(fn for fn in os.listdir(d) if fn.lower().endswith(".pdf"))
+
+
+def reconcile(subject, dpi):
+    """对照课件文件夹增量更新档：增→建、删→移除、改→重建。返回变更摘要。"""
+    ocr_engine = "tesseract" if _ocr_available() else "none"
+    fp = processing_fingerprint(dpi, ocr_engine)
+    base = subject_paths(subject)["base"]
+    manifest = load_manifest(subject)
+    # 处理指纹变了 → 清掉整个科目缓存（含旧 PNG/文字），整体重建（spec §3.5 红线）
+    if manifest.get("fingerprint") != fp:
+        if os.path.isdir(base):
+            shutil.rmtree(base, ignore_errors=True)
+        manifest = {"fingerprint": fp, "files": {}}
+
+    present = set(list_pdfs(subject))
+    known = set(manifest["files"].keys())
+    added, removed, changed, unchanged = [], [], [], []
+
+    for fn in sorted(known - present):          # 删除：从档移除 + 清掉它的渲染目录
+        slug = manifest["files"][fn].get("slug")
+        if slug:
+            shutil.rmtree(os.path.join(base, slug), ignore_errors=True)
+        del manifest["files"][fn]
+        removed.append(fn)
+
+    for fn in sorted(present):
+        path = os.path.join(slide_dir(subject), fn)
+        sig = quick_sig(path)
+        ent = manifest["files"].get(fn)
+        if ent and ent.get("indexed_complete") and ent.get("quick_sig") == sig:
+            unchanged.append(fn)                # 名/大小/时间都没变 → 跳过，不重读
+            continue
+        sha = file_sha256(path)
+        if ent and ent.get("indexed_complete") and ent.get("sha256") == sha:
+            ent["quick_sig"] = sig              # 仅 mtime 变、内容没变
+            unchanged.append(fn)
+            continue
+        _index_file(subject, fn, path, sha, sig, dpi, ocr_engine, manifest)
+        (changed if ent else added).append(fn)
+
+    _rebuild_vectors(subject, manifest)
+    _atomic_write_json(subject_paths(subject)["manifest"], manifest)
+    return {"added": added, "removed": removed, "changed": changed,
+            "unchanged": unchanged, "mode": _embed_state["mode"]}
+
+
+def _index_file(subject, fn, path, sha, sig, dpi, ocr_engine, manifest):
+    """建单个 PDF 的档：标 incomplete → 抽页 → 标 complete（中断不会被复用）。"""
+    slug = file_slug(fn)
+    png_dir = os.path.join(subject_paths(subject)["base"], slug)
+    # 重建该文件前清掉旧渲染/文字，避免内容变了仍返回旧 PNG（spec §3.5 红线）
+    if os.path.isdir(png_dir):
+        shutil.rmtree(png_dir, ignore_errors=True)
+    manifest["files"][fn] = {"sha256": sha, "quick_sig": sig,
+                             "indexed_complete": False, "slug": slug, "pages": []}
+    pages = extract_pages(path, png_dir, dpi, ocr_engine)
+    page_meta = []
+    for pr in pages:
+        page_meta.append({k: pr[k] for k in
+                          ("page", "visual_flag", "visual_only", "unreadable")})
+        # 文字写入每页缓存（供答题阶段加载，免重抽）
+        if not pr["unreadable"]:
+            os.makedirs(png_dir, exist_ok=True)
+            with open(os.path.join(png_dir, f"page-{pr['page']:03d}.text.txt"),
+                      "w", encoding="utf-8") as f:
+                f.write(pr["text"])
+    manifest["files"][fn]["pages"] = page_meta
+    manifest["files"][fn]["indexed_complete"] = True
+
+
+def _build_chunks(subject, manifest):
+    """从 manifest 收集每页 chunk（page-level）。坏页和全空页不入检索。"""
+    chunks = []
+    for fn, ent in manifest["files"].items():
+        if not ent.get("indexed_complete"):
+            continue                              # 半截档不参与检索（spec §3.4）
+        slug = ent["slug"]
+        png_dir = os.path.join(subject_paths(subject)["base"], slug)
+        for pm in ent["pages"]:
+            if pm["unreadable"]:
+                continue
+            tpath = os.path.join(png_dir, f"page-{pm['page']:03d}.text.txt")
+            text = ""
+            if os.path.exists(tpath):
+                with open(tpath, encoding="utf-8") as f:
+                    text = f.read()
+            lecture = _lecture_label(fn)
+            # visual_only 页文字稀少：用「讲次标题 + 文件名」兜底，靠所属讲次被检索到
+            embed_text = text.strip() or f"{lecture} {fn}"
+            chunks.append({
+                "file": fn, "source_file_exact": fn, "lecture": lecture,
+                "page": pm["page"], "visual_flag": pm["visual_flag"],
+                "visual_only": pm["visual_only"], "slug": slug,
+                "text": text, "embed_text": embed_text,
+            })
+    return chunks
+
+
+def _rebuild_vectors(subject, manifest):
+    import numpy as np
+    sp = subject_paths(subject)
+    chunks = _build_chunks(subject, manifest)
+    _atomic_write_json(sp["chunks"], chunks)
+    if not chunks:
+        _atomic_write_npy(sp["vectors"], np.zeros((0, EMBED_DIM), dtype="float32"))
+        return
+    vecs = embed_texts([c["embed_text"] for c in chunks], "passage")
+    if vecs is None:                              # 关键词模式：无向量
+        if os.path.exists(sp["vectors"]):
+            os.remove(sp["vectors"])
+        return
+    mat = np.vstack(vecs)
+    mat = mat / (np.linalg.norm(mat, axis=1, keepdims=True) + 1e-9)
+    _atomic_write_npy(sp["vectors"], mat)
+
+
+_LEC_RE = re.compile(r"(Lecture\s*\d+|Week\s*\d+|L\d+)", re.IGNORECASE)
+
+
+def _lecture_label(fn):
+    m = _LEC_RE.search(fn)
+    return m.group(1) if m else os.path.splitext(fn)[0]
+
+
+# ── 检索 ──────────────────────────────────────────────────────────────────────
+def _content_tokens(s):
+    """分词并去停用词，只留「内容词」（防 the/is/的/了 这类把无关页判成命中）。"""
+    return {t for t in _TOKEN_RE.findall(s.lower())
+            if t not in _EN_STOP and t not in _ZH_STOP}
+
+
+def _keyword_score(query, text):
+    qt = _content_tokens(query)
+    if not qt:
+        return 0.0                              # 问题里没有内容词 → 不可判命中
+    ds = _content_tokens(text)
+    if not ds:
+        return 0.0
+    hit = sum(1 for t in qt if t in ds)
+    return hit / len(qt)                         # 命中内容词占比（query 侧）
+
+
+def query(subject, question, k, dpi):
+    rec = reconcile(subject, dpi)
+    sp = subject_paths(subject)
+    if not os.path.exists(sp["chunks"]):
+        return {"subject": subject, "mode": rec["mode"], "reconcile": rec,
+                "miss": True, "reason": "no_index", "results": []}
+    with open(sp["chunks"], encoding="utf-8") as f:
+        chunks = json.load(f)
+    if not chunks:
+        return {"subject": subject, "mode": rec["mode"], "reconcile": rec,
+                "miss": True, "reason": "empty", "results": []}
+
+    mode = _embed_state["mode"]
+    scored = []
+    if mode == "semantic" and os.path.exists(sp["vectors"]):
+        import numpy as np
+        mat = np.load(sp["vectors"])
+        qv = embed_texts([question], "query")
+        if qv is not None and mat.shape[0] == len(chunks):
+            q = qv[0] / (np.linalg.norm(qv[0]) + 1e-9)
+            sims = mat @ q
+            order = np.argsort(-sims)[:k]
+            scored = [(float(sims[i]), chunks[i]) for i in order]
+        else:
+            mode = "keyword"
+    if not scored:                                # 关键词降级路径
+        mode = "keyword"
+        ranked = sorted(((_keyword_score(question, c["embed_text"]), c)
+                         for c in chunks), key=lambda x: -x[0])[:k]
+        scored = [(s, c) for s, c in ranked]
+
+    threshold = SIM_THRESHOLD if mode == "semantic" else KEYWORD_MIN_RATIO
+    top = scored[0][0] if scored else 0.0
+    miss = (not scored) or (top < threshold)
+
+    results = []
+    for score, c in scored:
+        png = None
+        if c["visual_flag"] or c["visual_only"]:  # 仅对含图的命中页渲染（top-k，开销小）
+            png = render_page(os.path.join(slide_dir(subject), c["file"]), c["page"],
+                              dpi, os.path.join(sp["base"], c["slug"]))
+        results.append({
+            "source_file_exact": c["source_file_exact"], "lecture": c["lecture"],
+            "page": c["page"], "score": round(score, 4),
+            "visual_flag": c["visual_flag"], "visual_only": c["visual_only"],
+            "png_path": png,
+            "text_path": os.path.join(sp["base"], c["slug"],
+                                      f"page-{c['page']:03d}.text.txt"),
+        })
+    return {"subject": subject, "mode": mode, "reconcile": rec,
+            "miss": miss, "threshold": threshold, "top_score": round(top, 4),
+            "results": results}
+
+
+# ── CLI ───────────────────────────────────────────────────────────────────────
+def main(argv=None):
+    p = argparse.ArgumentParser(description="课件 PDF 检索/建档/渲染引擎")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    pi = sub.add_parser("index", help="建/更新某科目课件档")
+    pi.add_argument("--subject", required=True)
+    pi.add_argument("--dpi", type=int, default=DEFAULT_DPI)
+
+    pq = sub.add_parser("query", help="检索（先 reconcile）")
+    pq.add_argument("--subject", required=True)
+    pq.add_argument("-q", "--question", required=True)
+    pq.add_argument("-k", type=int, default=5)
+    pq.add_argument("--dpi", type=int, default=DEFAULT_DPI)
+
+    pr = sub.add_parser("render", help="渲染单页 PNG")
+    pr.add_argument("--pdf", required=True, help="PDF 路径")
+    pr.add_argument("--page", type=int, required=True)
+    pr.add_argument("--dpi", type=int, default=DEFAULT_DPI)
+    pr.add_argument("--subject", help="可选，仅用于缓存归类")
+
+    args = p.parse_args(argv)
+    if args.cmd == "index":
+        out = reconcile(args.subject, args.dpi)
+    elif args.cmd == "query":
+        out = query(args.subject, args.question, args.k, args.dpi)
+    elif args.cmd == "render":
+        slug = file_slug(os.path.basename(args.pdf))   # 碰撞安全，与 index 路径一致
+        base = os.path.join(CACHE_ROOT, args.subject or "_adhoc", slug)
+        path = render_page(args.pdf, args.page, args.dpi, base)
+        out = {"png_path": path, "ok": path is not None}
+    print(json.dumps(out, ensure_ascii=False))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/template/.harness/scripts/requirements-pdf.txt

@@ -0,0 +1,6 @@
+# pdf-vision 依赖（pdf-rag.sh 安装到隔离 venv .harness/.venv-pdf）
+# OCR（Tesseract）是可选系统依赖，不在此处，运行时探测。
+pymupdf==1.24.*
+fastembed==0.7.*
+numpy>=1.26,<3
+pypdf>=4,<6