kc-beta 0.6.2 → 0.7.1

package/template/release/v1/kc_runtime/doc_parser.py

@@ -0,0 +1,127 @@
+"""
+Minimal document parser for the release runtime.
+
+Strategy: try native Python parsers first (pypdf, python-docx),
+fall back to LibreOffice CLI if natives unavailable AND lo is on
+PATH, finally fall back to UTF-8 plaintext read. Each strategy
+records what it tried via the result dict so workflows can decide
+whether to trust the text.
+
+This is a release-time helper — KC's CLI mode uses its own document
+parsing pipeline (src/agent/document-parser.js + LibreOffice).
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+from pathlib import Path
+
+
+def preflight(doc: Path) -> dict:
+    """
+    Verify a document is parseable; return a small status dict.
+    Lets workflows skip cleanly when the parse will fail rather than
+    burning a worker-LLM call on an unreadable file.
+    """
+    if not doc.exists():
+        return {"ok": False, "reason": "not_found", "path": str(doc)}
+    if not doc.is_file():
+        return {"ok": False, "reason": "not_file", "path": str(doc)}
+    if doc.stat().st_size == 0:
+        return {"ok": False, "reason": "empty", "path": str(doc)}
+    return {"ok": True, "path": str(doc), "size_bytes": doc.stat().st_size}
+
+
+def extract_text(doc: Path) -> dict:
+    """
+    Pull text out of a document. Returns:
+      { "text": "...", "via": "<strategy>", "ok": bool, "error"?: str }
+    Strategies tried in order:
+      1. Suffix-specific native parser (pypdf for .pdf, python-docx for .docx)
+      2. LibreOffice CLI (`soffice --headless --convert-to txt`) if on PATH
+      3. UTF-8 plaintext (.txt, .md, or any file with text-like bytes)
+    """
+    suffix = doc.suffix.lower()
+
+    if suffix == ".pdf":
+        text = _try_pypdf(doc)
+        if text is not None:
+            return {"text": text, "via": "pypdf", "ok": True}
+
+    if suffix in (".docx",):
+        text = _try_python_docx(doc)
+        if text is not None:
+            return {"text": text, "via": "python-docx", "ok": True}
+
+    # LibreOffice fallback for anything we couldn't parse natively
+    if suffix in (".pdf", ".doc", ".docx", ".odt", ".rtf"):
+        text = _try_libreoffice(doc)
+        if text is not None:
+            return {"text": text, "via": "libreoffice", "ok": True}
+
+    # Plaintext fallback (covers .txt, .md, .csv, .json, etc.)
+    try:
+        text = doc.read_text(encoding="utf-8")
+        return {"text": text, "via": "plaintext_utf8", "ok": True}
+    except UnicodeDecodeError:
+        try:
+            text = doc.read_text(encoding="gbk")  # common in Chinese corpora
+            return {"text": text, "via": "plaintext_gbk", "ok": True}
+        except Exception as exc:
+            return {"text": "", "via": "none", "ok": False, "error": str(exc)}
+
+
+# --- internals ---
+
+
+def _try_pypdf(doc: Path):
+    try:
+        import pypdf  # type: ignore
+    except ImportError:
+        return None
+    try:
+        reader = pypdf.PdfReader(str(doc))
+        return "\n".join(page.extract_text() or "" for page in reader.pages)
+    except Exception:
+        return None
+
+
+def _try_python_docx(doc: Path):
+    try:
+        import docx  # python-docx
+    except ImportError:
+        return None
+    try:
+        d = docx.Document(str(doc))
+        parts = [p.text for p in d.paragraphs]
+        for table in d.tables:
+            for row in table.rows:
+                for cell in row.cells:
+                    if cell.text:
+                        parts.append(cell.text)
+        return "\n".join(parts)
+    except Exception:
+        return None
+
+
+def _try_libreoffice(doc: Path):
+    soffice = shutil.which("soffice") or shutil.which("libreoffice")
+    if not soffice:
+        return None
+    out_dir = doc.parent / ".kc-lo-out"
+    out_dir.mkdir(exist_ok=True)
+    try:
+        subprocess.run(
+            [soffice, "--headless", "--convert-to", "txt", "--outdir", str(out_dir), str(doc)],
+            capture_output=True,
+            timeout=60,
+            check=True,
+        )
+        txt_path = out_dir / (doc.stem + ".txt")
+        if txt_path.exists():
+            return txt_path.read_text(encoding="utf-8")
+    except Exception:
+        return None
+    return None

package/template/release/v1/manifest.json.tmpl

@@ -0,0 +1,11 @@
+{
+  "release_version": "v1",
+  "kc_version": "{{kc_version}}",
+  "generated_at": "{{generated_at}}",
+  "session_id": "{{session_id}}",
+  "rules_count": {{rule_count}},
+  "workflows_count": {{workflow_count}},
+  "workflows": {},
+  "calibration_source": "confidence_calibration.json",
+  "documentation": "README.md"
+}

package/template/release/v1/render_dashboard.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""
+Render a single-page HTML dashboard from the run.py output directory.
+
+Usage:
+    python3 render_dashboard.py [output/results dir]
+    python3 render_dashboard.py output/results/  > dashboard.html
+
+Reads every <doc>.json under the given directory + summary.json, renders
+a self-contained HTML page (inline CSS, no external assets) showing per-
+document verdict tables + an overall summary. Designed to be opened
+directly in a browser.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from html import escape
+
+
+VERDICT_COLORS = {
+    "PASS": "#16a34a",
+    "FAIL": "#dc2626",
+    "PARTIAL": "#f59e0b",
+    "PARTIAL_PASS": "#f59e0b",
+    "NOT_APPLICABLE": "#6b7280",
+    "UNDETERMINED": "#9ca3af",
+    "INCONCLUSIVE": "#9ca3af",
+    "NO_WORKFLOW": "#7c3aed",
+    "ERROR": "#000000",
+}
+
+
+def _color(verdict: str) -> str:
+    return VERDICT_COLORS.get(verdict, "#374151")
+
+
+def _render_doc(doc_path: Path) -> str:
+    data = json.loads(doc_path.read_text(encoding="utf-8"))
+    rows = []
+    for rule_id, verdict in sorted(data.get("results", {}).items()):
+        v = verdict.get("verdict", "UNKNOWN")
+        conf = verdict.get("confidence", 0.0)
+        reason = verdict.get("reason", "") or verdict.get("error_type", "")
+        rows.append(
+            f"<tr>"
+            f"<td><code>{escape(rule_id)}</code></td>"
+            f"<td><span class='verdict' style='background:{_color(v)}'>{escape(v)}</span></td>"
+            f"<td>{conf:.2f}</td>"
+            f"<td>{escape(str(reason))[:200]}</td>"
+            f"</tr>"
+        )
+    name = escape(Path(data.get("document", doc_path.stem)).name)
+    return (
+        f"<section><h2>{name}</h2><table><thead><tr>"
+        f"<th>Rule</th><th>Verdict</th><th>Conf</th><th>Reason</th>"
+        f"</tr></thead><tbody>{''.join(rows)}</tbody></table></section>"
+    )
+
+
+def render(results_dir: Path) -> str:
+    doc_files = sorted(p for p in results_dir.glob("*.json") if p.name != "summary.json")
+    summary = {}
+    summary_path = results_dir / "summary.json"
+    if summary_path.exists():
+        summary = json.loads(summary_path.read_text(encoding="utf-8"))
+
+    summary_rows = "".join(
+        f"<li><b>{escape(v)}</b>: {n}</li>"
+        for v, n in sorted(summary.get("by_verdict", {}).items(), key=lambda kv: -kv[1])
+    )
+
+    body_sections = "\n".join(_render_doc(p) for p in doc_files)
+
+    return f"""<!doctype html>
+<html lang="en"><head><meta charset="utf-8">
+<title>KC verification dashboard</title>
+<style>
+  body {{ font: 14px system-ui, sans-serif; max-width: 1100px; margin: 2em auto; padding: 0 1em; color: #111; }}
+  h1 {{ font-size: 1.5em; }}
+  h2 {{ font-size: 1.1em; margin-top: 2em; border-bottom: 1px solid #e5e7eb; padding-bottom: .3em; }}
+  table {{ width: 100%; border-collapse: collapse; margin-top: .5em; }}
+  th, td {{ text-align: left; padding: .4em .6em; border-bottom: 1px solid #f3f4f6; vertical-align: top; }}
+  th {{ background: #f9fafb; font-weight: 600; }}
+  code {{ font-family: ui-monospace, monospace; }}
+  .verdict {{ display: inline-block; padding: 2px 8px; border-radius: 4px; color: white; font-weight: 600; font-size: .85em; }}
+  .summary {{ background: #f9fafb; padding: 1em; border-radius: 6px; }}
+  .summary ul {{ margin: 0; padding-left: 1.2em; }}
+</style>
+</head><body>
+<h1>KC verification — release v1</h1>
+<div class="summary">
+  <p><b>Total runs:</b> {summary.get("total_runs", 0)} ·
+     <b>Errors:</b> {summary.get("errors", 0)} ·
+     <b>Documents:</b> {len(doc_files)}</p>
+  <ul>{summary_rows or '<li>No verdicts</li>'}</ul>
+</div>
+{body_sections}
+</body></html>
+"""
+
+
+def main():
+    if len(sys.argv) > 1:
+        results_dir = Path(sys.argv[1]).resolve()
+    else:
+        results_dir = Path(__file__).resolve().parent / "output" / "results"
+    if not results_dir.is_dir():
+        print(f"results directory not found: {results_dir}", file=sys.stderr)
+        sys.exit(1)
+    print(render(results_dir))
+
+
+if __name__ == "__main__":
+    main()

package/template/release/v1/run.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+"""
+KC release runner v1.
+
+Entry point for a self-contained KC release bundle. Loads the manifest,
+iterates rules, dispatches each rule's workflow against the supplied
+input documents, writes per-document verdict JSONs to output/results/.
+
+Usage:
+    python3 run.py <input_dir>
+    python3 run.py <input_dir> --rules R001,R005,R012
+    python3 run.py --doc <single_file>           # single-doc smoke test
+
+The bundle is shipped from KC's finalization phase. KC's run-in-CLI
+mode is the source of truth; this is the ship-as-artifact form for
+re-running verification on new document batches without the full KC
+toolchain installed.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import subprocess
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+
+# Add kc_runtime to path so submodules import cleanly when run from any cwd.
+sys.path.insert(0, str(HERE))
+
+from kc_runtime import doc_parser, confidence  # noqa: E402
+
+
+def _load_json(path: Path, *, required: bool = False, default=None):
+    if not path.exists():
+        if required:
+            raise SystemExit(
+                f"required file missing: {path}\n"
+                f"this release bundle was shipped without a complete manifest.\n"
+                f"re-run KC finalization or contact the bundle author."
+            )
+        return default
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def _select_rules(catalog, rule_filter):
+    if not rule_filter:
+        return catalog
+    wanted = set(rule_filter.split(","))
+    return [r for r in catalog if r.get("id") in wanted]
+
+
+def _list_input_docs(input_dir: Path):
+    if not input_dir.is_dir():
+        raise SystemExit(f"input_dir not a directory: {input_dir}")
+    docs = []
+    for entry in sorted(input_dir.iterdir()):
+        if entry.is_file() and not entry.name.startswith("."):
+            docs.append(entry)
+    return docs
+
+
+def _run_workflow(rule_id: str, workflow_path: Path, doc_path: Path) -> dict:
+    """
+    Dispatch a single workflow against a single document.
+
+    Each workflow is a stand-alone Python script that takes a document
+    path on argv and emits a JSON verdict on stdout. Workflows are
+    sandbox-runnable: no shared module state, no special imports beyond
+    stdlib + kc_runtime.
+    """
+    if not workflow_path.exists():
+        return {
+            "rule_id": rule_id,
+            "verdict": "ERROR",
+            "confidence": 0.0,
+            "error_type": "workflow_missing",
+            "reason": f"workflow not found: {workflow_path.name}",
+        }
+    try:
+        proc = subprocess.run(
+            [sys.executable, str(workflow_path), str(doc_path)],
+            capture_output=True,
+            text=True,
+            timeout=180,
+        )
+        if proc.returncode != 0:
+            return {
+                "rule_id": rule_id,
+                "verdict": "ERROR",
+                "confidence": 0.0,
+                "error_type": "workflow_exit_nonzero",
+                "reason": (proc.stderr or proc.stdout or "").strip()[:500],
+            }
+        # Workflow contract: last stdout line is the verdict JSON.
+        last = next(
+            (line for line in reversed(proc.stdout.splitlines()) if line.strip()),
+            None,
+        )
+        if not last:
+            return {
+                "rule_id": rule_id,
+                "verdict": "ERROR",
+                "confidence": 0.0,
+                "error_type": "empty_workflow_output",
+            }
+        verdict = json.loads(last)
+        verdict.setdefault("rule_id", rule_id)
+        return verdict
+    except subprocess.TimeoutExpired:
+        return {
+            "rule_id": rule_id,
+            "verdict": "ERROR",
+            "confidence": 0.0,
+            "error_type": "workflow_timeout",
+        }
+    except json.JSONDecodeError as exc:
+        return {
+            "rule_id": rule_id,
+            "verdict": "ERROR",
+            "confidence": 0.0,
+            "error_type": "workflow_output_not_json",
+            "reason": str(exc),
+        }
+
+
+def main():
+    parser = argparse.ArgumentParser(prog="run.py", description="KC release runner")
+    parser.add_argument("input_dir", nargs="?", help="Directory of input documents")
+    parser.add_argument("--doc", help="Single document path (smoke-test mode)")
+    parser.add_argument("--rules", help="Comma-separated rule_ids to run (default: all)")
+    parser.add_argument("--output-dir", default=str(HERE / "output" / "results"))
+    args = parser.parse_args()
+
+    if not args.input_dir and not args.doc:
+        parser.error("either input_dir or --doc is required")
+
+    manifest = _load_json(HERE / "manifest.json", required=True)
+    catalog = _load_json(HERE / "catalog.json", required=False, default=[])
+    historical = _load_json(
+        HERE / "confidence_calibration.json",
+        required=False,
+        default={"historical_accuracy": {}},
+    )
+
+    rules = _select_rules(catalog, args.rules)
+    if not rules:
+        raise SystemExit("no rules to run (catalog empty or filter excluded all)")
+
+    if args.doc:
+        docs = [Path(args.doc).resolve()]
+    else:
+        docs = _list_input_docs(Path(args.input_dir).resolve())
+
+    output_dir = Path(args.output_dir).resolve()
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    workflows = manifest.get("workflows", {})
+    summary = {"total_runs": 0, "by_verdict": {}, "errors": 0}
+
+    for doc in docs:
+        # Lightweight parse — let the workflow do its own parse if needed,
+        # but offer a doc_parser preflight so workflows can rely on the
+        # text being available.
+        try:
+            doc_parser.preflight(doc)
+        except Exception as exc:
+            print(f"[run.py] preflight failed for {doc.name}: {exc}", file=sys.stderr)
+
+        results = {}
+        for rule in rules:
+            rule_id = rule.get("id")
+            if not rule_id:
+                continue
+            wf_relpath = workflows.get(rule_id)
+            if not wf_relpath:
+                results[rule_id] = {
+                    "rule_id": rule_id,
+                    "verdict": "NO_WORKFLOW",
+                    "confidence": 0.0,
+                }
+                continue
+            wf_path = HERE / wf_relpath
+            verdict = _run_workflow(rule_id, wf_path, doc)
+            verdict = confidence.calibrate(verdict, historical)
+            results[rule_id] = verdict
+            summary["total_runs"] += 1
+            v = verdict.get("verdict", "UNKNOWN")
+            summary["by_verdict"][v] = summary["by_verdict"].get(v, 0) + 1
+            if v == "ERROR":
+                summary["errors"] += 1
+
+        out_file = output_dir / f"{doc.stem}.json"
+        out_file.write_text(
+            json.dumps(
+                {"document": str(doc), "results": results}, ensure_ascii=False, indent=2
+            ),
+            encoding="utf-8",
+        )
+
+    summary_path = output_dir / "summary.json"
+    summary_path.write_text(
+        json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8"
+    )
+    print(json.dumps(summary, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()

package/template/release/v1/serve.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+# Minimal local-preview server for the KC release dashboard.
+# Renders dashboard.html if missing, then serves on PORT (default 8765).
+
+set -e
+HERE="$(cd "$(dirname "$0")" && pwd)"
+cd "$HERE"
+
+PORT="${PORT:-8765}"
+
+if [ ! -f dashboard.html ] || [ output/results/summary.json -nt dashboard.html ]; then
+  echo "rendering dashboard.html..."
+  python3 render_dashboard.py output/results/ > dashboard.html
+fi
+
+echo "serving on http://localhost:$PORT/dashboard.html"
+exec python3 -m http.server "$PORT"