novel-writer-cli 0.0.1

package/scripts/lib/calibrate_quality_judge.py

@@ -0,0 +1,312 @@
+"""QualityJudge calibration against human-labeled dataset (M3).
+
+Extracted from scripts/calibrate-quality-judge.sh heredoc.
+Shared helpers imported from _common.py (same directory).
+"""
+
+import json
+import math
+import os
+import sys
+from datetime import datetime, timezone
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
+
+import _common
+
+_SCRIPT = "calibrate-quality-judge.sh"
+
+
+def _die(msg: str, exit_code: int = 1) -> None:
+    _common.die(f"{_SCRIPT}: {msg}", exit_code)
+
+
+def _load_json(path: str) -> Any:
+    try:
+        return _common.load_json(path)
+    except Exception as e:
+        _die(f"invalid JSON at {path}: {e}", 1)
+
+
+def _iter_jsonl(path: str) -> Iterable[Tuple[int, Dict[str, Any]]]:
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            for line_no, raw in enumerate(f, start=1):
+                line = raw.strip()
+                if not line:
+                    continue
+                if line.startswith("#"):
+                    continue
+                try:
+                    obj = json.loads(line)
+                except Exception as e:
+                    _die(f"invalid JSONL at {path}:{line_no}: {e}", 1)
+                if not isinstance(obj, dict):
+                    _die(f"JSONL record must be an object at {path}:{line_no}", 1)
+                yield line_no, obj
+    except FileNotFoundError:
+        _die(f"labels file not found: {path}", 1)
+    except SystemExit:
+        raise
+    except Exception as e:
+        _die(f"failed to read labels file {path}: {e}", 1)
+
+
+def _pearson(x: Sequence[float], y: Sequence[float]) -> Optional[float]:
+    if len(x) != len(y) or len(x) < 2:
+        return None
+    mean_x = sum(x) / len(x)
+    mean_y = sum(y) / len(y)
+    num = 0.0
+    den_x = 0.0
+    den_y = 0.0
+    for a, b in zip(x, y):
+        dx = a - mean_x
+        dy = b - mean_y
+        num += dx * dy
+        den_x += dx * dx
+        den_y += dy * dy
+    if den_x <= 0.0 or den_y <= 0.0:
+        return None
+    return num / math.sqrt(den_x * den_y)
+
+
+def _linear_fit(x: Sequence[float], y: Sequence[float]) -> Optional[Dict[str, float]]:
+    if len(x) != len(y) or len(x) < 2:
+        return None
+    mean_x = sum(x) / len(x)
+    mean_y = sum(y) / len(y)
+    sxx = 0.0
+    sxy = 0.0
+    for a, b in zip(x, y):
+        dx = a - mean_x
+        sxx += dx * dx
+        sxy += dx * (b - mean_y)
+    if sxx <= 0.0:
+        return None
+    slope = sxy / sxx
+    intercept = mean_y - slope * mean_x
+    return {"slope": slope, "intercept": intercept}
+
+
+def _clamp(v: float, lo: float = 1.0, hi: float = 5.0) -> float:
+    return max(lo, min(hi, v))
+
+
+def _safe_round(v: Optional[float], ndigits: int = 4) -> Optional[float]:
+    if v is None:
+        return None
+    return round(float(v), ndigits)
+
+
+def main() -> None:
+    project_dir = sys.argv[1]
+    labels_path = sys.argv[2]
+    out_path = sys.argv[3].strip() if len(sys.argv) > 3 else ""
+
+    label_records: Dict[int, Dict[str, Any]] = {}
+    label_line_by_chapter: Dict[int, int] = {}
+
+    for line_no, obj in _iter_jsonl(labels_path):
+        chapter = obj.get("chapter")
+        if not isinstance(chapter, int) or chapter < 1:
+            _die(f"labels record missing valid chapter at {labels_path}:{line_no}", 1)
+        schema_version = obj.get("schema_version")
+        if schema_version != 1:
+            _die(
+                f"unsupported schema_version at {labels_path}:{line_no} (expected 1, got {schema_version})",
+                1,
+            )
+        human_scores = obj.get("human_scores")
+        if not isinstance(human_scores, dict) or _common.as_float(human_scores.get("overall")) is None:
+            _die(f"labels record missing human_scores.overall at {labels_path}:{line_no}", 1)
+        if chapter in label_records:
+            _die(
+                f"duplicate chapter {chapter} in labels (lines {label_line_by_chapter[chapter]} and {line_no})",
+                1,
+            )
+        label_records[chapter] = obj
+        label_line_by_chapter[chapter] = line_no
+
+    if not label_records:
+        _die("labels file has no records", 1)
+
+    eval_dir = os.path.join(project_dir, "evaluations")
+    if not os.path.isdir(eval_dir):
+        _die(f"evaluations/ not found under project dir: {project_dir}", 1)
+
+    eval_file_list = _common.find_eval_files(eval_dir)
+    eval_files: Dict[int, str] = {ch: path for ch, path in eval_file_list}
+
+    matched_chapters: List[int] = []
+    missing_eval_chapters: List[int] = []
+
+    human_overall: List[float] = []
+    judge_overall: List[float] = []
+    judge_overall_source: Dict[int, str] = {}
+
+    dim_pairs: Dict[str, Tuple[List[float], List[float]]] = {}
+
+    for chapter in sorted(label_records.keys()):
+        eval_path = eval_files.get(chapter)
+        if not eval_path:
+            missing_eval_chapters.append(chapter)
+            continue
+
+        eval_obj = _load_json(eval_path)
+        if not isinstance(eval_obj, dict):
+            _die(f"eval JSON must be an object at {eval_path}", 1)
+
+        human = _common.as_float(label_records[chapter]["human_scores"]["overall"])
+        judge = _common.extract_overall(eval_obj)
+        if human is None:
+            _die(f"labels human_scores.overall not a number for chapter {chapter}", 1)
+        if judge is None:
+            missing_eval_chapters.append(chapter)
+            continue
+
+        # Track which field provided judge overall.
+        src = "unknown"
+        if _common.as_float(eval_obj.get("overall_final")) is not None:
+            src = "overall_final"
+        elif _common.as_float(_common.extract_eval_used(eval_obj).get("overall_final")) is not None:
+            src = "eval_used.overall_final"
+        elif _common.as_float(_common.extract_eval_used(eval_obj).get("overall")) is not None:
+            src = "eval_used.overall"
+        elif _common.as_float(eval_obj.get("overall")) is not None:
+            src = "overall"
+        else:
+            meta = eval_obj.get("metadata")
+            if isinstance(meta, dict) and isinstance(meta.get("judges"), dict) and _common.as_float(meta["judges"].get("overall_final")) is not None:
+                src = "metadata.judges.overall_final"
+        judge_overall_source[chapter] = src
+
+        matched_chapters.append(chapter)
+        human_overall.append(float(human))
+        judge_overall.append(float(judge))
+
+        human_scores = label_records[chapter].get("human_scores")
+        human_dims: Dict[str, float] = {}
+        if isinstance(human_scores, dict):
+            for k, v in human_scores.items():
+                if k == "overall":
+                    continue
+                n = _common.as_float(v)
+                if n is not None:
+                    human_dims[str(k)] = float(n)
+
+        judge_dims = _common.extract_dimension_scores(eval_obj)
+        for dim_key, human_dim_score in human_dims.items():
+            judge_dim_score = judge_dims.get(dim_key)
+            if judge_dim_score is None:
+                continue
+            xs, ys = dim_pairs.setdefault(dim_key, ([], []))
+            xs.append(float(human_dim_score))
+            ys.append(float(judge_dim_score))
+
+    if len(matched_chapters) < 2:
+        _die(
+            f"need at least 2 matched chapters to compute Pearson (matched={len(matched_chapters)})",
+            1,
+        )
+
+    r_overall = _pearson(human_overall, judge_overall)
+    fit = _linear_fit(judge_overall, human_overall)  # human ~ slope * judge + intercept
+
+    errors = [j - h for h, j in zip(human_overall, judge_overall)]
+    mae = sum(abs(e) for e in errors) / len(errors)
+    rmse = math.sqrt(sum(e * e for e in errors) / len(errors))
+    bias = sum(errors) / len(errors)
+
+    # pause_for_user_force_rewrite is implicit (<2.0), no threshold to calibrate
+    default_thresholds = {"pass": 4.0, "polish": 3.5, "revise": 3.0, "pause_for_user": 2.0}
+
+    suggestions: Dict[str, Any] = {"defaults": default_thresholds, "methods": {}}
+
+    # Method 1: shift thresholds by mean judge-human bias.
+    shifted = {k: _safe_round(_clamp(v + bias), 3) for k, v in default_thresholds.items()}
+    suggestions["methods"]["shift_by_bias"] = {
+        "bias_judge_minus_human": _safe_round(bias, 4),
+        "suggested_thresholds": shifted,
+        "note": "若 judge 整体偏高（bias>0），建议上调阈值；偏低（bias<0）则下调。仅作启发式建议。",
+    }
+
+    # Method 2: linear fit inversion: find judge thresholds that map to target human thresholds.
+    if fit is not None and abs(float(fit["slope"])) > 1e-6:
+        slope = float(fit["slope"])
+        intercept = float(fit["intercept"])
+        inv = {}
+        for k, human_t in default_thresholds.items():
+            inv[k] = _safe_round(_clamp((human_t - intercept) / slope), 3)
+        suggestions["methods"]["linear_fit_inverse"] = {
+            "fit_human_equals_slope_times_judge_plus_intercept": {
+                "slope": _safe_round(slope, 6),
+                "intercept": _safe_round(intercept, 6),
+            },
+            "suggested_thresholds": inv,
+            "note": "基于最小二乘线性拟合的反解（用 judge 预测 human）。样本少/分布窄时可能不稳定，仅作建议。",
+        }
+    else:
+        suggestions["methods"]["linear_fit_inverse"] = {
+            "fit_human_equals_slope_times_judge_plus_intercept": None,
+            "suggested_thresholds": None,
+            "note": "judge 分布过窄或样本不足，无法稳定拟合。",
+        }
+
+    dims_report: Dict[str, Any] = {}
+    for dim_key in sorted(dim_pairs.keys()):
+        xs, ys = dim_pairs[dim_key]
+        r = _pearson(xs, ys)
+        dims_report[dim_key] = {"n": len(xs), "pearson_r": _safe_round(r, 4)}
+
+    now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+    out: Dict[str, Any] = {
+        "schema_version": 1,
+        "generated_at": now,
+        "project": {
+            "path": os.path.abspath(project_dir),
+            "evaluations_dir": os.path.join(os.path.abspath(project_dir), "evaluations"),
+        },
+        "labels": {
+            "path": os.path.abspath(labels_path),
+            "records": len(label_records),
+        },
+        "alignment": {
+            "matched_chapters": matched_chapters,
+            "missing_eval_chapters": missing_eval_chapters,
+            "judge_overall_source_by_chapter": {str(k): v for k, v in sorted(judge_overall_source.items())},
+        },
+        "overall": {
+            "n": len(matched_chapters),
+            "pearson_r": _safe_round(r_overall, 4),
+            "human_mean": _safe_round(sum(human_overall) / len(human_overall), 4),
+            "judge_mean": _safe_round(sum(judge_overall) / len(judge_overall), 4),
+            "mae": _safe_round(mae, 4),
+            "rmse": _safe_round(rmse, 4),
+            "bias_judge_minus_human": _safe_round(bias, 4),
+        },
+        "dimensions": dims_report,
+        "threshold_suggestions": suggestions,
+    }
+
+    out_json = json.dumps(out, ensure_ascii=False, sort_keys=True) + "\n"
+    sys.stdout.write(out_json)
+
+    if out_path:
+        out_dir = os.path.dirname(os.path.abspath(out_path))
+        if out_dir and not os.path.isdir(out_dir):
+            os.makedirs(out_dir, exist_ok=True)
+        try:
+            with open(out_path, "w", encoding="utf-8") as f:
+                f.write(out_json)
+        except Exception as e:
+            _die(f"failed to write report to {out_path}: {e}", 1)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except SystemExit:
+        raise
+    except Exception as e:
+        sys.stderr.write(f"calibrate-quality-judge.sh: unexpected error: {e}\n")
+        raise SystemExit(2)

package/scripts/lib/compare_regression_runs.py

@@ -0,0 +1,142 @@
+"""Compare two archived regression-run summaries (M3).
+
+Extracted from the heredoc in scripts/compare-regression-runs.sh.
+Reuses helpers from _common to avoid duplication.
+"""
+
+import json
+import os
+import sys
+from typing import Any, Dict, Optional
+
+import _common
+
+
+def _die(msg: str, exit_code: int = 1) -> None:
+    _common.die(f"compare-regression-runs.sh: {msg}", exit_code)
+
+
+def _load_json(path: str) -> Any:
+    try:
+        return _common.load_json(path)
+    except Exception as e:
+        _die(f"invalid JSON at {path}: {e}", 1)
+
+
+def _delta_number(a: Any, b: Any) -> Optional[float]:
+    na = _common.as_float(a)
+    nb = _common.as_float(b)
+    if na is None or nb is None:
+        return None
+    return nb - na
+
+
+def _delta_dim_means(a: Any, b: Any) -> Dict[str, Optional[float]]:
+    """Delta on score_dimensions: extract 'mean' from {n, mean} objects."""
+    if not isinstance(a, dict):
+        a = {}
+    if not isinstance(b, dict):
+        b = {}
+    keys = sorted(set(list(a.keys()) + list(b.keys())))
+    out: Dict[str, Optional[float]] = {}
+    for k in keys:
+        va = a.get(k)
+        vb = b.get(k)
+        ma = _common.as_float(va.get("mean")) if isinstance(va, dict) else None
+        mb = _common.as_float(vb.get("mean")) if isinstance(vb, dict) else None
+        if ma is None and mb is None:
+            continue
+        if ma is None or mb is None:
+            out[k] = None
+        else:
+            out[k] = round(float(mb - ma), 6)
+    return out
+
+
+def _delta_map(a: Any, b: Any) -> Dict[str, Optional[float]]:
+    if not isinstance(a, dict):
+        a = {}
+    if not isinstance(b, dict):
+        b = {}
+    keys = set([str(k) for k in a.keys()] + [str(k) for k in b.keys()])
+    out: Dict[str, Optional[float]] = {}
+    for k in sorted(keys):
+        da = _common.as_float(a.get(k))
+        db = _common.as_float(b.get(k))
+        if da is None and db is None:
+            continue
+        if da is None or db is None:
+            out[k] = None
+        else:
+            out[k] = float(db - da)
+    return out
+
+
+def main() -> None:
+    path_a = sys.argv[1]
+    path_b = sys.argv[2]
+    out_path = sys.argv[3].strip() if len(sys.argv) > 3 else ""
+
+    a = _load_json(path_a)
+    b = _load_json(path_b)
+    if not isinstance(a, dict) or not isinstance(b, dict):
+        _die("both summaries must be JSON objects", 1)
+
+    comp_a = a.get("compliance") if isinstance(a.get("compliance"), dict) else {}
+    comp_b = b.get("compliance") if isinstance(b.get("compliance"), dict) else {}
+
+    score_a = a.get("score_overall") if isinstance(a.get("score_overall"), dict) else {}
+    score_b = b.get("score_overall") if isinstance(b.get("score_overall"), dict) else {}
+
+    out = {
+        "schema_version": 1,
+        "generated_at": _common.iso_utc_now(),
+        "run_a": {"dir": os.path.dirname(os.path.abspath(path_a)), "summary_path": os.path.abspath(path_a), "run_id": a.get("run_id")},
+        "run_b": {"dir": os.path.dirname(os.path.abspath(path_b)), "summary_path": os.path.abspath(path_b), "run_id": b.get("run_id")},
+        "delta": {
+            "chapters_total": _delta_number(a.get("chapters_total"), b.get("chapters_total")),
+            "violations_total": _delta_number(a.get("violations_total"), b.get("violations_total")),
+            "compliance_rate_high_confidence": _delta_number(comp_a.get("compliance_rate_high_confidence"), comp_b.get("compliance_rate_high_confidence")),
+            "compliance_rate_any_violation": _delta_number(comp_a.get("compliance_rate_any_violation"), comp_b.get("compliance_rate_any_violation")),
+            "chapters_with_high_confidence_violation": _delta_number(
+                comp_a.get("chapters_with_high_confidence_violation"), comp_b.get("chapters_with_high_confidence_violation")
+            ),
+            "chapters_with_any_violation": _delta_number(comp_a.get("chapters_with_any_violation"), comp_b.get("chapters_with_any_violation")),
+            "violations_by_confidence": _delta_map(a.get("violations_by_confidence"), b.get("violations_by_confidence")),
+            "violations_by_layer": _delta_map(a.get("violations_by_layer"), b.get("violations_by_layer")),
+            "score_overall_mean": _delta_number(score_a.get("mean"), score_b.get("mean")),
+        },
+        "score_dimensions": {},
+        "notes": [],
+    }
+
+    if _common.as_float(a.get("chapters_total")) != _common.as_float(b.get("chapters_total")):
+        out["notes"].append("chapters_total differs; compare deltas with caution.")
+
+    dims_a = a.get("score_dimensions", {})
+    dims_b = b.get("score_dimensions", {})
+    if dims_a or dims_b:
+        out["score_dimensions"] = _delta_dim_means(dims_a, dims_b)
+
+    out_json = json.dumps(out, ensure_ascii=False, sort_keys=True) + "\n"
+    sys.stdout.write(out_json)
+
+    if out_path:
+        out_dir = os.path.dirname(os.path.abspath(out_path))
+        if out_dir and not os.path.isdir(out_dir):
+            os.makedirs(out_dir, exist_ok=True)
+        try:
+            with open(out_path, "w", encoding="utf-8") as f:
+                f.write(out_json)
+        except Exception as e:
+            _die(f"failed to write to {out_path}: {e}", 1)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except SystemExit:
+        raise
+    except Exception as e:
+        sys.stderr.write(f"compare-regression-runs.sh: unexpected error: {e}\n")
+        raise SystemExit(2)