@kernaut/kernaut-resources 0.1.17-beta.1 → 0.1.18

package/resources/kop-2026-10-23-0.1.17-beta.1/product_capabilities/kernel_optflow/skills/KernelWiki-private/agent/kernaut-experience-ledger/scripts/record_tools.py

@@ -1,773 +0,0 @@
-#!/usr/bin/env python3
-"""Deterministic helpers for Kernaut gateway recording analysis.
-
-The utility preserves evidence structure and provenance but does not make
-semantic conclusions about attempts, techniques, bottlenecks, or dead ends.
-"""
-
-from __future__ import annotations
-
-import argparse
-import gzip
-import hashlib
-import json
-import re
-import sys
-from collections import Counter
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Any, Iterator
-
-
-SUPPORTED_SUFFIXES = {".ndjson", ".jsonl", ".gz"}
-ARTIFACT_TYPE = "kernaut_evidence_candidate_v1"
-LEDGER_ARTIFACT_TYPE = "kernaut_experience_ledger_v1"
-ISOLATED_ARTIFACT_TYPE = "kernaut_unrecognized_measurement_v1"
-LEDGER_RECORD_TYPES = {"measurement", "lesson", "pitfall", "usage"}
-EVIDENCE_ORIGINS = {"measured", "source-reported", "observed", "inferred"}
-VERIFICATION_STATUSES = {"unverified", "triaged", "reproduced", "human-approved"}
-
-SENSITIVE_KEY = re.compile(
-    r"authorization|proxy[_-]?authorization|api[_-]?(?:key|token)|access[_-]?token|"
-    r"refresh[_-]?token|id[_-]?token|secret|password|passwd|cookie|private[_-]?key|"
-    r"client[_-]?secret|session[_-]?token",
-    re.IGNORECASE,
-)
-SENSITIVE_TEXT = (
-    re.compile(r"(?i)\b(?:bearer|basic)\s+[A-Za-z0-9._~+/=-]+"),
-    re.compile(
-        r"(?i)\b(?:api[_-]?(?:key|token)|access[_-]?token|refresh[_-]?token|secret|"
-        r"password|passwd|cookie(?:_value)?|private[_-]?key|client[_-]?secret)"
-        r"\s*[:=]\s*[^\s,;]+"
-    ),
-    re.compile(r"\bgh[opusr]_[A-Za-z0-9]{20,}\b"),
-    re.compile(r"\bsk-[A-Za-z0-9_-]{20,}\b"),
-    re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b"),
-    re.compile(r"\bAIza[0-9A-Za-z_-]{30,}\b"),
-    re.compile(r"\bAKIA[0-9A-Z]{16}\b"),
-    re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----", re.DOTALL),
-)
-
-NUMBER = r"(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?"
-MEASUREMENT_PATTERNS = (
-    re.compile(
-        rf"(?:e2e\s+)?baseline\s*=\s*(?P<baseline>{NUMBER})\s*us\s+"
-        rf"candidate\s*=\s*(?P<candidate>{NUMBER})\s*us\s+"
-        rf"speedup\s*=\s*(?P<speedup>{NUMBER})\s*x",
-        re.IGNORECASE,
-    ),
-    re.compile(
-        rf"kernel:\s*baseline\s*=\s*(?P<baseline>{NUMBER})\s*us\s+"
-        rf"candidate\s*=\s*(?P<candidate>{NUMBER})\s*us\s+"
-        rf"speedup\s*=\s*(?P<speedup>{NUMBER})\s*x",
-        re.IGNORECASE,
-    ),
-    re.compile(
-        rf"case\s+gate\s+failed\s*:\s*(?P<case>[^\n]+?)\s+"
-        rf"(?P<actual>{NUMBER})\s+(?P<required>{NUMBER})",
-        re.IGNORECASE,
-    ),
-)
-UNRECOGNIZED_MEASUREMENT = re.compile(
-    rf"(?<![\w.])(?P<left>{NUMBER})\s*(?P<left_unit>us|ms|x)\b[^\n]{{0,100}}?"
-    rf"(?P<right>{NUMBER})\s*(?P<right_unit>us|ms|x)\b",
-    re.IGNORECASE,
-)
-CATEGORY_PATTERNS = {
-    "patch": re.compile(r"apply_patch|diff --git|git\s+(?:apply|diff)", re.IGNORECASE),
-    "build": re.compile(r"\b(?:cmake|ninja|make|build|compile|bisheng)\b", re.IGNORECASE),
-    "correctness": re.compile(r"correctness|accuracy|precision|mismatch|pass_rate", re.IGNORECASE),
-    "benchmark": re.compile(r"kernel_eval|cann-bench|benchmark\.py|pipeline\.py\s+run|msprof|speedup", re.IGNORECASE),
-    "hardware": re.compile(r"npu-smi|Ascend(?:310|910)|910B2C|soc_version", re.IGNORECASE),
-    "wiki_usage": re.compile(r"wiki_query|wiki_get_page|wiki_grep", re.IGNORECASE),
-    "failure_signal": re.compile(r"error|failed|failure|mismatch|timeout|out of memory|OOM|失败|错误|超时", re.IGNORECASE),
-}
-READ_ONLY_COMMAND = re.compile(
-    r"(?:^|&&|\|\||;|\|)\s*(?:sudo\s+)?(?:cat|sed|grep|rg|head|tail|less|more|find|ls|wc)\b",
-    re.IGNORECASE,
-)
-EVALUATION_COMMAND = re.compile(
-    r"kernel_eval|cann-bench|benchmark\.py|correctness\.py|pipeline\.py\s+run|msprof|pytest.*benchmark|"
-    r"(?:^|[\s;&|/])(?:bench|benchmark)[-_a-zA-Z0-9.]*",
-    re.IGNORECASE,
-)
-
-
-@dataclass
-class ParseStats:
-    non_empty_lines: int = 0
-    records: int = 0
-    invalid_json: int = 0
-    non_object: int = 0
-    read_errors: int = 0
-    error_samples: list[str] = field(default_factory=list)
-
-    def add_error(self, message: str) -> None:
-        if len(self.error_samples) < 20:
-            self.error_samples.append(message)
-
-    @property
-    def errors(self) -> int:
-        return self.invalid_json + self.non_object + self.read_errors
-
-    def as_dict(self) -> dict[str, Any]:
-        return {
-            "non_empty_lines": self.non_empty_lines,
-            "records": self.records,
-            "invalid_json": self.invalid_json,
-            "non_object": self.non_object,
-            "read_errors": self.read_errors,
-            "error_samples": self.error_samples,
-        }
-
-
-def open_text(path: Path):
-    if path.suffix == ".gz":
-        return gzip.open(path, "rt", encoding="utf-8", errors="replace")
-    return path.open("r", encoding="utf-8", errors="replace")
-
-
-def looks_generated(path: Path) -> bool:
-    if path.name in {"evidence-candidates.jsonl", "ledger.jsonl", "attempt-map.jsonl"}:
-        return True
-    try:
-        with open_text(path) as handle:
-            for line in handle:
-                if not line.strip():
-                    continue
-                value = json.loads(line)
-                return isinstance(value, dict) and value.get("artifact_type") in {ARTIFACT_TYPE, LEDGER_ARTIFACT_TYPE}
-    except (OSError, EOFError, json.JSONDecodeError):
-        return False
-    return False
-
-
-def discover(inputs: list[Path], excluded: set[Path] | None = None) -> list[Path]:
-    excluded_resolved = {path.resolve() for path in (excluded or set())}
-    paths: list[Path] = []
-    for input_path in inputs:
-        if input_path.is_dir():
-            paths.extend(path for path in input_path.rglob("*") if path.is_file() and path.suffix in SUPPORTED_SUFFIXES)
-        elif input_path.is_file() and input_path.suffix in SUPPORTED_SUFFIXES:
-            paths.append(input_path)
-    return sorted(
-        {
-            path
-            for path in paths
-            if path.resolve() not in excluded_resolved and not looks_generated(path)
-        }
-    )
-
-
-def is_within(path: Path, directory: Path) -> bool:
-    try:
-        path.relative_to(directory)
-    except ValueError:
-        return False
-    return True
-
-
-def iter_records(paths: list[Path], stats: ParseStats) -> Iterator[tuple[Path, int, dict[str, Any]]]:
-    for path in paths:
-        try:
-            with open_text(path) as handle:
-                for line_number, line in enumerate(handle, 1):
-                    if not line.strip():
-                        continue
-                    stats.non_empty_lines += 1
-                    try:
-                        value = json.loads(line)
-                    except json.JSONDecodeError as error:
-                        stats.invalid_json += 1
-                        stats.add_error(f"{path}:{line_number}: invalid JSON: {error.msg}")
-                        continue
-                    if not isinstance(value, dict):
-                        stats.non_object += 1
-                        stats.add_error(f"{path}:{line_number}: top-level value is not an object")
-                        continue
-                    stats.records += 1
-                    yield path, line_number, value
-        except (OSError, EOFError) as error:
-            stats.read_errors += 1
-            stats.add_error(f"{path}: read error: {error}")
-
-
-def walk_nodes(value: Any, path: str = "$", parent: dict[str, Any] | None = None) -> Iterator[tuple[str, Any, dict[str, Any] | None]]:
-    yield path, value, parent
-    if isinstance(value, dict):
-        for key, child in value.items():
-            if SENSITIVE_KEY.search(str(key)):
-                continue
-            yield from walk_nodes(child, f"{path}.{key}", value)
-    elif isinstance(value, list):
-        for index, child in enumerate(value):
-            yield from walk_nodes(child, f"{path}[{index}]", parent)
-
-
-def evidence_roots(record: dict[str, Any]) -> list[tuple[str, Any]]:
-    request = record.get("request")
-    if isinstance(request, dict):
-        body = request.get("body")
-        value = body.get("value") if isinstance(body, dict) else None
-        inputs = value.get("input") if isinstance(value, dict) else None
-        if isinstance(inputs, list):
-            roots: list[tuple[str, Any]] = []
-            allowed_types = {
-                "message",
-                "reasoning",
-                "function_call",
-                "function_call_output",
-                "custom_tool_call",
-                "custom_tool_call_output",
-            }
-            for index, item in enumerate(inputs):
-                if not isinstance(item, dict) or str(item.get("type", "")) not in allowed_types:
-                    continue
-                if item.get("type") == "message" and str(item.get("role", "")) not in {"user", "assistant"}:
-                    continue
-                roots.append((f"$.request.body.value.input[{index}]", item))
-            return roots
-    return [("$", record)]
-
-
-def first_value(record: dict[str, Any], *keys: str) -> Any:
-    for _, node, _ in walk_nodes(record):
-        if isinstance(node, dict):
-            for key in keys:
-                if key in node and node[key] not in (None, "") and not SENSITIVE_KEY.search(key):
-                    return node[key]
-    return None
-
-
-def redact(text: str) -> str:
-    for pattern in SENSITIVE_TEXT:
-        text = pattern.sub("[REDACTED]", text)
-    return text
-
-
-def digest(value: Any) -> str:
-    serialized = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
-    return hashlib.sha256(serialized.encode("utf-8", errors="replace")).hexdigest()
-
-
-def fingerprint_value(value: Any) -> Any:
-    if isinstance(value, dict):
-        return {key: fingerprint_value(child) for key, child in value.items() if key != "source_path"}
-    if isinstance(value, list):
-        return [fingerprint_value(child) for child in value]
-    return value
-
-
-def parse_arguments(value: Any) -> dict[str, Any]:
-    if isinstance(value, dict):
-        return value
-    if isinstance(value, str):
-        try:
-            parsed = json.loads(value)
-        except json.JSONDecodeError:
-            return {}
-        return parsed if isinstance(parsed, dict) else {}
-    return {}
-
-
-def command_from_node(node: dict[str, Any]) -> str | None:
-    for key in ("command", "cmd"):
-        if isinstance(node.get(key), str):
-            return node[key]
-    arguments = parse_arguments(node.get("arguments"))
-    for key in ("command", "cmd"):
-        if isinstance(arguments.get(key), str):
-            return arguments[key]
-    return None
-
-
-def build_call_map(record: dict[str, Any]) -> dict[str, dict[str, Any]]:
-    calls: dict[str, dict[str, Any]] = {}
-    for root_path, root in evidence_roots(record):
-        for path, node, _ in walk_nodes(root, root_path):
-            if not isinstance(node, dict):
-                continue
-            call_id = node.get("call_id") or node.get("callId")
-            name = node.get("name") or node.get("tool_name") or node.get("toolName")
-            if call_id and name:
-                calls[str(call_id)] = {
-                    "tool_name": str(name),
-                    "command": command_from_node(node),
-                    "call_path": path,
-                }
-    return calls
-
-
-def source_kind(path: str, parent: dict[str, Any] | None) -> str:
-    event_type = str((parent or {}).get("type", "")).lower()
-    lowered = path.lower()
-    if "output" in event_type or any(token in lowered for token in (".output", ".result", "function_call_output")):
-        return "tool_output"
-    if "call" in event_type or any(token in lowered for token in (".arguments", ".command", ".cmd")):
-        return "tool_input"
-    return "text"
-
-
-def classify_command(command: str | None) -> str:
-    if not command:
-        return "unknown"
-    unwrapped = re.sub(r"^\s*(?:timeout\s+\S+\s+)?(?:bash|sh)\s+-lc\s+['\"]?", "", command, flags=re.IGNORECASE)
-    if READ_ONLY_COMMAND.search(unwrapped):
-        return "read_only"
-    if EVALUATION_COMMAND.search(command):
-        return "evaluation_candidate"
-    return "other"
-
-
-def extract_measurement_candidates(text: str, path: str, provenance: dict[str, Any]) -> list[dict[str, Any]]:
-    candidates: list[dict[str, Any]] = []
-    seen: set[tuple[float, float, float]] = set()
-    for pattern in MEASUREMENT_PATTERNS:
-        for match in pattern.finditer(text):
-            if "baseline" not in match.groupdict():
-                candidates.append(
-                    {
-                        "format": "gate_failure",
-                        "case": match.group("case").strip(),
-                        "actual": float(match.group("actual")),
-                        "required": float(match.group("required")),
-                        "parser": "gate_failure_v1",
-                        "source_path": path,
-                        "source_kind": provenance["source_kind"],
-                        "tool_name": provenance.get("tool_name"),
-                        "call_id": provenance.get("call_id"),
-                        "command_class": provenance["command_class"],
-                        "verification_status": "unverified",
-                        "measured_evidence_eligible": bool(
-                            provenance["source_kind"] == "tool_output"
-                            and provenance["command_class"] == "evaluation_candidate"
-                        ),
-                    }
-                )
-                continue
-            baseline = float(match.group("baseline"))
-            candidate = float(match.group("candidate"))
-            speedup = float(match.group("speedup"))
-            key = (baseline, candidate, speedup)
-            if key in seen:
-                continue
-            seen.add(key)
-            expected = baseline / candidate if candidate else None
-            candidates.append(
-                {
-                    "baseline_us": baseline,
-                    "candidate_us": candidate,
-                    "speedup_x": speedup,
-                    "speedup_consistent": bool(expected and abs(expected - speedup) / expected <= 0.02),
-                    "parser": "baseline_candidate_speedup_v2",
-                    "source_path": path,
-                    "source_kind": provenance["source_kind"],
-                    "tool_name": provenance.get("tool_name"),
-                    "call_id": provenance.get("call_id"),
-                    "command_class": provenance["command_class"],
-                    "verification_status": "unverified",
-                    "measured_evidence_eligible": bool(
-                        provenance["source_kind"] == "tool_output"
-                        and provenance["command_class"] == "evaluation_candidate"
-                    ),
-                }
-            )
-    return candidates
-
-
-def extract_unrecognized_measurements(text: str, path: str, provenance: dict[str, Any]) -> list[dict[str, Any]]:
-    if provenance["source_kind"] != "tool_output" or provenance["command_class"] != "evaluation_candidate":
-        return []
-    if any(candidate for candidate in extract_measurement_candidates(text, path, provenance)):
-        return []
-    candidates = []
-    for match in UNRECOGNIZED_MEASUREMENT.finditer(text):
-        units = {match.group("left_unit").lower(), match.group("right_unit").lower()}
-        if not units.intersection({"us", "ms"}):
-            continue
-        candidates.append(
-            {
-                "artifact_type": ISOLATED_ARTIFACT_TYPE,
-                "format": "unrecognized_pair",
-                "values": [float(match.group("left")), float(match.group("right"))],
-                "units": sorted(units),
-                "parser": "unrecognized_pair_v1",
-                "review_status": "review-required",
-                "source_path": path,
-                "source_kind": provenance["source_kind"],
-                "tool_name": provenance.get("tool_name"),
-                "call_id": provenance.get("call_id"),
-                "command_class": provenance["command_class"],
-            }
-        )
-    return candidates
-
-
-def evidence_items(record: dict[str, Any], excerpt_chars: int) -> list[dict[str, Any]]:
-    calls = build_call_map(record)
-    items: list[dict[str, Any]] = []
-    for root_path, root in evidence_roots(record):
-        for path, node, parent in walk_nodes(root, root_path):
-            if not isinstance(node, str) or not node.strip():
-                continue
-            categories = [name for name, pattern in CATEGORY_PATTERNS.items() if pattern.search(node)]
-            call_id = None
-            if parent:
-                call_id = parent.get("call_id") or parent.get("callId")
-            call = calls.get(str(call_id), {}) if call_id else {}
-            command = call.get("command") or (command_from_node(parent) if parent else None)
-            provenance = {
-                "source_kind": source_kind(path, parent),
-                "event_type": str((parent or {}).get("type", "")) or None,
-                "call_id": str(call_id) if call_id else None,
-                "tool_name": call.get("tool_name") or (parent or {}).get("name") or (parent or {}).get("tool_name"),
-                "command": redact(command)[:1000] if command else None,
-                "command_class": classify_command(command),
-            }
-            measurements = extract_measurement_candidates(node, path, provenance)
-            isolated_measurements = extract_unrecognized_measurements(node, path, provenance)
-            if not categories and not measurements and not isolated_measurements:
-                continue
-            items.append(
-                {
-                    "json_path": path,
-                    "categories": categories,
-                    "provenance": provenance,
-                    "measurement_candidates": measurements,
-                    "isolated_measurement_candidates": isolated_measurements,
-                    "excerpt": redact(node)[:excerpt_chars],
-                }
-            )
-    return items
-
-
-def inspect_inputs(args: argparse.Namespace) -> int:
-    paths = discover(args.input)
-    stats = ParseStats()
-    sessions: set[str] = set()
-    top_level_keys: Counter[str] = Counter()
-    samples: list[dict[str, Any]] = []
-    for path, line_number, record in iter_records(paths, stats):
-        top_level_keys.update(record.keys())
-        session_id = first_value(record, "session_id", "sessionId", "conversation_id", "conversationId")
-        if session_id is not None:
-            sessions.add(str(session_id))
-        if len(samples) < args.samples:
-            samples.append(
-                {
-                    "source": f"{path}:{line_number}",
-                    "top_level_keys": sorted(record.keys()),
-                    "session_field_found": session_id is not None,
-                }
-            )
-    summary = {
-        "files": len(paths),
-        "bytes": sum(path.stat().st_size for path in paths),
-        "suffixes": dict(Counter(path.suffix for path in paths)),
-        "sessions": len(sessions),
-        "top_level_keys": dict(top_level_keys.most_common(30)),
-        "sample_sources": samples,
-        "parse": stats.as_dict(),
-    }
-    print(json.dumps(summary, ensure_ascii=False, indent=2))
-    return 0 if (args.allow_parse_errors or stats.errors == 0) and paths else 2
-
-
-def extract_candidates(args: argparse.Namespace) -> int:
-    output = args.output.resolve()
-    for input_path in args.input:
-        if input_path.is_dir() and is_within(output, input_path.resolve()):
-            print(
-                json.dumps(
-                    {
-                        "error": "output must be outside every input directory",
-                        "input": str(input_path),
-                        "output": str(args.output),
-                    },
-                    ensure_ascii=False,
-                ),
-                file=sys.stderr,
-            )
-            return 2
-    paths = discover(args.input, {args.output})
-    stats = ParseStats()
-    grouped: dict[str, dict[str, Any]] = {}
-    isolated: dict[str, dict[str, Any]] = {}
-    category_counts: Counter[str] = Counter()
-    for path, line_number, record in iter_records(paths, stats):
-        session_id = first_value(record, "session_id", "sessionId", "conversation_id", "conversationId")
-        recorded_at = first_value(record, "recorded_at", "recordedAt", "timestamp", "created_at", "createdAt")
-        turn_index = first_value(record, "turn_index", "turnIndex")
-        record_id = first_value(record, "record_id", "recordId")
-        for item in evidence_items(record, args.excerpt_chars):
-            category_counts.update(item["categories"])
-            fingerprint = digest(
-                {
-                    "session_id": session_id,
-                    "call_id": item["provenance"].get("call_id"),
-                    "excerpt": item["excerpt"],
-                    "measurements": fingerprint_value(item["measurement_candidates"]),
-                    "isolated_measurements": fingerprint_value(item["isolated_measurement_candidates"]),
-                }
-            )
-            source = f"{path}:{line_number}#{item['json_path']}"
-            if fingerprint not in grouped:
-                grouped[fingerprint] = {
-                    "artifact_type": ARTIFACT_TYPE,
-                    "candidate_id": f"candidate-{fingerprint[:16]}",
-                    "session_id": session_id,
-                    "recorded_at": recorded_at,
-                    "turn_index": turn_index,
-                    "record_id": record_id,
-                    "categories": item["categories"],
-                    "provenance": item["provenance"],
-                    "measurement_candidates": item["measurement_candidates"],
-                    "isolated_measurement_candidates": item["isolated_measurement_candidates"],
-                    "evidence_excerpt": item["excerpt"],
-                    "source_records": [source],
-                    "source_record_sha256": [digest(record)],
-                }
-            else:
-                grouped[fingerprint]["source_records"].append(source)
-                grouped[fingerprint]["source_record_sha256"].append(digest(record))
-            for isolated_item in item["isolated_measurement_candidates"]:
-                isolated_fingerprint = digest(
-                    {
-                        "session_id": session_id,
-                        "call_id": isolated_item.get("call_id"),
-                        "values": isolated_item.get("values"),
-                    }
-                )
-                if isolated_fingerprint not in isolated:
-                    isolated[isolated_fingerprint] = {
-                        **isolated_item,
-                        "candidate_id": f"isolated-{isolated_fingerprint[:16]}",
-                        "session_id": session_id,
-                        "recorded_at": recorded_at,
-                        "turn_index": turn_index,
-                        "record_id": record_id,
-                        "json_path": item["json_path"],
-                        "evidence_excerpt": item["excerpt"],
-                        "source_records": [source],
-                        "source_record_sha256": [digest(record)],
-                    }
-                else:
-                    isolated[isolated_fingerprint]["source_records"].append(source)
-                    isolated[isolated_fingerprint]["source_record_sha256"].append(digest(record))
-    args.output.parent.mkdir(parents=True, exist_ok=True)
-    rows = sorted(grouped.values(), key=lambda row: (str(row.get("session_id")), str(row.get("recorded_at")), row["candidate_id"]))
-    with args.output.open("w", encoding="utf-8") as output:
-        for row in rows:
-            row["source_records"] = sorted(set(row["source_records"]))
-            row["source_record_sha256"] = sorted(set(row["source_record_sha256"]))
-            output.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n")
-    isolated_path = args.output.with_name("isolated-measurements.jsonl")
-    with isolated_path.open("w", encoding="utf-8") as output:
-        for row in sorted(isolated.values(), key=lambda value: value["candidate_id"]):
-            row["source_records"] = sorted(set(row["source_records"]))
-            row["source_record_sha256"] = sorted(set(row["source_record_sha256"]))
-            output.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n")
-    summary = {
-        "input_files": len(paths),
-        "candidates": len(rows),
-        "isolated_measurements": len(isolated),
-        "categories": dict(category_counts),
-        "parse": stats.as_dict(),
-        "output": str(args.output),
-        "isolated_output": str(isolated_path),
-    }
-    print(json.dumps(summary, ensure_ascii=False))
-    return 0 if (args.allow_parse_errors or stats.errors == 0) and paths else 2
-
-
-def load_jsonl(path: Path) -> Iterator[tuple[int, dict[str, Any]]]:
-    with path.open("r", encoding="utf-8") as handle:
-        for line_number, line in enumerate(handle, 1):
-            if not line.strip():
-                continue
-            value = json.loads(line)
-            if isinstance(value, dict):
-                yield line_number, value
-
-
-def summarize_jsonl(args: argparse.Namespace) -> int:
-    sessions: Counter[str] = Counter()
-    categories: Counter[str] = Counter()
-    record_types: Counter[str] = Counter()
-    outcomes: Counter[str] = Counter()
-    rows = 0
-    for _, row in load_jsonl(args.file):
-        rows += 1
-        sessions[str(row.get("session_id"))] += 1
-        categories.update(str(value) for value in row.get("categories", []))
-        if row.get("record_type"):
-            record_types[str(row["record_type"])] += 1
-        if row.get("outcome"):
-            outcomes[str(row["outcome"])] += 1
-    result = {
-        "rows": rows,
-        "sessions": len(sessions),
-        "largest_sessions": dict(sessions.most_common(args.top)),
-        "categories": dict(categories),
-        "record_types": dict(record_types),
-        "outcomes": dict(outcomes),
-    }
-    print(json.dumps(result, ensure_ascii=False, indent=2))
-    return 0
-
-
-def query_jsonl(args: argparse.Namespace) -> int:
-    emitted = 0
-    for _, row in load_jsonl(args.file):
-        if args.session_id is not None and str(row.get("session_id")) != args.session_id:
-            continue
-        if args.category is not None and args.category not in row.get("categories", []):
-            continue
-        operator = (row.get("context") or {}).get("operator")
-        if args.operator is not None and operator != args.operator:
-            continue
-        print(json.dumps(row, ensure_ascii=False))
-        emitted += 1
-        if args.limit and emitted >= args.limit:
-            break
-    return 0
-
-
-def list_sessions(args: argparse.Namespace) -> int:
-    sessions: Counter[str] = Counter()
-    for _, row in load_jsonl(args.file):
-        session_id = row.get("session_id")
-        sessions[str(session_id) if session_id is not None else "<unknown>"] += 1
-    for session_id in sorted(sessions):
-        print(json.dumps({"session_id": session_id, "rows": sessions[session_id]}, ensure_ascii=False))
-    return 0
-
-
-def validate_ledger(args: argparse.Namespace) -> int:
-    required = {
-        "artifact_type",
-        "entry_id",
-        "attempt_id",
-        "session_id",
-        "record_type",
-        "evidence_origin",
-        "verification_status",
-        "context",
-        "technique",
-        "observations",
-        "measurements",
-        "bottleneck",
-        "outcome",
-        "scope",
-        "uncertainties",
-        "source_records",
-    }
-    errors: list[str] = []
-    seen_ids: set[str] = set()
-    rows = 0
-    try:
-        for line_number, row in load_jsonl(args.ledger):
-            rows += 1
-            missing = sorted(required - row.keys())
-            if missing:
-                errors.append(f"line {line_number}: missing {', '.join(missing)}")
-            entry_id = row.get("entry_id")
-            if row.get("artifact_type") != LEDGER_ARTIFACT_TYPE:
-                errors.append(f"line {line_number}: invalid artifact_type")
-            if not isinstance(entry_id, str) or not entry_id:
-                errors.append(f"line {line_number}: entry_id must be a non-empty string")
-            elif entry_id in seen_ids:
-                errors.append(f"line {line_number}: duplicate entry_id {entry_id}")
-            else:
-                seen_ids.add(entry_id)
-            if row.get("record_type") not in LEDGER_RECORD_TYPES:
-                errors.append(f"line {line_number}: invalid record_type")
-            if row.get("evidence_origin") not in EVIDENCE_ORIGINS:
-                errors.append(f"line {line_number}: invalid evidence_origin")
-            if row.get("verification_status") not in VERIFICATION_STATUSES:
-                errors.append(f"line {line_number}: invalid verification_status")
-            for key in ("attempt_id", "session_id"):
-                if not isinstance(row.get(key), str) or not row.get(key):
-                    errors.append(f"line {line_number}: {key} must be a non-empty string")
-            for key in ("context", "technique", "bottleneck"):
-                if not isinstance(row.get(key), dict):
-                    errors.append(f"line {line_number}: {key} must be an object")
-            context = row.get("context")
-            if isinstance(context, dict):
-                for key in ("operator", "data_type", "shape", "data_scale", "implementation", "hardware", "environment", "baseline"):
-                    if key not in context:
-                        errors.append(f"line {line_number}: context missing {key}")
-            technique = row.get("technique")
-            if isinstance(technique, dict):
-                for key in ("family", "signature", "summary"):
-                    if key not in technique:
-                        errors.append(f"line {line_number}: technique missing {key}")
-            bottleneck = row.get("bottleneck")
-            if isinstance(bottleneck, dict):
-                for key in ("hypothesis", "evidence", "confidence"):
-                    if key not in bottleneck:
-                        errors.append(f"line {line_number}: bottleneck missing {key}")
-                if "evidence" in bottleneck and not isinstance(bottleneck["evidence"], list):
-                    errors.append(f"line {line_number}: bottleneck.evidence must be a list")
-            for key in ("observations", "measurements", "uncertainties"):
-                if not isinstance(row.get(key), list):
-                    errors.append(f"line {line_number}: {key} must be a list")
-            if isinstance(row.get("measurements"), list) and not all(isinstance(item, dict) for item in row["measurements"]):
-                errors.append(f"line {line_number}: measurements entries must be objects")
-            sources = row.get("source_records")
-            if not isinstance(sources, list) or not sources or not all(isinstance(source, str) and source for source in sources):
-                errors.append(f"line {line_number}: source_records must be a non-empty string list")
-            if row.get("evidence_origin") == "measured" and not row.get("measurements"):
-                errors.append(f"line {line_number}: measured evidence requires measurements")
-    except (OSError, json.JSONDecodeError) as error:
-        errors.append(f"unable to read ledger: {error}")
-    result = {"rows": rows, "errors": errors, "valid": not errors}
-    print(json.dumps(result, ensure_ascii=False, indent=2))
-    return 0 if not errors else 1
-
-
-def build_parser() -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser(description=__doc__)
-    commands = parser.add_subparsers(dest="command", required=True)
-    inspect_parser = commands.add_parser("inspect", help="summarize schemas without emitting raw content")
-    inspect_parser.add_argument("input", nargs="+", type=Path)
-    inspect_parser.add_argument("--samples", type=int, default=5)
-    inspect_parser.add_argument("--allow-parse-errors", action="store_true")
-    inspect_parser.set_defaults(func=inspect_inputs)
-    extract_parser = commands.add_parser("extract", help="write structured, redacted evidence candidates")
-    extract_parser.add_argument("input", nargs="+", type=Path)
-    extract_parser.add_argument("-o", "--output", type=Path, default=Path("out/evidence-candidates.jsonl"))
-    extract_parser.add_argument("--excerpt-chars", type=int, default=4000)
-    extract_parser.add_argument("--allow-parse-errors", action="store_true")
-    extract_parser.set_defaults(func=extract_candidates)
-    summarize_parser = commands.add_parser("summarize", help="summarize a candidate or ledger JSONL file")
-    summarize_parser.add_argument("file", type=Path)
-    summarize_parser.add_argument("--top", type=int, default=20)
-    summarize_parser.set_defaults(func=summarize_jsonl)
-    query_parser = commands.add_parser("query", help="stream filtered candidate or ledger rows")
-    query_parser.add_argument("file", type=Path)
-    query_parser.add_argument("--session-id")
-    query_parser.add_argument("--category")
-    query_parser.add_argument("--operator")
-    query_parser.add_argument("--limit", type=int)
-    query_parser.set_defaults(func=query_jsonl)
-    sessions_parser = commands.add_parser("sessions", help="list every session in a candidate or ledger JSONL file")
-    sessions_parser.add_argument("file", type=Path)
-    sessions_parser.set_defaults(func=list_sessions)
-    validate_parser = commands.add_parser("validate", help="validate the ledger envelope and enums")
-    validate_parser.add_argument("ledger", type=Path)
-    validate_parser.set_defaults(func=validate_ledger)
-    return parser
-
-
-def main(argv: list[str] | None = None) -> int:
-    args = build_parser().parse_args(argv)
-    try:
-        return args.func(args)
-    except (OSError, EOFError, json.JSONDecodeError) as error:
-        print(json.dumps({"error": str(error), "command": args.command}, ensure_ascii=False), file=sys.stderr)
-        return 2
-
-
-if __name__ == "__main__":
-    sys.exit(main())