@heretek-ai/epistemic-swarm 0.7.25 → 1.0.2

package/runner/path_safety.py

@@ -1,62 +0,0 @@
-#!/usr/bin/env python3
-"""Canonical path safety and containment helpers.
-
-Trust boundary
---------------
-These helpers exist for the real S8707 pattern: an **untrusted relative segment
-joined under a trusted base** (a cache key, a scope id, a filename from an LLM).
-That is where containment must be enforced.
-
-They are deliberately *not* applied to caller-chosen CLI/tool arguments such as
-``pcrb --out``, ``socratic_tree --file`` or ``--key-file``. Those are the
-principal's own paths: the user (or the orchestrating agent) is the trust root,
-and restricting them would break legitimate use like writing a brief to
-``~/reports/``. Those sites canonicalise with ``os.path.realpath`` only. Do not
-"tighten" them by routing through :func:`safe_resolve_path` — that is a behavior
-change, not a hardening.
-"""
-
-import os
-from pathlib import Path
-from typing import Union
-
-
-def safe_resolve_path(untrusted_path: Union[str, Path], base_dir: Union[str, Path]) -> Path:
-    """Canonicalise a path and require that it stays inside ``base_dir``.
-
-    Accepts either an absolute path or one relative to ``base_dir``. An absolute
-    path is allowed only when it already resolves inside the base; ``..``
-    segments and symlinks are resolved *before* the containment test, so neither
-    can be used to escape.
-
-    Raises ValueError if the resolved path escapes ``base_dir``.
-    """
-    base = os.path.realpath(str(base_dir))
-    raw = str(untrusted_path)
-    # os.path.join would silently drop `base` for an absolute second argument,
-    # so decide the candidate explicitly.
-    candidate = raw if os.path.isabs(raw) else os.path.join(base, raw)
-    target = os.path.realpath(candidate)
-    base_prefix = base if base.endswith(os.sep) else base + os.sep
-    if target != base and not target.startswith(base_prefix):
-        raise ValueError(f"Path traversal detected: {untrusted_path!r} escapes {base_dir!r}")
-    return Path(target)
-
-
-def safe_join(base_dir: Union[str, Path], *segments: str) -> Path:
-    """Join untrusted *relative* segments under a trusted base, with containment.
-
-    This is the S8707 shape: ``base_dir / f"{user_token}.md"``. Each segment must
-    be a bare name — no separators, no ``..`` — and the joined result must still
-    resolve inside ``base_dir``.
-
-    Raises ValueError on any segment that could escape.
-    """
-    for segment in segments:
-        if not segment or segment in (".", ".."):
-            raise ValueError(f"Unsafe path segment: {segment!r}")
-        if os.sep in segment or (os.altsep and os.altsep in segment):
-            raise ValueError(f"Path separator in segment: {segment!r}")
-        if Path(segment).name != segment:
-            raise ValueError(f"Unsafe path segment: {segment!r}")
-    return safe_resolve_path(os.path.join(*segments), base_dir)

package/runner/pcrb.py

@@ -1,290 +0,0 @@
-#!/usr/bin/env python3
-"""
-Proof-Carrying Research Brief (PCRB) — export side (Stream D).
-
-A PCRB is a self-contained signed bundle: the synthesis, every claim with its
-quote witness, the full text of every cited source, an integrity manifest, and
-an optional HMAC signature. A third party can re-verify everything WITHOUT
-trusting the LLM that produced it and WITHOUT access to the original
-.research/ workspace — verification runs against the bundled sources.
-
-Bundle shape (brief.pcrb.json):
-{
-  "pcrb_version": "1.0",
-  "objective": ..., "exported_at": ...,
-  "synthesis_markdown": ...,
-  "claims": [ {claim fields..., "witness": {quote, source_hash, confidence, verified_at}} ],
-  "sources": { "<hash>": {"url","title","tier","content"} },
-  "manifest": { "synthesis_markdown": sha, "claims": {id: sha}, "sources": {hash: sha} },
-  "signature": {"alg": "hmac-sha256" | "none", "key_id": ..., "sig": ...}
-}
-
-Member hashes use SourceHasher.compute_sha256 semantics so a bundled source's
-member hash equals its cache hash.
-
-Signing: stdlib hmac when a key is provided (env IUMBTEMS_PCRB_KEY or key
-file). alg="none" provides integrity (member-vs-manifest) but NOT authenticity
-— a coordinated rewrite of content + manifest is undetectable without a key.
-Real PKI (ed25519) is a gated follow-on; HMAC is sufficient for the tamper
-probe, not for the regulatory story.
-"""
-
-import hashlib
-import hmac
-import json
-import os
-import sys
-from datetime import datetime, timezone
-from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple
-
-SCRIPT_DIR = Path(__file__).resolve().parent
-PROJECT_ROOT = SCRIPT_DIR.parent
-if str(PROJECT_ROOT) not in sys.path:
-    sys.path.insert(0, str(PROJECT_ROOT))
-
-from runner.claim_witness import claims_from_dossier, load_dossier  # noqa: E402
-
-PCRB_VERSION = "1.0"
-ENV_KEY = "IUMBTEMS_PCRB_KEY"
-
-
-def canonical_json(obj: Any) -> str:
-    """Deterministic serialization for hashing/signing."""
-    return json.dumps(obj, sort_keys=True, separators=(",", ":"))
-
-
-def sha256_text(text: str) -> str:
-    """SourceHasher.compute_sha256 semantics (strip + utf-8)."""
-    return hashlib.sha256(text.strip().encode("utf-8")).hexdigest()
-
-
-def member_hash(obj: Any) -> str:
-    """Hash a structured member via canonical JSON."""
-    return sha256_text(canonical_json(obj))
-
-
-def _resolve_key(key: Optional[str], key_path: Optional[Path]) -> Optional[bytes]:
-    if key:
-        return key.encode("utf-8")
-    env = os.environ.get(ENV_KEY)
-    if env:
-        return env.encode("utf-8")
-    if key_path:
-        # An explicitly supplied key file that cannot be read must fail loudly.
-        # Returning None here would sign the brief with `alg: "none"` and report
-        # success — a typo'd path silently produces an unauthenticated bundle.
-        canonical_key_path = Path(os.path.realpath(str(key_path)))
-        if not canonical_key_path.is_file():
-            raise ValueError(
-                f"key file is not a readable file: {key_path!r} (resolved {canonical_key_path})"
-            )
-        return canonical_key_path.read_bytes().strip()
-    return None
-
-
-def sign_manifest(
-    manifest: Dict[str, Any], key: Optional[bytes], key_id: str = "local"
-) -> Dict[str, Any]:
-    if not key:
-        return {"alg": "none", "key_id": None, "sig": None}
-    sig = hmac.new(
-        key, canonical_json(manifest).encode("utf-8"), hashlib.sha256
-    ).hexdigest()
-    return {"alg": "hmac-sha256", "key_id": key_id, "sig": sig}
-
-
-def _resolve_synthesis(base_dir: Path) -> str:
-    final_md = base_dir / "final_synthesis.md"
-    if final_md.exists():
-        return final_md.read_text(encoding="utf-8")
-    parts = sorted((base_dir / "scratchpads").glob("*/scope_synthesis.md"))
-    return "\n\n---\n\n".join(p.read_text(encoding="utf-8") for p in parts)
-
-
-def _witness_for(c, hasher: Any, needed_hashes: set) -> Dict[str, Any]:
-    """Build one claim's witness record and track hashes worth bundling.
-
-    R8 (01-nk-hardening tiebreak-2 F1): source_hash/verbatim_quote are
-    untrusted-adjacent (hand-built witnesses may carry int/dict/list) —
-    isinstance-guard both before hasher/set. Non-string sides degrade to
-    reject (confidence None, no bundling), never TypeError via
-    `re.match` (hasher) or `set.add` (unhashable dict).
-    """
-    _sh = getattr(c, "source_hash", None)
-    _q = getattr(c, "verbatim_quote", None)
-    witness: Dict[str, Any] = {
-        "quote": _q,
-        "source_hash": _sh,
-        "confidence": None,
-        "verified_at": None,
-    }
-    if not isinstance(_sh, str) or not _sh:
-        return witness
-    if not isinstance(_q, str) or not _q:
-        try:
-            needed_hashes.add(_sh)
-        except TypeError:
-            pass
-        return witness
-    try:
-        passed, conf, _msg = hasher.verify_quote(_sh, _q)
-    except (TypeError, AttributeError):
-        return witness
-    try:
-        witness["confidence"] = float(conf)
-    except (TypeError, ValueError):
-        witness["confidence"] = None
-    witness["verified_at"] = datetime.now(timezone.utc).isoformat()
-    if passed:
-        try:
-            needed_hashes.add(_sh)
-        except TypeError:
-            pass
-    # Legacy: a present-but-unverified quote bundles nothing (verifier
-    # re-checks independently); a missing quote bundles the hash for audit.
-    return witness
-
-
-def _iter_dossier_claims(scope_dirs: List[Path]):
-    """Yield claim witnesses from every alpha/beta dossier under scopes."""
-    for scope_dir in scope_dirs:
-        for dossier_name in ("alpha_dossier.json", "beta_dossier.json"):
-            dpath = scope_dir / dossier_name
-            if not dpath.exists():
-                continue
-            for c in claims_from_dossier(load_dossier(dpath)):
-                yield c
-
-
-def _collect_claims(
-    scope_dirs: List[Path], hasher: Any
-) -> Tuple[List[Dict[str, Any]], set]:
-    claim_records: List[Dict[str, Any]] = []
-    needed_hashes: set = set()
-    for c in _iter_dossier_claims(scope_dirs):
-        rec = c.to_dict()
-        rec["witness"] = _witness_for(c, hasher, needed_hashes)
-        claim_records.append(rec)
-    return claim_records, needed_hashes
-
-
-def _resolve_objective(base_dir: Path, objective: Optional[str]) -> Optional[str]:
-    if objective:
-        return objective
-    from runner.state_machine import find_any_manifest
-
-    manifest_p = find_any_manifest(base_dir)
-    if manifest_p is not None and manifest_p.exists():
-        try:
-            return json.loads(manifest_p.read_text(encoding="utf-8")).get("objective")
-        except (OSError, json.JSONDecodeError):
-            return None
-    return None
-
-
-def export_brief(
-    base_dir: Path,
-    out_path: Optional[Path] = None,
-    scope_ids: Optional[List[str]] = None,
-    objective: Optional[str] = None,
-    key: Optional[str] = None,
-    key_path: Optional[Path] = None,
-    key_id: str = "local",
-) -> Path:
-    """Assemble a PCRB from a .research workspace. Returns the bundle path."""
-    from skills.research_cache.hasher import SourceHasher
-
-    base_dir = Path(os.path.realpath(str(base_dir)))
-    hasher = SourceHasher(base_dir)
-    synthesis = _resolve_synthesis(base_dir)
-
-    scratch = base_dir / "scratchpads"
-    if scope_ids:
-        scope_dirs = [scratch / s for s in scope_ids]
-    elif scratch.exists():
-        scope_dirs = sorted(p for p in scratch.iterdir() if p.is_dir())
-    else:
-        scope_dirs = []
-
-    claim_records, needed_hashes = _collect_claims(scope_dirs, hasher)
-
-    # --- bundled sources (self-contained: full text in the bundle) ---
-    sources: Dict[str, Dict[str, Any]] = {}
-    for src_hash in sorted(needed_hashes):
-        content = hasher.get_source_content(src_hash)
-        meta = hasher.get_source_metadata(src_hash) or {}
-        if content is None:
-            continue
-        sources[src_hash] = {
-            "url": meta.get("url"),
-            "title": meta.get("title"),
-            "tier": meta.get("tier"),
-            "content": content,
-        }
-
-    # --- manifest (integrity) ---
-    manifest: Dict[str, Any] = {
-        "synthesis_markdown": sha256_text(synthesis),
-        "claims": {rec["claim_id"]: member_hash(rec) for rec in claim_records},
-        "sources": {h: sha256_text(s["content"]) for h, s in sources.items()},
-    }
-
-    resolved_key = _resolve_key(key, key_path)
-    signature = sign_manifest(manifest, resolved_key, key_id=key_id)
-
-    bundle = {
-        "pcrb_version": PCRB_VERSION,
-        "objective": _resolve_objective(base_dir, objective),
-        "exported_at": datetime.now(timezone.utc).isoformat(),
-        "synthesis_markdown": synthesis,
-        "claims": claim_records,
-        "sources": sources,
-        "manifest": manifest,
-        "signature": signature,
-    }
-
-    out = (
-        Path(os.path.realpath(str(out_path)))
-        if out_path
-        else (base_dir / "brief.pcrb.json")
-    )
-    out.parent.mkdir(parents=True, exist_ok=True)
-    with open(out, "w", encoding="utf-8") as f:
-        json.dump(bundle, f, indent=2)
-    return out
-
-
-def main(argv: Optional[List[str]] = None) -> int:
-    import argparse
-
-    parser = argparse.ArgumentParser(
-        description="Export a proof-carrying research brief"
-    )
-    parser.add_argument("--dir", default=".research")
-    parser.add_argument("--out", default=None)
-    parser.add_argument("--scope", action="append", dest="scope_ids")
-    parser.add_argument("--objective", default=None)
-    parser.add_argument("--key-file", default=None)
-    parser.add_argument("--key-id", default="local")
-    args = parser.parse_args(argv)
-
-    canonical_dir = Path(os.path.realpath(str(args.dir)))
-    canonical_out = Path(os.path.realpath(str(args.out))) if args.out else None
-    canonical_key = (
-        Path(os.path.realpath(str(args.key_file))) if args.key_file else None
-    )
-    path = export_brief(
-        canonical_dir,
-        out_path=canonical_out,
-        scope_ids=args.scope_ids,
-        objective=args.objective,
-        key_path=canonical_key,
-        key_id=args.key_id,
-    )
-    print(json.dumps({"status": "exported", "path": str(path)}))
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

package/runner/pcrb_verify.py

@@ -1,267 +0,0 @@
-#!/usr/bin/env python3
-"""
-Proof-Carrying Research Brief (PCRB) — standalone verifier (Stream D).
-
-Verifies a brief.pcrb.json WITHOUT trusting the LLM that produced it and
-WITHOUT the original .research/ workspace: every quote is re-checked against
-the sources BUNDLED INSIDE the brief.
-
-Checks (all must pass for exit 0):
-  1. source identity  — sha256(source content) == its hash key in the bundle
-  2. manifest integrity — every member hash recomputes to the manifest value
-  3. signature — HMAC over the manifest (when alg != "none")
-  4. quotes — every claim's verbatim_quote re-verified against the bundled
-     source text, using the SAME verification stack as the auditor
-     (SourceHasher.verify_quote via a throwaway cache directory)
-
-Usage:
-  python3 runner/pcrb_verify.py brief.pcrb.json [--json] [--key-file ...]
-
-Exit codes: 0 = all checks passed, 1 = tampering/failure detected,
-2 = unreadable bundle.
-"""
-
-import argparse
-import hashlib
-import hmac
-import json
-import os
-import sys
-import tempfile
-import time
-from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple
-
-SCRIPT_DIR = Path(__file__).resolve().parent
-PROJECT_ROOT = SCRIPT_DIR.parent
-if str(PROJECT_ROOT) not in sys.path:
-    sys.path.insert(0, str(PROJECT_ROOT))
-
-# Reuse the export-side canonicalization so hashes always agree.
-from runner.pcrb import (  # noqa: E402
-    ENV_KEY,
-    canonical_json,
-    member_hash,
-    sha256_text,
-)
-
-
-def _check_source_identity(sources: Dict[str, Any]) -> List[Dict[str, str]]:
-    failures = []
-    for src_hash, meta in sources.items():
-        content = meta.get("content") or ""
-        actual = sha256_text(content)
-        if actual != src_hash:
-            failures.append({
-                "check": "source_identity",
-                "member": src_hash,
-                "message": f"bundled source content hashes to {actual}, not its key {src_hash}",
-            })
-    return failures
-
-
-def _check_manifest_integrity(
-    bundle: Dict[str, Any],
-    manifest: Dict[str, Any],
-    claims: List[Dict[str, Any]],
-    sources: Dict[str, Any],
-) -> List[Dict[str, str]]:
-    failures = []
-    synth_hash = sha256_text(bundle.get("synthesis_markdown") or "")
-    if manifest.get("synthesis_markdown") != synth_hash:
-        failures.append({
-            "check": "manifest_integrity",
-            "member": "synthesis_markdown",
-            "message": "synthesis_markdown does not match its manifest hash",
-        })
-
-    manifest_claims = manifest.get("claims") or {}
-    for rec in claims:
-        cid = rec.get("claim_id", "UNKNOWN")
-        expected = manifest_claims.get(cid)
-        actual = member_hash(rec)
-        if expected is None:
-            failures.append({"check": "manifest_integrity", "member": cid, "message": "claim missing from manifest"})
-        elif expected != actual:
-            failures.append({
-                "check": "manifest_integrity",
-                "member": cid,
-                "message": f"claim hash mismatch (manifest {expected[:16]}… vs actual {actual[:16]}…)",
-            })
-
-    manifest_sources = manifest.get("sources") or {}
-    for src_hash, meta in sources.items():
-        expected = manifest_sources.get(src_hash)
-        actual = sha256_text(meta.get("content") or "")
-        if expected is None:
-            failures.append({"check": "manifest_integrity", "member": src_hash, "message": "source missing from manifest"})
-        elif expected != actual:
-            failures.append({
-                "check": "manifest_integrity",
-                "member": src_hash,
-                "message": "source content does not match its manifest hash",
-            })
-    return failures
-
-
-def _check_signature(
-    signature: Dict[str, Any],
-    manifest: Dict[str, Any],
-    key: Optional[bytes],
-) -> Tuple[str, List[Dict[str, str]]]:
-    alg = signature.get("alg") or "none"
-    failures = []
-    if alg != "hmac-sha256":
-        return "unsigned", failures
-    if not key:
-        failures.append({
-            "check": "signature",
-            "member": "manifest",
-            "message": "brief is HMAC-signed but no key was provided (env IUMBTEMS_PCRB_KEY or --key-file)",
-        })
-        return "unverifiable", failures
-
-    expected_sig = hmac.new(key, canonical_json(manifest).encode("utf-8"), hashlib.sha256).hexdigest()
-    provided = signature.get("sig") or ""
-    if not hmac.compare_digest(expected_sig, provided):
-        failures.append({
-            "check": "signature",
-            "member": "manifest",
-            "message": "HMAC signature does not match manifest (manifest was rewritten or key differs)",
-        })
-        return "invalid", failures
-    return "valid", failures
-
-
-def _check_quotes(
-    claims: List[Dict[str, Any]],
-    sources: Dict[str, Any],
-) -> Tuple[int, int, List[Dict[str, str]]]:
-    quotes_verified = 0
-    quotes_failed = 0
-    failures = []
-    with tempfile.TemporaryDirectory(prefix="pcrb_verify_") as tmp:
-        from skills.research_cache.hasher import SourceHasher
-
-        hasher = SourceHasher(Path(tmp))
-        for src_hash, meta in sources.items():
-            content = meta.get("content") or ""
-            md_path = Path(tmp) / "sources" / f"{src_hash}.md"
-            md_path.parent.mkdir(parents=True, exist_ok=True)
-            md_path.write_text(content, encoding="utf-8")
-
-        for rec in claims:
-            cid = rec.get("claim_id", "UNKNOWN")
-            witness = rec.get("witness") or {}
-            src_hash = witness.get("source_hash") or rec.get("source_hash")
-            quote = witness.get("quote") or rec.get("verbatim_quote")
-            if not src_hash or not quote:
-                continue
-            if src_hash not in sources:
-                quotes_failed += 1
-                failures.append({
-                    "check": "quote",
-                    "member": cid,
-                    "message": f"claim cites source {src_hash} which is not bundled",
-                })
-                continue
-            passed, conf, msg = hasher.verify_quote(src_hash, quote)
-            if passed:
-                quotes_verified += 1
-            else:
-                quotes_failed += 1
-                failures.append({
-                    "check": "quote",
-                    "member": cid,
-                    "message": f"quote not found in bundled source (conf={conf}): {msg}",
-                })
-    return quotes_verified, quotes_failed, failures
-
-
-def verify_brief(
-    bundle: Dict[str, Any],
-    key: Optional[bytes] = None,
-) -> Dict[str, Any]:
-    """Run all checks. Returns a structured report dict."""
-    started = time.perf_counter()
-    manifest = bundle.get("manifest") or {}
-    signature = bundle.get("signature") or {}
-    claims = bundle.get("claims") or []
-    sources = bundle.get("sources") or {}
-
-    failures: List[Dict[str, str]] = []
-    failures.extend(_check_source_identity(sources))
-    failures.extend(_check_manifest_integrity(bundle, manifest, claims, sources))
-    sig_status, sig_failures = _check_signature(signature, manifest, key)
-    failures.extend(sig_failures)
-    quotes_verified, quotes_failed, quote_failures = _check_quotes(claims, sources)
-    failures.extend(quote_failures)
-
-    elapsed = time.perf_counter() - started
-    return {
-        "ok": not failures,
-        "failures": failures,
-        "checks": {
-            "source_identity": "pass" if not any(f["check"] == "source_identity" for f in failures) else "fail",
-            "manifest_integrity": "pass" if not any(f["check"] == "manifest_integrity" for f in failures) else "fail",
-            "signature": sig_status,
-            "quotes": "pass" if quotes_failed == 0 else "fail",
-        },
-        "stats": {
-            "claims": len(claims),
-            "sources": len(sources),
-            "quotes_verified": quotes_verified,
-            "quotes_failed": quotes_failed,
-            "elapsed_s": round(elapsed, 4),
-        },
-    }
-
-
-def main(argv: Optional[List[str]] = None) -> int:
-    parser = argparse.ArgumentParser(description="Verify a proof-carrying research brief")
-    parser.add_argument("brief", help="Path to brief.pcrb.json")
-    parser.add_argument("--json", action="store_true", help="Print JSON report")
-    parser.add_argument("--key-file", default=None)
-    args = parser.parse_args(argv)
-
-    path = Path(os.path.realpath(str(args.brief)))
-    if not path.is_file():
-        print(f"error: no such brief: {path}", file=sys.stderr)
-        return 2
-    try:
-        bundle = json.loads(path.read_text(encoding="utf-8"))
-    except (OSError, json.JSONDecodeError) as exc:
-        print(f"error: unreadable brief: {exc}", file=sys.stderr)
-        return 2
-
-    key = None
-    if args.key_file:
-        key_file_path = Path(os.path.realpath(str(args.key_file)))
-        # Fail loudly rather than fall through to an unsigned verification:
-        # a typo'd --key-file would otherwise silently verify with no key.
-        if not key_file_path.is_file():
-            print(f"error: key file is not a readable file: {args.key_file}", file=sys.stderr)
-            return 2
-        key = key_file_path.read_bytes().strip()
-    elif os.environ.get(ENV_KEY):
-        key = os.environ[ENV_KEY].encode("utf-8")
-
-    report = verify_brief(bundle, key=key)
-
-    if args.json:
-        print(json.dumps(report, indent=2))
-    else:
-        status = "PASS" if report["ok"] else "FAIL"
-        print(f"[{status}] {path}")
-        for name, st in report["checks"].items():
-            print(f"  {name}: {st}")
-        for f in report["failures"]:
-            print(f"  ✗ [{f['check']}] {f['member']}: {f['message']}")
-        s = report["stats"]
-        print(f"  quotes verified: {s['quotes_verified']}/{s['claims']} in {s['elapsed_s']}s")
-
-    return 0 if report["ok"] else 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())