@heretek-ai/epistemic-swarm 0.2.2 → 0.3.0

package/runner/living_dossiers.py

@@ -0,0 +1,355 @@
+#!/usr/bin/env python3
+"""
+Living Dossiers: claim degradation ledger (Stream C).
+
+Epistemic core idea: a `[VERIFIED]` claim is a time-bounded LOAN against its
+cached source, not a permanent fact. When a source is retracted or revised,
+the loan is called — regardless of whether the local cache still contains a
+matching quote.
+
+Source-of-truth discipline (same as the rest of IUMBTEMS):
+- Retraction events are FILES: `.research/retractions/<hash>.json`
+  `{hash, event: RETRACTED|REVISED, at, note, supersedes?}`
+- Status transitions are an append-only ledger: `.research/ledger/claim_status.json`
+- Dossiers are NEVER mutated in place — they are the historical record.
+- Requeue instructions land in `.research/requeue.json` for the next swarm pass.
+
+Degradation rules (applied after quote verification):
+  RETRACTED source -> claim.status = STALE   (loan called; quote match irrelevant)
+  REVISED    source -> claim.status = SUSPECT (needs human/agent re-review + requeue)
+"""
+
+import json
+import os
+import sys
+from dataclasses import dataclass, asdict
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional, Tuple
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+PROJECT_ROOT = SCRIPT_DIR.parent
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from runner.claim_witness import (  # noqa: E402
+    STATUS_LIVE,
+    STATUS_STALE,
+    STATUS_SUSPECT,
+    ClaimWitness,
+)
+
+EVENT_RETRACTED = "RETRACTED"
+EVENT_REVISED = "REVISED"
+VALID_EVENTS = (EVENT_RETRACTED, EVENT_REVISED)
+
+
+@dataclass
+class RetractionEvent:
+    hash: str
+    event: str
+    at: str
+    note: str = ""
+    supersedes: Optional[str] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass
+class StatusEvent:
+    scope_id: str
+    claim_id: str
+    from_status: str
+    to_status: str
+    reason: str
+    at: str
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+
+# ---- retraction events (files are the record) ---------------------------
+
+
+def write_retraction(
+    base_dir: Path,
+    source_hash: str,
+    event: str,
+    note: str = "",
+    at: Optional[str] = None,
+    supersedes: Optional[str] = None,
+) -> Path:
+    """Record a RETRACTED/REVISED event for a cached source."""
+    if event not in VALID_EVENTS:
+        raise ValueError(f"event must be one of {VALID_EVENTS}, got {event!r}")
+    retr_dir = Path(base_dir) / "retractions"
+    retr_dir.mkdir(parents=True, exist_ok=True)
+    payload = RetractionEvent(
+        hash=source_hash,
+        event=event,
+        at=at or datetime.now(timezone.utc).isoformat(),
+        note=note,
+        supersedes=supersedes,
+    )
+    path = retr_dir / f"{source_hash}.json"
+    # Last write wins per hash (a REVISED may follow a RETRACTED); the ledger
+    # keeps the full history of claim transitions either way.
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(payload.to_dict(), f, indent=2)
+    return path
+
+
+def load_retractions(base_dir: Path) -> Dict[str, RetractionEvent]:
+    """Load all retraction events, keyed by source hash."""
+    retr_dir = Path(base_dir) / "retractions"
+    out: Dict[str, RetractionEvent] = {}
+    if not retr_dir.exists():
+        return out
+    for path in sorted(retr_dir.glob("*.json")):
+        try:
+            with open(path, "r", encoding="utf-8") as f:
+                data = json.load(f)
+        except (OSError, json.JSONDecodeError):
+            continue
+        src_hash = data.get("hash") or path.stem
+        event = str(data.get("event", "")).upper()
+        if event not in VALID_EVENTS:
+            continue
+        out[src_hash] = RetractionEvent(
+            hash=src_hash,
+            event=event,
+            at=data.get("at") or "",
+            note=data.get("note") or "",
+            supersedes=data.get("supersedes"),
+        )
+    return out
+
+
+# ---- degradation -------------------------------------------------------
+
+
+def apply_degradation(
+    claims: Iterable[ClaimWitness],
+    retractions: Dict[str, RetractionEvent],
+    scope_id: str = "unknown",
+    now: Optional[str] = None,
+) -> Tuple[List[ClaimWitness], List[StatusEvent]]:
+    """Mark claims STALE/SUSPECT per retraction events.
+
+    Returns (degraded_claims, status_events). ClaimWitness objects are
+    updated in memory only — callers persist to the ledger, never into the
+    source dossier files.
+    """
+    now = now or datetime.now(timezone.utc).isoformat()
+    events: List[StatusEvent] = []
+    out: List[ClaimWitness] = []
+
+    for claim in claims:
+        prior = claim.status or STATUS_LIVE
+        src_hash = claim.source_hash
+        ev = retractions.get(src_hash) if src_hash else None
+
+        new_status = prior
+        reason = ""
+        if ev is not None:
+            if ev.event == EVENT_RETRACTED:
+                new_status = STATUS_STALE
+                reason = f"source {src_hash} RETRACTED: {ev.note or 'no note'}"
+            elif ev.event == EVENT_REVISED:
+                new_status = STATUS_SUSPECT
+                reason = f"source {src_hash} REVISED: {ev.note or 'no note'}"
+
+        if new_status != prior:
+            claim.status = new_status
+            events.append(
+                StatusEvent(
+                    scope_id=scope_id,
+                    claim_id=claim.claim_id,
+                    from_status=prior,
+                    to_status=new_status,
+                    reason=reason,
+                    at=now,
+                )
+            )
+        out.append(claim)
+
+    return out, events
+
+
+# ---- ledger + requeue sidecars -----------------------------------------
+
+
+def append_ledger(base_dir: Path, events: List[StatusEvent]) -> Path:
+    """Append status transitions to the append-only ledger (never rewrite)."""
+    ledger_dir = Path(base_dir) / "ledger"
+    ledger_dir.mkdir(parents=True, exist_ok=True)
+    path = ledger_dir / "claim_status.json"
+    existing: List[Dict[str, Any]] = []
+    if path.exists():
+        try:
+            with open(path, "r", encoding="utf-8") as f:
+                existing = json.load(f)
+            if not isinstance(existing, list):
+                existing = []
+        except (OSError, json.JSONDecodeError):
+            existing = []
+    existing.extend(e.to_dict() for e in events)
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(existing, f, indent=2)
+    return path
+
+
+def queue_requeue(base_dir: Path, scope_id: str, reason: str = "claim degradation") -> Path:
+    """Record that a scope needs a re-run on the next swarm pass."""
+    path = Path(base_dir) / "requeue.json"
+    entries: List[Dict[str, Any]] = []
+    if path.exists():
+        try:
+            with open(path, "r", encoding="utf-8") as f:
+                entries = json.load(f)
+            if not isinstance(entries, list):
+                entries = []
+        except (OSError, json.JSONDecodeError):
+            entries = []
+    if not any(e.get("scope_id") == scope_id for e in entries):
+        entries.append(
+            {
+                "scope_id": scope_id,
+                "reason": reason,
+                "queued_at": datetime.now(timezone.utc).isoformat(),
+            }
+        )
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(entries, f, indent=2)
+    return path
+
+
+def load_requeue(base_dir: Path) -> List[Dict[str, Any]]:
+    path = Path(base_dir) / "requeue.json"
+    if not path.exists():
+        return []
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        return data if isinstance(data, list) else []
+    except (OSError, json.JSONDecodeError):
+        return []
+
+
+# ---- one-pass staleness check (the probe's unit of work) ----------------
+
+
+def check_staleness(base_dir: Path, scope_ids: Optional[List[str]] = None) -> Dict[str, Any]:
+    """Run one degradation pass over scope dossiers.
+
+    For each scope with alpha/beta dossiers, fold claims, join against
+    retraction events, write the ledger, mirror into the claim store, and
+    queue re-runs for scopes that degraded. Dossiers themselves are left
+    untouched.
+
+    Returns a summary dict suitable for JSON tool output.
+    """
+    from runner.claim_witness import claims_from_dossier, load_dossier
+
+    base_dir = Path(base_dir)
+    retractions = load_retractions(base_dir)
+    scratch = base_dir / "scratchpads"
+
+    if scope_ids is None:
+        if not scratch.exists():
+            return {"scopes": [], "retractions": len(retractions), "degraded": 0}
+        scope_dirs = sorted(p for p in scratch.iterdir() if p.is_dir())
+    else:
+        scope_dirs = [scratch / s for s in scope_ids]
+
+    all_events: List[StatusEvent] = []
+    per_scope: List[Dict[str, Any]] = []
+    degraded_scopes = set()
+
+    for scope_dir in scope_dirs:
+        if not scope_dir.is_dir():
+            continue
+        scope_id = scope_dir.name
+        scope_event_count = 0
+        claim_sets = 0
+
+        for dossier_name in ("alpha_dossier.json", "beta_dossier.json"):
+            dpath = scope_dir / dossier_name
+            if not dpath.exists():
+                continue
+            claim_sets += 1
+            dossier = load_dossier(dpath)
+            claims = claims_from_dossier(dossier)
+            degraded, events = apply_degradation(
+                claims, retractions, scope_id=scope_id
+            )
+            all_events.extend(events)
+            scope_event_count += len(events)
+            if events:
+                degraded_scopes.add(scope_id)
+
+        per_scope.append(
+            {
+                "scope_id": scope_id,
+                "claim_sets": claim_sets,
+                "status_events": scope_event_count,
+                "degraded": scope_id in degraded_scopes,
+            }
+        )
+
+    if all_events:
+        append_ledger(base_dir, all_events)
+        # Mirror into the derived claim store (best-effort; never fatal).
+        try:
+            from runner.claim_store import ClaimStore
+
+            store = ClaimStore(base_dir)
+            for e in all_events:
+                store.record_status_event(
+                    e.scope_id, e.claim_id, e.from_status, e.to_status, e.reason, e.at
+                )
+            store.close()
+        except Exception as exc:  # noqa: BLE001
+            print(f"[living-dossiers] claim store mirror skipped: {exc}", file=sys.stderr)
+
+    for scope_id in sorted(degraded_scopes):
+        queue_requeue(base_dir, scope_id, reason="claim degradation (retraction event)")
+
+    return {
+        "retractions": len(retractions),
+        "scopes": per_scope,
+        "degraded_scopes": sorted(degraded_scopes),
+        "status_events": len(all_events),
+    }
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    import argparse
+
+    parser = argparse.ArgumentParser(description="IUMBTEMS living dossiers ledger")
+    parser.add_argument("--dir", default=".research")
+    sub = parser.add_subparsers(dest="command")
+
+    rep = sub.add_parser("report", help="Record a retraction/retraction event")
+    rep.add_argument("--hash", required=True)
+    rep.add_argument("--event", required=True, choices=list(VALID_EVENTS))
+    rep.add_argument("--note", default="")
+
+    sub.add_parser("check", help="Run one staleness/degradation pass")
+    args = parser.parse_args(argv)
+
+    base = Path(args.dir)
+    if args.command == "report":
+        path = write_retraction(base, args.hash, args.event, note=args.note)
+        print(json.dumps({"status": "recorded", "path": str(path)}))
+        return 0
+    if args.command == "check":
+        print(json.dumps(check_staleness(base), indent=2))
+        return 0
+    print(json.dumps(check_staleness(base), indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/runner/mcp_protocol.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+"""
+Shared MCP stdio JSON-RPC protocol loop for IUMBTEMS.
+
+Extracted from the duplicated hand-rolled servers in config/mcp_launcher.py
+(the zero-key fallback) and config/searxng_mcp.py so the first-party tool
+server (runner/mcp_server.py) can share one framing/dispatch implementation.
+
+Protocol contract (kept byte-compatible with the previous inline loops):
+- protocolVersion: "2024-11-05"
+- Methods: initialize, notifications/initialized (no reply), tools/list,
+  tools/call. Unknown methods get JSON-RPC error -32601.
+- Framing: Content-Length header frames AND newline-delimited JSON are both
+  accepted; a reply is framed the same way its request arrived.
+- tools/call success payload: {"content": [{"type": "text", "text": ...}]}.
+
+Resilience note (intentional delta from the old inline loops): a tool handler
+that raises now returns JSON-RPC error -32603 instead of killing the server.
+Malformed messages are logged and skipped; any other stream error ends the
+loop (Content-Length desync is not recoverable).
+"""
+
+import json
+import sys
+from typing import Any, Callable, Dict, List, Optional, TextIO, Tuple
+
+PROTOCOL_VERSION = "2024-11-05"
+
+
+class ToolSpec:
+    """One MCP tool: public manifest fields plus the handler behind it."""
+
+    def __init__(
+        self,
+        name: str,
+        description: str,
+        input_schema: Dict[str, Any],
+        handler: Callable[[Dict[str, Any]], str],
+        aliases: Tuple[str, ...] = (),
+    ):
+        self.name = name
+        self.description = description
+        self.input_schema = input_schema
+        self.handler = handler
+        self.aliases = tuple(aliases)
+
+    def to_manifest(self) -> Dict[str, Any]:
+        return {
+            "name": self.name,
+            "description": self.description,
+            "inputSchema": self.input_schema,
+        }
+
+
+def read_message(stream: TextIO) -> Optional[Tuple[Dict[str, Any], bool]]:
+    """Read one JSON-RPC message. Returns (request, use_headers); None on EOF."""
+    while True:
+        line = stream.readline()
+        if not line:
+            return None
+        line = line.strip()
+        if not line:
+            continue
+        if line.startswith("Content-Length:"):
+            length = int(line.split(":", 1)[1].strip())
+            stream.readline()  # blank separator line
+            body = stream.read(length)
+            return json.loads(body), True
+        return json.loads(line), False
+
+
+def write_message(stream: TextIO, message: Dict[str, Any], use_headers: bool) -> None:
+    """Write one JSON-RPC message, matching the request's framing."""
+    out = json.dumps(message)
+    if use_headers:
+        stream.write(f"Content-Length: {len(out)}\r\n\r\n{out}")
+    else:
+        stream.write(out + "\n")
+    stream.flush()
+
+
+class StdioJsonRpcServer:
+    """Dispatch loop over a fixed tool registry."""
+
+    def __init__(
+        self,
+        server_name: str,
+        version: str,
+        tools: List[ToolSpec],
+        protocol_version: str = PROTOCOL_VERSION,
+        log_prefix: Optional[str] = None,
+    ):
+        self.server_name = server_name
+        self.version = version
+        self.tools = list(tools)
+        self.protocol_version = protocol_version
+        self.log_prefix = log_prefix or f"[{server_name}]"
+        self._by_name: Dict[str, ToolSpec] = {}
+        for spec in self.tools:
+            self._by_name[spec.name] = spec
+            for alias in spec.aliases:
+                self._by_name[alias] = spec
+
+    # --- request handling -------------------------------------------------
+
+    def handle(self, req: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        """Return a response dict, or None for notifications (no reply)."""
+        req_id = req.get("id")
+        method = req.get("method")
+        params = req.get("params") or {}
+
+        if method is not None and str(method).startswith("notifications/"):
+            return None
+
+        if method == "initialize":
+            return self._reply(
+                req_id,
+                {
+                    "protocolVersion": self.protocol_version,
+                    "serverInfo": {
+                        "name": self.server_name,
+                        "version": self.version,
+                    },
+                    "capabilities": {"tools": {}},
+                },
+            )
+        if method == "tools/list":
+            return self._reply(
+                req_id, {"tools": [t.to_manifest() for t in self.tools]}
+            )
+        if method == "tools/call":
+            return self._call(req_id, params)
+        return self._error(req_id, -32601, f"Method {method} not handled")
+
+    def _call(self, req_id: Any, params: Dict[str, Any]) -> Dict[str, Any]:
+        tool_name = params.get("name")
+        args = params.get("arguments") or {}
+        spec = self._by_name.get(tool_name)
+        if spec is None:
+            return self._error(req_id, -32601, f"Tool {tool_name} not found")
+        try:
+            text = spec.handler(args)
+        except Exception as exc:  # noqa: BLE001 - surface, do not kill the server
+            return self._error(req_id, -32603, f"{type(exc).__name__}: {exc}")
+        return self._reply(
+            req_id, {"content": [{"type": "text", "text": text}]}
+        )
+
+    @staticmethod
+    def _reply(req_id: Any, result: Dict[str, Any]) -> Dict[str, Any]:
+        return {"jsonrpc": "2.0", "id": req_id, "result": result}
+
+    @staticmethod
+    def _error(req_id: Any, code: int, message: str) -> Dict[str, Any]:
+        return {
+            "jsonrpc": "2.0",
+            "id": req_id,
+            "error": {"code": code, "message": message},
+        }
+
+    # --- stdio loop -------------------------------------------------------
+
+    def serve_forever(
+        self,
+        stdin: Optional[TextIO] = None,
+        stdout: Optional[TextIO] = None,
+        stderr: Optional[TextIO] = None,
+    ) -> None:
+        stdin = stdin if stdin is not None else sys.stdin
+        stdout = stdout if stdout is not None else sys.stdout
+        stderr = stderr if stderr is not None else sys.stderr
+        while True:
+            try:
+                parsed = read_message(stdin)
+                if parsed is None:
+                    break
+                req, use_headers = parsed
+                res = self.handle(req)
+                if res is not None:
+                    write_message(stdout, res, use_headers)
+            except json.JSONDecodeError as exc:
+                stderr.write(f"{self.log_prefix} Skipping malformed message: {exc}\n")
+                stderr.flush()
+                continue
+            except Exception as exc:  # noqa: BLE001 - stream desync ends the loop
+                stderr.write(f"{self.log_prefix} Error handling request: {exc}\n")
+                stderr.flush()
+                break