create-egregore 0.20.2 → 0.21.0-runtime-mvp.2

package/runtime/pi/egregore_runtime/readiness.py

@@ -0,0 +1,379 @@
+"""Runtime MVP readiness surface.
+
+One staged, truthful view of what the local Runtime can do right now:
+trust boundary, authorization policy, pinned QMD, reranking posture, BM25,
+embeddings, the semantic build revision versus the canonical source
+revision, and whether search stays usable while vectors rebuild.
+
+Truth sources are Runtime state — ``RetrieverHealth``, a ``PolicyDecision``
+from the actor's authorizer, and the session boundary state computed at
+session start. Nothing is inferred from filenames, process names, or
+hardcoded organizations, no percentage progress is invented (QMD does not
+provide one), and building this report performs no network call.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Mapping
+
+from .contracts import Permission, PolicyDecision, RetrieverHealth
+from .adapters.qmd import RERANKING_ENABLED
+
+STAGE_READY = "ready"
+STAGE_BUILDING = "building"
+STAGE_UNAVAILABLE = "unavailable"
+STAGE_DISABLED = "disabled"
+
+_GLYPHS = {
+    STAGE_READY: "●",
+    STAGE_BUILDING: "◐",
+    STAGE_UNAVAILABLE: "○",
+    STAGE_DISABLED: "–",
+}
+
+
+@dataclass(frozen=True, slots=True)
+class ReadinessItem:
+    key: str
+    label: str
+    stage: str
+    detail: str
+
+    def to_dict(self) -> dict[str, str]:
+        return {
+            "key": self.key,
+            "label": self.label,
+            "stage": self.stage,
+            "detail": self.detail,
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class ReadinessReport:
+    items: tuple[ReadinessItem, ...]
+    generated_in_ms: int
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": "egregore-readiness/v1",
+            "items": [item.to_dict() for item in self.items],
+            "generated_in_ms": self.generated_in_ms,
+        }
+
+
+def _short_revision(revision: str | None) -> str:
+    """Compress a canonical revision to an honest, comparable short form."""
+    if not revision:
+        return "none"
+    dirty = "+dirty" if "+dirty" in revision else ""
+    head = revision.split("+", 1)[0]
+    if ":" in head:
+        scheme, value = head.split(":", 1)
+        return f"{scheme}:{value[:12]}{dirty}"
+    return f"{head[:12]}{dirty}"
+
+
+def session_boundary_state(root: Path) -> Mapping[str, Any] | None:
+    """The session boundary computed at session start, if any.
+
+    The cache key mirrors the shell tooling: the hex MD5 of the project
+    path. Reading state written by the boundary machinery is the runtime
+    truth for isolation readiness; absence means no session start has
+    computed a boundary here yet.
+    """
+    digest = hashlib.md5(str(root).encode("utf-8")).hexdigest()
+    cache = Path("/tmp") / f"egregore-boundary-{digest}.json"
+    try:
+        value = json.loads(cache.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+    return value if isinstance(value, dict) else None
+
+
+def _boundary_item(root: Path, boundary: Mapping[str, Any] | None) -> ReadinessItem:
+    label = "Trust boundary"
+    if boundary is None:
+        return ReadinessItem(
+            key="boundary",
+            label=label,
+            stage=STAGE_UNAVAILABLE,
+            detail="session boundary not computed yet — run a session start",
+        )
+    project_dir = str(boundary.get("project_dir") or "")
+    if Path(project_dir) != root:
+        return ReadinessItem(
+            key="boundary",
+            label=label,
+            stage=STAGE_UNAVAILABLE,
+            detail="boundary belongs to another checkout — run a session start here",
+        )
+    denied = boundary.get("denied_paths")
+    if not isinstance(denied, list):
+        return ReadinessItem(
+            key="boundary",
+            label=label,
+            stage=STAGE_UNAVAILABLE,
+            detail="boundary state is unreadable — run a session start",
+        )
+    posture = str(boundary.get("posture") or "standard")
+    return ReadinessItem(
+        key="boundary",
+        label=label,
+        stage=STAGE_READY,
+        detail=(
+            f"posture {posture} · {len(denied)} foreign instance"
+            f"{'s' if len(denied) != 1 else ''} isolated"
+        ),
+    )
+
+
+def _authorization_item(probe: PolicyDecision | Exception | None) -> ReadinessItem:
+    label = "Authorization"
+    if probe is None:
+        return ReadinessItem(
+            key="authorization",
+            label=label,
+            stage=STAGE_UNAVAILABLE,
+            detail="no policy answered the authorization probe",
+        )
+    if isinstance(probe, Exception):
+        return ReadinessItem(
+            key="authorization",
+            label=label,
+            stage=STAGE_UNAVAILABLE,
+            detail=f"policy probe failed: {str(probe)[:120]}",
+        )
+    if not probe.allowed:
+        reasons = "; ".join(probe.reasons) or "no reason given"
+        return ReadinessItem(
+            key="authorization",
+            label=label,
+            stage=STAGE_UNAVAILABLE,
+            detail=f"read denied by policy {probe.policy_epoch}: {reasons[:120]}",
+        )
+    return ReadinessItem(
+        key="authorization",
+        label=label,
+        stage=STAGE_READY,
+        detail=f"actor authorized · policy {probe.policy_epoch}",
+    )
+
+
+def _qmd_item(health: RetrieverHealth) -> ReadinessItem:
+    label = "QMD"
+    if health.available:
+        return ReadinessItem(
+            key="qmd",
+            label=label,
+            stage=STAGE_READY,
+            detail=f"{health.adapter} {health.adapter_version} pinned",
+        )
+    reason = health.warnings[0] if health.warnings else "retrieval adapter unavailable"
+    return ReadinessItem(
+        key="qmd", label=label, stage=STAGE_UNAVAILABLE, detail=str(reason)[:140]
+    )
+
+
+def _reranker_item() -> ReadinessItem:
+    if RERANKING_ENABLED:
+        return ReadinessItem(
+            key="reranker",
+            label="Reranker",
+            stage=STAGE_READY,
+            detail="model reranking enabled",
+        )
+    return ReadinessItem(
+        key="reranker",
+        label="Reranker",
+        stage=STAGE_DISABLED,
+        detail="disabled for the MVP — typed rank fusion only",
+    )
+
+
+def _bm25_item(health: RetrieverHealth) -> ReadinessItem:
+    label = "BM25"
+    if health.bm25_ready:
+        return ReadinessItem(
+            key="bm25",
+            label=label,
+            stage=STAGE_READY,
+            detail=f"lexical index usable · {_short_revision(health.index_source_revision)}",
+        )
+    if health.available and health.canonical_state_ready:
+        return ReadinessItem(
+            key="bm25",
+            label=label,
+            stage=STAGE_BUILDING,
+            detail="lexical index registering — first update pending",
+        )
+    return ReadinessItem(
+        key="bm25",
+        label=label,
+        stage=STAGE_UNAVAILABLE,
+        detail="no lexical index — retrieval adapter is not available",
+    )
+
+
+def _embeddings_item(health: RetrieverHealth) -> ReadinessItem:
+    label = "Embeddings"
+    state = health.embedding_state
+    if state == "ready":
+        return ReadinessItem(
+            key="embeddings", label=label, stage=STAGE_READY, detail="vector build complete"
+        )
+    if state in ("running", "pending"):
+        detail = (
+            "background embed running — search does not wait for it"
+            if state == "running"
+            else "documents pending embedding — background embed scheduled"
+        )
+        return ReadinessItem(
+            key="embeddings", label=label, stage=STAGE_BUILDING, detail=detail
+        )
+    if state == "missing_model":
+        return ReadinessItem(
+            key="embeddings",
+            label=label,
+            stage=STAGE_UNAVAILABLE,
+            detail="embedding model not installed — lexical retrieval only",
+        )
+    return ReadinessItem(
+        key="embeddings",
+        label=label,
+        stage=STAGE_UNAVAILABLE,
+        detail="no vector build yet",
+    )
+
+
+def _semantic_item(health: RetrieverHealth) -> ReadinessItem:
+    label = "Semantic build"
+    source = _short_revision(health.source_revision)
+    build = _short_revision(health.semantic_source_revision)
+    if health.semantic_ready:
+        return ReadinessItem(
+            key="semantic",
+            label=label,
+            stage=STAGE_READY,
+            detail=f"vectors current at {build}",
+        )
+    if health.semantic_source_revision:
+        return ReadinessItem(
+            key="semantic",
+            label=label,
+            stage=STAGE_BUILDING,
+            detail=f"serving {build} while {source} builds",
+        )
+    return ReadinessItem(
+        key="semantic",
+        label=label,
+        stage=STAGE_UNAVAILABLE,
+        detail=f"no complete vector build yet · source {source}",
+    )
+
+
+def _search_item(health: RetrieverHealth) -> ReadinessItem:
+    label = "Search"
+    if health.bm25_ready and health.semantic_ready:
+        return ReadinessItem(
+            key="search", label=label, stage=STAGE_READY, detail="hybrid current"
+        )
+    if health.bm25_ready and health.semantic_source_revision:
+        return ReadinessItem(
+            key="search",
+            label=label,
+            stage=STAGE_READY,
+            detail="hybrid stays available while embeddings rebuild",
+        )
+    if health.bm25_ready:
+        return ReadinessItem(
+            key="search",
+            label=label,
+            stage=STAGE_READY,
+            detail="lexical available until the first vector build completes",
+        )
+    reason = health.warnings[0] if health.warnings else "retrieval is not available"
+    return ReadinessItem(
+        key="search", label=label, stage=STAGE_UNAVAILABLE, detail=str(reason)[:140]
+    )
+
+
+def build_report(
+    *,
+    root: Path,
+    health: RetrieverHealth,
+    boundary: Mapping[str, Any] | None,
+    authorization: PolicyDecision | Exception | None,
+    started_monotonic: float | None = None,
+) -> ReadinessReport:
+    """Assemble the staged report from injected Runtime state (deterministic)."""
+    started = started_monotonic if started_monotonic is not None else time.monotonic()
+    items = (
+        _boundary_item(root, boundary),
+        _authorization_item(authorization),
+        _qmd_item(health),
+        _reranker_item(),
+        _bm25_item(health),
+        _embeddings_item(health),
+        _semantic_item(health),
+        _search_item(health),
+    )
+    elapsed_ms = max(0, round((time.monotonic() - started) * 1000))
+    return ReadinessReport(items=items, generated_in_ms=elapsed_ms)
+
+
+def collect_report(root: Path, runtime, *, harness: str, session_id: str) -> ReadinessReport:
+    """Gather live Runtime state and assemble the report. Local I/O only."""
+    started = time.monotonic()
+    health = runtime.retrieval_health()
+    boundary = session_boundary_state(root)
+    authorization: PolicyDecision | Exception | None
+    try:
+        actor = runtime.resolve_actor(session_id=session_id, harness=harness)
+        authorization = runtime.authorize(actor, Permission.READ)
+    except Exception as exc:  # readiness must report failures, not raise them
+        authorization = exc
+    return build_report(
+        root=root,
+        health=health,
+        boundary=boundary,
+        authorization=authorization,
+        started_monotonic=started,
+    )
+
+
+def render(report: ReadinessReport, *, tty: bool) -> str:
+    """Render the staged report for humans. No raw JSON, no command traces."""
+    lines: list[str] = []
+    if tty:
+        lines.append("Egregore Runtime readiness")
+        lines.append("┄" * 40)
+    else:
+        lines.append("egregore runtime readiness")
+    width = max(len(item.label) for item in report.items)
+    for item in report.items:
+        if tty:
+            glyph = _GLYPHS.get(item.stage, "○")
+            lines.append(f"  {glyph} {item.label:<{width}}  {item.stage:<11} {item.detail}")
+        else:
+            lines.append(f"{item.key}: {item.stage} — {item.detail}")
+    lines.append(f"rendered in {report.generated_in_ms} ms · local state only, no network")
+    return "\n".join(lines)
+
+
+__all__ = [
+    "STAGE_BUILDING",
+    "STAGE_DISABLED",
+    "STAGE_READY",
+    "STAGE_UNAVAILABLE",
+    "ReadinessItem",
+    "ReadinessReport",
+    "build_report",
+    "collect_report",
+    "render",
+    "session_boundary_state",
+]

package/runtime/pi/egregore_runtime/research_ingest.py

@@ -0,0 +1,256 @@
+"""Typed canonical write boundary for meetings and user interviews."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path, PurePosixPath
+from typing import Any, Mapping, Sequence
+
+from .artifacts import ARTIFACT_SCHEMA_VERSION, CanonicalDocument, artifact_id_from_path, content_digest
+from .contracts import (
+    ActorContext,
+    ArtifactRelationship,
+    CanonicalArtifact,
+    Permission,
+    SourceProvenance,
+    WritebackStatus,
+)
+from .runtime import EgregoreRuntime
+from .policy import scope_allows_path
+
+
+RESEARCH_INGEST_SCHEMA_VERSION = "egregore-research-ingest/v1"
+KINDS = frozenset({"meeting", "interview"})
+ARTIFACT_TYPES = frozenset(
+    {"meeting", "interview", "participant", "decision", "finding", "pattern"}
+)
+MAX_DOCUMENTS = 40
+MAX_BODY_BYTES = 250_000
+
+
+class ResearchIngestError(ValueError):
+    """An analysis package violates the canonical research contract."""
+
+
+@dataclass(frozen=True, slots=True)
+class ResearchIngestReceipt:
+    kind: str
+    source_id: str
+    status: str
+    artifacts: tuple[dict[str, Any], ...]
+    warnings: tuple[str, ...]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": RESEARCH_INGEST_SCHEMA_VERSION,
+            "kind": self.kind,
+            "source_id": self.source_id,
+            "status": self.status,
+            "artifacts": list(self.artifacts),
+            "warnings": list(self.warnings),
+        }
+
+
+def _required(mapping: Mapping[str, Any], key: str) -> str:
+    value = str(mapping.get(key) or "").strip()
+    if not value:
+        raise ResearchIngestError(f"{key} is required")
+    return value
+
+
+def _canonical_path(kind: str, value: str, artifact_type: str) -> str:
+    raw = value.strip().replace("\\", "/").removeprefix("memory/")
+    path = PurePosixPath(raw)
+    if path.is_absolute() or ".." in path.parts or path.suffix != ".md":
+        raise ResearchIngestError("canonical_path must be a relative Markdown path")
+    allowed = (
+        ("meetings/", "knowledge/decisions/", "knowledge/findings/", "knowledge/patterns/")
+        if kind == "meeting"
+        else (
+            "research/interviews/",
+            "research/participants/",
+            "knowledge/findings/",
+            "knowledge/patterns/",
+            "knowledge/decisions/",
+        )
+    )
+    if not raw.startswith(allowed):
+        raise ResearchIngestError(f"canonical_path is outside the {kind} research boundary")
+    if artifact_type == "meeting" and not raw.startswith("meetings/"):
+        raise ResearchIngestError("meeting briefing must be stored under meetings/")
+    if artifact_type == "interview" and not raw.startswith("research/interviews/"):
+        raise ResearchIngestError("interview briefing must be stored under research/interviews/")
+    if artifact_type == "participant" and not raw.startswith("research/participants/"):
+        raise ResearchIngestError("participant record must be stored under research/participants/")
+    return raw
+
+
+def _relationships(values: Any) -> tuple[ArtifactRelationship, ...]:
+    if values is None:
+        return ()
+    if not isinstance(values, Sequence) or isinstance(values, (str, bytes)):
+        raise ResearchIngestError("relationships must be a list")
+    rows = []
+    for item in values:
+        if not isinstance(item, Mapping):
+            raise ResearchIngestError("relationship entries must be objects")
+        rows.append(
+            ArtifactRelationship(
+                relation=_required(item, "relation"),
+                target_id=_required(item, "target_id"),
+            )
+        )
+    return tuple(rows)
+
+
+class CanonicalResearchIngestService:
+    """Validate metadata, authorize, then inspect and persist analysis content."""
+
+    def __init__(self, runtime: EgregoreRuntime, *, clock=None, content_loader=None) -> None:
+        self.runtime = runtime
+        self.clock = clock or (lambda: datetime.now(UTC))
+        self.content_loader = content_loader or (
+            lambda body_path: Path(body_path).read_text(encoding="utf-8")
+        )
+
+    def ingest(
+        self,
+        actor: ActorContext,
+        package: Mapping[str, Any],
+        *,
+        expected_kind: str | None = None,
+    ) -> ResearchIngestReceipt:
+        schema = _required(package, "schema_version")
+        if schema != RESEARCH_INGEST_SCHEMA_VERSION:
+            raise ResearchIngestError(f"unsupported schema_version: {schema}")
+        kind = _required(package, "kind").casefold()
+        if kind not in KINDS or (expected_kind and kind != expected_kind):
+            raise ResearchIngestError("research package kind does not match the requested adapter")
+        source = package.get("source")
+        if not isinstance(source, Mapping):
+            raise ResearchIngestError("source must be an object")
+        source_type = _required(source, "type")
+        source_id = _required(source, "id")
+        source_revision = _required(source, "revision")
+        source_content_hash = _required(source, "content_hash")
+        documents = package.get("documents")
+        if not isinstance(documents, Sequence) or isinstance(documents, (str, bytes)):
+            raise ResearchIngestError("documents must be a list")
+        if not documents or len(documents) > MAX_DOCUMENTS:
+            raise ResearchIngestError(f"documents must contain 1-{MAX_DOCUMENTS} entries")
+
+        # Metadata is intentionally validated before any body is accessed.
+        metadata: list[tuple[Mapping[str, Any], str, str, str, str, str]] = []
+        resources = []
+        for raw in documents:
+            if not isinstance(raw, Mapping):
+                raise ResearchIngestError("document entries must be objects")
+            artifact_type = _required(raw, "artifact_type").casefold()
+            if artifact_type not in ARTIFACT_TYPES:
+                raise ResearchIngestError(f"unsupported research artifact_type: {artifact_type}")
+            title = _required(raw, "title")
+            status = str(raw.get("status") or "active").strip()
+            body_path = _required(raw, "body_path")
+            canonical_path = _canonical_path(
+                kind, _required(raw, "canonical_path"), artifact_type
+            )
+            artifact_id = str(raw.get("id") or artifact_id_from_path(canonical_path)).strip()
+            metadata.append((raw, artifact_type, title, status, canonical_path, body_path))
+            resources.append(artifact_id)
+
+        decision = self.runtime.authorize(actor, Permission.WRITE, tuple(resources))
+        if not decision.allowed:
+            raise PermissionError("; ".join(decision.reasons) or "research ingest denied")
+
+        for _, _, _, _, canonical_path, _ in metadata:
+            if decision.scopes and not scope_allows_path(
+                f"memory/{canonical_path}", decision.scopes
+            ):
+                raise PermissionError(
+                    "research artifact path is outside the actor's authorized scope"
+                )
+
+        now = self.clock().astimezone(UTC)
+        provenance = SourceProvenance(
+            source_type=source_type,
+            source_id=source_id,
+            revision=source_revision,
+            content_hash=source_content_hash,
+            observed_at=now,
+            source_uri=str(source.get("uri")) if source.get("uri") else None,
+            imported_by=actor.actor.actor_id,
+        )
+        prepared_documents: list[CanonicalDocument] = []
+        for raw, artifact_type, title, status, canonical_path, body_path in metadata:
+            body = self.content_loader(body_path)
+            if not isinstance(body, str) or not body.strip():
+                raise ResearchIngestError("document body is required")
+            if len(body.encode("utf-8")) > MAX_BODY_BYTES:
+                raise ResearchIngestError("document body exceeds the research ingest limit")
+            digest = content_digest(body)
+            artifact_id = str(raw.get("id") or artifact_id_from_path(canonical_path)).strip()
+            artifact = CanonicalArtifact(
+                schema_version=ARTIFACT_SCHEMA_VERSION,
+                artifact_id=artifact_id,
+                org_id=actor.profile.org_id,
+                artifact_type=artifact_type,
+                title=title,
+                created_at=now,
+                created_by=actor.actor.actor_id,
+                status=status,
+                canonical_path=canonical_path,
+                revision=f"sha256:{digest[:16]}",
+                content_hash=digest,
+                workstream=str(raw.get("workstream")) if raw.get("workstream") else None,
+                visibility=tuple(str(item) for item in (raw.get("visibility") or ())),
+                relationships=_relationships(raw.get("relationships")),
+                supersedes=tuple(str(item) for item in (raw.get("supersedes") or ())),
+                provenance=(provenance,),
+            )
+            legacy_fields = dict(raw.get("metadata") or {})
+            legacy_fields.update(
+                {
+                    "research_ingest_schema": RESEARCH_INGEST_SCHEMA_VERSION,
+                    "research_kind": kind,
+                    "source_type": source_type,
+                    "source_id": source_id,
+                    "source_revision": source_revision,
+                }
+            )
+            prepared_documents.append(
+                CanonicalDocument(
+                    artifact=artifact,
+                    body=body,
+                    policy_hints=dict(raw.get("policy_hints") or {}),
+                    legacy_fields=legacy_fields,
+                )
+            )
+
+        receipts = self.runtime.write_documents(actor, tuple(prepared_documents))
+        rows = []
+        warnings: list[str] = []
+        for receipt in receipts:
+            rows.append(
+                {
+                    "artifact_id": receipt.artifact.artifact_id,
+                    "canonical_path": f"memory/{receipt.artifact.canonical_path}",
+                    "status": receipt.status.value,
+                    "git_revision": receipt.git_revision,
+                    "index_revision": receipt.index_revision,
+                    "embedding_state": receipt.embedding_state,
+                }
+            )
+            warnings.extend(
+                warning for warning in receipt.warnings if warning not in warnings
+            )
+
+        statuses = {row["status"] for row in rows}
+        overall = (
+            WritebackStatus.REJECTED.value
+            if statuses == {WritebackStatus.REJECTED.value}
+            else WritebackStatus.PARTIAL.value
+            if warnings or WritebackStatus.REJECTED.value in statuses or WritebackStatus.PARTIAL.value in statuses
+            else WritebackStatus.ACCEPTED.value
+        )
+        return ResearchIngestReceipt(kind, source_id, overall, tuple(rows), tuple(warnings))

package/runtime/pi/egregore_runtime/research_ingest_cli.py

@@ -0,0 +1,44 @@
+"""Shell adapter for canonical meeting and interview writeback."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+import sys
+
+from .research_ingest import CanonicalResearchIngestService, ResearchIngestError
+from .runtime import local_runtime
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(prog="egregore-research-ingest")
+    parser.add_argument("kind", choices=("meeting", "interview"))
+    parser.add_argument("--input", required=True, help="analysis package JSON")
+    parser.add_argument("--no-push", action="store_true")
+    args = parser.parse_args()
+    root = Path(os.environ.get("EGREGORE_ROOT") or Path.cwd()).resolve()
+    runtime = local_runtime(root, push_remote=not args.no_push)
+    session_path = root / ".egregore-session-id"
+    session_id = os.environ.get("EGREGORE_SESSION_ID")
+    if not session_id and session_path.is_file():
+        session_id = session_path.read_text(encoding="utf-8").splitlines()[0].strip()
+    actor = runtime.resolve_actor(
+        session_id=session_id or "research-ingest-shell",
+        harness=os.environ.get("EGREGORE_RUNTIME", "shell"),
+    )
+    package = json.loads(Path(args.input).read_text(encoding="utf-8"))
+    receipt = CanonicalResearchIngestService(runtime).ingest(
+        actor, package, expected_kind=args.kind
+    )
+    print(json.dumps(receipt.to_dict(), ensure_ascii=False, separators=(",", ":")))
+    return 0 if receipt.status in {"accepted", "partial"} else 1
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except (json.JSONDecodeError, OSError, PermissionError, ResearchIngestError) as exc:
+        print(f"research-ingest: {exc}", file=sys.stderr)
+        raise SystemExit(1) from exc