create-egregore 0.20.2 → 0.21.0-runtime-mvp.1

package/runtime/claude/egregore_runtime/observe.py

@@ -0,0 +1,342 @@
+"""Organizational context compilation independent of harness/infrastructure."""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, replace
+from datetime import UTC, datetime
+from uuid import uuid4
+
+from .contracts import (
+    ActionAuthorizer,
+    ActorContext,
+    EvidenceItem,
+    OrgContext,
+    OrgContextCompiler,
+    Permission,
+    RetrievalHit,
+    RetrievalRequest,
+    Retriever,
+    TelemetryEvent,
+    TelemetrySink,
+    TemporalScope,
+)
+from .admin_gate import AdminGate
+from .errors import AuthorizationDenied, ContextBudgetError
+from .lifecycle import AuthorityState, CanonicalLifecycleService, LifecycleRecord
+from .policy import scope_allows_path
+
+
+def _token_count(content: str) -> int:
+    # Deterministic, dependency-free planning estimate. Harness adapters may
+    # supply a tokenizer later without weakening the authorization boundary.
+    return max(1, (len(content) + 3) // 4)
+
+
+def _truncate_to_budget(content: str, budget: int) -> str:
+    return content[: max(0, budget * 4)]
+
+
+@dataclass(slots=True)
+class DefaultOrgContextCompiler(OrgContextCompiler):
+    authorizer: ActionAuthorizer
+    retriever: Retriever
+    telemetry: TelemetrySink
+    lifecycle: CanonicalLifecycleService | None = None
+    admin_gate: AdminGate | None = None
+
+    def observe(
+        self,
+        actor: ActorContext,
+        request: RetrievalRequest,
+        *,
+        token_budget: int,
+    ) -> OrgContext:
+        if token_budget <= 0:
+            raise ContextBudgetError("token budget must be positive")
+
+        started = time.monotonic()
+        decision = self.authorizer.authorize_observe(actor, request)
+        if not decision.allowed:
+            self._emit(
+                actor,
+                request,
+                "observe.denied",
+                metrics={"success": False, "failure_code": "authorization_denied"},
+            )
+            raise AuthorizationDenied("; ".join(decision.reasons))
+
+        scopes = tuple(
+            scope
+            for scope in (request.authorized_scopes or decision.scopes)
+            if not decision.scopes or scope_allows_path(scope, decision.scopes)
+        )
+        if not scopes:
+            raise AuthorizationDenied("policy produced no authorized retrieval scope")
+
+        allowed_ids = request.allowed_artifact_ids
+        if decision.resource_ids:
+            allowed_ids = tuple(
+                artifact_id
+                for artifact_id in (allowed_ids or decision.resource_ids)
+                if artifact_id in decision.resource_ids
+            )
+            if not allowed_ids:
+                raise AuthorizationDenied("policy produced no authorized artifact set")
+
+        authorized_request = replace(
+            request,
+            authorized_scopes=scopes,
+            allowed_artifact_ids=allowed_ids,
+        )
+        result = self.retriever.retrieve(authorized_request)
+
+        # Defense in depth: an adapter returning an out-of-scope candidate
+        # cannot place it in model context.
+        authorized_hits = tuple(
+            hit
+            for hit in result.hits
+            if scope_allows_path(hit.canonical_path, scopes)
+            and (not allowed_ids or hit.artifact_id in allowed_ids)
+        )
+
+        # Egregore-surface admin protection: QMD may match an admin-marked
+        # document internally, but for a non-admin actor it must contribute
+        # nothing to evidence — no title, path, snippet, count detail, or
+        # model context. Only an opaque withheld tally survives.
+        withheld_admin = 0
+        if self.admin_gate is not None and not self.admin_gate.actor_is_admin(actor):
+            visible_hits = []
+            for hit in authorized_hits:
+                if self.admin_gate.path_is_admin(hit.canonical_path):
+                    withheld_admin += 1
+                else:
+                    visible_hits.append(hit)
+            authorized_hits = tuple(visible_hits)
+
+        lifecycle_by_path: dict[str, LifecycleRecord] = {}
+        lifecycle_revision = "unavailable"
+        lifecycle_warning: str | None = None
+        if self.lifecycle is not None:
+            try:
+                resolved, lifecycle_revision = self.lifecycle.resolve_paths(
+                    tuple(hit.canonical_path for hit in authorized_hits),
+                    source_revision=result.source_revision,
+                    authorized_scopes=scopes,
+                    allowed_artifact_ids=allowed_ids,
+                )
+                lifecycle_by_path = dict(resolved)
+            except Exception as exc:
+                lifecycle_warning = f"lifecycle-unavailable:{type(exc).__name__}"
+
+        # Lifecycle events are organizational receipts, not normal answer
+        # evidence. Current authority is selected only after authorization and
+        # before any source is opened.
+        selected_hits: list[RetrievalHit] = []
+        for hit in authorized_hits:
+            record = lifecycle_by_path.get(hit.canonical_path)
+            if record is not None and record.artifact_type == "lifecycle-event":
+                if "lifecycle-event" not in request.artifact_types:
+                    continue
+            if record is not None:
+                historical = record.authority_state in {
+                    AuthorityState.SUPERSEDED,
+                    AuthorityState.HISTORICAL,
+                }
+                if request.temporal_scope is TemporalScope.CURRENT and historical:
+                    continue
+                if request.temporal_scope is TemporalScope.HISTORICAL and not historical:
+                    continue
+            observed_at = hit.observed_at or (record.created_at if record is not None else None)
+            if request.not_before is not None:
+                if observed_at is None:
+                    continue
+                if observed_at.tzinfo is None:
+                    observed_at = observed_at.replace(tzinfo=UTC)
+                if observed_at.astimezone(UTC) < request.not_before.astimezone(UTC):
+                    continue
+            selected_hits.append(hit)
+
+        selected_hits.sort(
+            key=lambda hit: (
+                1
+                if lifecycle_by_path.get(hit.canonical_path)
+                and lifecycle_by_path[hit.canonical_path].authority_state
+                is AuthorityState.AMBIGUOUS
+                else 2
+                if lifecycle_by_path.get(hit.canonical_path)
+                and lifecycle_by_path[hit.canonical_path].authority_state
+                in {AuthorityState.SUPERSEDED, AuthorityState.HISTORICAL}
+                else 0,
+                hit.rank,
+            )
+        )
+
+        evidence, opened_ids, omissions, tokens_used = self._package(
+            tuple(selected_hits),
+            request,
+            token_budget,
+            lifecycle_by_path,
+        )
+        if lifecycle_warning:
+            omissions = (*omissions, lifecycle_warning)
+        if withheld_admin:
+            # Tag-not-hide: existence is visible, identity is not. No id,
+            # title, or path of the withheld documents may appear here.
+            omissions = (*omissions, f"[admin] {withheld_admin} admin-marked item(s) withheld")
+        permissions = frozenset(
+            permission
+            for permission in Permission
+            if self.authorizer.authorize_action(actor, permission).allowed
+        )
+        context = OrgContext(
+            schema_version="egregore-context/v1",
+            context_id=f"ctx_{uuid4().hex}",
+            org_id=actor.profile.org_id,
+            actor_id=actor.actor.actor_id,
+            task=request.task,
+            profile_revision=actor.profile.revision,
+            source_revision=result.source_revision,
+            policy_epoch=decision.policy_epoch,
+            evidence=evidence,
+            action_permissions=permissions,
+            token_budget=token_budget,
+            tokens_used=tokens_used,
+            omissions=omissions,
+            freshness={
+                "source_revision": result.source_revision,
+                "index_revision": result.index_revision,
+                "index_spec_version": result.index_spec_version,
+                "lifecycle_revision": lifecycle_revision,
+                "temporal_scope": request.temporal_scope.value,
+                "temporal_authority": "canonical-markdown",
+            },
+        )
+        elapsed_ms = int((time.monotonic() - started) * 1000)
+        self._emit(
+            actor,
+            request,
+            "observe.completed",
+            # Artifact ids retain retrieval order, so array position is the
+            # privacy-safe rank without creating a content-bearing map.
+            artifact_ids=tuple(hit.artifact_id for hit in selected_hits),
+            metrics={
+                "duration_ms": elapsed_ms,
+                "latency_ms": result.latency_ms,
+                "count": len(selected_hits),
+                "opened_count": len(opened_ids),
+                "result": "context_produced",
+                "token_count": tokens_used,
+                "retrieval_type": result.mode.value,
+                "success": True,
+            },
+            index_spec_version=result.index_spec_version,
+        )
+        if opened_ids:
+            self._emit(
+                actor,
+                request,
+                "source.opened",
+                artifact_ids=opened_ids,
+                metrics={"opened_count": len(opened_ids)},
+                index_spec_version=result.index_spec_version,
+            )
+        return context
+
+    def _package(
+        self,
+        hits: tuple[RetrievalHit, ...],
+        request: RetrievalRequest,
+        token_budget: int,
+        lifecycle_by_path: dict[str, LifecycleRecord] | None = None,
+    ) -> tuple[tuple[EvidenceItem, ...], tuple[str, ...], tuple[str, ...], int]:
+        evidence: list[EvidenceItem] = []
+        opened_ids: list[str] = []
+        omissions: list[str] = []
+        remaining = token_budget
+
+        for hit in hits:
+            if remaining <= 0:
+                omissions.append(f"budget:{hit.artifact_id}")
+                continue
+            if request.open_sources:
+                content = self.retriever.open_source(hit)
+                opened_ids.append(hit.artifact_id)
+            else:
+                content = hit.passage or ""
+            if not content:
+                omissions.append(f"empty:{hit.artifact_id}")
+                continue
+
+            count = _token_count(content)
+            if count > remaining:
+                content = _truncate_to_budget(content, remaining)
+                count = _token_count(content)
+                omissions.append(f"truncated:{hit.artifact_id}")
+            evidence.append(
+                EvidenceItem(
+                    artifact_id=hit.artifact_id,
+                    canonical_path=hit.canonical_path,
+                    revision=hit.revision or "unknown",
+                    reason=self._evidence_reason(
+                        hit,
+                        (lifecycle_by_path or {}).get(hit.canonical_path),
+                    ),
+                    content=content,
+                    token_count=count,
+                )
+            )
+            remaining -= count
+
+        return (
+            tuple(evidence),
+            tuple(opened_ids),
+            tuple(omissions),
+            token_budget - remaining,
+        )
+
+    @staticmethod
+    def _evidence_reason(hit: RetrievalHit, lifecycle: LifecycleRecord | None) -> str:
+        observed_at = hit.observed_at or (lifecycle.created_at if lifecycle is not None else None)
+        observed = (
+            f"; observed={observed_at.date().isoformat()}" if observed_at is not None else ""
+        )
+        if lifecycle is None:
+            return f"retrieval rank {hit.rank}; temporal authority unknown{observed}"
+        return (
+            f"retrieval rank {hit.rank}; authority={lifecycle.authority_state.value}; "
+            f"attention={lifecycle.attention_state.value}; "
+            f"obligation={lifecycle.obligation_state.value}{observed}"
+        )
+
+    def _emit(
+        self,
+        actor: ActorContext,
+        request: RetrievalRequest,
+        event_type: str,
+        *,
+        artifact_ids: tuple[str, ...] = (),
+        metrics: dict[str, int | float | bool | str | None] | None = None,
+        index_spec_version: str | None = None,
+    ) -> None:
+        try:
+            self.telemetry.emit(
+                TelemetryEvent(
+                    schema_version="egregore.telemetry/v1",
+                    event_id=f"evt_{uuid4().hex}",
+                    event_type=event_type,
+                    occurred_at=datetime.now(UTC),
+                    org_id=actor.profile.org_id,
+                    actor_id=actor.actor.actor_id,
+                    session_id=actor.session_id,
+                    task_id=request.request_id,
+                    org_revision=actor.profile.revision,
+                    index_spec_version=index_spec_version,
+                    metrics=metrics or {},
+                    artifact_ids=artifact_ids,
+                )
+            )
+        except Exception:
+            # Telemetry is an observability projection. It cannot make a valid
+            # authorization/context result fail.
+            return

package/runtime/claude/egregore_runtime/onboarding.py

@@ -0,0 +1,290 @@
+"""Small resumable onboarding model and authorized state service."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+import json
+import os
+from pathlib import Path
+import re
+import tempfile
+from typing import Any, Mapping
+
+from .contracts import ActorContext, Permission, Serializable
+from .errors import AuthorizationDenied
+
+
+ONBOARDING_SCHEMA = "egregore-onboarding/v1"
+
+
+@dataclass(frozen=True, slots=True)
+class OnboardingStep:
+    step_id: str
+    required_fields: tuple[str, ...]
+    next_step: str | None = None
+    branches: Mapping[str, str] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True)
+class OnboardingPlan:
+    schema_version: str
+    intent: str
+    org_type: str
+    start_step: str
+    steps: tuple[OnboardingStep, ...]
+
+    def step(self, step_id: str) -> OnboardingStep:
+        for item in self.steps:
+            if item.step_id == step_id:
+                return item
+        raise KeyError(step_id)
+
+    def advance(self, step_id: str, *, choice: str | None = None) -> str | None:
+        current = self.step(step_id)
+        if choice is not None and choice in current.branches:
+            return current.branches[choice]
+        return current.next_step
+
+
+def onboarding_plan(*, intent: str = "collaborate", org_type: str = "team") -> OnboardingPlan:
+    """Return today's short flow represented as an evolvable decision tree."""
+
+    normalized_intent = intent.strip().lower() or "collaborate"
+    normalized_type = org_type.strip().lower() or "team"
+    return OnboardingPlan(
+        schema_version=ONBOARDING_SCHEMA,
+        intent=normalized_intent,
+        org_type=normalized_type,
+        start_step="identity",
+        steps=(
+            OnboardingStep("identity", ("display_name",), next_step="organization"),
+            OnboardingStep(
+                "organization",
+                ("org_name", "org_type", "intent"),
+                next_step="role",
+            ),
+            OnboardingStep("role", ("member_role",), next_step="workspace"),
+            OnboardingStep(
+                "workspace",
+                ("runtime",),
+                branches={"founder": "invite", "joiner": "complete"},
+                next_step="complete",
+            ),
+            OnboardingStep("invite", (), next_step="complete"),
+            OnboardingStep("complete", (), next_step=None),
+        ),
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class OnboardingSnapshot(Serializable):
+    schema_version: str
+    phase: str
+    complete: bool
+    mode: str
+    usage_type: str
+    display_name: str | None
+    member_role: str | None
+    profile_fields_collected: tuple[str, ...]
+    invite_skipped: bool | None
+    installer_captured: bool
+    invite_handled_by: str | None
+    profile_witness: bool
+    ready: bool
+    blockers: tuple[str, ...]
+
+
+class OnboardingService:
+    """Own resumable onboarding state behind ActorContext authorization."""
+
+    LEGACY_PHASES = frozenset(
+        {"welcome", "harvest_identity", "harvest_connection", "consent", "first_todo"}
+    )
+
+    def __init__(self, root: Path | str, runtime) -> None:
+        self.root = Path(root).resolve()
+        self.runtime = runtime
+        self.state_path = self.root / ".egregore-state.json"
+
+    def _authorize(self, actor: ActorContext, permission: Permission) -> None:
+        resource = f"identity:onboarding:{actor.actor.actor_id}"
+        decision = self.runtime.authorize(actor, permission, (resource,))
+        if not decision.allowed:
+            raise AuthorizationDenied("; ".join(decision.reasons))
+
+    def _load(self) -> dict[str, Any]:
+        try:
+            value = json.loads(self.state_path.read_text(encoding="utf-8"))
+        except FileNotFoundError:
+            value = {}
+        if not isinstance(value, dict):
+            raise ValueError("onboarding state must contain an object")
+        return value
+
+    def _write(self, state: Mapping[str, Any]) -> None:
+        target = self.state_path.resolve() if self.state_path.is_symlink() else self.state_path
+        target.parent.mkdir(parents=True, exist_ok=True)
+        descriptor, temporary = tempfile.mkstemp(prefix=f"{target.name}.", dir=target.parent)
+        try:
+            with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+                json.dump(dict(state), handle, ensure_ascii=False, indent=2)
+                handle.write("\n")
+            os.chmod(temporary, 0o600)
+            os.replace(temporary, target)
+        finally:
+            if os.path.exists(temporary):
+                os.unlink(temporary)
+
+    @staticmethod
+    def _fields(state: Mapping[str, Any]) -> list[str]:
+        values = state.get("profile_fields_collected") or []
+        return [str(value) for value in values if str(value)] if isinstance(values, list) else []
+
+    def _has_secret(self, name: str) -> bool:
+        if os.environ.get(name, "").strip():
+            return True
+        env_path = self.root / ".env"
+        if not env_path.is_file():
+            return False
+        prefix = f"{name}="
+        return any(
+            line.startswith(prefix) and bool(line[len(prefix) :].strip())
+            for line in env_path.read_text(encoding="utf-8").splitlines()
+        )
+
+    def _migrate(self, state: dict[str, Any]) -> tuple[dict[str, Any], bool]:
+        onboarding = dict(state.get("onboarding") or {})
+        phase = str(onboarding.get("phase") or "verify")
+        changed = False
+        if phase in self.LEGACY_PHASES:
+            onboarding["phase"] = "invite" if state.get("display_name") else "orient"
+            fields = self._fields(state)
+            if state.get("display_name") and "name" not in fields:
+                fields.append("name")
+            state["profile_fields_collected"] = fields
+            changed = True
+        if onboarding.get("schema_version") != ONBOARDING_SCHEMA:
+            onboarding["schema_version"] = ONBOARDING_SCHEMA
+            changed = True
+        state["onboarding"] = onboarding
+        return state, changed
+
+    def snapshot(self, actor: ActorContext) -> OnboardingSnapshot:
+        self._authorize(actor, Permission.READ)
+        # Reads may normalize legacy phases in the returned view, but only an
+        # ADMINISTER-authorized mutation is allowed to persist that migration.
+        state, _ = self._migrate(self._load())
+        config = json.loads((self.root / "egregore.json").read_text(encoding="utf-8"))
+        onboarding = dict(state.get("onboarding") or {})
+        username = str(state.get("github_username") or "")
+        profile = self.root / "memory" / "people" / f"{username}.md"
+        profile_lines = (
+            profile.read_text(encoding="utf-8").splitlines() if profile.is_file() else []
+        )
+        witness = bool(profile_lines) and (
+            profile_lines[0].startswith("# ")
+            or any(line.startswith("Onboarded:") for line in profile_lines)
+        )
+        connected = bool(config.get("api_url")) or config.get("mode") == "connected"
+        blockers: list[str] = []
+        if not (self.root / "memory").exists():
+            blockers.append("shared memory is unavailable; run the normal Egregore sync")
+        if not self._has_secret("GITHUB_TOKEN"):
+            blockers.append("GitHub authentication is missing; run bash bin/github-auth.sh")
+        if connected and not self._has_secret("EGREGORE_API_KEY"):
+            blockers.append("Connected authentication is missing")
+        return OnboardingSnapshot(
+            schema_version=ONBOARDING_SCHEMA,
+            phase=str(onboarding.get("phase") or "verify"),
+            complete=bool(state.get("onboarding_complete")),
+            mode=(
+                "connected"
+                if connected
+                else "local"
+            ),
+            usage_type=str(state.get("usage_type") or "joiner_group"),
+            display_name=str(state.get("display_name")) if state.get("display_name") else None,
+            member_role=str(state.get("member_role")) if state.get("member_role") else None,
+            profile_fields_collected=tuple(self._fields(state)),
+            invite_skipped=(
+                bool(onboarding["invite_skipped"])
+                if "invite_skipped" in onboarding else None
+            ),
+            installer_captured=bool(onboarding.get("installer_captured")),
+            invite_handled_by=(
+                str(onboarding["invite_handled_by"])
+                if onboarding.get("invite_handled_by") else None
+            ),
+            profile_witness=witness,
+            ready=not blockers,
+            blockers=tuple(blockers),
+        )
+
+    def set_name(self, actor: ActorContext, display_name: str) -> OnboardingSnapshot:
+        self._authorize(actor, Permission.ADMINISTER)
+        name = display_name.strip()
+        if not 1 <= len(name) <= 30 or not re.fullmatch(r"[\w -]+", name, re.UNICODE):
+            raise ValueError("display name must be 1-30 letters, numbers, spaces, or hyphens")
+        state, _ = self._migrate(self._load())
+        onboarding = dict(state.get("onboarding") or {})
+        onboarding.update(
+            {
+                "phase": "invite",
+                "started_at": onboarding.get("started_at")
+                or datetime.now(UTC).isoformat().replace("+00:00", "Z"),
+            }
+        )
+        fields = self._fields(state)
+        if "name" not in fields:
+            fields.append("name")
+        state.update(
+            {
+                "display_name": name,
+                "name": re.sub(r"-+", "-", re.sub(r"[^a-z0-9]+", "-", name.lower())).strip("-"),
+                "profile_fields_collected": fields,
+                "onboarding": onboarding,
+            }
+        )
+        self._write(state)
+        return self.snapshot(actor)
+
+    def set_role(self, actor: ActorContext, role: str) -> OnboardingSnapshot:
+        self._authorize(actor, Permission.ADMINISTER)
+        value = role.strip()
+        if not 1 <= len(value) <= 80:
+            raise ValueError("member role must be 1-80 characters")
+        state, _ = self._migrate(self._load())
+        fields = self._fields(state)
+        if "role" not in fields:
+            fields.append("role")
+        state["member_role"] = value
+        state["profile_fields_collected"] = fields
+        self._write(state)
+        return self.snapshot(actor)
+
+    def record_invite(self, actor: ActorContext, *, skipped: bool) -> OnboardingSnapshot:
+        self._authorize(actor, Permission.ADMINISTER)
+        state, _ = self._migrate(self._load())
+        onboarding = dict(state.get("onboarding") or {})
+        onboarding.update({"phase": "invite", "invite_skipped": skipped})
+        state["onboarding"] = onboarding
+        self._write(state)
+        return self.snapshot(actor)
+
+    def complete(self, actor: ActorContext, *, profile_witness: bool) -> OnboardingSnapshot:
+        self._authorize(actor, Permission.ADMINISTER)
+        if not profile_witness:
+            raise ValueError("canonical person profile witness is missing")
+        state, _ = self._migrate(self._load())
+        onboarding = dict(state.get("onboarding") or {})
+        onboarding.update(
+            {
+                "phase": "complete",
+                "completed_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
+            }
+        )
+        state["onboarding"] = onboarding
+        state["onboarding_complete"] = True
+        self._write(state)
+        return self.snapshot(actor)