create-egregore 0.21.6 → 0.22.0

package/runtime/pi/egregore_runtime/hosted_api.py

@@ -0,0 +1,137 @@
+"""The address every hosted request goes to.
+
+The hosted API moved to api.egregore.xyz after the October 2026 security
+reset. Its old Railway origin was retired and is no longer ours to trust: a
+credential sent there could reach whoever holds that address next. Every
+request that carries a token or key resolves its address here, so a
+configuration or environment saved before the move never reaches it.
+
+The rule is an allowlist, not a normalizer. Only a plain web address is used:
+http or https, a host of ASCII letters, digits, dots, hyphens and underscores,
+an optional port and a plain path. Every transport reads such an address the
+same way, so its host is the one the request reaches. A plain address on the
+retired host becomes the hosted API; any other plain address is kept.
+Anything else is refused, because curl, urllib and WHATWG parsers disagree
+about where it goes: userinfo, backslashes, percent-encoding, curl glob
+characters, a query or fragment, whitespace or invisible characters inside,
+a ``URL:`` prefix, non-ASCII. A refused address raises
+:class:`RefusedApiAddress` and nothing is sent.
+
+The shell (bin/lib/config.sh), Node (packages/create-egregore/lib/
+hosted-api.js) and connector resolvers follow the same table:
+bin/tests/fixtures/hosted-api-url-vectors.json.
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+from typing import Mapping
+from urllib.request import Request
+
+HOSTED_API_ORIGIN = "https://api.egregore.xyz"
+RETIRED_API_ORIGIN = "https://egregore-production-55f2.up.railway.app"
+REFUSED_API_ADDRESS_NOTICE = (
+    "The hosted services address isn't a plain web address, so nothing was sent. "
+    f"Correct api_url in egregore.json; the hosted address is {HOSTED_API_ORIGIN}."
+)
+
+_RETIRED_HOST = RETIRED_API_ORIGIN.split("://", 1)[1]
+_WHITESPACE = " \t\n\r\f\v"
+_PLAIN = re.compile(r"https?://([A-Za-z0-9._-]+)(:[0-9]{1,5})?(/[A-Za-z0-9._~/-]*)?")
+# The start of a request URL: a plain origin, then a path, query or nothing.
+_PLAIN_ORIGIN = re.compile(r"https?://([A-Za-z0-9._-]+)(:[0-9]{1,5})?(?=[/?#]|\Z)")
+_notice_shown = False
+
+
+class RefusedApiAddress(ValueError):
+    """The hosted address is not a plain web address, so nothing may be sent to it."""
+
+    def __init__(self) -> None:
+        super().__init__(REFUSED_API_ADDRESS_NOTICE)
+
+
+def _refuse() -> RefusedApiAddress:
+    """Say why once per process on stderr, and return the error to raise."""
+    global _notice_shown
+    if not _notice_shown:
+        _notice_shown = True
+        try:
+            print(REFUSED_API_ADDRESS_NOTICE, file=sys.stderr)
+        except (OSError, ValueError):
+            pass
+    return RefusedApiAddress()
+
+
+def _retired_host(host: str) -> bool:
+    return host.lower().rstrip(".") == _RETIRED_HOST
+
+
+def retired_api_url(value: object) -> bool:
+    """Whether ``value`` is a plain address on the retired hosted origin."""
+    text = ("" if value is None else str(value)).strip(_WHITESPACE)
+    plain = _PLAIN.fullmatch(text)
+    return bool(plain) and _retired_host(plain.group(1))
+
+
+def hosted_api_url(value: object, exact: bool = False) -> str:
+    """The address to send a request to.
+
+    A plain address on the retired origin becomes the hosted API, keeping its
+    path, with trailing slashes trimmed. Any other plain address keeps its text
+    apart from surrounding whitespace and trailing slashes; empty stays empty.
+    Anything else raises :class:`RefusedApiAddress`, and the caller sends
+    nothing.
+
+    ``exact`` is for the server's own settings, such as an OAuth redirect URI
+    that is compared exactly: only a plain retired address changes, and every
+    other value is returned byte for byte, never refused.
+    """
+    text = "" if value is None else str(value)
+    trimmed = text.strip(_WHITESPACE)
+    plain = _PLAIN.fullmatch(trimmed)
+    if plain and _retired_host(plain.group(1)):
+        return (HOSTED_API_ORIGIN + (plain.group(3) or "")).rstrip("/")
+    if exact:
+        return text
+    if not trimmed:
+        return ""
+    if not plain:
+        raise _refuse()
+    return trimmed.rstrip("/")
+
+
+_CREDENTIAL_HEADER = re.compile(r"authorization|cookie|token|key|secret|auth", re.IGNORECASE)
+
+
+def credential_request(
+    url: str,
+    *,
+    headers: Mapping[str, str] | None = None,
+    data: bytes | None = None,
+    method: str | None = None,
+) -> Request:
+    """A request whose credentials reach only a plain, current hosted address.
+
+    The URL must start with a plain origin that is not the retired one;
+    otherwise :class:`RefusedApiAddress` is raised and nothing is built. The
+    resolver already guarantees that for an address it returned; this holds it
+    for every URL built from one.
+
+    urllib's redirect handler copies only ``Request.headers`` to the next
+    request. Authorization, cookies and any token, key or secret header are
+    added with ``add_unredirected_header`` instead, so they stay with the
+    first request and cannot follow a redirect to another host. A redirected
+    POST loses its body (urllib refuses 307 and 308 for it), so a credential
+    in the body stays behind too.
+    """
+    origin = _PLAIN_ORIGIN.match(url) if isinstance(url, str) else None
+    if not origin or _retired_host(origin.group(1)):
+        raise _refuse()
+    request = Request(url, data=data, method=method)
+    for name, value in (headers or {}).items():
+        if _CREDENTIAL_HEADER.search(name):
+            request.add_unredirected_header(name, value)
+        else:
+            request.add_header(name, value)
+    return request

package/runtime/pi/egregore_runtime/identity.py

@@ -12,8 +12,9 @@ import tempfile
 import fcntl
 import hashlib
 import re
+import stat
 from contextlib import contextmanager
-from collections.abc import Callable, Mapping
+from collections.abc import Callable, Iterable, Mapping
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
@@ -21,12 +22,16 @@ from uuid import uuid4
 
 from .contracts import (
     AccountIdentity,
+    EgregoreProfile,
     ActorContext,
     ActorIdentity,
     ActorKind,
     OrgMembership,
+    PolicyDecision,
 )
+from . import held
 from .entitlements import resolve_entitlements
+from .policy import check
 from .profile import initialize_org_config, load_profile, render_identity_document
 
 
@@ -48,9 +53,10 @@ class OrganizationMemberPresentation:
     aliases: tuple[str, ...]
 
 
-def _profile_identity(path: Path) -> dict[str, Any]:
+def _profile_identity(path: Path, source: Path | None = None, data: bytes | None = None) -> dict[str, Any]:
     try:
-        text = path.read_text(encoding="utf-8", errors="replace")[:4_000]
+        text = (data.decode("utf-8", "replace") if data is not None
+                else (source or path).read_text(encoding="utf-8", errors="replace"))[:4_000]
     except OSError:
         return {}
     title = re.search(r"(?m)^#\s+(.+?)\s*$", text)
@@ -83,6 +89,10 @@ def _profile_identity(path: Path) -> dict[str, Any]:
 def organization_member_presentations(
     root: Path,
     actor: ActorContext | None = None,
+    *,
+    decision: PolicyDecision | None,
+    sources: Mapping[str, bytes] | None = None,
+    removed: Iterable[str] | None = None,
 ) -> tuple[OrganizationMemberPresentation, ...]:
     """Load stable member names without treating aliases as extra people.
 
@@ -90,31 +100,50 @@ def organization_member_presentations(
     reconciliation. Profiles with the same Person-ID collapse into one member;
     an explicit Alias-Of title is the preferred presentation witness for
     legacy canonical profiles whose title is still a provider handle.
+
+    ``decision`` is required: a profile is read only when the one predicate
+    admits it for that reader, from the path it checked, so prompt context and
+    query terms name nobody the reader may not see. ``decision=None`` is an
+    explicit opt-out for a caller that applies its own holds (the exporter's
+    hand-over chain); leaving it out is an error, never an unchecked read.
+    ``sources`` (canonical path → bytes a caller already read and authorized,
+    as a hand-over holds them) replaces reading the profiles from disk, and
+    ``removed`` the working tree's removal ledger (another principal's
+    hand-over passes the committed one).
     """
 
     people = root / "memory" / "people"
-    if not people.is_dir():
+    if sources is None and not people.is_dir():
         return ()
     # Durable removals: the committed ledger in egregore.json outranks file
     # presence — a person file resurrected by a session write or sync never
     # resurfaces a removed member in the roster.
-    removed: set[str] = set()
-    try:
-        config = json.loads((root / "egregore.json").read_text(encoding="utf-8"))
-        removed = {
-            str(handle).strip().casefold()
-            for handle in (config.get("people_removed") or [])
-            if str(handle).strip()
-        }
-    except (OSError, ValueError):
-        removed = set()
+    if removed is None:
+        try:
+            config = json.loads((root / "egregore.json").read_text(encoding="utf-8"))
+            removed = {
+                str(handle).strip().casefold()
+                for handle in (config.get("people_removed") or [])
+                if str(handle).strip()
+            }
+        except (OSError, ValueError):
+            removed = set()
+    removed = set(removed)
     grouped: dict[str, list[dict[str, Any]]] = {}
-    for path in sorted(people.glob("*.md")):
+    profiles = sorted(people.glob("*.md")) if sources is None else sorted(
+        root / name for name in sources if name.startswith("memory/people/") and name.count("/") == 2)
+    for path in profiles:
         if path.name == "index.md":
             continue
         if path.stem.casefold() in removed:
             continue
-        profile = _profile_identity(path)
+        source = path
+        if decision is not None:
+            verdict = check(decision, f"memory/people/{path.name}", "read", root=root / "memory")
+            if not verdict.allowed:
+                continue
+            source = verdict.resolved
+        profile = _profile_identity(path, source, None if sources is None else sources[f"memory/people/{path.name}"])
         if not profile:
             continue
         if str(profile.get("github") or "").strip().casefold() in removed:
@@ -211,7 +240,8 @@ def organization_member_presentations(
                 aliases=compact_aliases,
             )
         )
-    return tuple(sorted(members, key=lambda item: item.display_name.casefold()))
+    members.sort(key=lambda item: item.display_name.casefold())  # a readable profile can name held material too
+    return tuple(members if decision is None else held.redact(decision, members, root=root / "memory"))
 
 
 def resolve_actor_query(actor: ActorContext, query: str) -> str:
@@ -752,6 +782,102 @@ def _identity_lock(state_path: Path):
             fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
 
 
+def _existing_identity_json(path: Path) -> tuple[dict[str, Any], bytes]:
+    """Bounded, no-symlink read for the opt-in non-migrating resolver."""
+    path = Path(os.path.abspath(path))
+    parent = os.open(path.anchor, os.O_RDONLY | os.O_DIRECTORY)
+    descriptor = None
+    try:
+        for part in path.parts[1:-1]:
+            child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent)
+            os.close(parent)
+            parent = child
+        descriptor = os.open(path.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=parent)
+        info = os.fstat(descriptor)
+        if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_size > 200000:
+            raise ValueError("existing identity is unavailable")
+        raw = os.read(descriptor, 200001)
+        if len(raw) > 200000:
+            raise ValueError("existing identity is unavailable")
+    finally:
+        if descriptor is not None:
+            os.close(descriptor)
+        os.close(parent)
+    def unique(pairs):
+        result = {}
+        for key, value in pairs:
+            if key in result:
+                raise ValueError("existing identity is invalid")
+            result[key] = value
+        return result
+    try:
+        value = json.loads(raw, object_pairs_hook=unique)
+        pending = [(value, 0)]
+        while pending:
+            item, depth = pending.pop()
+            if depth > 64:
+                raise ValueError("existing identity is invalid")
+            if isinstance(item, str):
+                item.encode("utf-8")
+            elif isinstance(item, dict):
+                pending.extend((part, depth + 1) for pair in item.items() for part in pair)
+            elif isinstance(item, list):
+                pending.extend((part, depth + 1) for part in item)
+        if not isinstance(value, dict):
+            raise ValueError("existing identity is invalid")
+    except (ValueError, UnicodeError, RecursionError):
+        raise ValueError("existing identity is invalid") from None
+    return value, raw
+
+
+def _existing_identifier(value: Any) -> bool:
+    return (isinstance(value, str) and bool(value) and value == value.strip()
+            and len(value.encode("utf-8")) <= 2000
+            and not any(ord(character) < 33 or ord(character) == 127 for character in value))
+
+
+def _existing_identity(root: Path):
+    """Retained IDs and config only; deliberately never read identity documents.
+
+    The profile revision has a distinct config-only namespace and cannot stand
+    in for load_profile's revision of config plus declared identity documents.
+    """
+    from .profile import PROFILE_SCHEMA, DEFAULT_CONTEXT_BUDGET
+    config, raw = _existing_identity_json(root / "egregore.json")
+    state, _ = _existing_identity_json(root / ".egregore-state.json")
+    org = config.get("org_id")
+    if not _existing_identifier(org) or any(not _existing_identifier(state.get(key)) for key in ("account_id", "actor_id", "membership_id")):
+        raise ValueError("existing identity is required")
+    nested = state.get("identity", {})
+    if not isinstance(nested, dict) or any(key in nested and nested[key] != value for key, value in {
+        "account_id": state["account_id"], "actor_id": state["actor_id"],
+        "membership_id": state["membership_id"], "org_id": org, "actor_kind": "person",
+    }.items()) or "org_id" in state and state["org_id"] != org:
+        raise ValueError("existing identity is inconsistent")
+    if not isinstance(state.get("provider_identities", {}), dict):
+        raise ValueError("existing identity is invalid")
+    declared = config.get("profile") or {}
+    if not isinstance(declared, dict):
+        raise ValueError("existing profile is invalid")
+    schema = declared.get("schema_version") or PROFILE_SCHEMA
+    budget = declared.get("default_context_budget", DEFAULT_CONTEXT_BUDGET)
+    slug, name = str(config.get("slug") or "").strip(), str(config.get("org_name") or config.get("name") or "").strip()
+    if not slug or not name or schema != PROFILE_SCHEMA or type(budget) is not int or budget <= 0:
+        raise ValueError("existing profile is invalid")
+    for field in ("purpose_path", "principles_path", "conventions_path"):
+        relative = declared.get(field)
+        if relative is not None and (not isinstance(relative, str) or Path(relative.replace("\\", "/")).is_absolute() or ".." in Path(relative.replace("\\", "/")).parts):
+            raise ValueError("existing profile is invalid")
+    profile = EgregoreProfile(
+        schema_version=schema, org_id=org, slug=slug, name=name,
+        revision="profile-config-only-sha256:" + hashlib.sha256(raw).hexdigest(),
+        purpose_path=declared.get("purpose_path"), principles_path=declared.get("principles_path"),
+        conventions_path=declared.get("conventions_path"),
+        default_policy=str(declared.get("default_policy") or "all-access-v1"), default_context_budget=budget,
+    )
+    return profile, state, config
+
+
 class LocalIdentityResolver:
     """Resolve actor context from committed org config and local user state."""
 
@@ -761,62 +887,67 @@ class LocalIdentityResolver:
         *,
         persist_migrations: bool = True,
         entitlement_row: Mapping[str, Any] | None = None,
+        require_existing: bool = False,
     ) -> None:
         self.root = Path(root)
         self.persist_migrations = persist_migrations
         self.entitlement_row = entitlement_row
+        self.require_existing = require_existing
 
     def resolve(self, *, session_id: str, harness: str) -> ActorContext:
-        profile = load_profile(self.root / "egregore.json")
-        state_path = self.root / ".egregore-state.json"
-        try:
-            state = json.loads(state_path.read_text(encoding="utf-8"))
-        except FileNotFoundError as exc:
-            raise ValueError(f"local identity state is missing: {state_path}") from exc
-        if not isinstance(state, dict):
-            raise ValueError(f"local identity state must be an object: {state_path}")
-
-        config = json.loads((self.root / "egregore.json").read_text(encoding="utf-8"))
-        id_factory = new_identity_id
-        if (
-            profile.org_id.startswith("legacy-org:")
-            and (str(config.get("mode") or "local") == "connected" or config.get("api_url"))
-        ):
-            # A pre-migration Connected install must remain usable offline, but
-            # it must not mint a new random actor in each detached checkout.
-            # These explicitly legacy ids are deterministic compatibility
-            # placeholders and are replaced only by control-plane IDs.
-            legacy_seed = json.dumps(
-                {
-                    "org": profile.org_id,
-                    "person_id": state.get("person_id"),
-                    "github_id": state.get("github_id"),
-                    "github_username": state.get("github_username"),
-                    "email": state.get("email"),
-                },
-                sort_keys=True,
-                separators=(",", ":"),
-            )
+        if self.require_existing:
+            profile, migrated, config = _existing_identity(self.root)
+        else:
+            profile = load_profile(self.root / "egregore.json")
+            state_path = self.root / ".egregore-state.json"
+            try:
+                state = json.loads(state_path.read_text(encoding="utf-8"))
+            except FileNotFoundError as exc:
+                raise ValueError(f"local identity state is missing: {state_path}") from exc
+            if not isinstance(state, dict):
+                raise ValueError(f"local identity state must be an object: {state_path}")
+
+            config = json.loads((self.root / "egregore.json").read_text(encoding="utf-8"))
+            id_factory = new_identity_id
+            if (
+                profile.org_id.startswith("legacy-org:")
+                and (str(config.get("mode") or "local") == "connected" or config.get("api_url"))
+            ):
+                # A pre-migration Connected install must remain usable offline, but
+                # it must not mint a new random actor in each detached checkout.
+                # These explicitly legacy ids are deterministic compatibility
+                # placeholders and are replaced only by control-plane IDs.
+                legacy_seed = json.dumps(
+                    {
+                        "org": profile.org_id,
+                        "person_id": state.get("person_id"),
+                        "github_id": state.get("github_id"),
+                        "github_username": state.get("github_username"),
+                        "email": state.get("email"),
+                    },
+                    sort_keys=True,
+                    separators=(",", ":"),
+                )
 
-            def id_factory(prefix: str) -> str:
-                digest = hashlib.sha256(f"{prefix}:{legacy_seed}".encode()).hexdigest()[:24]
-                return f"legacy-{prefix}:{digest}"
+                def id_factory(prefix: str) -> str:
+                    digest = hashlib.sha256(f"{prefix}:{legacy_seed}".encode()).hexdigest()[:24]
+                    return f"legacy-{prefix}:{digest}"
 
-        migrated, changed = migrate_local_identity_state(
-            state, org_id=profile.org_id, id_factory=id_factory
-        )
-        if changed and self.persist_migrations:
-            # First-session races must not mint competing account/actor ids.
-            # Re-read after taking the instance lock, then persist once.
-            with _identity_lock(state_path):
-                current = json.loads(state_path.read_text(encoding="utf-8"))
-                if not isinstance(current, dict):
-                    raise ValueError(f"local identity state must be an object: {state_path}")
-                migrated, changed = migrate_local_identity_state(
-                    current, org_id=profile.org_id, id_factory=id_factory
-                )
-                if changed:
-                    _atomic_json(state_path, migrated)
+            migrated, changed = migrate_local_identity_state(
+                state, org_id=profile.org_id, id_factory=id_factory
+            )
+            if changed and self.persist_migrations:
+                # First-session races must not mint competing account/actor ids.
+                # Re-read after taking the instance lock, then persist once.
+                with _identity_lock(state_path):
+                    current = json.loads(state_path.read_text(encoding="utf-8"))
+                    if not isinstance(current, dict):
+                        raise ValueError(f"local identity state must be an object: {state_path}")
+                    migrated, changed = migrate_local_identity_state(
+                        current, org_id=profile.org_id, id_factory=id_factory
+                    )
+                    if changed:
+                        _atomic_json(state_path, migrated)
 
         provider_aliases: dict[str, str] = {}
         for provider, record in (migrated.get("provider_identities") or {}).items():

package/runtime/pi/egregore_runtime/identity_transfer.py

@@ -18,7 +18,7 @@ from typing import Any, Mapping
 
 from .artifacts import parse_canonical_markdown, render_canonical_markdown, split_frontmatter
 from .contracts import ActorContext, Permission
-from .policy import scope_allows_path
+from .policy import allows, action_for
 from .runtime import local_runtime
 
 SCHEMA = "egregore-identity-transfer/v1"
@@ -145,7 +145,7 @@ def _document(before: str, path: str, pair: Mapping[str, Any], transfer_id: str)
 
 def _authorize(runtime, source: ActorContext, path: str, permission: Permission) -> None:
     decision = runtime.authorize(source, permission, (f"memory/{path}",))
-    if not decision.allowed or not scope_allows_path(f"memory/{path}", decision.scopes):
+    if not decision.allowed or not allows(decision, f"memory/{path}", action_for(decision.permission), root=runtime.canonical_root):
         raise IdentityTransferError("current actor cannot authorize ownership transfer for the canonical path")
 
 
@@ -192,6 +192,10 @@ class _RevisionStore:
     def __init__(self, store, path: str, expected: str):
         self.store, self.path, self.expected = store, path, expected
 
+    @property
+    def root(self):
+        return self.store.root
+
     def check_write(self, document):
         if document.artifact.canonical_path != self.path:
             raise IdentityTransferError("ownership transfer attempted an unplanned write")

package/runtime/pi/egregore_runtime/ingest_cli.py

@@ -22,7 +22,7 @@ from .ingest import (
     LocalIngestJournal,
     LocalQuarantine,
 )
-from .policy import scope_allows_path
+from .policy import allows, action_for
 from .runtime import local_runtime, runtime_root
 from .scratch import read_and_consume
 
@@ -70,7 +70,7 @@ def _require(runtime, actor, permission: Permission, source_id: str):
         raise PermissionError(
             "; ".join(decision.reasons) or f"ingest {permission.value} denied"
         )
-    if not scope_allows_path(_canonical_directory(source_id), decision.scopes):
+    if not allows(decision, _canonical_directory(source_id), action_for(decision.permission), root=runtime.canonical_root):
         raise PermissionError("ingest path is outside the actor's authorized scope")
     return decision
 

package/runtime/pi/egregore_runtime/ingest_index_cli.py

@@ -8,6 +8,7 @@ legacy JSON shape consumed by the ingest boundary filter.
 from __future__ import annotations
 
 import argparse
+from dataclasses import replace
 import json
 import os
 import sys
@@ -17,9 +18,7 @@ from pathlib import Path, PurePosixPath
 from egregore_runtime.adapters.qmd import QmdAdapterError, QmdLocalRetriever
 from egregore_runtime.contracts import RetrievalMode, RetrievalRequest
 from egregore_runtime.identity import LocalIdentityResolver
-from egregore_runtime.observe import DefaultOrgContextCompiler
-from egregore_runtime.policy import all_access_dogfood_policy
-from egregore_runtime.telemetry import LocalTelemetrySink
+from egregore_runtime.runtime import local_runtime
 
 
 def _root() -> Path:
@@ -76,19 +75,8 @@ def _query(args: argparse.Namespace) -> int:
         mode=RetrievalMode.HYBRID,
         open_sources=False,
     )
-    policy = all_access_dogfood_policy(
-        org_id=actor.profile.org_id,
-        actor_id=actor.actor.actor_id,
-    )
-    compiler = DefaultOrgContextCompiler(
-        authorizer=policy,
-        retriever=retriever,
-        telemetry=LocalTelemetrySink(
-            state_file=root / ".egregore-state.json",
-            instance_root=(root / "memory").resolve(),
-        ),
-    )
-    context = compiler.observe(
+    runtime = replace(local_runtime(root, push_remote=False), retriever=retriever)
+    context = runtime.observe(
         actor,
         request,
         token_budget=actor.profile.default_context_budget,