create-egregore 0.21.6 → 0.22.0

package/runtime/pi/egregore_runtime/installation.py

@@ -0,0 +1,119 @@
+"""Which Egregore installation a checkout belongs to.
+
+Standard library only, no package imports: the upgrade engine, the map
+hand-over and the memory map notice writer import it, and the map's server
+(serve.py) loads this file by path from a checkout that may run another Runtime
+version.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import stat
+from pathlib import Path
+
+_SMALL = 4096  # a .git file or a git dir's back-link is one short path
+_UNSET = object()
+
+
+def _read_small(path: str) -> str:
+    """A short regular file's text; a link, folder, pipe or device raises OSError."""
+    descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
+    try:
+        if not stat.S_ISREG(os.fstat(descriptor).st_mode):
+            raise OSError(f"not a regular file: {path}")
+        return os.read(descriptor, _SMALL).decode("utf-8")
+    finally:
+        os.close(descriptor)
+
+
+def installation_root(repository_root) -> Path:
+    """The main installation directory a checkout belongs to.
+
+    A linked git worktree (``.git`` is a file) belongs to its main checkout;
+    upgrade state is per installation, so every worktree of one install
+    shares it. A standalone clone is its own installation.
+
+    The ``.git`` file sits in the checkout and anyone working there can rewrite
+    it, so its claim counts only when the git dir links back to this checkout,
+    as git's own ``<gitdir>/gitdir`` file does for a real linked worktree (the
+    check bin/boundary-guard.cjs makes). A forged or broken ``.git`` file, a git
+    dir outside ``<main>/.git/worktrees/<name>``, or any read error makes the
+    checkout its own installation.
+
+    So does a worktree moved with plain ``mv``: its back-link still names the
+    old place until ``git worktree repair`` is run in it. The shell copies of
+    this climb (bin/search.sh, bin/settings.sh, bin/graph.sh, bin/lib/greeting.sh,
+    bin/lib/runtime-owned.sh) skip the back-link and key such a checkout by its
+    main checkout. Every genuine layout gets the same answer from both.
+    """
+    root = Path(repository_root).resolve()
+    marker = root / ".git"
+    try:
+        if not stat.S_ISREG(os.lstat(marker).st_mode):
+            return root
+        for line in _read_small(str(marker)).splitlines():
+            if line.startswith("gitdir:"):
+                git_dir = Path(line.split(":", 1)[1].strip())
+                break
+        else:
+            return root
+        if not git_dir.is_absolute():
+            git_dir = (root / git_dir).resolve()
+        # <main>/.git/worktrees/<name> → <main>
+        if git_dir.parent.name != "worktrees" or git_dir.parent.parent.name != ".git":
+            return root
+        back = _read_small(str(git_dir / "gitdir")).strip()
+        # ``root`` is resolved and ``.git`` was read as a regular file, so ``marker`` is already its
+        # real path; resolving it again would follow a ``.git`` swapped for a link after the read.
+        if not back or os.path.realpath(os.path.join(git_dir, back)) != str(marker):
+            return root
+        main = git_dir.parent.parent.parent
+        return main.resolve() if main.is_dir() else root
+    except (OSError, ValueError):
+        return root
+
+
+class InstallationIdentityError(ValueError):
+    """The checkout has no stable organization identity, so per-installation state fails closed."""
+
+
+def installation_key(repository_root, *, existing_org_id=_UNSET) -> str:
+    """Durable per-installation key: stable org identity × installation identity.
+
+    It keys upgrade state and the map hand-over slot. Identity is never inferred
+    from a path alone: the stable organization ID from egregore.json is required
+    and leads the key, so two organizations that share, symlink, or otherwise
+    resolve to the same memory repository never share state. The installation
+    path is composed in so two installs of one organization stay independent,
+    while worktrees of one install share it. A checkout without a stable org_id
+    fails closed.
+
+    ``existing_org_id`` is an optional host-owned value already read and
+    validated by a caller. It avoids a second config read; it does not change
+    installation_root's existing path/backlink semantics or authenticate an org.
+    """
+    root = Path(repository_root).resolve()
+    if existing_org_id is _UNSET:
+        try:
+            org_id = str(json.loads((root / "egregore.json").read_text(encoding="utf-8")).get("org_id") or "").strip()
+        except (OSError, ValueError, AttributeError) as exc:
+            raise InstallationIdentityError(f"cannot read egregore.json for instance identity: {exc}") from exc
+    else:
+        org_id = existing_org_id
+        try:
+            valid = (isinstance(org_id, str) and bool(org_id) and org_id == org_id.strip()
+                     and len(org_id.encode("utf-8")) <= 2000
+                     and not any(ord(character) < 33 or ord(character) == 127 for character in org_id))
+        except UnicodeError:
+            valid = False
+        if not valid:
+            raise InstallationIdentityError("a canonical existing organization identity is required")
+    if not org_id:
+        raise InstallationIdentityError(
+            "this instance has no stable org_id; per-installation state fails closed "
+            "without a stable organization identity"
+        )
+    return hashlib.sha256(f"{org_id}|{installation_root(root)}".encode("utf-8")).hexdigest()[:16]

package/runtime/pi/egregore_runtime/investigation.py

@@ -21,9 +21,10 @@ from .contracts import RetrievalHit, RetrievalMode, RetrievalRequest, RetrievalR
 from .errors import AuthorizationDenied, EgregoreRuntimeError
 from .eligibility import is_internal_event
 from .excerpts import select_excerpt
+from .held import identity, package, redact
 from .identity import actor_lexical_formulations, resolve_actor_query
 from .investigation_contracts import InvestigationRequest
-from .policy import scope_allows_path
+from .policy import allows, check, within
 
 KINDS = {'hybrid':RetrievalMode.HYBRID, 'semantic':RetrievalMode.VEC, 'keyword':RetrievalMode.LEX}
 STATE_VERSION = 'egregore-investigation/v1'
@@ -34,6 +35,17 @@ MAX_STATE_BYTES = 16_000_000
 MAX_OPEN_BATCH_BYTES = 16_000
 
 
+def new_cursor():
+    """Mint a continuation cursor an agent can pass back as `--cursor TOKEN`.
+
+    Lowercase hex never starts with '-', so argparse cannot mistake the token
+    for an option. The length matches the 24-character size estimate used
+    when fitting a page into the output budget. Cursors minted earlier as
+    URL-safe base64 stay valid: lookup is by exact key.
+    """
+    return secrets.token_hex(12)
+
+
 def state_path(root, episode_id):
     key = hashlib.sha256(str(episode_id).encode()).hexdigest()
     return Path(root) / '.egregore/runtime/investigations' / (key + '.json')
@@ -108,9 +120,9 @@ def _save_state(path, value):
 
 class LookupAdapter:
     """Deterministic canonical lookup and filters, behind Runtime authorization."""
-    def __init__(self, original, kind, params, actor, admin_gate, *, question=''):
+    def __init__(self, original, kind, params, actor, *, question=''):
         self.original, self.kind, self.params = original, kind, params
-        self.actor, self.admin_gate = actor, admin_gate
+        self.actor = actor
         self.question = question or params.get('query', '')
         self.last_result = None
         self.scanned = 0
@@ -151,7 +163,7 @@ class LookupAdapter:
         if p.get('before') and (day is None or day > date.fromisoformat(p['before'])):
             return False
         prefix = p.get('prefix')
-        if prefix and not scope_allows_path(path, (prefix,)):
+        if prefix and not within(path, (prefix,)):
             return False
         if p.get('artifact_type') and metadata.artifact_type != p['artifact_type']:
             return False
@@ -160,12 +172,12 @@ class LookupAdapter:
         return True
 
     def retrieve(self, request):
-        return self._retrieve(request)
+        raise AuthorizationDenied("lookup requires an authorized decision")
 
-    def retrieve_authorized(self, request, *, actor, admin_gate=None, lifecycle=None):
-        return self._retrieve(request, lifecycle=lifecycle)
+    def retrieve_authorized(self, request, *, actor, decision, lifecycle=None):
+        return self._retrieve(request, decision=decision, lifecycle=lifecycle)
 
-    def _retrieve(self, request, *, lifecycle=None):
+    def _retrieve(self, request, *, decision, lifecycle=None):
         started = time.monotonic()
         if self.kind in KINDS:
             lexical = actor_lexical_formulations(self.actor, request.task, request.lex)
@@ -174,8 +186,8 @@ class LookupAdapter:
                 vec=tuple(resolve_actor_query(self.actor, value) for value in request.vec),
                 artifact_types=(self.params['artifact_type'],) if self.params.get('artifact_type') else request.artifact_types)
             scoped = getattr(self.original, 'retrieve_authorized', None)
-            raw = (scoped(ranked_request, actor=self.actor, admin_gate=self.admin_gate, lifecycle=lifecycle,
-                path_filter=lambda path: not self.params.get('prefix') or scope_allows_path(path, (self.params['prefix'],)),
+            raw = (scoped(ranked_request, actor=self.actor, decision=decision, lifecycle=lifecycle,
+                path_filter=lambda path: not self.params.get('prefix') or within(path, (self.params['prefix'],)),
                 metadata_filter=self._matches) if callable(scoped) else self.original.retrieve(ranked_request))
             candidates = raw.hits
             self.last_result = raw
@@ -185,15 +197,12 @@ class LookupAdapter:
             for path in sorted(root.rglob('*.md')):
                 canonical = 'memory/' + path.relative_to(root).as_posix()
                 # No metadata/content read outside the authorized canonical scope.
-                if not scope_allows_path(canonical, request.authorized_scopes):
-                    continue
-                resolved = path.resolve()
-                if not resolved.is_relative_to(root.resolve()):
-                    continue
-                if not scope_allows_path('memory/' + resolved.relative_to(root.resolve()).as_posix(), request.authorized_scopes):
+                verdict = check(decision, canonical, "read", root=root)
+                if not verdict.allowed:
                     continue
+                resolved = verdict.resolved
                 # Path lookup and explicit scope do not need to inspect every document.
-                if self.params.get('prefix') and not scope_allows_path(canonical, (self.params['prefix'],)):
+                if self.params.get('prefix') and not within(canonical, (self.params['prefix'],)):
                     continue
                 if self.kind == 'filename' and self.params['query'].casefold() not in canonical.casefold():
                     continue
@@ -204,8 +213,6 @@ class LookupAdapter:
                     continue
                 if request.allowed_artifact_ids and metadata.artifact_id not in request.allowed_artifact_ids:
                     continue
-                if self.admin_gate and not self.admin_gate.actor_is_admin(self.actor) and self.admin_gate.path_is_admin(canonical):
-                    continue
                 self.scanned += 1
                 if not self._matches(canonical, metadata):
                     continue
@@ -227,14 +234,14 @@ class LookupAdapter:
             candidates.sort(key=lambda hit: (-(hit.observed_at.timestamp() if hit.observed_at else 0), hit.canonical_path))
         filtered = []
         for hit in candidates:
-            if not scope_allows_path(hit.canonical_path, request.authorized_scopes):
+            if not allows(decision, hit.canonical_path, "read", root=self.original.memory_root):
                 continue
             metadata = self.original.describe_source(hit.canonical_path)
             if metadata.observed_at is None and hit.observed_at is not None:
                 metadata = replace(metadata, observed_at=hit.observed_at, date_provenance='lifecycle.created_at')
             if not self._matches(hit.canonical_path, metadata):
                 continue
-            # Stable, compact metadata plus a passage; no full-source dump.
+            # Stable, compact metadata plus a passage; no full-source dump. Observe redacts it field by field.
             passage = hit.passage if self.params.get('_context_projection') else json.dumps({
                 'title':metadata.title, 'document_date':str(self._day(metadata.observed_at)) if metadata.observed_at else None,
                 'date_provenance':getattr(metadata, 'date_provenance', None),
@@ -309,13 +316,34 @@ class Investigation:
                 if path and identifier and self._can_deliver(path,identifier):
                     self.ledger[path] = evidence.get('revision','unknown')
 
+    def _package(self, row):
+        """A cached row as it may be handed over now: its path and id decided again, every text redacted under
+        the held set in effect, not the one it was cached under. None when it is refused."""
+        from .contracts import Permission
+        decision = self.runtime.authorize(self.actor, Permission.READ, (row['artifact_id'],))
+        text = {key: value for key, value in row.items() if key not in ('path', 'artifact_id')}
+        kept = package(decision, text, root=self.runtime.canonical_root, paths=(row['path'],))
+        if kept is None:
+            return None
+        return {**row, **kept, 'artifact_id': identity(decision, row['artifact_id'], row['path'],
+                                                       root=self.runtime.canonical_root)}
+
+    def _redacted(self, value):
+        """A delivery kept from an earlier call (a replayed response, a reused context), redacted again."""
+        from .contracts import Permission
+        return redact(self.runtime.authorize(self.actor, Permission.READ), value, root=self.runtime.canonical_root)
+
     def _can_deliver(self, path, artifact_id):
         from .contracts import Permission
         decision = self.runtime.authorize(self.actor, Permission.READ, (artifact_id,))
-        if not decision.allowed or not scope_allows_path(path,decision.scopes):
+        return allows(decision, path, "read", root=self.runtime.canonical_root)
+
+    def _deliverable(self, path):
+        """Decide the path before reading it for its id, then decide the id."""
+        from .contracts import Permission
+        if not allows(self.runtime.authorize(self.actor, Permission.READ), path, "read", root=self.runtime.canonical_root):
             return False
-        gate = self.runtime.admin_gate
-        return not gate or gate.actor_is_admin(self.actor) or not gate.path_is_admin(path)
+        return self._can_deliver(path, self.runtime.retriever.describe_source(path).artifact_id)
 
     def _authorize(self):
         started=time.monotonic()
@@ -331,7 +359,6 @@ class Investigation:
             raise AuthorizationDenied('investigation requires current discover/read permission')
         signature = decision.to_dict()
         signature['profile_revision'] = self.actor.profile.revision
-        signature['admin'] = self.runtime.admin_gate.actor_is_admin(self.actor) if self.runtime.admin_gate else False
         digest = hashlib.sha256(json.dumps(signature,sort_keys=True).encode()).hexdigest()
         if self.episode_signature is not None and self.episode_signature != digest:
             raise AuthorizationDenied('authorization changed; start a new investigation')
@@ -385,7 +412,7 @@ class Investigation:
         key = key_for(self._revision(), self._index_revision())
         cached = self.context_cache.get(key)
         if cached and (not request.open_sources or cached['open_sources']):
-            context = cached['context']
+            context = self._redacted(cached['context'])
             self._check_context(context)
             return {'projection':'context', 'context':context, 'source_revision':context['source_revision'],
                     'reused':True, 'searches_remaining':self.search_budget-self.searches}
@@ -403,7 +430,7 @@ class Investigation:
             adapter = LookupAdapter(self.runtime.retriever,
                 {RetrievalMode.LEX:'keyword',RetrievalMode.VEC:'semantic',RetrievalMode.HYBRID:'hybrid'}[request.mode],
                 {**self.bounds, '_context_projection':True, '_candidate_limit':request.top_k},
-                self.actor,self.runtime.admin_gate)
+                self.actor)
             target = replace(self.runtime,retriever=adapter)
         try:
             context = target.observe(self.actor,request,token_budget=options.token_budget).to_dict()
@@ -446,20 +473,17 @@ class Investigation:
             if result['context'].get('freshness',{}).get('index_revision') != self._index_revision():
                 raise ValueError('index changed before delivery; retry with a new request_id')
         elif result.get('path'):
-            metadata=self.runtime.retriever.describe_source(result['path'])
-            if not self._can_deliver(result['path'],metadata.artifact_id):
+            if not self._deliverable(result['path']):
                 raise AuthorizationDenied('evidence authorization changed')
         elif result.get('sources'):
             for source in result['sources']:
-                metadata = self.runtime.retriever.describe_source(source['path'])
-                if not self._can_deliver(source['path'], metadata.artifact_id):
+                if not self._deliverable(source['path']):
                     raise AuthorizationDenied('evidence authorization changed')
         elif any(not self._can_deliver(row['path'],row['artifact_id']) for row in result.get('results',[])):
             raise AuthorizationDenied('evidence authorization changed')
         for key in ['seen_sources','opened_sources']:
             for path in result.get(key,{}):
-                metadata=self.runtime.retriever.describe_source(path)
-                if not self._can_deliver(path,metadata.artifact_id):
+                if not self._deliverable(path):
                     raise AuthorizationDenied('retained evidence authorization changed')
 
     def _request(self, text, mode, limit):
@@ -494,7 +518,7 @@ class Investigation:
             if params.get(key): date.fromisoformat(params[key])
         if params.get('after') and params.get('before') and params['after']>params['before']:
             raise ValueError('after must not exceed before')
-        if params.get('prefix') and (not params['prefix'].startswith('memory') or '\\' in params['prefix'] or not scope_allows_path(params['prefix'],('memory',))):
+        if params.get('prefix') and (not params['prefix'].startswith('memory') or '\\' in params['prefix'] or not within(params['prefix'],('memory',))):
             raise ValueError('prefix must be a canonical memory path')
         return kind,limit
 
@@ -542,7 +566,7 @@ class Investigation:
                     raise ValueError('investigation search budget exhausted; report remaining gaps')
                 self.searches+=1
                 self._checkpoint()
-                adapter=LookupAdapter(self.runtime.retriever,kind,params,self.actor,self.runtime.admin_gate,
+                adapter=LookupAdapter(self.runtime.retriever,kind,params,self.actor,
                                       question=self.question)
                 facade=replace(self.runtime,retriever=adapter)
                 request=self._request(str(params.get('query') or 'Documents within the requested date interval'),KINDS.get(kind,RetrievalMode.LEX).value,10000)
@@ -568,8 +592,8 @@ class Investigation:
                     'degraded':adapter.last_result.degraded,'warnings':list(adapter.last_result.warnings),'source_revision':context.source_revision,
                     'retrieval_seconds':adapter.last_result.latency_ms/1000,'scanned_authorized':adapter.scanned}
         cached=self.cache[fingerprint]
-        rows=cached['rows'][offset:offset+limit]
-        if any(not self._can_deliver(row['path'],row['artifact_id']) for row in rows):
+        rows=[self._package(row) for row in cached['rows'][offset:offset+limit]]
+        if any(row is None for row in rows):
             raise AuthorizationDenied('evidence authorization changed; start a new investigation')
         next_cursor=None
         def result_for(selected):
@@ -583,7 +607,7 @@ class Investigation:
         if not rows and offset<len(cached['rows']):
             raise ValueError('evidence output budget exhausted; report the remaining gap')
         if offset+len(rows)<len(cached['rows']):
-            next_cursor=secrets.token_urlsafe(18)
+            next_cursor=new_cursor()
             self.cursors[next_cursor]=(fingerprint,offset+len(rows),limit,signature,revision)
         for row in rows:self.ledger[row['path']]=row['revision']
         result=result_for(rows);result['next_cursor']=next_cursor;result['previous_evidence_retained']=len(self.ledger)
@@ -675,8 +699,7 @@ class Investigation:
         visible = {}
         for path, revision in sources.items():
             try:
-                metadata = self.runtime.retriever.describe_source(path)
-                if self._can_deliver(path, metadata.artifact_id):
+                if self._deliverable(path):
                     visible[path] = revision
             except FileNotFoundError:
                 continue
@@ -723,7 +746,7 @@ class Investigation:
                 self._authorize()
                 if cached.get('source_revision') and cached['source_revision'] != self._revision():
                     raise ValueError('sources changed since this response; retry with a new request_id')
-                response=copy.deepcopy(cached['response'])
+                response=self._redacted(copy.deepcopy(cached['response']))
                 if not response['ok']:
                     result=None
                 else:

package/runtime/pi/egregore_runtime/investigation_guidance.py

@@ -7,7 +7,7 @@ Use find for discovery (query is compatibility-only). Use search.sh open for sou
 reads; never use raw file tools or shell reads on memory. Answer from sufficient
 excerpts without opening again."""
 
-INVESTIGATION_GUIDANCE = RECALL_WORKFLOW_GUIDANCE + "\n" + """Search organizational memory through Runtime; reuse evidence already in context.
+INVESTIGATION_GUIDANCE = RECALL_WORKFLOW_GUIDANCE + "\n" + """Reuse evidence already in context; never bypass Runtime with raw memory reads or direct QMD access.
 For exact handoff status/order, use `bash bin/search.sh handoffs --mine --status open --order newest --limit 1 --open`; adjust filters to the question.
 Choose a lookup and call `bash bin/search.sh find "<query>" --kind <kind>`:
 filename for known path fragments; literal for exact wording; dates for dated
@@ -30,17 +30,11 @@ Up to 10 sources per call; --offset N is a nonnegative character offset and
 result budget, so windows may be shorter. Follow the printed per-source
 continuation only when that source needs more context; do not assume a partial
 window is the complete document. Keep each source batch in its own tool result.
-In Codex, set the outer functions.exec output budget with
-`// @exec: {"max_output_tokens": 20000}` and exec_command.max_output_tokens to
-20000 for source reads; a larger inner limit alone cannot prevent outer truncation.
 Known paths may be opened directly. Use find --cursor TOKEN only when another
 page is needed. Do not fetch or reread this contract when it is already attached.
 Runtime retains date bounds and resolves the native session automatically.
 Do not supply episode IDs, request IDs, or gap explanations in normal calls.
-If native binding is unavailable, use this prompt's supplied --episode reference;
-never borrow another prompt's reference. A new prompt resets retrieval limits.
-Use a normal completion wait (at least 10 seconds for yield_time_ms); do not
-create polling turns with one-second waits. Batch independent work in the harness.
+A new prompt resets retrieval limits. Batch independent work in the harness.
 Stop when the question is supported. Ranked matches are not exhaustive; verify
 coverage before claiming something is newest or absent. Document dates need not
 be event dates. Distinguish proposals/conditions from completed work and cite sources.
@@ -52,7 +46,18 @@ failure or index-coverage warning into the final answer. Say what could not be
 verified and how the gap limits that conclusion; do not turn a search gap into a fact.
 Authorization, source checks, retained evidence and budgets remain inside Runtime:
 8 discoveries, 20 source windows, 50 calls and 120,000 result characters per prompt.
-Never bypass Runtime with raw memory filesystem or direct QMD access. Code search
-and tests use ordinary tools. Emit the memory retrieval beat once per question.
-On Claude append --context-packet; never reproduce internal JSON or ranking traces.
+Never reproduce internal JSON or ranking traces.
 """
+
+# Codex-only tool budgets; other harnesses never see these names.
+CODEX_GUIDANCE = """In Codex, set the outer functions.exec output budget with
+`// @exec: {"max_output_tokens": 20000}` and exec_command.max_output_tokens to
+20000 for source reads; a larger inner limit alone cannot prevent outer truncation.
+Use a normal completion wait (at least 10 seconds for yield_time_ms); do not
+create polling turns with one-second waits."""
+
+
+def investigation_guidance(harness: str) -> str:
+    """The shared contract plus the lines only this harness can act on."""
+    base = INVESTIGATION_GUIDANCE.rstrip("\n")
+    return base + "\n" + CODEX_GUIDANCE if harness == "codex" else base