create-egregore 0.20.2 → 0.21.0-runtime-mvp.2

package/runtime/prime/egregore_runtime/ingest_cli.py

@@ -0,0 +1,254 @@
+"""Shell bridge for canonical ingest registration.
+
+Extraction remains connector-specific. This adapter owns the organizational
+boundary between normalized intake and disposable retrieval indexing.
+"""
+
+from __future__ import annotations
+
+import argparse
+from datetime import UTC, datetime
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+from uuid import uuid4
+
+from .contracts import IngestSource, Permission
+from .ingest import (
+    CanonicalIngestWorkflow,
+    IngestReview,
+    LocalIngestJournal,
+    LocalQuarantine,
+)
+from .policy import scope_allows_path
+from .runtime import local_runtime
+
+
+def _root() -> Path:
+    return Path(os.environ.get("EGREGORE_ROOT") or Path.cwd()).resolve()
+
+
+def _slug(value: str) -> str:
+    result = re.sub(r"[^a-z0-9]+", "-", value.casefold()).strip("-")
+    if not result:
+        raise ValueError("source id must contain a letter or number")
+    return result
+
+
+def _context(*, push_remote: bool = True):
+    root = _root()
+    runtime = local_runtime(root, push_remote=push_remote)
+    actor = runtime.resolve_actor(
+        session_id=os.environ.get("EGREGORE_SESSION_ID", f"session_{uuid4().hex}"),
+        harness=os.environ.get("EGREGORE_RUNTIME", "shell"),
+    )
+    state_root = Path(
+        os.environ.get("EGREGORE_INGEST_WORKFLOW_DIR")
+        or root / ".egregore" / "ingest-runtime"
+    ).resolve()
+    workflow = CanonicalIngestWorkflow(
+        org_id=actor.profile.org_id,
+        writeback=runtime,
+        journal=LocalIngestJournal(state_root),
+        quarantine_store=LocalQuarantine(state_root / "quarantine"),
+        default_actor_id=actor.actor.actor_id,
+    )
+    return root, runtime, actor, workflow
+
+
+def _canonical_directory(source_id: str) -> str:
+    return f"memory/ingest/sources/{_slug(source_id)}/documents"
+
+
+def _require(runtime, actor, permission: Permission, source_id: str):
+    resource = f"ingest-source:{_slug(source_id)}"
+    decision = runtime.authorize(actor, permission, (resource,))
+    if not decision.allowed:
+        raise PermissionError(
+            "; ".join(decision.reasons) or f"ingest {permission.value} denied"
+        )
+    if not scope_allows_path(_canonical_directory(source_id), decision.scopes):
+        raise PermissionError("ingest path is outside the actor's authorized scope")
+    return decision
+
+
+def _boundaries(values: list[str]) -> dict[str, str]:
+    rows: dict[str, str] = {}
+    for value in values:
+        key, separator, item = value.partition("=")
+        if not separator or not key.strip() or not item.strip():
+            raise ValueError("boundaries must use key=value")
+        rows[key.strip()] = item.strip()
+    return rows
+
+
+def _source(args, actor, payload: bytes) -> IngestSource:
+    digest = hashlib.sha256(payload).hexdigest()
+    observed = (
+        datetime.fromisoformat(args.observed_at.replace("Z", "+00:00"))
+        if args.observed_at
+        else datetime.now(UTC)
+    )
+    if observed.tzinfo is None:
+        observed = observed.replace(tzinfo=UTC)
+    return IngestSource(
+        source_type=args.source_type,
+        source_id=args.source,
+        revision=args.revision or f"sha256:{digest}",
+        content_hash=digest,
+        observed_at=observed.astimezone(UTC),
+        title=args.title,
+        source_uri=args.source_uri,
+        visibility=tuple(args.visibility),
+        metadata={
+            "source_path": args.source_path or args.source_uri or args.source,
+            "boundaries": _boundaries(args.boundary),
+            "imported_by": actor.actor.actor_id,
+            **({"workstream": args.workstream} if args.workstream else {}),
+        },
+    )
+
+
+def _input_bytes(path: str) -> bytes:
+    source = Path(path).expanduser().resolve()
+    if not source.is_file() or source.is_symlink():
+        raise ValueError("ingest input must be a regular non-symlink file")
+    return source.read_bytes()
+
+
+def _receipt(value) -> dict[str, object]:
+    row: dict[str, object] = {
+        "source": value.source_id,
+        "artifact_id": value.artifact_id,
+        "phase": value.phase.value,
+        "quarantine_path": value.quarantine_path,
+        "warnings": list(value.warnings),
+    }
+    if value.writeback is not None:
+        row["writeback"] = value.writeback.to_dict()
+        row["canonical_path"] = f"memory/{value.writeback.artifact.canonical_path}"
+    return row
+
+
+def _stage(args: argparse.Namespace) -> int:
+    _, runtime, actor, workflow = _context(push_remote=False)
+    _require(runtime, actor, Permission.WRITE, args.source)
+    payload = _input_bytes(args.input)
+    item = workflow.normalize_item(_source(args, actor, payload), payload)
+    receipt = workflow.quarantine(item)
+    print(json.dumps(_receipt(receipt), ensure_ascii=False))
+    return 0
+
+
+def _review(args: argparse.Namespace) -> int:
+    _, runtime, actor, workflow = _context(push_remote=not args.no_push)
+    _require(runtime, actor, Permission.READ, args.source)
+    _require(runtime, actor, Permission.PROMOTE, args.source)
+    item = workflow.load_quarantined(source_id=args.source, artifact_id=args.artifact)
+    receipt = workflow.admit(
+        actor,
+        item,
+        IngestReview(args.approve, actor.actor.actor_id, datetime.now(UTC), args.notes),
+    )
+    print(json.dumps(_receipt(receipt), ensure_ascii=False))
+    return 0 if receipt.phase.value in {"admitted", "partial", "rejected"} else 1
+
+
+def _add(args: argparse.Namespace) -> int:
+    _, runtime, actor, workflow = _context(push_remote=not args.no_push)
+    # Admission is explicit in this command. Resolve both permissions before
+    # opening external bytes so a denied source never enters local staging.
+    _require(runtime, actor, Permission.WRITE, args.source)
+    _require(runtime, actor, Permission.PROMOTE, args.source)
+    payload = _input_bytes(args.input)
+    item = workflow.normalize_item(_source(args, actor, payload), payload)
+    workflow.quarantine(item)
+    receipt = workflow.admit(
+        actor,
+        item,
+        IngestReview(True, actor.actor.actor_id, datetime.now(UTC), args.notes),
+    )
+    print(json.dumps(_receipt(receipt), ensure_ascii=False))
+    return 0 if receipt.phase.value in {"admitted", "partial"} else 1
+
+
+def _register(args: argparse.Namespace) -> int:
+    _, runtime, actor, _ = _context(push_remote=not args.no_push)
+    receipt = runtime.register_ingest_manifest(
+        actor,
+        source_id=args.source,
+        manifest_path=Path(args.manifest),
+        source_path=Path(args.source_record),
+    )
+    print(
+        json.dumps(
+            {
+                "status": "registered",
+                "source": receipt.source_id,
+                "documents": receipt.document_count,
+                "manifest_revision": receipt.manifest_revision,
+                "git_revision": receipt.sync.local_revision,
+                "warnings": list(receipt.sync.warnings),
+            },
+            ensure_ascii=False,
+        )
+    )
+    return 0
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="egregore-runtime-ingest")
+    commands = parser.add_subparsers(dest="command", required=True)
+    register = commands.add_parser("register")
+    register.add_argument("--source", required=True)
+    register.add_argument("--manifest", required=True)
+    register.add_argument("--source-record", required=True)
+    register.add_argument("--no-push", action="store_true")
+    register.set_defaults(handler=_register)
+
+    def add_source_arguments(command):
+        command.add_argument("--input", required=True)
+        command.add_argument("--source", required=True)
+        command.add_argument("--source-type", required=True)
+        command.add_argument("--title", required=True)
+        command.add_argument("--revision")
+        command.add_argument("--observed-at")
+        command.add_argument("--source-uri")
+        command.add_argument("--source-path")
+        command.add_argument("--visibility", action="append", default=[])
+        command.add_argument("--boundary", action="append", default=[])
+        command.add_argument("--workstream")
+
+    stage = commands.add_parser("stage")
+    add_source_arguments(stage)
+    stage.set_defaults(handler=_stage)
+    review = commands.add_parser("review")
+    review.add_argument("--source", required=True)
+    review.add_argument("--artifact", required=True)
+    outcome = review.add_mutually_exclusive_group(required=True)
+    outcome.add_argument("--approve", action="store_true")
+    outcome.add_argument("--reject", dest="approve", action="store_false")
+    review.add_argument("--notes")
+    review.add_argument("--no-push", action="store_true")
+    review.set_defaults(handler=_review)
+    add = commands.add_parser("add")
+    add_source_arguments(add)
+    add.add_argument("--notes")
+    add.add_argument("--no-push", action="store_true")
+    add.set_defaults(handler=_add)
+    return parser
+
+
+def main() -> int:
+    args = _parser().parse_args()
+    return args.handler(args)
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except (OSError, PermissionError, ValueError) as exc:
+        print(f"ingest: {exc}", file=os.sys.stderr)
+        raise SystemExit(1) from exc

package/runtime/prime/egregore_runtime/ingest_index_cli.py

@@ -0,0 +1,147 @@
+"""Compatibility bridge from ``bin/ingest.sh`` to the Local Retriever.
+
+All QMD-specific behavior remains in ``adapters.qmd``.  This module only
+selects the ingest corpus/collection and translates domain results into the
+legacy JSON shape consumed by the ingest boundary filter.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import uuid
+from pathlib import Path, PurePosixPath
+
+from egregore_runtime.adapters.qmd import QmdAdapterError, QmdLocalRetriever
+from egregore_runtime.contracts import RetrievalMode, RetrievalRequest
+from egregore_runtime.identity import LocalIdentityResolver
+from egregore_runtime.observe import DefaultOrgContextCompiler
+from egregore_runtime.policy import all_access_dogfood_policy
+from egregore_runtime.telemetry import LocalTelemetrySink
+
+
+def _root() -> Path:
+    configured = os.environ.get("EGREGORE_ROOT")
+    return Path(configured).resolve() if configured else Path.cwd().resolve()
+
+
+def _config(root: Path) -> dict[str, object]:
+    try:
+        return json.loads((root / "egregore.json").read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return {}
+
+
+def _ingest_root(root: Path) -> Path:
+    configured = os.environ.get("EGREGORE_INGEST_ROOT")
+    return Path(configured).resolve() if configured else (root / ".egregore" / "ingest").resolve()
+
+
+def _retriever(root: Path) -> QmdLocalRetriever:
+    return QmdLocalRetriever(
+        repository_root=root,
+        memory_root=_ingest_root(root),
+        collection_purpose="ingest",
+    )
+
+
+def _update(args: argparse.Namespace) -> int:
+    retriever = _retriever(_root())
+    health = retriever.update()
+    if args.background_embed:
+        health = retriever.embed_background()
+    print(json.dumps(health.to_dict(), ensure_ascii=False))
+    return 0
+
+
+def _query(args: argparse.Namespace) -> int:
+    root = _root()
+    retriever = _retriever(root)
+    actor = LocalIdentityResolver(root).resolve(
+        session_id=os.environ.get("EGREGORE_SESSION_ID", f"session_{uuid.uuid4().hex}"),
+        harness=os.environ.get("EGREGORE_RUNTIME", "shell"),
+    )
+    request = RetrievalRequest(
+        request_id=f"ingest-search-{uuid.uuid4()}",
+        org_id=actor.profile.org_id,
+        actor_id=actor.actor.actor_id,
+        task=args.query,
+        lex=(args.query,),
+        vec=(args.query,),
+        top_k=args.limit,
+        mode=RetrievalMode.HYBRID,
+        open_sources=False,
+    )
+    policy = all_access_dogfood_policy(
+        org_id=actor.profile.org_id,
+        actor_id=actor.actor.actor_id,
+    )
+    compiler = DefaultOrgContextCompiler(
+        authorizer=policy,
+        retriever=retriever,
+        telemetry=LocalTelemetrySink(
+            state_file=root / ".egregore-state.json",
+            instance_root=(root / "memory").resolve(),
+        ),
+    )
+    context = compiler.observe(
+        actor,
+        request,
+        token_budget=actor.profile.default_context_budget,
+    )
+    collection = retriever.collection
+    rows = []
+    for evidence in context.evidence:
+        relative = evidence.canonical_path.removeprefix("memory/")
+        rows.append(
+            {
+                "file": f"qmd://{collection}/{relative}",
+                "title": PurePosixPath(relative).stem,
+                "snippet": evidence.content,
+                "score": 0.0,
+                "artifact_id": evidence.artifact_id,
+                "rank": len(rows) + 1,
+                "retrieval_types": [RetrievalMode.HYBRID.value],
+                "index_spec_version": context.freshness["index_spec_version"],
+            }
+        )
+    print(json.dumps(rows, ensure_ascii=False))
+    return 0
+
+
+def _status(_: argparse.Namespace) -> int:
+    health = _retriever(_root()).health()
+    print(json.dumps(health.to_dict(), ensure_ascii=False))
+    return 0 if health.available else 1
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="egregore-runtime-ingest-index")
+    commands = parser.add_subparsers(dest="command", required=True)
+    update = commands.add_parser("update")
+    update.add_argument("--background-embed", action="store_true")
+    update.set_defaults(handler=_update)
+    query = commands.add_parser("query")
+    query.add_argument("--query", required=True)
+    query.add_argument("--limit", type=int, default=200)
+    query.set_defaults(handler=_query)
+    status = commands.add_parser("status")
+    status.set_defaults(handler=_status)
+    return parser
+
+
+def main() -> int:
+    args = _parser().parse_args()
+    if getattr(args, "limit", 1) < 1:
+        raise QmdAdapterError("retrieval limit must be positive")
+    return args.handler(args)
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except QmdAdapterError as exc:
+        print(f"ingest index: {exc}", file=sys.stderr)
+        raise SystemExit(1) from exc

package/runtime/prime/egregore_runtime/invitation_cli.py

@@ -0,0 +1,51 @@
+"""Runtime-neutral command adapter for organization invitations."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+from uuid import uuid4
+
+from .adapters.invitation import ShellInvitationTransport
+from .errors import AuthorizationDenied
+from .invitations import InvitationService
+from .runtime import local_runtime
+
+
+def _root() -> Path:
+    return Path(os.environ.get("EGREGORE_ROOT") or Path.cwd()).resolve()
+
+
+def _session_id(root: Path) -> str:
+    value = os.environ.get("EGREGORE_SESSION_ID", "").strip()
+    marker = root / ".egregore-session-id"
+    if not value and marker.is_file():
+        value = marker.read_text(encoding="utf-8").splitlines()[0].strip()
+    return value or f"invite_{uuid4().hex}"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(prog="egregore-invite")
+    parser.add_argument("username")
+    args = parser.parse_args()
+    root = _root()
+    runtime = local_runtime(root, push_remote=True)
+    actor = runtime.resolve_actor(
+        session_id=_session_id(root),
+        harness=os.environ.get("EGREGORE_RUNTIME", "shell"),
+    )
+    receipt = InvitationService(
+        runtime, ShellInvitationTransport(root, push_remote=True)
+    ).invite(actor, provider_username=args.username)
+    print(json.dumps(receipt.to_dict(), ensure_ascii=False, sort_keys=True))
+    return 0 if receipt.status == "accepted" else 1
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except (AuthorizationDenied, OSError, ValueError, json.JSONDecodeError) as exc:
+        print(f"invite: {exc}", file=os.sys.stderr)
+        raise SystemExit(1) from exc

package/runtime/prime/egregore_runtime/invitations.py

@@ -0,0 +1,116 @@
+"""Provider-independent organization invitation domain service.
+
+An invitation is an authorization to create a future membership.  It is not
+an AccountIdentity, ActorIdentity, or OrgMembership, and linked-provider
+handles remain transport aliases until the invitee authenticates.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+import re
+from typing import Mapping, Protocol
+
+from .contracts import ActorContext, Permission, Serializable
+from .errors import AuthorizationDenied
+
+
+_PROVIDER_USERNAME = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?$")
+
+
+@dataclass(frozen=True, slots=True)
+class InvitationTransportResult(Serializable):
+    status: str
+    provider_status: str
+    memory_status: str
+    managed_access: tuple[Mapping[str, str], ...] = ()
+    invite_url: str | None = None
+    join_command: str | None = None
+    group_link: str | None = None
+    manual_access_url: str | None = None
+    warnings: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class InvitationReceipt(Serializable):
+    status: str
+    org_id: str
+    org_name: str
+    invited_by_actor_id: str
+    provider: str
+    provider_username: str
+    provider_status: str
+    memory_status: str
+    managed_access: tuple[Mapping[str, str], ...] = ()
+    invite_url: str | None = None
+    join_command: str | None = None
+    group_link: str | None = None
+    manual_access_url: str | None = None
+    warnings: tuple[str, ...] = ()
+
+
+class InvitationTransport(Protocol):
+    """Current provider/control-plane transport behind the Runtime seam."""
+
+    def invite(
+        self,
+        actor: ActorContext,
+        *,
+        provider: str,
+        provider_username: str,
+    ) -> InvitationTransportResult: ...
+
+
+class InvitationService:
+    """Authorize one future membership before invoking any external effect."""
+
+    def __init__(self, runtime, transport: InvitationTransport) -> None:
+        self.runtime = runtime
+        self.transport = transport
+
+    @staticmethod
+    def normalize_provider_username(value: str) -> str:
+        username = value.strip()
+        if not _PROVIDER_USERNAME.fullmatch(username) or "--" in username:
+            raise ValueError("a valid GitHub username is required")
+        return username
+
+    def invite(
+        self,
+        actor: ActorContext,
+        *,
+        provider_username: str,
+        provider: str = "github",
+    ) -> InvitationReceipt:
+        username = self.normalize_provider_username(provider_username)
+        if provider != "github":
+            raise ValueError("the installed invitation transport supports GitHub aliases only")
+
+        resource = f"org:{actor.profile.org_id}:membership-invite"
+        decision = self.runtime.authorize(actor, Permission.ADMINISTER, (resource,))
+        if not decision.allowed:
+            raise AuthorizationDenied("; ".join(decision.reasons))
+        if actor.membership is None or actor.membership.status != "active":
+            raise AuthorizationDenied("an active organization membership is required")
+
+        result = self.transport.invite(
+            actor,
+            provider=provider,
+            provider_username=username,
+        )
+        return InvitationReceipt(
+            status=result.status,
+            org_id=actor.profile.org_id,
+            org_name=actor.profile.name,
+            invited_by_actor_id=actor.actor.actor_id,
+            provider=provider,
+            provider_username=username,
+            provider_status=result.provider_status,
+            memory_status=result.memory_status,
+            managed_access=result.managed_access,
+            invite_url=result.invite_url,
+            join_command=result.join_command,
+            group_link=result.group_link,
+            manual_access_url=result.manual_access_url,
+            warnings=result.warnings,
+        )