@agentikos/omega-os 0.2.0 → 0.19.6

package/omega/Agentik_Engine/omega_engine/vault.py

@@ -0,0 +1,342 @@
+"""Encrypted secrets vault — age if available, chmod 600 plaintext fallback.
+
+The vault is the single source for *how* a secret reaches disk. It exists so
+that the rest of the engine never duplicates the "if age then encrypt else
+chmod 600" logic.
+
+Backends
+--------
+* **age** (preferred) — uses the system `age` and `age-keygen` binaries.
+  At first write, we generate an age identity at
+  `Agentik_Extra/etc/secrets/.vault-key` (mode 600) plus its public recipient
+  at `.vault-pub` (mode 644). Every secret is written as `<ref>.age` next to
+  the key file.
+* **plain** (fallback) — when age is not installed, secrets are still stored
+  with mode 600 under `Agentik_Extra/etc/secrets/<ref>`. The doctor flags
+  this as "unencrypted" so the operator can install age and re-write.
+
+API
+---
+* ``vault_init(home)`` — idempotent. Ensures the secrets dir exists, mode 700.
+  If age is installed, ensures the keypair exists. Returns a status dict.
+* ``vault_write(home, ref, value)`` — write a secret. Returns the path.
+* ``vault_read(home, ref) -> str | None`` — read a secret, or None if absent.
+* ``vault_status(home)`` — summary for ``omega doctor``.
+
+The vault is intentionally narrow: one value per ref, plain string. Tokens
+are strings; structured secrets (mcp-<id>.env) keep their dotenv layout for
+now and can migrate later.
+"""
+from __future__ import annotations
+
+import os
+import shutil
+import stat
+import subprocess
+from pathlib import Path
+from typing import Any
+
+
+def _safe_ref(ref: str) -> str:
+    """Refuse path-traversal — `ref` is a flat filename only."""
+    if not ref or "/" in ref or "\\" in ref or ".." in ref:
+        raise ValueError(f"invalid secret ref: {ref!r}")
+    if ref.startswith("."):
+        raise ValueError(f"secret ref must not start with '.': {ref!r}")
+    return ref
+
+
+def _safe_project_slug(project: str) -> str:
+    """A project slug is a flat name — no traversal."""
+    if not project or "/" in project or "\\" in project or ".." in project:
+        raise ValueError(f"invalid project slug: {project!r}")
+    if project.startswith("."):
+        raise ValueError(f"project slug must not start with '.': {project!r}")
+    return project
+
+
+def _secrets_dir(home: Path, project: str | None = None) -> Path:
+    """Return the vault directory, creating it with mode 700.
+
+    When ``project`` is set, the directory is
+    ``Agentik_Coding/projects/<slug>/.secrets/`` — a per-project vault that
+    sits next to the project's code. Otherwise the global vault at
+    ``Agentik_Extra/etc/secrets/``.
+    """
+    if project is not None:
+        slug = _safe_project_slug(project)
+        d = (Path(home) / "Agentik_Coding" / "projects" / slug / ".secrets")
+    else:
+        d = Path(home) / "Agentik_Extra" / "etc" / "secrets"
+    d.mkdir(parents=True, exist_ok=True)
+    try:
+        os.chmod(d, 0o700)
+    except OSError:
+        pass
+    return d
+
+
+def _global_key_dir(home: Path) -> Path:
+    """Where the age keypair lives. Always the global secrets dir — every
+    per-project vault encrypts to the same recipient so the engine can
+    decrypt anything with one key."""
+    return _secrets_dir(home, project=None)
+
+
+def _age_available() -> bool:
+    return shutil.which("age") is not None and shutil.which("age-keygen") is not None
+
+
+def vault_init(home: str | Path) -> dict[str, Any]:
+    """Idempotent vault setup. Generates the age keypair on first call when
+    age is installed; no-op afterwards. Always safe to call.
+
+    Returns: ``{ "backend": "age" | "plain", "key_path"?, "pub_path"?, ... }``
+    """
+    home = Path(home)
+    sec = _global_key_dir(home)
+    if not _age_available():
+        return {"backend": "plain", "age_installed": False, "secrets_dir": str(sec)}
+
+    key_path = sec / ".vault-key"
+    pub_path = sec / ".vault-pub"
+    if key_path.exists() and pub_path.exists():
+        return {
+            "backend": "age", "age_installed": True,
+            "key_path": str(key_path), "pub_path": str(pub_path),
+            "secrets_dir": str(sec),
+        }
+
+    # Generate a fresh keypair.
+    proc = subprocess.run(
+        ["age-keygen", "-o", str(key_path)],
+        check=False, capture_output=True, text=True, timeout=30,
+    )
+    if proc.returncode != 0:
+        return {
+            "backend": "plain", "age_installed": True,
+            "secrets_dir": str(sec),
+            "warning": f"age-keygen failed: {proc.stderr.strip()[:200]}",
+        }
+    try:
+        os.chmod(key_path, 0o600)
+    except OSError:
+        pass
+
+    # age-keygen prints "Public key: age1xyz..." to stderr.
+    pub: str | None = None
+    for line in (proc.stderr or "").splitlines():
+        s = line.strip()
+        if s.startswith("Public key:"):
+            pub = s.split(":", 1)[1].strip()
+            break
+    if not pub:
+        # Some age-keygen versions write the public key as a comment in the
+        # private-key file. Try to extract it from there.
+        for line in key_path.read_text().splitlines():
+            s = line.strip()
+            if s.lower().startswith("# public key:"):
+                pub = s.split(":", 1)[1].strip()
+                break
+    if not pub:
+        return {
+            "backend": "plain", "age_installed": True,
+            "secrets_dir": str(sec),
+            "warning": "could not extract public key from age-keygen output",
+        }
+
+    pub_path.write_text(pub + "\n")
+    try:
+        os.chmod(pub_path, 0o644)
+    except OSError:
+        pass
+    return {
+        "backend": "age", "age_installed": True,
+        "key_path": str(key_path), "pub_path": str(pub_path),
+        "secrets_dir": str(sec),
+    }
+
+
+def vault_write(
+    home: str | Path,
+    ref: str,
+    value: str,
+    *,
+    project: str | None = None,
+) -> Path:
+    """Write a secret. age-encrypted when available, mode-600 plaintext otherwise.
+
+    With ``project=<slug>``, the secret is scoped to that project's vault at
+    ``Agentik_Coding/projects/<slug>/.secrets/`` (encrypted to the SAME age
+    recipient as the global vault, so one key decrypts everything). Without
+    ``project``, the global vault is used.
+
+    Returns the on-disk path written. Idempotent for a given (ref, value):
+    re-writing rotates the secret in place.
+    """
+    ref = _safe_ref(ref)
+    if not isinstance(value, str) or not value:
+        raise ValueError("secret value must be a non-empty string")
+
+    home = Path(home)
+    # vault_init always operates on the global dir so the keypair is
+    # discoverable from any project.
+    info = vault_init(home)
+    sec = _secrets_dir(home, project=project)
+
+    if info["backend"] == "age":
+        pub = Path(info["pub_path"]).read_text().strip()
+        target = sec / f"{ref}.age"
+        proc = subprocess.run(
+            ["age", "-r", pub, "-o", str(target)],
+            input=value, text=True, check=False, capture_output=True, timeout=30,
+        )
+        if proc.returncode != 0:
+            raise RuntimeError(
+                f"age encrypt failed for ref={ref}: {proc.stderr.strip()[:200]}"
+            )
+        try:
+            os.chmod(target, 0o600)
+        except OSError:
+            pass
+        # Best-effort cleanup of a previous plaintext for the same ref.
+        plain_old = sec / ref
+        if plain_old.exists():
+            try:
+                plain_old.unlink()
+            except OSError:
+                pass
+        return target
+
+    # Plain fallback.
+    target = sec / ref
+    fd = os.open(str(target), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    try:
+        with os.fdopen(fd, "w") as fh:
+            fh.write(value)
+    finally:
+        try:
+            os.close(fd)
+        except OSError:
+            pass
+    try:
+        os.chmod(target, stat.S_IRUSR | stat.S_IWUSR)  # 0o600
+    except OSError:
+        pass
+    return target
+
+
+def vault_read(
+    home: str | Path,
+    ref: str,
+    *,
+    project: str | None = None,
+) -> str | None:
+    """Read a secret. Tries age then plain; returns None if neither exists.
+
+    ``project=<slug>`` reads from the per-project vault.
+
+    Raises RuntimeError if an encrypted file exists but decryption fails
+    (so callers don't silently treat a broken vault as missing).
+    """
+    ref = _safe_ref(ref)
+    home = Path(home)
+    sec = _secrets_dir(home, project=project)
+    age_file = sec / f"{ref}.age"
+    plain_file = sec / ref
+
+    if age_file.exists():
+        # The age key always lives in the global dir.
+        key = _global_key_dir(home) / ".vault-key"
+        if not key.exists():
+            raise RuntimeError(
+                f"encrypted secret {age_file} exists but vault key {key} is missing"
+            )
+        if not _age_available():
+            raise RuntimeError(
+                f"encrypted secret {age_file} exists but `age` is not installed"
+            )
+        proc = subprocess.run(
+            ["age", "-d", "-i", str(key), str(age_file)],
+            check=False, capture_output=True, text=True, timeout=30,
+        )
+        if proc.returncode != 0:
+            raise RuntimeError(
+                f"age decrypt failed for ref={ref}: {proc.stderr.strip()[:200]}"
+            )
+        return proc.stdout
+    if plain_file.exists():
+        return plain_file.read_text()
+    return None
+
+
+def _file_has_real_secret(path: Path) -> bool:
+    """True iff the file contains at least one non-comment KEY=VALUE line.
+
+    A file with only commented placeholders like
+        # COMPOSIO_API_KEY=  # set this when ready
+    has nothing to leak and is not counted as a plaintext secret.
+    """
+    try:
+        text = path.read_text(errors="replace")
+    except OSError:
+        return False
+    if not text.strip():
+        return False
+    for raw in text.splitlines():
+        ln = raw.strip()
+        if not ln or ln.startswith("#"):
+            continue
+        if "=" not in ln:
+            # A bare opaque value (e.g. an OAuth token written as the whole
+            # file body) still counts as a real secret.
+            return True
+        k, _, v = ln.partition("=")
+        if v.strip().strip('"').strip("'"):
+            return True
+    return False
+
+
+def vault_status(home: str | Path, *, project: str | None = None) -> dict[str, Any]:
+    """Summary of the vault for `omega doctor`.
+
+    With ``project=<slug>``, summarises the per-project vault only.
+    """
+    home = Path(home)
+    sec = _secrets_dir(home, project=project)
+    info = vault_init(home)
+    files = [p for p in sec.iterdir() if p.is_file() and not p.name.startswith(".")]
+    encrypted = [p.name for p in files if p.suffix == ".age"]
+    plain_files = [p for p in files if p.suffix != ".age"]
+    # Discriminate real plaintext secrets from placeholder dotenv files.
+    real_plain = [p.name for p in plain_files if _file_has_real_secret(p)]
+    placeholders = [p.name for p in plain_files if p.name not in real_plain]
+    return {
+        "backend": info["backend"],
+        "age_installed": info.get("age_installed", False),
+        "encrypted_count": len(encrypted),
+        "plaintext_count": len(real_plain),
+        "placeholder_count": len(placeholders),
+        "secrets_dir": str(sec),
+        "project": project,
+    }
+
+
+def list_project_vaults(home: str | Path) -> list[dict[str, Any]]:
+    """Enumerate every per-project vault under
+    ``Agentik_Coding/projects/<slug>/.secrets/``. Returns a list of vault
+    statuses, one per project that has any vault (encrypted or plain).
+    """
+    home = Path(home)
+    projects_root = home / "Agentik_Coding" / "projects"
+    out: list[dict[str, Any]] = []
+    if not projects_root.is_dir():
+        return out
+    for proj_dir in sorted(projects_root.iterdir()):
+        if not proj_dir.is_dir():
+            continue
+        sec = proj_dir / ".secrets"
+        if not sec.is_dir():
+            continue
+        out.append(vault_status(home, project=proj_dir.name))
+    return out

package/omega/Agentik_Engine/omega_engine/webhooks.py

@@ -0,0 +1,262 @@
+"""Inbound webhooks — GitHub / Linear / generic HMAC-SHA256.
+
+The autonomous supervisor already has ``wake_webhook(path, payload)``. The
+gap was the *HTTP front door*: nothing was listening on a port, nothing
+verified signatures. This module is that front door.
+
+Specs live in ``Agentik_SSOT/webhooks/webhooks.yaml`` (one path = one
+spec). The engine daemon's HTTP server handles ``POST /webhook/<id>``:
+it locates the spec, reads the source-specific signature header, looks
+up the HMAC secret in the vault, verifies, and calls
+``supervisor.wake_webhook(path, payload)``. On a valid signature the
+caller gets ``202 Accepted`` immediately — the mission runs async.
+
+We never accept an unsigned webhook. The default is ``source: generic``
+which still requires HMAC-SHA256 in a configurable header.
+
+Why we ship this small (no fancy framework):
+
+  * The engine daemon is already on a loopback port. The operator's
+    reverse proxy (nginx, Cloudflare Tunnel) is responsible for the
+    public face. We do auth, not networking.
+  * Signature verification is the only thing that matters — without it,
+    a webhook endpoint is a remote code execution waiting to happen.
+"""
+from __future__ import annotations
+
+import hashlib
+import hmac
+import json
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+# Source-specific header names. The signature value's wire format is
+# different per source; see _verify_source for the parsing.
+_SIGNATURE_HEADER = {
+    "github":  "X-Hub-Signature-256",
+    "linear":  "Linear-Signature",
+    "generic": "X-Omega-Signature",
+}
+
+
+@dataclass
+class WebhookSpec:
+    """One inbound webhook endpoint."""
+    id: str
+    path: str                            # "/webhook/github-prs"
+    source: str = "generic"              # "github" | "linear" | "generic"
+    secret_ref: str = ""                 # vault ref for the HMAC secret
+    description: str = ""
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> "WebhookSpec":
+        return WebhookSpec(
+            id=str(d["id"]),
+            path=str(d.get("path") or f"/webhook/{d['id']}"),
+            source=str(d.get("source", "generic")),
+            secret_ref=str(d.get("secret_ref",
+                                  f"WEBHOOK_{d['id'].upper().replace('-', '_')}")),
+            description=str(d.get("description", "")),
+        )
+
+
+@dataclass
+class WebhookRegistry:
+    """All webhook specs known to the engine."""
+    specs: list[WebhookSpec] = field(default_factory=list)
+
+    def by_path(self, path: str) -> WebhookSpec | None:
+        for s in self.specs:
+            if s.path == path:
+                return s
+        return None
+
+    def by_id(self, id: str) -> WebhookSpec | None:
+        for s in self.specs:
+            if s.id == id:
+                return s
+        return None
+
+
+def load_registry(omega_home: str | Path) -> WebhookRegistry:
+    """Read ``Agentik_SSOT/webhooks/webhooks.yaml``. Empty if missing."""
+    path = (Path(omega_home) / "Agentik_SSOT" / "webhooks" / "webhooks.yaml")
+    if not path.exists():
+        return WebhookRegistry()
+    data = yaml.safe_load(path.read_text()) or {}
+    specs = [
+        WebhookSpec.from_dict(d)
+        for d in (data.get("webhooks") or [])
+        if isinstance(d, dict) and d.get("id")
+    ]
+    return WebhookRegistry(specs=specs)
+
+
+def signature_header_for(source: str) -> str:
+    """Return the header name the signature is expected in."""
+    return _SIGNATURE_HEADER.get(source.lower(), _SIGNATURE_HEADER["generic"])
+
+
+def verify_signature(
+    source: str,
+    body: bytes,
+    signature_header_value: str | None,
+    secret: str,
+    *,
+    timestamp_header_value: str | None = None,
+    replay_window_s: int = 300,
+    now_s: float | None = None,
+) -> tuple[bool, str]:
+    """Return (ok, reason) for a webhook signature.
+
+    Hard rule: missing signature → False. Missing secret → False. We
+    NEVER let an unsigned webhook through.
+
+    Replay protection
+    -----------------
+    When ``timestamp_header_value`` is provided, we require the timestamp
+    to be within ``replay_window_s`` seconds of ``now_s``. An attacker who
+    captured a valid signed body can otherwise replay it forever. Callers
+    SHOULD pass the timestamp header (X-Omega-Timestamp / X-Hub-Timestamp /
+    Linear-Delivery-Timestamp); when absent we accept the request (HMAC
+    alone is the v0.5 contract; timestamp is additive).
+
+    Source-specific notes:
+
+      * ``github`` ships ``X-Hub-Signature-256: sha256=<hex>``.
+      * ``linear`` ships ``Linear-Signature: <hex>`` (no prefix).
+      * ``generic`` expects ``X-Omega-Signature: sha256=<hex>``.
+    """
+    if not signature_header_value:
+        return False, "missing signature header"
+    if not secret:
+        return False, "secret not set in vault"
+
+    # Replay window check (additive, off when no timestamp supplied).
+    if timestamp_header_value is not None:
+        try:
+            ts = float(timestamp_header_value)
+        except (TypeError, ValueError):
+            return False, "timestamp header is not numeric"
+        import time as _t
+        wall = now_s if now_s is not None else _t.time()
+        if abs(wall - ts) > replay_window_s:
+            return False, (
+                f"timestamp outside ±{replay_window_s}s replay window "
+                f"(now={wall:.0f} hdr={ts:.0f})"
+            )
+
+    src = source.lower()
+    raw_sig = signature_header_value.strip()
+    if src in ("github", "generic"):
+        # `sha256=<hex>` form
+        if not raw_sig.startswith("sha256="):
+            return False, "signature missing sha256= prefix"
+        provided = raw_sig.split("=", 1)[1]
+    elif src == "linear":
+        # raw hex
+        provided = raw_sig
+    else:
+        return False, f"unknown source {source!r}"
+
+    mac = hmac.new(
+        secret.encode("utf-8"), body, hashlib.sha256,
+    ).hexdigest()
+    if not hmac.compare_digest(mac, provided):
+        return False, "signature mismatch"
+    return True, "ok"
+
+
+def _load_secret(omega_home: str | Path, secret_ref: str) -> str:
+    """Best-effort vault read."""
+    if not secret_ref:
+        return ""
+    try:
+        from omega_engine.vault import vault_read
+        v = vault_read(omega_home, secret_ref)
+        return v or ""
+    except Exception:  # noqa: BLE001
+        return ""
+
+
+def handle_webhook(
+    omega_home: str | Path,
+    *,
+    path: str,
+    headers: dict[str, str],
+    body: bytes,
+    supervisor=None,
+) -> dict[str, Any]:
+    """Run the gate for one inbound webhook.
+
+    Returns ``{status: int, ok: bool, reason: str, fired: [charter_ids]}``.
+    Status codes: 200 (already-known but no charter fired), 202 (charters
+    fired), 401 (bad signature), 404 (unknown path), 500 (internal).
+
+    ``supervisor`` is optional — pass an ``AutonomousSupervisor`` to
+    actually fire charters. Without one, we still verify but don't fire
+    (useful for ``omega webhook test``).
+    """
+    home = Path(omega_home)
+    registry = load_registry(home)
+    spec = registry.by_path(path)
+    if spec is None:
+        return {"status": 404, "ok": False, "reason": "no spec for path",
+                "fired": []}
+
+    secret = _load_secret(home, spec.secret_ref)
+    sig_header_name = signature_header_for(spec.source)
+    sig_value = None
+    # Headers come from a stdlib BaseHTTPRequestHandler which lower-cases
+    # access via dict-style only when wrapped; here we normalise both sides.
+    for k, v in headers.items():
+        if k.lower() == sig_header_name.lower():
+            sig_value = v
+            break
+    ok, reason = verify_signature(spec.source, body, sig_value, secret)
+    if not ok:
+        return {"status": 401, "ok": False, "reason": reason, "fired": []}
+
+    # Parse payload best-effort; webhooks SHOULD be JSON.
+    try:
+        payload = json.loads(body.decode("utf-8") or "{}")
+    except (json.JSONDecodeError, UnicodeDecodeError):
+        payload = {"_raw": body.decode("utf-8", errors="replace")[:1000]}
+
+    fired: list[str] = []
+    if supervisor is not None:
+        try:
+            fired = supervisor.wake_webhook(path, payload) or []
+        except Exception as exc:  # noqa: BLE001
+            return {
+                "status": 500, "ok": False,
+                "reason": f"supervisor raised: {exc}"[:300],
+                "fired": [],
+            }
+
+    return {
+        "status": 202 if fired else 200,
+        "ok": True,
+        "reason": "ok",
+        "fired": fired,
+    }
+
+
+def generate_test_signature(source: str, body: bytes, secret: str) -> str:
+    """Build a valid signature header value for the given source.
+
+    Used by ``omega webhook test`` and by the integration tests so we can
+    POST a synthetic webhook to the engine and confirm the gate works.
+    """
+    mac = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest()
+    src = source.lower()
+    if src in ("github", "generic"):
+        return f"sha256={mac}"
+    if src == "linear":
+        return mac
+    raise ValueError(f"unknown source: {source}")