arkaos 4.45.0 → 4.46.0

package/THE-ARKAOS-GUIDE.md

@@ -1,6 +1,6 @@
 # The ArkaOS Guide
 
-> v4.45.0 — 89 agents, 17 departments, 332 skills, 297 commands, 18 ADRs.
+> v4.46.0 — 89 agents, 17 departments, 332 skills, 297 commands, 19 ADRs.
 > One file, everything you need to start. Generated by `scripts/guide_gen.py` — never hand-edited.
 
 ## What it is

package/VERSION

@@ -1 +1 @@
-4.45.0
+4.46.0

package/core/egress/__init__.py

@@ -0,0 +1,20 @@
+"""Egress policy — deny-by-default guard for anything leaving the machine.
+
+Workstream D of the 2026-07 repair campaign integrates notebooklm-py,
+and everything uploaded there leaves for a Google account. The operator
+serves confidential clients, so this package ships FIRST and every
+other D-slice depends on it:
+
+- ``policy``    — ``evaluate`` (never raises) / ``enforce`` (raises
+  :class:`~core.egress.policy.EgressDeniedError`), ordered checks
+- ``redact``    — fail-closed redaction over
+  ``core.evals.sanitizer.sanitize_text``
+- ``allowlist`` — expiring exceptions; client identifiers are never
+  allowlistable
+- ``audit``     — hashed JSONL trail; no raw payloads, ever
+
+Fail-closed is the operator's standing directive (2026-07-26): with no
+``~/.arkaos/redaction-clients.json`` on disk there is nothing to prove
+a payload clean, so nothing leaves. ADR:
+``docs/adr/2026-07-31-egress-policy.md``.
+"""

package/core/egress/allowlist.py

@@ -0,0 +1,123 @@
+"""Expiring egress exceptions — narrow, auditable, never for clients.
+
+An allowlist entry permits ONE finding kind + token pair to leave,
+until it expires. Two kinds are unallowlistable by construction:
+``client-identifier`` (confidentiality is non-negotiable — the v2.18.0
+npm leak precedent) and ``redaction-config-missing`` (an exception to
+fail-closed would BE the open-by-default this package exists to
+prevent). Malformed or expired entries permit nothing; a broken
+allowlist degrades to deny, never to allow.
+
+File: ``~/.arkaos/egress/allowlist.json`` —
+``{"entries": [{"kind": "secret" | "home-path", "token": "<exact>",
+"reason": "...", "expires": "<ISO-8601>",
+"destinations": ["notebooklm"]}]}``. ``destinations`` is optional;
+absent means any destination.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+
+ALLOWLISTABLE_KINDS: frozenset[str] = frozenset({"secret", "home-path"})
+
+
+def default_allowlist_path(home: Path | None = None) -> Path:
+    return (home or Path.home()) / ".arkaos" / "egress" / "allowlist.json"
+
+
+@dataclass(frozen=True)
+class AllowlistEntry:
+    """One live exception, already validated and unexpired."""
+
+    kind: str
+    token: str
+    reason: str
+    expires: datetime
+    destinations: tuple[str, ...]  # empty = any destination
+
+    def permits(self, kind: str, token: str, destination: str) -> bool:
+        return (
+            self.kind == kind
+            and self.token == token
+            and (not self.destinations or destination in self.destinations)
+        )
+
+
+def active_entries(
+    path: Path, now: datetime | None = None
+) -> list[AllowlistEntry]:
+    """Live entries in the allowlist file; never raises.
+
+    Anything unparseable, incomplete, of an unallowlistable kind, or
+    past its expiry is dropped — a defect in the file narrows what is
+    permitted; it never widens it. The catch is deliberately broad
+    (QG D1 r1: a latin-1 file raised UnicodeDecodeError through the
+    original three-exception tuple; a naive ``now`` raised TypeError
+    at the expiry comparison): any failure here means "no exceptions
+    are in force", which is the deny direction.
+    """
+    moment = now or datetime.now(UTC)
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+        raw_entries = data.get("entries", [])
+        if not isinstance(raw_entries, list):
+            return []
+        return [
+            entry
+            for raw in raw_entries
+            if (entry := _parse_entry(raw)) is not None
+            and entry.expires > moment
+        ]
+    except Exception:
+        return []
+
+
+def permits(
+    entries: list[AllowlistEntry], kind: str, token: str, destination: str
+) -> bool:
+    """True when a live entry covers this exact finding."""
+    if kind not in ALLOWLISTABLE_KINDS:
+        return False
+    return any(e.permits(kind, token, destination) for e in entries)
+
+
+def _parse_entry(raw: object) -> AllowlistEntry | None:
+    if not isinstance(raw, dict):
+        return None
+    kind = raw.get("kind")
+    token = raw.get("token")
+    reason = raw.get("reason")
+    if kind not in ALLOWLISTABLE_KINDS or not token or not reason:
+        return None
+    expires = _parse_expiry(raw.get("expires"))
+    if expires is None:
+        return None
+    destinations = raw.get("destinations", [])
+    if not isinstance(destinations, list):
+        return None
+    return AllowlistEntry(
+        kind=str(kind),
+        token=str(token),
+        reason=str(reason),
+        expires=expires,
+        destinations=tuple(str(d) for d in destinations),
+    )
+
+
+def _parse_expiry(value: object) -> datetime | None:
+    """A usable, timezone-aware expiry — or None (entry dropped).
+
+    An entry with no expiry, a malformed one, or a naive timestamp is
+    invalid: an exception that cannot expire is a permanent hole.
+    """
+    if not isinstance(value, str):
+        return None
+    try:
+        expires = datetime.fromisoformat(value)
+    except ValueError:
+        return None
+    return expires if expires.tzinfo is not None else None

package/core/egress/audit.py

@@ -0,0 +1,127 @@
+"""Hashed egress audit trail — evidence of every decision, leak of none.
+
+One JSONL line per ``evaluate`` outcome (allow AND deny) in
+``~/.arkaos/egress/audit.jsonl`` (dir 0700, file 0600); a failed
+write is reported to the caller rather than swallowed. The line
+carries digests and finding KINDS with salted-hash tokens — never the
+payload, never a client name, never a secret value, never a raw path.
+The audit file itself must be safe to read aloud.
+
+``record`` returns False instead of raising: the caller
+(``policy.evaluate``) treats a failed audit as fail-closed — an ALLOW
+that cannot be audited flips to denied, and the flip itself is
+re-recorded best-effort.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import json
+import os
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+
+
+def default_audit_path(home: Path | None = None) -> Path:
+    return (home or Path.home()) / ".arkaos" / "egress" / "audit.jsonl"
+
+
+def default_salt_path(home: Path | None = None) -> Path:
+    return (home or Path.home()) / ".arkaos" / "egress" / ".audit-salt"
+
+
+def load_or_create_salt(path: Path) -> bytes:
+    """Per-install random salt for token hashing; created on first use.
+
+    The finding tokens this package hashes are LOW-ENTROPY and
+    enumerable — the client list, the fixed secret-label vocabulary,
+    paths under a known home — so an unsalted digest is a
+    confirm-a-guess lookup for anyone holding the candidate list (QG
+    D1 r2, Francisca M1: reversed in microseconds). Keyed with a salt,
+    the tokens confirm nothing to a reader who holds only the audit
+    file. A salt that cannot be read or created degrades to ``b""`` —
+    hashes stay raw-material-free; only the guess-confirmation
+    resistance is lost. An EMPTY read is treated as not-yet-created (a
+    concurrent creator's file exists before its bytes land — QG D1 r3
+    F-M1), so the create path retries briefly before degrading.
+    """
+    data = _read_salt(path)
+    if data:
+        return data
+    created = _create_salt(path)
+    if created is not None:
+        return created
+    for _ in range(3):  # a concurrent creator may have won; its write
+        time.sleep(0.005)  # may be in flight — bounded retry, degrade
+        data = _read_salt(path)
+        if data:
+            return data
+    return b""
+
+
+def _create_salt(path: Path) -> bytes | None:
+    """The new salt, or None when the file already exists (concurrent
+    creator) or the filesystem refuses."""
+    salt = os.urandom(16)
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        os.chmod(path.parent, 0o700)
+        fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+        with os.fdopen(fd, "wb") as handle:
+            handle.write(salt)
+        return salt
+    except OSError:
+        return None
+
+
+def _read_salt(path: Path) -> bytes:
+    try:
+        return path.read_bytes()
+    except OSError:
+        return b""
+
+
+def hash_token(token: str, salt: bytes = b"") -> str:
+    """Keyed 16-hex digest for a finding token.
+
+    Long enough to correlate entries across lines, and it carries no
+    raw material. With the per-install salt it is also not a
+    confirm-a-guess oracle: the tokens are low-entropy (client names,
+    secret labels, home paths), so an UNSALTED truncated hash would
+    confirm a candidate for anyone who already holds the list.
+    """
+    payload = token.encode("utf-8", errors="surrogatepass")
+    return hmac.new(salt, payload, hashlib.sha256).hexdigest()[:16]
+
+
+def record(entry: dict, path: Path, now: datetime | None = None) -> bool:
+    """Append one audit line; True on durable success, False otherwise.
+
+    The catch is broad by design: any failure — filesystem OR a
+    payload that will not serialize — reads as "not audited", and the
+    caller flips an unaudited ALLOW to denied. ``os.chmod(path, ...)``
+    runs on every write, not only at creation, so a pre-existing file
+    with looser permissions is repaired rather than trusted (QG D1 r1
+    M2).
+    """
+    try:
+        stamped = {
+            "ts": (now or datetime.now(UTC)).isoformat(),
+            **entry,
+        }
+        line = json.dumps(stamped, ensure_ascii=False, sort_keys=True)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        os.chmod(path.parent, 0o700)
+        fd = os.open(
+            path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600
+        )
+        with os.fdopen(fd, "a", encoding="utf-8") as handle:
+            os.chmod(path, 0o600)
+            handle.write(line + "\n")
+            handle.flush()
+            os.fsync(handle.fileno())
+        return True
+    except Exception:
+        return False

package/core/egress/policy.py

@@ -0,0 +1,261 @@
+"""Deny-by-default egress policy — the guard every upload passes.
+
+Check order, each one sufficient to deny:
+
+1. **Redaction, fail-closed** — no redaction config on disk means the
+   payload cannot be proven clean, so it does not leave (operator
+   directive 2026-07-26). What leaves is always the REDACTED text.
+2. **Residual client identifiers** — the redacted text is re-scanned
+   with the same pattern source; any hit means redaction failed.
+   Never allowlistable.
+3. **Secrets** — ``secret_labels`` vocabulary; allowlistable per exact
+   label, with expiry.
+4. **Operator home paths** — absolute paths under the operator's home
+   AND their tilde form, matched case-insensitively; both expose
+   local project structure; allowlistable per exact path.
+5. **Audit** — every decision is recorded; an ALLOW that cannot be
+   audited flips to denied (``audit-unavailable``).
+
+``evaluate`` never raises. ``enforce`` is the raising convenience for
+call sites that want the guard inline: redacted text back, or
+:class:`EgressDeniedError`.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+
+from core.egress import allowlist, audit, redact
+from core.governance.harness_scanner import secret_labels
+
+
+@dataclass(frozen=True)
+class Finding:
+    """One reason a payload may not leave."""
+
+    kind: str  # redaction-config-missing | redaction-failed
+    #          # | client-identifier | secret | home-path
+    #          # | audit-unavailable | payload-not-text | guard-failure
+    token: str
+
+    def to_audit(self, salt: bytes = b"") -> dict:
+        return {
+            "kind": self.kind,
+            "token_sha16": (
+                audit.hash_token(self.token, salt) if self.token else ""
+            ),
+        }
+
+
+@dataclass
+class EgressDecision:
+    """Outcome of one egress evaluation."""
+
+    allowed: bool
+    destination: str
+    payload_sha256: str
+    findings: list[Finding] = field(default_factory=list)
+    allowlisted: list[Finding] = field(default_factory=list)
+    redacted_text: str | None = None
+    redacted_sha256: str = ""
+    audited: bool = False
+
+    def to_audit(self, salt: bytes = b"") -> dict:
+        return {
+            "allowed": self.allowed,
+            "destination": self.destination,
+            "payload_sha256": self.payload_sha256,
+            "redacted_sha256": self.redacted_sha256,
+            "findings": [f.to_audit(salt) for f in self.findings],
+            "allowlisted": [f.to_audit(salt) for f in self.allowlisted],
+        }
+
+
+class EgressDeniedError(RuntimeError):
+    """Raised by ``enforce`` when the payload may not leave."""
+
+    def __init__(self, decision: EgressDecision):
+        reasons = ", ".join(f.kind for f in decision.findings) or "denied"
+        super().__init__(
+            f"egress to {decision.destination} denied: {reasons} "
+            f"(audit: {'recorded' if decision.audited else 'FAILED'})"
+        )
+        self.decision = decision
+
+
+def evaluate(
+    text: object,
+    destination: str,
+    *,
+    config_path: Path | None = None,
+    home: Path | None = None,
+    allowlist_path: Path | None = None,
+    audit_path: Path | None = None,
+    now: datetime | None = None,
+) -> EgressDecision:
+    """Judge one payload against the policy. Never raises.
+
+    ``home`` scopes the home-path check and the default allowlist /
+    audit locations; ``now`` pins allowlist expiry for tests.
+    """
+    destination = _safe_str(destination)
+    # Digest computed ONCE, before any failure path, so no handler
+    # ever re-executes the operation that failed (QG D1 r2 E-B1).
+    payload_digest = _sha256(text) if isinstance(text, str) else ""
+    try:
+        decision = _judge(
+            text, destination, config_path, home, allowlist_path, now
+        )
+    except Exception as exc:  # never-raises boundary — deny, not crash
+        decision = EgressDecision(
+            allowed=False, destination=destination,
+            payload_sha256=payload_digest,
+            findings=[Finding("guard-failure", type(exc).__name__)],
+        )
+    return _audited(decision, home, audit_path, now)
+
+
+def _safe_str(value: object) -> str:
+    try:
+        return str(value)
+    except Exception:
+        return f"<unprintable {type(value).__name__}>"
+
+
+def _audited(
+    decision: EgressDecision,
+    home: Path | None,
+    audit_path: Path | None,
+    now: datetime | None,
+) -> EgressDecision:
+    try:
+        target = audit_path or audit.default_audit_path(home)
+        salt = audit.load_or_create_salt(audit.default_salt_path(home))
+    except Exception:  # a home that cannot even form paths (M5)
+        target, salt = None, b""
+    decision.audited = target is not None and audit.record(
+        decision.to_audit(salt), target, now
+    )
+    if decision.allowed and not decision.audited:
+        # No audit trail, no egress — record the flip best-effort.
+        decision.allowed = False
+        decision.redacted_text = None
+        decision.findings.append(Finding("audit-unavailable", ""))
+        if target is not None:
+            audit.record(decision.to_audit(salt), target, now)
+    return decision
+
+
+def enforce(text: object, destination: str, **kwargs) -> str:
+    """The redacted text cleared to leave, or :class:`EgressDeniedError`."""
+    decision = evaluate(text, destination, **kwargs)
+    if not decision.allowed or decision.redacted_text is None:
+        raise EgressDeniedError(decision)
+    return decision.redacted_text
+
+
+def _judge(
+    text: object,
+    destination: str,
+    config_path: Path | None,
+    home: Path | None,
+    allowlist_path: Path | None,
+    now: datetime | None,
+) -> EgressDecision:
+    if not isinstance(text, str):
+        return EgressDecision(
+            allowed=False, destination=destination, payload_sha256="",
+            findings=[Finding("payload-not-text", type(text).__name__)],
+        )
+    decision = EgressDecision(
+        allowed=False, destination=destination,
+        payload_sha256=_sha256(text),
+    )
+    clean, failure = _redacted(text, config_path)
+    if failure is not None:
+        decision.findings.append(failure)
+        return decision
+    _collect_findings(
+        decision, clean, destination, config_path, home, allowlist_path, now
+    )
+    if not decision.findings:
+        decision.allowed = True
+        decision.redacted_text = clean
+        decision.redacted_sha256 = _sha256(clean)
+    return decision
+
+
+def _sha256(text: str) -> str:
+    """Total over any str — surrogatepass, because a payload holding a
+    lone surrogate (routine from errors="surrogateescape" decoding or
+    json.loads of an escape) must be DIGESTIBLE to be denied with an
+    audit line (QG D1 r2 E-B1)."""
+    payload = text.encode("utf-8", errors="surrogatepass")
+    return hashlib.sha256(payload).hexdigest()
+
+
+def _redacted(
+    text: str, config_path: Path | None
+) -> tuple[str | None, Finding | None]:
+    """``(clean_text, None)`` or ``(None, denial_finding)``.
+
+    A redaction that CRASHES proves nothing about the payload — same
+    posture as a missing config: denied, never allowlistable (QG D1
+    r1 B1: U+017F KeyError, non-list clients config, latin-1 file).
+    """
+    try:
+        clean, _counts = redact.redact(text, config_path)
+        return clean, None
+    except redact.SanitizerConfigMissing:
+        return None, Finding("redaction-config-missing", "")
+    except Exception as exc:
+        return None, Finding("redaction-failed", type(exc).__name__)
+
+
+def _collect_findings(
+    decision: EgressDecision,
+    clean: str,
+    destination: str,
+    config_path: Path | None,
+    home: Path | None,
+    allowlist_path: Path | None,
+    now: datetime | None,
+) -> None:
+    for token in redact.residual_identifiers(clean, config_path):
+        # Never allowlistable — a residual identifier is a redaction
+        # failure, not an operator choice.
+        decision.findings.append(Finding("client-identifier", token))
+    entries = allowlist.active_entries(
+        allowlist_path or allowlist.default_allowlist_path(home), now
+    )
+    candidates = [
+        Finding("secret", label) for label in secret_labels(clean)
+    ] + [Finding("home-path", p) for p in _home_paths(clean, home)]
+    for finding in candidates:
+        cleared = allowlist.permits(
+            entries, finding.kind, finding.token, destination
+        )
+        (decision.allowlisted if cleared else decision.findings).append(
+            finding
+        )
+
+
+def _home_paths(text: str, home: Path | None) -> list[str]:
+    """Distinct operator-home paths found in *text*.
+
+    Matches the absolute form AND the ``~/`` tilde form, case-
+    insensitively — macOS filesystems resolve case variants to the
+    same file, and a tilde path names the home just as precisely
+    (QG D1 r1 M3). Over-capture of trailing punctuation is accepted:
+    it is the deny direction.
+    """
+    root = re.escape(str(home or Path.home()))
+    pattern = re.compile(
+        rf"(?:{root}(?:/[^\s'\"`)\]]*)?|~/[^\s'\"`)\]]*)",
+        re.IGNORECASE,
+    )
+    return sorted({m.group(0) for m in pattern.finditer(text)})

package/core/egress/redact.py

@@ -0,0 +1,46 @@
+"""Fail-closed redaction for egress payloads.
+
+Thin, deliberate wrapper: the identifier list and the replacement
+semantics belong to ``core.evals.sanitizer`` (one source of truth with
+the release leak-scanner), and this module adds only the egress-side
+contract — redaction that cannot be skipped, plus a residual check
+that is DELIBERATELY looser than the sanitizer, so a payload the
+sanitizer cannot fully redact is denied instead of leaking.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from core.evals.sanitizer import SanitizerConfigMissing, sanitize_text
+from core.governance.leak_scanner import load_redaction_patterns
+
+__all__ = ["SanitizerConfigMissing", "redact", "residual_identifiers"]
+
+
+def redact(text: str, config_path: Path | None = None) -> tuple[str, dict[str, int]]:
+    """Redact client identifiers; ``(clean_text, placeholder_counts)``.
+
+    Raises :class:`SanitizerConfigMissing` when the redaction config is
+    absent or empty — the caller (``policy.evaluate``) converts that
+    into a denial, never into a pass-through.
+    """
+    return sanitize_text(text, config_path=config_path)
+
+
+def residual_identifiers(text: str, config_path: Path | None = None) -> list[str]:
+    """Configured identifiers still present in *text*, as substrings.
+
+    Defence in depth only counts when the second layer differs from
+    the first (QG D1 r1, Francisca B2): the sanitizer replaces
+    word-boundary matches, so a compound token like ``<client>2026``
+    survives redaction untouched. This check is therefore
+    SUBSTRING-based — no boundaries, case-insensitive. Any hit on the
+    redacted text denies; the false-positive cost of the looser match
+    is a denial, never a leak.
+    """
+    patterns = load_redaction_patterns(config_path)
+    if not patterns:
+        return []
+    lowered = text.lower()
+    return [p for p in patterns if p in lowered]