pantheon-opencode 1.0.0

package/scripts/scrub_secrets.py

@@ -0,0 +1,281 @@
+#!/usr/bin/env python3
+"""
+scrub-secrets.py — Credential scrubber for Memory Bank promotions.
+
+Scrubs credentials, tokens, and secrets from text before promoting it
+to permanent memory bank files. Uses Python 3.9+ stdlib only.
+
+Usage:
+    python3 scripts/scrub-secrets.py --file path/to/file
+        # scrub in-place (with .bak backup)
+    python3 scripts/scrub-secrets.py --stdin
+        # read stdin, write scrubbed to stdout
+    python3 scripts/scrub-secrets.py --audit path/to/file
+        # only report, don't modify
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import sys
+from re import Match
+
+# ---------------------------------------------------------------------------
+# Pattern definitions — order matters (private keys first to catch multi-line)
+# ---------------------------------------------------------------------------
+
+
+def _make_redactor(pattern: str, replacement: str) -> tuple[re.Pattern, str]:
+    """Compile a pattern with DOTALL for multi-line."""
+    return (re.compile(pattern, re.DOTALL), replacement)
+
+
+REDACTORS: list[tuple[re.Pattern, str]] = [
+    # SSH private keys (multi-line, before generic private key)
+    _make_redactor(
+        r"-----BEGIN\s+OPENSSH\s+PRIVATE\s+KEY-----"
+        r".*?-----END\s+OPENSSH\s+PRIVATE\s+KEY-----",
+        "[SSH PRIVATE KEY REDACTED]",
+    ),
+    # Private keys (multi-line, must run before Bearer/JWT)
+    _make_redactor(
+        r"-----BEGIN\s+.*?PRIVATE\s+KEY-----"
+        r".*?-----END\s+.*?PRIVATE\s+KEY-----",
+        "[PRIVATE KEY REDACTED]",
+    ),
+    # Certificates (multi-line)
+    _make_redactor(
+        r"-----BEGIN\s+CERTIFICATE-----"
+        r".*?-----END\s+CERTIFICATE-----",
+        "[CERTIFICATE REDACTED]",
+    ),
+    # Bearer tokens
+    _make_redactor(
+        r"Bearer\s+[A-Za-z0-9\-._~+/]+=*",
+        "Bearer [REDACTED]",
+    ),
+    # JWTs
+    _make_redactor(
+        r"eyJ[A-Za-z0-9\-_]+\.[A-Za-z0-9\-_]+\.[A-Za-z0-9\-_]+",
+        "[JWT REDACTED]",
+    ),
+    # GitHub PATs
+    _make_redactor(
+        r"ghp_[A-Za-z0-9]{36}",
+        "[GITHUB TOKEN REDACTED]",
+    ),
+    # OpenAI keys
+    _make_redactor(
+        r"sk-[A-Za-z0-9\-_]{10,}",
+        "[OPENAI KEY REDACTED]",
+    ),
+    # API keys (api_key, api-key, apikey)
+    _make_redactor(
+        r"(api[_-]?key|apikey)\s*[=:]\s*['\"]?[A-Za-z0-9\-_]{8,}",
+        r"\1=[REDACTED]",
+    ),
+    # Passwords / tokens / secrets
+    _make_redactor(
+        r"(token|secret|password)\s*[=:]\s*['\"]?[A-Za-z0-9\-_]{8,}",
+        r"\1=[REDACTED]",
+    ),
+    # AWS Access Keys
+    _make_redactor(
+        r"AKIA[0-9A-Z]{16}",
+        "[AWS KEY REDACTED]",
+    ),
+    # Google API Keys
+    _make_redactor(
+        r"AIza[0-9A-Za-z\-_]{35}",
+        "[GOOGLE API KEY REDACTED]",
+    ),
+    # Slack tokens
+    _make_redactor(
+        r"xox[baprs]-[0-9a-zA-Z-]{20,}",
+        "[SLACK TOKEN REDACTED]",
+    ),
+    # Heroku API Keys (UUID format)
+    _make_redactor(
+        r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}",
+        "[HEROKU API KEY REDACTED]",
+    ),
+    # Database connection strings (non-greedy to prevent over-matching)
+    _make_redactor(
+        r"postgresql\+?:\/\/.*?:.*?@",
+        "[POSTGRESQL CONNECTION REDACTED]",
+    ),
+    _make_redactor(
+        r"mysql\+?:\/\/.*?:.*?@",
+        "[MYSQL CONNECTION REDACTED]",
+    ),
+    _make_redactor(
+        r"redis\+?:\/\/.*?:.*?@",
+        "[REDIS CONNECTION REDACTED]",
+    ),
+    # .env export statements
+    _make_redactor(
+        r"export\s+\w+=['\"]?\S+",
+        "[ENV EXPORT REDACTED]",
+    ),
+]
+
+# Classification dispatch: prefix → label
+_CLASSIFIERS: list[tuple[str, str]] = [
+    ("-----BEGIN OPENSSH", "ssh_private_key"),
+    ("-----BEGIN CERTIFICATE", "certificate"),
+    ("-----BEGIN", "private_key"),
+    ("Bearer ", "bearer_token"),
+    ("eyJ", "jwt"),
+    ("ghp_", "github_pat"),
+    ("sk-", "openai_key"),
+    ("xox", "slack_token"),
+    ("AKIA", "aws_key"),
+    ("AIza", "google_api_key"),
+    ("postgresql://", "postgresql_conn"),
+    ("mysql://", "mysql_conn"),
+    ("redis://", "redis_conn"),
+    ("export ", "env_export"),
+]
+
+
+def _classify_match(matched: str) -> str:
+    """Return a human-readable label for a matched secret."""
+    for prefix, label in _CLASSIFIERS:
+        if matched.startswith(prefix):
+            return label
+    if re.match(r"api[_-]?key|apikey", matched, re.IGNORECASE):
+        return "api_key"
+    if re.match(r"(token|secret|password)\s*=", matched, re.IGNORECASE):
+        return "credential"
+    if re.match(
+        r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}",
+        matched,
+    ):
+        return "heroku_uuid"
+    return "unknown_secret"
+
+
+def scrub(content: str) -> tuple[str, list[dict]]:
+    """Scrub secrets from *content*.
+
+    Returns (scrubbed_content, redactions) where each redaction dict has:
+        type        — human-readable label for the matched secret
+        position    — (start, end) tuple of character offsets in original
+        replacement — what the secret was replaced with
+
+    Each redactor runs independently so every match in the input is
+    captured exactly once.  Overlapping matches across *different* patterns
+    are possible but extremely unlikely given the pattern design.
+    """
+    redactions: list[dict] = []
+    scrubbed = content
+
+    for pattern, replacement in REDACTORS:
+        # Walk matches in reverse offset order so replacements don't shift
+        # positions of yet-unreplaced matches.
+        matches: list[Match] = list(pattern.finditer(scrubbed))
+        for m in reversed(matches):
+            start, end = m.start(), m.end()
+            matched_text = m.group(0)
+            resolved = m.expand(replacement) if "\\" in replacement else replacement
+
+            redactions.append(
+                {
+                    "type": _classify_match(matched_text),
+                    "position": (start, end),
+                    "replacement": resolved,
+                }
+            )
+            scrubbed = scrubbed[:start] + resolved + scrubbed[end:]
+
+    # Restore original order (positional, ascending)
+    redactions.reverse()
+    return scrubbed, redactions
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+
+def _audit(path: str) -> None:
+    """Print what would be redacted without modifying the file."""
+    with open(path, encoding="utf-8") as fh:
+        content = fh.read()
+    _, redactions = scrub(content)
+    if not redactions:
+        print(f"[OK] {path}: no secrets found")
+        return
+    print(f"[SCAN] {path}: {len(redactions)} potential secret(s) found:")
+    for r in redactions:
+        print(f"   [{r['type']}] at {r['position']} -> {r['replacement']}")
+
+
+def _scrub_file(path: str) -> None:
+    """Scrub file in-place, creating a .bak backup."""
+    with open(path, encoding="utf-8") as fh:
+        content = fh.read()
+    scrubbed, redactions = scrub(content)
+    if not redactions:
+        print(f"[OK] {path}: no secrets found, file unchanged")
+        return
+    # Write backup
+    bak = path + ".bak"
+    if not os.path.exists(bak):
+        os.rename(path, bak)
+    else:
+        print(f"[WARN] {bak} already exists, skipping backup")
+    with open(path, "w", encoding="utf-8") as fh:
+        fh.write(scrubbed)
+    print(
+        f"[SECURE] {path}: {len(redactions)} secret(s) redacted "
+        f"({' + '.join(r['type'] for r in redactions)})"
+    )
+    print(f"   Backup: {bak}")
+
+
+def _scrub_stdin() -> None:
+    """Read stdin, write scrubbed output to stdout."""
+    content = sys.stdin.read()
+    scrubbed, redactions = scrub(content)
+    for r in redactions:
+        print(f"  [REDACTED {r['type']}]", file=sys.stderr)
+    sys.stdout.write(scrubbed)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Scrub secrets from text before promoting to "
+        "permanent memory bank files.",
+    )
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument(
+        "--file",
+        type=str,
+        help="Scrub file in-place (with .bak backup)",
+    )
+    group.add_argument(
+        "--stdin",
+        action="store_true",
+        help="Read stdin, write scrubbed to stdout",
+    )
+    group.add_argument(
+        "--audit",
+        type=str,
+        help="Only report what would be redacted",
+    )
+
+    args = parser.parse_args()
+
+    if args.file:
+        _scrub_file(args.file)
+    elif args.stdin:
+        _scrub_stdin()
+    elif args.audit:
+        _audit(args.audit)
+
+
+if __name__ == "__main__":
+    main()

package/scripts/test-context-compression.sh

@@ -0,0 +1,166 @@
+#!/bin/bash
+# Test: Context Compression Pipeline
+# Simulates Themis APPROVED → triggers compress_context handoff
+#
+# Usage: bash scripts/test-context-compression.sh
+#
+# This script validates the prerequisites for the Level 2 context compression
+# pipeline and creates mock artifacts for end-to-end testing.
+
+set -euo pipefail
+
+echo "=== Context Compression Test ==="
+echo ""
+
+# Step 1: Verify prerequisites
+echo "[1/4] Checking prerequisites..."
+errors=0
+
+test -f skills/context-compression/SKILL.md || { echo "   ❌ SKILL.md not found"; errors=$((errors + 1)); }
+test -f agents/mnemosyne.agent.md || { echo "   ❌ mnemosyne.agent.md not found"; errors=$((errors + 1)); }
+test -f agents/zeus.agent.md || { echo "   ❌ zeus.agent.md not found"; errors=$((errors + 1)); }
+test -f routing.yml || { echo "   ❌ routing.yml not found"; errors=$((errors + 1)); }
+test -f skills/artifact-management/SKILL.md || { echo "   ❌ artifact-protocol.instructions.md not found"; errors=$((errors + 1)); }
+test -f .pantheon/memory-bank/01-active-context.md || { echo "   ❌ active-context.md not found"; errors=$((errors + 1)); }
+test -d .pantheon/memory-bank/.tmp || mkdir -p .pantheon/memory-bank/.tmp
+
+if [ -f scripts/scrub-secrets.py ]; then
+    echo "   ✅ scrub-secrets.py found (217 lines, $(grep -c 'def ' scripts/scrub-secrets.py || echo 0) functions)"
+else
+    echo "   ⚠️  scrub-secrets.py not found (will skip scrubbing)"
+fi
+
+# Verify compress_context handoff in routing.yml
+if grep -q "compress_context" routing.yml; then
+    HANDOFF_LINE=$(grep -n "compress_context" routing.yml | head -1 | cut -d: -f1)
+    echo "   ✅ compress_context handoff found in routing.yml at line $HANDOFF_LINE"
+else
+    echo "   ❌ compress_context handoff missing from routing.yml"
+    errors=$((errors + 1))
+fi
+
+# Verify ZZ artifact mention in artifact-protocol
+if grep -q "ZZ-phase" skills/artifact-management/SKILL.md; then
+    echo "   ✅ ZZ artifact format documented in artifact-protocol.instructions.md"
+else
+    echo "   ❌ ZZ artifact format missing from artifact-protocol.instructions.md"
+    errors=$((errors + 1))
+fi
+
+# Verify context-compression trigger was added to zeus.agent.md
+if grep -q "Context Compression Trigger" agents/zeus.agent.md; then
+    echo "   ✅ Context Compression Trigger section present in zeus.agent.md"
+else
+    echo "   ⚠️  Context Compression Trigger section not yet in zeus.agent.md"
+fi
+
+if [ "$errors" -gt 0 ]; then
+    echo "   ❌ $errors prerequisite(s) missing — aborting"
+    exit 1
+fi
+echo "   ✅ All prerequisites met"
+echo ""
+
+# Step 2: Create a mock IMPL artifact for testing
+echo "[2/4] Creating mock IMPL artifact..."
+
+cat > .pantheon/memory-bank/.tmp/IMPL-test-phase-hermes.md << 'ARTIFACT'
+# IMPL-test-phase-hermes
+**Date:** 2026-06-26  **Status:** Awaiting Compression Test
+
+## What Was Implemented
+- test/file.py — mock implementation for compression test
+- test/schema.py — mock schema migration for priority scoring test
+
+## Tests
+- ✅ 5 tests passing / Coverage: 87%
+
+## Notes for Themis
+This is a test artifact to validate the compression pipeline end-to-end.
+ARTIFACT
+
+cat > .pantheon/memory-bank/.tmp/IMPL-test-phase-demeter.md << 'ARTIFACT'
+# IMPL-test-phase-demeter
+**Date:** 2026-06-26  **Status:** Awaiting Compression Test
+
+## What Was Implemented
+- backend/migrations/0013_add_refresh_tokens.py — new refresh_tokens table
+- backend/models/token.py — Token model with FK to users
+
+## Tests
+- ✅ Migration tested: upgrade + downgrade both verified
+
+## Notes for Themis
+New table: refresh_tokens with (user_id, token_hash, expires_at, revoked_at)
+ARTIFACT
+
+echo "   ✅ 2 mock artifacts created (hermes + demeter)"
+echo ""
+
+# Step 3: Create mock subtask_summary for compression
+echo "[3/4] Creating mock subtask summaries..."
+
+cat > .pantheon/memory-bank/.tmp/subtask-hermes.json << 'JSON'
+{
+  "files_changed": ["test/file.py", "test/schema.py"],
+  "summary": "Added JWT auth endpoint with refresh token rotation",
+  "tests": "✅ All passing",
+  "coverage": "87%",
+  "status": "complete",
+  "blockers": null
+}
+JSON
+
+cat > .pantheon/memory-bank/.tmp/subtask-demeter.json << 'JSON'
+{
+  "files_changed": ["backend/migrations/0013_add_refresh_tokens.py", "backend/models/token.py"],
+  "summary": "Created refresh_tokens table with FK to users, hashed token, expires_at, revoked_at",
+  "tests": "✅ Migration upgrade + downgrade verified",
+  "coverage": "100%",
+  "status": "complete",
+  "blockers": null
+}
+JSON
+
+echo "   ✅ 2 mock subtask summaries created"
+echo ""
+
+# Step 4: Verify .tmp structure is writable and correct
+echo "[4/4] Verifying .tmp structure..."
+ls -la .pantheon/memory-bank/.tmp/
+echo ""
+echo "   ✅ .tmp structure looks correct"
+echo ""
+
+# Step 5: Validate scrub-secrets.py works
+echo "[5/4] Testing scrub-secrets.py (bonus)..."
+if [ -f scripts/scrub-secrets.py ]; then
+    # Test with a simple secret
+    SECRET_TEST=$(echo "Bearer my-test-token-12345" | python3 scripts/scrub-secrets.py --stdin 2>/dev/null)
+    if echo "$SECRET_TEST" | grep -q "REDACTED"; then
+        echo "   ✅ scrub-secrets.py: secret detection working"
+    else
+        echo "   ⚠️  scrub-secrets.py: did not detect test secret (may need different pattern)"
+    fi
+fi
+echo ""
+
+echo "=== Test Complete ==="
+echo ""
+echo "Mock artifacts ready for compression pipeline test."
+echo ""
+echo "Next manual step: Trigger compress_context handoff:"
+echo "  @mnemosyne Run compression pipeline with test artifacts"
+echo ""
+echo "Expected output:"
+echo "  - .pantheon/memory-bank/.tmp/ZZ-test-context.md"
+echo "  - Updated 01-active-context.md with compressed entries"
+echo "  - Updated 02-progress-log.md with archived mock artifact"
+echo ""
+echo "Scoring verification (expected):"
+echo "  - subtask-hermes.json: JWT + endpoint + token keywords → HIGH (score ~0.71)"
+echo "  - subtask-demeter.json: migration + new table + FK keywords → CRITICAL (score ~0.83)"
+echo ""
+echo "To clean up test artifacts:"
+echo "  rm .pantheon/memory-bank/.tmp/IMPL-test-phase-*.md"
+echo "  rm .pantheon/memory-bank/.tmp/subtask-*.json"