delimit-cli 4.1.7 → 4.1.9

package/gateway/ai/ledger_propose.py

@@ -1,240 +0,0 @@
-"""Ledger Propose — AI-driven item generation from signals.
-
-Analyzes repo state, sensing signals, completed work, and venture priorities
-to propose 3-5 new ledger items with rationale. Runs at end of build loops
-when the queue is empty, or on-demand.
-
-Works across all AI models via MCP — no model-specific code.
-"""
-
-import json
-import time
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-
-SIGNALS_DIR = Path.home() / ".delimit" / "signals"
-PROPOSALS_DIR = Path.home() / ".delimit" / "proposals"
-
-
-def _gather_context(venture: str = "") -> Dict[str, Any]:
-    """Collect context from multiple sources for proposal generation."""
-    context = {
-        "timestamp": time.time(),
-        "venture": venture or "all",
-        "sources": {},
-    }
-
-    # 1. Recent ledger completions (what just shipped)
-    ledger_path = Path.home() / ".delimit" / "ledger.jsonl"
-    if ledger_path.exists():
-        recent_done = []
-        for line in ledger_path.read_text().strip().split("\n")[-100:]:
-            try:
-                item = json.loads(line)
-                if item.get("status") == "done":
-                    recent_done.append({
-                        "id": item.get("id", ""),
-                        "title": item.get("title", ""),
-                        "venture": item.get("venture", ""),
-                    })
-            except json.JSONDecodeError:
-                continue
-        context["sources"]["completed_work"] = recent_done[-10:]
-
-    # 2. Open items (what's already tracked)
-    open_items = []
-    if ledger_path.exists():
-        for line in ledger_path.read_text().strip().split("\n"):
-            try:
-                item = json.loads(line)
-                if item.get("status") == "open":
-                    open_items.append(item.get("title", ""))
-            except json.JSONDecodeError:
-                continue
-    context["sources"]["open_items_count"] = len(open_items)
-    context["sources"]["open_item_titles"] = open_items[:20]
-
-    # 3. Sensing signals (GitHub issues, Reddit, migrations)
-    signals_file = Path.home() / ".delimit" / "signals" / "recent.jsonl"
-    if signals_file.exists():
-        signals = []
-        for line in signals_file.read_text().strip().split("\n")[-20:]:
-            try:
-                sig = json.loads(line)
-                signals.append({
-                    "type": sig.get("type", ""),
-                    "title": sig.get("title", ""),
-                    "source": sig.get("source", ""),
-                    "relevance": sig.get("relevance", 0),
-                })
-            except json.JSONDecodeError:
-                continue
-        context["sources"]["signals"] = signals
-
-    # 4. Git recent activity
-    try:
-        import subprocess
-        result = subprocess.run(
-            ["git", "log", "--oneline", "-10"],
-            capture_output=True, text=True, timeout=5,
-        )
-        if result.returncode == 0:
-            context["sources"]["recent_commits"] = result.stdout.strip().split("\n")
-    except Exception:
-        pass
-
-    # 5. Swarm state
-    swarm_registry = Path.home() / ".delimit" / "swarm" / "agent_registry.json"
-    if swarm_registry.exists():
-        try:
-            reg = json.loads(swarm_registry.read_text())
-            context["sources"]["swarm_agents"] = len(reg.get("agents", {}))
-        except json.JSONDecodeError:
-            pass
-
-    return context
-
-
-def propose_items(
-    venture: str = "",
-    focus: str = "",
-    max_items: int = 5,
-) -> Dict[str, Any]:
-    """Generate proposed ledger items based on current context.
-
-    Analyzes completed work, open items, sensing signals, and repo state
-    to suggest what to work on next. Returns structured proposals that
-    can be added to the ledger with delimit_ledger_add.
-
-    This is the AI's "what should I do next?" engine. Works with any model.
-
-    Args:
-        venture: Focus proposals on a specific venture.
-        focus: Optional focus area (e.g., "outreach", "engineering", "security").
-        max_items: Maximum number of proposals to generate.
-    """
-    context = _gather_context(venture)
-
-    # Build proposal prompt from context
-    proposals = []
-    open_titles = set(context["sources"].get("open_item_titles", []))
-
-    # Strategy 1: Follow-through on completed work
-    for done in context["sources"].get("completed_work", []):
-        title = done.get("title", "")
-        if venture and done.get("venture", "") != venture:
-            continue
-        # Suggest follow-up actions
-        if "deploy" in title.lower() or "publish" in title.lower():
-            follow_up = f"Verify deployment: {title}"
-            if follow_up not in open_titles:
-                proposals.append({
-                    "title": follow_up,
-                    "rationale": f"Follow-through: '{title}' was shipped but may need verification",
-                    "priority": "P1",
-                    "type": "task",
-                    "source": "ledger_propose:follow_through",
-                })
-        if "outreach" in title.lower() or "issue" in title.lower():
-            follow_up = f"Monitor engagement: {title}"
-            if follow_up not in open_titles:
-                proposals.append({
-                    "title": follow_up,
-                    "rationale": f"Outreach needs monitoring for responses and engagement",
-                    "priority": "P1",
-                    "type": "task",
-                    "source": "ledger_propose:follow_through",
-                })
-
-    # Strategy 2: Act on unprocessed signals
-    for sig in context["sources"].get("signals", []):
-        sig_title = sig.get("title", "")
-        if sig_title and sig_title not in open_titles:
-            proposals.append({
-                "title": f"Evaluate signal: {sig_title[:80]}",
-                "rationale": f"Unprocessed {sig.get('type', 'signal')} from {sig.get('source', 'unknown')}",
-                "priority": "P1",
-                "type": "strategy",
-                "source": "ledger_propose:signal",
-            })
-
-    # Strategy 3: Detect gaps
-    has_tests = any("test" in t.lower() for t in open_titles)
-    has_docs = any("doc" in t.lower() or "readme" in t.lower() for t in open_titles)
-    has_security = any("security" in t.lower() or "audit" in t.lower() for t in open_titles)
-
-    if not has_tests and focus != "outreach":
-        proposals.append({
-            "title": f"Run test coverage analysis{f' for {venture}' if venture else ''}",
-            "rationale": "No test-related items in queue — ensure coverage hasn't regressed",
-            "priority": "P1",
-            "type": "task",
-            "source": "ledger_propose:gap_detection",
-        })
-
-    if not has_security and focus != "outreach":
-        proposals.append({
-            "title": f"Security audit{f' for {venture}' if venture else ''}",
-            "rationale": "No security items in queue — periodic audits prevent surprises",
-            "priority": "P2",
-            "type": "task",
-            "source": "ledger_propose:gap_detection",
-        })
-
-    if not has_docs:
-        proposals.append({
-            "title": f"Documentation freshness check{f' for {venture}' if venture else ''}",
-            "rationale": "No doc items in queue — ensure README/CHANGELOG reflect current state",
-            "priority": "P2",
-            "type": "task",
-            "source": "ledger_propose:gap_detection",
-        })
-
-    # Apply focus filter
-    if focus:
-        focus_lower = focus.lower()
-        proposals = [p for p in proposals if focus_lower in p.get("title", "").lower()
-                     or focus_lower in p.get("rationale", "").lower()
-                     or focus_lower in p.get("type", "").lower()]
-
-    # Deduplicate and limit
-    seen = set()
-    unique = []
-    for p in proposals:
-        key = p["title"][:50]
-        if key not in seen:
-            seen.add(key)
-            unique.append(p)
-    proposals = unique[:max_items]
-
-    # Save proposals for audit trail
-    PROPOSALS_DIR.mkdir(parents=True, exist_ok=True)
-    proposal_file = PROPOSALS_DIR / f"proposal_{int(time.time())}.json"
-    proposal_file.write_text(json.dumps({
-        "proposals": proposals,
-        "context_summary": {
-            "completed_work": len(context["sources"].get("completed_work", [])),
-            "open_items": context["sources"].get("open_items_count", 0),
-            "signals": len(context["sources"].get("signals", [])),
-            "swarm_agents": context["sources"].get("swarm_agents", 0),
-        },
-        "venture": venture,
-        "focus": focus,
-        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
-    }, indent=2))
-
-    return {
-        "status": "ok",
-        "proposals": proposals,
-        "total": len(proposals),
-        "venture": venture or "all",
-        "focus": focus or "none",
-        "context": {
-            "completed_work_analyzed": len(context["sources"].get("completed_work", [])),
-            "open_items": context["sources"].get("open_items_count", 0),
-            "signals_analyzed": len(context["sources"].get("signals", [])),
-        },
-        "message": f"Generated {len(proposals)} proposal(s). "
-                   "Use delimit_ledger_add to add approved items.",
-    }

package/gateway/ai/reddit_proxy.py

@@ -1,106 +0,0 @@
-import json
-import logging
-import os
-import urllib.parse
-import urllib.request
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-logger = logging.getLogger("delimit.ai.reddit_proxy")
-
-def _get_proxy_config() -> Dict[str, str]:
-    """Load proxy config from private secrets or environment."""
-    config = {"proxy_url": ""}
-    
-    # 1. Check environment variable
-    env_url = os.environ.get("DELIMIT_REDDIT_PROXY")
-    if env_url:
-        config["proxy_url"] = env_url
-        return config
-
-    # 2. Check private secrets file
-    secrets_path = Path.home() / ".delimit" / "secrets" / "reddit-proxy.json"
-    if secrets_path.exists():
-        try:
-            secrets = json.loads(secrets_path.read_text())
-            config["proxy_url"] = secrets.get("proxy_url", "")
-        except Exception as e:
-            logger.debug(f"Failed to load reddit-proxy secrets: {e}")
-            
-    return config
-
-def fetch_subreddit(subreddit: str, sort: str = "new", limit: int = 10) -> List[Dict[str, Any]]:
-    """
-    Fetch posts from a single subreddit with fallback chain.
-    Returns standardized post dicts.
-    """
-    reddit_url = f"https://www.reddit.com/r/{subreddit}/{sort}.json?limit={limit}&raw_json=1"
-    
-    # 1. Try Local Proxy (Residential IP)
-    proxy_cfg = _get_proxy_config()
-    proxy_url = proxy_cfg.get("proxy_url")
-    if proxy_url:
-        try:
-            fetch_url = f"{proxy_url}?url={urllib.parse.quote(reddit_url, safe='')}"
-            req = urllib.request.Request(fetch_url, headers={"User-Agent": "Delimit/1.0"})
-            with urllib.request.urlopen(req, timeout=10) as resp:
-                body = json.loads(resp.read().decode())
-                children = body.get("data", {}).get("children", [])
-                return [c.get("data", {}) for c in children if c.get("data")]
-        except Exception as e:
-            logger.debug(f"Local proxy failed for r/{subreddit}: {e}")
-
-    # 2. Fallback: PullPush API (Public Archive)
-    try:
-        pp_url = f"https://api.pullpush.io/reddit/search/submission/?subreddit={subreddit}&size={limit}&sort=desc"
-        req = urllib.request.Request(pp_url, headers={"User-Agent": "Delimit/1.0"})
-        with urllib.request.urlopen(req, timeout=10) as resp:
-            body = json.loads(resp.read().decode())
-            return body.get("data", [])
-    except Exception as e:
-        logger.debug(f"PullPush fallback failed for r/{subreddit}: {e}")
-
-    # 3. Fallback: Direct (Often blocked on servers)
-    try:
-        req = urllib.request.Request(reddit_url, headers={"User-Agent": "Mozilla/5.0 (Delimit)"})
-        with urllib.request.urlopen(req, timeout=5) as resp:
-            body = json.loads(resp.read().decode())
-            children = body.get("data", {}).get("children", [])
-            return [c.get("data", {}) for c in children if c.get("data")]
-    except Exception as e:
-        logger.warning(f"Direct fetch failed for r/{subreddit}: {e}")
-
-    return []
-
-def fetch_thread(thread_id: str) -> Optional[Dict[str, Any]]:
-    """
-    Fetch a single Reddit thread by ID with fallback chain.
-    """
-    reddit_url = f"https://www.reddit.com/comments/{thread_id}.json?raw_json=1"
-    
-    # 1. Try Local Proxy
-    proxy_cfg = _get_proxy_config()
-    proxy_url = proxy_cfg.get("proxy_url")
-    if proxy_url:
-        try:
-            fetch_url = f"{proxy_url}?url={urllib.parse.quote(reddit_url, safe='')}"
-            req = urllib.request.Request(fetch_url, headers={"User-Agent": "Delimit/1.0"})
-            with urllib.request.urlopen(req, timeout=10) as resp:
-                data = json.loads(resp.read().decode())
-                if isinstance(data, list) and len(data) > 0:
-                    return data[0].get("data", {}).get("children", [{}])[0].get("data", {})
-        except Exception as e:
-            logger.debug(f"Local proxy failed for thread {thread_id}: {e}")
-
-    # 2. Fallback: PullPush
-    try:
-        pp_url = f"https://api.pullpush.io/reddit/search/submission/?ids={thread_id}"
-        req = urllib.request.Request(pp_url, headers={"User-Agent": "Delimit/1.0"})
-        with urllib.request.urlopen(req, timeout=10) as resp:
-            body = json.loads(resp.read().decode())
-            data = body.get("data", [])
-            return data[0] if data else None
-    except Exception as e:
-        logger.debug(f"PullPush fallback failed for thread {thread_id}: {e}")
-
-    return None

package/gateway/ai/siem_streaming.py

@@ -1,290 +0,0 @@
-"""SIEM Streaming — forward audit trail events to external security tools.
-
-LED-280: Enterprise CISOs need audit logs in their existing SIEM, not just our dashboard.
-Supports Splunk HEC, Datadog Logs API, and AWS EventBridge.
-
-Config stored at ~/.delimit/siem.json. Each integration can be enabled independently.
-Events are forwarded in real-time as they're written to the audit trail.
-"""
-
-import json
-import logging
-import os
-import time
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-logger = logging.getLogger("delimit.ai.siem_streaming")
-
-SIEM_CONFIG_PATH = Path.home() / ".delimit" / "siem.json"
-SIEM_LOG_PATH = Path.home() / ".delimit" / "siem_delivery.jsonl"
-
-DEFAULT_CONFIG = {
-    "splunk": {
-        "enabled": False,
-        "hec_url": "",
-        "hec_token": "",
-        "index": "delimit",
-        "source": "delimit-governance",
-        "sourcetype": "_json",
-    },
-    "datadog": {
-        "enabled": False,
-        "api_key": "",
-        "site": "datadoghq.com",
-        "service": "delimit",
-        "source": "delimit-governance",
-        "tags": ["env:production"],
-    },
-    "eventbridge": {
-        "enabled": False,
-        "bus_name": "default",
-        "region": "us-east-1",
-        "source": "delimit.governance",
-        "detail_type": "GovernanceEvent",
-    },
-    "webhook": {
-        "enabled": False,
-        "url": "",
-        "headers": {},
-        "method": "POST",
-    },
-}
-
-
-def _load_config() -> Dict[str, Any]:
-    if SIEM_CONFIG_PATH.exists():
-        try:
-            return json.loads(SIEM_CONFIG_PATH.read_text())
-        except json.JSONDecodeError:
-            pass
-    return dict(DEFAULT_CONFIG)
-
-
-def _save_config(config: Dict[str, Any]) -> None:
-    SIEM_CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
-    SIEM_CONFIG_PATH.write_text(json.dumps(config, indent=2))
-
-
-def _log_delivery(integration: str, event_id: str, status: str, error: str = "") -> None:
-    try:
-        SIEM_LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
-        entry = {
-            "timestamp": time.time(),
-            "integration": integration,
-            "event_id": event_id,
-            "status": status,
-            "error": error,
-        }
-        with open(SIEM_LOG_PATH, "a") as f:
-            f.write(json.dumps(entry) + "\n")
-    except Exception:
-        pass
-
-
-def configure(
-    integration: str,
-    settings: Optional[Dict[str, Any]] = None,
-    enabled: Optional[bool] = None,
-) -> Dict[str, Any]:
-    """Configure a SIEM integration.
-
-    Args:
-        integration: splunk, datadog, eventbridge, or webhook
-        settings: Key-value pairs to update (e.g. {"hec_url": "...", "hec_token": "..."})
-        enabled: Enable or disable the integration
-    """
-    if integration not in DEFAULT_CONFIG:
-        return {"error": f"Unknown integration: {integration}. Choose: splunk, datadog, eventbridge, webhook"}
-
-    config = _load_config()
-    if integration not in config:
-        config[integration] = dict(DEFAULT_CONFIG[integration])
-
-    if settings:
-        config[integration].update(settings)
-    if enabled is not None:
-        config[integration]["enabled"] = enabled
-
-    _save_config(config)
-
-    # Mask secrets in response
-    safe = dict(config[integration])
-    for key in ("hec_token", "api_key"):
-        if key in safe and safe[key]:
-            safe[key] = safe[key][:4] + "***" + safe[key][-4:]
-
-    return {
-        "status": "configured",
-        "integration": integration,
-        "config": safe,
-    }
-
-
-def get_status() -> Dict[str, Any]:
-    """Get status of all SIEM integrations."""
-    config = _load_config()
-    integrations = {}
-
-    for name, settings in config.items():
-        if not isinstance(settings, dict):
-            continue
-        enabled = settings.get("enabled", False)
-        healthy = False
-
-        if enabled:
-            if name == "splunk":
-                healthy = bool(settings.get("hec_url") and settings.get("hec_token"))
-            elif name == "datadog":
-                healthy = bool(settings.get("api_key"))
-            elif name == "eventbridge":
-                healthy = bool(settings.get("bus_name"))
-            elif name == "webhook":
-                healthy = bool(settings.get("url"))
-
-        integrations[name] = {
-            "enabled": enabled,
-            "healthy": healthy if enabled else None,
-            "status": "active" if (enabled and healthy) else "misconfigured" if enabled else "disabled",
-        }
-
-    # Delivery stats
-    delivery_count = 0
-    delivery_errors = 0
-    if SIEM_LOG_PATH.exists():
-        for line in SIEM_LOG_PATH.read_text().strip().split("\n")[-100:]:
-            try:
-                entry = json.loads(line)
-                delivery_count += 1
-                if entry.get("status") == "error":
-                    delivery_errors += 1
-            except json.JSONDecodeError:
-                continue
-
-    return {
-        "integrations": integrations,
-        "active_count": sum(1 for i in integrations.values() if i["status"] == "active"),
-        "total_deliveries": delivery_count,
-        "delivery_errors": delivery_errors,
-    }
-
-
-def forward_event(event: Dict[str, Any]) -> Dict[str, Any]:
-    """Forward a governance event to all enabled SIEM integrations.
-
-    Called automatically by the audit trail when events are recorded.
-    Returns delivery status per integration.
-    """
-    config = _load_config()
-    results = {}
-    event_id = event.get("id", str(time.time()))
-
-    for name, settings in config.items():
-        if not isinstance(settings, dict) or not settings.get("enabled"):
-            continue
-
-        try:
-            if name == "splunk":
-                results[name] = _forward_splunk(event, settings, event_id)
-            elif name == "datadog":
-                results[name] = _forward_datadog(event, settings, event_id)
-            elif name == "eventbridge":
-                results[name] = _forward_eventbridge(event, settings, event_id)
-            elif name == "webhook":
-                results[name] = _forward_webhook(event, settings, event_id)
-        except Exception as e:
-            results[name] = {"status": "error", "error": str(e)}
-            _log_delivery(name, event_id, "error", str(e))
-
-    return {
-        "event_id": event_id,
-        "forwarded_to": list(results.keys()),
-        "results": results,
-    }
-
-
-def _forward_splunk(event: Dict, settings: Dict, event_id: str) -> Dict:
-    """Forward to Splunk via HTTP Event Collector (HEC)."""
-    import urllib.request
-
-    payload = json.dumps({
-        "event": event,
-        "source": settings.get("source", "delimit-governance"),
-        "sourcetype": settings.get("sourcetype", "_json"),
-        "index": settings.get("index", "delimit"),
-    }).encode()
-
-    req = urllib.request.Request(
-        settings["hec_url"],
-        data=payload,
-        headers={
-            "Authorization": f"Splunk {settings['hec_token']}",
-            "Content-Type": "application/json",
-        },
-    )
-    resp = urllib.request.urlopen(req, timeout=10)
-    _log_delivery("splunk", event_id, "ok")
-    return {"status": "ok", "http_code": resp.status}
-
-
-def _forward_datadog(event: Dict, settings: Dict, event_id: str) -> Dict:
-    """Forward to Datadog Logs API."""
-    import urllib.request
-
-    site = settings.get("site", "datadoghq.com")
-    payload = json.dumps([{
-        "ddsource": settings.get("source", "delimit-governance"),
-        "ddtags": ",".join(settings.get("tags", [])),
-        "service": settings.get("service", "delimit"),
-        "message": json.dumps(event),
-    }]).encode()
-
-    req = urllib.request.Request(
-        f"https://http-intake.logs.{site}/api/v2/logs",
-        data=payload,
-        headers={
-            "DD-API-KEY": settings["api_key"],
-            "Content-Type": "application/json",
-        },
-    )
-    resp = urllib.request.urlopen(req, timeout=10)
-    _log_delivery("datadog", event_id, "ok")
-    return {"status": "ok", "http_code": resp.status}
-
-
-def _forward_eventbridge(event: Dict, settings: Dict, event_id: str) -> Dict:
-    """Forward to AWS EventBridge."""
-    try:
-        import boto3
-        client = boto3.client("events", region_name=settings.get("region", "us-east-1"))
-        response = client.put_events(Entries=[{
-            "Source": settings.get("source", "delimit.governance"),
-            "DetailType": settings.get("detail_type", "GovernanceEvent"),
-            "Detail": json.dumps(event),
-            "EventBusName": settings.get("bus_name", "default"),
-        }])
-        failed = response.get("FailedEntryCount", 0)
-        status = "ok" if failed == 0 else "partial"
-        _log_delivery("eventbridge", event_id, status)
-        return {"status": status, "failed_entries": failed}
-    except ImportError:
-        _log_delivery("eventbridge", event_id, "error", "boto3 not installed")
-        return {"status": "error", "error": "boto3 not installed. Run: pip install boto3"}
-
-
-def _forward_webhook(event: Dict, settings: Dict, event_id: str) -> Dict:
-    """Forward to a generic webhook URL."""
-    import urllib.request
-
-    headers = {"Content-Type": "application/json"}
-    headers.update(settings.get("headers", {}))
-
-    req = urllib.request.Request(
-        settings["url"],
-        data=json.dumps(event).encode(),
-        headers=headers,
-        method=settings.get("method", "POST"),
-    )
-    resp = urllib.request.urlopen(req, timeout=10)
-    _log_delivery("webhook", event_id, "ok")
-    return {"status": "ok", "http_code": resp.status}