delimit-cli 4.5.13 → 4.6.0

package/gateway/ai/backends/tools_infra.py

@@ -64,12 +64,16 @@ _CREDENTIAL_FALSE_POSITIVES = re.compile(
     r"sk-ant-demo|sk-demo|AIza-demo|xai-demo|demo[_-]?(?:key|secret|token)|"
     r"-demo['\"]|"
     # Function-call RHS (reading from parsed JSON, env, getters, slicing strings)
-    r"json\.loads|\.read_text\(|\.slice\(|"
+    r"json\.loads|\.read_text\(|\.slice\(|\.split\(|"
     r"\w+\.get\(|token\s*=\s*_make_token|"
     # RHS that is a parameter reference like token=tokens.get("access_token"...
     r"=\s*\w+\.get\(|"
     # Dict index dereference: token_data["token"], result["secret"], etc.
     r"_data\[|_result\[|"
+    # LED-1278 (b): function-call RHS with leading underscore (e.g. _load_token())
+    r"=\s*_\w+\(|"
+    # LED-1278 (b): documentation/example placeholders in angle brackets
+    r"<[^>]*?(?:long|same|random|your|placeholder|example|secret|token|key)[^>]*?>|"
     # Bare `if not <var>:` and similar control-flow lines that mention
     # the credential variable name but contain no value.
     r"if\s+not\s+\w+:|"
@@ -98,6 +102,82 @@ SCAN_EXTENSIONS = {".py", ".js", ".ts", ".jsx", ".tsx", ".go", ".rb", ".java", "
 # Skip directories
 SKIP_DIRS = {"node_modules", ".git", "__pycache__", ".venv", "venv", ".tox", "dist", "build", ".next", ".nuxt", "vendor"}
 
+# LED-1278 (a): test-tree path patterns excluded by default. The scanner walks  # nosec
+# test directories with prod rules, so test fixtures (placeholder tokens,  # nosec
+# trivial JWT bodies, code-injection demos) get surfaced as critical findings  # nosec
+# on every audit. Default behavior now skips these; callers can pass  # nosec
+# include_tests=True to scan everything.  # nosec
+TEST_PATH_PATTERNS = (
+    re.compile(r"(?:^|[\\/])tests?[\\/]"),         # tests/ or test/ as a path component
+    re.compile(r"(?:^|[\\/])__tests__[\\/]"),      # JS __tests__/
+    re.compile(r"(?:^|[\\/])spec[\\/]"),           # spec/
+    re.compile(r"(?:^|[\\/])fixtures?[\\/]"),      # fixtures/ or fixture/
+    re.compile(r"(?:^|[\\/])test_[^\\/]+\.py$"),   # test_*.py
+    re.compile(r"_test\.(?:py|go|rb|java)$"),       # *_test.py / *_test.go
+    re.compile(r"\.(?:test|spec)\.(?:js|jsx|ts|tsx|mjs|cjs)$"),  # *.test.js, *.spec.tsx
+)
+
+
+def _is_test_path(path: str) -> bool:
+    """Return True if path looks like a test file/dir per TEST_PATH_PATTERNS."""
+    s = str(path)
+    return any(pat.search(s) for pat in TEST_PATH_PATTERNS)
+
+
+# LED-1278 (b): well-known dummy / fixture values. Even when include_tests=True
+# (or when production code intentionally embeds canonical placeholders in
+# docs/examples), these specific shapes should be suppressed as `info` log
+# lines, not raised as critical findings.
+#
+# Each entry: (regex applied to the matched secret text, human label).
+KNOWN_DUMMY_PATTERNS = [
+    # AWS canonical dummy from official AWS documentation.
+    (re.compile(r"AKIAIOSFODNN7EXAMPLE"), "aws_doc_dummy"),
+    # GitHub token placeholders that use the printable-alphabet pattern.
+    (re.compile(r"^gh[pousr]_ABCDEFGHIJKLMNOPQRSTUVWXYZ", re.IGNORECASE), "github_alphabet_dummy"),
+    # Slack tokens with the leading 1234567890 sequence.
+    (re.compile(r"^xox[baprs]-1234567890-"), "slack_seq_dummy"),
+    # JWT with the unsigned-HS256 header + trivial body. We match the literal
+    # eyJhbGciOiJIUzI1NiJ9 header and check the payload separately below.
+    (re.compile(r"^eyJhbGciOiJIUzI1NiJ9\."), "jwt_hs256_trivial"),
+    # Generic dict-credential placeholder values: fake/test/dummy/example/etc.
+    (re.compile(r"['\"](?:fake|test|dummy|example|placeholder|stale|from-)[A-Za-z0-9_\-]*['\"]\s*$", re.IGNORECASE),
+     "generic_placeholder_value"),
+    # Provider test-key shapes: xai-key-123, google-key-7, claude-key-2 etc.
+    (re.compile(r"['\"](?:xai|google|claude|gem|grok|codex|ollama)[-_]?key[-_]?\d+['\"]\s*$", re.IGNORECASE),
+     "provider_test_key"),
+]
+
+
+def _looks_like_known_dummy(secret_name: str, matched_text: str) -> Optional[str]:
+    """Return a label if matched_text is a known-dummy/fixture value, else None.
+
+    Used by the secret scanner to convert what would otherwise be a critical
+    finding into an `info`-level suppressed entry. Keeps the audit-trail
+    visible (so a future regression in the allowlist is detectable) while
+    eliminating the false-positive-storm noise.
+
+    For JWT, additionally checks that the body is the trivial `sub:1234567890`
+    payload — we don't want to suppress real signed JWTs that happen to use
+    HS256.
+    """
+    for pattern, label in KNOWN_DUMMY_PATTERNS:
+        if pattern.search(matched_text):
+            if label == "jwt_hs256_trivial":
+                # Only treat as dummy if the payload is the canonical demo
+                # body (`sub: "1234567890"` or trivial abc123 segment).
+                # The JWT pattern produces something like:
+                #   eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.abc123def456ghi789
+                # The middle segment base64-decodes to {"sub":"1234567890"}.
+                if (
+                    "eyJzdWIiOiIxMjM0NTY3ODkwIn0" in matched_text
+                    or re.search(r"\.[A-Za-z0-9_-]*abc123[A-Za-z0-9_-]*$", matched_text)
+                ):
+                    return label
+                continue
+            return label
+    return None
+
 
 def _run_cmd(cmd: List[str], timeout: int = 30, cwd: Optional[str] = None) -> Dict[str, Any]:
     """Run a command and return stdout, stderr, returncode.
@@ -144,8 +224,13 @@ def _bump_semver(version: str, bump: str) -> str:
     return f"{major}.{minor}.{patch}"
 
 
-def _scan_files(target: str) -> List[Path]:
-    """Collect scannable source files under target."""
+def _scan_files(target: str, include_tests: bool = False) -> List[Path]:
+    """Collect scannable source files under target.
+
+    LED-1278 (a): when include_tests=False (the new default), skip files that
+    match TEST_PATH_PATTERNS so test fixtures do not surface as findings.
+    Single-file targets are always scanned regardless (caller asked explicitly).
+    """
     root = Path(target).resolve()
     files = []
     if root.is_file():
@@ -154,10 +239,25 @@ def _scan_files(target: str) -> List[Path]:
         return []
     for dirpath, dirnames, filenames in os.walk(root, onerror=lambda _err: None):
         dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
+        if not include_tests:
+            # Prune obvious test directory names before recursing so we don't
+            # walk huge __tests__/ trees just to discard them later.
+            dirnames[:] = [
+                d for d in dirnames
+                if d not in ("tests", "test", "__tests__", "spec", "fixtures", "fixture")
+            ]
         for filename in filenames:
             p = Path(dirpath) / filename
-            if p.suffix in SCAN_EXTENSIONS:
-                files.append(p)
+            if p.suffix not in SCAN_EXTENSIONS:
+                continue
+            if not include_tests:
+                try:
+                    rel = str(p.relative_to(root))
+                except ValueError:
+                    rel = str(p)
+                if _is_test_path(rel):
+                    continue
+            files.append(p)
             # Cap to avoid scanning massive repos
             if len(files) >= 5000:
                 return files
@@ -166,11 +266,26 @@ def _scan_files(target: str) -> List[Path]:
 
 # ─── 5. security_audit ──────────────────────────────────────────────────
 
-def security_audit(target: str = ".") -> Dict[str, Any]:
+def security_audit(target: str = ".", include_tests: bool = False) -> Dict[str, Any]:
     """Audit security: dependency vulnerabilities + anti-patterns + secret detection.
 
     Default: runs pip-audit/npm-audit, regex scans for secrets and dangerous patterns.
     Optional upgrade: set SNYK_TOKEN or TRIVY_PATH for enhanced scanning.
+
+    LED-1278 fixes:
+      (a) include_tests defaults to False — test directories (tests/, __tests__/,
+          spec/, fixtures/, *_test.py, *.test.tsx, etc.) are skipped so
+          test fixtures don't get raised as critical production findings.
+          Pass include_tests=True to scan everything (legacy behavior).
+      (b) Well-known dummy/placeholder values (AWS canonical example,
+          alphabet-pattern GitHub tokens, leading-1234567890 Slack tokens,
+          trivial JWT, fake/test/dummy/placeholder dict values, provider
+          test-key shapes) are suppressed and recorded as `info`-severity
+          allowlist hits in `suppressed_findings` for audit visibility.
+
+    Args:
+        target: Repository or file path to audit.
+        include_tests: When True, scan test directories (default False).
     """
     target_path = Path(target).resolve()
     if not target_path.exists():
@@ -179,6 +294,7 @@ def security_audit(target: str = ".") -> Dict[str, Any]:
     vulnerabilities = []
     anti_patterns_found = []
     secrets_found = []
+    suppressed_findings: List[Dict[str, Any]] = []  # LED-1278 (b): allowlist log
     tools_used = []
     severity_counts = {"critical": 0, "high": 0, "medium": 0, "low": 0, "info": 0}
 
@@ -284,8 +400,10 @@ def security_audit(target: str = ".") -> Dict[str, Any]:
             pass
 
     # --- 2. Anti-pattern scan ---
-    files = _scan_files(target)
-    tools_used.append(f"pattern-scanner ({len(files)} files)")
+    files = _scan_files(target, include_tests=include_tests)
+    scan_label = f"pattern-scanner ({len(files)} files"
+    scan_label += ", include_tests=True" if include_tests else ", tests excluded"
+    tools_used.append(scan_label + ")")
 
     for fpath in files:
         try:
@@ -305,6 +423,25 @@ def security_audit(target: str = ".") -> Dict[str, Any]:
                 if secret_name in _FP_FILTERED and _CREDENTIAL_FALSE_POSITIVES.search(matched_text):
                     continue
                 line_num = content[:match.start()].count("\n") + 1
+                # LED-1278 (b): well-known dummy/placeholder values get
+                # suppressed to info-level rather than raised as critical.
+                # Logged in suppressed_findings so a future regression in the
+                # allowlist (e.g. real key matching by accident) is auditable.
+                dummy_label = _looks_like_known_dummy(secret_name, matched_text)
+                if dummy_label:
+                    suppressed_findings.append({
+                        "file": rel,
+                        "line": line_num,
+                        "type": secret_name,
+                        "reason": dummy_label,
+                        "severity": "info",
+                    })
+                    severity_counts["info"] += 1
+                    logger.info(
+                        "security_audit: suppressed known-dummy %s (%s) in %s:%d",
+                        secret_name, dummy_label, rel, line_num,
+                    )
+                    continue
                 # Redact actual secret values in snippet output
                 snippet_raw = content[max(0, match.start() - 10):match.end() + 10].strip()[:80]
                 secrets_found.append({
@@ -358,6 +495,9 @@ def security_audit(target: str = ".") -> Dict[str, Any]:
         "anti_patterns": anti_patterns_found,
         "secrets_detected": len(secrets_found),
         "secrets": secrets_found[:20],  # Cap output to avoid huge responses
+        "suppressed_findings": suppressed_findings[:20],  # LED-1278 (b): allowlist audit log
+        "suppressed_count": len(suppressed_findings),
+        "include_tests": include_tests,  # LED-1278 (a): expose scan scope
         "env_in_git": env_in_git,
         "severity_summary": severity_counts,
         "tools_used": tools_used,
@@ -765,9 +905,9 @@ def release_plan(environment: str = "production", version: str = "", repository:
 
     # Commits since last tag
     if last_tag:
-        r = _run_cmd(["git", "log", f"{last_tag}..HEAD", "--oneline", "--no-decorate"], cwd=cwd)
+        r = _run_cmd(["git", "log", f"{last_tag}..HEAD", "--format=%s"], cwd=cwd)
     else:
-        r = _run_cmd(["git", "log", "--oneline", "--no-decorate", "-50"], cwd=cwd)
+        r = _run_cmd(["git", "log", "--format=%s", "-50"], cwd=cwd)
     commits = [line.strip() for line in r["stdout"].strip().split("\n") if line.strip()] if r["stdout"].strip() else []
     result["commits_since_last_tag"] = len(commits)
     result["commits"] = commits[:30]  # Cap

package/gateway/ai/daemon.py

@@ -75,6 +75,7 @@ AUTO_PATTERNS = {
     "test": ["test", "coverage", "smoke"],
     "docs": ["docs", "documentation", "readme"],
     "governance": ["governance", "policy", "compliance"],
+    "build": ["feat", "fix", "task", "implementation"],
 }
 
 
@@ -263,6 +264,14 @@ def get_next_automatable_item(
     return None
 
 
+
+def _run_build(item_id: str, venture: str = "") -> dict:
+    """Run the governed build loop for a specific item (LED-1146)."""
+    from ai.loop_engine import run_governed_iteration
+    # Use a persistent session for the daemon
+    session_id = "daemon-build-loop"
+    return run_governed_iteration(session_id=session_id)
+
 def process_item(item: dict, log_path: Optional[Path] = None) -> dict:
     """Process a single ledger item by running the suggested tool.
 
@@ -293,6 +302,7 @@ def process_item(item: dict, log_path: Optional[Path] = None) -> dict:
         "test": _run_test,
         "governance": _run_governance,
         "docs": _run_docs,
+        "build": _run_build,
     }
 
     runner = tool_map.get(tool)

package/gateway/ai/daily_digest.py

@@ -20,11 +20,10 @@ Call via MCP: delimit_digest(action="run") or scheduled cron.
 from __future__ import annotations
 
 import json
-import time
 from collections import Counter
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict
 
 DIGEST_DIR = Path.home() / ".delimit" / "digest"
 LEDGER_DIR = Path.home() / ".delimit" / "ledger"

package/gateway/ai/delimit_daemon.py

@@ -0,0 +1,67 @@
+"""
+Unified Delimit Daemon (LED-193).
+
+Consolidates three long-running daemons into a single process:
+  - inbox_daemon (5m cadence)
+  - social_daemon (15m cadence)
+  - self_repair_daemon (1h cadence)
+
+Retains the individual modules' internal state files and thread-level
+encapsulation to minimize blast radius and ensure existing MCP interfaces
+(status checks) continue to work without modification.
+"""
+
+import time
+import logging
+import signal
+import sys
+
+from ai.inbox_daemon import start_daemon as start_inbox, stop_daemon as stop_inbox
+from ai.social_daemon import start_daemon as start_social, stop_daemon as stop_social
+from ai.self_repair_daemon import start_daemon as start_self_repair, stop_daemon as stop_self_repair
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger("delimit.daemon_runner")
+
+def _handle_sigterm(signum, frame):
+    logger.info("Received SIGTERM, shutting down all daemons...")
+    try:
+        stop_inbox()
+    except Exception as e:
+        logger.error(f"Error stopping inbox: {e}")
+    try:
+        stop_social()
+    except Exception as e:
+        logger.error(f"Error stopping social: {e}")
+    try:
+        stop_self_repair()
+    except Exception as e:
+        logger.error(f"Error stopping self_repair: {e}")
+    sys.exit(0)
+
+def main():
+    signal.signal(signal.SIGTERM, _handle_sigterm)
+    signal.signal(signal.SIGINT, _handle_sigterm)
+
+    logger.info("Starting unified delimit_daemon (LED-193)...")
+    
+    inbox_res = start_inbox()
+    logger.info(f"Inbox daemon: {inbox_res.get('status')}")
+    
+    social_res = start_social()
+    logger.info(f"Social daemon: {social_res.get('status')}")
+    
+    repair_res = start_self_repair()
+    logger.info(f"Self-repair daemon: {repair_res.get('status')}")
+
+    try:
+        while True:
+            time.sleep(60)
+    except KeyboardInterrupt:
+        _handle_sigterm(None, None)
+
+if __name__ == "__main__":
+    main()