delimit-cli 4.5.13 → 4.6.1

package/gateway/ai/heartbeat.py

@@ -0,0 +1,290 @@
+"""Heartbeat liveness framework — Phase 1 local file-based (LED-1412).
+
+Solves the silent-staleness class that the 2026-05-15 session exposed:
+delimit-reddit-proxy.service was inactive/disabled for 13 days, all
+reddit scans failed silently with 429/403, and the founder noticed via
+"3 day old posts" — not the system. There was no central liveness
+reporting and no alert.
+
+Phase 1 (this module): every scheduled task writes a heartbeat file
+when it runs. A central check tool walks the heartbeat directory and
+flags anything stale. Local-only — Codex's correct caveat that
+heartbeats can't catch a full-host outage motivates Phase 2 (external
+deadman ping, tracked separately as LED-1414).
+
+Heartbeat file format — one per service at ~/.delimit/heartbeats/<service>.json:
+{
+  "service": "delimit-reddit-proxy",
+  "last_run": "2026-05-15T14:23:51Z",
+  "last_success": "2026-05-15T14:23:51Z",  # may differ from last_run on partial failure
+  "status": "ok" | "degraded" | "failed",
+  "next_expected": "2026-05-15T15:23:51Z",
+  "detail": "string — optional one-line context for status != ok"
+}
+
+Memory anchor: feedback_corrupted_worktree_phantom_failures.md (sister
+failure class — both surface as "system reports stale data because no-one
+checks freshness").
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+# All heartbeats live under one directory. Override via env for tests.
+DEFAULT_HEARTBEAT_DIR = Path.home() / ".delimit" / "heartbeats"
+
+# Per-service staleness thresholds (seconds). Overridable via config file
+# at ~/.delimit/heartbeats/_thresholds.json. Service names match the
+# `service` key written by write_heartbeat().
+DEFAULT_STALENESS_THRESHOLDS: Dict[str, int] = {
+    # Reddit scanner: hourly social loop. >2 hours = stale.
+    "delimit-reddit-proxy": 7200,
+    "delimit-social-loop": 7200,
+    # Inbox daemon: 5-min poll. >30 min = stale.
+    "delimit-inbox": 1800,
+    # License watch: daily timer. >36 hours = stale.
+    "delimit-license-watch": 129600,
+    # Drift check: daily. >36 hours = stale.
+    "delimit-drift-check": 129600,
+    # stake.one INJ-claim: daily 13:00 UTC. >30 hours = stale.
+    "stakeone-inj-claim": 108000,
+}
+
+# Fallback for services not in the threshold map.
+DEFAULT_FALLBACK_STALENESS = 86400  # 24 hours
+
+
+def _heartbeat_dir(override: Optional[str] = None) -> Path:
+    """Resolve the heartbeat directory. Honors:
+    - explicit override arg
+    - DELIMIT_HEARTBEAT_DIR env var
+    - default ~/.delimit/heartbeats/
+    """
+    if override:
+        return Path(override)
+    env = os.environ.get("DELIMIT_HEARTBEAT_DIR")
+    if env:
+        return Path(env)
+    return DEFAULT_HEARTBEAT_DIR
+
+
+def _now_iso() -> str:
+    """Current UTC time as ISO 8601 with Z suffix (matches existing
+    delimit timestamp convention)."""
+    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+
+
+def _parse_iso(ts: str) -> Optional[float]:
+    """Parse an ISO 8601 timestamp to a unix epoch float. Returns None
+    on parse failure — callers treat None as 'unknown' (degraded but
+    not actionable)."""
+    if not ts:
+        return None
+    try:
+        # %Y-%m-%dT%H:%M:%SZ — UTC, no fractional seconds.
+        return time.mktime(time.strptime(ts, "%Y-%m-%dT%H:%M:%SZ")) - time.timezone
+    except (ValueError, TypeError):
+        return None
+
+
+def write_heartbeat(
+    service: str,
+    status: str = "ok",
+    next_expected_in: Optional[int] = None,
+    detail: str = "",
+    success: bool = True,
+    heartbeat_dir: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Write a heartbeat for `service`.
+
+    Called by every scheduled task at the end of its run. On success,
+    pass status='ok' and success=True (default). On partial failure
+    (e.g., one of N subreddits 429'd but most succeeded), pass
+    status='degraded'. On total failure, status='failed' + success=False.
+
+    Args:
+        service: stable service identifier (e.g., 'delimit-reddit-proxy').
+            Should match the systemd unit name where applicable.
+        status: 'ok' | 'degraded' | 'failed'.
+        next_expected_in: seconds until the next run is expected. Used
+            by check_staleness to compute next_expected timestamp.
+        detail: optional one-line context (printed to operators on stale).
+        success: True if the run achieved its primary purpose (independent
+            of `status` — a successful run can still be 'degraded' if
+            some optional sub-tasks failed). last_success only updates
+            when True.
+        heartbeat_dir: override the heartbeat directory (for tests).
+
+    Returns:
+        Dict with the written record (also persisted to disk).
+    """
+    target_dir = _heartbeat_dir(heartbeat_dir)
+    target_dir.mkdir(parents=True, exist_ok=True)
+    file_path = target_dir / f"{service}.json"
+
+    now = _now_iso()
+    next_expected = ""
+    if next_expected_in:
+        next_expected_epoch = time.time() + next_expected_in
+        next_expected = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(next_expected_epoch))
+
+    # Preserve last_success across runs (only update if this run succeeded).
+    last_success = now if success else ""
+    if not success and file_path.exists():
+        try:
+            prior = json.loads(file_path.read_text())
+            last_success = prior.get("last_success", "")
+        except (json.JSONDecodeError, OSError):
+            pass  # Ignore corrupted prior; treat as no last_success known.
+
+    record = {
+        "service": service,
+        "last_run": now,
+        "last_success": last_success,
+        "status": status,
+        "next_expected": next_expected,
+        "detail": detail,
+    }
+    file_path.write_text(json.dumps(record, indent=2) + "\n")
+    return record
+
+
+def read_heartbeats(heartbeat_dir: Optional[str] = None) -> List[Dict[str, Any]]:
+    """Read every heartbeat file in the directory. Skips files that
+    don't parse as JSON (corrupted heartbeats are reported as a separate
+    'parse_error' entry so the operator sees them)."""
+    target_dir = _heartbeat_dir(heartbeat_dir)
+    if not target_dir.exists():
+        return []
+    out: List[Dict[str, Any]] = []
+    for path in sorted(target_dir.glob("*.json")):
+        # Skip the threshold config file
+        if path.name == "_thresholds.json":
+            continue
+        try:
+            data = json.loads(path.read_text())
+            out.append(data)
+        except (json.JSONDecodeError, OSError) as e:
+            out.append({
+                "service": path.stem,
+                "status": "parse_error",
+                "detail": f"heartbeat file {path.name} unreadable: {type(e).__name__}: {e}",
+                "last_run": "",
+                "last_success": "",
+                "next_expected": "",
+            })
+    return out
+
+
+def _load_thresholds(heartbeat_dir: Optional[str] = None) -> Dict[str, int]:
+    """Merge defaults with the optional override at <dir>/_thresholds.json."""
+    thresholds = dict(DEFAULT_STALENESS_THRESHOLDS)
+    target_dir = _heartbeat_dir(heartbeat_dir)
+    override_path = target_dir / "_thresholds.json"
+    if override_path.exists():
+        try:
+            override = json.loads(override_path.read_text())
+            if isinstance(override, dict):
+                thresholds.update({k: int(v) for k, v in override.items() if isinstance(v, (int, float))})
+        except (json.JSONDecodeError, OSError, ValueError):
+            pass
+    return thresholds
+
+
+def check_staleness(heartbeat_dir: Optional[str] = None) -> Dict[str, Any]:
+    """Walk all heartbeats and classify each by staleness.
+
+    Returns:
+        {
+          "checked_at": ISO8601 string,
+          "summary": {"ok": N, "stale": N, "degraded": N, "failed": N, "parse_error": N},
+          "services": [{service, status, last_run, last_success, age_seconds,
+                        threshold_seconds, classification}],
+          "stale_services": [<service names that are stale>],  # convenience for alerts
+        }
+
+    Classification rules (most-severe-first):
+      - parse_error: heartbeat file unreadable
+      - failed: status='failed' in the record
+      - stale: last_run older than threshold
+      - degraded: status='degraded' in the record
+      - ok: status='ok' AND last_run within threshold
+      - never_seen: heartbeat directory exists but service has no file
+        (only reported when a service is configured in thresholds but
+        has never written a heartbeat — surfaces "scheduled task never
+        ran since heartbeat instrumentation landed")
+    """
+    now = time.time()
+    records = read_heartbeats(heartbeat_dir)
+    thresholds = _load_thresholds(heartbeat_dir)
+
+    by_service: Dict[str, Dict[str, Any]] = {}
+    for rec in records:
+        service = rec.get("service", "?unknown?")
+        last_run_epoch = _parse_iso(rec.get("last_run", ""))
+        threshold = thresholds.get(service, DEFAULT_FALLBACK_STALENESS)
+        if last_run_epoch is not None:
+            age_seconds = int(now - last_run_epoch)
+        else:
+            age_seconds = -1
+
+        # Classify (most-severe-first)
+        if rec.get("status") == "parse_error":
+            classification = "parse_error"
+        elif rec.get("status") == "failed":
+            classification = "failed"
+        elif age_seconds < 0:
+            classification = "unknown_age"
+        elif age_seconds > threshold:
+            classification = "stale"
+        elif rec.get("status") == "degraded":
+            classification = "degraded"
+        else:
+            classification = "ok"
+
+        by_service[service] = {
+            "service": service,
+            "status": rec.get("status", "?"),
+            "last_run": rec.get("last_run", ""),
+            "last_success": rec.get("last_success", ""),
+            "age_seconds": age_seconds,
+            "threshold_seconds": threshold,
+            "classification": classification,
+            "detail": rec.get("detail", ""),
+        }
+
+    # Add never_seen entries for configured services that have no record
+    for service in thresholds.keys():
+        if service not in by_service:
+            by_service[service] = {
+                "service": service,
+                "status": "never_seen",
+                "last_run": "",
+                "last_success": "",
+                "age_seconds": -1,
+                "threshold_seconds": thresholds[service],
+                "classification": "never_seen",
+                "detail": "no heartbeat file found — service may not be instrumented yet",
+            }
+
+    services = list(by_service.values())
+    summary = {"ok": 0, "stale": 0, "degraded": 0, "failed": 0, "parse_error": 0,
+               "never_seen": 0, "unknown_age": 0}
+    stale_services = []
+    for svc in services:
+        c = svc["classification"]
+        summary[c] = summary.get(c, 0) + 1
+        if c in ("stale", "failed", "parse_error", "never_seen"):
+            stale_services.append(svc["service"])
+
+    return {
+        "checked_at": _now_iso(),
+        "summary": summary,
+        "services": services,
+        "stale_services": stale_services,
+    }

package/gateway/ai/hot_reload.py

@@ -35,11 +35,10 @@ import logging
 import os
 import sys
 import threading
-import time
 import traceback
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional, Set
+from typing import Any, Dict, List, Optional, Set
 
 logger = logging.getLogger("delimit.ai.hot_reload")
 

package/gateway/ai/led193_daemon/executor.py

@@ -346,11 +346,17 @@ def execute_format_fix(
         return out
     ok, reason = _ensure_clean_worktree(repo_path, runner=runner)
     if not ok:
+        # Precondition mismatch (worktree dirty), not a profile failure.
+        # Mark skipped so the consecutive-failure breaker doesn't trip.
+        out.result = "skipped"
         out.reason = reason
         return out
 
     cmd = _detect_format_command(repo_path)
     if cmd is None:
+        # Repo has no formatter config — daemon can't run, but this is
+        # a setup gap not an executor failure. Skip rather than fail.
+        out.result = "skipped"
         out.reason = "no_formatter_detected"
         return out
 
@@ -429,11 +435,13 @@ def execute_lockfile_refresh(
         return out
     ok, reason = _ensure_clean_worktree(repo_path, runner=runner)
     if not ok:
+        out.result = "skipped"
         out.reason = reason
         return out
 
     detected = _detect_lockfile_command(repo_path)
     if detected is None:
+        out.result = "skipped"
         out.reason = "no_lockfile_or_manager_detected"
         return out
     cmd, _lockfile = detected
@@ -551,6 +559,7 @@ def execute_docs_typo(
         return out
     ok, reason = _ensure_clean_worktree(repo_path, runner=runner)
     if not ok:
+        out.result = "skipped"
         out.reason = reason
         return out
 

package/gateway/ai/ledger_manager.py

@@ -124,7 +124,16 @@ def _detect_venture(project_path: str = ".") -> Dict[str, str]:
 
 
 def _register_venture(info: Dict[str, str]):
-    """Silently register a venture in the global registry."""
+    """Silently register a venture in the global registry.
+
+    Phase C follow-up (2026-05-18): reject paths under /tmp/* or the
+    bare "/tmp" itself. Pytest tmp_path values leaked into the registry
+    as ventures (`tmp: /tmp`, `test_project: /tmp/pytest-of-root/...`),
+    causing every fresh tmp_path to match via path-prefix in
+    resolve_venture and breaking test_resolve_venture_unregistered_path.
+    The guard fails-silently — tests that pass tmp_path to functions
+    which auto-register simply don't pollute the registry going forward.
+    """
     GLOBAL_DIR.mkdir(parents=True, exist_ok=True)
     ventures = {}
     if VENTURES_FILE.exists():
@@ -134,9 +143,19 @@ def _register_venture(info: Dict[str, str]):
             pass
 
     name = info["name"]
+    path = info.get("path", "")
+    # Guard against the specific test-state pollution that broke
+    # test_resolve_venture_unregistered_path: a `tmp: /tmp` venture
+    # caught EVERY pytest tmp_path via path-prefix in resolve_venture.
+    # Reject bare "/tmp" only. Deeper /tmp/<X> paths are fine — they
+    # only path-prefix-match their own subtree, not every tmp_path,
+    # AND legitimate test fixtures (e.g. test_ledger_proof) register
+    # subpaths during a single test run and need that to work.
+    if path == "/tmp" or path.rstrip("/") == "/tmp":
+        return
     if name not in ventures:
         ventures[name] = {
-            "path": info.get("path", ""),
+            "path": path,
             "repo": info.get("repo", ""),
             "type": info.get("type", ""),
             "registered_at": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
@@ -565,8 +584,18 @@ def update_item(
     blocks: Optional[str] = None,
     project_path: str = ".",
     worked_by: str = "",
+    commit_sha: Optional[str] = None,
+    pr_url: Optional[str] = None,
 ) -> Dict[str, Any]:
-    """Update an existing ledger item's fields."""
+    """Update an existing ledger item's fields.
+
+    LED-1408 Phase 1: when `status="done"` is requested, callers MAY provide
+    `commit_sha` and/or `pr_url` as proof that the work shipped to main.
+    The proof is recorded on the update event under `ship_proof` with a
+    `verified: bool` flag. Phase 1 does NOT enforce — items still
+    transition to `done` even without proof — but the flag lets future
+    audits and the Phase 2 reconciler find unverified-done items.
+    """
     _ensure(project_path)
     ledger_dir = _project_ledger_dir(project_path)
 
@@ -619,11 +648,63 @@ def update_item(
         if due_date:
             update["due_date"] = due_date
         if labels is not None:
+            # LED-2221: write to both `labels` and `tags`. The list_items
+            # reconstruction (around line ~870) merges update events into
+            # current state by checking the `tags` key only. Writing only
+            # `labels` silently drops the update at read time, which in
+            # particular meant the build daemon's `autonomous-build` tag
+            # check could never see tags written through the MCP. Keeping
+            # `labels` for any external consumer that reads the raw event
+            # stream; adding `tags` so the live state aggregator picks it up.
             update["labels"] = labels
+            update["tags"] = labels
         if blocked_by:
             update["blocked_by"] = blocked_by
         if blocks:
             update["blocks"] = blocks
+
+        # LED-1408 Phase 1: attach ship_proof block when status transitions to
+        # `done` or `shipped_pending`. Verified=True iff commit_sha or pr_url
+        # was supplied (directly or scraped from the note). Phase 2's
+        # reconciler will use this to distinguish "trustworthy done" from
+        # "marked done but never verified on main."
+        if status in ("done", "shipped_pending"):
+            try:
+                from ai.ledger_proof import build_ship_proof
+                update["ship_proof"] = build_ship_proof(
+                    commit_sha=commit_sha,
+                    pr_url=pr_url,
+                    note=note,
+                )
+                # LED-1420 Phase 2 strict-mode flip: when DELIMIT_LEDGER_STRICT_DONE=1,
+                # an unverified `done` transition is downgraded to `shipped_pending`
+                # so the nightly reconciler (scripts/delimit_ledger_reconciler.py)
+                # can promote it to `done` once a commit-trailer match shows up on
+                # origin/main. Off by default so existing workflows keep closing
+                # items without hitting an unexpected gate; flip when the
+                # reconciler has been observed running for ~1 week without
+                # surprises.
+                strict = os.environ.get("DELIMIT_LEDGER_STRICT_DONE") == "1"
+                if (
+                    strict
+                    and status == "done"
+                    and not update["ship_proof"].get("verified")
+                ):
+                    update["status"] = "shipped_pending"
+                    existing_note = update.get("note") or ""
+                    suffix = (
+                        "[LED-1420 strict-mode: downgraded done → shipped_pending — "
+                        "no commit_sha/pr_url proof; reconciler will upgrade to "
+                        "done when it finds a Ledger-Item: " + item_id + " trailer "
+                        "on origin/main]"
+                    )
+                    update["note"] = (existing_note + " " + suffix).strip() if existing_note else suffix
+            except Exception:
+                # Soft-fail: a ship_proof bug must not break ledger close.
+                # The unverified state will be re-detectable from the missing
+                # key on the next audit pass.
+                pass
+
         _append(path, update)
 
         # Sync to Supabase for dashboard visibility
@@ -1297,7 +1378,12 @@ def session_history(limit: int = 5) -> Dict[str, Any]:
 # `archive` is a soft transition (status="archived", appended to JSONL); items
 # stay in replay forever. NO hard delete. Per-item failures don't block others.
 BULK_ACTIONS = ("archive", "set_status", "set_priority", "add_tag", "mark_done", "cancel")
-_VALID_BULK_STATUSES = ("open", "in_progress", "blocked", "done", "cancelled", "archived", "completed")
+# LED-1408: `shipped_pending` is the intermediate state between "committed" and
+# "verified on main." Items transition to shipped_pending when a worker reports
+# completion (commit exists somewhere) but the orchestrator hasn't yet verified
+# the commit is reachable from origin/main. The reconciler (Phase 2) promotes
+# shipped_pending → done once reachability is confirmed.
+_VALID_BULK_STATUSES = ("open", "in_progress", "blocked", "shipped_pending", "done", "cancelled", "archived", "completed")
 _VALID_BULK_PRIORITIES = ("P0", "P1", "P2", "P3")
 
 

package/gateway/ai/ledger_proof.py

@@ -0,0 +1,127 @@
+"""Ledger ship-state proof helpers (LED-1408 Phase 1).
+
+When a ledger item transitions to `done`, we want auditable evidence that the
+fix actually shipped. Two forms of proof:
+
+1. **Commit-trailer binding** — the merge commit carries a `Ledger-Item:
+   LED-NNNN` trailer. Lets a webhook or reconciler walk `git log
+   origin/main` and find every commit-to-ledger link without naming
+   conventions or fuzzy matching.
+
+2. **PR-URL linkage** — `https://github.com/<org>/<repo>/pull/<N>` plus a
+   verified `merged_at` timestamp. Fallback for items closed without a
+   trailer.
+
+Phase 1 (this module): parse + record. Items closed with proof get
+`verified: true` on the event; items closed without proof get
+`verified: false` so a future audit can find them.
+
+Phase 2 (separate LED): the reconciler enforces stricter semantics —
+items without proof default to `shipped_pending`, not `done`.
+
+Memory anchor: feedback_agent_dashboard_done_means_committed_not_merged.md
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Dict, Optional
+
+# Match `Ledger-Item: LED-1234` (case-insensitive, leading whitespace allowed).
+# Pattern intentionally tolerant of trailing whitespace + multiple LED IDs:
+# we extract the FIRST LED-N on the line.
+_LEDGER_TRAILER_RE = re.compile(
+    r"(?im)^\s*Ledger-Item\s*:\s*(LED-\d+)",
+)
+
+# Match a GitHub PR URL: https://github.com/<owner>/<repo>/pull/<N>
+_PR_URL_RE = re.compile(
+    r"https://github\.com/([\w.-]+)/([\w.-]+)/pull/(\d+)",
+)
+
+
+def parse_ledger_trailer(commit_message: str) -> Optional[str]:
+    """Extract the `Ledger-Item: LED-NNNN` trailer value from a commit message.
+
+    Returns the LED id (e.g. `LED-1408`) or None if no trailer is present.
+    The trailer must be on its own line; mentions inside prose (e.g.
+    `mentions LED-1408 in passing`) do NOT match.
+    """
+    if not commit_message:
+        return None
+    match = _LEDGER_TRAILER_RE.search(commit_message)
+    if match:
+        return match.group(1)
+    return None
+
+
+def parse_pr_url(text: str) -> Optional[Dict[str, str]]:
+    """Extract the first GitHub PR URL from any string.
+
+    Returns {owner, repo, number} or None.
+    """
+    if not text:
+        return None
+    match = _PR_URL_RE.search(text)
+    if match:
+        return {
+            "owner": match.group(1),
+            "repo": match.group(2),
+            "number": match.group(3),
+        }
+    return None
+
+
+def build_ship_proof(
+    commit_sha: Optional[str] = None,
+    pr_url: Optional[str] = None,
+    note: Optional[str] = None,
+) -> Dict[str, object]:
+    """Build a ship-proof block to attach to a ledger `done` event.
+
+    Inputs may come from explicit MCP-tool args OR from inline mentions
+    in the note (the caller might paste a PR URL into the note field
+    without realizing it's also queryable).
+
+    Returns a dict with keys:
+      - verified: bool — True iff commit_sha OR pr_url was provided
+      - commit_sha: str or None
+      - pr_url: str or None
+      - pr_owner / pr_repo / pr_number: str or None (parsed from pr_url)
+      - ledger_trailer: str or None (parsed from note, if present)
+
+    The `verified` flag is the primary downstream signal. A future
+    reconciler will refuse to transition `done` without verified=True;
+    Phase 1 only records the flag without enforcing.
+    """
+    proof: Dict[str, object] = {
+        "verified": bool(commit_sha or pr_url),
+        "commit_sha": commit_sha or None,
+        "pr_url": pr_url or None,
+    }
+
+    if pr_url:
+        parsed = parse_pr_url(pr_url)
+        if parsed:
+            proof["pr_owner"] = parsed["owner"]
+            proof["pr_repo"] = parsed["repo"]
+            proof["pr_number"] = parsed["number"]
+
+    # If pr_url not explicitly passed but appears in the note, capture it
+    if not pr_url and note:
+        parsed = parse_pr_url(note)
+        if parsed:
+            proof["pr_url"] = f"https://github.com/{parsed['owner']}/{parsed['repo']}/pull/{parsed['number']}"
+            proof["pr_owner"] = parsed["owner"]
+            proof["pr_repo"] = parsed["repo"]
+            proof["pr_number"] = parsed["number"]
+            proof["verified"] = True
+
+    # Capture any Ledger-Item trailer from the note (rare but possible —
+    # a worker might paste the commit message into the close note).
+    if note:
+        trailer = parse_ledger_trailer(note)
+        if trailer:
+            proof["ledger_trailer"] = trailer
+
+    return proof