delimit-cli 4.5.13 → 4.6.1

package/gateway/ai/social_capability/fit_floor.py

@@ -45,10 +45,19 @@ from typing import Any, Dict, Iterable, Optional, Set, Tuple
 
 logger = logging.getLogger(__name__)
 
-# Default cooldown window for topic coverage. Founder directive (2026-05-05):
-# "if we've already drafted on a similar topic in the last 7 days, abstain on
-# the next one to avoid spam-pattern detection."
-DEFAULT_COOLDOWN_DAYS = 7
+# Default cooldown window for topic coverage.
+# Original founder directive (2026-05-05): 7 days to avoid spam-pattern detection.
+# Revised 2026-05-12 (LED-1356, panel-unanimous): 7 days × cross-platform global
+# scope was producing 57% abstention rate (115/200 recent abstentions =
+# topic_cooldown) and effectively blocked every Delimit-relevant topic for a
+# full week after one draft. Reduced to 48h AND scoped per-platform/per-subreddit
+# so a Reddit/r/mcp draft no longer cools an HN draft on MCP, and a Reddit/r/mcp
+# draft no longer cools a Reddit/r/programming draft on MCP. Different audiences,
+# different scans.
+DEFAULT_COOLDOWN_HOURS = 48
+# Backwards-compat alias for any callers that still pass `cooldown_days=` —
+# converted to hours internally. Don't add new callers using this name.
+DEFAULT_COOLDOWN_DAYS = DEFAULT_COOLDOWN_HOURS / 24
 
 # Default high-engagement opportunity-cost threshold. Threads above this score
 # pass the fit floor even without keyword match, but with ``human_only=True``
@@ -179,18 +188,63 @@ def _parse_iso(value: Optional[str]) -> Optional[datetime]:
     return dt
 
 
+def _log_entry_scope_key(entry: dict) -> str:
+    """LED-1356 platform-scoped cooldown key. Reddit cooldowns are per-subreddit
+    (different subreddits = different audiences); HN/devto/X are global within
+    the platform. Cross-platform doesn't cool.
+
+    Returns:
+        - "reddit:r/<name>" for Reddit entries with a subreddit field
+        - "reddit" for Reddit entries without (legacy fallback)
+        - The platform name itself for hn / devto / x / twitter
+        - "unknown" when the entry doesn't carry a platform
+    """
+    platform = (entry.get("platform") or "").lower()
+    if not platform:
+        return "unknown"
+    if platform == "reddit":
+        sub = (entry.get("subreddit") or "").lower()
+        if sub:
+            sub = sub.lstrip("r/")
+            return f"reddit:r/{sub}"
+        return "reddit"
+    # twitter and x are aliases for the same audience
+    if platform == "twitter":
+        return "x"
+    return platform
+
+
 def _recent_topic_fingerprints(
-    cooldown_days: int = DEFAULT_COOLDOWN_DAYS,
+    cooldown_hours: float = DEFAULT_COOLDOWN_HOURS,
     log_path: Optional[Path] = None,
-) -> Set[str]:
-    """Return the union of topic fingerprints found in ``social_log.jsonl``
-    within the cooldown window. Tolerant of malformed lines.
+    cooldown_days: Optional[float] = None,
+) -> Dict[str, Set[str]]:
+    """Return platform-scoped topic fingerprints from ``social_log.jsonl``
+    within the cooldown window.
+
+    LED-1356 changes (2026-05-12, panel-unanimous):
+    - Window default 7d → 48h.
+    - Return shape Set[str] → Dict[scope_key, Set[str]] so callers can ask
+      "what topics has THIS audience seen recently" instead of the prior
+      "global union of every topic across every platform."
+    - `cooldown_days` is preserved as backwards-compat kwarg; converted to
+      hours internally.
+
+    Callers should look up the relevant set via the dict and pass it to
+    ``evaluate_fit(..., recent_topics=...)``. A target's scope is computed
+    via :func:`_log_entry_scope_key` on the target dict.
+
+    Returns dict mapping scope_key → set of canonical signal fingerprints
+    drafted on for that scope within the window.
     """
     p = log_path or _social_log_path()
     if not p.exists():
-        return set()
-    cutoff = datetime.now(timezone.utc) - timedelta(days=cooldown_days)
-    seen: Set[str] = set()
+        return {}
+    # Backwards compatibility: if cooldown_days passed, convert to hours.
+    if cooldown_days is not None:
+        cooldown_hours = float(cooldown_days) * 24.0
+    cutoff = datetime.now(timezone.utc) - timedelta(hours=cooldown_hours)
+    seen: Dict[str, Set[str]] = {}
     try:
         with open(p, "r", encoding="utf-8") as fh:
             for line in fh:
@@ -204,6 +258,7 @@ def _recent_topic_fingerprints(
                 ts = _parse_iso(entry.get("ts"))
                 if ts is None or ts < cutoff:
                     continue
+                scope_key = _log_entry_scope_key(entry)
                 # Use the post body + thread title (when present) as the
                 # topic surface. Reddit entries log the thread title
                 # separately; X entries don't have one but the post body
@@ -214,12 +269,55 @@ def _recent_topic_fingerprints(
                         entry.get("thread_title") or "",
                     ]
                 )
-                seen.update(_extract_topic_fingerprint(blob))
+                fps = _extract_topic_fingerprint(blob)
+                if not fps:
+                    continue
+                seen.setdefault(scope_key, set()).update(fps)
     except OSError as exc:
         logger.warning("fit_floor: failed to read %s: %s", p, exc)
     return seen
 
 
+def topics_for_scope(
+    recent_topics_dict: Dict[str, Set[str]],
+    scope_key: str,
+) -> Set[str]:
+    """Convenience helper: look up the topic set for a given scope.
+
+    Returns empty set if the scope has no recent activity in the window.
+    Callers should pass the result as ``evaluate_fit(..., recent_topics=...)``.
+    """
+    return recent_topics_dict.get(scope_key, set())
+
+
+def target_scope_key(target: dict) -> str:
+    """Compute the scope key for a target dict (for cooldown lookup).
+
+    Mirror of :func:`_log_entry_scope_key` for the inbound target side.
+    Targets may carry `platform` plus (for Reddit) `subreddit` extracted
+    from the thread URL.
+    """
+    platform = (target.get("platform") or "").lower()
+    if not platform:
+        return "unknown"
+    if platform == "reddit":
+        sub = (target.get("subreddit") or "").lower()
+        if not sub:
+            # Try to extract from canonical_url / thread_url
+            url = target.get("canonical_url") or target.get("thread_url") or ""
+            import re as _re
+            m = _re.search(r"/r/([A-Za-z0-9_]+)", url)
+            if m:
+                sub = m.group(1).lower()
+        if sub:
+            sub = sub.lstrip("r/")
+            return f"reddit:r/{sub}"
+        return "reddit"
+    if platform == "twitter":
+        return "x"
+    return platform
+
+
 # ── Fit-floor decision ──────────────────────────────────────────────
 
 
@@ -348,6 +446,7 @@ def append_jsonl(path: Path, payload: Dict[str, Any]) -> None:
 
 __all__ = [
     "DEFAULT_COOLDOWN_DAYS",
+    "DEFAULT_COOLDOWN_HOURS",
     "DEFAULT_HIGH_ENGAGEMENT_FLOOR",
     "SOCIAL_LOG",
     "DELIMIT_DOMAIN_SIGNALS",
@@ -357,4 +456,7 @@ __all__ = [
     "append_jsonl",
     "_extract_topic_fingerprint",
     "_recent_topic_fingerprints",
+    "_log_entry_scope_key",
+    "target_scope_key",
+    "topics_for_scope",
 ]

package/gateway/ai/social_queue.py

@@ -48,6 +48,55 @@ QUEUE_FILE = Path.home() / ".delimit" / "social_scan_queue.jsonl"
 DEFAULT_DEDUPE_HOURS = 24 * 7  # don't re-queue a fingerprint within 7 days
 DEFAULT_EXPIRE_HOURS = 24 * 7  # entries older than 7 days roll to expired
 
+# Per-platform freshness cap at claim_pending time. Reddit posts decay
+# in comment-visibility VERY fast (Boris-Cherny LED-1335: <6h high-yield,
+# <12h marginal, ~zero after 24h), so drafting on a 3-day-old post wastes
+# a brand-account engagement. Other platforms (github, devto) have longer
+# half-lives — issue threads can be relevant weeks later — so we don't
+# apply the freshness cap there.
+#
+# Founder regression report 2026-05-18: drafts were being generated on
+# posts queued 3+ days earlier because the queue is FIFO and the drafter
+# falls behind the scanner. This filter ensures claim_pending() never
+# returns reddit entries whose `queued_at` is more than CLAIM_FRESHNESS_HOURS
+# old, regardless of queue position.
+CLAIM_FRESHNESS_HOURS_BY_PLATFORM: Dict[str, int] = {
+    "reddit": 24,
+    # Phase C (2026-05-18): github targets fail ~96% of the time
+    # (historical 3256/3389 marked drafted_failed) and the queue grew
+    # to 1122 pending dominating FIFO order. 24h cap drains stale crud
+    # while preserving fresh github targets the drafter would actually
+    # process. Without this, github starves reddit/x/hn even with the
+    # round-robin claim (see CLAIM_MAX_PER_PLATFORM).
+    "github": 24,
+}
+
+# Phase C (2026-05-18): round-robin claim_pending across platforms so a
+# noisy platform doesn't starve quieter ones. Drafter calls claim_pending
+# with limit=10; pre-Phase-C this returned 10 oldest entries regardless
+# of platform, which with github=1122 pending meant 10/10 github and
+# reddit drafts never fired. With this cap, drafter sees a balanced mix.
+# Within-platform order is FIFO (oldest pending first) EXCEPT for
+# platforms in CLAIM_LIFO_PLATFORMS — see below.
+CLAIM_MAX_PER_PLATFORM: int = 3
+
+# Phase D (2026-05-18 founder request): "we need first-poster advantage."
+# For time-critical engagement platforms, the drafter should pick the
+# FRESHEST pending entry, not the oldest. Reddit comment visibility decays
+# sharply after the first 15-30 minutes of a thread (the first 5-10 visible
+# comments capture the bulk of upvotes + clickthrough). Pre-Phase-D's
+# within-reddit FIFO meant the drafter pulled 22-24h-old entries (near
+# the freshness cap) instead of brand-new ones.
+#
+# LIFO-within-platform reverses that for the listed platforms: within the
+# eligible bucket, sort newest queued_at first. Across platforms, round-
+# robin still applies. Entries that get displaced by newer ones are
+# naturally cleaned up by the freshness cap groomer.
+#
+# Other platforms (github, devto, etc.) keep FIFO — their content has a
+# longer half-life and oldest-first is the right discipline there.
+CLAIM_LIFO_PLATFORMS: set = {"reddit"}
+
 PENDING = "pending"
 DRAFTED = "drafted"
 DRAFTED_FAILED = "drafted_failed"
@@ -184,23 +233,130 @@ def enqueue(target: Dict[str, Any], dedupe_hours: int = DEFAULT_DEDUPE_HOURS) ->
 
 
 def claim_pending(platform: Optional[str] = None, limit: int = 20) -> List[Dict[str, Any]]:
-    """Return up to ``limit`` pending entries, optionally filtered by platform.
+    """Return up to ``limit`` pending entries, with round-robin balancing
+    across platforms when ``platform`` is None.
 
     Read-only — does NOT mutate state. The caller must call ``mark_drafted``
-    or ``mark_failed`` once it processes the entry. Returns oldest-first
-    (FIFO) so the queue drains in scan order.
+    or ``mark_failed`` once it processes the entry.
+
+    Round-robin (Phase C, 2026-05-18): without a platform filter, returns
+    at most CLAIM_MAX_PER_PLATFORM entries from any single platform per
+    call. Within each platform, oldest-first (FIFO). Across platforms,
+    interleaved so the drafter sees a balanced mix instead of saturating
+    on whichever platform has the deepest backlog.
+
+    With an explicit ``platform`` filter, behaves as FIFO over that single
+    platform's pending entries (no per-platform cap, since the caller is
+    already targeting one platform).
+
+    Freshness cap (Phase A, 2026-05-18): entries whose platform has a
+    CLAIM_FRESHNESS_HOURS_BY_PLATFORM cap and whose ``queued_at`` is
+    older than that cap are skipped silently. Those stale entries stay
+    in the file with status=pending; the separate ``expire_stale_for_
+    freshness_caps`` pass flips them so they don't pile up forever.
     """
-    out: List[Dict[str, Any]] = []
-    # Build a list because we want oldest-first; JSONL append order = FIFO.
-    for entry in _iter_entries():
+    now = datetime.now(timezone.utc)
+
+    def _is_eligible(entry: Dict[str, Any]) -> bool:
         if entry.get("status") != PENDING:
-            continue
+            return False
         if platform and entry.get("platform") != platform:
+            return False
+        cap = CLAIM_FRESHNESS_HOURS_BY_PLATFORM.get(entry.get("platform"))
+        if cap is not None:
+            qts = _parse_iso(entry.get("queued_at"))
+            if qts is not None and (now - qts) > timedelta(hours=cap):
+                return False
+        return True
+
+    # Single-platform filter path: keep legacy strict-FIFO behavior.
+    if platform:
+        out: List[Dict[str, Any]] = []
+        for entry in _iter_entries():
+            if not _is_eligible(entry):
+                continue
+            out.append(entry)
+            if len(out) >= limit:
+                break
+        return out
+
+    # Round-robin path: group by platform first (preserving within-platform
+    # FIFO via iteration order), then cap per-platform and interleave.
+    # CRITICAL Phase D change: for CLAIM_LIFO_PLATFORMS, collect the full
+    # eligible set per platform first, then sort newest-first BEFORE
+    # truncation — otherwise the early-break truncation in the FIFO path
+    # would keep the oldest entries even when we want the newest.
+    by_platform: Dict[str, List[Dict[str, Any]]] = {}
+    for entry in _iter_entries():
+        if not _is_eligible(entry):
             continue
-        out.append(entry)
-        if len(out) >= limit:
+        plat = entry.get("platform") or "unknown"
+        by_platform.setdefault(plat, []).append(entry)
+
+    # Sort + truncate each bucket.
+    for plat in list(by_platform.keys()):
+        if plat in CLAIM_LIFO_PLATFORMS:
+            # Newest queued_at first. Parse-failures sort last so a
+            # corrupted-timestamp entry doesn't block legitimate fresh
+            # entries from being claimed.
+            by_platform[plat].sort(
+                key=lambda e: _parse_iso(e.get("queued_at")) or datetime.min.replace(tzinfo=timezone.utc),
+                reverse=True,
+            )
+        # FIFO platforms keep insertion order (which is JSONL-append =
+        # oldest first); no sort needed.
+        by_platform[plat] = by_platform[plat][:CLAIM_MAX_PER_PLATFORM]
+
+    # Interleave: round-robin across platforms in alphabetical order for
+    # determinism. Stop when limit is reached or all buckets are drained.
+    out2: List[Dict[str, Any]] = []
+    plat_order = sorted(by_platform.keys())
+    idx = {p: 0 for p in plat_order}
+    while len(out2) < limit:
+        added = False
+        for p in plat_order:
+            if idx[p] < len(by_platform[p]):
+                out2.append(by_platform[p][idx[p]])
+                idx[p] += 1
+                added = True
+                if len(out2) >= limit:
+                    break
+        if not added:
             break
-    return out
+    return out2
+
+
+def expire_stale_for_freshness_caps() -> Dict[str, int]:
+    """Roll pending entries past their platform's claim freshness cap to expired.
+
+    Companion to ``claim_pending``'s in-flight skip: without this, the
+    queue file fills up with pending-but-permanently-skipped entries
+    that we still re-scan on every claim. Returns a dict
+    ``{platform: count_expired}`` for observability.
+    """
+    now = datetime.now(timezone.utc)
+    entries = list(_iter_entries())
+    if not entries:
+        return {}
+    flipped: Dict[str, int] = {}
+    changed = False
+    for entry in entries:
+        if entry.get("status") != PENDING:
+            continue
+        plat = entry.get("platform")
+        cap = CLAIM_FRESHNESS_HOURS_BY_PLATFORM.get(plat)
+        if cap is None:
+            continue
+        qts = _parse_iso(entry.get("queued_at"))
+        if qts is None or (now - qts) <= timedelta(hours=cap):
+            continue
+        entry["status"] = EXPIRED
+        entry["error"] = f"expired_freshness_cap_{cap}h"
+        flipped[plat] = flipped.get(plat, 0) + 1
+        changed = True
+    if changed:
+        _atomic_rewrite(entries)
+    return flipped
 
 
 def _update_entry(fingerprint: str, mutator) -> bool: