@m13v/s4l 1.7.5-rc.2 → 1.7.5-rc.21

package/scripts/engagement_signal_feedback.py

@@ -0,0 +1,256 @@
+#!/usr/bin/env python3
+"""engagement_signal_feedback.py — turn downstream reply engagement into two
+durable, whitelist-enforced learned_preferences_global entries:
+
+1. Warm-signal authors: repeat, constructive, low-risk authors (matching the
+   same repeat_constructive_author shape reply_risk_digest.py already tags)
+   get promoted into `audience_prefer` so future judging treats them as a
+   positive signal instead of getting rediscovered from scratch every day.
+
+2. Style performance: engagement_style values with enough sample size are
+   ranked by child-reply continuation rate (the strongest observed signal
+   for "this reply kept the thread going"); the current leader is recorded
+   as a `draft_style_notes` entry so the drafting model actually defaults to
+   the pattern that's been working, not just whatever the digest narrates in
+   a one-off email.
+
+Both write through learned_preferences.apply_mutations(), the SAME
+whitelist-enforced, flock+backup+atomic-replace writer the human review-card
+feedback loop uses (see scripts/learned_preferences.py). This script is a
+second, independent caller of that writer, not a new writer: the whitelist
+enforcement lives in apply_mutations() itself, so nothing here can touch
+config.json outside audience_prefer / draft_style_notes.
+
+All DB reads go through the s4l.ai HTTP API (http_api), never direct SQL, so
+this script works unmodified on shipped customer installs, not just this
+operator box. Risk-scoring reuses reply_risk_digest._skip_reason_risk_score
+by import (read-only; that file is intentionally not modified here) instead
+of forking a second copy of the skip-risk heuristics.
+
+Usage:
+    python3 scripts/engagement_signal_feedback.py --dry-run
+    python3 scripts/engagement_signal_feedback.py --platform x --days 30
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from collections import Counter, defaultdict
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+REPO_DIR = Path(__file__).resolve().parent.parent
+SCRIPT_DIR = REPO_DIR / "scripts"
+sys.path.insert(0, str(SCRIPT_DIR))
+
+from db import load_env  # noqa: E402
+from http_api import api_get  # noqa: E402
+from learned_preferences import apply_mutations, get_global_block  # noqa: E402
+from reply_risk_digest import _skip_reason_risk_score  # noqa: E402
+
+DEFAULT_DAYS = 30
+MIN_STYLE_SAMPLE = 10
+WARM_MIN_LAST_30D = 4
+WARM_MIN_CHILD_REPLIES = 1
+MAX_WARM_PROMOTIONS_PER_RUN = 3
+ENTRY_CHAR_LIMIT = 200  # mirrors learned_preferences.MAX_ENTRY_CHARS
+
+
+def fetch_window_rows(platform: str, days: int, limit_pages: int = 20) -> list[dict]:
+    """Page through /api/v1/replies for the lookback window, id-cursor style
+    (mirrors enrich_reply_parents.py's fetch_work loop). Capped at
+    limit_pages * 500 rows as a sane upper bound for a 30d window."""
+    since = (datetime.now(timezone.utc) - timedelta(days=int(days))).isoformat()
+    out: list[dict] = []
+    before_id = None
+    for _ in range(limit_pages):
+        query = {"platform": platform, "since": since, "order_by": "id", "limit": "500"}
+        if before_id:
+            query["before_id"] = str(before_id)
+        resp = api_get("/api/v1/replies", query=query)
+        rows = (resp.get("data") or {}).get("replies") or []
+        if not rows:
+            break
+        out.extend(rows)
+        min_id = min(r["id"] for r in rows)
+        if before_id is not None and min_id >= before_id:
+            break
+        before_id = min_id
+        if len(rows) < 500:
+            break
+    return out
+
+
+def compute_style_stats(rows: list[dict]) -> list[dict]:
+    by_style: dict[str, dict] = defaultdict(lambda: {"n": 0, "child_replies": 0, "upvotes": 0})
+    for r in rows:
+        if r.get("status") != "replied":
+            continue
+        style = r.get("engagement_style")
+        if not style:
+            continue
+        s = by_style[style]
+        s["n"] += 1
+        s["child_replies"] += int(r.get("comments_count") or 0)
+        s["upvotes"] += int(r.get("upvotes") or 0)
+
+    ranked = []
+    for style, s in by_style.items():
+        if s["n"] < MIN_STYLE_SAMPLE:
+            continue
+        ranked.append({
+            "style": style,
+            "n": s["n"],
+            "child_reply_rate": s["child_replies"] / s["n"],
+            "upvote_rate": s["upvotes"] / s["n"],
+        })
+    ranked.sort(key=lambda x: x["child_reply_rate"], reverse=True)
+    return ranked
+
+
+def compute_warm_authors(rows: list[dict]) -> list[dict]:
+    by_author: dict[str, dict] = defaultdict(lambda: {
+        "n": 0, "replied": 0, "riskish_skips": 0, "child_replies": 0,
+        "upvotes": 0, "projects": Counter(),
+    })
+    for r in rows:
+        handle = (r.get("their_author") or "").strip()
+        if not handle:
+            continue
+        a = by_author[handle]
+        a["n"] += 1
+        if r.get("status") == "replied":
+            a["replied"] += 1
+        if _skip_reason_risk_score(r.get("skip_reason") or "") >= 4:
+            a["riskish_skips"] += 1
+        a["child_replies"] += int(r.get("comments_count") or 0)
+        a["upvotes"] += int(r.get("upvotes") or 0)
+        if r.get("project_name"):
+            a["projects"][r["project_name"]] += 1
+
+    warm = []
+    for handle, a in by_author.items():
+        if a["n"] < WARM_MIN_LAST_30D:
+            continue
+        if a["riskish_skips"] > 0:
+            continue
+        if a["child_replies"] < WARM_MIN_CHILD_REPLIES:
+            continue
+        top_project = a["projects"].most_common(1)[0][0] if a["projects"] else None
+        warm.append({
+            "handle": handle,
+            "last_30d": a["n"],
+            "replied": a["replied"],
+            "child_replies": a["child_replies"],
+            "upvotes": a["upvotes"],
+            "project_name": top_project,
+        })
+    warm.sort(key=lambda x: (x["child_replies"], x["last_30d"]), reverse=True)
+    return warm
+
+
+def pick_default_project(cfg: dict, fallback_candidates: list[dict]) -> str | None:
+    for c in fallback_candidates:
+        if c.get("project_name"):
+            return c["project_name"]
+    projects = cfg.get("projects") or []
+    return projects[0].get("name") if projects else None
+
+
+def build_warm_entry(w: dict) -> str:
+    entry = (
+        f"@{w['handle']} — repeat constructive practitioner "
+        f"({w['replied']}/{w['last_30d']} replied in 30d, {w['child_replies']} child "
+        f"replies, 0 risk skips); warm to more engaged/product-relevant replies."
+    )
+    return entry[:ENTRY_CHAR_LIMIT]
+
+
+def build_style_note(top: dict) -> str:
+    note = (
+        f"engagement_style='{top['style']}' shows the best child-reply continuation rate "
+        f"({top['child_reply_rate']:.2f}/reply, n={top['n']}); prefer it for technical-"
+        "correction replies (concrete metric, name the exact failure mode)."
+    )
+    if len(note) > ENTRY_CHAR_LIMIT:
+        note = note[: ENTRY_CHAR_LIMIT - 1].rsplit(" ", 1)[0] + "…"
+    return note
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--platform", default="x")
+    ap.add_argument("--days", type=int, default=DEFAULT_DAYS)
+    ap.add_argument("--dry-run", action="store_true")
+    args = ap.parse_args()
+    load_env()
+
+    rows = fetch_window_rows(args.platform, args.days)
+    print(f"[engagement_signal_feedback] fetched {len(rows)} rows over last {args.days}d ({args.platform})", file=sys.stderr)
+
+    style_ranked = compute_style_stats(rows)
+    warm_authors = compute_warm_authors(rows)[:MAX_WARM_PROMOTIONS_PER_RUN]
+
+    cfg_path_val = None
+    try:
+        from learned_preferences import config_path
+        cfg_path_val = config_path()
+        cfg = json.loads(Path(cfg_path_val).read_text())
+    except Exception as e:
+        print(f"[engagement_signal_feedback] cannot read config: {e}", file=sys.stderr)
+        return 1
+
+    block = get_global_block(cfg)
+    existing_prefer = block.get("audience_prefer") or []
+    existing_notes = block.get("draft_style_notes") or []
+
+    audience_prefer_add = []
+    for w in warm_authors:
+        handle_lower = w["handle"].lower()
+        already = any(f"@{handle_lower}" in e.lower() for e in existing_prefer)
+        if already:
+            continue
+        audience_prefer_add.append(build_warm_entry(w))
+
+    draft_style_notes_add = []
+    if style_ranked:
+        top = style_ranked[0]
+        already_noted = any(f"engagement_style='{top['style']}'" in n for n in existing_notes)
+        if not already_noted:
+            draft_style_notes_add.append(build_style_note(top))
+
+    print("[engagement_signal_feedback] style ranking:", file=sys.stderr)
+    for s in style_ranked:
+        print(f"    {s['style']}: n={s['n']} child_reply_rate={s['child_reply_rate']:.2f} upvote_rate={s['upvote_rate']:.2f}", file=sys.stderr)
+    print(f"[engagement_signal_feedback] warm candidates this run: {[w['handle'] for w in warm_authors]}", file=sys.stderr)
+    print(f"[engagement_signal_feedback] audience_prefer additions: {audience_prefer_add}", file=sys.stderr)
+    print(f"[engagement_signal_feedback] draft_style_notes additions: {draft_style_notes_add}", file=sys.stderr)
+
+    if not audience_prefer_add and not draft_style_notes_add:
+        print("[engagement_signal_feedback] nothing new to promote", file=sys.stderr)
+        return 0
+
+    if args.dry_run:
+        print("[engagement_signal_feedback] --dry-run: not writing", file=sys.stderr)
+        return 0
+
+    project_name = pick_default_project(cfg, warm_authors)
+    if not project_name:
+        print("[engagement_signal_feedback] no project available to validate write; aborting", file=sys.stderr)
+        return 1
+
+    plan = {"changes": {}, "rationale": "engagement_signal_feedback.py: downstream reply engagement (child-reply/upvote rates)"}
+    if audience_prefer_add:
+        plan["changes"]["audience_prefer"] = {"add": audience_prefer_add}
+    if draft_style_notes_add:
+        plan["changes"]["draft_style_notes"] = {"add": draft_style_notes_add}
+
+    result = apply_mutations(project_name, plan, cfg_path=cfg_path_val)
+    print(f"[engagement_signal_feedback] apply_mutations result: {json.dumps(result)}", file=sys.stderr)
+    return 0 if result.get("ok") else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/learned_preferences.py

@@ -22,10 +22,12 @@ The feedback loop:
      `learned_preferences_global` block, whitelist-enforced, with flock +
      backup + atomic replace.
   4. Enforcement is SOFT (prompt-level, never a deterministic filter): the
-     twitter prep prompt embeds every project entry verbatim via
-     ALL_PROJECTS_JSON (each project dict is stamped with the same global
-     block at prompt-build time), so the block (with its self-describing
-     _instruction) reaches the judging/drafting model automatically.
+     twitter prep prompt embeds the block exactly ONCE via
+     GLOBAL_LEARNED_PREFS_JSON (the 'Global learned preferences' line under
+     PROJECT ROUTING; per-project stamping retired 2026-07-14), so the block
+     (with its self-describing _instruction) reaches the judging/drafting
+     model automatically. `history` is stripped at prompt-build time: it is
+     a config.json-only audit changelog with no reader in any prompt.
      prompt_block() renders the same content for callers that want an
      explicit section instead of the raw embed.
 
@@ -81,9 +83,9 @@ MAX_HISTORY = 50
 MAX_EDIT_EXAMPLES = 5
 MAX_EXAMPLE_CHARS = 600
 
-# Travels inside the JSON the prep prompt embeds (ALL_PROJECTS_JSON is the
-# full project entry), so the drafting model reads its own operating manual
-# for the block. Split semantics (2026-07-03, after the persona lane kept
+# Travels inside the global block the prep prompt embeds (once, via
+# GLOBAL_LEARNED_PREFS_JSON), so the drafting model reads its own operating
+# manual for the block. Split semantics (2026-07-03, after the persona lane kept
 # recycling a rejected draft structure): candidate JUDGING signals stay soft
 # (judgment, not a hard ban), but draft_style_notes is MANDATORY when
 # writing; if it doesn't beat the engagement style's structural template,
@@ -108,13 +110,34 @@ DEFAULT_INSTRUCTION = (
 
 
 def config_path() -> str:
-    explicit = os.environ.get("S4L_CONFIG_PATH")
-    if explicit:
-        return explicit
-    repo = os.environ.get("S4L_REPO_DIR")
-    if repo:
-        return os.path.join(repo, "config.json")
-    return os.path.expanduser("~/social-autoposter/config.json")
+    """Delegates to scripts/config.py, THE config resolver (state-dir first,
+    symlinks resolved to their real target). This module's os.replace()
+    writers previously flattened the operator Mac's config symlink into a
+    plain file whenever S4L_CONFIG_PATH was missing from the environment
+    (2026-07-11 and 2026-07-13 incidents); the shared resolver's realpath
+    makes that impossible regardless of env."""
+    try:
+        from config import config_path as _shared
+
+        return _shared()
+    except Exception:
+        # Direct-invocation fallback (scripts/ not importable): same order.
+        explicit = os.environ.get("S4L_CONFIG_PATH")
+        if explicit:
+            p = os.path.expanduser(explicit)
+        else:
+            state_dir = os.environ.get("S4L_STATE_DIR") or "~/.social-autoposter-mcp"
+            sp = os.path.join(os.path.expanduser(state_dir), "config.json")
+            if os.path.exists(sp):
+                p = sp
+            else:
+                repo = os.environ.get("S4L_REPO_DIR")
+                p = (
+                    os.path.join(repo, "config.json")
+                    if repo
+                    else os.path.expanduser("~/social-autoposter/config.json")
+                )
+        return os.path.realpath(p) if os.path.exists(p) else p
 
 
 def _now_iso() -> str:

package/scripts/memory_snapshot.py

@@ -701,6 +701,7 @@ def build_summary() -> dict[str, Any]:
         "twitter_cycle": twitter_cycle_status(),
         "draft_publish": draft_publish_wrapper_status(),
         "chrome_relaunches": chrome_relaunch_status(),
+        "chrome_crashpad": chrome_crashpad_status(),
         "cdp_health": cdp_health_status(),
         "process_count": len(rows),
         "mem": {
@@ -955,6 +956,50 @@ def chrome_relaunch_status() -> dict[str, Any] | None:
         return None
 
 
+def chrome_crashpad_status() -> dict[str, Any] | None:
+    """Chrome crash-dump rate from the GLOBAL Crashpad database.
+
+    macOS Chrome writes all crash dumps (any --user-data-dir) to
+    ~/Library/Application Support/Google/Chrome/Crashpad/completed, NOT the
+    profile-local Crashpad dir. Baseline on a healthy box is ~1 dump/week;
+    the 2026-07-12 Chrome 150 regression produced 11-31/day of renderer
+    CHECK-aborts from the harness profile, and each crash preceded a CDP
+    websocket wedge by minutes (memory
+    insights_chrome150_renderer_crash_wedge_2026_07_14). Carrying the counts
+    plus the running Chrome version on the heartbeat makes the NEXT silent
+    Chrome-update regression visible in hours instead of days of stalls."""
+    try:
+        d = (
+            Path.home()
+            / "Library/Application Support/Google/Chrome/Crashpad/completed"
+        )
+        if not d.is_dir():
+            return None
+        now = time.time()
+        count_24h = 0
+        total = 0
+        newest = 0.0
+        for f in d.iterdir():
+            if f.suffix != ".dmp":
+                continue
+            total += 1
+            try:
+                mt = f.stat().st_mtime
+            except OSError:
+                continue
+            if now - mt <= 86400:
+                count_24h += 1
+            newest = max(newest, mt)
+        out: dict[str, Any] = {"count_24h": count_24h, "total": total}
+        if newest:
+            out["last_dump_at"] = dt.datetime.fromtimestamp(
+                newest, dt.timezone.utc
+            ).strftime("%Y-%m-%dT%H:%M:%SZ")
+        return out
+    except Exception:
+        return None
+
+
 def _tail_lines(path: Path, n: int, approx_line_bytes: int = 4096) -> list[str]:
     """Return the last `n` lines of a possibly-large file without reading it all.
     Reads a bounded tail window (n * approx_line_bytes) from the end. Best-effort."""

package/scripts/merge_review_queue.py

@@ -123,6 +123,58 @@ def _thread_url(c: dict) -> str:
     return ""
 
 
+def _reddit_plan_to_candidates(plan: dict) -> list:
+    """Adapt a post_reddit.py draft-phase plan into review-queue card entries.
+
+    The reddit plan shape is {project_name, batch_id, decisions: [...]}, where
+    each decision is the full post-ready dict (_draft_iteration's merged
+    candidate: thread_url, text, engagement_style, ripen data, campaign
+    fields). The card carries the twitter-shaped display fields the menu bar
+    knows how to render, PLUS the verbatim decision + plan metadata under
+    reddit_* keys so an approval can reconstruct a one-decision plan and hand
+    it to `post_reddit.py --phase post` unchanged (locks, URL wrapping,
+    campaign suffixes, and log_post all stay in the one battle-tested poster).
+    """
+    import hashlib
+
+    out = []
+    for d in plan.get("decisions") or []:
+        if not isinstance(d, dict) or not d.get("text") or not d.get("thread_url"):
+            continue
+        # candidate_id gets an "rd-" prefix ON PURPOSE: bare integer ids here
+        # would collide with twitter_candidates ids in every id-keyed flow
+        # (review-events row flips, bulk-discard PATCH /twitter-candidates/
+        # by-id) and silently retire the WRONG twitter row. The prefixed id
+        # still satisfies index.ts's merge-by-candidate_id stamping (ids are
+        # compared as strings), while the twitter-table flips 404 harmlessly.
+        _rid = d.get("id") or d.get("candidate_id")
+        if not _rid:
+            _rid = hashlib.sha1(d["thread_url"].encode()).hexdigest()[:10]
+        out.append({
+            "platform": "reddit",
+            "candidate_id": f"rd-{_rid}",
+            "candidate_url": d.get("thread_url"),
+            "thread_url": d.get("thread_url"),
+            "thread_author": d.get("thread_author"),
+            "thread_text": d.get("thread_title") or "",
+            "matched_project": plan.get("project_name"),
+            "reply_text": d.get("text"),
+            "engagement_style": d.get("engagement_style"),
+            "assigned_style": d.get("engagement_style"),
+            "search_topic": d.get("search_topic"),
+            "reddit_batch_id": plan.get("batch_id"),
+            "reddit_decision": d,
+            "reddit_plan_meta": {
+                k: plan.get(k)
+                for k in ("project_name", "batch_id", "style_assignment",
+                          "generation_trace_path", "session_id",
+                          "draft_session_id")
+                if plan.get(k) is not None
+            },
+        })
+    return out
+
+
 # Discovery-time author/engagement fields stamped onto each plan candidate so the
 # approval card can show them. All already captured on the twitter_candidates row
 # by the discovery pipeline (and refreshed at T1); no scrape happens here.
@@ -174,6 +226,10 @@ def _sync_with_backend(cands: list) -> tuple[int, int]:
         if not c.get("posted")
         and not c.get("terminal")
         and not c.get("approved")
+        # The bulk lookup is /api/v1/twitter-candidates; reddit cards would
+        # never match a row there, so skip them (their freshness gating stays
+        # with the reddit pipeline's own salvage/expiry lanes).
+        and c.get("platform") != "reddit"
         and _thread_url(c)
     ]
     if not pending:
@@ -216,16 +272,21 @@ def main() -> int:
         "--plan-from-marker",
         help="text containing a DRAFT_ONLY_PLAN=<path> marker (e.g. cycle stdout)",
     )
+    ap.add_argument(
+        "--reddit-plan",
+        help="path to a post_reddit.py draft-phase plan (decisions[] shape); "
+        "adapted into reddit cards instead of the twitter candidates[] shape",
+    )
     ap.add_argument("--project", default=None, help="project name for the review-request marker")
     ns = ap.parse_args()
 
-    src = ns.plan
+    src = ns.plan or ns.reddit_plan
     if not src and ns.plan_from_marker:
         m = re.search(r"DRAFT_ONLY_PLAN=(\S+\.json)", ns.plan_from_marker)
         if m:
             src = m.group(1)
     if not src:
-        print("[merge_review_queue] no source plan (need --plan or a DRAFT_ONLY_PLAN marker)", file=sys.stderr)
+        print("[merge_review_queue] no source plan (need --plan, --reddit-plan or a DRAFT_ONLY_PLAN marker)", file=sys.stderr)
         return 2
     if not os.path.exists(src):
         print(f"[merge_review_queue] source plan not found: {src}", file=sys.stderr)
@@ -238,7 +299,10 @@ def main() -> int:
         print(f"[merge_review_queue] could not read source plan: {e}", file=sys.stderr)
         return 2
 
-    new_cands = batch.get("candidates") or []
+    if ns.reddit_plan:
+        new_cands = _reddit_plan_to_candidates(batch)
+    else:
+        new_cands = batch.get("candidates") or []
     if not new_cands:
         print("[merge_review_queue] source plan has 0 candidates; nothing to merge", file=sys.stderr)
         return 0