@m13v/s4l 1.7.2 → 1.7.4-rc.1

package/scripts/scan_x_profile.py

@@ -29,7 +29,16 @@ Usage:
 
 Output (stdout, last line is JSON):
   {"ok": true, "handle": "...", "profile": {...}, "posts": [...],
-   "comments": [...], "counts": {...}, "grounding_instructions": "..."}
+   "comments": [...], "top_posts": [...], "top_replies": [...],
+   "counts": {...}, "grounding_instructions": "..."}
+
+top_posts / top_replies are the same items ranked by real engagement
+(likes*3 + retweets*5 + replies*2), each stamped with rank + engagement_score.
+On /with_replies each reply also carries `parent` = {author, text, url} (the
+tweet it replied to, best-effort DOM-adjacent pairing), and the top posts get
+their thread continuation expanded (`thread`: [tweet texts]) by visiting the
+permalink. scripts/voice_exemplars.py turns these into voice.examples +
+persona_corpus.txt exemplars.
 """
 
 from __future__ import annotations
@@ -182,7 +191,11 @@ def scrape_profile(send) -> dict:
 # --------------------------------------------------------------------------- #
 _TIMELINE_JS_TMPL = r"""(function(){
   var ME=%s; // lowercase handle without @
+  var WITH_PARENTS=%s; // true only on /with_replies: pair each reply with the
+                       // other-author article directly above it (same
+                       // conversation cell renders parent then our reply)
   var out=[];
+  var lastOther=null;
   var arts=document.querySelectorAll('article');
   for(var i=0;i<arts.length;i++){
     var art=arts[i];
@@ -194,10 +207,8 @@ _TIMELINE_JS_TMPL = r"""(function(){
       var mm=hh.match(/^\/([A-Za-z0-9_]{1,15})$/);
       if(mm){authorHandle=mm[1].toLowerCase();break;}
     }
-    if(authorHandle && authorHandle!==ME) continue; // skip others' posts (reposts/quotes/threads)
     var tEl=art.querySelector('[data-testid="tweetText"]');
     var text=tEl?(tEl.innerText||'').trim():'';
-    if(!text) continue;
     // permalink + id
     var url='';var id='';
     var statusLinks=art.querySelectorAll('a[href*="/status/"]');
@@ -206,6 +217,16 @@ _TIMELINE_JS_TMPL = r"""(function(){
       var sm=sh.match(/\/status\/(\d+)/);
       if(sm){id=sm[1];url='https://x.com'+sh.split('?')[0];break;}
     }
+    if(authorHandle && authorHandle!==ME){
+      // Someone else's article. On /with_replies it is the parent of our next
+      // reply in the same conversation cell; remember it. On the posts tab it
+      // is a repost/quote we simply skip (WITH_PARENTS=false there).
+      if(WITH_PARENTS && text){
+        lastOther={author:'@'+authorHandle,text:text.slice(0,500),url:url};
+      }
+      continue;
+    }
+    if(!text) continue;
     // reply? presence of a "Replying to" header in the cell
     var isReply=false, replyTo='';
     var spans=art.querySelectorAll('span,div');
@@ -226,15 +247,21 @@ _TIMELINE_JS_TMPL = r"""(function(){
       var m=al.match(/([\d,]+)/);
       return m?parseInt(m[1].replace(/,/g,''),10):0;
     }
-    out.push({text:text,url:url,id:id,is_reply:isReply,reply_to:replyTo,
-              likes:metric('like'),replies:metric('reply'),retweets:metric('retweet')});
+    var item={text:text,url:url,id:id,is_reply:isReply,reply_to:replyTo,
+              likes:metric('like'),replies:metric('reply'),retweets:metric('retweet')};
+    if(WITH_PARENTS){
+      item.parent=lastOther; // may be null (their own standalone post)
+      lastOther=null;        // consume: never pair one parent with two replies
+    }
+    out.push(item);
   }
   return JSON.stringify(out);
 })()"""
 
 
 def scrape_timeline(send, me: str, want: int, max_scrolls: int = 30,
-                    exclude_ids: "set | None" = None) -> list:
+                    exclude_ids: "set | None" = None,
+                    capture_parents: bool = False) -> list:
     """Scroll the current timeline, collecting up to `want` of the user's OWN
     authored articles (in DOM order = newest first). `exclude_ids` drops items
     already captured elsewhere — that's how the comments pass (/with_replies)
@@ -250,7 +277,8 @@ def scrape_timeline(send, me: str, want: int, max_scrolls: int = 30,
     scrolling to the bottom each step to force the next lazy-load batch."""
     seen: dict[str, dict] = {}
     exclude_ids = exclude_ids or set()
-    expr = _TIMELINE_JS_TMPL % json.dumps(me.lower())
+    expr = _TIMELINE_JS_TMPL % (json.dumps(me.lower()),
+                                "true" if capture_parents else "false")
     STALL_LIMIT = 4
     stall = 0
     for n in range(max_scrolls):
@@ -283,6 +311,69 @@ def scrape_timeline(send, me: str, want: int, max_scrolls: int = 30,
     return items[:want]
 
 
+# --------------------------------------------------------------------------- #
+# Engagement ranking + thread expansion for exemplar extraction.
+# --------------------------------------------------------------------------- #
+def _engagement_score(item: dict) -> int:
+    """Same weighting everywhere exemplars are ranked (voice_exemplars.py
+    re-derives it if absent): a retweet is a stronger signal than a like,
+    a like stronger than a reply-back."""
+    return (int(item.get("likes") or 0) * 3
+            + int(item.get("retweets") or 0) * 5
+            + int(item.get("replies") or 0) * 2)
+
+
+def rank_top(items: list, n: int = 5) -> list:
+    """Top n items by engagement, stamped with rank + engagement_score.
+    Zero-engagement items still rank (small accounts often have nothing else);
+    the score on each entry lets downstream decide what to keep."""
+    ranked = sorted(items, key=_engagement_score, reverse=True)[:n]
+    out = []
+    for i, it in enumerate(ranked, 1):
+        e = dict(it)
+        e["rank"] = i
+        e["engagement_score"] = _engagement_score(it)
+        out.append(e)
+    return out
+
+
+_THREAD_JS_TMPL = r"""(function(){
+  var ME=%s;
+  var out=[];var started=false;
+  var arts=document.querySelectorAll('article');
+  for(var i=0;i<arts.length;i++){
+    var art=arts[i];
+    var authorHandle='';
+    var links=art.querySelectorAll('a[href^="/"]');
+    for(var j=0;j<links.length;j++){
+      var hh=links[j].getAttribute('href')||'';
+      var mm=hh.match(/^\/([A-Za-z0-9_]{1,15})$/);
+      if(mm){authorHandle=mm[1].toLowerCase();break;}
+    }
+    var tEl=art.querySelector('[data-testid="tweetText"]');
+    var text=tEl?(tEl.innerText||'').trim():'';
+    if(authorHandle===ME && text){out.push(text);started=true;}
+    else if(started){break;} // first non-ME article after the run = end of thread
+  }
+  return JSON.stringify(out);
+})()"""
+
+
+def expand_thread(send, me: str, url: str) -> list:
+    """Visit a post's permalink and return the consecutive run of the user's own
+    tweets starting at the focal one ([focal, continuation, ...]). Length 1 =
+    not a thread. Bonus: the permalink shows full text, so this also untruncates
+    posts the timeline cut with 'Show more'. Best-effort; [] on failure."""
+    if not url or not _navigate(send, url, settle=3.5, expect="/status/"):
+        return []
+    raw = _eval(send, _THREAD_JS_TMPL % json.dumps(me.lower())) or "[]"
+    try:
+        parts = json.loads(raw)
+    except Exception:
+        parts = []
+    return parts if isinstance(parts, list) else []
+
+
 GROUNDING_INSTRUCTIONS = (
     "You now have this user's real X profile (bio, original posts, and their own "
     "replies). Use it as GROUND TRUTH to draft their autoposter config fields in "
@@ -294,9 +385,15 @@ GROUNDING_INSTRUCTIONS = (
     "capitalization habits, emoji/punctuation usage, and recurring phrases or tics. "
     "Write the `voice` field so a reply drafted with it would be indistinguishable "
     "from something they'd actually type.\n"
-    "3. GOLDEN-RULE EXAMPLES: pick 2-4 of their strongest real replies/posts "
-    "verbatim and keep them as exemplars (these become few-shot anchors). Choose "
-    "ones that show the target reply behavior: helpful, specific, in-voice.\n"
+    "3. GOLDEN-RULE EXAMPLES: `top_replies` and `top_posts` are already ranked by "
+    "REAL engagement (each entry carries likes/retweets/replies, engagement_score, "
+    "the parent tweet it replied to, and thread continuations). From them keep up "
+    "to 5 replies verbatim as exemplars, skipping throwaway one-liners that show "
+    "no voice; on small accounts engagement is sparse, so judge voice quality too. "
+    "STORE them in the project's `voice.examples` (every drafter on every platform "
+    "mirrors that field), or run "
+    "`voice_exemplars.py apply --scan <scan.json> --project <name>` to write "
+    "voice.examples + the persona_corpus.txt exemplar section deterministically.\n"
     "4. PHRASE BANK: list the kinds of phrases / openers / sign-offs they reuse, "
     "and any words/claims they clearly AVOID (for `content_guardrails`).\n"
     "5. ICP: infer who they engage with (who they reply to, what communities) to "
@@ -314,6 +411,10 @@ def main() -> int:
     ap.add_argument("--handle", default=None, help="@handle to scan (default: live logged-in handle)")
     ap.add_argument("--posts", type=int, default=20, help="max original posts to collect")
     ap.add_argument("--comments", type=int, default=50, help="max replies/comments to collect")
+    ap.add_argument("--top", type=int, default=5, help="how many top posts/replies to rank")
+    ap.add_argument("--expand-threads", type=int, default=3,
+                    help="visit this many top posts' permalinks to capture thread "
+                         "continuations + full text (0 = off)")
     args = ap.parse_args()
 
     if create_connection is None:
@@ -354,10 +455,23 @@ def main() -> int:
         on_replies = _navigate(send, f"https://x.com/{handle}/with_replies",
                                settle=4.0, expect=f"/{handle}/with_replies")
         comments = (
-            scrape_timeline(send, handle, args.comments, exclude_ids=post_ids)
+            scrape_timeline(send, handle, args.comments, exclude_ids=post_ids,
+                            capture_parents=True)
             if on_replies else []
         )
 
+        # 4. Rank both surfaces by real engagement, then expand the top posts'
+        #    permalinks to capture thread continuations (and untruncated text).
+        top_posts = rank_top(posts, args.top)
+        top_replies = rank_top(comments, args.top)
+        for tp in top_posts[:max(args.expand_threads, 0)]:
+            parts = expand_thread(send, handle, tp.get("url") or "")
+            if parts:
+                if len(parts[0]) > len(tp.get("text") or ""):
+                    tp["text"] = parts[0]  # permalink text is never truncated
+                if len(parts) > 1:
+                    tp["thread"] = parts
+
         result = {
             "ok": True,
             "state": "scanned",
@@ -365,9 +479,27 @@ def main() -> int:
             "profile": profile,
             "posts": posts,
             "comments": comments,
+            "top_posts": top_posts,
+            "top_replies": top_replies,
             "counts": {"posts": len(posts), "comments": len(comments)},
             "grounding_instructions": GROUNDING_INSTRUCTIONS,
         }
+
+        # Persist the corpus beside config.json so the later project-save step
+        # (setup writes the project, then voice_exemplars.py auto-applies the
+        # exemplars) works without a re-scan. The scan can run BEFORE config.json
+        # exists on a fresh onboarding, hence mkdir. Best-effort: the scan is
+        # still fully usable from stdout if the write fails.
+        try:
+            sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+            import s4l_mode
+            sidecar = s4l_mode.config_path().parent / "last_profile_scan.json"
+            sidecar.parent.mkdir(parents=True, exist_ok=True)
+            sidecar.write_text(json.dumps(result, ensure_ascii=False))
+            result["scan_file"] = str(sidecar)
+        except Exception:
+            result["scan_file"] = None
+
         print(json.dumps(result, ensure_ascii=False))
         return 0
     except Exception as e:

package/scripts/tlh_pick_248.py

@@ -0,0 +1,92 @@
+import re, os, subprocess, json, hashlib
+from collections import Counter, defaultdict
+from itertools import combinations
+
+DATA = os.path.expanduser("~/social-autoposter/mixer/remotion/src/mixer/data.ts")
+PUB  = os.path.expanduser("~/social-autoposter/mixer/remotion/public/mixer")
+src = open(DATA).read()
+
+# Extract every clipsV2 block and legacy clips arrays referencing tlh-*.mp4
+# Find all "mixer/tlh-...mp4" occurrences grouped per variant via clipsV2:[ ... ] blocks
+sets = []
+for m in re.finditer(r'clipsV2:\s*\[(.*?)\]', src, re.S):
+    clips = re.findall(r'mixer/(tlh-[0-9a-z\-]+\.mp4)', m.group(1))
+    if clips:
+        sets.append(clips)
+# legacy v1: clips arrays inside TLH? lesson-1 uses clips[] with tlh-1..5
+for m in re.finditer(r'clips:\s*\[(.*?)\]', src, re.S):
+    clips = re.findall(r'"(tlh-[0-9a-z\-]+\.mp4)"', m.group(1))
+    if clips:
+        sets.append(clips)
+
+def family(fn):
+    # tlh-<firstint>-... -> first integer group
+    mm = re.match(r'tlh-(\d+)', fn)
+    return mm.group(1) if mm else fn
+
+usage = Counter()
+cooc = Counter()
+for s in sets:
+    for c in set(s):
+        usage[c]+=1
+    for a,b in combinations(sorted(set(s)),2):
+        cooc[(a,b)]+=1
+
+# available encoded tlh clips on disk
+avail = sorted([f for f in os.listdir(PUB) if re.match(r'tlh-[0-9a-z\-]+\.mp4$', f)])
+
+def probe(fn):
+    p = os.path.join(PUB, fn)
+    out = subprocess.run(["ffprobe","-v","error","-select_streams","v:0",
+        "-show_entries","stream=width,height:format=duration","-of","json",p],
+        capture_output=True,text=True).stdout
+    j = json.loads(out)
+    st = j["stream"][0] if "stream" in j else j["streams"][0]
+    dur = float(j["format"]["duration"])
+    return st["width"], st["height"], dur
+
+def blackframes(fn):
+    p = os.path.join(PUB, fn)
+    r = subprocess.run(["ffmpeg","-v","error","-i",p,"-vf","blackdetect=d=0.1:pic_th=0.98",
+        "-an","-f","null","-"],capture_output=True,text=True)
+    return "black_start" in r.stderr
+
+def sha(fn):
+    return hashlib.sha256(open(os.path.join(PUB,fn),'rb').read()).hexdigest()
+
+# Filter avail to valid 1080x1920, dur>=1.75 (holds a 2.0s slot acceptably or exact), non-black
+os.environ["PATH"]="/opt/homebrew/Cellar/ffmpeg/8.1.1/bin:"+os.environ["PATH"]
+valid=[]
+info={}
+for f in avail:
+    try:
+        w,h,d = probe(f)
+    except Exception as e:
+        continue
+    if (w,h)!=(1080,1920): continue
+    info[f]=(w,h,d)
+    valid.append(f)
+
+print("total prior sets:", len(sets))
+print("valid 1080x1920 clips on disk:", len(valid))
+
+# Build candidate 4-sets: distinct families, zero pairwise co-occurrence, low total usage.
+# Rank by (max pairwise cooc, total usage). Verify durations >=1.75 and non-black + distinct sha lazily on the winner.
+from itertools import combinations as comb
+# group valid by family, prefer low-usage clips
+valid_sorted = sorted(valid, key=lambda f:(usage[f], f))
+best=None
+# to keep it tractable, restrict candidate pool to the 40 lowest-usage valid clips across distinct families
+pool = valid_sorted[:60]
+results=[]
+for quad in comb(pool,4):
+    fams=[family(f) for f in quad]
+    if len(set(fams))!=4: continue
+    pairs=list(comb(sorted(quad),2))
+    maxco=max(cooc[p] for p in pairs)
+    tot=sum(usage[f] for f in quad)
+    results.append((maxco,tot,quad))
+results.sort(key=lambda x:(x[0],x[1]))
+print("\ntop 15 candidate quads (maxcooc, totalusage, clips):")
+for maxco,tot,quad in results[:15]:
+    print(maxco,tot,list(quad),"durs",[round(info[f][2],3) for f in quad])

package/scripts/top_performers.py

@@ -388,15 +388,17 @@ def format_report(summary, top, bottom, project=None, platform=None,
     # Projects with zero total_clicks across many posts are the canaries
     # for "this product/voice combination isn't landing" (the 'General'
     # bucket in the 7d audit on 2026-05-12: 56 posts, 0 clicks).
-    lines.append("### Posts per Project per Platform")
-    for row in summary:
-        lines.append(
-            f"  {row[0]:<20} {row[1]:<12} {row[2]:>5} posts  "
-            f"avg_clicks={row[5]}  avg_cm={row[4]}  avg_up={row[3]}  "
-            f"best_clicks={row[8]}  best_cm={row[7]}  best_up={row[6]}  "
-            f"total_clicks={row[9]}"
-        )
-    lines.append("")
+    # Empty summary (--no-project-sections) skips the section entirely.
+    if summary:
+        lines.append("### Posts per Project per Platform")
+        for row in summary:
+            lines.append(
+                f"  {row[0]:<20} {row[1]:<12} {row[2]:>5} posts  "
+                f"avg_clicks={row[5]}  avg_cm={row[4]}  avg_up={row[3]}  "
+                f"best_clicks={row[8]}  best_cm={row[7]}  best_up={row[6]}  "
+                f"total_clicks={row[9]}"
+            )
+        lines.append("")
 
     # Per-project top performers (when no project filter)
     if top_by_group:
@@ -425,6 +427,16 @@ def format_report(summary, top, bottom, project=None, platform=None,
         for p in fallback_top:
             lines.append(format_post(p, suffix_strip_list=suffix_strip_list))
             lines.append("")
+    elif project:
+        # Project filter given, nothing qualified, and no fallback rows
+        # either (or the caller suppressed them). Say so explicitly so the
+        # on-demand --brief caller sees a definitive answer, not a bare
+        # header it might re-query.
+        lines.append(
+            f"### No {project} posts meeting {threshold_label} in the recency window. "
+            "Draft from the thread and the project's config voice; there is no winner to ground on."
+        )
+        lines.append("")
 
     # Bottom posts with failure annotations
     if bottom:
@@ -540,6 +552,21 @@ def main():
                               "the few-shot exemplar section shows only the matching "
                               "high-scoring posts instead of every style. Summary, "
                               "fallback_top, and top_by_group are not affected."))
+    parser.add_argument("--no-project-sections", action="store_true",
+                        help=("Omit the per-project summary table and the Top "
+                              "Posts by Project section from the report. Added "
+                              "2026-07-10 for the cycle orchestrators: the full "
+                              "multi-project winner corpus is no longer bulk-"
+                              "injected into every draft prompt (it homogenized "
+                              "drafts). Instead the drafting session queries "
+                              "`--project <name> --top 3` on demand AFTER it "
+                              "has routed a candidate to a project."))
+    parser.add_argument("--brief", action="store_true",
+                        help=("Render ONLY the top-posts list for the given "
+                              "--project (no style table, no exemplars, no "
+                              "summary, no bottom posts). This is the lean "
+                              "on-demand shape the drafting session calls "
+                              "after routing a candidate to a project."))
     parser.add_argument("--json", action="store_true", help="Output as JSON")
     args = parser.parse_args()
 
@@ -557,6 +584,25 @@ def main():
             if row and len(row) > 12 and row[12] in wanted
         ]
 
+    if args.no_project_sections:
+        summary = []
+        top_by_group = None
+        # Also drop the flat top-posts list: with top_by_group gone,
+        # format_report's `elif top:` branch would otherwise render a
+        # platform-wide "Top N Posts" block, which is the same shared
+        # winner corpus under a different header.
+        top = []
+        fallback_top = None
+
+    if args.brief:
+        # Lean per-project view: keep `top` (and its fallback message),
+        # strip everything else.
+        style_perf = []
+        top_by_style = []
+        summary = []
+        top_by_group = None
+        bottom = []
+
     if args.json:
         output = {
             "summary": [list(row) for row in summary],

package/scripts/twitter_post_plan.py

@@ -43,12 +43,59 @@ import json
 import os
 import random
 import re
+import signal
 import subprocess
 import sys
 import time
 from datetime import datetime, timezone
 from pathlib import Path
 
+
+def _neuter_stream(stream) -> None:
+    """Point a dead pipe's fd at /dev/null so later writes (including the
+    interpreter-shutdown flush) can't raise BrokenPipeError again."""
+    try:
+        devnull = os.open(os.devnull, os.O_WRONLY)
+        os.dup2(devnull, stream.fileno())
+        os.close(devnull)
+    except Exception:
+        pass
+
+
+_builtin_print = print
+
+
+def print(*args, **kwargs):  # noqa: A001 -- deliberate builtins.print shadow
+    # When the parent (cycle shell tree or MCP server) is killed mid-batch, our
+    # stdout/stderr pipes close and the next print raises BrokenPipeError. The
+    # old behavior unwound through excepthook (Sentry S4L-8), killing the batch
+    # BETWEEN posting a reply and recording it — which is how live tweets ended
+    # up unlogged and candidates re-feedable. Output to a dead parent is
+    # worthless; the bookkeeping (log_post, update_candidate, audit JSONL) is
+    # not. Swallow the error, neuter the fds, keep going.
+    try:
+        _builtin_print(*args, **kwargs)
+    except BrokenPipeError:
+        _neuter_stream(sys.stdout)
+        _neuter_stream(sys.stderr)
+
+
+# Graceful SIGTERM: the MCP post_drafts timeout (and anything else that asks
+# nicely before SIGKILL) sends SIGTERM. Dying instantly reopens the same
+# posted-but-unrecorded window as the broken pipe, so instead flag the loop to
+# stop at the next candidate boundary: current candidate finishes its
+# bookkeeping, the summary and audit line still get written, exit stays 0.
+_terminate_requested = False
+
+
+def _on_sigterm(signum, frame):
+    global _terminate_requested
+    _terminate_requested = True
+    print("[post] SIGTERM received; stopping after current candidate", flush=True)
+
+
+signal.signal(signal.SIGTERM, _on_sigterm)
+
 # This pipeline ONLY posts (never scans), so mark every twitter_browser.py reply
 # subprocess it spawns as the high-priority "post" lock role. run_subprocess
 # inherits this process env, so the child twitter_browser.py reads S4L_LOCK_ROLE
@@ -1150,6 +1197,10 @@ def main() -> int:
 
     try:
         for _idx, c in enumerate(candidates, start=1):
+            if _terminate_requested:
+                print(f"[post] stopping early on SIGTERM: {_idx - 1}/{_total} "
+                      f"candidates processed, posted={posted}", flush=True)
+                break
             # Live per-post status for the S4L menu bar. LEAD with `posted` (the
             # REAL count of replies that actually landed), not `_idx` (the loop
             # position). _idx races through already-posted / deleted cards as instant
@@ -1224,6 +1275,22 @@ def main() -> int:
                 "reason": reason or "",
                 "our_url": c.get("our_url") or "",
             })
+            # Per-candidate durable record: the run-level audit at the bottom of
+            # main() is lost if this process is SIGKILLed mid-batch (browser-lock
+            # hijack), leaving no local trace of what already posted. Separate
+            # file from post-results.jsonl on purpose: that one is run-level and
+            # the menu bar/dashboard parse it; don't mix schemas.
+            try:
+                _prog_path = os.path.join(
+                    REPO_DIR, "skill", "logs", "post-candidates.jsonl")
+                with open(_prog_path, "a", encoding="utf-8") as _pf:
+                    _pf.write(json.dumps({
+                        "at": datetime.now(timezone.utc).isoformat(),
+                        "plan": plan_path.name,
+                        **candidate_results[-1],
+                    }) + "\n")
+            except Exception:
+                pass
     finally:
         _clear_activity()
         # Release the batch hold so the next scan/post can take the browser