@m13v/s4l 1.7.3 → 1.7.4-rc.10

package/scripts/recent_self_posts.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""scripts/recent_self_posts.py — cross-cycle anti-repetition context.
+
+Prints a prompt block listing OUR most recent posted replies on a platform
+(across ALL projects), so a drafting session can see what this account
+already sounds like and deliberately diverge. This is the cross-cycle
+complement to author_history_block.py (which is per-author only): before
+2026-07-10 the model never saw its own recent output across threads and
+kept recycling the same openers and sentence skeletons cycle after cycle.
+
+The block is explicitly NEGATIVE context ("do not sound like these"), the
+opposite of the top_performers few-shots. Keep it that way: never add
+engagement numbers or any "this one did well" framing here, or the model
+will read it as examples to imitate.
+
+Wired into (one callsite):
+  - skill/run-twitter-cycle.sh  (Phase 2b-prep PREP_PROMPT)
+
+CLI:
+  python3 scripts/recent_self_posts.py --platform twitter --limit 20
+
+Stdout is a ready-to-inject prompt block; EMPTY stdout when there are no
+rows or on any failure (the cycle must never block on this context).
+Stderr carries diagnostics only.
+"""
+
+import argparse
+import os
+import sys
+
+REPO_DIR = os.path.expanduser("~/social-autoposter")
+sys.path.insert(0, os.path.join(REPO_DIR, "scripts"))
+from http_api import api_get  # noqa: E402
+
+# Truncation length per reply. Long enough to expose the opener + skeleton
+# (the parts the model must avoid repeating), short enough that 20 rows stay
+# a small fraction of the prompt.
+SNIPPET_CHARS = 220
+
+
+def _load_active_campaign_suffixes():
+    """Best-effort list of active campaign suffix literals to strip.
+
+    Same contract as author_history_block._load_active_campaign_suffixes:
+    the block must never teach the model to echo a campaign suffix (the
+    tool layer appends its own copy at post time). On failure returns [].
+    """
+    suffixes = []
+    try:
+        resp = api_get(
+            "/api/v1/campaigns",
+            query={"status": "active", "has_suffix": "true", "limit": 500},
+        )
+        rows = ((resp or {}).get("data") or {}).get("campaigns") or []
+        for r in rows:
+            s = (r.get("suffix") or "").strip()
+            if s and s not in suffixes:
+                suffixes.append(s)
+    except Exception as e:
+        print(f"[recent_self_posts] suffix load failed: {e!r}", file=sys.stderr)
+    return suffixes
+
+
+def _strip_suffixes(text, suffixes):
+    """Trailing-only, idempotent strip (mirrors author_history_block)."""
+    if not text or not suffixes:
+        return text
+    cleaned = text.rstrip()
+    changed = True
+    while changed:
+        changed = False
+        for sfx in suffixes:
+            if sfx and cleaned.endswith(sfx):
+                cleaned = cleaned[: -len(sfx)].rstrip()
+                changed = True
+    return cleaned
+
+
+def _snippet(text, suffixes):
+    """One-line, suffix-stripped, truncated rendering of a reply."""
+    t = _strip_suffixes((text or "").strip(), suffixes)
+    t = " ".join(t.split())  # collapse newlines/runs of whitespace
+    if len(t) > SNIPPET_CHARS:
+        t = t[: SNIPPET_CHARS - 1].rstrip() + "…"
+    return t
+
+
+def build_block(platform, limit):
+    """Return the prompt block string, or "" when nothing to show."""
+    resp = api_get(
+        "/api/v1/posts",
+        query={
+            "platform": platform,
+            "status": "active",
+            "order_by": "posted_at",
+            "order_dir": "desc",
+            "limit": str(limit),
+            # Scope to THIS install's own posts (server filters on the
+            # authenticated X-Installation identity). Without this the
+            # default read returns the whole fleet's posts and the block
+            # would claim another account's replies as "yours"
+            # (found 2026-07-10: 8 of 20 rows were another install's).
+            "own_install": "true",
+        },
+    )
+    rows = ((resp or {}).get("data") or {}).get("posts") or []
+    suffixes = _load_active_campaign_suffixes()
+    items = []
+    for r in rows:
+        snip = _snippet(r.get("our_content"), suffixes)
+        if not snip:
+            continue
+        date = str(r.get("posted_at") or "")[:10]
+        proj = r.get("project_name") or "(no project)"
+        items.append(f"{len(items) + 1}. [{date} | {proj}] {snip}")
+        if len(items) >= limit:
+            break
+    if not items:
+        return ""
+    header = (
+        "## YOUR RECENT REPLIES (cross-cycle anti-repetition; NEGATIVE examples)\n"
+        f"The {len(items)} most recent replies this account posted, all projects. "
+        "This is what you ALREADY sound like. It is NOT a list to imitate. "
+        "Hard rules for every draft this cycle:\n"
+        "- Do NOT reuse any opener below (the first 6-8 words' shape counts, "
+        "not just the exact words).\n"
+        "- Do NOT reuse their sentence skeletons, rhetorical moves, or pet "
+        "phrases (recurring words like 'actually', 'the real X', copula "
+        "reframes 'X is the Y').\n"
+        "- If a draft you are writing starts to echo any entry below, stop "
+        "and rewrite it from a different entry point.\n"
+    )
+    return header + "\n".join(items)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Render the cross-cycle recent-self-replies prompt block")
+    parser.add_argument("--platform", default="twitter")
+    parser.add_argument("--limit", type=int, default=20)
+    args = parser.parse_args()
+    try:
+        block = build_block(args.platform, max(1, min(args.limit, 50)))
+    except Exception as e:
+        print(f"[recent_self_posts] failed: {e!r}", file=sys.stderr)
+        block = ""
+    if block:
+        print(block)
+
+
+if __name__ == "__main__":
+    main()

package/scripts/scan_x_profile.py

@@ -29,7 +29,16 @@ Usage:
 
 Output (stdout, last line is JSON):
   {"ok": true, "handle": "...", "profile": {...}, "posts": [...],
-   "comments": [...], "counts": {...}, "grounding_instructions": "..."}
+   "comments": [...], "top_posts": [...], "top_replies": [...],
+   "counts": {...}, "grounding_instructions": "..."}
+
+top_posts / top_replies are the same items ranked by real engagement
+(likes*3 + retweets*5 + replies*2), each stamped with rank + engagement_score.
+On /with_replies each reply also carries `parent` = {author, text, url} (the
+tweet it replied to, best-effort DOM-adjacent pairing), and the top posts get
+their thread continuation expanded (`thread`: [tweet texts]) by visiting the
+permalink. scripts/voice_exemplars.py turns these into voice.examples +
+persona_corpus.txt exemplars.
 """
 
 from __future__ import annotations
@@ -182,7 +191,11 @@ def scrape_profile(send) -> dict:
 # --------------------------------------------------------------------------- #
 _TIMELINE_JS_TMPL = r"""(function(){
   var ME=%s; // lowercase handle without @
+  var WITH_PARENTS=%s; // true only on /with_replies: pair each reply with the
+                       // other-author article directly above it (same
+                       // conversation cell renders parent then our reply)
   var out=[];
+  var lastOther=null;
   var arts=document.querySelectorAll('article');
   for(var i=0;i<arts.length;i++){
     var art=arts[i];
@@ -194,10 +207,8 @@ _TIMELINE_JS_TMPL = r"""(function(){
       var mm=hh.match(/^\/([A-Za-z0-9_]{1,15})$/);
       if(mm){authorHandle=mm[1].toLowerCase();break;}
     }
-    if(authorHandle && authorHandle!==ME) continue; // skip others' posts (reposts/quotes/threads)
     var tEl=art.querySelector('[data-testid="tweetText"]');
     var text=tEl?(tEl.innerText||'').trim():'';
-    if(!text) continue;
     // permalink + id
     var url='';var id='';
     var statusLinks=art.querySelectorAll('a[href*="/status/"]');
@@ -206,6 +217,16 @@ _TIMELINE_JS_TMPL = r"""(function(){
       var sm=sh.match(/\/status\/(\d+)/);
       if(sm){id=sm[1];url='https://x.com'+sh.split('?')[0];break;}
     }
+    if(authorHandle && authorHandle!==ME){
+      // Someone else's article. On /with_replies it is the parent of our next
+      // reply in the same conversation cell; remember it. On the posts tab it
+      // is a repost/quote we simply skip (WITH_PARENTS=false there).
+      if(WITH_PARENTS && text){
+        lastOther={author:'@'+authorHandle,text:text.slice(0,500),url:url};
+      }
+      continue;
+    }
+    if(!text) continue;
     // reply? presence of a "Replying to" header in the cell
     var isReply=false, replyTo='';
     var spans=art.querySelectorAll('span,div');
@@ -226,15 +247,21 @@ _TIMELINE_JS_TMPL = r"""(function(){
       var m=al.match(/([\d,]+)/);
       return m?parseInt(m[1].replace(/,/g,''),10):0;
     }
-    out.push({text:text,url:url,id:id,is_reply:isReply,reply_to:replyTo,
-              likes:metric('like'),replies:metric('reply'),retweets:metric('retweet')});
+    var item={text:text,url:url,id:id,is_reply:isReply,reply_to:replyTo,
+              likes:metric('like'),replies:metric('reply'),retweets:metric('retweet')};
+    if(WITH_PARENTS){
+      item.parent=lastOther; // may be null (their own standalone post)
+      lastOther=null;        // consume: never pair one parent with two replies
+    }
+    out.push(item);
   }
   return JSON.stringify(out);
 })()"""
 
 
 def scrape_timeline(send, me: str, want: int, max_scrolls: int = 30,
-                    exclude_ids: "set | None" = None) -> list:
+                    exclude_ids: "set | None" = None,
+                    capture_parents: bool = False) -> list:
     """Scroll the current timeline, collecting up to `want` of the user's OWN
     authored articles (in DOM order = newest first). `exclude_ids` drops items
     already captured elsewhere — that's how the comments pass (/with_replies)
@@ -250,7 +277,8 @@ def scrape_timeline(send, me: str, want: int, max_scrolls: int = 30,
     scrolling to the bottom each step to force the next lazy-load batch."""
     seen: dict[str, dict] = {}
     exclude_ids = exclude_ids or set()
-    expr = _TIMELINE_JS_TMPL % json.dumps(me.lower())
+    expr = _TIMELINE_JS_TMPL % (json.dumps(me.lower()),
+                                "true" if capture_parents else "false")
     STALL_LIMIT = 4
     stall = 0
     for n in range(max_scrolls):
@@ -283,6 +311,113 @@ def scrape_timeline(send, me: str, want: int, max_scrolls: int = 30,
     return items[:want]
 
 
+def _own_posted_ids() -> set:
+    """Status ids of everything S4L itself has posted from this install, via
+    /api/v1/posts (install-scoped by the X-Installation header). Used to keep
+    the bot's own output OUT of the author-voice exemplar pool: on accounts
+    where S4L has been active, a recency scan is dominated by S4L drafts, and
+    feeding those back as 'the author's voice' is a feedback loop. Best-effort:
+    any failure (offline, fresh install, no API) returns an empty set and the
+    scan proceeds unfiltered."""
+    try:
+        sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+        from http_api import api_get  # noqa: PLC0415
+        import re
+        ids: set = set()
+        # The route caps limit at 500, so walk the FULL posting history with a
+        # forward posted_at cursor (order_dir=asc + since>=). On prolific
+        # accounts a single newest-500 page misses older S4L posts that the
+        # profile scan still reaches (bit the operator account: 12.8k posted
+        # statuses, 2 of 5 picked exemplars were S4L's own). The since filter
+        # is inclusive, so overlap rows dedupe via the set; a page whose max
+        # posted_at equals the cursor would loop and breaks instead.
+        cursor = None
+        for _ in range(80):  # hard cap 40k statuses
+            q = {"platform": "twitter", "has_our_url": "true", "limit": 500,
+                 "order_by": "posted_at", "order_dir": "asc"}
+            if cursor:
+                q["since"] = cursor
+            resp = api_get("/api/v1/posts", q)
+            rows = ((resp or {}).get("data") or {}).get("posts") or []
+            for r in rows:
+                m = re.search(r"/status/(\d+)", str(r.get("our_url") or ""))
+                if m:
+                    ids.add(m.group(1))
+            if len(rows) < 500:
+                break
+            page_max = max((r.get("posted_at") or "" for r in rows), default="")
+            if not page_max or page_max == cursor:
+                break
+            cursor = page_max
+        return ids
+    except Exception as e:
+        print(f"[scan_x_profile] own-posts exclusion unavailable: {e}", file=sys.stderr)
+        return set()
+
+
+# --------------------------------------------------------------------------- #
+# Engagement ranking + thread expansion for exemplar extraction.
+# --------------------------------------------------------------------------- #
+def _engagement_score(item: dict) -> int:
+    """Same weighting everywhere exemplars are ranked (voice_exemplars.py
+    re-derives it if absent): a retweet is a stronger signal than a like,
+    a like stronger than a reply-back."""
+    return (int(item.get("likes") or 0) * 3
+            + int(item.get("retweets") or 0) * 5
+            + int(item.get("replies") or 0) * 2)
+
+
+def rank_top(items: list, n: int = 5) -> list:
+    """Top n items by engagement, stamped with rank + engagement_score.
+    Zero-engagement items still rank (small accounts often have nothing else);
+    the score on each entry lets downstream decide what to keep."""
+    ranked = sorted(items, key=_engagement_score, reverse=True)[:n]
+    out = []
+    for i, it in enumerate(ranked, 1):
+        e = dict(it)
+        e["rank"] = i
+        e["engagement_score"] = _engagement_score(it)
+        out.append(e)
+    return out
+
+
+_THREAD_JS_TMPL = r"""(function(){
+  var ME=%s;
+  var out=[];var started=false;
+  var arts=document.querySelectorAll('article');
+  for(var i=0;i<arts.length;i++){
+    var art=arts[i];
+    var authorHandle='';
+    var links=art.querySelectorAll('a[href^="/"]');
+    for(var j=0;j<links.length;j++){
+      var hh=links[j].getAttribute('href')||'';
+      var mm=hh.match(/^\/([A-Za-z0-9_]{1,15})$/);
+      if(mm){authorHandle=mm[1].toLowerCase();break;}
+    }
+    var tEl=art.querySelector('[data-testid="tweetText"]');
+    var text=tEl?(tEl.innerText||'').trim():'';
+    if(authorHandle===ME && text){out.push(text);started=true;}
+    else if(started){break;} // first non-ME article after the run = end of thread
+  }
+  return JSON.stringify(out);
+})()"""
+
+
+def expand_thread(send, me: str, url: str) -> list:
+    """Visit a post's permalink and return the consecutive run of the user's own
+    tweets starting at the focal one ([focal, continuation, ...]). Length 1 =
+    not a thread. Bonus: the permalink shows full text, so this also untruncates
+    posts the timeline cut with 'Show more'. Best-effort; [] on failure."""
+    if not url or not _navigate(send, url, settle=3.5, expect="/status/"):
+        return []
+    raw = _eval(send, _THREAD_JS_TMPL % json.dumps(me.lower())) or "[]"
+    try:
+        parts = json.loads(raw)
+    except Exception:
+        parts = []
+    return parts if isinstance(parts, list) else []
+
+
 GROUNDING_INSTRUCTIONS = (
     "You now have this user's real X profile (bio, original posts, and their own "
     "replies). Use it as GROUND TRUTH to draft their autoposter config fields in "
@@ -294,9 +429,15 @@ GROUNDING_INSTRUCTIONS = (
     "capitalization habits, emoji/punctuation usage, and recurring phrases or tics. "
     "Write the `voice` field so a reply drafted with it would be indistinguishable "
     "from something they'd actually type.\n"
-    "3. GOLDEN-RULE EXAMPLES: pick 2-4 of their strongest real replies/posts "
-    "verbatim and keep them as exemplars (these become few-shot anchors). Choose "
-    "ones that show the target reply behavior: helpful, specific, in-voice.\n"
+    "3. GOLDEN-RULE EXAMPLES: `top_replies` and `top_posts` are already ranked by "
+    "REAL engagement (each entry carries likes/retweets/replies, engagement_score, "
+    "the parent tweet it replied to, and thread continuations). From them keep up "
+    "to 5 replies verbatim as exemplars, skipping throwaway one-liners that show "
+    "no voice; on small accounts engagement is sparse, so judge voice quality too. "
+    "STORE them in the project's `voice.examples` (every drafter on every platform "
+    "mirrors that field), or run "
+    "`voice_exemplars.py apply --scan <scan.json> --project <name>` to write "
+    "voice.examples + the persona_corpus.txt exemplar section deterministically.\n"
     "4. PHRASE BANK: list the kinds of phrases / openers / sign-offs they reuse, "
     "and any words/claims they clearly AVOID (for `content_guardrails`).\n"
     "5. ICP: infer who they engage with (who they reply to, what communities) to "
@@ -312,8 +453,12 @@ GROUNDING_INSTRUCTIONS = (
 def main() -> int:
     ap = argparse.ArgumentParser()
     ap.add_argument("--handle", default=None, help="@handle to scan (default: live logged-in handle)")
-    ap.add_argument("--posts", type=int, default=20, help="max original posts to collect")
-    ap.add_argument("--comments", type=int, default=50, help="max replies/comments to collect")
+    ap.add_argument("--posts", type=int, default=60, help="max original posts to collect")
+    ap.add_argument("--comments", type=int, default=150, help="max replies/comments to collect")
+    ap.add_argument("--top", type=int, default=5, help="how many top posts/replies to rank")
+    ap.add_argument("--expand-threads", type=int, default=3,
+                    help="visit this many top posts' permalinks to capture thread "
+                         "continuations + full text (0 = off)")
     args = ap.parse_args()
 
     if create_connection is None:
@@ -345,19 +490,48 @@ def main() -> int:
                                expect=f"/{handle}")
         profile = scrape_profile(send) if on_profile else {}
 
-        # 2. Original posts (current page = posts tab).
-        posts = scrape_timeline(send, handle, args.posts) if on_profile else []
+        # 2. Everything S4L itself posted from this install gets excluded from
+        #    BOTH surfaces (posts and replies): the exemplars must be the
+        #    human's writing, not the bot's own output echoed back.
+        s4l_ids = _own_posted_ids()
+        if s4l_ids:
+            print(f"[scan_x_profile] excluding {len(s4l_ids)} s4l-posted statuses "
+                  "from the exemplar pool", file=sys.stderr)
+
+        # 3. Original posts (current page = posts tab). max_scrolls tracks the
+        #    requested depth: the scan is programmatic, so scrolling deeper
+        #    costs only time, and end-of-feed stall detection stops it early
+        #    on small accounts.
+        posts = (scrape_timeline(send, handle, args.posts,
+                                 max_scrolls=max(30, args.posts),
+                                 exclude_ids=s4l_ids)
+                 if on_profile else [])
         post_ids = {p.get("id") for p in posts if p.get("id")}
 
-        # 3. Replies / comments = the user's own articles on /with_replies that
+        # 4. Replies / comments = the user's own articles on /with_replies that
         #    are NOT among the original posts (set subtraction, not DOM text).
         on_replies = _navigate(send, f"https://x.com/{handle}/with_replies",
                                settle=4.0, expect=f"/{handle}/with_replies")
         comments = (
-            scrape_timeline(send, handle, args.comments, exclude_ids=post_ids)
+            scrape_timeline(send, handle, args.comments,
+                            max_scrolls=max(30, args.comments),
+                            exclude_ids=post_ids | s4l_ids,
+                            capture_parents=True)
             if on_replies else []
         )
 
+        # 5. Rank both surfaces by real engagement, then expand the top posts'
+        #    permalinks to capture thread continuations (and untruncated text).
+        top_posts = rank_top(posts, args.top)
+        top_replies = rank_top(comments, args.top)
+        for tp in top_posts[:max(args.expand_threads, 0)]:
+            parts = expand_thread(send, handle, tp.get("url") or "")
+            if parts:
+                if len(parts[0]) > len(tp.get("text") or ""):
+                    tp["text"] = parts[0]  # permalink text is never truncated
+                if len(parts) > 1:
+                    tp["thread"] = parts
+
         result = {
             "ok": True,
             "state": "scanned",
@@ -365,9 +539,28 @@ def main() -> int:
             "profile": profile,
             "posts": posts,
             "comments": comments,
-            "counts": {"posts": len(posts), "comments": len(comments)},
+            "top_posts": top_posts,
+            "top_replies": top_replies,
+            "counts": {"posts": len(posts), "comments": len(comments),
+                       "s4l_posted_excluded": len(s4l_ids)},
             "grounding_instructions": GROUNDING_INSTRUCTIONS,
         }
+
+        # Persist the corpus beside config.json so the later project-save step
+        # (setup writes the project, then voice_exemplars.py auto-applies the
+        # exemplars) works without a re-scan. The scan can run BEFORE config.json
+        # exists on a fresh onboarding, hence mkdir. Best-effort: the scan is
+        # still fully usable from stdout if the write fails.
+        try:
+            sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+            import s4l_mode
+            sidecar = s4l_mode.config_path().parent / "last_profile_scan.json"
+            sidecar.parent.mkdir(parents=True, exist_ok=True)
+            sidecar.write_text(json.dumps(result, ensure_ascii=False))
+            result["scan_file"] = str(sidecar)
+        except Exception:
+            result["scan_file"] = None
+
         print(json.dumps(result, ensure_ascii=False))
         return 0
     except Exception as e:

package/scripts/tlh_pick_248.py

@@ -0,0 +1,92 @@
+import re, os, subprocess, json, hashlib
+from collections import Counter, defaultdict
+from itertools import combinations
+
+DATA = os.path.expanduser("~/social-autoposter/mixer/remotion/src/mixer/data.ts")
+PUB  = os.path.expanduser("~/social-autoposter/mixer/remotion/public/mixer")
+src = open(DATA).read()
+
+# Extract every clipsV2 block and legacy clips arrays referencing tlh-*.mp4
+# Find all "mixer/tlh-...mp4" occurrences grouped per variant via clipsV2:[ ... ] blocks
+sets = []
+for m in re.finditer(r'clipsV2:\s*\[(.*?)\]', src, re.S):
+    clips = re.findall(r'mixer/(tlh-[0-9a-z\-]+\.mp4)', m.group(1))
+    if clips:
+        sets.append(clips)
+# legacy v1: clips arrays inside TLH? lesson-1 uses clips[] with tlh-1..5
+for m in re.finditer(r'clips:\s*\[(.*?)\]', src, re.S):
+    clips = re.findall(r'"(tlh-[0-9a-z\-]+\.mp4)"', m.group(1))
+    if clips:
+        sets.append(clips)
+
+def family(fn):
+    # tlh-<firstint>-... -> first integer group
+    mm = re.match(r'tlh-(\d+)', fn)
+    return mm.group(1) if mm else fn
+
+usage = Counter()
+cooc = Counter()
+for s in sets:
+    for c in set(s):
+        usage[c]+=1
+    for a,b in combinations(sorted(set(s)),2):
+        cooc[(a,b)]+=1
+
+# available encoded tlh clips on disk
+avail = sorted([f for f in os.listdir(PUB) if re.match(r'tlh-[0-9a-z\-]+\.mp4$', f)])
+
+def probe(fn):
+    p = os.path.join(PUB, fn)
+    out = subprocess.run(["ffprobe","-v","error","-select_streams","v:0",
+        "-show_entries","stream=width,height:format=duration","-of","json",p],
+        capture_output=True,text=True).stdout
+    j = json.loads(out)
+    st = j["stream"][0] if "stream" in j else j["streams"][0]
+    dur = float(j["format"]["duration"])
+    return st["width"], st["height"], dur
+
+def blackframes(fn):
+    p = os.path.join(PUB, fn)
+    r = subprocess.run(["ffmpeg","-v","error","-i",p,"-vf","blackdetect=d=0.1:pic_th=0.98",
+        "-an","-f","null","-"],capture_output=True,text=True)
+    return "black_start" in r.stderr
+
+def sha(fn):
+    return hashlib.sha256(open(os.path.join(PUB,fn),'rb').read()).hexdigest()
+
+# Filter avail to valid 1080x1920, dur>=1.75 (holds a 2.0s slot acceptably or exact), non-black
+os.environ["PATH"]="/opt/homebrew/Cellar/ffmpeg/8.1.1/bin:"+os.environ["PATH"]
+valid=[]
+info={}
+for f in avail:
+    try:
+        w,h,d = probe(f)
+    except Exception as e:
+        continue
+    if (w,h)!=(1080,1920): continue
+    info[f]=(w,h,d)
+    valid.append(f)
+
+print("total prior sets:", len(sets))
+print("valid 1080x1920 clips on disk:", len(valid))
+
+# Build candidate 4-sets: distinct families, zero pairwise co-occurrence, low total usage.
+# Rank by (max pairwise cooc, total usage). Verify durations >=1.75 and non-black + distinct sha lazily on the winner.
+from itertools import combinations as comb
+# group valid by family, prefer low-usage clips
+valid_sorted = sorted(valid, key=lambda f:(usage[f], f))
+best=None
+# to keep it tractable, restrict candidate pool to the 40 lowest-usage valid clips across distinct families
+pool = valid_sorted[:60]
+results=[]
+for quad in comb(pool,4):
+    fams=[family(f) for f in quad]
+    if len(set(fams))!=4: continue
+    pairs=list(comb(sorted(quad),2))
+    maxco=max(cooc[p] for p in pairs)
+    tot=sum(usage[f] for f in quad)
+    results.append((maxco,tot,quad))
+results.sort(key=lambda x:(x[0],x[1]))
+print("\ntop 15 candidate quads (maxcooc, totalusage, clips):")
+for maxco,tot,quad in results[:15]:
+    print(maxco,tot,list(quad),"durs",[round(info[f][2],3) for f in quad])