@m13v/s4l 1.6.197-rc.10

package/scripts/twitter_scan.py

@@ -0,0 +1,324 @@
+#!/usr/bin/env python3
+"""
+twitter_scan.py — deterministic X/Twitter search scrape.
+
+Runs inside the browser-harness CLI process (BU_NAME=twitter-harness,
+BU_CDP_URL=http://127.0.0.1:9555) and drives the live managed Chrome on 9555.
+Called once per drafted query by run-twitter-cycle.sh's Phase 1 lean loop:
+
+    BU_NAME=twitter-harness BU_CDP_URL=http://127.0.0.1:9555 \
+        browser-harness -c "
+    import sys; sys.path.insert(0, '/Users/matthewdi/social-autoposter/scripts')
+    from twitter_scan import scan
+    for q in <queries list>:
+        scan(query=q['query'], project=q['project'],
+             search_topic=q['search_topic'],
+             freshness_hours=<env FRESHNESS_HOURS_DISCOVER>,
+             skip_ids=<env ENGAGED_TWEET_IDS>)
+    "
+
+The cycle shell drafts queries via a small Claude call (no tools), then loops
+this function per query. Result tweets go to SCAN_TWEETS_FILE (env-set), which
+the cycle reads directly into $RAW_FILE + $QUERIES_FILE for the scorer.
+
+What scan() does:
+- Strips any since/until/since_time/until_time from the query so the
+  freshness window is operator-controlled, not caller-controlled.
+- Builds an x.com/search URL with `&f=live` (Latest tab forced) and appends
+  `since_time:<now - freshness_hours*3600>` to the query.
+- Reuses an existing real tab or opens one on the first call.
+- Scrapes the first ~8 article cards.
+- Applies a deterministic Python age gate behind the URL since_time
+  (belt-and-suspenders against cached / lazy-loaded stale viewports).
+- Drops skip_ids (recently-engaged tweets).
+- Stamps search_topic / matched_project / query on every kept tweet.
+- Appends a sidecar JSONL record to
+  ~/social-autoposter/skill/logs/twitter-scan-attempts.jsonl for operator
+  visibility, and a per-attempt record to SCAN_TWEETS_FILE for the shell.
+- Returns the kept tweet list.
+
+Standalone test (no cycle shell):
+
+    ~/.local/bin/browser-harness -c '
+    import sys; sys.path.insert(0, "/Users/matthewdi/social-autoposter/scripts")
+    from twitter_scan import scan
+    scan(query="AI agent min_faves:10",
+         project="WhatsApp MCP",
+         search_topic="AI agent",
+         freshness_hours=6)
+    '
+"""
+from __future__ import annotations
+
+import datetime
+import json
+import os
+import pathlib
+import re
+import time
+import urllib.parse
+
+# Pin the daemon socket BEFORE importing helpers — helpers.py reads BU_NAME
+# at module-load time (helpers.py:37). setdefault is a no-op when the bh_run
+# wrapper or the cycle shell already set these; required when invoked from a
+# bare `browser-harness -c` test invocation where the env happens to be empty.
+os.environ.setdefault("BU_NAME", "twitter-harness")
+os.environ.setdefault("BU_CDP_URL", "http://127.0.0.1:9555")
+
+from browser_harness.helpers import (  # noqa: E402  (env must be set first)
+    goto_url,
+    js,
+    list_tabs,
+    new_tab,
+    wait_for_load,
+)
+
+_SIDECAR = (
+    pathlib.Path.home()
+    / "social-autoposter"
+    / "skill"
+    / "logs"
+    / "twitter-scan-attempts.jsonl"
+)
+
+# Derived from skill/run-twitter-cycle.sh:666-708 (the legacy inline scan JS).
+# twitter_candidates fields (handle, text, tweetUrl, datetime, engagement
+# counters) land in the same shape the scorer and dashboard already consume.
+# 2026-06-04 DIVERGENCE: this copy adds repost awareness on top of the legacy
+# JS — `handle` is now taken from the status URL (authoritative original author,
+# since on a repost the first profile link is the REPOSTER), plus `is_repost`
+# and `reposted_by` from the "<X> reposted" socialContext banner. This is the
+# live data path (the cycle reads SCAN_TWEETS_FILE written here); the locked
+# shell's inline JS is the inert fallback and simply omits the two new fields.
+_SCRAPE_JS = r"""
+(() => {
+  const SNOWFLAKE = /\/status\/(\d{15,19})(?:[\/?#]|$)/;
+  const FAKE_TAIL = /0{6,}$/;
+  const results = [];
+  for (const article of [...document.querySelectorAll('article[data-testid="tweet"]')].slice(0, 8)) {
+    try {
+      let handle = '';
+      for (const link of article.querySelectorAll('a[role="link"]')) {
+        const href = link.getAttribute('href');
+        if (href && href.startsWith('/') && !href.includes('/status/') && !href.includes('/search') && href.length > 1 && href.split('/').length === 2) {
+          handle = href.replace('/', ''); break;
+        }
+      }
+      const tweetText = article.querySelector('[data-testid="tweetText"]');
+      const text = tweetText ? tweetText.textContent : '';
+      const timeEl = article.querySelector('time');
+      const timeParent = timeEl ? timeEl.closest('a') : null;
+      const tweetUrl = timeParent ? 'https://x.com' + timeParent.getAttribute('href') : '';
+      const datetime = timeEl ? timeEl.getAttribute('datetime') : '';
+      const sm = tweetUrl.match(SNOWFLAKE);
+      if (!sm || FAKE_TAIL.test(sm[1])) continue;
+      if (!datetime || !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/.test(datetime)) continue;
+      // Author = first path segment of the status URL. This is authoritative:
+      // on a repost the displayed/first-link handle above is the REPOSTER, not
+      // the original author, so prefer the URL author and keep the link-scan
+      // handle only as a fallback when the URL can't be parsed.
+      const authorM = tweetUrl.match(/x\.com\/([^\/]+)\/status\//);
+      if (authorM && authorM[1]) handle = authorM[1];
+      // Repost detection: a "<X> reposted" banner lives in socialContext. The
+      // SAME testid is reused for "Pinned", so match the text, not presence.
+      // reposted_by = the account whose profile link wraps the banner.
+      let is_repost = false, reposted_by = '';
+      const sc = article.querySelector('[data-testid="socialContext"]');
+      if (sc && /\breposted\b/i.test(sc.textContent || '')) {
+        is_repost = true;
+        const a = sc.closest('a');
+        const rh = a ? (a.getAttribute('href') || '') : '';
+        if (rh.startsWith('/') && rh.split('/').length === 2) reposted_by = rh.replace('/', '');
+      }
+      let replies=0, retweets=0, likes=0, views=0, bookmarks=0;
+      for (const btn of article.querySelectorAll('[role="group"] button')) {
+        const al = btn.getAttribute('aria-label') || '';
+        let m;
+        if (m=al.match(/([\d,]+)\s*repl/i)) replies=parseInt(m[1].replace(/,/g,''));
+        if (m=al.match(/([\d,]+)\s*repost/i)) retweets=parseInt(m[1].replace(/,/g,''));
+        if (m=al.match(/([\d,]+)\s*like/i)) likes=parseInt(m[1].replace(/,/g,''));
+        if (m=al.match(/([\d,]+)\s*view/i)) views=parseInt(m[1].replace(/,/g,''));
+        if (m=al.match(/([\d,]+)\s*bookmark/i)) bookmarks=parseInt(m[1].replace(/,/g,''));
+      }
+      results.push({handle, text, tweetUrl, datetime, replies, retweets, likes, views, bookmarks, is_repost, reposted_by});
+    } catch(e) {}
+  }
+  return results;
+})()
+"""
+
+# 2026-05-28: also matches the bash-arithmetic form
+# `since_time:$(( $(date +%s) - FRESHNESS_HOURS_DISCOVER * 3600 ))` that was
+# accidentally taught to the model when an escaping bug in the prompt sent
+# the literal template text instead of an evaluated epoch. \S+ alone stops
+# at the first space and leaves the tail (`$(date +%s) - ... ))`) behind as
+# keyword garbage that X searches for literally. The non-greedy `.*?` inside
+# `$((...))` matches up to the first `))` which is the template's own close.
+_DATE_OPS_RE = re.compile(
+    r"\b(since|until|since_time|until_time):(?:\$\(\(.*?\)\)|\S+)",
+    re.IGNORECASE,
+)
+# Belt + suspenders: even after _DATE_OPS_RE, residual orphan fragments could
+# remain if the model invents some other broken template. Strip common ones.
+_BASH_GARBAGE_RE = re.compile(
+    r"\$\(\(|\$\([^)]*\)|\bFRESHNESS_HOURS_DISCOVER\s*\*\s*\d+\b|\)\)"
+)
+_STATUS_ID_RE = re.compile(r"/status/(\d+)")
+
+
+def _build_url(query: str, freshness_hours: int) -> str:
+    """Force-build the Latest-tab URL with since_time pinned `freshness_hours` ago.
+
+    Stripping the model's date operators first is what closes the dodge: a
+    rogue `since:2020-01-01` in the model's query string can no longer widen
+    the window. `f=live` is what closes the Top-tab dodge: without it X may
+    serve the Top tab where the time operator is advisory."""
+    cleaned = _DATE_OPS_RE.sub("", query)
+    cleaned = _BASH_GARBAGE_RE.sub("", cleaned)
+    cleaned = re.sub(r"\s+", " ", cleaned).strip()
+    cap_epoch = int(time.time()) - int(freshness_hours) * 3600
+    full = f"{cleaned} since_time:{cap_epoch}".strip()
+    return "https://x.com/search?q=" + urllib.parse.quote(full) + "&src=typed_query&f=live"
+
+
+def _parse_dt_epoch(ds: str):
+    if not ds:
+        return None
+    try:
+        return int(
+            datetime.datetime.fromisoformat(ds.replace("Z", "+00:00")).timestamp()
+        )
+    except (ValueError, TypeError):
+        return None
+
+
+def _status_id(url: str):
+    m = _STATUS_ID_RE.search(url or "")
+    return m.group(1) if m else None
+
+
+def _write_sidecar(rec: dict) -> None:
+    try:
+        _SIDECAR.parent.mkdir(parents=True, exist_ok=True)
+        with _SIDECAR.open("a") as f:
+            f.write(json.dumps(rec) + "\n")
+    except OSError:
+        pass  # fail-open; sidecar is operator visibility only, not on the data path
+
+
+def _write_scan_tweets_record(rec: dict) -> None:
+    """Append one JSONL record per scan() call to the path in SCAN_TWEETS_FILE.
+
+    2026-05-28: shell-side data path. When the cycle exports SCAN_TWEETS_FILE,
+    run-twitter-cycle.sh reads this file after the scan claude session ends
+    and uses it as the source of truth for both $RAW_FILE (tweets fed to the
+    scorer) and $QUERIES_FILE (attempts fed to log_twitter_search_attempts.py),
+    skipping the model's structured_output relay entirely. This cuts the
+    relay-tokens bill (model no longer has to copy the tweets/queries_used
+    arrays from bh_run stdout into structured_output).
+
+    Inert when SCAN_TWEETS_FILE is unset; the model's structured_output path
+    remains the fallback so existing standalone test invocations (no cycle
+    env) and any session where the file write fails still produce candidates."""
+    path = os.environ.get("SCAN_TWEETS_FILE")
+    if not path:
+        return
+    try:
+        with open(path, "a") as f:
+            f.write(json.dumps(rec) + "\n")
+    except OSError:
+        pass  # fail-open; shell falls back to structured_output if file is missing
+
+
+def _navigate(url: str) -> None:
+    """Reuse the existing real tab if there is one (typical cycle behavior),
+    otherwise open one. The MCP-managed Chrome always has at least an
+    about:blank tab from launch, but be defensive: a hung tab close between
+    cycles can leave us with only chrome:// tabs."""
+    real = [
+        t for t in list_tabs(include_chrome=False)
+        if (t.get("url") or "").startswith(("http", "about:"))
+    ]
+    if real:
+        goto_url(url)
+    else:
+        new_tab(url)
+
+
+def scan(
+    query: str,
+    project: str,
+    search_topic: str,
+    freshness_hours: int = 6,
+    skip_ids=None,
+    settle_seconds: float = 4.0,
+) -> list:
+    """Deterministic scrape + age gate. Prints JSON between
+    ###TWEETS_BEGIN###/###TWEETS_END### sentinels for the scan model to relay
+    into StructuredOutput; also returns the kept list so direct callers (tests,
+    future shell-driven invocations) can consume it without parsing stdout."""
+    skip = {str(s) for s in (skip_ids or [])}
+    url = _build_url(query, int(freshness_hours))
+    _navigate(url)
+    wait_for_load(timeout=15.0)
+    # X lazy-loads the result list; settle briefly before scraping. Matches
+    # the legacy template's `time.sleep(4)`.
+    time.sleep(float(settle_seconds))
+
+    raw = js(_SCRAPE_JS)
+    tweets = raw if isinstance(raw, list) else []
+    pre_count = len(tweets)
+
+    cap_epoch = int(time.time()) - int(freshness_hours) * 3600
+    fresh = []
+    for t in tweets:
+        ep = _parse_dt_epoch(t.get("datetime", ""))
+        if ep is not None and ep >= cap_epoch:
+            fresh.append(t)
+    dropped_age = pre_count - len(fresh)
+
+    kept = [t for t in fresh if _status_id(t.get("tweetUrl", "")) not in skip]
+    dropped_skip = len(fresh) - len(kept)
+
+    for t in kept:
+        t["search_topic"] = search_topic
+        t["matched_project"] = project
+        t["query"] = query
+
+    _write_sidecar(
+        {
+            "ts": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
+            "ts_epoch": int(time.time()),
+            "query": query,
+            "project": project,
+            "search_topic": search_topic,
+            "freshness_hours": int(freshness_hours),
+            "url": url,
+            "pre_count": pre_count,
+            "kept_after_age": len(fresh),
+            "dropped_age": dropped_age,
+            "kept_after_skip": len(kept),
+            "dropped_skip": dropped_skip,
+            "batch_id": os.environ.get("BATCH_ID"),
+            "cycle_variant": os.environ.get("TWITTER_CYCLE_VARIANT"),
+        }
+    )
+
+    # Shell-side data path. The cycle (when it exports SCAN_TWEETS_FILE) reads
+    # this file directly instead of asking the scan model to relay tweets via
+    # structured_output, saving relay tokens. One JSONL record per scan() call;
+    # the cycle aggregates across all calls in one Phase 1 attempt.
+    _write_scan_tweets_record(
+        {
+            "ts": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
+            "query": query,
+            "project": project,
+            "search_topic": search_topic,
+            "tweets": kept,
+        }
+    )
+
+    # 2026-05-28 cleanup: sentinel-print removed. The cycle reads SCAN_TWEETS_FILE
+    # directly via _write_scan_tweets_record() above; the bh_run stdout relay path
+    # is no longer wired. scan() still returns `kept` so direct callers can use it.
+    return kept

package/scripts/twitter_supply_signal.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""
+twitter_supply_signal.py
+
+Per-project supply table: at each `min_faves:N` tier, what's the median
+number of tweets X actually returned for queries we ran for that project?
+
+This is the answer to the question the Phase 1 scanner has been guessing
+at since the cycle was written: "what min_faves should I use for this
+project?". Today the prompt says a flat "broad=50, narrow=20" rule, which
+works for tech-Twitter (mk0r, claude-meter, fazm) but starves student-
+Twitter (studyly), where even niche audience tweets rarely clear 20 likes.
+
+Output: JSON list of
+    {"project": "<name>", "tiers": [{"min_faves": N, "attempts": N,
+                                     "median_tweets_found": N,
+                                     "zero_result_pct": 0-100}, ...]}
+sorted by project. Within each project, tiers ordered ascending min_faves
+so the model can read "as I raise the floor, supply collapses; pick the
+lowest min_faves where supply is still ≥3".
+
+Usage:
+
+    python3 scripts/twitter_supply_signal.py [--window-days 14] [--project NAME]
+
+Migrated 2026-05-18: reads now go through /api/v1/twitter-search-attempts/
+supply-signal via scripts/http_api.py instead of psycopg2.
+"""
+import argparse
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from http_api import api_get  # noqa: E402
+
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument("--window-days", type=int, default=14)
+    p.add_argument("--project", default=None,
+                   help="If set, only return supply table for this project.")
+    args = p.parse_args()
+
+    query = {"window_days": args.window_days}
+    if args.project:
+        query["project"] = args.project
+
+    resp = api_get("/api/v1/twitter-search-attempts/supply-signal", query=query)
+    out = (resp.get("data") or {}).get("rows") or []
+
+    json.dump(out, sys.stdout)
+    print("", file=sys.stdout)
+
+
+if __name__ == "__main__":
+    main()

package/scripts/twitter_threads_helper.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""twitter_threads_helper.py — small CLI wrapper used by
+skill/run-twitter-threads.sh to replace the three `psql` one-liners that
+loaded recent posts / styles / top performers for the original-thread
+prompt context. Each subcommand prints exactly one value to stdout (raw
+newline-separated content or pipe-separated tuples) so the surrounding
+bash code can keep using $(...) capture unchanged.
+
+Subcommands:
+  recent-posts --project P [--days 14] [--limit 10]
+      -> GET /api/v1/posts?platform=twitter&project=P
+            &since=<now-14d>&limit=...
+      -> filter rows to platform='twitter' AND thread_url = our_url
+         (= our original posts, not mention placeholders) and print one
+         line per row containing the post's our_content (newlines and
+         pipes inside content survive because the legacy psql -t -A
+         output did the same).
+
+  recent-styles --project P [--limit 5]
+      -> Same source; print one engagement_style per line for the most
+         recent N our-original posts where engagement_style is set.
+
+  top-posts --project P [--limit 8]
+      -> Same source; print pipe-separated tuples:
+           our_content|upvotes|comments_count|views
+         filtered to rows with composite (upvotes + 3*comments + views/100)
+         > 5 and sorted by the same composite DESC.
+
+Migrated 2026-05-18: removes 3 direct psql calls from
+skill/run-twitter-threads.sh. The route at /api/v1/posts already supports
+platform + project + since + status filters server-side; this helper just
+shapes the response into the legacy line/pipe format the bash prompt
+consumes verbatim.
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from datetime import datetime, timedelta, timezone
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from http_api import api_get  # noqa: E402
+
+
+def _fetch_posts(project: str, days: int | None = None, limit: int = 500):
+    """Pull recent twitter posts for a project via /api/v1/posts. The
+    server-side WHERE handles platform/project/status/since; the
+    thread_url = our_url filter (= our original posts, not replies)
+    is applied client-side so the route stays general-purpose.
+
+    Post 2026-05-23, mentions live in the dedicated `mentions` table,
+    so we no longer need to filter '(mention%' placeholders client-side.
+    """
+    query: dict = {
+        "platform": "twitter",
+        "project": project,
+        "limit": limit,
+        "status": "active",
+        "has_our_url": "true",
+    }
+    if days is not None:
+        since = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat()
+        query["since"] = since
+    resp = api_get("/api/v1/posts", query=query)
+    rows = (resp.get("data") or {}).get("posts") or []
+    out = []
+    for r in rows:
+        if r.get("thread_url") != r.get("our_url"):
+            continue
+        out.append(r)
+    return out
+
+
+def cmd_recent_posts(project: str, days: int, limit: int) -> int:
+    rows = _fetch_posts(project, days=days, limit=max(limit * 5, 50))
+    # Posts come back posted_at DESC already; just take the first N.
+    for r in rows[:limit]:
+        content = (r.get("our_content") or "").replace("\n", " ").replace("\r", " ")
+        sys.stdout.write(content + "\n")
+    return 0
+
+
+def cmd_recent_styles(project: str, limit: int) -> int:
+    # No --days bound originally; pull a generous window so we don't miss
+    # styles when the account has been quiet recently.
+    rows = _fetch_posts(project, days=90, limit=max(limit * 10, 50))
+    n = 0
+    for r in rows:
+        style = (r.get("engagement_style") or "").strip()
+        if not style:
+            continue
+        sys.stdout.write(style + "\n")
+        n += 1
+        if n >= limit:
+            break
+    return 0
+
+
+def cmd_top_posts(project: str, limit: int) -> int:
+    rows = _fetch_posts(project, days=None, limit=500)
+
+    def composite(r):
+        return (
+            int(r.get("upvotes") or 0)
+            + int(r.get("comments_count") or 0) * 3
+            + int(r.get("views") or 0) // 100
+        )
+
+    # Same composite + threshold + sort as the legacy SQL: floor=5,
+    # ORDER BY composite DESC, LIMIT.
+    filtered = [r for r in rows if composite(r) > 5]
+    filtered.sort(key=composite, reverse=True)
+    for r in filtered[:limit]:
+        content = (r.get("our_content") or "").replace("\n", " ").replace("\r", " ")
+        upvotes = int(r.get("upvotes") or 0)
+        comments = int(r.get("comments_count") or 0)
+        views = int(r.get("views") or 0)
+        sys.stdout.write(f"{content}|{upvotes}|{comments}|{views}\n")
+    return 0
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="Helper for run-twitter-threads.sh")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    p_rp = sub.add_parser("recent-posts")
+    p_rp.add_argument("--project", required=True)
+    p_rp.add_argument("--days", type=int, default=14)
+    p_rp.add_argument("--limit", type=int, default=10)
+
+    p_rs = sub.add_parser("recent-styles")
+    p_rs.add_argument("--project", required=True)
+    p_rs.add_argument("--limit", type=int, default=5)
+
+    p_tp = sub.add_parser("top-posts")
+    p_tp.add_argument("--project", required=True)
+    p_tp.add_argument("--limit", type=int, default=8)
+
+    args = ap.parse_args()
+
+    if args.cmd == "recent-posts":
+        return cmd_recent_posts(args.project, args.days, args.limit)
+    if args.cmd == "recent-styles":
+        return cmd_recent_styles(args.project, args.limit)
+    if args.cmd == "top-posts":
+        return cmd_top_posts(args.project, args.limit)
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/unclaim_web_chat.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+"""Unclaim a web-chat thread (re-set unread_by_founder=1, clear cooldown).
+
+HTTP-only: POST /api/v1/web-chat/threads/<thread_id>/unclaim. Used when a Claude
+session fails so the next pipeline tick will retry.
+
+Usage:
+  python3 unclaim_web_chat.py <thread_id>
+"""
+
+import argparse
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from http_api import api_post
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("thread_id")
+    args = parser.parse_args()
+
+    api_post(f"/api/v1/web-chat/threads/{args.thread_id}/unclaim", {})
+    print(f"unclaimed thread {args.thread_id}")
+
+
+if __name__ == "__main__":
+    main()