@m13v/s4l 1.6.197-rc.10

package/scripts/scan_twitter_mentions_browser.py

@@ -0,0 +1,327 @@
+#!/usr/bin/env python3
+"""Scan Twitter notifications via the browser (no API cost) and insert new replies.
+
+Browser-based replacement for the old API-powered scan_twitter_mentions.py.
+Consumes JSON from `twitter_browser.py notifications [scroll] [tab]` which
+defaults to the /notifications (All) tab so we catch nested replies where the
+@-tag was dropped. Pass tab="mentions" to restrict to explicit @-mentions only.
+Companion: scan_twitter_thread_followups.py revisits our recent replies to
+pick up depth-2+ follow-ups that never surface in notifications at all.
+
+Usage:
+    python3 scripts/twitter_browser.py notifications 8 all > /tmp/twitter_notifs.json
+    python3 scripts/scan_twitter_mentions_browser.py --json-file /tmp/twitter_notifs.json
+
+Migrated 2026-05-18: reads/writes go through s4l.ai HTTP API (/api/v1/posts,
+/api/v1/posts/lookup, /api/v1/replies) via scripts/http_api.py instead of
+psycopg2. Note: the route enforces (platform, their_comment_id) uniqueness
+server-side, so the "existing_ids" prefetch is now a soft local cache used
+to short-circuit the POST loop; we still rely on the API's ON CONFLICT path
+as the source of truth.
+
+Migrated 2026-05-23: third-party mentions now write to the dedicated
+`mentions` table via /api/v1/mentions instead of a placeholder row in
+`posts`. The associated reply row carries `mention_id` instead of
+`post_id`, enforced by the replies_post_or_mention_exclusive_check DB
+constraint. See migrations/2026-05-23-mentions-table.sql and
+scripts/migrate_mentions_out_of_posts.py for the cutover history.
+"""
+
+import argparse
+import json
+import os
+import re
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from http_api import api_get, api_post  # noqa: E402
+from project_topics import topics_for_project  # noqa: E402
+try:
+    from account_resolver import resolve as _resolve_account  # noqa: E402
+except Exception:
+    def _resolve_account(_platform):  # type: ignore[unused-arg]
+        return None
+
+CONFIG_PATH = os.path.expanduser("~/social-autoposter/config.json")
+MIN_WORDS = 3
+OUR_HANDLE = _resolve_account("twitter")
+if not OUR_HANDLE:
+    # No hardcoded fallback: scanning/attributing under a default handle silently
+    # impersonates the repo owner. Refuse to run so the missing config surfaces.
+    sys.stderr.write(
+        "[scan_twitter_mentions] no Twitter handle configured "
+        "(accounts.twitter.handle / AUTOPOSTER_TWITTER_HANDLE); refusing to run "
+        "to avoid wrong-account attribution. Run connect_x first.\n")
+    sys.exit(1)
+
+# Paginate the replies prefetch in chunks so we never blow the route's max
+# limit. 500 is the per-call cap inside /api/v1/replies; we walk pages until
+# the response is short.
+REPLY_PAGE_LIMIT = 500
+REPLY_MAX_PAGES = 200  # 100k rows of headroom; plenty for the dedup cache.
+
+
+def load_config():
+    if os.path.exists(CONFIG_PATH):
+        with open(CONFIG_PATH) as f:
+            return json.load(f)
+    return {}
+
+
+def word_count(text):
+    return len(text.split()) if text else 0
+
+
+def get_existing_reply_ids():
+    """Pull every existing replies.their_comment_id for platform=x as a dedup cache.
+
+    The route caps responses at 500 rows per call; we paginate by id DESC and
+    keep walking until we exhaust the set. The route also handles uniqueness
+    on the server, so even if our local cache lags slightly we won't insert
+    duplicates — we'll just get ok_on_conflict back from POST.
+    """
+    cache = set()
+    max_id = None
+    for _ in range(REPLY_MAX_PAGES):
+        query = {
+            "platform": "x",
+            "limit": REPLY_PAGE_LIMIT,
+            "order_by": "id",
+        }
+        # We don't have an explicit max_id filter on the route today; walk by
+        # `since` instead is wrong (since acts on discovered_at). Easiest: ask
+        # for the first 500 most-recent rows and trust that older rows in DB
+        # already collided once at insert-time, so we don't need a perfect
+        # global cache — just a recency window deep enough to catch this
+        # cycle's incoming notifications.
+        resp = api_get("/api/v1/replies", query=query)
+        rows = (resp.get("data") or {}).get("replies") or []
+        if not rows:
+            break
+        for r in rows:
+            cid = r.get("their_comment_id")
+            if cid:
+                cache.add(cid)
+        if len(rows) < REPLY_PAGE_LIMIT:
+            break
+        # Today's route has no "id <" cursor parameter, so one page is all we
+        # get. That is enough: it caps memory + roundtrip and the server-side
+        # UNIQUE index is still the canonical dedup. Break out.
+        break
+        # Suppress unused-binding lint warning for max_id while we leave the
+        # placeholder in place; future route work may add an id-cursor.
+        _ = max_id
+    return cache
+
+
+def get_our_posts():
+    """Map tweet_id (last URL segment) -> post row for our active twitter posts."""
+    resp = api_get(
+        "/api/v1/posts",
+        query={"platform": "twitter", "status": "active", "limit": 500},
+    )
+    rows = (resp.get("data") or {}).get("posts") or []
+    posts = {}
+    for row in rows:
+        url = row.get("our_url")
+        if not url:
+            continue
+        m = re.search(r"/status/(\d+)", url)
+        if m:
+            posts[m.group(1)] = row
+    return posts
+
+
+def guess_project(text, config):
+    projects = config.get("projects", [])
+    text_lower = (text or "").lower()
+    for p in projects:
+        name = p.get("name", "")
+        # DB-backed seed list (post 2026-05-27 config.json removal).
+        topics = topics_for_project(name)
+        for topic in topics:
+            if topic.lower() in text_lower:
+                return name
+        if name.lower() in text_lower:
+            return name
+    return config.get("default_project", "General")
+
+
+def most_recent_active_project():
+    """Project_name of the most recent active twitter post we made.
+
+    Used as a fallback for replies-to-us where the notification feed doesn't
+    expose the parent tweet ID, so we can't identify *which* of our posts
+    the mention is under. Recency is a much stronger signal than
+    keyword-matching a 3-word reply body.
+
+    Post 2026-05-23 the "(mention - no original post)" placeholder rows no
+    longer exist in `posts` (they live in `mentions` now), so the SQL/
+    client-side filter that used to live here is gone.
+    """
+    resp = api_get(
+        "/api/v1/posts",
+        query={
+            "platform": "twitter",
+            "status": "active",
+            "limit": 50,
+        },
+    )
+    rows = (resp.get("data") or {}).get("posts") or []
+    for r in rows:
+        proj = r.get("project_name")
+        if not proj:
+            continue
+        return proj
+    return None
+
+
+def process_notifications(notifications, config):
+    exclusions = config.get("exclusions", {})
+    excluded_accounts = {a.lower() for a in exclusions.get("twitter_accounts", [])}
+    excluded_accounts.add(OUR_HANDLE.lower())
+
+    existing_ids = get_existing_reply_ids()
+    our_posts = get_our_posts()
+    recent_project = most_recent_active_project()
+
+    stats = {
+        "new": 0,
+        "already_tracked": 0,
+        "excluded_author": 0,
+        "own_account": 0,
+        "too_short": 0,
+        "no_tweet_id": 0,
+    }
+
+    for n in notifications:
+        tweet_id = n.get("tweet_id", "")
+        handle = (n.get("handle") or "").lstrip("@")
+        text = n.get("text") or ""
+        tweet_url = n.get("tweet_url") or (
+            f"https://x.com/{handle}/status/{tweet_id}" if handle and tweet_id else ""
+        )
+        replying_to = (n.get("replying_to") or "").lstrip("@").lower()
+
+        if not tweet_id:
+            stats["no_tweet_id"] += 1
+            continue
+
+        if tweet_id in existing_ids:
+            stats["already_tracked"] += 1
+            continue
+
+        if handle.lower() in excluded_accounts:
+            stats["own_account" if handle.lower() == OUR_HANDLE.lower() else "excluded_author"] += 1
+            continue
+
+        if word_count(text) < MIN_WORDS:
+            stats["too_short"] += 1
+            continue
+
+        # Resolve project for the mention. Reply-to-us inherits the project
+        # of our most recent active post (short reply text is unreliable for
+        # keyword matching); other mentions fall back to keyword guess.
+        is_reply_to_us = replying_to == OUR_HANDLE.lower() and bool(our_posts)
+        if is_reply_to_us and recent_project:
+            project = recent_project
+        else:
+            project = guess_project(text, config)
+        # _ = our_posts  # currently unused for direct post_id linkage; notifications
+        # don't expose conversation_id, so we attribute via mentions table only.
+
+        # Insert into /api/v1/mentions. Dedup on (platform, mentioning_url)
+        # — if the row already exists we get back existing_mention_id from
+        # the 409 body via ok_on_conflict.
+        mention_body = {
+            "platform": "twitter",
+            "mentioning_url": tweet_url,
+            "mentioning_handle": handle,
+            "mentioning_text": text,
+            "our_handle": OUR_HANDLE,
+            "project": project,
+            "status": "active",
+        }
+        mention_resp = api_post(
+            "/api/v1/mentions", mention_body, ok_on_conflict=True,
+        )
+        mention_data = mention_resp.get("data") or {}
+        mention_row = mention_data.get("mention") or {}
+        mention_id = mention_row.get("id")
+        if not mention_id and mention_resp.get("error"):
+            details = (mention_resp.get("error") or {}).get("details") or {}
+            mention_id = details.get("existing_mention_id")
+            if not mention_id:
+                inner = details.get("mention") or {}
+                mention_id = inner.get("id")
+        if not mention_id:
+            print(
+                f"  WARNING: could not resolve mention_id for {tweet_url!r}; skipping",
+                file=sys.stderr,
+            )
+            continue
+
+        reply_resp = api_post(
+            "/api/v1/replies",
+            {
+                "mention_id": mention_id,
+                "platform": "x",
+                "their_comment_id": tweet_id,
+                "their_author": handle,
+                "their_content": text,
+                "their_comment_url": tweet_url,
+                "depth": 1,
+                "status": "pending",
+                "our_account": OUR_HANDLE,
+            },
+            ok_on_conflict=True,
+        )
+        # 409 means the row already existed under the server-side UNIQUE
+        # (platform, their_comment_id) constraint; count it as already_tracked
+        # rather than new so the summary matches reality.
+        if (reply_resp.get("error") or {}).get("code") == "duplicate_reply":
+            stats["already_tracked"] += 1
+        else:
+            stats["new"] += 1
+            print(f"  NEW: @{handle}: {text[:80]}")
+        existing_ids.add(tweet_id)
+
+    return stats
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Process Twitter notification data from browser scanner"
+    )
+    parser.add_argument(
+        "--json-file",
+        required=True,
+        help="Path to JSON from twitter_browser.py notifications",
+    )
+    args = parser.parse_args()
+
+    with open(args.json_file) as f:
+        data = json.load(f)
+
+    if isinstance(data, dict) and data.get("error"):
+        print(f"ERROR from extractor: {data['error']}", file=sys.stderr)
+        sys.exit(1)
+
+    notifications = data.get("notifications", []) if isinstance(data, dict) else data
+    print(f"Processing {len(notifications)} mentions...")
+
+    config = load_config()
+    stats = process_notifications(notifications, config)
+
+    print(
+        f"\nSummary: {stats['new']} new, "
+        f"{stats['already_tracked']} already tracked, "
+        f"{stats['excluded_author']} excluded, "
+        f"{stats['own_account']} own account, "
+        f"{stats['too_short']} too short, "
+        f"{stats['no_tweet_id']} no tweet_id"
+    )
+
+
+if __name__ == "__main__":
+    main()

package/scripts/scan_twitter_thread_followups.py

@@ -0,0 +1,299 @@
+#!/usr/bin/env python3
+"""Scan our recent X replies for new public follow-ups and ingest them.
+
+Companion to scan_twitter_mentions_browser.py. The mentions tab only surfaces
+explicit @-mentions, so replies to our replies without a retagged handle are
+invisible. This script compensates by revisiting each of our recent X replies
+and scraping the page for depth-2+ comments that aren't yet in the DB.
+
+Flow:
+  1. Query `replies` for our X replies in last N days (default 14) where
+     `our_reply_url IS NOT NULL`. These are the threads we're subscribing to.
+  2. Write those URLs to a temp file.
+  3. Invoke `twitter_browser.py thread-followups <file>`, which scrapes each
+     URL and returns a `{results: [{thread_url, anchor_tweet_id, followups}]}`
+     JSON blob.
+  4. For each followup not already in `replies` (by platform+their_comment_id),
+     insert a new `replies` row with:
+       - platform = 'x'
+       - parent_reply_id = id of the original reply (the anchor)
+       - post_id = anchor.post_id
+       - depth = anchor.depth + 1
+       - status = 'pending'
+     Tweets we posted ourselves are skipped (OUR_HANDLE check). Own-account
+     replies from us get status='replied' with our_reply_id populated, mirroring
+     the mentions scanner.
+
+Usage:
+    python3 scripts/scan_twitter_thread_followups.py [--days N] [--max-urls N]
+
+Migrated 2026-05-18: reads/writes now route through the s4l.ai HTTP API
+(/api/v1/replies for both filter-list and insert) instead of psycopg2.
+The (platform, their_comment_id) dedup runs server-side; the local
+known_ids cache is now just for in-loop short-circuiting.
+"""
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+from datetime import datetime, timedelta, timezone
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from http_api import api_get, api_post  # noqa: E402
+try:
+    from account_resolver import resolve as _resolve_account  # noqa: E402
+except Exception:
+    def _resolve_account(_platform):  # type: ignore[unused-arg]
+        return None
+
+CONFIG_PATH = os.path.expanduser("~/social-autoposter/config.json")
+OUR_HANDLE = _resolve_account("twitter")
+if not OUR_HANDLE:
+    # No hardcoded fallback: scanning/attributing under a default handle silently
+    # impersonates the repo owner. Refuse to run so the missing config surfaces.
+    sys.stderr.write(
+        "[scan_twitter_followups] no Twitter handle configured "
+        "(accounts.twitter.handle / AUTOPOSTER_TWITTER_HANDLE); refusing to run "
+        "to avoid wrong-account attribution. Run connect_x first.\n")
+    sys.exit(1)
+DEFAULT_DAYS = 14
+DEFAULT_MAX_URLS = 40
+REPO_DIR = os.path.expanduser("~/social-autoposter")
+REPLY_PAGE_LIMIT = 500
+
+
+def load_config():
+    if os.path.exists(CONFIG_PATH):
+        with open(CONFIG_PATH) as f:
+            return json.load(f)
+    return {}
+
+
+def fetch_our_recent_x_replies(days, max_urls):
+    """Return list of (reply_id, our_reply_url, post_id, depth) for our recent X replies.
+
+    Filters live in the route as:
+      - platform = x
+      - status = replied (the route's WHERE)
+      - has_our_reply_content / has_our_reply_id NOT used here; we need
+        our_reply_url, but the route returns it on every row and we filter
+        client-side after the page comes back.
+      - replied_at >= NOW() - <days>d
+    """
+    since = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat()
+    resp = api_get(
+        "/api/v1/replies",
+        query={
+            "platform": "x",
+            "status": "replied",
+            "since": since,
+            "limit": max_urls,
+            "order_by": "replied_at",
+        },
+    )
+    rows = (resp.get("data") or {}).get("replies") or []
+    out = []
+    for r in rows:
+        url = r.get("our_reply_url")
+        if not url:
+            continue
+        out.append((r["id"], url, r.get("post_id"), int(r.get("depth") or 1)))
+    return out[:max_urls]
+
+
+def existing_comment_ids():
+    """First-page snapshot of replies.their_comment_id for platform=x.
+
+    The route's UNIQUE (platform, their_comment_id) index is the canonical
+    dedup; this cache short-circuits the per-followup POST loop and prints
+    accurate "already tracked" counts. Bounded at REPLY_PAGE_LIMIT (500) by
+    the route — fine because the most recent rows are the ones we'd
+    otherwise collide with.
+    """
+    resp = api_get(
+        "/api/v1/replies",
+        query={"platform": "x", "limit": REPLY_PAGE_LIMIT, "order_by": "id"},
+    )
+    rows = (resp.get("data") or {}).get("replies") or []
+    return {r.get("their_comment_id") for r in rows if r.get("their_comment_id")}
+
+
+def anchor_id_from_url(url):
+    m = re.search(r"/status/(\d+)", url or "")
+    return m.group(1) if m else None
+
+
+def run_browser_scrape(urls, scroll_count=3):
+    """Shell out to twitter_browser.py thread-followups and parse JSON."""
+    if not urls:
+        return {"results": [], "urls_visited": 0}
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:
+        urls_path = f.name
+        for u in urls:
+            f.write(u + "\n")
+    try:
+        proc = subprocess.run(
+            ["python3", os.path.join(REPO_DIR, "scripts/twitter_browser.py"),
+             "thread-followups", urls_path, str(scroll_count)],
+            capture_output=True, text=True, timeout=1800,
+        )
+        if proc.returncode != 0:
+            print(f"ERROR: twitter_browser.py exited {proc.returncode}", file=sys.stderr)
+            print(proc.stderr[-2000:], file=sys.stderr)
+            return {"results": [], "error": "browser_failed"}
+        try:
+            return json.loads(proc.stdout)
+        except json.JSONDecodeError as e:
+            print(f"ERROR: could not parse browser output as JSON: {e}", file=sys.stderr)
+            print(proc.stdout[-2000:], file=sys.stderr)
+            return {"results": [], "error": "json_parse_failed"}
+    finally:
+        try:
+            os.unlink(urls_path)
+        except OSError:
+            pass
+
+
+def insert_followup(followup, parent_reply_id, post_id, parent_depth, root_author=None):
+    """Insert one follow-up row via /api/v1/replies. Returns True if inserted,
+    False if skipped (own handle, missing required fields, or 409 duplicate)."""
+    tweet_id = followup.get("tweet_id") or ""
+    handle = (followup.get("handle") or "").lstrip("@")
+    text = followup.get("text") or ""
+    url = followup.get("tweet_url") or ""
+    if not tweet_id or not handle:
+        return False
+    if handle.lower() == OUR_HANDLE.lower():
+        return False
+    body = {
+        "post_id": post_id,
+        "platform": "x",
+        "their_comment_id": tweet_id,
+        "their_author": handle,
+        "their_content": text,
+        "their_comment_url": url,
+        "depth": (parent_depth or 1) + 1,
+        "status": "pending",
+        "parent_reply_id": parent_reply_id,
+        "our_account": OUR_HANDLE,
+    }
+    # OP of the thread our reply lives in, scraped for free from the conversation
+    # page (twitter_browser.scrape_many_thread_followups). Always set when known,
+    # including when the OP is the replier — that equality is the "OP replied"
+    # signal the analytic needs.
+    root_author = (root_author or "").lstrip("@")
+    if root_author:
+        body["thread_author_handle"] = root_author
+    # Media of the followup tweet itself (images/videos/GIFs/link-cards),
+    # captured for free during the same DOM pass in
+    # twitter_browser.scrape_thread_followups (2026-06-03 thread-media feature).
+    # The engage prompt reads this back via /api/v1/replies/next-pending so it
+    # can reply to what the comment VISUALLY shows, not just its text. An empty
+    # list [] is meaningful ("captured, none found"); only omit when the
+    # extractor returned nothing parseable (None). Harmless no-op against the
+    # pre-deploy API, which simply ignores the unknown field.
+    media = followup.get("media")
+    if isinstance(media, list):
+        body["their_media"] = media
+    resp = api_post("/api/v1/replies", body, ok_on_conflict=True)
+    if (resp.get("error") or {}).get("code") == "duplicate_reply":
+        return False
+    return True
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--days", type=int, default=DEFAULT_DAYS,
+                        help=f"Look back N days for our replies (default {DEFAULT_DAYS})")
+    parser.add_argument("--max-urls", type=int, default=DEFAULT_MAX_URLS,
+                        help=f"Max thread URLs to revisit per run (default {DEFAULT_MAX_URLS})")
+    parser.add_argument("--scroll-count", type=int, default=3,
+                        help="Scrolls per thread page (default 3)")
+    parser.add_argument("--dry-run", action="store_true",
+                        help="Print what would be inserted without writing")
+    args = parser.parse_args()
+
+    our_replies = fetch_our_recent_x_replies(args.days, args.max_urls)
+    print(f"Revisiting {len(our_replies)} of our recent X replies (last {args.days}d)")
+    if not our_replies:
+        return 0
+
+    url_to_meta = {url: (rid, pid, depth) for rid, url, pid, depth in our_replies}
+    urls = list(url_to_meta.keys())
+
+    print(f"Invoking browser scraper for {len(urls)} URLs...")
+    data = run_browser_scrape(urls, scroll_count=args.scroll_count)
+
+    results = data.get("results", [])
+    known_ids = existing_comment_ids()
+    new_count = 0
+    skip_own = 0
+    skip_existing = 0
+    skip_anchor = 0
+    skip_not_replying_to_us = 0
+
+    for r in results:
+        thread_url = r.get("thread_url") or ""
+        anchor_id = r.get("anchor_tweet_id") or anchor_id_from_url(thread_url)
+        root_author = (r.get("root_author") or "").lstrip("@")
+        meta = url_to_meta.get(thread_url)
+        if not meta:
+            continue
+        parent_reply_id, post_id, parent_depth = meta
+
+        for fu in r.get("followups", []):
+            tid = fu.get("tweet_id")
+            handle = (fu.get("handle") or "").lstrip("@")
+            if not tid:
+                continue
+            if tid == anchor_id:
+                skip_anchor += 1
+                continue
+            if handle.lower() == OUR_HANDLE.lower():
+                skip_own += 1
+                continue
+            # Filter: only keep tweets that are actually replying to us.
+            # X tweet permalink pages inject "more from this author" / "you might
+            # like" articles into the timeline. Without this check, those leak
+            # in as fake follow-ups (observed 2026-05: ~80% of captures were
+            # the seed author's later unrelated promotional tweets, not replies
+            # to our reply). The extractor in twitter_browser.py captures
+            # `replying_to` from the "Replying to @handle" block above each
+            # tweet; if it's empty or doesn't point at our handle, it's not a
+            # response to us.
+            replying_to = (fu.get("replying_to") or "").lstrip("@").lower()
+            if replying_to != OUR_HANDLE.lower():
+                skip_not_replying_to_us += 1
+                continue
+            if tid in known_ids:
+                skip_existing += 1
+                continue
+            if args.dry_run:
+                print(f"  [DRY] @{handle} (tid={tid}) op=@{root_author or '?'} parent_reply={parent_reply_id} depth={(parent_depth or 1) + 1}: {(fu.get('text') or '')[:80]}")
+                new_count += 1
+                known_ids.add(tid)
+                continue
+            inserted = insert_followup(fu, parent_reply_id, post_id, parent_depth, root_author=root_author)
+            if inserted:
+                new_count += 1
+                known_ids.add(tid)
+                print(f"  NEW follow-up: @{handle} (tid={tid}) parent_reply={parent_reply_id} depth={(parent_depth or 1) + 1}: {(fu.get('text') or '')[:80]}")
+            else:
+                # 409 duplicate (someone else inserted between our local cache
+                # and this POST). Count it as already-tracked, not new.
+                known_ids.add(tid)
+                skip_existing += 1
+
+    print(f"\nSummary: {new_count} new follow-ups ingested, "
+          f"{skip_existing} already tracked, {skip_own} own account, "
+          f"{skip_anchor} anchor skips, {skip_not_replying_to_us} not replying to us")
+    return new_count
+
+
+if __name__ == "__main__":
+    rc = main()
+    sys.exit(0 if rc >= 0 else 1)