@m13v/s4l 1.7.4-rc.2 → 1.7.4-rc.21

package/scripts/enrich_reply_parents.py

@@ -0,0 +1,312 @@
+#!/usr/bin/env python3
+"""Fill parent-thread linkage on X replies discovered via the notifications lane.
+
+The X notifications feed does not expose the parent tweet id, so
+scan_twitter_mentions_browser.py inserts `replies` rows with mention_id only:
+no post_id, no parent_reply_id, and often no project_name. This script
+resolves the parent chain AFTER the fact, deterministically, with no browser
+and no model: fxtwitter's public JSON (already used by fetch_twitter_t1.py)
+returns `replying_to_status` for any tweet.
+
+Per row with (post_id IS NULL AND parent_reply_id IS NULL):
+  1. fxtwitter GET on their_comment_id -> parent tweet id + handle.
+  2. Walk up the ancestor chain (bounded hops) until the root.
+  3. First ancestor that is one of OUR posts (/api/v1/posts/lookup, wide
+     window) -> PATCH replies.post_id (+ project_name when the row has none).
+  4. Immediate parent that is another tracked reply
+     (/api/v1/replies?their_comment_id=) -> PATCH parent_reply_id + depth.
+  5. Root author -> PATCH thread_author_handle.
+
+Terminal misses (tweet deleted, protected, not-a-reply with nothing to link)
+are remembered in a local state file so recurring runs don't refetch forever.
+
+Usage:
+  python3 scripts/enrich_reply_parents.py --limit 20            # recurring lane
+  python3 scripts/enrich_reply_parents.py --backfill            # all missing rows
+  python3 scripts/enrich_reply_parents.py --ids 546832 542579   # specific rows
+  python3 scripts/enrich_reply_parents.py --limit 5 --dry-run
+
+All DB I/O goes through the s4l.ai HTTP API (http_api), never direct SQL.
+"""
+import argparse
+import json
+import os
+import sys
+import time
+import urllib.error
+import urllib.request
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from http_api import api_get, api_patch  # noqa: E402
+
+STATE_PATH = os.environ.get(
+    "S4L_ENRICH_PARENTS_STATE",
+    os.path.expanduser("~/.social-autoposter-enrich-parents.json"),
+)
+MAX_HOPS = 6
+LOOKUP_DAYS = 3650
+
+
+def load_state():
+    try:
+        with open(STATE_PATH) as f:
+            return json.load(f)
+    except Exception:
+        return {}
+
+
+def save_state(state):
+    tmp = STATE_PATH + ".tmp"
+    with open(tmp, "w") as f:
+        json.dump(state, f)
+    os.replace(tmp, STATE_PATH)
+
+
+def fetch_fxtwitter(handle, tweet_id):
+    """Returns (status, tweet_dict). status: 'ok' | 'gone' | 'transient'."""
+    url = f"https://api.fxtwitter.com/{handle or 'i'}/status/{tweet_id}"
+    req = urllib.request.Request(url, headers={"User-Agent": "social-autoposter/1.0"})
+    try:
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            data = json.loads(resp.read())
+    except urllib.error.HTTPError as e:
+        # 401 = protected account, 404 = deleted. Both terminal.
+        if e.code in (401, 404):
+            return "gone", None
+        return "transient", None
+    except Exception:
+        return "transient", None
+    code = data.get("code")
+    if code == 200 and data.get("tweet"):
+        return "ok", data["tweet"]
+    if code in (401, 404):
+        return "gone", None
+    return "transient", None
+
+
+def walk_ancestors(handle, tweet_id, sleep_s):
+    """Ancestor chain bottom-up: [(id, handle), ...] parent first, root last.
+
+    Returns (focal, chain, terminal): focal is the fetched tweet object for
+    tweet_id itself (None when it is gone/unfetchable — needed for quote
+    linkage), terminal is 'root' when the walk reached a non-reply tweet,
+    'gone'/'transient' when a hop was deleted/protected/errored (the chain up
+    to that point is still usable, but root attribution is not), or 'hop_cap'.
+    """
+    chain = []
+    focal = None
+    cur_handle, cur_id = handle, tweet_id
+    for _ in range(MAX_HOPS):
+        status, tweet = fetch_fxtwitter(cur_handle, cur_id)
+        time.sleep(sleep_s)
+        if status != "ok":
+            # 'gone' is terminal (deleted/protected); 'transient' must NOT be
+            # remembered, the next run retries it.
+            return focal, chain, status
+        if focal is None:
+            focal = tweet
+        parent_id = tweet.get("replying_to_status")
+        parent_handle = tweet.get("replying_to") or ""
+        if not parent_id:
+            return focal, chain, "root"
+        chain.append((str(parent_id), parent_handle))
+        cur_handle, cur_id = parent_handle, parent_id
+    return focal, chain, "hop_cap"
+
+
+def lookup_our_post(tweet_id):
+    resp = api_get(
+        "/api/v1/posts/lookup",
+        query={"platform": "twitter", "post_id": str(tweet_id), "days": str(LOOKUP_DAYS)},
+    )
+    return (resp.get("data") or {}).get("post") or None
+
+
+def lookup_tracked_reply(tweet_id):
+    resp = api_get(
+        "/api/v1/replies",
+        query={"platform": "x", "their_comment_id": str(tweet_id), "limit": "1"},
+    )
+    rows = (resp.get("data") or {}).get("replies") or []
+    return rows[0] if rows else None
+
+
+def lookup_our_posted_reply(tweet_id):
+    """Reply row where WE authored the tweet (our_reply_id / our_reply_url)."""
+    resp = api_get(
+        "/api/v1/replies",
+        query={"platform": "x", "our_reply_status_id": str(tweet_id), "limit": "1"},
+    )
+    rows = (resp.get("data") or {}).get("replies") or []
+    return rows[0] if rows else None
+
+
+def fetch_work(limit, our_account=None, ids=None, before_id=None):
+    if ids:
+        out = []
+        for rid in ids:
+            resp = api_get(f"/api/v1/replies/{rid}")
+            row = (resp.get("data") or {}).get("reply")
+            if row:
+                out.append(row)
+        return out
+    query = {
+        "platform": "x",
+        "missing_parent": "1",
+        "order_by": "id",
+        "limit": str(limit),
+    }
+    if our_account:
+        query["our_account"] = our_account
+    if before_id:
+        query["before_id"] = str(before_id)
+    resp = api_get("/api/v1/replies", query=query)
+    return (resp.get("data") or {}).get("replies") or []
+
+
+def enrich_row(row, state, sleep_s, dry_run):
+    rid = row["id"]
+    tid = str(row.get("their_comment_id") or "")
+    handle = (row.get("their_author") or "").lstrip("@")
+    if not tid:
+        return "no_tweet_id"
+    if state.get(tid):
+        return "state_skip"
+
+    focal, chain, terminal = walk_ancestors(handle, tid, sleep_s)
+    if focal is None:
+        if terminal == "transient":
+            return "transient"  # retry next run, no state write
+        state[tid] = "gone"  # deleted/protected focal tweet
+        return "gone"
+
+    patch = {}
+    matched_post = None
+    for anc_id, _anc_handle in chain:
+        post = lookup_our_post(anc_id)
+        if post:
+            matched_post = post
+            break
+    if matched_post:
+        patch["post_id"] = matched_post["id"]
+        if not row.get("project_name") and matched_post.get("project_name"):
+            patch["project_name"] = matched_post["project_name"]
+
+    if chain:
+        parent_id, _parent_handle = chain[0]
+        # Immediate parent: another inbound reply we track, or a reply WE
+        # posted (the dominant case: a fan replying to our engagement reply).
+        tracked = lookup_tracked_reply(parent_id)
+        if tracked and tracked["id"] != rid:
+            patch["parent_reply_id"] = tracked["id"]
+            patch["depth"] = (tracked.get("depth") or 1) + 1
+        else:
+            ours = lookup_our_posted_reply(parent_id)
+            if ours and ours["id"] != rid:
+                patch["parent_reply_id"] = ours["id"]
+                patch["depth"] = (ours.get("depth") or 1) + 1
+                if "post_id" not in patch and ours.get("post_id"):
+                    patch["post_id"] = ours["post_id"]
+                if not row.get("project_name") and not patch.get("project_name") \
+                        and ours.get("project_name"):
+                    patch["project_name"] = ours["project_name"]
+
+    # Quote linkage: a quote-tweet of our post (or of a reply we posted) is
+    # engagement on our content even when replying_to_status is empty.
+    quote_id = str(((focal.get("quote") or {}).get("id")) or "")
+    if quote_id and "post_id" not in patch:
+        qpost = lookup_our_post(quote_id)
+        if qpost:
+            patch["post_id"] = qpost["id"]
+            if not row.get("project_name") and not patch.get("project_name") \
+                    and qpost.get("project_name"):
+                patch["project_name"] = qpost["project_name"]
+        elif "parent_reply_id" not in patch:
+            qours = lookup_our_posted_reply(quote_id)
+            if qours and qours["id"] != rid:
+                patch["parent_reply_id"] = qours["id"]
+                patch["depth"] = (qours.get("depth") or 1) + 1
+                if qours.get("post_id"):
+                    patch["post_id"] = qours["post_id"]
+
+    if terminal == "root":
+        root_handle = (chain[-1][1] if chain else (focal.get("author") or {}).get("screen_name") or "")
+        root_handle = (root_handle or "").lstrip("@")
+        if chain and root_handle and not row.get("thread_author_handle"):
+            patch["thread_author_handle"] = root_handle
+
+    if not patch:
+        if terminal == "transient":
+            return "transient"  # incomplete walk; retry next run
+        # Chain walked to root / cut by a deleted ancestor / not a reply at
+        # all, and nothing of ours anywhere in it: remember permanently.
+        state[tid] = "no_link" if chain else "not_a_reply"
+        return state[tid]
+
+    if dry_run:
+        print(f"  [DRY] reply {rid}: {json.dumps(patch)}")
+        return "would_patch"
+
+    resp = api_patch(f"/api/v1/replies/{rid}", patch)
+    if resp.get("error"):
+        print(f"  ERROR patching reply {rid}: {resp['error']}", file=sys.stderr)
+        return "patch_error"
+    state[tid] = "linked"
+    kinds = "+".join(k for k in ("post_id", "parent_reply_id", "thread_author_handle") if k in patch)
+    print(f"  linked reply {rid}: {kinds} {json.dumps(patch)}")
+    return "linked"
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Backfill parent-thread linkage on X replies")
+    ap.add_argument("--limit", type=int, default=20, help="rows per run (recurring lane)")
+    ap.add_argument("--backfill", action="store_true", help="keep paging until no work is left")
+    ap.add_argument("--ids", nargs="*", type=int, help="enrich specific reply ids")
+    ap.add_argument("--our-account", default=None, help="scope to one posting handle")
+    ap.add_argument("--all-accounts", action="store_true", help="do not scope by handle")
+    ap.add_argument("--sleep", type=float, default=0.5, help="seconds between fxtwitter calls")
+    ap.add_argument("--dry-run", action="store_true")
+    args = ap.parse_args()
+
+    our_account = args.our_account
+    if not our_account and not args.all_accounts and not args.ids:
+        try:
+            from account_resolver import resolve as _resolve_account
+            our_account = _resolve_account("twitter")
+        except Exception:
+            our_account = None
+        if not our_account:
+            print("No twitter account resolvable; pass --our-account or --all-accounts", file=sys.stderr)
+            sys.exit(1)
+
+    state = load_state()
+    totals = {}
+    seen_ids = set()
+    before_id = None
+    while True:
+        rows = fetch_work(
+            args.limit if not args.backfill else 200, our_account, args.ids, before_id
+        )
+        rows = [r for r in rows if r["id"] not in seen_ids]
+        if not rows:
+            break
+        for row in rows:
+            seen_ids.add(row["id"])
+            # Terminal misses (deleted tweet, foreign thread) never leave the
+            # missing_parent queue; the id cursor pages past them.
+            before_id = row["id"] if before_id is None else min(before_id, row["id"])
+            outcome = enrich_row(row, state, args.sleep, args.dry_run)
+            totals[outcome] = totals.get(outcome, 0) + 1
+            if len(seen_ids) % 25 == 0 and not args.dry_run:
+                save_state(state)
+        if not args.dry_run:
+            save_state(state)
+        if args.ids or not args.backfill:
+            break
+
+    print(f"[enrich_reply_parents] processed={len(seen_ids)} " +
+          " ".join(f"{k}={v}" for k, v in sorted(totals.items())))
+
+
+if __name__ == "__main__":
+    main()

package/scripts/feedback_digest.py

@@ -82,6 +82,13 @@ MAX_EVENTS_PER_RUN = 200
 # degraded (a bad search topic, a draft-quality regression), not that any
 # single draft was wrong in a way the digest model could articulate.
 BULK_NO_REASON_THRESHOLD = int(os.environ.get("S4L_BULK_NO_REASON_THRESHOLD", "3"))
+# Two-draft cards ship raw per-box pointer-dwell ms in draft_choice; this is
+# the read-vs-skim floor for the draft the reviewer did NOT pick. At or above
+# it, keeping the preselected Draft A counts as an informed keep (they read B
+# and stayed); below it the approval says nothing about B. Threshold lives
+# HERE, not in the menubar client, so it can be tuned without a client
+# release.
+DRAFT_READ_MS = int(os.environ.get("S4L_DRAFT_READ_MS", "1000"))
 
 DISALLOWED_TOOLS = (
     "ScheduleWakeup,CronCreate,CronDelete,CronList,EnterPlanMode,EnterWorktree,"
@@ -106,10 +113,54 @@ def load_config():
         return {"projects": []}
 
 
+def _draft_choice(e: dict) -> dict | None:
+    """Parsed draft_choice payload (two-draft cards only). The API returns
+    jsonb as a dict; a locally-buffered event may still carry it as a JSON
+    string. None when absent, unparseable, or missing the unchosen draft
+    (nothing pairwise to say without the loser)."""
+    dc = e.get("draft_choice")
+    if isinstance(dc, str):
+        try:
+            dc = json.loads(dc)
+        except Exception:
+            return None
+    if not isinstance(dc, dict) or not (dc.get("unchosen_text") or "").strip():
+        return None
+    return dc
+
+
 def _event_line(e: dict) -> str:
     """One compact evidence line per event for the prompt."""
     parts = [f"[{e.get('decision')}{'+loved' if e.get('loved') else ''}]"]
     note = (e.get("reject_note") or "").strip()
+    # Two-draft pairwise flags (approvals only: on a reject BOTH drafts died,
+    # so which box the caret sat in carries no preference). Weighting ladder,
+    # explained to the model in build_prompt: an active switch to Draft B is
+    # strong (they necessarily read both); keeping the preselected A counts
+    # only when hover dwell shows they actually read B; a fast approve with B
+    # unread is flagged as exactly that so no preference gets fabricated.
+    dc = _draft_choice(e) if e.get("decision") == "approved" else None
+    show_unchosen = False
+    if dc:
+        if not dc.get("auto_selected"):
+            parts.append("picked_draft_b_over_default_a")
+            show_unchosen = True
+        elif dc.get("visited_other"):
+            # Clicked into B, then came back and approved A: an explicit
+            # head-to-head choice of the default, same strength as a switch
+            # (2026-07-10 user rule: only a zero-interaction approve is
+            # no-signal).
+            parts.append("chose_default_a_after_trying_b")
+            show_unchosen = True
+        else:
+            other_ms = dc.get("hover_b_ms") or 0
+            if other_ms >= DRAFT_READ_MS:
+                parts.append(
+                    f"kept_default_a_after_reading_b={round(other_ms / 1000, 1)}s"
+                )
+                show_unchosen = True
+            else:
+                parts.append("second_draft_not_read")
     if e.get("reject_category"):
         parts.append(f"category={e['reject_category']}")
     elif e.get("decision") == "rejected" and not note:
@@ -147,6 +198,13 @@ def _event_line(e: dict) -> str:
         line += f"\n  user REWROTE it to: {draft[:300]}"
     elif draft:
         line += f"\n  our draft was: {draft[:200]}"
+    if dc and show_unchosen:
+        line += f"\n  the draft they did NOT pick was: {(dc.get('unchosen_text') or '')[:300]}"
+        if dc.get("style") or dc.get("unchosen_style"):
+            line += (
+                f"\n  styles: picked={dc.get('style') or '?'}"
+                f" not_picked={dc.get('unchosen_style') or '?'}"
+            )
     url = (e.get("thread_url") or "").strip()
     if url:
         line += f"\n  thread: {url}"
@@ -205,6 +263,8 @@ NEW REVIEW EVENTS since the last digest ({len(rejected)} rejected, {len(no_reaso
 
 Categories: wrong_author = the thread's author/audience was a bad fit; off_topic = the thread itself was a bad fit; bad_draft = thread was fine but the written reply was off; other = see the note. "no_reason_given" means the user rejected without picking a category or typing a note: the rejection itself is real, but WHY is your inference from the author/thread/draft context alone, so treat it as weak evidence. It can corroborate a pattern that reasoned events already show, but a no_reason_given reject never justifies a new entry or an author block on its own, and 2+ of them agreeing still only justify an entry when the shared pattern in their context is unmistakable. "edited_before_approving" with an ORIGINAL/REWROTE pair means the user hand-corrected our draft before posting: the rewrite is a direct statement of the voice they want. Diff the pair; when 2+ edits show the same correction (a phrase type removed, a structure replaced, tone shifted, length cut), distill that recurring pattern into draft_style_notes. Ignore edit content that is lead-specific or cosmetic (typo fixes, one-off facts); learn only what generalizes. "user_checked=profile_click" means the user opened the author's profile before deciding (a strong author-quality signal even without a note). "[approved+loved]" means the user picked the heart in the approve row ("this was a really good one"; approve_level_N in interactions carries the strength, 2 = best of the best): strong positive evidence for audience_prefer and thread selection, worth roughly two plain approvals.
 
+Two-draft cards show a "did NOT pick" pair. "picked_draft_b_over_default_a" means the card offered two drafts with A preselected and the user deliberately clicked into B and approved it: a direct head-to-head preference for the picked draft over the shown alternative, evidence on par with a hand rewrite. "chose_default_a_after_trying_b" means they clicked into B (trying it as the selection) and then came back and approved A: equally explicit, the same head-to-head strength as a switch, just with the default as the winner. "kept_default_a_after_reading_b=Xs" means they kept the preselected A but spent Xs with the pointer over B first: an informed keep, weaker than either explicit choice (reading B does not prove they weighed it; treat like no_reason_given, corroborating a pattern that stronger events already show rather than founding one). "second_draft_not_read" means they approved the default without touching or reading the alternative: NO pairwise signal, never infer anything against the unread draft. When 2+ pairwise events agree, diff the picked texts against the not-picked ones and distill WHAT recurs (tone, structure, length, opener type, directness) into draft_style_notes; the "styles:" line names each side's engagement style, useful when the same style keeps winning or losing.
+
 You can also block SPECIFIC authors via the plan's block_authors list. A block is a permanent hard exclusion of that one handle from all future thread selection, so it is YOUR judgment call, never automatic. Block when the evidence is strong: a wrong_author reject IS a direct human statement about that author (especially with profile_click), and the author context (author_followers, their post, found_via_topic) or the user's note confirms the account itself was the problem rather than the topic. Do NOT block when the reject looks topic-driven (off_topic/bad_draft on a reasonable account) or when you are unsure; the generalizable TYPE entry in audience_avoid is the softer tool for that.
 
 Propose changes to the block. RULES, in priority order:
@@ -333,6 +393,22 @@ def _is_actionable(e: dict) -> bool:
     # as actionable as a reject (and it feeds edit_examples).
     if e.get("edited"):
         return True
+    # Any pairwise draft signal triggers the digest (2026-07-10 user rule,
+    # widened same evening): an explicit choice (switched to B, or tried B
+    # and came back to A) is style evidence on par with an edit, and even a
+    # hover-read of the other draft (>= DRAFT_READ_MS) is a small but real
+    # signal that must flow through the pipeline rather than bank forever
+    # behind the plain-approvals gate. The PROMPT still weights the tiers
+    # (explicit strong, hover-read weak/corroborating); this gate only
+    # decides when a Claude turn is worth burning. A zero-interaction
+    # approve (second_draft_not_read) remains non-actionable: no signal.
+    dc = _draft_choice(e)
+    if dc and (
+        not dc.get("auto_selected")
+        or dc.get("visited_other")
+        or (dc.get("hover_b_ms") or 0) >= DRAFT_READ_MS
+    ):
+        return True
     return bool((e.get("reject_note") or "").strip())
 
 

package/scripts/generate_daily_human_style.py

@@ -193,11 +193,23 @@ def build_prompt(platform, replies, reserved_names):
     lines.append("")
     lines.append(
         "These replies all WON the thread (top of the conversation by likes). "
-        "Find the shared pattern that makes them work — the rhetorical move, "
+        "Find the shared pattern that makes them work: the rhetorical move, "
         "the structural shape, the relationship to the OP. Most winners "
         "share ONE pattern; that pattern is your new engagement style."
     )
     lines.append("")
+    lines.append(
+        "PRESERVE THE STRUCTURAL FINGERPRINT. Do not smooth the pattern "
+        "into a generic one-liner ('add a thoughtful counterpoint'); keep "
+        "what the winning replies actually DO on the page: how the first "
+        "words enter (lowercase noun, a number, the question itself, a "
+        "quoted phrase), how many sentences and of what shape (one clipped "
+        "line, two short + one long, a fragment), where the punch sits, "
+        "and what the reply refuses to do (no greeting, no hedge, no "
+        "summary). The char counts shown per reply are part of the "
+        "fingerprint; notice where the winners cluster."
+    )
+    lines.append("")
     lines.append(
         "Ignore: replies that win because of follower count, fame, or "
         "non-repeatable luck. Focus on the structural move that we (a "
@@ -209,14 +221,15 @@ def build_prompt(platform, replies, reserved_names):
     )
     lines.append("")
     for i, r in enumerate(replies, 1):
+        content = r.get("reply_content") or ""
         lines.append(
             f"### #{i} (likes={r['likes']}, replies={r['replies_count']}, "
-            f"rt={r['retweets']})"
+            f"rt={r['retweets']}, chars={len(content.strip())})"
         )
         lines.append(f"Thread: {r['thread_url']}")
         handle = r.get("reply_author_handle") or "(unknown)"
         lines.append(f"Reply by @{handle}:")
-        lines.append(f"> {r['reply_content']}")
+        lines.append(f"> {content}")
         lines.append("")
 
     lines.append("## Schema (match exactly)")
@@ -230,7 +243,12 @@ def build_prompt(platform, replies, reserved_names):
     lines.append("```")
     lines.append("{")
     lines.append('  "name": "<snake_case_name>",')
-    lines.append('  "description": "<one to three sentences describing the style>",')
+    lines.append(
+        '  "description": "<one to three sentences carrying the structural '
+        "fingerprint: the DEFINING MOVE (the one thing every draft in this "
+        "style must contain), the OPENING (how the first words enter), and "
+        'the sentence shape>",'
+    )
     lines.append('  "example": "<one short OP + reply pair demonstrating the style>",')
     lines.append('  "best_in": {')
     lines.append(f'    "{platform}": ["<short context label>", ...],')
@@ -255,8 +273,11 @@ def build_prompt(platform, replies, reserved_names):
         "MOVE (e.g. `mirror_and_extend`, `flip_to_alt`, not `good_reply`)."
     )
     lines.append(
-        "3. The description should make the style copyable: a future model "
-        "reading just that one sentence should know what to write."
+        "3. The description must make the style copyable AS A FORM: a future "
+        "model reading only the description must know the defining move, how "
+        "to open, and what sentence shape to write. 'Add a sharp "
+        "counterpoint' fails this test; 'one clipped sentence, no greeting, "
+        "opens with the concrete number the OP left out' passes."
     )
     lines.append(
         "4. The example should be a realistic OP + reply pair, not lifted "
@@ -270,7 +291,10 @@ def build_prompt(platform, replies, reserved_names):
     lines.append(
         "6. NEVER propose a style about including a product, a URL, or a "
         "mechanism. Our link-tail layer handles that downstream. The style "
-        "is about the text BEFORE the link."
+        "is about the text BEFORE the link. The note MAY state in one "
+        "clause how a plain product mention would enter this style if ever "
+        "(e.g. 'product name fits as the concrete example slot; never in "
+        "the opening'), but no URLs or link mechanics."
     )
     return "\n".join(lines)