@m13v/s4l 1.6.197-rc.7

package/scripts/top_search_topics.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""Return top-performing search_topic seeds per project + platform.
+
+This is the Reddit + GitHub feedback feed; the Twitter analog lives in
+`scripts/top_twitter_queries.py`. As of 2026-05-10 the Reddit path is at
+parity with the Twitter feed: it reads `reddit_candidates` instead of
+`posts`, surfaces the full conversion funnel (posted/skipped sample
+sizes, posted/skipped delta_score split, upvotes/comments/clicks),
+and ranks by clicks first.
+
+What changed (2026-05-10): the previous version scored only
+`comments_count*3 + upvotes` from the `posts` table, missed clicks
+entirely, had no posted-vs-skipped split, and could not tell the model
+"this query keeps surfacing viral threads we keep skipping" — i.e. a
+mismatch signal. Twitter has had that signal since the Phase 0 batch
+salvage rebuild; Reddit was flying blind on which subreddits/queries
+actually convert to clicks.
+
+Reddit path (platform='reddit'):
+  Source = reddit_candidates (one row per discovered thread, status in
+  pending/posted/skipped/expired/failed). Joins posts via post_id for
+  upvotes/comments_count and post_links via post_id for real_clicks
+  (clicks are only meaningful on posted rows; the FILTER clauses gate
+  to status='posted' inside the SUM).
+
+  Fields surfaced per (search_topic, project):
+    posts                    — distinct posted candidates
+    posted_n                 — count(*) FILTER (status='posted')
+    skipped_n                — count(*) FILTER (status IN ('skipped','expired','failed'))
+    avg_delta_posted         — avg reddit_candidates.delta_score for posted rows
+    avg_delta_skipped        — avg reddit_candidates.delta_score for skipped/expired/failed rows
+    upvotes_total            — sum upvotes on our replies (posted only)
+    comments_total           — sum comments_count on our replies (posted only)
+    clicks_total             — sum post_links.real_clicks on our replies (posted only)
+    composite_score          — clicks*100 + comments + upvotes (clicks dominate)
+
+  delta_score is reddit's velocity proxy (Δup + 4*Δcomments computed
+  during the T1 ripen step in ripen_reddit_plan.py). It is set on every
+  ripened row regardless of eventual status, which is what lets us
+  split the average by posted vs skipped — same diagnostic shape as
+  Twitter's avg_virality_posted / avg_virality_skipped:
+    high avg_delta_posted + many posts        → keep this query, mimic style
+    high avg_delta_skipped + few posts        → on-rank but off-topic, reword
+    low avg_delta_skipped + few posts         → dead supply, drop the seed
+
+Non-reddit path (platform='github' or unset):
+  Source = posts (search_topic stamped at INSERT time). Joins
+  post_links via posts.id for clicks_total. Same composite + clicks-DESC
+  ordering as the reddit path. Reddit-style status splits are not
+  available here because GitHub posts directly without a candidates
+  table.
+
+Usage:
+    python3 scripts/top_search_topics.py --project "fazm" --platform reddit
+    python3 scripts/top_search_topics.py --project "fazm" --platform github
+    python3 scripts/top_search_topics.py --project "fazm" --platform reddit --json
+"""
+import argparse
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from http_api import api_get  # noqa: E402
+
+
+def query(project=None, platform=None, window_days=30, limit=10):
+    """Top-performing search_topic seeds per (project, platform).
+
+    Migrated 2026-05-30 off direct DB (db.get_conn) onto the HTTP lane:
+    GET /api/v1/search-topics/ranked?platform=&project=&window_days=&limit=.
+    The route mirrors the four legacy `_query_*` SQL paths one-for-one
+    (reddit / twitter / linkedin / posts-fallback), including the 2026-05-29
+    cross-route guard on the twitter posted-conversion aggregates, and returns
+    rows already shaped as the dicts this function used to build, so we read
+    `data.rows` verbatim. There is intentionally NO direct-DB fallback.
+    """
+    q = {"window_days": int(window_days), "limit": int(limit)}
+    if platform:
+        q["platform"] = platform
+    if project:
+        q["project"] = project
+    resp = api_get("/api/v1/search-topics/ranked", q)
+    data = (resp or {}).get("data") or {}
+    return list(data.get("rows") or [])
+
+
+def format_text(results, project=None, platform=None, window_days=30):
+    plat = (platform or "").lower()
+    is_reddit = plat == "reddit"
+    is_twitter = plat == "twitter"
+    is_linkedin = plat == "linkedin"
+    if not results:
+        return (
+            f"(no search_topic data yet in the last {window_days}d"
+            + (f" for {project}" if project else "")
+            + (f" on {platform}" if platform else "")
+            + ")"
+        )
+    header = f"Top search_topic seeds (last {window_days}d"
+    if project:
+        header += f", project={project}"
+    if platform:
+        header += f", platform={platform}"
+    if is_reddit:
+        header += ", ranked by clicks_total DESC then composite (clicks×100 + comments + upvotes))"
+    elif is_twitter:
+        header += ", ranked by clicks_total DESC then composite (clicks×100 + likes + views×0.001))"
+    elif is_linkedin:
+        header += ", ranked by clicks_total DESC then composite (clicks×100 + likes + views×0.001 + velocity))"
+    else:
+        header += ", ranked by clicks_total DESC then composite (clicks×100 + comments×3 + upvotes))"
+    lines = [header]
+    if is_reddit:
+        lines.append(
+            f"  {'clicks':>6} {'comm':>5} {'upv':>5} "
+            f"{'posts':>5} {'pN':>3} {'sN':>3} "
+            f"{'Δpost':>6} {'Δskip':>6}  topic"
+        )
+        for r in results:
+            lines.append(
+                f"  {r['clicks_total']:>6} {r['comments_total']:>5} {r['upvotes_total']:>5} "
+                f"{r['posts']:>5} {r['posted_n']:>3} {r['skipped_n']:>3} "
+                f"{r['avg_delta_posted']:>6.1f} {r['avg_delta_skipped']:>6.1f}  {r['search_topic']}"
+            )
+        lines.append(
+            "  (Δpost = avg ripen delta_score on posted rows; "
+            "Δskip = avg ripen delta_score on skipped/expired/failed rows. "
+            "High Δskip + few posts = query is on-rank but off-topic — reword. "
+            "Low Δskip + few posts = dead supply, drop the seed.)"
+        )
+    elif is_twitter or is_linkedin:
+        lines.append(
+            f"  {'clicks':>6} {'views':>7} {'likes':>5} "
+            f"{'posts':>5} {'pN':>3} {'sN':>3} "
+            f"{'Vpost':>6} {'Vskip':>6}  topic"
+        )
+        for r in results:
+            lines.append(
+                f"  {r['clicks_total']:>6} {r['views_total']:>7} {r['likes_total']:>5} "
+                f"{r['posts']:>5} {r['posted_n']:>3} {r['skipped_n']:>3} "
+                f"{r['avg_virality_posted']:>6.1f} {r['avg_virality_skipped']:>6.1f}  {r['search_topic']}"
+            )
+        lines.append(
+            "  (Vpost = avg virality_score on posted rows; "
+            "Vskip = avg virality_score on skipped/expired/failed rows. "
+            "High Vskip + few posts = topic finds viral noise we keep skipping - reword. "
+            "Low Vskip + few posts = dead supply, drop the seed.)"
+        )
+    else:
+        lines.append(
+            f"  {'clicks':>6} {'comm':>5} {'upv':>5} {'posts':>5}  topic"
+        )
+        for r in results:
+            lines.append(
+                f"  {r['clicks_total']:>6} {r['comments_total']:>5} {r['upvotes_total']:>5} "
+                f"{r['posts']:>5}  {r['search_topic']}"
+            )
+    return "\n".join(lines)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--project", default=None)
+    ap.add_argument("--platform", default=None)
+    ap.add_argument("--window-days", type=int, default=30)
+    ap.add_argument("--limit", type=int, default=10)
+    ap.add_argument("--json", action="store_true", help="Output JSON instead of text")
+    args = ap.parse_args()
+
+    results = query(args.project, args.platform, args.window_days, args.limit)
+    if args.json:
+        json.dump(results, sys.stdout)
+        sys.stdout.write("\n")
+    else:
+        print(format_text(results, args.project, args.platform, args.window_days))
+
+
+if __name__ == "__main__":
+    main()

package/scripts/top_twitter_queries.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+"""
+top_twitter_queries.py
+
+Returns top-performing historical search queries scored by a composite of
+clicks, likes, views, posts produced, AND raw supply (tweets_found per
+attempt). Used as STYLE inspiration for the LLM that drafts new queries.
+
+Per-query fields, structured so the model can see the FULL conversion
+funnel AND distinguish "queries that find threads worth posting to" from
+"queries that find viral noise we keep skipping":
+
+    query                  , the literal X search string (with min_faves:N etc.)
+    project                , project the query was drafted for (matched_project)
+    tweets_found_avg       , SUPPLY: avg tweets X returned per attempt
+    posted_n               , count of candidates with status='posted'
+    skipped_n              , count of candidates with status IN ('skipped','expired')
+    post_rate              , posted_n / (posted_n + skipped_n); draft-gate acceptance ratio
+    avg_virality_posted    , avg source-thread virality_score for posted candidates
+    avg_virality_skipped   , avg source-thread virality_score for skipped/expired
+    views_total            , sum of views on OUR replies (downstream surface)
+    likes_total            , sum of likes on OUR replies
+    clicks_total           , sum of real_clicks attributed to our replies (CTA tracking)
+    composite_score        , clicks*100 + likes + views*0.001  (clicks dominate)
+
+The two virality fields together let the model diagnose query failure
+modes that pure conversion data misses:
+  - high avg_virality_posted + many posts → keep / mimic this query style
+  - high avg_virality_skipped + few posts → reword: query is on-rank but
+    semantically off-topic (e.g. studyly catching unrelated viral student
+    drama because keywords overlap with study-related slang)
+  - low avg_virality_skipped + few posts → query is just dead supply,
+    drop the keyword cluster entirely
+
+Source-thread virality_score is computed by score_twitter_candidates.py
+(engagement velocity + retweet ratio + reply weight + author followers,
+with 6h half-life decay). It's set on EVERY candidate at discovery time
+regardless of posted/skipped/expired status, which is why we can split
+the average by status group.
+
+Usage:
+
+    python3 scripts/top_twitter_queries.py [--limit 20] [--window-days 14] [--project NAME]
+
+The optional --project filter is what enables per-project surfacing in the
+Phase 1 scanner prompt: each cycle, the scanner can fetch the top queries
+specifically for the project it's currently drafting for.
+
+Migrated 2026-05-18: reads now go through /api/v1/twitter-search-attempts/
+top-queries via scripts/http_api.py instead of a direct psycopg2 query.
+The SQL composite-score join (cand_agg + supply_agg, click_total tiebreaker)
+runs server-side; this script just shapes the response into the legacy JSON.
+"""
+import argparse
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from http_api import api_get  # noqa: E402
+
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument("--limit", type=int, default=20)
+    p.add_argument("--window-days", type=int, default=14)
+    p.add_argument("--project", default=None,
+                   help="If set, only return top queries for this project (matched_project).")
+    args = p.parse_args()
+
+    query = {
+        "limit": args.limit,
+        "window_days": args.window_days,
+    }
+    if args.project:
+        query["project"] = args.project
+
+    resp = api_get("/api/v1/twitter-search-attempts/top-queries", query=query)
+    rows = (resp.get("data") or {}).get("rows") or []
+
+    # Pass-through shape, but float-coerce so the legacy JSON consumers
+    # (run-twitter-cycle.sh's Phase 1 prompt) see the same types as before.
+    # Derived field: post_rate = posted_n / (posted_n + skipped_n), the draft-gate
+    # acceptance ratio. Lets the model see whether a query that LOOKS productive
+    # by raw posts count actually clears the skip filter, or whether it surfaces
+    # 100 candidates and we reject 95 of them. Safe-divide on empty denominator.
+    # Dropped: 'posts' field (was identical to 'posted_n' in every observed row).
+    def _post_rate(posted: int, skipped: int) -> float:
+        denom = posted + skipped
+        if denom <= 0:
+            return 0.0
+        return round(posted / denom, 3)
+
+    # Derived field: posts_per_attempt = tweets_found_avg * post_rate.
+    # Algebraically equivalent to posted_n / attempts_n (the raw API doesn't
+    # expose attempts_n directly, but tweets_found_avg is tweets/attempt and
+    # post_rate is posted/(posted+skipped) so the product is posts/attempt).
+    # This is the headline efficiency number: how many posts does this query
+    # actually yield per Phase 1 search invocation? <0.1 means most attempts
+    # don't even produce one survivor.
+    def _posts_per_attempt(tweets_found_avg: float, post_rate: float) -> float:
+        return round(tweets_found_avg * post_rate, 3)
+
+    # Two-axis bucketing (2026-05-28). Drafter prompt uses these labels to
+    # decide whether to mimic, narrow, or broaden a past query's structure.
+    # Raw numbers stay in the payload; buckets are derived helpers that give
+    # the model a categorical hint instead of forcing it to threshold floats
+    # in-context every time.
+    #
+    # supply_bucket: raw tweet supply from X for this query phrasing.
+    #   low    = <1 tweets/attempt   (query is dying or freshness window too tight)
+    #   medium = 1-5 tweets/attempt  (healthy)
+    #   high   = >5 tweets/attempt   (lots of supply, often noisy)
+    #
+    # conversion_bucket: how often a found tweet survives the draft gate.
+    #   low    = <0.2 post_rate      (gate keeps rejecting; query is on-rank
+    #                                 but semantically off-target)
+    #   medium = 0.2-0.6 post_rate   (normal)
+    #   high   = >=0.6 post_rate     (high-fit query, mimic the structure)
+    def _supply_bucket(tweets_found_avg: float) -> str:
+        if tweets_found_avg < 1:
+            return "low"
+        if tweets_found_avg <= 5:
+            return "medium"
+        return "high"
+
+    def _conversion_bucket(post_rate: float) -> str:
+        if post_rate < 0.2:
+            return "low"
+        if post_rate < 0.6:
+            return "medium"
+        return "high"
+
+    # guidance: what should the drafter DO with this query as a reference?
+    #
+    #   BROADEN     — supply is dying. Shorten to 1-2 keywords, drop OR groups,
+    #                 step min_faves down a tier. Past query's OPERATORS are
+    #                 dead weight; only the topic-keyword is signal.
+    #   NARROW      — supply is abundant but conversion is bad. Past query
+    #                 fishes in a noisy pond. Add specificity (more OR terms,
+    #                 stricter min_faves, -term excludes) so the freshness
+    #                 window surfaces fewer, higher-fit candidates.
+    #   KEEP_STYLE  — middle of the road. Use the operator skeleton, swap
+    #                 keywords for the picker-assigned topic.
+    #   MIMIC       — gold tier. Reuse the full operator pattern verbatim,
+    #                 only swap the topic-keyword. This is what works.
+    def _guidance(supply_b: str, conv_b: str) -> str:
+        if supply_b == "low":
+            return "BROADEN"
+        if supply_b == "high" and conv_b == "low":
+            return "NARROW"
+        if conv_b == "high":
+            return "MIMIC"
+        if supply_b == "medium" and conv_b == "medium":
+            return "KEEP_STYLE"
+        # high supply + medium conversion, medium supply + low conversion
+        return "NARROW" if conv_b == "low" else "KEEP_STYLE"
+
+    out = []
+    for r in rows:
+        posted_n = int(r.get("posted_n") or 0)
+        skipped_n = int(r.get("skipped_n") or 0)
+        tweets_found_avg = float(r.get("tweets_found_avg") or 0)
+        post_rate = _post_rate(posted_n, skipped_n)
+        supply_b = _supply_bucket(tweets_found_avg)
+        conv_b = _conversion_bucket(post_rate)
+        out.append({
+            "query": r.get("query"),
+            "project": r.get("project") or "",
+            "posted_n": posted_n,
+            "skipped_n": skipped_n,
+            "post_rate": post_rate,
+            "posts_per_attempt": _posts_per_attempt(tweets_found_avg, post_rate),
+            "supply_bucket": supply_b,
+            "conversion_bucket": conv_b,
+            "guidance": _guidance(supply_b, conv_b),
+            "avg_virality_posted": round(float(r.get("avg_virality_posted") or 0), 2),
+            "avg_virality_skipped": round(float(r.get("avg_virality_skipped") or 0), 2),
+            "views_total": int(r.get("views_total") or 0),
+            "likes_total": int(r.get("likes_total") or 0),
+            "clicks_total": int(r.get("clicks_total") or 0),
+            "tweets_found_avg": tweets_found_avg,
+            "composite_score": round(float(r.get("composite_score") or 0), 2),
+        })
+    json.dump(out, sys.stdout)
+    print("", file=sys.stdout)
+
+
+if __name__ == "__main__":
+    main()