@m13v/s4l 1.6.197-rc.10

package/scripts/seed_search_queries.py

@@ -0,0 +1,453 @@
+#!/usr/bin/env python3
+"""seed_search_queries.py — convert a project's seeded search TOPICS into a
+cold-start QUERY bank (>=30 real X advanced-search strings) at setup time.
+
+Why
+---
+The Twitter cycle's deterministic Phase 1 (scripts/qualified_query_bank.py)
+replays a project's historically *qualified* queries — distinct X search
+strings that already produced an engaged post. A brand-new project has zero
+post history, so that bank is empty and the cycle falls back to ONE crude
+query (the single picked topic + "-filter:replies"). That's the "only one
+search query" cold-start symptom the user hit on chosenhq.
+
+This script fixes the supply side: it reads the project's ACTIVE topics from
+project_search_topics, fans each topic out into several distinct X queries via
+the SAME Claude drafting prompt invent_topics.py uses (reused, not duplicated),
+optionally supply-tests them against the live browser harness, and persists the
+survivors into project_search_queries with source='seed'. The bank backfills
+from these active rows when the proven+invented set is still thin, so a fresh
+project runs ~30 queries on day one and the seed rows fade as real winners
+accumulate.
+
+Reuse
+-----
+All drafting / parsing / dedup / supply-test logic is imported from
+invent_topics.py (build_query_prompt, extract_queries, call_claude,
+normalize_query, load_existing_query_cores, dedup_queries, supply_test,
+harness_alive). This file is only the orchestration + persistence layer.
+
+Topics + queries are read/written through the website API (/api/v1/*) per the
+"no direct SQL in pipeline Python" rule.
+
+CLI:
+    python3 scripts/seed_search_queries.py --project chosenhq
+    python3 scripts/seed_search_queries.py --project fazm --target 30
+    python3 scripts/seed_search_queries.py --project fazm --supply-test off
+    python3 scripts/seed_search_queries.py --project fazm --dry-run
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from http_api import api_get, api_post  # noqa: E402
+from pick_project import load_config  # noqa: E402
+from invent_topics import (  # noqa: E402
+    CDP_PORT,
+    FRESHNESS_HOURS,
+    build_query_prompt,
+    call_claude,
+    dedup_queries,
+    extract_queries,
+    harness_alive,
+    load_existing_query_cores,
+    normalize_query,
+    supply_test,
+)
+
+# How many real queries we want a fresh project to launch with. The user's
+# target: "convert these search topics into at least 30 search queries
+# altogether". 30 is enough to fan a typical 6-12 topic universe across
+# multiple angles without exploding setup cost.
+DEFAULT_TARGET = 30
+
+# Per-topic draft cap so a project with very few topics doesn't ask Claude for
+# 30 queries off one topic (they'd collapse into near-dupes). With >=4 topics
+# the ceil(target/topics) math stays under this anyway.
+MAX_PER_TOPIC = 8
+MIN_PER_TOPIC = 2
+
+# Query-draft Claude call timeout (one call per topic).
+DRAFT_TIMEOUT_SEC = 240
+
+
+def _load_active_topics(project: str) -> list[str]:
+    """Active topics for a project from project_search_topics (DB universe)."""
+    resp = api_get(
+        "/api/v1/project-search-topics",
+        {"project": project, "status": "active"},
+    )
+    data = (resp or {}).get("data") or {}
+    rows = data.get("topics") or []
+    out, seen = [], set()
+    for r in rows:
+        t = (r.get("topic") or "").strip()
+        if t and t.lower() not in seen:
+            seen.add(t.lower())
+            out.append(t)
+    return out
+
+
+def _load_existing_seed_cores(project: str) -> set[str]:
+    """Normalized cores of seed queries already persisted for this project, so
+    re-running the seeder is idempotent and never double-drafts the same core."""
+    try:
+        resp = api_get(
+            "/api/v1/project-search-queries",
+            {"project": project, "status": "all"},
+        )
+    except SystemExit as exc:
+        print(f"[seed_search_queries] existing-seed read failed for "
+              f"{project!r}: {exc} (proceeding without seed dedup)",
+              file=sys.stderr)
+        return set()
+    data = (resp or {}).get("data") or {}
+    rows = data.get("queries") or []
+    return {normalize_query(r.get("query") or "") for r in rows if r.get("query")}
+
+
+def _fetch_active_queries(project: str) -> list[dict]:
+    """The project's current ACTIVE seed-query bank (query + topic), so a caller
+    can show the user exactly what the cycle will fan out over. Best-effort:
+    returns [] on read failure."""
+    try:
+        resp = api_get(
+            "/api/v1/project-search-queries",
+            {"project": project, "status": "active"},
+        )
+    except SystemExit:
+        return []
+    data = (resp or {}).get("data") or {}
+    out = []
+    for r in (data.get("queries") or []):
+        q = (r.get("query") or "").strip()
+        if q:
+            out.append({"query": q, "topic": (r.get("topic") or "").strip()})
+    return out
+
+
+def _load_provided_queries(path: str) -> list[tuple[str, str]]:
+    """Read AGENT-SUPPLIED queries from a JSON file: returns (query, topic) pairs.
+
+    Accepts either {"queries": [...]} or a bare top-level list. Each item may be
+    a string (topic left blank) or an object {"query": ..., "topic": ...}.
+    This is the claude-free seed path: the in-session agent already expanded the
+    topics into queries, so there is nothing to draft."""
+    with open(path, "r", encoding="utf-8") as fh:
+        raw = json.load(fh)
+    items = raw.get("queries") if isinstance(raw, dict) else raw
+    out: list[tuple[str, str]] = []
+    seen: set[str] = set()
+    for it in (items or []):
+        if isinstance(it, str):
+            q, t = it.strip(), ""
+        elif isinstance(it, dict):
+            q, t = (it.get("query") or "").strip(), (it.get("topic") or "").strip()
+        else:
+            continue
+        if not q:
+            continue
+        core = normalize_query(q)
+        if core in seen:
+            continue
+        seen.add(core)
+        out.append((q, t))
+    return out
+
+
+def _find_project(cfg: dict, name: str) -> dict | None:
+    for p in cfg.get("projects", []):
+        if (p.get("name") or "").strip().lower() == name.strip().lower():
+            return p
+    return None
+
+
+def _persist(project: str, query: str, topic: str,
+             supply_tested: bool, tweets_found, dry_run: bool) -> str:
+    """POST one seed query; returns the action ('inserted'/'updated'/'fail')."""
+    if dry_run:
+        return "dry"
+    try:
+        resp = api_post(
+            "/api/v1/project-search-queries",
+            body={
+                "project": project,
+                "query": query,
+                "topic": topic,
+                "source": "seed",
+                "status": "active",
+                "supply_tested": supply_tested,
+                "tweets_found": tweets_found,
+            },
+        )
+    except SystemExit as e:
+        print(f"[FAIL] {project}: {query!r}: {e}", file=sys.stderr)
+        return "fail"
+    data = (resp or {}).get("data") or resp or {}
+    return data.get("action") or "unknown"
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--project", required=True,
+                    help="Project name (config.json casing).")
+    ap.add_argument("--target", type=int, default=DEFAULT_TARGET,
+                    help=f"Total queries to aim for (default {DEFAULT_TARGET}).")
+    ap.add_argument("--supply-test", choices=["auto", "on", "off"],
+                    default="auto",
+                    help="auto (default): supply-test only if the harness is "
+                         "up; on: require it; off: skip and seed all drafts.")
+    ap.add_argument("--dry-run", action="store_true",
+                    help="Draft + (maybe) supply-test, but do NOT persist.")
+    ap.add_argument("--emit-json", action="store_true",
+                    help="After seeding, print the project's full ACTIVE seed-query "
+                         "bank as JSON on a sentinel line (===QUERIES_JSON===) so a "
+                         "caller (e.g. the MCP setup tool) can hand the queries back "
+                         "to the user.")
+    ap.add_argument("--queries-json",
+                    help="Path to a JSON file of AGENT-SUPPLIED queries to seed "
+                         "directly, bypassing the `claude -p` drafting loop entirely. "
+                         "Shape: {\"queries\":[{\"query\":\"...\",\"topic\":\"...\"}]} "
+                         "or a bare list of strings. Use this from the in-session MCP "
+                         "setup path so the seed step never depends on the claude CLI.")
+    args = ap.parse_args()
+
+    cfg = load_config()
+    project_entry = _find_project(cfg, args.project)
+    if not project_entry:
+        print(f"seed_search_queries: project {args.project!r} not in config.json",
+              file=sys.stderr)
+        return 2
+    # canonical name as stored
+    project = (project_entry.get("name") or args.project).strip()
+
+    target = max(1, args.target)
+
+    # ---- Agent-supplied path (claude-free) ------------------------------
+    # When the caller (the in-session MCP setup tool) hands us queries the agent
+    # already expanded from the topics, we skip the `claude -p` drafting loop
+    # entirely and just dedup + (maybe) supply-test + persist them. This is the
+    # single setup path on machines without the claude CLI. (2026-06-19)
+    if args.queries_json:
+        topics = _load_active_topics(project)  # best-effort, summary only
+        provided = _load_provided_queries(args.queries_json)
+        existing = (load_existing_query_cores(project)
+                    | _load_existing_seed_cores(project))
+        drafted = [(q, t) for q, t in provided
+                   if normalize_query(q) not in existing]
+        active_now = len(_fetch_active_queries(project))
+        print(f"seed_search_queries: project={project!r} mode=agent-supplied "
+              f"provided={len(provided)} new={len(drafted)} "
+              f"active_now={active_now} (no claude)", file=sys.stderr)
+        if not drafted:
+            # Everything provided was already seeded (or nothing usable) —
+            # healthy no-op: re-emit the live bank and succeed.
+            print(
+                f"seed_search_queries: project={project} topics={len(topics)} "
+                f"drafted=0 supply_ran=0 dropped_zero=0 seeded=0 inserted=0 "
+                f"updated=0 failed=0 (agent_supplied_nothing_new)"
+            )
+            if args.emit_json:
+                queries = _fetch_active_queries(project)
+                print("===QUERIES_JSON===")
+                print(json.dumps({"project": project, "count": len(queries),
+                                  "queries": queries}))
+            return 0
+    else:
+        topics = _load_active_topics(project)
+        if not topics:
+            print(f"seed_search_queries: no active topics for {project!r} — seed "
+                  f"topics first (scripts/seed_search_topics.py).", file=sys.stderr)
+            return 3
+
+        # Idempotency: aim for `target` TOTAL active queries, not `target` NEW per
+        # run. A project that already has >= target active seed queries needs no
+        # drafting — re-running reseed just re-emits the existing bank instead of
+        # ballooning it. Otherwise we only draft the shortfall. (2026-06-04)
+        active_now = len(_fetch_active_queries(project))
+        need = max(0, target - active_now)
+        per_topic = math.ceil(need / len(topics)) if need else 0
+        per_topic = max(MIN_PER_TOPIC, min(MAX_PER_TOPIC, per_topic)) if need else 0
+
+        # Dedup against (a) every query EVER attempted for this project and (b) seed
+        # queries already persisted. Accumulates as we go so topics don't overlap.
+        avoid_cores = load_existing_query_cores(project) | _load_existing_seed_cores(project)
+
+        print(f"seed_search_queries: project={project!r} topics={len(topics)} "
+              f"target={target} active_now={active_now} need={need} "
+              f"per_topic={per_topic} existing_cores={len(avoid_cores)}",
+              file=sys.stderr)
+
+        # --- Draft queries topic by topic ------------------------------------
+        drafted = []  # (query, topic)
+        for topic in topics:
+            if need <= 0 or len({normalize_query(q) for q, _ in drafted}) >= need:
+                break
+            prompt = build_query_prompt(
+                project_entry, topic, per_topic, avoid_queries=avoid_cores
+            )
+            try:
+                out = call_claude(prompt, timeout_sec=DRAFT_TIMEOUT_SEC)
+            except SystemExit as e:
+                print(f"[seed_search_queries] draft failed for topic {topic!r}: {e}",
+                      file=sys.stderr)
+                continue
+            qs = extract_queries(out, per_topic)
+            new_qs, dupes = dedup_queries(qs, avoid_cores)
+            for q in new_qs:
+                avoid_cores.add(normalize_query(q))
+                drafted.append((q, topic))
+            print(f"  topic={topic!r}: drafted={len(qs)} new={len(new_qs)} "
+                  f"dupes={len(dupes)}", file=sys.stderr)
+
+    if not drafted:
+        # Two cases: (a) the bank already meets target (need==0) — that's a
+        # healthy no-op, re-emit the existing bank and succeed; (b) we wanted
+        # queries but drafting produced none (every draft failed/duped) — that's
+        # a real failure.
+        if need <= 0:
+            print(f"seed_search_queries: project={project} already has "
+                  f"{active_now} active queries (>= target {target}); nothing to "
+                  f"draft.", file=sys.stderr)
+            print(
+                f"seed_search_queries: project={project} topics={len(topics)} "
+                f"drafted=0 supply_ran=0 dropped_zero=0 seeded=0 inserted=0 "
+                f"updated=0 failed=0 (already_full)"
+            )
+            if args.emit_json:
+                queries = _fetch_active_queries(project)
+                print("===QUERIES_JSON===")
+                print(json.dumps({"project": project, "count": len(queries),
+                                  "queries": queries}))
+            return 0
+        print("seed_search_queries: drafted 0 queries — nothing to seed.",
+              file=sys.stderr)
+        return 1
+
+    # --- Decide whether to supply-test -----------------------------------
+    do_supply = False
+    if args.supply_test == "on":
+        do_supply = True
+    elif args.supply_test == "auto":
+        do_supply = harness_alive(CDP_PORT)
+    if args.supply_test == "on" and not harness_alive(CDP_PORT):
+        print("seed_search_queries: --supply-test on but harness not reachable "
+              f"on :{CDP_PORT}", file=sys.stderr)
+        return 4
+
+    supply_map: dict[str, int] = {}
+    supply_ran = False
+    if do_supply:
+        # Group by topic so supply_test gets a coherent (topic, queries) batch.
+        by_topic: dict[str, list[str]] = {}
+        for q, t in drafted:
+            by_topic.setdefault(t, []).append(q)
+        for t, qs in by_topic.items():
+            tested, results = supply_test(project, t, qs,
+                                          freshness_hours=FRESHNESS_HOURS)
+            if not tested:
+                # Lock timeout / browser down: do NOT treat as zero supply.
+                print(f"  supply-test topic={t!r}: not tested (harness "
+                      f"unavailable) — keeping queries untested",
+                      file=sys.stderr)
+                continue
+            supply_ran = True
+            for r in results:
+                supply_map[normalize_query(r.get("query") or "")] = int(
+                    r.get("tweets_found") or 0
+                )
+
+    # --- Persist ----------------------------------------------------------
+    # When supply ran, drop zero-supply queries (they surfaced nothing fresh),
+    # but never let the drop take us below MIN_KEEP — a thin real bank beats an
+    # empty one for cold start. When supply did NOT run, seed everything
+    # untested (the bank still beats the 1-query fallback).
+    MIN_KEEP = max(1, target // 2)
+    to_seed: list[tuple[str, str, bool, object]] = []
+    dropped_zero = 0
+    for q, t in drafted:
+        core = normalize_query(q)
+        tested = supply_ran and core in supply_map
+        tw = supply_map.get(core) if tested else None
+        if supply_ran and tested and tw == 0:
+            dropped_zero += 1
+            continue
+        to_seed.append((q, t, bool(tested), tw))
+
+    if supply_ran and len(to_seed) < MIN_KEEP:
+        # Too aggressive — restore the highest-supply zeros... actually all were
+        # zero, so restore drafted order until MIN_KEEP. Keep them as tested=0.
+        restored = 0
+        have = {normalize_query(q) for q, _, _, _ in to_seed}
+        for q, t in drafted:
+            if restored and len(to_seed) >= MIN_KEEP:
+                break
+            core = normalize_query(q)
+            if core in have:
+                continue
+            to_seed.append((q, t, True, supply_map.get(core, 0)))
+            have.add(core)
+            restored += 1
+        print(f"  supply-test dropped too many; restored {restored} to meet "
+              f"MIN_KEEP={MIN_KEEP}", file=sys.stderr)
+
+    inserted = updated = failed = 0
+    for q, t, tested, tw in to_seed:
+        action = _persist(project, q, t, tested, tw, args.dry_run)
+        if action == "inserted":
+            inserted += 1
+        elif action == "updated":
+            updated += 1
+        elif action == "fail":
+            failed += 1
+
+    # Machine-parseable summary line (consumed by mcp/src/index.ts setup hook).
+    print(
+        f"seed_search_queries: project={project} topics={len(topics)} "
+        f"drafted={len(drafted)} supply_ran={int(supply_ran)} "
+        f"dropped_zero={dropped_zero} seeded={inserted + updated} "
+        f"inserted={inserted} updated={updated} failed={failed}"
+        + (" [dry-run]" if args.dry_run else "")
+    )
+
+    # Hand the resulting bank back to the caller (MCP setup tool) so it can show
+    # the user exactly which queries the cycle will run. Sentinel-delimited so it
+    # survives alongside the human/stderr log noise. On --dry-run we report what
+    # we drafted (nothing persisted yet); otherwise the live active bank.
+    if args.emit_json:
+        if args.dry_run:
+            queries = [{"query": q, "topic": t} for q, t in drafted]
+        else:
+            queries = _fetch_active_queries(project)
+        print("===QUERIES_JSON===")
+        print(json.dumps({"project": project, "count": len(queries),
+                          "queries": queries}))
+
+    return 1 if (failed and not (inserted or updated)) else 0
+
+
+if __name__ == "__main__":
+    try:
+        _rc = main()
+    except BrokenPipeError:
+        # The MCP setup hook (our parent) closes stdout once it has read the
+        # sentinel ===QUERIES_JSON=== block; the trailing summary prints then hit
+        # a dead pipe and raise BrokenPipeError. All persistence already happened
+        # earlier in main(), so this is BENIGN. Previously it propagated as an
+        # uncaught exception and Sentry logged it as a "seeding failed" event
+        # (Karol, 2026-06-22) — a false positive that buried the real signal.
+        # Redirect stdout to devnull so interpreter shutdown doesn't re-raise on
+        # the final flush, then exit clean.
+        try:
+            _devnull = os.open(os.devnull, os.O_WRONLY)
+            os.dup2(_devnull, sys.stdout.fileno())
+        except Exception:
+            pass
+        _rc = 0
+    raise SystemExit(_rc)

package/scripts/seed_search_topics.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""Seed project_search_topics from config.json (one-time-per-install bootstrap).
+
+The DB is now the source of truth for the search-topic universe (see migration
+2026-05-27-project-search-topics.sql and pick_search_topic.py). This script
+mirrors the project_name -> search_topics[] block in ~/social-autoposter/config.json
+into /api/v1/project-search-topics with source='seed', status='active'. The API
+upserts on (install_id, project_name, topic), so re-running this script:
+
+  - Inserts rows added to config.json since the last seed.
+  - Leaves source unchanged for rows that already exist (server preserves
+    source on UPDATE; status is touched only because of upsert semantics, but
+    it lands on the same 'active' default we send).
+  - Never deletes rows (paused/excluded topics in the DB are protected from
+    config.json drift; explicit pause/exclude lives in the dashboard, not here).
+
+CLI:
+    python3 scripts/seed_search_topics.py            # seed every project
+    python3 scripts/seed_search_topics.py --project fazm
+    python3 scripts/seed_search_topics.py --dry-run  # show counts, don't POST
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from http_api import api_post  # noqa: E402
+
+CONFIG_PATH = os.path.expanduser("~/social-autoposter/config.json")
+
+
+def _load_projects(only_project: str | None = None):
+    with open(CONFIG_PATH) as f:
+        cfg = json.load(f)
+    out = []
+    for p in cfg.get("projects", []):
+        name = (p.get("name") or "").strip()
+        if not name:
+            continue
+        if only_project and name.lower() != only_project.lower():
+            continue
+        topics = []
+        seen = set()
+        for t in (p.get("search_topics") or []):
+            t = (t or "").strip()
+            if t and t not in seen:
+                seen.add(t)
+                topics.append(t)
+        out.append((name, topics))
+    return out
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--project", default=None,
+                    help="Only seed this project name (default: all)")
+    ap.add_argument("--dry-run", action="store_true",
+                    help="Print counts; do not POST")
+    args = ap.parse_args()
+
+    projects = _load_projects(args.project)
+    if not projects:
+        sys.stderr.write(
+            f"seed_search_topics: no projects matched "
+            f"(filter={args.project!r})\n"
+        )
+        sys.exit(2)
+
+    total_inserted = 0
+    total_updated = 0
+    total_failed = 0
+    total_planned = 0
+
+    for name, topics in projects:
+        total_planned += len(topics)
+        if args.dry_run:
+            print(f"[dry] {name}: {len(topics)} topics")
+            continue
+        inserted = 0
+        updated = 0
+        for topic in topics:
+            try:
+                resp = api_post(
+                    "/api/v1/project-search-topics",
+                    body={
+                        "project": name,
+                        "topic": topic,
+                        "source": "seed",
+                        "status": "active",
+                    },
+                )
+            except SystemExit as e:
+                total_failed += 1
+                print(f"[FAIL] {name}: {topic!r}: {e}", file=sys.stderr)
+                continue
+            data = (resp or {}).get("data") or resp or {}
+            action = data.get("action") or ""
+            if action == "inserted":
+                inserted += 1
+            elif action == "updated":
+                updated += 1
+        total_inserted += inserted
+        total_updated += updated
+        print(
+            f"{name}: planned={len(topics)} inserted={inserted} "
+            f"updated={updated}"
+        )
+
+    if args.dry_run:
+        print(f"[dry] total topics across {len(projects)} project(s): {total_planned}")
+        return
+
+    print(
+        f"\ndone. projects={len(projects)} planned={total_planned} "
+        f"inserted={total_inserted} updated={total_updated} "
+        f"failed={total_failed}"
+    )
+    if total_failed:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()