@m13v/s4l 1.6.197-rc.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (326) hide show
  1. package/README.md +143 -0
  2. package/SKILL.md +342 -0
  3. package/bin/cli.js +980 -0
  4. package/bin/cookie-helper.js +315 -0
  5. package/bin/platform.js +59 -0
  6. package/bin/scheduler/index.js +12 -0
  7. package/bin/scheduler/launchd.js +518 -0
  8. package/browser-agent-configs/all-agents-mcp.json +68 -0
  9. package/browser-agent-configs/linkedin-agent-mcp.json +16 -0
  10. package/browser-agent-configs/linkedin-agent.json +17 -0
  11. package/browser-agent-configs/linkedin-harness-mcp.json +21 -0
  12. package/browser-agent-configs/reddit-agent-mcp.json +16 -0
  13. package/browser-agent-configs/reddit-agent.json +17 -0
  14. package/browser-agent-configs/twitter-harness-mcp.json +18 -0
  15. package/config.example.json +45 -0
  16. package/mcp/dist/index.js +4212 -0
  17. package/mcp/dist/onboarding.js +200 -0
  18. package/mcp/dist/panel.html +176 -0
  19. package/mcp/dist/product-link.html +102 -0
  20. package/mcp/dist/repo.js +222 -0
  21. package/mcp/dist/runtime.js +1079 -0
  22. package/mcp/dist/screencast.js +323 -0
  23. package/mcp/dist/setup.js +545 -0
  24. package/mcp/dist/telemetry.js +306 -0
  25. package/mcp/dist/twitterAuth.js +138 -0
  26. package/mcp/dist/version.js +271 -0
  27. package/mcp/dist/version.json +4 -0
  28. package/mcp/install-runtime.mjs +70 -0
  29. package/mcp/install.mjs +169 -0
  30. package/mcp/manifest.json +80 -0
  31. package/mcp/menubar/dashboard_server.py +213 -0
  32. package/mcp/menubar/s4l_card.py +1336 -0
  33. package/mcp/menubar/s4l_log_relay.py +179 -0
  34. package/mcp/menubar/s4l_menubar.py +2439 -0
  35. package/mcp/menubar/s4l_state.py +891 -0
  36. package/mcp/package.json +34 -0
  37. package/mcp/shared/doctor.cjs +437 -0
  38. package/mcp/shared/onboarding-ledger.cjs +324 -0
  39. package/mcp-servers/browser-harness/server.py +968 -0
  40. package/package.json +160 -0
  41. package/requirements.txt +20 -0
  42. package/scripts/_compute_allowlist.py +58 -0
  43. package/scripts/_db_update.py +20 -0
  44. package/scripts/_filt.py +9 -0
  45. package/scripts/_li_notif_match.py +76 -0
  46. package/scripts/_li_notif_orchestrate.py +126 -0
  47. package/scripts/_lock_preempt_test.py +60 -0
  48. package/scripts/_run_icp_precheck.py +57 -0
  49. package/scripts/a16z_pearx_calendar_reminders.py +99 -0
  50. package/scripts/account_resolver.py +141 -0
  51. package/scripts/active_campaigns.py +114 -0
  52. package/scripts/active_users.py +190 -0
  53. package/scripts/amplitude_24h_signups.py +468 -0
  54. package/scripts/amplitude_signups.py +177 -0
  55. package/scripts/apply_onboarding_selections.py +131 -0
  56. package/scripts/audience_pages.py +243 -0
  57. package/scripts/audit_helper.py +120 -0
  58. package/scripts/author_history_block.py +353 -0
  59. package/scripts/autopilot_stall_watch.py +284 -0
  60. package/scripts/backfill_twitter_attempts_topic.py +81 -0
  61. package/scripts/backfill_twitter_log_post_no_id.py +322 -0
  62. package/scripts/bench_dashboard.sh +138 -0
  63. package/scripts/bh_send.py +39 -0
  64. package/scripts/build_persona.py +409 -0
  65. package/scripts/bulk_icp.py +18 -0
  66. package/scripts/campaign_bump.py +51 -0
  67. package/scripts/capture_thread_media.py +288 -0
  68. package/scripts/check_browser_lock_health.sh +81 -0
  69. package/scripts/check_external_pool_depth.py +253 -0
  70. package/scripts/check_unread_web_chats.py +28 -0
  71. package/scripts/claim_web_chat.py +47 -0
  72. package/scripts/classify_run_error.py +158 -0
  73. package/scripts/claude_job.py +988 -0
  74. package/scripts/clean_stale_singleton.sh +56 -0
  75. package/scripts/cleanup_harness_tabs.py +68 -0
  76. package/scripts/copy_browser_cookies.py +454 -0
  77. package/scripts/counterparty_history.py +350 -0
  78. package/scripts/db.py +57 -0
  79. package/scripts/discover_claude_profiles.py +120 -0
  80. package/scripts/discover_linkedin_candidates.py +984 -0
  81. package/scripts/dm_conversation.py +682 -0
  82. package/scripts/dm_db_update.py +69 -0
  83. package/scripts/dm_engage_helper.py +161 -0
  84. package/scripts/dm_outreach_helper.py +147 -0
  85. package/scripts/dm_outreach_twitter_helper.py +129 -0
  86. package/scripts/dm_send_log.py +106 -0
  87. package/scripts/dm_short_links.py +1084 -0
  88. package/scripts/dump_web_chat_history.py +47 -0
  89. package/scripts/engage_github.py +640 -0
  90. package/scripts/engage_reddit.py +1235 -0
  91. package/scripts/engage_twitter_helper.py +301 -0
  92. package/scripts/engagement_styles.py +1787 -0
  93. package/scripts/enrich_twitter_candidates.py +82 -0
  94. package/scripts/feedback_digest.py +448 -0
  95. package/scripts/fetch_prospect_profile.py +312 -0
  96. package/scripts/fetch_twitter_t1.py +134 -0
  97. package/scripts/find_threads.py +530 -0
  98. package/scripts/follow_gate_log.py +59 -0
  99. package/scripts/funnel_per_day.py +194 -0
  100. package/scripts/generate_daily_human_style.py +494 -0
  101. package/scripts/generation_trace.py +173 -0
  102. package/scripts/get_run_cost.py +107 -0
  103. package/scripts/github_engage_helper.py +93 -0
  104. package/scripts/github_tools.py +509 -0
  105. package/scripts/harness_overlay.py +556 -0
  106. package/scripts/harvest_twitter_following.py +243 -0
  107. package/scripts/heartbeat.sh +70 -0
  108. package/scripts/history_context.py +284 -0
  109. package/scripts/http_api.py +206 -0
  110. package/scripts/human_dm_replies_helper.py +169 -0
  111. package/scripts/identity.py +302 -0
  112. package/scripts/ig_batch_creator.sh +93 -0
  113. package/scripts/ig_post_type_picker.py +243 -0
  114. package/scripts/ig_scrape_transcribe.sh +91 -0
  115. package/scripts/ingest_human_dm_replies.py +271 -0
  116. package/scripts/ingest_web_chat_replies.py +229 -0
  117. package/scripts/install_fleet.py +187 -0
  118. package/scripts/invent_mcp_server.py +350 -0
  119. package/scripts/invent_topics.py +1462 -0
  120. package/scripts/learned_preferences.py +263 -0
  121. package/scripts/li_discovery.py +161 -0
  122. package/scripts/link_edit_helper.py +142 -0
  123. package/scripts/link_tail.py +592 -0
  124. package/scripts/linkedin_api.py +561 -0
  125. package/scripts/linkedin_browser.py +730 -0
  126. package/scripts/linkedin_cooldown.py +128 -0
  127. package/scripts/linkedin_exclusions.py +234 -0
  128. package/scripts/linkedin_killswitch.py +1333 -0
  129. package/scripts/linkedin_search_topic_schema.py +49 -0
  130. package/scripts/linkedin_unipile.py +658 -0
  131. package/scripts/linkedin_url.py +228 -0
  132. package/scripts/log_claude_session.py +636 -0
  133. package/scripts/log_draft.py +143 -0
  134. package/scripts/log_linkedin_search_attempts.py +126 -0
  135. package/scripts/log_post.py +651 -0
  136. package/scripts/log_run.py +364 -0
  137. package/scripts/log_thread_media.py +108 -0
  138. package/scripts/log_twitter_search_attempts.py +150 -0
  139. package/scripts/log_twitter_skips.py +211 -0
  140. package/scripts/lookup_post.py +78 -0
  141. package/scripts/mark_web_chat_processed.py +32 -0
  142. package/scripts/mcp_lock_proxy.py +370 -0
  143. package/scripts/memory_snapshot.py +972 -0
  144. package/scripts/merge_review_queue.py +215 -0
  145. package/scripts/mint_external_pool.py +182 -0
  146. package/scripts/mint_kent_pool.py +249 -0
  147. package/scripts/moltbook_post.py +320 -0
  148. package/scripts/moltbook_tools.py +159 -0
  149. package/scripts/pending_threads.py +188 -0
  150. package/scripts/pick_ig_account.py +177 -0
  151. package/scripts/pick_project.py +208 -0
  152. package/scripts/pick_search_topic.py +771 -0
  153. package/scripts/pick_thread_target.py +279 -0
  154. package/scripts/pick_twitter_thread_target.py +202 -0
  155. package/scripts/podlog_fetch_batch.sh +32 -0
  156. package/scripts/post_github.py +1311 -0
  157. package/scripts/post_reddit.py +2668 -0
  158. package/scripts/precompute_dashboard_stats.py +204 -0
  159. package/scripts/preflight.sh +297 -0
  160. package/scripts/progress.py +88 -0
  161. package/scripts/project_excludes.py +353 -0
  162. package/scripts/project_slugs.py +91 -0
  163. package/scripts/project_stats.py +241 -0
  164. package/scripts/project_stats_json.py +1563 -0
  165. package/scripts/project_topics.py +192 -0
  166. package/scripts/qualified_query_bank.py +436 -0
  167. package/scripts/reap_stale_claude_sessions.py +867 -0
  168. package/scripts/reddit_browser.py +2549 -0
  169. package/scripts/reddit_browser_fetch.py +141 -0
  170. package/scripts/reddit_browser_lock.py +593 -0
  171. package/scripts/reddit_chat_sync.py +710 -0
  172. package/scripts/reddit_query_bank.py +200 -0
  173. package/scripts/reddit_threads_helper.py +151 -0
  174. package/scripts/reddit_tools.py +956 -0
  175. package/scripts/refresh_instagram_tokens.py +280 -0
  176. package/scripts/release-mcpb.sh +513 -0
  177. package/scripts/reply_db.py +334 -0
  178. package/scripts/reply_insert.py +98 -0
  179. package/scripts/reply_risk_digest.py +761 -0
  180. package/scripts/reset-test-machine.sh +602 -0
  181. package/scripts/restore_twitter_session.py +177 -0
  182. package/scripts/ripen_reddit_plan.py +478 -0
  183. package/scripts/run_claude.sh +433 -0
  184. package/scripts/run_moltbook_cycle.py +555 -0
  185. package/scripts/s4l_box_update.sh +226 -0
  186. package/scripts/s4l_channel.py +103 -0
  187. package/scripts/s4l_ctl.sh +75 -0
  188. package/scripts/s4l_env.py +47 -0
  189. package/scripts/saps_activity.py +126 -0
  190. package/scripts/saps_mode.py +328 -0
  191. package/scripts/scan_dm_candidates.py +580 -0
  192. package/scripts/scan_github_replies.py +168 -0
  193. package/scripts/scan_instagram_comments.py +481 -0
  194. package/scripts/scan_moltbook_replies.py +252 -0
  195. package/scripts/scan_pii.py +190 -0
  196. package/scripts/scan_reddit_replies.py +377 -0
  197. package/scripts/scan_twitter_mentions_browser.py +327 -0
  198. package/scripts/scan_twitter_thread_followups.py +299 -0
  199. package/scripts/scan_x_profile.py +384 -0
  200. package/scripts/schedule_state.py +202 -0
  201. package/scripts/scheduled_tasks_snapshot.py +123 -0
  202. package/scripts/score_linkedin_candidates.py +419 -0
  203. package/scripts/score_twitter_candidates.py +718 -0
  204. package/scripts/scrape_linkedin_comment_stats.py +1755 -0
  205. package/scripts/scrape_linkedin_stats_browser.py +52 -0
  206. package/scripts/scrape_reddit_views.py +365 -0
  207. package/scripts/seed_search_queries.py +453 -0
  208. package/scripts/seed_search_topics.py +127 -0
  209. package/scripts/send_web_chat_reply.py +130 -0
  210. package/scripts/sentry_init.py +128 -0
  211. package/scripts/setup_twitter_auth.py +1320 -0
  212. package/scripts/snapshot.py +583 -0
  213. package/scripts/stats.py +2702 -0
  214. package/scripts/stats_helper.py +52 -0
  215. package/scripts/strike_alert.py +783 -0
  216. package/scripts/sweep_post_link_clicks.py +107 -0
  217. package/scripts/sync_ig_to_posts.py +147 -0
  218. package/scripts/test_browser_lock.py +189 -0
  219. package/scripts/test_installation_api.sh +52 -0
  220. package/scripts/test_percard_posting.py +142 -0
  221. package/scripts/top_dud_linkedin_queries.py +71 -0
  222. package/scripts/top_dud_reddit_queries.py +67 -0
  223. package/scripts/top_dud_twitter_queries.py +71 -0
  224. package/scripts/top_dud_twitter_topics.py +102 -0
  225. package/scripts/top_linkedin_queries.py +55 -0
  226. package/scripts/top_omitted_reddit_topics.py +91 -0
  227. package/scripts/top_performers.py +588 -0
  228. package/scripts/top_search_topics.py +180 -0
  229. package/scripts/top_twitter_queries.py +190 -0
  230. package/scripts/twitter_access_check.py +382 -0
  231. package/scripts/twitter_account.py +41 -0
  232. package/scripts/twitter_batch_phase.py +126 -0
  233. package/scripts/twitter_browser.py +2804 -0
  234. package/scripts/twitter_cookie_mirror.py +130 -0
  235. package/scripts/twitter_cycle_helper.py +310 -0
  236. package/scripts/twitter_gen_links.py +287 -0
  237. package/scripts/twitter_post_plan.py +1188 -0
  238. package/scripts/twitter_scan.py +324 -0
  239. package/scripts/twitter_supply_signal.py +57 -0
  240. package/scripts/twitter_threads_helper.py +152 -0
  241. package/scripts/unclaim_web_chat.py +29 -0
  242. package/scripts/update_instagram_stats.py +261 -0
  243. package/scripts/update_linkedin_stats_from_feed.py +328 -0
  244. package/scripts/version.py +72 -0
  245. package/scripts/watchdog_hung_runs.py +343 -0
  246. package/scripts/write_generation_trace.py +73 -0
  247. package/setup/SKILL.md +277 -0
  248. package/skill/amplitude-24h-signups.sh +38 -0
  249. package/skill/archive-old-logs.sh +40 -0
  250. package/skill/audit-dm-staleness.sh +42 -0
  251. package/skill/audit-linkedin.sh +14 -0
  252. package/skill/audit-moltbook.sh +4 -0
  253. package/skill/audit-reddit-resurrect.sh +67 -0
  254. package/skill/audit-reddit.sh +4 -0
  255. package/skill/audit-twitter.sh +4 -0
  256. package/skill/audit.sh +287 -0
  257. package/skill/backfill-twitter-attempts-topic.sh +19 -0
  258. package/skill/backfill-twitter-ghost-posts.sh +24 -0
  259. package/skill/check-external-pool-depth.sh +7 -0
  260. package/skill/check-web-chats.sh +203 -0
  261. package/skill/dm-outreach-linkedin.sh +250 -0
  262. package/skill/dm-outreach-reddit.sh +274 -0
  263. package/skill/dm-outreach-twitter.sh +265 -0
  264. package/skill/engage-dm-replies-linkedin.sh +4 -0
  265. package/skill/engage-dm-replies-reddit.sh +4 -0
  266. package/skill/engage-dm-replies-twitter.sh +4 -0
  267. package/skill/engage-dm-replies.sh +1597 -0
  268. package/skill/engage-linkedin.sh +581 -0
  269. package/skill/engage-moltbook.sh +36 -0
  270. package/skill/engage-reddit.sh +146 -0
  271. package/skill/engage-twitter.sh +467 -0
  272. package/skill/github-engage.sh +176 -0
  273. package/skill/ingest-web-chat-replies.sh +38 -0
  274. package/skill/invent-supply-test.sh +100 -0
  275. package/skill/invent-topics.sh +50 -0
  276. package/skill/lib/linkedin-backend.sh +364 -0
  277. package/skill/lib/platform.sh +48 -0
  278. package/skill/lib/reddit-backend.sh +234 -0
  279. package/skill/lib/twitter-backend.sh +314 -0
  280. package/skill/link-edit-github.sh +136 -0
  281. package/skill/link-edit-moltbook.sh +117 -0
  282. package/skill/link-edit-reddit.sh +201 -0
  283. package/skill/linkedin-presence.sh +182 -0
  284. package/skill/linkedin-recovery.sh +282 -0
  285. package/skill/lock.sh +647 -0
  286. package/skill/memory-snapshot.sh +39 -0
  287. package/skill/precompute-stats.sh +35 -0
  288. package/skill/prewarm-funnel.sh +104 -0
  289. package/skill/refresh-instagram-tokens.sh +57 -0
  290. package/skill/refresh-twitter-following.sh +52 -0
  291. package/skill/reply-risk-digest.sh +31 -0
  292. package/skill/run-cycle-update-guard.sh +44 -0
  293. package/skill/run-draft-and-publish.sh +123 -0
  294. package/skill/run-generate-daily-style.sh +50 -0
  295. package/skill/run-github-launchd.sh +62 -0
  296. package/skill/run-github.sh +102 -0
  297. package/skill/run-instagram-daily.sh +149 -0
  298. package/skill/run-instagram-render.sh +875 -0
  299. package/skill/run-linkedin-launchd.sh +81 -0
  300. package/skill/run-linkedin-unipile.sh +130 -0
  301. package/skill/run-linkedin.sh +1593 -0
  302. package/skill/run-moltbook-launchd.sh +61 -0
  303. package/skill/run-moltbook.sh +38 -0
  304. package/skill/run-overlay-watch.sh +100 -0
  305. package/skill/run-reddit-search-launchd.sh +64 -0
  306. package/skill/run-reddit-search.sh +505 -0
  307. package/skill/run-reddit-threads-double.sh +32 -0
  308. package/skill/run-reddit-threads.sh +847 -0
  309. package/skill/run-scan-moltbook-replies.sh +57 -0
  310. package/skill/run-twitter-cycle-launchd.sh +63 -0
  311. package/skill/run-twitter-cycle-singleton.sh +62 -0
  312. package/skill/run-twitter-cycle.sh +2408 -0
  313. package/skill/run-twitter-threads.sh +592 -0
  314. package/skill/scan-instagram-replies.sh +61 -0
  315. package/skill/scan-twitter-followups.sh +57 -0
  316. package/skill/social-autoposter-update.sh +66 -0
  317. package/skill/stats-instagram.sh +72 -0
  318. package/skill/stats-linkedin.sh +271 -0
  319. package/skill/stats-moltbook.sh +4 -0
  320. package/skill/stats-reddit.sh +4 -0
  321. package/skill/stats-twitter.sh +4 -0
  322. package/skill/stats.sh +521 -0
  323. package/skill/strike-alert.sh +18 -0
  324. package/skill/styles.sh +87 -0
  325. package/skill/sweep-link-clicks.sh +40 -0
  326. package/skill/topics.sh +51 -0
@@ -0,0 +1,2408 @@
1
+ #!/bin/bash
2
+ # run-twitter-cycle.sh — Combined Twitter scan + post cycle.
3
+ #
4
+ # Phase 1 (t=0):
5
+ # - select 8 projects via the shared inverse-recent-share picker
6
+ # (scripts/pick_project.py, same logic as github/reddit)
7
+ # - LLM drafts a search query per project (style from past top queries);
8
+ # if Phase 1 yields <RETRY_TARGET candidates that pass all filters
9
+ # (harness age gate + scorer dedupe + already-posted), the scan is
10
+ # re-invoked with the previously-tried queries injected as "do NOT
11
+ # repeat" — up to MAX_SCAN_ATTEMPTS total per cycle, same batch_id.
12
+ # - scrape tweets via twitter-harness, enrich via fxtwitter -> T0 snapshot
13
+ # - store all candidates with batch_id and search_topic
14
+ #
15
+ # No ripen wait (variant D won the A/B/C/D test 2026-05-31): the cycle goes
16
+ # straight from discovery to drafting. There is NO 5-min sleep and NO fxtwitter
17
+ # T1 re-poll anywhere in the Twitter pipeline. delta_score stays at its T0
18
+ # value and is no longer a gate. Do not re-introduce a ripen/sleep step here.
19
+ #
20
+ # Phase 2 (immediately after Phase 1):
21
+ # - sort candidates by virality_score DESC (composite predictor stamped at
22
+ # discovery by score_twitter_candidates.py); no delta floor, no T1 re-poll
23
+ # - Claude reads top 25 (raised from 15 so the long tail reaches the model),
24
+ # drops unsuitable, posts every candidate it judges genuinely on-brand
25
+ # (no per-cycle post cap, no per-project quota)
26
+ # - keep remaining pending rows: salvaged into the next cycle, hard-expired
27
+ # by Phase 0 once tweet age crosses FRESHNESS_HOURS
28
+ #
29
+ # Launchd cadence: every 20 minutes. One combined job, one browser lock.
30
+
31
+ set -uo pipefail
32
+
33
+ # SAPS_->S4L_ env mirror (brand rename 2026-07-03): old plists/tasks still
34
+ # export SAPS_*; new code reads S4L_*. Copy names, never values via eval.
35
+ while IFS='=' read -r _k _; do
36
+ case "$_k" in SAPS_*) _n="S4L_${_k#SAPS_}"; eval "[ -n \"\${$_n+x}\" ] || export $_n=\"\${$_k}\"";; esac
37
+ done <<EOF_ENV
38
+ $(env | grep '^SAPS_' | cut -d= -f1 | sed 's/$/=/')
39
+ EOF_ENV
40
+
41
+ # 2026-05-28: launchd inherits a default open-files limit of 256 on macOS,
42
+ # which is below the threshold the claude binary needs when it loads MCP
43
+ # servers from ~/.claude.json (50+ servers, each opening a stdio pipe pair).
44
+ # Without this bump, `claude -p` exits with code 1 and ZERO bytes of output
45
+ # (no stdout, no stderr, no archive) because the fd-exhaustion crash happens
46
+ # inside Node.js startup before any handler can run. The lean Phase 1 path
47
+ # (no --strict-mcp-config) was the first thing in the cycle to hit it.
48
+ # 4096 is well above what claude + uv + helpers need; soft-fail to original
49
+ # if the kernel/account caps below this.
50
+ ulimit -n 4096 2>/dev/null || true
51
+
52
+ # Honor S4L_REPO_DIR (set by the MCP wrapper + launchd plists) so a .mcpb
53
+ # install that materializes the repo under ~/.social-autoposter-mcp/repo/package
54
+ # resolves correctly. Falls back to the legacy ~/social-autoposter path for
55
+ # npm/git installs and direct invocations. Cascades to every $REPO_DIR/... ref
56
+ # below (sourced libs + child scripts inherit it), so this one line fixes the
57
+ # whole cycle's repo resolution on a bare .mcpb install.
58
+ REPO_DIR="${S4L_REPO_DIR:-$HOME/social-autoposter}"
59
+ SKILL_FILE="$REPO_DIR/SKILL.md"
60
+ LOG_DIR="$REPO_DIR/skill/logs"
61
+ mkdir -p "$LOG_DIR"
62
+
63
+ BATCH_ID="twcycle-$(date +%Y%m%d-%H%M%S)"
64
+ # Exported so twitter_post_plan.py (Phase 2b-post child process) can re-stamp
65
+ # the executing cycle's batch_id onto candidates at post time. Without this
66
+ # export, peer cycles' Phase 0 salvage can rewrite our candidates' batch_id
67
+ # mid-flight (documented edge case 2026-05-15); the re-stamp at post time is
68
+ # the structural fix so attribution always lands on the cycle that fired the
69
+ # browser, regardless of salvage timing.
70
+ export BATCH_ID
71
+ # Export the same id as SA_CYCLE_ID so every Claude session spawned by this
72
+ # cycle (via run_claude.sh -> log_claude_session.py) stamps its claude_sessions
73
+ # row with cycle_id=$BATCH_ID. Enables exact per-cycle cost accounting via
74
+ # get_run_cost.py --cycle-id, instead of the legacy script+since query which
75
+ # bleeds costs across concurrent stacked cycles. See 2026-05-10 cycle_id
76
+ # rollout (started on reddit, extended here).
77
+ export SA_CYCLE_ID="$BATCH_ID"
78
+
79
+ # LENGTH A/B CONCLUDED 2026-06-04: control won the configured primary metric
80
+ # (avg_clicks) and the enforcement branch is retired. New cycles no longer
81
+ # export LENGTH_ARM, so engagement_styles.py renders the legacy "keep it tight"
82
+ # prompt and twitter_post_plan.py does not stamp posts.length_arm. Historical
83
+ # arm stats stay preserved as a frozen shipped card in /api/experiments.
84
+
85
+ LOG_FILE="$LOG_DIR/twitter-cycle-$(date +%Y-%m-%d_%H%M%S).log"
86
+ RAW_FILE="/tmp/twitter_cycle_raw_$(date +%s).json"
87
+ QUERIES_FILE="/tmp/twitter_cycle_queries_$(date +%s).json"
88
+ # log_twitter_search_attempts.py writes [{query, project, attempt_id}, ...]
89
+ # here so score_twitter_candidates.py can stamp the exact discovering
90
+ # attempt_id onto each twitter_candidates row (2026-05-21).
91
+ ATTEMPTS_FILE="/tmp/twitter_cycle_attempts_$(date +%s).json"
92
+ RUN_START=$(date +%s)
93
+
94
+ # ----------------------------------------------------------------------------
95
+ # Browser: CDP-driven real Google Chrome on port 9555 via the twitter-harness
96
+ # MCP. Profile lives at ~/.claude/browser-profiles/browser-harness.
97
+ # TW_MCP_CONFIG / TW_ENGINE_PREFIX are placeholders, the real values get set
98
+ # below when lib/twitter-backend.sh is sourced (overwriting both).
99
+ # ----------------------------------------------------------------------------
100
+ TW_MCP_CONFIG=""
101
+ TW_ENGINE_PREFIX=""
102
+ # Tweets older than this are no longer worth replying to. Pending rows older
103
+ # than this are hard-expired by Phase 0; younger pending rows are salvaged
104
+ # from prior cycles into this batch.
105
+ # 2026-06-01: tightened 6h -> 2h. The pending pool had bloated to 636 rows,
106
+ # 523 of them >6h old (median virality 0.44, far below the ~5.8 posted median),
107
+ # because the salvage loop kept re-carrying stale low-virality junk. A 2h
108
+ # ceiling drops that carry runway so aged-out junk expires instead of riding
109
+ # ~80 cycles. Discovery is already capped at 1h (FRESHNESS_HOURS_DISCOVER).
110
+ #
111
+ # 2026-06-17 (per user request): DRAFT mode (DRAFT_ONLY=1, the MCP draft_cycle
112
+ # tool) widens both freshness knobs to 24h so human review surfaces more (and
113
+ # older) candidates. Autopilot is untouched: it keeps the experiment-concluded
114
+ # 2h expire ceiling + 1h discovery window (variant D). The branch is on
115
+ # DRAFT_ONLY, an external env var set by the draft_cycle tool, available here.
116
+ # 2026-07-02 (first-run onboarding boost, per user request): the draft-mode
117
+ # value accepts an env override, S4L_DRAFT_FRESHNESS_HOURS, so the kicker
118
+ # wrapper (run-draft-and-publish.sh) can widen a brand-new user's FIRST draft
119
+ # cycle to 48h and surface multiple review cards. Unset = the standard 24h
120
+ # draft window. Autopilot (DRAFT_ONLY=0) ignores the override entirely.
121
+ if [ "${DRAFT_ONLY:-0}" = "1" ]; then
122
+ FRESHNESS_HOURS="${S4L_DRAFT_FRESHNESS_HOURS:-24}"
123
+ else
124
+ FRESHNESS_HOURS=2
125
+ fi
126
+
127
+ # ----------------------------------------------------------------------------
128
+ # EXPERIMENT CONCLUDED 2026-05-31: variant D won the ripen+freshness A/B/C/D
129
+ # test (shipped 2026-05-22, D added 2026-05-25). D = no ripen wait + 1h Phase 1
130
+ # freshness window + drop parent threads with T0 views > 2000. Over the 60-day
131
+ # window D cut thread-age-at-discover p50 to 21 min (vs 173-277 for A/B/C) and
132
+ # led on avg views (91) and avg clicks (0.45), trading post-rate for fresher,
133
+ # higher-converting replies. A/B/C logic has been ripped out; D is now the
134
+ # permanent, hardcoded behavior. The cycle_variant column is still stamped 'D'
135
+ # below so historical analytics keep a consistent label.
136
+ #
137
+ # Phase 0 hard-expire uses FRESHNESS_HOURS (the union ceiling, tightened to 2h
138
+ # on 2026-06-01, see above) so peer cycles don't accidentally expire each
139
+ # other's still-pending rows. FRESHNESS_HOURS_DISCOVER (Phase 1 prompt +
140
+ # since-rewrite hook) stays tightened to 1h, the winning D setting.
141
+ TWITTER_CYCLE_VARIANT=D
142
+ # DRAFT mode widens discovery to 24h by default; autopilot keeps the winning D
143
+ # setting of 1h. S4L_DRAFT_FRESHNESS_HOURS (first-run onboarding boost, see the
144
+ # FRESHNESS_HOURS branch above) can widen the draft-mode value further (48h on a
145
+ # brand-new install's first cycle). The lean Phase 1 CDP scraper reads
146
+ # FRESHNESS_HOURS_DISCOVER directly and honors any value.
147
+ if [ "${DRAFT_ONLY:-0}" = "1" ]; then
148
+ FRESHNESS_HOURS_DISCOVER="${S4L_DRAFT_FRESHNESS_HOURS:-24}"
149
+ else
150
+ FRESHNESS_HOURS_DISCOVER=1
151
+ fi
152
+ # Export FRESHNESS_HOURS too so score_twitter_candidates.py inherits it and
153
+ # drives the expire-stale gate from the same knob (was hardcoded 18h there).
154
+ export TWITTER_CYCLE_VARIANT FRESHNESS_HOURS_DISCOVER FRESHNESS_HOURS
155
+ # Hook env: ~/.claude/hooks/twitter-search-since-rewrite.py reads this and
156
+ # uses it in place of its hardcoded 6h default when present. The hook accepts
157
+ # only a 1-24h range and silently falls back to its 6h default on anything
158
+ # bigger, so cap the exported value at 24: during the 48h first-run boost the
159
+ # CDP scraper still gets the full window via FRESHNESS_HOURS_DISCOVER, while
160
+ # any hook-rewritten query keeps the widest value the hook can honor.
161
+ if [ "$FRESHNESS_HOURS_DISCOVER" -gt 24 ] 2>/dev/null; then
162
+ export FRESHNESS_HOURS_OVERRIDE=24
163
+ else
164
+ export FRESHNESS_HOURS_OVERRIDE=$FRESHNESS_HOURS_DISCOVER
165
+ fi
166
+
167
+ # `set -a` auto-exports every variable assigned by `source .env`, so DATABASE_URL
168
+ # and friends propagate to subprocess env (python3 scripts use os.environ at
169
+ # import time and would otherwise see empty strings — silently breaking
170
+ # update_candidate_posted in twitter_post_plan.py and creating duplicate posts
171
+ # under parallel cycles, observed 2026-05-01 batches 02-08).
172
+ if [ -f "$REPO_DIR/.env" ]; then
173
+ set -a
174
+ source "$REPO_DIR/.env"
175
+ set +a
176
+ fi
177
+
178
+ log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$LOG_FILE"; }
179
+
180
+ log "=== Twitter Cycle (batch=$BATCH_ID): $(date) ==="
181
+ log "Logic=D (no-ripen + 1h freshness + 2k_view_cap; experiment concluded 2026-05-31); discover_freshness=${FRESHNESS_HOURS_DISCOVER}h"
182
+ log "Length-control experiment concluded 2026-06-04; winner=control; LENGTH_ARM retired"
183
+
184
+ # --- Preflight (added 2026-05-02) -----------------------------------------
185
+ # Three early-exit gates BEFORE we open the DB, set up traps, or touch the
186
+ # browser. Each gate emits a `[skipped: <reason>]` stderr line and exits 0
187
+ # so launchd treats the slot as cleanly consumed and fires the next one
188
+ # on schedule.
189
+ #
190
+ # 1. Memory pressure: 2026-05-01 a JetsamEvent at 19:26 swallowed two
191
+ # consecutive launchd fires (19:38, 19:53). The wrappers fired, but the
192
+ # grandchild bash never produced output — most likely jetsam-killed or
193
+ # starved during the system's crash-cleanup spike. Skipping when
194
+ # pressure_level >= 2 (warn) avoids piling more Chrome+Claude+Python
195
+ # work onto an already-thrashing system.
196
+ #
197
+ # 2. Claude quota stamp: prior run_claude.sh invocation hit a fatal cap
198
+ # (monthly cap, daily cap, context-window, credit balance, persistent
199
+ # 429). Skip until the stamp expires (default 10 min). When the cap
200
+ # lifts and the next post-expiry fire succeeds, run_claude.sh clears
201
+ # the stamp automatically.
202
+ #
203
+ # 3. Single-cycle gate: exactly 1 concurrent run-twitter-cycle.sh, enforced
204
+ # HERE in the script itself so EVERY launch path is covered — the launchd
205
+ # singleton wrapper's snapshot copy, the MCP draft_cycle tool's direct
206
+ # `bash skill/run-twitter-cycle.sh`, and any manual/agent invocation all
207
+ # run this preflight and compete for the same /tmp/sa-twitter-cycle-slot-1
208
+ # mkdir lock. History: 2026-05-03 introduced this as a max-4 cap; on
209
+ # 2026-05-22 we added run-twitter-cycle-singleton.sh to enforce one-at-a-
210
+ # time, but that wrapper only governs the launchd path and never kills
211
+ # (per user instruction), so out-of-band literal launches (MCP/manual)
212
+ # sailed past it while this cap still permitted 4. On 2026-06-01 a launchd
213
+ # studyly cycle + an out-of-band fazm cycle overlapped, fought over the
214
+ # twitter-browser lock, and the fazm one got watchdog-killed (logged as
215
+ # phase2b_silent). Cutting the cap to 1 unifies the gate across all paths.
216
+ # The slot's dead-holder GC (kill -0 in preflight.sh) still reclaims slots
217
+ # orphaned by SIGKILL/OOM, so a crashed cycle never wedges the gate.
218
+ #
219
+ # preflight.sh exposes a small set of helpers; we call them in order
220
+ # (cheapest first) so a fast-path skip (already-blocked) doesn't even
221
+ # spend the sysctl read for the next check.
222
+ source "$REPO_DIR/scripts/preflight.sh"
223
+ SA_PREFLIGHT_SCRIPT="run-twitter-cycle"
224
+ if [ "${SCAN_ONLY:-0}" = "1" ]; then
225
+ # SCAN_ONLY (the Desktop-session autopilot's scan step) gets its OWN slot pool
226
+ # so it is NOT blocked by a live launchd autopilot cycle; the two then serialize
227
+ # on the twitter-browser lock (acquired in Phase 1) instead of being mutually
228
+ # exclusive. It also SKIPS the claude-blocked gate: SCAN_ONLY drives no
229
+ # `claude -p`, so a prior claude cap must not suppress a pure scan.
230
+ SA_PREFLIGHT_SCRIPT="run-twitter-cycle-scan"
231
+ preflight_skip_if_jetsam_pressure
232
+ preflight_acquire_slot_or_skip "twitter-scan" 1
233
+ else
234
+ preflight_skip_if_claude_blocked
235
+ preflight_skip_if_jetsam_pressure
236
+ preflight_acquire_slot_or_skip "twitter-cycle" 1
237
+ fi
238
+
239
+ # Source lock helpers (functions only, no lock acquired here). Phase 0 + the
240
+ # project/queries setup below run lock-free against DB and config files;
241
+ # the twitter-browser lock is acquired later, immediately before the Phase 1
242
+ # Claude scan that actually drives the browser (line ~177). Pre-2026-05-01
243
+ # this acquire was here at script start and held the lock through Phase 0
244
+ # (~3-10s of pure DB/Python work that doesn't touch the browser), starving
245
+ # peer cycles' Phase 2b-post under parallel-cycle contention.
246
+ source "$REPO_DIR/skill/lock.sh"
247
+
248
+ # Harness-only browser bootstrap (twitter-agent path fully removed 2026-05-19).
249
+ # Sets MCP_CONFIG_FILE, BROWSER_INSTRUCTIONS, exports TWITTER_CDP_URL=9555.
250
+ # Phase 2b-post's twitter_post_plan.py shells out to twitter_browser.py, which
251
+ # honors TWITTER_CDP_URL exported by this lib.
252
+ source "$REPO_DIR/skill/lib/twitter-backend.sh"
253
+ TW_MCP_CONFIG="$MCP_CONFIG_FILE"
254
+ # 2026-06-26: the model-facing steps (Phase 1 query draft, Phase 2b prep draft) are
255
+ # TOOL-FREE. All browser work is done deterministically by the shell's CDP scan
256
+ # (browser-harness over port 9555) + Phase 2b-post's twitter_browser.py, NOT by the
257
+ # model. The old BROWSER BACKEND / bh_run "translation table" block is no longer
258
+ # injected: prep drafts purely from the inlined candidate context (Text: $ctext per
259
+ # candidate) + MEDIA_BLOCK, which is exactly what the model's rare bh_run fallback
260
+ # used to re-fetch (1 call/week vs ~18.5k/wk deterministic CDP scans). The 9555 Chrome
261
+ # is still launched by twitter-backend.sh above for the shell scan + post step; only
262
+ # the model's browser fallback is removed. This also drops the hardcoded "logged in as
263
+ # m13v_" identity that the block carried, so prompts are no longer single-tenant.
264
+ TW_ENGINE_PREFIX=""
265
+
266
+ # --- Phase tracking: start the twitter_batches row + chain into lock.sh trap -
267
+ # Per-cycle phase row (twitter_batches.current_phase + phase_started_at) is
268
+ # read by peer cycles' Phase 0 to decide salvage timing per-phase instead of
269
+ # the old flat 20-min wall-clock cutoff. Phase 2b-gen (SEO landing-page build)
270
+ # legitimately runs 10-40 min and was being salvaged out from under live
271
+ # owners under the old rule. See migration 2026-05-01_twitter_batches.sql and
272
+ # scripts/twitter_batch_phase.py.
273
+ #
274
+ # Trap design: lock.sh installs `_sa_release_locks` on EXIT/INT/TERM/HUP. We
275
+ # wrap that into `_sa_combined_exit` so a clean exit ALSO deletes our
276
+ # twitter_batches row. SIGKILL / OOM / hard crash bypasses traps and
277
+ # intentionally leaves the row stale — that's the salvage recovery path.
278
+ _sa_cleanup_batch_row() {
279
+ if [ -n "${BATCH_ID:-}" ]; then
280
+ python3 "$REPO_DIR/scripts/twitter_batch_phase.py" end "$BATCH_ID" 2>/dev/null || true
281
+ fi
282
+ }
283
+ _sa_combined_exit() {
284
+ # Emit run_monitor.log summary FIRST, before any cleanup. Without this,
285
+ # SIGTERM landing between Phase 2b-post (where twitter_post_plan.py has
286
+ # already committed to the `posts` table) and the historical inline
287
+ # summary write at the bottom of the script silently drops the run from
288
+ # run_monitor.log. Mirrors the same fix shipped to run-reddit-search.sh.
289
+ # Idempotent: a flag-guarded one-shot, so the happy-path explicit call at
290
+ # the bottom and the trap firing on EXIT do not double-write.
291
+ _sa_emit_run_summary_oneshot
292
+ _sa_cleanup_batch_row
293
+ # Release the parallel-cycle slot acquired by preflight.sh. Without this,
294
+ # this trap (which OVERWRITES the preflight trap installed at source-time)
295
+ # would leak the slot until the next launchd fire's GC pass — capping
296
+ # effective throughput at 1/cycle even though the slot pool is 4 wide.
297
+ if command -v _preflight_release_slots >/dev/null 2>&1; then
298
+ _preflight_release_slots
299
+ fi
300
+ _sa_release_locks
301
+ }
302
+
303
+ # Idempotent run_monitor.log emitter wired into _sa_combined_exit (which is
304
+ # trap'd to EXIT INT TERM HUP). On the happy path the bottom of the script
305
+ # calls this directly; on SIGTERM the trap calls it. Either order is a no-op
306
+ # after first emission via _SA_RUN_SUMMARY_EMITTED.
307
+ #
308
+ # Reads counters from globals the cycle has been accumulating (BATCH_ID,
309
+ # RUN_START, EXEC_FAILED, EXEC_REASONS, EXEC_SKIPPED, CANDIDATE_COUNT,
310
+ # SALVAGED, QUERIES_TOTAL, DUDS_TOTAL, TWEETS_PULLED, BATCH_COUNT,
311
+ # HIGH_DELTA_COUNT). Re-derives POSTED_CT/SKIPPED_CT from the
312
+ # twitter_candidates table directly so a SIGTERM mid-Phase-2b still gets
313
+ # accurate counts (the row was committed inside twitter_post_plan.py before
314
+ # the kill). All psql / get_run_cost.py calls are wrapped in `timeout 10`
315
+ # so a Postgres hang during shutdown can't wedge the trap.
316
+ #
317
+ # Early-exit failure paths (Phase 1 abort, empty batch, etc.) write their
318
+ # own dedicated log_run.py line with custom failure_reasons and then set
319
+ # _SA_RUN_SUMMARY_EMITTED=1 to short-circuit this function — they keep
320
+ # their tailored error reason, this fallback skips.
321
+ _SA_RUN_SUMMARY_EMITTED=0
322
+ _sa_emit_run_summary_oneshot() {
323
+ [ "${_SA_RUN_SUMMARY_EMITTED:-0}" = "1" ] && return 0
324
+ _SA_RUN_SUMMARY_EMITTED=1
325
+
326
+ local posted_ct=0 skipped_ct=0 cost="0.0000" failed_ct failure_reasons
327
+ # Prefer the in-memory counters captured from twitter_post_plan.py's JSON
328
+ # summary (EXEC_POSTED / EXEC_SKIPPED). Those are the ground truth for what
329
+ # THIS cycle did. The fallback SQL count is needed when SIGTERM hits before
330
+ # Phase 2b-post records a count, but it's UNRELIABLE during normal exit:
331
+ # peer cycles' Phase 0 may have salvaged this batch's candidates into a new
332
+ # batch_id mid-Phase-2b (documented edge case, mitigated by the phase2b-*
333
+ # advance stamps but not 100% eliminated under heavy parallel load), in
334
+ # which case the WHERE batch_id='$BATCH_ID' query returns 0 even though we
335
+ # successfully posted N replies. That false-zero is what historically
336
+ # synthesized phase2b_silent failure_reasons against successful runs.
337
+ if [ -n "${EXEC_POSTED:-}" ] || [ -n "${EXEC_SKIPPED:-}" ]; then
338
+ posted_ct="${EXEC_POSTED:-0}"
339
+ skipped_ct="${EXEC_SKIPPED:-0}"
340
+ elif [ -n "${BATCH_ID:-}" ]; then
341
+ # /api/v1/twitter-candidates/counts-by-batch returns posted +
342
+ # skipped_or_expired in one roundtrip; helper prints them space-
343
+ # separated so this stays a single $() capture.
344
+ _SC=$(timeout 10 python3 "$REPO_DIR/scripts/twitter_cycle_helper.py" \
345
+ status-counts --batch-id "$BATCH_ID" 2>/dev/null || echo "0 0")
346
+ posted_ct=$(echo "$_SC" | awk '{print $1}')
347
+ skipped_ct=$(echo "$_SC" | awk '{print $2}')
348
+ : "${posted_ct:=0}"
349
+ : "${skipped_ct:=0}"
350
+ fi
351
+ cost=$(timeout 10 python3 "$REPO_DIR/scripts/get_run_cost.py" \
352
+ --cycle-id "${BATCH_ID}" \
353
+ 2>/dev/null || echo "0.0000")
354
+
355
+ failed_ct="${EXEC_FAILED:-0}"
356
+ failure_reasons="${EXEC_REASONS:-}"
357
+ # Reproduce the failure-reason synthesis block so SIGTERM cycles still
358
+ # get a useful reason instead of a silent "—". Same conditions as the
359
+ # historical inline block: cycle ended with zero progress despite having
360
+ # candidates pending.
361
+ if [ "${posted_ct:-0}" = "0" ] \
362
+ && [ "${failed_ct:-0}" = "0" ] \
363
+ && [ "${EXEC_SKIPPED:-0}" = "0" ] \
364
+ && [ -z "$failure_reasons" ] \
365
+ && [ "${CANDIDATE_COUNT:-0}" -gt 0 ]; then
366
+ local phase2b_log
367
+ phase2b_log=$(awk '/Phase 1: drafting queries|Phase 2b-prep: Claude reading|Phase 2b-post:/,EOF' "$LOG_FILE" 2>/dev/null || echo "")
368
+ # Inline reason-add: bash doesn't support `local` on function decls,
369
+ # and a free-standing nested function would leak into the outer
370
+ # scope, so we just expand the assignments at each call site.
371
+ # Run the shared API-error classifier first — catches monthly_limit,
372
+ # stream_idle_timeout, api_overloaded, context_overflow, credit_balance,
373
+ # etc. uniformly so the dashboard pill reads with the actual error
374
+ # class instead of falling through to the generic phase2b_silent.
375
+ local classifier_reason
376
+ classifier_reason=$(echo "$phase2b_log" | python3 "$REPO_DIR/scripts/classify_run_error.py" 2>/dev/null)
377
+ if [ -n "$classifier_reason" ]; then
378
+ failure_reasons="${failure_reasons:+$failure_reasons,}${classifier_reason}:1"
379
+ failed_ct=$(( failed_ct + 1 ))
380
+ fi
381
+ if echo "$phase2b_log" | grep -qiE 'auth redirect|re-authenticat|browser profile.*auth|profile.*needs.*re-auth'; then
382
+ failure_reasons="${failure_reasons:+$failure_reasons,}auth_redirect:1"
383
+ failed_ct=$(( failed_ct + 1 ))
384
+ fi
385
+ if echo "$phase2b_log" | grep -qiE '"error":"rate_limited"|RATE_LIMITED_TWITTER'; then
386
+ failure_reasons="${failure_reasons:+$failure_reasons,}rate_limited:1"
387
+ failed_ct=$(( failed_ct + 1 ))
388
+ fi
389
+ if echo "$phase2b_log" | grep -qiE 'page.load.timeout|navigation timeout|timed out|Timeout exceeded'; then
390
+ failure_reasons="${failure_reasons:+$failure_reasons,}timeout:1"
391
+ failed_ct=$(( failed_ct + 1 ))
392
+ fi
393
+ if echo "$phase2b_log" | grep -qiE 'reply_box_not_found|tweet_not_found'; then
394
+ failure_reasons="${failure_reasons:+$failure_reasons,}posting_blocked:1"
395
+ failed_ct=$(( failed_ct + 1 ))
396
+ fi
397
+ if [ -z "$failure_reasons" ]; then
398
+ failure_reasons="phase2b_silent:1"
399
+ failed_ct=$(( failed_ct + 1 ))
400
+ fi
401
+ fi
402
+
403
+ local args
404
+ args=(--script "post_twitter" \
405
+ --posted "${posted_ct:-0}" \
406
+ --skipped "${skipped_ct:-0}" \
407
+ --failed "$failed_ct" \
408
+ --salvaged "${SALVAGED:-0}" \
409
+ --queries "${QUERIES_TOTAL:-0}" --duds "${DUDS_TOTAL:-0}" \
410
+ --tweets-pulled "${TWEETS_PULLED:-0}" \
411
+ --candidates "${BATCH_COUNT:-0}" --above-floor "${HIGH_DELTA_COUNT:-0}" \
412
+ --cost "$cost" \
413
+ --elapsed $(( $(date +%s) - ${RUN_START:-$(date +%s)} )))
414
+ [ -n "$failure_reasons" ] && args+=(--failure-reasons "$failure_reasons")
415
+ [ -n "${EXEC_SKIP_REASONS:-}" ] && args+=(--skip-reasons "$EXEC_SKIP_REASONS")
416
+ python3 "$REPO_DIR/scripts/log_run.py" "${args[@]}" 2>/dev/null || true
417
+ }
418
+
419
+ trap _sa_combined_exit EXIT INT TERM HUP
420
+
421
+ python3 "$REPO_DIR/scripts/twitter_batch_phase.py" start "$BATCH_ID" --phase phase0 2>&1 | tee -a "$LOG_FILE" || true
422
+
423
+ # --- Phase 0: hard-expire stale pending + salvage truly-orphaned rows --------
424
+ # Pending rows from prior cycles fall into two buckets:
425
+ # - tweet_posted_at older than FRESHNESS_HOURS -> hard-expire (lost the
426
+ # replying window, no value in retrying)
427
+ # - still-fresh AND owning batch is dead -> re-assign to this batch
428
+ # so Phase 2a re-measures T1 and Phase 2b reconsiders them. This is the
429
+ # recovery path for cycles whose Phase 2b died on Anthropic org quota,
430
+ # X rate limit, browser crash, or any other infra failure.
431
+ #
432
+ # Two safety guards make this safe under parallel cycles (post 2026-04-30
433
+ # detach refactor: launchd no longer suppresses overlapping fires, so 2-3
434
+ # run-twitter-cycle.sh can be in Phase 0/1/2 simultaneously):
435
+ #
436
+ # 1. pg_advisory_xact_lock(7472346) serializes Phase 0 transactions, so
437
+ # two cycles can't race on the salvage UPDATE.
438
+ #
439
+ # 2. PHASE-AWARE BUDGET (post 2026-05-01): salvage timing is per-phase,
440
+ # read from the owner's twitter_batches row:
441
+ # phase0 -> 5 min (just the salvage SQL)
442
+ # phase1 -> 20 min (Claude scan + scrape)
443
+ # phase2a -> 20 min (browser-lock handoff window; no ripen wait since 2026-05-31)
444
+ # phase2b-prep -> 45 min (Claude reads threads + drafts; bumped 2026-05-15
445
+ # 15 -> 30 after 17:15 cycle was wrongly salvaged
446
+ # while queued behind 17:30's 42-min lock-hold;
447
+ # bumped 2026-05-22 30 -> 45 to leave more
448
+ # headroom for big-batch Claude reads after the
449
+ # Variant A wake re-stamp fix)
450
+ # phase2b-gen -> 60 min (SEO landing-page build, the slow phase)
451
+ # phase2b-post -> 15 min (browser reply + log)
452
+ # Pre-2026-05-01 the rule was a flat 20-min wall-clock cutoff against
453
+ # batch_id, which salvaged live cycles whose Phase 2b-gen step (10-40
454
+ # min in normal operation) hadn't finished. Observed 2026-05-01: cycle
455
+ # 16:23's candidate 7994 was salvaged into 16:53 while 16:23 was still
456
+ # generating the SEO page; both cycles raced on the post and the
457
+ # late-arriving owner logged failed=1.
458
+ #
459
+ # 3. LEGACY FALLBACK: rows whose batch has no twitter_batches entry (any
460
+ # cycle that ran before this migration, OR a cycle whose start helper
461
+ # failed) fall back to the original flat 20-min batch_id heuristic.
462
+ # Self-cleans within FRESHNESS_HOURS of migration.
463
+ #
464
+ # batch_id format is `twcycle-YYYYMMDD-HHMMSS` (assigned at script start
465
+ # from `date +%Y%m%d-%H%M%S`, local time). Since the format is fixed-width
466
+ # and lexicographically sortable, we compute the cutoff in the shell
467
+ # (same TZ as batch_id) and do a string comparison in SQL — sidesteps the
468
+ # Postgres session-TZ trap that would otherwise mis-interpret batch_id.
469
+ LEGACY_SALVAGE_CUTOFF_MIN=20
470
+ LEGACY_SALVAGE_CUTOFF_BATCH_ID="twcycle-$(python3 -c "import datetime; print((datetime.datetime.now() - datetime.timedelta(minutes=${LEGACY_SALVAGE_CUTOFF_MIN})).strftime('%Y%m%d-%H%M%S'))")"
471
+ # Single-transaction Phase 0 salvage now lives server-side at
472
+ # /api/v1/twitter-candidates/phase0-salvage. Same advisory lock (7472346),
473
+ # same expire + salvage CTE, same phase-aware budget table. The helper
474
+ # prints "<expired_count>|<salvaged_count>" so the legacy cut/cut shape
475
+ # downstream still works.
476
+ PHASE0_RESULT=$(python3 "$REPO_DIR/scripts/twitter_cycle_helper.py" \
477
+ phase0-salvage \
478
+ --batch-id "$BATCH_ID" \
479
+ --freshness-hours "$FRESHNESS_HOURS" \
480
+ --legacy-cutoff "$LEGACY_SALVAGE_CUTOFF_BATCH_ID" \
481
+ 2>/dev/null | tail -1 | tr -d ' ')
482
+ EXPIRED_STALE=$(echo "$PHASE0_RESULT" | cut -d'|' -f1)
483
+ SALVAGED=$(echo "$PHASE0_RESULT" | cut -d'|' -f2)
484
+ [ "${EXPIRED_STALE:-0}" -gt 0 ] && log "Phase 0: hard-expired $EXPIRED_STALE pending rows older than ${FRESHNESS_HOURS}h"
485
+ [ "${SALVAGED:-0}" -gt 0 ] && log "Phase 0: salvaged $SALVAGED orphaned pending rows (phase-aware budget) into $BATCH_ID"
486
+
487
+ # Advance our own batch row from phase0 -> phase1 now that the salvage SQL
488
+ # committed. Subsequent phase transitions are stamped right before the work
489
+ # they cover begins.
490
+ python3 "$REPO_DIR/scripts/twitter_batch_phase.py" advance "$BATCH_ID" --phase phase1 2>&1 | tee -a "$LOG_FILE" || true
491
+
492
+ # --- Shared project selection (inverse-recent-share) -------------------------
493
+ # Project selection is shared across twitter/github/reddit via
494
+ # scripts/pick_project.py (pick_projects): inverse-recent-share weighting,
495
+ # weight / (1 + posts in the last 7d), sampled without replacement. This
496
+ # replaced the old inline weighted sample on 2026-05-15 so all platforms
497
+ # pick projects the same way.
498
+ # Each chosen project is then enriched here with an `excludes_for_search`
499
+ # array sourced from project_search_excludes (only terms past the
500
+ # 2-distinct-batch activation gate). The Phase 1 scanner is required to
501
+ # mechanically append these as `-term` operators to whatever query it drafts
502
+ # for the project. See scripts/project_excludes.py for proposal/activation/
503
+ # decay rules.
504
+ _PJ_ERR="$(mktemp)"
505
+ PROJECTS_JSON=$(python3 - 2>"$_PJ_ERR" <<'PY'
506
+ import json, os, subprocess, sys
507
+ REPO = os.path.expanduser('~/social-autoposter')
508
+ sys.path.insert(0, os.path.join(REPO, 'scripts'))
509
+ import project_excludes as pe
510
+
511
+ _pp_args = ['python3', os.path.join(REPO, 'scripts', 'pick_project.py'),
512
+ '--platform', 'twitter', '--count', '1', '--json']
513
+ # Manual-mode (MCP draft_cycle) single-project scoping: when S4L_FORCE_PROJECT
514
+ # is set, force that exact project instead of the weighted-random autopilot
515
+ # pick, so a customer's interactive cycle only ever touches their own project.
516
+ _force_project = os.environ.get('S4L_FORCE_PROJECT')
517
+ if _force_project:
518
+ _pp_args += ['--project', _force_project]
519
+ res = subprocess.run(
520
+ _pp_args,
521
+ capture_output=True, text=True, timeout=30,
522
+ )
523
+ picked = []
524
+ if res.returncode == 0 and res.stdout.strip():
525
+ try:
526
+ picked = json.loads(res.stdout)
527
+ except Exception:
528
+ picked = []
529
+
530
+ # pick_project.py returns a single dict when --count=1, a list when --count>1.
531
+ # Normalize to a list so the rest of the heredoc works either way.
532
+ if isinstance(picked, dict):
533
+ picked = [picked]
534
+
535
+ from pick_search_topic import pick_topic_for_project, PickerError
536
+
537
+ chosen = []
538
+ for p in picked:
539
+ try:
540
+ excludes = pe.active_excludes('twitter', p.get('name'))
541
+ except Exception:
542
+ excludes = []
543
+ # 2026-05-26: force-pick ONE search_topic per project via the Python
544
+ # picker so end-to-end attribution (topic -> query -> candidate ->
545
+ # post -> click) is clean. Mirrors the engagement_styles flow.
546
+ #
547
+ # Single mode (post-2026-05-28): picker returns search_topic=<string>,
548
+ # weighted-random over the FULL universe with log-smoothed weights
549
+ # (top ~20-30%, cold ~0.5-1%). Claude must use the assigned topic
550
+ # verbatim. EXPLORE_INVENT was removed in favor of the standalone
551
+ # invent_topics.py job that writes new topics directly into
552
+ # project_search_topics outside the cycle.
553
+ #
554
+ # 2026-05-27: NO fallback. The DB is the only source of truth for
555
+ # the universe. If pick_topic_for_project raises (DB unreachable or
556
+ # zero active topics for this project), let the heredoc crash so
557
+ # PROJECTS_JSON is empty, the bash trap fires, and launchd records
558
+ # a hard failure. Silent fallback to config.json or to the first
559
+ # legacy search_topics[] entry would post against a stale seed list
560
+ # and corrupt attribution; the rule is "stop the pipeline".
561
+ topic_pick = pick_topic_for_project(p.get('name'), platform='twitter')
562
+ picked_topic = topic_pick.get('search_topic')
563
+ reference_topics = topic_pick.get('reference_topics') or []
564
+ picked_weight_pct = topic_pick.get('picked_weight_pct')
565
+ chosen.append({
566
+ 'name': p.get('name'),
567
+ 'description': p.get('description', ''),
568
+ # Force-picked single topic (2026-05-26). Replaces the legacy
569
+ # `search_topics: [...]` array. Claude draws its query from THIS
570
+ # topic and must echo it verbatim on every tweet object via the
571
+ # bh_run scrape script's `search_topic` Python variable.
572
+ 'search_topic': picked_topic,
573
+ 'picked_weight_pct': picked_weight_pct,
574
+ # Per-project pool stats (top by composite_score). Surfaced as
575
+ # context to help Claude understand the topic's history.
576
+ 'reference_topics': reference_topics,
577
+ # Self-improving exclusion list (2026-05-09): MUST be appended
578
+ # as `-term` to every query drafted for this project.
579
+ 'excludes_for_search': excludes,
580
+ })
581
+ print(json.dumps(chosen, indent=2))
582
+ PY
583
+ )
584
+ _PJ_RC=$?
585
+ # Fail loud when the project/topic universe can't be built. The heredoc above
586
+ # exits non-zero (PROJECTS_JSON empty) when pick_topic_for_project finds zero
587
+ # active rows in project_search_topics for the selected project, or the topics
588
+ # API is unreachable; it also yields "[]" when no project is eligible. Without
589
+ # this guard the empty PROJECTS_JSON silently falls through to "0 queries -> 0
590
+ # tweets -> batch expired -> zero", which reads to the user as "nothing to post"
591
+ # when the real cause is "this project was never seeded with search topics".
592
+ # Seeding now happens in the MCP setup tool; this is the defense-in-depth net
593
+ # so a missing universe is surfaced, never swallowed. (2026-06-02)
594
+ if [ "$_PJ_RC" -ne 0 ] || ! printf '%s' "$PROJECTS_JSON" | python3 -c 'import json,sys; d=json.load(sys.stdin); sys.exit(0 if isinstance(d,list) and d else 1)' 2>/dev/null; then
595
+ _PJ_REASON="project_selection_failed"
596
+ if grep -q "no active search topics" "$_PJ_ERR" 2>/dev/null; then
597
+ _PJ_REASON="no_search_topics"
598
+ elif grep -qiE "project-search-topics API|API unreachable" "$_PJ_ERR" 2>/dev/null; then
599
+ _PJ_REASON="topics_api_unreachable"
600
+ fi
601
+ log "Project/topic universe build FAILED (reason=$_PJ_REASON); stopping cycle before scan. Last error lines:"
602
+ tail -15 "$_PJ_ERR" 2>/dev/null | sed 's/^/ /' | tee -a "$LOG_FILE"
603
+ rm -f "$_PJ_ERR"
604
+ # Surface the reason to the MCP draft_cycle wrapper (stdout marker) in manual mode.
605
+ if [ "${DRAFT_ONLY:-0}" = "1" ]; then
606
+ echo "DRAFT_ONLY_BLOCKED=$_PJ_REASON"
607
+ fi
608
+ # Record a dashboard-visible failure row (best-effort) and exit cleanly.
609
+ python3 "$REPO_DIR/scripts/log_run.py" --script "post_twitter" --posted 0 --skipped 0 --failed 1 \
610
+ --failure-reasons "${_PJ_REASON}:1" --cost "0.0000" --elapsed $(( $(date +%s) - RUN_START )) 2>/dev/null || true
611
+ _SA_RUN_SUMMARY_EMITTED=1
612
+ exit 0
613
+ fi
614
+ rm -f "$_PJ_ERR"
615
+
616
+ log "Selected projects: $(echo "$PROJECTS_JSON" | python3 -c 'import json,sys; print(", ".join(p["name"] for p in json.load(sys.stdin)))')"
617
+ EXCLUDES_TOTAL=$(echo "$PROJECTS_JSON" | python3 -c 'import json,sys; d=json.load(sys.stdin); print(sum(len(p.get("excludes_for_search") or []) for p in d))')
618
+ [ "${EXCLUDES_TOTAL:-0}" -gt 0 ] && log "Active project-wide excludes loaded across selected projects: $EXCLUDES_TOTAL"
619
+
620
+ # --- Top past queries for style inspiration (PER-PROJECT, 2026-05-26) -------
621
+ # Now scored by composite (clicks×100 + likes + views×0.001) AND filtered by
622
+ # project so the model sees ITS OWN historical winners, not a global pool.
623
+ # Each query carries the full conversion funnel: tweets_found_avg, posted_n,
624
+ # skipped_n, post_rate, views, likes, clicks. Clicks are the ultimate signal;
625
+ # composite weights them ×100 so one click outvalues 100 likes of vibes.
626
+ #
627
+ # Why per-project (vs the previous global TOP_QUERIES_JSON): the global list
628
+ # let a thin niche (paperback-expert) cross-mimic a stronger project's
629
+ # min_faves tier from the gold list, even when paperback-expert had ZERO
630
+ # historical rows of its own. Per-project routing isolates the signal so
631
+ # each project's prompt sees only queries it ran itself.
632
+ #
633
+ # Cold-start projects (zero historical rows): no cross-project fallback. They
634
+ # get an empty project_queries array and rely on PER-PROJECT SUPPLY SIGNAL
635
+ # (for min_faves) + their config.json description (for keyword phrasing). A
636
+ # cross-project "structural inspiration" fallback contradicts the whole point
637
+ # of the per-project routing; explicitly removed 2026-05-26.
638
+ TOP_QUERIES_PER_PROJECT_JSON=$(echo "$PROJECTS_JSON" | python3 -c "
639
+ import json, sys, subprocess
640
+ projects = json.load(sys.stdin)
641
+ repo_dir = '$REPO_DIR'
642
+
643
+ def run_q(args):
644
+ try:
645
+ r = subprocess.run(['python3', f'{repo_dir}/scripts/top_twitter_queries.py'] + args,
646
+ capture_output=True, text=True, timeout=30)
647
+ return json.loads((r.stdout or '[]').strip() or '[]') if r.returncode == 0 else []
648
+ except Exception:
649
+ return []
650
+
651
+ out = {}
652
+ for p in projects:
653
+ name = (p.get('name') or '').strip()
654
+ if not name:
655
+ continue
656
+ rows = run_q(['--limit', '20', '--window-days', '7', '--project', name])
657
+ out[name] = {'project_queries': rows}
658
+ print(json.dumps(out))
659
+ " 2>/dev/null || echo "{}")
660
+ TOP_QUERIES_SUMMARY=$(echo "$TOP_QUERIES_PER_PROJECT_JSON" | python3 -c '
661
+ import json, sys
662
+ d = json.load(sys.stdin)
663
+ parts = []
664
+ cold = 0
665
+ for name, entry in d.items():
666
+ n = len(entry.get("project_queries") or [])
667
+ parts.append(f"{name}={n}")
668
+ if n == 0:
669
+ cold += 1
670
+ print(", ".join(parts) + f" (cold_start_projects={cold})")
671
+ ')
672
+ log "Per-project top queries loaded: $TOP_QUERIES_SUMMARY"
673
+
674
+ # --- Top performing search topics (topic-universe evolution, 2026-05-25) ----
675
+ # Sibling signal to TOP_QUERIES_JSON, one level up the funnel: where queries
676
+ # are the literal X search strings, search_topics are the conceptual seeds
677
+ # they were drafted from (e.g. "MCP client desktop", "AI agent that takes
678
+ # actions"). top_search_topics.py reads twitter_candidates (sidesteps
679
+ # posts.search_topic which was 0% covered for Twitter until this cycle) and
680
+ # returns, per topic: posted vs skipped count, avg virality posted vs
681
+ # skipped, total clicks/likes/views, composite_score. The model uses this to
682
+ # evolve the TOPIC UNIVERSE itself (drop topics with high skipped/posted
683
+ # ratio, mimic topics with non-zero clicks, invent variants of winning
684
+ # topics) rather than just rephrasing within the same fixed set of topics.
685
+ TOP_TOPICS_JSON=$(python3 "$REPO_DIR/scripts/top_search_topics.py" --platform twitter --limit 20 --window-days 14 --json 2>/dev/null || echo "[]")
686
+ TOP_TOPICS_COUNT=$(echo "$TOP_TOPICS_JSON" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))')
687
+ log "Top search topics loaded: $TOP_TOPICS_COUNT (Twitter, 14d window)"
688
+
689
+ # --- Dud queries: phrasings that returned 0 tweets in the last 48h ----------
690
+ # Fed into the prompt as a negative-signal anti-list so the LLM stops
691
+ # redrafting the same flat queries every 20-min cycle. Source is
692
+ # twitter_search_attempts, populated below from this run's queries_used.
693
+ # Now also surfaces the parsed `min_faves` value per dud so the model can
694
+ # spot patterns like "every studyly dud last 48h used min_faves:20 — drop
695
+ # the floor for that project".
696
+ DUD_QUERIES_JSON=$(python3 "$REPO_DIR/scripts/top_dud_twitter_queries.py" --limit 30 --window-hours 48 2>/dev/null || echo "[]")
697
+ DUD_COUNT=$(echo "$DUD_QUERIES_JSON" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))')
698
+ log "Dud queries loaded: $DUD_COUNT (last 48h, 0-result, with min_faves parsed)"
699
+
700
+ # --- Dud topics: CONCEPT SEEDS that find viral but off-fit candidates -------
701
+ # One level up from dud queries. DUD_QUERIES_JSON says "this exact phrasing
702
+ # returns 0 tweets, do not reuse it"; DUD_TOPICS_JSON says "this CONCEPT
703
+ # SEED finds viral tweets but Phase 2b keeps skipping them — the seed is
704
+ # mismatched to your buyers; reword the queries narrower or drop the seed".
705
+ # Surfaces sample_skip_reasons so the model can see WHY (audience mismatch,
706
+ # competitor launch, spam-flagged author, etc.) rather than just numeric
707
+ # skip counts. 7d window so we accumulate enough skips for action thresholds
708
+ # without dragging in stale topics.
709
+ DUD_TOPICS_JSON=$(python3 "$REPO_DIR/scripts/top_dud_twitter_topics.py" --limit 12 --window-hours 168 --min-skips 5 2>/dev/null || echo "[]")
710
+ DUD_TOPICS_COUNT=$(echo "$DUD_TOPICS_JSON" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))')
711
+ log "Dud topics loaded: $DUD_TOPICS_COUNT (7d window, min_skips=5)"
712
+
713
+ # --- Per-project supply signal: what min_faves tier returns tweets? ----------
714
+ # Replaces the old flat "broad=50 / narrow=20" rule. For each project the
715
+ # model is currently drafting for, this table shows the median tweets_found
716
+ # at each min_faves tier we've ever tried, plus zero-result %. The model
717
+ # is instructed to pick the LOWEST min_faves tier that historically yields
718
+ # >=3 median tweets for that project (or step down one tier if every tier
719
+ # is >=3 — supply signal trumps the flat rule). For studyly this auto-
720
+ # selects min_faves:15; for mk0r it stays at 30-50.
721
+ SUPPLY_SIGNAL_JSON=$(python3 "$REPO_DIR/scripts/twitter_supply_signal.py" --window-days 14 2>/dev/null || echo "[]")
722
+ SUPPLY_COUNT=$(echo "$SUPPLY_SIGNAL_JSON" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))')
723
+ log "Per-project supply signal loaded: $SUPPLY_COUNT projects"
724
+
725
+ # --- Recently-engaged tweet IDs: scanner skips tweets we already replied to -
726
+ # The scanner re-searches stable hot topics every cycle, so the same fresh
727
+ # tweets resurface. Once we've replied to one it's a dead candidate
728
+ # (score_twitter_candidates.py dedups it downstream). Injecting the last 48h
729
+ # of engaged status IDs into the scan prompt lets the model skip them while
730
+ # scraping instead of spending tokens evaluating tweets it can't post to.
731
+ # 48h is ample: the 6h freshness wall means any dup is necessarily a recent
732
+ # reply. Scoring remains the backstop; this is purely a token cleanup.
733
+ ENGAGED_TWEET_IDS=$(python3 "$REPO_DIR/scripts/twitter_cycle_helper.py" engaged-tweet-ids --window-hours 48 2>/dev/null || echo "[]")
734
+ [ -z "$ENGAGED_TWEET_IDS" ] && ENGAGED_TWEET_IDS="[]"
735
+ ENGAGED_COUNT=$(echo "$ENGAGED_TWEET_IDS" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))')
736
+ log "Recently-engaged tweet IDs loaded: $ENGAGED_COUNT (last 48h; scanner will skip them)"
737
+
738
+
739
+ # --- Phase 1: Claude drafts queries, scrapes tweets -------------------------
740
+ # JSON schema forces structured output. Eliminates the prose-drift failure mode
741
+ # Lean Phase 1 schema (2026-05-28): the scan session no longer scrapes,
742
+ # it only drafts queries. The Python pipeline runs each query via headless
743
+ # Chrome and writes the tweets directly to SCAN_TWEETS_FILE for the shell.
744
+ SCAN_SCHEMA_LEAN='{"type":"object","properties":{"queries":{"type":"array","items":{"type":"object","properties":{"project":{"type":"string"},"query":{"type":"string"},"search_topic":{"type":"string"}},"required":["project","query","search_topic"]}}},"required":["queries"]}'
745
+
746
+ log "Acquiring twitter-browser lock for Phase 1 Claude scan..."
747
+ acquire_lock "twitter-browser" 3600 2>>"$LOG_FILE"
748
+ log "twitter-browser lock held (pid=$$) Phase 1"
749
+ # Drop stale Chrome singleton symlinks before launch. Background ungraceful-
750
+ # exits (SIGKILL, jetsam, force quit) leave Singleton{Lock,Cookie,Socket}
751
+ # pointing at dead PIDs / vanished sockets; without this, Chrome pops "Something
752
+ # went wrong when opening your profile" 7x and the pipeline hangs. Helper
753
+ # refuses to clean if the lock PID is alive.
754
+ ensure_twitter_browser_for_backend 2>&1 | tee -a "$LOG_FILE"
755
+
756
+ # --- Pre-flight: live X session probe (added 2026-06-02) --------------------
757
+ # Before drafting/scraping anything, confirm the harness Chrome actually has a
758
+ # valid x.com session. One CDP Network.getCookies call (<1s) catches the
759
+ # "import never ran, evaporated after a hard restart, or auth_token expired"
760
+ # cases that previously surfaced as "Phase 1 returned 0 tweets" mysteries.
761
+ # Failing fast here turns a wasted ~7-minute scan + Claude bill into a clear
762
+ # "reconnect X" message in the log.
763
+ # Probe the harness Chrome for a live x.com auth_token. Echoes a single
764
+ # PREFLIGHT_OK / PREFLIGHT_FAIL / PREFLIGHT_CDP_ERROR line. Used pre-cycle and
765
+ # again after an auto-restore from the local cookie mirror.
766
+ _xsession_probe() {
767
+ BU_NAME=twitter-harness BU_CDP_URL=http://127.0.0.1:9555 \
768
+ "$HOME/.local/bin/browser-harness" <<'PY' 2>&1
769
+ import sys, time
770
+ try:
771
+ raw = cdp('Network.getCookies', urls=['https://x.com/', 'https://twitter.com/'])
772
+ except Exception as e:
773
+ print('PREFLIGHT_CDP_ERROR ' + type(e).__name__ + ': ' + str(e))
774
+ sys.exit(0)
775
+ ck = raw.get('cookies', [])
776
+ auth = [c for c in ck if c.get('name') == 'auth_token']
777
+ if not auth:
778
+ print('PREFLIGHT_FAIL no_auth_token cookies_total=' + str(len(ck)))
779
+ sys.exit(0)
780
+ exp = auth[0].get('expires')
781
+ domain = auth[0].get('domain', '?')
782
+ if exp in (None, -1, 0):
783
+ print('PREFLIGHT_OK session domain=' + domain)
784
+ else:
785
+ now = time.time()
786
+ if exp < now:
787
+ print('PREFLIGHT_FAIL auth_token_expired exp=' + str(int(exp)) + ' now=' + str(int(now)))
788
+ sys.exit(0)
789
+ print('PREFLIGHT_OK exp=' + str(int(exp)) + ' domain=' + domain)
790
+ PY
791
+ }
792
+
793
+ log "Pre-flight: probing harness Chrome for a live x.com auth_token..."
794
+ _PREFLIGHT_OUT=$(_xsession_probe)
795
+ if ! printf '%s\n' "$_PREFLIGHT_OUT" | grep -q '^PREFLIGHT_OK'; then
796
+ # Gap B auto-recovery: the harness Chrome lost its x.com session — its cookie
797
+ # store was wiped on a hard restart or a macOS keychain re-lock, or never
798
+ # persisted to disk. Re-inject from the durable 0600 local cookie mirror
799
+ # (written on every connect, keychain-independent) via CDP, then re-probe
800
+ # before giving up. This is what makes the session survive app/Chrome
801
+ # restarts without a manual reconnect.
802
+ log " Pre-flight FAILED ($(printf '%s\n' "$_PREFLIGHT_OUT" | tail -1)); auto-restoring from local cookie mirror..."
803
+ _RESTORE_OUT=$(TWITTER_CDP_URL="${TWITTER_CDP_URL:-http://127.0.0.1:9555}" \
804
+ python3 "$REPO_DIR/scripts/restore_twitter_session.py" 2>&1)
805
+ log " Restore: $(printf '%s\n' "$_RESTORE_OUT" | tail -2 | tr '\n' '|')"
806
+ _PREFLIGHT_OUT=$(_xsession_probe)
807
+ fi
808
+ if printf '%s\n' "$_PREFLIGHT_OUT" | grep -q '^PREFLIGHT_OK'; then
809
+ log " Pre-flight OK: $(printf '%s\n' "$_PREFLIGHT_OUT" | grep '^PREFLIGHT_OK' | head -1)"
810
+ else
811
+ log " Pre-flight FAILED. The harness Chrome has no live X session (auto-restore from the local cookie mirror did not recover it)."
812
+ log " Details: $(printf '%s\n' "$_PREFLIGHT_OUT" | tail -3 | tr '\n' '|')"
813
+ log " Action: run \`python3 scripts/setup_twitter_auth.py connect\` (or call the connect_x MCP tool) to import a fresh X session from your everyday browser, then re-run the cycle. If the import fails with 'access denied', unlock the macOS keychain first: \`security unlock-keychain ~/Library/Keychains/login.keychain-db\`."
814
+ echo "twitter_batches: ended $BATCH_ID"
815
+ release_lock "twitter-browser" 2>/dev/null || true
816
+ exit 1
817
+ fi
818
+
819
+ # --- Pre-flight 2: live access-gate probe + backoff (added 2026-06-29) -------
820
+ # The cookie probe above only proves an auth_token EXISTS. X can still gate a
821
+ # perfectly valid session: from a datacenter IP (e.g. the MacStadium box) it
822
+ # commonly 302s authenticated routes to /account/access ("verify it's you") or
823
+ # fronts them with a Cloudflare "security verification" interstitial. A gated
824
+ # session renders real, public tweets as "this page doesn't exist", so the scan
825
+ # silently returns 0-few candidates and we'd draft/post against phantom
826
+ # emptiness (this is one root of the old "Phase 1 returned 0 tweets" mysteries
827
+ # that the cookie probe alone never caught). Navigate ONE authenticated route
828
+ # and STOP the cycle if X is gating us. Fails OPEN: a probe error or an
829
+ # ok/unknown render never blocks, so a transient hydration miss can't halt
830
+ # posting — only a positively-detected gate (gated:true) stops the cycle.
831
+ #
832
+ # BACKOFF: this launchd job fires every 5 min, and a gated cycle exits in ~2s,
833
+ # so without backoff we'd hit Cloudflare /account/access ~12x/hr (~288/day),
834
+ # which only deepens the datacenter-IP trust penalty. A state marker records the
835
+ # gate and an exponential cooldown (15m -> 30m -> 60m -> cap 120m). While the
836
+ # cooldown is live we skip the cycle WITHOUT navigating (no flagged traffic);
837
+ # once it elapses we re-probe; an 'ok' probe clears the marker and resumes. Only
838
+ # gated:true ever writes the marker, so fail-open is preserved.
839
+ _S4L_STATE_DIR="${S4L_STATE_DIR:-$HOME/.social-autoposter-mcp}"
840
+ _GATE_FILE="$_S4L_STATE_DIR/x-access-gate.json"
841
+ _NOW=$(date +%s)
842
+
843
+ # Backoff short-circuit: still inside a cooldown window -> skip without probing.
844
+ if [ -f "$_GATE_FILE" ]; then
845
+ _CD_UNTIL=$(python3 -c 'import json,sys
846
+ try: print(int(json.load(open(sys.argv[1])).get("cooldown_until",0)))
847
+ except Exception: print(0)' "$_GATE_FILE" 2>/dev/null || echo 0)
848
+ if [ "${_CD_UNTIL:-0}" -gt "$_NOW" ]; then
849
+ _MINS=$(( (_CD_UNTIL - _NOW + 59) / 60 ))
850
+ log "Pre-flight: X access-gate backoff active (~${_MINS}m left); skipping cycle without re-probing to avoid adding flagged Cloudflare traffic."
851
+ echo "twitter_batches: ended $BATCH_ID"
852
+ release_lock "twitter-browser" 2>/dev/null || true
853
+ exit 0
854
+ fi
855
+ fi
856
+
857
+ log "Pre-flight: probing for an X access gate (/account/access, Cloudflare)..."
858
+ _ACCESS_OUT=$(TWITTER_CDP_URL="${TWITTER_CDP_URL:-http://127.0.0.1:9555}" \
859
+ python3 "$REPO_DIR/scripts/twitter_access_check.py" --session-probe --wait-ms 12000 2>/dev/null)
860
+ if printf '%s' "$_ACCESS_OUT" | grep -q '"gated": *true'; then
861
+ # Write/refresh the backoff marker with an exponential cooldown. Python
862
+ # prints "<next_mins> <consecutive> <cooldown_secs> <gate_age_secs>".
863
+ _GATE_FIELDS=$(python3 -c 'import json,sys
864
+ gf, now = sys.argv[1], int(sys.argv[2])
865
+ base, cap, factor = 900, 7200, 2
866
+ try: prev = json.load(open(gf))
867
+ except Exception: prev = {}
868
+ cd = prev.get("cooldown_secs")
869
+ cd = base if not cd else min(int(cd)*factor, cap)
870
+ fs = int(prev.get("first_seen", now))
871
+ cons = int(prev.get("consecutive", 0)) + 1
872
+ out = {"first_seen": fs, "last_seen": now, "reason": "access_gated",
873
+ "consecutive": cons, "cooldown_secs": cd, "cooldown_until": now+cd}
874
+ json.dump(out, open(gf, "w"))
875
+ print(cd//60, cons, cd, max(0, now-fs))' "$_GATE_FILE" "$_NOW" 2>/dev/null || echo "15 1 900 0")
876
+ read -r _NEXT_MINS _CONS _CD_SECS _AGE_SECS <<< "$_GATE_FIELDS"
877
+ log " Pre-flight FAILED: X is gating this session (access gate detected)."
878
+ log " Probe: $(printf '%s' "$_ACCESS_OUT" | tr '\n' ' ' | tr -s ' ' | sed 's/^ *//')"
879
+ log " X redirected an authenticated route to /account/access or served a Cloudflare verification page. This is usually datacenter-IP trust degradation: the session cookie is still valid but X hides content from it, so a scan would return phantom 'doesn't exist' results."
880
+ log " Backoff engaged: next access re-probe in ~${_NEXT_MINS}m (intervening 5-min firings skip without touching Cloudflare)."
881
+ log " Action: open the harness Chrome (CDP :9555) and complete the verification at https://x.com/account/access once, or route the box through a residential/clean IP. The cycle auto-resumes within one cooldown of the gate lifting."
882
+ # Machine-greppable marker (additive; mirrors the stderr-marker convention
883
+ # bin/server.js parses). Pairs with twitter_access_gate:recovered below.
884
+ echo "twitter_access_gate: gated consecutive=${_CONS} age_s=${_AGE_SECS} next_reprobe_s=${_CD_SECS}" >&2
885
+ echo "twitter_batches: ended $BATCH_ID"
886
+ release_lock "twitter-browser" 2>/dev/null || true
887
+ exit 1
888
+ fi
889
+ # Probe came back clean. If a backoff marker exists we were gated: record the
890
+ # recovery (how long the gate lasted, since first_seen) BEFORE deleting it, so
891
+ # the lift event + duration survive in the log even though the marker is gone.
892
+ if [ -f "$_GATE_FILE" ]; then
893
+ _REC=$(python3 -c 'import json,sys
894
+ try: d = json.load(open(sys.argv[1]))
895
+ except Exception: d = {}
896
+ now = int(sys.argv[2]); fs = int(d.get("first_seen", now)); cons = int(d.get("consecutive", 0))
897
+ dur = max(0, now-fs)
898
+ print(dur, dur//60, cons)' "$_GATE_FILE" "$_NOW" 2>/dev/null || echo "0 0 0")
899
+ read -r _DUR_S _DUR_M _RCONS <<< "$_REC"
900
+ rm -f "$_GATE_FILE"
901
+ echo "twitter_access_gate: recovered_after_s=${_DUR_S} consecutive=${_RCONS}" >&2
902
+ log " X access gate lifted after ~${_DUR_M}m (${_RCONS} consecutive gated probes); cleared backoff marker and resuming normal cycle."
903
+ fi
904
+ log " Pre-flight access OK: $(printf '%s' "$_ACCESS_OUT" | tr '\n' ' ' | tr -s ' ' | sed 's/^ *//')"
905
+
906
+ # --- Phase 1 retry loop (2026-05-27) ----------------------------------------
907
+ # When a single scan produces fewer than RETRY_TARGET candidates that survive
908
+ # all Phase 1 filters (harness age gate, scorer stale_age cutoff, already-
909
+ # posted dedupe, fabricated_id check), re-invoke the Claude scan with the
910
+ # queries already tried this cycle injected as a "do NOT repeat" block.
911
+ # Each iteration upserts into the SAME batch_id so survivors accumulate.
912
+ # Cap at MAX_SCAN_ATTEMPTS to stay inside the 20-min Phase 1 budget; if the
913
+ # cap is hit before target, proceed with whatever we have (even 1 candidate
914
+ # is better than 0). When BATCH_COUNT is still 0 after the loop, the
915
+ # post-loop empty_batch branch fires.
916
+ # DEFAULT Phase 1 is the deterministic qualified-query bank (no Claude): the
917
+ # bank replays every historically qualified query for the picked project in a
918
+ # single pass, so there is nothing to "retry-draft" and one attempt is enough.
919
+ # The legacy LLM-draft path (TWITTER_PHASE1_LLM_DRAFT=1) keeps the 5-attempt
920
+ # retry loop, because LLM queries frequently return empty and need re-drafting.
921
+ if [ "${TWITTER_PHASE1_LLM_DRAFT:-0}" = "1" ]; then
922
+ MAX_SCAN_ATTEMPTS=5
923
+ else
924
+ MAX_SCAN_ATTEMPTS=1
925
+ fi
926
+ RETRY_TARGET=5
927
+ SCAN_ATTEMPT=0
928
+ BATCH_COUNT=0
929
+ # Cumulative counters across iterations — feed log_run.py once at end so the
930
+ # dashboard shows the total work the cycle did, not just the last attempt.
931
+ CUMULATIVE_QUERIES=0
932
+ CUMULATIVE_DUDS=0
933
+ CUMULATIVE_TWEETS_PULLED=0
934
+ # Running list of queries the model has already tried THIS cycle, injected
935
+ # into each retry's prompt as "do NOT repeat these phrasings". Extended after
936
+ # every scan from QUERIES_FILE before log_twitter_search_attempts deletes it.
937
+ TRIED_QUERIES_JSON='[]'
938
+ # Running list of SEARCH TOPICS already tried this cycle. Each retry calls
939
+ # pick_topic_for_project again with this list as exclude_topics so the model
940
+ # isn't pinned to one assigned topic for all 5 attempts. When the filtered
941
+ # universe empties (small-project case), the picker raises
942
+ # UniverseExhaustedError and the retry loop breaks — no invent fallback
943
+ # (invention is the standalone invent_topics.py job's responsibility).
944
+ TRIED_TOPICS_JSON='[]'
945
+ # Latest Anthropic-side error classification for the post-loop log_run when
946
+ # every attempt returned zero tweets (stream_idle_timeout vs phase1_no_tweets
947
+ # vs api_overloaded, etc.). Falls back to phase1_no_tweets when unset.
948
+ LAST_PHASE1_REASON=""
949
+ # Set to 1 by the in-loop repick when pick_search_topic raises
950
+ # UniverseExhaustedError (the project ran out of un-tried active topics).
951
+ # Used by the post-loop empty-batch branch to emit `universe_exhausted:1`
952
+ # instead of `empty_batch:1` so the dashboard shows the right cause.
953
+ UNIVERSE_EXHAUSTED=0
954
+
955
+ while [ "$SCAN_ATTEMPT" -lt "$MAX_SCAN_ATTEMPTS" ]; do
956
+ SCAN_ATTEMPT=$((SCAN_ATTEMPT + 1))
957
+ # Snapshot the pre-attempt batch size so the verdict step below can compute
958
+ # kept_after_skip as a delta after the scorer finishes this attempt (2026-05-28
959
+ # retry-feedback: turns TRIED_QUERIES_JSON from bare phrasings into per-query
960
+ # verdicts the drafter can use to choose broaden vs narrow vs new-topic).
961
+ BATCH_COUNT_BEFORE_ATTEMPT="${BATCH_COUNT:-0}"
962
+
963
+ # --- Per-attempt topic (re)pick (2026-05-27) ---------------------------------
964
+ # Attempt 1 keeps the pre-loop topic that PROJECTS_JSON already carries.
965
+ # Attempts 2+ call pick_topic_for_project again with TRIED_TOPICS_JSON as
966
+ # exclude_topics, then rewrite PROJECTS_JSON in place with the new topic and
967
+ # its reference_topics. This makes the retry genuinely end-to-end programmatic:
968
+ # new topic -> new query -> new tweets, not just "model rephrases the same
969
+ # assigned topic 5 times". When the project's filtered universe empties
970
+ # (small project, all topics tried this cycle), the picker raises
971
+ # UniverseExhaustedError and the shell breaks the retry loop cleanly
972
+ # (post-2026-05-28: no invent fallback; invention lives in invent_topics.py).
973
+ if [ "$SCAN_ATTEMPT" -gt 1 ]; then
974
+ log "Phase 1 attempt $SCAN_ATTEMPT: re-picking search_topic via pick_topic_for_project (exclude=$(echo "$TRIED_TOPICS_JSON" | python3 -c 'import json,sys;print(len(json.load(sys.stdin)))' 2>/dev/null || echo 0) tried)..."
975
+ # The exhaustion marker file is the cross-boundary signal back to
976
+ # bash: when pick_topic_for_project raises UniverseExhaustedError for
977
+ # ANY selected project, the Python writes this file and the shell
978
+ # breaks the retry loop after the heredoc returns. No invent fallback
979
+ # (2026-05-28 architecture: invention is the standalone
980
+ # invent_topics.py job's responsibility, not the cycle's).
981
+ UNIVERSE_EXHAUSTED_MARKER="/tmp/twitter_cycle_universe_exhausted_${BATCH_ID}"
982
+ rm -f "$UNIVERSE_EXHAUSTED_MARKER"
983
+ PROJECTS_JSON=$(python3 - "$PROJECTS_JSON" "$TRIED_TOPICS_JSON" "$UNIVERSE_EXHAUSTED_MARKER" <<'PY' 2>>"$LOG_FILE"
984
+ import json, os, sys
985
+ sys.path.insert(0, os.path.expanduser('~/social-autoposter/scripts'))
986
+ from pick_search_topic import pick_topic_for_project, PickerError, UniverseExhaustedError
987
+
988
+ projects = json.loads(sys.argv[1] or '[]')
989
+ excluded = json.loads(sys.argv[2] or '[]')
990
+ marker_path = sys.argv[3]
991
+ exhausted_for = []
992
+ for p in projects:
993
+ name = p.get('name')
994
+ if not name:
995
+ continue
996
+ try:
997
+ new_pick = pick_topic_for_project(
998
+ name, platform='twitter', exclude_topics=excluded,
999
+ )
1000
+ except UniverseExhaustedError as exc:
1001
+ # All active topics for this project already tried this cycle.
1002
+ # Stamp the marker file so the shell breaks the retry loop and
1003
+ # logs `universe_exhausted:1` as the failure reason. Leave the
1004
+ # project entry as-is (the loop will exit before scanning anyway).
1005
+ sys.stderr.write(f"repick_universe_exhausted project={name!r} error={exc}\n")
1006
+ exhausted_for.append(name)
1007
+ continue
1008
+ except PickerError as exc:
1009
+ # On repick PickerError (DB unreachable, etc) keep the previous
1010
+ # topic; the scan will still run. Strictly better than aborting.
1011
+ sys.stderr.write(f"repick_failed project={name!r} error={exc}\n")
1012
+ continue
1013
+ p['search_topic'] = new_pick.get('search_topic')
1014
+ p['picked_weight_pct'] = new_pick.get('picked_weight_pct')
1015
+ p['reference_topics'] = new_pick.get('reference_topics') or []
1016
+ if exhausted_for:
1017
+ with open(marker_path, 'w') as fh:
1018
+ fh.write(','.join(exhausted_for) + '\n')
1019
+ print(json.dumps(projects))
1020
+ PY
1021
+ )
1022
+ if [ -f "$UNIVERSE_EXHAUSTED_MARKER" ]; then
1023
+ UNIVERSE_EXHAUSTED=1
1024
+ _EXH_PROJECTS=$(cat "$UNIVERSE_EXHAUSTED_MARKER" 2>/dev/null | tr -d '\n')
1025
+ log " Universe exhausted for project(s)=$_EXH_PROJECTS after $((SCAN_ATTEMPT - 1)) prior attempt(s); breaking retry loop"
1026
+ rm -f "$UNIVERSE_EXHAUSTED_MARKER"
1027
+ break
1028
+ fi
1029
+ fi
1030
+
1031
+ # Snapshot this attempt's topic(s) into TRIED_TOPICS_JSON so the NEXT
1032
+ # iteration's repick excludes them. Runs every attempt (incl. attempt 1)
1033
+ # so the initial pre-loop topic also goes into the exclude list before
1034
+ # attempt 2's repick. Idempotent: same topic added twice is a no-op.
1035
+ TRIED_TOPICS_JSON=$(python3 - "$TRIED_TOPICS_JSON" "$PROJECTS_JSON" <<'PY' 2>>"$LOG_FILE"
1036
+ import json, sys
1037
+ cur = json.loads(sys.argv[1] or '[]')
1038
+ projects = json.loads(sys.argv[2] or '[]')
1039
+ seen = {(t or '').strip().lower() for t in cur if t}
1040
+ for p in projects:
1041
+ t = (p.get('search_topic') or '').strip()
1042
+ if t and t.lower() not in seen:
1043
+ cur.append(t)
1044
+ seen.add(t.lower())
1045
+ print(json.dumps(cur))
1046
+ PY
1047
+ )
1048
+
1049
+ _CURRENT_TOPICS=$(echo "$PROJECTS_JSON" | python3 -c 'import json,sys; ps=json.load(sys.stdin); print(", ".join((p.get("search_topic") or "?") for p in ps))' 2>/dev/null || echo "")
1050
+ log "Phase 1 scan attempt $SCAN_ATTEMPT/$MAX_SCAN_ATTEMPTS (batch=$BATCH_ID, candidates so far=$BATCH_COUNT/$RETRY_TARGET, topic(s)=$_CURRENT_TOPICS)"
1051
+
1052
+ log "Phase 1: drafting queries and scraping tweets..."
1053
+
1054
+ # Shell-side data path. scripts/twitter_scan.scan() appends one JSONL record
1055
+ # per call to this file. After the claude scan session ends we parse it
1056
+ # directly into $RAW_FILE and $QUERIES_FILE, bypassing the model's
1057
+ # structured_output relay so the model no longer pays per-tweet copy tokens.
1058
+ # One file per Phase 1 attempt so retry iterations do not share state. The
1059
+ # rm -f makes each attempt's accumulation start clean. Falls back to the
1060
+ # structured_output parse below when the file is empty (e.g. every bh_run was
1061
+ # denied by the stub-enforcement hook so scan() never executed).
1062
+ SCAN_TWEETS_FILE="/tmp/twcycle-${BATCH_ID}-attempt-${SCAN_ATTEMPT}-tweets.jsonl"
1063
+ rm -f "$SCAN_TWEETS_FILE"
1064
+ export SCAN_TWEETS_FILE
1065
+
1066
+ # === LEAN PHASE 1 (2026-05-28) =============================================
1067
+ # Replaces the model-driven Twitter scrape with: small Claude call that only
1068
+ # DRAFTS queries (no tools, no MCP, no browser), then a Python loop that runs
1069
+ # each query via the operator-owned twitter_scan.scan() function over the same
1070
+ # CDP daemon. Cuts per-cycle scan cost roughly 10x by removing:
1071
+ # - MCP bh_run tool roundtrips
1072
+ # - structured_output tweet relay (was hundreds of tweet objects)
1073
+ # - draft-deny-retry churn (model used to try inline scrapes and bounce off
1074
+ # the PreToolUse stub-enforcement hook every session)
1075
+ # Output downstream is identical: $RAW_FILE + $QUERIES_FILE feed the scorer
1076
+ # and twitter_search_attempts logger the same way as before.
1077
+ #
1078
+ if [ "${TWITTER_PHASE1_LLM_DRAFT:-0}" = "1" ]; then
1079
+ # === LLM QUERY-DRAFT PATH (legacy, behind TWITTER_PHASE1_LLM_DRAFT=1) ========
1080
+ log "Lean Phase 1: drafting queries (no browser tools)..."
1081
+
1082
+ QUERIES_OUTPUT=$("$REPO_DIR/scripts/run_claude.sh" "run-twitter-cycle-queries" --strict-mcp-config --mcp-config "$TW_MCP_CONFIG" -p --output-format json --json-schema "$SCAN_SCHEMA_LEAN" "${TW_ENGINE_PREFIX}You are a Twitter query drafter. Your ONLY job is to draft fresh X advanced-search queries that surface tweets relevant to our projects. You do NOT post, you do NOT call any tools, you do NOT scrape. A separate Python pipeline runs your queries over the same CDP-driven Chrome and applies a strict freshness gate; you only return the query strings.
1083
+
1084
+ ## Step 1: Draft one search query per project
1085
+
1086
+ You have $(echo "$PROJECTS_JSON" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))') projects. Draft exactly ONE Twitter search query for each, tailored to that project's ASSIGNED search_topic.
1087
+
1088
+ Each project entry carries TWO fields that drive your behavior: \`topic_picked_mode\` (either \`use\` or \`explore_invent\`) and \`search_topic\` (a string in \`use\` mode, NULL in \`explore_invent\` mode).
1089
+
1090
+ USE mode (~90% of cycles, indicated by \`topic_picked_mode: \"use\"\` and a non-null \`search_topic\`):
1091
+ The Python picker has already chosen this project's search_topic by weighted-random sampling over the FULL universe in config.json. Your job is to translate that ASSIGNED topic into the best Twitter advanced-search query that will surface fresh, on-topic tweets. Do NOT substitute a different topic; do NOT paraphrase the topic. End-to-end attribution joins on the exact string.
1092
+
1093
+ EXPLORE_INVENT mode (~10% of cycles, indicated by \`topic_picked_mode: \"explore_invent\"\` and \`search_topic: null\`):
1094
+ The picker is asking you to INVENT a brand-new search_topic. Look at the project's own \`reference_topics\` array and propose ONE new topic concept that does NOT appear there and is NOT a paraphrase of anything in it. Use your invented topic as the query's \`search_topic\` AND drive the keyword phrasing from it (one consistent string per project).
1095
+
1096
+ Projects:
1097
+ $PROJECTS_JSON
1098
+
1099
+ Top past queries FOR THE PROJECT YOU'RE DRAFTING FOR (7-day window, per-project, sorted by clicks DESC first, then composite-scored: clicks×100 + likes + views×0.001). CLICKS ARE THE PRIORITY SIGNAL. Each row carries THREE labels that tell you what to do with it as a reference:
1100
+
1101
+ - \`supply_bucket\`: low (<1 tweet/attempt), medium (1-5), high (>5). Raw supply X returned for this phrasing.
1102
+ - \`conversion_bucket\`: low (<0.2 post_rate), medium (0.2-0.6), high (>=0.6). How often a found tweet survived the draft gate.
1103
+ - \`guidance\`: one of MIMIC, KEEP_STYLE, NARROW, BROADEN — the action to take when drawing from this query.
1104
+ - \`posts_per_attempt\`: posts produced per Phase 1 search invocation; <0.1 means most attempts produce zero survivors.
1105
+
1106
+ How to act on \`guidance\`:
1107
+ - MIMIC — gold tier. Reuse the operator skeleton verbatim, swap only the topic keyword for the picker-assigned topic.
1108
+ - KEEP_STYLE — solid. Use the operator pattern as inspiration; small phrasing tweaks OK.
1109
+ - NARROW — high supply, low conversion (noisy pond). If you draw from it, ADD specificity: more OR alternates, stricter min_faves, extra -term excludes.
1110
+ - BROADEN — low supply (query dying or topic running dry). The OPERATORS are dead weight. Shorten to 1-2 keywords, drop OR groups, step min_faves down a tier. Do NOT inherit operators from a BROADEN-tagged row.
1111
+
1112
+ The canonical source for \`min_faves:N\` selection is the PER-PROJECT SUPPLY SIGNAL block below.
1113
+ $TOP_QUERIES_PER_PROJECT_JSON
1114
+
1115
+ TOP-PERFORMING SEARCH TOPICS (conceptual seeds, 14d window) — context for query phrasing only; you draft a query for the picker-assigned topic, you do NOT swap topics here:
1116
+ $TOP_TOPICS_JSON
1117
+
1118
+ DUD QUERIES — DO NOT REUSE these phrasings or close variants. They returned ZERO tweets in the last 48h:
1119
+ $DUD_QUERIES_JSON
1120
+
1121
+ DUD CONCEPT SEEDS — these search_topic seeds pulled in tweets that Phase 2b's draft gate kept skipping over the last 7d. Per entry: \`omit_rate\` = skipped_n / (posted_n + skipped_n), \`sample_skip_reasons\` are the top reject reasons. If \`omit_rate >= 0.6\` AND \`skipped_n >= 5\`, REWORD the query narrower or drop the seed and pick a different config.json seed for that project:
1122
+ $DUD_TOPICS_JSON
1123
+
1124
+ PER-PROJECT SUPPLY SIGNAL — for each project, the historical median tweets_found at each \`min_faves:N\` tier you've drafted in the last 14d. Pick the LOWEST tier where \`median_tweets_found >= 3\`; if every tier is below 3, drop one tier lower than the lowest you've tried. Trust this table over priors:
1125
+ $SUPPLY_SIGNAL_JSON
1126
+
1127
+ ALREADY-ENGAGED TWEET IDS (last 48h) — the Python scraper skips these regardless, but knowing them helps you avoid drafting a query that would predominantly surface dead candidates:
1128
+ $ENGAGED_TWEET_IDS
1129
+
1130
+ THIS-CYCLE QUERIES ALREADY TRIED with per-query outcomes (attempt $SCAN_ATTEMPT/$MAX_SCAN_ATTEMPTS, target=$RETRY_TARGET candidates after filters). Do NOT repeat any of these phrasings or close variants. Read each entry's \`verdict\` field and respond directionally (do NOT default to generic "broaden"):
1131
+ - \`dead_supply\` (raw_tweets=0): the phrasing returned ZERO tweets from X. The query was too narrow for X's index. HARD RULE: attempt N+1 MUST execute at least ONE of these THREE concrete broadening moves, NOT a topic rotation. Pick exactly one and apply it visibly: (a) lower \`min_faves\` by ONE FULL TIER (e.g. 20→5, 5→1, 1→0); (b) reduce the OR alternates inside any parenthesized group to AT MOST 2 terms (e.g. \`(A OR B OR C OR D)\` → \`(A OR B)\`); (c) drop ALL \`-term\` excludes EXCEPT those listed in this project's \`excludes_for_search\` (which remain mandatory). The PER-PROJECT SUPPLY SIGNAL block is OVERRIDDEN by \`dead_supply\` THIS CYCLE — do not appeal to historical min_faves when the current attempt returned 0. Swapping the topic noun while keeping the same operator skeleton is NOT broadening and is FORBIDDEN as a response to \`dead_supply\`.
1132
+ - \`all_aged_out\` (raw>0, kept_after_age=0): topic is supply-limited at the current freshness window; every tweet was older than the cap. Pick a structurally adjacent topic; do NOT rephrase the same one (it will just hit the cap again).
1133
+ - \`all_engaged_or_skipped\` (kept_after_age>0, kept_after_skip=0): query phrasing is fine, but the surviving tweets were already engaged on prior cycles. Pick a DIFFERENT topic, not a rephrase.
1134
+ - \`found_some\` (kept_after_skip>0 but below target): query is on-target. Raise min_faves one tier OR add a semantic constraint to lift quality. Do NOT broaden.
1135
+ $TRIED_QUERIES_JSON
1136
+
1137
+ Query guidelines:
1138
+ - MANDATORY: do NOT add any date or time-window operator to your query (no \`since:\`, \`until:\`, \`since_time:\`, \`until_time:\`). The Python scraper enforces the freshness window at the URL level after you return; any time operator you include is stripped and overwritten. Including raw bash arithmetic, format strings, or placeholder text in place of a real epoch will be sent to X as a literal keyword and produce zero results.
1139
+ - MANDATORY EVEN IF YOUR QUERY KEYWORDS DO NOT NAME THE EXCLUDED TOPIC: if a project's \`excludes_for_search\` array is non-empty, append \`-term\` for EVERY listed term to that project's query, verbatim, no exceptions.
1140
+ - MANDATORY: pick \`min_faves:N\` per the PER-PROJECT SUPPLY SIGNAL above. If a project has no entry there (new / first cycle), start at min_faves:20.
1141
+ - Favor discussions/opinions (people sharing experience, asking questions), not news/promos/giveaways.
1142
+ - Pick a query likely to surface tweets RELEVANT to that project's actual domain.
1143
+ - Mix it up each run; don't always use the same query for the same project.
1144
+ - Use the project's ASSIGNED \`search_topic\` plus its \`description\` as grounding for query phrasing.
1145
+ - The \`search_topic\` you emit in the output JSON MUST be the project's assigned \`search_topic\` field pasted VERBATIM (NOT the query string, NOT a paraphrase). The scoring pipeline stamps \`twitter_candidates.search_topic\` from this for end-to-end attribution.
1146
+
1147
+ ## Output
1148
+
1149
+ Return ONLY the structured_output JSON with this shape:
1150
+ {\"queries\": [{\"project\": \"PROJECT_NAME\", \"query\": \"X advanced search string with operators\", \"search_topic\": \"assigned or invented topic, verbatim\"}, ...]}
1151
+
1152
+ One entry per project. Do NOT include tweets, do NOT include tweets_found, do NOT call any tool, do NOT scrape. The shell pipeline runs each query via headless Chrome with a strict freshness gate after you return." 2>&1)
1153
+
1154
+
1155
+ # Dump the captured envelope to the cycle log for offline inspection.
1156
+ echo "$QUERIES_OUTPUT" >> "$LOG_FILE"
1157
+
1158
+ # Extract the drafted queries to a temp file.
1159
+ QUERIES_TMP="/tmp/twcycle-${BATCH_ID}-attempt-${SCAN_ATTEMPT}-queries.json"
1160
+ python3 -c "
1161
+ import json, sys
1162
+ text = sys.stdin.read().strip()
1163
+ try:
1164
+ env, _ = json.JSONDecoder().raw_decode(text)
1165
+ except Exception as e:
1166
+ print(f'lean phase 1: envelope parse error: {e}', file=sys.stderr)
1167
+ json.dump([], open('$QUERIES_TMP', 'w'))
1168
+ sys.exit(0)
1169
+ so = env.get('structured_output')
1170
+ if so is None:
1171
+ so = env.get('result')
1172
+ if isinstance(so, str):
1173
+ try: so = json.loads(so)
1174
+ except Exception: pass
1175
+ qs = so.get('queries', []) if isinstance(so, dict) else []
1176
+ json.dump(qs, open('$QUERIES_TMP', 'w'))
1177
+ print(f'lean phase 1: drafted {len(qs)} queries', flush=True)
1178
+ " <<< "$QUERIES_OUTPUT" 2>&1 | tee -a "$LOG_FILE"
1179
+
1180
+ else
1181
+ # === DETERMINISTIC QUALIFIED-QUERY-BANK PATH (default, 2026-05-28) ==========
1182
+ # No Claude call. Replay every historically qualified query for the picked
1183
+ # project(s): every distinct query that ever produced a posted reply with
1184
+ # >=1 like OR >=1 non-bot link click, regardless of the per-cycle
1185
+ # search_topic. This makes Phase 1 fully deterministic; the only remaining
1186
+ # Claude session in the cycle is Phase 2b (reply drafting). The bank is
1187
+ # exhaustive on attempt 1, so MAX_SCAN_ATTEMPTS is forced to 1 above; the
1188
+ # attempt>1 guard here is belt-and-suspenders for the legacy retry loop.
1189
+ QUERIES_TMP="/tmp/twcycle-${BATCH_ID}-attempt-${SCAN_ATTEMPT}-queries.json"
1190
+ if [ "$SCAN_ATTEMPT" -gt 1 ]; then
1191
+ echo "[]" > "$QUERIES_TMP"
1192
+ log "Phase 1 (bank): attempt $SCAN_ATTEMPT no-op (full bank already run on attempt 1)"
1193
+ else
1194
+ log "Phase 1 (bank): building qualified query bank from PROJECTS_JSON (deterministic, no Claude)..."
1195
+ echo "$PROJECTS_JSON" | python3 "$REPO_DIR/scripts/qualified_query_bank.py" --from-projects-json > "$QUERIES_TMP" 2>>"$LOG_FILE"
1196
+ fi
1197
+ fi
1198
+
1199
+ QUERIES_COUNT=$(python3 -c "
1200
+ import json
1201
+ try: print(len(json.load(open('$QUERIES_TMP'))))
1202
+ except Exception: print(0)
1203
+ " 2>/dev/null || echo 0)
1204
+
1205
+ # Loop: for each drafted query, run scan() over the same browser-harness daemon
1206
+ # the cycle already keeps alive (port 9555, BU_NAME=twitter-harness). One
1207
+ # browser-harness invocation handles the full loop so we don't pay the CLI
1208
+ # startup cost N times. Each scan() call appends one JSONL record to
1209
+ # $SCAN_TWEETS_FILE, which the existing shell-side parse below consumes.
1210
+ if [ "$QUERIES_COUNT" -gt 0 ]; then
1211
+ log "Lean Phase 1: executing $QUERIES_COUNT queries via browser-harness CDP"
1212
+ # browser-harness upstream main reads the script from STDIN (the `-c` flag was
1213
+ # removed). Feed the body via a quoted heredoc and pass $REPO_DIR / $QUERIES_TMP
1214
+ # through the environment so the Python reads them from os.environ (no shell
1215
+ # expansion inside the heredoc). Keep the local CLI in sync with upstream main:
1216
+ # `uv tool install -e ~/Developer/browser-harness --force` after a git pull.
1217
+ BU_NAME=twitter-harness BU_CDP_URL=http://127.0.0.1:9555 \
1218
+ SCAN_TWEETS_FILE="$SCAN_TWEETS_FILE" \
1219
+ BATCH_ID="$BATCH_ID" \
1220
+ TWITTER_CYCLE_VARIANT="$TWITTER_CYCLE_VARIANT" \
1221
+ FRESHNESS_HOURS_DISCOVER="$FRESHNESS_HOURS_DISCOVER" \
1222
+ ENGAGED_TWEET_IDS="$ENGAGED_TWEET_IDS" \
1223
+ REPO_DIR="$REPO_DIR" \
1224
+ QUERIES_TMP="$QUERIES_TMP" \
1225
+ "$HOME/.local/bin/browser-harness" <<'PY' 2>&1 | tee -a "$LOG_FILE"
1226
+ import sys, json, os, time
1227
+ sys.path.insert(0, os.environ['REPO_DIR'] + '/scripts')
1228
+ from twitter_scan import scan
1229
+ queries = json.load(open(os.environ['QUERIES_TMP']))
1230
+ freshness = int(os.environ.get('FRESHNESS_HOURS_DISCOVER', '6'))
1231
+ skip_ids = json.loads(os.environ.get('ENGAGED_TWEET_IDS', '[]'))
1232
+ for q in queries:
1233
+ project = q.get('project', '')
1234
+ query = q.get('query', '')
1235
+ topic = q.get('search_topic', '')
1236
+ t0 = time.time()
1237
+ try:
1238
+ kept = scan(
1239
+ query=query,
1240
+ project=project,
1241
+ search_topic=topic,
1242
+ freshness_hours=freshness,
1243
+ skip_ids=skip_ids,
1244
+ )
1245
+ dt = time.time() - t0
1246
+ print(f' ok project={project!r} q={query[:50]!r} kept={len(kept)} in {dt:.1f}s', flush=True)
1247
+ except Exception as e:
1248
+ dt = time.time() - t0
1249
+ print(f' err project={project!r} q={query[:50]!r} in {dt:.1f}s {type(e).__name__}: {e}', flush=True)
1250
+ PY
1251
+ fi
1252
+ rm -f "$QUERIES_TMP"
1253
+
1254
+ # Shell-side parse of $SCAN_TWEETS_FILE -> $RAW_FILE + $QUERIES_FILE. Identical
1255
+ # to the prior shell-side branch; the structured_output fallback is no longer
1256
+ # wired because the lean flow always produces SCAN_TWEETS_FILE (scan() writes
1257
+ # even on zero-tweet calls). If SCAN_TWEETS_FILE is missing entirely (e.g. the
1258
+ # Claude call returned no queries), write empty arrays so downstream scoring
1259
+ # treats this attempt as a zero-result Phase 1 and the retry loop fires.
1260
+ if [ -s "$SCAN_TWEETS_FILE" ]; then
1261
+ log "Parsing tweets from $SCAN_TWEETS_FILE"
1262
+ python3 -c "
1263
+ import json, sys
1264
+ recs = []
1265
+ for ln in open('$SCAN_TWEETS_FILE'):
1266
+ ln = ln.strip()
1267
+ if not ln:
1268
+ continue
1269
+ try:
1270
+ recs.append(json.loads(ln))
1271
+ except json.JSONDecodeError:
1272
+ print(f'shell-side: skipping bad JSONL line', file=sys.stderr)
1273
+ tweets = []
1274
+ queries_used = []
1275
+ for r in recs:
1276
+ ts = r.get('tweets') or []
1277
+ tweets.extend(ts)
1278
+ queries_used.append({
1279
+ 'query': r.get('query', ''),
1280
+ 'project': r.get('project', ''),
1281
+ 'tweets_found': len(ts),
1282
+ 'search_topic': r.get('search_topic', ''),
1283
+ })
1284
+ json.dump(queries_used, open('$QUERIES_FILE', 'w'))
1285
+ json.dump(tweets, open('$RAW_FILE', 'w'))
1286
+ print(f'shell-side parse: {len(tweets)} tweets, {len(queries_used)} attempts from SCAN_TWEETS_FILE', flush=True)
1287
+ sys.exit(0 if tweets else 1)
1288
+ " 2>&1 | tee -a "$LOG_FILE"
1289
+ EXTRACT_EXIT=${PIPESTATUS[0]:-1}
1290
+ else
1291
+ log "no SCAN_TWEETS_FILE this attempt (0 queries drafted or every scrape errored)"
1292
+ : > "$QUERIES_FILE"
1293
+ : > "$RAW_FILE"
1294
+ EXTRACT_EXIT=1
1295
+ fi
1296
+ # --- Discovery-stage counters ------------------------------------------------
1297
+ # Capture queries-run / duds / raw-tweets-pulled BEFORE any early-exit branch
1298
+ # so every log_run.py call below can pass --queries/--duds/--tweets-pulled.
1299
+ # QUERIES_FILE is the array Claude returned (one row per drafted query incl.
1300
+ # zero-result ones); RAW_FILE is the deduped tweet array. Use python3 inline so
1301
+ # we get the exact in-memory counts the rest of the pipeline operates on.
1302
+ QUERIES_TOTAL=0
1303
+ DUDS_TOTAL=0
1304
+ TWEETS_PULLED=0
1305
+ if [ -f "$QUERIES_FILE" ]; then
1306
+ QUERIES_TOTAL=$(python3 -c "
1307
+ import json, sys
1308
+ try:
1309
+ d = json.load(open(sys.argv[1]))
1310
+ print(len(d) if isinstance(d, list) else 0)
1311
+ except Exception:
1312
+ print(0)
1313
+ " "$QUERIES_FILE" 2>/dev/null || echo 0)
1314
+ DUDS_TOTAL=$(python3 -c "
1315
+ import json, sys
1316
+ try:
1317
+ d = json.load(open(sys.argv[1]))
1318
+ n = sum(1 for q in (d if isinstance(d, list) else []) if (q.get('tweets_found') or 0) == 0)
1319
+ print(n)
1320
+ except Exception:
1321
+ print(0)
1322
+ " "$QUERIES_FILE" 2>/dev/null || echo 0)
1323
+ fi
1324
+ if [ -f "$RAW_FILE" ]; then
1325
+ TWEETS_PULLED=$(python3 -c "
1326
+ import json, sys
1327
+ try:
1328
+ d = json.load(open(sys.argv[1]))
1329
+ print(len(d) if isinstance(d, list) else 0)
1330
+ except Exception:
1331
+ print(0)
1332
+ " "$RAW_FILE" 2>/dev/null || echo 0)
1333
+ fi
1334
+
1335
+ # Accumulate per-iteration counts into cycle-level totals for the post-loop
1336
+ # log_run.py call (otherwise the dashboard would show only the last attempt's
1337
+ # queries/duds/tweets-pulled, hiding the retry work).
1338
+ CUMULATIVE_QUERIES=$((CUMULATIVE_QUERIES + QUERIES_TOTAL))
1339
+ CUMULATIVE_DUDS=$((CUMULATIVE_DUDS + DUDS_TOTAL))
1340
+ CUMULATIVE_TWEETS_PULLED=$((CUMULATIVE_TWEETS_PULLED + TWEETS_PULLED))
1341
+
1342
+ # Snapshot this iteration's queries WITH per-query verdicts into
1343
+ # TRIED_QUERIES_JSON BEFORE log_twitter_search_attempts.py deletes QUERIES_FILE.
1344
+ # Verdicts come from joining QUERIES_FILE (drafted queries) with SCAN_TWEETS_FILE
1345
+ # (raw scrape per query record) and the BATCH_COUNT delta this attempt
1346
+ # (kept_after_skip approximation). kept_after_age comes from the optional
1347
+ # scorer sidecar at /tmp/twcycle-${BATCH_ID}-attempt-${SCAN_ATTEMPT}-scored.json
1348
+ # (written by score_twitter_candidates.py --scored-sidecar); when the sidecar
1349
+ # is missing we assume kept_after_age == raw_tweets, which collapses the
1350
+ # all_aged_out branch into dead_supply / found_some — still useful, just less
1351
+ # directional. Output entry shape:
1352
+ # {query, project, search_topic, raw_tweets, kept_after_age,
1353
+ # kept_after_skip, verdict}
1354
+ # verdict ∈ {dead_supply, all_aged_out, all_engaged_or_skipped, found_some}.
1355
+ if [ -f "$QUERIES_FILE" ]; then
1356
+ TRIED_QUERIES_JSON=$(python3 - \
1357
+ "$TRIED_QUERIES_JSON" "$QUERIES_FILE" "$SCAN_TWEETS_FILE" \
1358
+ "/tmp/twcycle-${BATCH_ID}-attempt-${SCAN_ATTEMPT}-scored.json" \
1359
+ "$BATCH_COUNT_BEFORE_ATTEMPT" "$BATCH_COUNT" <<'PY' 2>/dev/null || echo "$TRIED_QUERIES_JSON"
1360
+ import json, os, sys
1361
+ from collections import Counter
1362
+
1363
+ cur = json.loads(sys.argv[1] or '[]')
1364
+ queries_path = sys.argv[2]
1365
+ scan_path = sys.argv[3]
1366
+ scored_path = sys.argv[4]
1367
+ try:
1368
+ pre = int(sys.argv[5] or 0)
1369
+ except Exception:
1370
+ pre = 0
1371
+ try:
1372
+ post = int(sys.argv[6] or 0)
1373
+ except Exception:
1374
+ post = 0
1375
+
1376
+ try:
1377
+ new = json.load(open(queries_path))
1378
+ if not isinstance(new, list):
1379
+ new = []
1380
+ except Exception:
1381
+ new = []
1382
+
1383
+ # raw_tweets per query from SCAN_TWEETS_FILE (one JSONL record per scan call).
1384
+ # Multiple records can share a query if the harness retried; we sum.
1385
+ raw_by_query = Counter()
1386
+ if scan_path and os.path.exists(scan_path):
1387
+ with open(scan_path) as fh:
1388
+ for line in fh:
1389
+ try:
1390
+ rec = json.loads(line)
1391
+ except Exception:
1392
+ continue
1393
+ q = (rec.get('query') or '').strip()
1394
+ n = len(rec.get('tweets') or [])
1395
+ if q:
1396
+ raw_by_query[q] += n
1397
+
1398
+ # kept_after_age per query from the scorer sidecar (optional). Falls back to
1399
+ # raw_tweets when the sidecar is absent.
1400
+ age_by_query = {}
1401
+ if scored_path and os.path.exists(scored_path):
1402
+ try:
1403
+ scored = json.load(open(scored_path))
1404
+ for q, counts in (scored or {}).items():
1405
+ age_by_query[(q or '').strip()] = int(counts.get('kept_after_age') or 0)
1406
+ except Exception:
1407
+ pass
1408
+
1409
+ # kept_after_skip is the cycle-level delta this attempt. The scorer doesn't
1410
+ # tag the per-tweet survivor with its source query upstream, so we split the
1411
+ # delta evenly across queries that actually returned raw tweets. We mostly
1412
+ # care about zero vs nonzero per query, not the exact split.
1413
+ delta = max(0, post - pre)
1414
+ queries_with_raw = [e for e in new if raw_by_query.get((e.get('query') or '').strip(), 0) > 0]
1415
+ share = (delta / max(1, len(queries_with_raw))) if queries_with_raw else 0
1416
+
1417
+ for entry in new:
1418
+ q = (entry.get('query') or '').strip()
1419
+ raw = raw_by_query.get(q, 0)
1420
+ # When sidecar present, trust it; else assume freshness gate passed all raw.
1421
+ kept_age = age_by_query[q] if q in age_by_query else raw
1422
+ kept_skip = int(round(share)) if raw > 0 else 0
1423
+ if raw == 0:
1424
+ verdict = 'dead_supply'
1425
+ elif kept_age == 0:
1426
+ verdict = 'all_aged_out'
1427
+ elif kept_skip == 0:
1428
+ verdict = 'all_engaged_or_skipped'
1429
+ else:
1430
+ verdict = 'found_some'
1431
+ entry['raw_tweets'] = raw
1432
+ entry['kept_after_age'] = kept_age
1433
+ entry['kept_after_skip'] = kept_skip
1434
+ entry['verdict'] = verdict
1435
+
1436
+ cur.extend(new)
1437
+ print(json.dumps(cur))
1438
+ PY
1439
+ )
1440
+ fi
1441
+
1442
+ # Log every drafted query (incl. zero-result ones) to twitter_search_attempts
1443
+ # BEFORE any early-exit branches. Runs even when the tweets array is empty
1444
+ # so dud queries actually accumulate in the negative-signal table.
1445
+ if [ -f "$QUERIES_FILE" ]; then
1446
+ python3 "$REPO_DIR/scripts/log_twitter_search_attempts.py" --batch-id "$BATCH_ID" \
1447
+ --attempts-out "$ATTEMPTS_FILE" \
1448
+ < "$QUERIES_FILE" 2>&1 | tee -a "$LOG_FILE"
1449
+ rm -f "$QUERIES_FILE"
1450
+ fi
1451
+
1452
+ # Stamp last_used_at on every active project-wide exclude we surfaced to
1453
+ # Claude this cycle. These are the terms Claude was REQUIRED to append as
1454
+ # `-term` to its drafted queries, so even if Claude omits one, the term is
1455
+ # still considered "in use" for decay purposes — drafter compliance is its
1456
+ # own problem, not a reason to prune a learned exclude. Done after the
1457
+ # search_attempts log so a Phase 1 abort still leaves the marks behind.
1458
+ python3 - "$PROJECTS_JSON" <<'PY' 2>&1 | tee -a "$LOG_FILE" || true
1459
+ import json, os, sys
1460
+ sys.path.insert(0, os.path.expanduser('~/social-autoposter/scripts'))
1461
+ import project_excludes as pe
1462
+ projects = json.loads(sys.argv[1] or '[]')
1463
+ total = 0
1464
+ for p in projects:
1465
+ terms = p.get('excludes_for_search') or []
1466
+ if not terms:
1467
+ continue
1468
+ try:
1469
+ n = pe.mark_used('twitter', p.get('name'), terms)
1470
+ except Exception as exc:
1471
+ print(f"mark_used error for {p.get('name')}: {exc}", file=sys.stderr)
1472
+ continue
1473
+ total += n
1474
+ if total:
1475
+ print(f"project_excludes: marked {total} term(s) used across selected projects")
1476
+ PY
1477
+ if [ "$EXTRACT_EXIT" -ne 0 ] || [ ! -f "$RAW_FILE" ]; then
1478
+ # Claude returned no usable tweet array this attempt. Could be a real
1479
+ # Anthropic error (stream_idle_timeout, api_overloaded, monthly_limit,
1480
+ # context_overflow) or just "model found nothing relevant". Classify
1481
+ # the failure for the post-loop log_run summary; the loop control below
1482
+ # decides whether to retry or give up.
1483
+ # SCAN_OUTPUT was a stale leftover from the pre-lean design (when the scan's
1484
+ # stdout was captured into a shell var); the lean Phase 1 loop now tees its
1485
+ # output to $LOG_FILE instead, so an empty-scan attempt hit `set -u` and
1486
+ # aborted the whole cycle here. Feed the classifier the recent log tail (the
1487
+ # actual scan output, where harness/Anthropic error signatures land) so we
1488
+ # still distinguish a real error from "found nothing relevant".
1489
+ SCAN_OUTPUT=$(tail -n 80 "$LOG_FILE" 2>/dev/null || true)
1490
+ PHASE1_REASON_LATEST=$(echo "$SCAN_OUTPUT" | python3 "$REPO_DIR/scripts/classify_run_error.py" 2>/dev/null)
1491
+ [ -z "$PHASE1_REASON_LATEST" ] && PHASE1_REASON_LATEST="phase1_no_tweets"
1492
+ LAST_PHASE1_REASON="$PHASE1_REASON_LATEST"
1493
+ log " Phase 1 attempt $SCAN_ATTEMPT returned no tweets (reason=$PHASE1_REASON_LATEST); falling through to loop control"
1494
+ else
1495
+ # --- Phase 1 finalize: enrich + score with T0 + batch_id ----------------
1496
+ log "Enriching via fxtwitter + scoring with T0 snapshot (batch=$BATCH_ID, attempt=$SCAN_ATTEMPT)..."
1497
+ cat "$RAW_FILE" \
1498
+ | python3 "$REPO_DIR/scripts/enrich_twitter_candidates.py" \
1499
+ | python3 "$REPO_DIR/scripts/score_twitter_candidates.py" --batch-id "$BATCH_ID" \
1500
+ ${ATTEMPTS_FILE:+--attempts "$ATTEMPTS_FILE"} \
1501
+ --scored-sidecar "/tmp/twcycle-${BATCH_ID}-attempt-${SCAN_ATTEMPT}-scored.json" \
1502
+ 2>&1 | tee -a "$LOG_FILE"
1503
+ rm -f "$RAW_FILE" "$ATTEMPTS_FILE"
1504
+ fi
1505
+
1506
+ BATCH_COUNT=$(python3 "$REPO_DIR/scripts/twitter_cycle_helper.py" batch-count --batch-id "$BATCH_ID" 2>/dev/null || echo 0)
1507
+ log "Phase 1 attempt $SCAN_ATTEMPT complete. Batch has $BATCH_COUNT/$RETRY_TARGET candidates with T0 snapshot."
1508
+
1509
+ # --- Retry-loop control ------------------------------------------------------
1510
+ # Break out if we hit the target; else either retry or give up at the cap.
1511
+ if [ "$BATCH_COUNT" -ge "$RETRY_TARGET" ]; then
1512
+ log " Reached target ($BATCH_COUNT >= $RETRY_TARGET) after $SCAN_ATTEMPT scan(s); proceeding to Phase 2"
1513
+ break
1514
+ fi
1515
+ if [ "$SCAN_ATTEMPT" -ge "$MAX_SCAN_ATTEMPTS" ]; then
1516
+ log " Hit scan cap ($MAX_SCAN_ATTEMPTS); proceeding with $BATCH_COUNT candidate(s)"
1517
+ break
1518
+ fi
1519
+ _TRIED_N=$(echo "$TRIED_QUERIES_JSON" | python3 -c 'import json,sys;print(len(json.load(sys.stdin)))' 2>/dev/null || echo 0)
1520
+ log " Below target ($BATCH_COUNT/$RETRY_TARGET); $_TRIED_N queries tried so far this cycle; looping for attempt $((SCAN_ATTEMPT + 1))..."
1521
+ done
1522
+
1523
+ # --- Post-loop bookkeeping ---------------------------------------------------
1524
+ # Stamp cycle_variant='D' onto every candidate in this batch. The A/B/C/D
1525
+ # experiment concluded 2026-05-31 (D won); this is now a constant label kept so
1526
+ # downstream analytics (post-rate, thread-age-at-discover, lag-after-thread,
1527
+ # top-reply ratio) stay continuous with the historical experiment rows.
1528
+ # Idempotent: same value would be written if the batch is salvaged into a peer
1529
+ # cycle.
1530
+ # HTTP-only (2026-06-01): the cycle_variant stamp routes through
1531
+ # /api/v1/twitter-candidates/stamp-cycle-variant via twitter_cycle_helper.py.
1532
+ # No DATABASE_URL, no psycopg, no fallback. Idempotent: only NULL rows touched.
1533
+ python3 "$REPO_DIR/scripts/twitter_cycle_helper.py" stamp-cycle-variant \
1534
+ --batch-id "$BATCH_ID" --variant "$TWITTER_CYCLE_VARIANT" \
1535
+ >/dev/null 2>>"$LOG_FILE" || log "Phase 1: cycle_variant stamp failed (non-fatal)"
1536
+
1537
+ # Promote cumulative totals onto the per-iteration names so every downstream
1538
+ # log_run.py / trap handler picks up the cycle-level work (not just the last
1539
+ # attempt's counts). Keeps all the existing "${QUERIES_TOTAL:-0}" etc. call
1540
+ # sites correct without touching them individually.
1541
+ QUERIES_TOTAL="$CUMULATIVE_QUERIES"
1542
+ DUDS_TOTAL="$CUMULATIVE_DUDS"
1543
+ TWEETS_PULLED="$CUMULATIVE_TWEETS_PULLED"
1544
+
1545
+ log "Phase 1 complete after $SCAN_ATTEMPT scan attempt(s). Final batch has $BATCH_COUNT candidates with T0 snapshot."
1546
+
1547
+ if [ "$BATCH_COUNT" = "0" ]; then
1548
+ # Distinguish "Claude returned no tweets at all" from "Claude returned
1549
+ # tweets but enrichment dropped them all" from "we exhausted the topic
1550
+ # universe mid-retry" so the dashboard can surface the right failure
1551
+ # mode. Priority order: universe_exhausted (the picker said stop) >
1552
+ # Anthropic-side classified error > generic empty_batch.
1553
+ if [ "${UNIVERSE_EXHAUSTED:-0}" = "1" ]; then
1554
+ _FAILURE_REASON="universe_exhausted:1"
1555
+ elif [ -n "$LAST_PHASE1_REASON" ] && [ "$CUMULATIVE_TWEETS_PULLED" = "0" ]; then
1556
+ _FAILURE_REASON="${LAST_PHASE1_REASON}:1"
1557
+ else
1558
+ _FAILURE_REASON="empty_batch:1"
1559
+ fi
1560
+ log "Empty batch after $SCAN_ATTEMPT attempt(s) (reason=$_FAILURE_REASON). Nothing to re-score. Exiting."
1561
+ _COST=$(python3 "$REPO_DIR/scripts/get_run_cost.py" --cycle-id "$BATCH_ID" 2>/dev/null || echo "0.0000")
1562
+ python3 "$REPO_DIR/scripts/log_run.py" --script "post_twitter" --posted 0 --skipped 0 --failed 1 \
1563
+ --salvaged "${SALVAGED:-0}" \
1564
+ --queries "${CUMULATIVE_QUERIES:-0}" --duds "${CUMULATIVE_DUDS:-0}" \
1565
+ --tweets-pulled "${CUMULATIVE_TWEETS_PULLED:-0}" \
1566
+ --failure-reasons "$_FAILURE_REASON" \
1567
+ --cost "$_COST" --elapsed $(( $(date +%s) - RUN_START ))
1568
+ _SA_RUN_SUMMARY_EMITTED=1
1569
+ exit 0
1570
+ fi
1571
+
1572
+ # Stamp phase2a before releasing the lock so the salvage budget covers the
1573
+ # browser-lock handoff window (phase2a budget = 20 min).
1574
+ python3 "$REPO_DIR/scripts/twitter_batch_phase.py" advance "$BATCH_ID" --phase phase2a 2>&1 | tee -a "$LOG_FILE" || true
1575
+
1576
+ # Release the twitter-browser lock between Phase 1 scrape and Phase 2b posting.
1577
+ # Other pipelines (engage-twitter, dm-outreach-twitter, link-edit-twitter,
1578
+ # stats.sh) can run their browser steps in this window instead of waiting for us
1579
+ # to finish. We re-acquire just before Phase 2b posts, blocking up to the
1580
+ # acquire_lock timeout if another pipeline is mid-run.
1581
+ log "Releasing twitter-browser lock between Phase 1 scrape and Phase 2b posting..."
1582
+ release_lock "twitter-browser" 2>>"$LOG_FILE"
1583
+ # (2026-06-16) NO `rm -f twitter-browser-lock.json` here. The blind rm was
1584
+ # ownership-unaware and ran AFTER release_lock, so under a pipeline handoff it
1585
+ # deleted a LIVE peer's session mutex (defect b) -> two browser ops on one X
1586
+ # tab. Dead python:PID holders are now reclaimed by _acquire_browser_lock in
1587
+ # scripts/twitter_browser.py (os.kill liveness), so the workaround is obsolete
1588
+ # AND unsafe. Do NOT re-add it. See docs/twitter_browser_lock.md.
1589
+
1590
+ # --- No ripen wait (winning variant D) --------------------------------------
1591
+ # The 20-min ripen sleep + fetch_twitter_t1 re-measurement was removed when
1592
+ # variant D won the A/B/C/D test (2026-05-31). The wait was originally a
1593
+ # velocity gate; the gate floor was removed 2026-05-15 so it only fed
1594
+ # delta_score into the LLM prompt, and the experiment showed eliminating that
1595
+ # ~20 min thread->post lag improves engagement more than delta_score helps the
1596
+ # draft. We go straight from candidate discovery to Phase 2b; delta_score stays
1597
+ # at its T0 value.
1598
+ log "No ripen wait (logic D): skipping sleep + T1 fetch, delta_score stays at T0 value"
1599
+
1600
+ # --- Phase 2b: top 25 by virality_score, no post cap ---------------------
1601
+ # Sort key (2026-05-27): virality_score DESC. This is the composite predictor
1602
+ # stamped at discovery by score_twitter_candidates.py:
1603
+ # virality_score = velocity * reach_mult * age_decay * rt_bonus
1604
+ # * (1 + reply_bonus) * (1 + discussion_bonus)
1605
+ # It folds in engagement velocity, author reach (follower-tier multiplier),
1606
+ # age decay (6h half-life), retweet ratio, reply count, and discussion
1607
+ # quality (reply:like ratio). Cohort analysis on 30d posted data: the
1608
+ # [10k+) virality bucket gets ~36x the median reply views of the [0-10)
1609
+ # bucket, which is much steeper than what raw 5-min delta predicts.
1610
+ # Replaces the prior `delta + flat-5 intent-regex boost` sort: the intent
1611
+ # regex was a crutch for delta_score (a raw growth count that ignored
1612
+ # reach + decay); the model reads tweet text directly in the prep prompt
1613
+ # and detects intent itself, so the lexical layer is redundant.
1614
+ # 2026-05-15: ripening floor removed entirely (was `delta_score >= 0`).
1615
+ # The model already sees per-candidate Virality + Delta in CANDIDATE_BLOCK
1616
+ # below and can weigh velocity against topical fit itself. Letting
1617
+ # negative-delta tweets through means a thoughtful comment can still ride
1618
+ # an on-theme but cooling thread to the right audience. LIMIT 25 stays as
1619
+ # a draft-budget cap, not a ripening gate.
1620
+ # Candidate list comes through /api/v1/twitter-candidates (route returns
1621
+ # all pending rows for the batch); the helper applies the virality_score
1622
+ # sort + 25-row cap client-side and emits the SAME pipe-separated columns
1623
+ # the legacy psql -F '|' query produced. Pipe shape is documented in
1624
+ # scripts/twitter_cycle_helper.py:cmd_candidates.
1625
+ CANDIDATES=$(python3 "$REPO_DIR/scripts/twitter_cycle_helper.py" candidates --batch-id "$BATCH_ID" 2>/dev/null || echo "")
1626
+
1627
+ if [ -z "$CANDIDATES" ]; then
1628
+ log "No candidates with delta scores. Marking batch expired."
1629
+ # /api/v1/twitter-candidates/expire-batch performs the same status-flip
1630
+ # UPDATE atomically and prints the resulting expired_count integer that
1631
+ # the EXPIRED_BATCH variable previously got from a second COUNT(*) query.
1632
+ EXPIRED_BATCH=$(python3 "$REPO_DIR/scripts/twitter_cycle_helper.py" expire-batch --batch-id "$BATCH_ID" 2>/dev/null || echo 0)
1633
+ _COST=$(python3 "$REPO_DIR/scripts/get_run_cost.py" --cycle-id "$BATCH_ID" 2>/dev/null || echo "0.0000")
1634
+ # Not a hard error — batch had candidates but none remained 'pending' after
1635
+ # Phase 2a (typically: every row already flipped to posted/skipped/expired
1636
+ # by an earlier salvage pass). With the ripening floor removed (2026-05-15),
1637
+ # this no longer fires on low-delta rows; only on empty/exhausted batches.
1638
+ # Report as skipped (not failed) so the row reads "skipped: N" rather than
1639
+ # the silent "—" we used to render. failure_reasons stays empty.
1640
+ python3 "$REPO_DIR/scripts/log_run.py" --script "post_twitter" --posted 0 --skipped "${EXPIRED_BATCH:-0}" --failed 0 \
1641
+ --salvaged "${SALVAGED:-0}" \
1642
+ --queries "${QUERIES_TOTAL:-0}" --duds "${DUDS_TOTAL:-0}" \
1643
+ --tweets-pulled "${TWEETS_PULLED:-0}" --candidates "${BATCH_COUNT:-0}" \
1644
+ --cost "$_COST" --elapsed $(( $(date +%s) - RUN_START ))
1645
+ _SA_RUN_SUMMARY_EMITTED=1
1646
+ exit 0
1647
+ fi
1648
+
1649
+ # --- SCAN_ONLY gate: stop after scoring, emit candidates, skip drafting -------
1650
+ # When SCAN_ONLY=1 the cycle runs scan -> score -> top-N select, writes the chosen
1651
+ # candidates as JSON, and STOPS before the claude drafting step. The MCP
1652
+ # scan_candidates tool reads this so a Claude Desktop scheduled-task session can do
1653
+ # the drafting ITSELF (on the user's plan, no `claude -p`) and hand the drafts back
1654
+ # via submit_drafts. Candidates stay 'pending' (drafted+posted via submit_drafts ->
1655
+ # post_drafts, or salvaged by a later cycle). The browser lock was already released
1656
+ # at the Phase 1 handoff, so this exits clean via the EXIT trap. NO current caller
1657
+ # sets SCAN_ONLY, so the autopilot/draft_cycle paths are byte-for-byte unchanged.
1658
+ if [ "${SCAN_ONLY:-0}" = "1" ]; then
1659
+ SCAN_FILE="/tmp/saps_scan_candidates_${BATCH_ID}.json"
1660
+ # $CANDIDATES is the same pipe-separated top-N the drafting step consumes (cols
1661
+ # documented in twitter_cycle_helper.py:cmd_candidates; tweet_text/draft fields
1662
+ # are pipe+newline sanitized there, so a field split is safe). Batch id + out
1663
+ # path travel via env so the single-quoted python needs no shell interpolation.
1664
+ printf '%s\n' "$CANDIDATES" | S4L_SCAN_FILE="$SCAN_FILE" S4L_SCAN_BATCH="$BATCH_ID" python3 -c '
1665
+ import json, os, sys
1666
+ def _i(x):
1667
+ try:
1668
+ return int(float(x or 0))
1669
+ except Exception:
1670
+ return 0
1671
+ def _f(x):
1672
+ try:
1673
+ return float(x or 0)
1674
+ except Exception:
1675
+ return 0.0
1676
+ out = []
1677
+ for line in sys.stdin:
1678
+ line = line.rstrip("\n")
1679
+ if not line.strip():
1680
+ continue
1681
+ p = line.split("|")
1682
+ if len(p) < 14 or not p[0].isdigit():
1683
+ continue
1684
+ out.append({
1685
+ "id": int(p[0]), "tweet_url": p[1], "author_handle": p[2], "tweet_text": p[3],
1686
+ "virality_score": _f(p[4]), "delta_score": _f(p[5]), "matched_project": p[6],
1687
+ "search_topic": p[7], "likes": _i(p[8]), "retweets": _i(p[9]), "replies": _i(p[10]),
1688
+ "views": _i(p[11]), "author_followers": _i(p[12]), "age_hours": _f(p[13]),
1689
+ "existing_draft": p[14] if len(p) > 14 else "", "existing_draft_style": p[15] if len(p) > 15 else "",
1690
+ })
1691
+ json.dump({"batch_id": os.environ["S4L_SCAN_BATCH"], "candidates": out}, open(os.environ["S4L_SCAN_FILE"], "w"))
1692
+ ' 2>/dev/null || printf '{"batch_id": "%s", "candidates": []}' "$BATCH_ID" > "$SCAN_FILE"
1693
+ SCAN_N=$(python3 -c "import json; print(len(json.load(open('$SCAN_FILE')).get('candidates') or []))" 2>/dev/null || echo 0)
1694
+ log "SCAN_ONLY=1: $SCAN_N candidate(s) scored and written to $SCAN_FILE. Stopping before drafting (agent drafts next)."
1695
+ _SA_RUN_SUMMARY_EMITTED=1
1696
+ echo "SCAN_ONLY_RESULT=$SCAN_FILE"
1697
+ exit 0
1698
+ fi
1699
+
1700
+ CANDIDATE_COUNT=$(printf '%s\n' "$CANDIDATES" | grep -c '^[0-9]')
1701
+ log "Top $CANDIDATE_COUNT candidates by virality_score selected for post review."
1702
+
1703
+ # No post cap: Phase 2b-prep posts every candidate it judges genuinely
1704
+ # on-brand. HIGH_DELTA_COUNT is still computed, but ONLY as a dashboard
1705
+ # diagnostic (the "Δ≥10 N" stat, fed to log_run.py --above-floor). It no
1706
+ # longer gates how many replies the cycle is allowed to post.
1707
+ HIGH_DELTA_COUNT=$(printf '%s\n' "$CANDIDATES" | awk -F'|' '$1 ~ /^[0-9]+$/ && $6+0 >= 10 {n++} END {print n+0}')
1708
+ log "Candidates with Δ≥10 (momentum diagnostic only, not a cap): $HIGH_DELTA_COUNT"
1709
+
1710
+ CANDIDATE_BLOCK=""
1711
+ # Thread-media capture (2026-06-03): collect each candidate's id|url so that,
1712
+ # AFTER the browser lock is acquired, we can deterministically pre-fetch the
1713
+ # media (images/videos/GIFs/link-cards) of every thread the model is about to
1714
+ # draft against and feed it into the prep prompt. Gated by
1715
+ # S4L_TWITTER_CAPTURE_MEDIA so it stays a no-op until the website API (with the
1716
+ # set_media action + thread_media column) deploys. Populated in the loop below.
1717
+ MEDIA_URLS_FILE=$(mktemp -t saps_twitter_media_urls_XXXXXX.tsv)
1718
+ while IFS='|' read -r cid curl cauthor ctext cscore cdelta cproject ctopic clikes crts creplies cviews cfollowers cage cdraft cdraftstyle cdraftage; do
1719
+ if [ -n "$cid" ] && [ -n "$curl" ]; then
1720
+ printf '%s\t%s\n' "$cid" "$curl" >> "$MEDIA_URLS_FILE"
1721
+ fi
1722
+ DRAFT_LINE=""
1723
+ if [ -n "$cdraft" ] && [ "$cdraftage" != "-1" ]; then
1724
+ # Round draft age to whole minutes for the prompt.
1725
+ DRAFT_MIN=$(printf '%.0f' "$cdraftage")
1726
+ DRAFT_LINE="
1727
+ EXISTING DRAFT (style=$cdraftstyle, age=${DRAFT_MIN}m): $cdraft"
1728
+ fi
1729
+ # Per-candidate prior-interaction context: surface our last 5 comments to
1730
+ # this author in the past 30 days (soft context only — vary angle, don't
1731
+ # repeat phrasing). Empty when we have no history. Failure is silent.
1732
+ AUTHOR_HISTORY_LINE=""
1733
+ if [ -n "$cauthor" ]; then
1734
+ _AH=$(python3 "$REPO_DIR/scripts/author_history_block.py" --platform twitter --author "$cauthor" --days 30 --limit 5 2>>"$LOG_FILE" || true)
1735
+ if [ -n "$_AH" ]; then
1736
+ AUTHOR_HISTORY_LINE="
1737
+ $_AH"
1738
+ fi
1739
+ fi
1740
+ CANDIDATE_BLOCK="${CANDIDATE_BLOCK}
1741
+ ---
1742
+ Candidate ID: $cid
1743
+ URL: $curl
1744
+ Author: @$cauthor (${cfollowers} followers)
1745
+ Text: $ctext
1746
+ Virality: $cscore | Delta (5min): $cdelta | Likes: $clikes | RTs: $crts | Replies: $creplies | Views: $cviews | Age: ${cage}h
1747
+ Search query: $ctopic
1748
+ Project match: $cproject${DRAFT_LINE}${AUTHOR_HISTORY_LINE}
1749
+ "
1750
+ done <<< "$CANDIDATES"
1751
+
1752
+ ALL_PROJECTS_JSON=$(python3 -c "
1753
+ import json, os
1754
+ config = json.load(open(os.path.expanduser('~/social-autoposter/config.json')))
1755
+ projects = config.get('projects', [])
1756
+ lane = os.environ.get('S4L_ACTIVE_LANE', '')
1757
+ if lane == 'personal_brand':
1758
+ # Personal-brand lane is pure organic growth: the drafter must NOT see any
1759
+ # product config at all (no website, links, booking_link, get_started_link,
1760
+ # features, pricing, CTAs). We emit ONLY the persona project, and ONLY the
1761
+ # drafting-relevant fields, so there is literally no product context in the
1762
+ # prompt to accidentally pitch, quote, or link. This also kills cross-routing
1763
+ # (no 'other project' exists to route a candidate to). Whitelist, not
1764
+ # denylist: any field added to the persona entry later stays out unless
1765
+ # explicitly allowed here.
1766
+ ALLOWED = {
1767
+ 'name', 'description', 'content_angle', 'voice',
1768
+ 'voice_relationship', 'content_guardrails',
1769
+ }
1770
+ persona = next((p for p in projects if p.get('persona') is True), None)
1771
+ out = {}
1772
+ if persona:
1773
+ out[persona['name']] = {k: v for k, v in persona.items() if k in ALLOWED}
1774
+ print(json.dumps(out, indent=2))
1775
+ else:
1776
+ print(json.dumps({p['name']: p for p in projects}, indent=2))
1777
+ " 2>/dev/null || echo "{}")
1778
+
1779
+ # Engagement-style picker (2026-05-19): pick ONE assigned style for this
1780
+ # cycle. The picked style flows two places: (1) --style filter for
1781
+ # top_performers.py so the per-style exemplars section shows only posts
1782
+ # matching the assigned style, (2) saps_render_style_block (below) so the
1783
+ # prompt block embeds the same assignment. On invent mode picked_style is
1784
+ # empty and top_performers stays unfiltered (model sees full landscape).
1785
+ source "$REPO_DIR/skill/styles.sh"
1786
+ STYLE_ASSIGN_FILE=$(mktemp -t saps_twitter_assign_XXXXXX.json)
1787
+ saps_pick_style twitter posting "$STYLE_ASSIGN_FILE" >/dev/null 2>&1 || true
1788
+ PICKED_STYLE=$(python3 -c "
1789
+ import json
1790
+ try:
1791
+ with open('$STYLE_ASSIGN_FILE') as f:
1792
+ d = json.load(f)
1793
+ print(d.get('style') or '')
1794
+ except Exception:
1795
+ print('')
1796
+ " 2>/dev/null)
1797
+ PICKED_MODE=$(python3 -c "
1798
+ import json
1799
+ try:
1800
+ with open('$STYLE_ASSIGN_FILE') as f:
1801
+ d = json.load(f)
1802
+ print(d.get('mode') or 'use')
1803
+ except Exception:
1804
+ print('use')
1805
+ " 2>/dev/null)
1806
+ log "Engagement style assigned: mode=$PICKED_MODE style=${PICKED_STYLE:-(invent)}"
1807
+
1808
+ # --- Draft-prompt A/B: decouple product pivot (2026-06-29) -------------------
1809
+ # Per-CYCLE arm (the prep session drafts the whole batch from ONE prompt, so
1810
+ # assignment is at cycle granularity, not per post; the whole batch shares it).
1811
+ # control = the current draft directive verbatim.
1812
+ # treatment = "decoupled" wording: the reply must stand on its own and NOT be
1813
+ # built as a concede-the-obvious-then-pivot-to-the-product setup;
1814
+ # product mentioned only when genuinely relevant.
1815
+ # The arm is stamped onto every post this cycle via S4L_DRAFT_PROMPT_VARIANT
1816
+ # (read by twitter_post_plan.py -> log_post.py -> posts.draft_prompt_variant),
1817
+ # mirroring the tail_link_variant plumbing. Split tunable via
1818
+ # TWITTER_DRAFT_PROMPT_AB_RATE = fraction of cycles assigned to 'treatment' (the
1819
+ # decoupled directive). CODE DEFAULT 1 = 100% treatment: a fresh plugin install /
1820
+ # new user with no override drafts in the DECOUPLED style by default (the old
1821
+ # concede->pivot 'control' is opt-in, not the default for customers). OUR own
1822
+ # install pins this to 0.5 in .env for a 50/50 holdback so we can measure
1823
+ # decoupled vs the old behavior on our account. The dashboard reads the SAME var
1824
+ # with the SAME default (bin/server.js), so display and routing never diverge.
1825
+ DRAFT_PROMPT_AB_RATE="${TWITTER_DRAFT_PROMPT_AB_RATE:-1}"
1826
+ S4L_DRAFT_PROMPT_VARIANT=$(python3 -c "
1827
+ import random
1828
+ try:
1829
+ rate = float('$DRAFT_PROMPT_AB_RATE')
1830
+ except Exception:
1831
+ rate = 1.0
1832
+ rate = min(1.0, max(0.0, rate))
1833
+ print('treatment' if random.random() < rate else 'control')
1834
+ " 2>/dev/null || echo treatment)
1835
+ export S4L_DRAFT_PROMPT_VARIANT
1836
+ log "Draft-prompt A/B arm: $S4L_DRAFT_PROMPT_VARIANT (rate=$DRAFT_PROMPT_AB_RATE)"
1837
+ if [ "$S4L_DRAFT_PROMPT_VARIANT" = "treatment" ]; then
1838
+ DRAFT_DIRECTIVE="Otherwise: draft a direct, natural reply that stands on its own as a useful contribution to the thread. Mention the matched project only when it is genuinely the most relevant thing to say, and state it plainly in one clause; most replies will not need it. Do NOT build the reply as a concede-the-obvious-then-pivot-to-the-product setup. Length is governed ENTIRELY by the per-style LENGTH LIMIT in the style block above; obey that target and ceiling, do not apply any other length rule here. NEVER em dashes. Apply the matched project's \`voice\` block from ALL_PROJECTS_JSON: follow voice.tone, never violate voice.never, mirror voice.examples / voice.examples_good when present."
1839
+ else
1840
+ DRAFT_DIRECTIVE="Otherwise: draft a reply using the best engagement style. Length is governed ENTIRELY by the per-style LENGTH LIMIT in the style block above; obey that target and ceiling, do not apply any other length rule here. NEVER em dashes. Apply the matched project's \`voice\` block from ALL_PROJECTS_JSON: follow voice.tone, never violate voice.never, mirror voice.examples / voice.examples_good when present."
1841
+ fi
1842
+ # Personal-brand lane (S4L_ACTIVE_LANE=personal_brand, set by saps_mode.py):
1843
+ # replace the product-framed directive entirely. This lane is pure organic
1844
+ # growth: no product, no link, no CTA. The reply must add real value grounded in
1845
+ # the persona's first-hand material (the PERSONA CORPUS block + the persona voice
1846
+ # block), not concede-and-agree filler. Overrides both A/B arms above.
1847
+ if [ "${S4L_ACTIVE_LANE:-}" = "personal_brand" ]; then
1848
+ DRAFT_DIRECTIVE="Otherwise: draft a reply that stands on its own as a genuinely useful contribution to THIS thread. Ground it in the persona's real, first-hand experience from the PERSONA CORPUS block below (specific projects, real numbers, sharp opinions, actual failures) and in the persona's \`voice\` block from ALL_PROJECTS_JSON. Add exactly ONE of: a concrete specific from that lived experience, a sharp non-obvious opinion, a useful pointer, or a question that genuinely moves the thread forward. NEVER generic agreement ('makes sense', 'this is spot on', 'great point', 'the nuance here is'). This is a personal account, not a brand: sound like a real person in the thread. If web search is available and the thread hinges on a current fact, verify it before drafting rather than guessing. Length is governed ENTIRELY by the per-style LENGTH LIMIT in the style block above; obey that target and ceiling. NEVER em dashes. Follow voice.tone, never violate voice.never, mirror voice.examples / voice.examples_good when present."
1849
+ fi
1850
+
1851
+ if [ -n "$PICKED_STYLE" ]; then
1852
+ TOP_REPORT=$(python3 "$REPO_DIR/scripts/top_performers.py" --platform twitter --style "$PICKED_STYLE" 2>/dev/null || echo "(top performers report unavailable)")
1853
+ else
1854
+ TOP_REPORT=$(python3 "$REPO_DIR/scripts/top_performers.py" --platform twitter 2>/dev/null || echo "(top performers report unavailable)")
1855
+ fi
1856
+
1857
+ # --- Generation trace -------------------------------------------------------
1858
+ # Snapshot the few-shot context this cycle will feed to Claude — top_performers
1859
+ # report, top_queries from Phase 1, supply signal, dud queries — and write to a
1860
+ # tempfile. Path travels via env var to twitter_post_plan.py (Phase 2b-post),
1861
+ # which forwards it as --generation-trace to log_post.py so every post landed
1862
+ # this cycle gets posts.generation_trace JSONB pointing to the same snapshot.
1863
+ # This is what answers "which examples produced post #N" later. See
1864
+ # migrations/2026-05-12_generation_trace.sql for the shape contract.
1865
+ #
1866
+ # Failure is non-fatal: empty string means downstream skips --generation-trace
1867
+ # and the row gets NULL trace. We never block the cycle on the audit row.
1868
+ TRACE_INPUT=$(python3 -c "
1869
+ import json, sys
1870
+ payload = {
1871
+ 'platform': 'twitter',
1872
+ 'project_name': 'all',
1873
+ 'prompt_chars': len(sys.argv[1]) + len(sys.argv[2]) + len(sys.argv[3]) + len(sys.argv[4]) + len(sys.argv[7]),
1874
+ 'top_performers_text': sys.argv[1],
1875
+ 'top_search_topics_text': sys.argv[7],
1876
+ 'recent_comment_ids': [],
1877
+ 'extras': {
1878
+ 'top_queries_per_project': json.loads(sys.argv[2] or '{}'),
1879
+ 'supply_signal': json.loads(sys.argv[3] or '[]'),
1880
+ 'dud_queries': json.loads(sys.argv[4] or '[]'),
1881
+ 'auto_picked_style': sys.argv[5] or None,
1882
+ 'auto_picked_mode': sys.argv[6] or 'use',
1883
+ 'top_search_topics': json.loads(sys.argv[7] or '[]'),
1884
+ },
1885
+ 'min_score_floor': 5,
1886
+ }
1887
+ print(json.dumps(payload))
1888
+ " "$TOP_REPORT" "$TOP_QUERIES_PER_PROJECT_JSON" "$SUPPLY_SIGNAL_JSON" "$DUD_QUERIES_JSON" "$PICKED_STYLE" "$PICKED_MODE" "$TOP_TOPICS_JSON" 2>/dev/null || echo '{}')
1889
+ S4L_TWITTER_GEN_TRACE_PATH=$(printf '%s' "$TRACE_INPUT" | python3 "$REPO_DIR/scripts/write_generation_trace.py" --prefix twitter_gen_trace_ 2>/dev/null || echo "")
1890
+ export S4L_TWITTER_GEN_TRACE_PATH
1891
+ if [ -n "$S4L_TWITTER_GEN_TRACE_PATH" ] && [ -f "$S4L_TWITTER_GEN_TRACE_PATH" ]; then
1892
+ log "Generation trace: $S4L_TWITTER_GEN_TRACE_PATH ($(wc -c < "$S4L_TWITTER_GEN_TRACE_PATH") bytes)"
1893
+ else
1894
+ log "WARN: generation_trace build returned empty path; posts this cycle will have NULL trace"
1895
+ fi
1896
+
1897
+ STYLES_BLOCK=$(saps_render_style_block "$STYLE_ASSIGN_FILE" twitter posting)
1898
+ # Style assignment file is the same one we picked above; styles.sh already sourced.
1899
+ # Cleanup at cycle end (best effort).
1900
+ trap 'rm -f "$STYLE_ASSIGN_FILE" 2>/dev/null || true' EXIT
1901
+
1902
+ # Phase 2b is split into three sub-phases so the twitter-browser lock is only
1903
+ # held during actual browser work. The killer in the old single-session flow
1904
+ # was generate_page.py running inside the Claude session: 10-40 minutes of
1905
+ # Cloud Run deploy chain time, all under the browser lock, blocking every
1906
+ # other twitter pipeline. The new flow:
1907
+ # 2b-prep (lock held): Claude reads threads, drafts replies, saves drafts,
1908
+ # emits a JSON plan listing chosen candidates.
1909
+ # <release lock>
1910
+ # 2b-gen (no lock): twitter_gen_links.py runs generate_page.py per
1911
+ # candidate; falls back to plain project URL on failure.
1912
+ # <re-acquire lock>
1913
+ # 2b-post (lock held): twitter_post_plan.py calls twitter_browser.py reply,
1914
+ # log_post.py, campaign_bump.py, marks link_edited_at.
1915
+
1916
+ PLAN_FILE="/tmp/twitter_cycle_plan_${BATCH_ID}.json"
1917
+ SKIP_FILE="/tmp/twitter_cycle_skips_${BATCH_ID}.json"
1918
+
1919
+ # --- Phase 2b-prep: pick + draft + plan -------------------------------------
1920
+ # Stamp phase2b-prep BEFORE the long-running Claude read/draft so peer cycles'
1921
+ # Phase 0 salvage SQL sees current_phase='phase2b-prep' (45-min budget) instead
1922
+ # of stale phase2a (20-min budget). Without this stamp, mid-Phase-2b runs get
1923
+ # wrongly salvaged once 20 min elapse past phase2a's start, creating false
1924
+ # phase2b_silent run-monitor rows even when posts succeeded.
1925
+ python3 "$REPO_DIR/scripts/twitter_batch_phase.py" advance "$BATCH_ID" --phase phase2b-prep 2>&1 | tee -a "$LOG_FILE" || true
1926
+ log "Re-acquiring twitter-browser lock for Phase 2b-prep (read+draft only)..."
1927
+ acquire_lock "twitter-browser" 3600 2>>"$LOG_FILE"
1928
+ log "twitter-browser lock held (pid=$$) Phase 2b-prep"
1929
+ # Drop stale singleton locks (see clean_stale_singleton.sh, also called in Phase 1).
1930
+ ensure_twitter_browser_for_backend 2>&1 | tee -a "$LOG_FILE"
1931
+
1932
+ # Thread-media capture (2026-06-03, gated by S4L_TWITTER_CAPTURE_MEDIA, default
1933
+ # OFF). Now that the browser lock is held and the harness Chrome is up, do ONE
1934
+ # cheap deterministic pass over every candidate thread to pull its media
1935
+ # (images/videos/GIFs/link-cards), persist each into
1936
+ # twitter_candidates.thread_media, and build a MEDIA CONTEXT block injected into
1937
+ # the prep prompt so the reply-writer can react to what the tweet visually shows
1938
+ # instead of replying text-blind. Must be deterministic (Python pre-fetch) because
1939
+ # the prep prompt forbids the model from calling twitter_browser.py. Entirely
1940
+ # best-effort: any failure leaves MEDIA_BLOCK empty and the cycle proceeds.
1941
+ MEDIA_BLOCK=""
1942
+ if [ "${S4L_TWITTER_CAPTURE_MEDIA:-0}" = "1" ] || [ "${S4L_TWITTER_CAPTURE_MEDIA:-}" = "true" ]; then
1943
+ if [ -s "$MEDIA_URLS_FILE" ]; then
1944
+ log "Phase 2b-prep: capturing thread media for $(wc -l < "$MEDIA_URLS_FILE" | tr -d ' ') candidate(s)..."
1945
+ MEDIA_BLOCK=$(python3 "$REPO_DIR/scripts/capture_thread_media.py" --urls-file "$MEDIA_URLS_FILE" --scroll 1 2>>"$LOG_FILE" || true)
1946
+ if [ -n "$MEDIA_BLOCK" ]; then
1947
+ log "Phase 2b-prep: media context captured ($(printf '%s' "$MEDIA_BLOCK" | grep -c '^Candidate ') thread(s) with media)."
1948
+ else
1949
+ log "Phase 2b-prep: no media captured (none found or capture skipped)."
1950
+ fi
1951
+ fi
1952
+ else
1953
+ log "Phase 2b-prep: thread-media capture disabled (S4L_TWITTER_CAPTURE_MEDIA not set)."
1954
+ fi
1955
+ rm -f "$MEDIA_URLS_FILE" 2>/dev/null || true
1956
+
1957
+ # --- PERSONA CORPUS injection (personal_brand lane only) --------------------
1958
+ # build_persona.py apply writes a raw first-hand corpus sidecar next to
1959
+ # config.json. In the personal_brand lane we inline it so the drafter grounds
1960
+ # replies in real specifics (projects, numbers, opinions) instead of the single
1961
+ # synthesized content_angle paragraph. Empty string in the promotion lane, so
1962
+ # promotion prompts stay lean and config.json is never bloated with the corpus.
1963
+ CORPUS_BLOCK=""
1964
+ if [ "${S4L_ACTIVE_LANE:-}" = "personal_brand" ] && [ -f "$REPO_DIR/persona_corpus.txt" ]; then
1965
+ CORPUS_BLOCK="## PERSONA CORPUS (raw first-hand material — ground your reply in THIS)
1966
+ This is the persona's own public writing and work, verbatim. Quote and draw real specifics from it: actual projects, real numbers, sharp opinions, real failures. Do NOT invent anything not supported here or in the persona voice block. Use it to make the reply concrete and unmistakably human.
1967
+ $(cat "$REPO_DIR/persona_corpus.txt")
1968
+ "
1969
+ log "Phase 2b-prep: injected persona corpus ($(wc -c < "$REPO_DIR/persona_corpus.txt" | tr -d ' ') bytes)."
1970
+ fi
1971
+
1972
+ log "Phase 2b-prep: Claude reading threads and drafting replies (no post cap)..."
1973
+
1974
+ # Pre-assign the prep session UUID in the parent shell so it survives the
1975
+ # command-substitution subshell run_claude.sh runs in. We write it into the
1976
+ # plan JSON below so Phase 2b-post can re-export it for log_post.py, which
1977
+ # stamps posts.claude_session_id and lets the dashboard activity feed join
1978
+ # to claude_sessions for cost. Without this, twitter posts get NULL session
1979
+ # ids and blank cost cells.
1980
+ CLAUDE_SESSION_ID="$(uuidgen | tr 'A-Z' 'a-z')"
1981
+ export CLAUDE_SESSION_ID
1982
+
1983
+ # PREP_SCHEMA — strict JSON schema for the prep envelope. Includes
1984
+ # optional `new_style` per candidate (an inner object) that the model
1985
+ # MUST populate when it chooses a brand-new engagement style name (i.e.
1986
+ # the picker set mode=invent and the model invented a snake_case name).
1987
+ # Fields mirror engagement_styles.py::_REQUIRED_NEW_STYLE_FIELDS so the
1988
+ # downstream validate_or_register call accepts the block without a
1989
+ # second schema layer.
1990
+ PREP_SCHEMA='{"type":"object","properties":{"candidates":{"type":"array","items":{"type":"object","properties":{"candidate_id":{"type":"integer"},"candidate_url":{"type":"string"},"thread_author":{"type":"string"},"thread_text":{"type":"string"},"matched_project":{"type":"string"},"reply_text":{"type":"string"},"engagement_style":{"type":"string"},"new_style":{"type":["object","null"],"properties":{"description":{"type":"string"},"example":{"type":"string"},"why_existing_didnt_fit":{"type":"string"},"note":{"type":"string"},"target_chars":{"type":"integer"}}},"language":{"type":"string"},"has_landing_pages":{"type":"boolean"},"link_keyword":{"type":"string"},"link_slug":{"type":"string"},"search_topic":{"type":"string"}},"required":["candidate_id","candidate_url","matched_project","reply_text","engagement_style","language","has_landing_pages","search_topic"]}},"rejected":{"type":"array","items":{"type":"object","properties":{"candidate_id":{"type":"integer"},"reason":{"type":"string"},"proposed_excludes":{"type":"array","items":{"type":"string"}}},"required":["candidate_id","reason"]}}},"required":["candidates","rejected"]}'
1991
+
1992
+ PREP_PROMPT="${TW_ENGINE_PREFIX}You are the Social Autoposter prep step.
1993
+
1994
+ Your ONLY job in THIS session:
1995
+ 1. Read each candidate's thread context from the PRE-SCORED CANDIDATES block below (each entry's 'Text:' field is the parent tweet). You have WebSearch and WebFetch available: use them ONLY when a thread hinges on a current fact, a name, a release, or a claim you are not sure about, so your reply is specific and correct instead of vague. You do NOT have the Twitter/X browser this session — never fetch, navigate, or open a tweet/x.com URL, and never try to load the thread itself; the thread text you need is already inlined below. Most replies need no search at all; reach for it only when it materially improves the reply.
1996
+ 2. Draft a reply for each.
1997
+ 3. Persist each fresh draft via log_draft.py.
1998
+ 4. Emit a structured plan describing the chosen candidates, the reply text, and (when applicable) the SEO link keyword + slug.
1999
+
2000
+ You will NOT post anything. You will NOT generate landing pages. You will NOT call log_post.py. The shell handles all of that AFTER your session ends, with the browser lock released for the long landing-page build.
2001
+
2002
+ Read $SKILL_FILE for content rules and voice context.
2003
+ Read $REPO_DIR/config.json for project metadata.
2004
+
2005
+ ## PRE-SCORED CANDIDATES (sorted by Virality DESC, highest first)
2006
+ Virality is a composite predictor of how big this thread will get AFTER you reply: it combines engagement velocity (eng/hour), author reach (follower tier), age decay (6h half-life), retweet ratio, reply count, and discussion quality (reply:like ratio). On historical posted data the highest-Virality cohort (score >= 10000) received ~36x the median reply views of the lowest cohort (score < 10), so prioritize on-brand candidates with HIGH Virality. Rule of thumb: Virality >= 100 = strong thread on a real growth curve, your reply is likely to land 10-100x more eyeballs than a low-Virality thread. Delta (5min) is the raw T1-T0 engagement count and is shown for context only; do not re-rank on Delta.
2007
+ $CANDIDATE_BLOCK
2008
+ $MEDIA_BLOCK
2009
+ $CORPUS_BLOCK
2010
+
2011
+ ## PROJECT ROUTING (per-candidate)
2012
+ Each candidate has a 'Project match' field. Use that project unless the thread content clearly better fits another project.
2013
+ All project configs: $ALL_PROJECTS_JSON
2014
+
2015
+ ## FEEDBACK FROM PAST PERFORMANCE:
2016
+ $TOP_REPORT
2017
+
2018
+ $STYLES_BLOCK
2019
+
2020
+ ## WORKFLOW
2021
+ There is NO cap on how many candidates you may pick this cycle. Pick EVERY candidate whose thread is genuinely on-brand and worth a substantive reply. Skip a candidate ONLY when its thread is off-topic for the matched project, toxic / hateful, low-quality / spam, an audience mismatch, or a near-duplicate of something already replied to. Do NOT cap, quota, or balance picks by project: if the strongest candidates this cycle all belong to one project, pick all of them. Project routing matters; project diversification does not. Never force a weak entry just to add volume, and never drop a strong on-brand entry just to limit volume.
2022
+
2023
+ For each chosen candidate:
2024
+ 1. Read the candidate's parent tweet from its 'Text:' field in the PRE-SCORED CANDIDATES block above.
2025
+ 2. Understand the context from that inlined text (the thread text is already in this prompt; you do NOT have the Twitter browser, but you MAY use WebSearch/WebFetch for external facts when a thread needs them to be answered well).
2026
+ 3. DRAFT HANDLING (existing vs fresh):
2027
+ - If the candidate block shows an EXISTING DRAFT line AND draft age < 30 minutes, REUSE the draft text verbatim. Set engagement_style to the existing style. Do NOT call log_draft.py; do NOT redraft. Reason: prior cycle paid the LLM cost.
2028
+ - $DRAFT_DIRECTIVE
2029
+ 3a. PERSIST FRESH DRAFTS (skip for reused drafts):
2030
+ python3 $REPO_DIR/scripts/log_draft.py --candidate-id CANDIDATE_ID --text 'YOUR_REPLY_TEXT' --style STYLE --assigned-style '$PICKED_STYLE' --assigned-mode '$PICKED_MODE'
2031
+ The --assigned-style / --assigned-mode flags carry the orchestrator's picker output (this cycle: mode=$PICKED_MODE style='${PICKED_STYLE:-(invent)}') into the candidate row so the post pipeline can coerce drift and register invented styles. Pass them VERBATIM as shown.
2032
+ If you are inventing a brand-new style this cycle (i.e. \$PICKED_MODE=invent and your STYLE is a new snake_case name not in the style block above), ALSO pass:
2033
+ --new-style '{\"description\":\"...\",\"example\":\"...\",\"why_existing_didnt_fit\":\"...\"}'
2034
+ with the same description/example/why_existing_didnt_fit you put in the 'new_style' field of your output JSON for this candidate.
2035
+ Failure here is non-fatal, log a warning and continue.
2036
+ 4. EMIT one entry in the structured 'candidates' array with these fields:
2037
+ - candidate_id (int): from the candidate block
2038
+ - candidate_url (string): the parent tweet URL
2039
+ - thread_author (string): the @handle (no leading @)
2040
+ - thread_text (string): the parent tweet's text, condensed to <=500 chars if needed
2041
+ - matched_project (string): the project name to attribute this post to
2042
+ - reply_text (string): the FINAL reply text WITHOUT any URL appended (the shell appends the URL later). 250 chars is the hard ceiling (leaves room for a 23-char t.co link inside the 280-char cap) — stay well under it, not up to it.
2043
+ - engagement_style (string): style name applied (or 'reused' for an unchanged stale draft). In USE mode ($PICKED_MODE=use) this MUST be the assigned style name '${PICKED_STYLE}' verbatim; the orchestrator silently coerces drift back. In INVENT mode ($PICKED_MODE=invent) this MUST be a NEW snake_case style name not in the curated style block.
2044
+ - new_style (object, REQUIRED iff INVENT mode produced a new name; OMIT or set null otherwise): {description (string), example (string), why_existing_didnt_fit (string), note (string, optional), target_chars (integer, REQUIRED)}. Same shape you passed to --new-style in step 3a. The post pipeline reads this and POSTs to /api/v1/engagement-styles/registry so the new style lands in engagement_styles_registry alongside Reddit/GitHub/Moltbook inventions. target_chars is the comment length THIS new style wins at, in characters. IMPORTANT: the example you write must be EXACTLY that length — the example IS the canonical length reference, and the hard ceiling is target_chars × 1.5. Write the example first, count its characters, then set target_chars to that count. Bias SHORT: one-liner style ~45, story-arc style up to ~180, never above 220.
2045
+ - language (string): ISO 639-1 code (en, ja, zh, es, ...)
2046
+ - has_landing_pages (bool): true iff the matched project has BOTH landing_pages.repo AND landing_pages.base_url set in config.json. Otherwise false.
2047
+ - link_keyword (string, REQUIRED when has_landing_pages=true; OMIT otherwise): a SHORT 3-6 word phrase that captures the ESSENCE OF YOUR REPLY (not just the thread topic). Think: what would a reader search to find a useful page about what you just said?
2048
+ - link_slug (string, REQUIRED when has_landing_pages=true; OMIT otherwise): kebab-case, alphanumeric+hyphens only, max 50 chars.
2049
+ - search_topic (string, REQUIRED): normally the EXACT 'Search query' value from this candidate's block above, copied verbatim (do not paraphrase, normalise, or trim). EXCEPTION (cross-route): if the matched_project you chose for this candidate is DIFFERENT from the candidate's 'Project match' field (i.e. you re-routed the thread to a better-fitting project), set search_topic to an empty string \"\" instead. The origin query's topic belongs to the project that ISSUED that query, not the one you routed to; copying it onto the new project's post miscredits the new project's topic ranking and the issuing project's query bank. When matched_project equals the 'Project match' field, copy the topic verbatim as before. The shell stamps this onto posts.search_topic so the next cycle's Phase 1 can rank which topics convert (clicks per post) and evolve the universe accordingly.
2050
+
2051
+ 5. CLASSIFY EVERY PRE-SCORED CANDIDATE into ONE of THREE outcomes. There is NO post cap and NO per-project quota: post EVERY thread you judge genuinely on-brand.
2052
+ (a) 'candidates' — an on-brand pick you are replying to this cycle (step 4 above). No cap.
2053
+ (b) 'rejected' — ONLY for a PERMANENT, thread-intrinsic reason this thread should NEVER be replied to for the matched project: off-topic for the project, toxic / hateful, low-quality / spam / promo / shill, audience or ICP mismatch, our own account, or stale. Reason must be <=200 chars, plain text, no quotes. CRITICAL: the shell marks every 'rejected' entry status='skipped', and a skipped (thread, project) is filtered out of ALL future scans for this account PERMANENTLY. Only reject things that will never be a good fit.
2054
+ (c) OMIT from BOTH arrays — for a TIMING-ONLY reason where the thread itself is fine but you are simply not posting to it right NOW. Omitting keeps it 'pending' so a later cycle can re-judge it. ALWAYS omit (NEVER reject) when your only reason is one of:
2055
+ - you preferred a stronger candidate this cycle (there is no cap, so ideally just post this one too; if you still defer, omit it),
2056
+ - it is a near-duplicate of another thread you are already picking THIS cycle,
2057
+ - you already engaged this author / a similar thread this cycle and want to avoid back-to-back over-engagement.
2058
+ These are DEFERRALS, not rejections. Putting any of them in 'rejected' would permanently blacklist a thread that is actually fine. Do NOT do that.
2059
+ It is fine for 'candidates' to be empty (nothing on-brand) and fine for 'rejected' to be empty (nothing permanently unsuitable).
2060
+ Do NOT update twitter_candidates yourself; the shell will mark every entry of 'rejected' as status='skipped' with the reason, and Phase 0 will salvage anything you omit or forget.
2061
+
2062
+ 5a. SELF-IMPROVING PROJECT-WIDE EXCLUSION LIST (optional, on rejected entries only):
2063
+ When you put a candidate in 'rejected' BECAUSE of a stable, recurring CLASS of false-positive (not a one-off bad tweet), you MAY include a 'proposed_excludes' array of 1-3 specific keywords. If you do, the pipeline will (after a 2-distinct-batch activation gate) automatically append \`-keyword\` to ALL future Twitter searches for the matched_project, project-wide and persistent. This is the ONLY upstream block against the entire class of false-positive that a tighter Phase 1 query alone cannot prevent.
2064
+
2065
+ USE THIS POWER NARROWLY. False-negatives (legit tweets being filtered out) are far worse than the cost of seeing one more cricket tweet. Apply ALL of these rules:
2066
+
2067
+ - DO emit when: the false-positive is caused by a SPECIFIC ambiguous proper noun, brand, or domain term that has a wholly unrelated meaning collisional with the project. Example for Vipassana: an IPL/cricket thread surfaced because the search query included \`Goenka\` (the meditation teacher S.N. Goenka shares a surname with Sanjiv Goenka, owner of an IPL team). Right proposed_excludes: ['cricket','kohli','ipl','lsg','rcb']. WRONG proposed_excludes: ['goenka'] (would mute legit S.N. Goenka tweets).
2068
+
2069
+ - DO NOT emit when: the candidate is just personally low-quality (spam, low engagement, generic), the language is wrong, the author is bot-like, or the thread is just slightly off-topic. Those are one-offs, NOT classes. Use the 'reason' field instead.
2070
+
2071
+ - Each proposed term must be:
2072
+ * a SINGLE token, lowercase, ascii letters/digits/hyphen only, no spaces, length 3-32. (e.g. 'cricket', 'kohli', 'ipl', 'lsg', 'rcb-fan', 'crypto', 'memecoin').
2073
+ * SPECIFIC and unambiguous in the project's domain. Proper nouns, brand names, narrow jargon, sport/team/franchise terms preferred. Generic words like 'practice', 'retreat', 'meditation', 'work', 'tips', 'app', 'tool', 'help' are FORBIDDEN — they will produce false-negatives.
2074
+ * NOT a core search topic of the matched_project (the validator rejects any term in the project's search_topics, so don't waste tokens proposing one).
2075
+
2076
+ - Cap: at most 3 terms per rejected entry. If you need more, you're probably proposing too generically — narrow the list.
2077
+
2078
+ - Activation gate: each term needs >=2 SEPARATE batches to propose it before it goes live, so a single false-rejection cannot mute a search. You don't need to think about this — propose if you'd be confident a future cycle's Claude would also propose it; if not, leave proposed_excludes off.
2079
+
2080
+ - When in doubt, omit the field entirely. The default behavior (no proposed_excludes) is safe; over-proposing is not.
2081
+
2082
+ CRITICAL:
2083
+ - DO NOT post anything. The shell handles posting.
2084
+ - DO NOT call twitter_browser.py.
2085
+ - DO NOT call generate_page.py (the shell runs it AFTER your session, outside the lock).
2086
+ - DO NOT call log_post.py or campaign_bump.py.
2087
+ - You do NOT have the Twitter/X browser this session: never navigate, fetch, or open a tweet/x.com URL, and never try to reload the thread. WebSearch/WebFetch ARE available for external fact-checking only; use them sparingly and never to open the tweet itself.
2088
+ - NEVER use em dashes. Use commas, periods, or regular dashes (-).
2089
+ - Reply in the SAME LANGUAGE as the parent tweet."
2090
+
2091
+ # Pipe the prep prompt via stdin instead of passing as a shell argument.
2092
+ # On Linux ARG_MAX is 2MB; the assembled prompt (config.json + top_report +
2093
+ # styles + schema + candidates) busts that on the VM, dying with E2BIG
2094
+ # "Argument list too long". stdin has no such cap.
2095
+ # --allowedTools: restore external fact-checking to the prep drafter (removed
2096
+ # 2026-06-26). --strict-mcp-config stays so the twitter-harness browser MCP is NOT
2097
+ # loaded: the model can search the web but can never touch the CDP Chrome that
2098
+ # Phase 2b-post drives (that would break the two-lock). The tools are passed as a
2099
+ # SINGLE comma-separated token on purpose: claude_job.py's queue parser (box
2100
+ # installs) treats --allowedTools as a one-value flag, so a space-separated second
2101
+ # tool would leak in as the prompt. On the box these flags ride through
2102
+ # claude_job.py; Desktop's own web search + the reworded prompt enable it there.
2103
+ PREP_OUTPUT=$(printf '%s' "$PREP_PROMPT" | "$REPO_DIR/scripts/run_claude.sh" "run-twitter-cycle-prep" --strict-mcp-config --mcp-config "$TW_MCP_CONFIG" --allowedTools WebSearch,WebFetch -p --output-format json --json-schema "$PREP_SCHEMA" 2>&1)
2104
+
2105
+ echo "$PREP_OUTPUT" >> "$LOG_FILE"
2106
+
2107
+ # --- TOP-N POST CAP (2026-06-29) -------------------------------------------
2108
+ # The prep model still drafts EVERY on-brand candidate, but autopilot now posts
2109
+ # only the single highest-Virality one per cycle. This caps per-account reply
2110
+ # volume (the May-June ~10x ramp that collapsed per-post reach ~15x) while
2111
+ # keeping the strongest thread. Deferred picks are dropped from the plan so they
2112
+ # stay status='pending' (NOT 'skipped'); Phase 0 salvage re-judges them next
2113
+ # cycle and reuses their fresh drafts. (2026-06-30) The cap is now the SINGLE
2114
+ # standard for BOTH lanes: autopilot direct-post AND DRAFT_ONLY manual MCP review.
2115
+ # The old DRAFT_ONLY=1 -> POST_TOP_N=0 special-case was removed on purpose, so the
2116
+ # human reviews the exact same one highest-Virality draft the autopilot would post.
2117
+ # Override with S4L_TWITTER_POST_TOP_N (default 1; 0 = no cap, env opt-out only).
2118
+ POST_TOP_N="${S4L_TWITTER_POST_TOP_N:-1}"
2119
+
2120
+ # --- ROLLING VIRALITY BAR (2026-07-02) --------------------------------------
2121
+ # Fetch THIS install's trailing-24h virality percentile so the parse step posts
2122
+ # the top-1 ONLY if it clears the bar. This holds the post rate near the target
2123
+ # (~20-30 / 8h) with NO hard cap: the bar is the Nth percentile of the install's
2124
+ # OWN recent candidate pool (via /api/v1/twitter-candidates/virality-threshold),
2125
+ # so it self-calibrates to cadence and niche instead of being a fixed number.
2126
+ # The bar is OFF (empty threshold) when:
2127
+ # - DRAFT_ONLY: new users / manual review see every draft (we don't even fetch).
2128
+ # - Cold start: sample_count < min, so a fresh pool posts ungated until it fills.
2129
+ # - Fetch failure: fail-open, never silence posting on a transient API blip.
2130
+ # Tunables: S4L_TWITTER_VIRALITY_PCTILE (default 0.97),
2131
+ # S4L_TWITTER_VIRALITY_MIN_SAMPLE (default 200).
2132
+ VIRALITY_THRESHOLD=""
2133
+ if [ "${DRAFT_ONLY:-0}" != "1" ]; then
2134
+ VIRALITY_THRESHOLD=$(S4L_VPCTILE="${S4L_TWITTER_VIRALITY_PCTILE:-0.97}" \
2135
+ S4L_VMIN="${S4L_TWITTER_VIRALITY_MIN_SAMPLE:-200}" \
2136
+ python3 -c "
2137
+ import os, sys
2138
+ sys.path.insert(0, os.path.expanduser('~/social-autoposter/scripts'))
2139
+ from http_api import api_get
2140
+ try:
2141
+ r = api_get('/api/v1/twitter-candidates/virality-threshold',
2142
+ {'pctile': os.environ['S4L_VPCTILE'], 'hours': 24})
2143
+ d = (r or {}).get('data') or {}
2144
+ thr = d.get('threshold')
2145
+ n = int(d.get('sample_count') or 0)
2146
+ mn = int(os.environ['S4L_VMIN'])
2147
+ if thr is not None and n >= mn:
2148
+ print(f'{float(thr):.4f}')
2149
+ except BaseException as e:
2150
+ sys.stderr.write(f'virality-bar fetch failed (bar OFF this cycle): {e}\n')
2151
+ " 2>>"$LOG_FILE" || echo "")
2152
+ if [ -n "$VIRALITY_THRESHOLD" ]; then
2153
+ log "Virality bar ACTIVE: p${S4L_TWITTER_VIRALITY_PCTILE:-0.97} = $VIRALITY_THRESHOLD (this install, trailing 24h); top-1 posts only if it clears the bar."
2154
+ else
2155
+ log "Virality bar OFF this cycle (cold-start/thin pool or fetch failed); top-1 posts ungated."
2156
+ fi
2157
+ fi
2158
+
2159
+ # Parse the prep envelope and write the plan to \$PLAN_FILE; also extract the
2160
+ # 'rejected' array into \$SKIP_FILE so log_twitter_skips.py can persist a
2161
+ # reason against every twitter_candidates row Claude reviewed but didn't pick.
2162
+ S4L_CAND_VIR="$CANDIDATES" S4L_POST_TOP_N="$POST_TOP_N" VIRALITY_THRESHOLD="$VIRALITY_THRESHOLD" python3 -c "
2163
+ import json, sys, os
2164
+ text = sys.stdin.read().strip()
2165
+ try:
2166
+ env, _ = json.JSONDecoder().raw_decode(text)
2167
+ except Exception as e:
2168
+ print(f'prep: envelope parse error: {e}', file=sys.stderr); sys.exit(1)
2169
+ so = env.get('structured_output')
2170
+ if so is None:
2171
+ so = env.get('result')
2172
+ if isinstance(so, str):
2173
+ try: so = json.loads(so)
2174
+ except Exception: pass
2175
+ candidates = so.get('candidates', []) if isinstance(so, dict) else []
2176
+ rejected = so.get('rejected', []) if isinstance(so, dict) else []
2177
+ # Build candidate_id -> virality_score from the pre-scored CANDIDATES block
2178
+ # (pipe cols: id|url|author|text|virality|delta|...). Shared by the top-N cap
2179
+ # and the rolling virality bar below.
2180
+ _vir = {}
2181
+ for _ln in (os.environ.get('S4L_CAND_VIR', '') or '').splitlines():
2182
+ _p = _ln.split('|')
2183
+ if len(_p) >= 5 and _p[0].isdigit():
2184
+ try: _vir[int(_p[0])] = float(_p[4] or 0)
2185
+ except Exception: pass
2186
+ # TOP-N POST CAP (2026-06-29): keep only the highest-Virality on-brand pick(s).
2187
+ # S4L_POST_TOP_N=0 disables the cap (env opt-out only; the cap applies to both
2188
+ # autopilot and DRAFT_ONLY lanes as of 2026-06-30). Truncated picks are dropped
2189
+ # from the plan, so they stay status='pending' (NOT 'rejected'); Phase 0 salvages.
2190
+ _top_n = int(os.environ.get('S4L_POST_TOP_N', '1') or '1')
2191
+ _deferred = 0
2192
+ if _top_n > 0 and len(candidates) > _top_n:
2193
+ candidates.sort(key=lambda c: _vir.get(c.get('candidate_id'), 0.0), reverse=True)
2194
+ _deferred = len(candidates) - _top_n
2195
+ candidates = candidates[:_top_n]
2196
+ # ROLLING VIRALITY BAR (2026-07-02): drop kept pick(s) below the trailing-24h
2197
+ # percentile of THIS install's candidate pool (VIRALITY_THRESHOLD, from /api/v1).
2198
+ # Empty env = bar OFF: DRAFT_ONLY (new users see every draft), cold start (thin
2199
+ # pool), or fetch failure. Below-bar picks are dropped like deferrals -> stay
2200
+ # 'pending', never 'rejected', so Phase 0 re-judges them next cycle.
2201
+ _bar = (os.environ.get('VIRALITY_THRESHOLD', '') or '').strip()
2202
+ _below_bar = 0
2203
+ if _bar and candidates:
2204
+ try:
2205
+ _thr = float(_bar)
2206
+ _kept = [c for c in candidates if _vir.get(c.get('candidate_id'), 0.0) >= _thr]
2207
+ _below_bar = len(candidates) - len(_kept)
2208
+ candidates = _kept
2209
+ except Exception:
2210
+ pass
2211
+ # The picker assignment travels through the plan envelope so
2212
+ # twitter_post_plan.py can call validate_or_register(...) with the
2213
+ # original (assigned_style, assigned_mode) and coerce USE-mode drift
2214
+ # back to the picker's choice (or accept the INVENT-mode invention +
2215
+ # POST it to /api/v1/engagement-styles/registry). Without this, the
2216
+ # post pipeline can't tell which style the picker actually assigned
2217
+ # vs. what the model picked. Empty string means INVENT mode (NULL
2218
+ # assigned_style in the registry-coercion contract).
2219
+ json.dump({'candidates': candidates,
2220
+ 'session_id': '$CLAUDE_SESSION_ID',
2221
+ 'assigned_style': '$PICKED_STYLE' or None,
2222
+ 'assigned_mode': '$PICKED_MODE' or 'use'}, open('$PLAN_FILE', 'w'), indent=2)
2223
+ json.dump({'skips': rejected}, open('$SKIP_FILE', 'w'), indent=2)
2224
+ print(f'prep: wrote {len(candidates)} candidate(s) (deferred {_deferred} lower-virality, {_below_bar} below bar) and {len(rejected)} skips to $PLAN_FILE / $SKIP_FILE', file=sys.stderr)
2225
+ " <<< "$PREP_OUTPUT" 2>&1 | tee -a "$LOG_FILE"
2226
+
2227
+ PREP_PARSE_EXIT=${PIPESTATUS[0]:-1}
2228
+
2229
+ # Persist the rejected list to twitter_candidates (status='skipped' with reason)
2230
+ # scoped to this batch so we never clobber rows from peer cycles. Non-fatal.
2231
+ if [ -f "$SKIP_FILE" ]; then
2232
+ python3 "$REPO_DIR/scripts/log_twitter_skips.py" \
2233
+ --file "$SKIP_FILE" --require-batch-id "$BATCH_ID" 2>&1 | tee -a "$LOG_FILE" || true
2234
+ rm -f "$SKIP_FILE"
2235
+ fi
2236
+
2237
+ # Classify Anthropic-side error in the prep envelope so the dashboard
2238
+ # surfaces a specific reason (monthly_limit, stream_idle_timeout, api_overloaded,
2239
+ # context_overflow, etc.) rather than a silent failure when prep returns no
2240
+ # plan. Empty plan with NO classified API error falls through to the historical
2241
+ # "empty plan, no failure logged" branch below (salvage retries next cycle).
2242
+ PREP_REASON=$(echo "$PREP_OUTPUT" | python3 "$REPO_DIR/scripts/classify_run_error.py" 2>/dev/null)
2243
+
2244
+ PLAN_COUNT=0
2245
+ if [ "$PREP_PARSE_EXIT" -eq 0 ] && [ -f "$PLAN_FILE" ]; then
2246
+ PLAN_COUNT=$(python3 -c "import json; print(len(json.load(open('$PLAN_FILE')).get('candidates') or []))" 2>/dev/null || echo 0)
2247
+ fi
2248
+ log "Phase 2b-prep complete. plan_count=$PLAN_COUNT"
2249
+
2250
+ # Determine if Phase 2b-gen will be a no-op. When TWITTER_PAGE_GEN_RATE=0
2251
+ # globally, scripts/twitter_gen_links.py rewrites the plan with plain URLs in
2252
+ # <1s. In that case the release-now + re-acquire-after-gen dance is pure waste:
2253
+ # under cycle overlap the re-acquire can sit in the FIFO ticket queue for
2254
+ # 30-90s behind the very `engage-twitter` / next `run-twitter-cycle` we just
2255
+ # handed the lock to. We keep the lock through 2b-gen instead and skip the
2256
+ # dance entirely.
2257
+ GEN_RATE_RAW="${TWITTER_PAGE_GEN_RATE:-0.0}"
2258
+ GEN_IS_NOOP=false
2259
+ case "$GEN_RATE_RAW" in
2260
+ 0|0.0|0.00|0.000|"") GEN_IS_NOOP=true ;;
2261
+ esac
2262
+
2263
+ # Release the lock unless (a) plan is non-empty AND (b) gen is a no-op. The
2264
+ # empty-plan early-exit below still needs the release for a clean handoff, so
2265
+ # we cannot just skip when GEN_IS_NOOP=true unconditionally.
2266
+ if [ "${PLAN_COUNT:-0}" = "0" ] || ! $GEN_IS_NOOP; then
2267
+ log "Releasing twitter-browser lock (gen step is lock-free)..."
2268
+ release_lock "twitter-browser" 2>>"$LOG_FILE"
2269
+ # (2026-06-16) session-lock rm removed (defect b); dead holders self-reclaim
2270
+ # in twitter_browser.py now. Do NOT re-add. See Phase 1 note + docs/twitter_browser_lock.md.
2271
+ else
2272
+ log "Keeping twitter-browser lock through Phase 2b-gen (TWITTER_PAGE_GEN_RATE=$GEN_RATE_RAW, gen is a no-op; skipping release/re-acquire dance)"
2273
+ fi
2274
+
2275
+ if [ "${PLAN_COUNT:-0}" = "0" ]; then
2276
+ log "Empty plan from prep step. Exiting cycle without posting (pending rows salvaged next cycle)."
2277
+ rm -f "$PLAN_FILE"
2278
+ _COST=$(python3 "$REPO_DIR/scripts/get_run_cost.py" --cycle-id "$BATCH_ID" 2>/dev/null || echo "0.0000")
2279
+ # If the classifier identified a real Anthropic error (any non-empty reason
2280
+ # key), log as failed=1 with that reason so the dashboard pill reads
2281
+ # "failed: stream_idle_timeout" / "failed: monthly_limit" / etc. Otherwise
2282
+ # keep the historical failed=0 behaviour for "empty plan, no API error"
2283
+ # (salvage retries the candidates next cycle, nothing to surface).
2284
+ if [ -n "$PREP_REASON" ]; then
2285
+ python3 "$REPO_DIR/scripts/log_run.py" --script "post_twitter" --posted 0 --skipped "${CANDIDATE_COUNT:-0}" --failed 1 --salvaged "${SALVAGED:-0}" \
2286
+ --queries "${QUERIES_TOTAL:-0}" --duds "${DUDS_TOTAL:-0}" \
2287
+ --tweets-pulled "${TWEETS_PULLED:-0}" --candidates "${BATCH_COUNT:-0}" --above-floor "${HIGH_DELTA_COUNT:-0}" \
2288
+ --failure-reasons "${PREP_REASON}:1" --cost "$_COST" --elapsed $(( $(date +%s) - RUN_START ))
2289
+ else
2290
+ python3 "$REPO_DIR/scripts/log_run.py" --script "post_twitter" --posted 0 --skipped "${CANDIDATE_COUNT:-0}" --failed 0 --salvaged "${SALVAGED:-0}" \
2291
+ --queries "${QUERIES_TOTAL:-0}" --duds "${DUDS_TOTAL:-0}" \
2292
+ --tweets-pulled "${TWEETS_PULLED:-0}" --candidates "${BATCH_COUNT:-0}" --above-floor "${HIGH_DELTA_COUNT:-0}" \
2293
+ --cost "$_COST" --elapsed $(( $(date +%s) - RUN_START ))
2294
+ fi
2295
+ # In DRAFT_ONLY (MCP draft_cycle) mode, a non-empty PREP_REASON means the
2296
+ # prep step FAILED for a real reason (e.g. claude_not_logged_in) rather than
2297
+ # genuinely finding nothing on-brand. Surface it on stdout so the MCP wrapper
2298
+ # can tell the user the actual problem (e.g. "run claude /login") instead of
2299
+ # mis-reporting it as "all threads already engaged".
2300
+ if [ "${DRAFT_ONLY:-0}" = "1" ] && [ -n "$PREP_REASON" ]; then
2301
+ echo "DRAFT_ONLY_BLOCKED=$PREP_REASON"
2302
+ fi
2303
+ _SA_RUN_SUMMARY_EMITTED=1
2304
+ exit 0
2305
+ fi
2306
+
2307
+ # --- Phase 2b-gen: SEO landing pages (no browser lock) ----------------------
2308
+ # phase2b-gen has the longest budget (60 min) because the SEO landing-page
2309
+ # build can legitimately run 10-40 min. Stamping it here is what protects
2310
+ # this cycle from being salvaged out from under itself.
2311
+ python3 "$REPO_DIR/scripts/twitter_batch_phase.py" advance "$BATCH_ID" --phase phase2b-gen 2>&1 | tee -a "$LOG_FILE" || true
2312
+ log "Phase 2b-gen: generating SEO pages for $PLAN_COUNT candidate(s) without holding the browser lock..."
2313
+ python3 "$REPO_DIR/scripts/twitter_gen_links.py" --plan "$PLAN_FILE" 2>&1 | tee -a "$LOG_FILE"
2314
+ GEN_EXIT=${PIPESTATUS[0]:-1}
2315
+ if [ "$GEN_EXIT" -ne 0 ]; then
2316
+ log "WARN: twitter_gen_links.py exited $GEN_EXIT, continuing with whatever links it set (per-candidate fallback to plain project URL on gen failure)."
2317
+ fi
2318
+
2319
+ # --- DRAFT_ONLY gate: stop after drafting for human review (MCP manual mode) -
2320
+ # When DRAFT_ONLY=1 the cycle runs scan -> score -> draft -> link-gen, leaves the
2321
+ # fully-baked plan (links already stamped into reply_text) at $PLAN_FILE, and
2322
+ # STOPS before posting. The social-autoposter MCP draft_cycle tool reads that
2323
+ # plan, walks the human through approve/skip per draft, then posts the approved
2324
+ # subset via twitter_post_plan.py. Nothing is posted from this script in that
2325
+ # mode. The gate sits AFTER 2b-gen on purpose: twitter_post_plan.py does not run
2326
+ # link-gen itself, so the plan must already carry baked links before we hand it
2327
+ # off. Run with TWITTER_PAGE_GEN_RATE=0 (the default) so gen is a sub-second
2328
+ # plain-URL rewrite, not a 10-40 min SEO build, in the interactive path.
2329
+ if [ "${DRAFT_ONLY:-0}" = "1" ]; then
2330
+ # Not posting, so the browser lock isn't needed; release it if still held.
2331
+ release_lock "twitter-browser" 2>>"$LOG_FILE" || true
2332
+ # (2026-06-16) session-lock rm removed (defect b); dead holders self-reclaim
2333
+ # in twitter_browser.py now. Do NOT re-add. See Phase 1 note + docs/twitter_browser_lock.md.
2334
+ log "DRAFT_ONLY=1: $PLAN_COUNT draft(s) ready for review at $PLAN_FILE. Stopping before post."
2335
+ # Emit a clean posted=0 run row and suppress the EXIT-trap summary oneshot, so
2336
+ # a draft-only run is NOT mis-synthesized as a phase2b_silent failure (the
2337
+ # trap's fallback would do that for posted=0 with candidates pending).
2338
+ _COST=$(python3 "$REPO_DIR/scripts/get_run_cost.py" --cycle-id "$BATCH_ID" 2>/dev/null || echo "0.0000")
2339
+ python3 "$REPO_DIR/scripts/log_run.py" --script "post_twitter" --posted 0 --skipped 0 --failed 0 --salvaged "${SALVAGED:-0}" \
2340
+ --queries "${QUERIES_TOTAL:-0}" --duds "${DUDS_TOTAL:-0}" \
2341
+ --tweets-pulled "${TWEETS_PULLED:-0}" --candidates "${BATCH_COUNT:-0}" --above-floor "${HIGH_DELTA_COUNT:-0}" \
2342
+ --cost "$_COST" --elapsed $(( $(date +%s) - RUN_START )) 2>/dev/null || true
2343
+ _SA_RUN_SUMMARY_EMITTED=1
2344
+ # IMPORTANT: do NOT rm -f "$PLAN_FILE" here; the reviewer needs it. Print a
2345
+ # machine-readable marker so the MCP wrapper can locate the plan deterministically.
2346
+ echo "DRAFT_ONLY_PLAN=$PLAN_FILE"
2347
+ exit 0
2348
+ fi
2349
+
2350
+ # --- Phase 2b-post: re-acquire browser lock and post ------------------------
2351
+ # Stamp phase2b-post (15-min budget) before the browser-side reply loop. After
2352
+ # 2b-gen's potentially long run, peer cycles' 20-min phase2a fallback would
2353
+ # already be tripping if we left the row at phase2a.
2354
+ python3 "$REPO_DIR/scripts/twitter_batch_phase.py" advance "$BATCH_ID" --phase phase2b-post 2>&1 | tee -a "$LOG_FILE" || true
2355
+ # Re-acquire only if we actually released for gen (see GEN_IS_NOOP above).
2356
+ # When the lock was kept through 2b-gen there's nothing to re-acquire.
2357
+ if ! $GEN_IS_NOOP; then
2358
+ log "Re-acquiring twitter-browser lock for Phase 2b-post..."
2359
+ acquire_lock "twitter-browser" 3600 2>>"$LOG_FILE"
2360
+ fi
2361
+ log "twitter-browser lock held (pid=$$) Phase 2b-post"
2362
+ # Drop stale singleton locks (see clean_stale_singleton.sh, also called in Phase 1 / 2b-prep).
2363
+ ensure_twitter_browser_for_backend 2>&1 | tee -a "$LOG_FILE"
2364
+
2365
+ log "Phase 2b-post: posting $PLAN_COUNT candidate(s)..."
2366
+ POST_OUTPUT=$("${S4L_PYTHON:-python3}" "$REPO_DIR/scripts/twitter_post_plan.py" --plan "$PLAN_FILE" 2>&1)
2367
+ echo "$POST_OUTPUT" >> "$LOG_FILE"
2368
+
2369
+ # The post helper prints a JSON summary on its last stdout line.
2370
+ POST_SUMMARY=$(printf '%s\n' "$POST_OUTPUT" | tail -n 1)
2371
+ EXEC_POSTED=$(python3 -c "import json,sys; d=json.loads(sys.argv[1] or '{}'); print(d.get('posted', 0))" "$POST_SUMMARY" 2>/dev/null || echo 0)
2372
+ EXEC_SKIPPED=$(python3 -c "import json,sys; d=json.loads(sys.argv[1] or '{}'); print(d.get('skipped', 0))" "$POST_SUMMARY" 2>/dev/null || echo 0)
2373
+ EXEC_FAILED=$(python3 -c "import json,sys; d=json.loads(sys.argv[1] or '{}'); print(d.get('failed', 0))" "$POST_SUMMARY" 2>/dev/null || echo 0)
2374
+ EXEC_REASONS=$(python3 -c "import json,sys; d=json.loads(sys.argv[1] or '{}'); print(d.get('failure_reasons', ''))" "$POST_SUMMARY" 2>/dev/null || echo "")
2375
+ EXEC_SKIP_REASONS=$(python3 -c "import json,sys; d=json.loads(sys.argv[1] or '{}'); print(d.get('skip_reasons', ''))" "$POST_SUMMARY" 2>/dev/null || echo "")
2376
+ log "Phase 2b-post summary: posted=$EXEC_POSTED skipped=$EXEC_SKIPPED failed=$EXEC_FAILED reasons=$EXEC_REASONS skip_reasons=$EXEC_SKIP_REASONS"
2377
+
2378
+ rm -f "$PLAN_FILE"
2379
+
2380
+ # Generation trace tempfile cleanup. By now every post in this cycle that
2381
+ # made it to log_post.py has the trace persisted to posts.generation_trace
2382
+ # JSONB, so the on-disk JSON is redundant. Best-effort delete.
2383
+ if [ -n "$S4L_TWITTER_GEN_TRACE_PATH" ] && [ -f "$S4L_TWITTER_GEN_TRACE_PATH" ]; then
2384
+ rm -f "$S4L_TWITTER_GEN_TRACE_PATH"
2385
+ fi
2386
+
2387
+ # --- No end-of-cycle expire ------------------------------------------------
2388
+ # Pending rows are intentionally left alone. They are either:
2389
+ # - candidates Phase 2b never reached (e.g., org quota, browser crash, or
2390
+ # a phase budget elapsing before the long tail was reviewed), and the
2391
+ # next cycle's Phase 0 will salvage them while still fresh
2392
+ # - hard-expired by the next cycle's Phase 0 once they cross FRESHNESS_HOURS
2393
+ # This avoids losing work to transient infra failures.
2394
+
2395
+ # --- Summary ---------------------------------------------------------------
2396
+ # Per-run-log human readout. The persistent run_monitor.log row is written
2397
+ # by _sa_emit_run_summary_oneshot (defined near the top of this script) so
2398
+ # SIGTERM during the summary block still produces a dashboard-visible row.
2399
+ # Summary now comes from /api/v1/twitter-candidates/counts-by-batch via
2400
+ # the helper, formatted as "status|count\nstatus|count..." to match the
2401
+ # legacy psql -F '|' shape this log line consumed.
2402
+ SUMMARY=$(python3 "$REPO_DIR/scripts/twitter_cycle_helper.py" batch-summary --batch-id "$BATCH_ID" 2>/dev/null)
2403
+ log "Batch summary: $SUMMARY"
2404
+
2405
+ _sa_emit_run_summary_oneshot
2406
+
2407
+ log "=== Cycle complete: $(date) ==="
2408
+ find "$LOG_DIR" -name "twitter-cycle-*.log" -mtime +7 -delete 2>/dev/null || true