@m13v/s4l 1.6.197-rc.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +143 -0
- package/SKILL.md +342 -0
- package/bin/cli.js +980 -0
- package/bin/cookie-helper.js +315 -0
- package/bin/platform.js +59 -0
- package/bin/scheduler/index.js +12 -0
- package/bin/scheduler/launchd.js +518 -0
- package/browser-agent-configs/all-agents-mcp.json +68 -0
- package/browser-agent-configs/linkedin-agent-mcp.json +16 -0
- package/browser-agent-configs/linkedin-agent.json +17 -0
- package/browser-agent-configs/linkedin-harness-mcp.json +21 -0
- package/browser-agent-configs/reddit-agent-mcp.json +16 -0
- package/browser-agent-configs/reddit-agent.json +17 -0
- package/browser-agent-configs/twitter-harness-mcp.json +18 -0
- package/config.example.json +45 -0
- package/mcp/dist/index.js +4212 -0
- package/mcp/dist/onboarding.js +200 -0
- package/mcp/dist/panel.html +176 -0
- package/mcp/dist/product-link.html +102 -0
- package/mcp/dist/repo.js +222 -0
- package/mcp/dist/runtime.js +1079 -0
- package/mcp/dist/screencast.js +323 -0
- package/mcp/dist/setup.js +545 -0
- package/mcp/dist/telemetry.js +306 -0
- package/mcp/dist/twitterAuth.js +138 -0
- package/mcp/dist/version.js +271 -0
- package/mcp/dist/version.json +4 -0
- package/mcp/install-runtime.mjs +70 -0
- package/mcp/install.mjs +169 -0
- package/mcp/manifest.json +80 -0
- package/mcp/menubar/dashboard_server.py +213 -0
- package/mcp/menubar/s4l_card.py +1336 -0
- package/mcp/menubar/s4l_log_relay.py +179 -0
- package/mcp/menubar/s4l_menubar.py +2439 -0
- package/mcp/menubar/s4l_state.py +891 -0
- package/mcp/package.json +34 -0
- package/mcp/shared/doctor.cjs +437 -0
- package/mcp/shared/onboarding-ledger.cjs +324 -0
- package/mcp-servers/browser-harness/server.py +968 -0
- package/package.json +160 -0
- package/requirements.txt +20 -0
- package/scripts/_compute_allowlist.py +58 -0
- package/scripts/_db_update.py +20 -0
- package/scripts/_filt.py +9 -0
- package/scripts/_li_notif_match.py +76 -0
- package/scripts/_li_notif_orchestrate.py +126 -0
- package/scripts/_lock_preempt_test.py +60 -0
- package/scripts/_run_icp_precheck.py +57 -0
- package/scripts/a16z_pearx_calendar_reminders.py +99 -0
- package/scripts/account_resolver.py +141 -0
- package/scripts/active_campaigns.py +114 -0
- package/scripts/active_users.py +190 -0
- package/scripts/amplitude_24h_signups.py +468 -0
- package/scripts/amplitude_signups.py +177 -0
- package/scripts/apply_onboarding_selections.py +131 -0
- package/scripts/audience_pages.py +243 -0
- package/scripts/audit_helper.py +120 -0
- package/scripts/author_history_block.py +353 -0
- package/scripts/autopilot_stall_watch.py +284 -0
- package/scripts/backfill_twitter_attempts_topic.py +81 -0
- package/scripts/backfill_twitter_log_post_no_id.py +322 -0
- package/scripts/bench_dashboard.sh +138 -0
- package/scripts/bh_send.py +39 -0
- package/scripts/build_persona.py +409 -0
- package/scripts/bulk_icp.py +18 -0
- package/scripts/campaign_bump.py +51 -0
- package/scripts/capture_thread_media.py +288 -0
- package/scripts/check_browser_lock_health.sh +81 -0
- package/scripts/check_external_pool_depth.py +253 -0
- package/scripts/check_unread_web_chats.py +28 -0
- package/scripts/claim_web_chat.py +47 -0
- package/scripts/classify_run_error.py +158 -0
- package/scripts/claude_job.py +988 -0
- package/scripts/clean_stale_singleton.sh +56 -0
- package/scripts/cleanup_harness_tabs.py +68 -0
- package/scripts/copy_browser_cookies.py +454 -0
- package/scripts/counterparty_history.py +350 -0
- package/scripts/db.py +57 -0
- package/scripts/discover_claude_profiles.py +120 -0
- package/scripts/discover_linkedin_candidates.py +984 -0
- package/scripts/dm_conversation.py +682 -0
- package/scripts/dm_db_update.py +69 -0
- package/scripts/dm_engage_helper.py +161 -0
- package/scripts/dm_outreach_helper.py +147 -0
- package/scripts/dm_outreach_twitter_helper.py +129 -0
- package/scripts/dm_send_log.py +106 -0
- package/scripts/dm_short_links.py +1084 -0
- package/scripts/dump_web_chat_history.py +47 -0
- package/scripts/engage_github.py +640 -0
- package/scripts/engage_reddit.py +1235 -0
- package/scripts/engage_twitter_helper.py +301 -0
- package/scripts/engagement_styles.py +1787 -0
- package/scripts/enrich_twitter_candidates.py +82 -0
- package/scripts/feedback_digest.py +448 -0
- package/scripts/fetch_prospect_profile.py +312 -0
- package/scripts/fetch_twitter_t1.py +134 -0
- package/scripts/find_threads.py +530 -0
- package/scripts/follow_gate_log.py +59 -0
- package/scripts/funnel_per_day.py +194 -0
- package/scripts/generate_daily_human_style.py +494 -0
- package/scripts/generation_trace.py +173 -0
- package/scripts/get_run_cost.py +107 -0
- package/scripts/github_engage_helper.py +93 -0
- package/scripts/github_tools.py +509 -0
- package/scripts/harness_overlay.py +556 -0
- package/scripts/harvest_twitter_following.py +243 -0
- package/scripts/heartbeat.sh +70 -0
- package/scripts/history_context.py +284 -0
- package/scripts/http_api.py +206 -0
- package/scripts/human_dm_replies_helper.py +169 -0
- package/scripts/identity.py +302 -0
- package/scripts/ig_batch_creator.sh +93 -0
- package/scripts/ig_post_type_picker.py +243 -0
- package/scripts/ig_scrape_transcribe.sh +91 -0
- package/scripts/ingest_human_dm_replies.py +271 -0
- package/scripts/ingest_web_chat_replies.py +229 -0
- package/scripts/install_fleet.py +187 -0
- package/scripts/invent_mcp_server.py +350 -0
- package/scripts/invent_topics.py +1462 -0
- package/scripts/learned_preferences.py +263 -0
- package/scripts/li_discovery.py +161 -0
- package/scripts/link_edit_helper.py +142 -0
- package/scripts/link_tail.py +592 -0
- package/scripts/linkedin_api.py +561 -0
- package/scripts/linkedin_browser.py +730 -0
- package/scripts/linkedin_cooldown.py +128 -0
- package/scripts/linkedin_exclusions.py +234 -0
- package/scripts/linkedin_killswitch.py +1333 -0
- package/scripts/linkedin_search_topic_schema.py +49 -0
- package/scripts/linkedin_unipile.py +658 -0
- package/scripts/linkedin_url.py +228 -0
- package/scripts/log_claude_session.py +636 -0
- package/scripts/log_draft.py +143 -0
- package/scripts/log_linkedin_search_attempts.py +126 -0
- package/scripts/log_post.py +651 -0
- package/scripts/log_run.py +364 -0
- package/scripts/log_thread_media.py +108 -0
- package/scripts/log_twitter_search_attempts.py +150 -0
- package/scripts/log_twitter_skips.py +211 -0
- package/scripts/lookup_post.py +78 -0
- package/scripts/mark_web_chat_processed.py +32 -0
- package/scripts/mcp_lock_proxy.py +370 -0
- package/scripts/memory_snapshot.py +972 -0
- package/scripts/merge_review_queue.py +215 -0
- package/scripts/mint_external_pool.py +182 -0
- package/scripts/mint_kent_pool.py +249 -0
- package/scripts/moltbook_post.py +320 -0
- package/scripts/moltbook_tools.py +159 -0
- package/scripts/pending_threads.py +188 -0
- package/scripts/pick_ig_account.py +177 -0
- package/scripts/pick_project.py +208 -0
- package/scripts/pick_search_topic.py +771 -0
- package/scripts/pick_thread_target.py +279 -0
- package/scripts/pick_twitter_thread_target.py +202 -0
- package/scripts/podlog_fetch_batch.sh +32 -0
- package/scripts/post_github.py +1311 -0
- package/scripts/post_reddit.py +2668 -0
- package/scripts/precompute_dashboard_stats.py +204 -0
- package/scripts/preflight.sh +297 -0
- package/scripts/progress.py +88 -0
- package/scripts/project_excludes.py +353 -0
- package/scripts/project_slugs.py +91 -0
- package/scripts/project_stats.py +241 -0
- package/scripts/project_stats_json.py +1563 -0
- package/scripts/project_topics.py +192 -0
- package/scripts/qualified_query_bank.py +436 -0
- package/scripts/reap_stale_claude_sessions.py +867 -0
- package/scripts/reddit_browser.py +2549 -0
- package/scripts/reddit_browser_fetch.py +141 -0
- package/scripts/reddit_browser_lock.py +593 -0
- package/scripts/reddit_chat_sync.py +710 -0
- package/scripts/reddit_query_bank.py +200 -0
- package/scripts/reddit_threads_helper.py +151 -0
- package/scripts/reddit_tools.py +956 -0
- package/scripts/refresh_instagram_tokens.py +280 -0
- package/scripts/release-mcpb.sh +513 -0
- package/scripts/reply_db.py +334 -0
- package/scripts/reply_insert.py +98 -0
- package/scripts/reply_risk_digest.py +761 -0
- package/scripts/reset-test-machine.sh +602 -0
- package/scripts/restore_twitter_session.py +177 -0
- package/scripts/ripen_reddit_plan.py +478 -0
- package/scripts/run_claude.sh +433 -0
- package/scripts/run_moltbook_cycle.py +555 -0
- package/scripts/s4l_box_update.sh +226 -0
- package/scripts/s4l_channel.py +103 -0
- package/scripts/s4l_ctl.sh +75 -0
- package/scripts/s4l_env.py +47 -0
- package/scripts/saps_activity.py +126 -0
- package/scripts/saps_mode.py +328 -0
- package/scripts/scan_dm_candidates.py +580 -0
- package/scripts/scan_github_replies.py +168 -0
- package/scripts/scan_instagram_comments.py +481 -0
- package/scripts/scan_moltbook_replies.py +252 -0
- package/scripts/scan_pii.py +190 -0
- package/scripts/scan_reddit_replies.py +377 -0
- package/scripts/scan_twitter_mentions_browser.py +327 -0
- package/scripts/scan_twitter_thread_followups.py +299 -0
- package/scripts/scan_x_profile.py +384 -0
- package/scripts/schedule_state.py +202 -0
- package/scripts/scheduled_tasks_snapshot.py +123 -0
- package/scripts/score_linkedin_candidates.py +419 -0
- package/scripts/score_twitter_candidates.py +718 -0
- package/scripts/scrape_linkedin_comment_stats.py +1755 -0
- package/scripts/scrape_linkedin_stats_browser.py +52 -0
- package/scripts/scrape_reddit_views.py +365 -0
- package/scripts/seed_search_queries.py +453 -0
- package/scripts/seed_search_topics.py +127 -0
- package/scripts/send_web_chat_reply.py +130 -0
- package/scripts/sentry_init.py +128 -0
- package/scripts/setup_twitter_auth.py +1320 -0
- package/scripts/snapshot.py +583 -0
- package/scripts/stats.py +2702 -0
- package/scripts/stats_helper.py +52 -0
- package/scripts/strike_alert.py +783 -0
- package/scripts/sweep_post_link_clicks.py +107 -0
- package/scripts/sync_ig_to_posts.py +147 -0
- package/scripts/test_browser_lock.py +189 -0
- package/scripts/test_installation_api.sh +52 -0
- package/scripts/test_percard_posting.py +142 -0
- package/scripts/top_dud_linkedin_queries.py +71 -0
- package/scripts/top_dud_reddit_queries.py +67 -0
- package/scripts/top_dud_twitter_queries.py +71 -0
- package/scripts/top_dud_twitter_topics.py +102 -0
- package/scripts/top_linkedin_queries.py +55 -0
- package/scripts/top_omitted_reddit_topics.py +91 -0
- package/scripts/top_performers.py +588 -0
- package/scripts/top_search_topics.py +180 -0
- package/scripts/top_twitter_queries.py +190 -0
- package/scripts/twitter_access_check.py +382 -0
- package/scripts/twitter_account.py +41 -0
- package/scripts/twitter_batch_phase.py +126 -0
- package/scripts/twitter_browser.py +2804 -0
- package/scripts/twitter_cookie_mirror.py +130 -0
- package/scripts/twitter_cycle_helper.py +310 -0
- package/scripts/twitter_gen_links.py +287 -0
- package/scripts/twitter_post_plan.py +1188 -0
- package/scripts/twitter_scan.py +324 -0
- package/scripts/twitter_supply_signal.py +57 -0
- package/scripts/twitter_threads_helper.py +152 -0
- package/scripts/unclaim_web_chat.py +29 -0
- package/scripts/update_instagram_stats.py +261 -0
- package/scripts/update_linkedin_stats_from_feed.py +328 -0
- package/scripts/version.py +72 -0
- package/scripts/watchdog_hung_runs.py +343 -0
- package/scripts/write_generation_trace.py +73 -0
- package/setup/SKILL.md +277 -0
- package/skill/amplitude-24h-signups.sh +38 -0
- package/skill/archive-old-logs.sh +40 -0
- package/skill/audit-dm-staleness.sh +42 -0
- package/skill/audit-linkedin.sh +14 -0
- package/skill/audit-moltbook.sh +4 -0
- package/skill/audit-reddit-resurrect.sh +67 -0
- package/skill/audit-reddit.sh +4 -0
- package/skill/audit-twitter.sh +4 -0
- package/skill/audit.sh +287 -0
- package/skill/backfill-twitter-attempts-topic.sh +19 -0
- package/skill/backfill-twitter-ghost-posts.sh +24 -0
- package/skill/check-external-pool-depth.sh +7 -0
- package/skill/check-web-chats.sh +203 -0
- package/skill/dm-outreach-linkedin.sh +250 -0
- package/skill/dm-outreach-reddit.sh +274 -0
- package/skill/dm-outreach-twitter.sh +265 -0
- package/skill/engage-dm-replies-linkedin.sh +4 -0
- package/skill/engage-dm-replies-reddit.sh +4 -0
- package/skill/engage-dm-replies-twitter.sh +4 -0
- package/skill/engage-dm-replies.sh +1597 -0
- package/skill/engage-linkedin.sh +581 -0
- package/skill/engage-moltbook.sh +36 -0
- package/skill/engage-reddit.sh +146 -0
- package/skill/engage-twitter.sh +467 -0
- package/skill/github-engage.sh +176 -0
- package/skill/ingest-web-chat-replies.sh +38 -0
- package/skill/invent-supply-test.sh +100 -0
- package/skill/invent-topics.sh +50 -0
- package/skill/lib/linkedin-backend.sh +364 -0
- package/skill/lib/platform.sh +48 -0
- package/skill/lib/reddit-backend.sh +234 -0
- package/skill/lib/twitter-backend.sh +314 -0
- package/skill/link-edit-github.sh +136 -0
- package/skill/link-edit-moltbook.sh +117 -0
- package/skill/link-edit-reddit.sh +201 -0
- package/skill/linkedin-presence.sh +182 -0
- package/skill/linkedin-recovery.sh +282 -0
- package/skill/lock.sh +647 -0
- package/skill/memory-snapshot.sh +39 -0
- package/skill/precompute-stats.sh +35 -0
- package/skill/prewarm-funnel.sh +104 -0
- package/skill/refresh-instagram-tokens.sh +57 -0
- package/skill/refresh-twitter-following.sh +52 -0
- package/skill/reply-risk-digest.sh +31 -0
- package/skill/run-cycle-update-guard.sh +44 -0
- package/skill/run-draft-and-publish.sh +123 -0
- package/skill/run-generate-daily-style.sh +50 -0
- package/skill/run-github-launchd.sh +62 -0
- package/skill/run-github.sh +102 -0
- package/skill/run-instagram-daily.sh +149 -0
- package/skill/run-instagram-render.sh +875 -0
- package/skill/run-linkedin-launchd.sh +81 -0
- package/skill/run-linkedin-unipile.sh +130 -0
- package/skill/run-linkedin.sh +1593 -0
- package/skill/run-moltbook-launchd.sh +61 -0
- package/skill/run-moltbook.sh +38 -0
- package/skill/run-overlay-watch.sh +100 -0
- package/skill/run-reddit-search-launchd.sh +64 -0
- package/skill/run-reddit-search.sh +505 -0
- package/skill/run-reddit-threads-double.sh +32 -0
- package/skill/run-reddit-threads.sh +847 -0
- package/skill/run-scan-moltbook-replies.sh +57 -0
- package/skill/run-twitter-cycle-launchd.sh +63 -0
- package/skill/run-twitter-cycle-singleton.sh +62 -0
- package/skill/run-twitter-cycle.sh +2408 -0
- package/skill/run-twitter-threads.sh +592 -0
- package/skill/scan-instagram-replies.sh +61 -0
- package/skill/scan-twitter-followups.sh +57 -0
- package/skill/social-autoposter-update.sh +66 -0
- package/skill/stats-instagram.sh +72 -0
- package/skill/stats-linkedin.sh +271 -0
- package/skill/stats-moltbook.sh +4 -0
- package/skill/stats-reddit.sh +4 -0
- package/skill/stats-twitter.sh +4 -0
- package/skill/stats.sh +521 -0
- package/skill/strike-alert.sh +18 -0
- package/skill/styles.sh +87 -0
- package/skill/sweep-link-clicks.sh +40 -0
- package/skill/topics.sh +51 -0
|
@@ -0,0 +1,2408 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# run-twitter-cycle.sh — Combined Twitter scan + post cycle.
|
|
3
|
+
#
|
|
4
|
+
# Phase 1 (t=0):
|
|
5
|
+
# - select 8 projects via the shared inverse-recent-share picker
|
|
6
|
+
# (scripts/pick_project.py, same logic as github/reddit)
|
|
7
|
+
# - LLM drafts a search query per project (style from past top queries);
|
|
8
|
+
# if Phase 1 yields <RETRY_TARGET candidates that pass all filters
|
|
9
|
+
# (harness age gate + scorer dedupe + already-posted), the scan is
|
|
10
|
+
# re-invoked with the previously-tried queries injected as "do NOT
|
|
11
|
+
# repeat" — up to MAX_SCAN_ATTEMPTS total per cycle, same batch_id.
|
|
12
|
+
# - scrape tweets via twitter-harness, enrich via fxtwitter -> T0 snapshot
|
|
13
|
+
# - store all candidates with batch_id and search_topic
|
|
14
|
+
#
|
|
15
|
+
# No ripen wait (variant D won the A/B/C/D test 2026-05-31): the cycle goes
|
|
16
|
+
# straight from discovery to drafting. There is NO 5-min sleep and NO fxtwitter
|
|
17
|
+
# T1 re-poll anywhere in the Twitter pipeline. delta_score stays at its T0
|
|
18
|
+
# value and is no longer a gate. Do not re-introduce a ripen/sleep step here.
|
|
19
|
+
#
|
|
20
|
+
# Phase 2 (immediately after Phase 1):
|
|
21
|
+
# - sort candidates by virality_score DESC (composite predictor stamped at
|
|
22
|
+
# discovery by score_twitter_candidates.py); no delta floor, no T1 re-poll
|
|
23
|
+
# - Claude reads top 25 (raised from 15 so the long tail reaches the model),
|
|
24
|
+
# drops unsuitable, posts every candidate it judges genuinely on-brand
|
|
25
|
+
# (no per-cycle post cap, no per-project quota)
|
|
26
|
+
# - keep remaining pending rows: salvaged into the next cycle, hard-expired
|
|
27
|
+
# by Phase 0 once tweet age crosses FRESHNESS_HOURS
|
|
28
|
+
#
|
|
29
|
+
# Launchd cadence: every 20 minutes. One combined job, one browser lock.
|
|
30
|
+
|
|
31
|
+
set -uo pipefail
|
|
32
|
+
|
|
33
|
+
# SAPS_->S4L_ env mirror (brand rename 2026-07-03): old plists/tasks still
|
|
34
|
+
# export SAPS_*; new code reads S4L_*. Copy names, never values via eval.
|
|
35
|
+
while IFS='=' read -r _k _; do
|
|
36
|
+
case "$_k" in SAPS_*) _n="S4L_${_k#SAPS_}"; eval "[ -n \"\${$_n+x}\" ] || export $_n=\"\${$_k}\"";; esac
|
|
37
|
+
done <<EOF_ENV
|
|
38
|
+
$(env | grep '^SAPS_' | cut -d= -f1 | sed 's/$/=/')
|
|
39
|
+
EOF_ENV
|
|
40
|
+
|
|
41
|
+
# 2026-05-28: launchd inherits a default open-files limit of 256 on macOS,
|
|
42
|
+
# which is below the threshold the claude binary needs when it loads MCP
|
|
43
|
+
# servers from ~/.claude.json (50+ servers, each opening a stdio pipe pair).
|
|
44
|
+
# Without this bump, `claude -p` exits with code 1 and ZERO bytes of output
|
|
45
|
+
# (no stdout, no stderr, no archive) because the fd-exhaustion crash happens
|
|
46
|
+
# inside Node.js startup before any handler can run. The lean Phase 1 path
|
|
47
|
+
# (no --strict-mcp-config) was the first thing in the cycle to hit it.
|
|
48
|
+
# 4096 is well above what claude + uv + helpers need; soft-fail to original
|
|
49
|
+
# if the kernel/account caps below this.
|
|
50
|
+
ulimit -n 4096 2>/dev/null || true
|
|
51
|
+
|
|
52
|
+
# Honor S4L_REPO_DIR (set by the MCP wrapper + launchd plists) so a .mcpb
|
|
53
|
+
# install that materializes the repo under ~/.social-autoposter-mcp/repo/package
|
|
54
|
+
# resolves correctly. Falls back to the legacy ~/social-autoposter path for
|
|
55
|
+
# npm/git installs and direct invocations. Cascades to every $REPO_DIR/... ref
|
|
56
|
+
# below (sourced libs + child scripts inherit it), so this one line fixes the
|
|
57
|
+
# whole cycle's repo resolution on a bare .mcpb install.
|
|
58
|
+
REPO_DIR="${S4L_REPO_DIR:-$HOME/social-autoposter}"
|
|
59
|
+
SKILL_FILE="$REPO_DIR/SKILL.md"
|
|
60
|
+
LOG_DIR="$REPO_DIR/skill/logs"
|
|
61
|
+
mkdir -p "$LOG_DIR"
|
|
62
|
+
|
|
63
|
+
BATCH_ID="twcycle-$(date +%Y%m%d-%H%M%S)"
|
|
64
|
+
# Exported so twitter_post_plan.py (Phase 2b-post child process) can re-stamp
|
|
65
|
+
# the executing cycle's batch_id onto candidates at post time. Without this
|
|
66
|
+
# export, peer cycles' Phase 0 salvage can rewrite our candidates' batch_id
|
|
67
|
+
# mid-flight (documented edge case 2026-05-15); the re-stamp at post time is
|
|
68
|
+
# the structural fix so attribution always lands on the cycle that fired the
|
|
69
|
+
# browser, regardless of salvage timing.
|
|
70
|
+
export BATCH_ID
|
|
71
|
+
# Export the same id as SA_CYCLE_ID so every Claude session spawned by this
|
|
72
|
+
# cycle (via run_claude.sh -> log_claude_session.py) stamps its claude_sessions
|
|
73
|
+
# row with cycle_id=$BATCH_ID. Enables exact per-cycle cost accounting via
|
|
74
|
+
# get_run_cost.py --cycle-id, instead of the legacy script+since query which
|
|
75
|
+
# bleeds costs across concurrent stacked cycles. See 2026-05-10 cycle_id
|
|
76
|
+
# rollout (started on reddit, extended here).
|
|
77
|
+
export SA_CYCLE_ID="$BATCH_ID"
|
|
78
|
+
|
|
79
|
+
# LENGTH A/B CONCLUDED 2026-06-04: control won the configured primary metric
|
|
80
|
+
# (avg_clicks) and the enforcement branch is retired. New cycles no longer
|
|
81
|
+
# export LENGTH_ARM, so engagement_styles.py renders the legacy "keep it tight"
|
|
82
|
+
# prompt and twitter_post_plan.py does not stamp posts.length_arm. Historical
|
|
83
|
+
# arm stats stay preserved as a frozen shipped card in /api/experiments.
|
|
84
|
+
|
|
85
|
+
LOG_FILE="$LOG_DIR/twitter-cycle-$(date +%Y-%m-%d_%H%M%S).log"
|
|
86
|
+
RAW_FILE="/tmp/twitter_cycle_raw_$(date +%s).json"
|
|
87
|
+
QUERIES_FILE="/tmp/twitter_cycle_queries_$(date +%s).json"
|
|
88
|
+
# log_twitter_search_attempts.py writes [{query, project, attempt_id}, ...]
|
|
89
|
+
# here so score_twitter_candidates.py can stamp the exact discovering
|
|
90
|
+
# attempt_id onto each twitter_candidates row (2026-05-21).
|
|
91
|
+
ATTEMPTS_FILE="/tmp/twitter_cycle_attempts_$(date +%s).json"
|
|
92
|
+
RUN_START=$(date +%s)
|
|
93
|
+
|
|
94
|
+
# ----------------------------------------------------------------------------
|
|
95
|
+
# Browser: CDP-driven real Google Chrome on port 9555 via the twitter-harness
|
|
96
|
+
# MCP. Profile lives at ~/.claude/browser-profiles/browser-harness.
|
|
97
|
+
# TW_MCP_CONFIG / TW_ENGINE_PREFIX are placeholders, the real values get set
|
|
98
|
+
# below when lib/twitter-backend.sh is sourced (overwriting both).
|
|
99
|
+
# ----------------------------------------------------------------------------
|
|
100
|
+
TW_MCP_CONFIG=""
|
|
101
|
+
TW_ENGINE_PREFIX=""
|
|
102
|
+
# Tweets older than this are no longer worth replying to. Pending rows older
|
|
103
|
+
# than this are hard-expired by Phase 0; younger pending rows are salvaged
|
|
104
|
+
# from prior cycles into this batch.
|
|
105
|
+
# 2026-06-01: tightened 6h -> 2h. The pending pool had bloated to 636 rows,
|
|
106
|
+
# 523 of them >6h old (median virality 0.44, far below the ~5.8 posted median),
|
|
107
|
+
# because the salvage loop kept re-carrying stale low-virality junk. A 2h
|
|
108
|
+
# ceiling drops that carry runway so aged-out junk expires instead of riding
|
|
109
|
+
# ~80 cycles. Discovery is already capped at 1h (FRESHNESS_HOURS_DISCOVER).
|
|
110
|
+
#
|
|
111
|
+
# 2026-06-17 (per user request): DRAFT mode (DRAFT_ONLY=1, the MCP draft_cycle
|
|
112
|
+
# tool) widens both freshness knobs to 24h so human review surfaces more (and
|
|
113
|
+
# older) candidates. Autopilot is untouched: it keeps the experiment-concluded
|
|
114
|
+
# 2h expire ceiling + 1h discovery window (variant D). The branch is on
|
|
115
|
+
# DRAFT_ONLY, an external env var set by the draft_cycle tool, available here.
|
|
116
|
+
# 2026-07-02 (first-run onboarding boost, per user request): the draft-mode
|
|
117
|
+
# value accepts an env override, S4L_DRAFT_FRESHNESS_HOURS, so the kicker
|
|
118
|
+
# wrapper (run-draft-and-publish.sh) can widen a brand-new user's FIRST draft
|
|
119
|
+
# cycle to 48h and surface multiple review cards. Unset = the standard 24h
|
|
120
|
+
# draft window. Autopilot (DRAFT_ONLY=0) ignores the override entirely.
|
|
121
|
+
if [ "${DRAFT_ONLY:-0}" = "1" ]; then
|
|
122
|
+
FRESHNESS_HOURS="${S4L_DRAFT_FRESHNESS_HOURS:-24}"
|
|
123
|
+
else
|
|
124
|
+
FRESHNESS_HOURS=2
|
|
125
|
+
fi
|
|
126
|
+
|
|
127
|
+
# ----------------------------------------------------------------------------
|
|
128
|
+
# EXPERIMENT CONCLUDED 2026-05-31: variant D won the ripen+freshness A/B/C/D
|
|
129
|
+
# test (shipped 2026-05-22, D added 2026-05-25). D = no ripen wait + 1h Phase 1
|
|
130
|
+
# freshness window + drop parent threads with T0 views > 2000. Over the 60-day
|
|
131
|
+
# window D cut thread-age-at-discover p50 to 21 min (vs 173-277 for A/B/C) and
|
|
132
|
+
# led on avg views (91) and avg clicks (0.45), trading post-rate for fresher,
|
|
133
|
+
# higher-converting replies. A/B/C logic has been ripped out; D is now the
|
|
134
|
+
# permanent, hardcoded behavior. The cycle_variant column is still stamped 'D'
|
|
135
|
+
# below so historical analytics keep a consistent label.
|
|
136
|
+
#
|
|
137
|
+
# Phase 0 hard-expire uses FRESHNESS_HOURS (the union ceiling, tightened to 2h
|
|
138
|
+
# on 2026-06-01, see above) so peer cycles don't accidentally expire each
|
|
139
|
+
# other's still-pending rows. FRESHNESS_HOURS_DISCOVER (Phase 1 prompt +
|
|
140
|
+
# since-rewrite hook) stays tightened to 1h, the winning D setting.
|
|
141
|
+
TWITTER_CYCLE_VARIANT=D
|
|
142
|
+
# DRAFT mode widens discovery to 24h by default; autopilot keeps the winning D
|
|
143
|
+
# setting of 1h. S4L_DRAFT_FRESHNESS_HOURS (first-run onboarding boost, see the
|
|
144
|
+
# FRESHNESS_HOURS branch above) can widen the draft-mode value further (48h on a
|
|
145
|
+
# brand-new install's first cycle). The lean Phase 1 CDP scraper reads
|
|
146
|
+
# FRESHNESS_HOURS_DISCOVER directly and honors any value.
|
|
147
|
+
if [ "${DRAFT_ONLY:-0}" = "1" ]; then
|
|
148
|
+
FRESHNESS_HOURS_DISCOVER="${S4L_DRAFT_FRESHNESS_HOURS:-24}"
|
|
149
|
+
else
|
|
150
|
+
FRESHNESS_HOURS_DISCOVER=1
|
|
151
|
+
fi
|
|
152
|
+
# Export FRESHNESS_HOURS too so score_twitter_candidates.py inherits it and
|
|
153
|
+
# drives the expire-stale gate from the same knob (was hardcoded 18h there).
|
|
154
|
+
export TWITTER_CYCLE_VARIANT FRESHNESS_HOURS_DISCOVER FRESHNESS_HOURS
|
|
155
|
+
# Hook env: ~/.claude/hooks/twitter-search-since-rewrite.py reads this and
|
|
156
|
+
# uses it in place of its hardcoded 6h default when present. The hook accepts
|
|
157
|
+
# only a 1-24h range and silently falls back to its 6h default on anything
|
|
158
|
+
# bigger, so cap the exported value at 24: during the 48h first-run boost the
|
|
159
|
+
# CDP scraper still gets the full window via FRESHNESS_HOURS_DISCOVER, while
|
|
160
|
+
# any hook-rewritten query keeps the widest value the hook can honor.
|
|
161
|
+
if [ "$FRESHNESS_HOURS_DISCOVER" -gt 24 ] 2>/dev/null; then
|
|
162
|
+
export FRESHNESS_HOURS_OVERRIDE=24
|
|
163
|
+
else
|
|
164
|
+
export FRESHNESS_HOURS_OVERRIDE=$FRESHNESS_HOURS_DISCOVER
|
|
165
|
+
fi
|
|
166
|
+
|
|
167
|
+
# `set -a` auto-exports every variable assigned by `source .env`, so DATABASE_URL
|
|
168
|
+
# and friends propagate to subprocess env (python3 scripts use os.environ at
|
|
169
|
+
# import time and would otherwise see empty strings — silently breaking
|
|
170
|
+
# update_candidate_posted in twitter_post_plan.py and creating duplicate posts
|
|
171
|
+
# under parallel cycles, observed 2026-05-01 batches 02-08).
|
|
172
|
+
if [ -f "$REPO_DIR/.env" ]; then
|
|
173
|
+
set -a
|
|
174
|
+
source "$REPO_DIR/.env"
|
|
175
|
+
set +a
|
|
176
|
+
fi
|
|
177
|
+
|
|
178
|
+
log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$LOG_FILE"; }
|
|
179
|
+
|
|
180
|
+
log "=== Twitter Cycle (batch=$BATCH_ID): $(date) ==="
|
|
181
|
+
log "Logic=D (no-ripen + 1h freshness + 2k_view_cap; experiment concluded 2026-05-31); discover_freshness=${FRESHNESS_HOURS_DISCOVER}h"
|
|
182
|
+
log "Length-control experiment concluded 2026-06-04; winner=control; LENGTH_ARM retired"
|
|
183
|
+
|
|
184
|
+
# --- Preflight (added 2026-05-02) -----------------------------------------
|
|
185
|
+
# Three early-exit gates BEFORE we open the DB, set up traps, or touch the
|
|
186
|
+
# browser. Each gate emits a `[skipped: <reason>]` stderr line and exits 0
|
|
187
|
+
# so launchd treats the slot as cleanly consumed and fires the next one
|
|
188
|
+
# on schedule.
|
|
189
|
+
#
|
|
190
|
+
# 1. Memory pressure: 2026-05-01 a JetsamEvent at 19:26 swallowed two
|
|
191
|
+
# consecutive launchd fires (19:38, 19:53). The wrappers fired, but the
|
|
192
|
+
# grandchild bash never produced output — most likely jetsam-killed or
|
|
193
|
+
# starved during the system's crash-cleanup spike. Skipping when
|
|
194
|
+
# pressure_level >= 2 (warn) avoids piling more Chrome+Claude+Python
|
|
195
|
+
# work onto an already-thrashing system.
|
|
196
|
+
#
|
|
197
|
+
# 2. Claude quota stamp: prior run_claude.sh invocation hit a fatal cap
|
|
198
|
+
# (monthly cap, daily cap, context-window, credit balance, persistent
|
|
199
|
+
# 429). Skip until the stamp expires (default 10 min). When the cap
|
|
200
|
+
# lifts and the next post-expiry fire succeeds, run_claude.sh clears
|
|
201
|
+
# the stamp automatically.
|
|
202
|
+
#
|
|
203
|
+
# 3. Single-cycle gate: exactly 1 concurrent run-twitter-cycle.sh, enforced
|
|
204
|
+
# HERE in the script itself so EVERY launch path is covered — the launchd
|
|
205
|
+
# singleton wrapper's snapshot copy, the MCP draft_cycle tool's direct
|
|
206
|
+
# `bash skill/run-twitter-cycle.sh`, and any manual/agent invocation all
|
|
207
|
+
# run this preflight and compete for the same /tmp/sa-twitter-cycle-slot-1
|
|
208
|
+
# mkdir lock. History: 2026-05-03 introduced this as a max-4 cap; on
|
|
209
|
+
# 2026-05-22 we added run-twitter-cycle-singleton.sh to enforce one-at-a-
|
|
210
|
+
# time, but that wrapper only governs the launchd path and never kills
|
|
211
|
+
# (per user instruction), so out-of-band literal launches (MCP/manual)
|
|
212
|
+
# sailed past it while this cap still permitted 4. On 2026-06-01 a launchd
|
|
213
|
+
# studyly cycle + an out-of-band fazm cycle overlapped, fought over the
|
|
214
|
+
# twitter-browser lock, and the fazm one got watchdog-killed (logged as
|
|
215
|
+
# phase2b_silent). Cutting the cap to 1 unifies the gate across all paths.
|
|
216
|
+
# The slot's dead-holder GC (kill -0 in preflight.sh) still reclaims slots
|
|
217
|
+
# orphaned by SIGKILL/OOM, so a crashed cycle never wedges the gate.
|
|
218
|
+
#
|
|
219
|
+
# preflight.sh exposes a small set of helpers; we call them in order
|
|
220
|
+
# (cheapest first) so a fast-path skip (already-blocked) doesn't even
|
|
221
|
+
# spend the sysctl read for the next check.
|
|
222
|
+
source "$REPO_DIR/scripts/preflight.sh"
|
|
223
|
+
SA_PREFLIGHT_SCRIPT="run-twitter-cycle"
|
|
224
|
+
if [ "${SCAN_ONLY:-0}" = "1" ]; then
|
|
225
|
+
# SCAN_ONLY (the Desktop-session autopilot's scan step) gets its OWN slot pool
|
|
226
|
+
# so it is NOT blocked by a live launchd autopilot cycle; the two then serialize
|
|
227
|
+
# on the twitter-browser lock (acquired in Phase 1) instead of being mutually
|
|
228
|
+
# exclusive. It also SKIPS the claude-blocked gate: SCAN_ONLY drives no
|
|
229
|
+
# `claude -p`, so a prior claude cap must not suppress a pure scan.
|
|
230
|
+
SA_PREFLIGHT_SCRIPT="run-twitter-cycle-scan"
|
|
231
|
+
preflight_skip_if_jetsam_pressure
|
|
232
|
+
preflight_acquire_slot_or_skip "twitter-scan" 1
|
|
233
|
+
else
|
|
234
|
+
preflight_skip_if_claude_blocked
|
|
235
|
+
preflight_skip_if_jetsam_pressure
|
|
236
|
+
preflight_acquire_slot_or_skip "twitter-cycle" 1
|
|
237
|
+
fi
|
|
238
|
+
|
|
239
|
+
# Source lock helpers (functions only, no lock acquired here). Phase 0 + the
|
|
240
|
+
# project/queries setup below run lock-free against DB and config files;
|
|
241
|
+
# the twitter-browser lock is acquired later, immediately before the Phase 1
|
|
242
|
+
# Claude scan that actually drives the browser (line ~177). Pre-2026-05-01
|
|
243
|
+
# this acquire was here at script start and held the lock through Phase 0
|
|
244
|
+
# (~3-10s of pure DB/Python work that doesn't touch the browser), starving
|
|
245
|
+
# peer cycles' Phase 2b-post under parallel-cycle contention.
|
|
246
|
+
source "$REPO_DIR/skill/lock.sh"
|
|
247
|
+
|
|
248
|
+
# Harness-only browser bootstrap (twitter-agent path fully removed 2026-05-19).
|
|
249
|
+
# Sets MCP_CONFIG_FILE, BROWSER_INSTRUCTIONS, exports TWITTER_CDP_URL=9555.
|
|
250
|
+
# Phase 2b-post's twitter_post_plan.py shells out to twitter_browser.py, which
|
|
251
|
+
# honors TWITTER_CDP_URL exported by this lib.
|
|
252
|
+
source "$REPO_DIR/skill/lib/twitter-backend.sh"
|
|
253
|
+
TW_MCP_CONFIG="$MCP_CONFIG_FILE"
|
|
254
|
+
# 2026-06-26: the model-facing steps (Phase 1 query draft, Phase 2b prep draft) are
|
|
255
|
+
# TOOL-FREE. All browser work is done deterministically by the shell's CDP scan
|
|
256
|
+
# (browser-harness over port 9555) + Phase 2b-post's twitter_browser.py, NOT by the
|
|
257
|
+
# model. The old BROWSER BACKEND / bh_run "translation table" block is no longer
|
|
258
|
+
# injected: prep drafts purely from the inlined candidate context (Text: $ctext per
|
|
259
|
+
# candidate) + MEDIA_BLOCK, which is exactly what the model's rare bh_run fallback
|
|
260
|
+
# used to re-fetch (1 call/week vs ~18.5k/wk deterministic CDP scans). The 9555 Chrome
|
|
261
|
+
# is still launched by twitter-backend.sh above for the shell scan + post step; only
|
|
262
|
+
# the model's browser fallback is removed. This also drops the hardcoded "logged in as
|
|
263
|
+
# m13v_" identity that the block carried, so prompts are no longer single-tenant.
|
|
264
|
+
TW_ENGINE_PREFIX=""
|
|
265
|
+
|
|
266
|
+
# --- Phase tracking: start the twitter_batches row + chain into lock.sh trap -
|
|
267
|
+
# Per-cycle phase row (twitter_batches.current_phase + phase_started_at) is
|
|
268
|
+
# read by peer cycles' Phase 0 to decide salvage timing per-phase instead of
|
|
269
|
+
# the old flat 20-min wall-clock cutoff. Phase 2b-gen (SEO landing-page build)
|
|
270
|
+
# legitimately runs 10-40 min and was being salvaged out from under live
|
|
271
|
+
# owners under the old rule. See migration 2026-05-01_twitter_batches.sql and
|
|
272
|
+
# scripts/twitter_batch_phase.py.
|
|
273
|
+
#
|
|
274
|
+
# Trap design: lock.sh installs `_sa_release_locks` on EXIT/INT/TERM/HUP. We
|
|
275
|
+
# wrap that into `_sa_combined_exit` so a clean exit ALSO deletes our
|
|
276
|
+
# twitter_batches row. SIGKILL / OOM / hard crash bypasses traps and
|
|
277
|
+
# intentionally leaves the row stale — that's the salvage recovery path.
|
|
278
|
+
_sa_cleanup_batch_row() {
|
|
279
|
+
if [ -n "${BATCH_ID:-}" ]; then
|
|
280
|
+
python3 "$REPO_DIR/scripts/twitter_batch_phase.py" end "$BATCH_ID" 2>/dev/null || true
|
|
281
|
+
fi
|
|
282
|
+
}
|
|
283
|
+
_sa_combined_exit() {
|
|
284
|
+
# Emit run_monitor.log summary FIRST, before any cleanup. Without this,
|
|
285
|
+
# SIGTERM landing between Phase 2b-post (where twitter_post_plan.py has
|
|
286
|
+
# already committed to the `posts` table) and the historical inline
|
|
287
|
+
# summary write at the bottom of the script silently drops the run from
|
|
288
|
+
# run_monitor.log. Mirrors the same fix shipped to run-reddit-search.sh.
|
|
289
|
+
# Idempotent: a flag-guarded one-shot, so the happy-path explicit call at
|
|
290
|
+
# the bottom and the trap firing on EXIT do not double-write.
|
|
291
|
+
_sa_emit_run_summary_oneshot
|
|
292
|
+
_sa_cleanup_batch_row
|
|
293
|
+
# Release the parallel-cycle slot acquired by preflight.sh. Without this,
|
|
294
|
+
# this trap (which OVERWRITES the preflight trap installed at source-time)
|
|
295
|
+
# would leak the slot until the next launchd fire's GC pass — capping
|
|
296
|
+
# effective throughput at 1/cycle even though the slot pool is 4 wide.
|
|
297
|
+
if command -v _preflight_release_slots >/dev/null 2>&1; then
|
|
298
|
+
_preflight_release_slots
|
|
299
|
+
fi
|
|
300
|
+
_sa_release_locks
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
# Idempotent run_monitor.log emitter wired into _sa_combined_exit (which is
|
|
304
|
+
# trap'd to EXIT INT TERM HUP). On the happy path the bottom of the script
|
|
305
|
+
# calls this directly; on SIGTERM the trap calls it. Either order is a no-op
|
|
306
|
+
# after first emission via _SA_RUN_SUMMARY_EMITTED.
|
|
307
|
+
#
|
|
308
|
+
# Reads counters from globals the cycle has been accumulating (BATCH_ID,
|
|
309
|
+
# RUN_START, EXEC_FAILED, EXEC_REASONS, EXEC_SKIPPED, CANDIDATE_COUNT,
|
|
310
|
+
# SALVAGED, QUERIES_TOTAL, DUDS_TOTAL, TWEETS_PULLED, BATCH_COUNT,
|
|
311
|
+
# HIGH_DELTA_COUNT). Re-derives POSTED_CT/SKIPPED_CT from the
|
|
312
|
+
# twitter_candidates table directly so a SIGTERM mid-Phase-2b still gets
|
|
313
|
+
# accurate counts (the row was committed inside twitter_post_plan.py before
|
|
314
|
+
# the kill). All psql / get_run_cost.py calls are wrapped in `timeout 10`
|
|
315
|
+
# so a Postgres hang during shutdown can't wedge the trap.
|
|
316
|
+
#
|
|
317
|
+
# Early-exit failure paths (Phase 1 abort, empty batch, etc.) write their
|
|
318
|
+
# own dedicated log_run.py line with custom failure_reasons and then set
|
|
319
|
+
# _SA_RUN_SUMMARY_EMITTED=1 to short-circuit this function — they keep
|
|
320
|
+
# their tailored error reason, this fallback skips.
|
|
321
|
+
_SA_RUN_SUMMARY_EMITTED=0
|
|
322
|
+
_sa_emit_run_summary_oneshot() {
|
|
323
|
+
[ "${_SA_RUN_SUMMARY_EMITTED:-0}" = "1" ] && return 0
|
|
324
|
+
_SA_RUN_SUMMARY_EMITTED=1
|
|
325
|
+
|
|
326
|
+
local posted_ct=0 skipped_ct=0 cost="0.0000" failed_ct failure_reasons
|
|
327
|
+
# Prefer the in-memory counters captured from twitter_post_plan.py's JSON
|
|
328
|
+
# summary (EXEC_POSTED / EXEC_SKIPPED). Those are the ground truth for what
|
|
329
|
+
# THIS cycle did. The fallback SQL count is needed when SIGTERM hits before
|
|
330
|
+
# Phase 2b-post records a count, but it's UNRELIABLE during normal exit:
|
|
331
|
+
# peer cycles' Phase 0 may have salvaged this batch's candidates into a new
|
|
332
|
+
# batch_id mid-Phase-2b (documented edge case, mitigated by the phase2b-*
|
|
333
|
+
# advance stamps but not 100% eliminated under heavy parallel load), in
|
|
334
|
+
# which case the WHERE batch_id='$BATCH_ID' query returns 0 even though we
|
|
335
|
+
# successfully posted N replies. That false-zero is what historically
|
|
336
|
+
# synthesized phase2b_silent failure_reasons against successful runs.
|
|
337
|
+
if [ -n "${EXEC_POSTED:-}" ] || [ -n "${EXEC_SKIPPED:-}" ]; then
|
|
338
|
+
posted_ct="${EXEC_POSTED:-0}"
|
|
339
|
+
skipped_ct="${EXEC_SKIPPED:-0}"
|
|
340
|
+
elif [ -n "${BATCH_ID:-}" ]; then
|
|
341
|
+
# /api/v1/twitter-candidates/counts-by-batch returns posted +
|
|
342
|
+
# skipped_or_expired in one roundtrip; helper prints them space-
|
|
343
|
+
# separated so this stays a single $() capture.
|
|
344
|
+
_SC=$(timeout 10 python3 "$REPO_DIR/scripts/twitter_cycle_helper.py" \
|
|
345
|
+
status-counts --batch-id "$BATCH_ID" 2>/dev/null || echo "0 0")
|
|
346
|
+
posted_ct=$(echo "$_SC" | awk '{print $1}')
|
|
347
|
+
skipped_ct=$(echo "$_SC" | awk '{print $2}')
|
|
348
|
+
: "${posted_ct:=0}"
|
|
349
|
+
: "${skipped_ct:=0}"
|
|
350
|
+
fi
|
|
351
|
+
cost=$(timeout 10 python3 "$REPO_DIR/scripts/get_run_cost.py" \
|
|
352
|
+
--cycle-id "${BATCH_ID}" \
|
|
353
|
+
2>/dev/null || echo "0.0000")
|
|
354
|
+
|
|
355
|
+
failed_ct="${EXEC_FAILED:-0}"
|
|
356
|
+
failure_reasons="${EXEC_REASONS:-}"
|
|
357
|
+
# Reproduce the failure-reason synthesis block so SIGTERM cycles still
|
|
358
|
+
# get a useful reason instead of a silent "—". Same conditions as the
|
|
359
|
+
# historical inline block: cycle ended with zero progress despite having
|
|
360
|
+
# candidates pending.
|
|
361
|
+
if [ "${posted_ct:-0}" = "0" ] \
|
|
362
|
+
&& [ "${failed_ct:-0}" = "0" ] \
|
|
363
|
+
&& [ "${EXEC_SKIPPED:-0}" = "0" ] \
|
|
364
|
+
&& [ -z "$failure_reasons" ] \
|
|
365
|
+
&& [ "${CANDIDATE_COUNT:-0}" -gt 0 ]; then
|
|
366
|
+
local phase2b_log
|
|
367
|
+
phase2b_log=$(awk '/Phase 1: drafting queries|Phase 2b-prep: Claude reading|Phase 2b-post:/,EOF' "$LOG_FILE" 2>/dev/null || echo "")
|
|
368
|
+
# Inline reason-add: bash doesn't support `local` on function decls,
|
|
369
|
+
# and a free-standing nested function would leak into the outer
|
|
370
|
+
# scope, so we just expand the assignments at each call site.
|
|
371
|
+
# Run the shared API-error classifier first — catches monthly_limit,
|
|
372
|
+
# stream_idle_timeout, api_overloaded, context_overflow, credit_balance,
|
|
373
|
+
# etc. uniformly so the dashboard pill reads with the actual error
|
|
374
|
+
# class instead of falling through to the generic phase2b_silent.
|
|
375
|
+
local classifier_reason
|
|
376
|
+
classifier_reason=$(echo "$phase2b_log" | python3 "$REPO_DIR/scripts/classify_run_error.py" 2>/dev/null)
|
|
377
|
+
if [ -n "$classifier_reason" ]; then
|
|
378
|
+
failure_reasons="${failure_reasons:+$failure_reasons,}${classifier_reason}:1"
|
|
379
|
+
failed_ct=$(( failed_ct + 1 ))
|
|
380
|
+
fi
|
|
381
|
+
if echo "$phase2b_log" | grep -qiE 'auth redirect|re-authenticat|browser profile.*auth|profile.*needs.*re-auth'; then
|
|
382
|
+
failure_reasons="${failure_reasons:+$failure_reasons,}auth_redirect:1"
|
|
383
|
+
failed_ct=$(( failed_ct + 1 ))
|
|
384
|
+
fi
|
|
385
|
+
if echo "$phase2b_log" | grep -qiE '"error":"rate_limited"|RATE_LIMITED_TWITTER'; then
|
|
386
|
+
failure_reasons="${failure_reasons:+$failure_reasons,}rate_limited:1"
|
|
387
|
+
failed_ct=$(( failed_ct + 1 ))
|
|
388
|
+
fi
|
|
389
|
+
if echo "$phase2b_log" | grep -qiE 'page.load.timeout|navigation timeout|timed out|Timeout exceeded'; then
|
|
390
|
+
failure_reasons="${failure_reasons:+$failure_reasons,}timeout:1"
|
|
391
|
+
failed_ct=$(( failed_ct + 1 ))
|
|
392
|
+
fi
|
|
393
|
+
if echo "$phase2b_log" | grep -qiE 'reply_box_not_found|tweet_not_found'; then
|
|
394
|
+
failure_reasons="${failure_reasons:+$failure_reasons,}posting_blocked:1"
|
|
395
|
+
failed_ct=$(( failed_ct + 1 ))
|
|
396
|
+
fi
|
|
397
|
+
if [ -z "$failure_reasons" ]; then
|
|
398
|
+
failure_reasons="phase2b_silent:1"
|
|
399
|
+
failed_ct=$(( failed_ct + 1 ))
|
|
400
|
+
fi
|
|
401
|
+
fi
|
|
402
|
+
|
|
403
|
+
local args
|
|
404
|
+
args=(--script "post_twitter" \
|
|
405
|
+
--posted "${posted_ct:-0}" \
|
|
406
|
+
--skipped "${skipped_ct:-0}" \
|
|
407
|
+
--failed "$failed_ct" \
|
|
408
|
+
--salvaged "${SALVAGED:-0}" \
|
|
409
|
+
--queries "${QUERIES_TOTAL:-0}" --duds "${DUDS_TOTAL:-0}" \
|
|
410
|
+
--tweets-pulled "${TWEETS_PULLED:-0}" \
|
|
411
|
+
--candidates "${BATCH_COUNT:-0}" --above-floor "${HIGH_DELTA_COUNT:-0}" \
|
|
412
|
+
--cost "$cost" \
|
|
413
|
+
--elapsed $(( $(date +%s) - ${RUN_START:-$(date +%s)} )))
|
|
414
|
+
[ -n "$failure_reasons" ] && args+=(--failure-reasons "$failure_reasons")
|
|
415
|
+
[ -n "${EXEC_SKIP_REASONS:-}" ] && args+=(--skip-reasons "$EXEC_SKIP_REASONS")
|
|
416
|
+
python3 "$REPO_DIR/scripts/log_run.py" "${args[@]}" 2>/dev/null || true
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
trap _sa_combined_exit EXIT INT TERM HUP
|
|
420
|
+
|
|
421
|
+
python3 "$REPO_DIR/scripts/twitter_batch_phase.py" start "$BATCH_ID" --phase phase0 2>&1 | tee -a "$LOG_FILE" || true
|
|
422
|
+
|
|
423
|
+
# --- Phase 0: hard-expire stale pending + salvage truly-orphaned rows --------
|
|
424
|
+
# Pending rows from prior cycles fall into two buckets:
|
|
425
|
+
# - tweet_posted_at older than FRESHNESS_HOURS -> hard-expire (lost the
|
|
426
|
+
# replying window, no value in retrying)
|
|
427
|
+
# - still-fresh AND owning batch is dead -> re-assign to this batch
|
|
428
|
+
# so Phase 2a re-measures T1 and Phase 2b reconsiders them. This is the
|
|
429
|
+
# recovery path for cycles whose Phase 2b died on Anthropic org quota,
|
|
430
|
+
# X rate limit, browser crash, or any other infra failure.
|
|
431
|
+
#
|
|
432
|
+
# Two safety guards make this safe under parallel cycles (post 2026-04-30
|
|
433
|
+
# detach refactor: launchd no longer suppresses overlapping fires, so 2-3
|
|
434
|
+
# run-twitter-cycle.sh can be in Phase 0/1/2 simultaneously):
|
|
435
|
+
#
|
|
436
|
+
# 1. pg_advisory_xact_lock(7472346) serializes Phase 0 transactions, so
|
|
437
|
+
# two cycles can't race on the salvage UPDATE.
|
|
438
|
+
#
|
|
439
|
+
# 2. PHASE-AWARE BUDGET (post 2026-05-01): salvage timing is per-phase,
|
|
440
|
+
# read from the owner's twitter_batches row:
|
|
441
|
+
# phase0 -> 5 min (just the salvage SQL)
|
|
442
|
+
# phase1 -> 20 min (Claude scan + scrape)
|
|
443
|
+
# phase2a -> 20 min (browser-lock handoff window; no ripen wait since 2026-05-31)
|
|
444
|
+
# phase2b-prep -> 45 min (Claude reads threads + drafts; bumped 2026-05-15
|
|
445
|
+
# 15 -> 30 after 17:15 cycle was wrongly salvaged
|
|
446
|
+
# while queued behind 17:30's 42-min lock-hold;
|
|
447
|
+
# bumped 2026-05-22 30 -> 45 to leave more
|
|
448
|
+
# headroom for big-batch Claude reads after the
|
|
449
|
+
# Variant A wake re-stamp fix)
|
|
450
|
+
# phase2b-gen -> 60 min (SEO landing-page build, the slow phase)
|
|
451
|
+
# phase2b-post -> 15 min (browser reply + log)
|
|
452
|
+
# Pre-2026-05-01 the rule was a flat 20-min wall-clock cutoff against
|
|
453
|
+
# batch_id, which salvaged live cycles whose Phase 2b-gen step (10-40
|
|
454
|
+
# min in normal operation) hadn't finished. Observed 2026-05-01: cycle
|
|
455
|
+
# 16:23's candidate 7994 was salvaged into 16:53 while 16:23 was still
|
|
456
|
+
# generating the SEO page; both cycles raced on the post and the
|
|
457
|
+
# late-arriving owner logged failed=1.
|
|
458
|
+
#
|
|
459
|
+
# 3. LEGACY FALLBACK: rows whose batch has no twitter_batches entry (any
|
|
460
|
+
# cycle that ran before this migration, OR a cycle whose start helper
|
|
461
|
+
# failed) fall back to the original flat 20-min batch_id heuristic.
|
|
462
|
+
# Self-cleans within FRESHNESS_HOURS of migration.
|
|
463
|
+
#
|
|
464
|
+
# batch_id format is `twcycle-YYYYMMDD-HHMMSS` (assigned at script start
|
|
465
|
+
# from `date +%Y%m%d-%H%M%S`, local time). Since the format is fixed-width
|
|
466
|
+
# and lexicographically sortable, we compute the cutoff in the shell
|
|
467
|
+
# (same TZ as batch_id) and do a string comparison in SQL — sidesteps the
|
|
468
|
+
# Postgres session-TZ trap that would otherwise mis-interpret batch_id.
|
|
469
|
+
LEGACY_SALVAGE_CUTOFF_MIN=20
|
|
470
|
+
LEGACY_SALVAGE_CUTOFF_BATCH_ID="twcycle-$(python3 -c "import datetime; print((datetime.datetime.now() - datetime.timedelta(minutes=${LEGACY_SALVAGE_CUTOFF_MIN})).strftime('%Y%m%d-%H%M%S'))")"
|
|
471
|
+
# Single-transaction Phase 0 salvage now lives server-side at
|
|
472
|
+
# /api/v1/twitter-candidates/phase0-salvage. Same advisory lock (7472346),
|
|
473
|
+
# same expire + salvage CTE, same phase-aware budget table. The helper
|
|
474
|
+
# prints "<expired_count>|<salvaged_count>" so the legacy cut/cut shape
|
|
475
|
+
# downstream still works.
|
|
476
|
+
PHASE0_RESULT=$(python3 "$REPO_DIR/scripts/twitter_cycle_helper.py" \
|
|
477
|
+
phase0-salvage \
|
|
478
|
+
--batch-id "$BATCH_ID" \
|
|
479
|
+
--freshness-hours "$FRESHNESS_HOURS" \
|
|
480
|
+
--legacy-cutoff "$LEGACY_SALVAGE_CUTOFF_BATCH_ID" \
|
|
481
|
+
2>/dev/null | tail -1 | tr -d ' ')
|
|
482
|
+
EXPIRED_STALE=$(echo "$PHASE0_RESULT" | cut -d'|' -f1)
|
|
483
|
+
SALVAGED=$(echo "$PHASE0_RESULT" | cut -d'|' -f2)
|
|
484
|
+
[ "${EXPIRED_STALE:-0}" -gt 0 ] && log "Phase 0: hard-expired $EXPIRED_STALE pending rows older than ${FRESHNESS_HOURS}h"
|
|
485
|
+
[ "${SALVAGED:-0}" -gt 0 ] && log "Phase 0: salvaged $SALVAGED orphaned pending rows (phase-aware budget) into $BATCH_ID"
|
|
486
|
+
|
|
487
|
+
# Advance our own batch row from phase0 -> phase1 now that the salvage SQL
|
|
488
|
+
# committed. Subsequent phase transitions are stamped right before the work
|
|
489
|
+
# they cover begins.
|
|
490
|
+
python3 "$REPO_DIR/scripts/twitter_batch_phase.py" advance "$BATCH_ID" --phase phase1 2>&1 | tee -a "$LOG_FILE" || true
|
|
491
|
+
|
|
492
|
+
# --- Shared project selection (inverse-recent-share) -------------------------
|
|
493
|
+
# Project selection is shared across twitter/github/reddit via
|
|
494
|
+
# scripts/pick_project.py (pick_projects): inverse-recent-share weighting,
|
|
495
|
+
# weight / (1 + posts in the last 7d), sampled without replacement. This
|
|
496
|
+
# replaced the old inline weighted sample on 2026-05-15 so all platforms
|
|
497
|
+
# pick projects the same way.
|
|
498
|
+
# Each chosen project is then enriched here with an `excludes_for_search`
|
|
499
|
+
# array sourced from project_search_excludes (only terms past the
|
|
500
|
+
# 2-distinct-batch activation gate). The Phase 1 scanner is required to
|
|
501
|
+
# mechanically append these as `-term` operators to whatever query it drafts
|
|
502
|
+
# for the project. See scripts/project_excludes.py for proposal/activation/
|
|
503
|
+
# decay rules.
|
|
504
|
+
_PJ_ERR="$(mktemp)"
|
|
505
|
+
PROJECTS_JSON=$(python3 - 2>"$_PJ_ERR" <<'PY'
|
|
506
|
+
import json, os, subprocess, sys
|
|
507
|
+
REPO = os.path.expanduser('~/social-autoposter')
|
|
508
|
+
sys.path.insert(0, os.path.join(REPO, 'scripts'))
|
|
509
|
+
import project_excludes as pe
|
|
510
|
+
|
|
511
|
+
_pp_args = ['python3', os.path.join(REPO, 'scripts', 'pick_project.py'),
|
|
512
|
+
'--platform', 'twitter', '--count', '1', '--json']
|
|
513
|
+
# Manual-mode (MCP draft_cycle) single-project scoping: when S4L_FORCE_PROJECT
|
|
514
|
+
# is set, force that exact project instead of the weighted-random autopilot
|
|
515
|
+
# pick, so a customer's interactive cycle only ever touches their own project.
|
|
516
|
+
_force_project = os.environ.get('S4L_FORCE_PROJECT')
|
|
517
|
+
if _force_project:
|
|
518
|
+
_pp_args += ['--project', _force_project]
|
|
519
|
+
res = subprocess.run(
|
|
520
|
+
_pp_args,
|
|
521
|
+
capture_output=True, text=True, timeout=30,
|
|
522
|
+
)
|
|
523
|
+
picked = []
|
|
524
|
+
if res.returncode == 0 and res.stdout.strip():
|
|
525
|
+
try:
|
|
526
|
+
picked = json.loads(res.stdout)
|
|
527
|
+
except Exception:
|
|
528
|
+
picked = []
|
|
529
|
+
|
|
530
|
+
# pick_project.py returns a single dict when --count=1, a list when --count>1.
|
|
531
|
+
# Normalize to a list so the rest of the heredoc works either way.
|
|
532
|
+
if isinstance(picked, dict):
|
|
533
|
+
picked = [picked]
|
|
534
|
+
|
|
535
|
+
from pick_search_topic import pick_topic_for_project, PickerError
|
|
536
|
+
|
|
537
|
+
chosen = []
|
|
538
|
+
for p in picked:
|
|
539
|
+
try:
|
|
540
|
+
excludes = pe.active_excludes('twitter', p.get('name'))
|
|
541
|
+
except Exception:
|
|
542
|
+
excludes = []
|
|
543
|
+
# 2026-05-26: force-pick ONE search_topic per project via the Python
|
|
544
|
+
# picker so end-to-end attribution (topic -> query -> candidate ->
|
|
545
|
+
# post -> click) is clean. Mirrors the engagement_styles flow.
|
|
546
|
+
#
|
|
547
|
+
# Single mode (post-2026-05-28): picker returns search_topic=<string>,
|
|
548
|
+
# weighted-random over the FULL universe with log-smoothed weights
|
|
549
|
+
# (top ~20-30%, cold ~0.5-1%). Claude must use the assigned topic
|
|
550
|
+
# verbatim. EXPLORE_INVENT was removed in favor of the standalone
|
|
551
|
+
# invent_topics.py job that writes new topics directly into
|
|
552
|
+
# project_search_topics outside the cycle.
|
|
553
|
+
#
|
|
554
|
+
# 2026-05-27: NO fallback. The DB is the only source of truth for
|
|
555
|
+
# the universe. If pick_topic_for_project raises (DB unreachable or
|
|
556
|
+
# zero active topics for this project), let the heredoc crash so
|
|
557
|
+
# PROJECTS_JSON is empty, the bash trap fires, and launchd records
|
|
558
|
+
# a hard failure. Silent fallback to config.json or to the first
|
|
559
|
+
# legacy search_topics[] entry would post against a stale seed list
|
|
560
|
+
# and corrupt attribution; the rule is "stop the pipeline".
|
|
561
|
+
topic_pick = pick_topic_for_project(p.get('name'), platform='twitter')
|
|
562
|
+
picked_topic = topic_pick.get('search_topic')
|
|
563
|
+
reference_topics = topic_pick.get('reference_topics') or []
|
|
564
|
+
picked_weight_pct = topic_pick.get('picked_weight_pct')
|
|
565
|
+
chosen.append({
|
|
566
|
+
'name': p.get('name'),
|
|
567
|
+
'description': p.get('description', ''),
|
|
568
|
+
# Force-picked single topic (2026-05-26). Replaces the legacy
|
|
569
|
+
# `search_topics: [...]` array. Claude draws its query from THIS
|
|
570
|
+
# topic and must echo it verbatim on every tweet object via the
|
|
571
|
+
# bh_run scrape script's `search_topic` Python variable.
|
|
572
|
+
'search_topic': picked_topic,
|
|
573
|
+
'picked_weight_pct': picked_weight_pct,
|
|
574
|
+
# Per-project pool stats (top by composite_score). Surfaced as
|
|
575
|
+
# context to help Claude understand the topic's history.
|
|
576
|
+
'reference_topics': reference_topics,
|
|
577
|
+
# Self-improving exclusion list (2026-05-09): MUST be appended
|
|
578
|
+
# as `-term` to every query drafted for this project.
|
|
579
|
+
'excludes_for_search': excludes,
|
|
580
|
+
})
|
|
581
|
+
print(json.dumps(chosen, indent=2))
|
|
582
|
+
PY
|
|
583
|
+
)
|
|
584
|
+
_PJ_RC=$?
|
|
585
|
+
# Fail loud when the project/topic universe can't be built. The heredoc above
|
|
586
|
+
# exits non-zero (PROJECTS_JSON empty) when pick_topic_for_project finds zero
|
|
587
|
+
# active rows in project_search_topics for the selected project, or the topics
|
|
588
|
+
# API is unreachable; it also yields "[]" when no project is eligible. Without
|
|
589
|
+
# this guard the empty PROJECTS_JSON silently falls through to "0 queries -> 0
|
|
590
|
+
# tweets -> batch expired -> zero", which reads to the user as "nothing to post"
|
|
591
|
+
# when the real cause is "this project was never seeded with search topics".
|
|
592
|
+
# Seeding now happens in the MCP setup tool; this is the defense-in-depth net
|
|
593
|
+
# so a missing universe is surfaced, never swallowed. (2026-06-02)
|
|
594
|
+
if [ "$_PJ_RC" -ne 0 ] || ! printf '%s' "$PROJECTS_JSON" | python3 -c 'import json,sys; d=json.load(sys.stdin); sys.exit(0 if isinstance(d,list) and d else 1)' 2>/dev/null; then
|
|
595
|
+
_PJ_REASON="project_selection_failed"
|
|
596
|
+
if grep -q "no active search topics" "$_PJ_ERR" 2>/dev/null; then
|
|
597
|
+
_PJ_REASON="no_search_topics"
|
|
598
|
+
elif grep -qiE "project-search-topics API|API unreachable" "$_PJ_ERR" 2>/dev/null; then
|
|
599
|
+
_PJ_REASON="topics_api_unreachable"
|
|
600
|
+
fi
|
|
601
|
+
log "Project/topic universe build FAILED (reason=$_PJ_REASON); stopping cycle before scan. Last error lines:"
|
|
602
|
+
tail -15 "$_PJ_ERR" 2>/dev/null | sed 's/^/ /' | tee -a "$LOG_FILE"
|
|
603
|
+
rm -f "$_PJ_ERR"
|
|
604
|
+
# Surface the reason to the MCP draft_cycle wrapper (stdout marker) in manual mode.
|
|
605
|
+
if [ "${DRAFT_ONLY:-0}" = "1" ]; then
|
|
606
|
+
echo "DRAFT_ONLY_BLOCKED=$_PJ_REASON"
|
|
607
|
+
fi
|
|
608
|
+
# Record a dashboard-visible failure row (best-effort) and exit cleanly.
|
|
609
|
+
python3 "$REPO_DIR/scripts/log_run.py" --script "post_twitter" --posted 0 --skipped 0 --failed 1 \
|
|
610
|
+
--failure-reasons "${_PJ_REASON}:1" --cost "0.0000" --elapsed $(( $(date +%s) - RUN_START )) 2>/dev/null || true
|
|
611
|
+
_SA_RUN_SUMMARY_EMITTED=1
|
|
612
|
+
exit 0
|
|
613
|
+
fi
|
|
614
|
+
rm -f "$_PJ_ERR"
|
|
615
|
+
|
|
616
|
+
log "Selected projects: $(echo "$PROJECTS_JSON" | python3 -c 'import json,sys; print(", ".join(p["name"] for p in json.load(sys.stdin)))')"
|
|
617
|
+
EXCLUDES_TOTAL=$(echo "$PROJECTS_JSON" | python3 -c 'import json,sys; d=json.load(sys.stdin); print(sum(len(p.get("excludes_for_search") or []) for p in d))')
|
|
618
|
+
[ "${EXCLUDES_TOTAL:-0}" -gt 0 ] && log "Active project-wide excludes loaded across selected projects: $EXCLUDES_TOTAL"
|
|
619
|
+
|
|
620
|
+
# --- Top past queries for style inspiration (PER-PROJECT, 2026-05-26) -------
|
|
621
|
+
# Now scored by composite (clicks×100 + likes + views×0.001) AND filtered by
|
|
622
|
+
# project so the model sees ITS OWN historical winners, not a global pool.
|
|
623
|
+
# Each query carries the full conversion funnel: tweets_found_avg, posted_n,
|
|
624
|
+
# skipped_n, post_rate, views, likes, clicks. Clicks are the ultimate signal;
|
|
625
|
+
# composite weights them ×100 so one click outvalues 100 likes of vibes.
|
|
626
|
+
#
|
|
627
|
+
# Why per-project (vs the previous global TOP_QUERIES_JSON): the global list
|
|
628
|
+
# let a thin niche (paperback-expert) cross-mimic a stronger project's
|
|
629
|
+
# min_faves tier from the gold list, even when paperback-expert had ZERO
|
|
630
|
+
# historical rows of its own. Per-project routing isolates the signal so
|
|
631
|
+
# each project's prompt sees only queries it ran itself.
|
|
632
|
+
#
|
|
633
|
+
# Cold-start projects (zero historical rows): no cross-project fallback. They
|
|
634
|
+
# get an empty project_queries array and rely on PER-PROJECT SUPPLY SIGNAL
|
|
635
|
+
# (for min_faves) + their config.json description (for keyword phrasing). A
|
|
636
|
+
# cross-project "structural inspiration" fallback contradicts the whole point
|
|
637
|
+
# of the per-project routing; explicitly removed 2026-05-26.
|
|
638
|
+
TOP_QUERIES_PER_PROJECT_JSON=$(echo "$PROJECTS_JSON" | python3 -c "
|
|
639
|
+
import json, sys, subprocess
|
|
640
|
+
projects = json.load(sys.stdin)
|
|
641
|
+
repo_dir = '$REPO_DIR'
|
|
642
|
+
|
|
643
|
+
def run_q(args):
|
|
644
|
+
try:
|
|
645
|
+
r = subprocess.run(['python3', f'{repo_dir}/scripts/top_twitter_queries.py'] + args,
|
|
646
|
+
capture_output=True, text=True, timeout=30)
|
|
647
|
+
return json.loads((r.stdout or '[]').strip() or '[]') if r.returncode == 0 else []
|
|
648
|
+
except Exception:
|
|
649
|
+
return []
|
|
650
|
+
|
|
651
|
+
out = {}
|
|
652
|
+
for p in projects:
|
|
653
|
+
name = (p.get('name') or '').strip()
|
|
654
|
+
if not name:
|
|
655
|
+
continue
|
|
656
|
+
rows = run_q(['--limit', '20', '--window-days', '7', '--project', name])
|
|
657
|
+
out[name] = {'project_queries': rows}
|
|
658
|
+
print(json.dumps(out))
|
|
659
|
+
" 2>/dev/null || echo "{}")
|
|
660
|
+
TOP_QUERIES_SUMMARY=$(echo "$TOP_QUERIES_PER_PROJECT_JSON" | python3 -c '
|
|
661
|
+
import json, sys
|
|
662
|
+
d = json.load(sys.stdin)
|
|
663
|
+
parts = []
|
|
664
|
+
cold = 0
|
|
665
|
+
for name, entry in d.items():
|
|
666
|
+
n = len(entry.get("project_queries") or [])
|
|
667
|
+
parts.append(f"{name}={n}")
|
|
668
|
+
if n == 0:
|
|
669
|
+
cold += 1
|
|
670
|
+
print(", ".join(parts) + f" (cold_start_projects={cold})")
|
|
671
|
+
')
|
|
672
|
+
log "Per-project top queries loaded: $TOP_QUERIES_SUMMARY"
|
|
673
|
+
|
|
674
|
+
# --- Top performing search topics (topic-universe evolution, 2026-05-25) ----
|
|
675
|
+
# Sibling signal to TOP_QUERIES_JSON, one level up the funnel: where queries
|
|
676
|
+
# are the literal X search strings, search_topics are the conceptual seeds
|
|
677
|
+
# they were drafted from (e.g. "MCP client desktop", "AI agent that takes
|
|
678
|
+
# actions"). top_search_topics.py reads twitter_candidates (sidesteps
|
|
679
|
+
# posts.search_topic which was 0% covered for Twitter until this cycle) and
|
|
680
|
+
# returns, per topic: posted vs skipped count, avg virality posted vs
|
|
681
|
+
# skipped, total clicks/likes/views, composite_score. The model uses this to
|
|
682
|
+
# evolve the TOPIC UNIVERSE itself (drop topics with high skipped/posted
|
|
683
|
+
# ratio, mimic topics with non-zero clicks, invent variants of winning
|
|
684
|
+
# topics) rather than just rephrasing within the same fixed set of topics.
|
|
685
|
+
TOP_TOPICS_JSON=$(python3 "$REPO_DIR/scripts/top_search_topics.py" --platform twitter --limit 20 --window-days 14 --json 2>/dev/null || echo "[]")
|
|
686
|
+
TOP_TOPICS_COUNT=$(echo "$TOP_TOPICS_JSON" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))')
|
|
687
|
+
log "Top search topics loaded: $TOP_TOPICS_COUNT (Twitter, 14d window)"
|
|
688
|
+
|
|
689
|
+
# --- Dud queries: phrasings that returned 0 tweets in the last 48h ----------
|
|
690
|
+
# Fed into the prompt as a negative-signal anti-list so the LLM stops
|
|
691
|
+
# redrafting the same flat queries every 20-min cycle. Source is
|
|
692
|
+
# twitter_search_attempts, populated below from this run's queries_used.
|
|
693
|
+
# Now also surfaces the parsed `min_faves` value per dud so the model can
|
|
694
|
+
# spot patterns like "every studyly dud last 48h used min_faves:20 — drop
|
|
695
|
+
# the floor for that project".
|
|
696
|
+
DUD_QUERIES_JSON=$(python3 "$REPO_DIR/scripts/top_dud_twitter_queries.py" --limit 30 --window-hours 48 2>/dev/null || echo "[]")
|
|
697
|
+
DUD_COUNT=$(echo "$DUD_QUERIES_JSON" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))')
|
|
698
|
+
log "Dud queries loaded: $DUD_COUNT (last 48h, 0-result, with min_faves parsed)"
|
|
699
|
+
|
|
700
|
+
# --- Dud topics: CONCEPT SEEDS that find viral but off-fit candidates -------
|
|
701
|
+
# One level up from dud queries. DUD_QUERIES_JSON says "this exact phrasing
|
|
702
|
+
# returns 0 tweets, do not reuse it"; DUD_TOPICS_JSON says "this CONCEPT
|
|
703
|
+
# SEED finds viral tweets but Phase 2b keeps skipping them — the seed is
|
|
704
|
+
# mismatched to your buyers; reword the queries narrower or drop the seed".
|
|
705
|
+
# Surfaces sample_skip_reasons so the model can see WHY (audience mismatch,
|
|
706
|
+
# competitor launch, spam-flagged author, etc.) rather than just numeric
|
|
707
|
+
# skip counts. 7d window so we accumulate enough skips for action thresholds
|
|
708
|
+
# without dragging in stale topics.
|
|
709
|
+
DUD_TOPICS_JSON=$(python3 "$REPO_DIR/scripts/top_dud_twitter_topics.py" --limit 12 --window-hours 168 --min-skips 5 2>/dev/null || echo "[]")
|
|
710
|
+
DUD_TOPICS_COUNT=$(echo "$DUD_TOPICS_JSON" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))')
|
|
711
|
+
log "Dud topics loaded: $DUD_TOPICS_COUNT (7d window, min_skips=5)"
|
|
712
|
+
|
|
713
|
+
# --- Per-project supply signal: what min_faves tier returns tweets? ----------
|
|
714
|
+
# Replaces the old flat "broad=50 / narrow=20" rule. For each project the
|
|
715
|
+
# model is currently drafting for, this table shows the median tweets_found
|
|
716
|
+
# at each min_faves tier we've ever tried, plus zero-result %. The model
|
|
717
|
+
# is instructed to pick the LOWEST min_faves tier that historically yields
|
|
718
|
+
# >=3 median tweets for that project (or step down one tier if every tier
|
|
719
|
+
# is >=3 — supply signal trumps the flat rule). For studyly this auto-
|
|
720
|
+
# selects min_faves:15; for mk0r it stays at 30-50.
|
|
721
|
+
SUPPLY_SIGNAL_JSON=$(python3 "$REPO_DIR/scripts/twitter_supply_signal.py" --window-days 14 2>/dev/null || echo "[]")
|
|
722
|
+
SUPPLY_COUNT=$(echo "$SUPPLY_SIGNAL_JSON" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))')
|
|
723
|
+
log "Per-project supply signal loaded: $SUPPLY_COUNT projects"
|
|
724
|
+
|
|
725
|
+
# --- Recently-engaged tweet IDs: scanner skips tweets we already replied to -
|
|
726
|
+
# The scanner re-searches stable hot topics every cycle, so the same fresh
|
|
727
|
+
# tweets resurface. Once we've replied to one it's a dead candidate
|
|
728
|
+
# (score_twitter_candidates.py dedups it downstream). Injecting the last 48h
|
|
729
|
+
# of engaged status IDs into the scan prompt lets the model skip them while
|
|
730
|
+
# scraping instead of spending tokens evaluating tweets it can't post to.
|
|
731
|
+
# 48h is ample: the 6h freshness wall means any dup is necessarily a recent
|
|
732
|
+
# reply. Scoring remains the backstop; this is purely a token cleanup.
|
|
733
|
+
ENGAGED_TWEET_IDS=$(python3 "$REPO_DIR/scripts/twitter_cycle_helper.py" engaged-tweet-ids --window-hours 48 2>/dev/null || echo "[]")
|
|
734
|
+
[ -z "$ENGAGED_TWEET_IDS" ] && ENGAGED_TWEET_IDS="[]"
|
|
735
|
+
ENGAGED_COUNT=$(echo "$ENGAGED_TWEET_IDS" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))')
|
|
736
|
+
log "Recently-engaged tweet IDs loaded: $ENGAGED_COUNT (last 48h; scanner will skip them)"
|
|
737
|
+
|
|
738
|
+
|
|
739
|
+
# --- Phase 1: Claude drafts queries, scrapes tweets -------------------------
|
|
740
|
+
# JSON schema forces structured output. Eliminates the prose-drift failure mode
|
|
741
|
+
# Lean Phase 1 schema (2026-05-28): the scan session no longer scrapes,
|
|
742
|
+
# it only drafts queries. The Python pipeline runs each query via headless
|
|
743
|
+
# Chrome and writes the tweets directly to SCAN_TWEETS_FILE for the shell.
|
|
744
|
+
SCAN_SCHEMA_LEAN='{"type":"object","properties":{"queries":{"type":"array","items":{"type":"object","properties":{"project":{"type":"string"},"query":{"type":"string"},"search_topic":{"type":"string"}},"required":["project","query","search_topic"]}}},"required":["queries"]}'
|
|
745
|
+
|
|
746
|
+
log "Acquiring twitter-browser lock for Phase 1 Claude scan..."
|
|
747
|
+
acquire_lock "twitter-browser" 3600 2>>"$LOG_FILE"
|
|
748
|
+
log "twitter-browser lock held (pid=$$) Phase 1"
|
|
749
|
+
# Drop stale Chrome singleton symlinks before launch. Background ungraceful-
|
|
750
|
+
# exits (SIGKILL, jetsam, force quit) leave Singleton{Lock,Cookie,Socket}
|
|
751
|
+
# pointing at dead PIDs / vanished sockets; without this, Chrome pops "Something
|
|
752
|
+
# went wrong when opening your profile" 7x and the pipeline hangs. Helper
|
|
753
|
+
# refuses to clean if the lock PID is alive.
|
|
754
|
+
ensure_twitter_browser_for_backend 2>&1 | tee -a "$LOG_FILE"
|
|
755
|
+
|
|
756
|
+
# --- Pre-flight: live X session probe (added 2026-06-02) --------------------
|
|
757
|
+
# Before drafting/scraping anything, confirm the harness Chrome actually has a
|
|
758
|
+
# valid x.com session. One CDP Network.getCookies call (<1s) catches the
|
|
759
|
+
# "import never ran, evaporated after a hard restart, or auth_token expired"
|
|
760
|
+
# cases that previously surfaced as "Phase 1 returned 0 tweets" mysteries.
|
|
761
|
+
# Failing fast here turns a wasted ~7-minute scan + Claude bill into a clear
|
|
762
|
+
# "reconnect X" message in the log.
|
|
763
|
+
# Probe the harness Chrome for a live x.com auth_token. Echoes a single
|
|
764
|
+
# PREFLIGHT_OK / PREFLIGHT_FAIL / PREFLIGHT_CDP_ERROR line. Used pre-cycle and
|
|
765
|
+
# again after an auto-restore from the local cookie mirror.
|
|
766
|
+
_xsession_probe() {
|
|
767
|
+
BU_NAME=twitter-harness BU_CDP_URL=http://127.0.0.1:9555 \
|
|
768
|
+
"$HOME/.local/bin/browser-harness" <<'PY' 2>&1
|
|
769
|
+
import sys, time
|
|
770
|
+
try:
|
|
771
|
+
raw = cdp('Network.getCookies', urls=['https://x.com/', 'https://twitter.com/'])
|
|
772
|
+
except Exception as e:
|
|
773
|
+
print('PREFLIGHT_CDP_ERROR ' + type(e).__name__ + ': ' + str(e))
|
|
774
|
+
sys.exit(0)
|
|
775
|
+
ck = raw.get('cookies', [])
|
|
776
|
+
auth = [c for c in ck if c.get('name') == 'auth_token']
|
|
777
|
+
if not auth:
|
|
778
|
+
print('PREFLIGHT_FAIL no_auth_token cookies_total=' + str(len(ck)))
|
|
779
|
+
sys.exit(0)
|
|
780
|
+
exp = auth[0].get('expires')
|
|
781
|
+
domain = auth[0].get('domain', '?')
|
|
782
|
+
if exp in (None, -1, 0):
|
|
783
|
+
print('PREFLIGHT_OK session domain=' + domain)
|
|
784
|
+
else:
|
|
785
|
+
now = time.time()
|
|
786
|
+
if exp < now:
|
|
787
|
+
print('PREFLIGHT_FAIL auth_token_expired exp=' + str(int(exp)) + ' now=' + str(int(now)))
|
|
788
|
+
sys.exit(0)
|
|
789
|
+
print('PREFLIGHT_OK exp=' + str(int(exp)) + ' domain=' + domain)
|
|
790
|
+
PY
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
log "Pre-flight: probing harness Chrome for a live x.com auth_token..."
|
|
794
|
+
_PREFLIGHT_OUT=$(_xsession_probe)
|
|
795
|
+
if ! printf '%s\n' "$_PREFLIGHT_OUT" | grep -q '^PREFLIGHT_OK'; then
|
|
796
|
+
# Gap B auto-recovery: the harness Chrome lost its x.com session — its cookie
|
|
797
|
+
# store was wiped on a hard restart or a macOS keychain re-lock, or never
|
|
798
|
+
# persisted to disk. Re-inject from the durable 0600 local cookie mirror
|
|
799
|
+
# (written on every connect, keychain-independent) via CDP, then re-probe
|
|
800
|
+
# before giving up. This is what makes the session survive app/Chrome
|
|
801
|
+
# restarts without a manual reconnect.
|
|
802
|
+
log " Pre-flight FAILED ($(printf '%s\n' "$_PREFLIGHT_OUT" | tail -1)); auto-restoring from local cookie mirror..."
|
|
803
|
+
_RESTORE_OUT=$(TWITTER_CDP_URL="${TWITTER_CDP_URL:-http://127.0.0.1:9555}" \
|
|
804
|
+
python3 "$REPO_DIR/scripts/restore_twitter_session.py" 2>&1)
|
|
805
|
+
log " Restore: $(printf '%s\n' "$_RESTORE_OUT" | tail -2 | tr '\n' '|')"
|
|
806
|
+
_PREFLIGHT_OUT=$(_xsession_probe)
|
|
807
|
+
fi
|
|
808
|
+
if printf '%s\n' "$_PREFLIGHT_OUT" | grep -q '^PREFLIGHT_OK'; then
|
|
809
|
+
log " Pre-flight OK: $(printf '%s\n' "$_PREFLIGHT_OUT" | grep '^PREFLIGHT_OK' | head -1)"
|
|
810
|
+
else
|
|
811
|
+
log " Pre-flight FAILED. The harness Chrome has no live X session (auto-restore from the local cookie mirror did not recover it)."
|
|
812
|
+
log " Details: $(printf '%s\n' "$_PREFLIGHT_OUT" | tail -3 | tr '\n' '|')"
|
|
813
|
+
log " Action: run \`python3 scripts/setup_twitter_auth.py connect\` (or call the connect_x MCP tool) to import a fresh X session from your everyday browser, then re-run the cycle. If the import fails with 'access denied', unlock the macOS keychain first: \`security unlock-keychain ~/Library/Keychains/login.keychain-db\`."
|
|
814
|
+
echo "twitter_batches: ended $BATCH_ID"
|
|
815
|
+
release_lock "twitter-browser" 2>/dev/null || true
|
|
816
|
+
exit 1
|
|
817
|
+
fi
|
|
818
|
+
|
|
819
|
+
# --- Pre-flight 2: live access-gate probe + backoff (added 2026-06-29) -------
|
|
820
|
+
# The cookie probe above only proves an auth_token EXISTS. X can still gate a
|
|
821
|
+
# perfectly valid session: from a datacenter IP (e.g. the MacStadium box) it
|
|
822
|
+
# commonly 302s authenticated routes to /account/access ("verify it's you") or
|
|
823
|
+
# fronts them with a Cloudflare "security verification" interstitial. A gated
|
|
824
|
+
# session renders real, public tweets as "this page doesn't exist", so the scan
|
|
825
|
+
# silently returns 0-few candidates and we'd draft/post against phantom
|
|
826
|
+
# emptiness (this is one root of the old "Phase 1 returned 0 tweets" mysteries
|
|
827
|
+
# that the cookie probe alone never caught). Navigate ONE authenticated route
|
|
828
|
+
# and STOP the cycle if X is gating us. Fails OPEN: a probe error or an
|
|
829
|
+
# ok/unknown render never blocks, so a transient hydration miss can't halt
|
|
830
|
+
# posting — only a positively-detected gate (gated:true) stops the cycle.
|
|
831
|
+
#
|
|
832
|
+
# BACKOFF: this launchd job fires every 5 min, and a gated cycle exits in ~2s,
|
|
833
|
+
# so without backoff we'd hit Cloudflare /account/access ~12x/hr (~288/day),
|
|
834
|
+
# which only deepens the datacenter-IP trust penalty. A state marker records the
|
|
835
|
+
# gate and an exponential cooldown (15m -> 30m -> 60m -> cap 120m). While the
|
|
836
|
+
# cooldown is live we skip the cycle WITHOUT navigating (no flagged traffic);
|
|
837
|
+
# once it elapses we re-probe; an 'ok' probe clears the marker and resumes. Only
|
|
838
|
+
# gated:true ever writes the marker, so fail-open is preserved.
|
|
839
|
+
_S4L_STATE_DIR="${S4L_STATE_DIR:-$HOME/.social-autoposter-mcp}"
|
|
840
|
+
_GATE_FILE="$_S4L_STATE_DIR/x-access-gate.json"
|
|
841
|
+
_NOW=$(date +%s)
|
|
842
|
+
|
|
843
|
+
# Backoff short-circuit: still inside a cooldown window -> skip without probing.
|
|
844
|
+
if [ -f "$_GATE_FILE" ]; then
|
|
845
|
+
_CD_UNTIL=$(python3 -c 'import json,sys
|
|
846
|
+
try: print(int(json.load(open(sys.argv[1])).get("cooldown_until",0)))
|
|
847
|
+
except Exception: print(0)' "$_GATE_FILE" 2>/dev/null || echo 0)
|
|
848
|
+
if [ "${_CD_UNTIL:-0}" -gt "$_NOW" ]; then
|
|
849
|
+
_MINS=$(( (_CD_UNTIL - _NOW + 59) / 60 ))
|
|
850
|
+
log "Pre-flight: X access-gate backoff active (~${_MINS}m left); skipping cycle without re-probing to avoid adding flagged Cloudflare traffic."
|
|
851
|
+
echo "twitter_batches: ended $BATCH_ID"
|
|
852
|
+
release_lock "twitter-browser" 2>/dev/null || true
|
|
853
|
+
exit 0
|
|
854
|
+
fi
|
|
855
|
+
fi
|
|
856
|
+
|
|
857
|
+
log "Pre-flight: probing for an X access gate (/account/access, Cloudflare)..."
|
|
858
|
+
_ACCESS_OUT=$(TWITTER_CDP_URL="${TWITTER_CDP_URL:-http://127.0.0.1:9555}" \
|
|
859
|
+
python3 "$REPO_DIR/scripts/twitter_access_check.py" --session-probe --wait-ms 12000 2>/dev/null)
|
|
860
|
+
if printf '%s' "$_ACCESS_OUT" | grep -q '"gated": *true'; then
|
|
861
|
+
# Write/refresh the backoff marker with an exponential cooldown. Python
|
|
862
|
+
# prints "<next_mins> <consecutive> <cooldown_secs> <gate_age_secs>".
|
|
863
|
+
_GATE_FIELDS=$(python3 -c 'import json,sys
|
|
864
|
+
gf, now = sys.argv[1], int(sys.argv[2])
|
|
865
|
+
base, cap, factor = 900, 7200, 2
|
|
866
|
+
try: prev = json.load(open(gf))
|
|
867
|
+
except Exception: prev = {}
|
|
868
|
+
cd = prev.get("cooldown_secs")
|
|
869
|
+
cd = base if not cd else min(int(cd)*factor, cap)
|
|
870
|
+
fs = int(prev.get("first_seen", now))
|
|
871
|
+
cons = int(prev.get("consecutive", 0)) + 1
|
|
872
|
+
out = {"first_seen": fs, "last_seen": now, "reason": "access_gated",
|
|
873
|
+
"consecutive": cons, "cooldown_secs": cd, "cooldown_until": now+cd}
|
|
874
|
+
json.dump(out, open(gf, "w"))
|
|
875
|
+
print(cd//60, cons, cd, max(0, now-fs))' "$_GATE_FILE" "$_NOW" 2>/dev/null || echo "15 1 900 0")
|
|
876
|
+
read -r _NEXT_MINS _CONS _CD_SECS _AGE_SECS <<< "$_GATE_FIELDS"
|
|
877
|
+
log " Pre-flight FAILED: X is gating this session (access gate detected)."
|
|
878
|
+
log " Probe: $(printf '%s' "$_ACCESS_OUT" | tr '\n' ' ' | tr -s ' ' | sed 's/^ *//')"
|
|
879
|
+
log " X redirected an authenticated route to /account/access or served a Cloudflare verification page. This is usually datacenter-IP trust degradation: the session cookie is still valid but X hides content from it, so a scan would return phantom 'doesn't exist' results."
|
|
880
|
+
log " Backoff engaged: next access re-probe in ~${_NEXT_MINS}m (intervening 5-min firings skip without touching Cloudflare)."
|
|
881
|
+
log " Action: open the harness Chrome (CDP :9555) and complete the verification at https://x.com/account/access once, or route the box through a residential/clean IP. The cycle auto-resumes within one cooldown of the gate lifting."
|
|
882
|
+
# Machine-greppable marker (additive; mirrors the stderr-marker convention
|
|
883
|
+
# bin/server.js parses). Pairs with twitter_access_gate:recovered below.
|
|
884
|
+
echo "twitter_access_gate: gated consecutive=${_CONS} age_s=${_AGE_SECS} next_reprobe_s=${_CD_SECS}" >&2
|
|
885
|
+
echo "twitter_batches: ended $BATCH_ID"
|
|
886
|
+
release_lock "twitter-browser" 2>/dev/null || true
|
|
887
|
+
exit 1
|
|
888
|
+
fi
|
|
889
|
+
# Probe came back clean. If a backoff marker exists we were gated: record the
|
|
890
|
+
# recovery (how long the gate lasted, since first_seen) BEFORE deleting it, so
|
|
891
|
+
# the lift event + duration survive in the log even though the marker is gone.
|
|
892
|
+
if [ -f "$_GATE_FILE" ]; then
|
|
893
|
+
_REC=$(python3 -c 'import json,sys
|
|
894
|
+
try: d = json.load(open(sys.argv[1]))
|
|
895
|
+
except Exception: d = {}
|
|
896
|
+
now = int(sys.argv[2]); fs = int(d.get("first_seen", now)); cons = int(d.get("consecutive", 0))
|
|
897
|
+
dur = max(0, now-fs)
|
|
898
|
+
print(dur, dur//60, cons)' "$_GATE_FILE" "$_NOW" 2>/dev/null || echo "0 0 0")
|
|
899
|
+
read -r _DUR_S _DUR_M _RCONS <<< "$_REC"
|
|
900
|
+
rm -f "$_GATE_FILE"
|
|
901
|
+
echo "twitter_access_gate: recovered_after_s=${_DUR_S} consecutive=${_RCONS}" >&2
|
|
902
|
+
log " X access gate lifted after ~${_DUR_M}m (${_RCONS} consecutive gated probes); cleared backoff marker and resuming normal cycle."
|
|
903
|
+
fi
|
|
904
|
+
log " Pre-flight access OK: $(printf '%s' "$_ACCESS_OUT" | tr '\n' ' ' | tr -s ' ' | sed 's/^ *//')"
|
|
905
|
+
|
|
906
|
+
# --- Phase 1 retry loop (2026-05-27) ----------------------------------------
|
|
907
|
+
# When a single scan produces fewer than RETRY_TARGET candidates that survive
|
|
908
|
+
# all Phase 1 filters (harness age gate, scorer stale_age cutoff, already-
|
|
909
|
+
# posted dedupe, fabricated_id check), re-invoke the Claude scan with the
|
|
910
|
+
# queries already tried this cycle injected as a "do NOT repeat" block.
|
|
911
|
+
# Each iteration upserts into the SAME batch_id so survivors accumulate.
|
|
912
|
+
# Cap at MAX_SCAN_ATTEMPTS to stay inside the 20-min Phase 1 budget; if the
|
|
913
|
+
# cap is hit before target, proceed with whatever we have (even 1 candidate
|
|
914
|
+
# is better than 0). When BATCH_COUNT is still 0 after the loop, the
|
|
915
|
+
# post-loop empty_batch branch fires.
|
|
916
|
+
# DEFAULT Phase 1 is the deterministic qualified-query bank (no Claude): the
|
|
917
|
+
# bank replays every historically qualified query for the picked project in a
|
|
918
|
+
# single pass, so there is nothing to "retry-draft" and one attempt is enough.
|
|
919
|
+
# The legacy LLM-draft path (TWITTER_PHASE1_LLM_DRAFT=1) keeps the 5-attempt
|
|
920
|
+
# retry loop, because LLM queries frequently return empty and need re-drafting.
|
|
921
|
+
if [ "${TWITTER_PHASE1_LLM_DRAFT:-0}" = "1" ]; then
|
|
922
|
+
MAX_SCAN_ATTEMPTS=5
|
|
923
|
+
else
|
|
924
|
+
MAX_SCAN_ATTEMPTS=1
|
|
925
|
+
fi
|
|
926
|
+
RETRY_TARGET=5
|
|
927
|
+
SCAN_ATTEMPT=0
|
|
928
|
+
BATCH_COUNT=0
|
|
929
|
+
# Cumulative counters across iterations — feed log_run.py once at end so the
|
|
930
|
+
# dashboard shows the total work the cycle did, not just the last attempt.
|
|
931
|
+
CUMULATIVE_QUERIES=0
|
|
932
|
+
CUMULATIVE_DUDS=0
|
|
933
|
+
CUMULATIVE_TWEETS_PULLED=0
|
|
934
|
+
# Running list of queries the model has already tried THIS cycle, injected
|
|
935
|
+
# into each retry's prompt as "do NOT repeat these phrasings". Extended after
|
|
936
|
+
# every scan from QUERIES_FILE before log_twitter_search_attempts deletes it.
|
|
937
|
+
TRIED_QUERIES_JSON='[]'
|
|
938
|
+
# Running list of SEARCH TOPICS already tried this cycle. Each retry calls
|
|
939
|
+
# pick_topic_for_project again with this list as exclude_topics so the model
|
|
940
|
+
# isn't pinned to one assigned topic for all 5 attempts. When the filtered
|
|
941
|
+
# universe empties (small-project case), the picker raises
|
|
942
|
+
# UniverseExhaustedError and the retry loop breaks — no invent fallback
|
|
943
|
+
# (invention is the standalone invent_topics.py job's responsibility).
|
|
944
|
+
TRIED_TOPICS_JSON='[]'
|
|
945
|
+
# Latest Anthropic-side error classification for the post-loop log_run when
|
|
946
|
+
# every attempt returned zero tweets (stream_idle_timeout vs phase1_no_tweets
|
|
947
|
+
# vs api_overloaded, etc.). Falls back to phase1_no_tweets when unset.
|
|
948
|
+
LAST_PHASE1_REASON=""
|
|
949
|
+
# Set to 1 by the in-loop repick when pick_search_topic raises
|
|
950
|
+
# UniverseExhaustedError (the project ran out of un-tried active topics).
|
|
951
|
+
# Used by the post-loop empty-batch branch to emit `universe_exhausted:1`
|
|
952
|
+
# instead of `empty_batch:1` so the dashboard shows the right cause.
|
|
953
|
+
UNIVERSE_EXHAUSTED=0
|
|
954
|
+
|
|
955
|
+
while [ "$SCAN_ATTEMPT" -lt "$MAX_SCAN_ATTEMPTS" ]; do
|
|
956
|
+
SCAN_ATTEMPT=$((SCAN_ATTEMPT + 1))
|
|
957
|
+
# Snapshot the pre-attempt batch size so the verdict step below can compute
|
|
958
|
+
# kept_after_skip as a delta after the scorer finishes this attempt (2026-05-28
|
|
959
|
+
# retry-feedback: turns TRIED_QUERIES_JSON from bare phrasings into per-query
|
|
960
|
+
# verdicts the drafter can use to choose broaden vs narrow vs new-topic).
|
|
961
|
+
BATCH_COUNT_BEFORE_ATTEMPT="${BATCH_COUNT:-0}"
|
|
962
|
+
|
|
963
|
+
# --- Per-attempt topic (re)pick (2026-05-27) ---------------------------------
|
|
964
|
+
# Attempt 1 keeps the pre-loop topic that PROJECTS_JSON already carries.
|
|
965
|
+
# Attempts 2+ call pick_topic_for_project again with TRIED_TOPICS_JSON as
|
|
966
|
+
# exclude_topics, then rewrite PROJECTS_JSON in place with the new topic and
|
|
967
|
+
# its reference_topics. This makes the retry genuinely end-to-end programmatic:
|
|
968
|
+
# new topic -> new query -> new tweets, not just "model rephrases the same
|
|
969
|
+
# assigned topic 5 times". When the project's filtered universe empties
|
|
970
|
+
# (small project, all topics tried this cycle), the picker raises
|
|
971
|
+
# UniverseExhaustedError and the shell breaks the retry loop cleanly
|
|
972
|
+
# (post-2026-05-28: no invent fallback; invention lives in invent_topics.py).
|
|
973
|
+
if [ "$SCAN_ATTEMPT" -gt 1 ]; then
|
|
974
|
+
log "Phase 1 attempt $SCAN_ATTEMPT: re-picking search_topic via pick_topic_for_project (exclude=$(echo "$TRIED_TOPICS_JSON" | python3 -c 'import json,sys;print(len(json.load(sys.stdin)))' 2>/dev/null || echo 0) tried)..."
|
|
975
|
+
# The exhaustion marker file is the cross-boundary signal back to
|
|
976
|
+
# bash: when pick_topic_for_project raises UniverseExhaustedError for
|
|
977
|
+
# ANY selected project, the Python writes this file and the shell
|
|
978
|
+
# breaks the retry loop after the heredoc returns. No invent fallback
|
|
979
|
+
# (2026-05-28 architecture: invention is the standalone
|
|
980
|
+
# invent_topics.py job's responsibility, not the cycle's).
|
|
981
|
+
UNIVERSE_EXHAUSTED_MARKER="/tmp/twitter_cycle_universe_exhausted_${BATCH_ID}"
|
|
982
|
+
rm -f "$UNIVERSE_EXHAUSTED_MARKER"
|
|
983
|
+
PROJECTS_JSON=$(python3 - "$PROJECTS_JSON" "$TRIED_TOPICS_JSON" "$UNIVERSE_EXHAUSTED_MARKER" <<'PY' 2>>"$LOG_FILE"
|
|
984
|
+
import json, os, sys
|
|
985
|
+
sys.path.insert(0, os.path.expanduser('~/social-autoposter/scripts'))
|
|
986
|
+
from pick_search_topic import pick_topic_for_project, PickerError, UniverseExhaustedError
|
|
987
|
+
|
|
988
|
+
projects = json.loads(sys.argv[1] or '[]')
|
|
989
|
+
excluded = json.loads(sys.argv[2] or '[]')
|
|
990
|
+
marker_path = sys.argv[3]
|
|
991
|
+
exhausted_for = []
|
|
992
|
+
for p in projects:
|
|
993
|
+
name = p.get('name')
|
|
994
|
+
if not name:
|
|
995
|
+
continue
|
|
996
|
+
try:
|
|
997
|
+
new_pick = pick_topic_for_project(
|
|
998
|
+
name, platform='twitter', exclude_topics=excluded,
|
|
999
|
+
)
|
|
1000
|
+
except UniverseExhaustedError as exc:
|
|
1001
|
+
# All active topics for this project already tried this cycle.
|
|
1002
|
+
# Stamp the marker file so the shell breaks the retry loop and
|
|
1003
|
+
# logs `universe_exhausted:1` as the failure reason. Leave the
|
|
1004
|
+
# project entry as-is (the loop will exit before scanning anyway).
|
|
1005
|
+
sys.stderr.write(f"repick_universe_exhausted project={name!r} error={exc}\n")
|
|
1006
|
+
exhausted_for.append(name)
|
|
1007
|
+
continue
|
|
1008
|
+
except PickerError as exc:
|
|
1009
|
+
# On repick PickerError (DB unreachable, etc) keep the previous
|
|
1010
|
+
# topic; the scan will still run. Strictly better than aborting.
|
|
1011
|
+
sys.stderr.write(f"repick_failed project={name!r} error={exc}\n")
|
|
1012
|
+
continue
|
|
1013
|
+
p['search_topic'] = new_pick.get('search_topic')
|
|
1014
|
+
p['picked_weight_pct'] = new_pick.get('picked_weight_pct')
|
|
1015
|
+
p['reference_topics'] = new_pick.get('reference_topics') or []
|
|
1016
|
+
if exhausted_for:
|
|
1017
|
+
with open(marker_path, 'w') as fh:
|
|
1018
|
+
fh.write(','.join(exhausted_for) + '\n')
|
|
1019
|
+
print(json.dumps(projects))
|
|
1020
|
+
PY
|
|
1021
|
+
)
|
|
1022
|
+
if [ -f "$UNIVERSE_EXHAUSTED_MARKER" ]; then
|
|
1023
|
+
UNIVERSE_EXHAUSTED=1
|
|
1024
|
+
_EXH_PROJECTS=$(cat "$UNIVERSE_EXHAUSTED_MARKER" 2>/dev/null | tr -d '\n')
|
|
1025
|
+
log " Universe exhausted for project(s)=$_EXH_PROJECTS after $((SCAN_ATTEMPT - 1)) prior attempt(s); breaking retry loop"
|
|
1026
|
+
rm -f "$UNIVERSE_EXHAUSTED_MARKER"
|
|
1027
|
+
break
|
|
1028
|
+
fi
|
|
1029
|
+
fi
|
|
1030
|
+
|
|
1031
|
+
# Snapshot this attempt's topic(s) into TRIED_TOPICS_JSON so the NEXT
|
|
1032
|
+
# iteration's repick excludes them. Runs every attempt (incl. attempt 1)
|
|
1033
|
+
# so the initial pre-loop topic also goes into the exclude list before
|
|
1034
|
+
# attempt 2's repick. Idempotent: same topic added twice is a no-op.
|
|
1035
|
+
TRIED_TOPICS_JSON=$(python3 - "$TRIED_TOPICS_JSON" "$PROJECTS_JSON" <<'PY' 2>>"$LOG_FILE"
|
|
1036
|
+
import json, sys
|
|
1037
|
+
cur = json.loads(sys.argv[1] or '[]')
|
|
1038
|
+
projects = json.loads(sys.argv[2] or '[]')
|
|
1039
|
+
seen = {(t or '').strip().lower() for t in cur if t}
|
|
1040
|
+
for p in projects:
|
|
1041
|
+
t = (p.get('search_topic') or '').strip()
|
|
1042
|
+
if t and t.lower() not in seen:
|
|
1043
|
+
cur.append(t)
|
|
1044
|
+
seen.add(t.lower())
|
|
1045
|
+
print(json.dumps(cur))
|
|
1046
|
+
PY
|
|
1047
|
+
)
|
|
1048
|
+
|
|
1049
|
+
_CURRENT_TOPICS=$(echo "$PROJECTS_JSON" | python3 -c 'import json,sys; ps=json.load(sys.stdin); print(", ".join((p.get("search_topic") or "?") for p in ps))' 2>/dev/null || echo "")
|
|
1050
|
+
log "Phase 1 scan attempt $SCAN_ATTEMPT/$MAX_SCAN_ATTEMPTS (batch=$BATCH_ID, candidates so far=$BATCH_COUNT/$RETRY_TARGET, topic(s)=$_CURRENT_TOPICS)"
|
|
1051
|
+
|
|
1052
|
+
log "Phase 1: drafting queries and scraping tweets..."
|
|
1053
|
+
|
|
1054
|
+
# Shell-side data path. scripts/twitter_scan.scan() appends one JSONL record
|
|
1055
|
+
# per call to this file. After the claude scan session ends we parse it
|
|
1056
|
+
# directly into $RAW_FILE and $QUERIES_FILE, bypassing the model's
|
|
1057
|
+
# structured_output relay so the model no longer pays per-tweet copy tokens.
|
|
1058
|
+
# One file per Phase 1 attempt so retry iterations do not share state. The
|
|
1059
|
+
# rm -f makes each attempt's accumulation start clean. Falls back to the
|
|
1060
|
+
# structured_output parse below when the file is empty (e.g. every bh_run was
|
|
1061
|
+
# denied by the stub-enforcement hook so scan() never executed).
|
|
1062
|
+
SCAN_TWEETS_FILE="/tmp/twcycle-${BATCH_ID}-attempt-${SCAN_ATTEMPT}-tweets.jsonl"
|
|
1063
|
+
rm -f "$SCAN_TWEETS_FILE"
|
|
1064
|
+
export SCAN_TWEETS_FILE
|
|
1065
|
+
|
|
1066
|
+
# === LEAN PHASE 1 (2026-05-28) =============================================
|
|
1067
|
+
# Replaces the model-driven Twitter scrape with: small Claude call that only
|
|
1068
|
+
# DRAFTS queries (no tools, no MCP, no browser), then a Python loop that runs
|
|
1069
|
+
# each query via the operator-owned twitter_scan.scan() function over the same
|
|
1070
|
+
# CDP daemon. Cuts per-cycle scan cost roughly 10x by removing:
|
|
1071
|
+
# - MCP bh_run tool roundtrips
|
|
1072
|
+
# - structured_output tweet relay (was hundreds of tweet objects)
|
|
1073
|
+
# - draft-deny-retry churn (model used to try inline scrapes and bounce off
|
|
1074
|
+
# the PreToolUse stub-enforcement hook every session)
|
|
1075
|
+
# Output downstream is identical: $RAW_FILE + $QUERIES_FILE feed the scorer
|
|
1076
|
+
# and twitter_search_attempts logger the same way as before.
|
|
1077
|
+
#
|
|
1078
|
+
if [ "${TWITTER_PHASE1_LLM_DRAFT:-0}" = "1" ]; then
|
|
1079
|
+
# === LLM QUERY-DRAFT PATH (legacy, behind TWITTER_PHASE1_LLM_DRAFT=1) ========
|
|
1080
|
+
log "Lean Phase 1: drafting queries (no browser tools)..."
|
|
1081
|
+
|
|
1082
|
+
QUERIES_OUTPUT=$("$REPO_DIR/scripts/run_claude.sh" "run-twitter-cycle-queries" --strict-mcp-config --mcp-config "$TW_MCP_CONFIG" -p --output-format json --json-schema "$SCAN_SCHEMA_LEAN" "${TW_ENGINE_PREFIX}You are a Twitter query drafter. Your ONLY job is to draft fresh X advanced-search queries that surface tweets relevant to our projects. You do NOT post, you do NOT call any tools, you do NOT scrape. A separate Python pipeline runs your queries over the same CDP-driven Chrome and applies a strict freshness gate; you only return the query strings.
|
|
1083
|
+
|
|
1084
|
+
## Step 1: Draft one search query per project
|
|
1085
|
+
|
|
1086
|
+
You have $(echo "$PROJECTS_JSON" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))') projects. Draft exactly ONE Twitter search query for each, tailored to that project's ASSIGNED search_topic.
|
|
1087
|
+
|
|
1088
|
+
Each project entry carries TWO fields that drive your behavior: \`topic_picked_mode\` (either \`use\` or \`explore_invent\`) and \`search_topic\` (a string in \`use\` mode, NULL in \`explore_invent\` mode).
|
|
1089
|
+
|
|
1090
|
+
USE mode (~90% of cycles, indicated by \`topic_picked_mode: \"use\"\` and a non-null \`search_topic\`):
|
|
1091
|
+
The Python picker has already chosen this project's search_topic by weighted-random sampling over the FULL universe in config.json. Your job is to translate that ASSIGNED topic into the best Twitter advanced-search query that will surface fresh, on-topic tweets. Do NOT substitute a different topic; do NOT paraphrase the topic. End-to-end attribution joins on the exact string.
|
|
1092
|
+
|
|
1093
|
+
EXPLORE_INVENT mode (~10% of cycles, indicated by \`topic_picked_mode: \"explore_invent\"\` and \`search_topic: null\`):
|
|
1094
|
+
The picker is asking you to INVENT a brand-new search_topic. Look at the project's own \`reference_topics\` array and propose ONE new topic concept that does NOT appear there and is NOT a paraphrase of anything in it. Use your invented topic as the query's \`search_topic\` AND drive the keyword phrasing from it (one consistent string per project).
|
|
1095
|
+
|
|
1096
|
+
Projects:
|
|
1097
|
+
$PROJECTS_JSON
|
|
1098
|
+
|
|
1099
|
+
Top past queries FOR THE PROJECT YOU'RE DRAFTING FOR (7-day window, per-project, sorted by clicks DESC first, then composite-scored: clicks×100 + likes + views×0.001). CLICKS ARE THE PRIORITY SIGNAL. Each row carries THREE labels that tell you what to do with it as a reference:
|
|
1100
|
+
|
|
1101
|
+
- \`supply_bucket\`: low (<1 tweet/attempt), medium (1-5), high (>5). Raw supply X returned for this phrasing.
|
|
1102
|
+
- \`conversion_bucket\`: low (<0.2 post_rate), medium (0.2-0.6), high (>=0.6). How often a found tweet survived the draft gate.
|
|
1103
|
+
- \`guidance\`: one of MIMIC, KEEP_STYLE, NARROW, BROADEN — the action to take when drawing from this query.
|
|
1104
|
+
- \`posts_per_attempt\`: posts produced per Phase 1 search invocation; <0.1 means most attempts produce zero survivors.
|
|
1105
|
+
|
|
1106
|
+
How to act on \`guidance\`:
|
|
1107
|
+
- MIMIC — gold tier. Reuse the operator skeleton verbatim, swap only the topic keyword for the picker-assigned topic.
|
|
1108
|
+
- KEEP_STYLE — solid. Use the operator pattern as inspiration; small phrasing tweaks OK.
|
|
1109
|
+
- NARROW — high supply, low conversion (noisy pond). If you draw from it, ADD specificity: more OR alternates, stricter min_faves, extra -term excludes.
|
|
1110
|
+
- BROADEN — low supply (query dying or topic running dry). The OPERATORS are dead weight. Shorten to 1-2 keywords, drop OR groups, step min_faves down a tier. Do NOT inherit operators from a BROADEN-tagged row.
|
|
1111
|
+
|
|
1112
|
+
The canonical source for \`min_faves:N\` selection is the PER-PROJECT SUPPLY SIGNAL block below.
|
|
1113
|
+
$TOP_QUERIES_PER_PROJECT_JSON
|
|
1114
|
+
|
|
1115
|
+
TOP-PERFORMING SEARCH TOPICS (conceptual seeds, 14d window) — context for query phrasing only; you draft a query for the picker-assigned topic, you do NOT swap topics here:
|
|
1116
|
+
$TOP_TOPICS_JSON
|
|
1117
|
+
|
|
1118
|
+
DUD QUERIES — DO NOT REUSE these phrasings or close variants. They returned ZERO tweets in the last 48h:
|
|
1119
|
+
$DUD_QUERIES_JSON
|
|
1120
|
+
|
|
1121
|
+
DUD CONCEPT SEEDS — these search_topic seeds pulled in tweets that Phase 2b's draft gate kept skipping over the last 7d. Per entry: \`omit_rate\` = skipped_n / (posted_n + skipped_n), \`sample_skip_reasons\` are the top reject reasons. If \`omit_rate >= 0.6\` AND \`skipped_n >= 5\`, REWORD the query narrower or drop the seed and pick a different config.json seed for that project:
|
|
1122
|
+
$DUD_TOPICS_JSON
|
|
1123
|
+
|
|
1124
|
+
PER-PROJECT SUPPLY SIGNAL — for each project, the historical median tweets_found at each \`min_faves:N\` tier you've drafted in the last 14d. Pick the LOWEST tier where \`median_tweets_found >= 3\`; if every tier is below 3, drop one tier lower than the lowest you've tried. Trust this table over priors:
|
|
1125
|
+
$SUPPLY_SIGNAL_JSON
|
|
1126
|
+
|
|
1127
|
+
ALREADY-ENGAGED TWEET IDS (last 48h) — the Python scraper skips these regardless, but knowing them helps you avoid drafting a query that would predominantly surface dead candidates:
|
|
1128
|
+
$ENGAGED_TWEET_IDS
|
|
1129
|
+
|
|
1130
|
+
THIS-CYCLE QUERIES ALREADY TRIED with per-query outcomes (attempt $SCAN_ATTEMPT/$MAX_SCAN_ATTEMPTS, target=$RETRY_TARGET candidates after filters). Do NOT repeat any of these phrasings or close variants. Read each entry's \`verdict\` field and respond directionally (do NOT default to generic "broaden"):
|
|
1131
|
+
- \`dead_supply\` (raw_tweets=0): the phrasing returned ZERO tweets from X. The query was too narrow for X's index. HARD RULE: attempt N+1 MUST execute at least ONE of these THREE concrete broadening moves, NOT a topic rotation. Pick exactly one and apply it visibly: (a) lower \`min_faves\` by ONE FULL TIER (e.g. 20→5, 5→1, 1→0); (b) reduce the OR alternates inside any parenthesized group to AT MOST 2 terms (e.g. \`(A OR B OR C OR D)\` → \`(A OR B)\`); (c) drop ALL \`-term\` excludes EXCEPT those listed in this project's \`excludes_for_search\` (which remain mandatory). The PER-PROJECT SUPPLY SIGNAL block is OVERRIDDEN by \`dead_supply\` THIS CYCLE — do not appeal to historical min_faves when the current attempt returned 0. Swapping the topic noun while keeping the same operator skeleton is NOT broadening and is FORBIDDEN as a response to \`dead_supply\`.
|
|
1132
|
+
- \`all_aged_out\` (raw>0, kept_after_age=0): topic is supply-limited at the current freshness window; every tweet was older than the cap. Pick a structurally adjacent topic; do NOT rephrase the same one (it will just hit the cap again).
|
|
1133
|
+
- \`all_engaged_or_skipped\` (kept_after_age>0, kept_after_skip=0): query phrasing is fine, but the surviving tweets were already engaged on prior cycles. Pick a DIFFERENT topic, not a rephrase.
|
|
1134
|
+
- \`found_some\` (kept_after_skip>0 but below target): query is on-target. Raise min_faves one tier OR add a semantic constraint to lift quality. Do NOT broaden.
|
|
1135
|
+
$TRIED_QUERIES_JSON
|
|
1136
|
+
|
|
1137
|
+
Query guidelines:
|
|
1138
|
+
- MANDATORY: do NOT add any date or time-window operator to your query (no \`since:\`, \`until:\`, \`since_time:\`, \`until_time:\`). The Python scraper enforces the freshness window at the URL level after you return; any time operator you include is stripped and overwritten. Including raw bash arithmetic, format strings, or placeholder text in place of a real epoch will be sent to X as a literal keyword and produce zero results.
|
|
1139
|
+
- MANDATORY EVEN IF YOUR QUERY KEYWORDS DO NOT NAME THE EXCLUDED TOPIC: if a project's \`excludes_for_search\` array is non-empty, append \`-term\` for EVERY listed term to that project's query, verbatim, no exceptions.
|
|
1140
|
+
- MANDATORY: pick \`min_faves:N\` per the PER-PROJECT SUPPLY SIGNAL above. If a project has no entry there (new / first cycle), start at min_faves:20.
|
|
1141
|
+
- Favor discussions/opinions (people sharing experience, asking questions), not news/promos/giveaways.
|
|
1142
|
+
- Pick a query likely to surface tweets RELEVANT to that project's actual domain.
|
|
1143
|
+
- Mix it up each run; don't always use the same query for the same project.
|
|
1144
|
+
- Use the project's ASSIGNED \`search_topic\` plus its \`description\` as grounding for query phrasing.
|
|
1145
|
+
- The \`search_topic\` you emit in the output JSON MUST be the project's assigned \`search_topic\` field pasted VERBATIM (NOT the query string, NOT a paraphrase). The scoring pipeline stamps \`twitter_candidates.search_topic\` from this for end-to-end attribution.
|
|
1146
|
+
|
|
1147
|
+
## Output
|
|
1148
|
+
|
|
1149
|
+
Return ONLY the structured_output JSON with this shape:
|
|
1150
|
+
{\"queries\": [{\"project\": \"PROJECT_NAME\", \"query\": \"X advanced search string with operators\", \"search_topic\": \"assigned or invented topic, verbatim\"}, ...]}
|
|
1151
|
+
|
|
1152
|
+
One entry per project. Do NOT include tweets, do NOT include tweets_found, do NOT call any tool, do NOT scrape. The shell pipeline runs each query via headless Chrome with a strict freshness gate after you return." 2>&1)
|
|
1153
|
+
|
|
1154
|
+
|
|
1155
|
+
# Dump the captured envelope to the cycle log for offline inspection.
|
|
1156
|
+
echo "$QUERIES_OUTPUT" >> "$LOG_FILE"
|
|
1157
|
+
|
|
1158
|
+
# Extract the drafted queries to a temp file.
|
|
1159
|
+
QUERIES_TMP="/tmp/twcycle-${BATCH_ID}-attempt-${SCAN_ATTEMPT}-queries.json"
|
|
1160
|
+
python3 -c "
|
|
1161
|
+
import json, sys
|
|
1162
|
+
text = sys.stdin.read().strip()
|
|
1163
|
+
try:
|
|
1164
|
+
env, _ = json.JSONDecoder().raw_decode(text)
|
|
1165
|
+
except Exception as e:
|
|
1166
|
+
print(f'lean phase 1: envelope parse error: {e}', file=sys.stderr)
|
|
1167
|
+
json.dump([], open('$QUERIES_TMP', 'w'))
|
|
1168
|
+
sys.exit(0)
|
|
1169
|
+
so = env.get('structured_output')
|
|
1170
|
+
if so is None:
|
|
1171
|
+
so = env.get('result')
|
|
1172
|
+
if isinstance(so, str):
|
|
1173
|
+
try: so = json.loads(so)
|
|
1174
|
+
except Exception: pass
|
|
1175
|
+
qs = so.get('queries', []) if isinstance(so, dict) else []
|
|
1176
|
+
json.dump(qs, open('$QUERIES_TMP', 'w'))
|
|
1177
|
+
print(f'lean phase 1: drafted {len(qs)} queries', flush=True)
|
|
1178
|
+
" <<< "$QUERIES_OUTPUT" 2>&1 | tee -a "$LOG_FILE"
|
|
1179
|
+
|
|
1180
|
+
else
|
|
1181
|
+
# === DETERMINISTIC QUALIFIED-QUERY-BANK PATH (default, 2026-05-28) ==========
|
|
1182
|
+
# No Claude call. Replay every historically qualified query for the picked
|
|
1183
|
+
# project(s): every distinct query that ever produced a posted reply with
|
|
1184
|
+
# >=1 like OR >=1 non-bot link click, regardless of the per-cycle
|
|
1185
|
+
# search_topic. This makes Phase 1 fully deterministic; the only remaining
|
|
1186
|
+
# Claude session in the cycle is Phase 2b (reply drafting). The bank is
|
|
1187
|
+
# exhaustive on attempt 1, so MAX_SCAN_ATTEMPTS is forced to 1 above; the
|
|
1188
|
+
# attempt>1 guard here is belt-and-suspenders for the legacy retry loop.
|
|
1189
|
+
QUERIES_TMP="/tmp/twcycle-${BATCH_ID}-attempt-${SCAN_ATTEMPT}-queries.json"
|
|
1190
|
+
if [ "$SCAN_ATTEMPT" -gt 1 ]; then
|
|
1191
|
+
echo "[]" > "$QUERIES_TMP"
|
|
1192
|
+
log "Phase 1 (bank): attempt $SCAN_ATTEMPT no-op (full bank already run on attempt 1)"
|
|
1193
|
+
else
|
|
1194
|
+
log "Phase 1 (bank): building qualified query bank from PROJECTS_JSON (deterministic, no Claude)..."
|
|
1195
|
+
echo "$PROJECTS_JSON" | python3 "$REPO_DIR/scripts/qualified_query_bank.py" --from-projects-json > "$QUERIES_TMP" 2>>"$LOG_FILE"
|
|
1196
|
+
fi
|
|
1197
|
+
fi
|
|
1198
|
+
|
|
1199
|
+
QUERIES_COUNT=$(python3 -c "
|
|
1200
|
+
import json
|
|
1201
|
+
try: print(len(json.load(open('$QUERIES_TMP'))))
|
|
1202
|
+
except Exception: print(0)
|
|
1203
|
+
" 2>/dev/null || echo 0)
|
|
1204
|
+
|
|
1205
|
+
# Loop: for each drafted query, run scan() over the same browser-harness daemon
|
|
1206
|
+
# the cycle already keeps alive (port 9555, BU_NAME=twitter-harness). One
|
|
1207
|
+
# browser-harness invocation handles the full loop so we don't pay the CLI
|
|
1208
|
+
# startup cost N times. Each scan() call appends one JSONL record to
|
|
1209
|
+
# $SCAN_TWEETS_FILE, which the existing shell-side parse below consumes.
|
|
1210
|
+
if [ "$QUERIES_COUNT" -gt 0 ]; then
|
|
1211
|
+
log "Lean Phase 1: executing $QUERIES_COUNT queries via browser-harness CDP"
|
|
1212
|
+
# browser-harness upstream main reads the script from STDIN (the `-c` flag was
|
|
1213
|
+
# removed). Feed the body via a quoted heredoc and pass $REPO_DIR / $QUERIES_TMP
|
|
1214
|
+
# through the environment so the Python reads them from os.environ (no shell
|
|
1215
|
+
# expansion inside the heredoc). Keep the local CLI in sync with upstream main:
|
|
1216
|
+
# `uv tool install -e ~/Developer/browser-harness --force` after a git pull.
|
|
1217
|
+
BU_NAME=twitter-harness BU_CDP_URL=http://127.0.0.1:9555 \
|
|
1218
|
+
SCAN_TWEETS_FILE="$SCAN_TWEETS_FILE" \
|
|
1219
|
+
BATCH_ID="$BATCH_ID" \
|
|
1220
|
+
TWITTER_CYCLE_VARIANT="$TWITTER_CYCLE_VARIANT" \
|
|
1221
|
+
FRESHNESS_HOURS_DISCOVER="$FRESHNESS_HOURS_DISCOVER" \
|
|
1222
|
+
ENGAGED_TWEET_IDS="$ENGAGED_TWEET_IDS" \
|
|
1223
|
+
REPO_DIR="$REPO_DIR" \
|
|
1224
|
+
QUERIES_TMP="$QUERIES_TMP" \
|
|
1225
|
+
"$HOME/.local/bin/browser-harness" <<'PY' 2>&1 | tee -a "$LOG_FILE"
|
|
1226
|
+
import sys, json, os, time
|
|
1227
|
+
sys.path.insert(0, os.environ['REPO_DIR'] + '/scripts')
|
|
1228
|
+
from twitter_scan import scan
|
|
1229
|
+
queries = json.load(open(os.environ['QUERIES_TMP']))
|
|
1230
|
+
freshness = int(os.environ.get('FRESHNESS_HOURS_DISCOVER', '6'))
|
|
1231
|
+
skip_ids = json.loads(os.environ.get('ENGAGED_TWEET_IDS', '[]'))
|
|
1232
|
+
for q in queries:
|
|
1233
|
+
project = q.get('project', '')
|
|
1234
|
+
query = q.get('query', '')
|
|
1235
|
+
topic = q.get('search_topic', '')
|
|
1236
|
+
t0 = time.time()
|
|
1237
|
+
try:
|
|
1238
|
+
kept = scan(
|
|
1239
|
+
query=query,
|
|
1240
|
+
project=project,
|
|
1241
|
+
search_topic=topic,
|
|
1242
|
+
freshness_hours=freshness,
|
|
1243
|
+
skip_ids=skip_ids,
|
|
1244
|
+
)
|
|
1245
|
+
dt = time.time() - t0
|
|
1246
|
+
print(f' ok project={project!r} q={query[:50]!r} kept={len(kept)} in {dt:.1f}s', flush=True)
|
|
1247
|
+
except Exception as e:
|
|
1248
|
+
dt = time.time() - t0
|
|
1249
|
+
print(f' err project={project!r} q={query[:50]!r} in {dt:.1f}s {type(e).__name__}: {e}', flush=True)
|
|
1250
|
+
PY
|
|
1251
|
+
fi
|
|
1252
|
+
rm -f "$QUERIES_TMP"
|
|
1253
|
+
|
|
1254
|
+
# Shell-side parse of $SCAN_TWEETS_FILE -> $RAW_FILE + $QUERIES_FILE. Identical
|
|
1255
|
+
# to the prior shell-side branch; the structured_output fallback is no longer
|
|
1256
|
+
# wired because the lean flow always produces SCAN_TWEETS_FILE (scan() writes
|
|
1257
|
+
# even on zero-tweet calls). If SCAN_TWEETS_FILE is missing entirely (e.g. the
|
|
1258
|
+
# Claude call returned no queries), write empty arrays so downstream scoring
|
|
1259
|
+
# treats this attempt as a zero-result Phase 1 and the retry loop fires.
|
|
1260
|
+
if [ -s "$SCAN_TWEETS_FILE" ]; then
|
|
1261
|
+
log "Parsing tweets from $SCAN_TWEETS_FILE"
|
|
1262
|
+
python3 -c "
|
|
1263
|
+
import json, sys
|
|
1264
|
+
recs = []
|
|
1265
|
+
for ln in open('$SCAN_TWEETS_FILE'):
|
|
1266
|
+
ln = ln.strip()
|
|
1267
|
+
if not ln:
|
|
1268
|
+
continue
|
|
1269
|
+
try:
|
|
1270
|
+
recs.append(json.loads(ln))
|
|
1271
|
+
except json.JSONDecodeError:
|
|
1272
|
+
print(f'shell-side: skipping bad JSONL line', file=sys.stderr)
|
|
1273
|
+
tweets = []
|
|
1274
|
+
queries_used = []
|
|
1275
|
+
for r in recs:
|
|
1276
|
+
ts = r.get('tweets') or []
|
|
1277
|
+
tweets.extend(ts)
|
|
1278
|
+
queries_used.append({
|
|
1279
|
+
'query': r.get('query', ''),
|
|
1280
|
+
'project': r.get('project', ''),
|
|
1281
|
+
'tweets_found': len(ts),
|
|
1282
|
+
'search_topic': r.get('search_topic', ''),
|
|
1283
|
+
})
|
|
1284
|
+
json.dump(queries_used, open('$QUERIES_FILE', 'w'))
|
|
1285
|
+
json.dump(tweets, open('$RAW_FILE', 'w'))
|
|
1286
|
+
print(f'shell-side parse: {len(tweets)} tweets, {len(queries_used)} attempts from SCAN_TWEETS_FILE', flush=True)
|
|
1287
|
+
sys.exit(0 if tweets else 1)
|
|
1288
|
+
" 2>&1 | tee -a "$LOG_FILE"
|
|
1289
|
+
EXTRACT_EXIT=${PIPESTATUS[0]:-1}
|
|
1290
|
+
else
|
|
1291
|
+
log "no SCAN_TWEETS_FILE this attempt (0 queries drafted or every scrape errored)"
|
|
1292
|
+
: > "$QUERIES_FILE"
|
|
1293
|
+
: > "$RAW_FILE"
|
|
1294
|
+
EXTRACT_EXIT=1
|
|
1295
|
+
fi
|
|
1296
|
+
# --- Discovery-stage counters ------------------------------------------------
|
|
1297
|
+
# Capture queries-run / duds / raw-tweets-pulled BEFORE any early-exit branch
|
|
1298
|
+
# so every log_run.py call below can pass --queries/--duds/--tweets-pulled.
|
|
1299
|
+
# QUERIES_FILE is the array Claude returned (one row per drafted query incl.
|
|
1300
|
+
# zero-result ones); RAW_FILE is the deduped tweet array. Use python3 inline so
|
|
1301
|
+
# we get the exact in-memory counts the rest of the pipeline operates on.
|
|
1302
|
+
QUERIES_TOTAL=0
|
|
1303
|
+
DUDS_TOTAL=0
|
|
1304
|
+
TWEETS_PULLED=0
|
|
1305
|
+
if [ -f "$QUERIES_FILE" ]; then
|
|
1306
|
+
QUERIES_TOTAL=$(python3 -c "
|
|
1307
|
+
import json, sys
|
|
1308
|
+
try:
|
|
1309
|
+
d = json.load(open(sys.argv[1]))
|
|
1310
|
+
print(len(d) if isinstance(d, list) else 0)
|
|
1311
|
+
except Exception:
|
|
1312
|
+
print(0)
|
|
1313
|
+
" "$QUERIES_FILE" 2>/dev/null || echo 0)
|
|
1314
|
+
DUDS_TOTAL=$(python3 -c "
|
|
1315
|
+
import json, sys
|
|
1316
|
+
try:
|
|
1317
|
+
d = json.load(open(sys.argv[1]))
|
|
1318
|
+
n = sum(1 for q in (d if isinstance(d, list) else []) if (q.get('tweets_found') or 0) == 0)
|
|
1319
|
+
print(n)
|
|
1320
|
+
except Exception:
|
|
1321
|
+
print(0)
|
|
1322
|
+
" "$QUERIES_FILE" 2>/dev/null || echo 0)
|
|
1323
|
+
fi
|
|
1324
|
+
if [ -f "$RAW_FILE" ]; then
|
|
1325
|
+
TWEETS_PULLED=$(python3 -c "
|
|
1326
|
+
import json, sys
|
|
1327
|
+
try:
|
|
1328
|
+
d = json.load(open(sys.argv[1]))
|
|
1329
|
+
print(len(d) if isinstance(d, list) else 0)
|
|
1330
|
+
except Exception:
|
|
1331
|
+
print(0)
|
|
1332
|
+
" "$RAW_FILE" 2>/dev/null || echo 0)
|
|
1333
|
+
fi
|
|
1334
|
+
|
|
1335
|
+
# Accumulate per-iteration counts into cycle-level totals for the post-loop
|
|
1336
|
+
# log_run.py call (otherwise the dashboard would show only the last attempt's
|
|
1337
|
+
# queries/duds/tweets-pulled, hiding the retry work).
|
|
1338
|
+
CUMULATIVE_QUERIES=$((CUMULATIVE_QUERIES + QUERIES_TOTAL))
|
|
1339
|
+
CUMULATIVE_DUDS=$((CUMULATIVE_DUDS + DUDS_TOTAL))
|
|
1340
|
+
CUMULATIVE_TWEETS_PULLED=$((CUMULATIVE_TWEETS_PULLED + TWEETS_PULLED))
|
|
1341
|
+
|
|
1342
|
+
# Snapshot this iteration's queries WITH per-query verdicts into
|
|
1343
|
+
# TRIED_QUERIES_JSON BEFORE log_twitter_search_attempts.py deletes QUERIES_FILE.
|
|
1344
|
+
# Verdicts come from joining QUERIES_FILE (drafted queries) with SCAN_TWEETS_FILE
|
|
1345
|
+
# (raw scrape per query record) and the BATCH_COUNT delta this attempt
|
|
1346
|
+
# (kept_after_skip approximation). kept_after_age comes from the optional
|
|
1347
|
+
# scorer sidecar at /tmp/twcycle-${BATCH_ID}-attempt-${SCAN_ATTEMPT}-scored.json
|
|
1348
|
+
# (written by score_twitter_candidates.py --scored-sidecar); when the sidecar
|
|
1349
|
+
# is missing we assume kept_after_age == raw_tweets, which collapses the
|
|
1350
|
+
# all_aged_out branch into dead_supply / found_some — still useful, just less
|
|
1351
|
+
# directional. Output entry shape:
|
|
1352
|
+
# {query, project, search_topic, raw_tweets, kept_after_age,
|
|
1353
|
+
# kept_after_skip, verdict}
|
|
1354
|
+
# verdict ∈ {dead_supply, all_aged_out, all_engaged_or_skipped, found_some}.
|
|
1355
|
+
if [ -f "$QUERIES_FILE" ]; then
|
|
1356
|
+
TRIED_QUERIES_JSON=$(python3 - \
|
|
1357
|
+
"$TRIED_QUERIES_JSON" "$QUERIES_FILE" "$SCAN_TWEETS_FILE" \
|
|
1358
|
+
"/tmp/twcycle-${BATCH_ID}-attempt-${SCAN_ATTEMPT}-scored.json" \
|
|
1359
|
+
"$BATCH_COUNT_BEFORE_ATTEMPT" "$BATCH_COUNT" <<'PY' 2>/dev/null || echo "$TRIED_QUERIES_JSON"
|
|
1360
|
+
import json, os, sys
|
|
1361
|
+
from collections import Counter
|
|
1362
|
+
|
|
1363
|
+
cur = json.loads(sys.argv[1] or '[]')
|
|
1364
|
+
queries_path = sys.argv[2]
|
|
1365
|
+
scan_path = sys.argv[3]
|
|
1366
|
+
scored_path = sys.argv[4]
|
|
1367
|
+
try:
|
|
1368
|
+
pre = int(sys.argv[5] or 0)
|
|
1369
|
+
except Exception:
|
|
1370
|
+
pre = 0
|
|
1371
|
+
try:
|
|
1372
|
+
post = int(sys.argv[6] or 0)
|
|
1373
|
+
except Exception:
|
|
1374
|
+
post = 0
|
|
1375
|
+
|
|
1376
|
+
try:
|
|
1377
|
+
new = json.load(open(queries_path))
|
|
1378
|
+
if not isinstance(new, list):
|
|
1379
|
+
new = []
|
|
1380
|
+
except Exception:
|
|
1381
|
+
new = []
|
|
1382
|
+
|
|
1383
|
+
# raw_tweets per query from SCAN_TWEETS_FILE (one JSONL record per scan call).
|
|
1384
|
+
# Multiple records can share a query if the harness retried; we sum.
|
|
1385
|
+
raw_by_query = Counter()
|
|
1386
|
+
if scan_path and os.path.exists(scan_path):
|
|
1387
|
+
with open(scan_path) as fh:
|
|
1388
|
+
for line in fh:
|
|
1389
|
+
try:
|
|
1390
|
+
rec = json.loads(line)
|
|
1391
|
+
except Exception:
|
|
1392
|
+
continue
|
|
1393
|
+
q = (rec.get('query') or '').strip()
|
|
1394
|
+
n = len(rec.get('tweets') or [])
|
|
1395
|
+
if q:
|
|
1396
|
+
raw_by_query[q] += n
|
|
1397
|
+
|
|
1398
|
+
# kept_after_age per query from the scorer sidecar (optional). Falls back to
|
|
1399
|
+
# raw_tweets when the sidecar is absent.
|
|
1400
|
+
age_by_query = {}
|
|
1401
|
+
if scored_path and os.path.exists(scored_path):
|
|
1402
|
+
try:
|
|
1403
|
+
scored = json.load(open(scored_path))
|
|
1404
|
+
for q, counts in (scored or {}).items():
|
|
1405
|
+
age_by_query[(q or '').strip()] = int(counts.get('kept_after_age') or 0)
|
|
1406
|
+
except Exception:
|
|
1407
|
+
pass
|
|
1408
|
+
|
|
1409
|
+
# kept_after_skip is the cycle-level delta this attempt. The scorer doesn't
|
|
1410
|
+
# tag the per-tweet survivor with its source query upstream, so we split the
|
|
1411
|
+
# delta evenly across queries that actually returned raw tweets. We mostly
|
|
1412
|
+
# care about zero vs nonzero per query, not the exact split.
|
|
1413
|
+
delta = max(0, post - pre)
|
|
1414
|
+
queries_with_raw = [e for e in new if raw_by_query.get((e.get('query') or '').strip(), 0) > 0]
|
|
1415
|
+
share = (delta / max(1, len(queries_with_raw))) if queries_with_raw else 0
|
|
1416
|
+
|
|
1417
|
+
for entry in new:
|
|
1418
|
+
q = (entry.get('query') or '').strip()
|
|
1419
|
+
raw = raw_by_query.get(q, 0)
|
|
1420
|
+
# When sidecar present, trust it; else assume freshness gate passed all raw.
|
|
1421
|
+
kept_age = age_by_query[q] if q in age_by_query else raw
|
|
1422
|
+
kept_skip = int(round(share)) if raw > 0 else 0
|
|
1423
|
+
if raw == 0:
|
|
1424
|
+
verdict = 'dead_supply'
|
|
1425
|
+
elif kept_age == 0:
|
|
1426
|
+
verdict = 'all_aged_out'
|
|
1427
|
+
elif kept_skip == 0:
|
|
1428
|
+
verdict = 'all_engaged_or_skipped'
|
|
1429
|
+
else:
|
|
1430
|
+
verdict = 'found_some'
|
|
1431
|
+
entry['raw_tweets'] = raw
|
|
1432
|
+
entry['kept_after_age'] = kept_age
|
|
1433
|
+
entry['kept_after_skip'] = kept_skip
|
|
1434
|
+
entry['verdict'] = verdict
|
|
1435
|
+
|
|
1436
|
+
cur.extend(new)
|
|
1437
|
+
print(json.dumps(cur))
|
|
1438
|
+
PY
|
|
1439
|
+
)
|
|
1440
|
+
fi
|
|
1441
|
+
|
|
1442
|
+
# Log every drafted query (incl. zero-result ones) to twitter_search_attempts
|
|
1443
|
+
# BEFORE any early-exit branches. Runs even when the tweets array is empty
|
|
1444
|
+
# so dud queries actually accumulate in the negative-signal table.
|
|
1445
|
+
if [ -f "$QUERIES_FILE" ]; then
|
|
1446
|
+
python3 "$REPO_DIR/scripts/log_twitter_search_attempts.py" --batch-id "$BATCH_ID" \
|
|
1447
|
+
--attempts-out "$ATTEMPTS_FILE" \
|
|
1448
|
+
< "$QUERIES_FILE" 2>&1 | tee -a "$LOG_FILE"
|
|
1449
|
+
rm -f "$QUERIES_FILE"
|
|
1450
|
+
fi
|
|
1451
|
+
|
|
1452
|
+
# Stamp last_used_at on every active project-wide exclude we surfaced to
|
|
1453
|
+
# Claude this cycle. These are the terms Claude was REQUIRED to append as
|
|
1454
|
+
# `-term` to its drafted queries, so even if Claude omits one, the term is
|
|
1455
|
+
# still considered "in use" for decay purposes — drafter compliance is its
|
|
1456
|
+
# own problem, not a reason to prune a learned exclude. Done after the
|
|
1457
|
+
# search_attempts log so a Phase 1 abort still leaves the marks behind.
|
|
1458
|
+
python3 - "$PROJECTS_JSON" <<'PY' 2>&1 | tee -a "$LOG_FILE" || true
|
|
1459
|
+
import json, os, sys
|
|
1460
|
+
sys.path.insert(0, os.path.expanduser('~/social-autoposter/scripts'))
|
|
1461
|
+
import project_excludes as pe
|
|
1462
|
+
projects = json.loads(sys.argv[1] or '[]')
|
|
1463
|
+
total = 0
|
|
1464
|
+
for p in projects:
|
|
1465
|
+
terms = p.get('excludes_for_search') or []
|
|
1466
|
+
if not terms:
|
|
1467
|
+
continue
|
|
1468
|
+
try:
|
|
1469
|
+
n = pe.mark_used('twitter', p.get('name'), terms)
|
|
1470
|
+
except Exception as exc:
|
|
1471
|
+
print(f"mark_used error for {p.get('name')}: {exc}", file=sys.stderr)
|
|
1472
|
+
continue
|
|
1473
|
+
total += n
|
|
1474
|
+
if total:
|
|
1475
|
+
print(f"project_excludes: marked {total} term(s) used across selected projects")
|
|
1476
|
+
PY
|
|
1477
|
+
if [ "$EXTRACT_EXIT" -ne 0 ] || [ ! -f "$RAW_FILE" ]; then
|
|
1478
|
+
# Claude returned no usable tweet array this attempt. Could be a real
|
|
1479
|
+
# Anthropic error (stream_idle_timeout, api_overloaded, monthly_limit,
|
|
1480
|
+
# context_overflow) or just "model found nothing relevant". Classify
|
|
1481
|
+
# the failure for the post-loop log_run summary; the loop control below
|
|
1482
|
+
# decides whether to retry or give up.
|
|
1483
|
+
# SCAN_OUTPUT was a stale leftover from the pre-lean design (when the scan's
|
|
1484
|
+
# stdout was captured into a shell var); the lean Phase 1 loop now tees its
|
|
1485
|
+
# output to $LOG_FILE instead, so an empty-scan attempt hit `set -u` and
|
|
1486
|
+
# aborted the whole cycle here. Feed the classifier the recent log tail (the
|
|
1487
|
+
# actual scan output, where harness/Anthropic error signatures land) so we
|
|
1488
|
+
# still distinguish a real error from "found nothing relevant".
|
|
1489
|
+
SCAN_OUTPUT=$(tail -n 80 "$LOG_FILE" 2>/dev/null || true)
|
|
1490
|
+
PHASE1_REASON_LATEST=$(echo "$SCAN_OUTPUT" | python3 "$REPO_DIR/scripts/classify_run_error.py" 2>/dev/null)
|
|
1491
|
+
[ -z "$PHASE1_REASON_LATEST" ] && PHASE1_REASON_LATEST="phase1_no_tweets"
|
|
1492
|
+
LAST_PHASE1_REASON="$PHASE1_REASON_LATEST"
|
|
1493
|
+
log " Phase 1 attempt $SCAN_ATTEMPT returned no tweets (reason=$PHASE1_REASON_LATEST); falling through to loop control"
|
|
1494
|
+
else
|
|
1495
|
+
# --- Phase 1 finalize: enrich + score with T0 + batch_id ----------------
|
|
1496
|
+
log "Enriching via fxtwitter + scoring with T0 snapshot (batch=$BATCH_ID, attempt=$SCAN_ATTEMPT)..."
|
|
1497
|
+
cat "$RAW_FILE" \
|
|
1498
|
+
| python3 "$REPO_DIR/scripts/enrich_twitter_candidates.py" \
|
|
1499
|
+
| python3 "$REPO_DIR/scripts/score_twitter_candidates.py" --batch-id "$BATCH_ID" \
|
|
1500
|
+
${ATTEMPTS_FILE:+--attempts "$ATTEMPTS_FILE"} \
|
|
1501
|
+
--scored-sidecar "/tmp/twcycle-${BATCH_ID}-attempt-${SCAN_ATTEMPT}-scored.json" \
|
|
1502
|
+
2>&1 | tee -a "$LOG_FILE"
|
|
1503
|
+
rm -f "$RAW_FILE" "$ATTEMPTS_FILE"
|
|
1504
|
+
fi
|
|
1505
|
+
|
|
1506
|
+
BATCH_COUNT=$(python3 "$REPO_DIR/scripts/twitter_cycle_helper.py" batch-count --batch-id "$BATCH_ID" 2>/dev/null || echo 0)
|
|
1507
|
+
log "Phase 1 attempt $SCAN_ATTEMPT complete. Batch has $BATCH_COUNT/$RETRY_TARGET candidates with T0 snapshot."
|
|
1508
|
+
|
|
1509
|
+
# --- Retry-loop control ------------------------------------------------------
|
|
1510
|
+
# Break out if we hit the target; else either retry or give up at the cap.
|
|
1511
|
+
if [ "$BATCH_COUNT" -ge "$RETRY_TARGET" ]; then
|
|
1512
|
+
log " Reached target ($BATCH_COUNT >= $RETRY_TARGET) after $SCAN_ATTEMPT scan(s); proceeding to Phase 2"
|
|
1513
|
+
break
|
|
1514
|
+
fi
|
|
1515
|
+
if [ "$SCAN_ATTEMPT" -ge "$MAX_SCAN_ATTEMPTS" ]; then
|
|
1516
|
+
log " Hit scan cap ($MAX_SCAN_ATTEMPTS); proceeding with $BATCH_COUNT candidate(s)"
|
|
1517
|
+
break
|
|
1518
|
+
fi
|
|
1519
|
+
_TRIED_N=$(echo "$TRIED_QUERIES_JSON" | python3 -c 'import json,sys;print(len(json.load(sys.stdin)))' 2>/dev/null || echo 0)
|
|
1520
|
+
log " Below target ($BATCH_COUNT/$RETRY_TARGET); $_TRIED_N queries tried so far this cycle; looping for attempt $((SCAN_ATTEMPT + 1))..."
|
|
1521
|
+
done
|
|
1522
|
+
|
|
1523
|
+
# --- Post-loop bookkeeping ---------------------------------------------------
|
|
1524
|
+
# Stamp cycle_variant='D' onto every candidate in this batch. The A/B/C/D
|
|
1525
|
+
# experiment concluded 2026-05-31 (D won); this is now a constant label kept so
|
|
1526
|
+
# downstream analytics (post-rate, thread-age-at-discover, lag-after-thread,
|
|
1527
|
+
# top-reply ratio) stay continuous with the historical experiment rows.
|
|
1528
|
+
# Idempotent: same value would be written if the batch is salvaged into a peer
|
|
1529
|
+
# cycle.
|
|
1530
|
+
# HTTP-only (2026-06-01): the cycle_variant stamp routes through
|
|
1531
|
+
# /api/v1/twitter-candidates/stamp-cycle-variant via twitter_cycle_helper.py.
|
|
1532
|
+
# No DATABASE_URL, no psycopg, no fallback. Idempotent: only NULL rows touched.
|
|
1533
|
+
python3 "$REPO_DIR/scripts/twitter_cycle_helper.py" stamp-cycle-variant \
|
|
1534
|
+
--batch-id "$BATCH_ID" --variant "$TWITTER_CYCLE_VARIANT" \
|
|
1535
|
+
>/dev/null 2>>"$LOG_FILE" || log "Phase 1: cycle_variant stamp failed (non-fatal)"
|
|
1536
|
+
|
|
1537
|
+
# Promote cumulative totals onto the per-iteration names so every downstream
|
|
1538
|
+
# log_run.py / trap handler picks up the cycle-level work (not just the last
|
|
1539
|
+
# attempt's counts). Keeps all the existing "${QUERIES_TOTAL:-0}" etc. call
|
|
1540
|
+
# sites correct without touching them individually.
|
|
1541
|
+
QUERIES_TOTAL="$CUMULATIVE_QUERIES"
|
|
1542
|
+
DUDS_TOTAL="$CUMULATIVE_DUDS"
|
|
1543
|
+
TWEETS_PULLED="$CUMULATIVE_TWEETS_PULLED"
|
|
1544
|
+
|
|
1545
|
+
log "Phase 1 complete after $SCAN_ATTEMPT scan attempt(s). Final batch has $BATCH_COUNT candidates with T0 snapshot."
|
|
1546
|
+
|
|
1547
|
+
if [ "$BATCH_COUNT" = "0" ]; then
|
|
1548
|
+
# Distinguish "Claude returned no tweets at all" from "Claude returned
|
|
1549
|
+
# tweets but enrichment dropped them all" from "we exhausted the topic
|
|
1550
|
+
# universe mid-retry" so the dashboard can surface the right failure
|
|
1551
|
+
# mode. Priority order: universe_exhausted (the picker said stop) >
|
|
1552
|
+
# Anthropic-side classified error > generic empty_batch.
|
|
1553
|
+
if [ "${UNIVERSE_EXHAUSTED:-0}" = "1" ]; then
|
|
1554
|
+
_FAILURE_REASON="universe_exhausted:1"
|
|
1555
|
+
elif [ -n "$LAST_PHASE1_REASON" ] && [ "$CUMULATIVE_TWEETS_PULLED" = "0" ]; then
|
|
1556
|
+
_FAILURE_REASON="${LAST_PHASE1_REASON}:1"
|
|
1557
|
+
else
|
|
1558
|
+
_FAILURE_REASON="empty_batch:1"
|
|
1559
|
+
fi
|
|
1560
|
+
log "Empty batch after $SCAN_ATTEMPT attempt(s) (reason=$_FAILURE_REASON). Nothing to re-score. Exiting."
|
|
1561
|
+
_COST=$(python3 "$REPO_DIR/scripts/get_run_cost.py" --cycle-id "$BATCH_ID" 2>/dev/null || echo "0.0000")
|
|
1562
|
+
python3 "$REPO_DIR/scripts/log_run.py" --script "post_twitter" --posted 0 --skipped 0 --failed 1 \
|
|
1563
|
+
--salvaged "${SALVAGED:-0}" \
|
|
1564
|
+
--queries "${CUMULATIVE_QUERIES:-0}" --duds "${CUMULATIVE_DUDS:-0}" \
|
|
1565
|
+
--tweets-pulled "${CUMULATIVE_TWEETS_PULLED:-0}" \
|
|
1566
|
+
--failure-reasons "$_FAILURE_REASON" \
|
|
1567
|
+
--cost "$_COST" --elapsed $(( $(date +%s) - RUN_START ))
|
|
1568
|
+
_SA_RUN_SUMMARY_EMITTED=1
|
|
1569
|
+
exit 0
|
|
1570
|
+
fi
|
|
1571
|
+
|
|
1572
|
+
# Stamp phase2a before releasing the lock so the salvage budget covers the
|
|
1573
|
+
# browser-lock handoff window (phase2a budget = 20 min).
|
|
1574
|
+
python3 "$REPO_DIR/scripts/twitter_batch_phase.py" advance "$BATCH_ID" --phase phase2a 2>&1 | tee -a "$LOG_FILE" || true
|
|
1575
|
+
|
|
1576
|
+
# Release the twitter-browser lock between Phase 1 scrape and Phase 2b posting.
|
|
1577
|
+
# Other pipelines (engage-twitter, dm-outreach-twitter, link-edit-twitter,
|
|
1578
|
+
# stats.sh) can run their browser steps in this window instead of waiting for us
|
|
1579
|
+
# to finish. We re-acquire just before Phase 2b posts, blocking up to the
|
|
1580
|
+
# acquire_lock timeout if another pipeline is mid-run.
|
|
1581
|
+
log "Releasing twitter-browser lock between Phase 1 scrape and Phase 2b posting..."
|
|
1582
|
+
release_lock "twitter-browser" 2>>"$LOG_FILE"
|
|
1583
|
+
# (2026-06-16) NO `rm -f twitter-browser-lock.json` here. The blind rm was
|
|
1584
|
+
# ownership-unaware and ran AFTER release_lock, so under a pipeline handoff it
|
|
1585
|
+
# deleted a LIVE peer's session mutex (defect b) -> two browser ops on one X
|
|
1586
|
+
# tab. Dead python:PID holders are now reclaimed by _acquire_browser_lock in
|
|
1587
|
+
# scripts/twitter_browser.py (os.kill liveness), so the workaround is obsolete
|
|
1588
|
+
# AND unsafe. Do NOT re-add it. See docs/twitter_browser_lock.md.
|
|
1589
|
+
|
|
1590
|
+
# --- No ripen wait (winning variant D) --------------------------------------
|
|
1591
|
+
# The 20-min ripen sleep + fetch_twitter_t1 re-measurement was removed when
|
|
1592
|
+
# variant D won the A/B/C/D test (2026-05-31). The wait was originally a
|
|
1593
|
+
# velocity gate; the gate floor was removed 2026-05-15 so it only fed
|
|
1594
|
+
# delta_score into the LLM prompt, and the experiment showed eliminating that
|
|
1595
|
+
# ~20 min thread->post lag improves engagement more than delta_score helps the
|
|
1596
|
+
# draft. We go straight from candidate discovery to Phase 2b; delta_score stays
|
|
1597
|
+
# at its T0 value.
|
|
1598
|
+
log "No ripen wait (logic D): skipping sleep + T1 fetch, delta_score stays at T0 value"
|
|
1599
|
+
|
|
1600
|
+
# --- Phase 2b: top 25 by virality_score, no post cap ---------------------
|
|
1601
|
+
# Sort key (2026-05-27): virality_score DESC. This is the composite predictor
|
|
1602
|
+
# stamped at discovery by score_twitter_candidates.py:
|
|
1603
|
+
# virality_score = velocity * reach_mult * age_decay * rt_bonus
|
|
1604
|
+
# * (1 + reply_bonus) * (1 + discussion_bonus)
|
|
1605
|
+
# It folds in engagement velocity, author reach (follower-tier multiplier),
|
|
1606
|
+
# age decay (6h half-life), retweet ratio, reply count, and discussion
|
|
1607
|
+
# quality (reply:like ratio). Cohort analysis on 30d posted data: the
|
|
1608
|
+
# [10k+) virality bucket gets ~36x the median reply views of the [0-10)
|
|
1609
|
+
# bucket, which is much steeper than what raw 5-min delta predicts.
|
|
1610
|
+
# Replaces the prior `delta + flat-5 intent-regex boost` sort: the intent
|
|
1611
|
+
# regex was a crutch for delta_score (a raw growth count that ignored
|
|
1612
|
+
# reach + decay); the model reads tweet text directly in the prep prompt
|
|
1613
|
+
# and detects intent itself, so the lexical layer is redundant.
|
|
1614
|
+
# 2026-05-15: ripening floor removed entirely (was `delta_score >= 0`).
|
|
1615
|
+
# The model already sees per-candidate Virality + Delta in CANDIDATE_BLOCK
|
|
1616
|
+
# below and can weigh velocity against topical fit itself. Letting
|
|
1617
|
+
# negative-delta tweets through means a thoughtful comment can still ride
|
|
1618
|
+
# an on-theme but cooling thread to the right audience. LIMIT 25 stays as
|
|
1619
|
+
# a draft-budget cap, not a ripening gate.
|
|
1620
|
+
# Candidate list comes through /api/v1/twitter-candidates (route returns
|
|
1621
|
+
# all pending rows for the batch); the helper applies the virality_score
|
|
1622
|
+
# sort + 25-row cap client-side and emits the SAME pipe-separated columns
|
|
1623
|
+
# the legacy psql -F '|' query produced. Pipe shape is documented in
|
|
1624
|
+
# scripts/twitter_cycle_helper.py:cmd_candidates.
|
|
1625
|
+
CANDIDATES=$(python3 "$REPO_DIR/scripts/twitter_cycle_helper.py" candidates --batch-id "$BATCH_ID" 2>/dev/null || echo "")
|
|
1626
|
+
|
|
1627
|
+
if [ -z "$CANDIDATES" ]; then
|
|
1628
|
+
log "No candidates with delta scores. Marking batch expired."
|
|
1629
|
+
# /api/v1/twitter-candidates/expire-batch performs the same status-flip
|
|
1630
|
+
# UPDATE atomically and prints the resulting expired_count integer that
|
|
1631
|
+
# the EXPIRED_BATCH variable previously got from a second COUNT(*) query.
|
|
1632
|
+
EXPIRED_BATCH=$(python3 "$REPO_DIR/scripts/twitter_cycle_helper.py" expire-batch --batch-id "$BATCH_ID" 2>/dev/null || echo 0)
|
|
1633
|
+
_COST=$(python3 "$REPO_DIR/scripts/get_run_cost.py" --cycle-id "$BATCH_ID" 2>/dev/null || echo "0.0000")
|
|
1634
|
+
# Not a hard error — batch had candidates but none remained 'pending' after
|
|
1635
|
+
# Phase 2a (typically: every row already flipped to posted/skipped/expired
|
|
1636
|
+
# by an earlier salvage pass). With the ripening floor removed (2026-05-15),
|
|
1637
|
+
# this no longer fires on low-delta rows; only on empty/exhausted batches.
|
|
1638
|
+
# Report as skipped (not failed) so the row reads "skipped: N" rather than
|
|
1639
|
+
# the silent "—" we used to render. failure_reasons stays empty.
|
|
1640
|
+
python3 "$REPO_DIR/scripts/log_run.py" --script "post_twitter" --posted 0 --skipped "${EXPIRED_BATCH:-0}" --failed 0 \
|
|
1641
|
+
--salvaged "${SALVAGED:-0}" \
|
|
1642
|
+
--queries "${QUERIES_TOTAL:-0}" --duds "${DUDS_TOTAL:-0}" \
|
|
1643
|
+
--tweets-pulled "${TWEETS_PULLED:-0}" --candidates "${BATCH_COUNT:-0}" \
|
|
1644
|
+
--cost "$_COST" --elapsed $(( $(date +%s) - RUN_START ))
|
|
1645
|
+
_SA_RUN_SUMMARY_EMITTED=1
|
|
1646
|
+
exit 0
|
|
1647
|
+
fi
|
|
1648
|
+
|
|
1649
|
+
# --- SCAN_ONLY gate: stop after scoring, emit candidates, skip drafting -------
|
|
1650
|
+
# When SCAN_ONLY=1 the cycle runs scan -> score -> top-N select, writes the chosen
|
|
1651
|
+
# candidates as JSON, and STOPS before the claude drafting step. The MCP
|
|
1652
|
+
# scan_candidates tool reads this so a Claude Desktop scheduled-task session can do
|
|
1653
|
+
# the drafting ITSELF (on the user's plan, no `claude -p`) and hand the drafts back
|
|
1654
|
+
# via submit_drafts. Candidates stay 'pending' (drafted+posted via submit_drafts ->
|
|
1655
|
+
# post_drafts, or salvaged by a later cycle). The browser lock was already released
|
|
1656
|
+
# at the Phase 1 handoff, so this exits clean via the EXIT trap. NO current caller
|
|
1657
|
+
# sets SCAN_ONLY, so the autopilot/draft_cycle paths are byte-for-byte unchanged.
|
|
1658
|
+
if [ "${SCAN_ONLY:-0}" = "1" ]; then
|
|
1659
|
+
SCAN_FILE="/tmp/saps_scan_candidates_${BATCH_ID}.json"
|
|
1660
|
+
# $CANDIDATES is the same pipe-separated top-N the drafting step consumes (cols
|
|
1661
|
+
# documented in twitter_cycle_helper.py:cmd_candidates; tweet_text/draft fields
|
|
1662
|
+
# are pipe+newline sanitized there, so a field split is safe). Batch id + out
|
|
1663
|
+
# path travel via env so the single-quoted python needs no shell interpolation.
|
|
1664
|
+
printf '%s\n' "$CANDIDATES" | S4L_SCAN_FILE="$SCAN_FILE" S4L_SCAN_BATCH="$BATCH_ID" python3 -c '
|
|
1665
|
+
import json, os, sys
|
|
1666
|
+
def _i(x):
|
|
1667
|
+
try:
|
|
1668
|
+
return int(float(x or 0))
|
|
1669
|
+
except Exception:
|
|
1670
|
+
return 0
|
|
1671
|
+
def _f(x):
|
|
1672
|
+
try:
|
|
1673
|
+
return float(x or 0)
|
|
1674
|
+
except Exception:
|
|
1675
|
+
return 0.0
|
|
1676
|
+
out = []
|
|
1677
|
+
for line in sys.stdin:
|
|
1678
|
+
line = line.rstrip("\n")
|
|
1679
|
+
if not line.strip():
|
|
1680
|
+
continue
|
|
1681
|
+
p = line.split("|")
|
|
1682
|
+
if len(p) < 14 or not p[0].isdigit():
|
|
1683
|
+
continue
|
|
1684
|
+
out.append({
|
|
1685
|
+
"id": int(p[0]), "tweet_url": p[1], "author_handle": p[2], "tweet_text": p[3],
|
|
1686
|
+
"virality_score": _f(p[4]), "delta_score": _f(p[5]), "matched_project": p[6],
|
|
1687
|
+
"search_topic": p[7], "likes": _i(p[8]), "retweets": _i(p[9]), "replies": _i(p[10]),
|
|
1688
|
+
"views": _i(p[11]), "author_followers": _i(p[12]), "age_hours": _f(p[13]),
|
|
1689
|
+
"existing_draft": p[14] if len(p) > 14 else "", "existing_draft_style": p[15] if len(p) > 15 else "",
|
|
1690
|
+
})
|
|
1691
|
+
json.dump({"batch_id": os.environ["S4L_SCAN_BATCH"], "candidates": out}, open(os.environ["S4L_SCAN_FILE"], "w"))
|
|
1692
|
+
' 2>/dev/null || printf '{"batch_id": "%s", "candidates": []}' "$BATCH_ID" > "$SCAN_FILE"
|
|
1693
|
+
SCAN_N=$(python3 -c "import json; print(len(json.load(open('$SCAN_FILE')).get('candidates') or []))" 2>/dev/null || echo 0)
|
|
1694
|
+
log "SCAN_ONLY=1: $SCAN_N candidate(s) scored and written to $SCAN_FILE. Stopping before drafting (agent drafts next)."
|
|
1695
|
+
_SA_RUN_SUMMARY_EMITTED=1
|
|
1696
|
+
echo "SCAN_ONLY_RESULT=$SCAN_FILE"
|
|
1697
|
+
exit 0
|
|
1698
|
+
fi
|
|
1699
|
+
|
|
1700
|
+
CANDIDATE_COUNT=$(printf '%s\n' "$CANDIDATES" | grep -c '^[0-9]')
|
|
1701
|
+
log "Top $CANDIDATE_COUNT candidates by virality_score selected for post review."
|
|
1702
|
+
|
|
1703
|
+
# No post cap: Phase 2b-prep posts every candidate it judges genuinely
|
|
1704
|
+
# on-brand. HIGH_DELTA_COUNT is still computed, but ONLY as a dashboard
|
|
1705
|
+
# diagnostic (the "Δ≥10 N" stat, fed to log_run.py --above-floor). It no
|
|
1706
|
+
# longer gates how many replies the cycle is allowed to post.
|
|
1707
|
+
HIGH_DELTA_COUNT=$(printf '%s\n' "$CANDIDATES" | awk -F'|' '$1 ~ /^[0-9]+$/ && $6+0 >= 10 {n++} END {print n+0}')
|
|
1708
|
+
log "Candidates with Δ≥10 (momentum diagnostic only, not a cap): $HIGH_DELTA_COUNT"
|
|
1709
|
+
|
|
1710
|
+
CANDIDATE_BLOCK=""
|
|
1711
|
+
# Thread-media capture (2026-06-03): collect each candidate's id|url so that,
|
|
1712
|
+
# AFTER the browser lock is acquired, we can deterministically pre-fetch the
|
|
1713
|
+
# media (images/videos/GIFs/link-cards) of every thread the model is about to
|
|
1714
|
+
# draft against and feed it into the prep prompt. Gated by
|
|
1715
|
+
# S4L_TWITTER_CAPTURE_MEDIA so it stays a no-op until the website API (with the
|
|
1716
|
+
# set_media action + thread_media column) deploys. Populated in the loop below.
|
|
1717
|
+
MEDIA_URLS_FILE=$(mktemp -t saps_twitter_media_urls_XXXXXX.tsv)
|
|
1718
|
+
while IFS='|' read -r cid curl cauthor ctext cscore cdelta cproject ctopic clikes crts creplies cviews cfollowers cage cdraft cdraftstyle cdraftage; do
|
|
1719
|
+
if [ -n "$cid" ] && [ -n "$curl" ]; then
|
|
1720
|
+
printf '%s\t%s\n' "$cid" "$curl" >> "$MEDIA_URLS_FILE"
|
|
1721
|
+
fi
|
|
1722
|
+
DRAFT_LINE=""
|
|
1723
|
+
if [ -n "$cdraft" ] && [ "$cdraftage" != "-1" ]; then
|
|
1724
|
+
# Round draft age to whole minutes for the prompt.
|
|
1725
|
+
DRAFT_MIN=$(printf '%.0f' "$cdraftage")
|
|
1726
|
+
DRAFT_LINE="
|
|
1727
|
+
EXISTING DRAFT (style=$cdraftstyle, age=${DRAFT_MIN}m): $cdraft"
|
|
1728
|
+
fi
|
|
1729
|
+
# Per-candidate prior-interaction context: surface our last 5 comments to
|
|
1730
|
+
# this author in the past 30 days (soft context only — vary angle, don't
|
|
1731
|
+
# repeat phrasing). Empty when we have no history. Failure is silent.
|
|
1732
|
+
AUTHOR_HISTORY_LINE=""
|
|
1733
|
+
if [ -n "$cauthor" ]; then
|
|
1734
|
+
_AH=$(python3 "$REPO_DIR/scripts/author_history_block.py" --platform twitter --author "$cauthor" --days 30 --limit 5 2>>"$LOG_FILE" || true)
|
|
1735
|
+
if [ -n "$_AH" ]; then
|
|
1736
|
+
AUTHOR_HISTORY_LINE="
|
|
1737
|
+
$_AH"
|
|
1738
|
+
fi
|
|
1739
|
+
fi
|
|
1740
|
+
CANDIDATE_BLOCK="${CANDIDATE_BLOCK}
|
|
1741
|
+
---
|
|
1742
|
+
Candidate ID: $cid
|
|
1743
|
+
URL: $curl
|
|
1744
|
+
Author: @$cauthor (${cfollowers} followers)
|
|
1745
|
+
Text: $ctext
|
|
1746
|
+
Virality: $cscore | Delta (5min): $cdelta | Likes: $clikes | RTs: $crts | Replies: $creplies | Views: $cviews | Age: ${cage}h
|
|
1747
|
+
Search query: $ctopic
|
|
1748
|
+
Project match: $cproject${DRAFT_LINE}${AUTHOR_HISTORY_LINE}
|
|
1749
|
+
"
|
|
1750
|
+
done <<< "$CANDIDATES"
|
|
1751
|
+
|
|
1752
|
+
ALL_PROJECTS_JSON=$(python3 -c "
|
|
1753
|
+
import json, os
|
|
1754
|
+
config = json.load(open(os.path.expanduser('~/social-autoposter/config.json')))
|
|
1755
|
+
projects = config.get('projects', [])
|
|
1756
|
+
lane = os.environ.get('S4L_ACTIVE_LANE', '')
|
|
1757
|
+
if lane == 'personal_brand':
|
|
1758
|
+
# Personal-brand lane is pure organic growth: the drafter must NOT see any
|
|
1759
|
+
# product config at all (no website, links, booking_link, get_started_link,
|
|
1760
|
+
# features, pricing, CTAs). We emit ONLY the persona project, and ONLY the
|
|
1761
|
+
# drafting-relevant fields, so there is literally no product context in the
|
|
1762
|
+
# prompt to accidentally pitch, quote, or link. This also kills cross-routing
|
|
1763
|
+
# (no 'other project' exists to route a candidate to). Whitelist, not
|
|
1764
|
+
# denylist: any field added to the persona entry later stays out unless
|
|
1765
|
+
# explicitly allowed here.
|
|
1766
|
+
ALLOWED = {
|
|
1767
|
+
'name', 'description', 'content_angle', 'voice',
|
|
1768
|
+
'voice_relationship', 'content_guardrails',
|
|
1769
|
+
}
|
|
1770
|
+
persona = next((p for p in projects if p.get('persona') is True), None)
|
|
1771
|
+
out = {}
|
|
1772
|
+
if persona:
|
|
1773
|
+
out[persona['name']] = {k: v for k, v in persona.items() if k in ALLOWED}
|
|
1774
|
+
print(json.dumps(out, indent=2))
|
|
1775
|
+
else:
|
|
1776
|
+
print(json.dumps({p['name']: p for p in projects}, indent=2))
|
|
1777
|
+
" 2>/dev/null || echo "{}")
|
|
1778
|
+
|
|
1779
|
+
# Engagement-style picker (2026-05-19): pick ONE assigned style for this
|
|
1780
|
+
# cycle. The picked style flows two places: (1) --style filter for
|
|
1781
|
+
# top_performers.py so the per-style exemplars section shows only posts
|
|
1782
|
+
# matching the assigned style, (2) saps_render_style_block (below) so the
|
|
1783
|
+
# prompt block embeds the same assignment. On invent mode picked_style is
|
|
1784
|
+
# empty and top_performers stays unfiltered (model sees full landscape).
|
|
1785
|
+
source "$REPO_DIR/skill/styles.sh"
|
|
1786
|
+
STYLE_ASSIGN_FILE=$(mktemp -t saps_twitter_assign_XXXXXX.json)
|
|
1787
|
+
saps_pick_style twitter posting "$STYLE_ASSIGN_FILE" >/dev/null 2>&1 || true
|
|
1788
|
+
PICKED_STYLE=$(python3 -c "
|
|
1789
|
+
import json
|
|
1790
|
+
try:
|
|
1791
|
+
with open('$STYLE_ASSIGN_FILE') as f:
|
|
1792
|
+
d = json.load(f)
|
|
1793
|
+
print(d.get('style') or '')
|
|
1794
|
+
except Exception:
|
|
1795
|
+
print('')
|
|
1796
|
+
" 2>/dev/null)
|
|
1797
|
+
PICKED_MODE=$(python3 -c "
|
|
1798
|
+
import json
|
|
1799
|
+
try:
|
|
1800
|
+
with open('$STYLE_ASSIGN_FILE') as f:
|
|
1801
|
+
d = json.load(f)
|
|
1802
|
+
print(d.get('mode') or 'use')
|
|
1803
|
+
except Exception:
|
|
1804
|
+
print('use')
|
|
1805
|
+
" 2>/dev/null)
|
|
1806
|
+
log "Engagement style assigned: mode=$PICKED_MODE style=${PICKED_STYLE:-(invent)}"
|
|
1807
|
+
|
|
1808
|
+
# --- Draft-prompt A/B: decouple product pivot (2026-06-29) -------------------
|
|
1809
|
+
# Per-CYCLE arm (the prep session drafts the whole batch from ONE prompt, so
|
|
1810
|
+
# assignment is at cycle granularity, not per post; the whole batch shares it).
|
|
1811
|
+
# control = the current draft directive verbatim.
|
|
1812
|
+
# treatment = "decoupled" wording: the reply must stand on its own and NOT be
|
|
1813
|
+
# built as a concede-the-obvious-then-pivot-to-the-product setup;
|
|
1814
|
+
# product mentioned only when genuinely relevant.
|
|
1815
|
+
# The arm is stamped onto every post this cycle via S4L_DRAFT_PROMPT_VARIANT
|
|
1816
|
+
# (read by twitter_post_plan.py -> log_post.py -> posts.draft_prompt_variant),
|
|
1817
|
+
# mirroring the tail_link_variant plumbing. Split tunable via
|
|
1818
|
+
# TWITTER_DRAFT_PROMPT_AB_RATE = fraction of cycles assigned to 'treatment' (the
|
|
1819
|
+
# decoupled directive). CODE DEFAULT 1 = 100% treatment: a fresh plugin install /
|
|
1820
|
+
# new user with no override drafts in the DECOUPLED style by default (the old
|
|
1821
|
+
# concede->pivot 'control' is opt-in, not the default for customers). OUR own
|
|
1822
|
+
# install pins this to 0.5 in .env for a 50/50 holdback so we can measure
|
|
1823
|
+
# decoupled vs the old behavior on our account. The dashboard reads the SAME var
|
|
1824
|
+
# with the SAME default (bin/server.js), so display and routing never diverge.
|
|
1825
|
+
DRAFT_PROMPT_AB_RATE="${TWITTER_DRAFT_PROMPT_AB_RATE:-1}"
|
|
1826
|
+
S4L_DRAFT_PROMPT_VARIANT=$(python3 -c "
|
|
1827
|
+
import random
|
|
1828
|
+
try:
|
|
1829
|
+
rate = float('$DRAFT_PROMPT_AB_RATE')
|
|
1830
|
+
except Exception:
|
|
1831
|
+
rate = 1.0
|
|
1832
|
+
rate = min(1.0, max(0.0, rate))
|
|
1833
|
+
print('treatment' if random.random() < rate else 'control')
|
|
1834
|
+
" 2>/dev/null || echo treatment)
|
|
1835
|
+
export S4L_DRAFT_PROMPT_VARIANT
|
|
1836
|
+
log "Draft-prompt A/B arm: $S4L_DRAFT_PROMPT_VARIANT (rate=$DRAFT_PROMPT_AB_RATE)"
|
|
1837
|
+
if [ "$S4L_DRAFT_PROMPT_VARIANT" = "treatment" ]; then
|
|
1838
|
+
DRAFT_DIRECTIVE="Otherwise: draft a direct, natural reply that stands on its own as a useful contribution to the thread. Mention the matched project only when it is genuinely the most relevant thing to say, and state it plainly in one clause; most replies will not need it. Do NOT build the reply as a concede-the-obvious-then-pivot-to-the-product setup. Length is governed ENTIRELY by the per-style LENGTH LIMIT in the style block above; obey that target and ceiling, do not apply any other length rule here. NEVER em dashes. Apply the matched project's \`voice\` block from ALL_PROJECTS_JSON: follow voice.tone, never violate voice.never, mirror voice.examples / voice.examples_good when present."
|
|
1839
|
+
else
|
|
1840
|
+
DRAFT_DIRECTIVE="Otherwise: draft a reply using the best engagement style. Length is governed ENTIRELY by the per-style LENGTH LIMIT in the style block above; obey that target and ceiling, do not apply any other length rule here. NEVER em dashes. Apply the matched project's \`voice\` block from ALL_PROJECTS_JSON: follow voice.tone, never violate voice.never, mirror voice.examples / voice.examples_good when present."
|
|
1841
|
+
fi
|
|
1842
|
+
# Personal-brand lane (S4L_ACTIVE_LANE=personal_brand, set by saps_mode.py):
|
|
1843
|
+
# replace the product-framed directive entirely. This lane is pure organic
|
|
1844
|
+
# growth: no product, no link, no CTA. The reply must add real value grounded in
|
|
1845
|
+
# the persona's first-hand material (the PERSONA CORPUS block + the persona voice
|
|
1846
|
+
# block), not concede-and-agree filler. Overrides both A/B arms above.
|
|
1847
|
+
if [ "${S4L_ACTIVE_LANE:-}" = "personal_brand" ]; then
|
|
1848
|
+
DRAFT_DIRECTIVE="Otherwise: draft a reply that stands on its own as a genuinely useful contribution to THIS thread. Ground it in the persona's real, first-hand experience from the PERSONA CORPUS block below (specific projects, real numbers, sharp opinions, actual failures) and in the persona's \`voice\` block from ALL_PROJECTS_JSON. Add exactly ONE of: a concrete specific from that lived experience, a sharp non-obvious opinion, a useful pointer, or a question that genuinely moves the thread forward. NEVER generic agreement ('makes sense', 'this is spot on', 'great point', 'the nuance here is'). This is a personal account, not a brand: sound like a real person in the thread. If web search is available and the thread hinges on a current fact, verify it before drafting rather than guessing. Length is governed ENTIRELY by the per-style LENGTH LIMIT in the style block above; obey that target and ceiling. NEVER em dashes. Follow voice.tone, never violate voice.never, mirror voice.examples / voice.examples_good when present."
|
|
1849
|
+
fi
|
|
1850
|
+
|
|
1851
|
+
if [ -n "$PICKED_STYLE" ]; then
|
|
1852
|
+
TOP_REPORT=$(python3 "$REPO_DIR/scripts/top_performers.py" --platform twitter --style "$PICKED_STYLE" 2>/dev/null || echo "(top performers report unavailable)")
|
|
1853
|
+
else
|
|
1854
|
+
TOP_REPORT=$(python3 "$REPO_DIR/scripts/top_performers.py" --platform twitter 2>/dev/null || echo "(top performers report unavailable)")
|
|
1855
|
+
fi
|
|
1856
|
+
|
|
1857
|
+
# --- Generation trace -------------------------------------------------------
|
|
1858
|
+
# Snapshot the few-shot context this cycle will feed to Claude — top_performers
|
|
1859
|
+
# report, top_queries from Phase 1, supply signal, dud queries — and write to a
|
|
1860
|
+
# tempfile. Path travels via env var to twitter_post_plan.py (Phase 2b-post),
|
|
1861
|
+
# which forwards it as --generation-trace to log_post.py so every post landed
|
|
1862
|
+
# this cycle gets posts.generation_trace JSONB pointing to the same snapshot.
|
|
1863
|
+
# This is what answers "which examples produced post #N" later. See
|
|
1864
|
+
# migrations/2026-05-12_generation_trace.sql for the shape contract.
|
|
1865
|
+
#
|
|
1866
|
+
# Failure is non-fatal: empty string means downstream skips --generation-trace
|
|
1867
|
+
# and the row gets NULL trace. We never block the cycle on the audit row.
|
|
1868
|
+
TRACE_INPUT=$(python3 -c "
|
|
1869
|
+
import json, sys
|
|
1870
|
+
payload = {
|
|
1871
|
+
'platform': 'twitter',
|
|
1872
|
+
'project_name': 'all',
|
|
1873
|
+
'prompt_chars': len(sys.argv[1]) + len(sys.argv[2]) + len(sys.argv[3]) + len(sys.argv[4]) + len(sys.argv[7]),
|
|
1874
|
+
'top_performers_text': sys.argv[1],
|
|
1875
|
+
'top_search_topics_text': sys.argv[7],
|
|
1876
|
+
'recent_comment_ids': [],
|
|
1877
|
+
'extras': {
|
|
1878
|
+
'top_queries_per_project': json.loads(sys.argv[2] or '{}'),
|
|
1879
|
+
'supply_signal': json.loads(sys.argv[3] or '[]'),
|
|
1880
|
+
'dud_queries': json.loads(sys.argv[4] or '[]'),
|
|
1881
|
+
'auto_picked_style': sys.argv[5] or None,
|
|
1882
|
+
'auto_picked_mode': sys.argv[6] or 'use',
|
|
1883
|
+
'top_search_topics': json.loads(sys.argv[7] or '[]'),
|
|
1884
|
+
},
|
|
1885
|
+
'min_score_floor': 5,
|
|
1886
|
+
}
|
|
1887
|
+
print(json.dumps(payload))
|
|
1888
|
+
" "$TOP_REPORT" "$TOP_QUERIES_PER_PROJECT_JSON" "$SUPPLY_SIGNAL_JSON" "$DUD_QUERIES_JSON" "$PICKED_STYLE" "$PICKED_MODE" "$TOP_TOPICS_JSON" 2>/dev/null || echo '{}')
|
|
1889
|
+
S4L_TWITTER_GEN_TRACE_PATH=$(printf '%s' "$TRACE_INPUT" | python3 "$REPO_DIR/scripts/write_generation_trace.py" --prefix twitter_gen_trace_ 2>/dev/null || echo "")
|
|
1890
|
+
export S4L_TWITTER_GEN_TRACE_PATH
|
|
1891
|
+
if [ -n "$S4L_TWITTER_GEN_TRACE_PATH" ] && [ -f "$S4L_TWITTER_GEN_TRACE_PATH" ]; then
|
|
1892
|
+
log "Generation trace: $S4L_TWITTER_GEN_TRACE_PATH ($(wc -c < "$S4L_TWITTER_GEN_TRACE_PATH") bytes)"
|
|
1893
|
+
else
|
|
1894
|
+
log "WARN: generation_trace build returned empty path; posts this cycle will have NULL trace"
|
|
1895
|
+
fi
|
|
1896
|
+
|
|
1897
|
+
STYLES_BLOCK=$(saps_render_style_block "$STYLE_ASSIGN_FILE" twitter posting)
|
|
1898
|
+
# Style assignment file is the same one we picked above; styles.sh already sourced.
|
|
1899
|
+
# Cleanup at cycle end (best effort).
|
|
1900
|
+
trap 'rm -f "$STYLE_ASSIGN_FILE" 2>/dev/null || true' EXIT
|
|
1901
|
+
|
|
1902
|
+
# Phase 2b is split into three sub-phases so the twitter-browser lock is only
|
|
1903
|
+
# held during actual browser work. The killer in the old single-session flow
|
|
1904
|
+
# was generate_page.py running inside the Claude session: 10-40 minutes of
|
|
1905
|
+
# Cloud Run deploy chain time, all under the browser lock, blocking every
|
|
1906
|
+
# other twitter pipeline. The new flow:
|
|
1907
|
+
# 2b-prep (lock held): Claude reads threads, drafts replies, saves drafts,
|
|
1908
|
+
# emits a JSON plan listing chosen candidates.
|
|
1909
|
+
# <release lock>
|
|
1910
|
+
# 2b-gen (no lock): twitter_gen_links.py runs generate_page.py per
|
|
1911
|
+
# candidate; falls back to plain project URL on failure.
|
|
1912
|
+
# <re-acquire lock>
|
|
1913
|
+
# 2b-post (lock held): twitter_post_plan.py calls twitter_browser.py reply,
|
|
1914
|
+
# log_post.py, campaign_bump.py, marks link_edited_at.
|
|
1915
|
+
|
|
1916
|
+
PLAN_FILE="/tmp/twitter_cycle_plan_${BATCH_ID}.json"
|
|
1917
|
+
SKIP_FILE="/tmp/twitter_cycle_skips_${BATCH_ID}.json"
|
|
1918
|
+
|
|
1919
|
+
# --- Phase 2b-prep: pick + draft + plan -------------------------------------
|
|
1920
|
+
# Stamp phase2b-prep BEFORE the long-running Claude read/draft so peer cycles'
|
|
1921
|
+
# Phase 0 salvage SQL sees current_phase='phase2b-prep' (45-min budget) instead
|
|
1922
|
+
# of stale phase2a (20-min budget). Without this stamp, mid-Phase-2b runs get
|
|
1923
|
+
# wrongly salvaged once 20 min elapse past phase2a's start, creating false
|
|
1924
|
+
# phase2b_silent run-monitor rows even when posts succeeded.
|
|
1925
|
+
python3 "$REPO_DIR/scripts/twitter_batch_phase.py" advance "$BATCH_ID" --phase phase2b-prep 2>&1 | tee -a "$LOG_FILE" || true
|
|
1926
|
+
log "Re-acquiring twitter-browser lock for Phase 2b-prep (read+draft only)..."
|
|
1927
|
+
acquire_lock "twitter-browser" 3600 2>>"$LOG_FILE"
|
|
1928
|
+
log "twitter-browser lock held (pid=$$) Phase 2b-prep"
|
|
1929
|
+
# Drop stale singleton locks (see clean_stale_singleton.sh, also called in Phase 1).
|
|
1930
|
+
ensure_twitter_browser_for_backend 2>&1 | tee -a "$LOG_FILE"
|
|
1931
|
+
|
|
1932
|
+
# Thread-media capture (2026-06-03, gated by S4L_TWITTER_CAPTURE_MEDIA, default
|
|
1933
|
+
# OFF). Now that the browser lock is held and the harness Chrome is up, do ONE
|
|
1934
|
+
# cheap deterministic pass over every candidate thread to pull its media
|
|
1935
|
+
# (images/videos/GIFs/link-cards), persist each into
|
|
1936
|
+
# twitter_candidates.thread_media, and build a MEDIA CONTEXT block injected into
|
|
1937
|
+
# the prep prompt so the reply-writer can react to what the tweet visually shows
|
|
1938
|
+
# instead of replying text-blind. Must be deterministic (Python pre-fetch) because
|
|
1939
|
+
# the prep prompt forbids the model from calling twitter_browser.py. Entirely
|
|
1940
|
+
# best-effort: any failure leaves MEDIA_BLOCK empty and the cycle proceeds.
|
|
1941
|
+
MEDIA_BLOCK=""
|
|
1942
|
+
if [ "${S4L_TWITTER_CAPTURE_MEDIA:-0}" = "1" ] || [ "${S4L_TWITTER_CAPTURE_MEDIA:-}" = "true" ]; then
|
|
1943
|
+
if [ -s "$MEDIA_URLS_FILE" ]; then
|
|
1944
|
+
log "Phase 2b-prep: capturing thread media for $(wc -l < "$MEDIA_URLS_FILE" | tr -d ' ') candidate(s)..."
|
|
1945
|
+
MEDIA_BLOCK=$(python3 "$REPO_DIR/scripts/capture_thread_media.py" --urls-file "$MEDIA_URLS_FILE" --scroll 1 2>>"$LOG_FILE" || true)
|
|
1946
|
+
if [ -n "$MEDIA_BLOCK" ]; then
|
|
1947
|
+
log "Phase 2b-prep: media context captured ($(printf '%s' "$MEDIA_BLOCK" | grep -c '^Candidate ') thread(s) with media)."
|
|
1948
|
+
else
|
|
1949
|
+
log "Phase 2b-prep: no media captured (none found or capture skipped)."
|
|
1950
|
+
fi
|
|
1951
|
+
fi
|
|
1952
|
+
else
|
|
1953
|
+
log "Phase 2b-prep: thread-media capture disabled (S4L_TWITTER_CAPTURE_MEDIA not set)."
|
|
1954
|
+
fi
|
|
1955
|
+
rm -f "$MEDIA_URLS_FILE" 2>/dev/null || true
|
|
1956
|
+
|
|
1957
|
+
# --- PERSONA CORPUS injection (personal_brand lane only) --------------------
|
|
1958
|
+
# build_persona.py apply writes a raw first-hand corpus sidecar next to
|
|
1959
|
+
# config.json. In the personal_brand lane we inline it so the drafter grounds
|
|
1960
|
+
# replies in real specifics (projects, numbers, opinions) instead of the single
|
|
1961
|
+
# synthesized content_angle paragraph. Empty string in the promotion lane, so
|
|
1962
|
+
# promotion prompts stay lean and config.json is never bloated with the corpus.
|
|
1963
|
+
CORPUS_BLOCK=""
|
|
1964
|
+
if [ "${S4L_ACTIVE_LANE:-}" = "personal_brand" ] && [ -f "$REPO_DIR/persona_corpus.txt" ]; then
|
|
1965
|
+
CORPUS_BLOCK="## PERSONA CORPUS (raw first-hand material — ground your reply in THIS)
|
|
1966
|
+
This is the persona's own public writing and work, verbatim. Quote and draw real specifics from it: actual projects, real numbers, sharp opinions, real failures. Do NOT invent anything not supported here or in the persona voice block. Use it to make the reply concrete and unmistakably human.
|
|
1967
|
+
$(cat "$REPO_DIR/persona_corpus.txt")
|
|
1968
|
+
"
|
|
1969
|
+
log "Phase 2b-prep: injected persona corpus ($(wc -c < "$REPO_DIR/persona_corpus.txt" | tr -d ' ') bytes)."
|
|
1970
|
+
fi
|
|
1971
|
+
|
|
1972
|
+
log "Phase 2b-prep: Claude reading threads and drafting replies (no post cap)..."
|
|
1973
|
+
|
|
1974
|
+
# Pre-assign the prep session UUID in the parent shell so it survives the
|
|
1975
|
+
# command-substitution subshell run_claude.sh runs in. We write it into the
|
|
1976
|
+
# plan JSON below so Phase 2b-post can re-export it for log_post.py, which
|
|
1977
|
+
# stamps posts.claude_session_id and lets the dashboard activity feed join
|
|
1978
|
+
# to claude_sessions for cost. Without this, twitter posts get NULL session
|
|
1979
|
+
# ids and blank cost cells.
|
|
1980
|
+
CLAUDE_SESSION_ID="$(uuidgen | tr 'A-Z' 'a-z')"
|
|
1981
|
+
export CLAUDE_SESSION_ID
|
|
1982
|
+
|
|
1983
|
+
# PREP_SCHEMA — strict JSON schema for the prep envelope. Includes
|
|
1984
|
+
# optional `new_style` per candidate (an inner object) that the model
|
|
1985
|
+
# MUST populate when it chooses a brand-new engagement style name (i.e.
|
|
1986
|
+
# the picker set mode=invent and the model invented a snake_case name).
|
|
1987
|
+
# Fields mirror engagement_styles.py::_REQUIRED_NEW_STYLE_FIELDS so the
|
|
1988
|
+
# downstream validate_or_register call accepts the block without a
|
|
1989
|
+
# second schema layer.
|
|
1990
|
+
PREP_SCHEMA='{"type":"object","properties":{"candidates":{"type":"array","items":{"type":"object","properties":{"candidate_id":{"type":"integer"},"candidate_url":{"type":"string"},"thread_author":{"type":"string"},"thread_text":{"type":"string"},"matched_project":{"type":"string"},"reply_text":{"type":"string"},"engagement_style":{"type":"string"},"new_style":{"type":["object","null"],"properties":{"description":{"type":"string"},"example":{"type":"string"},"why_existing_didnt_fit":{"type":"string"},"note":{"type":"string"},"target_chars":{"type":"integer"}}},"language":{"type":"string"},"has_landing_pages":{"type":"boolean"},"link_keyword":{"type":"string"},"link_slug":{"type":"string"},"search_topic":{"type":"string"}},"required":["candidate_id","candidate_url","matched_project","reply_text","engagement_style","language","has_landing_pages","search_topic"]}},"rejected":{"type":"array","items":{"type":"object","properties":{"candidate_id":{"type":"integer"},"reason":{"type":"string"},"proposed_excludes":{"type":"array","items":{"type":"string"}}},"required":["candidate_id","reason"]}}},"required":["candidates","rejected"]}'
|
|
1991
|
+
|
|
1992
|
+
PREP_PROMPT="${TW_ENGINE_PREFIX}You are the Social Autoposter prep step.
|
|
1993
|
+
|
|
1994
|
+
Your ONLY job in THIS session:
|
|
1995
|
+
1. Read each candidate's thread context from the PRE-SCORED CANDIDATES block below (each entry's 'Text:' field is the parent tweet). You have WebSearch and WebFetch available: use them ONLY when a thread hinges on a current fact, a name, a release, or a claim you are not sure about, so your reply is specific and correct instead of vague. You do NOT have the Twitter/X browser this session — never fetch, navigate, or open a tweet/x.com URL, and never try to load the thread itself; the thread text you need is already inlined below. Most replies need no search at all; reach for it only when it materially improves the reply.
|
|
1996
|
+
2. Draft a reply for each.
|
|
1997
|
+
3. Persist each fresh draft via log_draft.py.
|
|
1998
|
+
4. Emit a structured plan describing the chosen candidates, the reply text, and (when applicable) the SEO link keyword + slug.
|
|
1999
|
+
|
|
2000
|
+
You will NOT post anything. You will NOT generate landing pages. You will NOT call log_post.py. The shell handles all of that AFTER your session ends, with the browser lock released for the long landing-page build.
|
|
2001
|
+
|
|
2002
|
+
Read $SKILL_FILE for content rules and voice context.
|
|
2003
|
+
Read $REPO_DIR/config.json for project metadata.
|
|
2004
|
+
|
|
2005
|
+
## PRE-SCORED CANDIDATES (sorted by Virality DESC, highest first)
|
|
2006
|
+
Virality is a composite predictor of how big this thread will get AFTER you reply: it combines engagement velocity (eng/hour), author reach (follower tier), age decay (6h half-life), retweet ratio, reply count, and discussion quality (reply:like ratio). On historical posted data the highest-Virality cohort (score >= 10000) received ~36x the median reply views of the lowest cohort (score < 10), so prioritize on-brand candidates with HIGH Virality. Rule of thumb: Virality >= 100 = strong thread on a real growth curve, your reply is likely to land 10-100x more eyeballs than a low-Virality thread. Delta (5min) is the raw T1-T0 engagement count and is shown for context only; do not re-rank on Delta.
|
|
2007
|
+
$CANDIDATE_BLOCK
|
|
2008
|
+
$MEDIA_BLOCK
|
|
2009
|
+
$CORPUS_BLOCK
|
|
2010
|
+
|
|
2011
|
+
## PROJECT ROUTING (per-candidate)
|
|
2012
|
+
Each candidate has a 'Project match' field. Use that project unless the thread content clearly better fits another project.
|
|
2013
|
+
All project configs: $ALL_PROJECTS_JSON
|
|
2014
|
+
|
|
2015
|
+
## FEEDBACK FROM PAST PERFORMANCE:
|
|
2016
|
+
$TOP_REPORT
|
|
2017
|
+
|
|
2018
|
+
$STYLES_BLOCK
|
|
2019
|
+
|
|
2020
|
+
## WORKFLOW
|
|
2021
|
+
There is NO cap on how many candidates you may pick this cycle. Pick EVERY candidate whose thread is genuinely on-brand and worth a substantive reply. Skip a candidate ONLY when its thread is off-topic for the matched project, toxic / hateful, low-quality / spam, an audience mismatch, or a near-duplicate of something already replied to. Do NOT cap, quota, or balance picks by project: if the strongest candidates this cycle all belong to one project, pick all of them. Project routing matters; project diversification does not. Never force a weak entry just to add volume, and never drop a strong on-brand entry just to limit volume.
|
|
2022
|
+
|
|
2023
|
+
For each chosen candidate:
|
|
2024
|
+
1. Read the candidate's parent tweet from its 'Text:' field in the PRE-SCORED CANDIDATES block above.
|
|
2025
|
+
2. Understand the context from that inlined text (the thread text is already in this prompt; you do NOT have the Twitter browser, but you MAY use WebSearch/WebFetch for external facts when a thread needs them to be answered well).
|
|
2026
|
+
3. DRAFT HANDLING (existing vs fresh):
|
|
2027
|
+
- If the candidate block shows an EXISTING DRAFT line AND draft age < 30 minutes, REUSE the draft text verbatim. Set engagement_style to the existing style. Do NOT call log_draft.py; do NOT redraft. Reason: prior cycle paid the LLM cost.
|
|
2028
|
+
- $DRAFT_DIRECTIVE
|
|
2029
|
+
3a. PERSIST FRESH DRAFTS (skip for reused drafts):
|
|
2030
|
+
python3 $REPO_DIR/scripts/log_draft.py --candidate-id CANDIDATE_ID --text 'YOUR_REPLY_TEXT' --style STYLE --assigned-style '$PICKED_STYLE' --assigned-mode '$PICKED_MODE'
|
|
2031
|
+
The --assigned-style / --assigned-mode flags carry the orchestrator's picker output (this cycle: mode=$PICKED_MODE style='${PICKED_STYLE:-(invent)}') into the candidate row so the post pipeline can coerce drift and register invented styles. Pass them VERBATIM as shown.
|
|
2032
|
+
If you are inventing a brand-new style this cycle (i.e. \$PICKED_MODE=invent and your STYLE is a new snake_case name not in the style block above), ALSO pass:
|
|
2033
|
+
--new-style '{\"description\":\"...\",\"example\":\"...\",\"why_existing_didnt_fit\":\"...\"}'
|
|
2034
|
+
with the same description/example/why_existing_didnt_fit you put in the 'new_style' field of your output JSON for this candidate.
|
|
2035
|
+
Failure here is non-fatal, log a warning and continue.
|
|
2036
|
+
4. EMIT one entry in the structured 'candidates' array with these fields:
|
|
2037
|
+
- candidate_id (int): from the candidate block
|
|
2038
|
+
- candidate_url (string): the parent tweet URL
|
|
2039
|
+
- thread_author (string): the @handle (no leading @)
|
|
2040
|
+
- thread_text (string): the parent tweet's text, condensed to <=500 chars if needed
|
|
2041
|
+
- matched_project (string): the project name to attribute this post to
|
|
2042
|
+
- reply_text (string): the FINAL reply text WITHOUT any URL appended (the shell appends the URL later). 250 chars is the hard ceiling (leaves room for a 23-char t.co link inside the 280-char cap) — stay well under it, not up to it.
|
|
2043
|
+
- engagement_style (string): style name applied (or 'reused' for an unchanged stale draft). In USE mode ($PICKED_MODE=use) this MUST be the assigned style name '${PICKED_STYLE}' verbatim; the orchestrator silently coerces drift back. In INVENT mode ($PICKED_MODE=invent) this MUST be a NEW snake_case style name not in the curated style block.
|
|
2044
|
+
- new_style (object, REQUIRED iff INVENT mode produced a new name; OMIT or set null otherwise): {description (string), example (string), why_existing_didnt_fit (string), note (string, optional), target_chars (integer, REQUIRED)}. Same shape you passed to --new-style in step 3a. The post pipeline reads this and POSTs to /api/v1/engagement-styles/registry so the new style lands in engagement_styles_registry alongside Reddit/GitHub/Moltbook inventions. target_chars is the comment length THIS new style wins at, in characters. IMPORTANT: the example you write must be EXACTLY that length — the example IS the canonical length reference, and the hard ceiling is target_chars × 1.5. Write the example first, count its characters, then set target_chars to that count. Bias SHORT: one-liner style ~45, story-arc style up to ~180, never above 220.
|
|
2045
|
+
- language (string): ISO 639-1 code (en, ja, zh, es, ...)
|
|
2046
|
+
- has_landing_pages (bool): true iff the matched project has BOTH landing_pages.repo AND landing_pages.base_url set in config.json. Otherwise false.
|
|
2047
|
+
- link_keyword (string, REQUIRED when has_landing_pages=true; OMIT otherwise): a SHORT 3-6 word phrase that captures the ESSENCE OF YOUR REPLY (not just the thread topic). Think: what would a reader search to find a useful page about what you just said?
|
|
2048
|
+
- link_slug (string, REQUIRED when has_landing_pages=true; OMIT otherwise): kebab-case, alphanumeric+hyphens only, max 50 chars.
|
|
2049
|
+
- search_topic (string, REQUIRED): normally the EXACT 'Search query' value from this candidate's block above, copied verbatim (do not paraphrase, normalise, or trim). EXCEPTION (cross-route): if the matched_project you chose for this candidate is DIFFERENT from the candidate's 'Project match' field (i.e. you re-routed the thread to a better-fitting project), set search_topic to an empty string \"\" instead. The origin query's topic belongs to the project that ISSUED that query, not the one you routed to; copying it onto the new project's post miscredits the new project's topic ranking and the issuing project's query bank. When matched_project equals the 'Project match' field, copy the topic verbatim as before. The shell stamps this onto posts.search_topic so the next cycle's Phase 1 can rank which topics convert (clicks per post) and evolve the universe accordingly.
|
|
2050
|
+
|
|
2051
|
+
5. CLASSIFY EVERY PRE-SCORED CANDIDATE into ONE of THREE outcomes. There is NO post cap and NO per-project quota: post EVERY thread you judge genuinely on-brand.
|
|
2052
|
+
(a) 'candidates' — an on-brand pick you are replying to this cycle (step 4 above). No cap.
|
|
2053
|
+
(b) 'rejected' — ONLY for a PERMANENT, thread-intrinsic reason this thread should NEVER be replied to for the matched project: off-topic for the project, toxic / hateful, low-quality / spam / promo / shill, audience or ICP mismatch, our own account, or stale. Reason must be <=200 chars, plain text, no quotes. CRITICAL: the shell marks every 'rejected' entry status='skipped', and a skipped (thread, project) is filtered out of ALL future scans for this account PERMANENTLY. Only reject things that will never be a good fit.
|
|
2054
|
+
(c) OMIT from BOTH arrays — for a TIMING-ONLY reason where the thread itself is fine but you are simply not posting to it right NOW. Omitting keeps it 'pending' so a later cycle can re-judge it. ALWAYS omit (NEVER reject) when your only reason is one of:
|
|
2055
|
+
- you preferred a stronger candidate this cycle (there is no cap, so ideally just post this one too; if you still defer, omit it),
|
|
2056
|
+
- it is a near-duplicate of another thread you are already picking THIS cycle,
|
|
2057
|
+
- you already engaged this author / a similar thread this cycle and want to avoid back-to-back over-engagement.
|
|
2058
|
+
These are DEFERRALS, not rejections. Putting any of them in 'rejected' would permanently blacklist a thread that is actually fine. Do NOT do that.
|
|
2059
|
+
It is fine for 'candidates' to be empty (nothing on-brand) and fine for 'rejected' to be empty (nothing permanently unsuitable).
|
|
2060
|
+
Do NOT update twitter_candidates yourself; the shell will mark every entry of 'rejected' as status='skipped' with the reason, and Phase 0 will salvage anything you omit or forget.
|
|
2061
|
+
|
|
2062
|
+
5a. SELF-IMPROVING PROJECT-WIDE EXCLUSION LIST (optional, on rejected entries only):
|
|
2063
|
+
When you put a candidate in 'rejected' BECAUSE of a stable, recurring CLASS of false-positive (not a one-off bad tweet), you MAY include a 'proposed_excludes' array of 1-3 specific keywords. If you do, the pipeline will (after a 2-distinct-batch activation gate) automatically append \`-keyword\` to ALL future Twitter searches for the matched_project, project-wide and persistent. This is the ONLY upstream block against the entire class of false-positive that a tighter Phase 1 query alone cannot prevent.
|
|
2064
|
+
|
|
2065
|
+
USE THIS POWER NARROWLY. False-negatives (legit tweets being filtered out) are far worse than the cost of seeing one more cricket tweet. Apply ALL of these rules:
|
|
2066
|
+
|
|
2067
|
+
- DO emit when: the false-positive is caused by a SPECIFIC ambiguous proper noun, brand, or domain term that has a wholly unrelated meaning collisional with the project. Example for Vipassana: an IPL/cricket thread surfaced because the search query included \`Goenka\` (the meditation teacher S.N. Goenka shares a surname with Sanjiv Goenka, owner of an IPL team). Right proposed_excludes: ['cricket','kohli','ipl','lsg','rcb']. WRONG proposed_excludes: ['goenka'] (would mute legit S.N. Goenka tweets).
|
|
2068
|
+
|
|
2069
|
+
- DO NOT emit when: the candidate is just personally low-quality (spam, low engagement, generic), the language is wrong, the author is bot-like, or the thread is just slightly off-topic. Those are one-offs, NOT classes. Use the 'reason' field instead.
|
|
2070
|
+
|
|
2071
|
+
- Each proposed term must be:
|
|
2072
|
+
* a SINGLE token, lowercase, ascii letters/digits/hyphen only, no spaces, length 3-32. (e.g. 'cricket', 'kohli', 'ipl', 'lsg', 'rcb-fan', 'crypto', 'memecoin').
|
|
2073
|
+
* SPECIFIC and unambiguous in the project's domain. Proper nouns, brand names, narrow jargon, sport/team/franchise terms preferred. Generic words like 'practice', 'retreat', 'meditation', 'work', 'tips', 'app', 'tool', 'help' are FORBIDDEN — they will produce false-negatives.
|
|
2074
|
+
* NOT a core search topic of the matched_project (the validator rejects any term in the project's search_topics, so don't waste tokens proposing one).
|
|
2075
|
+
|
|
2076
|
+
- Cap: at most 3 terms per rejected entry. If you need more, you're probably proposing too generically — narrow the list.
|
|
2077
|
+
|
|
2078
|
+
- Activation gate: each term needs >=2 SEPARATE batches to propose it before it goes live, so a single false-rejection cannot mute a search. You don't need to think about this — propose if you'd be confident a future cycle's Claude would also propose it; if not, leave proposed_excludes off.
|
|
2079
|
+
|
|
2080
|
+
- When in doubt, omit the field entirely. The default behavior (no proposed_excludes) is safe; over-proposing is not.
|
|
2081
|
+
|
|
2082
|
+
CRITICAL:
|
|
2083
|
+
- DO NOT post anything. The shell handles posting.
|
|
2084
|
+
- DO NOT call twitter_browser.py.
|
|
2085
|
+
- DO NOT call generate_page.py (the shell runs it AFTER your session, outside the lock).
|
|
2086
|
+
- DO NOT call log_post.py or campaign_bump.py.
|
|
2087
|
+
- You do NOT have the Twitter/X browser this session: never navigate, fetch, or open a tweet/x.com URL, and never try to reload the thread. WebSearch/WebFetch ARE available for external fact-checking only; use them sparingly and never to open the tweet itself.
|
|
2088
|
+
- NEVER use em dashes. Use commas, periods, or regular dashes (-).
|
|
2089
|
+
- Reply in the SAME LANGUAGE as the parent tweet."
|
|
2090
|
+
|
|
2091
|
+
# Pipe the prep prompt via stdin instead of passing as a shell argument.
|
|
2092
|
+
# On Linux ARG_MAX is 2MB; the assembled prompt (config.json + top_report +
|
|
2093
|
+
# styles + schema + candidates) busts that on the VM, dying with E2BIG
|
|
2094
|
+
# "Argument list too long". stdin has no such cap.
|
|
2095
|
+
# --allowedTools: restore external fact-checking to the prep drafter (removed
|
|
2096
|
+
# 2026-06-26). --strict-mcp-config stays so the twitter-harness browser MCP is NOT
|
|
2097
|
+
# loaded: the model can search the web but can never touch the CDP Chrome that
|
|
2098
|
+
# Phase 2b-post drives (that would break the two-lock). The tools are passed as a
|
|
2099
|
+
# SINGLE comma-separated token on purpose: claude_job.py's queue parser (box
|
|
2100
|
+
# installs) treats --allowedTools as a one-value flag, so a space-separated second
|
|
2101
|
+
# tool would leak in as the prompt. On the box these flags ride through
|
|
2102
|
+
# claude_job.py; Desktop's own web search + the reworded prompt enable it there.
|
|
2103
|
+
PREP_OUTPUT=$(printf '%s' "$PREP_PROMPT" | "$REPO_DIR/scripts/run_claude.sh" "run-twitter-cycle-prep" --strict-mcp-config --mcp-config "$TW_MCP_CONFIG" --allowedTools WebSearch,WebFetch -p --output-format json --json-schema "$PREP_SCHEMA" 2>&1)
|
|
2104
|
+
|
|
2105
|
+
echo "$PREP_OUTPUT" >> "$LOG_FILE"
|
|
2106
|
+
|
|
2107
|
+
# --- TOP-N POST CAP (2026-06-29) -------------------------------------------
|
|
2108
|
+
# The prep model still drafts EVERY on-brand candidate, but autopilot now posts
|
|
2109
|
+
# only the single highest-Virality one per cycle. This caps per-account reply
|
|
2110
|
+
# volume (the May-June ~10x ramp that collapsed per-post reach ~15x) while
|
|
2111
|
+
# keeping the strongest thread. Deferred picks are dropped from the plan so they
|
|
2112
|
+
# stay status='pending' (NOT 'skipped'); Phase 0 salvage re-judges them next
|
|
2113
|
+
# cycle and reuses their fresh drafts. (2026-06-30) The cap is now the SINGLE
|
|
2114
|
+
# standard for BOTH lanes: autopilot direct-post AND DRAFT_ONLY manual MCP review.
|
|
2115
|
+
# The old DRAFT_ONLY=1 -> POST_TOP_N=0 special-case was removed on purpose, so the
|
|
2116
|
+
# human reviews the exact same one highest-Virality draft the autopilot would post.
|
|
2117
|
+
# Override with S4L_TWITTER_POST_TOP_N (default 1; 0 = no cap, env opt-out only).
|
|
2118
|
+
POST_TOP_N="${S4L_TWITTER_POST_TOP_N:-1}"
|
|
2119
|
+
|
|
2120
|
+
# --- ROLLING VIRALITY BAR (2026-07-02) --------------------------------------
|
|
2121
|
+
# Fetch THIS install's trailing-24h virality percentile so the parse step posts
|
|
2122
|
+
# the top-1 ONLY if it clears the bar. This holds the post rate near the target
|
|
2123
|
+
# (~20-30 / 8h) with NO hard cap: the bar is the Nth percentile of the install's
|
|
2124
|
+
# OWN recent candidate pool (via /api/v1/twitter-candidates/virality-threshold),
|
|
2125
|
+
# so it self-calibrates to cadence and niche instead of being a fixed number.
|
|
2126
|
+
# The bar is OFF (empty threshold) when:
|
|
2127
|
+
# - DRAFT_ONLY: new users / manual review see every draft (we don't even fetch).
|
|
2128
|
+
# - Cold start: sample_count < min, so a fresh pool posts ungated until it fills.
|
|
2129
|
+
# - Fetch failure: fail-open, never silence posting on a transient API blip.
|
|
2130
|
+
# Tunables: S4L_TWITTER_VIRALITY_PCTILE (default 0.97),
|
|
2131
|
+
# S4L_TWITTER_VIRALITY_MIN_SAMPLE (default 200).
|
|
2132
|
+
VIRALITY_THRESHOLD=""
|
|
2133
|
+
if [ "${DRAFT_ONLY:-0}" != "1" ]; then
|
|
2134
|
+
VIRALITY_THRESHOLD=$(S4L_VPCTILE="${S4L_TWITTER_VIRALITY_PCTILE:-0.97}" \
|
|
2135
|
+
S4L_VMIN="${S4L_TWITTER_VIRALITY_MIN_SAMPLE:-200}" \
|
|
2136
|
+
python3 -c "
|
|
2137
|
+
import os, sys
|
|
2138
|
+
sys.path.insert(0, os.path.expanduser('~/social-autoposter/scripts'))
|
|
2139
|
+
from http_api import api_get
|
|
2140
|
+
try:
|
|
2141
|
+
r = api_get('/api/v1/twitter-candidates/virality-threshold',
|
|
2142
|
+
{'pctile': os.environ['S4L_VPCTILE'], 'hours': 24})
|
|
2143
|
+
d = (r or {}).get('data') or {}
|
|
2144
|
+
thr = d.get('threshold')
|
|
2145
|
+
n = int(d.get('sample_count') or 0)
|
|
2146
|
+
mn = int(os.environ['S4L_VMIN'])
|
|
2147
|
+
if thr is not None and n >= mn:
|
|
2148
|
+
print(f'{float(thr):.4f}')
|
|
2149
|
+
except BaseException as e:
|
|
2150
|
+
sys.stderr.write(f'virality-bar fetch failed (bar OFF this cycle): {e}\n')
|
|
2151
|
+
" 2>>"$LOG_FILE" || echo "")
|
|
2152
|
+
if [ -n "$VIRALITY_THRESHOLD" ]; then
|
|
2153
|
+
log "Virality bar ACTIVE: p${S4L_TWITTER_VIRALITY_PCTILE:-0.97} = $VIRALITY_THRESHOLD (this install, trailing 24h); top-1 posts only if it clears the bar."
|
|
2154
|
+
else
|
|
2155
|
+
log "Virality bar OFF this cycle (cold-start/thin pool or fetch failed); top-1 posts ungated."
|
|
2156
|
+
fi
|
|
2157
|
+
fi
|
|
2158
|
+
|
|
2159
|
+
# Parse the prep envelope and write the plan to \$PLAN_FILE; also extract the
|
|
2160
|
+
# 'rejected' array into \$SKIP_FILE so log_twitter_skips.py can persist a
|
|
2161
|
+
# reason against every twitter_candidates row Claude reviewed but didn't pick.
|
|
2162
|
+
S4L_CAND_VIR="$CANDIDATES" S4L_POST_TOP_N="$POST_TOP_N" VIRALITY_THRESHOLD="$VIRALITY_THRESHOLD" python3 -c "
|
|
2163
|
+
import json, sys, os
|
|
2164
|
+
text = sys.stdin.read().strip()
|
|
2165
|
+
try:
|
|
2166
|
+
env, _ = json.JSONDecoder().raw_decode(text)
|
|
2167
|
+
except Exception as e:
|
|
2168
|
+
print(f'prep: envelope parse error: {e}', file=sys.stderr); sys.exit(1)
|
|
2169
|
+
so = env.get('structured_output')
|
|
2170
|
+
if so is None:
|
|
2171
|
+
so = env.get('result')
|
|
2172
|
+
if isinstance(so, str):
|
|
2173
|
+
try: so = json.loads(so)
|
|
2174
|
+
except Exception: pass
|
|
2175
|
+
candidates = so.get('candidates', []) if isinstance(so, dict) else []
|
|
2176
|
+
rejected = so.get('rejected', []) if isinstance(so, dict) else []
|
|
2177
|
+
# Build candidate_id -> virality_score from the pre-scored CANDIDATES block
|
|
2178
|
+
# (pipe cols: id|url|author|text|virality|delta|...). Shared by the top-N cap
|
|
2179
|
+
# and the rolling virality bar below.
|
|
2180
|
+
_vir = {}
|
|
2181
|
+
for _ln in (os.environ.get('S4L_CAND_VIR', '') or '').splitlines():
|
|
2182
|
+
_p = _ln.split('|')
|
|
2183
|
+
if len(_p) >= 5 and _p[0].isdigit():
|
|
2184
|
+
try: _vir[int(_p[0])] = float(_p[4] or 0)
|
|
2185
|
+
except Exception: pass
|
|
2186
|
+
# TOP-N POST CAP (2026-06-29): keep only the highest-Virality on-brand pick(s).
|
|
2187
|
+
# S4L_POST_TOP_N=0 disables the cap (env opt-out only; the cap applies to both
|
|
2188
|
+
# autopilot and DRAFT_ONLY lanes as of 2026-06-30). Truncated picks are dropped
|
|
2189
|
+
# from the plan, so they stay status='pending' (NOT 'rejected'); Phase 0 salvages.
|
|
2190
|
+
_top_n = int(os.environ.get('S4L_POST_TOP_N', '1') or '1')
|
|
2191
|
+
_deferred = 0
|
|
2192
|
+
if _top_n > 0 and len(candidates) > _top_n:
|
|
2193
|
+
candidates.sort(key=lambda c: _vir.get(c.get('candidate_id'), 0.0), reverse=True)
|
|
2194
|
+
_deferred = len(candidates) - _top_n
|
|
2195
|
+
candidates = candidates[:_top_n]
|
|
2196
|
+
# ROLLING VIRALITY BAR (2026-07-02): drop kept pick(s) below the trailing-24h
|
|
2197
|
+
# percentile of THIS install's candidate pool (VIRALITY_THRESHOLD, from /api/v1).
|
|
2198
|
+
# Empty env = bar OFF: DRAFT_ONLY (new users see every draft), cold start (thin
|
|
2199
|
+
# pool), or fetch failure. Below-bar picks are dropped like deferrals -> stay
|
|
2200
|
+
# 'pending', never 'rejected', so Phase 0 re-judges them next cycle.
|
|
2201
|
+
_bar = (os.environ.get('VIRALITY_THRESHOLD', '') or '').strip()
|
|
2202
|
+
_below_bar = 0
|
|
2203
|
+
if _bar and candidates:
|
|
2204
|
+
try:
|
|
2205
|
+
_thr = float(_bar)
|
|
2206
|
+
_kept = [c for c in candidates if _vir.get(c.get('candidate_id'), 0.0) >= _thr]
|
|
2207
|
+
_below_bar = len(candidates) - len(_kept)
|
|
2208
|
+
candidates = _kept
|
|
2209
|
+
except Exception:
|
|
2210
|
+
pass
|
|
2211
|
+
# The picker assignment travels through the plan envelope so
|
|
2212
|
+
# twitter_post_plan.py can call validate_or_register(...) with the
|
|
2213
|
+
# original (assigned_style, assigned_mode) and coerce USE-mode drift
|
|
2214
|
+
# back to the picker's choice (or accept the INVENT-mode invention +
|
|
2215
|
+
# POST it to /api/v1/engagement-styles/registry). Without this, the
|
|
2216
|
+
# post pipeline can't tell which style the picker actually assigned
|
|
2217
|
+
# vs. what the model picked. Empty string means INVENT mode (NULL
|
|
2218
|
+
# assigned_style in the registry-coercion contract).
|
|
2219
|
+
json.dump({'candidates': candidates,
|
|
2220
|
+
'session_id': '$CLAUDE_SESSION_ID',
|
|
2221
|
+
'assigned_style': '$PICKED_STYLE' or None,
|
|
2222
|
+
'assigned_mode': '$PICKED_MODE' or 'use'}, open('$PLAN_FILE', 'w'), indent=2)
|
|
2223
|
+
json.dump({'skips': rejected}, open('$SKIP_FILE', 'w'), indent=2)
|
|
2224
|
+
print(f'prep: wrote {len(candidates)} candidate(s) (deferred {_deferred} lower-virality, {_below_bar} below bar) and {len(rejected)} skips to $PLAN_FILE / $SKIP_FILE', file=sys.stderr)
|
|
2225
|
+
" <<< "$PREP_OUTPUT" 2>&1 | tee -a "$LOG_FILE"
|
|
2226
|
+
|
|
2227
|
+
PREP_PARSE_EXIT=${PIPESTATUS[0]:-1}
|
|
2228
|
+
|
|
2229
|
+
# Persist the rejected list to twitter_candidates (status='skipped' with reason)
|
|
2230
|
+
# scoped to this batch so we never clobber rows from peer cycles. Non-fatal.
|
|
2231
|
+
if [ -f "$SKIP_FILE" ]; then
|
|
2232
|
+
python3 "$REPO_DIR/scripts/log_twitter_skips.py" \
|
|
2233
|
+
--file "$SKIP_FILE" --require-batch-id "$BATCH_ID" 2>&1 | tee -a "$LOG_FILE" || true
|
|
2234
|
+
rm -f "$SKIP_FILE"
|
|
2235
|
+
fi
|
|
2236
|
+
|
|
2237
|
+
# Classify Anthropic-side error in the prep envelope so the dashboard
|
|
2238
|
+
# surfaces a specific reason (monthly_limit, stream_idle_timeout, api_overloaded,
|
|
2239
|
+
# context_overflow, etc.) rather than a silent failure when prep returns no
|
|
2240
|
+
# plan. Empty plan with NO classified API error falls through to the historical
|
|
2241
|
+
# "empty plan, no failure logged" branch below (salvage retries next cycle).
|
|
2242
|
+
PREP_REASON=$(echo "$PREP_OUTPUT" | python3 "$REPO_DIR/scripts/classify_run_error.py" 2>/dev/null)
|
|
2243
|
+
|
|
2244
|
+
PLAN_COUNT=0
|
|
2245
|
+
if [ "$PREP_PARSE_EXIT" -eq 0 ] && [ -f "$PLAN_FILE" ]; then
|
|
2246
|
+
PLAN_COUNT=$(python3 -c "import json; print(len(json.load(open('$PLAN_FILE')).get('candidates') or []))" 2>/dev/null || echo 0)
|
|
2247
|
+
fi
|
|
2248
|
+
log "Phase 2b-prep complete. plan_count=$PLAN_COUNT"
|
|
2249
|
+
|
|
2250
|
+
# Determine if Phase 2b-gen will be a no-op. When TWITTER_PAGE_GEN_RATE=0
|
|
2251
|
+
# globally, scripts/twitter_gen_links.py rewrites the plan with plain URLs in
|
|
2252
|
+
# <1s. In that case the release-now + re-acquire-after-gen dance is pure waste:
|
|
2253
|
+
# under cycle overlap the re-acquire can sit in the FIFO ticket queue for
|
|
2254
|
+
# 30-90s behind the very `engage-twitter` / next `run-twitter-cycle` we just
|
|
2255
|
+
# handed the lock to. We keep the lock through 2b-gen instead and skip the
|
|
2256
|
+
# dance entirely.
|
|
2257
|
+
GEN_RATE_RAW="${TWITTER_PAGE_GEN_RATE:-0.0}"
|
|
2258
|
+
GEN_IS_NOOP=false
|
|
2259
|
+
case "$GEN_RATE_RAW" in
|
|
2260
|
+
0|0.0|0.00|0.000|"") GEN_IS_NOOP=true ;;
|
|
2261
|
+
esac
|
|
2262
|
+
|
|
2263
|
+
# Release the lock unless (a) plan is non-empty AND (b) gen is a no-op. The
|
|
2264
|
+
# empty-plan early-exit below still needs the release for a clean handoff, so
|
|
2265
|
+
# we cannot just skip when GEN_IS_NOOP=true unconditionally.
|
|
2266
|
+
if [ "${PLAN_COUNT:-0}" = "0" ] || ! $GEN_IS_NOOP; then
|
|
2267
|
+
log "Releasing twitter-browser lock (gen step is lock-free)..."
|
|
2268
|
+
release_lock "twitter-browser" 2>>"$LOG_FILE"
|
|
2269
|
+
# (2026-06-16) session-lock rm removed (defect b); dead holders self-reclaim
|
|
2270
|
+
# in twitter_browser.py now. Do NOT re-add. See Phase 1 note + docs/twitter_browser_lock.md.
|
|
2271
|
+
else
|
|
2272
|
+
log "Keeping twitter-browser lock through Phase 2b-gen (TWITTER_PAGE_GEN_RATE=$GEN_RATE_RAW, gen is a no-op; skipping release/re-acquire dance)"
|
|
2273
|
+
fi
|
|
2274
|
+
|
|
2275
|
+
if [ "${PLAN_COUNT:-0}" = "0" ]; then
|
|
2276
|
+
log "Empty plan from prep step. Exiting cycle without posting (pending rows salvaged next cycle)."
|
|
2277
|
+
rm -f "$PLAN_FILE"
|
|
2278
|
+
_COST=$(python3 "$REPO_DIR/scripts/get_run_cost.py" --cycle-id "$BATCH_ID" 2>/dev/null || echo "0.0000")
|
|
2279
|
+
# If the classifier identified a real Anthropic error (any non-empty reason
|
|
2280
|
+
# key), log as failed=1 with that reason so the dashboard pill reads
|
|
2281
|
+
# "failed: stream_idle_timeout" / "failed: monthly_limit" / etc. Otherwise
|
|
2282
|
+
# keep the historical failed=0 behaviour for "empty plan, no API error"
|
|
2283
|
+
# (salvage retries the candidates next cycle, nothing to surface).
|
|
2284
|
+
if [ -n "$PREP_REASON" ]; then
|
|
2285
|
+
python3 "$REPO_DIR/scripts/log_run.py" --script "post_twitter" --posted 0 --skipped "${CANDIDATE_COUNT:-0}" --failed 1 --salvaged "${SALVAGED:-0}" \
|
|
2286
|
+
--queries "${QUERIES_TOTAL:-0}" --duds "${DUDS_TOTAL:-0}" \
|
|
2287
|
+
--tweets-pulled "${TWEETS_PULLED:-0}" --candidates "${BATCH_COUNT:-0}" --above-floor "${HIGH_DELTA_COUNT:-0}" \
|
|
2288
|
+
--failure-reasons "${PREP_REASON}:1" --cost "$_COST" --elapsed $(( $(date +%s) - RUN_START ))
|
|
2289
|
+
else
|
|
2290
|
+
python3 "$REPO_DIR/scripts/log_run.py" --script "post_twitter" --posted 0 --skipped "${CANDIDATE_COUNT:-0}" --failed 0 --salvaged "${SALVAGED:-0}" \
|
|
2291
|
+
--queries "${QUERIES_TOTAL:-0}" --duds "${DUDS_TOTAL:-0}" \
|
|
2292
|
+
--tweets-pulled "${TWEETS_PULLED:-0}" --candidates "${BATCH_COUNT:-0}" --above-floor "${HIGH_DELTA_COUNT:-0}" \
|
|
2293
|
+
--cost "$_COST" --elapsed $(( $(date +%s) - RUN_START ))
|
|
2294
|
+
fi
|
|
2295
|
+
# In DRAFT_ONLY (MCP draft_cycle) mode, a non-empty PREP_REASON means the
|
|
2296
|
+
# prep step FAILED for a real reason (e.g. claude_not_logged_in) rather than
|
|
2297
|
+
# genuinely finding nothing on-brand. Surface it on stdout so the MCP wrapper
|
|
2298
|
+
# can tell the user the actual problem (e.g. "run claude /login") instead of
|
|
2299
|
+
# mis-reporting it as "all threads already engaged".
|
|
2300
|
+
if [ "${DRAFT_ONLY:-0}" = "1" ] && [ -n "$PREP_REASON" ]; then
|
|
2301
|
+
echo "DRAFT_ONLY_BLOCKED=$PREP_REASON"
|
|
2302
|
+
fi
|
|
2303
|
+
_SA_RUN_SUMMARY_EMITTED=1
|
|
2304
|
+
exit 0
|
|
2305
|
+
fi
|
|
2306
|
+
|
|
2307
|
+
# --- Phase 2b-gen: SEO landing pages (no browser lock) ----------------------
|
|
2308
|
+
# phase2b-gen has the longest budget (60 min) because the SEO landing-page
|
|
2309
|
+
# build can legitimately run 10-40 min. Stamping it here is what protects
|
|
2310
|
+
# this cycle from being salvaged out from under itself.
|
|
2311
|
+
python3 "$REPO_DIR/scripts/twitter_batch_phase.py" advance "$BATCH_ID" --phase phase2b-gen 2>&1 | tee -a "$LOG_FILE" || true
|
|
2312
|
+
log "Phase 2b-gen: generating SEO pages for $PLAN_COUNT candidate(s) without holding the browser lock..."
|
|
2313
|
+
python3 "$REPO_DIR/scripts/twitter_gen_links.py" --plan "$PLAN_FILE" 2>&1 | tee -a "$LOG_FILE"
|
|
2314
|
+
GEN_EXIT=${PIPESTATUS[0]:-1}
|
|
2315
|
+
if [ "$GEN_EXIT" -ne 0 ]; then
|
|
2316
|
+
log "WARN: twitter_gen_links.py exited $GEN_EXIT, continuing with whatever links it set (per-candidate fallback to plain project URL on gen failure)."
|
|
2317
|
+
fi
|
|
2318
|
+
|
|
2319
|
+
# --- DRAFT_ONLY gate: stop after drafting for human review (MCP manual mode) -
|
|
2320
|
+
# When DRAFT_ONLY=1 the cycle runs scan -> score -> draft -> link-gen, leaves the
|
|
2321
|
+
# fully-baked plan (links already stamped into reply_text) at $PLAN_FILE, and
|
|
2322
|
+
# STOPS before posting. The social-autoposter MCP draft_cycle tool reads that
|
|
2323
|
+
# plan, walks the human through approve/skip per draft, then posts the approved
|
|
2324
|
+
# subset via twitter_post_plan.py. Nothing is posted from this script in that
|
|
2325
|
+
# mode. The gate sits AFTER 2b-gen on purpose: twitter_post_plan.py does not run
|
|
2326
|
+
# link-gen itself, so the plan must already carry baked links before we hand it
|
|
2327
|
+
# off. Run with TWITTER_PAGE_GEN_RATE=0 (the default) so gen is a sub-second
|
|
2328
|
+
# plain-URL rewrite, not a 10-40 min SEO build, in the interactive path.
|
|
2329
|
+
if [ "${DRAFT_ONLY:-0}" = "1" ]; then
|
|
2330
|
+
# Not posting, so the browser lock isn't needed; release it if still held.
|
|
2331
|
+
release_lock "twitter-browser" 2>>"$LOG_FILE" || true
|
|
2332
|
+
# (2026-06-16) session-lock rm removed (defect b); dead holders self-reclaim
|
|
2333
|
+
# in twitter_browser.py now. Do NOT re-add. See Phase 1 note + docs/twitter_browser_lock.md.
|
|
2334
|
+
log "DRAFT_ONLY=1: $PLAN_COUNT draft(s) ready for review at $PLAN_FILE. Stopping before post."
|
|
2335
|
+
# Emit a clean posted=0 run row and suppress the EXIT-trap summary oneshot, so
|
|
2336
|
+
# a draft-only run is NOT mis-synthesized as a phase2b_silent failure (the
|
|
2337
|
+
# trap's fallback would do that for posted=0 with candidates pending).
|
|
2338
|
+
_COST=$(python3 "$REPO_DIR/scripts/get_run_cost.py" --cycle-id "$BATCH_ID" 2>/dev/null || echo "0.0000")
|
|
2339
|
+
python3 "$REPO_DIR/scripts/log_run.py" --script "post_twitter" --posted 0 --skipped 0 --failed 0 --salvaged "${SALVAGED:-0}" \
|
|
2340
|
+
--queries "${QUERIES_TOTAL:-0}" --duds "${DUDS_TOTAL:-0}" \
|
|
2341
|
+
--tweets-pulled "${TWEETS_PULLED:-0}" --candidates "${BATCH_COUNT:-0}" --above-floor "${HIGH_DELTA_COUNT:-0}" \
|
|
2342
|
+
--cost "$_COST" --elapsed $(( $(date +%s) - RUN_START )) 2>/dev/null || true
|
|
2343
|
+
_SA_RUN_SUMMARY_EMITTED=1
|
|
2344
|
+
# IMPORTANT: do NOT rm -f "$PLAN_FILE" here; the reviewer needs it. Print a
|
|
2345
|
+
# machine-readable marker so the MCP wrapper can locate the plan deterministically.
|
|
2346
|
+
echo "DRAFT_ONLY_PLAN=$PLAN_FILE"
|
|
2347
|
+
exit 0
|
|
2348
|
+
fi
|
|
2349
|
+
|
|
2350
|
+
# --- Phase 2b-post: re-acquire browser lock and post ------------------------
|
|
2351
|
+
# Stamp phase2b-post (15-min budget) before the browser-side reply loop. After
|
|
2352
|
+
# 2b-gen's potentially long run, peer cycles' 20-min phase2a fallback would
|
|
2353
|
+
# already be tripping if we left the row at phase2a.
|
|
2354
|
+
python3 "$REPO_DIR/scripts/twitter_batch_phase.py" advance "$BATCH_ID" --phase phase2b-post 2>&1 | tee -a "$LOG_FILE" || true
|
|
2355
|
+
# Re-acquire only if we actually released for gen (see GEN_IS_NOOP above).
|
|
2356
|
+
# When the lock was kept through 2b-gen there's nothing to re-acquire.
|
|
2357
|
+
if ! $GEN_IS_NOOP; then
|
|
2358
|
+
log "Re-acquiring twitter-browser lock for Phase 2b-post..."
|
|
2359
|
+
acquire_lock "twitter-browser" 3600 2>>"$LOG_FILE"
|
|
2360
|
+
fi
|
|
2361
|
+
log "twitter-browser lock held (pid=$$) Phase 2b-post"
|
|
2362
|
+
# Drop stale singleton locks (see clean_stale_singleton.sh, also called in Phase 1 / 2b-prep).
|
|
2363
|
+
ensure_twitter_browser_for_backend 2>&1 | tee -a "$LOG_FILE"
|
|
2364
|
+
|
|
2365
|
+
log "Phase 2b-post: posting $PLAN_COUNT candidate(s)..."
|
|
2366
|
+
POST_OUTPUT=$("${S4L_PYTHON:-python3}" "$REPO_DIR/scripts/twitter_post_plan.py" --plan "$PLAN_FILE" 2>&1)
|
|
2367
|
+
echo "$POST_OUTPUT" >> "$LOG_FILE"
|
|
2368
|
+
|
|
2369
|
+
# The post helper prints a JSON summary on its last stdout line.
|
|
2370
|
+
POST_SUMMARY=$(printf '%s\n' "$POST_OUTPUT" | tail -n 1)
|
|
2371
|
+
EXEC_POSTED=$(python3 -c "import json,sys; d=json.loads(sys.argv[1] or '{}'); print(d.get('posted', 0))" "$POST_SUMMARY" 2>/dev/null || echo 0)
|
|
2372
|
+
EXEC_SKIPPED=$(python3 -c "import json,sys; d=json.loads(sys.argv[1] or '{}'); print(d.get('skipped', 0))" "$POST_SUMMARY" 2>/dev/null || echo 0)
|
|
2373
|
+
EXEC_FAILED=$(python3 -c "import json,sys; d=json.loads(sys.argv[1] or '{}'); print(d.get('failed', 0))" "$POST_SUMMARY" 2>/dev/null || echo 0)
|
|
2374
|
+
EXEC_REASONS=$(python3 -c "import json,sys; d=json.loads(sys.argv[1] or '{}'); print(d.get('failure_reasons', ''))" "$POST_SUMMARY" 2>/dev/null || echo "")
|
|
2375
|
+
EXEC_SKIP_REASONS=$(python3 -c "import json,sys; d=json.loads(sys.argv[1] or '{}'); print(d.get('skip_reasons', ''))" "$POST_SUMMARY" 2>/dev/null || echo "")
|
|
2376
|
+
log "Phase 2b-post summary: posted=$EXEC_POSTED skipped=$EXEC_SKIPPED failed=$EXEC_FAILED reasons=$EXEC_REASONS skip_reasons=$EXEC_SKIP_REASONS"
|
|
2377
|
+
|
|
2378
|
+
rm -f "$PLAN_FILE"
|
|
2379
|
+
|
|
2380
|
+
# Generation trace tempfile cleanup. By now every post in this cycle that
|
|
2381
|
+
# made it to log_post.py has the trace persisted to posts.generation_trace
|
|
2382
|
+
# JSONB, so the on-disk JSON is redundant. Best-effort delete.
|
|
2383
|
+
if [ -n "$S4L_TWITTER_GEN_TRACE_PATH" ] && [ -f "$S4L_TWITTER_GEN_TRACE_PATH" ]; then
|
|
2384
|
+
rm -f "$S4L_TWITTER_GEN_TRACE_PATH"
|
|
2385
|
+
fi
|
|
2386
|
+
|
|
2387
|
+
# --- No end-of-cycle expire ------------------------------------------------
|
|
2388
|
+
# Pending rows are intentionally left alone. They are either:
|
|
2389
|
+
# - candidates Phase 2b never reached (e.g., org quota, browser crash, or
|
|
2390
|
+
# a phase budget elapsing before the long tail was reviewed), and the
|
|
2391
|
+
# next cycle's Phase 0 will salvage them while still fresh
|
|
2392
|
+
# - hard-expired by the next cycle's Phase 0 once they cross FRESHNESS_HOURS
|
|
2393
|
+
# This avoids losing work to transient infra failures.
|
|
2394
|
+
|
|
2395
|
+
# --- Summary ---------------------------------------------------------------
|
|
2396
|
+
# Per-run-log human readout. The persistent run_monitor.log row is written
|
|
2397
|
+
# by _sa_emit_run_summary_oneshot (defined near the top of this script) so
|
|
2398
|
+
# SIGTERM during the summary block still produces a dashboard-visible row.
|
|
2399
|
+
# Summary now comes from /api/v1/twitter-candidates/counts-by-batch via
|
|
2400
|
+
# the helper, formatted as "status|count\nstatus|count..." to match the
|
|
2401
|
+
# legacy psql -F '|' shape this log line consumed.
|
|
2402
|
+
SUMMARY=$(python3 "$REPO_DIR/scripts/twitter_cycle_helper.py" batch-summary --batch-id "$BATCH_ID" 2>/dev/null)
|
|
2403
|
+
log "Batch summary: $SUMMARY"
|
|
2404
|
+
|
|
2405
|
+
_sa_emit_run_summary_oneshot
|
|
2406
|
+
|
|
2407
|
+
log "=== Cycle complete: $(date) ==="
|
|
2408
|
+
find "$LOG_DIR" -name "twitter-cycle-*.log" -mtime +7 -delete 2>/dev/null || true
|