@m13v/s4l 1.6.197-rc.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (326) hide show
  1. package/README.md +143 -0
  2. package/SKILL.md +342 -0
  3. package/bin/cli.js +980 -0
  4. package/bin/cookie-helper.js +315 -0
  5. package/bin/platform.js +59 -0
  6. package/bin/scheduler/index.js +12 -0
  7. package/bin/scheduler/launchd.js +518 -0
  8. package/browser-agent-configs/all-agents-mcp.json +68 -0
  9. package/browser-agent-configs/linkedin-agent-mcp.json +16 -0
  10. package/browser-agent-configs/linkedin-agent.json +17 -0
  11. package/browser-agent-configs/linkedin-harness-mcp.json +21 -0
  12. package/browser-agent-configs/reddit-agent-mcp.json +16 -0
  13. package/browser-agent-configs/reddit-agent.json +17 -0
  14. package/browser-agent-configs/twitter-harness-mcp.json +18 -0
  15. package/config.example.json +45 -0
  16. package/mcp/dist/index.js +4212 -0
  17. package/mcp/dist/onboarding.js +200 -0
  18. package/mcp/dist/panel.html +176 -0
  19. package/mcp/dist/product-link.html +102 -0
  20. package/mcp/dist/repo.js +222 -0
  21. package/mcp/dist/runtime.js +1079 -0
  22. package/mcp/dist/screencast.js +323 -0
  23. package/mcp/dist/setup.js +545 -0
  24. package/mcp/dist/telemetry.js +306 -0
  25. package/mcp/dist/twitterAuth.js +138 -0
  26. package/mcp/dist/version.js +271 -0
  27. package/mcp/dist/version.json +4 -0
  28. package/mcp/install-runtime.mjs +70 -0
  29. package/mcp/install.mjs +169 -0
  30. package/mcp/manifest.json +80 -0
  31. package/mcp/menubar/dashboard_server.py +213 -0
  32. package/mcp/menubar/s4l_card.py +1336 -0
  33. package/mcp/menubar/s4l_log_relay.py +179 -0
  34. package/mcp/menubar/s4l_menubar.py +2439 -0
  35. package/mcp/menubar/s4l_state.py +891 -0
  36. package/mcp/package.json +34 -0
  37. package/mcp/shared/doctor.cjs +437 -0
  38. package/mcp/shared/onboarding-ledger.cjs +324 -0
  39. package/mcp-servers/browser-harness/server.py +968 -0
  40. package/package.json +160 -0
  41. package/requirements.txt +20 -0
  42. package/scripts/_compute_allowlist.py +58 -0
  43. package/scripts/_db_update.py +20 -0
  44. package/scripts/_filt.py +9 -0
  45. package/scripts/_li_notif_match.py +76 -0
  46. package/scripts/_li_notif_orchestrate.py +126 -0
  47. package/scripts/_lock_preempt_test.py +60 -0
  48. package/scripts/_run_icp_precheck.py +57 -0
  49. package/scripts/a16z_pearx_calendar_reminders.py +99 -0
  50. package/scripts/account_resolver.py +141 -0
  51. package/scripts/active_campaigns.py +114 -0
  52. package/scripts/active_users.py +190 -0
  53. package/scripts/amplitude_24h_signups.py +468 -0
  54. package/scripts/amplitude_signups.py +177 -0
  55. package/scripts/apply_onboarding_selections.py +131 -0
  56. package/scripts/audience_pages.py +243 -0
  57. package/scripts/audit_helper.py +120 -0
  58. package/scripts/author_history_block.py +353 -0
  59. package/scripts/autopilot_stall_watch.py +284 -0
  60. package/scripts/backfill_twitter_attempts_topic.py +81 -0
  61. package/scripts/backfill_twitter_log_post_no_id.py +322 -0
  62. package/scripts/bench_dashboard.sh +138 -0
  63. package/scripts/bh_send.py +39 -0
  64. package/scripts/build_persona.py +409 -0
  65. package/scripts/bulk_icp.py +18 -0
  66. package/scripts/campaign_bump.py +51 -0
  67. package/scripts/capture_thread_media.py +288 -0
  68. package/scripts/check_browser_lock_health.sh +81 -0
  69. package/scripts/check_external_pool_depth.py +253 -0
  70. package/scripts/check_unread_web_chats.py +28 -0
  71. package/scripts/claim_web_chat.py +47 -0
  72. package/scripts/classify_run_error.py +158 -0
  73. package/scripts/claude_job.py +988 -0
  74. package/scripts/clean_stale_singleton.sh +56 -0
  75. package/scripts/cleanup_harness_tabs.py +68 -0
  76. package/scripts/copy_browser_cookies.py +454 -0
  77. package/scripts/counterparty_history.py +350 -0
  78. package/scripts/db.py +57 -0
  79. package/scripts/discover_claude_profiles.py +120 -0
  80. package/scripts/discover_linkedin_candidates.py +984 -0
  81. package/scripts/dm_conversation.py +682 -0
  82. package/scripts/dm_db_update.py +69 -0
  83. package/scripts/dm_engage_helper.py +161 -0
  84. package/scripts/dm_outreach_helper.py +147 -0
  85. package/scripts/dm_outreach_twitter_helper.py +129 -0
  86. package/scripts/dm_send_log.py +106 -0
  87. package/scripts/dm_short_links.py +1084 -0
  88. package/scripts/dump_web_chat_history.py +47 -0
  89. package/scripts/engage_github.py +640 -0
  90. package/scripts/engage_reddit.py +1235 -0
  91. package/scripts/engage_twitter_helper.py +301 -0
  92. package/scripts/engagement_styles.py +1787 -0
  93. package/scripts/enrich_twitter_candidates.py +82 -0
  94. package/scripts/feedback_digest.py +448 -0
  95. package/scripts/fetch_prospect_profile.py +312 -0
  96. package/scripts/fetch_twitter_t1.py +134 -0
  97. package/scripts/find_threads.py +530 -0
  98. package/scripts/follow_gate_log.py +59 -0
  99. package/scripts/funnel_per_day.py +194 -0
  100. package/scripts/generate_daily_human_style.py +494 -0
  101. package/scripts/generation_trace.py +173 -0
  102. package/scripts/get_run_cost.py +107 -0
  103. package/scripts/github_engage_helper.py +93 -0
  104. package/scripts/github_tools.py +509 -0
  105. package/scripts/harness_overlay.py +556 -0
  106. package/scripts/harvest_twitter_following.py +243 -0
  107. package/scripts/heartbeat.sh +70 -0
  108. package/scripts/history_context.py +284 -0
  109. package/scripts/http_api.py +206 -0
  110. package/scripts/human_dm_replies_helper.py +169 -0
  111. package/scripts/identity.py +302 -0
  112. package/scripts/ig_batch_creator.sh +93 -0
  113. package/scripts/ig_post_type_picker.py +243 -0
  114. package/scripts/ig_scrape_transcribe.sh +91 -0
  115. package/scripts/ingest_human_dm_replies.py +271 -0
  116. package/scripts/ingest_web_chat_replies.py +229 -0
  117. package/scripts/install_fleet.py +187 -0
  118. package/scripts/invent_mcp_server.py +350 -0
  119. package/scripts/invent_topics.py +1462 -0
  120. package/scripts/learned_preferences.py +263 -0
  121. package/scripts/li_discovery.py +161 -0
  122. package/scripts/link_edit_helper.py +142 -0
  123. package/scripts/link_tail.py +592 -0
  124. package/scripts/linkedin_api.py +561 -0
  125. package/scripts/linkedin_browser.py +730 -0
  126. package/scripts/linkedin_cooldown.py +128 -0
  127. package/scripts/linkedin_exclusions.py +234 -0
  128. package/scripts/linkedin_killswitch.py +1333 -0
  129. package/scripts/linkedin_search_topic_schema.py +49 -0
  130. package/scripts/linkedin_unipile.py +658 -0
  131. package/scripts/linkedin_url.py +228 -0
  132. package/scripts/log_claude_session.py +636 -0
  133. package/scripts/log_draft.py +143 -0
  134. package/scripts/log_linkedin_search_attempts.py +126 -0
  135. package/scripts/log_post.py +651 -0
  136. package/scripts/log_run.py +364 -0
  137. package/scripts/log_thread_media.py +108 -0
  138. package/scripts/log_twitter_search_attempts.py +150 -0
  139. package/scripts/log_twitter_skips.py +211 -0
  140. package/scripts/lookup_post.py +78 -0
  141. package/scripts/mark_web_chat_processed.py +32 -0
  142. package/scripts/mcp_lock_proxy.py +370 -0
  143. package/scripts/memory_snapshot.py +972 -0
  144. package/scripts/merge_review_queue.py +215 -0
  145. package/scripts/mint_external_pool.py +182 -0
  146. package/scripts/mint_kent_pool.py +249 -0
  147. package/scripts/moltbook_post.py +320 -0
  148. package/scripts/moltbook_tools.py +159 -0
  149. package/scripts/pending_threads.py +188 -0
  150. package/scripts/pick_ig_account.py +177 -0
  151. package/scripts/pick_project.py +208 -0
  152. package/scripts/pick_search_topic.py +771 -0
  153. package/scripts/pick_thread_target.py +279 -0
  154. package/scripts/pick_twitter_thread_target.py +202 -0
  155. package/scripts/podlog_fetch_batch.sh +32 -0
  156. package/scripts/post_github.py +1311 -0
  157. package/scripts/post_reddit.py +2668 -0
  158. package/scripts/precompute_dashboard_stats.py +204 -0
  159. package/scripts/preflight.sh +297 -0
  160. package/scripts/progress.py +88 -0
  161. package/scripts/project_excludes.py +353 -0
  162. package/scripts/project_slugs.py +91 -0
  163. package/scripts/project_stats.py +241 -0
  164. package/scripts/project_stats_json.py +1563 -0
  165. package/scripts/project_topics.py +192 -0
  166. package/scripts/qualified_query_bank.py +436 -0
  167. package/scripts/reap_stale_claude_sessions.py +867 -0
  168. package/scripts/reddit_browser.py +2549 -0
  169. package/scripts/reddit_browser_fetch.py +141 -0
  170. package/scripts/reddit_browser_lock.py +593 -0
  171. package/scripts/reddit_chat_sync.py +710 -0
  172. package/scripts/reddit_query_bank.py +200 -0
  173. package/scripts/reddit_threads_helper.py +151 -0
  174. package/scripts/reddit_tools.py +956 -0
  175. package/scripts/refresh_instagram_tokens.py +280 -0
  176. package/scripts/release-mcpb.sh +513 -0
  177. package/scripts/reply_db.py +334 -0
  178. package/scripts/reply_insert.py +98 -0
  179. package/scripts/reply_risk_digest.py +761 -0
  180. package/scripts/reset-test-machine.sh +602 -0
  181. package/scripts/restore_twitter_session.py +177 -0
  182. package/scripts/ripen_reddit_plan.py +478 -0
  183. package/scripts/run_claude.sh +433 -0
  184. package/scripts/run_moltbook_cycle.py +555 -0
  185. package/scripts/s4l_box_update.sh +226 -0
  186. package/scripts/s4l_channel.py +103 -0
  187. package/scripts/s4l_ctl.sh +75 -0
  188. package/scripts/s4l_env.py +47 -0
  189. package/scripts/saps_activity.py +126 -0
  190. package/scripts/saps_mode.py +328 -0
  191. package/scripts/scan_dm_candidates.py +580 -0
  192. package/scripts/scan_github_replies.py +168 -0
  193. package/scripts/scan_instagram_comments.py +481 -0
  194. package/scripts/scan_moltbook_replies.py +252 -0
  195. package/scripts/scan_pii.py +190 -0
  196. package/scripts/scan_reddit_replies.py +377 -0
  197. package/scripts/scan_twitter_mentions_browser.py +327 -0
  198. package/scripts/scan_twitter_thread_followups.py +299 -0
  199. package/scripts/scan_x_profile.py +384 -0
  200. package/scripts/schedule_state.py +202 -0
  201. package/scripts/scheduled_tasks_snapshot.py +123 -0
  202. package/scripts/score_linkedin_candidates.py +419 -0
  203. package/scripts/score_twitter_candidates.py +718 -0
  204. package/scripts/scrape_linkedin_comment_stats.py +1755 -0
  205. package/scripts/scrape_linkedin_stats_browser.py +52 -0
  206. package/scripts/scrape_reddit_views.py +365 -0
  207. package/scripts/seed_search_queries.py +453 -0
  208. package/scripts/seed_search_topics.py +127 -0
  209. package/scripts/send_web_chat_reply.py +130 -0
  210. package/scripts/sentry_init.py +128 -0
  211. package/scripts/setup_twitter_auth.py +1320 -0
  212. package/scripts/snapshot.py +583 -0
  213. package/scripts/stats.py +2702 -0
  214. package/scripts/stats_helper.py +52 -0
  215. package/scripts/strike_alert.py +783 -0
  216. package/scripts/sweep_post_link_clicks.py +107 -0
  217. package/scripts/sync_ig_to_posts.py +147 -0
  218. package/scripts/test_browser_lock.py +189 -0
  219. package/scripts/test_installation_api.sh +52 -0
  220. package/scripts/test_percard_posting.py +142 -0
  221. package/scripts/top_dud_linkedin_queries.py +71 -0
  222. package/scripts/top_dud_reddit_queries.py +67 -0
  223. package/scripts/top_dud_twitter_queries.py +71 -0
  224. package/scripts/top_dud_twitter_topics.py +102 -0
  225. package/scripts/top_linkedin_queries.py +55 -0
  226. package/scripts/top_omitted_reddit_topics.py +91 -0
  227. package/scripts/top_performers.py +588 -0
  228. package/scripts/top_search_topics.py +180 -0
  229. package/scripts/top_twitter_queries.py +190 -0
  230. package/scripts/twitter_access_check.py +382 -0
  231. package/scripts/twitter_account.py +41 -0
  232. package/scripts/twitter_batch_phase.py +126 -0
  233. package/scripts/twitter_browser.py +2804 -0
  234. package/scripts/twitter_cookie_mirror.py +130 -0
  235. package/scripts/twitter_cycle_helper.py +310 -0
  236. package/scripts/twitter_gen_links.py +287 -0
  237. package/scripts/twitter_post_plan.py +1188 -0
  238. package/scripts/twitter_scan.py +324 -0
  239. package/scripts/twitter_supply_signal.py +57 -0
  240. package/scripts/twitter_threads_helper.py +152 -0
  241. package/scripts/unclaim_web_chat.py +29 -0
  242. package/scripts/update_instagram_stats.py +261 -0
  243. package/scripts/update_linkedin_stats_from_feed.py +328 -0
  244. package/scripts/version.py +72 -0
  245. package/scripts/watchdog_hung_runs.py +343 -0
  246. package/scripts/write_generation_trace.py +73 -0
  247. package/setup/SKILL.md +277 -0
  248. package/skill/amplitude-24h-signups.sh +38 -0
  249. package/skill/archive-old-logs.sh +40 -0
  250. package/skill/audit-dm-staleness.sh +42 -0
  251. package/skill/audit-linkedin.sh +14 -0
  252. package/skill/audit-moltbook.sh +4 -0
  253. package/skill/audit-reddit-resurrect.sh +67 -0
  254. package/skill/audit-reddit.sh +4 -0
  255. package/skill/audit-twitter.sh +4 -0
  256. package/skill/audit.sh +287 -0
  257. package/skill/backfill-twitter-attempts-topic.sh +19 -0
  258. package/skill/backfill-twitter-ghost-posts.sh +24 -0
  259. package/skill/check-external-pool-depth.sh +7 -0
  260. package/skill/check-web-chats.sh +203 -0
  261. package/skill/dm-outreach-linkedin.sh +250 -0
  262. package/skill/dm-outreach-reddit.sh +274 -0
  263. package/skill/dm-outreach-twitter.sh +265 -0
  264. package/skill/engage-dm-replies-linkedin.sh +4 -0
  265. package/skill/engage-dm-replies-reddit.sh +4 -0
  266. package/skill/engage-dm-replies-twitter.sh +4 -0
  267. package/skill/engage-dm-replies.sh +1597 -0
  268. package/skill/engage-linkedin.sh +581 -0
  269. package/skill/engage-moltbook.sh +36 -0
  270. package/skill/engage-reddit.sh +146 -0
  271. package/skill/engage-twitter.sh +467 -0
  272. package/skill/github-engage.sh +176 -0
  273. package/skill/ingest-web-chat-replies.sh +38 -0
  274. package/skill/invent-supply-test.sh +100 -0
  275. package/skill/invent-topics.sh +50 -0
  276. package/skill/lib/linkedin-backend.sh +364 -0
  277. package/skill/lib/platform.sh +48 -0
  278. package/skill/lib/reddit-backend.sh +234 -0
  279. package/skill/lib/twitter-backend.sh +314 -0
  280. package/skill/link-edit-github.sh +136 -0
  281. package/skill/link-edit-moltbook.sh +117 -0
  282. package/skill/link-edit-reddit.sh +201 -0
  283. package/skill/linkedin-presence.sh +182 -0
  284. package/skill/linkedin-recovery.sh +282 -0
  285. package/skill/lock.sh +647 -0
  286. package/skill/memory-snapshot.sh +39 -0
  287. package/skill/precompute-stats.sh +35 -0
  288. package/skill/prewarm-funnel.sh +104 -0
  289. package/skill/refresh-instagram-tokens.sh +57 -0
  290. package/skill/refresh-twitter-following.sh +52 -0
  291. package/skill/reply-risk-digest.sh +31 -0
  292. package/skill/run-cycle-update-guard.sh +44 -0
  293. package/skill/run-draft-and-publish.sh +123 -0
  294. package/skill/run-generate-daily-style.sh +50 -0
  295. package/skill/run-github-launchd.sh +62 -0
  296. package/skill/run-github.sh +102 -0
  297. package/skill/run-instagram-daily.sh +149 -0
  298. package/skill/run-instagram-render.sh +875 -0
  299. package/skill/run-linkedin-launchd.sh +81 -0
  300. package/skill/run-linkedin-unipile.sh +130 -0
  301. package/skill/run-linkedin.sh +1593 -0
  302. package/skill/run-moltbook-launchd.sh +61 -0
  303. package/skill/run-moltbook.sh +38 -0
  304. package/skill/run-overlay-watch.sh +100 -0
  305. package/skill/run-reddit-search-launchd.sh +64 -0
  306. package/skill/run-reddit-search.sh +505 -0
  307. package/skill/run-reddit-threads-double.sh +32 -0
  308. package/skill/run-reddit-threads.sh +847 -0
  309. package/skill/run-scan-moltbook-replies.sh +57 -0
  310. package/skill/run-twitter-cycle-launchd.sh +63 -0
  311. package/skill/run-twitter-cycle-singleton.sh +62 -0
  312. package/skill/run-twitter-cycle.sh +2408 -0
  313. package/skill/run-twitter-threads.sh +592 -0
  314. package/skill/scan-instagram-replies.sh +61 -0
  315. package/skill/scan-twitter-followups.sh +57 -0
  316. package/skill/social-autoposter-update.sh +66 -0
  317. package/skill/stats-instagram.sh +72 -0
  318. package/skill/stats-linkedin.sh +271 -0
  319. package/skill/stats-moltbook.sh +4 -0
  320. package/skill/stats-reddit.sh +4 -0
  321. package/skill/stats-twitter.sh +4 -0
  322. package/skill/stats.sh +521 -0
  323. package/skill/strike-alert.sh +18 -0
  324. package/skill/styles.sh +87 -0
  325. package/skill/sweep-link-clicks.sh +40 -0
  326. package/skill/topics.sh +51 -0
@@ -0,0 +1,204 @@
1
+ #!/usr/bin/env python3
2
+ """Precompute dashboard stat snapshots to disk so the dashboard never cold-starts.
3
+
4
+ Writes atomic JSON snapshots under ~/social-autoposter/skill/cache/:
5
+ - funnel_stats_<N>d.json for N in {1, 7, 14, 30, 90} (Top -> Pages + funnel)
6
+ - activity_stats_<H>h.json for H in {24, 168, 336, 720} (Activity tab counts)
7
+ - style_stats_<H>h.json for H in {24, 168, 336, 720} (Style tab, all/all)
8
+
9
+ Run on a launchd timer (see com.m13v.social-precompute-stats.plist). The
10
+ /api/funnel/stats, /api/activity/stats, and /api/style/stats endpoints in
11
+ bin/server.js read these files when fresh; live queries only run on miss.
12
+ """
13
+
14
+ import json
15
+ import os
16
+ import subprocess
17
+ import sys
18
+ import tempfile
19
+ import time
20
+ from datetime import datetime, timezone
21
+
22
+ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
23
+
24
+ from http_api import api_get, api_post, load_env
25
+
26
+ REPO_DIR = os.path.expanduser("~/social-autoposter")
27
+ CACHE_DIR = os.path.join(REPO_DIR, "skill", "cache")
28
+ SCRIPTS_DIR = os.path.join(REPO_DIR, "scripts")
29
+
30
+
31
+ def upsert_cache(key, payload):
32
+ """Mirror a snapshot to dashboard_cache over HTTP so Cloud Run (which has no
33
+ access to the operator's filesystem) can serve it. Tolerant: a mirror
34
+ failure logs and continues, since local disk is still the primary path."""
35
+ try:
36
+ api_post(
37
+ "/api/v1/dashboard/cache-upsert",
38
+ {"cache_key": key, "payload": payload},
39
+ )
40
+ except SystemExit as e:
41
+ print(f" [api] cache-upsert {key} failed: {e}", file=sys.stderr)
42
+ except Exception as e:
43
+ print(f" [api] cache-upsert {key} failed: {e}", file=sys.stderr)
44
+
45
+
46
+ def atomic_write_json(path, payload):
47
+ """Write JSON to `path` atomically (temp file + rename). Also mirrors
48
+ to Postgres dashboard_cache under the filename stem so hosted deploys can
49
+ read the same snapshot."""
50
+ os.makedirs(os.path.dirname(path), exist_ok=True)
51
+ fd, tmp = tempfile.mkstemp(dir=os.path.dirname(path), prefix=".tmp-", suffix=".json")
52
+ try:
53
+ with os.fdopen(fd, "w") as f:
54
+ json.dump(payload, f)
55
+ os.replace(tmp, path)
56
+ except Exception:
57
+ try: os.unlink(tmp)
58
+ except Exception: pass
59
+ raise
60
+ key = os.path.splitext(os.path.basename(path))[0]
61
+ upsert_cache(key, payload)
62
+
63
+
64
+ def precompute_funnel(days):
65
+ """Shell out to project_stats_json.py (it already knows how to build the
66
+ payload and hits PostHog + bookings DB). Returns parsed JSON or None."""
67
+ script = os.path.join(SCRIPTS_DIR, "project_stats_json.py")
68
+ t0 = time.time()
69
+ try:
70
+ out = subprocess.check_output(
71
+ ["python3", script, "--days", str(days)],
72
+ cwd=REPO_DIR,
73
+ env=os.environ.copy(),
74
+ timeout=180,
75
+ )
76
+ except subprocess.CalledProcessError as e:
77
+ print(f" funnel days={days} FAILED exit={e.returncode}: {e.stderr or e.output!r}", file=sys.stderr)
78
+ return None
79
+ except subprocess.TimeoutExpired:
80
+ print(f" funnel days={days} TIMEOUT after 180s", file=sys.stderr)
81
+ return None
82
+ try:
83
+ data = json.loads(out)
84
+ except Exception as e:
85
+ print(f" funnel days={days} JSON decode failed: {e}", file=sys.stderr)
86
+ return None
87
+ # Match the wire shape /api/funnel/stats returns: { days, ...data, cachedAt }
88
+ payload = {"days": days, **data, "cachedAt": int(time.time() * 1000)}
89
+ path = os.path.join(CACHE_DIR, f"funnel_stats_{days}d.json")
90
+ atomic_write_json(path, payload)
91
+ elapsed = time.time() - t0
92
+ print(f" funnel days={days} ok ({elapsed:.1f}s) -> {path}")
93
+ return payload
94
+
95
+
96
+ def precompute_activity(hours=24):
97
+ """Mirror the 15-way activity UNION (now served by
98
+ GET /api/v1/dashboard/activity-stats)."""
99
+ t0 = time.time()
100
+ resp = api_get("/api/v1/dashboard/activity-stats", query={"hours": int(hours)})
101
+ value = (resp.get("data") or {}).get("rows") or []
102
+ payload = {
103
+ "windowHours": int(hours),
104
+ "rows": value,
105
+ "cachedAt": int(time.time() * 1000),
106
+ }
107
+ path = os.path.join(CACHE_DIR, f"activity_stats_{int(hours)}h.json")
108
+ atomic_write_json(path, payload)
109
+ elapsed = time.time() - t0
110
+ print(f" activity hours={hours} ok ({elapsed:.1f}s) -> {path}")
111
+ return payload
112
+
113
+
114
+ def precompute_style(hours=24):
115
+ """Mirror the engagement-style aggregate (now served by
116
+ GET /api/v1/dashboard/style-stats) for the default all/all filter the
117
+ dashboard asks for on load."""
118
+ t0 = time.time()
119
+ resp = api_get("/api/v1/dashboard/style-stats", query={"hours": int(hours)})
120
+ data = resp.get("data") or {}
121
+ payload = {
122
+ "windowHours": int(hours),
123
+ "platform": "all",
124
+ "project": "all",
125
+ "rows": data.get("rows") or [],
126
+ "platforms": data.get("platforms") or [],
127
+ "projects": data.get("projects") or [],
128
+ "cachedAt": int(time.time() * 1000),
129
+ }
130
+ path = os.path.join(CACHE_DIR, f"style_stats_{int(hours)}h.json")
131
+ atomic_write_json(path, payload)
132
+ elapsed = time.time() - t0
133
+ print(f" style hours={hours} ok ({elapsed:.1f}s) -> {path}")
134
+ return payload
135
+
136
+
137
+ def main():
138
+ load_env()
139
+ os.makedirs(CACHE_DIR, exist_ok=True)
140
+
141
+ started = datetime.now(timezone.utc).isoformat()
142
+ print(f"=== precompute_dashboard_stats: {started} ===")
143
+ overall_t0 = time.time()
144
+
145
+ # Activity + style snapshots, one per Stats-tab window pill
146
+ # (24h / 7d / 14d / 30d = 24 / 168 / 336 / 720 hours). The dashboard's
147
+ # readSnapshotCached gate rejects anything older than 15 min, so every
148
+ # window must refresh every cycle or it falls through to the live query
149
+ # (the 15-way activity UNION costs ~15s under load and blocks Node's
150
+ # single event loop, freezing the whole dashboard). Pre-2026-05-30 only
151
+ # 24h was precomputed, so 7d/14d/30d hit the live path on every switch.
152
+ STATS_WINDOW_HOURS = (24, 168, 336, 720)
153
+ for h in STATS_WINDOW_HOURS:
154
+ try:
155
+ precompute_activity(h)
156
+ except Exception as e:
157
+ print(f" activity hours={h} FAILED: {e}", file=sys.stderr)
158
+ try:
159
+ precompute_style(h)
160
+ except Exception as e:
161
+ print(f" style hours={h} FAILED: {e}", file=sys.stderr)
162
+
163
+ # Funnel snapshots: one per window the dashboard pills can show.
164
+ #
165
+ # The job fires every 5 min, but each funnel window re-queries every
166
+ # PostHog bucket (~10 HogQL queries each). Recomputing all 5 windows
167
+ # every cycle = ~5x the query burst, which trips PostHog's short-window
168
+ # rate limiter (429 "throttled") and leaves whole buckets errored ('err'
169
+ # on the dashboard). The longer windows barely move between 5-min cycles,
170
+ # so only 1d + 7d refresh every cycle; 14/30/90d refresh at most every
171
+ # ~25 min (skipped while their snapshot is still fresh). This cuts the
172
+ # steady-state PostHog query volume by ~3/5 with no meaningful staleness.
173
+ HEAVY_WINDOW_MIN_AGE_S = 25 * 60
174
+ # Small gap between window runs so two adjacent window subprocesses don't
175
+ # stack their bursts back-to-back into the rate limiter.
176
+ INTER_WINDOW_SLEEP_S = 3
177
+ windows = (1, 7, 14, 30, 90)
178
+ for d in windows:
179
+ if d >= 14:
180
+ snap_path = os.path.join(CACHE_DIR, f"funnel_stats_{d}d.json")
181
+ try:
182
+ age = time.time() - os.path.getmtime(snap_path)
183
+ except OSError:
184
+ age = None # missing -> always compute
185
+ if age is not None and age < HEAVY_WINDOW_MIN_AGE_S:
186
+ print(f" funnel days={d} skipped (snapshot {age/60:.0f}m old < 25m)")
187
+ continue
188
+ try:
189
+ precompute_funnel(d)
190
+ except Exception as e:
191
+ print(f" funnel days={d} FAILED: {e}", file=sys.stderr)
192
+ time.sleep(INTER_WINDOW_SLEEP_S)
193
+
194
+ # Stamp a marker so ops can see when the last full cycle finished.
195
+ atomic_write_json(
196
+ os.path.join(CACHE_DIR, "_last_run.json"),
197
+ {"finished_at": datetime.now(timezone.utc).isoformat(),
198
+ "elapsed_sec": round(time.time() - overall_t0, 2)},
199
+ )
200
+ print(f"=== done in {time.time() - overall_t0:.1f}s ===")
201
+
202
+
203
+ if __name__ == "__main__":
204
+ main()
@@ -0,0 +1,297 @@
1
+ #!/bin/bash
2
+ # preflight.sh — sourced helper for launchd-fired run-*.sh wrappers.
3
+ #
4
+ # Three checks, each emits a `[skipped: <reason>]` stderr line and exits 0
5
+ # (so launchd treats the slot as cleanly consumed and fires the next one
6
+ # on schedule, rather than thinking the job is broken):
7
+ #
8
+ # 1. preflight_skip_if_jetsam_pressure
9
+ # Reads kern.memorystatus_vm_pressure_level (1=normal, 2=warn,
10
+ # 4=urgent, 8=critical). Skips when >= 2. Background: 2026-05-01
11
+ # a JetsamEvent at 19:26 swallowed two consecutive launchd fires
12
+ # of run-twitter-cycle (19:38, 19:53) — wrappers fired but the
13
+ # grandchild bash never produced output, presumably jetsam-killed
14
+ # or starved during the system's crash-cleanup spike. Skipping
15
+ # cleanly when pressure is already elevated avoids stacking more
16
+ # Chrome+Claude+Python work onto an already-thrashing system.
17
+ #
18
+ # 2. preflight_skip_if_claude_blocked
19
+ # Reads /tmp/sa-claude-blocked.json. If `blocked_until > now`,
20
+ # skips. Stamp is written by scripts/run_claude.sh when claude
21
+ # emits a recognized fatal-quota error (monthly cap, daily cap,
22
+ # org budget, context-window exceeded, credit balance, persistent
23
+ # 429). Default block window: 600s; once expired, the next fire
24
+ # proceeds normally and either (a) succeeds, in which case the
25
+ # stamp is auto-cleared, or (b) hits the same error and refreshes
26
+ # the stamp for another 600s. This prevents launchd from burning
27
+ # a fire every cadence-tick during a multi-hour outage while
28
+ # still recovering automatically within 10 min of the underlying
29
+ # cap being lifted.
30
+ #
31
+ # 3. preflight_acquire_slot_or_skip <pool_name> [max_slots=4]
32
+ # Slot-pool admission control via mkdir on
33
+ # /tmp/sa-${pool_name}-slot-{1..max_slots}.lock. If all slots are
34
+ # held by live PIDs, skips. Stale slots (PID dead) are GC'd before
35
+ # the acquire pass. Used by run-twitter-cycle.sh to cap concurrent
36
+ # cycles at 4 (post 2026-04-30 the launchd wrapper double-forks
37
+ # and no longer suppresses overlapping fires, so this is the only
38
+ # guardrail against ramp-up under sustained pressure).
39
+ #
40
+ # Sourcing requirements:
41
+ # - Source AFTER skill/lock.sh if you want both. preflight.sh chains
42
+ # its slot cleanup with lock.sh's _sa_release_locks via a combined
43
+ # EXIT trap (replaces lock.sh's trap; calls _sa_release_locks if
44
+ # defined, then releases preflight slots).
45
+ # - Source BEFORE the script-specific cleanup trap if any (the script
46
+ # can install its own trap that calls _preflight_release_slots
47
+ # itself; see run-twitter-cycle.sh for that pattern).
48
+
49
+ # Slot-pool array — initialised once per shell so multiple acquire calls
50
+ # in the same script stack cleanly.
51
+ if [ -z "${_SA_PREFLIGHT_SLOTS+x}" ]; then
52
+ declare -a _SA_PREFLIGHT_SLOTS=()
53
+ fi
54
+
55
+ _preflight_release_slots() {
56
+ local d
57
+ for d in ${_SA_PREFLIGHT_SLOTS[@]+"${_SA_PREFLIGHT_SLOTS[@]}"}; do
58
+ rm -rf "$d" 2>/dev/null || true
59
+ done
60
+ }
61
+
62
+ # Combined exit handler: clean preflight slots AND chain lock.sh cleanup
63
+ # if it's been sourced. Installed unconditionally on first source so
64
+ # slot leaks never outlive the script even if the caller forgets to
65
+ # install its own trap.
66
+ _preflight_combined_exit() {
67
+ _preflight_release_slots
68
+ if command -v _sa_release_locks >/dev/null 2>&1; then
69
+ _sa_release_locks
70
+ fi
71
+ }
72
+ trap _preflight_combined_exit EXIT INT TERM HUP
73
+
74
+ # ---------------------------------------------------------------------------
75
+ # 1. Memory-pressure preflight
76
+ # ---------------------------------------------------------------------------
77
+ preflight_skip_if_jetsam_pressure() {
78
+ # DISABLED (2026-06-04, by user request): this guard kept false-positive-
79
+ # skipping healthy cycles because kern.memorystatus_vm_pressure_level is
80
+ # sticky (latches at 2 for hours after a transient spike while RAM is
81
+ # actually fine). Now a no-op so it never blocks a cycle. Kept as a
82
+ # named function because run-twitter-cycle.sh (locked) still calls it.
83
+ return 0
84
+ }
85
+
86
+ # ---------------------------------------------------------------------------
87
+ # 2. Claude-quota stamp preflight
88
+ # ---------------------------------------------------------------------------
89
+ # Stamp file is JSON:
90
+ # {
91
+ # "reason": "monthly_limit|daily_limit|context_window|credit_balance|...",
92
+ # "stamped_at": "2026-05-02T18:00:00Z",
93
+ # "blocked_until": "2026-05-02T18:10:00Z",
94
+ # "stamped_by_session": "<uuid>",
95
+ # "stamped_by_script": "run-twitter-cycle"
96
+ # }
97
+ # Single source of truth across all pipelines. Written by run_claude.sh,
98
+ # read by every launchd wrapper. Blocking is per-machine, not per-pipeline,
99
+ # because every pipeline shares the same Anthropic org quota.
100
+ SA_CLAUDE_BLOCK_STAMP="${SA_CLAUDE_BLOCK_STAMP:-/tmp/sa-claude-blocked.json}"
101
+
102
+ preflight_skip_if_claude_blocked() {
103
+ [ -f "$SA_CLAUDE_BLOCK_STAMP" ] || return 0
104
+
105
+ # Pull blocked_until + reason in one python invocation. Falls through
106
+ # to "not blocked" on any parse failure (corrupt stamp -> recover).
107
+ local payload
108
+ payload=$(/usr/bin/python3 - <<'PY' "$SA_CLAUDE_BLOCK_STAMP" 2>/dev/null
109
+ import json, sys, os
110
+ from datetime import datetime, timezone
111
+ try:
112
+ with open(sys.argv[1]) as f:
113
+ d = json.load(f)
114
+ bu = d.get("blocked_until", "")
115
+ if not bu:
116
+ sys.exit(0)
117
+ # Tolerate trailing Z or +00:00.
118
+ bu_norm = bu.replace("Z", "+00:00")
119
+ until = datetime.fromisoformat(bu_norm)
120
+ now = datetime.now(timezone.utc)
121
+ remaining = int((until - now).total_seconds())
122
+ print(f"{remaining}|{d.get('reason','unknown')}|{d.get('stamped_by_script','?')}|{bu}")
123
+ except Exception:
124
+ pass
125
+ PY
126
+ )
127
+ [ -z "$payload" ] && return 0
128
+
129
+ local remaining reason stamped_by stamped_until
130
+ IFS='|' read -r remaining reason stamped_by stamped_until <<< "$payload"
131
+
132
+ if [ -z "$remaining" ]; then
133
+ return 0
134
+ fi
135
+
136
+ if [ "$remaining" -gt 0 ]; then
137
+ local script_tag="${SA_PREFLIGHT_SCRIPT:-${SCRIPT_TAG:-$(basename "$0")}}"
138
+ echo "[skipped: claude_blocked reason=$reason expires_in=${remaining}s stamped_by=$stamped_by until=$stamped_until script=$script_tag] $(date)" >&2
139
+ exit 0
140
+ fi
141
+
142
+ # Stamp expired. Best-effort cleanup so the next pipeline doesn't
143
+ # repeat the parse. If a parallel script is mid-write of a fresh
144
+ # stamp we lose the race harmlessly — they'll just re-write below.
145
+ rm -f "$SA_CLAUDE_BLOCK_STAMP" 2>/dev/null || true
146
+ }
147
+
148
+ # ---------------------------------------------------------------------------
149
+ # 3. Slot-pool admission (parallel-cycle cap)
150
+ # ---------------------------------------------------------------------------
151
+ preflight_acquire_slot_or_skip() {
152
+ local pool_name="$1"
153
+ local max_slots="${2:-4}"
154
+ local pass i pid slot_dir
155
+
156
+ if [ -z "$pool_name" ]; then
157
+ echo "preflight_acquire_slot_or_skip: pool_name required" >&2
158
+ return 1
159
+ fi
160
+
161
+ # Two passes:
162
+ # pass=1: GC slots whose holder PID is dead (clean SIGKILL / OOM).
163
+ # pass=2: try to claim the first free slot.
164
+ for pass in 1 2; do
165
+ for i in $(seq 1 "$max_slots"); do
166
+ slot_dir="/tmp/sa-${pool_name}-slot-${i}.lock"
167
+ if [ "$pass" = "1" ]; then
168
+ if [ -d "$slot_dir" ]; then
169
+ pid=$(cat "$slot_dir/pid" 2>/dev/null || echo "")
170
+ if [ -z "$pid" ] || ! kill -0 "$pid" 2>/dev/null; then
171
+ rm -rf "$slot_dir" 2>/dev/null || true
172
+ fi
173
+ fi
174
+ else
175
+ if mkdir "$slot_dir" 2>/dev/null; then
176
+ echo $$ > "$slot_dir/pid"
177
+ _SA_PREFLIGHT_SLOTS+=("$slot_dir")
178
+ return 0
179
+ fi
180
+ fi
181
+ done
182
+ done
183
+
184
+ # All slots taken — count + report and skip.
185
+ local active=0
186
+ for i in $(seq 1 "$max_slots"); do
187
+ [ -d "/tmp/sa-${pool_name}-slot-${i}.lock" ] && active=$((active + 1))
188
+ done
189
+ local script_tag="${SA_PREFLIGHT_SCRIPT:-${SCRIPT_TAG:-$(basename "$0")}}"
190
+ echo "[skipped: too_many_inflight pool=$pool_name max=$max_slots active=$active script=$script_tag] $(date)" >&2
191
+ exit 0
192
+ }
193
+
194
+ # ---------------------------------------------------------------------------
195
+ # Stamp helpers (used by run_claude.sh after claude exits with quota error).
196
+ # Exposed here so any caller can also stamp manually if it detects a quota
197
+ # signal outside of run_claude.sh (e.g. python script directly hitting the
198
+ # Anthropic API).
199
+ # ---------------------------------------------------------------------------
200
+
201
+ # Write/refresh the block stamp.
202
+ # $1 = reason (monthly_limit | daily_limit | context_window | credit_balance | rate_limit_persistent | unknown)
203
+ # $2 = duration_seconds (default 600)
204
+ # $3 = optional script tag
205
+ # $4 = optional session id
206
+ preflight_stamp_claude_blocked() {
207
+ local reason="${1:-unknown}"
208
+ local duration="${2:-600}"
209
+ local script_tag="${3:-${SA_PREFLIGHT_SCRIPT:-${SCRIPT_TAG:-unknown}}}"
210
+ local session="${4:-${CLAUDE_SESSION_ID:-unknown}}"
211
+
212
+ /usr/bin/python3 - "$SA_CLAUDE_BLOCK_STAMP" "$reason" "$duration" "$script_tag" "$session" <<'PY'
213
+ import json, sys, os, tempfile
214
+ from datetime import datetime, timezone, timedelta
215
+ path, reason, duration, script_tag, session = sys.argv[1:6]
216
+ duration = int(duration)
217
+ now = datetime.now(timezone.utc)
218
+ until = now + timedelta(seconds=duration)
219
+ payload = {
220
+ "reason": reason,
221
+ "stamped_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
222
+ "blocked_until": until.strftime("%Y-%m-%dT%H:%M:%SZ"),
223
+ "stamped_by_session": session,
224
+ "stamped_by_script": script_tag,
225
+ "duration_seconds": duration,
226
+ }
227
+ # If a stamp already exists with later expiry, keep the later one.
228
+ try:
229
+ with open(path) as f:
230
+ existing = json.load(f)
231
+ eu = existing.get("blocked_until", "")
232
+ if eu:
233
+ e_until = datetime.fromisoformat(eu.replace("Z", "+00:00"))
234
+ if e_until > until:
235
+ # Existing stamp blocks longer; preserve it but bump reason.
236
+ existing["reason"] = reason
237
+ payload = existing
238
+ except Exception:
239
+ pass
240
+ # Atomic write.
241
+ tmp = tempfile.NamedTemporaryFile("w", dir=os.path.dirname(path) or "/tmp",
242
+ delete=False, prefix=".sa-claude-blocked.", suffix=".tmp")
243
+ json.dump(payload, tmp)
244
+ tmp.close()
245
+ os.replace(tmp.name, path)
246
+ print(f"[claude_quota] stamped reason={reason} until={payload['blocked_until']} duration={duration}s", file=sys.stderr)
247
+ PY
248
+ }
249
+
250
+ # Clear the stamp (called when a fresh claude run succeeds, signalling
251
+ # the underlying cap has lifted).
252
+ preflight_clear_claude_block() {
253
+ [ -f "$SA_CLAUDE_BLOCK_STAMP" ] || return 0
254
+ rm -f "$SA_CLAUDE_BLOCK_STAMP" 2>/dev/null || true
255
+ echo "[claude_quota] cleared block stamp (claude run succeeded)" >&2
256
+ }
257
+
258
+ # Inspect a claude transcript / log for known fatal-quota error patterns.
259
+ # Reads from path argument or stdin. Echoes the matched reason on stdout
260
+ # (one of: monthly_limit | daily_limit | context_window | credit_balance |
261
+ # rate_limit_persistent | empty if no match). Exit 0 always.
262
+ #
263
+ # Patterns are intentionally broad — false positives stamp a 10-min skip
264
+ # which self-clears on next try. False negatives let the caller burn an
265
+ # entire cycle's budget on a doomed run, which is the worse failure.
266
+ preflight_classify_claude_error() {
267
+ local source_file="${1:-/dev/stdin}"
268
+ /usr/bin/python3 - "$source_file" <<'PY'
269
+ import sys, re, os
270
+ path = sys.argv[1]
271
+ try:
272
+ if path == "/dev/stdin":
273
+ text = sys.stdin.read()
274
+ else:
275
+ with open(path, "r", errors="replace") as f:
276
+ text = f.read()
277
+ except Exception:
278
+ sys.exit(0)
279
+
280
+ low = text.lower()
281
+
282
+ # Order matters — most-specific first.
283
+ patterns = [
284
+ ("monthly_limit", [r"monthly\s+usage\s+limit", r"hit your org's monthly", r"monthly\s+limit\s+reached", r"month'?s\s+(?:usage|allowance|cap)"]),
285
+ ("daily_limit", [r"daily\s+rate\s+limit", r"daily\s+usage\s+limit", r"daily\s+limit\s+reached", r"day'?s\s+(?:usage|allowance|cap)"]),
286
+ ("credit_balance", [r"credit\s+balance\s+is\s+too\s+low", r"insufficient\s+credit", r"out\s+of\s+credits"]),
287
+ ("context_window", [r"context[\s_-]?length\s+(?:exceeded|too\s+long)", r"context[\s_-]?window\s+(?:exceeded|too\s+long)", r"prompt\s+is\s+too\s+long", r"max(?:imum)?\s+context\s+(?:length|window)"]),
288
+ ("rate_limit_persistent", [r"5[\s-]?hour\s+(?:rate\s+)?limit", r"rate_limit_5h", r"per[\s-]?5h\s+(?:rate\s+)?limit"]),
289
+ ]
290
+ for reason, regexes in patterns:
291
+ for rx in regexes:
292
+ if re.search(rx, low):
293
+ print(reason)
294
+ sys.exit(0)
295
+ sys.exit(0)
296
+ PY
297
+ }
@@ -0,0 +1,88 @@
1
+ #!/usr/bin/env python3
2
+ """Tiny progress heartbeat writer for long-running stats jobs.
3
+
4
+ Each call atomically replaces skill/cache/progress_<platform>.json with a
5
+ snapshot of where the job is. Readers (dashboard, CLI status check, humans)
6
+ can cat the file at any time - or if the job dies mid-run, the last heartbeat
7
+ survives so we know how far it got before being killed (watchdog, Claude
8
+ rate limit, OS OOM, etc.).
9
+
10
+ Writes are best-effort: any failure here is swallowed so a broken disk or
11
+ permission issue never breaks the stats job itself.
12
+
13
+ CLI usage:
14
+ python3 scripts/progress.py # show all current heartbeats
15
+ python3 scripts/progress.py github # show only github
16
+ """
17
+
18
+ import json
19
+ import os
20
+ import sys
21
+ import tempfile
22
+ import time
23
+ from pathlib import Path
24
+
25
+ CACHE_DIR = Path(__file__).resolve().parent.parent / "skill" / "cache"
26
+
27
+
28
+ def tick(platform, done, total, **extras):
29
+ """Write a heartbeat showing `done`/`total` for `platform`.
30
+
31
+ Extra fields (updated, errors, deleted, state, etc.) are merged in.
32
+ """
33
+ try:
34
+ CACHE_DIR.mkdir(parents=True, exist_ok=True)
35
+ path = CACHE_DIR / f"progress_{platform}.json"
36
+ now = time.time()
37
+ payload = {
38
+ "platform": platform,
39
+ "done": done,
40
+ "total": total,
41
+ "pid": os.getpid(),
42
+ "updated_at": time.strftime("%Y-%m-%dT%H:%M:%S%z", time.localtime(now)),
43
+ "updated_at_ts": int(now),
44
+ **extras,
45
+ }
46
+ fd, tmp = tempfile.mkstemp(prefix=f".progress_{platform}_",
47
+ suffix=".json", dir=str(CACHE_DIR))
48
+ try:
49
+ with os.fdopen(fd, "w") as f:
50
+ json.dump(payload, f)
51
+ os.replace(tmp, path)
52
+ except Exception:
53
+ try:
54
+ os.unlink(tmp)
55
+ except Exception:
56
+ pass
57
+ raise
58
+ except Exception:
59
+ pass
60
+
61
+
62
+ def done(platform, total, **extras):
63
+ """Mark `platform` as completed. Final tick, done==total, state=done."""
64
+ tick(platform, total, total, state="done", **extras)
65
+
66
+
67
+ def _show(platform=None):
68
+ if not CACHE_DIR.exists():
69
+ return
70
+ files = sorted(CACHE_DIR.glob("progress_*.json"))
71
+ for f in files:
72
+ name = f.stem.replace("progress_", "")
73
+ if platform and name != platform:
74
+ continue
75
+ try:
76
+ data = json.loads(f.read_text())
77
+ except Exception:
78
+ continue
79
+ age = int(time.time() - data.get("updated_at_ts", 0))
80
+ state = data.get("state", "running")
81
+ done_n = data.get("done", 0)
82
+ total_n = data.get("total", 0)
83
+ pct = f"{100 * done_n / total_n:.1f}%" if total_n else "?"
84
+ print(f"{name:10} {state:8} {done_n}/{total_n} ({pct}) pid={data.get('pid')} {age}s ago @ {data.get('updated_at')}")
85
+
86
+
87
+ if __name__ == "__main__":
88
+ _show(sys.argv[1] if len(sys.argv) > 1 else None)