@m13v/s4l 1.6.197-rc.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +143 -0
- package/SKILL.md +342 -0
- package/bin/cli.js +980 -0
- package/bin/cookie-helper.js +315 -0
- package/bin/platform.js +59 -0
- package/bin/scheduler/index.js +12 -0
- package/bin/scheduler/launchd.js +518 -0
- package/browser-agent-configs/all-agents-mcp.json +68 -0
- package/browser-agent-configs/linkedin-agent-mcp.json +16 -0
- package/browser-agent-configs/linkedin-agent.json +17 -0
- package/browser-agent-configs/linkedin-harness-mcp.json +21 -0
- package/browser-agent-configs/reddit-agent-mcp.json +16 -0
- package/browser-agent-configs/reddit-agent.json +17 -0
- package/browser-agent-configs/twitter-harness-mcp.json +18 -0
- package/config.example.json +45 -0
- package/mcp/dist/index.js +4212 -0
- package/mcp/dist/onboarding.js +200 -0
- package/mcp/dist/panel.html +176 -0
- package/mcp/dist/product-link.html +102 -0
- package/mcp/dist/repo.js +222 -0
- package/mcp/dist/runtime.js +1079 -0
- package/mcp/dist/screencast.js +323 -0
- package/mcp/dist/setup.js +545 -0
- package/mcp/dist/telemetry.js +306 -0
- package/mcp/dist/twitterAuth.js +138 -0
- package/mcp/dist/version.js +271 -0
- package/mcp/dist/version.json +4 -0
- package/mcp/install-runtime.mjs +70 -0
- package/mcp/install.mjs +169 -0
- package/mcp/manifest.json +80 -0
- package/mcp/menubar/dashboard_server.py +213 -0
- package/mcp/menubar/s4l_card.py +1314 -0
- package/mcp/menubar/s4l_log_relay.py +179 -0
- package/mcp/menubar/s4l_menubar.py +2439 -0
- package/mcp/menubar/s4l_state.py +891 -0
- package/mcp/package.json +34 -0
- package/mcp/shared/doctor.cjs +437 -0
- package/mcp/shared/onboarding-ledger.cjs +324 -0
- package/mcp-servers/browser-harness/server.py +968 -0
- package/package.json +160 -0
- package/requirements.txt +20 -0
- package/scripts/_compute_allowlist.py +58 -0
- package/scripts/_db_update.py +20 -0
- package/scripts/_filt.py +9 -0
- package/scripts/_li_notif_match.py +76 -0
- package/scripts/_li_notif_orchestrate.py +126 -0
- package/scripts/_lock_preempt_test.py +60 -0
- package/scripts/_run_icp_precheck.py +57 -0
- package/scripts/a16z_pearx_calendar_reminders.py +99 -0
- package/scripts/account_resolver.py +141 -0
- package/scripts/active_campaigns.py +114 -0
- package/scripts/active_users.py +190 -0
- package/scripts/amplitude_24h_signups.py +468 -0
- package/scripts/amplitude_signups.py +177 -0
- package/scripts/apply_onboarding_selections.py +131 -0
- package/scripts/audience_pages.py +243 -0
- package/scripts/audit_helper.py +120 -0
- package/scripts/author_history_block.py +353 -0
- package/scripts/autopilot_stall_watch.py +284 -0
- package/scripts/backfill_twitter_attempts_topic.py +81 -0
- package/scripts/backfill_twitter_log_post_no_id.py +322 -0
- package/scripts/bench_dashboard.sh +138 -0
- package/scripts/bh_send.py +39 -0
- package/scripts/build_persona.py +409 -0
- package/scripts/bulk_icp.py +18 -0
- package/scripts/campaign_bump.py +51 -0
- package/scripts/capture_thread_media.py +288 -0
- package/scripts/check_browser_lock_health.sh +81 -0
- package/scripts/check_external_pool_depth.py +253 -0
- package/scripts/check_unread_web_chats.py +28 -0
- package/scripts/claim_web_chat.py +47 -0
- package/scripts/classify_run_error.py +158 -0
- package/scripts/claude_job.py +988 -0
- package/scripts/clean_stale_singleton.sh +56 -0
- package/scripts/cleanup_harness_tabs.py +68 -0
- package/scripts/copy_browser_cookies.py +454 -0
- package/scripts/counterparty_history.py +350 -0
- package/scripts/db.py +57 -0
- package/scripts/discover_claude_profiles.py +120 -0
- package/scripts/discover_linkedin_candidates.py +984 -0
- package/scripts/dm_conversation.py +682 -0
- package/scripts/dm_db_update.py +69 -0
- package/scripts/dm_engage_helper.py +161 -0
- package/scripts/dm_outreach_helper.py +147 -0
- package/scripts/dm_outreach_twitter_helper.py +129 -0
- package/scripts/dm_send_log.py +106 -0
- package/scripts/dm_short_links.py +1084 -0
- package/scripts/dump_web_chat_history.py +47 -0
- package/scripts/engage_github.py +640 -0
- package/scripts/engage_reddit.py +1235 -0
- package/scripts/engage_twitter_helper.py +301 -0
- package/scripts/engagement_styles.py +1787 -0
- package/scripts/enrich_twitter_candidates.py +82 -0
- package/scripts/feedback_digest.py +448 -0
- package/scripts/fetch_prospect_profile.py +312 -0
- package/scripts/fetch_twitter_t1.py +134 -0
- package/scripts/find_threads.py +530 -0
- package/scripts/follow_gate_log.py +59 -0
- package/scripts/funnel_per_day.py +194 -0
- package/scripts/generate_daily_human_style.py +494 -0
- package/scripts/generation_trace.py +173 -0
- package/scripts/get_run_cost.py +107 -0
- package/scripts/github_engage_helper.py +93 -0
- package/scripts/github_tools.py +509 -0
- package/scripts/harness_overlay.py +556 -0
- package/scripts/harvest_twitter_following.py +243 -0
- package/scripts/heartbeat.sh +70 -0
- package/scripts/history_context.py +284 -0
- package/scripts/http_api.py +206 -0
- package/scripts/human_dm_replies_helper.py +169 -0
- package/scripts/identity.py +302 -0
- package/scripts/ig_batch_creator.sh +93 -0
- package/scripts/ig_post_type_picker.py +243 -0
- package/scripts/ig_scrape_transcribe.sh +91 -0
- package/scripts/ingest_human_dm_replies.py +271 -0
- package/scripts/ingest_web_chat_replies.py +229 -0
- package/scripts/install_fleet.py +187 -0
- package/scripts/invent_mcp_server.py +350 -0
- package/scripts/invent_topics.py +1462 -0
- package/scripts/learned_preferences.py +263 -0
- package/scripts/li_discovery.py +161 -0
- package/scripts/link_edit_helper.py +142 -0
- package/scripts/link_tail.py +592 -0
- package/scripts/linkedin_api.py +561 -0
- package/scripts/linkedin_browser.py +730 -0
- package/scripts/linkedin_cooldown.py +128 -0
- package/scripts/linkedin_exclusions.py +234 -0
- package/scripts/linkedin_killswitch.py +1333 -0
- package/scripts/linkedin_search_topic_schema.py +49 -0
- package/scripts/linkedin_unipile.py +658 -0
- package/scripts/linkedin_url.py +228 -0
- package/scripts/log_claude_session.py +636 -0
- package/scripts/log_draft.py +143 -0
- package/scripts/log_linkedin_search_attempts.py +126 -0
- package/scripts/log_post.py +651 -0
- package/scripts/log_run.py +364 -0
- package/scripts/log_thread_media.py +108 -0
- package/scripts/log_twitter_search_attempts.py +150 -0
- package/scripts/log_twitter_skips.py +211 -0
- package/scripts/lookup_post.py +78 -0
- package/scripts/mark_web_chat_processed.py +32 -0
- package/scripts/mcp_lock_proxy.py +370 -0
- package/scripts/memory_snapshot.py +972 -0
- package/scripts/merge_review_queue.py +215 -0
- package/scripts/mint_external_pool.py +182 -0
- package/scripts/mint_kent_pool.py +249 -0
- package/scripts/moltbook_post.py +320 -0
- package/scripts/moltbook_tools.py +159 -0
- package/scripts/pending_threads.py +188 -0
- package/scripts/pick_ig_account.py +177 -0
- package/scripts/pick_project.py +208 -0
- package/scripts/pick_search_topic.py +771 -0
- package/scripts/pick_thread_target.py +279 -0
- package/scripts/pick_twitter_thread_target.py +202 -0
- package/scripts/podlog_fetch_batch.sh +32 -0
- package/scripts/post_github.py +1311 -0
- package/scripts/post_reddit.py +2668 -0
- package/scripts/precompute_dashboard_stats.py +204 -0
- package/scripts/preflight.sh +297 -0
- package/scripts/progress.py +88 -0
- package/scripts/project_excludes.py +353 -0
- package/scripts/project_slugs.py +91 -0
- package/scripts/project_stats.py +241 -0
- package/scripts/project_stats_json.py +1563 -0
- package/scripts/project_topics.py +192 -0
- package/scripts/qualified_query_bank.py +436 -0
- package/scripts/reap_stale_claude_sessions.py +867 -0
- package/scripts/reddit_browser.py +2549 -0
- package/scripts/reddit_browser_fetch.py +141 -0
- package/scripts/reddit_browser_lock.py +593 -0
- package/scripts/reddit_chat_sync.py +710 -0
- package/scripts/reddit_query_bank.py +200 -0
- package/scripts/reddit_threads_helper.py +151 -0
- package/scripts/reddit_tools.py +956 -0
- package/scripts/refresh_instagram_tokens.py +280 -0
- package/scripts/release-mcpb.sh +497 -0
- package/scripts/reply_db.py +334 -0
- package/scripts/reply_insert.py +98 -0
- package/scripts/reply_risk_digest.py +761 -0
- package/scripts/reset-test-machine.sh +602 -0
- package/scripts/restore_twitter_session.py +177 -0
- package/scripts/ripen_reddit_plan.py +478 -0
- package/scripts/run_claude.sh +433 -0
- package/scripts/run_moltbook_cycle.py +555 -0
- package/scripts/s4l_box_update.sh +226 -0
- package/scripts/s4l_channel.py +103 -0
- package/scripts/s4l_ctl.sh +75 -0
- package/scripts/s4l_env.py +47 -0
- package/scripts/saps_activity.py +126 -0
- package/scripts/saps_mode.py +328 -0
- package/scripts/scan_dm_candidates.py +580 -0
- package/scripts/scan_github_replies.py +168 -0
- package/scripts/scan_instagram_comments.py +481 -0
- package/scripts/scan_moltbook_replies.py +252 -0
- package/scripts/scan_pii.py +190 -0
- package/scripts/scan_reddit_replies.py +377 -0
- package/scripts/scan_twitter_mentions_browser.py +327 -0
- package/scripts/scan_twitter_thread_followups.py +299 -0
- package/scripts/scan_x_profile.py +384 -0
- package/scripts/schedule_state.py +202 -0
- package/scripts/scheduled_tasks_snapshot.py +123 -0
- package/scripts/score_linkedin_candidates.py +419 -0
- package/scripts/score_twitter_candidates.py +718 -0
- package/scripts/scrape_linkedin_comment_stats.py +1755 -0
- package/scripts/scrape_linkedin_stats_browser.py +52 -0
- package/scripts/scrape_reddit_views.py +365 -0
- package/scripts/seed_search_queries.py +453 -0
- package/scripts/seed_search_topics.py +127 -0
- package/scripts/send_web_chat_reply.py +130 -0
- package/scripts/sentry_init.py +128 -0
- package/scripts/setup_twitter_auth.py +1320 -0
- package/scripts/snapshot.py +583 -0
- package/scripts/stats.py +2702 -0
- package/scripts/stats_helper.py +52 -0
- package/scripts/strike_alert.py +783 -0
- package/scripts/sweep_post_link_clicks.py +107 -0
- package/scripts/sync_ig_to_posts.py +147 -0
- package/scripts/test_browser_lock.py +189 -0
- package/scripts/test_installation_api.sh +52 -0
- package/scripts/test_percard_posting.py +142 -0
- package/scripts/top_dud_linkedin_queries.py +71 -0
- package/scripts/top_dud_reddit_queries.py +67 -0
- package/scripts/top_dud_twitter_queries.py +71 -0
- package/scripts/top_dud_twitter_topics.py +102 -0
- package/scripts/top_linkedin_queries.py +55 -0
- package/scripts/top_omitted_reddit_topics.py +91 -0
- package/scripts/top_performers.py +588 -0
- package/scripts/top_search_topics.py +180 -0
- package/scripts/top_twitter_queries.py +190 -0
- package/scripts/twitter_access_check.py +382 -0
- package/scripts/twitter_account.py +41 -0
- package/scripts/twitter_batch_phase.py +126 -0
- package/scripts/twitter_browser.py +2804 -0
- package/scripts/twitter_cookie_mirror.py +130 -0
- package/scripts/twitter_cycle_helper.py +310 -0
- package/scripts/twitter_gen_links.py +287 -0
- package/scripts/twitter_post_plan.py +1188 -0
- package/scripts/twitter_scan.py +324 -0
- package/scripts/twitter_supply_signal.py +57 -0
- package/scripts/twitter_threads_helper.py +152 -0
- package/scripts/unclaim_web_chat.py +29 -0
- package/scripts/update_instagram_stats.py +261 -0
- package/scripts/update_linkedin_stats_from_feed.py +328 -0
- package/scripts/version.py +72 -0
- package/scripts/watchdog_hung_runs.py +343 -0
- package/scripts/write_generation_trace.py +73 -0
- package/setup/SKILL.md +277 -0
- package/skill/amplitude-24h-signups.sh +38 -0
- package/skill/archive-old-logs.sh +40 -0
- package/skill/audit-dm-staleness.sh +42 -0
- package/skill/audit-linkedin.sh +14 -0
- package/skill/audit-moltbook.sh +4 -0
- package/skill/audit-reddit-resurrect.sh +67 -0
- package/skill/audit-reddit.sh +4 -0
- package/skill/audit-twitter.sh +4 -0
- package/skill/audit.sh +287 -0
- package/skill/backfill-twitter-attempts-topic.sh +19 -0
- package/skill/backfill-twitter-ghost-posts.sh +24 -0
- package/skill/check-external-pool-depth.sh +7 -0
- package/skill/check-web-chats.sh +203 -0
- package/skill/dm-outreach-linkedin.sh +250 -0
- package/skill/dm-outreach-reddit.sh +274 -0
- package/skill/dm-outreach-twitter.sh +265 -0
- package/skill/engage-dm-replies-linkedin.sh +4 -0
- package/skill/engage-dm-replies-reddit.sh +4 -0
- package/skill/engage-dm-replies-twitter.sh +4 -0
- package/skill/engage-dm-replies.sh +1597 -0
- package/skill/engage-linkedin.sh +581 -0
- package/skill/engage-moltbook.sh +36 -0
- package/skill/engage-reddit.sh +146 -0
- package/skill/engage-twitter.sh +467 -0
- package/skill/github-engage.sh +176 -0
- package/skill/ingest-web-chat-replies.sh +38 -0
- package/skill/invent-supply-test.sh +100 -0
- package/skill/invent-topics.sh +50 -0
- package/skill/lib/linkedin-backend.sh +364 -0
- package/skill/lib/platform.sh +48 -0
- package/skill/lib/reddit-backend.sh +234 -0
- package/skill/lib/twitter-backend.sh +314 -0
- package/skill/link-edit-github.sh +136 -0
- package/skill/link-edit-moltbook.sh +117 -0
- package/skill/link-edit-reddit.sh +201 -0
- package/skill/linkedin-presence.sh +182 -0
- package/skill/linkedin-recovery.sh +282 -0
- package/skill/lock.sh +647 -0
- package/skill/memory-snapshot.sh +39 -0
- package/skill/precompute-stats.sh +35 -0
- package/skill/prewarm-funnel.sh +104 -0
- package/skill/refresh-instagram-tokens.sh +57 -0
- package/skill/refresh-twitter-following.sh +52 -0
- package/skill/reply-risk-digest.sh +31 -0
- package/skill/run-cycle-update-guard.sh +44 -0
- package/skill/run-draft-and-publish.sh +123 -0
- package/skill/run-generate-daily-style.sh +50 -0
- package/skill/run-github-launchd.sh +62 -0
- package/skill/run-github.sh +102 -0
- package/skill/run-instagram-daily.sh +149 -0
- package/skill/run-instagram-render.sh +875 -0
- package/skill/run-linkedin-launchd.sh +81 -0
- package/skill/run-linkedin-unipile.sh +130 -0
- package/skill/run-linkedin.sh +1593 -0
- package/skill/run-moltbook-launchd.sh +61 -0
- package/skill/run-moltbook.sh +38 -0
- package/skill/run-overlay-watch.sh +100 -0
- package/skill/run-reddit-search-launchd.sh +64 -0
- package/skill/run-reddit-search.sh +505 -0
- package/skill/run-reddit-threads-double.sh +32 -0
- package/skill/run-reddit-threads.sh +847 -0
- package/skill/run-scan-moltbook-replies.sh +57 -0
- package/skill/run-twitter-cycle-launchd.sh +63 -0
- package/skill/run-twitter-cycle-singleton.sh +62 -0
- package/skill/run-twitter-cycle.sh +2408 -0
- package/skill/run-twitter-threads.sh +592 -0
- package/skill/scan-instagram-replies.sh +61 -0
- package/skill/scan-twitter-followups.sh +57 -0
- package/skill/social-autoposter-update.sh +66 -0
- package/skill/stats-instagram.sh +72 -0
- package/skill/stats-linkedin.sh +271 -0
- package/skill/stats-moltbook.sh +4 -0
- package/skill/stats-reddit.sh +4 -0
- package/skill/stats-twitter.sh +4 -0
- package/skill/stats.sh +521 -0
- package/skill/strike-alert.sh +18 -0
- package/skill/styles.sh +87 -0
- package/skill/sweep-link-clicks.sh +40 -0
- package/skill/topics.sh +51 -0
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Precompute dashboard stat snapshots to disk so the dashboard never cold-starts.
|
|
3
|
+
|
|
4
|
+
Writes atomic JSON snapshots under ~/social-autoposter/skill/cache/:
|
|
5
|
+
- funnel_stats_<N>d.json for N in {1, 7, 14, 30, 90} (Top -> Pages + funnel)
|
|
6
|
+
- activity_stats_<H>h.json for H in {24, 168, 336, 720} (Activity tab counts)
|
|
7
|
+
- style_stats_<H>h.json for H in {24, 168, 336, 720} (Style tab, all/all)
|
|
8
|
+
|
|
9
|
+
Run on a launchd timer (see com.m13v.social-precompute-stats.plist). The
|
|
10
|
+
/api/funnel/stats, /api/activity/stats, and /api/style/stats endpoints in
|
|
11
|
+
bin/server.js read these files when fresh; live queries only run on miss.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import subprocess
|
|
17
|
+
import sys
|
|
18
|
+
import tempfile
|
|
19
|
+
import time
|
|
20
|
+
from datetime import datetime, timezone
|
|
21
|
+
|
|
22
|
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
23
|
+
|
|
24
|
+
from http_api import api_get, api_post, load_env
|
|
25
|
+
|
|
26
|
+
REPO_DIR = os.path.expanduser("~/social-autoposter")
|
|
27
|
+
CACHE_DIR = os.path.join(REPO_DIR, "skill", "cache")
|
|
28
|
+
SCRIPTS_DIR = os.path.join(REPO_DIR, "scripts")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def upsert_cache(key, payload):
|
|
32
|
+
"""Mirror a snapshot to dashboard_cache over HTTP so Cloud Run (which has no
|
|
33
|
+
access to the operator's filesystem) can serve it. Tolerant: a mirror
|
|
34
|
+
failure logs and continues, since local disk is still the primary path."""
|
|
35
|
+
try:
|
|
36
|
+
api_post(
|
|
37
|
+
"/api/v1/dashboard/cache-upsert",
|
|
38
|
+
{"cache_key": key, "payload": payload},
|
|
39
|
+
)
|
|
40
|
+
except SystemExit as e:
|
|
41
|
+
print(f" [api] cache-upsert {key} failed: {e}", file=sys.stderr)
|
|
42
|
+
except Exception as e:
|
|
43
|
+
print(f" [api] cache-upsert {key} failed: {e}", file=sys.stderr)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def atomic_write_json(path, payload):
|
|
47
|
+
"""Write JSON to `path` atomically (temp file + rename). Also mirrors
|
|
48
|
+
to Postgres dashboard_cache under the filename stem so hosted deploys can
|
|
49
|
+
read the same snapshot."""
|
|
50
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
51
|
+
fd, tmp = tempfile.mkstemp(dir=os.path.dirname(path), prefix=".tmp-", suffix=".json")
|
|
52
|
+
try:
|
|
53
|
+
with os.fdopen(fd, "w") as f:
|
|
54
|
+
json.dump(payload, f)
|
|
55
|
+
os.replace(tmp, path)
|
|
56
|
+
except Exception:
|
|
57
|
+
try: os.unlink(tmp)
|
|
58
|
+
except Exception: pass
|
|
59
|
+
raise
|
|
60
|
+
key = os.path.splitext(os.path.basename(path))[0]
|
|
61
|
+
upsert_cache(key, payload)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def precompute_funnel(days):
|
|
65
|
+
"""Shell out to project_stats_json.py (it already knows how to build the
|
|
66
|
+
payload and hits PostHog + bookings DB). Returns parsed JSON or None."""
|
|
67
|
+
script = os.path.join(SCRIPTS_DIR, "project_stats_json.py")
|
|
68
|
+
t0 = time.time()
|
|
69
|
+
try:
|
|
70
|
+
out = subprocess.check_output(
|
|
71
|
+
["python3", script, "--days", str(days)],
|
|
72
|
+
cwd=REPO_DIR,
|
|
73
|
+
env=os.environ.copy(),
|
|
74
|
+
timeout=180,
|
|
75
|
+
)
|
|
76
|
+
except subprocess.CalledProcessError as e:
|
|
77
|
+
print(f" funnel days={days} FAILED exit={e.returncode}: {e.stderr or e.output!r}", file=sys.stderr)
|
|
78
|
+
return None
|
|
79
|
+
except subprocess.TimeoutExpired:
|
|
80
|
+
print(f" funnel days={days} TIMEOUT after 180s", file=sys.stderr)
|
|
81
|
+
return None
|
|
82
|
+
try:
|
|
83
|
+
data = json.loads(out)
|
|
84
|
+
except Exception as e:
|
|
85
|
+
print(f" funnel days={days} JSON decode failed: {e}", file=sys.stderr)
|
|
86
|
+
return None
|
|
87
|
+
# Match the wire shape /api/funnel/stats returns: { days, ...data, cachedAt }
|
|
88
|
+
payload = {"days": days, **data, "cachedAt": int(time.time() * 1000)}
|
|
89
|
+
path = os.path.join(CACHE_DIR, f"funnel_stats_{days}d.json")
|
|
90
|
+
atomic_write_json(path, payload)
|
|
91
|
+
elapsed = time.time() - t0
|
|
92
|
+
print(f" funnel days={days} ok ({elapsed:.1f}s) -> {path}")
|
|
93
|
+
return payload
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def precompute_activity(hours=24):
|
|
97
|
+
"""Mirror the 15-way activity UNION (now served by
|
|
98
|
+
GET /api/v1/dashboard/activity-stats)."""
|
|
99
|
+
t0 = time.time()
|
|
100
|
+
resp = api_get("/api/v1/dashboard/activity-stats", query={"hours": int(hours)})
|
|
101
|
+
value = (resp.get("data") or {}).get("rows") or []
|
|
102
|
+
payload = {
|
|
103
|
+
"windowHours": int(hours),
|
|
104
|
+
"rows": value,
|
|
105
|
+
"cachedAt": int(time.time() * 1000),
|
|
106
|
+
}
|
|
107
|
+
path = os.path.join(CACHE_DIR, f"activity_stats_{int(hours)}h.json")
|
|
108
|
+
atomic_write_json(path, payload)
|
|
109
|
+
elapsed = time.time() - t0
|
|
110
|
+
print(f" activity hours={hours} ok ({elapsed:.1f}s) -> {path}")
|
|
111
|
+
return payload
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def precompute_style(hours=24):
|
|
115
|
+
"""Mirror the engagement-style aggregate (now served by
|
|
116
|
+
GET /api/v1/dashboard/style-stats) for the default all/all filter the
|
|
117
|
+
dashboard asks for on load."""
|
|
118
|
+
t0 = time.time()
|
|
119
|
+
resp = api_get("/api/v1/dashboard/style-stats", query={"hours": int(hours)})
|
|
120
|
+
data = resp.get("data") or {}
|
|
121
|
+
payload = {
|
|
122
|
+
"windowHours": int(hours),
|
|
123
|
+
"platform": "all",
|
|
124
|
+
"project": "all",
|
|
125
|
+
"rows": data.get("rows") or [],
|
|
126
|
+
"platforms": data.get("platforms") or [],
|
|
127
|
+
"projects": data.get("projects") or [],
|
|
128
|
+
"cachedAt": int(time.time() * 1000),
|
|
129
|
+
}
|
|
130
|
+
path = os.path.join(CACHE_DIR, f"style_stats_{int(hours)}h.json")
|
|
131
|
+
atomic_write_json(path, payload)
|
|
132
|
+
elapsed = time.time() - t0
|
|
133
|
+
print(f" style hours={hours} ok ({elapsed:.1f}s) -> {path}")
|
|
134
|
+
return payload
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def main():
|
|
138
|
+
load_env()
|
|
139
|
+
os.makedirs(CACHE_DIR, exist_ok=True)
|
|
140
|
+
|
|
141
|
+
started = datetime.now(timezone.utc).isoformat()
|
|
142
|
+
print(f"=== precompute_dashboard_stats: {started} ===")
|
|
143
|
+
overall_t0 = time.time()
|
|
144
|
+
|
|
145
|
+
# Activity + style snapshots, one per Stats-tab window pill
|
|
146
|
+
# (24h / 7d / 14d / 30d = 24 / 168 / 336 / 720 hours). The dashboard's
|
|
147
|
+
# readSnapshotCached gate rejects anything older than 15 min, so every
|
|
148
|
+
# window must refresh every cycle or it falls through to the live query
|
|
149
|
+
# (the 15-way activity UNION costs ~15s under load and blocks Node's
|
|
150
|
+
# single event loop, freezing the whole dashboard). Pre-2026-05-30 only
|
|
151
|
+
# 24h was precomputed, so 7d/14d/30d hit the live path on every switch.
|
|
152
|
+
STATS_WINDOW_HOURS = (24, 168, 336, 720)
|
|
153
|
+
for h in STATS_WINDOW_HOURS:
|
|
154
|
+
try:
|
|
155
|
+
precompute_activity(h)
|
|
156
|
+
except Exception as e:
|
|
157
|
+
print(f" activity hours={h} FAILED: {e}", file=sys.stderr)
|
|
158
|
+
try:
|
|
159
|
+
precompute_style(h)
|
|
160
|
+
except Exception as e:
|
|
161
|
+
print(f" style hours={h} FAILED: {e}", file=sys.stderr)
|
|
162
|
+
|
|
163
|
+
# Funnel snapshots: one per window the dashboard pills can show.
|
|
164
|
+
#
|
|
165
|
+
# The job fires every 5 min, but each funnel window re-queries every
|
|
166
|
+
# PostHog bucket (~10 HogQL queries each). Recomputing all 5 windows
|
|
167
|
+
# every cycle = ~5x the query burst, which trips PostHog's short-window
|
|
168
|
+
# rate limiter (429 "throttled") and leaves whole buckets errored ('err'
|
|
169
|
+
# on the dashboard). The longer windows barely move between 5-min cycles,
|
|
170
|
+
# so only 1d + 7d refresh every cycle; 14/30/90d refresh at most every
|
|
171
|
+
# ~25 min (skipped while their snapshot is still fresh). This cuts the
|
|
172
|
+
# steady-state PostHog query volume by ~3/5 with no meaningful staleness.
|
|
173
|
+
HEAVY_WINDOW_MIN_AGE_S = 25 * 60
|
|
174
|
+
# Small gap between window runs so two adjacent window subprocesses don't
|
|
175
|
+
# stack their bursts back-to-back into the rate limiter.
|
|
176
|
+
INTER_WINDOW_SLEEP_S = 3
|
|
177
|
+
windows = (1, 7, 14, 30, 90)
|
|
178
|
+
for d in windows:
|
|
179
|
+
if d >= 14:
|
|
180
|
+
snap_path = os.path.join(CACHE_DIR, f"funnel_stats_{d}d.json")
|
|
181
|
+
try:
|
|
182
|
+
age = time.time() - os.path.getmtime(snap_path)
|
|
183
|
+
except OSError:
|
|
184
|
+
age = None # missing -> always compute
|
|
185
|
+
if age is not None and age < HEAVY_WINDOW_MIN_AGE_S:
|
|
186
|
+
print(f" funnel days={d} skipped (snapshot {age/60:.0f}m old < 25m)")
|
|
187
|
+
continue
|
|
188
|
+
try:
|
|
189
|
+
precompute_funnel(d)
|
|
190
|
+
except Exception as e:
|
|
191
|
+
print(f" funnel days={d} FAILED: {e}", file=sys.stderr)
|
|
192
|
+
time.sleep(INTER_WINDOW_SLEEP_S)
|
|
193
|
+
|
|
194
|
+
# Stamp a marker so ops can see when the last full cycle finished.
|
|
195
|
+
atomic_write_json(
|
|
196
|
+
os.path.join(CACHE_DIR, "_last_run.json"),
|
|
197
|
+
{"finished_at": datetime.now(timezone.utc).isoformat(),
|
|
198
|
+
"elapsed_sec": round(time.time() - overall_t0, 2)},
|
|
199
|
+
)
|
|
200
|
+
print(f"=== done in {time.time() - overall_t0:.1f}s ===")
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
if __name__ == "__main__":
|
|
204
|
+
main()
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# preflight.sh — sourced helper for launchd-fired run-*.sh wrappers.
|
|
3
|
+
#
|
|
4
|
+
# Three checks, each emits a `[skipped: <reason>]` stderr line and exits 0
|
|
5
|
+
# (so launchd treats the slot as cleanly consumed and fires the next one
|
|
6
|
+
# on schedule, rather than thinking the job is broken):
|
|
7
|
+
#
|
|
8
|
+
# 1. preflight_skip_if_jetsam_pressure
|
|
9
|
+
# Reads kern.memorystatus_vm_pressure_level (1=normal, 2=warn,
|
|
10
|
+
# 4=urgent, 8=critical). Skips when >= 2. Background: 2026-05-01
|
|
11
|
+
# a JetsamEvent at 19:26 swallowed two consecutive launchd fires
|
|
12
|
+
# of run-twitter-cycle (19:38, 19:53) — wrappers fired but the
|
|
13
|
+
# grandchild bash never produced output, presumably jetsam-killed
|
|
14
|
+
# or starved during the system's crash-cleanup spike. Skipping
|
|
15
|
+
# cleanly when pressure is already elevated avoids stacking more
|
|
16
|
+
# Chrome+Claude+Python work onto an already-thrashing system.
|
|
17
|
+
#
|
|
18
|
+
# 2. preflight_skip_if_claude_blocked
|
|
19
|
+
# Reads /tmp/sa-claude-blocked.json. If `blocked_until > now`,
|
|
20
|
+
# skips. Stamp is written by scripts/run_claude.sh when claude
|
|
21
|
+
# emits a recognized fatal-quota error (monthly cap, daily cap,
|
|
22
|
+
# org budget, context-window exceeded, credit balance, persistent
|
|
23
|
+
# 429). Default block window: 600s; once expired, the next fire
|
|
24
|
+
# proceeds normally and either (a) succeeds, in which case the
|
|
25
|
+
# stamp is auto-cleared, or (b) hits the same error and refreshes
|
|
26
|
+
# the stamp for another 600s. This prevents launchd from burning
|
|
27
|
+
# a fire every cadence-tick during a multi-hour outage while
|
|
28
|
+
# still recovering automatically within 10 min of the underlying
|
|
29
|
+
# cap being lifted.
|
|
30
|
+
#
|
|
31
|
+
# 3. preflight_acquire_slot_or_skip <pool_name> [max_slots=4]
|
|
32
|
+
# Slot-pool admission control via mkdir on
|
|
33
|
+
# /tmp/sa-${pool_name}-slot-{1..max_slots}.lock. If all slots are
|
|
34
|
+
# held by live PIDs, skips. Stale slots (PID dead) are GC'd before
|
|
35
|
+
# the acquire pass. Used by run-twitter-cycle.sh to cap concurrent
|
|
36
|
+
# cycles at 4 (post 2026-04-30 the launchd wrapper double-forks
|
|
37
|
+
# and no longer suppresses overlapping fires, so this is the only
|
|
38
|
+
# guardrail against ramp-up under sustained pressure).
|
|
39
|
+
#
|
|
40
|
+
# Sourcing requirements:
|
|
41
|
+
# - Source AFTER skill/lock.sh if you want both. preflight.sh chains
|
|
42
|
+
# its slot cleanup with lock.sh's _sa_release_locks via a combined
|
|
43
|
+
# EXIT trap (replaces lock.sh's trap; calls _sa_release_locks if
|
|
44
|
+
# defined, then releases preflight slots).
|
|
45
|
+
# - Source BEFORE the script-specific cleanup trap if any (the script
|
|
46
|
+
# can install its own trap that calls _preflight_release_slots
|
|
47
|
+
# itself; see run-twitter-cycle.sh for that pattern).
|
|
48
|
+
|
|
49
|
+
# Slot-pool array — initialised once per shell so multiple acquire calls
|
|
50
|
+
# in the same script stack cleanly.
|
|
51
|
+
if [ -z "${_SA_PREFLIGHT_SLOTS+x}" ]; then
|
|
52
|
+
declare -a _SA_PREFLIGHT_SLOTS=()
|
|
53
|
+
fi
|
|
54
|
+
|
|
55
|
+
_preflight_release_slots() {
|
|
56
|
+
local d
|
|
57
|
+
for d in ${_SA_PREFLIGHT_SLOTS[@]+"${_SA_PREFLIGHT_SLOTS[@]}"}; do
|
|
58
|
+
rm -rf "$d" 2>/dev/null || true
|
|
59
|
+
done
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
# Combined exit handler: clean preflight slots AND chain lock.sh cleanup
|
|
63
|
+
# if it's been sourced. Installed unconditionally on first source so
|
|
64
|
+
# slot leaks never outlive the script even if the caller forgets to
|
|
65
|
+
# install its own trap.
|
|
66
|
+
_preflight_combined_exit() {
|
|
67
|
+
_preflight_release_slots
|
|
68
|
+
if command -v _sa_release_locks >/dev/null 2>&1; then
|
|
69
|
+
_sa_release_locks
|
|
70
|
+
fi
|
|
71
|
+
}
|
|
72
|
+
trap _preflight_combined_exit EXIT INT TERM HUP
|
|
73
|
+
|
|
74
|
+
# ---------------------------------------------------------------------------
|
|
75
|
+
# 1. Memory-pressure preflight
|
|
76
|
+
# ---------------------------------------------------------------------------
|
|
77
|
+
preflight_skip_if_jetsam_pressure() {
|
|
78
|
+
# DISABLED (2026-06-04, by user request): this guard kept false-positive-
|
|
79
|
+
# skipping healthy cycles because kern.memorystatus_vm_pressure_level is
|
|
80
|
+
# sticky (latches at 2 for hours after a transient spike while RAM is
|
|
81
|
+
# actually fine). Now a no-op so it never blocks a cycle. Kept as a
|
|
82
|
+
# named function because run-twitter-cycle.sh (locked) still calls it.
|
|
83
|
+
return 0
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
# ---------------------------------------------------------------------------
|
|
87
|
+
# 2. Claude-quota stamp preflight
|
|
88
|
+
# ---------------------------------------------------------------------------
|
|
89
|
+
# Stamp file is JSON:
|
|
90
|
+
# {
|
|
91
|
+
# "reason": "monthly_limit|daily_limit|context_window|credit_balance|...",
|
|
92
|
+
# "stamped_at": "2026-05-02T18:00:00Z",
|
|
93
|
+
# "blocked_until": "2026-05-02T18:10:00Z",
|
|
94
|
+
# "stamped_by_session": "<uuid>",
|
|
95
|
+
# "stamped_by_script": "run-twitter-cycle"
|
|
96
|
+
# }
|
|
97
|
+
# Single source of truth across all pipelines. Written by run_claude.sh,
|
|
98
|
+
# read by every launchd wrapper. Blocking is per-machine, not per-pipeline,
|
|
99
|
+
# because every pipeline shares the same Anthropic org quota.
|
|
100
|
+
SA_CLAUDE_BLOCK_STAMP="${SA_CLAUDE_BLOCK_STAMP:-/tmp/sa-claude-blocked.json}"
|
|
101
|
+
|
|
102
|
+
preflight_skip_if_claude_blocked() {
|
|
103
|
+
[ -f "$SA_CLAUDE_BLOCK_STAMP" ] || return 0
|
|
104
|
+
|
|
105
|
+
# Pull blocked_until + reason in one python invocation. Falls through
|
|
106
|
+
# to "not blocked" on any parse failure (corrupt stamp -> recover).
|
|
107
|
+
local payload
|
|
108
|
+
payload=$(/usr/bin/python3 - <<'PY' "$SA_CLAUDE_BLOCK_STAMP" 2>/dev/null
|
|
109
|
+
import json, sys, os
|
|
110
|
+
from datetime import datetime, timezone
|
|
111
|
+
try:
|
|
112
|
+
with open(sys.argv[1]) as f:
|
|
113
|
+
d = json.load(f)
|
|
114
|
+
bu = d.get("blocked_until", "")
|
|
115
|
+
if not bu:
|
|
116
|
+
sys.exit(0)
|
|
117
|
+
# Tolerate trailing Z or +00:00.
|
|
118
|
+
bu_norm = bu.replace("Z", "+00:00")
|
|
119
|
+
until = datetime.fromisoformat(bu_norm)
|
|
120
|
+
now = datetime.now(timezone.utc)
|
|
121
|
+
remaining = int((until - now).total_seconds())
|
|
122
|
+
print(f"{remaining}|{d.get('reason','unknown')}|{d.get('stamped_by_script','?')}|{bu}")
|
|
123
|
+
except Exception:
|
|
124
|
+
pass
|
|
125
|
+
PY
|
|
126
|
+
)
|
|
127
|
+
[ -z "$payload" ] && return 0
|
|
128
|
+
|
|
129
|
+
local remaining reason stamped_by stamped_until
|
|
130
|
+
IFS='|' read -r remaining reason stamped_by stamped_until <<< "$payload"
|
|
131
|
+
|
|
132
|
+
if [ -z "$remaining" ]; then
|
|
133
|
+
return 0
|
|
134
|
+
fi
|
|
135
|
+
|
|
136
|
+
if [ "$remaining" -gt 0 ]; then
|
|
137
|
+
local script_tag="${SA_PREFLIGHT_SCRIPT:-${SCRIPT_TAG:-$(basename "$0")}}"
|
|
138
|
+
echo "[skipped: claude_blocked reason=$reason expires_in=${remaining}s stamped_by=$stamped_by until=$stamped_until script=$script_tag] $(date)" >&2
|
|
139
|
+
exit 0
|
|
140
|
+
fi
|
|
141
|
+
|
|
142
|
+
# Stamp expired. Best-effort cleanup so the next pipeline doesn't
|
|
143
|
+
# repeat the parse. If a parallel script is mid-write of a fresh
|
|
144
|
+
# stamp we lose the race harmlessly — they'll just re-write below.
|
|
145
|
+
rm -f "$SA_CLAUDE_BLOCK_STAMP" 2>/dev/null || true
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
# ---------------------------------------------------------------------------
|
|
149
|
+
# 3. Slot-pool admission (parallel-cycle cap)
|
|
150
|
+
# ---------------------------------------------------------------------------
|
|
151
|
+
preflight_acquire_slot_or_skip() {
|
|
152
|
+
local pool_name="$1"
|
|
153
|
+
local max_slots="${2:-4}"
|
|
154
|
+
local pass i pid slot_dir
|
|
155
|
+
|
|
156
|
+
if [ -z "$pool_name" ]; then
|
|
157
|
+
echo "preflight_acquire_slot_or_skip: pool_name required" >&2
|
|
158
|
+
return 1
|
|
159
|
+
fi
|
|
160
|
+
|
|
161
|
+
# Two passes:
|
|
162
|
+
# pass=1: GC slots whose holder PID is dead (clean SIGKILL / OOM).
|
|
163
|
+
# pass=2: try to claim the first free slot.
|
|
164
|
+
for pass in 1 2; do
|
|
165
|
+
for i in $(seq 1 "$max_slots"); do
|
|
166
|
+
slot_dir="/tmp/sa-${pool_name}-slot-${i}.lock"
|
|
167
|
+
if [ "$pass" = "1" ]; then
|
|
168
|
+
if [ -d "$slot_dir" ]; then
|
|
169
|
+
pid=$(cat "$slot_dir/pid" 2>/dev/null || echo "")
|
|
170
|
+
if [ -z "$pid" ] || ! kill -0 "$pid" 2>/dev/null; then
|
|
171
|
+
rm -rf "$slot_dir" 2>/dev/null || true
|
|
172
|
+
fi
|
|
173
|
+
fi
|
|
174
|
+
else
|
|
175
|
+
if mkdir "$slot_dir" 2>/dev/null; then
|
|
176
|
+
echo $$ > "$slot_dir/pid"
|
|
177
|
+
_SA_PREFLIGHT_SLOTS+=("$slot_dir")
|
|
178
|
+
return 0
|
|
179
|
+
fi
|
|
180
|
+
fi
|
|
181
|
+
done
|
|
182
|
+
done
|
|
183
|
+
|
|
184
|
+
# All slots taken — count + report and skip.
|
|
185
|
+
local active=0
|
|
186
|
+
for i in $(seq 1 "$max_slots"); do
|
|
187
|
+
[ -d "/tmp/sa-${pool_name}-slot-${i}.lock" ] && active=$((active + 1))
|
|
188
|
+
done
|
|
189
|
+
local script_tag="${SA_PREFLIGHT_SCRIPT:-${SCRIPT_TAG:-$(basename "$0")}}"
|
|
190
|
+
echo "[skipped: too_many_inflight pool=$pool_name max=$max_slots active=$active script=$script_tag] $(date)" >&2
|
|
191
|
+
exit 0
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
# ---------------------------------------------------------------------------
|
|
195
|
+
# Stamp helpers (used by run_claude.sh after claude exits with quota error).
|
|
196
|
+
# Exposed here so any caller can also stamp manually if it detects a quota
|
|
197
|
+
# signal outside of run_claude.sh (e.g. python script directly hitting the
|
|
198
|
+
# Anthropic API).
|
|
199
|
+
# ---------------------------------------------------------------------------
|
|
200
|
+
|
|
201
|
+
# Write/refresh the block stamp.
|
|
202
|
+
# $1 = reason (monthly_limit | daily_limit | context_window | credit_balance | rate_limit_persistent | unknown)
|
|
203
|
+
# $2 = duration_seconds (default 600)
|
|
204
|
+
# $3 = optional script tag
|
|
205
|
+
# $4 = optional session id
|
|
206
|
+
preflight_stamp_claude_blocked() {
|
|
207
|
+
local reason="${1:-unknown}"
|
|
208
|
+
local duration="${2:-600}"
|
|
209
|
+
local script_tag="${3:-${SA_PREFLIGHT_SCRIPT:-${SCRIPT_TAG:-unknown}}}"
|
|
210
|
+
local session="${4:-${CLAUDE_SESSION_ID:-unknown}}"
|
|
211
|
+
|
|
212
|
+
/usr/bin/python3 - "$SA_CLAUDE_BLOCK_STAMP" "$reason" "$duration" "$script_tag" "$session" <<'PY'
|
|
213
|
+
import json, sys, os, tempfile
|
|
214
|
+
from datetime import datetime, timezone, timedelta
|
|
215
|
+
path, reason, duration, script_tag, session = sys.argv[1:6]
|
|
216
|
+
duration = int(duration)
|
|
217
|
+
now = datetime.now(timezone.utc)
|
|
218
|
+
until = now + timedelta(seconds=duration)
|
|
219
|
+
payload = {
|
|
220
|
+
"reason": reason,
|
|
221
|
+
"stamped_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
222
|
+
"blocked_until": until.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
223
|
+
"stamped_by_session": session,
|
|
224
|
+
"stamped_by_script": script_tag,
|
|
225
|
+
"duration_seconds": duration,
|
|
226
|
+
}
|
|
227
|
+
# If a stamp already exists with later expiry, keep the later one.
|
|
228
|
+
try:
|
|
229
|
+
with open(path) as f:
|
|
230
|
+
existing = json.load(f)
|
|
231
|
+
eu = existing.get("blocked_until", "")
|
|
232
|
+
if eu:
|
|
233
|
+
e_until = datetime.fromisoformat(eu.replace("Z", "+00:00"))
|
|
234
|
+
if e_until > until:
|
|
235
|
+
# Existing stamp blocks longer; preserve it but bump reason.
|
|
236
|
+
existing["reason"] = reason
|
|
237
|
+
payload = existing
|
|
238
|
+
except Exception:
|
|
239
|
+
pass
|
|
240
|
+
# Atomic write.
|
|
241
|
+
tmp = tempfile.NamedTemporaryFile("w", dir=os.path.dirname(path) or "/tmp",
|
|
242
|
+
delete=False, prefix=".sa-claude-blocked.", suffix=".tmp")
|
|
243
|
+
json.dump(payload, tmp)
|
|
244
|
+
tmp.close()
|
|
245
|
+
os.replace(tmp.name, path)
|
|
246
|
+
print(f"[claude_quota] stamped reason={reason} until={payload['blocked_until']} duration={duration}s", file=sys.stderr)
|
|
247
|
+
PY
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
# Clear the stamp (called when a fresh claude run succeeds, signalling
|
|
251
|
+
# the underlying cap has lifted).
|
|
252
|
+
preflight_clear_claude_block() {
|
|
253
|
+
[ -f "$SA_CLAUDE_BLOCK_STAMP" ] || return 0
|
|
254
|
+
rm -f "$SA_CLAUDE_BLOCK_STAMP" 2>/dev/null || true
|
|
255
|
+
echo "[claude_quota] cleared block stamp (claude run succeeded)" >&2
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
# Inspect a claude transcript / log for known fatal-quota error patterns.
|
|
259
|
+
# Reads from path argument or stdin. Echoes the matched reason on stdout
|
|
260
|
+
# (one of: monthly_limit | daily_limit | context_window | credit_balance |
|
|
261
|
+
# rate_limit_persistent | empty if no match). Exit 0 always.
|
|
262
|
+
#
|
|
263
|
+
# Patterns are intentionally broad — false positives stamp a 10-min skip
|
|
264
|
+
# which self-clears on next try. False negatives let the caller burn an
|
|
265
|
+
# entire cycle's budget on a doomed run, which is the worse failure.
|
|
266
|
+
preflight_classify_claude_error() {
|
|
267
|
+
local source_file="${1:-/dev/stdin}"
|
|
268
|
+
/usr/bin/python3 - "$source_file" <<'PY'
|
|
269
|
+
import sys, re, os
|
|
270
|
+
path = sys.argv[1]
|
|
271
|
+
try:
|
|
272
|
+
if path == "/dev/stdin":
|
|
273
|
+
text = sys.stdin.read()
|
|
274
|
+
else:
|
|
275
|
+
with open(path, "r", errors="replace") as f:
|
|
276
|
+
text = f.read()
|
|
277
|
+
except Exception:
|
|
278
|
+
sys.exit(0)
|
|
279
|
+
|
|
280
|
+
low = text.lower()
|
|
281
|
+
|
|
282
|
+
# Order matters — most-specific first.
|
|
283
|
+
patterns = [
|
|
284
|
+
("monthly_limit", [r"monthly\s+usage\s+limit", r"hit your org's monthly", r"monthly\s+limit\s+reached", r"month'?s\s+(?:usage|allowance|cap)"]),
|
|
285
|
+
("daily_limit", [r"daily\s+rate\s+limit", r"daily\s+usage\s+limit", r"daily\s+limit\s+reached", r"day'?s\s+(?:usage|allowance|cap)"]),
|
|
286
|
+
("credit_balance", [r"credit\s+balance\s+is\s+too\s+low", r"insufficient\s+credit", r"out\s+of\s+credits"]),
|
|
287
|
+
("context_window", [r"context[\s_-]?length\s+(?:exceeded|too\s+long)", r"context[\s_-]?window\s+(?:exceeded|too\s+long)", r"prompt\s+is\s+too\s+long", r"max(?:imum)?\s+context\s+(?:length|window)"]),
|
|
288
|
+
("rate_limit_persistent", [r"5[\s-]?hour\s+(?:rate\s+)?limit", r"rate_limit_5h", r"per[\s-]?5h\s+(?:rate\s+)?limit"]),
|
|
289
|
+
]
|
|
290
|
+
for reason, regexes in patterns:
|
|
291
|
+
for rx in regexes:
|
|
292
|
+
if re.search(rx, low):
|
|
293
|
+
print(reason)
|
|
294
|
+
sys.exit(0)
|
|
295
|
+
sys.exit(0)
|
|
296
|
+
PY
|
|
297
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Tiny progress heartbeat writer for long-running stats jobs.
|
|
3
|
+
|
|
4
|
+
Each call atomically replaces skill/cache/progress_<platform>.json with a
|
|
5
|
+
snapshot of where the job is. Readers (dashboard, CLI status check, humans)
|
|
6
|
+
can cat the file at any time - or if the job dies mid-run, the last heartbeat
|
|
7
|
+
survives so we know how far it got before being killed (watchdog, Claude
|
|
8
|
+
rate limit, OS OOM, etc.).
|
|
9
|
+
|
|
10
|
+
Writes are best-effort: any failure here is swallowed so a broken disk or
|
|
11
|
+
permission issue never breaks the stats job itself.
|
|
12
|
+
|
|
13
|
+
CLI usage:
|
|
14
|
+
python3 scripts/progress.py # show all current heartbeats
|
|
15
|
+
python3 scripts/progress.py github # show only github
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
import os
|
|
20
|
+
import sys
|
|
21
|
+
import tempfile
|
|
22
|
+
import time
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
CACHE_DIR = Path(__file__).resolve().parent.parent / "skill" / "cache"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def tick(platform, done, total, **extras):
|
|
29
|
+
"""Write a heartbeat showing `done`/`total` for `platform`.
|
|
30
|
+
|
|
31
|
+
Extra fields (updated, errors, deleted, state, etc.) are merged in.
|
|
32
|
+
"""
|
|
33
|
+
try:
|
|
34
|
+
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
|
35
|
+
path = CACHE_DIR / f"progress_{platform}.json"
|
|
36
|
+
now = time.time()
|
|
37
|
+
payload = {
|
|
38
|
+
"platform": platform,
|
|
39
|
+
"done": done,
|
|
40
|
+
"total": total,
|
|
41
|
+
"pid": os.getpid(),
|
|
42
|
+
"updated_at": time.strftime("%Y-%m-%dT%H:%M:%S%z", time.localtime(now)),
|
|
43
|
+
"updated_at_ts": int(now),
|
|
44
|
+
**extras,
|
|
45
|
+
}
|
|
46
|
+
fd, tmp = tempfile.mkstemp(prefix=f".progress_{platform}_",
|
|
47
|
+
suffix=".json", dir=str(CACHE_DIR))
|
|
48
|
+
try:
|
|
49
|
+
with os.fdopen(fd, "w") as f:
|
|
50
|
+
json.dump(payload, f)
|
|
51
|
+
os.replace(tmp, path)
|
|
52
|
+
except Exception:
|
|
53
|
+
try:
|
|
54
|
+
os.unlink(tmp)
|
|
55
|
+
except Exception:
|
|
56
|
+
pass
|
|
57
|
+
raise
|
|
58
|
+
except Exception:
|
|
59
|
+
pass
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def done(platform, total, **extras):
|
|
63
|
+
"""Mark `platform` as completed. Final tick, done==total, state=done."""
|
|
64
|
+
tick(platform, total, total, state="done", **extras)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _show(platform=None):
|
|
68
|
+
if not CACHE_DIR.exists():
|
|
69
|
+
return
|
|
70
|
+
files = sorted(CACHE_DIR.glob("progress_*.json"))
|
|
71
|
+
for f in files:
|
|
72
|
+
name = f.stem.replace("progress_", "")
|
|
73
|
+
if platform and name != platform:
|
|
74
|
+
continue
|
|
75
|
+
try:
|
|
76
|
+
data = json.loads(f.read_text())
|
|
77
|
+
except Exception:
|
|
78
|
+
continue
|
|
79
|
+
age = int(time.time() - data.get("updated_at_ts", 0))
|
|
80
|
+
state = data.get("state", "running")
|
|
81
|
+
done_n = data.get("done", 0)
|
|
82
|
+
total_n = data.get("total", 0)
|
|
83
|
+
pct = f"{100 * done_n / total_n:.1f}%" if total_n else "?"
|
|
84
|
+
print(f"{name:10} {state:8} {done_n}/{total_n} ({pct}) pid={data.get('pid')} {age}s ago @ {data.get('updated_at')}")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
if __name__ == "__main__":
|
|
88
|
+
_show(sys.argv[1] if len(sys.argv) > 1 else None)
|