@m13v/s4l 1.6.197-rc.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +143 -0
- package/SKILL.md +342 -0
- package/bin/cli.js +980 -0
- package/bin/cookie-helper.js +315 -0
- package/bin/platform.js +59 -0
- package/bin/scheduler/index.js +12 -0
- package/bin/scheduler/launchd.js +518 -0
- package/browser-agent-configs/all-agents-mcp.json +68 -0
- package/browser-agent-configs/linkedin-agent-mcp.json +16 -0
- package/browser-agent-configs/linkedin-agent.json +17 -0
- package/browser-agent-configs/linkedin-harness-mcp.json +21 -0
- package/browser-agent-configs/reddit-agent-mcp.json +16 -0
- package/browser-agent-configs/reddit-agent.json +17 -0
- package/browser-agent-configs/twitter-harness-mcp.json +18 -0
- package/config.example.json +45 -0
- package/mcp/dist/index.js +4212 -0
- package/mcp/dist/onboarding.js +200 -0
- package/mcp/dist/panel.html +176 -0
- package/mcp/dist/product-link.html +102 -0
- package/mcp/dist/repo.js +222 -0
- package/mcp/dist/runtime.js +1079 -0
- package/mcp/dist/screencast.js +323 -0
- package/mcp/dist/setup.js +545 -0
- package/mcp/dist/telemetry.js +306 -0
- package/mcp/dist/twitterAuth.js +138 -0
- package/mcp/dist/version.js +271 -0
- package/mcp/dist/version.json +4 -0
- package/mcp/install-runtime.mjs +70 -0
- package/mcp/install.mjs +169 -0
- package/mcp/manifest.json +80 -0
- package/mcp/menubar/dashboard_server.py +213 -0
- package/mcp/menubar/s4l_card.py +1336 -0
- package/mcp/menubar/s4l_log_relay.py +179 -0
- package/mcp/menubar/s4l_menubar.py +2439 -0
- package/mcp/menubar/s4l_state.py +891 -0
- package/mcp/package.json +34 -0
- package/mcp/shared/doctor.cjs +437 -0
- package/mcp/shared/onboarding-ledger.cjs +324 -0
- package/mcp-servers/browser-harness/server.py +968 -0
- package/package.json +160 -0
- package/requirements.txt +20 -0
- package/scripts/_compute_allowlist.py +58 -0
- package/scripts/_db_update.py +20 -0
- package/scripts/_filt.py +9 -0
- package/scripts/_li_notif_match.py +76 -0
- package/scripts/_li_notif_orchestrate.py +126 -0
- package/scripts/_lock_preempt_test.py +60 -0
- package/scripts/_run_icp_precheck.py +57 -0
- package/scripts/a16z_pearx_calendar_reminders.py +99 -0
- package/scripts/account_resolver.py +141 -0
- package/scripts/active_campaigns.py +114 -0
- package/scripts/active_users.py +190 -0
- package/scripts/amplitude_24h_signups.py +468 -0
- package/scripts/amplitude_signups.py +177 -0
- package/scripts/apply_onboarding_selections.py +131 -0
- package/scripts/audience_pages.py +243 -0
- package/scripts/audit_helper.py +120 -0
- package/scripts/author_history_block.py +353 -0
- package/scripts/autopilot_stall_watch.py +284 -0
- package/scripts/backfill_twitter_attempts_topic.py +81 -0
- package/scripts/backfill_twitter_log_post_no_id.py +322 -0
- package/scripts/bench_dashboard.sh +138 -0
- package/scripts/bh_send.py +39 -0
- package/scripts/build_persona.py +409 -0
- package/scripts/bulk_icp.py +18 -0
- package/scripts/campaign_bump.py +51 -0
- package/scripts/capture_thread_media.py +288 -0
- package/scripts/check_browser_lock_health.sh +81 -0
- package/scripts/check_external_pool_depth.py +253 -0
- package/scripts/check_unread_web_chats.py +28 -0
- package/scripts/claim_web_chat.py +47 -0
- package/scripts/classify_run_error.py +158 -0
- package/scripts/claude_job.py +988 -0
- package/scripts/clean_stale_singleton.sh +56 -0
- package/scripts/cleanup_harness_tabs.py +68 -0
- package/scripts/copy_browser_cookies.py +454 -0
- package/scripts/counterparty_history.py +350 -0
- package/scripts/db.py +57 -0
- package/scripts/discover_claude_profiles.py +120 -0
- package/scripts/discover_linkedin_candidates.py +984 -0
- package/scripts/dm_conversation.py +682 -0
- package/scripts/dm_db_update.py +69 -0
- package/scripts/dm_engage_helper.py +161 -0
- package/scripts/dm_outreach_helper.py +147 -0
- package/scripts/dm_outreach_twitter_helper.py +129 -0
- package/scripts/dm_send_log.py +106 -0
- package/scripts/dm_short_links.py +1084 -0
- package/scripts/dump_web_chat_history.py +47 -0
- package/scripts/engage_github.py +640 -0
- package/scripts/engage_reddit.py +1235 -0
- package/scripts/engage_twitter_helper.py +301 -0
- package/scripts/engagement_styles.py +1787 -0
- package/scripts/enrich_twitter_candidates.py +82 -0
- package/scripts/feedback_digest.py +448 -0
- package/scripts/fetch_prospect_profile.py +312 -0
- package/scripts/fetch_twitter_t1.py +134 -0
- package/scripts/find_threads.py +530 -0
- package/scripts/follow_gate_log.py +59 -0
- package/scripts/funnel_per_day.py +194 -0
- package/scripts/generate_daily_human_style.py +494 -0
- package/scripts/generation_trace.py +173 -0
- package/scripts/get_run_cost.py +107 -0
- package/scripts/github_engage_helper.py +93 -0
- package/scripts/github_tools.py +509 -0
- package/scripts/harness_overlay.py +556 -0
- package/scripts/harvest_twitter_following.py +243 -0
- package/scripts/heartbeat.sh +70 -0
- package/scripts/history_context.py +284 -0
- package/scripts/http_api.py +206 -0
- package/scripts/human_dm_replies_helper.py +169 -0
- package/scripts/identity.py +302 -0
- package/scripts/ig_batch_creator.sh +93 -0
- package/scripts/ig_post_type_picker.py +243 -0
- package/scripts/ig_scrape_transcribe.sh +91 -0
- package/scripts/ingest_human_dm_replies.py +271 -0
- package/scripts/ingest_web_chat_replies.py +229 -0
- package/scripts/install_fleet.py +187 -0
- package/scripts/invent_mcp_server.py +350 -0
- package/scripts/invent_topics.py +1462 -0
- package/scripts/learned_preferences.py +263 -0
- package/scripts/li_discovery.py +161 -0
- package/scripts/link_edit_helper.py +142 -0
- package/scripts/link_tail.py +592 -0
- package/scripts/linkedin_api.py +561 -0
- package/scripts/linkedin_browser.py +730 -0
- package/scripts/linkedin_cooldown.py +128 -0
- package/scripts/linkedin_exclusions.py +234 -0
- package/scripts/linkedin_killswitch.py +1333 -0
- package/scripts/linkedin_search_topic_schema.py +49 -0
- package/scripts/linkedin_unipile.py +658 -0
- package/scripts/linkedin_url.py +228 -0
- package/scripts/log_claude_session.py +636 -0
- package/scripts/log_draft.py +143 -0
- package/scripts/log_linkedin_search_attempts.py +126 -0
- package/scripts/log_post.py +651 -0
- package/scripts/log_run.py +364 -0
- package/scripts/log_thread_media.py +108 -0
- package/scripts/log_twitter_search_attempts.py +150 -0
- package/scripts/log_twitter_skips.py +211 -0
- package/scripts/lookup_post.py +78 -0
- package/scripts/mark_web_chat_processed.py +32 -0
- package/scripts/mcp_lock_proxy.py +370 -0
- package/scripts/memory_snapshot.py +972 -0
- package/scripts/merge_review_queue.py +215 -0
- package/scripts/mint_external_pool.py +182 -0
- package/scripts/mint_kent_pool.py +249 -0
- package/scripts/moltbook_post.py +320 -0
- package/scripts/moltbook_tools.py +159 -0
- package/scripts/pending_threads.py +188 -0
- package/scripts/pick_ig_account.py +177 -0
- package/scripts/pick_project.py +208 -0
- package/scripts/pick_search_topic.py +771 -0
- package/scripts/pick_thread_target.py +279 -0
- package/scripts/pick_twitter_thread_target.py +202 -0
- package/scripts/podlog_fetch_batch.sh +32 -0
- package/scripts/post_github.py +1311 -0
- package/scripts/post_reddit.py +2668 -0
- package/scripts/precompute_dashboard_stats.py +204 -0
- package/scripts/preflight.sh +297 -0
- package/scripts/progress.py +88 -0
- package/scripts/project_excludes.py +353 -0
- package/scripts/project_slugs.py +91 -0
- package/scripts/project_stats.py +241 -0
- package/scripts/project_stats_json.py +1563 -0
- package/scripts/project_topics.py +192 -0
- package/scripts/qualified_query_bank.py +436 -0
- package/scripts/reap_stale_claude_sessions.py +867 -0
- package/scripts/reddit_browser.py +2549 -0
- package/scripts/reddit_browser_fetch.py +141 -0
- package/scripts/reddit_browser_lock.py +593 -0
- package/scripts/reddit_chat_sync.py +710 -0
- package/scripts/reddit_query_bank.py +200 -0
- package/scripts/reddit_threads_helper.py +151 -0
- package/scripts/reddit_tools.py +956 -0
- package/scripts/refresh_instagram_tokens.py +280 -0
- package/scripts/release-mcpb.sh +513 -0
- package/scripts/reply_db.py +334 -0
- package/scripts/reply_insert.py +98 -0
- package/scripts/reply_risk_digest.py +761 -0
- package/scripts/reset-test-machine.sh +602 -0
- package/scripts/restore_twitter_session.py +177 -0
- package/scripts/ripen_reddit_plan.py +478 -0
- package/scripts/run_claude.sh +433 -0
- package/scripts/run_moltbook_cycle.py +555 -0
- package/scripts/s4l_box_update.sh +226 -0
- package/scripts/s4l_channel.py +103 -0
- package/scripts/s4l_ctl.sh +75 -0
- package/scripts/s4l_env.py +47 -0
- package/scripts/saps_activity.py +126 -0
- package/scripts/saps_mode.py +328 -0
- package/scripts/scan_dm_candidates.py +580 -0
- package/scripts/scan_github_replies.py +168 -0
- package/scripts/scan_instagram_comments.py +481 -0
- package/scripts/scan_moltbook_replies.py +252 -0
- package/scripts/scan_pii.py +190 -0
- package/scripts/scan_reddit_replies.py +377 -0
- package/scripts/scan_twitter_mentions_browser.py +327 -0
- package/scripts/scan_twitter_thread_followups.py +299 -0
- package/scripts/scan_x_profile.py +384 -0
- package/scripts/schedule_state.py +202 -0
- package/scripts/scheduled_tasks_snapshot.py +123 -0
- package/scripts/score_linkedin_candidates.py +419 -0
- package/scripts/score_twitter_candidates.py +718 -0
- package/scripts/scrape_linkedin_comment_stats.py +1755 -0
- package/scripts/scrape_linkedin_stats_browser.py +52 -0
- package/scripts/scrape_reddit_views.py +365 -0
- package/scripts/seed_search_queries.py +453 -0
- package/scripts/seed_search_topics.py +127 -0
- package/scripts/send_web_chat_reply.py +130 -0
- package/scripts/sentry_init.py +128 -0
- package/scripts/setup_twitter_auth.py +1320 -0
- package/scripts/snapshot.py +583 -0
- package/scripts/stats.py +2702 -0
- package/scripts/stats_helper.py +52 -0
- package/scripts/strike_alert.py +783 -0
- package/scripts/sweep_post_link_clicks.py +107 -0
- package/scripts/sync_ig_to_posts.py +147 -0
- package/scripts/test_browser_lock.py +189 -0
- package/scripts/test_installation_api.sh +52 -0
- package/scripts/test_percard_posting.py +142 -0
- package/scripts/top_dud_linkedin_queries.py +71 -0
- package/scripts/top_dud_reddit_queries.py +67 -0
- package/scripts/top_dud_twitter_queries.py +71 -0
- package/scripts/top_dud_twitter_topics.py +102 -0
- package/scripts/top_linkedin_queries.py +55 -0
- package/scripts/top_omitted_reddit_topics.py +91 -0
- package/scripts/top_performers.py +588 -0
- package/scripts/top_search_topics.py +180 -0
- package/scripts/top_twitter_queries.py +190 -0
- package/scripts/twitter_access_check.py +382 -0
- package/scripts/twitter_account.py +41 -0
- package/scripts/twitter_batch_phase.py +126 -0
- package/scripts/twitter_browser.py +2804 -0
- package/scripts/twitter_cookie_mirror.py +130 -0
- package/scripts/twitter_cycle_helper.py +310 -0
- package/scripts/twitter_gen_links.py +287 -0
- package/scripts/twitter_post_plan.py +1188 -0
- package/scripts/twitter_scan.py +324 -0
- package/scripts/twitter_supply_signal.py +57 -0
- package/scripts/twitter_threads_helper.py +152 -0
- package/scripts/unclaim_web_chat.py +29 -0
- package/scripts/update_instagram_stats.py +261 -0
- package/scripts/update_linkedin_stats_from_feed.py +328 -0
- package/scripts/version.py +72 -0
- package/scripts/watchdog_hung_runs.py +343 -0
- package/scripts/write_generation_trace.py +73 -0
- package/setup/SKILL.md +277 -0
- package/skill/amplitude-24h-signups.sh +38 -0
- package/skill/archive-old-logs.sh +40 -0
- package/skill/audit-dm-staleness.sh +42 -0
- package/skill/audit-linkedin.sh +14 -0
- package/skill/audit-moltbook.sh +4 -0
- package/skill/audit-reddit-resurrect.sh +67 -0
- package/skill/audit-reddit.sh +4 -0
- package/skill/audit-twitter.sh +4 -0
- package/skill/audit.sh +287 -0
- package/skill/backfill-twitter-attempts-topic.sh +19 -0
- package/skill/backfill-twitter-ghost-posts.sh +24 -0
- package/skill/check-external-pool-depth.sh +7 -0
- package/skill/check-web-chats.sh +203 -0
- package/skill/dm-outreach-linkedin.sh +250 -0
- package/skill/dm-outreach-reddit.sh +274 -0
- package/skill/dm-outreach-twitter.sh +265 -0
- package/skill/engage-dm-replies-linkedin.sh +4 -0
- package/skill/engage-dm-replies-reddit.sh +4 -0
- package/skill/engage-dm-replies-twitter.sh +4 -0
- package/skill/engage-dm-replies.sh +1597 -0
- package/skill/engage-linkedin.sh +581 -0
- package/skill/engage-moltbook.sh +36 -0
- package/skill/engage-reddit.sh +146 -0
- package/skill/engage-twitter.sh +467 -0
- package/skill/github-engage.sh +176 -0
- package/skill/ingest-web-chat-replies.sh +38 -0
- package/skill/invent-supply-test.sh +100 -0
- package/skill/invent-topics.sh +50 -0
- package/skill/lib/linkedin-backend.sh +364 -0
- package/skill/lib/platform.sh +48 -0
- package/skill/lib/reddit-backend.sh +234 -0
- package/skill/lib/twitter-backend.sh +314 -0
- package/skill/link-edit-github.sh +136 -0
- package/skill/link-edit-moltbook.sh +117 -0
- package/skill/link-edit-reddit.sh +201 -0
- package/skill/linkedin-presence.sh +182 -0
- package/skill/linkedin-recovery.sh +282 -0
- package/skill/lock.sh +647 -0
- package/skill/memory-snapshot.sh +39 -0
- package/skill/precompute-stats.sh +35 -0
- package/skill/prewarm-funnel.sh +104 -0
- package/skill/refresh-instagram-tokens.sh +57 -0
- package/skill/refresh-twitter-following.sh +52 -0
- package/skill/reply-risk-digest.sh +31 -0
- package/skill/run-cycle-update-guard.sh +44 -0
- package/skill/run-draft-and-publish.sh +123 -0
- package/skill/run-generate-daily-style.sh +50 -0
- package/skill/run-github-launchd.sh +62 -0
- package/skill/run-github.sh +102 -0
- package/skill/run-instagram-daily.sh +149 -0
- package/skill/run-instagram-render.sh +875 -0
- package/skill/run-linkedin-launchd.sh +81 -0
- package/skill/run-linkedin-unipile.sh +130 -0
- package/skill/run-linkedin.sh +1593 -0
- package/skill/run-moltbook-launchd.sh +61 -0
- package/skill/run-moltbook.sh +38 -0
- package/skill/run-overlay-watch.sh +100 -0
- package/skill/run-reddit-search-launchd.sh +64 -0
- package/skill/run-reddit-search.sh +505 -0
- package/skill/run-reddit-threads-double.sh +32 -0
- package/skill/run-reddit-threads.sh +847 -0
- package/skill/run-scan-moltbook-replies.sh +57 -0
- package/skill/run-twitter-cycle-launchd.sh +63 -0
- package/skill/run-twitter-cycle-singleton.sh +62 -0
- package/skill/run-twitter-cycle.sh +2408 -0
- package/skill/run-twitter-threads.sh +592 -0
- package/skill/scan-instagram-replies.sh +61 -0
- package/skill/scan-twitter-followups.sh +57 -0
- package/skill/social-autoposter-update.sh +66 -0
- package/skill/stats-instagram.sh +72 -0
- package/skill/stats-linkedin.sh +271 -0
- package/skill/stats-moltbook.sh +4 -0
- package/skill/stats-reddit.sh +4 -0
- package/skill/stats-twitter.sh +4 -0
- package/skill/stats.sh +521 -0
- package/skill/strike-alert.sh +18 -0
- package/skill/styles.sh +87 -0
- package/skill/sweep-link-clicks.sh +40 -0
- package/skill/topics.sh +51 -0
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Ingest human replies to DM escalation emails from Gmail into human_dm_replies.
|
|
3
|
+
|
|
4
|
+
Flow:
|
|
5
|
+
1. flag_human() in dm_conversation.py sends an escalation email with subject
|
|
6
|
+
`[DM #<id>] <author> [<platform>]: <reason>` FROM matt@s4l.ai TO
|
|
7
|
+
NOTIFICATION_EMAIL (i@m13v.com).
|
|
8
|
+
2. The human reads it in i@m13v.com, hits Reply in Gmail, writes what they want
|
|
9
|
+
to say, sends. Gmail keeps `[DM #<id>]` in the subject (prefixed with Re:).
|
|
10
|
+
Because the escalation's From is matt@s4l.ai, the reply is delivered TO the
|
|
11
|
+
matt@s4l.ai mailbox as a fresh unread inbound message (the reply is sent from
|
|
12
|
+
i@m13v.com, a different account, so it lands in matt@s4l.ai's inbox, not Sent).
|
|
13
|
+
3. This script polls the matt@s4l.ai mailbox for messages matching that subject
|
|
14
|
+
token that are unread. For each, it extracts the dm_id, strips the quoted
|
|
15
|
+
history from the reply body, and inserts a row into human_dm_replies with
|
|
16
|
+
status='pending' (unique on gmail message id).
|
|
17
|
+
4. It marks the Gmail message as read so we don't re-ingest.
|
|
18
|
+
5. Phase 0 of skill/engage-dm-replies.sh then picks up pending rows and sends
|
|
19
|
+
them as DMs on the target platform.
|
|
20
|
+
|
|
21
|
+
Auth: matt@s4l.ai is reached via the keyless Domain-Wide Delegation lane (the
|
|
22
|
+
service account gmail-dwd-impersonator impersonates it; s4l.ai is a secondary
|
|
23
|
+
domain in the mediar.ai Workspace). The short-lived access token is kept warm by
|
|
24
|
+
launchd job com.m13v.gmail-dwd-keepalive-s4l; this script also refreshes inline
|
|
25
|
+
if the token is missing or about to expire.
|
|
26
|
+
|
|
27
|
+
Usage:
|
|
28
|
+
python3 scripts/ingest_human_dm_replies.py # ingest and report
|
|
29
|
+
python3 scripts/ingest_human_dm_replies.py --dry-run # print what would be ingested, no DB writes, no label changes
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
import argparse
|
|
33
|
+
import base64
|
|
34
|
+
import json
|
|
35
|
+
import os
|
|
36
|
+
import re
|
|
37
|
+
import subprocess
|
|
38
|
+
import sys
|
|
39
|
+
import time
|
|
40
|
+
from email import message_from_bytes
|
|
41
|
+
from email.policy import default as email_default_policy
|
|
42
|
+
|
|
43
|
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
44
|
+
# HTTP-only: dms lookup + human_dm_replies dedup/insert go through the s4l.ai
|
|
45
|
+
# HTTP API (scripts/http_api.py). The direct-Postgres lane was removed
|
|
46
|
+
# 2026-06-01; DATABASE_URL is deliberately ignored, no DB path, no fallback.
|
|
47
|
+
from http_api import api_get, api_post
|
|
48
|
+
|
|
49
|
+
from google.oauth2.credentials import Credentials
|
|
50
|
+
from googleapiclient.discovery import build
|
|
51
|
+
|
|
52
|
+
GMAIL_SCOPES = ["https://mail.google.com/"]
|
|
53
|
+
|
|
54
|
+
# matt@s4l.ai via keyless Domain-Wide Delegation. The keepalive launchd job
|
|
55
|
+
# (com.m13v.gmail-dwd-keepalive-s4l) keeps this access token warm; we also
|
|
56
|
+
# refresh inline below if it is missing or within 60s of expiry.
|
|
57
|
+
DWD_CREDS_PATH = os.path.expanduser("~/.gmail-mcp-s4l/credentials.json")
|
|
58
|
+
DWD_REFRESHER = os.path.expanduser("~/gmail-dwd-keepalive/refresh_token_s4l.py")
|
|
59
|
+
DWD_PYTHON = os.path.expanduser("~/gmail-dwd-keepalive/.venv/bin/python")
|
|
60
|
+
|
|
61
|
+
SELF_ADDRESSES = {"matt@s4l.ai"}
|
|
62
|
+
DM_ID_RE = re.compile(r"\[DM\s*#(\d+)\]", re.IGNORECASE)
|
|
63
|
+
RE_PREFIX_RE = re.compile(r"^\s*re\s*:", re.IGNORECASE)
|
|
64
|
+
|
|
65
|
+
# Only fetch Gmail replies (subject starts with "Re:"). This excludes both our
|
|
66
|
+
# own outgoing escalation emails (original subject, no Re:) and stale historical
|
|
67
|
+
# escalations that pre-date the rewire.
|
|
68
|
+
GMAIL_QUERY = 'is:unread subject:"Re: [DM #"'
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _run_refresher():
|
|
72
|
+
subprocess.run([DWD_PYTHON, DWD_REFRESHER], check=True, timeout=60)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def gmail_service():
|
|
76
|
+
"""Build the Gmail service for matt@s4l.ai from the DWD access token.
|
|
77
|
+
|
|
78
|
+
Reads the short-lived token minted by the keepalive job. If the file is
|
|
79
|
+
missing or the token is within 60s of expiry, runs the refresher inline.
|
|
80
|
+
"""
|
|
81
|
+
if not os.path.exists(DWD_CREDS_PATH):
|
|
82
|
+
_run_refresher()
|
|
83
|
+
with open(DWD_CREDS_PATH) as f:
|
|
84
|
+
payload = json.load(f)
|
|
85
|
+
expiry_ms = payload.get("expiry_date", 0)
|
|
86
|
+
if not payload.get("access_token") or (expiry_ms and expiry_ms / 1000 <= time.time() + 60):
|
|
87
|
+
_run_refresher()
|
|
88
|
+
with open(DWD_CREDS_PATH) as f:
|
|
89
|
+
payload = json.load(f)
|
|
90
|
+
creds = Credentials(token=payload["access_token"], scopes=GMAIL_SCOPES)
|
|
91
|
+
return build("gmail", "v1", credentials=creds, cache_discovery=False)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def list_candidate_messages(service):
|
|
95
|
+
resp = service.users().messages().list(userId="me", q=GMAIL_QUERY, maxResults=50).execute()
|
|
96
|
+
return resp.get("messages", []) or []
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def fetch_raw(service, message_id):
|
|
100
|
+
msg = service.users().messages().get(userId="me", id=message_id, format="raw").execute()
|
|
101
|
+
raw = base64.urlsafe_b64decode(msg["raw"].encode("ASCII"))
|
|
102
|
+
return message_from_bytes(raw, policy=email_default_policy), msg.get("labelIds", [])
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def pick_plain_body(email_msg):
|
|
106
|
+
if email_msg.is_multipart():
|
|
107
|
+
text_part = None
|
|
108
|
+
for part in email_msg.walk():
|
|
109
|
+
ctype = part.get_content_type()
|
|
110
|
+
if ctype == "text/plain":
|
|
111
|
+
text_part = part
|
|
112
|
+
break
|
|
113
|
+
if text_part is None:
|
|
114
|
+
for part in email_msg.walk():
|
|
115
|
+
if part.get_content_type() == "text/html":
|
|
116
|
+
text_part = part
|
|
117
|
+
break
|
|
118
|
+
if text_part is None:
|
|
119
|
+
return ""
|
|
120
|
+
try:
|
|
121
|
+
return text_part.get_content()
|
|
122
|
+
except Exception:
|
|
123
|
+
return text_part.get_payload(decode=True).decode("utf-8", errors="replace")
|
|
124
|
+
try:
|
|
125
|
+
return email_msg.get_content()
|
|
126
|
+
except Exception:
|
|
127
|
+
payload = email_msg.get_payload(decode=True)
|
|
128
|
+
if payload:
|
|
129
|
+
return payload.decode("utf-8", errors="replace")
|
|
130
|
+
return email_msg.get_payload() or ""
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
# Common "On Mon, Apr 20, 2026 at 5:30 PM X wrote:" patterns across clients.
|
|
134
|
+
QUOTE_MARKER_RES = [
|
|
135
|
+
re.compile(r"^On .{5,200}\s+wrote:\s*$", re.MULTILINE | re.IGNORECASE),
|
|
136
|
+
re.compile(r"^-{2,}\s*Original Message\s*-{2,}\s*$", re.MULTILINE | re.IGNORECASE),
|
|
137
|
+
re.compile(r"^From:\s.+<.+>\s*$", re.MULTILINE),
|
|
138
|
+
]
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def strip_quoted_history(body):
|
|
142
|
+
if not body:
|
|
143
|
+
return ""
|
|
144
|
+
earliest = len(body)
|
|
145
|
+
for pat in QUOTE_MARKER_RES:
|
|
146
|
+
m = pat.search(body)
|
|
147
|
+
if m and m.start() < earliest:
|
|
148
|
+
earliest = m.start()
|
|
149
|
+
trimmed = body[:earliest]
|
|
150
|
+
|
|
151
|
+
lines = []
|
|
152
|
+
for line in trimmed.splitlines():
|
|
153
|
+
if line.lstrip().startswith(">"):
|
|
154
|
+
break
|
|
155
|
+
lines.append(line)
|
|
156
|
+
return "\n".join(lines).strip()
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def extract_sender_addr(raw_from):
|
|
160
|
+
if not raw_from:
|
|
161
|
+
return ""
|
|
162
|
+
m = re.search(r"<([^>]+)>", raw_from)
|
|
163
|
+
return (m.group(1) if m else raw_from).strip().lower()
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def main():
|
|
167
|
+
parser = argparse.ArgumentParser()
|
|
168
|
+
parser.add_argument("--dry-run", action="store_true", help="Print what would be ingested, do not touch DB or labels")
|
|
169
|
+
args = parser.parse_args()
|
|
170
|
+
|
|
171
|
+
try:
|
|
172
|
+
service = gmail_service()
|
|
173
|
+
except Exception as e:
|
|
174
|
+
print(f"FATAL: could not build Gmail service: {e}", file=sys.stderr)
|
|
175
|
+
sys.exit(2)
|
|
176
|
+
|
|
177
|
+
candidates = list_candidate_messages(service)
|
|
178
|
+
if not candidates:
|
|
179
|
+
print("No candidate Gmail messages for DM escalation replies.")
|
|
180
|
+
return
|
|
181
|
+
|
|
182
|
+
ingested = 0
|
|
183
|
+
skipped = 0
|
|
184
|
+
for c in candidates:
|
|
185
|
+
gmail_id = c["id"]
|
|
186
|
+
try:
|
|
187
|
+
email_msg, labels = fetch_raw(service, gmail_id)
|
|
188
|
+
except Exception as e:
|
|
189
|
+
print(f" SKIP {gmail_id}: fetch failed: {e}")
|
|
190
|
+
skipped += 1
|
|
191
|
+
continue
|
|
192
|
+
|
|
193
|
+
subject = email_msg.get("Subject", "") or ""
|
|
194
|
+
sender = extract_sender_addr(email_msg.get("From", ""))
|
|
195
|
+
m = DM_ID_RE.search(subject)
|
|
196
|
+
if not m:
|
|
197
|
+
print(f" SKIP {gmail_id}: subject has no [DM #N] token ({subject!r})")
|
|
198
|
+
skipped += 1
|
|
199
|
+
continue
|
|
200
|
+
dm_id = int(m.group(1))
|
|
201
|
+
|
|
202
|
+
# Belt-and-suspenders: reject anything where the subject doesn't start
|
|
203
|
+
# with Re:. The Gmail query should already filter this, but if someone
|
|
204
|
+
# forwards an escalation, is:unread + [DM #N] would match without Re:.
|
|
205
|
+
if not RE_PREFIX_RE.match(subject):
|
|
206
|
+
print(f" SKIP {gmail_id}: subject not a reply ({subject!r})")
|
|
207
|
+
skipped += 1
|
|
208
|
+
continue
|
|
209
|
+
|
|
210
|
+
dm_resp = api_get(f"/api/v1/dms/{dm_id}", ok_on_404=True)
|
|
211
|
+
dm_row = (dm_resp.get("data") or {}).get("dm") if dm_resp.get("ok") else None
|
|
212
|
+
if not dm_row:
|
|
213
|
+
print(f" SKIP {gmail_id}: DM #{dm_id} not found in dms table")
|
|
214
|
+
skipped += 1
|
|
215
|
+
continue
|
|
216
|
+
|
|
217
|
+
body_raw = pick_plain_body(email_msg)
|
|
218
|
+
reply_text = strip_quoted_history(body_raw)
|
|
219
|
+
if not reply_text:
|
|
220
|
+
print(f" SKIP {gmail_id}: empty reply after stripping quoted history")
|
|
221
|
+
skipped += 1
|
|
222
|
+
continue
|
|
223
|
+
|
|
224
|
+
project = dm_row.get("target_project") or dm_row.get("project_name")
|
|
225
|
+
|
|
226
|
+
print(f" MATCH {gmail_id}: DM #{dm_id} ({dm_row['platform']}/{dm_row['their_author']}) reply {reply_text!r}")
|
|
227
|
+
|
|
228
|
+
if args.dry_run:
|
|
229
|
+
ingested += 1
|
|
230
|
+
continue
|
|
231
|
+
|
|
232
|
+
# Dedup-on-gmail-id + insert happen server-side in one POST: the
|
|
233
|
+
# endpoint SELECTs by resend_email_id and returns reused=true if the
|
|
234
|
+
# reply was already ingested, otherwise inserts status='pending'.
|
|
235
|
+
try:
|
|
236
|
+
resp = api_post("/api/v1/human-dm-replies", {
|
|
237
|
+
"dm_id": dm_id,
|
|
238
|
+
"platform": dm_row["platform"],
|
|
239
|
+
"their_author": dm_row["their_author"],
|
|
240
|
+
"project_name": project,
|
|
241
|
+
"instructions": reply_text,
|
|
242
|
+
"email_subject": subject,
|
|
243
|
+
"resend_email_id": gmail_id,
|
|
244
|
+
})
|
|
245
|
+
except SystemExit as e:
|
|
246
|
+
print(f" ERROR {gmail_id}: insert failed: {e}")
|
|
247
|
+
skipped += 1
|
|
248
|
+
continue
|
|
249
|
+
data = (resp.get("data") or {}) if isinstance(resp, dict) else {}
|
|
250
|
+
reused = bool(data.get("reused"))
|
|
251
|
+
if reused:
|
|
252
|
+
print(f" SKIP {gmail_id}: already ingested as human_dm_replies #{data.get('id')}")
|
|
253
|
+
skipped += 1
|
|
254
|
+
|
|
255
|
+
# Mark as read in both cases so the Gmail query excludes it next run.
|
|
256
|
+
try:
|
|
257
|
+
service.users().messages().modify(
|
|
258
|
+
userId="me", id=gmail_id,
|
|
259
|
+
body={"removeLabelIds": ["UNREAD"]},
|
|
260
|
+
).execute()
|
|
261
|
+
except Exception as e:
|
|
262
|
+
print(f" WARN {gmail_id}: could not mark as read: {e}")
|
|
263
|
+
|
|
264
|
+
if not reused:
|
|
265
|
+
ingested += 1
|
|
266
|
+
|
|
267
|
+
print(f"Done. Ingested={ingested} skipped={skipped} candidates={len(candidates)}")
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
if __name__ == "__main__":
|
|
271
|
+
main()
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Ingest human (Matt) replies to web-chat escalation emails from Gmail into Postgres.
|
|
3
|
+
|
|
4
|
+
Mirror of scripts/ingest_human_dm_replies.py for the web-chat rail.
|
|
5
|
+
|
|
6
|
+
Flow:
|
|
7
|
+
1. The Claude session ends a conversation by sending an escalation email with
|
|
8
|
+
subject `[WEB-CHAT #<thread_db_id>] <project>: <visitor_email>` from
|
|
9
|
+
i@m13v.com to the project's notify_email (or i@m13v.com).
|
|
10
|
+
2. Matt reads it, hits Reply in Gmail, types what he actually wants the
|
|
11
|
+
visitor to see, sends. Gmail keeps `[WEB-CHAT #<id>]` in the subject.
|
|
12
|
+
3. This script polls i@m13v.com for unread replies matching that token,
|
|
13
|
+
extracts the thread_db_id, strips quoted history, INSERTs as a
|
|
14
|
+
sender='founder' message AND fires Resend to the visitor's email.
|
|
15
|
+
4. Marks the Gmail message as read so we don't re-ingest.
|
|
16
|
+
|
|
17
|
+
Usage:
|
|
18
|
+
python3 ingest_web_chat_replies.py # ingest and report
|
|
19
|
+
python3 ingest_web_chat_replies.py --dry-run # print, no DB writes
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
import argparse
|
|
23
|
+
import base64
|
|
24
|
+
import os
|
|
25
|
+
import re
|
|
26
|
+
import subprocess
|
|
27
|
+
import sys
|
|
28
|
+
from email import message_from_bytes
|
|
29
|
+
from email.policy import default as email_default_policy
|
|
30
|
+
|
|
31
|
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
32
|
+
from http_api import api_get
|
|
33
|
+
|
|
34
|
+
from google.auth.transport.requests import Request
|
|
35
|
+
from google.oauth2.credentials import Credentials
|
|
36
|
+
from googleapiclient.discovery import build
|
|
37
|
+
|
|
38
|
+
GMAIL_TOKEN_PATH = os.path.expanduser("~/gmail-api/token_i_at_m13v.com.json")
|
|
39
|
+
GMAIL_SCOPES = ["https://mail.google.com/"]
|
|
40
|
+
|
|
41
|
+
WEB_CHAT_ID_RE = re.compile(r"\[WEB-CHAT\s*#(\d+)\]", re.IGNORECASE)
|
|
42
|
+
RE_PREFIX_RE = re.compile(r"^\s*re\s*:", re.IGNORECASE)
|
|
43
|
+
GMAIL_QUERY = 'is:unread subject:"Re: [WEB-CHAT #"'
|
|
44
|
+
|
|
45
|
+
QUOTE_MARKER_RES = [
|
|
46
|
+
re.compile(r"^On .{5,200}\s+wrote:\s*$", re.MULTILINE | re.IGNORECASE),
|
|
47
|
+
re.compile(r"^-{2,}\s*Original Message\s*-{2,}\s*$", re.MULTILINE | re.IGNORECASE),
|
|
48
|
+
re.compile(r"^From:\s.+<.+>\s*$", re.MULTILINE),
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
NODE_BIN = os.path.expanduser("~/.nvm/versions/node/v20.19.4/bin/node")
|
|
52
|
+
SEND_REPLY = os.path.expanduser("~/social-autoposter/scripts/send_web_chat_reply.py")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def gmail_service():
|
|
56
|
+
creds = Credentials.from_authorized_user_file(GMAIL_TOKEN_PATH, GMAIL_SCOPES)
|
|
57
|
+
if creds.expired and creds.refresh_token:
|
|
58
|
+
creds.refresh(Request())
|
|
59
|
+
with open(GMAIL_TOKEN_PATH, "w") as f:
|
|
60
|
+
f.write(creds.to_json())
|
|
61
|
+
return build("gmail", "v1", credentials=creds)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def list_candidates(service):
|
|
65
|
+
resp = service.users().messages().list(userId="me", q=GMAIL_QUERY, maxResults=50).execute()
|
|
66
|
+
return resp.get("messages", []) or []
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def fetch(service, message_id):
|
|
70
|
+
msg = service.users().messages().get(userId="me", id=message_id, format="raw").execute()
|
|
71
|
+
raw = base64.urlsafe_b64decode(msg["raw"].encode("ASCII"))
|
|
72
|
+
return message_from_bytes(raw, policy=email_default_policy)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def pick_plain_body(email_msg):
|
|
76
|
+
if email_msg.is_multipart():
|
|
77
|
+
text_part = None
|
|
78
|
+
for part in email_msg.walk():
|
|
79
|
+
if part.get_content_type() == "text/plain":
|
|
80
|
+
text_part = part
|
|
81
|
+
break
|
|
82
|
+
if text_part is None:
|
|
83
|
+
for part in email_msg.walk():
|
|
84
|
+
if part.get_content_type() == "text/html":
|
|
85
|
+
text_part = part
|
|
86
|
+
break
|
|
87
|
+
if text_part is None:
|
|
88
|
+
return ""
|
|
89
|
+
try:
|
|
90
|
+
return text_part.get_content()
|
|
91
|
+
except Exception:
|
|
92
|
+
return text_part.get_payload(decode=True).decode("utf-8", errors="replace")
|
|
93
|
+
try:
|
|
94
|
+
return email_msg.get_content()
|
|
95
|
+
except Exception:
|
|
96
|
+
payload = email_msg.get_payload(decode=True)
|
|
97
|
+
if payload:
|
|
98
|
+
return payload.decode("utf-8", errors="replace")
|
|
99
|
+
return email_msg.get_payload() or ""
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def strip_quoted(body):
|
|
103
|
+
if not body:
|
|
104
|
+
return ""
|
|
105
|
+
earliest = len(body)
|
|
106
|
+
for pat in QUOTE_MARKER_RES:
|
|
107
|
+
m = pat.search(body)
|
|
108
|
+
if m and m.start() < earliest:
|
|
109
|
+
earliest = m.start()
|
|
110
|
+
trimmed = body[:earliest]
|
|
111
|
+
lines = []
|
|
112
|
+
for line in trimmed.splitlines():
|
|
113
|
+
if line.lstrip().startswith(">"):
|
|
114
|
+
break
|
|
115
|
+
lines.append(line)
|
|
116
|
+
return "\n".join(lines).strip()
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def main():
|
|
120
|
+
parser = argparse.ArgumentParser()
|
|
121
|
+
parser.add_argument("--dry-run", action="store_true")
|
|
122
|
+
args = parser.parse_args()
|
|
123
|
+
|
|
124
|
+
try:
|
|
125
|
+
service = gmail_service()
|
|
126
|
+
except Exception as e:
|
|
127
|
+
print(f"FATAL: could not build Gmail service: {e}", file=sys.stderr)
|
|
128
|
+
sys.exit(2)
|
|
129
|
+
|
|
130
|
+
candidates = list_candidates(service)
|
|
131
|
+
if not candidates:
|
|
132
|
+
print("no candidate Gmail replies for [WEB-CHAT #...]")
|
|
133
|
+
return
|
|
134
|
+
|
|
135
|
+
ingested = skipped = 0
|
|
136
|
+
for c in candidates:
|
|
137
|
+
gmail_id = c["id"]
|
|
138
|
+
try:
|
|
139
|
+
email_msg = fetch(service, gmail_id)
|
|
140
|
+
except Exception as e:
|
|
141
|
+
print(f" SKIP {gmail_id}: fetch failed: {e}")
|
|
142
|
+
skipped += 1
|
|
143
|
+
continue
|
|
144
|
+
|
|
145
|
+
subject = email_msg.get("Subject", "") or ""
|
|
146
|
+
m = WEB_CHAT_ID_RE.search(subject)
|
|
147
|
+
if not m:
|
|
148
|
+
print(f" SKIP {gmail_id}: no [WEB-CHAT #N] token ({subject!r})")
|
|
149
|
+
skipped += 1
|
|
150
|
+
continue
|
|
151
|
+
if not RE_PREFIX_RE.match(subject):
|
|
152
|
+
print(f" SKIP {gmail_id}: not a reply ({subject!r})")
|
|
153
|
+
skipped += 1
|
|
154
|
+
continue
|
|
155
|
+
|
|
156
|
+
thread_db_id = int(m.group(1))
|
|
157
|
+
thread_resp = api_get(
|
|
158
|
+
f"/api/v1/web-chat/thread-by-id/{thread_db_id}", ok_on_404=True
|
|
159
|
+
)
|
|
160
|
+
if thread_resp.get("_not_found"):
|
|
161
|
+
print(f" SKIP {gmail_id}: thread #{thread_db_id} not found")
|
|
162
|
+
skipped += 1
|
|
163
|
+
continue
|
|
164
|
+
thread = thread_resp.get("data") or {}
|
|
165
|
+
|
|
166
|
+
body = pick_plain_body(email_msg)
|
|
167
|
+
reply_text = strip_quoted(body)
|
|
168
|
+
if not reply_text:
|
|
169
|
+
print(f" SKIP {gmail_id}: empty after stripping quotes")
|
|
170
|
+
skipped += 1
|
|
171
|
+
continue
|
|
172
|
+
|
|
173
|
+
# Dedup on gmail id (partial unique index in schema, but pre-check anyway).
|
|
174
|
+
dedup = api_get(
|
|
175
|
+
"/api/v1/web-chat/gmail-ingested", query={"gmail_id": gmail_id}
|
|
176
|
+
)
|
|
177
|
+
already_id = (dedup.get("data") or {}).get("ingested_message_id")
|
|
178
|
+
if already_id:
|
|
179
|
+
print(f" SKIP {gmail_id}: already ingested as msg #{already_id}")
|
|
180
|
+
skipped += 1
|
|
181
|
+
try:
|
|
182
|
+
service.users().messages().modify(
|
|
183
|
+
userId="me", id=gmail_id, body={"removeLabelIds": ["UNREAD"]}
|
|
184
|
+
).execute()
|
|
185
|
+
except Exception:
|
|
186
|
+
pass
|
|
187
|
+
continue
|
|
188
|
+
|
|
189
|
+
print(f" MATCH {gmail_id}: WEB-CHAT #{thread_db_id} ({thread['project_name']}/{thread['visitor_email']}) reply {reply_text[:80]!r}")
|
|
190
|
+
|
|
191
|
+
if args.dry_run:
|
|
192
|
+
ingested += 1
|
|
193
|
+
continue
|
|
194
|
+
|
|
195
|
+
# Insert as sender='founder' AND fire visitor email via send-email.js
|
|
196
|
+
# (use send_web_chat_reply.py so the dedup + email logic stays in one place).
|
|
197
|
+
# Pass --ingested-gmail-id so the reply endpoint stamps it on the
|
|
198
|
+
# inserted row directly, keeping dedup honest on re-runs.
|
|
199
|
+
try:
|
|
200
|
+
subprocess.run(
|
|
201
|
+
[sys.executable, SEND_REPLY,
|
|
202
|
+
"--thread", thread["thread_id"],
|
|
203
|
+
"--text", reply_text,
|
|
204
|
+
"--name", "matt",
|
|
205
|
+
"--sender", "founder",
|
|
206
|
+
"--ingested-gmail-id", gmail_id],
|
|
207
|
+
check=True,
|
|
208
|
+
timeout=60,
|
|
209
|
+
)
|
|
210
|
+
except Exception as e:
|
|
211
|
+
print(f" ERROR {gmail_id}: send_web_chat_reply.py failed: {e}")
|
|
212
|
+
skipped += 1
|
|
213
|
+
continue
|
|
214
|
+
|
|
215
|
+
# Mark gmail as read.
|
|
216
|
+
try:
|
|
217
|
+
service.users().messages().modify(
|
|
218
|
+
userId="me", id=gmail_id, body={"removeLabelIds": ["UNREAD"]}
|
|
219
|
+
).execute()
|
|
220
|
+
except Exception as e:
|
|
221
|
+
print(f" WARN {gmail_id}: could not mark read: {e}")
|
|
222
|
+
|
|
223
|
+
ingested += 1
|
|
224
|
+
|
|
225
|
+
print(f"done. ingested={ingested} skipped={skipped} candidates={len(candidates)}")
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
if __name__ == "__main__":
|
|
229
|
+
main()
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Operator fleet report: per-install plugin version + latest resource sample.
|
|
3
|
+
|
|
4
|
+
Answers the two questions we kept not being able to answer:
|
|
5
|
+
- which version is each user on?
|
|
6
|
+
- who is leaking RAM / still lacks the agent-mode-session reaper (<1.6.111)?
|
|
7
|
+
|
|
8
|
+
Reads the installations table directly via psql + DATABASE_URL from .env (this
|
|
9
|
+
is an operator-box tool, not shipped to customers). app_version and the latest
|
|
10
|
+
resource_sample only populate once a box runs a build that ships them, so rows
|
|
11
|
+
predating this feature show "?" until their next heartbeat on the new build.
|
|
12
|
+
|
|
13
|
+
Usage:
|
|
14
|
+
python3 scripts/install_fleet.py # all installs, newest-seen first
|
|
15
|
+
python3 scripts/install_fleet.py --active 7 # only seen in the last 7 days
|
|
16
|
+
python3 scripts/install_fleet.py --leaking # only rows over the RAM threshold
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import argparse
|
|
21
|
+
import json
|
|
22
|
+
import os
|
|
23
|
+
import subprocess
|
|
24
|
+
import sys
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
|
|
27
|
+
REPO_DIR = Path(__file__).resolve().parents[1]
|
|
28
|
+
REAPER_FIX_VERSION = (1, 6, 111) # social-autoposter@1.6.111 shipped the leak reaper
|
|
29
|
+
LEAK_RAM_MB = 12000 # our-process RSS over this on a box smells like the session leak
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _database_url() -> str:
|
|
33
|
+
env = REPO_DIR / ".env"
|
|
34
|
+
if env.exists():
|
|
35
|
+
for line in env.read_text().splitlines():
|
|
36
|
+
if line.startswith("DATABASE_URL="):
|
|
37
|
+
return line.split("=", 1)[1].strip()
|
|
38
|
+
if os.environ.get("DATABASE_URL"):
|
|
39
|
+
return os.environ["DATABASE_URL"]
|
|
40
|
+
sys.exit("DATABASE_URL not found in .env or environment")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _ver_tuple(v: str | None) -> tuple[int, ...] | None:
|
|
44
|
+
if not v:
|
|
45
|
+
return None
|
|
46
|
+
parts = v.strip().lstrip("v").split(".")
|
|
47
|
+
try:
|
|
48
|
+
return tuple(int(p) for p in parts[:3])
|
|
49
|
+
except ValueError:
|
|
50
|
+
return None
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _reaper_flag(v: str | None) -> str:
|
|
54
|
+
t = _ver_tuple(v)
|
|
55
|
+
if t is None:
|
|
56
|
+
return "?(pre-telemetry)"
|
|
57
|
+
return "ok" if t >= REAPER_FIX_VERSION else "BEHIND<1.6.111"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _fnum(x) -> float | None:
|
|
61
|
+
try:
|
|
62
|
+
return float(x)
|
|
63
|
+
except (TypeError, ValueError):
|
|
64
|
+
return None
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def main() -> int:
|
|
68
|
+
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
69
|
+
ap.add_argument("--active", type=int, default=0, help="only installs seen in the last N days")
|
|
70
|
+
ap.add_argument("--leaking", action="store_true", help="only rows whose our-process RSS exceeds the leak threshold")
|
|
71
|
+
ap.add_argument("--json", action="store_true", help="emit raw JSON rows instead of the table")
|
|
72
|
+
args = ap.parse_args()
|
|
73
|
+
|
|
74
|
+
where = ""
|
|
75
|
+
if args.active > 0:
|
|
76
|
+
where = f"WHERE last_seen_at > NOW() - interval '{args.active} days'"
|
|
77
|
+
|
|
78
|
+
# Tab-separated so we can parse jsonb columns intact (no embedded tabs in our samples).
|
|
79
|
+
query = f"""
|
|
80
|
+
SELECT
|
|
81
|
+
install_id,
|
|
82
|
+
COALESCE(hostname, '?'),
|
|
83
|
+
COALESCE(app_version, ''),
|
|
84
|
+
COALESCE(git_email, '?'),
|
|
85
|
+
COALESCE(last_city, '?') || '/' || COALESCE(last_country, '?'),
|
|
86
|
+
EXTRACT(EPOCH FROM (NOW() - last_seen_at))::bigint,
|
|
87
|
+
request_count,
|
|
88
|
+
COALESCE(resource_sample::text, ''),
|
|
89
|
+
COALESCE(EXTRACT(EPOCH FROM (NOW() - resource_sampled_at))::bigint::text, '')
|
|
90
|
+
FROM installations
|
|
91
|
+
{where}
|
|
92
|
+
ORDER BY last_seen_at DESC;
|
|
93
|
+
"""
|
|
94
|
+
|
|
95
|
+
out = subprocess.run(
|
|
96
|
+
["psql", _database_url(), "-t", "-A", "-F", "\t", "-c", query],
|
|
97
|
+
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=30,
|
|
98
|
+
)
|
|
99
|
+
if out.returncode != 0:
|
|
100
|
+
sys.exit(f"psql failed: {out.stderr.strip()}")
|
|
101
|
+
|
|
102
|
+
rows = []
|
|
103
|
+
for line in out.stdout.splitlines():
|
|
104
|
+
if not line.strip():
|
|
105
|
+
continue
|
|
106
|
+
cols = line.split("\t")
|
|
107
|
+
if len(cols) < 9:
|
|
108
|
+
continue
|
|
109
|
+
iid, host, appv, email, geo, age_s, reqs, sample_json, sample_age_s = cols[:9]
|
|
110
|
+
sample = {}
|
|
111
|
+
if sample_json:
|
|
112
|
+
try:
|
|
113
|
+
sample = json.loads(sample_json)
|
|
114
|
+
except Exception:
|
|
115
|
+
sample = {}
|
|
116
|
+
mem = sample.get("mem") or {}
|
|
117
|
+
groups = sample.get("groups") or {}
|
|
118
|
+
|
|
119
|
+
def grp_mb(name: str) -> float | None:
|
|
120
|
+
g = groups.get(name) or {}
|
|
121
|
+
return _fnum(g.get("rss_mb"))
|
|
122
|
+
|
|
123
|
+
# "our footprint" = the S4L MCP servers + the Claude Desktop sessions that
|
|
124
|
+
# have our MCP loaded (the bucket Karol saw as "the S4L scheduled task").
|
|
125
|
+
our_mb = sum(
|
|
126
|
+
v for v in (
|
|
127
|
+
grp_mb("social_autoposter_mcp_servers"),
|
|
128
|
+
grp_mb("sessions_configured_social_autoposter_mcp"),
|
|
129
|
+
) if v is not None
|
|
130
|
+
) or None
|
|
131
|
+
claude_grp = groups.get("claude_cli") or {}
|
|
132
|
+
rows.append({
|
|
133
|
+
"install_id": iid,
|
|
134
|
+
"hostname": host,
|
|
135
|
+
"app_version": appv or None,
|
|
136
|
+
"reaper": _reaper_flag(appv or None),
|
|
137
|
+
"git_email": email,
|
|
138
|
+
"geo": geo,
|
|
139
|
+
"last_seen_age_h": round(int(age_s) / 3600, 1) if age_s else None,
|
|
140
|
+
"requests": int(reqs) if reqs.isdigit() else reqs,
|
|
141
|
+
"mem_used_mb": _fnum(mem.get("used_mb")),
|
|
142
|
+
"mem_total_mb": _fnum(mem.get("total_mb")),
|
|
143
|
+
"our_mb": round(our_mb, 1) if our_mb is not None else None,
|
|
144
|
+
"claude_cli_mb": _fnum(claude_grp.get("rss_mb")),
|
|
145
|
+
"claude_cli_n": claude_grp.get("count"),
|
|
146
|
+
"sample_age_min": round(int(sample_age_s) / 60, 1) if sample_age_s else None,
|
|
147
|
+
})
|
|
148
|
+
|
|
149
|
+
if args.leaking:
|
|
150
|
+
rows = [r for r in rows if (r["our_mb"] or 0) >= LEAK_RAM_MB]
|
|
151
|
+
|
|
152
|
+
if args.json:
|
|
153
|
+
print(json.dumps(rows, indent=2))
|
|
154
|
+
return 0
|
|
155
|
+
|
|
156
|
+
if not rows:
|
|
157
|
+
print("no installs match")
|
|
158
|
+
return 0
|
|
159
|
+
|
|
160
|
+
hdr = (
|
|
161
|
+
f"{'hostname':<24} {'ver':<9} {'reaper':<16} {'used/total GB':<14} "
|
|
162
|
+
f"{'ourGB':<7} {'claude(n)':<11} {'seen':<8} {'sample':<8} email"
|
|
163
|
+
)
|
|
164
|
+
print(hdr)
|
|
165
|
+
print("-" * len(hdr))
|
|
166
|
+
for r in rows:
|
|
167
|
+
used = r["mem_used_mb"]
|
|
168
|
+
total = r["mem_total_mb"]
|
|
169
|
+
used_total = f"{used/1024:.1f}/{total/1024:.0f}" if used and total else "?"
|
|
170
|
+
our = f"{r['our_mb']/1024:.1f}" if r["our_mb"] else "?"
|
|
171
|
+
claude = (
|
|
172
|
+
f"{r['claude_cli_mb']/1024:.1f}({r['claude_cli_n']})"
|
|
173
|
+
if r["claude_cli_mb"] is not None else "?"
|
|
174
|
+
)
|
|
175
|
+
seen = f"{r['last_seen_age_h']}h" if r["last_seen_age_h"] is not None else "?"
|
|
176
|
+
sample = f"{r['sample_age_min']}m" if r["sample_age_min"] is not None else "none"
|
|
177
|
+
print(
|
|
178
|
+
f"{r['hostname'][:24]:<24} {(r['app_version'] or '?'):<9} {r['reaper']:<16} "
|
|
179
|
+
f"{used_total:<14} {our:<7} {claude:<11} {seen:<8} {sample:<8} {r['git_email']}"
|
|
180
|
+
)
|
|
181
|
+
print(f"\n{len(rows)} install(s). 'reaper=BEHIND' lacks the 1.6.111 session-leak fix; "
|
|
182
|
+
f"'?' = hasn't heartbeat'd on a telemetry-capable build yet.")
|
|
183
|
+
return 0
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
if __name__ == "__main__":
|
|
187
|
+
sys.exit(main())
|