@m13v/s4l 1.7.4-rc.4 → 1.7.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +134 -101
  2. package/bin/cli.js +23 -0
  3. package/mcp/dist/index.js +24 -5
  4. package/mcp/dist/runtime.js +110 -0
  5. package/mcp/dist/version.js +163 -54
  6. package/mcp/dist/version.json +2 -2
  7. package/mcp/manifest.json +1 -1
  8. package/mcp/menubar/s4l_card.py +55 -19
  9. package/mcp/menubar/s4l_menubar.py +179 -56
  10. package/mcp/menubar/s4l_state.py +41 -1
  11. package/mcp/package.json +1 -1
  12. package/package.json +4 -2
  13. package/scripts/active_experiments.py +30 -6
  14. package/scripts/autopilot_stall_watch.py +185 -63
  15. package/scripts/cdp_ready_check.py +94 -0
  16. package/scripts/engage_twitter_helper.py +83 -17
  17. package/scripts/engagement_styles.py +221 -26
  18. package/scripts/enrich_reply_parents.py +68 -22
  19. package/scripts/feedback_digest.py +24 -1
  20. package/scripts/generate_daily_human_style.py +31 -7
  21. package/scripts/invent_styles.py +103 -26
  22. package/scripts/linkedin_cadence.py +234 -0
  23. package/scripts/linkedin_killswitch.py +43 -0
  24. package/scripts/memory_snapshot.py +57 -0
  25. package/scripts/patches/browser-harness-loopback-cdp-proxy.patch +118 -0
  26. package/scripts/recent_self_posts.py +6 -0
  27. package/scripts/reddit_browser.py +57 -3
  28. package/scripts/s4l_box_update.sh +20 -1
  29. package/scripts/s4l_mode.py +3 -4
  30. package/scripts/scan_x_profile.py +25 -7
  31. package/scripts/schedule_state.py +23 -0
  32. package/scripts/scheduled_task_selfheal.py +213 -16
  33. package/scripts/setup_twitter_auth.py +6 -1
  34. package/scripts/snapshot.py +179 -61
  35. package/scripts/stall_guard.py +70 -0
  36. package/scripts/test_scheduled_task_selfheal.py +121 -0
  37. package/scripts/top_performers.py +34 -0
  38. package/scripts/twitter_browser.py +146 -8
  39. package/scripts/twitter_post_plan.py +10 -1
  40. package/scripts/twitter_scan.py +26 -0
  41. package/scripts/watchdog_hung_runs.py +117 -3
  42. package/skill/amplitude-24h-signups.sh +1 -1
  43. package/skill/audit-dm-staleness.sh +1 -1
  44. package/skill/audit-linkedin.sh +1 -1
  45. package/skill/audit-reddit-resurrect.sh +1 -1
  46. package/skill/audit.sh +5 -5
  47. package/skill/check-web-chats.sh +1 -1
  48. package/skill/dm-outreach-linkedin.sh +4 -4
  49. package/skill/dm-outreach-reddit.sh +1 -1
  50. package/skill/dm-outreach-twitter.sh +1 -1
  51. package/skill/engage-dm-replies.sh +16 -3
  52. package/skill/engage-linkedin.sh +2 -2
  53. package/skill/engage-moltbook.sh +1 -1
  54. package/skill/engage-twitter.sh +4 -4
  55. package/skill/github-engage.sh +1 -1
  56. package/skill/lib/browser-launch.sh +54 -0
  57. package/skill/lib/linkedin-backend.sh +22 -15
  58. package/skill/lib/reddit-backend.sh +17 -11
  59. package/skill/lib/twitter-backend.sh +137 -16
  60. package/skill/link-edit-github.sh +1 -1
  61. package/skill/link-edit-moltbook.sh +1 -1
  62. package/skill/link-edit-reddit.sh +1 -1
  63. package/skill/linkedin-cadence.sh +20 -0
  64. package/skill/linkedin-presence.sh +1 -1
  65. package/skill/lock.sh +30 -6
  66. package/skill/precompute-stats.sh +1 -1
  67. package/skill/refresh-instagram-tokens.sh +1 -1
  68. package/skill/refresh-twitter-following.sh +1 -1
  69. package/skill/run-generate-daily-style.sh +1 -1
  70. package/skill/run-instagram-daily.sh +1 -1
  71. package/skill/run-instagram-render.sh +1 -1
  72. package/skill/run-linkedin-launchd.sh +1 -1
  73. package/skill/run-linkedin-unipile.sh +1 -1
  74. package/skill/run-linkedin.sh +1 -1
  75. package/skill/run-reddit-search.sh +2 -2
  76. package/skill/run-reddit-threads.sh +1 -1
  77. package/skill/run-twitter-cycle.sh +190 -55
  78. package/skill/scan-instagram-replies.sh +1 -1
  79. package/skill/sentry-digest.sh +1 -1
  80. package/skill/stats-instagram.sh +1 -1
  81. package/skill/stats-linkedin.sh +2 -2
  82. package/skill/stats.sh +1 -1
  83. package/skill/sweep-link-clicks.sh +1 -1
  84. package/scripts/_lock_preempt_test.py +0 -60
  85. package/scripts/saps_activity.py +0 -18
  86. package/scripts/saps_mode.py +0 -18
@@ -68,6 +68,15 @@ PREEMPT_KILL_WAIT = 5 # secs to wait for a preempted scan holder to die before
68
68
  LOCK_ROLE = (os.environ.get("S4L_LOCK_ROLE") or "scan").strip() or "scan"
69
69
  VIEWPORT = {"width": 911, "height": 1016}
70
70
 
71
+ # CDP connect ceiling. Playwright's connect_over_cdp default is 180s; a wedged
72
+ # harness Chrome (process alive, /json/version answering, websocket upgrade
73
+ # completing, but the browser loop never servicing the CDP handshake) made
74
+ # every reply attempt eat the full 3 minutes WHILE HOLDING the browser lock
75
+ # (S4L-4H, Karol 2026-07-11; identical wedge locally 2026-07-09). A localhost
76
+ # socket either completes the handshake in milliseconds or is wedged; 15s is
77
+ # generous headroom for a loaded box without feeding the lock-contention family.
78
+ CDP_CONNECT_TIMEOUT_MS = int(os.environ.get("S4L_CDP_CONNECT_TIMEOUT_MS") or "15000")
79
+
71
80
  # Posting handle. Resolved at call time from AUTOPOSTER_TWITTER_HANDLE env
72
81
  # var (set by per-account launchd/systemd units) or config.json
73
82
  # accounts.twitter.handle. Returns None when neither source is set.
@@ -226,6 +235,7 @@ def find_twitter_cdp_port():
226
235
 
227
236
  _LOCK_SESSION_ID = f"python:{os.getpid()}"
228
237
  _LOCK_INHERITED = False
238
+ _LOCK_ACQUIRED_AT = 0.0
229
239
  _UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
230
240
 
231
241
 
@@ -243,6 +253,19 @@ def _release_browser_lock():
243
253
  lock = json.load(f)
244
254
  if lock.get("session_id") == _LOCK_SESSION_ID:
245
255
  os.remove(LOCK_FILE)
256
+ # Quantify hold time: long holds are what starve peers into the
257
+ # "locked by session ... waited 45s, giving up" family (S4L-B),
258
+ # and the 2026-07-09/11 wedge showed a hung CDP connect holding
259
+ # this lock for 3 minutes per attempt. >=5s filter keeps the
260
+ # healthy-path noise out of the logs.
261
+ if _LOCK_ACQUIRED_AT:
262
+ _held = time.time() - _LOCK_ACQUIRED_AT
263
+ if _held >= 5:
264
+ print(
265
+ f"[browser-lock] held {_held:.0f}s "
266
+ f"(role={LOCK_ROLE}, pid={os.getpid()})",
267
+ file=sys.stderr,
268
+ )
246
269
  except (json.JSONDecodeError, OSError):
247
270
  pass
248
271
 
@@ -392,7 +415,7 @@ def _acquire_browser_lock():
392
415
  LIVE peers' locks (defect b) and was removed 2026-06-16. Dead holders are
393
416
  reclaimed here instead. See docs/twitter_browser_lock.md.
394
417
  """
395
- global _LOCK_SESSION_ID, _LOCK_INHERITED
418
+ global _LOCK_SESSION_ID, _LOCK_INHERITED, _LOCK_ACQUIRED_AT
396
419
  deadline = time.time() + LOCK_WAIT_MAX
397
420
  # Guarantee the lock dir exists so _try_take_lock's O_EXCL create can't fail
398
421
  # for a missing-parent reason (which would otherwise spin the no-file path).
@@ -548,6 +571,7 @@ def _acquire_browser_lock():
548
571
  sys.exit(1)
549
572
  time.sleep(LOCK_POLL_INTERVAL)
550
573
  continue
574
+ _LOCK_ACQUIRED_AT = time.time()
551
575
 
552
576
 
553
577
  def _refresh_browser_lock():
@@ -559,7 +583,88 @@ def _refresh_browser_lock():
559
583
  pass
560
584
 
561
585
 
586
+ def _cdp_diagnostics(cdp_url):
587
+ """Cheap post-mortem probe after a failed CDP attach.
588
+
589
+ Distinguishes "Chrome down" (HTTP dead) from "Chrome wedged" (HTTP still
590
+ answering /json/version but the CDP session handshake never completing) in
591
+ the error JSON itself, so the Sentry event carries the diagnosis instead of
592
+ requiring log archaeology on the customer box (S4L-4H took exactly that).
593
+ """
594
+ import urllib.request
595
+ diag = {"http_alive": False}
596
+ base = cdp_url if "://" in cdp_url else f"http://{cdp_url}"
597
+ try:
598
+ with urllib.request.urlopen(f"{base}/json/version", timeout=3) as r:
599
+ v = json.loads(r.read().decode())
600
+ diag["http_alive"] = True
601
+ diag["browser"] = v.get("Browser")
602
+ except Exception as e:
603
+ diag["version_error"] = str(e)[:120]
604
+ return diag
605
+ try:
606
+ with urllib.request.urlopen(f"{base}/json/list", timeout=3) as r:
607
+ diag["targets"] = len(json.loads(r.read().decode()))
608
+ except Exception as e:
609
+ diag["list_error"] = str(e)[:120]
610
+ return diag
611
+
612
+
613
+ # --- L1/L2 browser-hang guards (2026-07-13) ----------------------------------
614
+ # L1: no single Playwright operation may block forever. Until today NO script
615
+ # set a default timeout, so any page op could hang indefinitely — a twitter
616
+ # cycle wedged 60+ min in Phase 2b-prep media capture during a network flap
617
+ # (2026-07-13 08:02, pid 1380) while holding the twitter-browser lock, blocking
618
+ # every peer pipeline until the 180-min watchdog cap. Healthy ops here finish
619
+ # in <10s; navigation on the x.com SPA gets extra slack. Callers all have
620
+ # retry layers above, so failing fast is strictly better than hanging.
621
+ BROWSER_OP_TIMEOUT_MS = 30_000
622
+ BROWSER_NAV_TIMEOUT_MS = 60_000
623
+ # L2: liveness heartbeat for the SHELL browser lock (the cross-pipeline mutex,
624
+ # /tmp/social-autoposter-twitter-browser.lock). Touched on attach and on every
625
+ # page network request (throttled), so "lock held + heartbeat stale" means the
626
+ # holder is wedged, not just slow — a healthy 65-min phase2b-gen hold shows
627
+ # constant traffic. watchdog_hung_runs.py reads this to TERM wedged holders in
628
+ # ~30 min instead of the 180-min age cap. Best-effort: never fails the caller.
629
+ _BROWSER_LOCK_HEARTBEAT = "/tmp/social-autoposter-twitter-browser.lock/heartbeat"
630
+ _HB_THROTTLE_S = 5
631
+ _hb_last_touch = 0.0
632
+
633
+
634
+ def _touch_browser_heartbeat(*_args):
635
+ global _hb_last_touch
636
+ now = time.time()
637
+ if now - _hb_last_touch < _HB_THROTTLE_S:
638
+ return
639
+ _hb_last_touch = now
640
+ try:
641
+ if os.path.isdir(os.path.dirname(_BROWSER_LOCK_HEARTBEAT)):
642
+ with open(_BROWSER_LOCK_HEARTBEAT, "w") as f:
643
+ f.write(str(int(now)))
644
+ except Exception:
645
+ pass
646
+
647
+
562
648
  def get_browser_and_page(playwright):
649
+ """Instrumented entry point: attach via _get_browser_and_page_raw, then
650
+ (L1) set default per-op/navigation timeouts so no page call can hang
651
+ forever, and (L2) wire the browser-lock liveness heartbeat. All new code
652
+ should use THIS, never the raw variant."""
653
+ browser, page, is_cdp = _get_browser_and_page_raw(playwright)
654
+ try:
655
+ page.set_default_timeout(BROWSER_OP_TIMEOUT_MS)
656
+ page.set_default_navigation_timeout(BROWSER_NAV_TIMEOUT_MS)
657
+ except Exception:
658
+ pass
659
+ _touch_browser_heartbeat()
660
+ try:
661
+ page.on("request", _touch_browser_heartbeat)
662
+ except Exception:
663
+ pass
664
+ return browser, page, is_cdp
665
+
666
+
667
+ def _get_browser_and_page_raw(playwright):
563
668
  """Connect to the running twitter-harness Chrome via CDP.
564
669
 
565
670
  Returns (browser, page, is_cdp=True). `page` is a reused existing Twitter
@@ -581,8 +686,11 @@ def get_browser_and_page(playwright):
581
686
 
582
687
  cdp_url_override = os.environ.get("TWITTER_CDP_URL", "").strip()
583
688
  if cdp_url_override:
689
+ _t0 = time.time()
584
690
  try:
585
- browser = playwright.chromium.connect_over_cdp(cdp_url_override)
691
+ browser = playwright.chromium.connect_over_cdp(
692
+ cdp_url_override, timeout=CDP_CONNECT_TIMEOUT_MS
693
+ )
586
694
  contexts = browser.contexts
587
695
  if contexts:
588
696
  context = contexts[0]
@@ -601,16 +709,19 @@ def get_browser_and_page(playwright):
601
709
  _release_browser_lock()
602
710
  print(json.dumps({
603
711
  "success": False,
604
- "error": f"TWITTER_CDP_URL connect failed ({cdp_url_override}): {e}"
712
+ "error": f"TWITTER_CDP_URL connect failed ({cdp_url_override}): {e}",
713
+ "connect_elapsed_s": round(time.time() - _t0, 1),
714
+ "cdp_diag": _cdp_diagnostics(cdp_url_override),
605
715
  }))
606
716
  sys.exit(1)
607
717
 
608
718
  cdp_port = find_twitter_cdp_port()
609
719
 
610
720
  if cdp_port:
721
+ _t0 = time.time()
611
722
  try:
612
723
  browser = playwright.chromium.connect_over_cdp(
613
- f"http://localhost:{cdp_port}"
724
+ f"http://localhost:{cdp_port}", timeout=CDP_CONNECT_TIMEOUT_MS
614
725
  )
615
726
  contexts = browser.contexts
616
727
  if contexts:
@@ -625,7 +736,9 @@ def get_browser_and_page(playwright):
625
736
  _release_browser_lock()
626
737
  print(json.dumps({
627
738
  "success": False,
628
- "error": f"harness CDP attach failed (port {cdp_port}): {e}"
739
+ "error": f"harness CDP attach failed (port {cdp_port}): {e}",
740
+ "connect_elapsed_s": round(time.time() - _t0, 1),
741
+ "cdp_diag": _cdp_diagnostics(f"http://localhost:{cdp_port}"),
629
742
  }))
630
743
  sys.exit(1)
631
744
 
@@ -2035,7 +2148,7 @@ def read_conversation(thread_url, max_messages=20):
2035
2148
  browser.close()
2036
2149
 
2037
2150
 
2038
- def send_dm(thread_url, message, dm_id=None):
2151
+ def send_dm(thread_url, message, dm_id=None, apply_campaigns=True):
2039
2152
  """Send a message in a Twitter/X DM conversation.
2040
2153
 
2041
2154
  Navigates to the thread URL, types the message in the compose box,
@@ -2073,8 +2186,16 @@ def send_dm(thread_url, message, dm_id=None):
2073
2186
  message = wrap_res["text"]
2074
2187
  minted_link_codes = wrap_res.get("minted_codes", [])
2075
2188
 
2189
+ # apply_campaigns=False (via S4L_SKIP_CAMPAIGN_SUFFIX on the send-dm CLI)
2190
+ # opts this DM out of active-campaign suffixes. Set ONLY when delivering a
2191
+ # HUMAN-drafted escalation reply (engage-dm-replies Phase 0): a human
2192
+ # answering an escalation — often an AI-callout — must not get a bot tell
2193
+ # coin-flipped onto their words (2026-07-13: the ' written with ai' suffix
2194
+ # landing on a hand-approved recovery DM to ngeloxyz helped burn the lead).
2195
+ # The autopilot never sets it, so the A/B experiment continues everywhere
2196
+ # else. Mirrors reply_to_tweet's apply_campaigns plumbing.
2076
2197
  applied_campaigns = []
2077
- for cid, suffix, sample_rate in _load_active_twitter_campaigns():
2198
+ for cid, suffix, sample_rate in (_load_active_twitter_campaigns() if apply_campaigns else []):
2078
2199
  if random.random() < sample_rate:
2079
2200
  # Wrap any URLs in the suffix through dm_short_links (DM rail) so
2080
2201
  # clicks attribute to this DM. Falls back to raw suffix if dm_id
@@ -2611,6 +2732,19 @@ def _anchor_repost_from_tweets(tweets, anchor_tweet_id):
2611
2732
 
2612
2733
 
2613
2734
  def scrape_thread_media(thread_url, scroll_count=1):
2735
+ """Deadline wrapper: media capture relays CDP to the harness Chrome, and
2736
+ against a half-wedged browser or dead renderer a single evaluate can hang
2737
+ FOREVER (2026-07-13: hung 34+ min at candidate 24/92 holding the browser
2738
+ lock; only the watchdog's 90-min lock-liveness net would have reaped it).
2739
+ stall_guard aborts the process fast, stamps the wedge strike, and lets the
2740
+ cycle-entry heal chain converge. Shared implementation — see
2741
+ scripts/stall_guard.py; do NOT re-implement per step."""
2742
+ from stall_guard import stall_guard
2743
+ with stall_guard("thread_media", thread_url):
2744
+ return _scrape_thread_media_inner(thread_url, scroll_count)
2745
+
2746
+
2747
+ def _scrape_thread_media_inner(thread_url, scroll_count=1):
2614
2748
  """Navigate to a tweet's permalink and return the media of the anchor tweet.
2615
2749
 
2616
2750
  Deterministic, model-free media capture for the MAIN posting cycle: the
@@ -2814,7 +2948,11 @@ def main():
2814
2948
  except ValueError:
2815
2949
  print(f"send-dm: dm_id must be int, got {sys.argv[4]!r}", file=sys.stderr)
2816
2950
  sys.exit(1)
2817
- result = send_dm(sys.argv[2], sys.argv[3], dm_id=dm_id_arg)
2951
+ # S4L_SKIP_CAMPAIGN_SUFFIX=1 opts this DM out of campaign suffixes.
2952
+ # Set ONLY by engage-dm-replies Phase 0 when delivering a human-drafted
2953
+ # escalation reply — see send_dm's apply_campaigns comment.
2954
+ _skip_camp = os.environ.get("S4L_SKIP_CAMPAIGN_SUFFIX", "").strip().lower() in ("1", "true", "yes")
2955
+ result = send_dm(sys.argv[2], sys.argv[3], dm_id=dm_id_arg, apply_campaigns=not _skip_camp)
2818
2956
  print(json.dumps(result, indent=2))
2819
2957
 
2820
2958
  elif cmd == "notifications":
@@ -104,7 +104,16 @@ signal.signal(signal.SIGTERM, _on_sigterm)
104
104
  # since both shell out to this script. Set before any child is spawned.
105
105
  os.environ["S4L_LOCK_ROLE"] = "post"
106
106
 
107
- REPO_DIR = os.path.expanduser("~/social-autoposter")
107
+ # Resolve the repo root the way the rest of the pipeline does (run_claude.sh,
108
+ # identity.py): S4L_REPO_DIR when the caller sets it, else this script's own
109
+ # parent tree. The old hardcoded ~/social-autoposter pointed OUTSIDE the
110
+ # managed package on customer boxes, so every child spawned below
111
+ # (twitter_browser.py, log_post.py, ...) ran code auto-update could never
112
+ # reach (S4L-4H triage 2026-07-12, Karol's install).
113
+ REPO_DIR = (
114
+ os.environ.get("S4L_REPO_DIR")
115
+ or os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
116
+ )
108
117
  TWITTER_BROWSER = os.path.join(REPO_DIR, "scripts", "twitter_browser.py")
109
118
  LOG_POST = os.path.join(REPO_DIR, "scripts", "log_post.py")
110
119
  CAMPAIGN_BUMP = os.path.join(REPO_DIR, "scripts", "campaign_bump.py")
@@ -55,6 +55,7 @@ import json
55
55
  import os
56
56
  import pathlib
57
57
  import re
58
+ import sys
58
59
  import time
59
60
  import urllib.parse
60
61
 
@@ -245,6 +246,15 @@ def _navigate(url: str) -> None:
245
246
  new_tab(url)
246
247
 
247
248
 
249
+ # Per-query stall guard: shared implementation (scripts/stall_guard.py) used
250
+ # by every browser-touching pipeline step — do NOT re-implement per step.
251
+ # S4L_SCAN_QUERY_DEADLINE_S kept as a scan-specific override for continuity
252
+ # with rc.36-38; unset, the shared default applies.
253
+ from stall_guard import stall_guard # noqa: E402
254
+
255
+ _SCAN_DEADLINE_OVERRIDE = os.environ.get("S4L_SCAN_QUERY_DEADLINE_S")
256
+
257
+
248
258
  def scan(
249
259
  query: str,
250
260
  project: str,
@@ -257,6 +267,22 @@ def scan(
257
267
  ###TWEETS_BEGIN###/###TWEETS_END### sentinels for the scan model to relay
258
268
  into StructuredOutput; also returns the kept list so direct callers (tests,
259
269
  future shell-driven invocations) can consume it without parsing stdout."""
270
+ with stall_guard(
271
+ "scan_query",
272
+ query,
273
+ deadline_s=float(_SCAN_DEADLINE_OVERRIDE) if _SCAN_DEADLINE_OVERRIDE else None,
274
+ ):
275
+ return _scan_inner(query, project, search_topic, freshness_hours, skip_ids, settle_seconds)
276
+
277
+
278
+ def _scan_inner(
279
+ query: str,
280
+ project: str,
281
+ search_topic: str,
282
+ freshness_hours: int = 6,
283
+ skip_ids=None,
284
+ settle_seconds: float = 4.0,
285
+ ) -> list:
260
286
  skip = {str(s) for s in (skip_ids or [])}
261
287
  url = _build_url(query, int(freshness_hours))
262
288
  _navigate(url)
@@ -17,8 +17,44 @@ from pathlib import Path
17
17
 
18
18
  REPO = Path("/Users/matthewdi/social-autoposter")
19
19
  LOG_RUN_PY = REPO / "scripts" / "log_run.py"
20
- SKILL_PATH_MARKER = "/social-autoposter/skill/"
20
+ # Both homes of skill/*.sh: the operator repo AND the installed-app package.
21
+ # The single operator-repo marker left every kicker-driven cycle (which runs
22
+ # from ~/.social-autoposter-mcp/repo/package/skill/) invisible to this
23
+ # watchdog — a scan stuck on a half-wedged Chrome sat 23+ minutes at zero
24
+ # progress with nothing entitled to kill it (2026-07-13).
25
+ SKILL_PATH_MARKERS = (
26
+ "/social-autoposter/skill/",
27
+ "/.social-autoposter-mcp/repo/package/skill/",
28
+ )
21
29
  MAX_AGE_SEC = 45 * 60
30
+
31
+ # --- L2: browser-lock liveness (2026-07-13) ----------------------------------
32
+ # The age caps below are backstops sized ABOVE the worst-case duration of a
33
+ # HEALTHY run, so they can't tell "90 min in and progressing" from "wedged
34
+ # since minute 4". This check can: the shared browser libs (twitter_browser.py
35
+ # / reddit_browser.py get_browser_and_page) touch <lock_dir>/heartbeat on every
36
+ # attach and page network event, so a *-browser lock held by a LIVE process
37
+ # whose heartbeat has gone stale means the holder is wedged mid-browser-work
38
+ # (2026-07-13: pid 1380 hung 60+ min in Phase 2b-prep media capture during a
39
+ # network flap, blocking every peer pipeline until the 180-min age cap).
40
+ # Thresholds: WARN at 30 min stale (log-only). KILL at 90 min stale — above
41
+ # the gen phase's documented 60-min worst-case ceiling, so a legit long hold
42
+ # with no python browser invocations can't be false-killed — and ONLY when
43
+ # peers are actually queued waiting (an unwanted lock harms nobody). The
44
+ # 2026-07-12 lock.sh signal fix makes the TERM safe: the holder releases its
45
+ # locks and exits instead of continuing lock-protected work.
46
+ # ONLY the heartbeat-instrumented platforms. linkedin-browser holders write no
47
+ # heartbeat (the linkedin pipeline drives Chrome via the bh harness, not a
48
+ # shared python lib), so on pid-file age alone a healthy long linkedin run
49
+ # would be indistinguishable from a wedge — add it here only once its driver
50
+ # touches <lock_dir>/heartbeat too.
51
+ BROWSER_LOCKS_WATCHED = (
52
+ "/tmp/social-autoposter-twitter-browser.lock",
53
+ "/tmp/social-autoposter-reddit-browser.lock",
54
+ )
55
+ HB_WARN_SEC = 30 * 60
56
+ HB_KILL_SEC = 90 * 60
57
+ HB_REWARN_SEC = 30 * 60 # re-log the warning at most this often per holder
22
58
  # Per-script cap overrides for pipelines that legitimately run longer than
23
59
  # the 45 min global (stats.py over ~4-5k posts + rate-limit sleeps).
24
60
  # Key is (script_file, platform_or_None). Lookup order: (script, platform),
@@ -184,12 +220,12 @@ def list_skill_shell_processes():
184
220
  ppid = int(ppid_s)
185
221
  except ValueError:
186
222
  continue
187
- if SKILL_PATH_MARKER not in command:
223
+ if not any(m in command for m in SKILL_PATH_MARKERS):
188
224
  continue
189
225
  script_name = None
190
226
  tokens = command.split()
191
227
  for tok in tokens:
192
- if tok.endswith(".sh") and SKILL_PATH_MARKER in tok:
228
+ if tok.endswith(".sh") and any(m in tok for m in SKILL_PATH_MARKERS):
193
229
  script_name = os.path.basename(tok)
194
230
  break
195
231
  if not script_name:
@@ -317,6 +353,83 @@ def emit_job_log(label, elapsed_sec):
317
353
  )
318
354
 
319
355
 
356
+ def _holder_cmdline(pid: int) -> str:
357
+ try:
358
+ out = subprocess.run(
359
+ ["ps", "-p", str(pid), "-o", "command="],
360
+ capture_output=True, text=True, timeout=5,
361
+ ).stdout.strip()
362
+ return out[:200]
363
+ except Exception:
364
+ return ""
365
+
366
+
367
+ def check_browser_lock_liveness() -> None:
368
+ """L2: TERM a live *-browser lock holder whose liveness heartbeat has gone
369
+ stale for HB_KILL_SEC while peers wait. See the constants block up top."""
370
+ now = time.time()
371
+ for lock_dir in BROWSER_LOCKS_WATCHED:
372
+ if not os.path.isdir(lock_dir):
373
+ continue
374
+ try:
375
+ with open(os.path.join(lock_dir, "pid")) as f:
376
+ holder = int((f.read().strip() or "0"))
377
+ except (OSError, ValueError):
378
+ continue
379
+ if holder <= 0:
380
+ continue
381
+ try:
382
+ os.kill(holder, 0) # liveness probe only
383
+ except ProcessLookupError:
384
+ continue # dead holder: acquire_lock's dead_pid reclaim owns this
385
+ except OSError:
386
+ continue
387
+ # Last liveness = freshest of the heartbeat (browser libs) and the pid
388
+ # file (stamped at acquire), so a just-acquired lock is never "stale".
389
+ marks = []
390
+ for name in ("heartbeat", "pid"):
391
+ try:
392
+ marks.append(os.path.getmtime(os.path.join(lock_dir, name)))
393
+ except OSError:
394
+ pass
395
+ if not marks:
396
+ continue
397
+ stale = now - max(marks)
398
+ if stale < HB_WARN_SEC:
399
+ continue
400
+ try:
401
+ waiters = len(os.listdir(f"{lock_dir}.queue"))
402
+ except OSError:
403
+ waiters = 0
404
+ lock_name = os.path.basename(lock_dir)
405
+ if stale >= HB_KILL_SEC and waiters >= 1:
406
+ cmd = _holder_cmdline(holder)
407
+ watchdog_log(
408
+ f"KILL-STALE-HOLDER {lock_name} pid={holder} heartbeat_stale={int(stale)}s "
409
+ f"waiters={waiters} cmd={cmd!r}"
410
+ )
411
+ killed = kill_tree(holder)
412
+ watchdog_log(f" killed pids: {killed}")
413
+ continue
414
+ # Warn tier: log-only, rate-limited via a marker inside the lock dir
415
+ # (dies with the lock, so each hold re-warns independently).
416
+ marker = os.path.join(lock_dir, "hb-warned")
417
+ try:
418
+ if now - os.path.getmtime(marker) < HB_REWARN_SEC:
419
+ continue
420
+ except OSError:
421
+ pass
422
+ try:
423
+ with open(marker, "w") as f:
424
+ f.write(str(int(now)))
425
+ except OSError:
426
+ pass
427
+ watchdog_log(
428
+ f"WARN-STALE-HOLDER {lock_name} pid={holder} heartbeat_stale={int(stale)}s "
429
+ f"waiters={waiters} cmd={_holder_cmdline(holder)!r}"
430
+ )
431
+
432
+
320
433
  def main() -> None:
321
434
  procs = list_skill_shell_processes()
322
435
  for pid, ppid, etimes, script_file, platform in procs:
@@ -337,6 +450,7 @@ def main() -> None:
337
450
  watchdog_log(f" script trap already logged {label} — skipping watchdog emit")
338
451
  else:
339
452
  emit_job_log(label, etimes)
453
+ check_browser_lock_liveness()
340
454
 
341
455
 
342
456
  if __name__ == "__main__":
@@ -34,5 +34,5 @@ RUN_START=$(date +%s)
34
34
  EXIT_CODE=$?
35
35
  RUN_ELAPSED=$(( $(date +%s) - RUN_START ))
36
36
 
37
- echo "[$(date +%H:%M:%S)] === done in ${RUN_ELAPSED}s (exit=${EXIT_CODE}) ==="
37
+ echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] === done in ${RUN_ELAPSED}s (exit=${EXIT_CODE}) ==="
38
38
  exit "$EXIT_CODE"
@@ -14,7 +14,7 @@ LOG_DIR="$REPO_DIR/skill/logs"
14
14
  mkdir -p "$LOG_DIR"
15
15
  LOG_FILE="$LOG_DIR/audit-dm-staleness-$(date +%Y-%m-%d_%H%M%S).log"
16
16
 
17
- log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$LOG_FILE"; }
17
+ log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG_FILE"; }
18
18
 
19
19
  # HTTP-only lane (2026-06-01): both staleness UPDATEs run server-side via the
20
20
  # s4l.ai API (POST /api/v1/dms/staleness-sweep). No DATABASE_URL, no psql.
@@ -6,7 +6,7 @@
6
6
  # State: ~/.claude/social-autoposter/linkedin.killswitch
7
7
  # Clear: python3 ~/social-autoposter/scripts/linkedin_killswitch.py clear
8
8
  if [ -f "$HOME/.claude/social-autoposter/linkedin.killswitch" ]; then
9
- echo "[$(date +%H:%M:%S)] LINKEDIN_KILLSWITCH active. Aborting LinkedIn pipeline."
9
+ echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] LINKEDIN_KILLSWITCH active. Aborting LinkedIn pipeline."
10
10
  echo " Re-auth LinkedIn in harness Chrome, then: python3 ~/social-autoposter/scripts/linkedin_killswitch.py clear"
11
11
  exit 0
12
12
  fi
@@ -38,7 +38,7 @@ AUDIT_HELPER="$REPO_DIR/scripts/audit_helper.py"
38
38
  mkdir -p "$LOG_DIR"
39
39
  LOG_FILE="$LOG_DIR/audit-reddit-resurrect-$(date +%Y-%m-%d_%H%M%S).log"
40
40
 
41
- log() { echo "[$(date +%H:%M:%S)] $*" >> "$LOG_FILE"; echo "[$(date +%H:%M:%S)] $*"; }
41
+ log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" >> "$LOG_FILE"; echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*"; }
42
42
 
43
43
  RUN_START=$(date +%s)
44
44
  log "=== Reddit resurrect audit: $(date) ==="
package/skill/audit.sh CHANGED
@@ -62,10 +62,10 @@ case "${PLATFORM:-all}" in
62
62
  _LI_BOOT_RC=0
63
63
  ( source "$(dirname "$0")/lib/linkedin-backend.sh"; ensure_linkedin_browser_for_backend ) || _LI_BOOT_RC=$?
64
64
  if [ "$_LI_BOOT_RC" -eq 78 ]; then
65
- echo "[$(date +%H:%M:%S)] linkedin-pipeline lock: peer pipeline is driving the 9556 Chrome; skipping this fire"
65
+ echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] linkedin-pipeline lock: peer pipeline is driving the 9556 Chrome; skipping this fire"
66
66
  exit 0
67
67
  elif [ "$_LI_BOOT_RC" -ne 0 ]; then
68
- echo "[$(date +%H:%M:%S)] ERROR: linkedin browser bootstrap failed (rc=$_LI_BOOT_RC)"
68
+ echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] ERROR: linkedin browser bootstrap failed (rc=$_LI_BOOT_RC)"
69
69
  exit "$_LI_BOOT_RC"
70
70
  fi
71
71
  ;;
@@ -84,10 +84,10 @@ case "${PLATFORM:-all}" in
84
84
  _LI_BOOT_RC=0
85
85
  ( source "$(dirname "$0")/lib/linkedin-backend.sh"; ensure_linkedin_browser_for_backend ) || _LI_BOOT_RC=$?
86
86
  if [ "$_LI_BOOT_RC" -eq 78 ]; then
87
- echo "[$(date +%H:%M:%S)] linkedin-pipeline lock: peer pipeline is driving the 9556 Chrome; skipping this fire (all-platform audit)"
87
+ echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] linkedin-pipeline lock: peer pipeline is driving the 9556 Chrome; skipping this fire (all-platform audit)"
88
88
  exit 0
89
89
  elif [ "$_LI_BOOT_RC" -ne 0 ]; then
90
- echo "[$(date +%H:%M:%S)] ERROR: linkedin browser bootstrap failed (rc=$_LI_BOOT_RC)"
90
+ echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] ERROR: linkedin browser bootstrap failed (rc=$_LI_BOOT_RC)"
91
91
  exit "$_LI_BOOT_RC"
92
92
  fi
93
93
  python3 "$REPO_DIR_FOR_LOCK/scripts/reddit_browser_lock.py" acquire --timeout 3600 --ttl 90 2>&1 || \
@@ -113,7 +113,7 @@ mkdir -p "$LOG_DIR"
113
113
  LOG_TAG="${PLATFORM:-all}"
114
114
  LOG_FILE="$LOG_DIR/audit-${LOG_TAG}-$(date +%Y-%m-%d_%H%M%S).log"
115
115
 
116
- log() { echo "[$(date +%H:%M:%S)] $*" >> "$LOG_FILE"; echo "[$(date +%H:%M:%S)] $*"; }
116
+ log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" >> "$LOG_FILE"; echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*"; }
117
117
 
118
118
  RUN_START=$(date +%s)
119
119
  log "=== Audit Pipeline Run (${LOG_TAG}): $(date) ==="
@@ -35,7 +35,7 @@ mkdir -p "$LOG_DIR"
35
35
  PYTHON_BIN="${PYTHON_BIN:-/opt/homebrew/bin/python3.11}"
36
36
  [ -x "$PYTHON_BIN" ] || PYTHON_BIN="/usr/bin/python3"
37
37
 
38
- log() { echo "[$(date +%H:%M:%S)] $*" >> "$LOG_DIR/web-chat.log"; }
38
+ log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" >> "$LOG_DIR/web-chat.log"; }
39
39
 
40
40
  # Step 1: query Postgres for unread threads.
41
41
  CHATS=$("$PYTHON_BIN" "$SCRIPTS_DIR/check_unread_web_chats.py" 2>>"$LOG_DIR/web-chat.log")
@@ -12,7 +12,7 @@ set -euo pipefail
12
12
  # State: ~/.claude/social-autoposter/linkedin.killswitch
13
13
  # Clear: python3 ~/social-autoposter/scripts/linkedin_killswitch.py clear
14
14
  if [ -f "$HOME/.claude/social-autoposter/linkedin.killswitch" ]; then
15
- echo "[$(date +%H:%M:%S)] LINKEDIN_KILLSWITCH active. Aborting LinkedIn pipeline."
15
+ echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] LINKEDIN_KILLSWITCH active. Aborting LinkedIn pipeline."
16
16
  echo " Re-auth LinkedIn in harness Chrome, then: python3 ~/social-autoposter/scripts/linkedin_killswitch.py clear"
17
17
  exit 0
18
18
  fi
@@ -37,10 +37,10 @@ acquire_lock "linkedin-browser" 3600
37
37
  _LI_BOOT_RC=0
38
38
  ensure_linkedin_browser_for_backend || _LI_BOOT_RC=$?
39
39
  if [ "$_LI_BOOT_RC" -eq 78 ]; then
40
- echo "[$(date +%H:%M:%S)] linkedin-pipeline lock: peer pipeline is driving the 9556 Chrome; skipping this fire"
40
+ echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] linkedin-pipeline lock: peer pipeline is driving the 9556 Chrome; skipping this fire"
41
41
  exit 0
42
42
  elif [ "$_LI_BOOT_RC" -ne 0 ]; then
43
- echo "[$(date +%H:%M:%S)] ERROR: linkedin browser bootstrap failed (rc=$_LI_BOOT_RC)"
43
+ echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] ERROR: linkedin browser bootstrap failed (rc=$_LI_BOOT_RC)"
44
44
  exit "$_LI_BOOT_RC"
45
45
  fi
46
46
  acquire_lock "dm-outreach-linkedin" 2700
@@ -56,7 +56,7 @@ LOG_DIR="$REPO_DIR/skill/logs"
56
56
  mkdir -p "$LOG_DIR"
57
57
  LOG_FILE="$LOG_DIR/dm-outreach-linkedin-$(date +%Y-%m-%d_%H%M%S).log"
58
58
 
59
- log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$LOG_FILE"; }
59
+ log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG_FILE"; }
60
60
 
61
61
  RUN_START=$(date +%s)
62
62
  log "=== LinkedIn DM Outreach Run: $(date) ==="
@@ -50,7 +50,7 @@ LOG_DIR="$REPO_DIR/skill/logs"
50
50
  mkdir -p "$LOG_DIR"
51
51
  LOG_FILE="$LOG_DIR/dm-outreach-reddit-$(date +%Y-%m-%d_%H%M%S).log"
52
52
 
53
- log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$LOG_FILE"; }
53
+ log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG_FILE"; }
54
54
 
55
55
  RUN_START=$(date +%s)
56
56
  log "=== Reddit DM Outreach Run: $(date) ==="
@@ -44,7 +44,7 @@ SKILL_FILE="$REPO_DIR/SKILL.md"
44
44
  # dm-outreach-linkedin.sh (migrated 2026-05-12).
45
45
  # (LOG_DIR/LOG_FILE bootstrapped at top of script.)
46
46
 
47
- log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$LOG_FILE"; }
47
+ log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG_FILE"; }
48
48
 
49
49
  RUN_START=$(date +%s)
50
50
  log "=== Twitter DM Outreach Run: $(date) ==="
@@ -35,7 +35,7 @@ fi
35
35
  # State: ~/.claude/social-autoposter/linkedin.killswitch
36
36
  # Clear: python3 ~/social-autoposter/scripts/linkedin_killswitch.py clear
37
37
  if [ "$PLATFORM" = "linkedin" ] && [ -f "$HOME/.claude/social-autoposter/linkedin.killswitch" ]; then
38
- echo "[$(date +%H:%M:%S)] LINKEDIN_KILLSWITCH active. Aborting LinkedIn DM-replies pipeline."
38
+ echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] LINKEDIN_KILLSWITCH active. Aborting LinkedIn DM-replies pipeline."
39
39
  echo " Re-auth LinkedIn in harness Chrome, then: python3 ~/social-autoposter/scripts/linkedin_killswitch.py clear"
40
40
  exit 0
41
41
  fi
@@ -116,7 +116,7 @@ LOG_SUFFIX=""
116
116
  [ -n "$PLATFORM" ] && LOG_SUFFIX="-$PLATFORM"
117
117
  LOG_FILE="$LOG_DIR/engage-dm-replies${LOG_SUFFIX}-$(date +%Y-%m-%d_%H%M%S).log"
118
118
 
119
- log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$LOG_FILE"; }
119
+ log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG_FILE"; }
120
120
 
121
121
  RUN_START=$(date +%s)
122
122
  log "=== DM Reply Engagement Run: $(date) (platform: ${PLATFORM:-all}) ==="
@@ -302,12 +302,25 @@ The \`public_target_url\` field is THEIR public comment that originally led to t
302
302
 
303
303
  ### Step C. If \`reply_channel\` is \`dm\` or \`both\`: deliver the DM
304
304
 
305
+ **NO CAMPAIGN SUFFIX ON HUMAN REPLIES.** A row in this queue is a HUMAN answering
306
+ an escalation (often an AI-callout), so the campaign suffix (e.g. ' written with
307
+ ai') must NEVER be coin-flipped onto it — it re-burns the exact lead the human is
308
+ trying to recover (2026-07-13: the suffix landed on the hand-approved ngeloxyz
309
+ recovery DM and helped kill the thread). When sending via the python CLIs, ALWAYS
310
+ prefix the env opt-out:
311
+ \`\`\`bash
312
+ S4L_SKIP_CAMPAIGN_SUFFIX=1 python3 scripts/twitter_browser.py send-dm THREAD_URL "MESSAGE" DM_ID
313
+ S4L_SKIP_CAMPAIGN_SUFFIX=1 python3 scripts/reddit_browser.py send-dm CHAT_URL "MESSAGE" DM_ID
314
+ \`\`\`
315
+ This applies ONLY inside Phase 0 (human replies). Auto-replies in the later phases
316
+ keep the normal campaign behavior — do NOT carry this env var forward.
317
+
305
318
  1. Craft a natural DM based on the human's instructions and the conversation context.
306
319
  2. Navigate to the conversation on the correct platform using \`chat_url\` (or find the conversation with their_author).
307
320
  - **Reddit Chat** (mcp__reddit-harness__bh_run tool, CDP-driven real Chrome on port 9557)
308
321
  - **LinkedIn Messages** (mcp__linkedin-harness__bh_run tool, CDP-driven real Chrome on port 9556)
309
322
  - **X/Twitter DMs** (mcp__twitter-harness__bh_run tool), if encrypted DM passcode dialog appears, enter: $TWITTER_DM_PASSCODE
310
- 3. Type and send the crafted DM.
323
+ 3. Type and send the crafted DM (with the suffix opt-out env var when using the python send-dm CLIs, per the rule above).
311
324
  4. Log the outbound message (log what you ACTUALLY SENT, not the human's instructions). Pass --verified ONLY when the browser tool returned verified=true. If verification failed, log nothing and let the next cycle retry; never pass --verified speculatively:
312
325
  \`\`\`bash
313
326
  cd ~/social-autoposter && python3 scripts/dm_conversation.py log-outbound --dm-id DM_ID --content "THE_CRAFTED_DM_YOU_SENT" --verified