@m13v/s4l 1.7.1-rc.2 → 1.7.1-rc.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/bin/cli.js +23 -0
  2. package/mcp/dist/index.js +332 -24
  3. package/mcp/dist/onboarding.js +3 -10
  4. package/mcp/dist/panel.html +42 -31
  5. package/mcp/dist/product-link.html +1 -1
  6. package/mcp/dist/runtime.js +101 -1
  7. package/mcp/dist/telemetry.js +80 -3
  8. package/mcp/dist/version.json +2 -2
  9. package/mcp/manifest.json +9 -1
  10. package/mcp/menubar/dashboard_server.py +12 -0
  11. package/mcp/menubar/s4l_card.py +472 -96
  12. package/mcp/menubar/s4l_menubar.py +269 -26
  13. package/mcp/menubar/s4l_state.py +169 -23
  14. package/mcp/package.json +1 -1
  15. package/mcp/shared/doctor.cjs +8 -0
  16. package/mcp/shared/onboarding-ledger.cjs +1 -0
  17. package/package.json +3 -1
  18. package/scripts/active_users.py +3 -6
  19. package/scripts/autopilot_stall_watch.py +200 -16
  20. package/scripts/claude_job.py +6 -0
  21. package/scripts/feedback_digest.py +34 -6
  22. package/scripts/get-latest-staging-mcpb.sh +50 -0
  23. package/scripts/harness_overlay.py +45 -0
  24. package/scripts/identity.py +26 -0
  25. package/scripts/learned_preferences.py +241 -32
  26. package/scripts/link_tail.py +167 -91
  27. package/scripts/mark_event.py +128 -0
  28. package/scripts/memory_snapshot.py +62 -0
  29. package/scripts/pick_twitter_thread_target.py +1 -1
  30. package/scripts/release-mcpb.sh +48 -0
  31. package/scripts/reset-test-machine.sh +85 -3
  32. package/scripts/s4l_mode.py +7 -5
  33. package/scripts/scheduled_tasks_snapshot.py +107 -0
  34. package/scripts/send_gmail_report.py +76 -0
  35. package/scripts/sentry_digest.py +81 -189
  36. package/scripts/sentry_init.py +43 -15
  37. package/scripts/setup_twitter_auth.py +76 -38
  38. package/scripts/snapshot.py +20 -0
  39. package/scripts/twitter_post_plan.py +39 -13
  40. package/skill/dm-outreach-twitter.sh +19 -0
  41. package/skill/engage-twitter.sh +27 -0
  42. package/skill/run-draft-and-publish.sh +13 -12
  43. package/skill/run-linkedin.sh +9 -1
  44. package/skill/run-twitter-cycle.sh +199 -27
  45. package/skill/sentry-digest.sh +125 -12
@@ -82,6 +82,85 @@ def state_dir() -> str:
82
82
  )
83
83
 
84
84
 
85
+ # ---- pause/resume: stop drafting/posting without touching Claude Desktop ---
86
+ # Real mechanics (flag file + launchctl bootout/bootstrap), NOT a loopback call
87
+ # to the MCP: this needs to work from BOTH the menu bar (s4l_menubar.py) and
88
+ # the Claude-independent dashboard_server.py, which share this process but
89
+ # are a different process from the MCP server, so there's no way to delegate
90
+ # to mcp/src/index.ts's pauseS4L/resumeS4L here. Mirrors those functions
91
+ # (same 4 labels, same flag path) for the common pause->resume case. What it
92
+ # deliberately does NOT do: mcp/src/index.ts's resumeS4L also re-runs the
93
+ # readiness-gated install logic (persona/project/X-verified checks, content-
94
+ # aware plist rewrite) via ensure*Installed() — that only matters for a box
95
+ # that was never fully set up before Pause, an edge case the next Claude
96
+ # boot / project_config call naturally heals once unpaused. This resume just
97
+ # reloads the exact plists Pause unloaded (Pause never deletes anything).
98
+ PAUSE_TARGET_LABELS = (
99
+ AUTOPILOT_LABEL,
100
+ "com.m13v.social-claude-reaper",
101
+ "com.m13v.social-autopilot-stall-watch",
102
+ "com.m13v.social-memory-snapshot",
103
+ )
104
+
105
+
106
+ def pause_flag_path() -> str:
107
+ return os.path.join(state_dir(), "paused.flag")
108
+
109
+
110
+ def is_paused() -> bool:
111
+ return os.path.exists(pause_flag_path())
112
+
113
+
114
+ def pause_s4l() -> dict:
115
+ if is_paused():
116
+ return {"ok": True, "paused": True, "detail": "already paused"}
117
+ try:
118
+ os.makedirs(state_dir(), exist_ok=True)
119
+ with open(pause_flag_path(), "w") as fh:
120
+ fh.write(f"paused at {time.strftime('%Y-%m-%dT%H:%M:%S%z')}\n")
121
+ except Exception as e:
122
+ return {"ok": False, "paused": is_paused(), "detail": f"could not write pause flag: {e}"}
123
+ uid = os.getuid()
124
+ results = []
125
+ for label in PAUSE_TARGET_LABELS:
126
+ try:
127
+ subprocess.run(
128
+ ["launchctl", "bootout", f"gui/{uid}/{label}"],
129
+ capture_output=True, timeout=15,
130
+ )
131
+ results.append(f"{label}: unloaded")
132
+ except Exception as e:
133
+ results.append(f"{label}: {e}")
134
+ return {"ok": True, "paused": True, "detail": "; ".join(results)}
135
+
136
+
137
+ def resume_s4l() -> dict:
138
+ try:
139
+ os.remove(pause_flag_path())
140
+ except FileNotFoundError:
141
+ pass
142
+ except Exception:
143
+ pass
144
+ uid = os.getuid()
145
+ la_dir = str(Path.home() / "Library" / "LaunchAgents")
146
+ results = []
147
+ for label in PAUSE_TARGET_LABELS:
148
+ plist = os.path.join(la_dir, f"{label}.plist")
149
+ if not os.path.exists(plist):
150
+ results.append(f"{label}: no plist, skip")
151
+ continue
152
+ try:
153
+ subprocess.run(["launchctl", "enable", f"gui/{uid}/{label}"], capture_output=True, timeout=15)
154
+ r = subprocess.run(
155
+ ["launchctl", "bootstrap", f"gui/{uid}", plist],
156
+ capture_output=True, timeout=15,
157
+ )
158
+ results.append(f"{label}: rc={r.returncode}")
159
+ except Exception as e:
160
+ results.append(f"{label}: {e}")
161
+ return {"ok": True, "paused": is_paused(), "detail": "; ".join(results)}
162
+
163
+
85
164
  def read_json(name: str):
86
165
  try:
87
166
  return json.loads((Path(state_dir()) / name).read_text())
@@ -162,7 +241,9 @@ def _endpoint_url():
162
241
  with urllib.request.urlopen(url + "health", timeout=1.5) as r:
163
242
  if r.status == 200:
164
243
  return url
165
- except Exception:
244
+ except Exception as e:
245
+ sys.stderr.write(f"[s4l-state] loopback health check failed: {type(e).__name__}: {e}\n")
246
+ sys.stderr.flush()
166
247
  return None
167
248
  return None
168
249
 
@@ -208,7 +289,9 @@ def loopback_tool(name: str, args=None, timeout: float = 20.0):
208
289
  )
209
290
  with urllib.request.urlopen(req, timeout=timeout) as r:
210
291
  return _parse_tool_result(json.loads(r.read().decode()))
211
- except Exception:
292
+ except Exception as e:
293
+ sys.stderr.write(f"[s4l-state] loopback_tool({name!r}) failed: {type(e).__name__}: {e}\n")
294
+ sys.stderr.flush()
212
295
  return None
213
296
 
214
297
 
@@ -246,6 +329,12 @@ def ver_key(v):
246
329
  return _snapshot_module()._ver_key(v)
247
330
 
248
331
 
332
+ def version():
333
+ """Installed version, delegated to scripts/snapshot.py::_resolve_version so
334
+ there is exactly ONE Python implementation, same pattern as ver_key above."""
335
+ return _snapshot_module()._resolve_version()
336
+
337
+
249
338
  def _refresh_snapshot_bg():
250
339
  try:
251
340
  snap = _compute_snapshot_full()
@@ -259,6 +348,43 @@ def _refresh_snapshot_bg():
259
348
  _snap_refreshing[0] = False
260
349
 
261
350
 
351
+ # ---- version/update check: refreshed independently of the full snapshot ----
352
+ # compute() calls _x_status() FIRST, which can block up to 90s in a subprocess
353
+ # (setup_twitter_auth.py -> CDP to Chrome) whenever a scan job has the browser
354
+ # busy. That stalls _refresh_snapshot_bg() for the whole duration, so
355
+ # update_available/latest_version stay frozen on a busy box even though the
356
+ # GitHub check itself is a ~1s curl. Poll scripts/snapshot.py::version_status()
357
+ # on its own short cadence, gated by its own lock, so the "Update available"
358
+ # banner never waits on X/Chrome state.
359
+ _ver_cache = {"val": None, "at": 0.0}
360
+ _ver_refreshing = [False]
361
+ _VER_REFRESH_TTL = 20.0
362
+
363
+
364
+ def _refresh_version_bg():
365
+ try:
366
+ val = _snapshot_module().version_status()
367
+ if isinstance(val, dict) and "update_available" in val:
368
+ with _snap_lock:
369
+ _ver_cache["val"] = val
370
+ _ver_cache["at"] = time.time()
371
+ except Exception:
372
+ pass
373
+ finally:
374
+ _ver_refreshing[0] = False
375
+
376
+
377
+ def _version_overlay():
378
+ now = time.time()
379
+ with _snap_lock:
380
+ cached = _ver_cache["val"]
381
+ age = now - _ver_cache["at"]
382
+ if (cached is None or age > _VER_REFRESH_TTL) and not _ver_refreshing[0]:
383
+ _ver_refreshing[0] = True
384
+ threading.Thread(target=_refresh_version_bg, daemon=True).start()
385
+ return cached
386
+
387
+
262
388
  def snapshot():
263
389
  """Full snapshot computed DIRECTLY from the stateful files via
264
390
  scripts/snapshot.py — the SAME single-source module the MCP shells out to, so
@@ -268,7 +394,9 @@ def snapshot():
268
394
  a BACKGROUND thread; this returns the last cached result instantly.
269
395
 
270
396
  Tiers: (1) the background-computed local snapshot; (2) the server's last
271
- persisted `status-summary.json`; (3) the onboarding ledger."""
397
+ persisted `status-summary.json`; (3) the onboarding ledger. The version/update
398
+ fields are always overlaid from the independent _version_overlay() cache
399
+ (see above) so they stay live even while tier (1)'s compute() is blocked."""
272
400
  now = time.time()
273
401
  with _snap_lock:
274
402
  cached = _snap_cache["val"]
@@ -276,29 +404,40 @@ def snapshot():
276
404
  if (cached is None or age > 4.0) and not _snap_refreshing[0]:
277
405
  _snap_refreshing[0] = True
278
406
  threading.Thread(target=_refresh_snapshot_bg, daemon=True).start()
407
+
279
408
  if cached is not None:
280
409
  out = dict(cached)
281
410
  out["_live"] = True
282
- return out
283
- summ = read_json("status-summary.json")
284
- if isinstance(summ, dict) and "projects_total" in summ:
285
- summ["_live"] = False
286
- summ["_from_summary"] = True
287
- return summ
288
- ob = read_onboarding()
289
- return {
290
- "_live": False,
291
- "runtime_ready": runtime_ready(),
292
- "onboarding": ob,
293
- "autopilot_on": autopilot_loaded(),
294
- "x_connected": False, # unknowable offline; State derives from onboarding
295
- "x_handle": None,
296
- "projects_ready": 0,
297
- "projects_total": 0,
298
- "version": version(),
299
- "update_available": False,
300
- "latest_version": None,
301
- }
411
+ else:
412
+ summ = read_json("status-summary.json")
413
+ if isinstance(summ, dict) and "projects_total" in summ:
414
+ out = dict(summ)
415
+ out["_live"] = False
416
+ out["_from_summary"] = True
417
+ else:
418
+ ob = read_onboarding()
419
+ out = {
420
+ "_live": False,
421
+ "runtime_ready": runtime_ready(),
422
+ "onboarding": ob,
423
+ "autopilot_on": autopilot_loaded(),
424
+ "x_connected": False, # unknowable offline; State derives from onboarding
425
+ "x_handle": None,
426
+ "projects_ready": 0,
427
+ "projects_total": 0,
428
+ "version": version(),
429
+ "update_available": False,
430
+ "latest_version": None,
431
+ }
432
+
433
+ ver = _version_overlay()
434
+ if ver is not None:
435
+ out["version"] = ver.get("version", out.get("version"))
436
+ out["channel"] = ver.get("channel", out.get("channel"))
437
+ out["latest_version"] = ver.get("latest_version", out.get("latest_version"))
438
+ out["latest_tag"] = ver.get("latest_tag", out.get("latest_tag"))
439
+ out["update_available"] = ver.get("update_available", out.get("update_available"))
440
+ return out
302
441
 
303
442
 
304
443
  def stats_7d():
@@ -836,6 +975,13 @@ def review_drafts(plan, batch="review-queue"):
836
975
  # scripts/active_experiments.py. Generic: the card renders
837
976
  # whatever is here, so new experiments surface with no UI work.
838
977
  "experiments": c.get("experiments") or {},
978
+ # Two-draft cards (2026-07-07; no-recommendation pass
979
+ # 2026-07-08): present only when the prep step wrote two
980
+ # independent drafts (fresh candidate, not a reused stale
981
+ # draft). The card shows both, defaulting to Draft A (slot 0,
982
+ # no model recommendation involved); absent on reused/legacy
983
+ # candidates, which fall back to the single-draft UI above.
984
+ "drafts": c.get("drafts"),
839
985
  }
840
986
  )
841
987
  # The review queue is append-only, so the highest stable index is newest and
package/mcp/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@m13v/s4l-mcp",
3
- "version": "1.7.1-rc.2",
3
+ "version": "1.7.1-rc.21",
4
4
  "private": true,
5
5
  "description": "Desktop MCP client for social-autoposter (X/Twitter rail): manual draft/review/approve loop, autopilot control, and stats. Thin wrapper over the existing pipeline scripts.",
6
6
  "license": "MIT",
@@ -149,6 +149,14 @@ function runDoctorSync(options = {}) {
149
149
  "deferred until X connection to avoid an unexpected keychain prompt"
150
150
  );
151
151
  }
152
+ // Once the durable cookie mirror is populated, the X session no longer
153
+ // depends on this keychain item (see twitter_cookie_mirror.py — it exists
154
+ // precisely so a locked/missing Safe Storage key isn't fatal). Skip the
155
+ // probe instead of re-triggering + re-failing it on every doctor run;
156
+ // same mirrorCount() gate keychain_autolock already uses below.
157
+ if (mirrorCount() > 0) {
158
+ return result("pass", "skipped — durable cookie mirror already covers this");
159
+ }
152
160
  const r = spawnSync(
153
161
  "security",
154
162
  [
@@ -10,6 +10,7 @@ const MILESTONES = [
10
10
  "environment_checked",
11
11
  "runtime_ready",
12
12
  "x_connected",
13
+ "x_verified",
13
14
  "profile_scanned",
14
15
  "mode_chosen",
15
16
  "project_ready",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@m13v/s4l",
3
- "version": "1.7.1-rc.2",
3
+ "version": "1.7.1-rc.21",
4
4
  "description": "Automated social posting pipeline for Reddit, X/Twitter, LinkedIn, and Moltbook. Install as a Claude Code agent skill.",
5
5
  "bin": {
6
6
  "social-autoposter": "bin/cli.js",
@@ -18,6 +18,7 @@
18
18
  "!bin/auth.js",
19
19
  "scripts/*.py",
20
20
  "!scripts/db_direct.py",
21
+ "!scripts/sync_blocked_authors.py",
21
22
  "!scripts/backfill_real_clicks.py",
22
23
  "!scripts/historical_engagement.py",
23
24
  "!scripts/style_length_report.py",
@@ -117,6 +118,7 @@
117
118
  "requirements.txt",
118
119
  "SKILL.md",
119
120
  "skill/*.sh",
121
+ "!skill/sync-blocked-authors.sh",
120
122
  "skill/lib/*.sh",
121
123
  "setup/SKILL.md",
122
124
  "browser-agent-configs/",
@@ -32,13 +32,10 @@ from urllib.parse import unquote
32
32
 
33
33
  sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
34
34
  from db import load_env, get_conn # noqa: E402
35
-
36
35
  # Our own installs, hidden by default so the roster is real external users.
37
- INTERNAL_EMAILS = {"i@m13v.com", "agent@mk0r.com", "matt@mediar.ai"}
38
- INTERNAL_HOSTNAME_SUBSTR = ("e2b.local", "71522") # mk0r E2B sandboxes; MacStadium QA box
39
- # MacStadium remote QA box (hostname "71522", no git_email). It actively runs the
40
- # pipeline and posts, so without this it masquerades as our only posting customer.
41
- INTERNAL_HARDWARE_UUIDS = {"07CB793D-6E32-5EF8-82E2-7CDEABD47FBC"}
36
+ # Shared with autopilot_stall_watch.py via identity.py so the two lists can't
37
+ # drift apart (see identity.is_internal_install).
38
+ from identity import INTERNAL_EMAILS, INTERNAL_HOSTNAME_SUBSTR, INTERNAL_HARDWARE_UUIDS # noqa: E402
42
39
 
43
40
  # Connected X handle resolves only from posts.our_account; drop scaffolding values.
44
41
  PLACEHOLDER_HANDLES = {"your-twitter-handle", "your_handle", "your-handle", "none", "null", ""}
@@ -42,6 +42,7 @@ import time
42
42
  # SAPS_->S4L_ env mirror (brand rename 2026-07-03): old launchd plists and
43
43
  # scheduled-task prompts still export SAPS_*; this process reads S4L_*.
44
44
  import s4l_env # noqa: E402 (lives next to this file in scripts/)
45
+ import identity # noqa: E402 (lives next to this file in scripts/)
45
46
 
46
47
  s4l_env.mirror()
47
48
 
@@ -141,6 +142,45 @@ def _recent_rate_limit(window: int = 1200) -> bool:
141
142
  return False
142
143
 
143
144
 
145
+ BATCH_PROGRESSION_MIN_BATCHES = 5
146
+ BATCH_PROGRESSED_PHASES = {"phase2b-gen", "phase2b-post"}
147
+
148
+
149
+ def _recent_batches_not_progressing(min_batches: int = BATCH_PROGRESSION_MIN_BATCHES) -> bool:
150
+ """True if the last `min_batches` twitter_batches for THIS install ALL failed
151
+ to reach phase2b-gen (real draft generation) or later. A second, independent
152
+ detection layer: the queue's consecutive_timeouts counter is one level
153
+ removed from the actual product outcome (did a batch progress). During the
154
+ Karol 2026-07-07 investigation the DB showed the stall the whole time — 46
155
+ consecutive batches dying at phase2b-prep — while this watchdog's other
156
+ signals took hours longer to latch. Requires network (via http_api); any
157
+ failure (offline, endpoint missing, not enough batch history yet) returns
158
+ False so this signal can only ever ADD confidence, never false-positive on
159
+ its own from a transient API hiccup.
160
+
161
+ NOTE: catches BaseException, not just Exception — scripts/http_api.py's
162
+ _request() deliberately raises SystemExit (not a plain Exception; it
163
+ inherits from BaseException) on a terminal 4xx/5xx, which is correct for
164
+ most one-shot pipeline callers but would silently kill this entire
165
+ best-effort watchdog process if left uncaught here. Caught by testing this
166
+ against a not-yet-deployed endpoint version during development."""
167
+ try:
168
+ repo = os.environ.get("S4L_REPO_DIR")
169
+ if repo:
170
+ scripts_dir = os.path.join(repo, "scripts")
171
+ if scripts_dir not in sys.path:
172
+ sys.path.insert(0, scripts_dir)
173
+ import http_api # noqa: E402
174
+
175
+ resp = http_api.api_get("/api/v1/twitter-batches", {"list": "1", "limit": str(min_batches)})
176
+ batches = (resp or {}).get("batches") or []
177
+ if len(batches) < min_batches:
178
+ return False # not enough history yet to conclude a stall
179
+ return all((b or {}).get("current_phase") not in BATCH_PROGRESSED_PHASES for b in batches)
180
+ except BaseException:
181
+ return False
182
+
183
+
144
184
  def _oldest_pending_age() -> float | None:
145
185
  """Seconds since the oldest unclaimed pending draft job was written, or None
146
186
  if nothing is pending (idle queue). The FAST signal: catches a fresh stall
@@ -162,6 +202,25 @@ def _oldest_pending_age() -> float | None:
162
202
  return time.time() - oldest
163
203
 
164
204
 
205
+ def _pending_count() -> int:
206
+ """Number of unclaimed pending draft jobs across all types. Cheap: same glob
207
+ shape as _oldest_pending_age, just counted instead of min'd. Feeds the
208
+ queue_health_samples row (see _report_queue_health_sample)."""
209
+ pend_root = os.path.join(_queue_root(), "pending")
210
+ n = 0
211
+ for sub in glob.glob(os.path.join(pend_root, "*")):
212
+ for jf in glob.glob(os.path.join(sub, "*.json")):
213
+ if not jf.endswith(".tmp"):
214
+ n += 1
215
+ return n
216
+
217
+
218
+ def _running_count() -> int:
219
+ """Number of claimed-but-unfinished jobs. See _pending_count."""
220
+ run_root = os.path.join(_queue_root(), "running")
221
+ return sum(1 for jf in glob.glob(os.path.join(run_root, "*.json")) if not jf.endswith(".tmp"))
222
+
223
+
165
224
  def _oldest_running_age() -> float | None:
166
225
  """Seconds since the oldest CLAIMED-but-unfinished job was written, or None if
167
226
  nothing is in flight. A worker claims by moving a job pending/ -> running/ and
@@ -217,18 +276,97 @@ def _sentry():
217
276
  return sentry_init
218
277
 
219
278
 
279
+ def _report_recovery(total_duration_s: float, paged_duration_s: float | None) -> None:
280
+ """Log a recovery milestone (with duration) once per stall episode that
281
+ actually paged. Previously this had to be hand-reconstructed per incident by
282
+ diffing DB/log timestamps after the fact (see the Karol 2026-07-07
283
+ update-orphan investigation); this makes it a first-class, queryable Sentry
284
+ event so incident severity is trackable fleet-wide over time instead of
285
+ per-investigation. Only fires for episodes that crossed ALERT_AFTER — a
286
+ stall that self-clears before paging is normal jitter, not an incident."""
287
+ try:
288
+ sentry = _sentry()
289
+ sentry.init()
290
+ total_min = total_duration_s / 60.0
291
+ paged_str = f"{paged_duration_s / 60.0:.1f} min" if paged_duration_s is not None else "n/a"
292
+ sentry.capture_message(
293
+ "social-autoposter autopilot recovered: draft jobs draining again "
294
+ f"after {total_min:.1f} min stalled ({paged_str} since it paged).",
295
+ level="info",
296
+ tags={"component": "autopilot", "issue": "stall_recovered"},
297
+ extra={
298
+ "stall_total_duration_seconds": round(total_duration_s, 1),
299
+ "stall_paged_duration_seconds": (
300
+ round(paged_duration_s, 1) if paged_duration_s is not None else None
301
+ ),
302
+ },
303
+ )
304
+ sentry.flush()
305
+ except Exception:
306
+ sys.stderr.write(f"[stall-watch] recovered after {total_duration_s:.0f}s but Sentry report failed\n")
307
+
308
+
309
+ def _report_queue_health_sample(
310
+ pending: int,
311
+ running: int,
312
+ timeouts: int,
313
+ age: float | None,
314
+ run_age: float | None,
315
+ stalled: bool,
316
+ batches_stuck: bool,
317
+ ) -> None:
318
+ """Best-effort POST of one queue_health_samples row, on EVERY tick (not
319
+ just while stalled) — a continuous baseline is the point: it's what makes
320
+ "was this box actually healthy at time T" a one-line SQL query instead of
321
+ absence-of-evidence. See migrations/2026-07-07-queue-health-samples.sql.
322
+
323
+ Catches BaseException, not just Exception — see _recent_batches_not_progressing
324
+ for why (http_api._request raises SystemExit on a terminal 4xx/5xx, which
325
+ must never be allowed to kill this watchdog process)."""
326
+ try:
327
+ repo = os.environ.get("S4L_REPO_DIR")
328
+ if repo:
329
+ scripts_dir = os.path.join(repo, "scripts")
330
+ if scripts_dir not in sys.path:
331
+ sys.path.insert(0, scripts_dir)
332
+ import http_api # noqa: E402
333
+
334
+ http_api.api_post(
335
+ "/api/v1/queue-health-samples",
336
+ {
337
+ "pending": pending,
338
+ "running": running,
339
+ "consecutive_timeouts": timeouts,
340
+ "oldest_pending_age_s": int(age) if age is not None else None,
341
+ "oldest_running_age_s": int(run_age) if run_age is not None else None,
342
+ "stalled": stalled,
343
+ "batches_stuck": batches_stuck,
344
+ },
345
+ )
346
+ except BaseException:
347
+ pass
348
+
349
+
220
350
  def main() -> int:
221
351
  age = _oldest_pending_age()
222
352
  run_age = _oldest_running_age()
223
353
  timeouts = _consecutive_timeouts()
224
- # Three complementary signals, OR'd; all gated on the autopilot actually being
354
+ configured = _autopilot_configured()
355
+ # batches_stuck is deliberately computed unconditionally (not short-circuited
356
+ # by the other signals) — it's a genuinely independent, outcome-level
357
+ # backstop, not a cheaper proxy for the same thing. It's what would have
358
+ # caught the Karol 2026-07-07 stall immediately instead of hours later.
359
+ batches_stuck = configured and _recent_batches_not_progressing()
360
+ # Four complementary signals, OR'd; all gated on the autopilot actually being
225
361
  # configured here. (1) durable producer drain latch, (2) fast pending-age (job
226
- # never claimed), (3) running-age (job claimed then wedged mid-run). (3) is the
227
- # only one that catches a worker dying after it picked up the job.
228
- stalled = _autopilot_configured() and (
362
+ # never claimed), (3) running-age (job claimed then wedged mid-run) — (3) is
363
+ # the only one of the first three that catches a worker dying after it picked
364
+ # up the job — (4) batches_stuck, the outcome-level backstop above.
365
+ stalled = configured and (
229
366
  timeouts >= 1
230
367
  or (age is not None and age > STALL_SECONDS)
231
368
  or (run_age is not None and run_age > RUNNING_STALL_SECONDS)
369
+ or batches_stuck
232
370
  )
233
371
  # A rate-limit stall is expected and self-heals at the quota reset — never page
234
372
  # for it (and re-arm can't fix it). Treat it as "not an actionable stall" so the
@@ -236,45 +374,84 @@ def main() -> int:
236
374
  if stalled and _recent_rate_limit():
237
375
  stalled = False
238
376
 
377
+ # Record this tick regardless of outcome — see _report_queue_health_sample.
378
+ _report_queue_health_sample(
379
+ _pending_count(), _running_count(), timeouts, age, run_age, stalled, batches_stuck
380
+ )
381
+
239
382
  st = _read_state()
240
383
  consecutive = int(st.get("consecutive", 0))
241
384
  alerted = bool(st.get("alerted", False))
385
+ # first_seen_at: first check this episode looked stalled at all (predates
386
+ # paging by ALERT_AFTER checks). alerted_at: the single moment we paged.
387
+ # Both are stamped once and carried forward untouched for the rest of the
388
+ # episode — NOT refreshed every check — so they measure the episode's
389
+ # start, not "last time we saw it stalled".
390
+ first_seen_at = st.get("first_seen_at")
391
+ alerted_at = st.get("alerted_at")
242
392
 
243
393
  if not stalled:
244
394
  # Recovered (or never stalled) -> reset the episode so the next stall pages.
245
395
  if consecutive or alerted:
396
+ if alerted and first_seen_at:
397
+ total_duration = time.time() - float(first_seen_at)
398
+ paged_duration = (time.time() - float(alerted_at)) if alerted_at else None
399
+ _report_recovery(total_duration, paged_duration)
246
400
  _write_state({"consecutive": 0, "alerted": False})
247
401
  return 0
248
402
 
249
403
  consecutive += 1
404
+ if first_seen_at is None:
405
+ first_seen_at = time.time()
250
406
  age_str = f"{int(age)}s" if age is not None else "n/a (between cycles)"
251
407
  run_age_str = f"{int(run_age)}s" if run_age is not None else "n/a (none in flight)"
252
- # Distinguish the two shapes so the alert points at the right cause: a claimed-
253
- # but-wedged job (running-age) is a mid-run worker death, not an orphaned routine.
408
+ # Distinguish the shapes so the alert points at the right cause: a claimed-
409
+ # but-wedged job (running-age) is a mid-run worker death, batches_stuck is
410
+ # the outcome-level backstop firing independently of the queue signals, and
411
+ # the fallback is the classic orphaned-routine case.
254
412
  wedged_inflight = run_age is not None and run_age > RUNNING_STALL_SECONDS
255
413
  if consecutive >= ALERT_AFTER and not alerted:
256
414
  try:
257
415
  sentry = _sentry()
258
416
  sentry.init()
259
- cause = (
260
- "a worker claimed a draft job and then died mid-run (claude -p child "
261
- "never came up / crashed)"
262
- if wedged_inflight
263
- else "scheduled-task routines likely orphaned — Claude Desktop account change?"
264
- )
417
+ if wedged_inflight:
418
+ cause = (
419
+ "a worker claimed a draft job and then died mid-run (claude -p child "
420
+ "never came up / crashed)"
421
+ )
422
+ stall_shape = "inflight_wedged"
423
+ elif batches_stuck:
424
+ cause = (
425
+ f"last {BATCH_PROGRESSION_MIN_BATCHES} twitter_batches all failed to "
426
+ "reach phase2b-gen — drafting is not actually happening even if the "
427
+ "queue-level counters look ambiguous"
428
+ )
429
+ stall_shape = "batches_not_progressing"
430
+ else:
431
+ cause = "scheduled-task routines likely orphaned — Claude Desktop account change?"
432
+ stall_shape = "not_draining"
433
+ # Our own staging/QA/dev boxes (identity.is_internal_install) get set
434
+ # up and rebuilt with nobody actively feeding the queue, which looks
435
+ # identical to a real stall on the signals above. Downgrade those to
436
+ # warning instead of error so they don't page as a customer incident
437
+ # (the digest only scans error/fatal) while still leaving a Sentry
438
+ # record if we ever need to look one up by hand.
439
+ is_internal = identity.is_internal_install()
265
440
  sentry.capture_message(
266
441
  "social-autoposter autopilot stalled: draft jobs are not being "
267
442
  f"drained ({cause}). producer consecutive timeouts={timeouts}, "
268
443
  f"oldest pending job age={age_str}, oldest in-flight (running) job "
269
444
  f"age={run_age_str}, sustained {consecutive} checks.",
270
- level="error",
445
+ level=("warning" if is_internal else "error"),
271
446
  tags={
272
447
  "component": "autopilot",
273
448
  "issue": "stall",
274
- "stall_shape": "inflight_wedged" if wedged_inflight else "not_draining",
449
+ "stall_shape": stall_shape,
275
450
  "consecutive_timeouts": str(timeouts),
276
451
  "oldest_pending_age_s": str(int(age)) if age is not None else "",
277
452
  "oldest_running_age_s": str(int(run_age)) if run_age is not None else "",
453
+ "batches_stuck": str(batches_stuck),
454
+ "internal_install": str(is_internal),
278
455
  },
279
456
  )
280
457
  sentry.flush()
@@ -285,8 +462,15 @@ def main() -> int:
285
462
  f"pending_age={age_str}, running_age={run_age_str}) but Sentry report failed\n"
286
463
  )
287
464
  alerted = True
288
-
289
- _write_state({"consecutive": consecutive, "alerted": alerted, "at": time.time()})
465
+ alerted_at = time.time()
466
+
467
+ _write_state({
468
+ "consecutive": consecutive,
469
+ "alerted": alerted,
470
+ "first_seen_at": first_seen_at,
471
+ "alerted_at": alerted_at,
472
+ "at": time.time(),
473
+ })
290
474
  return 0
291
475
 
292
476
 
@@ -920,6 +920,12 @@ def _attempt_claim(ns, qtype: str) -> bool:
920
920
 
921
921
  def cmd_next(ns) -> int:
922
922
  _apply_state_dir_override(ns)
923
+ # S4L paused (mcp/src/index.ts pauseAutopilot): report no work instead of
924
+ # claiming a job, so an already-queued draft can't drain/post while paused
925
+ # even though the scheduled task worker still fires on its own cadence.
926
+ if os.path.exists(os.path.join(state_dir(), "paused.flag")):
927
+ print(json.dumps({}))
928
+ return 0
923
929
  qtype = ns.type
924
930
  wait_seconds = max(0, ns.wait_seconds or 0)
925
931
  # wait_seconds=0 (default) is the legacy single-shot behavior: one pass,