@m13v/s4l 1.7.1-rc.2 → 1.7.1-rc.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +23 -0
- package/mcp/dist/index.js +332 -24
- package/mcp/dist/onboarding.js +3 -10
- package/mcp/dist/panel.html +42 -31
- package/mcp/dist/product-link.html +1 -1
- package/mcp/dist/runtime.js +101 -1
- package/mcp/dist/telemetry.js +80 -3
- package/mcp/dist/version.json +2 -2
- package/mcp/manifest.json +9 -1
- package/mcp/menubar/dashboard_server.py +12 -0
- package/mcp/menubar/s4l_card.py +472 -96
- package/mcp/menubar/s4l_menubar.py +269 -26
- package/mcp/menubar/s4l_state.py +169 -23
- package/mcp/package.json +1 -1
- package/mcp/shared/doctor.cjs +8 -0
- package/mcp/shared/onboarding-ledger.cjs +1 -0
- package/package.json +3 -1
- package/scripts/active_users.py +3 -6
- package/scripts/autopilot_stall_watch.py +200 -16
- package/scripts/claude_job.py +6 -0
- package/scripts/feedback_digest.py +34 -6
- package/scripts/get-latest-staging-mcpb.sh +50 -0
- package/scripts/harness_overlay.py +45 -0
- package/scripts/identity.py +26 -0
- package/scripts/learned_preferences.py +241 -32
- package/scripts/link_tail.py +167 -91
- package/scripts/mark_event.py +128 -0
- package/scripts/memory_snapshot.py +62 -0
- package/scripts/pick_twitter_thread_target.py +1 -1
- package/scripts/release-mcpb.sh +48 -0
- package/scripts/reset-test-machine.sh +85 -3
- package/scripts/s4l_mode.py +7 -5
- package/scripts/scheduled_tasks_snapshot.py +107 -0
- package/scripts/send_gmail_report.py +76 -0
- package/scripts/sentry_digest.py +81 -189
- package/scripts/sentry_init.py +43 -15
- package/scripts/setup_twitter_auth.py +76 -38
- package/scripts/snapshot.py +20 -0
- package/scripts/twitter_post_plan.py +39 -13
- package/skill/dm-outreach-twitter.sh +19 -0
- package/skill/engage-twitter.sh +27 -0
- package/skill/run-draft-and-publish.sh +13 -12
- package/skill/run-linkedin.sh +9 -1
- package/skill/run-twitter-cycle.sh +199 -27
- package/skill/sentry-digest.sh +125 -12
package/mcp/menubar/s4l_state.py
CHANGED
|
@@ -82,6 +82,85 @@ def state_dir() -> str:
|
|
|
82
82
|
)
|
|
83
83
|
|
|
84
84
|
|
|
85
|
+
# ---- pause/resume: stop drafting/posting without touching Claude Desktop ---
|
|
86
|
+
# Real mechanics (flag file + launchctl bootout/bootstrap), NOT a loopback call
|
|
87
|
+
# to the MCP: this needs to work from BOTH the menu bar (s4l_menubar.py) and
|
|
88
|
+
# the Claude-independent dashboard_server.py, which share this process but
|
|
89
|
+
# are a different process from the MCP server, so there's no way to delegate
|
|
90
|
+
# to mcp/src/index.ts's pauseS4L/resumeS4L here. Mirrors those functions
|
|
91
|
+
# (same 4 labels, same flag path) for the common pause->resume case. What it
|
|
92
|
+
# deliberately does NOT do: mcp/src/index.ts's resumeS4L also re-runs the
|
|
93
|
+
# readiness-gated install logic (persona/project/X-verified checks, content-
|
|
94
|
+
# aware plist rewrite) via ensure*Installed() — that only matters for a box
|
|
95
|
+
# that was never fully set up before Pause, an edge case the next Claude
|
|
96
|
+
# boot / project_config call naturally heals once unpaused. This resume just
|
|
97
|
+
# reloads the exact plists Pause unloaded (Pause never deletes anything).
|
|
98
|
+
PAUSE_TARGET_LABELS = (
|
|
99
|
+
AUTOPILOT_LABEL,
|
|
100
|
+
"com.m13v.social-claude-reaper",
|
|
101
|
+
"com.m13v.social-autopilot-stall-watch",
|
|
102
|
+
"com.m13v.social-memory-snapshot",
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def pause_flag_path() -> str:
|
|
107
|
+
return os.path.join(state_dir(), "paused.flag")
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def is_paused() -> bool:
|
|
111
|
+
return os.path.exists(pause_flag_path())
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def pause_s4l() -> dict:
|
|
115
|
+
if is_paused():
|
|
116
|
+
return {"ok": True, "paused": True, "detail": "already paused"}
|
|
117
|
+
try:
|
|
118
|
+
os.makedirs(state_dir(), exist_ok=True)
|
|
119
|
+
with open(pause_flag_path(), "w") as fh:
|
|
120
|
+
fh.write(f"paused at {time.strftime('%Y-%m-%dT%H:%M:%S%z')}\n")
|
|
121
|
+
except Exception as e:
|
|
122
|
+
return {"ok": False, "paused": is_paused(), "detail": f"could not write pause flag: {e}"}
|
|
123
|
+
uid = os.getuid()
|
|
124
|
+
results = []
|
|
125
|
+
for label in PAUSE_TARGET_LABELS:
|
|
126
|
+
try:
|
|
127
|
+
subprocess.run(
|
|
128
|
+
["launchctl", "bootout", f"gui/{uid}/{label}"],
|
|
129
|
+
capture_output=True, timeout=15,
|
|
130
|
+
)
|
|
131
|
+
results.append(f"{label}: unloaded")
|
|
132
|
+
except Exception as e:
|
|
133
|
+
results.append(f"{label}: {e}")
|
|
134
|
+
return {"ok": True, "paused": True, "detail": "; ".join(results)}
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def resume_s4l() -> dict:
|
|
138
|
+
try:
|
|
139
|
+
os.remove(pause_flag_path())
|
|
140
|
+
except FileNotFoundError:
|
|
141
|
+
pass
|
|
142
|
+
except Exception:
|
|
143
|
+
pass
|
|
144
|
+
uid = os.getuid()
|
|
145
|
+
la_dir = str(Path.home() / "Library" / "LaunchAgents")
|
|
146
|
+
results = []
|
|
147
|
+
for label in PAUSE_TARGET_LABELS:
|
|
148
|
+
plist = os.path.join(la_dir, f"{label}.plist")
|
|
149
|
+
if not os.path.exists(plist):
|
|
150
|
+
results.append(f"{label}: no plist, skip")
|
|
151
|
+
continue
|
|
152
|
+
try:
|
|
153
|
+
subprocess.run(["launchctl", "enable", f"gui/{uid}/{label}"], capture_output=True, timeout=15)
|
|
154
|
+
r = subprocess.run(
|
|
155
|
+
["launchctl", "bootstrap", f"gui/{uid}", plist],
|
|
156
|
+
capture_output=True, timeout=15,
|
|
157
|
+
)
|
|
158
|
+
results.append(f"{label}: rc={r.returncode}")
|
|
159
|
+
except Exception as e:
|
|
160
|
+
results.append(f"{label}: {e}")
|
|
161
|
+
return {"ok": True, "paused": is_paused(), "detail": "; ".join(results)}
|
|
162
|
+
|
|
163
|
+
|
|
85
164
|
def read_json(name: str):
|
|
86
165
|
try:
|
|
87
166
|
return json.loads((Path(state_dir()) / name).read_text())
|
|
@@ -162,7 +241,9 @@ def _endpoint_url():
|
|
|
162
241
|
with urllib.request.urlopen(url + "health", timeout=1.5) as r:
|
|
163
242
|
if r.status == 200:
|
|
164
243
|
return url
|
|
165
|
-
except Exception:
|
|
244
|
+
except Exception as e:
|
|
245
|
+
sys.stderr.write(f"[s4l-state] loopback health check failed: {type(e).__name__}: {e}\n")
|
|
246
|
+
sys.stderr.flush()
|
|
166
247
|
return None
|
|
167
248
|
return None
|
|
168
249
|
|
|
@@ -208,7 +289,9 @@ def loopback_tool(name: str, args=None, timeout: float = 20.0):
|
|
|
208
289
|
)
|
|
209
290
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
210
291
|
return _parse_tool_result(json.loads(r.read().decode()))
|
|
211
|
-
except Exception:
|
|
292
|
+
except Exception as e:
|
|
293
|
+
sys.stderr.write(f"[s4l-state] loopback_tool({name!r}) failed: {type(e).__name__}: {e}\n")
|
|
294
|
+
sys.stderr.flush()
|
|
212
295
|
return None
|
|
213
296
|
|
|
214
297
|
|
|
@@ -246,6 +329,12 @@ def ver_key(v):
|
|
|
246
329
|
return _snapshot_module()._ver_key(v)
|
|
247
330
|
|
|
248
331
|
|
|
332
|
+
def version():
|
|
333
|
+
"""Installed version, delegated to scripts/snapshot.py::_resolve_version so
|
|
334
|
+
there is exactly ONE Python implementation, same pattern as ver_key above."""
|
|
335
|
+
return _snapshot_module()._resolve_version()
|
|
336
|
+
|
|
337
|
+
|
|
249
338
|
def _refresh_snapshot_bg():
|
|
250
339
|
try:
|
|
251
340
|
snap = _compute_snapshot_full()
|
|
@@ -259,6 +348,43 @@ def _refresh_snapshot_bg():
|
|
|
259
348
|
_snap_refreshing[0] = False
|
|
260
349
|
|
|
261
350
|
|
|
351
|
+
# ---- version/update check: refreshed independently of the full snapshot ----
|
|
352
|
+
# compute() calls _x_status() FIRST, which can block up to 90s in a subprocess
|
|
353
|
+
# (setup_twitter_auth.py -> CDP to Chrome) whenever a scan job has the browser
|
|
354
|
+
# busy. That stalls _refresh_snapshot_bg() for the whole duration, so
|
|
355
|
+
# update_available/latest_version stay frozen on a busy box even though the
|
|
356
|
+
# GitHub check itself is a ~1s curl. Poll scripts/snapshot.py::version_status()
|
|
357
|
+
# on its own short cadence, gated by its own lock, so the "Update available"
|
|
358
|
+
# banner never waits on X/Chrome state.
|
|
359
|
+
_ver_cache = {"val": None, "at": 0.0}
|
|
360
|
+
_ver_refreshing = [False]
|
|
361
|
+
_VER_REFRESH_TTL = 20.0
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def _refresh_version_bg():
|
|
365
|
+
try:
|
|
366
|
+
val = _snapshot_module().version_status()
|
|
367
|
+
if isinstance(val, dict) and "update_available" in val:
|
|
368
|
+
with _snap_lock:
|
|
369
|
+
_ver_cache["val"] = val
|
|
370
|
+
_ver_cache["at"] = time.time()
|
|
371
|
+
except Exception:
|
|
372
|
+
pass
|
|
373
|
+
finally:
|
|
374
|
+
_ver_refreshing[0] = False
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def _version_overlay():
|
|
378
|
+
now = time.time()
|
|
379
|
+
with _snap_lock:
|
|
380
|
+
cached = _ver_cache["val"]
|
|
381
|
+
age = now - _ver_cache["at"]
|
|
382
|
+
if (cached is None or age > _VER_REFRESH_TTL) and not _ver_refreshing[0]:
|
|
383
|
+
_ver_refreshing[0] = True
|
|
384
|
+
threading.Thread(target=_refresh_version_bg, daemon=True).start()
|
|
385
|
+
return cached
|
|
386
|
+
|
|
387
|
+
|
|
262
388
|
def snapshot():
|
|
263
389
|
"""Full snapshot computed DIRECTLY from the stateful files via
|
|
264
390
|
scripts/snapshot.py — the SAME single-source module the MCP shells out to, so
|
|
@@ -268,7 +394,9 @@ def snapshot():
|
|
|
268
394
|
a BACKGROUND thread; this returns the last cached result instantly.
|
|
269
395
|
|
|
270
396
|
Tiers: (1) the background-computed local snapshot; (2) the server's last
|
|
271
|
-
persisted `status-summary.json`; (3) the onboarding ledger.
|
|
397
|
+
persisted `status-summary.json`; (3) the onboarding ledger. The version/update
|
|
398
|
+
fields are always overlaid from the independent _version_overlay() cache
|
|
399
|
+
(see above) so they stay live even while tier (1)'s compute() is blocked."""
|
|
272
400
|
now = time.time()
|
|
273
401
|
with _snap_lock:
|
|
274
402
|
cached = _snap_cache["val"]
|
|
@@ -276,29 +404,40 @@ def snapshot():
|
|
|
276
404
|
if (cached is None or age > 4.0) and not _snap_refreshing[0]:
|
|
277
405
|
_snap_refreshing[0] = True
|
|
278
406
|
threading.Thread(target=_refresh_snapshot_bg, daemon=True).start()
|
|
407
|
+
|
|
279
408
|
if cached is not None:
|
|
280
409
|
out = dict(cached)
|
|
281
410
|
out["_live"] = True
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
411
|
+
else:
|
|
412
|
+
summ = read_json("status-summary.json")
|
|
413
|
+
if isinstance(summ, dict) and "projects_total" in summ:
|
|
414
|
+
out = dict(summ)
|
|
415
|
+
out["_live"] = False
|
|
416
|
+
out["_from_summary"] = True
|
|
417
|
+
else:
|
|
418
|
+
ob = read_onboarding()
|
|
419
|
+
out = {
|
|
420
|
+
"_live": False,
|
|
421
|
+
"runtime_ready": runtime_ready(),
|
|
422
|
+
"onboarding": ob,
|
|
423
|
+
"autopilot_on": autopilot_loaded(),
|
|
424
|
+
"x_connected": False, # unknowable offline; State derives from onboarding
|
|
425
|
+
"x_handle": None,
|
|
426
|
+
"projects_ready": 0,
|
|
427
|
+
"projects_total": 0,
|
|
428
|
+
"version": version(),
|
|
429
|
+
"update_available": False,
|
|
430
|
+
"latest_version": None,
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
ver = _version_overlay()
|
|
434
|
+
if ver is not None:
|
|
435
|
+
out["version"] = ver.get("version", out.get("version"))
|
|
436
|
+
out["channel"] = ver.get("channel", out.get("channel"))
|
|
437
|
+
out["latest_version"] = ver.get("latest_version", out.get("latest_version"))
|
|
438
|
+
out["latest_tag"] = ver.get("latest_tag", out.get("latest_tag"))
|
|
439
|
+
out["update_available"] = ver.get("update_available", out.get("update_available"))
|
|
440
|
+
return out
|
|
302
441
|
|
|
303
442
|
|
|
304
443
|
def stats_7d():
|
|
@@ -836,6 +975,13 @@ def review_drafts(plan, batch="review-queue"):
|
|
|
836
975
|
# scripts/active_experiments.py. Generic: the card renders
|
|
837
976
|
# whatever is here, so new experiments surface with no UI work.
|
|
838
977
|
"experiments": c.get("experiments") or {},
|
|
978
|
+
# Two-draft cards (2026-07-07; no-recommendation pass
|
|
979
|
+
# 2026-07-08): present only when the prep step wrote two
|
|
980
|
+
# independent drafts (fresh candidate, not a reused stale
|
|
981
|
+
# draft). The card shows both, defaulting to Draft A (slot 0,
|
|
982
|
+
# no model recommendation involved); absent on reused/legacy
|
|
983
|
+
# candidates, which fall back to the single-draft UI above.
|
|
984
|
+
"drafts": c.get("drafts"),
|
|
839
985
|
}
|
|
840
986
|
)
|
|
841
987
|
# The review queue is append-only, so the highest stable index is newest and
|
package/mcp/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@m13v/s4l-mcp",
|
|
3
|
-
"version": "1.7.1-rc.
|
|
3
|
+
"version": "1.7.1-rc.21",
|
|
4
4
|
"private": true,
|
|
5
5
|
"description": "Desktop MCP client for social-autoposter (X/Twitter rail): manual draft/review/approve loop, autopilot control, and stats. Thin wrapper over the existing pipeline scripts.",
|
|
6
6
|
"license": "MIT",
|
package/mcp/shared/doctor.cjs
CHANGED
|
@@ -149,6 +149,14 @@ function runDoctorSync(options = {}) {
|
|
|
149
149
|
"deferred until X connection to avoid an unexpected keychain prompt"
|
|
150
150
|
);
|
|
151
151
|
}
|
|
152
|
+
// Once the durable cookie mirror is populated, the X session no longer
|
|
153
|
+
// depends on this keychain item (see twitter_cookie_mirror.py — it exists
|
|
154
|
+
// precisely so a locked/missing Safe Storage key isn't fatal). Skip the
|
|
155
|
+
// probe instead of re-triggering + re-failing it on every doctor run;
|
|
156
|
+
// same mirrorCount() gate keychain_autolock already uses below.
|
|
157
|
+
if (mirrorCount() > 0) {
|
|
158
|
+
return result("pass", "skipped — durable cookie mirror already covers this");
|
|
159
|
+
}
|
|
152
160
|
const r = spawnSync(
|
|
153
161
|
"security",
|
|
154
162
|
[
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@m13v/s4l",
|
|
3
|
-
"version": "1.7.1-rc.
|
|
3
|
+
"version": "1.7.1-rc.21",
|
|
4
4
|
"description": "Automated social posting pipeline for Reddit, X/Twitter, LinkedIn, and Moltbook. Install as a Claude Code agent skill.",
|
|
5
5
|
"bin": {
|
|
6
6
|
"social-autoposter": "bin/cli.js",
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
"!bin/auth.js",
|
|
19
19
|
"scripts/*.py",
|
|
20
20
|
"!scripts/db_direct.py",
|
|
21
|
+
"!scripts/sync_blocked_authors.py",
|
|
21
22
|
"!scripts/backfill_real_clicks.py",
|
|
22
23
|
"!scripts/historical_engagement.py",
|
|
23
24
|
"!scripts/style_length_report.py",
|
|
@@ -117,6 +118,7 @@
|
|
|
117
118
|
"requirements.txt",
|
|
118
119
|
"SKILL.md",
|
|
119
120
|
"skill/*.sh",
|
|
121
|
+
"!skill/sync-blocked-authors.sh",
|
|
120
122
|
"skill/lib/*.sh",
|
|
121
123
|
"setup/SKILL.md",
|
|
122
124
|
"browser-agent-configs/",
|
package/scripts/active_users.py
CHANGED
|
@@ -32,13 +32,10 @@ from urllib.parse import unquote
|
|
|
32
32
|
|
|
33
33
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
34
34
|
from db import load_env, get_conn # noqa: E402
|
|
35
|
-
|
|
36
35
|
# Our own installs, hidden by default so the roster is real external users.
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
# pipeline and posts, so without this it masquerades as our only posting customer.
|
|
41
|
-
INTERNAL_HARDWARE_UUIDS = {"07CB793D-6E32-5EF8-82E2-7CDEABD47FBC"}
|
|
36
|
+
# Shared with autopilot_stall_watch.py via identity.py so the two lists can't
|
|
37
|
+
# drift apart (see identity.is_internal_install).
|
|
38
|
+
from identity import INTERNAL_EMAILS, INTERNAL_HOSTNAME_SUBSTR, INTERNAL_HARDWARE_UUIDS # noqa: E402
|
|
42
39
|
|
|
43
40
|
# Connected X handle resolves only from posts.our_account; drop scaffolding values.
|
|
44
41
|
PLACEHOLDER_HANDLES = {"your-twitter-handle", "your_handle", "your-handle", "none", "null", ""}
|
|
@@ -42,6 +42,7 @@ import time
|
|
|
42
42
|
# SAPS_->S4L_ env mirror (brand rename 2026-07-03): old launchd plists and
|
|
43
43
|
# scheduled-task prompts still export SAPS_*; this process reads S4L_*.
|
|
44
44
|
import s4l_env # noqa: E402 (lives next to this file in scripts/)
|
|
45
|
+
import identity # noqa: E402 (lives next to this file in scripts/)
|
|
45
46
|
|
|
46
47
|
s4l_env.mirror()
|
|
47
48
|
|
|
@@ -141,6 +142,45 @@ def _recent_rate_limit(window: int = 1200) -> bool:
|
|
|
141
142
|
return False
|
|
142
143
|
|
|
143
144
|
|
|
145
|
+
BATCH_PROGRESSION_MIN_BATCHES = 5
|
|
146
|
+
BATCH_PROGRESSED_PHASES = {"phase2b-gen", "phase2b-post"}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _recent_batches_not_progressing(min_batches: int = BATCH_PROGRESSION_MIN_BATCHES) -> bool:
|
|
150
|
+
"""True if the last `min_batches` twitter_batches for THIS install ALL failed
|
|
151
|
+
to reach phase2b-gen (real draft generation) or later. A second, independent
|
|
152
|
+
detection layer: the queue's consecutive_timeouts counter is one level
|
|
153
|
+
removed from the actual product outcome (did a batch progress). During the
|
|
154
|
+
Karol 2026-07-07 investigation the DB showed the stall the whole time — 46
|
|
155
|
+
consecutive batches dying at phase2b-prep — while this watchdog's other
|
|
156
|
+
signals took hours longer to latch. Requires network (via http_api); any
|
|
157
|
+
failure (offline, endpoint missing, not enough batch history yet) returns
|
|
158
|
+
False so this signal can only ever ADD confidence, never false-positive on
|
|
159
|
+
its own from a transient API hiccup.
|
|
160
|
+
|
|
161
|
+
NOTE: catches BaseException, not just Exception — scripts/http_api.py's
|
|
162
|
+
_request() deliberately raises SystemExit (not a plain Exception; it
|
|
163
|
+
inherits from BaseException) on a terminal 4xx/5xx, which is correct for
|
|
164
|
+
most one-shot pipeline callers but would silently kill this entire
|
|
165
|
+
best-effort watchdog process if left uncaught here. Caught by testing this
|
|
166
|
+
against a not-yet-deployed endpoint version during development."""
|
|
167
|
+
try:
|
|
168
|
+
repo = os.environ.get("S4L_REPO_DIR")
|
|
169
|
+
if repo:
|
|
170
|
+
scripts_dir = os.path.join(repo, "scripts")
|
|
171
|
+
if scripts_dir not in sys.path:
|
|
172
|
+
sys.path.insert(0, scripts_dir)
|
|
173
|
+
import http_api # noqa: E402
|
|
174
|
+
|
|
175
|
+
resp = http_api.api_get("/api/v1/twitter-batches", {"list": "1", "limit": str(min_batches)})
|
|
176
|
+
batches = (resp or {}).get("batches") or []
|
|
177
|
+
if len(batches) < min_batches:
|
|
178
|
+
return False # not enough history yet to conclude a stall
|
|
179
|
+
return all((b or {}).get("current_phase") not in BATCH_PROGRESSED_PHASES for b in batches)
|
|
180
|
+
except BaseException:
|
|
181
|
+
return False
|
|
182
|
+
|
|
183
|
+
|
|
144
184
|
def _oldest_pending_age() -> float | None:
|
|
145
185
|
"""Seconds since the oldest unclaimed pending draft job was written, or None
|
|
146
186
|
if nothing is pending (idle queue). The FAST signal: catches a fresh stall
|
|
@@ -162,6 +202,25 @@ def _oldest_pending_age() -> float | None:
|
|
|
162
202
|
return time.time() - oldest
|
|
163
203
|
|
|
164
204
|
|
|
205
|
+
def _pending_count() -> int:
|
|
206
|
+
"""Number of unclaimed pending draft jobs across all types. Cheap: same glob
|
|
207
|
+
shape as _oldest_pending_age, just counted instead of min'd. Feeds the
|
|
208
|
+
queue_health_samples row (see _report_queue_health_sample)."""
|
|
209
|
+
pend_root = os.path.join(_queue_root(), "pending")
|
|
210
|
+
n = 0
|
|
211
|
+
for sub in glob.glob(os.path.join(pend_root, "*")):
|
|
212
|
+
for jf in glob.glob(os.path.join(sub, "*.json")):
|
|
213
|
+
if not jf.endswith(".tmp"):
|
|
214
|
+
n += 1
|
|
215
|
+
return n
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _running_count() -> int:
|
|
219
|
+
"""Number of claimed-but-unfinished jobs. See _pending_count."""
|
|
220
|
+
run_root = os.path.join(_queue_root(), "running")
|
|
221
|
+
return sum(1 for jf in glob.glob(os.path.join(run_root, "*.json")) if not jf.endswith(".tmp"))
|
|
222
|
+
|
|
223
|
+
|
|
165
224
|
def _oldest_running_age() -> float | None:
|
|
166
225
|
"""Seconds since the oldest CLAIMED-but-unfinished job was written, or None if
|
|
167
226
|
nothing is in flight. A worker claims by moving a job pending/ -> running/ and
|
|
@@ -217,18 +276,97 @@ def _sentry():
|
|
|
217
276
|
return sentry_init
|
|
218
277
|
|
|
219
278
|
|
|
279
|
+
def _report_recovery(total_duration_s: float, paged_duration_s: float | None) -> None:
|
|
280
|
+
"""Log a recovery milestone (with duration) once per stall episode that
|
|
281
|
+
actually paged. Previously this had to be hand-reconstructed per incident by
|
|
282
|
+
diffing DB/log timestamps after the fact (see the Karol 2026-07-07
|
|
283
|
+
update-orphan investigation); this makes it a first-class, queryable Sentry
|
|
284
|
+
event so incident severity is trackable fleet-wide over time instead of
|
|
285
|
+
per-investigation. Only fires for episodes that crossed ALERT_AFTER — a
|
|
286
|
+
stall that self-clears before paging is normal jitter, not an incident."""
|
|
287
|
+
try:
|
|
288
|
+
sentry = _sentry()
|
|
289
|
+
sentry.init()
|
|
290
|
+
total_min = total_duration_s / 60.0
|
|
291
|
+
paged_str = f"{paged_duration_s / 60.0:.1f} min" if paged_duration_s is not None else "n/a"
|
|
292
|
+
sentry.capture_message(
|
|
293
|
+
"social-autoposter autopilot recovered: draft jobs draining again "
|
|
294
|
+
f"after {total_min:.1f} min stalled ({paged_str} since it paged).",
|
|
295
|
+
level="info",
|
|
296
|
+
tags={"component": "autopilot", "issue": "stall_recovered"},
|
|
297
|
+
extra={
|
|
298
|
+
"stall_total_duration_seconds": round(total_duration_s, 1),
|
|
299
|
+
"stall_paged_duration_seconds": (
|
|
300
|
+
round(paged_duration_s, 1) if paged_duration_s is not None else None
|
|
301
|
+
),
|
|
302
|
+
},
|
|
303
|
+
)
|
|
304
|
+
sentry.flush()
|
|
305
|
+
except Exception:
|
|
306
|
+
sys.stderr.write(f"[stall-watch] recovered after {total_duration_s:.0f}s but Sentry report failed\n")
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def _report_queue_health_sample(
|
|
310
|
+
pending: int,
|
|
311
|
+
running: int,
|
|
312
|
+
timeouts: int,
|
|
313
|
+
age: float | None,
|
|
314
|
+
run_age: float | None,
|
|
315
|
+
stalled: bool,
|
|
316
|
+
batches_stuck: bool,
|
|
317
|
+
) -> None:
|
|
318
|
+
"""Best-effort POST of one queue_health_samples row, on EVERY tick (not
|
|
319
|
+
just while stalled) — a continuous baseline is the point: it's what makes
|
|
320
|
+
"was this box actually healthy at time T" a one-line SQL query instead of
|
|
321
|
+
absence-of-evidence. See migrations/2026-07-07-queue-health-samples.sql.
|
|
322
|
+
|
|
323
|
+
Catches BaseException, not just Exception — see _recent_batches_not_progressing
|
|
324
|
+
for why (http_api._request raises SystemExit on a terminal 4xx/5xx, which
|
|
325
|
+
must never be allowed to kill this watchdog process)."""
|
|
326
|
+
try:
|
|
327
|
+
repo = os.environ.get("S4L_REPO_DIR")
|
|
328
|
+
if repo:
|
|
329
|
+
scripts_dir = os.path.join(repo, "scripts")
|
|
330
|
+
if scripts_dir not in sys.path:
|
|
331
|
+
sys.path.insert(0, scripts_dir)
|
|
332
|
+
import http_api # noqa: E402
|
|
333
|
+
|
|
334
|
+
http_api.api_post(
|
|
335
|
+
"/api/v1/queue-health-samples",
|
|
336
|
+
{
|
|
337
|
+
"pending": pending,
|
|
338
|
+
"running": running,
|
|
339
|
+
"consecutive_timeouts": timeouts,
|
|
340
|
+
"oldest_pending_age_s": int(age) if age is not None else None,
|
|
341
|
+
"oldest_running_age_s": int(run_age) if run_age is not None else None,
|
|
342
|
+
"stalled": stalled,
|
|
343
|
+
"batches_stuck": batches_stuck,
|
|
344
|
+
},
|
|
345
|
+
)
|
|
346
|
+
except BaseException:
|
|
347
|
+
pass
|
|
348
|
+
|
|
349
|
+
|
|
220
350
|
def main() -> int:
|
|
221
351
|
age = _oldest_pending_age()
|
|
222
352
|
run_age = _oldest_running_age()
|
|
223
353
|
timeouts = _consecutive_timeouts()
|
|
224
|
-
|
|
354
|
+
configured = _autopilot_configured()
|
|
355
|
+
# batches_stuck is deliberately computed unconditionally (not short-circuited
|
|
356
|
+
# by the other signals) — it's a genuinely independent, outcome-level
|
|
357
|
+
# backstop, not a cheaper proxy for the same thing. It's what would have
|
|
358
|
+
# caught the Karol 2026-07-07 stall immediately instead of hours later.
|
|
359
|
+
batches_stuck = configured and _recent_batches_not_progressing()
|
|
360
|
+
# Four complementary signals, OR'd; all gated on the autopilot actually being
|
|
225
361
|
# configured here. (1) durable producer drain latch, (2) fast pending-age (job
|
|
226
|
-
# never claimed), (3) running-age (job claimed then wedged mid-run)
|
|
227
|
-
# only one that catches a worker dying after it picked
|
|
228
|
-
|
|
362
|
+
# never claimed), (3) running-age (job claimed then wedged mid-run) — (3) is
|
|
363
|
+
# the only one of the first three that catches a worker dying after it picked
|
|
364
|
+
# up the job — (4) batches_stuck, the outcome-level backstop above.
|
|
365
|
+
stalled = configured and (
|
|
229
366
|
timeouts >= 1
|
|
230
367
|
or (age is not None and age > STALL_SECONDS)
|
|
231
368
|
or (run_age is not None and run_age > RUNNING_STALL_SECONDS)
|
|
369
|
+
or batches_stuck
|
|
232
370
|
)
|
|
233
371
|
# A rate-limit stall is expected and self-heals at the quota reset — never page
|
|
234
372
|
# for it (and re-arm can't fix it). Treat it as "not an actionable stall" so the
|
|
@@ -236,45 +374,84 @@ def main() -> int:
|
|
|
236
374
|
if stalled and _recent_rate_limit():
|
|
237
375
|
stalled = False
|
|
238
376
|
|
|
377
|
+
# Record this tick regardless of outcome — see _report_queue_health_sample.
|
|
378
|
+
_report_queue_health_sample(
|
|
379
|
+
_pending_count(), _running_count(), timeouts, age, run_age, stalled, batches_stuck
|
|
380
|
+
)
|
|
381
|
+
|
|
239
382
|
st = _read_state()
|
|
240
383
|
consecutive = int(st.get("consecutive", 0))
|
|
241
384
|
alerted = bool(st.get("alerted", False))
|
|
385
|
+
# first_seen_at: first check this episode looked stalled at all (predates
|
|
386
|
+
# paging by ALERT_AFTER checks). alerted_at: the single moment we paged.
|
|
387
|
+
# Both are stamped once and carried forward untouched for the rest of the
|
|
388
|
+
# episode — NOT refreshed every check — so they measure the episode's
|
|
389
|
+
# start, not "last time we saw it stalled".
|
|
390
|
+
first_seen_at = st.get("first_seen_at")
|
|
391
|
+
alerted_at = st.get("alerted_at")
|
|
242
392
|
|
|
243
393
|
if not stalled:
|
|
244
394
|
# Recovered (or never stalled) -> reset the episode so the next stall pages.
|
|
245
395
|
if consecutive or alerted:
|
|
396
|
+
if alerted and first_seen_at:
|
|
397
|
+
total_duration = time.time() - float(first_seen_at)
|
|
398
|
+
paged_duration = (time.time() - float(alerted_at)) if alerted_at else None
|
|
399
|
+
_report_recovery(total_duration, paged_duration)
|
|
246
400
|
_write_state({"consecutive": 0, "alerted": False})
|
|
247
401
|
return 0
|
|
248
402
|
|
|
249
403
|
consecutive += 1
|
|
404
|
+
if first_seen_at is None:
|
|
405
|
+
first_seen_at = time.time()
|
|
250
406
|
age_str = f"{int(age)}s" if age is not None else "n/a (between cycles)"
|
|
251
407
|
run_age_str = f"{int(run_age)}s" if run_age is not None else "n/a (none in flight)"
|
|
252
|
-
# Distinguish the
|
|
253
|
-
# but-wedged job (running-age) is a mid-run worker death,
|
|
408
|
+
# Distinguish the shapes so the alert points at the right cause: a claimed-
|
|
409
|
+
# but-wedged job (running-age) is a mid-run worker death, batches_stuck is
|
|
410
|
+
# the outcome-level backstop firing independently of the queue signals, and
|
|
411
|
+
# the fallback is the classic orphaned-routine case.
|
|
254
412
|
wedged_inflight = run_age is not None and run_age > RUNNING_STALL_SECONDS
|
|
255
413
|
if consecutive >= ALERT_AFTER and not alerted:
|
|
256
414
|
try:
|
|
257
415
|
sentry = _sentry()
|
|
258
416
|
sentry.init()
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
417
|
+
if wedged_inflight:
|
|
418
|
+
cause = (
|
|
419
|
+
"a worker claimed a draft job and then died mid-run (claude -p child "
|
|
420
|
+
"never came up / crashed)"
|
|
421
|
+
)
|
|
422
|
+
stall_shape = "inflight_wedged"
|
|
423
|
+
elif batches_stuck:
|
|
424
|
+
cause = (
|
|
425
|
+
f"last {BATCH_PROGRESSION_MIN_BATCHES} twitter_batches all failed to "
|
|
426
|
+
"reach phase2b-gen — drafting is not actually happening even if the "
|
|
427
|
+
"queue-level counters look ambiguous"
|
|
428
|
+
)
|
|
429
|
+
stall_shape = "batches_not_progressing"
|
|
430
|
+
else:
|
|
431
|
+
cause = "scheduled-task routines likely orphaned — Claude Desktop account change?"
|
|
432
|
+
stall_shape = "not_draining"
|
|
433
|
+
# Our own staging/QA/dev boxes (identity.is_internal_install) get set
|
|
434
|
+
# up and rebuilt with nobody actively feeding the queue, which looks
|
|
435
|
+
# identical to a real stall on the signals above. Downgrade those to
|
|
436
|
+
# warning instead of error so they don't page as a customer incident
|
|
437
|
+
# (the digest only scans error/fatal) while still leaving a Sentry
|
|
438
|
+
# record if we ever need to look one up by hand.
|
|
439
|
+
is_internal = identity.is_internal_install()
|
|
265
440
|
sentry.capture_message(
|
|
266
441
|
"social-autoposter autopilot stalled: draft jobs are not being "
|
|
267
442
|
f"drained ({cause}). producer consecutive timeouts={timeouts}, "
|
|
268
443
|
f"oldest pending job age={age_str}, oldest in-flight (running) job "
|
|
269
444
|
f"age={run_age_str}, sustained {consecutive} checks.",
|
|
270
|
-
level="error",
|
|
445
|
+
level=("warning" if is_internal else "error"),
|
|
271
446
|
tags={
|
|
272
447
|
"component": "autopilot",
|
|
273
448
|
"issue": "stall",
|
|
274
|
-
"stall_shape":
|
|
449
|
+
"stall_shape": stall_shape,
|
|
275
450
|
"consecutive_timeouts": str(timeouts),
|
|
276
451
|
"oldest_pending_age_s": str(int(age)) if age is not None else "",
|
|
277
452
|
"oldest_running_age_s": str(int(run_age)) if run_age is not None else "",
|
|
453
|
+
"batches_stuck": str(batches_stuck),
|
|
454
|
+
"internal_install": str(is_internal),
|
|
278
455
|
},
|
|
279
456
|
)
|
|
280
457
|
sentry.flush()
|
|
@@ -285,8 +462,15 @@ def main() -> int:
|
|
|
285
462
|
f"pending_age={age_str}, running_age={run_age_str}) but Sentry report failed\n"
|
|
286
463
|
)
|
|
287
464
|
alerted = True
|
|
288
|
-
|
|
289
|
-
|
|
465
|
+
alerted_at = time.time()
|
|
466
|
+
|
|
467
|
+
_write_state({
|
|
468
|
+
"consecutive": consecutive,
|
|
469
|
+
"alerted": alerted,
|
|
470
|
+
"first_seen_at": first_seen_at,
|
|
471
|
+
"alerted_at": alerted_at,
|
|
472
|
+
"at": time.time(),
|
|
473
|
+
})
|
|
290
474
|
return 0
|
|
291
475
|
|
|
292
476
|
|
package/scripts/claude_job.py
CHANGED
|
@@ -920,6 +920,12 @@ def _attempt_claim(ns, qtype: str) -> bool:
|
|
|
920
920
|
|
|
921
921
|
def cmd_next(ns) -> int:
|
|
922
922
|
_apply_state_dir_override(ns)
|
|
923
|
+
# S4L paused (mcp/src/index.ts pauseAutopilot): report no work instead of
|
|
924
|
+
# claiming a job, so an already-queued draft can't drain/post while paused
|
|
925
|
+
# even though the scheduled task worker still fires on its own cadence.
|
|
926
|
+
if os.path.exists(os.path.join(state_dir(), "paused.flag")):
|
|
927
|
+
print(json.dumps({}))
|
|
928
|
+
return 0
|
|
923
929
|
qtype = ns.type
|
|
924
930
|
wait_seconds = max(0, ns.wait_seconds or 0)
|
|
925
931
|
# wait_seconds=0 (default) is the legacy single-shot behavior: one pass,
|