@m13v/s4l 1.7.4-rc.4 → 1.7.4

package/scripts/snapshot.py

@@ -70,7 +70,7 @@ PERSONA_REQUIRED_FIELDS = ["name", "description", "voice", "search_topics"]
 CURRENT_WORKER_TASK_IDS = ("s4l-worker", "saps-worker")
 LEGACY_WORKER_TASK_IDS = ("saps-phase1-query", "saps-phase2b-draft")
 UPDATER_LABEL = "com.m13v.social-autoposter-update"
-AUTOPILOT_STALL_MS = 180_000
+AUTOPILOT_STALL_MS = 1_200_000
 
 # Milestones overlaid with LIVE state for display (the rest keep their ledger
 # value). Mirrors the overlay in buildSnapshot().
@@ -325,6 +325,81 @@ def _x_status():
 _ver_cache = {"at": 0.0, "latest": None, "tag": None, "channel": None, "checked": False}
 _VER_TTL = 55.0
 
+# SHARED CROSS-PROCESS CACHE (2026-07-13): <state dir>/latest-release.json.
+# Every surface that resolves the newest release (this module, mcp/src/
+# version.ts, scripts/s4l_box_update.sh) reads and writes THIS one file, so a
+# box makes at most one real GitHub probe per _SHARED_TTL no matter how many
+# short-lived processes spin up (MCP servers respawn per s4l-worker session,
+# buildSnapshot shells this module as a fresh subprocess, the menu bar ticks).
+# The persisted ETag makes even those probes quota-free between releases (304s
+# do not count against the anonymous 60/h-per-IP quota, and before this cache
+# each process held its OWN in-process ETag that died with the process, so
+# every respawn paid a full 200). Added after the box's aggregate probing
+# (~80-100 req/h across processes) burned the quota on 2026-07-13 and silenced
+# the update banner. Failures (version=None) are cached too, same rationale as
+# _ver_cache. Keep the file shape in lockstep with version.ts::SharedCache:
+# {"at": <epoch s>, "channel": ..., "version": ..., "tag": ..., "etag": ...}
+_SHARED_TTL = 600.0  # banner latency ceiling: a new release surfaces within 10 min
+_PROBE_LOCK_STALE = 30.0
+
+
+def _shared_cache_path():
+    return os.path.join(_state_dir(), "latest-release.json")
+
+
+def _read_shared_cache(channel):
+    d = _read_json(_shared_cache_path())
+    if not isinstance(d, dict) or d.get("channel") != channel:
+        return None
+    if not isinstance(d.get("at"), (int, float)):
+        return None
+    return {
+        "at": float(d["at"]),
+        "channel": channel,
+        "version": d.get("version") if isinstance(d.get("version"), str) else None,
+        "tag": d.get("tag") if isinstance(d.get("tag"), str) else None,
+        "etag": d.get("etag") if isinstance(d.get("etag"), str) else None,
+    }
+
+
+def _write_shared_cache(channel, version, tag, etag):
+    try:
+        os.makedirs(_state_dir(), exist_ok=True)
+        p = _shared_cache_path()
+        tmp = "%s.tmp.%d" % (p, os.getpid())
+        with open(tmp, "w") as f:
+            json.dump({"at": time.time(), "channel": channel,
+                       "version": version, "tag": tag, "etag": etag}, f)
+        os.replace(tmp, p)
+    except Exception:
+        pass  # best effort; worst case another process re-probes
+
+
+def _try_probe_lock():
+    """Single-flight guard so N concurrent processes with an expired shared
+    cache don't all probe at once. Returns (acquired, lock_path). acquired is
+    False ONLY when another probe holds a FRESH lock (< _PROBE_LOCK_STALE s);
+    any other failure returns (True, None) so we probe anyway rather than go
+    blind on a broken state dir."""
+    lock = os.path.join(_state_dir(), "latest-release.lock")
+    try:
+        os.makedirs(_state_dir(), exist_ok=True)
+        fd = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+        os.close(fd)
+        return True, lock
+    except FileExistsError:
+        try:
+            if time.time() - os.path.getmtime(lock) > _PROBE_LOCK_STALE:
+                os.unlink(lock)  # stale: holder died mid-probe
+                fd = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+                os.close(fd)
+                return True, lock
+        except Exception:
+            pass
+        return False, None
+    except Exception:
+        return True, None
+
 _RELEASES_LATEST_URL = "https://github.com/m13v/s4l/releases/latest"
 _RELEASES_LATEST_API = "https://api.github.com/repos/m13v/s4l/releases/latest"
 # Staging resolves the newest release OVERALL from the releases LIST, since
@@ -367,56 +442,64 @@ def _latest_from_github_redirect():
         return None
 
 
-# In-process conditional-request state. Long-lived processes (the menu bar) send
-# If-None-Match on every probe and get free 304s; short-lived shell-outs pay one
-# 200 per process, which is rare enough to stay far under quota.
-_api_state = {"etag": None, "latest": None}
-
-
-def _latest_from_github_api():
+# Conditional-request state lives in the SHARED cache file (latest-release.json)
+# so the ETag survives process boundaries: short-lived shell-outs used to pay a
+# full 200 per process; now every probe sends If-None-Match and gets a free 304
+# between releases.
+def _curl_conditional(url, etag):
+    """GET url with optional If-None-Match. Returns (status, new_etag, body)."""
+    args = ["/usr/bin/curl", "-sS", "-m", "10",
+            "-H", "Accept: application/vnd.github+json"]
+    if etag:
+        args += ["-H", "If-None-Match: %s" % etag]
+    args += ["-w", "\n__CURL_STATUS__:%{http_code}\n__CURL_ETAG__:%header{etag}", url]
+    res = subprocess.run(args, capture_output=True, text=True, timeout=12)
+    status, new_etag, body = 0, None, []
+    for line in (res.stdout or "").splitlines():
+        if line.startswith("__CURL_STATUS__:"):
+            status = int(line.split(":", 1)[1].strip() or 0)
+        elif line.startswith("__CURL_ETAG__:"):
+            new_etag = line.split(":", 1)[1].strip() or None
+        else:
+            body.append(line)
+    return status, new_etag, "\n".join(body)
+
+
+def _latest_from_github_api(etag=None, cached=None):
+    """Stable probe (releases/latest). Returns (version, etag). On 304 serves
+    the caller-supplied cached version with the same etag; If-None-Match is only
+    sent when there IS a cached value to serve."""
     try:
-        args = ["/usr/bin/curl", "-sS", "-m", "10",
-                "-H", "Accept: application/vnd.github+json"]
-        if _api_state["etag"]:
-            args += ["-H", "If-None-Match: %s" % _api_state["etag"]]
-        args += ["-w", "\n__CURL_STATUS__:%{http_code}\n__CURL_ETAG__:%header{etag}",
-                 _RELEASES_LATEST_API]
-        res = subprocess.run(args, capture_output=True, text=True, timeout=12)
-        status, etag, body = 0, None, []
-        for line in (res.stdout or "").splitlines():
-            if line.startswith("__CURL_STATUS__:"):
-                status = int(line.split(":", 1)[1].strip() or 0)
-            elif line.startswith("__CURL_ETAG__:"):
-                etag = line.split(":", 1)[1].strip() or None
-            else:
-                body.append(line)
+        status, new_etag, body = _curl_conditional(
+            _RELEASES_LATEST_API, etag if cached else None)
         if status == 304:
-            return _api_state["latest"]
+            return cached, etag
         if status != 200:
-            return None
-        tag = (json.loads("\n".join(body)) or {}).get("tag_name")
+            return None, None
+        tag = (json.loads(body) or {}).get("tag_name")
         v = _parse_semverish(tag.lstrip("v").strip()) if isinstance(tag, str) else None
-        if v:
-            _api_state.update(etag=etag, latest=v)
-        return v
+        return (v, new_etag) if v else (None, None)
     except Exception:
-        return None
+        return None, None
 
 
-def _latest_from_github_list_staging():
+def _latest_from_github_list_staging(etag=None, cached=None):
     """Staging channel: newest release OVERALL (prereleases included) from the
-    releases LIST endpoint. Returns (version, tag) or (None, None). Drafts are
-    skipped. 'Newest' is by the rc-aware key so 1.6.193 outranks 1.6.193-rc.N and
-    rc.2 outranks rc.1."""
+    releases LIST endpoint. Returns (version, tag, etag) or (None, None, None).
+    Drafts are skipped. 'Newest' is by the rc-aware key so 1.6.193 outranks
+    1.6.193-rc.N and rc.2 outranks rc.1. On 304 serves the caller-supplied
+    cached (version, tag) with the same etag."""
     try:
-        res = subprocess.run(
-            ["/usr/bin/curl", "-sS", "-m", "10",
-             "-H", "Accept: application/vnd.github+json",
-             _RELEASES_LIST_API],
-            capture_output=True, text=True, timeout=12)
-        rels = json.loads(res.stdout or "[]")
+        cached_v, cached_tag = cached if isinstance(cached, tuple) else (None, None)
+        status, new_etag, body = _curl_conditional(
+            _RELEASES_LIST_API, etag if (cached_v and cached_tag) else None)
+        if status == 304:
+            return cached_v, cached_tag, etag
+        if status != 200:
+            return None, None, None
+        rels = json.loads(body or "[]")
         if not isinstance(rels, list):
-            return None, None
+            return None, None, None
         best_v, best_tag, best_key = None, None, None
         for r in rels:
             if not isinstance(r, dict) or r.get("draft"):
@@ -430,9 +513,11 @@ def _latest_from_github_list_staging():
             k = _ver_key(v)
             if best_key is None or k > best_key:
                 best_v, best_tag, best_key = v, tag, k
-        return best_v, best_tag
+        if best_v is None:
+            return None, None, None
+        return best_v, best_tag, new_etag
     except Exception:
-        return None, None
+        return None, None, None
 
 
 def _latest_from_npm():
@@ -460,31 +545,64 @@ def _resolve_version() -> str:
 def _latest_published(channel=None):
     """(version, tag) for the newest release on this box's channel. The tag is
     what the staging download URL is built from; stable callers can ignore it and
-    use releases/latest/download. Cached with the channel so a mid-process
-    channel flip re-probes instead of serving the other channel's cached value."""
+    use releases/latest/download. Two cache layers, both keyed by channel so a
+    mid-process flip re-probes instead of serving the other channel's value:
+      1. in-process _ver_cache (55s) — spares the file read on tight tick loops;
+      2. shared latest-release.json (_SHARED_TTL) — spares the NETWORK probe
+         across ALL processes on the box.
+    Failures (latest=None) are cached in both, like version.ts: a tick loop
+    re-probing an unreachable/rate-limited GitHub every few seconds would burn
+    the unauthenticated API quota (60/h per IP) and lock itself out for good."""
     if channel is None:
         channel = _channel()
     now = time.time()
-    # Cache failures (latest=None) too, like version.ts: a menu-bar tick loop
-    # re-probing an unreachable/rate-limited GitHub every few seconds would burn
-    # the unauthenticated API quota (60/h per IP) and lock itself out for good.
     if (_ver_cache["checked"] and _ver_cache["channel"] == channel
             and now - _ver_cache["at"] < _VER_TTL):
         return _ver_cache["latest"], _ver_cache["tag"]
-    if channel == "staging":
-        latest, tag = _latest_from_github_list_staging()
-        # Fallback to the stable probes if the list endpoint fails, so a staging
-        # box degrades to "at least track stable" rather than going blind.
-        if latest is None:
-            latest = _latest_from_github_api() or _latest_from_github_redirect()
+    shared = _read_shared_cache(channel)
+    if shared and now - shared["at"] < _SHARED_TTL:
+        _ver_cache.update(at=now, latest=shared["version"], tag=shared["tag"],
+                          channel=channel, checked=True)
+        return shared["version"], shared["tag"]
+    acquired, lock = _try_probe_lock()
+    if not acquired:
+        # Another process is probing right now; serve the stale shared value
+        # (or None) instead of doubling the request. Short in-process TTL means
+        # we pick up its fresh result within a minute.
+        latest = shared["version"] if shared else None
+        tag = shared["tag"] if shared else None
+        _ver_cache.update(at=now, latest=latest, tag=tag, channel=channel, checked=True)
+        return latest, tag
+    etag = shared["etag"] if shared else None
+    try:
+        if channel == "staging":
+            latest, tag, new_etag = _latest_from_github_list_staging(
+                etag, (shared["version"], shared["tag"]) if shared else None)
+            # Fallback to the stable probes if the list endpoint fails, so a
+            # staging box degrades to "at least track stable" than going blind.
+            # new_etag stays None: a releases/latest etag must never be replayed
+            # against the LIST endpoint on the next staging probe.
+            if latest is None:
+                latest, _ = _latest_from_github_api()
+                new_etag = None
+                if latest is None:
+                    latest = _latest_from_github_redirect()
+                tag = ("v" + latest) if latest else None
+        else:
+            latest, new_etag = _latest_from_github_api(
+                etag, shared["version"] if shared else None)
+            if latest is None:
+                latest, new_etag = _latest_from_github_redirect(), None
+            if latest is None:
+                latest = _latest_from_npm()
             tag = ("v" + latest) if latest else None
-    else:
-        latest = _latest_from_github_api()
-        if latest is None:
-            latest = _latest_from_github_redirect()
-        if latest is None:
-            latest = _latest_from_npm()
-        tag = ("v" + latest) if latest else None
+        _write_shared_cache(channel, latest, tag, new_etag)
+    finally:
+        if lock:
+            try:
+                os.unlink(lock)
+            except Exception:
+                pass
     _ver_cache.update(at=now, latest=latest, tag=tag, channel=channel, checked=True)
     return latest, tag
 

package/scripts/stall_guard.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""Shared zero-progress deadline for browser-touching pipeline steps.
+
+Any step that relays CDP to the harness Chrome (scan queries, thread media
+capture, ...) can hang FOREVER against a half-wedged browser or a dead
+renderer: the call never returns, the step holds the browser lock, and the
+cycle-entry health checks that could heal Chrome can't run because the next
+cycle can't start. 2026-07-13: a scan hung 23 min at zero queries; the same
+night media capture hung 34+ min at candidate 24/92 and only the watchdog's
+90-min lock-liveness net was ever going to reap it.
+
+One implementation, used by every step (twitter_scan.py Phase-1 queries,
+twitter_browser.py media capture). Usage:
+
+    from stall_guard import stall_guard
+    with stall_guard("thread_media", url):   # default deadline 240s
+        <one bounded unit of browser work>
+
+If the guarded block does not finish inside the deadline the guard:
+  1. prints a loud machine-greppable marker to stderr
+     ([stall_guard] STALL_ABORT step=<step> ...)
+  2. stamps the port's wedge strike file, so the next cycle's two-strike
+     gate in skill/lib/twitter-backend.sh converges to a kill+relaunch if
+     Chrome is genuinely sick
+  3. hard-exits the process with EX_TEMPFAIL (75) so the cycle fails fast
+     and releases the browser lock + cycle slot.
+
+A completed block cancels its timer; a healthy pipeline never notices the
+guard exists. Deadline override: S4L_BROWSER_OP_DEADLINE_S (seconds).
+"""
+from __future__ import annotations
+
+import contextlib
+import os
+import pathlib
+import sys
+import threading
+
+DEFAULT_DEADLINE_S = float(os.environ.get("S4L_BROWSER_OP_DEADLINE_S", "240"))
+
+
+def _strike_file(port: int) -> str:
+    # Must match the two-strike gate in skill/lib/twitter-backend.sh.
+    return f"/tmp/s4l_cdp_wedge_strike_{port}"
+
+
+@contextlib.contextmanager
+def stall_guard(step: str, detail: str = "", deadline_s: float | None = None, port: int = 9555):
+    limit = DEFAULT_DEADLINE_S if deadline_s is None else float(deadline_s)
+
+    def _fire():
+        sys.stderr.write(
+            f"[stall_guard] STALL_ABORT step={step} detail={detail!r} "
+            f"deadline={limit:.0f}s — browser call hung (wedged Chrome or dead "
+            "renderer); stamping wedge strike and aborting so the lock frees\n"
+        )
+        sys.stderr.flush()
+        try:
+            pathlib.Path(_strike_file(port)).touch()
+        except OSError:
+            pass
+        os._exit(75)  # EX_TEMPFAIL
+
+    t = threading.Timer(limit, _fire)
+    t.daemon = True
+    t.start()
+    try:
+        yield
+    finally:
+        t.cancel()

package/scripts/test_scheduled_task_selfheal.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Focused tests for fresh-account scheduled-task registry creation."""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import sqlite3
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+sys.path.insert(0, os.path.dirname(__file__))
+import scheduled_task_selfheal as selfheal  # noqa: E402
+
+
+ACCOUNT_UUID = "f598e7b5-55be-46e8-815e-e8ddb394fc73"
+ORG_UUID = "d03d2d69-12d0-45bf-9f67-ae085d6848e5"
+
+
+def _write_cookie_db(root: str, *, value: str = "", encrypted: bytes = b"v10fixture"):
+    db = os.path.join(root, "Cookies")
+    conn = sqlite3.connect(db)
+    try:
+        conn.execute(
+            """
+            CREATE TABLE cookies (
+                host_key TEXT,
+                name TEXT,
+                value TEXT,
+                encrypted_value BLOB,
+                last_access_utc INTEGER
+            )
+            """
+        )
+        conn.execute(
+            "INSERT INTO cookies VALUES (?, 'lastActiveOrg', ?, ?, 1)",
+            (selfheal.CLAUDE_COOKIE_HOST, value, encrypted),
+        )
+        conn.commit()
+    finally:
+        conn.close()
+
+
+class ScheduledTaskSelfhealTest(unittest.TestCase):
+    def test_decrypted_cookie_strips_chromium_host_digest(self):
+        with tempfile.TemporaryDirectory() as root:
+            _write_cookie_db(root)
+            clear = (
+                hashlib.sha256(selfheal.CLAUDE_COOKIE_HOST.encode()).digest()
+                + ORG_UUID.encode()
+            )
+            with (
+                mock.patch.object(
+                    selfheal, "_claude_safe_storage_password", return_value=b"fixture"
+                ),
+                mock.patch.object(selfheal, "_aes_128_cbc_decrypt", return_value=clear),
+            ):
+                self.assertEqual(selfheal._active_org_uuid_from_cookie(root), ORG_UUID)
+
+    def test_capability_probe_does_not_read_keychain(self):
+        with tempfile.TemporaryDirectory() as root:
+            os.makedirs(root, exist_ok=True)
+            config = os.path.join(root, "config.json")
+            with open(config, "w") as fh:
+                json.dump({"lastKnownAccountUuid": ACCOUNT_UUID}, fh)
+            _write_cookie_db(root)
+            with (
+                mock.patch.object(
+                    selfheal.schedule_state, "_config_json_paths", return_value=[config]
+                ),
+                mock.patch.object(
+                    selfheal,
+                    "_claude_safe_storage_password",
+                    side_effect=AssertionError("passive probe touched Keychain"),
+                ),
+            ):
+                self.assertTrue(selfheal.can_create_for_active_account())
+
+    def test_heal_creates_exact_account_org_registry_without_session_dir(self):
+        with tempfile.TemporaryDirectory() as root:
+            config = os.path.join(root, "config.json")
+            with open(config, "w") as fh:
+                json.dump({"lastKnownAccountUuid": ACCOUNT_UUID}, fh)
+            worker_cwd = os.path.join(root, "worker")
+            registry_glob = os.path.join(
+                root, "claude-code-sessions", "*", "*", "scheduled-tasks.json"
+            )
+            expected = os.path.join(
+                root,
+                "claude-code-sessions",
+                ACCOUNT_UUID,
+                ORG_UUID,
+                "scheduled-tasks.json",
+            )
+            with (
+                mock.patch.object(
+                    selfheal.schedule_state, "_config_json_paths", return_value=[config]
+                ),
+                mock.patch.object(
+                    selfheal, "_active_org_uuid_from_cookie", return_value=ORG_UUID
+                ),
+                mock.patch.object(selfheal, "_ensure_worker_skill_md", return_value=True),
+                mock.patch.object(selfheal, "WORKER_CWD", worker_cwd),
+                mock.patch.object(selfheal, "SCHED_REGISTRY_GLOB", registry_glob),
+                mock.patch.object(selfheal, "DEPRECATED_TASK_IDS", ()),
+                mock.patch.object(selfheal, "LEGACY_WORKER_TASK_IDS", ()),
+            ):
+                summary = selfheal.heal()
+
+            self.assertEqual(summary["created"], [expected])
+            with open(expected) as fh:
+                registry = json.load(fh)
+            self.assertEqual(registry["recordedSkips"], [])
+            self.assertEqual(registry["scheduledTasks"][0]["id"], selfheal.WORKER_TASK_ID)
+            self.assertEqual(registry["scheduledTasks"][0]["cwd"], worker_cwd)
+
+
+if __name__ == "__main__":
+    unittest.main()

package/scripts/top_performers.py

@@ -567,9 +567,43 @@ def main():
                               "summary, no bottom posts). This is the lean "
                               "on-demand shape the drafting session calls "
                               "after routing a candidate to a project."))
+    parser.add_argument("--invoked-by", default=None,
+                        help=("Caller tag recorded in the on-demand invocation "
+                              "ledger (the drafting prompt passes the cycle's "
+                              "batch_id). Tracking only; no behavior change."))
     parser.add_argument("--json", action="store_true", help="Output as JSON")
     args = parser.parse_args()
 
+    # On-demand invocation ledger (2026-07-10). Any --project call is the
+    # on-demand per-project winners lookup the draft prompt offers; record it
+    # at the TOOL level (model self-reports are unreliable) so we can measure
+    # whether drafting sessions actually use the query. One JSON line per
+    # call in $S4L_STATE_DIR/top-performers-invocations.jsonl; the cycle
+    # counts lines for its batch_id after prep and logs a
+    # [project_top_performers] marker. Best-effort: never block the report.
+    if args.project:
+        try:
+            import datetime
+            state_dir = os.environ.get(
+                "S4L_STATE_DIR",
+                os.path.expanduser("~/.social-autoposter-mcp"))
+            os.makedirs(state_dir, exist_ok=True)
+            with open(os.path.join(state_dir,
+                                   "top-performers-invocations.jsonl"),
+                      "a") as fh:
+                fh.write(json.dumps({
+                    "ts": datetime.datetime.now(
+                        datetime.timezone.utc).isoformat(),
+                    "project": args.project,
+                    "platform": args.platform,
+                    "brief": bool(args.brief),
+                    "top": args.top,
+                    "invoked_by": args.invoked_by,
+                }) + "\n")
+        except Exception as exc:
+            print(f"[top_performers] invocation ledger write failed: {exc!r}",
+                  file=sys.stderr)
+
     (summary, style_perf, top, bottom, fallback_top,
      top_by_group, top_by_style) = _fetch_report_via_api(
         platform=args.platform, project=args.project, top=args.top, bottom=args.bottom,