@m13v/s4l 1.7.7-rc.5 → 1.7.7-rc.7

package/mcp/dist/index.js

@@ -131,10 +131,13 @@ const STALL_WATCH_INTERVAL_SECS = 120;
 // S4L_OVERLAY_WATCH=0.
 const OVERLAY_WATCH_LABEL = "com.m13v.social-overlay-watch";
 const OVERLAY_WATCH_PLIST = path.join(os.homedir(), "Library", "LaunchAgents", `${OVERLAY_WATCH_LABEL}.plist`);
-// Daily self-updater. Enabled alongside autopilot so a hands-free (headless)
-// install keeps itself current — the interactive `runtime` tool (action:'update')
-// only helps when
-// a human-facing agent session is open, which an autopilot box never has.
+// Legacy daily self-updater launchd job. BANNED (user decision 2026-07-31):
+// it ran s4l_box_update.sh unattended, which kills and relaunches Claude
+// Desktop with no human in the loop. Updates are user-initiated only (menubar
+// "Update now & restart Claude Desktop", the panel button, `runtime`
+// action:'update', or an operator running s4l_box_update.sh explicitly). The
+// label/plist constants survive so ensureUpdaterRemoved() can clean the job
+// off every box that got it from v1.7.1..v1.7.7-rc.4.
 const UPDATER_LABEL = "com.m13v.social-autoposter-update";
 const UPDATER_PLIST = path.join(os.homedir(), "Library", "LaunchAgents", `${UPDATER_LABEL}.plist`);
 // A sane PATH for launchd jobs (launchd starts with a bare PATH). Include the
@@ -4874,68 +4877,21 @@ async function ensureOverlayWatchInstalled() {
         return { ok: false, detail: e?.message || String(e) };
     }
 }
-// Install/refresh the daily self-updater launchd job. This used to be bundled
-// into the now-deleted `autopilot` MCP tool (removed 2026-06-19, 88bd1cb9):
-// calling `autopilot enable` installed this job as a side effect, so removing
-// the tool silently orphaned it — UPDATER_LABEL/UPDATER_PLIST and the
-// auto_update_on status check survived (buildSnapshot still reports them), but
-// nothing has installed the plist since, on ANY box provisioned after that
-// commit (auto_update_on reads false forever). Restored here as its own
-// deterministic boot-time job, same pattern as its five siblings above.
-//
-// Points at scripts/s4l_box_update.sh, NOT skill/social-autoposter-update.sh:
-// the latter is the npm-lane updater (npm view + npx update) and is a silent
-// no-op on a .mcpb box, which has no npm/npx on PATH (see version.ts). The
-// .mcpb-lane equivalent downloads the .mcpb directly from the channel-resolved
-// GitHub release and unpacks it over the extension dir — see that script's own
-// header for the channel/no-downgrade/retry guards. Default mode there
-// downloads + unpacks + restarts Claude Desktop with NO human in the loop,
-// matching the original bundled updater's intent ("keeps a headless install
-// current"); RunAtLoad so a box that boots already-behind checks promptly.
-async function ensureUpdaterInstalled() {
+// Remove the legacy daily self-updater launchd job (see UPDATER_LABEL's
+// comment). Boot-time, idempotent, best-effort: unload the job if launchd has
+// it and delete the plist so the daily silent Claude Desktop restart can never
+// fire again. Do NOT reintroduce an install path for this job — any future
+// auto-update mechanism must be user-visible and user-confirmed.
+async function ensureUpdaterRemoved() {
     try {
         if (process.platform !== "darwin")
-            return { ok: false, detail: "not macOS" };
-        if ((process.env.S4L_AUTO_UPDATE) === "0")
-            return { ok: false, detail: "disabled (S4L_AUTO_UPDATE=0)" };
-        const logDir = path.join(repoDir(), "skill", "logs");
-        try {
-            fs.mkdirSync(logDir, { recursive: true });
-        }
-        catch {
-            /* best-effort */
-        }
-        const xml = plistXml({
-            label: UPDATER_LABEL,
-            programArgs: ["/bin/bash", path.join(repoDir(), "scripts", "s4l_box_update.sh")],
-            intervalSecs: 86_400,
-            runAtLoad: true,
-            stdoutLog: path.join(logDir, "launchd-self-update-stdout.log"),
-            stderrLog: path.join(logDir, "launchd-self-update-stderr.log"),
-        });
+            return { ok: true, detail: "not macOS" };
         const uid = process.getuid ? process.getuid() : 0;
-        let cur = null;
-        try {
-            cur = fs.readFileSync(UPDATER_PLIST, "utf-8");
-        }
-        catch {
-            cur = null;
-        }
-        let detail;
-        if (cur === xml) {
-            const res = await loadPlist(UPDATER_LABEL, UPDATER_PLIST, uid);
-            detail = `current (load rc=${res.code})`;
-        }
-        else {
-            if (cur !== null) {
-                await unloadPlist(UPDATER_LABEL, UPDATER_PLIST, uid);
-            }
-            fs.mkdirSync(path.dirname(UPDATER_PLIST), { recursive: true });
-            fs.writeFileSync(UPDATER_PLIST, xml, "utf-8");
-            const res = await loadPlist(UPDATER_LABEL, UPDATER_PLIST, uid);
-            detail = cur === null ? "installed + loaded" : `rewritten + reloaded (rc=${res.code})`;
-        }
-        return { ok: true, detail };
+        const existed = fs.existsSync(UPDATER_PLIST);
+        await unloadPlist(UPDATER_LABEL, UPDATER_PLIST, uid);
+        if (existed)
+            fs.rmSync(UPDATER_PLIST, { force: true });
+        return { ok: true, detail: existed ? "removed" : "absent" };
     }
     catch (e) {
         return { ok: false, detail: e?.message || String(e) };
@@ -6134,13 +6090,12 @@ async function main() {
     void ensureOverlayWatchInstalled()
         .then((r) => console.error(`[overlay-watch] launchd supervisor: ${r.ok ? "ok" : "skip"} (${r.detail})`))
         .catch((e) => console.error("[overlay-watch] supervisor install failed:", e?.message || e));
-    // Daily self-updater: restored 2026-07-08 after 88bd1cb9 ("Remove autopilot
-    // tool") silently dropped its only install path (it used to be bundled into
-    // the deleted `autopilot enable` action). Best-effort; never blocks boot.
-    // Disable with S4L_AUTO_UPDATE=0.
-    void ensureUpdaterInstalled()
-        .then((r) => console.error(`[self-update] launchd updater: ${r.ok ? "ok" : "skip"} (${r.detail})`))
-        .catch((e) => console.error("[self-update] updater install failed:", e?.message || e));
+    // Silent daily self-updates are banned (user decision 2026-07-31): the job
+    // restarted Claude Desktop unattended. Clean it off every box that got it
+    // from v1.7.1..v1.7.7-rc.4. Best-effort; never blocks boot.
+    void ensureUpdaterRemoved()
+        .then((r) => console.error(`[self-update] legacy launchd updater: ${r.ok ? "ok" : "skip"} (${r.detail})`))
+        .catch((e) => console.error("[self-update] updater removal failed:", e?.message || e));
     // Heal installs onboarded before short_links_live defaulted to false: such a
     // project wraps short links against the customer's own domain, which has no
     // /r/[code] resolver, so every minted link 404s. Re-point them at the s4l.ai

package/mcp/dist/repo.js

@@ -211,7 +211,9 @@ export function readPlan(batchId) {
     }
 }
 export function writePlan(batchId, plan) {
-    fs.writeFileSync(planPath(batchId), JSON.stringify(plan, null, 2), "utf-8");
+    // Compact (no indent): the review store reached 65 MB with pretty-printing
+    // (2026-08-02 lag incident) and every reader pays the parse.
+    fs.writeFileSync(planPath(batchId), JSON.stringify(plan), "utf-8");
 }
 // Find the newest plan file when no batch id is supplied.
 export function latestBatchId() {

package/mcp/dist/version.json

@@ -1,4 +1,4 @@
 {
-  "version": "1.7.7-rc.5",
-  "installedAt": "2026-07-31T22:56:07.561Z"
+  "version": "1.7.7-rc.7",
+  "installedAt": "2026-08-02T22:02:14.223Z"
 }

package/mcp/manifest.json

@@ -2,7 +2,7 @@
   "dxt_version": "0.1",
   "name": "social-autoposter",
   "display_name": "S4L",
-  "version": "1.7.7-rc.5",
+  "version": "1.7.7-rc.7",
   "description": "Draft, review, approve, and autopilot X/Twitter posts.",
   "long_description": "## **⚠️ The disclaimer above is generic Claude boilerplate.** Anthropic shows the same warning on every plugin regardless of what it does; any plugin has the same level of access as any app you download from the internet.\n\nS4L is an open source product developed by Mediar.ai Incorporated, a VC-backed San Francisco-based startup.\n\nTo get started:\n\n1\\. Copy this prompt: **Set me up on S4L plugin end to end**\n\n2\\. Quit with CMD+Q, reopen Claude, paste into a new chat.\n\nWhat happens next:\n\n* About every 5 minutes S4L scans X for posts that match your topics and drafts replies in your voice.\n* Drafts show up as review cards, usually the first within a few minutes. Nothing is posted automatically; you approve each one.\n* Posting autopilot stays off until you explicitly turn it on.",
   "author": {

package/mcp/menubar/s4l_menubar.py

@@ -27,6 +27,20 @@ import sys
 import tempfile
 import threading
 import time
+import warnings
+
+# PyObjC emits an ObjCPointerWarning for every CGColor() handed to a CALayer
+# (s4l_card's tile styling). The message embeds the pointer address, so the
+# warnings module's once-per-location dedup never matches and a single card
+# render sprayed ~100 lines/sec into menubar.err.log (13k+ lines by the
+# 2026-08-02 lag incident). The pointers are handled correctly; silence the
+# category before any card module loads.
+try:
+    import objc
+
+    warnings.filterwarnings("ignore", category=objc.ObjCPointerWarning)
+except Exception:
+    pass
 
 # --- stderr timestamp wrapper -------------------------------------------------
 # menubar.err.log (this process's stderr, redirected by the launchd plist) has
@@ -715,8 +729,28 @@ class S4LMenuBar(rumps.App):
         self._tick_stats_at = 0.0
         self._reloc_timer = rumps.Timer(self._maybe_relocate_tasks, 90)
         self._reloc_timer.start()
+        # Store compaction: archive heavy fields off settled candidates so the
+        # review-queue file (and every 1s/5s poll that parses it) stays small.
+        # Boot pass runs off the main thread — the first pass after an
+        # un-compacted stretch can chew through tens of MB.
+        self._compacted_at = 0.0
+        self._compact_store_async()
         self._tick(None)
 
+    def _compact_store_async(self):
+        self._compacted_at = time.time()
+
+        def run():
+            try:
+                n = st.compact_store()
+                if n:
+                    sys.stderr.write(f"[s4l-menubar] store compaction archived {n} candidate(s)\n")
+                    sys.stderr.flush()
+            except Exception:
+                pass
+
+        threading.Thread(target=run, daemon=True, name="s4l-compact-store").start()
+
     # ---- side effects -----------------------------------------------------
     def _open_claude(self, _=None):
         subprocess.run(["open", "-a", CLAUDE_APP], capture_output=True,
@@ -3009,6 +3043,10 @@ class S4LMenuBar(rumps.App):
                     sys.stderr.flush()
             except Exception:
                 pass
+        # Hourly store compaction keeps the review-queue file small as posts
+        # settle (boot already ran one pass; see _compact_store_async).
+        if now_rc - self._compacted_at >= 3600:
+            self._compact_store_async()
 
     # ---- draft review pop-ups ---------------------------------------------
     def _posting_activity_label_locked(self):

package/mcp/menubar/s4l_state.py

@@ -721,7 +721,9 @@ def _store_update(mutate):
                         data = {"candidates": []}
                     rv = mutate(data)
                     tmp = f"{sp}.tmp.{os.getpid()}"
-                    Path(tmp).write_text(json.dumps(data, indent=2))
+                    # Compact separators: this file reached 65 MB with indent=2
+                    # (2026-08-02 lag incident) and every reader pays the parse.
+                    Path(tmp).write_text(json.dumps(data, separators=(",", ":")))
                     os.replace(tmp, sp)
                     return rv
                 finally:
@@ -779,6 +781,97 @@ def candidate_state(c):
     return "awaiting_review"
 
 
+# Draft-time payloads that are only needed while a candidate can still POST
+# (awaiting_review: card rendering; approved/post_failed: the drain path
+# rebuilds a mini-plan from reddit_decision/reddit_plan_meta). Once a candidate
+# is posted or terminal these blobs are dead weight — and they dominated the
+# 65 MB store that caused the 2026-08-02 UI lag (single reddit candidates
+# carried ~180 KB). Compaction moves them to an append-only sidecar so the
+# append-forever ledger keeps every byte while the hot file stays small.
+HEAVY_ARCHIVE_FIELDS = ("reddit_plan_meta", "reddit_decision", "thread_selftext")
+
+# Audit-only blobs inside reddit_plan_meta.style_assignment, stamped by
+# engagement_styles.pick_style_for_post "for audit" and copied onto EVERY
+# reddit candidate by merge_review_queue. distribution_snapshot alone was
+# ~107 KB per candidate (7.6 MB across one pending backlog). Nothing on the
+# card-render or posting path reads them (post_reddit.py --phase post uses
+# only .style/.mode; index.ts forwards the dict opaquely), so they are safe
+# to archive off candidates in ANY state, pending included.
+STYLE_AUDIT_FIELDS = ("distribution_snapshot", "reference_styles")
+
+ARCHIVE_STORE = "review-queue-archive.jsonl"
+
+
+def compact_store():
+    """Archive heavy payloads out of the hot store into review-queue-archive
+    .jsonl, then rewrite the store compactly. Two passes over the candidates:
+    HEAVY_ARCHIVE_FIELDS come off SETTLED (posted/terminal) candidates only —
+    approved/post_failed rows still need reddit_decision/reddit_plan_meta for
+    a (re-)approval drain — while STYLE_AUDIT_FIELDS come off every candidate.
+    Runs under the same store lock as every other python writer. The archive
+    lines land (flush+fsync) BEFORE any field is stripped, so no data is ever
+    lost — a crash in between only risks a duplicate archive line, never a
+    missing one. Returns the number of candidates compacted (0 when there was
+    nothing to do), None on failure."""
+
+    ap = str(Path(state_dir()) / ARCHIVE_STORE)
+
+    def mutate(data):
+        records = []  # (candidate, archived-fields dict, strip callback)
+        for c in data.get("candidates") or []:
+            fields = {}
+            settled = candidate_state(c) in ("posted", "terminal")
+            if settled:
+                fields.update(
+                    {k: c[k] for k in HEAVY_ARCHIVE_FIELDS if c.get(k)}
+                )
+            sa = (c.get("reddit_plan_meta") or {}).get("style_assignment")
+            audit = (
+                {}
+                if (settled and "reddit_plan_meta" in fields) or not isinstance(sa, dict)
+                else {k: sa[k] for k in STYLE_AUDIT_FIELDS if sa.get(k)}
+            )
+            if audit:
+                fields["style_assignment_audit"] = audit
+
+            def strip(c=c, settled=settled, sa=sa, audit=audit):
+                if settled:
+                    for k in HEAVY_ARCHIVE_FIELDS:
+                        c.pop(k, None)
+                for k in audit:
+                    sa.pop(k, None)
+
+            if fields:
+                records.append((c, fields, strip))
+        if not records:
+            return 0
+        with open(ap, "a") as f:
+            for c, fields, _ in records:
+                f.write(
+                    json.dumps(
+                        {
+                            "candidate_id": c.get("candidate_id"),
+                            "our_url": c.get("our_url"),
+                            "thread_url": c.get("thread_url"),
+                            "archived_at": time_iso(),
+                            "fields": fields,
+                        },
+                        separators=(",", ":"),
+                    )
+                    + "\n"
+                )
+            f.flush()
+            os.fsync(f.fileno())
+        for c, fields, strip in records:
+            strip()
+            c["archived_fields"] = sorted(
+                set(c.get("archived_fields") or []) | set(fields)
+            )
+        return len(records)
+
+    return _store_update(mutate)
+
+
 def store_stamp_decision(batch, decision):
     """Write a card decision INTO the store the instant the user clicks. This is
     the durable record (the old approved-queue.json ledger is no longer
@@ -1001,13 +1094,22 @@ def store_reconcile_decisions(batch, decisions):
     return fixed
 
 
+# (path, mtime_ns, size) -> posted count. review_queue_posted_count() is on the
+# menubar's 1-second activity poll; without this cache that poll re-parsed the
+# whole store every tick, which saturated the AppKit main thread once the file
+# grew (2026-08-02: 65 MB, 98% CPU, multi-second card lag).
+_posted_count_cache = {"key": None, "count": None}
+
+
 def review_queue_posted_count():
     """Posts that have LANDED in the review-queue plan — the durable, cross-process
     truth. Independent of the menu bar's in-memory burst queue (which dies on a
     restart) and of WHICH process is posting (the menu bar worker, the autopilot,
     or a host agent draining via approve_drafts). Returns the posted count, or None
     when the plan can't be read. Drives the menu-bar posting indicator so progress
-    stays visible regardless of how the drain is driven."""
+    stays visible regardless of how the drain is driven. Cached on the plan
+    file's (mtime, size): the caller polls every second, the file changes only
+    when a writer lands."""
     plan_path = None
     req = read_review_request()
     if req:
@@ -1016,11 +1118,22 @@ def review_queue_posted_count():
         plan_path = store_path()
     if not Path(plan_path).exists():
         plan_path = "/tmp/twitter_cycle_plan_review-queue.json"
+    try:
+        stt = os.stat(plan_path)
+        cache_key = (os.path.realpath(plan_path), stt.st_mtime_ns, stt.st_size)
+    except OSError:
+        cache_key = None
+    if cache_key is not None and _posted_count_cache["key"] == cache_key:
+        return _posted_count_cache["count"]
     plan = read_plan(plan_path)
     cands = (plan or {}).get("candidates")
-    if not cands:
-        return None
-    return sum(1 for c in cands if candidate_state(c) == "posted")
+    count = None if not cands else sum(
+        1 for c in cands if candidate_state(c) == "posted"
+    )
+    if cache_key is not None:
+        _posted_count_cache["key"] = cache_key
+        _posted_count_cache["count"] = count
+    return count
 
 
 def review_drafts(plan, batch="review-queue"):

package/mcp/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@m13v/s4l-mcp",
-  "version": "1.7.7-rc.5",
+  "version": "1.7.7-rc.7",
   "private": true,
   "description": "Desktop MCP client for social-autoposter (X/Twitter rail): manual draft/review/approve loop, autopilot control, and stats. Thin wrapper over the existing pipeline scripts.",
   "license": "MIT",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@m13v/s4l",
-  "version": "1.7.7-rc.5",
+  "version": "1.7.7-rc.7",
   "description": "Automated social posting pipeline for Reddit, X/Twitter, LinkedIn, and Moltbook. Install as a Claude Code agent skill.",
   "bin": {
     "social-autoposter": "bin/cli.js",

package/scripts/claude_job.py

@@ -76,6 +76,12 @@ TAG_TO_TYPE = {
     # Python and inlined; no Bash tools), mirroring the twitter Phase 2b
     # tool-free conversion, so it rides the same universal worker.
     "post-reddit-draft": "reddit-draft",
+    # Context mining (2026-07-31): scripts/context_mining.py inlines compacted
+    # transcripts + the corpus into one pure text->JSON turn. Execution
+    # guidance lives at the top of the prompt itself (not only in
+    # TYPE_TO_WORKER_NOTES) because an installed-package worker claims with
+    # ITS copy of this map, which may predate this type.
+    "context-mining": "context-mining",
 }
 
 # queue type -> (activity state, label) the menu bar shows while the job is in
@@ -92,6 +98,7 @@ TYPE_TO_ACTIVITY = {
     # must stay concise (user rule 2026-07-14); the platform shows on the
     # review card itself, not in the tray.
     "reddit-draft": ("drafting", "draft"),
+    "context-mining": ("learning", "mining context"),
 }
 
 # queue type -> execution notes PREPENDED to the prompt sidecar at claim time.
@@ -140,6 +147,13 @@ TYPE_TO_WORKER_NOTES = {
         "absent from posts (add a rejects entry only for the structural "
         "false-positive cases the prompt describes)."
     ),
+    "context-mining": (
+        "WORKER EXECUTION NOTES (queue metadata; follow while executing the "
+        "prompt below): single-turn, tool-free job. Everything you need is "
+        "inlined in the prompt (transcripts, corpus, prior decisions). Never "
+        "use tools. Read, then submit ONE result object matching the schema: "
+        "{\"proposals\": [...]}. An empty proposals list is a valid answer."
+    ),
 }
 
 

package/scripts/context_mining.py

@@ -0,0 +1,664 @@
+#!/usr/bin/env python3
+"""
+context_mining.py -- PROTOTYPE: mine the user's own Claude conversation
+transcripts for insights worth adding to a personal context corpus, using the
+existing claude_job.py file queue as the LLM lane (design agreed 2026-07-31).
+
+Flow (test-run shape; toggle/watermark/cards come later):
+  gather  -> deterministically collect recent human-interactive sessions,
+             compact each FULL transcript (user prose kept, tool plumbing
+             dropped), inline the current corpus (numbered lines) plus the
+             already-considered ledger, enqueue ONE pure text->JSON job via
+             run_claude.sh (tag: context-mining), block for the result, and
+             write proposals to pending.json.
+  review  -> print pending proposals.
+  approve -> apply a proposal to corpus.md (append or replace a line) and
+             record it in the considered ledger.
+  skip    -> record the proposal as skipped so it is never re-pitched.
+
+State lives under <state_dir>/context-mining/ where state_dir is
+$S4L_STATE_DIR or ~/.social-autoposter-mcp (same convention as the queue):
+  corpus.md         the corpus; every non-blank line is one numbered memory
+  pending.json      proposals awaiting a human decision
+  considered.jsonl  append-only ledger of every approve/skip decision
+
+Sources walked (same two stores as extract_user_messages_today.py):
+  ~/.claude/projects/*/*.jsonl                      Claude Code (CLI + desktop)
+  ~/Library/Application Support/Claude/
+      local-agent-mode-sessions/**/.claude/projects/**/*.jsonl   Cowork
+
+Eligibility: a session must contain >= MIN_HUMAN_MSGS user messages classified
+HUMAN (typed prose, not command echoes / task notifications / reminders), and
+its cwd must not be an S4L worker sandbox. S4L's own automation transcripts are
+never mined -- that would be a feedback loop.
+
+Usage:
+  python3 scripts/context_mining.py gather --days 3
+  python3 scripts/context_mining.py gather --days 3 --dry-run   # build, no enqueue
+  python3 scripts/context_mining.py review
+  python3 scripts/context_mining.py approve cm-1a2b3c4d [id...]
+  python3 scripts/context_mining.py skip cm-1a2b3c4d [id...]
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+HOME = Path.home()
+REPO_DIR = Path(__file__).resolve().parent.parent
+
+CLAUDE_CODE_PROJECTS_ROOT = HOME / ".claude" / "projects"
+COWORK_SESSIONS_ROOT = (
+    HOME / "Library" / "Application Support" / "Claude" / "local-agent-mode-sessions"
+)
+
+
+def state_dir() -> Path:
+    base = os.environ.get("S4L_STATE_DIR") or str(HOME / ".social-autoposter-mcp")
+    d = Path(base) / "context-mining"
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+
+def corpus_path() -> Path:
+    return state_dir() / "corpus.md"
+
+
+def pending_path() -> Path:
+    return state_dir() / "pending.json"
+
+
+def ledger_path() -> Path:
+    return state_dir() / "considered.jsonl"
+
+
+# ---------------------------------------------------------------- gathering
+
+MIN_HUMAN_MSGS = 2
+PER_USER_MSG_CAP = 2500
+PER_ASSISTANT_MSG_CAP = 700
+PER_SESSION_CAP = 14_000
+DEFAULT_TOTAL_BUDGET = 80_000
+
+# cwd substrings that mark a session as S4L's own automation, never mined
+EXCLUDED_CWD_MARKERS = (".s4l-worker", ".saps-worker")
+
+
+def _classify(text: str) -> str:
+    """HUMAN vs harness-injected user-role messages (same taxonomy as
+    extract_user_messages_today.py)."""
+    stripped = text.lstrip()
+    if stripped.startswith("<task-notification>"):
+        return "TASK_NOTIF"
+    if stripped.startswith("<command-name>") or stripped.startswith("<command-message>"):
+        return "COMMAND"
+    if stripped.startswith("<local-command-stdout>") or stripped.startswith(
+        "<local-command-stderr>"
+    ):
+        return "CMD_STDOUT"
+    if "<<autonomous-loop" in stripped or stripped.startswith("<loop-"):
+        return "SCHED_WAKE"
+    if stripped.startswith("<user-prompt-submit-hook>"):
+        return "HOOK"
+    if stripped.startswith("<system-reminder>"):
+        without = re.sub(
+            r"<system-reminder>.*?</system-reminder>", "", stripped, flags=re.DOTALL
+        ).strip()
+        if not without:
+            return "SYS_REMIND"
+    return "HUMAN"
+
+
+def _strip_reminders(text: str) -> str:
+    return re.sub(r"<system-reminder>.*?</system-reminder>", "", text, flags=re.DOTALL).strip()
+
+
+def _text_blocks(content) -> str | None:
+    """Textual payload of a message; None for tool_result plumbing."""
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        texts = []
+        for block in content:
+            if not isinstance(block, dict):
+                continue
+            btype = block.get("type")
+            if btype == "tool_result":
+                return None
+            if btype == "text" and isinstance(block.get("text"), str):
+                texts.append(block["text"])
+        if texts:
+            return "\n".join(texts)
+    return None
+
+
+def _iter_transcript_files():
+    if CLAUDE_CODE_PROJECTS_ROOT.is_dir():
+        for p in CLAUDE_CODE_PROJECTS_ROOT.glob("*/*.jsonl"):
+            yield "claude_code", p
+    if COWORK_SESSIONS_ROOT.is_dir():
+        for p in COWORK_SESSIONS_ROOT.glob("**/.claude/projects/**/*.jsonl"):
+            yield "cowork", p
+
+
+def _compact_session(path: Path) -> dict | None:
+    """Read one full transcript and compact it to the conversation arc:
+    HUMAN user messages (near-full) + assistant prose (trimmed). Returns None
+    if the session is not human-interactive or is S4L automation."""
+    cwd = None
+    first_ts = last_ts = None
+    human_count = 0
+    turns: list[str] = []
+    try:
+        with path.open("r", encoding="utf-8", errors="replace") as f:
+            for line in f:
+                try:
+                    d = json.loads(line)
+                except Exception:
+                    continue
+                ts = d.get("timestamp")
+                if ts:
+                    first_ts = first_ts or ts
+                    last_ts = ts
+                if cwd is None and d.get("cwd"):
+                    cwd = d["cwd"]
+                if d.get("isSidechain"):
+                    continue
+                msg = d.get("message") or {}
+                role = msg.get("role")
+                if d.get("type") == "user" and role == "user":
+                    text = _text_blocks(msg.get("content"))
+                    if text is None:
+                        continue
+                    text = text.strip()
+                    if not text or _classify(text) != "HUMAN":
+                        continue
+                    text = _strip_reminders(text)
+                    if not text:
+                        continue
+                    human_count += 1
+                    turns.append("USER: " + text[:PER_USER_MSG_CAP])
+                elif d.get("type") == "assistant" and role == "assistant":
+                    text = _text_blocks(msg.get("content"))
+                    if not text:
+                        continue
+                    text = text.strip()
+                    if len(text) < 40:  # skip one-liner tool narration
+                        continue
+                    turns.append("ASSISTANT: " + text[:PER_ASSISTANT_MSG_CAP])
+    except OSError:
+        return None
+
+    if human_count < MIN_HUMAN_MSGS:
+        return None
+    if cwd and any(m in cwd for m in EXCLUDED_CWD_MARKERS):
+        return None
+
+    body = "\n\n".join(turns)
+    if len(body) > PER_SESSION_CAP:
+        # keep the head and the tail; the middle is elided
+        half = PER_SESSION_CAP // 2
+        body = body[:half] + "\n\n[... middle of session elided ...]\n\n" + body[-half:]
+    return {
+        "session_id": path.stem,
+        "path": str(path),
+        "cwd": cwd or "?",
+        "first_ts": first_ts or "?",
+        "last_ts": last_ts or "?",
+        "human_msgs": human_count,
+        "body": body,
+    }
+
+
+def gather_sessions(days: int) -> list[dict]:
+    """ALL eligible human sessions within the window, most recent first."""
+    cutoff = time.time() - days * 86400
+    candidates = []
+    for _source, p in _iter_transcript_files():
+        try:
+            mtime = p.stat().st_mtime
+        except OSError:
+            continue
+        if mtime < cutoff:
+            continue
+        candidates.append((mtime, p))
+    candidates.sort(reverse=True)
+
+    kept: list[dict] = []
+    seen_ids: set[str] = set()
+    for _mtime, p in candidates:
+        if p.stem in seen_ids:
+            continue
+        sess = _compact_session(p)
+        if sess is None:
+            continue
+        seen_ids.add(p.stem)
+        kept.append(sess)
+    return kept
+
+
+def chunk_by_budget(sessions: list[dict], budget: int) -> list[list[dict]]:
+    """Split into batches whose combined body size fits the budget. A batch
+    always takes at least one session, so oversized sessions still ship."""
+    batches: list[list[dict]] = []
+    cur: list[dict] = []
+    used = 0
+    for s in sessions:
+        cost = len(s["body"])
+        if cur and used + cost > budget:
+            batches.append(cur)
+            cur, used = [], 0
+        cur.append(s)
+        used += cost
+    if cur:
+        batches.append(cur)
+    return batches
+
+
+# ------------------------------------------------------------------- corpus
+
+
+def read_corpus_lines() -> list[str]:
+    if not corpus_path().exists():
+        return []
+    return [ln.rstrip("\n") for ln in corpus_path().read_text().splitlines() if ln.strip()]
+
+
+def write_corpus_lines(lines: list[str]) -> None:
+    corpus_path().write_text("\n".join(lines) + ("\n" if lines else ""))
+
+
+def read_ledger() -> list[dict]:
+    if not ledger_path().exists():
+        return []
+    out = []
+    for ln in ledger_path().read_text().splitlines():
+        try:
+            out.append(json.loads(ln))
+        except Exception:
+            continue
+    return out
+
+
+def append_ledger(rec: dict) -> None:
+    with ledger_path().open("a") as f:
+        f.write(json.dumps(rec, ensure_ascii=False) + "\n")
+
+
+# ------------------------------------------------------------------- prompt
+
+SCHEMA = {
+    "type": "object",
+    "required": ["proposals"],
+    "properties": {
+        "proposals": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "required": [
+                    "action",
+                    "title",
+                    "story",
+                    "conclusion",
+                    "source_session",
+                    "source_date",
+                ],
+                "properties": {
+                    "action": {"type": "string", "enum": ["add", "revise"]},
+                    "revises_line": {"type": ["integer", "null"]},
+                    "title": {"type": "string"},
+                    "story": {"type": "string"},
+                    "conclusion": {"type": "string"},
+                    "source_session": {"type": "string"},
+                    "source_date": {"type": "string"},
+                },
+            },
+        }
+    },
+}
+
+
+def corpus_entry(p: dict) -> str:
+    """Render one approved proposal as a single corpus line (numbering is
+    line-based, so the entry must not contain newlines)."""
+    story = " ".join((p.get("story") or "").split())
+    concl = " ".join((p.get("conclusion") or "").split())
+    title = " ".join((p.get("title") or "").split())
+    return f"{title}: {story} Conclusion: {concl}"
+
+
+def build_prompt(sessions: list[dict], pending_props: list[dict] | None = None) -> str:
+    corpus_lines = read_corpus_lines()
+    corpus_block = (
+        "\n".join(f"[{i + 1}] {ln}" for i, ln in enumerate(corpus_lines))
+        if corpus_lines
+        else "(the corpus is currently empty)"
+    )
+    def _gist(r: dict) -> str:
+        # works for both the old one-line shape (text) and the story shape
+        return r.get("text") or f"{r.get('title', '')}: {r.get('conclusion', '')}"
+
+    considered = [
+        {"status": r.get("status"), "gist": _gist(r)} for r in read_ledger()[-60:]
+    ] + [{"status": "pending", "gist": _gist(p)} for p in (pending_props or [])]
+    considered_block = (
+        "\n".join(f"- ({r['status']}) {r['gist'][:200]}" for r in considered)
+        if considered
+        else "(nothing has been considered yet)"
+    )
+    transcript_parts = []
+    for s in sessions:
+        transcript_parts.append(
+            f"### Session {s['session_id'][:8]} | {s['first_ts'][:10]} | cwd: {s['cwd']}\n\n{s['body']}"
+        )
+    transcripts_block = "\n\n---\n\n".join(transcript_parts)
+
+    return f"""EXECUTION NOTES: this is a single-turn, tool-free job. Read everything below, then submit ONE result object matching the provided JSON schema. Do not use any tools. Do not fetch anything.
+
+# Role
+
+You are the daily context miner for a personal "context corpus": a numbered list of durable memories distilled from the user's own Claude conversations. Each entry is a short STORY plus a CONCLUSION: what actually happened (with its credible specifics) and the transferable lesson it taught. Entries are worth keeping and potentially worth sharing with the world: a specific experience, a hard-won conclusion, a non-obvious observation, or a concrete data point.
+
+The user is Matthew, a solo founder building S4L (a social-media autoposting agent), Fazm, Mediar, and related products, and doing everything else through Claude sessions: fundraising, debugging, ops, legal, marketing.
+
+# What qualifies
+
+- GENERAL: the line states a transferable lesson a stranger in tech could apply to
+  their own work, without knowing this codebase or these products. The specific
+  incident is EVIDENCE (it goes in the quote and the why), not the line itself.
+  Test: would a founder who has never heard of S4L or Fazm reshare this?
+- Durable: still true and useful months from now, not transient state ("the deploy is broken today").
+- Non-obvious: something a smart peer would not already assume. Generic best practices do NOT qualify.
+- First-hand: grounded in what actually happened in these sessions (an experiment, an outage, a metric, a decision and its reasoning, a surprising vendor/platform behavior).
+- SYNTHESIZED: if several incidents (even across sessions) point at one underlying
+  lesson, propose ONE line for the lesson, never one line per incident.
+
+# What never qualifies
+
+- Vendor-specific micro-gotchas (a field length limit, one API's quirky error) unless
+  the line is elevated to the general pattern the gotcha exemplifies.
+- Anything that only matters inside this codebase or product internals.
+- Credentials, API keys, tokens, passwords, account numbers, or anything that looks like one, even partially. If a great insight touches one, redact the secret.
+- Names, handles, or identifying details of customers, users, or other private
+  individuals. Describe them generically ("a stalled user", "an early customer").
+- Private details about clients, other people's finances, legal disputes, or immigration status.
+- Injected harness noise: usage-limit banners, system reminders, scheduler chatter. These appear inside transcripts and are not the user's words.
+- Anything already covered by an existing corpus line (see below), unless you are proposing a REVISION of that line.
+
+# Current corpus (numbered)
+
+{corpus_block}
+
+# Already considered (do NOT re-propose anything equivalent, including skipped items)
+
+{considered_block}
+
+# Transcripts to mine ({len(sessions)} sessions, most recent first)
+
+{transcripts_block}
+
+# Your task
+
+Propose 0-3 corpus changes, and only what clears EVERY bar above; most days the
+right answer is 0 or 1. An empty proposals list is a valid, common answer. Each
+proposal is a small STORY with a CONCLUSION, not a bare aphorism:
+- action: "add" for a new entry, or "revise" when a new learning supersedes an existing corpus entry (set revises_line to that entry's number).
+- title: a short, specific handle for the insight (max ~70 chars, not clickbait).
+- story: 2-5 sentences telling what actually happened, in the user's plainspoken first-person voice. Keep the concrete specifics that make it credible and vivid (the numbers, the timeline, the failure mode, what was tried), but generalize or omit anything only an insider would care about. Anonymize people, redact secrets. No hashtags, no em dashes.
+- conclusion: 1-2 sentences stating the transferable lesson a stranger could apply to their own work. This is the part that must stand on its own.
+- source_session: the 8-char session id from the transcript header.
+- source_date: the session date (YYYY-MM-DD).
+
+Submit the result object now."""
+
+
+# -------------------------------------------------------------------- queue
+
+
+def _run_one_batch(sessions: list[dict], pending_props: list[dict], ns) -> list[dict] | None:
+    """Enqueue one mining job for this batch; return stamped proposals, or None
+    on failure (caller stops the loop and keeps what it has)."""
+    prompt = build_prompt(sessions, pending_props)
+    schema_file = state_dir() / "schema.json"
+    schema_file.write_text(json.dumps(SCHEMA))
+
+    env = dict(os.environ)
+    env["S4L_REPO_DIR"] = str(REPO_DIR)  # route through THIS repo's claude_job.py
+    cmd = [
+        "bash",
+        str(REPO_DIR / "scripts" / "run_claude.sh"),
+        "context-mining",
+        "--json-schema",
+        str(schema_file),
+        "-p",
+    ]
+    try:
+        proc = subprocess.run(
+            cmd,
+            input=prompt,
+            capture_output=True,
+            text=True,
+            env=env,
+            timeout=ns.timeout + 120,
+        )
+    except subprocess.TimeoutExpired:
+        print("[gather] hard timeout waiting for the queue result", file=sys.stderr)
+        return None
+
+    if proc.returncode == 79:
+        print(
+            "[gather] queue timed out (rc=79): no worker claimed the job. Is Claude "
+            "Desktop open and the s4l-worker task firing?",
+            file=sys.stderr,
+        )
+        return None
+    if proc.returncode != 0:
+        print(f"[gather] provider failed rc={proc.returncode}", file=sys.stderr)
+        sys.stderr.write((proc.stderr or "")[-2000:] + "\n")
+        return None
+
+    try:
+        envelope = json.loads(proc.stdout[proc.stdout.index("{"):])
+        obj = envelope.get("structured_output")
+        if obj is None:
+            obj = json.loads(envelope.get("result") or "{}")
+    except Exception as e:
+        print(f"[gather] could not parse result envelope: {e}", file=sys.stderr)
+        sys.stderr.write((proc.stdout or "")[-2000:] + "\n")
+        return None
+
+    stamped = []
+    for p in obj.get("proposals") or []:
+        key = (p.get("title") or p.get("text") or "") + p.get("source_session", "")
+        pid = "cm-" + hashlib.sha1(key.encode()).hexdigest()[:8]
+        p["id"] = pid
+        p["mined_at"] = datetime.now(timezone.utc).isoformat(timespec="seconds")
+        stamped.append(p)
+    return stamped
+
+
+def run_gather(ns) -> int:
+    sessions = gather_sessions(ns.days)
+    if not sessions:
+        print(json.dumps({"ok": False, "reason": "no_eligible_sessions", "days": ns.days}))
+        return 1
+    batches = chunk_by_budget(sessions, ns.budget)
+    total_chars = sum(len(s["body"]) for s in sessions)
+    print(
+        f"[gather] {len(sessions)} sessions in window ({ns.days}d), "
+        f"{total_chars:,} chars -> {len(batches)} batches (budget {ns.budget:,})",
+        file=sys.stderr,
+    )
+
+    if ns.dry_run:
+        out = state_dir() / "last_prompt.txt"
+        out.write_text(build_prompt(batches[0], []))
+        for i, b in enumerate(batches, 1):
+            ids = ", ".join(s["session_id"][:8] for s in b)
+            print(f"[gather]   batch {i}/{len(batches)}: {len(b)} sessions ({ids})", file=sys.stderr)
+        print(f"[gather] dry-run: batch-1 prompt written to {out}, nothing enqueued", file=sys.stderr)
+        return 0
+
+    # carry over any still-pending proposals so batches dedup against them too
+    all_props: list[dict] = _load_pending()
+    known_ids = {p["id"] for p in all_props}
+    failed = 0
+    for i, batch in enumerate(batches, 1):
+        ids = ", ".join(s["session_id"][:8] for s in batch)
+        print(
+            f"[gather] batch {i}/{len(batches)}: {len(batch)} sessions "
+            f"({sum(len(s['body']) for s in batch):,} chars) [{ids}]",
+            file=sys.stderr,
+        )
+        stamped = _run_one_batch(batch, all_props, ns)
+        if stamped is None:
+            failed += 1
+            print(f"[gather] batch {i} failed; stopping loop, keeping prior results", file=sys.stderr)
+            break
+        fresh = [p for p in stamped if p["id"] not in known_ids]
+        known_ids.update(p["id"] for p in fresh)
+        all_props.extend(fresh)
+        pending_path().write_text(
+            json.dumps({"proposals": all_props}, indent=2, ensure_ascii=False)
+        )
+        print(f"[gather] batch {i} done: +{len(fresh)} proposals ({len(all_props)} total)", file=sys.stderr)
+
+    print(
+        f"[gather] run complete: {len(batches) - failed}/{len(batches)} batches, "
+        f"{len(all_props)} pending proposals -> {pending_path()}",
+        file=sys.stderr,
+    )
+    cmd_review(None)
+    return 0 if not failed else 1
+
+
+# ------------------------------------------------------------------- review
+
+
+def _load_pending() -> list[dict]:
+    if not pending_path().exists():
+        return []
+    try:
+        return json.loads(pending_path().read_text()).get("proposals", [])
+    except Exception:
+        return []
+
+
+def cmd_review(_ns) -> int:
+    props = _load_pending()
+    if not props:
+        print("no pending proposals")
+        return 0
+    corpus_lines = read_corpus_lines()
+    for p in props:
+        print("=" * 72)
+        head = p["id"]
+        if p.get("action") == "revise" and p.get("revises_line"):
+            n = p["revises_line"]
+            old = corpus_lines[n - 1] if 0 < n <= len(corpus_lines) else "(missing line)"
+            head += f"  REVISES [{n}]: {old[:120]}"
+        else:
+            head += "  ADD"
+        print(head)
+        if p.get("title"):
+            print(f"  title: {p['title']}")
+            print(f"  story: {p.get('story', '')}")
+            print(f"  concl: {p.get('conclusion', '')}")
+        else:  # legacy one-line shape
+            print(f"  text : {p.get('text', '')}")
+            print(f"  why  : {p.get('why', '')}")
+        print(f"  src  : {p.get('source_session', '?')} @ {p.get('source_date', '?')}")
+    print("=" * 72)
+    print(f"{len(props)} pending. approve/skip with: context_mining.py approve <id...>")
+    return 0
+
+
+def _decide(ids: list[str], status: str) -> int:
+    props = _load_pending()
+    if not props:
+        print("no pending proposals")
+        return 1
+    if ids == ["all"]:
+        ids = [p["id"] for p in props]
+    by_id = {p["id"]: p for p in props}
+    corpus_lines = read_corpus_lines()
+    done = []
+    for pid in ids:
+        p = by_id.get(pid)
+        if not p:
+            print(f"unknown id: {pid}")
+            continue
+        if status == "approved":
+            entry = corpus_entry(p) if p.get("title") else p.get("text", "")
+            if p.get("action") == "revise" and p.get("revises_line"):
+                n = p["revises_line"]
+                if 0 < n <= len(corpus_lines):
+                    corpus_lines[n - 1] = entry
+                else:
+                    corpus_lines.append(entry)
+            else:
+                corpus_lines.append(entry)
+        append_ledger(
+            {
+                "id": pid,
+                "status": status,
+                "title": p.get("title", ""),
+                "conclusion": p.get("conclusion", ""),
+                "text": p.get("text", ""),
+                "source_session": p.get("source_session"),
+                "decided_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+            }
+        )
+        done.append(pid)
+    if status == "approved":
+        write_corpus_lines(corpus_lines)
+    remaining = [p for p in props if p["id"] not in set(done)]
+    pending_path().write_text(
+        json.dumps({"proposals": remaining}, indent=2, ensure_ascii=False)
+    )
+    print(f"{status}: {', '.join(done) if done else 'nothing'}; {len(remaining)} still pending")
+    if status == "approved":
+        print(f"corpus now has {len(corpus_lines)} lines: {corpus_path()}")
+    return 0
+
+
+# ---------------------------------------------------------------------- CLI
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="mine Claude transcripts into a context corpus")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    g = sub.add_parser("gather", help="collect sessions, enqueue one mining job, save proposals")
+    g.add_argument("--days", type=int, default=3)
+    g.add_argument("--budget", type=int, default=DEFAULT_TOTAL_BUDGET)
+    g.add_argument("--timeout", type=int, default=1800)
+    g.add_argument("--dry-run", action="store_true", help="build the prompt, don't enqueue")
+    g.set_defaults(func=run_gather)
+
+    r = sub.add_parser("review", help="print pending proposals")
+    r.set_defaults(func=cmd_review)
+
+    a = sub.add_parser("approve", help="apply proposal(s) to the corpus")
+    a.add_argument("ids", nargs="+")
+    a.set_defaults(func=lambda ns: _decide(ns.ids, "approved"))
+
+    s = sub.add_parser("skip", help="reject proposal(s), never re-pitched")
+    s.add_argument("ids", nargs="+")
+    s.set_defaults(func=lambda ns: _decide(ns.ids, "skipped"))
+
+    ns = ap.parse_args()
+    return ns.func(ns)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/merge_review_queue.py

@@ -101,7 +101,9 @@ def _atomic_write(path: str, obj) -> None:
     os.makedirs(os.path.dirname(path), exist_ok=True)
     tmp = f"{path}.tmp.{os.getpid()}"
     with open(tmp, "w") as f:
-        json.dump(obj, f, indent=2)
+        # Compact separators: the review store reached 65 MB with indent=2
+        # (2026-08-02 lag incident) and every reader pays the parse.
+        json.dump(obj, f, separators=(",", ":"))
     os.replace(tmp, path)