@m13v/s4l 1.6.197-rc.10

package/scripts/harvest_twitter_following.py

@@ -0,0 +1,243 @@
+#!/usr/bin/env python3
+"""harvest_twitter_following.py — cache the list of accounts WE follow on X.
+
+The Twitter reply pipeline (score_twitter_candidates.py) drops candidate threads
+whose author is someone we already follow. fxtwitter can't supply that edge — it's
+an unauthenticated public API with no concept of "us" — so the follow relationship
+has to be read from our own logged-in session. This script scrapes
+`x.com/<handle>/following` via the harness Chrome (CDP, port 9555, same browser the
+cycle uses) and uploads the set to /api/v1/followed-accounts.
+
+Read-only: ONE navigation + DOM reads + scrolls. No clicks, no posting, no
+/voyager. Runs under the shared "twitter-browser" lock (held by the shell wrapper
+skill/refresh-twitter-following.sh) so it never races a live cycle.
+
+Completeness guard: we only upload when the scroll reached the end of the list
+(the deduped set stopped growing for STABLE_PASSES passes). A partial scrape is
+discarded, never uploaded — otherwise the un-scrolled tail would wrongly age out
+of the server's freshness window.
+
+Usage:
+    python3 scripts/harvest_twitter_following.py            # scrape + upload
+    python3 scripts/harvest_twitter_following.py --dry-run  # scrape + print, no upload
+    python3 scripts/harvest_twitter_following.py --out /tmp/following.json
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import time
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+CDP_URL = os.environ.get("TWITTER_CDP_URL", "http://127.0.0.1:9555").strip()
+PLATFORM = "twitter"
+
+# Scroll/scrape tuning (env-overridable for slow boxes / very large lists).
+STABLE_PASSES = int(os.environ.get("FOLLOW_HARVEST_STABLE_PASSES", "5"))
+MAX_PASSES = int(os.environ.get("FOLLOW_HARVEST_MAX_PASSES", "800"))
+PAUSE_MS = int(os.environ.get("FOLLOW_HARVEST_PAUSE_MS", "900"))
+UPLOAD_CHUNK = int(os.environ.get("FOLLOW_HARVEST_UPLOAD_CHUNK", "1000"))
+
+# Each row on the Following tab is a [data-testid="UserCell"]. The profile link
+# href is exactly `/<screen_name>`; grab the first anchor matching that shape
+# (X handles are 1-15 chars of [A-Za-z0-9_]) that isn't a reserved app route.
+SCRAPE_JS = r"""
+(() => {
+  const RESERVED = new Set(['home','explore','notifications','messages','i',
+    'settings','search','compose','hashtag','intent','login','signup','tos',
+    'privacy','about']);
+  const cells = Array.from(document.querySelectorAll('[data-testid="UserCell"]'));
+  const out = [];
+  for (const c of cells) {
+    let handle = null;
+    for (const a of c.querySelectorAll('a[href^="/"]')) {
+      const m = (a.getAttribute('href') || '').match(/^\/([A-Za-z0-9_]{1,15})$/);
+      if (m && !RESERVED.has(m[1].toLowerCase())) { handle = m[1]; break; }
+    }
+    if (!handle) continue;
+    let name = null;
+    const un = c.querySelector('[data-testid="User-Name"]');
+    if (un) {
+      // User-Name mashes "Display Name@handle…"; the display name is the text
+      // before the first '@'.
+      name = ((un.textContent || '').split('@')[0]).trim().slice(0, 120) || null;
+    }
+    out.push({ screen_name: handle, name });
+  }
+  return JSON.stringify(out);
+})()
+"""
+
+
+def _resolve_handle() -> str:
+    try:
+        import account_resolver
+        h = account_resolver.resolve("twitter")
+        if h:
+            return h.lstrip("@").strip().lower()
+    except Exception as e:
+        print(f"[harvest] account_resolver failed ({e})", file=sys.stderr)
+    # No hardcoded fallback: stamping a default handle here silently harvests /
+    # attributes under the wrong account. Refuse to run instead so the missing
+    # config surfaces. Run connect_x (or `setup_twitter_auth.py resolve-handle`)
+    # to persist accounts.twitter.handle.
+    print("[harvest] no Twitter handle configured (accounts.twitter.handle / "
+          "AUTOPOSTER_TWITTER_HANDLE); refusing to run to avoid wrong-account "
+          "attribution.", file=sys.stderr)
+    sys.exit(1)
+
+
+def _looks_logged_out(url: str) -> bool:
+    u = (url or "").lower()
+    return ("/login" in u) or ("i/flow/login" in u) or ("/account/access" in u)
+
+
+def scrape_following(handle: str) -> tuple[dict, bool]:
+    """Return (handle->name dict, complete). complete=True means the scroll
+    reached the end (set stopped growing) rather than hitting the pass cap."""
+    from playwright.sync_api import sync_playwright
+
+    seen: dict[str, str] = {}
+    complete = False
+    with sync_playwright() as p:
+        browser = p.chromium.connect_over_cdp(CDP_URL)
+        contexts = browser.contexts
+        if not contexts:
+            raise RuntimeError("no browser context on harness Chrome — is it logged in?")
+        context = contexts[0]
+        # Reuse an existing tab (tab hygiene); fall back to a fresh page.
+        page = context.pages[0] if context.pages else context.new_page()
+
+        url = f"https://x.com/{handle}/following"
+        page.goto(url, wait_until="domcontentloaded", timeout=45000)
+        page.wait_for_timeout(2500)
+
+        if _looks_logged_out(page.url):
+            raise RuntimeError(f"session looks logged out (url={page.url})")
+
+        # Wait for at least one row to render before scrolling.
+        try:
+            page.wait_for_selector('[data-testid="UserCell"]', timeout=20000)
+        except Exception:
+            # No cells at all — empty list, protected, or a block page. Treat as
+            # incomplete so we never upload an empty/partial set.
+            print(f"[harvest] no UserCell rendered for @{handle} (url={page.url})",
+                  file=sys.stderr)
+            return seen, False
+
+        last = 0
+        stable = 0
+        for i in range(MAX_PASSES):
+            try:
+                raw = page.evaluate(SCRAPE_JS)
+                rows = json.loads(raw) if isinstance(raw, str) else (raw or [])
+            except Exception as e:
+                print(f"[harvest] evaluate failed on pass {i} ({e})", file=sys.stderr)
+                rows = []
+            for r in rows:
+                sn = (r.get("screen_name") or "").strip().lower()
+                if not sn or sn == handle:  # never list ourselves
+                    continue
+                if sn not in seen:
+                    seen[sn] = r.get("name") or ""
+
+            if len(seen) == last:
+                stable += 1
+                if stable >= STABLE_PASSES:
+                    complete = True
+                    break
+            else:
+                stable = 0
+                last = len(seen)
+
+            page.evaluate(
+                "window.scrollBy(0, Math.round(document.documentElement.clientHeight * 0.85));"
+            )
+            page.wait_for_timeout(PAUSE_MS)
+
+        # Disconnect the CDP client without closing the shared Chrome/tab.
+        try:
+            browser.close()
+        except Exception:
+            pass
+
+    print(
+        f"[harvest] @{handle}: collected {len(seen)} followed handles "
+        f"(complete={complete}, passes_stable={stable}/{STABLE_PASSES})",
+        file=sys.stderr,
+    )
+    return seen, complete
+
+
+def upload(handle: str, seen: dict) -> int:
+    from http_api import api_post
+
+    accounts = [{"handle": h, "name": n} for h, n in seen.items()]
+    posted = 0
+    for i in range(0, len(accounts), UPLOAD_CHUNK):
+        chunk = accounts[i:i + UPLOAD_CHUNK]
+        api_post(
+            "/api/v1/followed-accounts",
+            {
+                "platform": PLATFORM,
+                "our_account": handle,
+                "accounts": chunk,
+                "complete": True,
+            },
+        )
+        posted += len(chunk)
+    return posted
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--dry-run", action="store_true",
+                        help="Scrape and report but do not upload.")
+    parser.add_argument("--out", help="Also write the scraped set to this JSON path.")
+    parser.add_argument("--handle", help="Override the resolved posting handle.")
+    args = parser.parse_args()
+
+    handle = (args.handle or _resolve_handle()).lstrip("@").strip().lower()
+    print(f"[harvest] resolving following list for @{handle} via {CDP_URL}",
+          file=sys.stderr)
+
+    try:
+        seen, complete = scrape_following(handle)
+    except Exception as e:
+        print(f"[harvest] FAILED: {e}", file=sys.stderr)
+        return 1
+
+    if args.out:
+        try:
+            with open(args.out, "w") as fh:
+                json.dump({"handle": handle, "complete": complete,
+                           "accounts": seen}, fh, indent=2)
+            print(f"[harvest] wrote scrape to {args.out}", file=sys.stderr)
+        except OSError as e:
+            print(f"[harvest] could not write {args.out}: {e}", file=sys.stderr)
+
+    if not seen:
+        print("[harvest] scraped 0 handles; nothing to upload.", file=sys.stderr)
+        return 2
+    if not complete:
+        print(
+            f"[harvest] scrape INCOMPLETE (hit {MAX_PASSES}-pass cap at "
+            f"{len(seen)} handles); NOT uploading, to avoid aging out the "
+            f"un-scrolled tail. Re-run will retry.",
+            file=sys.stderr,
+        )
+        return 3
+    if args.dry_run:
+        print(f"[harvest] dry-run: would upload {len(seen)} handles for @{handle}.")
+        return 0
+
+    posted = upload(handle, seen)
+    print(f"[harvest] uploaded {posted} followed handles for @{handle}.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/heartbeat.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# Continuous heartbeat to /api/v1/installations/heartbeat.
+#
+# Independent of any reply traffic: even when github-engage is quiet, this
+# proves the install lane (identity.py + Vercel + Postgres) is round-tripping.
+# A gap in installations.last_seen_at on the server is a leading signal of
+# Vercel outage / DNS / cert / migration drift before any user-facing
+# pipeline notices.
+#
+# Schedule: every 15 minutes via launchd.
+# Logs:     ~/social-autoposter/skill/logs/heartbeat-*.log
+
+set -euo pipefail
+
+# SAPS_->S4L_ env mirror (brand rename 2026-07-03): old plists/tasks still
+# export SAPS_*; new code reads S4L_*. Copy names, never values via eval.
+while IFS='=' read -r _k _; do
+  case "$_k" in SAPS_*) _n="S4L_${_k#SAPS_}"; eval "[ -n \"\${$_n+x}\" ] || export $_n=\"\${$_k}\"";; esac
+done <<EOF_ENV
+$(env | grep '^SAPS_' | cut -d= -f1 | sed 's/$/=/')
+EOF_ENV
+
+REPO_DIR="${REPO_DIR:-$HOME/social-autoposter}"
+BASE_URL="${AUTOPOSTER_API_BASE:-https://s4l.ai}"
+LOG_DIR="$REPO_DIR/skill/logs"
+LOG_FILE="$LOG_DIR/heartbeat.log"
+
+mkdir -p "$LOG_DIR"
+
+ts() { date -u "+%Y-%m-%dT%H:%M:%SZ"; }
+log() { printf '[%s] %s\n' "$(ts)" "$1" >> "$LOG_FILE"; }
+
+PYTHON_BIN="${PYTHON_BIN:-/usr/bin/python3}"
+HDR=$("$PYTHON_BIN" "$REPO_DIR/scripts/identity.py" header 2>>"$LOG_FILE") || {
+  log "FAIL identity.py header non-zero"
+  exit 1
+}
+
+# Attach the S4L autopilot scheduled-task folder state (parity with the .mcpb
+# heartbeat) so the server can tell centrally whether the queue-worker tasks are
+# running from ~/.s4l-worker or are still mislocated. Best-effort: any failure
+# falls back to an empty body so the heartbeat itself never depends on it.
+BODY='{}'
+if ST=$("$PYTHON_BIN" "$REPO_DIR/scripts/scheduled_tasks_snapshot.py" --summary 2>>"$LOG_FILE"); then
+  if [ -n "$ST" ]; then
+    BODY="{\"scheduled_tasks\":$ST}"
+  fi
+fi
+
+# POST so the server can refresh the volatile fields (last_ip, last_seen_at).
+RESP=$(curl -fsS -m 20 \
+  -X POST \
+  -H "X-Installation: $HDR" \
+  -H "content-type: application/json" \
+  -d "$BODY" \
+  -w "\n__HTTP__%{http_code}__%{time_total}s" \
+  "$BASE_URL/api/v1/installations/heartbeat" 2>>"$LOG_FILE") || {
+  log "FAIL curl exit=$?"
+  exit 1
+}
+
+CODE=$(echo "$RESP" | sed -n 's/.*__HTTP__\([0-9]*\)__.*/\1/p')
+DUR=$(echo "$RESP"  | sed -n 's/.*__HTTP__[0-9]*__\(.*\)/\1/p')
+
+if [ "$CODE" = "200" ]; then
+  log "ok 200 ${DUR}"
+else
+  log "FAIL http=$CODE dur=$DUR"
+  exit 1
+fi

package/scripts/history_context.py

@@ -0,0 +1,284 @@
+#!/usr/bin/env python3
+"""
+history_context.py -- PROTOTYPE: consent-gated, cwd-scoped pull of prior Claude
+Code conversation context to prefill S4L onboarding fields.
+
+Design (agreed 2026-07-02):
+  * READ-ONLY. Never mutates Claude session state (no archive, no writes to the
+    desktop app's session store).
+  * Consent-gated. Requires a one-time opt-in. The gate is checked on every call;
+    without it the pull refuses and returns nothing.
+  * cwd-scoped. Only reads sessions whose working dir is under the product's own
+    repos (local_repo + landing_pages.repo from config.json). This is BOTH the
+    relevance filter and the privacy boundary -- other clients' repos are never
+    touched.
+  * Enrichment only. Output is CANDIDATE fields for the user to confirm, never
+    saved silently.
+
+Sources (all local, no network, no approval card in the running session's mode):
+  * ~/.claude/claude_sessions.db          FTS5 index over msg_preview/tool_names/cwd
+  * ~/.claude/projects/<esc-cwd>/*.jsonl  full transcripts, read only on --expand
+
+CLI:
+  python3 scripts/history_context.py --project fazm
+  python3 scripts/history_context.py --project fazm --terms "icp,positioning,voice"
+  python3 scripts/history_context.py --project fazm --expand
+  python3 scripts/history_context.py --optin-status
+  python3 scripts/history_context.py --set-optin yes
+
+Production note: the opt-in flag is stored here in a sidecar (~/.claude/
+s4l_history_optin.json) to avoid mutating the live config.json from a prototype.
+In production this becomes a top-level `history_context_optin` key in config.json
+so it's one global, persisted decision reused across every product onboarding.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+from pathlib import Path
+
+HOME = Path.home()
+# NOTE: ~/.claude/claude_sessions.db is NOT a general index -- on this machine it
+# covered only 5 claude-meter sessions. The authoritative, complete source is the
+# per-cwd transcript tree below, so the pull reads that directly (the same
+# card-free file path used to read an archived session).
+PROJECTS_DIR = HOME / ".claude" / "projects"
+CONFIG_PATH = Path(__file__).resolve().parent.parent / "config.json"
+OPTIN_SIDECAR = HOME / ".claude" / "s4l_history_optin.json"
+
+MAX_FILES_PER_DIR = 80
+MAX_SNIPPETS_PER_SESSION = 6
+PREVIEW_LEN = 220
+
+# ---------------------------------------------------------------- consent gate
+
+
+def optin_status() -> dict:
+    """Return the persisted one-time opt-in. Absent => not yet asked."""
+    if OPTIN_SIDECAR.exists():
+        try:
+            return json.loads(OPTIN_SIDECAR.read_text())
+        except Exception:
+            pass
+    return {"allowed": None, "ts": None}  # None => never asked
+
+
+def set_optin(allowed: bool) -> dict:
+    rec = {"allowed": bool(allowed), "ts": _now_iso()}
+    OPTIN_SIDECAR.write_text(json.dumps(rec, indent=2))
+    return rec
+
+
+def _now_iso() -> str:
+    # Prototype-safe: avoid importing datetime.now semantics into workflow harness.
+    import datetime
+
+    return datetime.datetime.now().replace(microsecond=0).isoformat()
+
+
+# ------------------------------------------------------------- scope resolution
+
+
+def load_config() -> dict:
+    return json.loads(CONFIG_PATH.read_text())
+
+
+def resolve_scope(project_name: str) -> list[str]:
+    """Return absolute repo paths that define this product's cwd scope."""
+    cfg = load_config()
+    proj = next(
+        (p for p in cfg.get("projects", []) if p.get("name") == project_name), None
+    )
+    if not proj:
+        raise SystemExit(f"project '{project_name}' not found in {CONFIG_PATH}")
+    paths = []
+    for key in ("local_repo",):
+        v = proj.get(key)
+        if v:
+            paths.append(os.path.expanduser(v))
+    lp = proj.get("landing_pages") or {}
+    if lp.get("repo"):
+        paths.append(os.path.expanduser(lp["repo"]))
+    # de-dupe, keep order
+    seen, out = set(), []
+    for p in paths:
+        p = p.rstrip("/")
+        if p and p not in seen:
+            seen.add(p)
+            out.append(p)
+    return out
+
+
+# ------------------------------------------------------------------ the lookup
+
+
+def _escaped_prefix(path: str) -> str:
+    # Claude Code encodes a cwd as its path with every '/' and '.' -> '-'.
+    return re.sub(r"[/.]", "-", path)
+
+
+def _text_of(msg: dict) -> str:
+    c = (msg or {}).get("content")
+    if isinstance(c, str):
+        return c
+    if isinstance(c, list):
+        return "\n".join(
+            b.get("text", "")
+            for b in c
+            if isinstance(b, dict) and b.get("type") == "text"
+        )
+    return ""
+
+
+def _iter_session_files(scope: list[str]):
+    """Yield (transcript_path, scope_root) for every session dir whose escaped
+    name starts with a scope root. The prefix glob can over-match a sibling
+    (…-website vs …-websitex); the caller re-checks each file's real cwd field."""
+    seen = set()
+    for root in scope:
+        prefix = _escaped_prefix(root)
+        for d in sorted(PROJECTS_DIR.glob(prefix + "*")):
+            if not d.is_dir():
+                continue
+            files = sorted(
+                d.glob("*.jsonl"), key=lambda p: p.stat().st_mtime, reverse=True
+            )
+            for f in files[:MAX_FILES_PER_DIR]:
+                if f in seen:
+                    continue
+                seen.add(f)
+                yield f, root
+
+
+def pull(project_name: str, terms: list[str] | None = None, limit: int = 40) -> dict:
+    """Consent-gated, cwd-scoped candidate-context pull. Read-only, filesystem."""
+    status = optin_status()
+    if not status.get("allowed"):
+        return {
+            "ok": False,
+            "reason": "not_opted_in",
+            "hint": "run --set-optin yes (or ask the user) before pulling history",
+        }
+
+    scope = resolve_scope(project_name)
+    if not scope:
+        return {"ok": False, "reason": "no_repo_scope", "project": project_name}
+    if not PROJECTS_DIR.exists():
+        return {"ok": False, "reason": "no_transcripts_dir", "dir": str(PROJECTS_DIR)}
+
+    term_list = [t.strip().lower() for t in (terms or []) if t.strip()]
+    sessions: dict[str, list] = {}
+    snippet_total = 0
+
+    for f, root in _iter_session_files(scope):
+        file_cwd = None
+        picks: list[dict] = []
+        try:
+            with f.open() as fh:
+                for line in fh:
+                    try:
+                        d = json.loads(line)
+                    except Exception:
+                        continue
+                    if file_cwd is None and d.get("cwd"):
+                        file_cwd = d["cwd"]
+                    m = d.get("message") or {}
+                    role = m.get("role")
+                    if role not in ("user", "assistant"):
+                        continue
+                    txt = _text_of(m).strip()
+                    # skip tool plumbing / system-injected blocks, keep real prose
+                    if not txt or txt.startswith("<") or "tool_result" in txt[:24]:
+                        continue
+                    if term_list and not any(t in txt.lower() for t in term_list):
+                        continue
+                    picks.append({"role": role, "preview": txt[:PREVIEW_LEN]})
+        except Exception:
+            continue
+
+        # verify the transcript really belongs to this scope root (exact or subdir)
+        if not file_cwd or not (file_cwd == root or file_cwd.startswith(root + "/")):
+            continue
+        if not picks:
+            continue
+        sessions[f.stem] = picks[-MAX_SNIPPETS_PER_SESSION:]  # most recent few
+        snippet_total += len(sessions[f.stem])
+        if len(sessions) >= limit:
+            break
+
+    return {
+        "ok": True,
+        "project": project_name,
+        "scope": scope,
+        "session_count": len(sessions),
+        "snippet_count": snippet_total,
+        "sessions": sessions,
+    }
+
+
+def expand_span(file_path: str, line_no: int, radius: int = 0) -> str:
+    """Read the full message text for a matched preview from its .jsonl line.
+    Only invoked on --expand, so full transcripts are never bulk-read."""
+    p = Path(file_path)
+    if not p.exists():
+        return ""
+    with p.open() as fh:
+        for i, line in enumerate(fh):
+            if i == line_no:
+                try:
+                    d = json.loads(line)
+                except Exception:
+                    return ""
+                content = (d.get("message") or {}).get("content")
+                if isinstance(content, str):
+                    return content
+                if isinstance(content, list):
+                    return "\n".join(
+                        b.get("text", "")
+                        for b in content
+                        if isinstance(b, dict) and b.get("type") == "text"
+                    )
+    return ""
+
+
+# ------------------------------------------------------------------------- CLI
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--project")
+    ap.add_argument("--terms", help="comma-separated FTS terms")
+    ap.add_argument("--limit", type=int, default=40)
+    ap.add_argument("--expand", action="store_true", help="read full matched spans")
+    ap.add_argument("--optin-status", action="store_true")
+    ap.add_argument("--set-optin", choices=["yes", "no"])
+    args = ap.parse_args()
+
+    if args.set_optin:
+        print(json.dumps(set_optin(args.set_optin == "yes"), indent=2))
+        return
+    if args.optin_status:
+        print(json.dumps(optin_status(), indent=2))
+        return
+    if not args.project:
+        ap.error("--project is required for a pull")
+
+    terms = args.terms.split(",") if args.terms else None
+    result = pull(args.project, terms, limit=args.limit)
+
+    if result.get("ok") and args.expand:
+        for sess in result["sessions"].values():
+            for snip in sess:
+                fp, ln = snip.get("file_path"), snip.get("line_no")
+                if fp is not None and ln is not None:
+                    snip["full"] = expand_span(fp, ln)
+
+    print(json.dumps(result, indent=2)[:8000])
+
+
+if __name__ == "__main__":
+    main()