@m13v/s4l 1.6.197-rc.7

package/scripts/http_api.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+"""Shared HTTP helper for the s4l.ai API endpoints.
+
+All Reddit-pipeline (and friends) writes/reads route through here, carrying
+either:
+  - X-Installation header (default lane, open-source identity), or
+  - Authorization: Bearer $AUTOPOSTER_API_KEY when the key is set in env
+    (server-internal callers / endpoints that still use requireApiKey).
+
+Both headers are sent on every request when both are available, so a route
+that uses resolveAuth picks the install lane while a route that uses
+requireApiKey picks the bearer lane. No caller-side branching needed.
+"""
+from __future__ import annotations
+
+import json
+import os
+import ssl
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+# Best-effort Sentry init (no-op if sentry-sdk missing or DSN unset). http_api
+# is the central HTTP-lane client (~100 pipeline scripts import it), so this one
+# hook gives the whole Python pipeline error capture. Mirrors mcp/src/telemetry.ts.
+try:
+    import sentry_init as _sentry_init
+
+    _sentry_init.init()
+except Exception:
+    pass
+
+
+def _build_ssl_context() -> ssl.SSLContext:
+    """Pin a known-good trust store, immune to a bad inherited SSL_CERT_FILE.
+
+    The MCP/host app can inject SSL_CERT_FILE / SSL_CERT_DIR pointing at a
+    bundle that lacks the right roots, which makes urllib raise
+    CERTIFICATE_VERIFY_FAILED only inside the spawned subprocess (TLS works
+    fine in a normal shell). We resolve the trust store deliberately here
+    instead of trusting whatever env we inherit:
+
+      1. inherited SSL_CERT_FILE, but only if the path exists AND yields a
+         context with at least one trusted root;
+      2. the platform default store;
+      3. certifi, if installed.
+
+    The get_ca_certs() check is what rejects a bad inherited path: an empty
+    trust store silently falls through to the next candidate.
+    """
+    candidates = []
+    env_file = os.environ.get("SSL_CERT_FILE")
+    if env_file and os.path.exists(env_file):
+        candidates.append(env_file)
+    candidates.append(None)  # platform default
+    for cafile in candidates:
+        try:
+            ctx = ssl.create_default_context(cafile=cafile)
+            if ctx.get_ca_certs():
+                return ctx
+        except Exception:
+            continue
+    try:
+        import certifi
+        return ssl.create_default_context(cafile=certifi.where())
+    except Exception:
+        return ssl.create_default_context()
+
+
+_SSL_CONTEXT = _build_ssl_context()
+
+ENV_PATH = os.path.expanduser("~/social-autoposter/.env")
+
+
+def load_env():
+    """Load ~/social-autoposter/.env into os.environ (setdefault, never clobber).
+
+    Generic dotenv loader, not DB-specific: callers need it for keys like
+    MOLTBOOK_API_KEY / AUTOPOSTER_API_KEY / AUTOPOSTER_API_BASE. Lives here
+    (rather than the now-removed db.py) so HTTP-only scripts have one import.
+    """
+    if os.path.exists(ENV_PATH):
+        with open(ENV_PATH) as f:
+            for line in f:
+                line = line.strip()
+                if line and not line.startswith("#") and "=" in line:
+                    k, v = line.split("=", 1)
+                    os.environ.setdefault(k.strip(), v.strip())
+
+
+def _base_url():
+    return os.environ.get("AUTOPOSTER_API_BASE", "https://s4l.ai").rstrip("/")
+
+
+def _headers():
+    from identity import get_identity_header
+    headers = {
+        "Content-Type": "application/json",
+        "X-Installation": get_identity_header(),
+    }
+    bearer = (os.environ.get("AUTOPOSTER_API_KEY") or "").strip()
+    if bearer:
+        headers["Authorization"] = f"Bearer {bearer}"
+        headers["x-api-key"] = bearer
+    return headers
+
+
+def _request(method: str, path: str, body: dict | None = None,
+             query: dict | None = None, ok_on_conflict: bool = False,
+             ok_on_404: bool = False):
+    """Generic request runner with retries.
+
+    Returns parsed JSON. Raises SystemExit on terminal failure.
+
+    method: GET | POST | PATCH | DELETE
+    body: JSON body for write methods
+    query: optional dict for ?k=v query-string (GET / DELETE)
+    ok_on_conflict: when True, a 409 body is returned (not raised)
+    ok_on_404: when True, a 404 returns {"_not_found": True}
+    """
+    url = f"{_base_url()}{path}"
+    if query:
+        # Drop None values so we don't send 'key=None' string.
+        qs = urllib.parse.urlencode(
+            {k: v for k, v in query.items() if v is not None},
+            doseq=True,
+        )
+        if qs:
+            sep = "&" if "?" in url else "?"
+            url = f"{url}{sep}{qs}"
+
+    data = None
+    if body is not None:
+        data = json.dumps(body).encode()
+
+    delays = [1, 3, 9]
+    last_err = None
+    for attempt, delay in enumerate(delays, 1):
+        try:
+            req = urllib.request.Request(
+                url, data=data, headers=_headers(), method=method,
+            )
+            with urllib.request.urlopen(
+                req, timeout=30, context=_SSL_CONTEXT
+            ) as resp:
+                raw = resp.read()
+                if not raw:
+                    return {}
+                return json.loads(raw)
+        except urllib.error.HTTPError as e:
+            body_txt = e.read().decode(errors="replace")
+            if e.code == 409 and ok_on_conflict:
+                try:
+                    return json.loads(body_txt)
+                except Exception:
+                    return {"error": "conflict"}
+            if e.code == 404 and ok_on_404:
+                return {"_not_found": True}
+            if 400 <= e.code < 500:
+                raise SystemExit(
+                    f"[http_api] {method} {path} HTTP {e.code}: {body_txt}"
+                )
+            last_err = e
+            print(
+                f"[http_api] {method} {path} HTTP {e.code} attempt {attempt}: "
+                f"{body_txt[:120]}",
+                file=sys.stderr,
+            )
+        except Exception as e:
+            last_err = e
+            print(
+                f"[http_api] {method} {path} attempt {attempt}: {e}",
+                file=sys.stderr,
+            )
+        if attempt < len(delays):
+            time.sleep(delay)
+    raise SystemExit(
+        f"[http_api] {method} {path} failed after {len(delays)} attempts: "
+        f"{last_err}"
+    )
+
+
+def api_get(path: str, query: dict | None = None, ok_on_404: bool = False):
+    """GET path with optional query dict. Returns parsed JSON."""
+    return _request("GET", path, query=query, ok_on_404=ok_on_404)
+
+
+def api_post(path: str, body: dict, ok_on_conflict: bool = False, ok_on_404: bool = False):
+    """POST body to path. ok_on_conflict=True returns the 409 body;
+    ok_on_404=True returns {_not_found: True} on 404 instead of raising."""
+    return _request("POST", path, body=body, ok_on_conflict=ok_on_conflict, ok_on_404=ok_on_404)
+
+
+def api_patch(path: str, body: dict, ok_on_conflict: bool = False, ok_on_404: bool = False):
+    """PATCH body to path. ok_on_conflict=True returns the 409 body;
+    ok_on_404=True returns {_not_found: True} on 404 instead of raising."""
+    return _request("PATCH", path, body=body, ok_on_conflict=ok_on_conflict, ok_on_404=ok_on_404)
+
+
+def api_delete(path: str, query: dict | None = None):
+    """DELETE path. Optional query string."""
+    return _request("DELETE", path, query=query)

package/scripts/human_dm_replies_helper.py

@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+"""human_dm_replies_helper.py — shell-friendly entrypoints for Phase 0 of
+engage-dm-replies.sh, replacing the inline `psql "$DATABASE_URL"` blocks that
+read and mutate the `human_dm_replies` queue.
+
+Everything routes through /api/v1/human-dm-replies* on the website. No
+psycopg2, no DATABASE_URL — there is intentionally NO direct-DB fallback.
+
+Subcommands:
+
+  pending [--platform reddit|twitter|x|linkedin]
+    GET /api/v1/human-dm-replies?mode=pending[&platform=...]
+    -> prints the pending+retry queue as a JSON array (the same shape the
+       legacy `SELECT json_agg(q) FROM (... human_dm_replies h JOIN dms ...)`
+       produced). Prints NOTHING (empty output) when zero rows, so the bash
+       `[ -n "$HUMAN_REPLIES" ]` guard falls through to the "no replies"
+       branch exactly like psql's NULL -> empty string did.
+
+  kb [--limit 20]
+    GET /api/v1/human-dm-replies?mode=kb&limit=N
+    -> prints the last N SENT instructions as a JSON array (the Human Reply
+       Knowledge Base). Empty -> empty output (same as the legacy query).
+
+  patch --id N [--status S] [--last-error E] [--public-reply-id M]
+        [--increment-attempts] [--stamp-sent]
+    PATCH /api/v1/human-dm-replies/N
+    -> the four Phase 0 status transitions:
+         cancelled : --status cancelled --last-error "human reclassified: ..."
+         paired    : --public-reply-id M
+         sent      : --status sent
+         failed    : --status failed --increment-attempts --last-error "..."
+       (sent_at auto-stamps server-side on sent/cancelled; --stamp-sent forces
+        it otherwise.)
+
+  insert-public-reply --post-id N|--no-post-id --platform P --comment-id C
+        --author A --comment-url U --our-content TEXT --our-url URL [--depth 2]
+    POST /api/v1/replies
+    -> inserts the public reply row the delivery bot just posted and prints
+       the new replies.id to stdout (so it can be paired back via
+       `patch --public-reply-id`). 409 (duplicate their_comment_id) returns the
+       existing row's id, never an error.
+"""
+import argparse
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from http_api import api_get, api_patch, api_post
+
+
+def _print_rows_or_empty(rows):
+    """Mirror psql `json_agg` -t -A output: a JSON array when rows exist,
+    an empty string (nothing) when there are none."""
+    if not rows:
+        return
+    json.dump(rows, sys.stdout)
+    print("")
+
+
+def _cmd_pending(args):
+    query = {"mode": "pending"}
+    if args.platform:
+        query["platform"] = args.platform
+    resp = api_get("/api/v1/human-dm-replies", query=query)
+    rows = ((resp or {}).get("data") or {}).get("rows") or []
+    _print_rows_or_empty(rows)
+
+
+def _cmd_kb(args):
+    query = {"mode": "kb", "limit": args.limit}
+    resp = api_get("/api/v1/human-dm-replies", query=query)
+    rows = ((resp or {}).get("data") or {}).get("rows") or []
+    _print_rows_or_empty(rows)
+
+
+def _cmd_patch(args):
+    body = {}
+    if args.status is not None:
+        body["status"] = args.status
+    if args.last_error is not None:
+        body["last_error"] = args.last_error
+    if args.public_reply_id is not None:
+        body["public_reply_id"] = int(args.public_reply_id)
+    if args.increment_attempts:
+        body["increment_attempts"] = True
+    if args.stamp_sent:
+        body["stamp_sent_now"] = True
+    if not body:
+        print("patch: nothing to update (pass at least one field)", file=sys.stderr)
+        sys.exit(2)
+    resp = api_patch(f"/api/v1/human-dm-replies/{int(args.id)}", body)
+    row = ((resp or {}).get("data") or {}).get("human_dm_reply") or {}
+    print(json.dumps({"id": row.get("id"), "status": row.get("status"),
+                      "attempts": row.get("attempts"),
+                      "public_reply_id": row.get("public_reply_id")}))
+
+
+def _cmd_insert_public_reply(args):
+    body = {
+        "platform": args.platform,
+        "their_comment_id": args.comment_id,
+        "their_author": args.author,
+        "their_comment_url": args.comment_url,
+        "our_reply_content": args.our_content,
+        "our_reply_url": args.our_url,
+        "depth": args.depth,
+        "status": "replied",
+        "replied_at": "now",
+    }
+    if args.post_id is not None:
+        body["post_id"] = int(args.post_id)
+    # ok_on_conflict so a duplicate their_comment_id returns the existing row
+    # instead of raising; we still want its id to pair back.
+    resp = api_post("/api/v1/replies", body, ok_on_conflict=True)
+    data = (resp or {}).get("data") or {}
+    row = data.get("reply") or data.get("row") or data
+    rid = row.get("id") if isinstance(row, dict) else None
+    if rid is None and isinstance(data, dict):
+        # 409 path may nest the existing row under error.details
+        err = (resp or {}).get("error") or {}
+        details = err.get("details") if isinstance(err, dict) else None
+        if isinstance(details, dict):
+            rid = (details.get("reply") or details).get("id")
+    if rid is None:
+        print("insert-public-reply: no id in response: "
+              + json.dumps(resp)[:300], file=sys.stderr)
+        sys.exit(1)
+    print(int(rid))
+
+
+def main():
+    p = argparse.ArgumentParser(description="human_dm_replies Phase 0 helper (HTTP-only)")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    sp = sub.add_parser("pending")
+    sp.add_argument("--platform", default=None)
+    sp.set_defaults(func=_cmd_pending)
+
+    sk = sub.add_parser("kb")
+    sk.add_argument("--limit", type=int, default=20)
+    sk.set_defaults(func=_cmd_kb)
+
+    spt = sub.add_parser("patch")
+    spt.add_argument("--id", required=True)
+    spt.add_argument("--status", default=None)
+    spt.add_argument("--last-error", dest="last_error", default=None)
+    spt.add_argument("--public-reply-id", dest="public_reply_id", default=None)
+    spt.add_argument("--increment-attempts", dest="increment_attempts", action="store_true")
+    spt.add_argument("--stamp-sent", dest="stamp_sent", action="store_true")
+    spt.set_defaults(func=_cmd_patch)
+
+    si = sub.add_parser("insert-public-reply")
+    si.add_argument("--post-id", dest="post_id", default=None)
+    si.add_argument("--platform", required=True)
+    si.add_argument("--comment-id", dest="comment_id", required=True)
+    si.add_argument("--author", required=True)
+    si.add_argument("--comment-url", dest="comment_url", required=True)
+    si.add_argument("--our-content", dest="our_content", required=True)
+    si.add_argument("--our-url", dest="our_url", required=True)
+    si.add_argument("--depth", type=int, default=2)
+    si.set_defaults(func=_cmd_insert_public_reply)
+
+    args = p.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

package/scripts/identity.py

@@ -0,0 +1,302 @@
+#!/usr/bin/env python3
+"""Passive installation identity for the open-source social-autoposter client.
+
+Generates and stores a stable install_id on first run plus a snapshot of
+machine fingerprint fields. Every API call to social-autoposter-website
+carries this as an X-Installation header (base64 JSON) so the server can
+attribute writes per install, rate-limit, and surface usage without
+requiring any user signup.
+
+NO data is sent until the pipeline actually calls the API. NO secrets are
+collected. Every field captured is documented in PRIVACY.md at the repo
+root.
+
+CLI:
+    python3 scripts/identity.py show     # print identity JSON
+    python3 scripts/identity.py header   # print base64 X-Installation value
+    python3 scripts/identity.py reset    # delete identity.json
+    python3 scripts/identity.py path     # print path to identity.json
+
+Library:
+    from scripts.identity import get_identity, get_identity_header
+    headers = {"X-Installation": get_identity_header()}
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import os
+import platform
+import subprocess
+import sys
+import time
+import uuid
+from pathlib import Path
+
+IDENTITY_DIR = Path.home() / ".social-autoposter"
+IDENTITY_FILE = IDENTITY_DIR / "identity.json"
+
+
+def _safe(fn, *args, **kwargs):
+    try:
+        return fn(*args, **kwargs)
+    except Exception:
+        return None
+
+
+def _hardware_uuid_macos():
+    out = _safe(
+        subprocess.check_output,
+        ["ioreg", "-d2", "-c", "IOPlatformExpertDevice"],
+        stderr=subprocess.DEVNULL, timeout=5,
+    )
+    if not out:
+        return None
+    for line in out.decode("utf8", errors="ignore").splitlines():
+        if "IOPlatformUUID" in line:
+            parts = line.split('"')
+            if len(parts) >= 4:
+                return parts[3].strip()
+    return None
+
+
+def _hardware_uuid_linux():
+    for p in ("/etc/machine-id", "/var/lib/dbus/machine-id"):
+        try:
+            with open(p) as f:
+                v = f.read().strip()
+                if v:
+                    return v
+        except Exception:
+            continue
+    return None
+
+
+def _hardware_uuid_windows():
+    out = _safe(
+        subprocess.check_output,
+        ["reg", "query",
+         r"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Cryptography",
+         "/v", "MachineGuid"],
+        stderr=subprocess.DEVNULL, timeout=5,
+    )
+    if not out:
+        return None
+    for line in out.decode("utf8", errors="ignore").splitlines():
+        if "MachineGuid" in line:
+            tokens = line.split()
+            if tokens:
+                return tokens[-1].strip()
+    return None
+
+
+def _hardware_uuid():
+    sys_name = platform.system().lower()
+    if sys_name == "darwin":
+        return _hardware_uuid_macos()
+    if sys_name == "linux":
+        return _hardware_uuid_linux()
+    if sys_name == "windows":
+        return _hardware_uuid_windows()
+    return None
+
+
+def _hostname():
+    sys_name = platform.system().lower()
+    if sys_name == "darwin":
+        out = _safe(
+            subprocess.check_output,
+            ["scutil", "--get", "ComputerName"],
+            stderr=subprocess.DEVNULL, timeout=5,
+        )
+        if out:
+            v = out.decode("utf8", errors="ignore").strip()
+            if v:
+                return v
+    try:
+        import socket
+        return socket.gethostname() or None
+    except Exception:
+        return None
+
+
+def _git_email():
+    out = _safe(
+        subprocess.check_output,
+        ["git", "config", "--global", "user.email"],
+        stderr=subprocess.DEVNULL, timeout=3,
+    )
+    if not out:
+        return None
+    v = out.decode("utf8", errors="ignore").strip()
+    return v or None
+
+
+def _node_version():
+    out = _safe(
+        subprocess.check_output,
+        ["node", "--version"],
+        stderr=subprocess.DEVNULL, timeout=3,
+    )
+    if not out:
+        return None
+    v = out.decode("utf8", errors="ignore").strip()
+    return v.lstrip("v") or None
+
+
+def _app_version():
+    """Version of the installed S4L plugin.
+
+    On a .mcpb box the extension dir has manifest.json + package.json at its
+    root (one level above scripts/); read whichever resolves first. Honors
+    S4L_REPO_DIR / REPO_DIR when the pipeline sets it (launchd plists do).
+    """
+    root = Path(
+        os.environ.get("S4L_REPO_DIR")
+        or os.environ.get("REPO_DIR")
+        or Path(__file__).resolve().parents[1]
+    )
+    for name in ("manifest.json", "package.json"):
+        try:
+            data = json.loads((root / name).read_text())
+        except Exception:
+            continue
+        v = data.get("version")
+        if v:
+            return str(v).strip() or None
+    return None
+
+
+def _claude_desktop_version():
+    """CFBundleShortVersionString of the Claude Desktop app (macOS), or None.
+
+    Stamped into the install identity (and thus the X-Installation header on every
+    heartbeat) so the install-lane digest can correlate leaks/regressions with the
+    Desktop version. This is the variable we could not answer for Karol's box. Reads
+    Info.plist directly via plistlib; best-effort, never raises."""
+    if (platform.system() or "").lower() != "darwin":
+        return None
+    candidates = [
+        Path("/Applications/Claude.app/Contents/Info.plist"),
+        Path.home() / "Applications" / "Claude.app" / "Contents" / "Info.plist",
+    ]
+    for plist in candidates:
+        try:
+            if not plist.exists():
+                continue
+            import plistlib
+
+            with plist.open("rb") as f:
+                data = plistlib.load(f)
+            v = data.get("CFBundleShortVersionString") or data.get("CFBundleVersion")
+            if v:
+                return str(v).strip() or None
+        except Exception:
+            continue
+    return None
+
+
+def _tz():
+    try:
+        from datetime import datetime
+        tz = datetime.now().astimezone().tzinfo
+        if tz is not None:
+            name = tz.tzname(datetime.now())
+            if name:
+                return name
+    except Exception:
+        pass
+    return os.environ.get("TZ") or None
+
+
+def _build_fresh_identity():
+    return {
+        "install_id": str(uuid.uuid4()),
+        "hardware_uuid": _hardware_uuid(),
+        "hostname": _hostname(),
+        "os": (platform.system() or "").lower() or None,
+        "os_version": platform.release() or None,
+        "cpu_arch": platform.machine() or None,
+        "python_version": platform.python_version() or None,
+        "node_version": _node_version(),
+        "app_version": _app_version(),
+        "claude_desktop_version": _claude_desktop_version(),
+        "git_email": _git_email(),
+        "tz": _tz(),
+        "first_seen_at": int(time.time()),
+    }
+
+
+def get_identity(refresh: bool = False) -> dict:
+    """Read identity.json, creating it on first call.
+
+    refresh=True re-snapshots the volatile fields (versions, hostname,
+    git_email, tz) while preserving install_id and first_seen_at.
+    """
+    IDENTITY_DIR.mkdir(parents=True, exist_ok=True)
+
+    if not IDENTITY_FILE.exists():
+        ident = _build_fresh_identity()
+        IDENTITY_FILE.write_text(json.dumps(ident, indent=2))
+        try:
+            os.chmod(IDENTITY_FILE, 0o600)
+        except Exception:
+            pass
+        return ident
+
+    try:
+        ident = json.loads(IDENTITY_FILE.read_text())
+    except Exception:
+        # Corrupt file; rebuild rather than crashing the pipeline.
+        ident = _build_fresh_identity()
+        IDENTITY_FILE.write_text(json.dumps(ident, indent=2))
+        return ident
+
+    if refresh:
+        snap = _build_fresh_identity()
+        # preserve stable identifiers across refresh
+        snap["install_id"] = ident.get("install_id") or snap["install_id"]
+        snap["first_seen_at"] = ident.get("first_seen_at") or snap["first_seen_at"]
+        if snap != ident:
+            try:
+                IDENTITY_FILE.write_text(json.dumps(snap, indent=2))
+            except Exception:
+                pass
+        return snap
+    return ident
+
+
+def get_identity_header(refresh: bool = False) -> str:
+    """Return the base64 value to put in the X-Installation HTTP header."""
+    ident = get_identity(refresh=refresh)
+    payload = {
+        k: v for k, v in ident.items()
+        if k != "first_seen_at" and v is not None
+    }
+    raw = json.dumps(payload, separators=(",", ":")).encode("utf8")
+    return base64.b64encode(raw).decode("ascii")
+
+
+def main():
+    cmd = sys.argv[1] if len(sys.argv) > 1 else "show"
+    if cmd == "show":
+        print(json.dumps(get_identity(refresh=True), indent=2))
+    elif cmd == "header":
+        print(get_identity_header(refresh=True))
+    elif cmd == "reset":
+        if IDENTITY_FILE.exists():
+            IDENTITY_FILE.unlink()
+            print(f"deleted {IDENTITY_FILE}")
+        else:
+            print(f"no identity at {IDENTITY_FILE}")
+    elif cmd == "path":
+        print(str(IDENTITY_FILE))
+    else:
+        print(f"unknown cmd: {cmd}", file=sys.stderr)
+        print("usage: identity.py [show|header|reset|path]", file=sys.stderr)
+        sys.exit(2)
+
+
+if __name__ == "__main__":
+    main()