@m13v/s4l 1.6.197-rc.7

package/scripts/scan_moltbook_replies.py

@@ -0,0 +1,252 @@
+#!/usr/bin/env python3
+"""Scan Moltbook notifications for new replies to our content.
+
+Uses /api/v1/notifications (inbox-style), mirroring the Reddit scanner.
+Replaces the legacy per-post comment-polling scan, which broke when
+/api/v1/posts/{uuid} stopped embedding the `comments` array and moved
+them to /api/v1/posts/{uuid}/comments (~2026-03-18).
+
+Handles notification types `comment_reply` and `mention`.
+`dm_request` and `new_follower` are ignored (not engagement we reply to).
+
+Inserts into the `replies` table as 'pending' (fresh) or 'skipped'
+(backfill_old / too_short / deleted_or_spam). Matches posts by
+`relatedPostId` against `posts.thread_url`. Dedupe key is
+`relatedCommentId` via `reply_insert.already_tracked`.
+
+Usage:
+    python3 scripts/scan_moltbook_replies.py
+
+Requires MOLTBOOK_API_KEY in ~/social-autoposter/.env.
+"""
+
+import json
+import os
+import sys
+import time
+from datetime import datetime, timezone
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from http_api import api_get, load_env
+from reply_insert import insert_reply as _insert_reply
+from moltbook_tools import (
+    fetch_moltbook_json,
+    HttpNotFoundError,
+    MoltbookRateLimitedError,
+)
+
+CONFIG_PATH = os.path.expanduser("~/social-autoposter/config.json")
+
+PAGE_LIMIT = 100
+MAX_PAGES = 20  # caps pagination at ~2000 items per run
+BACKFILL_HOURS = 48
+CONSECUTIVE_KNOWN_STOP = 50
+ENGAGE_TYPES = {"comment_reply", "mention"}
+MIN_WORDS = 5
+PAGE_PAUSE_SECONDS = 1.0
+
+
+def load_config():
+    if os.path.exists(CONFIG_PATH):
+        with open(CONFIG_PATH) as f:
+            return json.load(f)
+    return {}
+
+
+def word_count(text):
+    return len(text.split()) if text else 0
+
+
+def parse_iso(ts):
+    if not ts:
+        return None
+    if ts.endswith("Z"):
+        ts = ts[:-1] + "+00:00"
+    try:
+        return datetime.fromisoformat(ts)
+    except ValueError:
+        return None
+
+
+class MoltbookNotificationScanner:
+    def __init__(self, api_key, self_username, self_agent_id, excluded_authors):
+        self.api_key = api_key
+        self.self_username_lower = (self_username or "").lower()
+        self.self_agent_id = (self_agent_id or "").lower()
+        self.excluded = {a.lower() for a in (excluded_authors or [])}
+        self.excluded.update({"[deleted]", self.self_username_lower})
+        # reply_insert.insert_reply ignores the `db` arg (HTTP-only) but still
+        # accepts it positionally; pass None to satisfy the signature.
+        self.db = None
+        self.discovered = 0
+        self.skipped_backfill = 0
+        self.skipped_short = 0
+        self.skipped_moderated = 0
+        self.skipped_excluded = 0
+        self.skipped_self = 0
+        self.unmatched = 0
+        self.total_seen = 0
+        self.consecutive_known = 0
+
+    def _post_id_for_moltbook(self, related_post_id):
+        if not related_post_id:
+            return None
+        resp = api_get(
+            "/api/v1/posts",
+            query={
+                "platform": "moltbook",
+                "thread_url_contains": related_post_id,
+                "order_by": "id",
+                "order_dir": "desc",
+                "limit": 1,
+            },
+        )
+        rows = ((resp or {}).get("data") or {}).get("posts") or []
+        return rows[0].get("id") if rows else None
+
+    def _insert(self, post_id, comment_id, author, content, comment_url,
+                post_uuid, status, skip_reason=None):
+        counters_before = (self.discovered + self.skipped_backfill
+                           + self.skipped_short + self.skipped_moderated)
+        result = _insert_reply(
+            self.db, post_id, "moltbook", comment_id, author, content, comment_url,
+            status=status, skip_reason=skip_reason,
+            moltbook_post_uuid=post_uuid,
+        )
+        if result == "pending":
+            self.discovered += 1
+        elif result == "skipped":
+            if skip_reason == "backfill_old":
+                self.skipped_backfill += 1
+            elif skip_reason == "moderated":
+                self.skipped_moderated += 1
+            elif skip_reason and skip_reason.startswith("too_short"):
+                self.skipped_short += 1
+        counters_after = (self.discovered + self.skipped_backfill
+                          + self.skipped_short + self.skipped_moderated)
+        if counters_after == counters_before:
+            self.consecutive_known += 1
+        else:
+            self.consecutive_known = 0
+
+    def scan(self):
+        if not self.api_key:
+            print("MOLTBOOK_API_KEY not set, skipping Moltbook notification scan")
+            return
+        print("Scanning Moltbook notifications...")
+        backfill_cutoff = datetime.now(timezone.utc).timestamp() - BACKFILL_HOURS * 3600
+        cursor = None
+        for page in range(1, MAX_PAGES + 1):
+            url = f"https://www.moltbook.com/api/v1/notifications?limit={PAGE_LIMIT}"
+            if cursor:
+                url += f"&cursor={cursor}"
+            try:
+                data = fetch_moltbook_json(url, api_key=self.api_key)
+            except MoltbookRateLimitedError as e:
+                print(f"  Stopping scan: Moltbook rate-limited for {int(e.reset_seconds)}s")
+                break
+            except HttpNotFoundError:
+                print("  Notifications endpoint returned 404; aborting scan")
+                break
+            if not data:
+                print("  Empty response; aborting scan")
+                break
+            notifs = data.get("notifications") or []
+            print(f"  page {page}: {len(notifs)} notifications (has_more={data.get('has_more')})")
+            if not notifs:
+                break
+            for n in notifs:
+                self.total_seen += 1
+                ntype = n.get("type")
+                if ntype not in ENGAGE_TYPES:
+                    continue
+                comment_id = n.get("relatedCommentId")
+                post_uuid = n.get("relatedPostId")
+                if not comment_id or not post_uuid:
+                    continue
+                post_id = self._post_id_for_moltbook(post_uuid)
+                if not post_id:
+                    self.unmatched += 1
+                    continue
+                comment = n.get("comment") or {}
+                author_id = (comment.get("authorId") or "").strip()
+                if author_id and author_id.lower() == self.self_agent_id:
+                    self.skipped_self += 1
+                    continue
+                author = author_id or "[unknown]"
+                if author.lower() in self.excluded:
+                    self.skipped_excluded += 1
+                    continue
+                content = comment.get("content") or ""
+                comment_url = f"https://www.moltbook.com/post/{post_uuid}#{comment_id}"
+
+                if comment.get("isDeleted") or comment.get("isSpam") or comment.get("isFlagged"):
+                    self._insert(post_id, comment_id, author, content, comment_url,
+                                 post_uuid=post_uuid,
+                                 status="skipped", skip_reason="moderated")
+                    continue
+
+                created_at = parse_iso(n.get("createdAt") or comment.get("createdAt"))
+                is_old = bool(created_at and created_at.timestamp() < backfill_cutoff)
+
+                if word_count(content) < MIN_WORDS:
+                    self._insert(post_id, comment_id, author, content, comment_url,
+                                 post_uuid=post_uuid,
+                                 status="skipped",
+                                 skip_reason=f"too_short ({word_count(content)} words)")
+                elif is_old:
+                    self._insert(post_id, comment_id, author, content, comment_url,
+                                 post_uuid=post_uuid,
+                                 status="skipped", skip_reason="backfill_old")
+                else:
+                    self._insert(post_id, comment_id, author, content, comment_url,
+                                 post_uuid=post_uuid,
+                                 status="pending")
+                    print(f"  NEW: [{post_id}] author={author[:8]} {content[:80]}...")
+                if self.consecutive_known >= CONSECUTIVE_KNOWN_STOP:
+                    print(f"  hit {self.consecutive_known} consecutive already-known items, stopping pagination")
+                    return
+            if not data.get("has_more"):
+                break
+            cursor = data.get("next_cursor")
+            if not cursor:
+                break
+            if page < MAX_PAGES:
+                time.sleep(PAGE_PAUSE_SECONDS)
+
+    def finish(self):
+        print(
+            f"Notification scan complete: seen={self.total_seen} "
+            f"new_pending={self.discovered} backfill_skipped={self.skipped_backfill} "
+            f"too_short_skipped={self.skipped_short} moderated_skipped={self.skipped_moderated} "
+            f"excluded_author={self.skipped_excluded} self_filtered={self.skipped_self} "
+            f"unmatched_thread={self.unmatched}"
+        )
+        return {
+            "discovered": self.discovered,
+            "backfill_skipped": self.skipped_backfill,
+            "too_short_skipped": self.skipped_short,
+            "moderated_skipped": self.skipped_moderated,
+            "excluded": self.skipped_excluded,
+            "self_filtered": self.skipped_self,
+            "unmatched": self.unmatched,
+            "total_seen": self.total_seen,
+        }
+
+
+def main():
+    load_env()
+    api_key = os.environ.get("MOLTBOOK_API_KEY", "")
+    config = load_config()
+    acct = config.get("accounts", {}).get("moltbook", {}) or {}
+    self_username = acct.get("username", "")
+    self_agent_id = acct.get("agent_id", "")  # optional; filters own replies by agentId if set
+    excluded_authors = config.get("exclusions", {}).get("authors", [])
+    scanner = MoltbookNotificationScanner(api_key, self_username, self_agent_id, excluded_authors)
+    scanner.scan()
+    result = scanner.finish()
+    sys.exit(0 if result["discovered"] > 0 else 1)
+
+
+if __name__ == "__main__":
+    main()

package/scripts/scan_pii.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+"""Repo hygiene scanner: block PII, secrets, images, and absolute home paths.
+
+This is a PUBLIC repo. The scanner is the single source of truth reused by:
+  - the shared pre-commit hook (scripts/git-hooks/pre-commit), mode --staged
+  - the gitleaks-adjacent CI job (.github/workflows/secret-scan.yml), mode --all
+  - on-demand audits: `python3 scripts/scan_pii.py --all`
+
+What it flags in staged/tracked content:
+  1. Secret-shaped literals (tokens, private keys, db URLs with passwords).
+  2. Real client/operator PII from a gitignored denylist (pii_denylist.local.txt):
+     one term per line (email, name, handle, phone). Never commit that file.
+  3. Tracked images / media (this repo stores none; see .gitignore).
+  4. Absolute /Users/<name>/ home paths (leaks layout, breaks other operators).
+
+Exit code 0 = clean, 1 = violations found (prints them). Override a specific
+commit with `git commit --no-verify`, but fix the finding instead when you can.
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parent.parent
+DENYLIST_FILE = REPO / "pii_denylist.local.txt"
+
+# Files/paths the scanner must not scan (it would flag its own rules/examples).
+ALLOWLIST_SUFFIXES = (
+    "scripts/scan_pii.py",
+    "scripts/git-hooks/pre-commit",
+    ".github/workflows/secret-scan.yml",
+    ".gitignore",
+    ".env.example",
+    "config.example.json",
+    "pii_denylist.local.txt",
+)
+
+SECRET_PATTERNS = [
+    (r"ghp_[A-Za-z0-9]{36}", "GitHub personal access token"),
+    (r"gh[opsu]_[A-Za-z0-9]{36}", "GitHub token"),
+    (r"github_pat_[A-Za-z0-9_]{50,}", "GitHub fine-grained PAT"),
+    (r"npm_[A-Za-z0-9]{36}", "npm token"),
+    (r"sk-(?:proj-)?[A-Za-z0-9]{20,}", "OpenAI-style secret key"),
+    (r"xox[baprs]-[A-Za-z0-9-]{10,}", "Slack token"),
+    (r"AKIA[0-9A-Z]{16}", "AWS access key id"),
+    (r"AIza[0-9A-Za-z_\-]{35}", "Google API key"),
+    (r"-----BEGIN (?:RSA |EC |OPENSSH |PGP )?PRIVATE KEY", "Private key block"),
+    (r"(?:postgres(?:ql)?|mysql|mongodb)://[^\s:/'\"]+:[^\s@'\"]+@", "DB URL with inline password"),
+    (r"(?i)(?:api[_-]?key|secret|passwd|password|auth[_-]?token)\s*[=:]\s*[\"'][^\"'\s]{16,}[\"']", "Hardcoded credential"),
+]
+
+# Absolute home path leak. Placeholder forms (/Users/<you>, /Users/USERNAME) pass.
+HOME_PATH_RE = re.compile(r"/Users/(?!<|USER|USERNAME|you\b|me\b|name\b)[a-z0-9._-]{2,}", re.I)
+
+IMAGE_EXT = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".heic", ".bmp", ".tiff"}
+
+
+def _run(cmd: list[str]) -> str:
+    return subprocess.run(cmd, cwd=REPO, capture_output=True, text=True).stdout
+
+
+def _staged_files() -> list[str]:
+    out = _run(["git", "diff", "--cached", "--name-only", "--diff-filter=ACM"])
+    return [f for f in out.splitlines() if f.strip()]
+
+
+def _tracked_files() -> list[str]:
+    return [f for f in _run(["git", "ls-files"]).splitlines() if f.strip()]
+
+
+def _added_lines(path: str) -> list[tuple[int, str]]:
+    """Return (lineno, text) for + lines in the staged diff of one file."""
+    diff = _run(["git", "diff", "--cached", "--unified=0", "--", path])
+    lines: list[tuple[int, str]] = []
+    new_ln = 0
+    for ln in diff.splitlines():
+        if ln.startswith("@@"):
+            m = re.search(r"\+(\d+)", ln)
+            new_ln = int(m.group(1)) if m else new_ln
+            continue
+        if ln.startswith("+") and not ln.startswith("+++"):
+            lines.append((new_ln, ln[1:]))
+            new_ln += 1
+        elif not ln.startswith("-"):
+            new_ln += 1
+    return lines
+
+
+def _full_lines(path: str) -> list[tuple[int, str]]:
+    p = REPO / path
+    try:
+        text = p.read_text(errors="replace")
+    except (OSError, UnicodeDecodeError):
+        return []
+    return list(enumerate(text.splitlines(), start=1))
+
+
+def _load_denylist() -> list[str]:
+    if not DENYLIST_FILE.exists():
+        return []
+    terms = []
+    for raw in DENYLIST_FILE.read_text().splitlines():
+        t = raw.strip()
+        if t and not t.startswith("#"):
+            terms.append(t)
+    return terms
+
+
+def _is_allowlisted(path: str) -> bool:
+    return any(path == s or path.endswith("/" + s) for s in ALLOWLIST_SUFFIXES)
+
+
+def scan(paths: list[str], staged: bool) -> tuple[list[str], list[str]]:
+    """Return (hard, soft) findings.
+
+    hard  = secrets, denylist PII, images. Fail in every mode.
+    soft  = absolute home paths. Hard-block NEW ones (--staged) but only warn on
+            the existing tree (--all), since there is pre-existing debt to burn
+            down gradually rather than block CI on day one.
+    """
+    denylist = _load_denylist()
+    deny_re = None
+    if denylist:
+        deny_re = re.compile("|".join(re.escape(t) for t in denylist), re.I)
+    secret_res = [(re.compile(p), label) for p, label in SECRET_PATTERNS]
+    hard: list[str] = []
+    soft: list[str] = []
+
+    for path in paths:
+        # Image / media files (path-based, no content read).
+        if Path(path).suffix.lower() in IMAGE_EXT:
+            hard.append(f"{path}: image/media file (this repo tracks no images)")
+            continue
+        if _is_allowlisted(path):
+            continue
+
+        lines = _added_lines(path) if staged else _full_lines(path)
+        for lineno, text in lines:
+            for rx, label in secret_res:
+                if rx.search(text):
+                    hard.append(f"{path}:{lineno}: possible {label}")
+            if deny_re and deny_re.search(text):
+                m = deny_re.search(text)
+                hard.append(f"{path}:{lineno}: PII denylist match ('{m.group(0)}')")
+            if HOME_PATH_RE.search(text):
+                m = HOME_PATH_RE.search(text)
+                (hard if staged else soft).append(
+                    f"{path}:{lineno}: absolute home path ('{m.group(0)}...')"
+                )
+    return hard, soft
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    g = ap.add_mutually_exclusive_group()
+    g.add_argument("--staged", action="store_true", help="scan staged diff (pre-commit)")
+    g.add_argument("--all", action="store_true", help="scan whole tracked tree (CI/audit)")
+    args = ap.parse_args()
+
+    staged = args.staged or not args.all
+    paths = _staged_files() if staged else _tracked_files()
+    hard, soft = scan(paths, staged=staged)
+
+    if soft:
+        print("Repo hygiene warnings (not blocking):\n", file=sys.stderr)
+        for f in soft:
+            print("  warn: " + f, file=sys.stderr)
+        print("", file=sys.stderr)
+
+    if not hard:
+        return 0
+    print("Repo hygiene scan found blocking issues:\n", file=sys.stderr)
+    for f in hard:
+        print("  " + f, file=sys.stderr)
+    print(
+        "\nFix the finding (move PII to a *.local.* gitignored file, drop the image, "
+        "use an env/config lookup instead of an absolute path).\n"
+        "If this is a genuine false positive, override this ONE commit with "
+        "`git commit --no-verify`.",
+        file=sys.stderr,
+    )
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())