@m13v/s4l 1.6.197-rc.7

package/scripts/generation_trace.py

@@ -0,0 +1,173 @@
+"""Shared helpers for writing the generation_trace audit blob.
+
+Every post-drafting pipeline (post_github.py, post_reddit.py,
+run-twitter-cycle.sh + twitter_post_plan.py) builds a small JSON snapshot
+of "what Claude saw" — top_performers report, top_search_topics report,
+recent-comments cluster, prompt size, model, scoring formula — and
+persists it to posts.generation_trace JSONB. This module owns the shape
+and the file-handoff dance so the three pipelines stay consistent.
+
+Why a shared module instead of duplicating the dict in each pipeline:
+the trace shape is a contract with the audit consumer (a future "show
+me which examples produced post #123" query). Drift between pipelines
+makes that query impossible. Centralizing here means a `version`
+bump migrates every writer at once.
+
+Shape v1 — must match the comment block at the top of
+migrations/2026-05-12_generation_trace.sql. Bump `TRACE_SHAPE_VERSION`
+and add a migration if the contract changes.
+"""
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+import uuid
+from datetime import datetime
+from typing import Iterable, Optional
+
+TRACE_SHAPE_VERSION = 1
+
+# API cap (matches src/app/api/v1/posts/route.ts). We do NOT truncate
+# the report bodies — top_performers.py and top_search_topics.py
+# already pre-summarize their output and rarely cross 10 KB combined.
+# Cap raised from 64 KB to 1 MB on 2026-05-13: real Twitter cycles were
+# producing ~85 KB traces and the API was rejecting every log_post call
+# with HTTP 400 (posted=0 / failed=log_post_no_id on the dashboard while
+# replies still landed on x.com). The "log_post just drops the trace
+# field and warns" fallback in the older comment was aspirational, not
+# implemented; bumping the cap is the correct lever here.
+MAX_TRACE_BYTES = 1024 * 1024
+
+
+def build_trace(
+    *,
+    platform: str,
+    project_name: str,
+    prompt_chars: int,
+    top_performers_text: str = "",
+    top_search_topics_text: str = "",
+    recent_comment_ids: Optional[Iterable[int]] = None,
+    model: Optional[str] = None,
+    min_score_floor: Optional[int] = None,
+    extras: Optional[dict] = None,
+) -> dict:
+    """Construct the canonical trace dict for one Claude drafting run.
+
+    All examples-strings are stored verbatim. We deliberately do NOT
+    re-derive structure (e.g. "parse top_performers_text and pull out
+    each example as a sub-object") — the bytes-for-bytes report is the
+    only audit-faithful representation of what landed in the prompt.
+
+    `extras` is a per-pipeline escape hatch (twitter passes top_queries
+    + supply_signal; reddit passes dud_queries). Stored under
+    `examples.extras` so the schema stays stable.
+    """
+    return {
+        "version": TRACE_SHAPE_VERSION,
+        "generated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
+        "model": model or "",
+        "platform": platform,
+        "project": project_name,
+        "prompt_chars": int(prompt_chars or 0),
+        "examples": {
+            "top_performers_text": top_performers_text or "",
+            "top_search_topics_text": top_search_topics_text or "",
+            "recent_comment_ids": [int(x) for x in (recent_comment_ids or [])],
+            "extras": dict(extras or {}),
+        },
+        "scoring": {
+            "score_formula": "clicks*10 + comments*3 + upvotes_net",
+            "min_score_floor": int(min_score_floor or 0),
+            "click_aware_since": "2026-05-12",
+        },
+    }
+
+
+_GEN_TRACE_DIR_ENV = "SOCIAL_AUTOPOSTER_GEN_TRACE_DIR"
+
+
+def _gen_trace_dir() -> Optional[str]:
+    """Resolve the persistent trace directory.
+
+    Order: env override > <repo>/log/gen_trace/ > None (fall back to tempfile).
+    """
+    override = os.environ.get(_GEN_TRACE_DIR_ENV)
+    if override:
+        try:
+            os.makedirs(override, exist_ok=True)
+            return override
+        except OSError:
+            return None
+    repo_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    target = os.path.join(repo_dir, "log", "gen_trace")
+    try:
+        os.makedirs(target, exist_ok=True)
+        return target
+    except OSError:
+        return None
+
+
+def write_trace_tempfile(trace: dict, *, prefix: str = "gen_trace_") -> Optional[str]:
+    """Persist trace dict to disk and return the path.
+
+    Was a /tmp NamedTemporaryFile until 2026-05-14; /tmp didn't survive
+    e2b sandbox pause/resume, losing per-thread draft decisions on the
+    very runs we needed to postmortem. Now writes to <repo>/log/gen_trace/
+    (or $SOCIAL_AUTOPOSTER_GEN_TRACE_DIR), persistent across the run.
+
+    Returns None on any IO failure — the caller must treat the trace as
+    nice-to-have, never block the post on a failed trace write.
+    """
+    trace_dir = _gen_trace_dir()
+    try:
+        if trace_dir:
+            ts = datetime.utcnow().strftime("%Y%m%d-%H%M%S")
+            fname = f"{prefix}{ts}_{uuid.uuid4().hex[:8]}.json"
+            path = os.path.join(trace_dir, fname)
+            with open(path, "w", encoding="utf-8") as fh:
+                json.dump(trace, fh, ensure_ascii=False)
+            return path
+        # Fallback: tempfile (read-only repo, sandbox edge cases).
+        tf = tempfile.NamedTemporaryFile(
+            prefix=prefix, suffix=".json",
+            mode="w", delete=False, encoding="utf-8",
+        )
+        try:
+            json.dump(trace, tf, ensure_ascii=False)
+        finally:
+            tf.close()
+        return tf.name
+    except (OSError, TypeError, ValueError):
+        return None
+
+
+def cleanup_trace_tempfile(path: Optional[str]) -> None:
+    """Historically deleted trace files at end-of-run. Now a no-op for files
+    under <repo>/log/ so failure postmortems retain trace JSON; still deletes
+    legacy /tmp paths to avoid filling /tmp on long-running hosts.
+    """
+    if not path:
+        return
+    try:
+        # Persistent log/ files: keep. Anything else (e.g. /tmp fallback): delete.
+        norm = os.path.abspath(path)
+        repo_log = os.path.abspath(os.path.join(
+            os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+            "log",
+        ))
+        override = os.environ.get(_GEN_TRACE_DIR_ENV)
+        keep_dirs = [repo_log] + ([os.path.abspath(override)] if override else [])
+        if any(norm.startswith(d + os.sep) for d in keep_dirs):
+            return
+        os.unlink(path)
+    except OSError:
+        pass
+
+
+def trace_bytes(trace: dict) -> int:
+    """Serialized size in bytes. Useful for guard checks before write."""
+    try:
+        return len(json.dumps(trace, ensure_ascii=False).encode("utf-8"))
+    except (TypeError, ValueError):
+        return 0

package/scripts/get_run_cost.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""Query claude_sessions to get total cost for a pipeline cycle.
+
+Two modes:
+
+1) Cycle mode (preferred): `--cycle-id rdcycle-...`
+   Sums orchestrator_cost_usd (native SDK billing) for every row stamped with
+   this cycle_id. This is the authoritative Anthropic bill, NOT the inflated
+   transcript-derived estimate (total_cost_usd) we used to report. Accurate
+   even when multiple cycles of the same script overlap in wall-clock time
+   (run-reddit-search.sh, run-twitter-cycle.sh double-fork their work so
+   stacked cycles are normal).
+
+2) Legacy time-window mode: `--since <unix_ts> --scripts tag1 tag2 ...`
+   Filters by script + started_at. Kept for backward compatibility with
+   callers that don't pass cycle_id (older pipelines, historical reports).
+   IMPORTANT: this mode over-counts when multiple cycles of the same script
+   overlap, because it has no way to distinguish them. Migrate callers to
+   --cycle-id when possible.
+
+Either mode is acceptable; --cycle-id wins if both are passed. Prints the
+total cost as a float (4 decimal places), or 0.0000 on any error.
+Designed to be called from shell script EXIT traps to get real cost per run.
+"""
+import argparse
+import os
+import sys
+from datetime import datetime, timezone
+
+ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+def _load_env():
+    env_path = os.path.join(ROOT_DIR, '.env')
+    if not os.path.exists(env_path):
+        return
+    with open(env_path) as f:
+        for line in f:
+            line = line.strip()
+            if line and not line.startswith('#') and '=' in line:
+                k, v = line.split('=', 1)
+                os.environ.setdefault(k.strip(), v.strip())
+
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument('--cycle-id', default=None,
+                   help='Pipeline cycle batch id (e.g. rdcycle-20260510-110005). '
+                        'Sums cost across every claude_sessions row stamped with '
+                        'this id. Wins over --since/--scripts when both passed.')
+    p.add_argument('--since', type=int, default=None,
+                   help='Unix timestamp of run start (legacy mode; use cycle-id '
+                        'instead when possible).')
+    p.add_argument('--scripts', nargs='*', default=None,
+                   help='claude_sessions.script values to sum (legacy mode).')
+    p.add_argument('--breakdown', action='store_true',
+                   help='Print "parent_cost subagent_cost task_count subagent_count" '
+                        'instead of just the parent total. Useful when investigating '
+                        'whether Task() subagents are inflating cost.')
+    args = p.parse_args()
+
+    _load_env()
+
+    # Resolve which mode we're running in. Cycle id is authoritative if
+    # given (and non-empty). Otherwise require the legacy pair.
+    cycle_id = args.cycle_id.strip() if args.cycle_id else None
+    if not cycle_id and (args.since is None or not args.scripts):
+        # Bash EXIT traps shell out blind; keep the contract simple: any
+        # missing-arg condition prints 0.0000 and exits 0 so the caller
+        # never crashes its log_run.py emit on a malformed cost call.
+        print("0.0000")
+        return
+
+    try:
+        sys.path.insert(0, os.path.join(ROOT_DIR, 'scripts'))
+        # HTTP-only: cost is always read from the s4l.ai HTTP API. The direct
+        # Postgres path (and the SOCIAL_AUTOPOSTER_LEGACY_NEON escape hatch) was
+        # removed 2026-06-01 — no DB path, no fallback.
+        parent_cost, subagent_cost, task_count, subagent_count = _fetch_via_api(
+            cycle_id=cycle_id, since=args.since, scripts=args.scripts,
+        )
+        if args.breakdown:
+            print(f"{parent_cost:.4f} {subagent_cost:.4f} {task_count} {subagent_count}")
+        else:
+            print(f"{parent_cost:.4f}")
+    except Exception:
+        print("0.0000")
+
+
+def _fetch_via_api(*, cycle_id, since, scripts):
+    from http_api import api_get
+    if cycle_id:
+        query = {"cycle_id": cycle_id}
+    else:
+        query = {"since_ts": str(int(since)), "scripts": ",".join(scripts)}
+    resp = api_get("/api/v1/claude-sessions/cost", query=query)
+    data = (resp or {}).get("data") or {}
+    return (
+        float(data.get("parent_cost") or 0),
+        float(data.get("subagent_cost") or 0),
+        int(data.get("task_count") or 0),
+        int(data.get("subagent_count") or 0),
+    )
+
+
+if __name__ == '__main__':
+    main()

package/scripts/github_engage_helper.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""github_engage_helper.py — small CLI wrapper used by skill/github-engage.sh
+to replace the five `psql "$DATABASE_URL" -t -A -c "..."` one-liners the shell
+used to embed inline. The direct-Postgres lane was removed 2026-06-01;
+DATABASE_URL is deliberately ignored, no DB, no fallback. Every subcommand
+prints exactly what the corresponding psql call printed so the surrounding
+shell capture ($(...)) and integer compares are unchanged.
+
+Subcommands:
+  posts-active-count
+      -> GET /api/v1/posts/count?platform=github&status=active
+      -> prints the integer count (was: SELECT COUNT(*) FROM posts
+         WHERE platform='github' AND status='active')
+  pending-count
+      -> GET /api/v1/replies/counts?platform=github
+      -> prints the integer pending count (was: SELECT COUNT(*) FROM replies
+         WHERE platform='github' AND status='pending')
+  reply-counts
+      -> GET /api/v1/replies/counts?platform=github
+      -> prints JSON {pending, replied, skipped} (replaces the three trailing
+         psql COUNT one-liners in Phase C)
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from http_api import api_get  # noqa: E402
+
+
+def _counts_dict() -> dict[str, int]:
+    resp = api_get("/api/v1/replies/counts", query={"platform": "github"})
+    data = resp.get("data") or {}
+    # github has no mentions/orphan nuance; the raw `counts` field is the
+    # authoritative per-status tally. (eligible_counts would also work but
+    # for github every reply is post-rooted.)
+    rows = data.get("counts") or []
+    out: dict[str, int] = {}
+    for r in rows:
+        s = r.get("status")
+        if s is None:
+            continue
+        try:
+            out[str(s)] = int(r.get("count") or 0)
+        except (TypeError, ValueError):
+            out[str(s)] = 0
+    return out
+
+
+def cmd_posts_active_count() -> int:
+    resp = api_get("/api/v1/posts/count", query={"platform": "github", "status": "active"})
+    print(int((resp.get("data") or {}).get("count") or 0))
+    return 0
+
+
+def cmd_pending_count() -> int:
+    print(int(_counts_dict().get("pending") or 0))
+    return 0
+
+
+def cmd_reply_counts() -> int:
+    counts = _counts_dict()
+    out = {
+        "pending": int(counts.get("pending") or 0),
+        "replied": int(counts.get("replied") or 0),
+        "skipped": int(counts.get("skipped") or 0),
+    }
+    json.dump(out, sys.stdout, separators=(",", ":"))
+    sys.stdout.write("\n")
+    return 0
+
+
+def main() -> int:
+    p = argparse.ArgumentParser()
+    sub = p.add_subparsers(dest="cmd", required=True)
+    sub.add_parser("posts-active-count")
+    sub.add_parser("pending-count")
+    sub.add_parser("reply-counts")
+    args = p.parse_args()
+    if args.cmd == "posts-active-count":
+        return cmd_posts_active_count()
+    if args.cmd == "pending-count":
+        return cmd_pending_count()
+    if args.cmd == "reply-counts":
+        return cmd_reply_counts()
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())