@m13v/s4l 1.6.197-rc.10

package/skill/github-engage.sh

@@ -0,0 +1,176 @@
+#!/usr/bin/env bash
+# github-engage.sh — GitHub Issues engagement loop
+# Scan our GitHub issue comments for replies, respond to substantive ones.
+# Called by launchd every 6 hours.
+
+
+set -euo pipefail
+
+# GitHub engage lock: wait up to 60min for previous run to finish, then skip
+source "$(dirname "$0")/lock.sh"
+acquire_lock "github" 3600
+
+# Load secrets
+# shellcheck source=/dev/null
+[ -f "$HOME/social-autoposter/.env" ] && source "$HOME/social-autoposter/.env"
+
+REPO_DIR="$HOME/social-autoposter"
+SKILL_FILE="$REPO_DIR/SKILL.md"
+LOG_DIR="$REPO_DIR/skill/logs"
+mkdir -p "$LOG_DIR"
+LOG_FILE="$LOG_DIR/github-engage-$(date +%Y-%m-%d_%H%M%S).log"
+
+log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$LOG_FILE"; }
+
+# Per-cycle batch id stamped onto every claude_sessions row spawned by this
+# engagement run (via SA_CYCLE_ID env -> log_claude_session.py). 2026-05-10
+# cycle_id rollout.
+BATCH_ID="engh-$(date +%Y%m%d-%H%M%S)-$$"
+export SA_CYCLE_ID="$BATCH_ID"
+
+RUN_START=$(date +%s)
+log "=== GitHub Engagement Run: $(date) (cycle=$BATCH_ID) ==="
+
+# HTTP-only lane (2026-06-01): all read-side counts route through the s4l.ai
+# HTTP API via scripts/github_engage_helper.py. The direct-Postgres lane was
+# removed; DATABASE_URL, if present in .env, is deliberately ignored. No psql,
+# no DB fallback.
+GH_HELPER="$REPO_DIR/scripts/github_engage_helper.py"
+
+# ═══════════════════════════════════════════════════════
+# PHASE A: Scan for replies to our GitHub comments
+# ═══════════════════════════════════════════════════════
+log "Phase A: Scanning GitHub issues for replies..."
+python3 "$REPO_DIR/scripts/scan_github_replies.py" 2>&1 | tee -a "$LOG_FILE"
+
+# ═══════════════════════════════════════════════════════
+# PHASE A.5: Refresh engagement stats on our GitHub comments
+# Reactions pulled via gh api; reply counts tallied from the replies
+# table that Phase A just refreshed. Stored on posts.upvotes +
+# posts.comments_count. Per-reply stats also refreshed (same call), and
+# the count is forwarded to a stats_github row in the dashboard Jobs table.
+# ═══════════════════════════════════════════════════════
+log "Phase A.5: Updating github engagement stats (reactions + reply counts)..."
+# Best-effort: stats failures (Postgres disconnects, gh rate limits) must not block
+# Phase B reply handling. Subshell scopes the set-flags, `|| true` absorbs rc.
+PHASE_A5_START=$(date +%s)
+GH_REPLY_SUMMARY=$(mktemp -t fazm-gh-reply-summary.XXXXXX)
+# Chain lock cleanup into our cleanup. A plain `trap '...' EXIT` here would
+# REPLACE lock.sh's `trap _sa_release_locks EXIT INT TERM HUP`, orphaning
+# /tmp/social-autoposter-github.lock across runs (root cause of the stale
+# github-lock orphans seen 2026-04-29). All four signals must be covered so
+# watchdog SIGTERM also frees the lock.
+trap 'rm -f "$GH_REPLY_SUMMARY"; _sa_release_locks' EXIT INT TERM HUP
+( set +e +o pipefail
+  python3 "$REPO_DIR/scripts/stats.py" --github-only --reply-summary "$GH_REPLY_SUMMARY" 2>&1 | tee -a "$LOG_FILE"
+) || true
+PHASE_A5_ELAPSED=$(( $(date +%s) - PHASE_A5_START ))
+
+GH_REPLIES_REFRESHED=0
+if [ -s "$GH_REPLY_SUMMARY" ]; then
+    GH_REPLIES_REFRESHED=$(python3 -c "import json; print(json.load(open('$GH_REPLY_SUMMARY')).get('github', 0))" 2>/dev/null || echo 0)
+fi
+GH_ACTIVE=$(python3 "$GH_HELPER" posts-active-count 2>/dev/null | tr -d '[:space:]')
+[ -z "$GH_ACTIVE" ] && GH_ACTIVE=0
+# Emit a stats_github row so the dashboard Jobs table shows the github stats run
+# the same way it shows stats_reddit / stats_twitter.
+python3 "$REPO_DIR/scripts/log_run.py" --script "stats_github" --posted "$GH_ACTIVE" --skipped 0 --failed 0 --replies-refreshed "$GH_REPLIES_REFRESHED" --cost 0 --elapsed "$PHASE_A5_ELAPSED" || true
+log "Phase A.5: done (replies_refreshed=$GH_REPLIES_REFRESHED)"
+
+# ═══════════════════════════════════════════════════════
+# PHASE B: Respond to pending GitHub replies
+# ═══════════════════════════════════════════════════════
+PENDING_COUNT=$(python3 "$GH_HELPER" pending-count 2>/dev/null | tr -d '[:space:]')
+[ -z "$PENDING_COUNT" ] && PENDING_COUNT=0
+
+if [ "$PENDING_COUNT" -eq 0 ]; then
+    log "Phase B: No pending GitHub replies. Done!"
+    RUN_ELAPSED=$(( $(date +%s) - RUN_START ))
+    # Pull scan-stage counters from Phase A so the empty-engage row still shows
+    # "scanned N / 0 new" instead of all-zeros. scan_github_replies.py prints:
+    #   Scanning N GitHub issues for replies...
+    #   GitHub scan complete: N new pending, N skipped, N errors
+    GH_SCAN_PROC_LINE=$(grep -m1 -E "^Scanning [0-9]+ GitHub issues" "$LOG_FILE" 2>/dev/null || true)
+    GH_SCAN_DONE_LINE=$(grep -m1 "^GitHub scan complete:" "$LOG_FILE" 2>/dev/null || true)
+    GH_SCAN_ARG=""
+    if [ -n "$GH_SCAN_PROC_LINE" ] || [ -n "$GH_SCAN_DONE_LINE" ]; then
+      gh_scanned=$(echo "$GH_SCAN_PROC_LINE" | grep -oE "[0-9]+" | head -1)
+      gh_new=$(echo "$GH_SCAN_DONE_LINE" | grep -oE "[0-9]+ new pending" | grep -oE "[0-9]+" | head -1)
+      gh_skip=$(echo "$GH_SCAN_DONE_LINE" | grep -oE "[0-9]+ skipped" | grep -oE "[0-9]+" | head -1)
+      gh_err=$(echo "$GH_SCAN_DONE_LINE" | grep -oE "[0-9]+ errors" | grep -oE "[0-9]+" | head -1)
+      parts=""
+      [ -n "$gh_scanned" ] && parts="${parts}scanned=${gh_scanned},"
+      [ -n "$gh_new" ]     && parts="${parts}new=${gh_new},"
+      [ -n "$gh_skip" ] && [ "$gh_skip" -gt 0 ] && parts="${parts}backfill=${gh_skip},"
+      [ -n "$gh_err"  ] && [ "$gh_err"  -gt 0 ] && parts="${parts}unmatched=${gh_err},"
+      GH_SCAN_ARG="${parts%,}"
+    fi
+    if [ -n "$GH_SCAN_ARG" ]; then
+      python3 "$REPO_DIR/scripts/log_run.py" --script "engage_github" --posted 0 --skipped 0 --failed 0 --cost 0 --elapsed "$RUN_ELAPSED" --scan "$GH_SCAN_ARG"
+    else
+      python3 "$REPO_DIR/scripts/log_run.py" --script "engage_github" --posted 0 --skipped 0 --failed 0 --cost 0 --elapsed "$RUN_ELAPSED"
+    fi
+    find "$LOG_DIR" -name "github-engage-*.log" -mtime +7 -delete 2>/dev/null || true
+    exit 0
+fi
+
+log "Phase B: $PENDING_COUNT pending GitHub replies to process"
+
+# One-at-a-time thread-aware orchestrator. Each reply gets its own Claude session
+# with the full issue thread fetched via gh CLI, so Claude can see our prior
+# comments and decide reply-or-skip with a JSON escape hatch. See
+# scripts/engage_github.py for the prompt and skip-reason contract.
+python3 "$REPO_DIR/scripts/engage_github.py" --timeout 3000 2>&1 | tee -a "$LOG_FILE"
+
+# ═══════════════════════════════════════════════════════
+# PHASE C: Summary
+# ═══════════════════════════════════════════════════════
+# engage_github.py prints a canonical LOG_RUN_SUMMARY line; we parse it and
+# write ONE log_run.py row that also carries Phase A scan counters. Previously
+# engage_github.py wrote its own row with no scan info and the shell wrote
+# nothing on the has-work branch -- so productive cycles lost scan visibility
+# and empty cycles wrote two rows (one with scan, one without).
+RUN_ELAPSED=$(( $(date +%s) - RUN_START ))
+GH_SCAN_PROC_LINE=$(grep -m1 -E "^Scanning [0-9]+ GitHub issues" "$LOG_FILE" 2>/dev/null || true)
+GH_SCAN_DONE_LINE=$(grep -m1 "^GitHub scan complete:" "$LOG_FILE" 2>/dev/null || true)
+GH_SCAN_ARG=""
+if [ -n "$GH_SCAN_PROC_LINE" ] || [ -n "$GH_SCAN_DONE_LINE" ]; then
+  gh_scanned=$(echo "$GH_SCAN_PROC_LINE" | grep -oE "[0-9]+" | head -1)
+  gh_new=$(echo "$GH_SCAN_DONE_LINE" | grep -oE "[0-9]+ new pending" | grep -oE "[0-9]+" | head -1)
+  gh_skip=$(echo "$GH_SCAN_DONE_LINE" | grep -oE "[0-9]+ skipped" | grep -oE "[0-9]+" | head -1)
+  gh_err=$(echo "$GH_SCAN_DONE_LINE" | grep -oE "[0-9]+ errors" | grep -oE "[0-9]+" | head -1)
+  parts=""
+  [ -n "$gh_scanned" ] && parts="${parts}scanned=${gh_scanned},"
+  [ -n "$gh_new" ]     && parts="${parts}new=${gh_new},"
+  [ -n "$gh_skip" ] && [ "$gh_skip" -gt 0 ] && parts="${parts}backfill=${gh_skip},"
+  [ -n "$gh_err"  ] && [ "$gh_err"  -gt 0 ] && parts="${parts}unmatched=${gh_err},"
+  GH_SCAN_ARG="${parts%,}"
+fi
+
+GH_SUMMARY_LINE=$(grep -m1 "^\[engage_github\] LOG_RUN_SUMMARY" "$LOG_FILE" 2>/dev/null || true)
+GH_POSTED=0; GH_SKIPPED=0; GH_FAILED=0; GH_COST="0.0000"
+if [ -n "$GH_SUMMARY_LINE" ]; then
+  GH_POSTED=$(echo "$GH_SUMMARY_LINE"  | grep -oE "posted=[0-9]+"  | head -1 | cut -d= -f2)
+  GH_SKIPPED=$(echo "$GH_SUMMARY_LINE" | grep -oE "skipped=[0-9]+" | head -1 | cut -d= -f2)
+  GH_FAILED=$(echo "$GH_SUMMARY_LINE"  | grep -oE "failed=[0-9]+"  | head -1 | cut -d= -f2)
+  GH_COST=$(echo "$GH_SUMMARY_LINE"    | grep -oE "cost=[0-9.]+"   | head -1 | cut -d= -f2)
+  : "${GH_POSTED:=0}" "${GH_SKIPPED:=0}" "${GH_FAILED:=0}" "${GH_COST:=0.0000}"
+fi
+
+GH_LOG_RUN_ARGS=(--script "engage_github" --posted "$GH_POSTED" --skipped "$GH_SKIPPED" --failed "$GH_FAILED" --cost "$GH_COST" --elapsed "$RUN_ELAPSED")
+[ -n "$GH_SCAN_ARG" ] && GH_LOG_RUN_ARGS+=(--scan "$GH_SCAN_ARG")
+python3 "$REPO_DIR/scripts/log_run.py" "${GH_LOG_RUN_ARGS[@]}" || true
+
+# Print cumulative status for visibility in the log file. One HTTP roundtrip
+# for all three counts instead of three psql one-liners.
+GH_COUNTS_JSON=$(python3 "$GH_HELPER" reply-counts 2>/dev/null || echo '{"pending":0,"replied":0,"skipped":0}')
+TOTAL_PENDING=$(echo "$GH_COUNTS_JSON" | python3 -c "import sys,json; print(json.load(sys.stdin).get('pending',0))" 2>/dev/null || echo "0")
+TOTAL_REPLIED=$(echo "$GH_COUNTS_JSON" | python3 -c "import sys,json; print(json.load(sys.stdin).get('replied',0))" 2>/dev/null || echo "0")
+TOTAL_SKIPPED=$(echo "$GH_COUNTS_JSON" | python3 -c "import sys,json; print(json.load(sys.stdin).get('skipped',0))" 2>/dev/null || echo "0")
+
+log "GitHub replies cumulative: pending=$TOTAL_PENDING replied=$TOTAL_REPLIED skipped=$TOTAL_SKIPPED"
+
+log "=== GitHub Engagement complete: $(date) ==="
+
+# Clean up old logs
+find "$LOG_DIR" -name "github-engage-*.log" -mtime +7 -delete 2>/dev/null || true

package/skill/ingest-web-chat-replies.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# ingest-web-chat-replies.sh — Poll Gmail for [WEB-CHAT #N] replies and forward
+# them to visitors via Resend. Called by launchd every 5 minutes.
+#
+# Mirror of ~/social-autoposter/skill/dm-replies-ingest pattern (the
+# ingest_human_dm_replies.py launchd rail), specialised for the web-chat thread.
+
+set -euo pipefail
+
+source "$(dirname "$0")/lock.sh"
+acquire_lock "ingest-web-chat-replies" 60
+
+# DB access is HTTP-only via scripts/http_api.py -> s4l.ai /api/v1/web-chat/*.
+# No DATABASE_URL needed here any more.
+
+ANALYTICS_ENV="$HOME/analytics/.env.production.local"
+if [ -f "$ANALYTICS_ENV" ]; then
+    export RESEND_API_KEY=$(grep '^RESEND_API_KEY=' "$ANALYTICS_ENV" | sed 's/^RESEND_API_KEY=//' | tr -d '"' | tr -d '\\n')
+fi
+export NODE_PATH="$HOME/analytics/node_modules"
+
+REPO_DIR="$HOME/social-autoposter"
+LOG_DIR="$REPO_DIR/skill/logs"
+mkdir -p "$LOG_DIR"
+
+PYTHON_BIN="${PYTHON_BIN:-/opt/homebrew/bin/python3.11}"
+[ -x "$PYTHON_BIN" ] || PYTHON_BIN="/usr/bin/python3"
+
+LOG_FILE="$LOG_DIR/web-chat-ingest.log"
+echo "[$(date)] starting ingest" >> "$LOG_FILE"
+
+"$PYTHON_BIN" "$REPO_DIR/scripts/ingest_web_chat_replies.py" >> "$LOG_FILE" 2>&1 || \
+    echo "[$(date)] ERROR: ingest_web_chat_replies.py failed" >> "$LOG_FILE"
+
+# Trim log.
+if [ -f "$LOG_FILE" ]; then
+    tail -2000 "$LOG_FILE" > "$LOG_FILE.tmp" 2>/dev/null && mv "$LOG_FILE.tmp" "$LOG_FILE" || true
+fi

package/skill/invent-supply-test.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+# invent-supply-test.sh — supply-test a batch of drafted Twitter queries for the
+# topic-invention job (scripts/invent_topics.py).
+#
+# Mirrors run-twitter-cycle.sh's Phase 1 lean scan loop, but stands alone so the
+# hourly invent job can measure how much FRESH (6h) supply each freshly-drafted
+# query returns BEFORE committing its parent topic. It:
+#
+#   1. Acquires the same "twitter-browser" mkdir-lock the cycle uses, so an
+#      invent run and a 15-min cycle never fight over the managed Chrome on
+#      port 9555. Short timeout (default 600s): if the cycle is mid-scan we'd
+#      rather skip this hour than block. acquire_lock exits 0 on timeout, which
+#      leaves SCAN_OUT empty and signals "untested" back to Python.
+#   2. Runs ONE browser-harness -c invocation that loops twitter_scan.scan()
+#      over every query, writing one JSONL record per query to SCAN_OUT (via the
+#      SCAN_TWEETS_FILE env the scan module already honors). scan() writes a
+#      record even on a zero-tweet result, so a present-but-empty tweets array
+#      is a real "tested, 0 supply" signal, distinct from a missing record.
+#   3. Releases the lock.
+#
+# Usage:
+#   invent-supply-test.sh <queries_json> <scan_out> [freshness_hours] [lock_timeout_s]
+#
+#   queries_json : path to a JSON array of {project, query, search_topic}
+#   scan_out     : path the per-query JSONL results are written to (truncated first)
+#   freshness_hours (default 6)
+#   lock_timeout_s  (default 600)
+#
+# Exit status is always 0 on a clean run (tested or skipped); Python decides
+# tested-vs-untested by whether SCAN_OUT has records. Mirrors the cycle's
+# fail-open posture so a transient browser hiccup never crashes the invent job.
+set -uo pipefail
+
+REPO_DIR="$HOME/social-autoposter"
+HARNESS_BIN="$HOME/.local/bin/browser-harness"
+
+QUERIES_JSON="${1:?queries_json path required}"
+SCAN_OUT="${2:?scan_out path required}"
+FRESHNESS_HOURS="${3:-6}"
+LOCK_TIMEOUT="${4:-600}"
+
+if [ ! -s "$QUERIES_JSON" ]; then
+  echo "[invent-supply-test] queries json missing/empty: $QUERIES_JSON" >&2
+  exit 0
+fi
+if [ ! -x "$HARNESS_BIN" ]; then
+  echo "[invent-supply-test] browser-harness not found at $HARNESS_BIN" >&2
+  exit 0
+fi
+
+# Truncate the output so a stale file from a prior run can't masquerade as
+# fresh results. Python treats an empty SCAN_OUT as "untested this run".
+: > "$SCAN_OUT"
+
+# Source the shared lock helpers (functions only; no lock acquired on source).
+# shellcheck disable=SC1091
+source "$REPO_DIR/skill/lock.sh"
+
+# acquire_lock exits 0 if the lock can't be taken within LOCK_TIMEOUT, which
+# unwinds this whole script — leaving SCAN_OUT empty. That's the intended
+# "skip this hour" path: the cycle owns the browser right now.
+echo "[invent-supply-test] acquiring twitter-browser lock (timeout=${LOCK_TIMEOUT}s)..." >&2
+acquire_lock "twitter-browser" "$LOCK_TIMEOUT"
+echo "[invent-supply-test] twitter-browser lock held (pid=$$)" >&2
+
+# One harness invocation handles every query so we pay the CLI startup once.
+# Each scan() call appends a JSONL record to SCAN_TWEETS_FILE=$SCAN_OUT.
+# browser-harness upstream main reads the script from STDIN (the `-c` flag was
+# removed). Feed the body via a quoted heredoc and pass $REPO_DIR / $QUERIES_JSON
+# through the environment so the Python reads them from os.environ.
+BU_NAME=twitter-harness BU_CDP_URL=http://127.0.0.1:9555 \
+SCAN_TWEETS_FILE="$SCAN_OUT" \
+BATCH_ID="${BATCH_ID:-}" \
+FRESHNESS_HOURS_DISCOVER="$FRESHNESS_HOURS" \
+REPO_DIR="$REPO_DIR" \
+QUERIES_JSON="$QUERIES_JSON" \
+  "$HARNESS_BIN" <<'PY' 2>&1
+import sys, json, os, time
+sys.path.insert(0, os.environ['REPO_DIR'] + '/scripts')
+from twitter_scan import scan
+queries = json.load(open(os.environ['QUERIES_JSON']))
+freshness = int(os.environ.get('FRESHNESS_HOURS_DISCOVER', '6'))
+for q in queries:
+    project = q.get('project', '')
+    query = q.get('query', '')
+    topic = q.get('search_topic', '')
+    t0 = time.time()
+    try:
+        kept = scan(query=query, project=project, search_topic=topic,
+                    freshness_hours=freshness)
+        dt = time.time() - t0
+        print(f'  ok  project={project!r}  q={query[:50]!r}  kept={len(kept)}  in {dt:.1f}s', flush=True)
+    except Exception as e:
+        dt = time.time() - t0
+        print(f'  err project={project!r}  q={query[:50]!r}  in {dt:.1f}s  {type(e).__name__}: {e}', flush=True)
+PY
+
+release_lock "twitter-browser"
+echo "[invent-supply-test] done; results in $SCAN_OUT" >&2
+exit 0

package/skill/invent-topics.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# invent-topics.sh — hourly topic invention job (replaces in-cycle EXPLORE_INVENT).
+#
+# Picks ONE project via pick_projects() (same inverse-recent-share weighting
+# the post-comments cycle uses), then calls Claude to propose N new
+# search_topic candidates given that project's ledger. Validates each
+# proposal against the universe (exact-match + Jaccard similarity), commits
+# survivors to project_search_topics with source='invented', status='active',
+# and appends an audit row to state/invented_topics_audit.jsonl.
+#
+# Fires hourly via com.m13v.social-invent-topics.plist. Deliberately runs
+# outside the 15-min cycle cadence because invention doesn't need realtime;
+# new topics added at minute :00 vs :30 make no difference to engagement.
+
+set -uo pipefail
+
+# shellcheck source=/dev/null
+[ -f "$HOME/social-autoposter/.env" ] && source "$HOME/social-autoposter/.env"
+
+REPO_DIR="$HOME/social-autoposter"
+LOG_DIR="$REPO_DIR/skill/logs"
+mkdir -p "$LOG_DIR"
+LOG_FILE="$LOG_DIR/invent-topics-$(date +%Y-%m-%d_%H%M%S).log"
+
+# Number of candidate topics to ask Claude for per attempt. One topic per loop
+# matches the new supply-test rhythm: invent ONE topic, draft its queries,
+# supply-test, gate on supply, decide whether to loop again. With the post-
+# 2026-05-29 dupe-retry-doesn't-burn-attempts behavior, asking for more than
+# one is wasteful (a dupe-only Claude call retries cost-free anyway).
+# Override via INVENT_PROPOSALS_PER_RUN.
+PROPOSALS="${INVENT_PROPOSALS_PER_RUN:-1}"
+
+# Stop the run as soon as ONE topic clears the supply floor — the qualifying
+# tweet count IS the real target, not "how many topics qualified." A single
+# topic with supply >= SUPPLY_FLOOR fresh tweets is enough; no need to keep
+# burning Claude calls on additional topics that hour. MAX_ATTEMPTS caps the
+# loop only if the project is genuinely dry (no qualifier in N tries).
+TARGET="${INVENT_TARGET:-1}"
+MAX_ATTEMPTS="${INVENT_MAX_ATTEMPTS:-5}"
+
+{
+    echo "[$(date +%Y-%m-%dT%H:%M:%S%z)] invent-topics start (proposals=$PROPOSALS target=$TARGET max_attempts=$MAX_ATTEMPTS)"
+    /usr/bin/python3 "$REPO_DIR/scripts/invent_topics.py" \
+        --proposals "$PROPOSALS" \
+        --target "$TARGET" \
+        --max-attempts "$MAX_ATTEMPTS"
+    rc=$?
+    echo "[$(date +%Y-%m-%dT%H:%M:%S%z)] invent-topics done rc=$rc"
+    exit $rc
+} 2>&1 | tee -a "$LOG_FILE"