@m13v/s4l 1.7.6-rc.2 → 1.7.6-rc.21

package/skill/run-twitter-cycle.sh

@@ -1675,6 +1675,66 @@ if [ -z "$CANDIDATES" ]; then
     exit 0
 fi
 
+# --- ROLLING VIRALITY BAR fetch + candidate-level PREFILTER (2026-07-17) -----
+# The p0.99 bar (see the PRE-DRAFT VIRALITY GATE below for history) is fetched
+# HERE, immediately after the candidate fetch, and applied as a deterministic
+# per-candidate filter BEFORE the prompt is ever built: when the bar is active,
+# below-bar candidates are removed from $CANDIDATES so the drafting session
+# only sees candidates that can actually enter the plan (plan-parse keeps the
+# top-1 on-brand pick ONLY if it clears this same bar). Measured 2026-07-17,
+# before this prefilter: 125 below-bar draft-reuse offers across 50 prompts
+# converted into 0 of 24 plan entries, so drafting below-bar candidates bought
+# nothing. Filtered rows are NOT expired or rejected: they stay 'pending', are
+# salvaged by later cycles, and re-enter the prompt automatically if their
+# thread's virality rises above the bar within the freshness window. Placement
+# AFTER the empty-batch expire check above is deliberate: a batch that is
+# non-empty but fully below-bar must NOT be expired, only skipped this cycle.
+# Fail-open: bar OFF (cold start / thin pool / fetch failure / sandbox) means
+# no filtering and drafting behaves exactly as before.
+if [ -n "${S4L_SANDBOX_CANDIDATES_FILE:-}" ]; then
+    # The fetch below reads /api/v1/twitter-candidates/virality-threshold under
+    # THIS machine's own identity (http_api.py's X-Installation), so it would
+    # reflect the OPERATOR's trailing-24h candidate pool -- the wrong baseline
+    # entirely when replaying another install's (e.g. Karol's, Nhat's)
+    # candidates. A sandbox test also wants every model-approved draft
+    # visible, not gated to only the top-1 that would clear a live posting
+    # cadence bar which, again, doesn't apply since nothing is ever posted.
+    VIRALITY_THRESHOLD=""
+    log "Virality bar OFF for sandbox mode (would reflect the operator's own pool, not the replayed install's; every model-approved draft stays visible)."
+else
+    VIRALITY_THRESHOLD=$(S4L_VPCTILE="0.99" \
+        S4L_VMIN="${S4L_TWITTER_VIRALITY_MIN_SAMPLE:-200}" \
+        S4L_SCRIPTS_DIR="$REPO_DIR/scripts" \
+        python3 -c "
+import os, sys
+_repo = os.path.expanduser(os.environ.get('S4L_REPO_DIR') or os.environ.get('REPO_DIR') or '~/social-autoposter')
+sys.path.insert(0, os.environ.get('S4L_SCRIPTS_DIR') or os.path.join(_repo, 'scripts'))
+from http_api import api_get
+try:
+    r = api_get('/api/v1/twitter-candidates/virality-threshold',
+                {'pctile': os.environ['S4L_VPCTILE'], 'hours': 24})
+    d = (r or {}).get('data') or {}
+    thr = d.get('threshold')
+    n = int(d.get('sample_count') or 0)
+    mn = int(os.environ['S4L_VMIN'])
+    if thr is not None and n >= mn:
+        print(f'{float(thr):.4f}')
+except BaseException as e:
+    sys.stderr.write(f'virality-bar fetch failed (bar OFF this cycle): {e}\n')
+" 2>>"$LOG_FILE" || echo "")
+    if [ -n "$VIRALITY_THRESHOLD" ]; then
+        log "Virality bar ACTIVE: p0.99 = $VIRALITY_THRESHOLD (this install, trailing 24h); top-1 kept only if it clears the bar."
+    else
+        log "Virality bar OFF this cycle (cold-start/thin pool or fetch failed); top-1 kept ungated."
+    fi
+fi
+if [ -n "$VIRALITY_THRESHOLD" ] && [ -n "$CANDIDATES" ]; then
+    _VF_TOTAL=$(printf '%s\n' "$CANDIDATES" | grep -c '^[0-9]' || true)
+    CANDIDATES=$(printf '%s\n' "$CANDIDATES" | awk -F'|' -v b="$VIRALITY_THRESHOLD" '$1 ~ /^[0-9]+$/ && NF>=5 && $5+0 >= b+0')
+    _VF_KEPT=$(printf '%s\n' "$CANDIDATES" | grep -c '^[0-9]' || true)
+    log "[virality_prefilter] kept ${_VF_KEPT:-0} of ${_VF_TOTAL:-0} candidate(s) at/above bar=$VIRALITY_THRESHOLD; below-bar rows stay pending and are never sent to the drafter"
+fi
+
 # --- SCAN_ONLY gate: stop after scoring, emit candidates, skip drafting -------
 # When SCAN_ONLY=1 the cycle runs scan -> score -> top-N select, writes the chosen
 # candidates as JSON, and STOPS before the claude drafting step. The MCP
@@ -1784,13 +1844,43 @@ done <<< "$CANDIDATES"
 # (honoring the same S4L_SANDBOX_CONFIG_DIR override and lane whitelist/ops
 # denylist). See the render-twitter call below.
 
+# --- Draft-prompt A/B arm (v4, voice-first) ----------------------------------
+# Owned by scripts/draft_prompt_core.py (single source of truth, shared with
+# the Reddit draft lane): the full experiment history, both arm texts, the
+# Draft-B divergence note, the persona-lane directive, and the pick
+# precedence (pre-set env for sandbox forcing -> server-side per-install pin
+# -> 50/50 coin flip via TWITTER_DRAFT_PROMPT_AB_RATE) ALL live there. The
+# arm is stamped onto every post this cycle via S4L_DRAFT_PROMPT_VARIANT
+# (active_experiments.collect -> plan candidates -> twitter_post_plan.py ->
+# posts.draft_prompt_variant), and engagement_styles.get_assigned_style_prompt
+# reads the SAME env var at render time in this process.
+#
+# MOVED HERE, before the style picker, on 2026-07-17: treatment_v4 assigns no
+# engagement style at all (see the style-picker block below), so the arm must
+# be known BEFORE deciding whether to run the picker. Do not move this back
+# below the style picker.
+source "$REPO_DIR/skill/styles.sh"
+S4L_DRAFT_PROMPT_VARIANT=$(python3 "$REPO_DIR/scripts/draft_prompt_core.py" pick-arm 2>>"$LOG_FILE" || echo treatment_v4)
+export S4L_DRAFT_PROMPT_VARIANT
+log "Draft-prompt A/B arm: $S4L_DRAFT_PROMPT_VARIANT (draft_prompt_core pick-arm: env>pin>coin)"
+
 # Engagement-style picker (2026-05-19): pick ONE assigned style for this
 # cycle. The picked style flows two places: (1) --style filter for
 # top_performers.py so the per-style exemplars section shows only posts
 # matching the assigned style, (2) s4l_render_style_block (below) so the
 # prompt block embeds the same assignment. On invent mode picked_style is
 # empty and top_performers stays unfiltered (model sees full landscape).
-source "$REPO_DIR/skill/styles.sh"
+#
+# Called UNCONDITIONALLY regardless of arm (2026-07-17 consolidation): under
+# treatment_v4, pick_style_for_post()/pick_exploration_style() themselves
+# (scripts/engagement_styles.py) refuse to assign a real style and return
+# the "voice_first" sentinel directly -- no network round-trip, and no arm
+# check needed here in the shell. This used to be a Twitter-only branch
+# here that skipped the picker and blanked PICKED_STYLE by hand; that logic
+# moved into the picker functions themselves so every caller (this bash
+# driver AND scripts/post_reddit.py, which calls the same functions
+# directly and was never covered by the old Twitter-only branch) gets the
+# same behavior automatically. Do not reintroduce an arm check in this file.
 STYLE_ASSIGN_FILE=$(mktemp -t s4l_twitter_assign_XXXXXX.json)
 s4l_pick_style twitter posting "$STYLE_ASSIGN_FILE" >/dev/null 2>&1 || true
 PICKED_STYLE=$(python3 -c "
@@ -1821,7 +1911,7 @@ log "Engagement style assigned: mode=$PICKED_MODE style=${PICKED_STYLE:-(invent)
 # are actually different (a same-name pair defeats the whole point); INVENT
 # mode on either side is accepted immediately since an invented name is
 # definitionally distinct from a pinned one. This is orthogonal to the
-# treatment_v4/control_v4 draft-prompt A/B below (that varies WORDING of the
+# treatment_v4/control_v4 draft-prompt A/B above (that varies WORDING of the
 # directive for the whole cycle; this varies STYLE per draft slot), so neither
 # experiment disturbs the other.
 STYLE_ASSIGN_FILE_B=$(mktemp -t s4l_twitter_assign_b_XXXXXX.json)
@@ -1889,20 +1979,6 @@ fi
 export S4L_EXP_DRAFT_B_SOURCE="$DRAFT_B_SOURCE"
 log "Engagement style B assigned: mode=$PICKED_MODE_B style=${PICKED_STYLE_B:-(invent)} source=$DRAFT_B_SOURCE"
 
-# --- Draft-prompt A/B arm (v4, voice-first) ----------------------------------
-# Owned by scripts/draft_prompt_core.py (single source of truth, shared with
-# the Reddit draft lane): the full experiment history, both arm texts, the
-# Draft-B divergence note, the persona-lane directive, and the pick
-# precedence (pre-set env for sandbox forcing -> server-side per-install pin
-# -> 50/50 coin flip via TWITTER_DRAFT_PROMPT_AB_RATE) ALL live there. The
-# arm is stamped onto every post this cycle via S4L_DRAFT_PROMPT_VARIANT
-# (active_experiments.collect -> plan candidates -> twitter_post_plan.py ->
-# posts.draft_prompt_variant), and engagement_styles.get_assigned_style_prompt
-# reads the SAME env var at render time in this process.
-S4L_DRAFT_PROMPT_VARIANT=$(python3 "$REPO_DIR/scripts/draft_prompt_core.py" pick-arm 2>>"$LOG_FILE" || echo treatment_v4)
-export S4L_DRAFT_PROMPT_VARIANT
-log "Draft-prompt A/B arm: $S4L_DRAFT_PROMPT_VARIANT (draft_prompt_core pick-arm: env>pin>coin)"
-
 # 2026-07-10 anti-sameness: --no-project-sections strips the multi-project
 # winner corpus (~400 lines of "Top Posts by Project" + summary table) that
 # used to ride along in EVERY draft prompt and homogenize drafts. The prep
@@ -2031,6 +2107,28 @@ SKIP_FILE="/tmp/twitter_cycle_skips_${BATCH_ID}.json"
 if [ -z "${S4L_SANDBOX_CANDIDATES_FILE:-}" ]; then
     python3 "$REPO_DIR/scripts/twitter_batch_phase.py" advance "$BATCH_ID" --phase phase2b-prep 2>&1 | tee -a "$LOG_FILE" || true
 fi
+
+# --- PRE-DRAFT VIRALITY GATE (2026-07-15; prefilter degenerate case 2026-07-17) ---
+# The bar is now fetched and applied as a candidate-level PREFILTER right after
+# the candidate fetch (see [virality_prefilter] above), so the drafting session
+# never sees a candidate that cannot enter the plan. This gate is the leftover
+# degenerate case: NOTHING survived the prefilter => skip media capture and the
+# whole Claude drafting session and synthesize an empty prep envelope through
+# the existing plan_count=0 path. Candidates stay status='pending' (never
+# expired or rejected here); Phase 0 salvages and re-judges them next cycle.
+# With the bar OFF the prefilter is a no-op and this gate can only fire on an
+# empty candidate list, which the expire check above already handles.
+PREGATE_SKIP=0
+_PG_N=$(printf '%s\n' "$CANDIDATES" | grep -c '^[0-9]' || true)
+if [ "${_PG_N:-0}" = "0" ]; then
+    PREGATE_SKIP=1
+    if [ -n "$VIRALITY_THRESHOLD" ]; then
+        log "[virality_pregate] SKIP drafting: no candidate at/above bar=$VIRALITY_THRESHOLD survived the prefilter; rows stay pending for next-cycle salvage"
+    else
+        log "[virality_pregate] SKIP drafting: candidate list is empty"
+    fi
+fi
+
 # Thread-media capture (2026-06-03, gated by S4L_TWITTER_CAPTURE_MEDIA, default
 # OFF). When enabled, does ONE cheap deterministic pass over every candidate
 # thread to pull its media (images/videos/GIFs/link-cards), persist each into
@@ -2047,7 +2145,11 @@ fi
 # capture off). Found live: a sandbox test run waited ~6 minutes in the FIFO
 # lock queue behind a real cycle for a lock it never ended up needing.
 MEDIA_BLOCK=""
-if [ "${S4L_TWITTER_CAPTURE_MEDIA:-0}" = "1" ] || [ "${S4L_TWITTER_CAPTURE_MEDIA:-}" = "true" ]; then
+if [ "$PREGATE_SKIP" = "1" ]; then
+    # Virality pre-gate: no draft will be produced this cycle, so thread-media
+    # capture would be pure waste; skip it along with the drafting session.
+    rm -f "$MEDIA_URLS_FILE" 2>/dev/null || true
+elif [ "${S4L_TWITTER_CAPTURE_MEDIA:-0}" = "1" ] || [ "${S4L_TWITTER_CAPTURE_MEDIA:-}" = "true" ]; then
     log "Re-acquiring twitter-browser lock for Phase 2b-prep (read+draft only)..."
     acquire_lock "twitter-browser" 3600 2>>"$LOG_FILE"
     log "twitter-browser lock held (pid=$$) Phase 2b-prep"
@@ -2166,20 +2268,29 @@ fi
 # only the exemplar context, so this is the ONLY durable full-prompt record.
 # Local-only, newest 50 kept (file cleanup, not candidate-row retention; the
 # no-retention rule covers DB *_candidates rows). Never blocks the run.
-PREP_PROMPT_DIR="${S4L_STATE_DIR:-$HOME/.social-autoposter-mcp}/prep-prompts"
-if mkdir -p "$PREP_PROMPT_DIR" 2>/dev/null; then
-    _PP_FILE="$PREP_PROMPT_DIR/prep-prompt-$BATCH_ID.md"
-    if printf '%s' "$PREP_PROMPT" > "$_PP_FILE" 2>/dev/null; then
-        ls -t "$PREP_PROMPT_DIR"/prep-prompt-*.md 2>/dev/null | tail -n +51 | while IFS= read -r _pp_old; do
-            rm -f "$_pp_old"
-        done
-        log "[prep_prompt_snapshot] batch=$BATCH_ID bytes=$(wc -c < "$_PP_FILE" | tr -d ' ') path=$_PP_FILE"
-    else
-        log "WARN: prep-prompt snapshot write failed for batch=$BATCH_ID (non-fatal)"
+if [ "$PREGATE_SKIP" = "1" ]; then
+    # Virality pre-gate fired: skip the prompt snapshot (nothing is sent) and
+    # the whole Claude drafting session, and synthesize the empty envelope the
+    # plan parser already handles (identical outcome to "drafted, all picks
+    # dropped below bar": plan_count=0, candidates stay pending for salvage).
+    log "[virality_pregate] drafting session skipped; synthesizing empty prep envelope"
+    PREP_OUTPUT='{"type":"result","subtype":"success","is_error":false,"structured_output":{"candidates":[],"rejected":[]}}'
+else
+    PREP_PROMPT_DIR="${S4L_STATE_DIR:-$HOME/.social-autoposter-mcp}/prep-prompts"
+    if mkdir -p "$PREP_PROMPT_DIR" 2>/dev/null; then
+        _PP_FILE="$PREP_PROMPT_DIR/prep-prompt-$BATCH_ID.md"
+        if printf '%s' "$PREP_PROMPT" > "$_PP_FILE" 2>/dev/null; then
+            ls -t "$PREP_PROMPT_DIR"/prep-prompt-*.md 2>/dev/null | tail -n +51 | while IFS= read -r _pp_old; do
+                rm -f "$_pp_old"
+            done
+            log "[prep_prompt_snapshot] batch=$BATCH_ID bytes=$(wc -c < "$_PP_FILE" | tr -d ' ') path=$_PP_FILE"
+        else
+            log "WARN: prep-prompt snapshot write failed for batch=$BATCH_ID (non-fatal)"
+        fi
     fi
-fi
 
-PREP_OUTPUT=$(printf '%s' "$PREP_PROMPT" | "$REPO_DIR/scripts/run_claude.sh" "run-twitter-cycle-prep" --strict-mcp-config --mcp-config "$TW_MCP_CONFIG" --allowedTools WebSearch,WebFetch -p --output-format json --json-schema "$PREP_SCHEMA" 2>&1)
+    PREP_OUTPUT=$(printf '%s' "$PREP_PROMPT" | "$REPO_DIR/scripts/run_claude.sh" "run-twitter-cycle-prep" --strict-mcp-config --mcp-config "$TW_MCP_CONFIG" --allowedTools WebSearch,WebFetch -p --output-format json --json-schema "$PREP_SCHEMA" 2>&1)
+fi
 
 echo "$PREP_OUTPUT" >> "$LOG_FILE"
 
@@ -2227,43 +2338,12 @@ fi
 # reach-recovery: quality over volume): single source of truth, no env
 # var, no fallback, one path (every install behaves identically regardless of how
 # its plist was generated). Sample floor S4L_TWITTER_VIRALITY_MIN_SAMPLE default 200.
-if [ -n "${S4L_SANDBOX_CANDIDATES_FILE:-}" ]; then
-    # The fetch below reads /api/v1/twitter-candidates/virality-threshold under
-    # THIS machine's own identity (http_api.py's X-Installation), so it would
-    # reflect the OPERATOR's trailing-24h candidate pool -- the wrong baseline
-    # entirely when replaying another install's (e.g. Karol's, Nhat's)
-    # candidates. A sandbox test also wants every model-approved draft
-    # visible, not gated to only the top-1 that would clear a live posting
-    # cadence bar which, again, doesn't apply since nothing is ever posted.
-    VIRALITY_THRESHOLD=""
-    log "Virality bar OFF for sandbox mode (would reflect the operator's own pool, not the replayed install's; every model-approved draft stays visible)."
-else
-    VIRALITY_THRESHOLD=$(S4L_VPCTILE="0.99" \
-        S4L_VMIN="${S4L_TWITTER_VIRALITY_MIN_SAMPLE:-200}" \
-        S4L_SCRIPTS_DIR="$REPO_DIR/scripts" \
-        python3 -c "
-import os, sys
-_repo = os.path.expanduser(os.environ.get('S4L_REPO_DIR') or os.environ.get('REPO_DIR') or '~/social-autoposter')
-sys.path.insert(0, os.environ.get('S4L_SCRIPTS_DIR') or os.path.join(_repo, 'scripts'))
-from http_api import api_get
-try:
-    r = api_get('/api/v1/twitter-candidates/virality-threshold',
-                {'pctile': os.environ['S4L_VPCTILE'], 'hours': 24})
-    d = (r or {}).get('data') or {}
-    thr = d.get('threshold')
-    n = int(d.get('sample_count') or 0)
-    mn = int(os.environ['S4L_VMIN'])
-    if thr is not None and n >= mn:
-        print(f'{float(thr):.4f}')
-except BaseException as e:
-    sys.stderr.write(f'virality-bar fetch failed (bar OFF this cycle): {e}\n')
-" 2>>"$LOG_FILE" || echo "")
-    if [ -n "$VIRALITY_THRESHOLD" ]; then
-        log "Virality bar ACTIVE: p0.99 = $VIRALITY_THRESHOLD (this install, trailing 24h); top-1 kept only if it clears the bar."
-    else
-        log "Virality bar OFF this cycle (cold-start/thin pool or fetch failed); top-1 kept ungated."
-    fi
-fi
+# VIRALITY_THRESHOLD is fetched ONCE per cycle by the candidate-level
+# PREFILTER right after the candidate fetch (2026-07-17; before that by the
+# 2026-07-15 pre-draft gate; originally fetched here, post-draft). The parse
+# below reuses the value; with the prefilter active its below-bar drop should
+# never fire and remains as belt-and-suspenders. Sandbox runs carry an empty
+# threshold (bar OFF) from the same block.
 
 # Parse the prep envelope and write the plan to \$PLAN_FILE; also extract the
 # 'rejected' array into \$SKIP_FILE so log_twitter_skips.py can persist a