@m13v/s4l 1.6.197-rc.7

package/scripts/ig_batch_creator.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+# Download + transcribe every IG post for one creator.
+# Usage: ./ig_batch_creator.sh <ig_handle>
+#
+# Reads URLs from scripts/ig_creators_run/<handle>/urls.txt (one post URL per line).
+# Writes <shortcode>.{mp4,m4a,info.json,deepgram.json} into the same dir.
+# Idempotent: skips a post if both .mp4 and .deepgram.json already exist.
+# Stops the loop after N consecutive download failures (likely IG rate-limit).
+set -uo pipefail
+
+HANDLE="${1:?ig handle required}"
+ROOT="/Users/matthewdi/social-autoposter/scripts/ig_creators_run"
+OUT="$ROOT/$HANDLE"
+URLS="$OUT/urls.txt"
+LOG="$OUT/run.log"
+FAIL_STREAK_LIMIT="${FAIL_STREAK_LIMIT:-5}"
+SLEEP_BETWEEN="${SLEEP_BETWEEN:-3}"
+
+[[ -f "$URLS" ]] || { echo "ERROR: $URLS missing"; exit 2; }
+
+DEEPGRAM_API_KEY="$(grep -E '^DEEPGRAM_API_KEY=' /Users/matthewdi/fazm/web/.env.local | head -1 | cut -d= -f2-)"
+[[ -n "$DEEPGRAM_API_KEY" ]] || { echo "ERROR: no DEEPGRAM_API_KEY"; exit 1; }
+
+TOTAL=$(wc -l < "$URLS" | tr -d ' ')
+echo "[start] handle=$HANDLE total=$TOTAL out=$OUT" | tee -a "$LOG"
+
+i=0
+ok=0
+skipped=0
+failed=0
+streak=0
+
+while IFS= read -r URL; do
+  [[ -z "$URL" ]] && continue
+  i=$((i+1))
+  SHORT=$(echo "$URL" | sed -E 's|.*/(reel\|p)/([^/]+)/?.*|\2|')
+  MP4="$OUT/${SHORT}.mp4"
+  DGM="$OUT/${SHORT}.deepgram.json"
+  printf "[%2d/%d] %s " "$i" "$TOTAL" "$SHORT"
+
+  if [[ -f "$MP4" && -f "$DGM" ]]; then
+    echo "skip (already done)" | tee -a "$LOG"
+    skipped=$((skipped+1))
+    continue
+  fi
+
+  # download
+  if [[ ! -f "$MP4" ]]; then
+    if ! yt-dlp --cookies-from-browser chrome --no-warnings -q \
+           -o "$OUT/${SHORT}.%(ext)s" --write-info-json "$URL" \
+           >>"$LOG" 2>&1; then
+      echo "FAIL download" | tee -a "$LOG"
+      failed=$((failed+1))
+      streak=$((streak+1))
+      if (( streak >= FAIL_STREAK_LIMIT )); then
+        echo "[stop] $streak consecutive failures, likely rate-limited" | tee -a "$LOG"
+        break
+      fi
+      sleep "$SLEEP_BETWEEN"
+      continue
+    fi
+  fi
+  streak=0
+
+  # audio
+  AUDIO="$OUT/${SHORT}.m4a"
+  if [[ ! -f "$AUDIO" ]]; then
+    ffmpeg -y -loglevel error -i "$MP4" -vn -c:a copy "$AUDIO" 2>>"$LOG" \
+      || ffmpeg -y -loglevel error -i "$MP4" -vn -c:a aac -b:a 96k "$AUDIO" 2>>"$LOG" \
+      || { echo "FAIL audio" | tee -a "$LOG"; failed=$((failed+1)); continue; }
+  fi
+
+  # transcribe
+  if [[ ! -f "$DGM" ]]; then
+    HTTP=$(curl -sS -o "$DGM" -w "%{http_code}" -X POST \
+      -H "Authorization: Token ${DEEPGRAM_API_KEY}" \
+      -H "Content-Type: audio/m4a" \
+      --data-binary "@${AUDIO}" \
+      "https://api.deepgram.com/v1/listen?model=nova-3&smart_format=true&punctuate=true&detect_language=true")
+    if [[ "$HTTP" != "200" ]]; then
+      echo "FAIL deepgram http=$HTTP" | tee -a "$LOG"
+      failed=$((failed+1))
+      continue
+    fi
+  fi
+
+  DUR=$(python3 -c "import json,sys; d=json.load(open('$DGM')); print(d['metadata'].get('duration',''))" 2>/dev/null)
+  echo "ok dur=${DUR}s" | tee -a "$LOG"
+  ok=$((ok+1))
+  sleep "$SLEEP_BETWEEN"
+done < "$URLS"
+
+echo "[done] ok=$ok skipped=$skipped failed=$failed processed=$i/$TOTAL" | tee -a "$LOG"

package/scripts/ig_post_type_picker.py

@@ -0,0 +1,243 @@
+#!/opt/homebrew/bin/python3.11
+"""
+Pick the next IG post type (organic vs product) and the next pending video of
+that type. Writes one JSON line to stdout for the shell harness to read.
+
+Algorithm: inverse-recent-share weighting, identical to the Twitter pipeline's
+scripts/pick_project.py. effective_weight = config_weight / (1 + posts in the
+last RECENT_WINDOW_DAYS). Configured via the `instagram` block in config.json:
+  post_type_weights: { organic: N, product: M }   # relative target shares
+  recent_window_days: 7                            # rolling window
+A type that has been posting heavily is dampened toward under-posted ones, but
+never selected above its raw config weight. Settles toward the target ratio
+over time.
+
+Usage:
+  ig_post_type_picker.py                  # pick across all accounts (legacy)
+  ig_post_type_picker.py --account NAME   # scope to one target_account
+
+Output:
+  {"post_type": "organic", "video_path": "...", "post_number": 4,
+   "target_account": "matt_diak", "reason": "...", "fallback": false}
+
+Exit codes:
+  0  — picked successfully
+  2  — no draft videos of either type (queue exhausted for the scoped account)
+  3  — config error / DB error
+"""
+
+import argparse
+import json
+import os
+import random
+import sys
+from pathlib import Path
+
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__))))
+from http_api import api_get
+
+CONFIG_FILE = Path.home() / "social-autoposter" / "config.json"
+
+
+def load_ig_config():
+    """Return (post_type_weights dict, recent_window_days int, product_cooldown_posts int) from config.json."""
+    cfg = json.loads(CONFIG_FILE.read_text())
+    ig = cfg.get("instagram") or {}
+    weights = ig.get("post_type_weights") or ig.get("post_type_ratio") or {
+        "organic": 4,
+        "product": 1,
+    }
+    days = int(ig.get("recent_window_days", 7))
+    # Project diversity cooldown: how many of the most recent posted rows on
+    # this account to look at when deciding which project_names are "recently
+    # posted" and therefore ineligible for the next product draft. Default 6
+    # means the same project cannot appear twice within any 6-post sliding
+    # window per account. Hard rule (no cascade relaxation): if every product
+    # draft is blocked, the picker falls back to organic (which has NULL
+    # project_name and is never on cooldown).
+    cooldown = int(ig.get("product_cooldown_posts", 6))
+    return weights, days, cooldown
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--account", help="scope all queries to target_account (default: pick across all)")
+    args = ap.parse_args()
+
+    type_weights_cfg, window_days, cooldown_posts = load_ig_config()
+    account = args.account
+
+    # Single round trip: raw rows for weighting + cooldown + draft selection.
+    # All weighting / cooldown / fallback logic stays local (HTTP-only).
+    _ctx = api_get(
+        "/api/v1/media-posts/ig-picker-context",
+        query={
+            "target_account": account,
+            "window_days": window_days,
+            "cooldown_posts": cooldown_posts,
+        },
+    )
+    ctx = (_ctx.get("data") or {})
+    recent_counts = dict(ctx.get("recent_type_counts") or {})
+    recent_posted_projects = list(ctx.get("recent_posted_projects") or [])
+    all_drafts = list(ctx.get("drafts") or [])
+    for t in ("organic", "product"):
+        recent_counts.setdefault(t, 0)
+
+    eligible = {
+        t: float(type_weights_cfg.get(t, 0))
+        for t in ("organic", "product")
+        if float(type_weights_cfg.get(t, 0)) > 0
+    }
+    if not eligible:
+        sys.stderr.write("instagram.post_type_weights is empty in config.json\n")
+        sys.exit(3)
+
+    effective = {t: w / (1 + recent_counts[t]) for t, w in eligible.items()}
+    names = list(effective.keys())
+    ws = [effective[n] for n in names]
+    target = random.choices(names, weights=ws, k=1)[0]
+
+    def _recent_project_names(window):
+        # Project names appearing in the last `window` posted IG rows on this
+        # account (already account-scoped + NULL-excluded by the endpoint).
+        # Returns a set; empty when window<=0 or no account scoping.
+        if window <= 0 or not account:
+            return set()
+        return set(recent_posted_projects)
+
+    def _draft_query(type_, blocked_projects=None):
+        # First draft of this type from the endpoint's account-scoped list
+        # (already ordered by post_number ASC). For product drafts, exclude
+        # rows whose project_name is in blocked_projects (project diversity
+        # cooldown). NULL project_name rows always pass (organic-shaped, never
+        # on cooldown). Returns (post_number, video_path, project_name) or None.
+        for d in all_drafts:
+            if d.get("post_type") != type_:
+                continue
+            pn = d.get("project_name")
+            if blocked_projects and pn is not None and pn in blocked_projects:
+                continue
+            return (d.get("post_number"), d.get("video_path"), pn)
+        return None
+
+    # Project diversity cooldown applies to product drafts only. Hard rule:
+    # same project_name cannot appear within the last N posted rows on this
+    # account. If no product draft survives the filter, fall back to organic
+    # (which is not subject to the cooldown since its project_name is NULL).
+    # We do NOT relax the cooldown window, because the whole point is to
+    # prevent the exact same product from posting twice in a short stretch.
+    row = None
+    fallback = False
+    fallback_from = None
+    cooldown_blocked = set()
+    cooldown_window_used = 0
+    cooldown_skipped_drafts = []
+
+    def _pick_product_with_cooldown():
+        nonlocal cooldown_blocked, cooldown_window_used, cooldown_skipped_drafts
+        cooldown_blocked = _recent_project_names(cooldown_posts) if cooldown_posts > 0 else set()
+        cooldown_window_used = cooldown_posts
+        if cooldown_blocked:
+            # Diagnostic: which product drafts got filtered out by the cooldown
+            # (computed locally from the endpoint's draft list).
+            cooldown_skipped_drafts = [
+                (d.get("post_number"), d.get("project_name"))
+                for d in all_drafts
+                if d.get("post_type") == "product" and d.get("project_name") in cooldown_blocked
+            ]
+        return _draft_query("product", blocked_projects=cooldown_blocked)
+
+    if target == "product":
+        row = _pick_product_with_cooldown()
+    else:
+        row = _draft_query(target)
+
+    if row is None:
+        # Fall back to the other type if this one has no drafts (either truly
+        # empty for organic, or all-cooldown-blocked for product). For
+        # product->organic fallback this preserves cadence without weakening
+        # the cooldown.
+        other = "product" if target == "organic" else "organic"
+        # ...but ONLY if the other type is still eligible (config weight > 0).
+        # A type explicitly disabled via post_type_weights=0 (e.g. product
+        # paused during an IG link-sharing restriction) must NEVER be posted
+        # through the empty-queue fallback, otherwise weight=0 is not a real
+        # off switch. If the only eligible type's queue is empty, exit 2
+        # (nothing to post this fire) rather than leak the disabled type.
+        if other not in eligible:
+            sys.stderr.write(
+                f"queue empty for eligible type '{target}'; fallback type "
+                f"'{other}' is disabled (config_weights={type_weights_cfg}), "
+                "not leaking it"
+                + (f" (target_account={account})" if account else "")
+                + "\n"
+            )
+            sys.exit(2)
+        if other == "product":
+            row = _pick_product_with_cooldown()
+        else:
+            row = _draft_query(other)
+        if row is None:
+            sys.stderr.write(
+                "queue empty: no draft rows for either organic or product"
+                + (f" (target_account={account})" if account else "")
+                + (
+                    f" (cooldown blocked product drafts: {cooldown_skipped_drafts})"
+                    if cooldown_skipped_drafts else ""
+                )
+                + "\n"
+            )
+            sys.exit(2)
+        sys.stderr.write(
+            f"queue imbalance: target={target} has 0 drafts"
+            + (
+                f" (cooldown blocked product drafts: {cooldown_skipped_drafts})"
+                if target == "product" and cooldown_skipped_drafts else ""
+            )
+            + f", falling back to {other}"
+            + (f" (target_account={account})" if account else "")
+            + "\n"
+        )
+        fallback_from = target
+        target = other
+        fallback = True
+
+    post_number, video_path, project_name = row
+
+    if target == "product":
+        sys.stderr.write(
+            f"[ig_picker] cooldown account={account or '<global>'} "
+            f"window={cooldown_window_used} "
+            f"blocked={sorted(cooldown_blocked) or '[]'} "
+            f"chose=project_name={project_name} post_number={post_number}\n"
+        )
+
+    out = {
+        "post_type": target,
+        "video_path": video_path,
+        "post_number": post_number,
+        "target_account": account,
+        "project_name": project_name,
+        "reason": (
+            f"window={window_days}d account={account or '<global>'} "
+            f"recent={recent_counts} config_weights={type_weights_cfg} "
+            f"effective={effective} chose={target}"
+            + (f" (fallback_from={fallback_from})" if fallback else "")
+            + (
+                f" cooldown_window={cooldown_window_used}"
+                f" cooldown_blocked={sorted(cooldown_blocked)}"
+                + (f" cooldown_skipped_drafts={cooldown_skipped_drafts}" if cooldown_skipped_drafts else "")
+                if target == "product" else ""
+            )
+        ),
+        "fallback": fallback,
+        "cooldown_window": cooldown_window_used if target == "product" else None,
+        "cooldown_blocked_projects": sorted(cooldown_blocked) if target == "product" else [],
+        "cooldown_skipped_drafts": cooldown_skipped_drafts if target == "product" else [],
+    }
+    print(json.dumps(out))
+
+
+if __name__ == "__main__":
+    main()

package/scripts/ig_scrape_transcribe.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# End-to-end: download latest IG post from a creator, transcribe with Deepgram.
+# Usage: ./ig_scrape_transcribe.sh <ig_handle> [N_CANDIDATES=3]
+#
+# How it works:
+#   1. Loads DEEPGRAM_API_KEY from ~/fazm/web/.env.local (sibling repo).
+#   2. Asks the user's logged-in Chrome (via playwright-extension MCP)
+#      for the first N reel/post URLs on the profile grid. (Caller must
+#      pass the URLs in via stdin, one per line — see ig_pick_latest.py
+#      for the picker that does this end-to-end via MCP.)
+#   3. yt-dlp fetches metadata for each candidate, sorts by upload_date,
+#      keeps the most recent (this skips pinned-but-old posts).
+#   4. Downloads the chosen post, extracts audio, sends to Deepgram nova-3.
+#
+# Standalone fallback: if no URLs on stdin, pass a single post URL as $2.
+set -euo pipefail
+
+HANDLE="${1:?ig handle required}"
+SINGLE_URL="${2:-}"
+OUT_DIR="/tmp/ig_scrape/${HANDLE}"
+mkdir -p "$OUT_DIR"
+
+DEEPGRAM_API_KEY="$(grep -E '^DEEPGRAM_API_KEY=' /Users/matthewdi/fazm/web/.env.local | head -1 | cut -d= -f2-)"
+[[ -z "${DEEPGRAM_API_KEY}" ]] && { echo "ERROR: no DEEPGRAM_API_KEY"; exit 1; }
+echo "[1/5] Deepgram key loaded (len=${#DEEPGRAM_API_KEY})"
+
+# Collect candidate URLs
+if [[ -n "$SINGLE_URL" ]]; then
+  CANDIDATES=("$SINGLE_URL")
+elif [[ ! -t 0 ]]; then
+  mapfile -t CANDIDATES < <(grep -E 'instagram\.com/.*/(p|reel)/' || true)
+else
+  echo "ERROR: pass post URL as \$2 OR pipe candidate URLs on stdin" >&2
+  exit 2
+fi
+echo "[2/5] ${#CANDIDATES[@]} candidate URL(s)"
+
+# Sort candidates by upload_date desc using yt-dlp metadata only
+BEST_URL=""
+BEST_DATE=""
+for U in "${CANDIDATES[@]}"; do
+  D=$(yt-dlp --cookies-from-browser chrome --no-warnings -q --skip-download \
+        --print "%(upload_date)s" "$U" 2>/dev/null || true)
+  echo "    $D  $U"
+  if [[ -n "$D" && "$D" > "${BEST_DATE:-}" ]]; then
+    BEST_DATE="$D"
+    BEST_URL="$U"
+  fi
+done
+[[ -z "$BEST_URL" ]] && { echo "ERROR: no usable candidate"; exit 3; }
+echo "[3/5] picked $BEST_URL (upload_date=$BEST_DATE)"
+
+# Download the chosen post
+yt-dlp --cookies-from-browser chrome --no-warnings -q \
+  -o "${OUT_DIR}/%(id)s.%(ext)s" --write-info-json "$BEST_URL"
+VIDEO="$(ls -t "$OUT_DIR"/*.mp4 | head -1)"
+echo "    file: $VIDEO ($(du -h "$VIDEO" | cut -f1))"
+
+# Extract audio
+AUDIO="${VIDEO%.mp4}.m4a"
+echo "[4/5] extracting audio"
+ffmpeg -y -loglevel error -i "$VIDEO" -vn -c:a copy "$AUDIO" 2>/dev/null || \
+  ffmpeg -y -loglevel error -i "$VIDEO" -vn -c:a aac -b:a 96k "$AUDIO"
+
+# Transcribe
+echo "[5/5] Deepgram nova-3"
+TJ="${VIDEO%.mp4}.deepgram.json"
+curl -sS -X POST \
+  -H "Authorization: Token ${DEEPGRAM_API_KEY}" \
+  -H "Content-Type: audio/m4a" \
+  --data-binary "@${AUDIO}" \
+  "https://api.deepgram.com/v1/listen?model=nova-3&smart_format=true&punctuate=true&detect_language=true" \
+  -o "$TJ"
+
+python3 - <<PY
+import json, os
+d = json.load(open("$TJ"))
+info = json.load(open(os.path.splitext("$VIDEO")[0] + ".info.json"))
+ch = d["results"]["channels"][0]
+print()
+print("==================== RESULT ====================")
+print(f"creator   : @${HANDLE}")
+print(f"url       : {info.get('webpage_url') or info.get('original_url')}")
+print(f"upload_dt : {info.get('upload_date')}")
+print(f"duration  : {d['metadata'].get('duration')}s")
+print(f"language  : {ch.get('detected_language','?')}")
+print(f"caption   : {(info.get('description') or '')[:240]}")
+print("------------------- TRANSCRIPT ------------------")
+print(ch['alternatives'][0]['transcript'] or '(no speech detected)')
+print("=================================================")
+PY