@m13v/s4l 1.6.197-rc.7

package/scripts/merge_review_queue.py

@@ -0,0 +1,215 @@
+#!/usr/bin/env python3
+"""
+merge_review_queue.py — deliver a DRAFT_ONLY cycle's plan into the approval cards.
+
+The deterministic pipeline (run-twitter-cycle.sh DRAFT_ONLY) writes its drafts to
+a per-batch plan file (/tmp/twitter_cycle_plan_<batch>.json) and prints
+`DRAFT_ONLY_PLAN=<path>`. On a customer box NOTHING used to consume that — the
+only writer of the review-queue cards was the (now-removed) host-draft
+submit_drafts path. This script closes that gap: it merges the batch plan's
+candidates into the single review-queue plan the menu-bar cards read, deduped by
+thread/candidate URL, and refreshes the review-request marker the menu bar polls.
+
+This is the SAME merge submit_drafts did, reimplemented in Python so the launchd
+kicker (no node/MCP) can run it after the cycle. ONE pipeline, one set of cards.
+
+Usage:
+  merge_review_queue.py --plan /tmp/twitter_cycle_plan_<batch>.json [--project NAME]
+  merge_review_queue.py --plan-from-marker '<stdout containing DRAFT_ONLY_PLAN=...>'
+
+State dir (for review-request.json) honors $S4L_STATE_DIR; the review-queue plan
+lives in $S4L_TMP_DIR or /tmp (matching the MCP's planPath()).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+import time
+
+REVIEW_QUEUE_ID = "review-queue"
+
+
+def tmp_dir() -> str:
+    return os.environ.get("S4L_TMP_DIR") or "/tmp"
+
+
+def state_dir() -> str:
+    return os.environ.get("S4L_STATE_DIR") or os.path.join(
+        os.path.expanduser("~"), ".social-autoposter-mcp"
+    )
+
+
+def plan_path(batch_id: str) -> str:
+    return os.path.join(tmp_dir(), f"twitter_cycle_plan_{batch_id}.json")
+
+
+def review_request_path() -> str:
+    return os.path.join(state_dir(), "review-request.json")
+
+
+def _atomic_write(path: str, obj) -> None:
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    tmp = f"{path}.tmp.{os.getpid()}"
+    with open(tmp, "w") as f:
+        json.dump(obj, f, indent=2)
+    os.replace(tmp, path)
+
+
+def _dedup_key(c: dict) -> str:
+    """Match submit_drafts: dedup by the thread/candidate URL, else candidate_id."""
+    for k in ("candidate_url", "tweet_url", "thread_url", "candidate_id"):
+        v = c.get(k)
+        if v:
+            return str(v)
+    # last resort: the reply text, so identical drafts don't double up
+    return (c.get("reply_text") or "")[:120]
+
+
+def _thread_url(c: dict) -> str:
+    for k in ("candidate_url", "tweet_url", "thread_url"):
+        v = c.get(k)
+        if v:
+            return str(v)
+    return ""
+
+
+# Discovery-time author/engagement fields stamped onto each plan candidate so the
+# approval card can show them. All already captured on the twitter_candidates row
+# by the discovery pipeline (and refreshed at T1); no scrape happens here.
+STATS_KEYS = (
+    "author_handle",
+    "author_followers",
+    "likes",
+    "retweets",
+    "replies",
+    "views",
+    "virality_score",
+    "tweet_posted_at",
+)
+
+
+def _enrich_with_stats(cands: list) -> int:
+    """Stamp a `stats` sidecar onto plan candidates that lack one, from the
+    twitter_candidates rows the discovery pipeline already wrote. ONE listing
+    call (/api/v1/twitter-candidates?tweet_urls=...) covers the whole queue.
+    Best-effort: any failure (offline box, missing identity, API error) leaves
+    candidates unstamped and NEVER blocks card delivery. Returns count stamped."""
+    want = [c for c in cands if not c.get("stats") and not c.get("posted") and _thread_url(c)]
+    if not want:
+        return 0
+    urls = sorted({_thread_url(c) for c in want})[:500]
+    try:
+        sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+        from http_api import api_get
+
+        resp = api_get(
+            "/api/v1/twitter-candidates",
+            query={"tweet_urls": ",".join(urls), "limit": 500},
+        )
+        rows = (resp.get("data") or {}).get("candidates") or []
+    except BaseException as e:  # http_api raises SystemExit on terminal failure
+        print(f"[merge_review_queue] stats enrichment skipped: {e}", file=sys.stderr)
+        return 0
+    by_url = {str(r.get("tweet_url")): r for r in rows if r.get("tweet_url")}
+    stamped = 0
+    for c in want:
+        row = by_url.get(_thread_url(c))
+        if not row:
+            continue
+        c["stats"] = {k: row.get(k) for k in STATS_KEYS}
+        stamped += 1
+    return stamped
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="Merge a DRAFT_ONLY plan into the review-queue cards")
+    ap.add_argument("--plan", help="path to the per-batch DRAFT_ONLY plan file")
+    ap.add_argument(
+        "--plan-from-marker",
+        help="text containing a DRAFT_ONLY_PLAN=<path> marker (e.g. cycle stdout)",
+    )
+    ap.add_argument("--project", default=None, help="project name for the review-request marker")
+    ns = ap.parse_args()
+
+    src = ns.plan
+    if not src and ns.plan_from_marker:
+        m = re.search(r"DRAFT_ONLY_PLAN=(\S+\.json)", ns.plan_from_marker)
+        if m:
+            src = m.group(1)
+    if not src:
+        print("[merge_review_queue] no source plan (need --plan or a DRAFT_ONLY_PLAN marker)", file=sys.stderr)
+        return 2
+    if not os.path.exists(src):
+        print(f"[merge_review_queue] source plan not found: {src}", file=sys.stderr)
+        return 2
+
+    try:
+        with open(src) as f:
+            batch = json.load(f)
+    except Exception as e:
+        print(f"[merge_review_queue] could not read source plan: {e}", file=sys.stderr)
+        return 2
+
+    new_cands = batch.get("candidates") or []
+    if not new_cands:
+        print("[merge_review_queue] source plan has 0 candidates; nothing to merge", file=sys.stderr)
+        return 0
+
+    dst = plan_path(REVIEW_QUEUE_ID)
+    existing = []
+    if os.path.exists(dst):
+        try:
+            with open(dst) as f:
+                existing = json.load(f).get("candidates") or []
+        except Exception:
+            existing = []
+
+    seen = {_dedup_key(c) for c in existing}
+    added = 0
+    merged = list(existing)
+    for c in new_cands:
+        k = _dedup_key(c)
+        if k in seen:
+            continue
+        seen.add(k)
+        merged.append(c)
+        added += 1
+
+    stamped = _enrich_with_stats(merged)
+    if stamped:
+        print(f"[merge_review_queue] stamped stats on {stamped} candidate(s)", file=sys.stderr)
+
+    _atomic_write(dst, {"candidates": merged})
+
+    # Refresh the review-request marker the menu bar polls (count = pending, not posted).
+    pending = len([c for c in merged if not c.get("posted")])
+    project = ns.project or batch.get("project") or (new_cands[0].get("matched_project") if new_cands else None)
+    _atomic_write(
+        review_request_path(),
+        {
+            "batch_id": REVIEW_QUEUE_ID,
+            "project": project,
+            "count": pending,
+            "plan_path": dst,
+            "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        },
+    )
+    print(
+        f"[merge_review_queue] merged {added} new draft(s) into {REVIEW_QUEUE_ID} "
+        f"({pending} pending total) from {os.path.basename(src)}",
+        file=sys.stderr,
+    )
+    # Clean up the consumed batch plan so /tmp doesn't fill with orphans.
+    try:
+        os.remove(src)
+    except Exception:
+        pass
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/mint_external_pool.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+"""Pre-mint a pool of `post_links` codes for projects whose redirector lives on
+the CLIENT'S domain (external_short_links=true in config.json).
+
+Why this exists: for projects where we own the domain (fazm.ai, cyrano.systems,
+etc.) we ship a /r/[code] route via @m13v/seo-components and resolve codes
+live by hitting our DB. For projects where the CLIENT owns the domain and
+doesn't want a PR (Kent: runner.now, agora.xyz, podlog.io), we hand them a
+static CSV of `code -> destination` pairs they drop into their own redirector.
+
+The pool is rows in `post_links` with post_id IS NULL AND reply_id IS NULL
+and minted_session LIKE 'pool:%'. When a pipeline posts for a project with
+external_short_links=true, wrap_text_for_post pops the next unclaimed pool
+row matching (project_name, platform) instead of minting a fresh code, so the
+client's CSV stays valid forever (until the pool runs dry, then we top up).
+
+Usage:
+  python3 scripts/mint_external_pool.py \
+    --project Runner --platforms reddit,twitter,linkedin,github_issues,moltbook \
+    --per-platform 250
+
+  python3 scripts/mint_external_pool.py --status                 # show pool depth
+  python3 scripts/mint_external_pool.py --export-csv DIR         # write CSVs
+"""
+
+from __future__ import annotations
+import argparse
+import csv
+import json
+import os
+import secrets
+import sys
+from urllib.parse import urlencode
+
+REPO_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, os.path.join(REPO_DIR, 'scripts'))
+
+from http_api import api_get, api_post  # noqa: E402
+from dm_short_links import CODE_ALPHABET, CODE_LEN, _load_projects  # noqa: E402
+
+POOL_SESSION_PREFIX = 'pool:'
+
+
+def _slug(name: str) -> str:
+    return ''.join(c.lower() if c.isalnum() else '-' for c in name).strip('-')
+
+
+def _website(projects: list, project_name: str) -> str:
+    for p in projects:
+        if p.get('name') == project_name:
+            return (p.get('website') or '').rstrip('/')
+    raise SystemExit(f"project '{project_name}' not found in config.json")
+
+
+def _gen_code() -> str:
+    return ''.join(secrets.choice(CODE_ALPHABET) for _ in range(CODE_LEN))
+
+
+def _build_target(homepage: str, *, platform: str, slug: str, code: str) -> str:
+    # Canonical UTM scheme: utm_source='s4l' identifies the agency, utm_term
+    # carries the platform. Keep aligned with dm_short_links._build_target_url
+    # / _build_target_url_for_post. utm_content stays as <code> so the customer's
+    # static-CSV redirector can PostHog-join clicks back to post_links.code.
+    params = {
+        'utm_source': 's4l',
+        'utm_medium': 'post',
+        'utm_campaign': slug,
+        'utm_term': platform,
+        'utm_content': code,
+    }
+    sep = '&' if '?' in homepage else '?'
+    return f"{homepage}{sep}{urlencode(params)}"
+
+
+def mint_pool(*, project_name: str, platforms: list, per_platform: int,
+              session_tag: str | None = None) -> dict:
+    projects = _load_projects()
+    homepage = _website(projects, project_name)
+    slug = _slug(project_name)
+    session = session_tag or f"{POOL_SESSION_PREFIX}{slug}-{platforms[0] if len(platforms)==1 else 'multi'}"
+
+    minted = {plat: 0 for plat in platforms}
+    skipped = {plat: 0 for plat in platforms}
+    for platform in platforms:
+        tries = 0
+        while minted[platform] < per_platform and tries < per_platform * 3:
+            tries += 1
+            code = _gen_code()
+            target = _build_target(homepage, platform=platform, slug=slug, code=code)
+            result = api_post(
+                "/api/v1/post-links/mint",
+                {
+                    "code": code,
+                    "platform": platform,
+                    "project_name": project_name,
+                    "target_url": target,
+                    "kind": "website",
+                    "project_at_mint": project_name,
+                    "minted_session": f"{POOL_SESSION_PREFIX}{slug}-{platform}",
+                },
+                ok_on_conflict=True,
+            )
+            if not result.get("ok", True):
+                # code_collision (409): retry with a fresh random code.
+                skipped[platform] += 1
+                continue
+            minted[platform] += 1
+    return {
+        'project': project_name,
+        'homepage': homepage,
+        'minted': minted,
+        'skipped_collisions': skipped,
+        'total_minted': sum(minted.values()),
+    }
+
+
+def pool_status(project_filter: str | None = None) -> list[dict]:
+    query = {"project_name": project_filter} if project_filter else None
+    resp = api_get("/api/v1/post-links/pool-status", query=query)
+    return (resp.get("data") or {}).get("rows") or []
+
+
+def export_csv(out_dir: str, project_filter: str | None = None) -> dict:
+    os.makedirs(out_dir, exist_ok=True)
+    query = {"project_name": project_filter} if project_filter else None
+    resp = api_get("/api/v1/post-links/pool-export", query=query)
+    all_rows = (resp.get("data") or {}).get("rows") or []
+
+    by_project: dict[str, list[dict]] = {}
+    for r in all_rows:
+        by_project.setdefault(r['project_name'], []).append(r)
+
+    written = {}
+    for project_name, rows in by_project.items():
+        slug = _slug(project_name)
+        path = os.path.join(out_dir, f"kent-shortlinks-{slug}.csv")
+        with open(path, 'w', newline='') as f:
+            w = csv.writer(f)
+            w.writerow(['short_path', 'destination_url', 'platform', 'project'])
+            for r in rows:
+                w.writerow([f"/r/{r['code']}", r['target_url'], r['platform'], project_name])
+        written[project_name] = {'path': path, 'rows': len(rows)}
+    return written
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument('--project', help='project_name from config.json')
+    ap.add_argument('--platforms', default='reddit,twitter,linkedin,github_issues,moltbook',
+                    help='comma-separated platforms to seed')
+    ap.add_argument('--per-platform', type=int, default=250,
+                    help='codes to mint per platform (default 250)')
+    ap.add_argument('--status', action='store_true', help='print pool depth per project/platform')
+    ap.add_argument('--export-csv', metavar='DIR', help='export CSVs to DIR (one per project)')
+    args = ap.parse_args()
+
+    if args.status:
+        rows = pool_status(args.project)
+        if not rows:
+            print('no pool rows found')
+            return
+        print(f"{'project':<22} {'platform':<14} {'avail':>6} {'claim':>6} {'total':>6} {'last_mint'}")
+        for r in rows:
+            print(f"{r['project_name']:<22} {r['platform']:<14} {r['available']:>6} {r['claimed']:>6} {r['total']:>6} {r['last_minted']}")
+        return
+
+    if args.export_csv:
+        out = export_csv(args.export_csv, args.project)
+        print(json.dumps(out, indent=2, default=str))
+        return
+
+    if not args.project:
+        ap.error('--project required (unless using --status or --export-csv)')
+
+    platforms = [p.strip() for p in args.platforms.split(',') if p.strip()]
+    result = mint_pool(project_name=args.project, platforms=platforms,
+                      per_platform=args.per_platform)
+    print(json.dumps(result, indent=2, default=str))
+
+
+if __name__ == '__main__':
+    main()

package/scripts/mint_kent_pool.py

@@ -0,0 +1,249 @@
+#!/usr/bin/env python3
+"""Mint Kent's external short-link pool: 10k codes per site, 75% homepage,
+25% across discovered subpages, distributed evenly across 5 platforms.
+
+Designed to be fast (bulk INSERT + ON CONFLICT DO NOTHING, no per-row commit)
+so the full 30k mint completes in seconds rather than the ~20min the legacy
+mint_external_pool.py took for 3,750 rows.
+
+Usage:
+  python3 scripts/mint_kent_pool.py --dry-run     # preview the plan
+  python3 scripts/mint_kent_pool.py               # mint it
+  python3 scripts/mint_kent_pool.py --status      # show pool depth by destination
+"""
+
+from __future__ import annotations
+import argparse
+import json
+import os
+import secrets
+import sys
+from datetime import date
+from typing import Any
+from urllib.parse import urlencode
+
+REPO_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, os.path.join(REPO_DIR, 'scripts'))
+
+from http_api import api_get, api_post  # noqa: E402
+from dm_short_links import CODE_ALPHABET, CODE_LEN  # noqa: E402
+
+PLATFORMS = ['reddit', 'twitter', 'linkedin', 'github_issues', 'moltbook']
+TOTAL_PER_SITE = 10_000
+HOME_FRACTION = 0.75
+
+SITE_CONFIG: dict[str, dict[str, Any]] = {
+    'Runner': {
+        'origin': 'https://runner.now',
+        'slug': 'runner',
+        'subpages': [
+            '/download/',
+            '/workflows/',
+            '/apps/',
+            '/runner-for-business/',
+            '/blog/',
+            '/changelog/',
+            '/workflows/ai-executive-assistant/',
+            '/workflows/morning-founder-briefing/',
+            '/workflows/meeting-notes-to-action-items/',
+            '/workflows/ai-email-assistant/',
+            '/workflows/qualify-inbound-leads-from-gmail-hubspot/',
+            '/workflows/stakeholder-research-after-sales-call/',
+            '/apps/gmail/',
+            '/apps/google-calendar/',
+            '/apps/slack/',
+            '/apps/notion/',
+            '/apps/hubspot/',
+            '/apps/granola/',
+            '/apps/linear/',
+            '/blog/best-ai-apps-2026/',
+        ],
+    },
+    'Agora': {
+        'origin': 'https://www.agora.xyz',
+        'slug': 'agora',
+        'subpages': [
+            '/about',
+            '/blogs',
+            '/talk-to-our-team',
+            '/jobs',
+        ],
+    },
+    'Podlog': {
+        'origin': 'https://podlog.io',
+        'slug': 'podlog',
+        'subpages': [],
+    },
+}
+
+POOL_PREFIX = 'pool:'
+
+
+def _slug_path(path: str) -> str:
+    if not path or path == '/':
+        return 'home'
+    cleaned = path.strip('/').replace('/', '-')
+    return cleaned or 'home'
+
+
+def _build_target(origin: str, path: str, *, platform: str, campaign_slug: str, code: str) -> str:
+    base = origin.rstrip('/') + path
+    # Canonical UTM scheme: see dm_short_links._build_target_url for rationale.
+    params = {
+        'utm_source': 's4l',
+        'utm_medium': 'post',
+        'utm_campaign': campaign_slug,
+        'utm_term': platform,
+        'utm_content': code,
+    }
+    sep = '&' if '?' in base else '?'
+    return f"{base}{sep}{urlencode(params)}"
+
+
+def _session_tag(today_iso: str, slug: str, path: str, platform: str) -> str:
+    return f"{POOL_PREFIX}kent-{today_iso}:{slug}:{_slug_path(path)}:{platform}"
+
+
+def _gen_unique_codes(n: int, existing: set[str]) -> list[str]:
+    out: set[str] = set()
+    while len(out) < n:
+        c = ''.join(secrets.choice(CODE_ALPHABET) for _ in range(CODE_LEN))
+        if c not in existing and c not in out:
+            out.add(c)
+    return list(out)
+
+
+def _plan(per_site: int = TOTAL_PER_SITE, home_frac: float = HOME_FRACTION) -> list[dict]:
+    """Compute (project, platform, path, count) tuples for the full mint."""
+    rows = []
+    per_platform = per_site // len(PLATFORMS)
+    home_per_platform = int(per_platform * home_frac)
+    subpage_per_platform = per_platform - home_per_platform
+    for project, cfg in SITE_CONFIG.items():
+        subpages = cfg['subpages']
+        if not subpages:
+            actual_home = per_platform
+            actual_sub_each = 0
+        else:
+            actual_home = home_per_platform
+            actual_sub_each = subpage_per_platform // len(subpages)
+            remainder = subpage_per_platform - actual_sub_each * len(subpages)
+            actual_home += remainder
+        for platform in PLATFORMS:
+            rows.append({
+                'project': project,
+                'platform': platform,
+                'path': '/',
+                'count': actual_home,
+            })
+            for path in subpages:
+                rows.append({
+                    'project': project,
+                    'platform': platform,
+                    'path': path,
+                    'count': actual_sub_each,
+                })
+    return rows
+
+
+def mint_all(*, dry_run: bool = False) -> dict:
+    plan = _plan()
+    if dry_run:
+        per_site_totals: dict[str, int] = {}
+        for r in plan:
+            per_site_totals[r['project']] = per_site_totals.get(r['project'], 0) + r['count']
+        return {
+            'plan_rows': len(plan),
+            'codes_by_site': per_site_totals,
+            'sample': plan[:3] + plan[-3:],
+        }
+
+    today = date.today().isoformat()
+    minted = {p: 0 for p in SITE_CONFIG}
+    total = 0
+    # Server enforces uniqueness via ON CONFLICT (code) DO NOTHING; we only
+    # dedup within this run so a single batch never carries two identical codes.
+    existing: set[str] = set()
+    BATCH = 1000
+    for entry in plan:
+        project = entry['project']
+        platform = entry['platform']
+        path = entry['path']
+        n = entry['count']
+        if n <= 0:
+            continue
+        cfg = SITE_CONFIG[project]
+        slug = cfg['slug']
+        session_tag = _session_tag(today, slug, path, platform)
+        codes = _gen_unique_codes(n, existing)
+        existing.update(codes)
+        rows = []
+        for code in codes:
+            target = _build_target(
+                cfg['origin'], path,
+                platform=platform, campaign_slug=slug, code=code,
+            )
+            rows.append({
+                "code": code,
+                "platform": platform,
+                "project_name": project,
+                "target_url": target,
+                "kind": "website",
+                "project_at_mint": project,
+                "minted_session": session_tag,
+            })
+        inserted_here = 0
+        for i in range(0, len(rows), BATCH):
+            chunk = rows[i:i + BATCH]
+            resp = api_post("/api/v1/post-links/mint-batch", {"rows": chunk})
+            inserted_here += int((resp.get("data") or {}).get("inserted") or 0)
+        minted[project] += inserted_here
+        total += inserted_here
+        print(f"  + {project:<8} {platform:<14} {path:<60} count={inserted_here}", flush=True)
+    return {
+        'minted_total': total,
+        'minted_by_project': minted,
+        'session_date': today,
+    }
+
+
+def pool_status_detailed() -> list[dict]:
+    resp = api_get(
+        "/api/v1/post-links/pool-status",
+        query={"session_like": "pool:kent-%", "group_by_session": "1"},
+    )
+    return (resp.get("data") or {}).get("rows") or []
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument('--dry-run', action='store_true', help='print the plan, do not write')
+    ap.add_argument('--status', action='store_true', help='print pool depth grouped by destination')
+    args = ap.parse_args()
+
+    if args.status:
+        rows = pool_status_detailed()
+        if not rows:
+            print('no kent pool rows found')
+            return
+        print(f"{'project':<10} {'platform':<14} {'session':<70} {'avail':>6} {'claim':>6}")
+        for r in rows:
+            sess = (r['minted_session'] or '')[-70:]
+            print(f"{r['project_name']:<10} {r['platform']:<14} {sess:<70} {r['available']:>6} {r['claimed']:>6}")
+        totals: dict[str, int] = {}
+        for r in rows:
+            totals[r['project_name']] = totals.get(r['project_name'], 0) + r['available']
+        print('---')
+        for k, v in sorted(totals.items()):
+            print(f"  {k}: {v} available")
+        return
+
+    if args.dry_run:
+        print(json.dumps(mint_all(dry_run=True), indent=2))
+        return
+
+    print(json.dumps(mint_all(), indent=2, default=str))
+
+
+if __name__ == '__main__':
+    main()