norn-cli 2.6.1 → 2.7.0

package/scripts/reddit_signal_miner.py

@@ -1,490 +0,0 @@
-#!/usr/bin/env python3
-"""
-Reddit signal miner for the Norn LinkedIn campaign.
-
-Anthropic's crawler is blocked by Reddit, so Claude can't fetch it — but this
-script runs from YOUR machine/IP against Reddit's public JSON, so it can.
-
-What it does:
-  - searches the campaign's target subreddits for the campaign's pain-terms
-  - pulls matching posts AND their top comments (the Post-5 gold was a comment)
-  - scores every bit of text by "juiciness" (spine-weighted keyword hits)
-  - isolates the exact verbatim sentences that carry the pain
-  - writes a ranked, paste-ready digest you can triage into Docs/market_signals.md
-
-Discipline (from the campaign skill): quote VERBATIM + source URL + date.
-Mining must not displace posting. Log it, pick one, go draft.
-
-Where things live:
-  - This script's output (`Docs/reddit_signal_harvest.md`) is GITIGNORED SCRATCH —
-    overwritten by every run, never the source of truth.
-  - Permanent harvest archives live in `Docs/harvests/YYYY-MM-DD-*.md` (committed).
-    Copy a useful run there before the next one blows the scratch away.
-  - Curated wins (the actual market signals) live in `Docs/market_signals.md` as
-    numbered Signal entries — check those first to avoid re-mining the same ground.
-
-Usage:
-  python3 scripts/reddit_signal_miner.py
-  python3 scripts/reddit_signal_miner.py --time year --limit 25 --top-threads 15
-  python3 scripts/reddit_signal_miner.py --subs QualityAssurance devops --out harvest.md
-
-Set REDDIT_USERNAME below to your handle (Reddit asks for it in the User-Agent).
-Uses the stdlib + certifi (already installed) for TLS; no other packages needed.
-
-Troubleshooting:
-  - 403 on every request: Reddit is blocking your egress IP. Datacenter / VPN /
-    cloud IPs are refused wholesale — run it from a normal home connection, VPN off.
-    (This is why Claude can't run it for you: its sandbox IP is in a blocked range.)
-  - 429: you're going too fast — raise REQUEST_PAUSE.
-  - still 403 from home: anonymous JSON has gotten flaky; create a Reddit "script"
-    app and switch to the OAuth endpoint. Ask Claude to add that path if you need it.
-"""
-
-import argparse
-import html
-import json
-import re
-import ssl
-import sys
-import time
-import urllib.error
-import urllib.parse
-import urllib.request
-from datetime import datetime, timezone
-
-# python.org builds on macOS don't trust the system keychain; use certifi's
-# bundle if present, otherwise fall back to the interpreter default.
-try:
-    import certifi
-    SSL_CONTEXT = ssl.create_default_context(cafile=certifi.where())
-except ImportError:
-    SSL_CONTEXT = ssl.create_default_context()
-
-# --- config you can tweak --------------------------------------------------
-
-REDDIT_USERNAME = "your_reddit_handle"  # <- put your handle here (UA courtesy)
-
-SUBREDDITS = [
-    "QualityAssurance",
-    "devops",
-    "ExperiencedDevs",
-    "webdev",
-    "programming",
-    "softwaretesting",
-    "QualityAssurance",
-]
-
-QUERIES = [
-    "leaving Postman",
-    "Postman alternative",
-    ".http files",
-    "API testing",
-    "contract testing",
-    "flaky API tests",
-    "schema drift",
-    "tests passed but broke",
-    "staging didn't catch",
-    "lost my collection",
-    "tests out of date",
-    "nobody runs the tests",
-    "integration tests green",
-    "200 OK error body",
-]
-
-# spine-weighted scoring. higher weight = closer to "tests on loan / rot / drift".
-PAIN_PHRASES = {
-    # the ownership / loss wound (Post 5 territory) — heaviest
-    "lost years": 6, "lost my": 4, "lost all": 4, "evaporat": 5, "gone forever": 5,
-    "disappeared": 4, "wiped": 4, "no backup": 4, "couldn't recover": 4,
-    # rot / drift / staleness — the core thesis
-    "out of date": 5, "outdated": 4, "rot": 5, "stale": 5, "drift": 6,
-    "nobody updates": 6, "nobody maintains": 6, "nobody runs": 6, "never updated": 5,
-    "haven't touched": 4, "bit rot": 5, "abandoned": 4,
-    # passed-but-broke / staging-vs-prod
-    "passed but": 6, "tests passed": 5, "green but": 6, "didn't catch": 6,
-    "broke prod": 6, "broke in prod": 6, "worked on staging": 5, "works on staging": 5,
-    "only in production": 5, "false sense": 5, "lying": 5, "lied": 4,
-    # location / ownership / lock-in / friction
-    "source of truth": 5, "two sources": 5, "on someone": 4, "their laptop": 5,
-    "their account": 5, "behind a login": 5, "sign in": 3, "login wall": 5,
-    "paywall": 4, "enshittif": 5, "vendor lock": 5, "cloud sync": 4,
-    "fans": 3, "slow to open": 4, "bloat": 4, "enterprise monster": 5,
-    # 200-OK genre
-    "200 ok": 4, "success: false": 5, "status code": 2,
-}
-
-# leading word-boundary match: "rot" hits "rotten" but not "protocols"; stems
-# like "evaporat"/"enshittif" still catch their variants.
-_PAIN_PATTERNS = [(re.compile(r"\b" + re.escape(k)), k, w)
-                  for k, w in PAIN_PHRASES.items()]
-
-# a thread must actually be about APIs/testing to count — kills off-domain noise
-# (a "schema drift" hit on Terraform, a "lost my collection" hit on WordPress).
-DOMAIN_TERMS = [
-    "api", "endpoint", "postman", "graphql", "rest client", "request",
-    "response", "contract test", "mock", "openapi", "swagger", "insomnia",
-    "bruno", ".http", "integration test", "test suite", "payload", "qa ",
-    "automation", "regression",
-]
-_DOMAIN_PATTERNS = [re.compile(r"\b" + re.escape(t)) for t in DOMAIN_TERMS]
-
-REQUEST_PAUSE = 2.0        # seconds between requests (be polite)
-COMMENT_FETCH_PAUSE = 2.0
-MAX_RETRIES = 4
-
-# --- http ------------------------------------------------------------------
-
-
-def _ua() -> str:
-    return f"python:norn-signal-miner:1.0 (by /u/{REDDIT_USERNAME})"
-
-
-def fetch_json(url: str) -> dict:
-    backoff = 3.0
-    for attempt in range(1, MAX_RETRIES + 1):
-        req = urllib.request.Request(url, headers={
-            "User-Agent": _ua(),
-            "Accept": "application/json",
-            "Accept-Language": "en-GB,en;q=0.9",
-        })
-        try:
-            with urllib.request.urlopen(req, timeout=30, context=SSL_CONTEXT) as resp:
-                return json.loads(resp.read().decode("utf-8"))
-        except urllib.error.HTTPError as e:
-            if e.code in (429, 500, 502, 503) and attempt < MAX_RETRIES:
-                wait = backoff * attempt
-                print(f"  [{e.code}] backing off {wait:.0f}s "
-                      f"(attempt {attempt}/{MAX_RETRIES})", file=sys.stderr)
-                time.sleep(wait)
-                continue
-            print(f"  [http {e.code}] {url}", file=sys.stderr)
-            return {}
-        except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as e:
-            print(f"  [err] {e} :: {url}", file=sys.stderr)
-            if attempt < MAX_RETRIES:
-                time.sleep(backoff * attempt)
-                continue
-            return {}
-    return {}
-
-
-# pluggable fetcher: defaults to urllib, swapped to the browser in --browser mode
-_FETCH = fetch_json
-
-
-def get(url: str) -> dict:
-    return _FETCH(url)
-
-
-class BrowserSession:
-    """Drives real Chrome so Reddit's anti-bot WAF serves us like a human.
-
-    The key move is the homepage warmup: landing on reddit.com first banks the
-    session cookie that the WAF then accepts on the .json endpoints. Without it
-    every request is a 'blocked by network security' 403.
-    """
-
-    def __init__(self, headless: bool = False):
-        from playwright.sync_api import sync_playwright
-        self._pw = sync_playwright().start()
-        self.browser = self._pw.chromium.launch(
-            channel="chrome", headless=headless,
-            args=["--disable-blink-features=AutomationControlled"])
-        self.ctx = self.browser.new_context(
-            locale="en-GB", timezone_id="Europe/London",
-            viewport={"width": 1280, "height": 900})
-        self.ctx.add_init_script(
-            "Object.defineProperty(navigator,'webdriver',{get:()=>undefined})")
-        self.page = self.ctx.new_page()
-        self._warmup()
-
-    def _warmup(self):
-        print("  [browser] homepage warmup (clearing WAF challenge)...", file=sys.stderr)
-        try:
-            self.page.goto("https://www.reddit.com/", wait_until="domcontentloaded",
-                           timeout=45000)
-            time.sleep(6)
-            try:
-                self.page.goto("https://www.reddit.com/", wait_until="networkidle",
-                               timeout=20000)
-            except Exception:
-                pass
-        except Exception as e:
-            print(f"  [browser] warmup error: {e}", file=sys.stderr)
-
-    def fetch(self, url: str) -> dict:
-        try:
-            self.page.goto(url, wait_until="domcontentloaded", timeout=30000)
-            body = self.page.evaluate("document.body ? document.body.innerText : ''")
-            return json.loads(body)
-        except Exception as e:
-            print(f"  [browser err] {e} :: {url}", file=sys.stderr)
-            return {}
-
-    def close(self):
-        try:
-            self.browser.close()
-        finally:
-            self._pw.stop()
-
-
-# --- scoring ---------------------------------------------------------------
-
-
-def clean(text: str) -> str:
-    text = html.unescape(text or "")
-    return re.sub(r"\s+", " ", text).strip()
-
-
-def score_text(text: str):
-    low = text.lower()
-    hits = []
-    total = 0
-    for pattern, phrase, weight in _PAIN_PATTERNS:
-        if pattern.search(low):
-            total += weight
-            hits.append(phrase)
-    return total, hits
-
-
-def is_on_domain(post: dict) -> bool:
-    blob = (post.get("title", "") + " " + post.get("selftext", "") + " " +
-            " ".join(c["body"] for c in post.get("juicy_comments", []))).lower()
-    return any(p.search(blob) for p in _DOMAIN_PATTERNS)
-
-
-def juicy_sentences(text: str, max_n: int = 3):
-    """Return the verbatim sentences that actually carry the pain, best first."""
-    parts = re.split(r"(?<=[.!?])\s+|\n+", text)
-    scored = []
-    for s in parts:
-        s = s.strip()
-        if 25 <= len(s) <= 320:
-            sc, _ = score_text(s)
-            if sc > 0:
-                scored.append((sc, s))
-    scored.sort(key=lambda x: x[0], reverse=True)
-    seen, out = set(), []
-    for _, s in scored:
-        if s not in seen:
-            seen.add(s)
-            out.append(s)
-        if len(out) >= max_n:
-            break
-    return out
-
-
-def when(ts) -> str:
-    try:
-        return datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d")
-    except Exception:
-        return "????-??-??"
-
-
-# --- reddit ----------------------------------------------------------------
-
-
-def search(sub: str, query: str, t: str, limit: int):
-    q = urllib.parse.quote_plus(query)
-    url = (f"https://www.reddit.com/r/{sub}/search.json?"
-           f"q={q}&restrict_sr=on&sort=relevance&t={t}&limit={limit}")
-    data = get(url)
-    out = []
-    for child in data.get("data", {}).get("children", []):
-        d = child.get("data", {})
-        out.append({
-            "id": d.get("id"),
-            "sub": d.get("subreddit"),
-            "title": clean(d.get("title", "")),
-            "selftext": clean(d.get("selftext", "")),
-            "score": d.get("score", 0),
-            "num_comments": d.get("num_comments", 0),
-            "permalink": "https://www.reddit.com" + d.get("permalink", ""),
-            "created": when(d.get("created_utc", 0)),
-            "matched_query": query,
-        })
-    return out
-
-
-def _walk_comments(children, out, depth, max_depth):
-    for child in children:
-        if child.get("kind") != "t1":
-            continue
-        d = child.get("data", {})
-        body = clean(d.get("body", ""))
-        if body and body not in ("[deleted]", "[removed]"):
-            out.append({
-                "author": d.get("author", "?"),
-                "body": body,
-                "score": d.get("score", 0),
-                "created": when(d.get("created_utc", 0)),
-                "depth": depth,
-            })
-        if depth < max_depth:
-            replies = d.get("replies")
-            if isinstance(replies, dict):
-                _walk_comments(replies.get("data", {}).get("children", []),
-                               out, depth + 1, max_depth)
-
-
-def top_comments(permalink: str, limit: int = 100, max_depth: int = 2):
-    # depth traversal captures the back-and-forth, not just top-level answers —
-    # a real argument in the replies is exactly the signal we want.
-    url = permalink.rstrip("/") + f"/.json?limit={limit}&sort=top"
-    data = get(url)
-    out = []
-    if not isinstance(data, list) or len(data) < 2:
-        return out
-    _walk_comments(data[1].get("data", {}).get("children", []), out, 0, max_depth)
-    return out
-
-
-# --- main ------------------------------------------------------------------
-
-
-def main():
-    ap = argparse.ArgumentParser(description="Mine Reddit for Norn campaign pain signals.")
-    ap.add_argument("--time", default="year", choices=["day", "week", "month", "year", "all"])
-    ap.add_argument("--limit", type=int, default=25, help="results per (sub, query)")
-    ap.add_argument("--top-threads", type=int, default=25,
-                    help="how many threads to show in detail in the digest")
-    ap.add_argument("--dive", type=int, default=35,
-                    help="how many most-discussed threads to fetch comments for")
-    ap.add_argument("--min-comments", type=int, default=15,
-                    help="drop threads with fewer comments than this (dead posts)")
-    ap.add_argument("--subs", nargs="*", default=None, help="override subreddit list")
-    ap.add_argument("--queries", nargs="*", default=None, help="override query list")
-    ap.add_argument("--out", default="Docs/reddit_signal_harvest.md")
-    ap.add_argument("--browser", action="store_true",
-                    help="drive real Chrome (beats Reddit's anti-bot WAF)")
-    ap.add_argument("--headless", action="store_true",
-                    help="with --browser, run Chrome headless (less reliable vs WAF)")
-    args = ap.parse_args()
-
-    if REDDIT_USERNAME == "your_reddit_handle":
-        print("note: set REDDIT_USERNAME at the top of the script (Reddit UA courtesy).\n",
-              file=sys.stderr)
-
-    subs = list(dict.fromkeys(args.subs or SUBREDDITS))
-    queries = args.queries or QUERIES
-
-    print(f"mining {len(subs)} subs x {len(queries)} queries (t={args.time})"
-          f"{' [browser]' if args.browser else ''}...", file=sys.stderr)
-
-    global _FETCH
-    session = None
-    if args.browser:
-        session = BrowserSession(headless=args.headless)
-        _FETCH = session.fetch
-    try:
-        ranked = harvest(subs, queries, args)
-    finally:
-        if session:
-            session.close()
-
-    write_digest(ranked, args)
-    print(f"\ndone. ranked {len(ranked)} juicy threads -> {args.out}", file=sys.stderr)
-
-
-def harvest(subs, queries, args):
-    posts = {}
-    for sub in subs:
-        for query in queries:
-            print(f"  r/{sub} :: {query!r}", file=sys.stderr)
-            for p in search(sub, query, args.time, args.limit):
-                if not p["id"]:
-                    continue
-                ps, phits = score_text(p["title"] + " " + p["selftext"])
-                p["score_title"] = ps
-                p["hits"] = phits
-                # keep the higher-scoring sighting if seen via multiple queries
-                if p["id"] not in posts or ps > posts[p["id"]].get("score_title", -1):
-                    posts[p["id"]] = p
-            time.sleep(REQUEST_PAUSE)
-
-    # ENGAGEMENT-FIRST: the signal is a live debate, not a keyword match. Dive
-    # the most-discussed topical threads; dead posts (few comments) are useless
-    # even when they match the spine perfectly.
-    engaging = [p for p in posts.values() if p["num_comments"] >= args.min_comments]
-    engaging.sort(key=lambda p: (p["num_comments"], p["score"]), reverse=True)
-    dive = engaging[: args.dive]
-    print(f"  {len(engaging)} threads >= {args.min_comments} comments; "
-          f"diving top {len(dive)}", file=sys.stderr)
-    for p in dive:
-        print(f"  comments ({p['num_comments']}c): {p['title'][:55]!r}", file=sys.stderr)
-        scored = []
-        for c in top_comments(p["permalink"]):
-            cs, chits = score_text(c["body"])
-            if cs > 0:
-                c["score_juicy"] = cs
-                c["hits"] = chits
-                c["quotes"] = juicy_sentences(c["body"])
-                scored.append(c)
-        scored.sort(key=lambda c: (c["score_juicy"], c["score"]), reverse=True)
-        p["juicy_comments"] = scored[:6]
-        p["n_juicy_comments"] = len(scored)
-        p["score_comments"] = sum(c["score_juicy"] for c in scored[:3])
-        time.sleep(COMMENT_FETCH_PAUSE)
-
-    ranked = []
-    for p in dive:
-        p.setdefault("juicy_comments", [])
-        p.setdefault("score_comments", 0)
-        p.setdefault("n_juicy_comments", 0)
-        # engagement leads (comments weighted over upvotes); on-spine discussion boosts
-        p["engagement"] = p["num_comments"] + 0.25 * p["score"]
-        p["rank_score"] = p["engagement"] + 3 * p["score_comments"] + p["score_title"]
-        # keep only threads that are genuinely about APIs/testing AND have spine signal
-        if is_on_domain(p) and (p["score_comments"] > 0 or p["score_title"] > 0):
-            ranked.append(p)
-
-    ranked.sort(key=lambda p: (p["rank_score"], p["n_juicy_comments"]), reverse=True)
-    return ranked
-
-
-def write_digest(ranked, args):
-    lines = []
-    lines.append("# Reddit signal harvest")
-    lines.append("")
-    lines.append(f"> Generated {datetime.now().strftime('%Y-%m-%d %H:%M')} · "
-                 f"t={args.time} · ranked by LIVE ENGAGEMENT (comments) + on-spine discussion.")
-    lines.append("> Paste-ready for Docs/market_signals.md triage. Verbatim quotes only.")
-    lines.append("")
-
-    for i, p in enumerate(ranked[: args.top_threads], 1):
-        lines.append(f"## {i}. [{p['num_comments']}c · ↑{p['score']}] "
-                     f"r/{p['sub']} — {p['title']}")
-        lines.append("")
-        lines.append(f"- **URL:** {p['permalink']}")
-        lines.append(f"- **Date:** {p['created']} · **Comments:** {p['num_comments']} · "
-                     f"**Upvotes:** {p['score']} · **On-spine comments:** "
-                     f"{p.get('n_juicy_comments', 0)} · **Found via:** {p['matched_query']!r}")
-        lines.append(f"- **Pain hits:** {', '.join(p['hits']) or '—'}")
-        if p.get("selftext"):
-            for q in juicy_sentences(p["selftext"]):
-                lines.append(f"  - > {q}")
-        for c in p.get("juicy_comments", []):
-            lines.append(f"- **comment** (↑{c['score']}, {c['created']}) "
-                         f"hits: {', '.join(c['hits'])}")
-            for q in c["quotes"]:
-                lines.append(f"  - > {q}")
-        lines.append("")
-
-    # remaining threads, compact
-    rest = ranked[args.top_threads:]
-    if rest:
-        lines.append("---")
-        lines.append("")
-        lines.append("### More live candidates")
-        lines.append("")
-        for p in rest:
-            lines.append(f"- [{p['num_comments']}c ↑{p['score']}] r/{p['sub']} "
-                         f"({p['created']}) — {p['title']} — {p['permalink']}")
-        lines.append("")
-
-    with open(args.out, "w", encoding="utf-8") as f:
-        f.write("\n".join(lines))
-
-
-if __name__ == "__main__":
-    main()

package/scripts/validate-skills.mjs

@@ -1,50 +0,0 @@
-#!/usr/bin/env node
-
-import { spawnSync } from 'node:child_process';
-import { existsSync, readdirSync, statSync } from 'node:fs';
-import os from 'node:os';
-import path from 'node:path';
-
-const repoRoot = process.cwd();
-const skillsRoot = path.join(repoRoot, '.github', 'skills');
-const codexHome = process.env.CODEX_HOME || path.join(os.homedir(), '.codex');
-const validatorPath = path.join(
-  codexHome,
-  'skills',
-  '.system',
-  'skill-creator',
-  'scripts',
-  'quick_validate.py'
-);
-
-if (!existsSync(skillsRoot)) {
-  console.error(`Skills directory not found: ${skillsRoot}`);
-  process.exit(1);
-}
-
-if (!existsSync(validatorPath)) {
-  console.error(`Skill validator not found: ${validatorPath}`);
-  console.error('Install the Codex skill-creator system skill or set CODEX_HOME correctly.');
-  process.exit(1);
-}
-
-const skillDirs = readdirSync(skillsRoot)
-  .map((name) => path.join(skillsRoot, name))
-  .filter((dir) => statSync(dir).isDirectory() && existsSync(path.join(dir, 'SKILL.md')))
-  .sort();
-
-if (skillDirs.length === 0) {
-  console.log('No skills found to validate.');
-  process.exit(0);
-}
-
-for (const skillDir of skillDirs) {
-  const label = path.relative(repoRoot, skillDir);
-  console.log(`Validating ${label}`);
-  const result = spawnSync('python3', [validatorPath, skillDir], { stdio: 'inherit' });
-  if (result.status !== 0) {
-    process.exit(result.status ?? 1);
-  }
-}
-
-console.log(`Validated ${skillDirs.length} skills.`);