luckiest-co 1.0.52 → 1.0.53

package/skills/trends/scripts/lib/render.py

@@ -598,17 +598,20 @@ def _defang_corpus_sentinels(value: str) -> str:
 def _defang_engine_sentinels(value: str) -> str:
     """Source content must not be able to forge the engine's own block markers.
 
-    The EVIDENCE FOR SYNTHESIS and PASS-THROUGH FOOTER envelopes are HTML
-    comments, and LAW 5 tells the host model to emit the footer block
-    verbatim. Scraped text carrying those markers could therefore close the
-    evidence envelope early or open a footer the model relays to the user
-    unmodified. Breaking the comment delimiters is what actually neutralizes
-    them; the phrase substitutions are defense in depth for a model that
-    pattern-matches on the wording rather than the comment syntax.
+    The EVIDENCE FOR SYNTHESIS envelope is an HTML comment pair and the engine
+    footer is a fenced ``text`` code block, which LAW 5 tells the host model to
+    emit verbatim. Scraped text carrying those markers could therefore close
+    the evidence envelope early or open a footer the model relays to the user
+    unmodified. Breaking the comment delimiters and the code-fence runs
+    (backtick and tilde, the two CommonMark fence characters) is what actually
+    neutralizes them; the phrase substitutions are defense in depth for a model
+    that pattern-matches on the wording rather than the syntax.
     """
     return (
         value.replace("<!--", "<!- -")
         .replace("-->", "- ->")
+        .replace("```", "'''")
+        .replace("~~~", "~ ~ ~")
         .replace("PASS-THROUGH FOOTER", "PASS-THROUGH-FOOTER")
         .replace("EVIDENCE FOR SYNTHESIS", "EVIDENCE-FOR-SYNTHESIS")
     )
@@ -845,8 +848,8 @@ def render_compact(
     # evidence for the model to READ, not output to emit. LAW 6 in SKILL.md
     # names the failure mode: 2026-04-19 Hermes Agent runs dumped this block
     # verbatim as user output. The envelope comments give the model an
-    # unambiguous scope for "pass through verbatim" (the PASS-THROUGH FOOTER
-    # block below) vs "synthesize from" (this block).
+    # unambiguous scope for "pass through verbatim" (the fenced footer block
+    # below) vs "synthesize from" (this block).
     lines.append(
         "<!-- EVIDENCE FOR SYNTHESIS: read this, do not emit verbatim. Transform into `What I learned:` prose per LAW 2. -->"
     )
@@ -936,13 +939,7 @@ def render_compact(
         lines.extend(comparison_scaffold)
 
     footer = _render_emoji_footer(report, save_path)
-    if footer:
-        lines.append("")
-        lines.append(
-            "<!-- PASS-THROUGH FOOTER: emit verbatim in the model response per LAW 5. -->"
-        )
-        lines.extend(footer)
-        lines.append("<!-- END PASS-THROUGH FOOTER -->")
+    _append_pass_through_footer(lines, footer)
 
     lines.extend(_render_canonical_boundary())
 
@@ -1168,12 +1165,24 @@ def _append_html_footer(
     lines: list[str], report: schema.Report, save_path: str | None
 ) -> None:
     footer = _render_emoji_footer(report, save_path)
+    _append_pass_through_footer(lines, footer)
+
+
+def _append_pass_through_footer(lines: list[str], footer: list[str]) -> None:
+    """Emit the emoji footer inside the fenced ``text`` block LAW 5 relays.
+
+    Footer rows interpolate source-controlled values (handles, subreddits,
+    market questions, advertiser names). A value carrying a newline plus a
+    fence run would close the trusted block early and leave the rest of the
+    row outside it, so every row is flattened to one line and defanged here,
+    at the one place all footer paths share.
+    """
+    if not footer:
+        return
     lines.append("")
-    lines.append(
-        "<!-- PASS-THROUGH FOOTER: emit verbatim in the model response per LAW 5. -->"
-    )
-    lines.extend(footer)
-    lines.append("<!-- END PASS-THROUGH FOOTER -->")
+    lines.append("```text")
+    lines.extend(_defang_engine_sentinels(" ".join(row.splitlines())) for row in footer)
+    lines.append("```")
 
 
 def _render_synthesis_directive() -> list[str]:
@@ -1199,7 +1208,7 @@ def _render_synthesis_directive() -> list[str]:
         "> READ, not text to emit. Transform it into `What I learned:` prose paragraphs",
         "> per LAW 2. Do NOT pass the `### N.` evidence clusters or the stats and",
         "> source-coverage blocks through verbatim. The ONLY block you emit verbatim is",
-        "> the PASS-THROUGH FOOTER (the emoji tree) lower down. The full contract repeats",
+        "> the fenced `text` footer block (the emoji tree) lower down. The full contract repeats",
         "> at the end-of-output boundary near the bottom; if your captured output was",
         "> truncated and never reached it, this contract still binds.",
         "",
@@ -1219,7 +1228,7 @@ def _render_canonical_boundary() -> list[str]:
     "Pass through the lines ABOVE this boundary verbatim" phrasing was
     ambiguous about scope and led two consecutive runs to dump the
     `## Ranked Evidence Clusters` scratchpad as user output. The current
-    phrasing scopes pass-through to the PASS-THROUGH FOOTER block only and
+    phrasing scopes pass-through to the fenced footer block only and
     gives the model a concrete self-check string (`### 1.` + score tuple).
     """
     return [
@@ -1227,7 +1236,9 @@ def _render_canonical_boundary() -> list[str]:
         "---",
         "# END OF last30days CANONICAL OUTPUT",
         "",
-        "Pass through ONLY the PASS-THROUGH FOOTER block verbatim (emoji-tree stats).",
+        "Pass through ONLY the fenced `text` footer block above (emoji-tree stats).",
+        "Keep the code fence, so Markdown does not turn `---` into horizontal",
+        "rules or distort the tree.",
         "The EVIDENCE FOR SYNTHESIS block above it is raw evidence for your synthesis,",
         "not output. Transform it into `What I learned:` prose paragraphs per LAW 2.",
         "",
@@ -1525,13 +1536,7 @@ def render_comparison_multi(
     lines.extend(scaffold)
 
     footer = _render_emoji_footer(main_report, save_path)
-    if footer:
-        lines.append("")
-        lines.append(
-            "<!-- PASS-THROUGH FOOTER: emit verbatim in the model response per LAW 5. -->"
-        )
-        lines.extend(footer)
-        lines.append("<!-- END PASS-THROUGH FOOTER -->")
+    _append_pass_through_footer(lines, footer)
 
     lines.extend(_render_canonical_boundary())
 
@@ -2007,9 +2012,7 @@ def render_full(report: schema.Report, save_path: str | None = None) -> str:
     lines.extend(_render_stats(evidence_report))
     lines.extend(_render_source_coverage(evidence_report))
     if save_path:
-        footer_lines = _render_emoji_footer(evidence_report, save_path)
-        if footer_lines:
-            lines.extend(["", *footer_lines])
+        _append_pass_through_footer(lines, _render_emoji_footer(evidence_report, save_path))
     return "\n".join(lines).strip() + "\n"
 
 
@@ -3424,17 +3427,7 @@ def _format_date(item: schema.SourceItem | None) -> str:
 def _format_actor(item: schema.SourceItem | None) -> str | None:
     if not item:
         return None
-    if item.source == "reddit" and item.container:
-        return f"r/{item.container}"
-    if item.source in {"x", "bluesky", "truthsocial"} and item.author:
-        return f"@{item.author.lstrip('@')}"
-    if item.source == "youtube" and item.author:
-        return item.author
-    if item.container and item.container != "Polymarket":
-        return item.container
-    if item.author:
-        return item.author
-    return None
+    return _stats_actor(item)
 
 
 # Per-source engagement display fields: list of (field_name, label) tuples.
@@ -3556,6 +3549,17 @@ def _top_voices_overall(
 
 
 def _stats_actor(item: schema.SourceItem) -> str | None:
+    """The handle, subreddit or channel an item is attributed to, on one line.
+
+    Author and container come from the source, and the result is interpolated
+    into evidence rows, ``## Stats`` and the pass-through footer, so it gets the
+    same flattening and sentinel defanging as a scraped title.
+    """
+    actor = _raw_actor(item)
+    return _safe_title(actor) if actor else None
+
+
+def _raw_actor(item: schema.SourceItem) -> str | None:
     if item.source == "reddit" and item.container:
         return f"r/{item.container}"
     if item.source in {"x", "bluesky", "truthsocial"} and item.author:

package/skills/trends/scripts/lib/youtube_yt.py

@@ -82,8 +82,8 @@ _ytdlp_slots = threading.Semaphore(_YTDLP_MAX_CONCURRENT)
 # every entity sub-run; cache hits avoid the redundant expensive --dump-json work.
 # Inflight coalescing prevents N concurrent identical searches from all missing
 # the cache and stampeding YouTube together.
-_search_cache: Dict[Tuple[str, int, str], Dict[str, Any]] = {}
-_search_inflight: Dict[Tuple[str, int, str], tuple[threading.Event, list]] = {}
+_search_cache: Dict[Tuple[str, int, str, str, Optional[str]], Dict[str, Any]] = {}
+_search_inflight: Dict[Tuple[str, int, str, str, Optional[str]], tuple[threading.Event, list]] = {}
 _search_cache_lock = threading.Lock()
 # Comments are enrichment, not core evidence: keep the budget tight so a slow
 # comment API can never dominate a run's wall clock (bounded to 3 videos).
@@ -198,7 +198,7 @@ def _run_ytdlp(cmd: List[str], *, timeout: float) -> subproc.SubprocResult:
 
 
 def _claim_search_slot(
-    cache_key: Tuple[str, int, str],
+    cache_key: Tuple[str, int, str, str, Optional[str]],
 ) -> tuple[Optional[Dict[str, Any]], Optional[threading.Event], Optional[list], bool]:
     """Return ``(cached, event, slot, is_leader)`` for search coalesce.
 
@@ -220,7 +220,7 @@ def _claim_search_slot(
 
 
 def _finish_search_slot(
-    cache_key: Tuple[str, int, str],
+    cache_key: Tuple[str, int, str, str, Optional[str]],
     payload: Dict[str, Any],
     *,
     event: threading.Event,
@@ -489,14 +489,17 @@ _RECENT_FRESHNESS_MODES = frozenset({"strict_recent", "balanced_recent"})
 def _apply_date_window(
     items: List[Dict[str, Any]],
     from_date: str,
+    to_date: str,
     freshness_mode: Optional[str],
 ) -> List[Dict[str, Any]]:
     """Prefer in-window videos; scope the keep-all fallback by freshness mode.
 
+    Videos published after ``to_date`` (an --as-of window) are never kept.
     3+ in-window videos: keep only those (every mode). Fewer: recent modes
     keep only the in-window ones; evergreen_ok and unknown/None keep all so
     evergreen topics can still fall back to older videos.
     """
+    items = [i for i in items if not (i["date"] and i["date"] > to_date)]
     recent = [i for i in items if i["date"] and i["date"] >= from_date]
     if len(recent) >= 3:
         _log(f"Found {len(recent)} videos within date range")
@@ -536,7 +539,7 @@ def search_youtube(
 
     count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
     core_topic = _extract_core_subject(topic)
-    cache_key = (core_topic, count, from_date, freshness_mode)
+    cache_key = (core_topic, count, from_date, to_date, freshness_mode)
     timeout = _search_timeout()
 
     cached, event, slot, is_leader = _claim_search_slot(cache_key)
@@ -645,7 +648,7 @@ def search_youtube(
 
         # Date filter: prefer recent items; the keep-all fallback is scoped
         # by freshness mode (evergreen only), see _apply_date_window.
-        items = _apply_date_window(items, from_date, freshness_mode)
+        items = _apply_date_window(items, from_date, to_date, freshness_mode)
 
         # Sort by views descending
         items.sort(key=lambda x: x["engagement"]["views"], reverse=True)
@@ -1667,7 +1670,7 @@ def search_youtube_sc(
         })
 
     # Date filter (keep-all fallback scoped by freshness mode)
-    items = _apply_date_window(items, from_date, freshness_mode)
+    items = _apply_date_window(items, from_date, to_date, freshness_mode)
 
     # Sort by views
     items.sort(key=lambda x: x["engagement"]["views"], reverse=True)

package/skills/trends/scripts/test-v1-vs-v2.sh

@@ -0,0 +1,220 @@
+#!/bin/bash
+set -euo pipefail
+
+# === V1 vs V2 Skill Test Harness ===
+# Runs all 17 test queries through both v1 and v2 SKILL.md
+# using `claude --print` to capture real end-to-end output.
+
+SKILL_DIR="$HOME/.claude/skills/last30days"
+REPO_DIR="${REPO_DIR:-$(cd "$(dirname "$0")/.." && pwd)}"
+CLAUDE="${CLAUDE:-$(command -v claude || echo claude)}"
+
+# Safety: always restore V2 SKILL.md on exit/crash
+cleanup() {
+  if [ -f "$SKILL_DIR/SKILL.md.v2.bak" ]; then
+    echo ""
+    echo "⚠️  Restoring V2 SKILL.md from backup (script interrupted)..."
+    cp "$SKILL_DIR/SKILL.md.v2.bak" "$SKILL_DIR/SKILL.md"
+    rm -f "$SKILL_DIR/SKILL.md.v2.bak"
+    echo "  ✅ V2 restored"
+  fi
+}
+trap cleanup EXIT
+TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+OUT_DIR="$REPO_DIR/docs/test-results/v1-vs-v2-${TIMESTAMP}"
+V1_DIR="$OUT_DIR/v1"
+V2_DIR="$OUT_DIR/v2"
+
+mkdir -p "$V1_DIR" "$V2_DIR"
+
+echo "📁 Output directory: $OUT_DIR"
+echo ""
+
+# All 17 test queries
+QUERIES=(
+  "prompting techniques for chatgpt for legal questions"
+  "best clawdbot use cases"
+  "how to best setup clawdbot"
+  "prompting tips for nano banana pro for ios designs"
+  "top claude code skills"
+  "using ChatGPT to make images of dogs"
+  "research best practices for beautiful remotion animation videos in claude code"
+  "photorealistic people in nano banana pro"
+  "What are the best rap songs lately"
+  "what are people saying about DeepSeek R1"
+  "best practices for cursor rules files for Cursor"
+  "prompt advice for using suno to make killer songs in simple mode"
+  "how do I use Codex with Claude Code on same app to make it better"
+  "kanye west"
+  "howie.ai"
+  "open claw"
+  "nano banana pro prompting"
+)
+
+TYPES=(
+  "PROMPTING+TOOL"
+  "RECOMMENDATIONS"
+  "HOW-TO"
+  "PROMPTING+TOOL"
+  "RECOMMENDATIONS"
+  "GENERAL"
+  "PROMPTING"
+  "PROMPTING"
+  "RECOMMENDATIONS"
+  "NEWS"
+  "PROMPTING"
+  "PROMPTING"
+  "HOW-TO"
+  "NEWS"
+  "GENERAL"
+  "GENERAL"
+  "PROMPTING"
+)
+
+slugify() {
+  echo "$1" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g' | sed 's/--*/-/g' | cut -c1-50
+}
+
+run_version() {
+  local version="$1"
+  local outdir="$2"
+  local total=${#QUERIES[@]}
+
+  echo ""
+  echo "=========================================="
+  echo "  Running $version — $total queries"
+  echo "=========================================="
+  echo ""
+
+  for i in "${!QUERIES[@]}"; do
+    local query="${QUERIES[$i]}"
+    local type="${TYPES[$i]}"
+    local slug
+    slug=$(slugify "$query")
+    local num=$((i + 1))
+    local outfile="$outdir/${num}-${slug}.txt"
+    local errfile="$outdir/${num}-${slug}.stderr.txt"
+
+    echo "[$version] ($num/$total) $query [$type]"
+
+    local start_time
+    start_time=$(date +%s)
+
+    # Run claude --print with the skill invocation
+    # No timeout — claude --print exits on its own; kill manually if stuck
+    if "$CLAUDE" --print \
+      "/last30days $query" \
+      > "$outfile" 2>"$errfile"; then
+      local end_time
+      end_time=$(date +%s)
+      local duration=$((end_time - start_time))
+      local lines
+      lines=$(wc -l < "$outfile")
+      echo "  ✅ Done — ${lines} lines, ${duration}s"
+    else
+      local exit_code=$?
+      echo "  ❌ Failed (exit $exit_code)" | tee -a "$outfile"
+    fi
+
+    # Brief pause between queries to avoid rate limits
+    sleep 3
+  done
+}
+
+# === Phase 1: Test V1 ===
+echo "📦 Backing up current V2 SKILL.md..."
+cp "$SKILL_DIR/SKILL.md" "$SKILL_DIR/SKILL.md.v2.bak"
+
+echo "📥 Installing V1 SKILL.md from upstream..."
+cd "$REPO_DIR"
+git show upstream/main:SKILL.md | sed '/^context: fork$/d; /^agent: Explore$/d; /^disable-model-invocation: true$/d' > "$SKILL_DIR/SKILL.md"
+cp "$SKILL_DIR/SKILL.md" "$OUT_DIR/v1-SKILL.md"
+echo "  ✅ V1 installed (stripped: context:fork, agent:Explore, disable-model-invocation)"
+
+run_version "V1" "$V1_DIR"
+
+# === Phase 2: Test V2 ===
+echo ""
+echo "📥 Restoring V2 SKILL.md..."
+cp "$SKILL_DIR/SKILL.md.v2.bak" "$SKILL_DIR/SKILL.md"
+cp "$SKILL_DIR/SKILL.md" "$OUT_DIR/v2-SKILL.md"
+echo "  ✅ V2 restored"
+
+run_version "V2" "$V2_DIR"
+
+# === Phase 3: Generate summary ===
+echo ""
+echo "=========================================="
+echo "  Generating comparison summary"
+echo "=========================================="
+
+SUMMARY="$OUT_DIR/comparison-summary.md"
+
+cat > "$SUMMARY" << EOF
+# V1 vs V2 Comparison Results
+
+Generated: $(date)
+Output directory: $OUT_DIR
+
+## Output Files
+
+| # | Query | Type | V1 Lines | V2 Lines | V1 Time | V2 Time |
+|---|-------|------|----------|----------|---------|---------|
+EOF
+
+for i in "${!QUERIES[@]}"; do
+  query="${QUERIES[$i]}"
+  type="${TYPES[$i]}"
+  slug=$(slugify "$query")
+  num=$((i + 1))
+
+  v1file="$V1_DIR/${num}-${slug}.txt"
+  v2file="$V2_DIR/${num}-${slug}.txt"
+
+  v1lines=$(wc -l < "$v1file" 2>/dev/null || echo "ERR")
+  v2lines=$(wc -l < "$v2file" 2>/dev/null || echo "ERR")
+
+  echo "| $num | \`$query\` | $type | $v1lines | $v2lines | — | — |" >> "$SUMMARY"
+done
+
+cat >> "$SUMMARY" << 'EOF'
+
+## Quick Check: Key Features
+
+For each query, check these v2 improvements:
+
+- [ ] Query parsing display (`🔍 **{TOPIC}** · {QUERY_TYPE}`)
+- [ ] Sparse citations (not every sentence)
+- [ ] Bold topic headers in summary
+- [ ] Emoji stats tree (`├─ 🟠 Reddit:`)
+- [ ] Quality checklist applied to prompts
+- [ ] Self-check (research grounding, not generic)
+
+## Scoring Guide
+
+Use the full scoring rubric from:
+`docs/plans/2026-02-06-test-v1-vs-v2-comparison-plan.md`
+
+## Next Step
+
+Have Claude read all 34 output files and generate scored comparison:
+```
+Read all files in docs/test-results/v1-vs-v2-*/v1/ and v2/
+Score each on the 7 dimensions from the test plan
+Write the final analysis to docs/test-results/v1-vs-v2-*/analysis.md
+```
+EOF
+
+# Cleanup backup
+rm -f "$SKILL_DIR/SKILL.md.v2.bak"
+
+echo ""
+echo "✅ All done!"
+echo ""
+echo "📁 Results:  $OUT_DIR"
+echo "📊 Summary:  $SUMMARY"
+echo "📄 V1 files: $V1_DIR/"
+echo "📄 V2 files: $V2_DIR/"
+echo ""
+echo "To review:"
+echo "  open $OUT_DIR"

package/skills/trends/scripts/test_device_auth.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+"""Test ScrapeCreators GitHub device auth flow from the CLI.
+
+Usage:
+    python3 scripts/test_device_auth.py
+
+Flow:
+    1. Starts device code request
+    2. Shows user code + opens GitHub auth URL in browser
+    3. Polls for token until you complete auth
+    4. Fetches your profile and prints your API key
+"""
+
+import json
+import sys
+import time
+import webbrowser
+from urllib.request import Request, urlopen
+from urllib.error import HTTPError, URLError
+
+BASE = "https://api.scrapecreators.com/v1/github/device"
+
+
+def _post(url, data=None):
+    body = json.dumps(data).encode() if data else None
+    req = Request(url, data=body, method="POST")
+    req.add_header("Content-Type", "application/json")
+    with urlopen(req, timeout=15) as resp:
+        return json.loads(resp.read())
+
+
+def _get(url, token):
+    req = Request(url)
+    req.add_header("Authorization", f"Bearer {token}")
+    with urlopen(req, timeout=15) as resp:
+        return json.loads(resp.read())
+
+
+def main():
+    # Step 1: Start device flow
+    print("Starting ScrapeCreators GitHub device auth...\n")
+    try:
+        code_resp = _post(f"{BASE}/code")
+    except (HTTPError, URLError) as e:
+        print(f"Failed to start device flow: {e}")
+        sys.exit(1)
+
+    device_code = code_resp.get("device_code")
+    user_code = code_resp.get("user_code")
+    verification_uri = code_resp.get("verification_uri")
+    interval = code_resp.get("interval", 5)
+    expires_in = code_resp.get("expires_in", 900)
+
+    if not device_code or not user_code:
+        print(f"Unexpected response: {json.dumps(code_resp, indent=2)}")
+        sys.exit(1)
+
+    print(f"Your code:  {user_code}")
+    print(f"Open:       {verification_uri}")
+    print(f"Expires in: {expires_in}s\n")
+
+    # Open browser
+    if verification_uri:
+        webbrowser.open(verification_uri)
+        print("Opened browser. Enter the code above, then authorize.\n")
+
+    # Step 2: Poll for token
+    print("Waiting for authorization", end="", flush=True)
+    deadline = time.time() + expires_in
+    access_token = None
+
+    while time.time() < deadline:
+        time.sleep(interval)
+        print(".", end="", flush=True)
+        try:
+            token_resp = _post(f"{BASE}/token", {"device_code": device_code})
+        except HTTPError as e:
+            # Some APIs return 4xx while pending
+            if e.code in (400, 403, 428):
+                continue
+            print(f"\nPoll error: {e}")
+            sys.exit(1)
+        except URLError:
+            continue
+
+        if token_resp.get("access_token"):
+            access_token = token_resp["access_token"]
+            break
+
+        # Check for explicit error states
+        error = token_resp.get("error")
+        if error == "authorization_pending" or error == "slow_down":
+            if error == "slow_down":
+                interval = min(interval + 2, 30)
+            continue
+        if error in ("expired_token", "access_denied"):
+            print(f"\n\nAuth failed: {error}")
+            sys.exit(1)
+
+    if not access_token:
+        print("\n\nTimed out waiting for authorization.")
+        sys.exit(1)
+
+    print(f"\n\nAuthorized! Access token: {access_token[:12]}...\n")
+
+    # Step 3: Fetch profile
+    print("Fetching profile...")
+    try:
+        profile = _get(f"{BASE}/profile", access_token)
+    except (HTTPError, URLError) as e:
+        print(f"Failed to fetch profile: {e}")
+        print(f"(access_token was: {access_token})")
+        sys.exit(1)
+
+    print(f"\nProfile response:\n{json.dumps(profile, indent=2)}\n")
+
+    api_key = profile.get("api_key")
+    if api_key:
+        print("=" * 50)
+        print(f"Your ScrapeCreators API key: {api_key}")
+        print("=" * 50)
+        print(f"\nTo use it: echo 'SCRAPECREATORS_API_KEY={api_key}' >> ~/.config/last30days/.env")
+    else:
+        print("No api_key in profile response. Full response printed above.")
+
+
+if __name__ == "__main__":
+    main()