luckiest-co 1.0.47 → 1.0.49

package/skills/trends/scripts/lib/instagram.py

@@ -271,7 +271,15 @@ def _user_reels(
 
     raw_items = data.get("items") or data.get("reels") or data.get("data") or []
     _log(f"  -> {len(raw_items)} reels from @{handle}")
-    return raw_items
+
+    # User-reels responses wrap each reel in a ``media`` envelope, while
+    # other Instagram endpoints already return flat reel dictionaries.
+    return [
+        item["media"]
+        if isinstance(item, dict) and isinstance(item.get("media"), dict)
+        else item
+        for item in raw_items
+    ]
 
 
 def search_instagram(
@@ -415,13 +423,17 @@ def fetch_captions(
         if not url:
             continue
         try:
-            data = http.get(
-                f"{SCRAPECREATORS_BASE}/v2/instagram/media/transcript",
-                params={"url": url},
-                headers=http.scrapecreators_headers(token),
-                timeout=transcript_timeout,
-                retries=1,
-            )
+            # Isolate transcript fetch errors from the pipeline-level
+            # capture_failures() context so an individual reel's 400 doesn't
+            # poison the entire source outcome (#829).
+            with http.capture_failures() as _tf:
+                data = http.get(
+                    f"{SCRAPECREATORS_BASE}/v2/instagram/media/transcript",
+                    params={"url": url},
+                    headers=http.scrapecreators_headers(token),
+                    timeout=transcript_timeout,
+                    retries=1,
+                )
             transcripts = data.get("transcripts") or []
             if transcripts and isinstance(transcripts, list):
                 transcript_text = " ".join(
@@ -494,7 +506,7 @@ def search_and_enrich(
                 items.append(item)
 
     # Sort merged results by views descending
-    items.sort(key=lambda x: x.get("engagement", {}).get("views", 0), reverse=True)
+    items.sort(key=lambda x: x.get("engagement", {}).get("views") or 0, reverse=True)
 
     if not items:
         return {"items": [], "error": last_error}
@@ -519,3 +531,114 @@ def parse_instagram_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
         List of item dicts ready for normalization.
     """
     return response.get("items", [])
+
+
+# ---------------------------------------------------------------------------
+# Comments (ScrapeCreators, opt-in via INCLUDE_SOURCES=instagram_comments)
+# ---------------------------------------------------------------------------
+
+
+def _ig_total_engagement(item: Dict[str, Any]) -> int:
+    """Sum an Instagram item's engagement for picking which posts to enrich."""
+    eng = item.get("engagement", {}) or {}
+    return (eng.get("views") or 0) + (eng.get("likes") or 0) + (eng.get("comments") or 0)
+
+
+def enrich_with_comments(
+    items: List[Dict[str, Any]],
+    token: str,
+    max_posts: int = 3,
+    max_comments: int = 5,
+) -> List[Dict[str, Any]]:
+    """Enrich top Instagram posts with comment data from ScrapeCreators.
+
+    Mirrors ``tiktok.enrich_with_comments`` / ``youtube_yt.enrich_with_comments``:
+    for the top N posts by engagement, fetch comments and attach them as a
+    ``top_comments`` field (highest-liked first). Failures never crash the run.
+    """
+    if not items or not token or max_posts <= 0:
+        return items
+
+    ranked = sorted(items, key=_ig_total_engagement, reverse=True)
+    top_items = ranked[:max_posts]
+    _log(f"Enriching comments for {len(top_items)} Instagram posts")
+
+    from concurrent.futures import ThreadPoolExecutor, as_completed
+
+    def _enrich_one(item: dict) -> bool:
+        post_url = item.get("url", "")
+        if not post_url:
+            return False
+        try:
+            comments = _fetch_post_comments(post_url, token, max_comments)
+            if comments:
+                item["top_comments"] = comments
+                return True
+        except Exception as exc:
+            _log(f"Comment enrichment failed for {post_url}: {exc}")
+        return False
+
+    enriched_count = 0
+    with ThreadPoolExecutor(max_workers=min(4, len(top_items))) as executor:
+        futures = {http.submit_with_context(executor, _enrich_one, item): item for item in top_items}
+        for future in as_completed(futures):
+            if future.result():
+                enriched_count += 1
+
+    _log(f"Enriched {enriched_count}/{len(top_items)} posts with comments")
+    return items
+
+
+def _fetch_post_comments(
+    post_url: str,
+    token: str,
+    max_comments: int = 5,
+) -> List[Dict[str, Any]]:
+    """Fetch comments for a single Instagram post/reel via ScrapeCreators.
+
+    SC endpoint: GET /v2/instagram/post/comments?url=<post_or_reel_url>
+    Response shape: { comments: [{text, comment_like_count, child_comment_count,
+    created_at, user{username, ...}}], cursor }
+
+    Returns:
+        List of comment dicts with author, text, comment_like_count (likes), date,
+        highest-liked first. Empty list on any error — never crashes the pipeline.
+    """
+    try:
+        data = http.get(
+            f"{SCRAPECREATORS_BASE}/v2/instagram/post/comments",
+            params={"url": post_url},
+            headers=http.scrapecreators_headers(token),
+            timeout=30,
+            retries=2,
+        )
+    except Exception as exc:
+        _log(f"Comment fetch error for {post_url}: {exc}")
+        return []
+
+    raw_comments = data.get("comments") or data.get("data") or []
+    # Sort by like count desc so normalize sees the highest-signal first.
+    raw_comments = sorted(
+        raw_comments,
+        key=lambda c: c.get("comment_like_count", 0) or 0,
+        reverse=True,
+    )
+    out: List[Dict[str, Any]] = []
+    for c in raw_comments[:max_comments]:
+        if not isinstance(c, dict):
+            continue
+        text = c.get("text") or ""
+        if not text:
+            continue
+        user = c.get("user") if isinstance(c.get("user"), dict) else {}
+        author = user.get("username") or ""
+        created_at = c.get("created_at") or ""
+        # created_at is ISO 8601 (e.g. "2026-07-04T14:27:58.000Z"); take the date.
+        date_str = created_at[:10] if isinstance(created_at, str) and len(created_at) >= 10 else ""
+        out.append({
+            "author": author,
+            "text": text[:400],
+            "comment_like_count": c.get("comment_like_count", 0) or 0,
+            "date": date_str,
+        })
+    return out

package/skills/trends/scripts/lib/jobs.py

@@ -141,7 +141,8 @@ def _resolve_careers_page(
             candidates.extend([f"https://{host}/careers", f"https://{host}/jobs"])
 
     for url in candidates:
-        html = http.get_text(url, accept="text/html", retries=1)
+        with http.expected_misses(403, 404):
+            html = http.get_text(url, accept="text/html", retries=1)
         if html and _looks_like_careers_html(html):
             return html, url
 
@@ -209,7 +210,8 @@ def _probe_ats(company: str) -> tuple[str | None, str | None, list[str]]:
         for provider in (ATS_PROVIDER_GREENHOUSE, ATS_PROVIDER_ASHBY, ATS_PROVIDER_LEVER):
             attempts.append(f"probe:{provider}:{slug}")
             try:
-                items = _fetch_ats(provider, slug)
+                with http.expected_misses(400, 401, 403, 404):
+                    items = _fetch_ats(provider, slug)
             except http.HTTPError as exc:
                 if exc.status_code in {400, 401, 403, 404}:
                     continue

package/skills/trends/scripts/lib/library.py

@@ -0,0 +1,308 @@
+"""Scan saved last30days research artifacts into a deterministic library."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import uuid
+from dataclasses import dataclass
+from datetime import date, datetime, timezone
+from pathlib import Path
+
+
+DEFAULT_MEMORY_DIR = Path.home() / "Documents" / "Last30Days"
+DEFAULT_BRIEFS_DIR = Path.home() / ".local" / "share" / "last30days" / "briefs"
+LIBRARY_ID_FILENAME = ".last30days-library-id"
+
+_REPORT_TITLE = re.compile(r"^#\s+last30days(?:\s+v[^:]+)?:\s*(.+?)\s*$", re.MULTILINE | re.IGNORECASE)
+_FIRST_TITLE = re.compile(r"^#\s+(.+?)\s*$", re.MULTILINE)
+_DATE_RANGE = re.compile(
+    r"^-\s*Date range:\s*\d{4}-\d{2}-\d{2}\s+to\s+(\d{4}-\d{2}-\d{2})\s*$",
+    re.MULTILINE | re.IGNORECASE,
+)
+_DATED_FILENAME = re.compile(r"-(\d{4}-\d{2}-\d{2})(?:-\d+)?$")
+_RANKED_HEADLINE = re.compile(r"^###\s+1[.)]\s+(.+?)\s*$", re.MULTILINE)
+_SCORE_SUFFIX = re.compile(r"\s+\(score\s+[^)]*\)\s*$", re.IGNORECASE)
+_MARKDOWN_LINK = re.compile(r"\[([^]]+)]\([^)]+\)")
+_LIBRARY_ID = re.compile(r"[0-9a-f]{32}")
+_GENERATED_BRIEF_NAME = re.compile(
+    r"[a-z0-9]+(?:-[a-z0-9]+)*-[0-9a-f]{8}-\d{4}-\d{2}-\d{2}\.html"
+)
+_PRIVATE_CORPUS_BLOCK = re.compile(
+    r"<!-- LAST30DAYS_PRIVATE_CORPUS_START -->.*?"
+    r"<!-- LAST30DAYS_PRIVATE_CORPUS_END -->\s*",
+    re.DOTALL,
+)
+
+
+@dataclass(frozen=True, slots=True)
+class LibraryEntry:
+    """Metadata and source content for one saved research artifact."""
+
+    slug: str
+    topic: str
+    published_date: date
+    headline: str
+    summary: str
+    source_path: Path
+    content: str
+    source_updated_at: datetime
+    source_format: str = "markdown"
+
+    @property
+    def entry_id(self) -> str:
+        return f"urn:last30days:{self.slug}:{self.identity_hash}:{self.published_date.isoformat()}"
+
+    @property
+    def output_name(self) -> str:
+        return f"{self.slug}-{self.identity_hash}-{self.published_date.isoformat()}.html"
+
+    @property
+    def identity_hash(self) -> str:
+        # Include the source filename stem so per-suffix runs of the same
+        # topic on the same date (--save-suffix per-client workflow) stay
+        # distinct entries instead of collapsing to one.
+        seed = f"{self.topic}\n{self.source_path.stem}"
+        return hashlib.sha256(seed.encode("utf-8")).hexdigest()[:8]
+
+
+def slugify(value: str) -> str:
+    slug = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")
+    return slug or "last30days"
+
+
+def get_or_create_library_id(memory_dir: Path | str) -> str:
+    """Return the persisted random namespace for one research library."""
+    memory_path = Path(memory_dir).expanduser()
+    memory_path.mkdir(parents=True, exist_ok=True)
+    id_path = memory_path / LIBRARY_ID_FILENAME
+    try:
+        library_id = id_path.read_text(encoding="utf-8").strip()
+    except FileNotFoundError:
+        library_id = uuid.uuid4().hex
+        try:
+            with id_path.open("x", encoding="utf-8") as handle:
+                handle.write(f"{library_id}\n")
+        except FileExistsError:
+            library_id = id_path.read_text(encoding="utf-8").strip()
+    if not _LIBRARY_ID.fullmatch(library_id):
+        raise ValueError(f"invalid library ID in {id_path}")
+    return library_id
+
+
+def is_generated_brief_name(name: str) -> bool:
+    """Return whether a filename has the exact library-renderer output shape."""
+    return _GENERATED_BRIEF_NAME.fullmatch(name) is not None
+
+
+def scan_library(
+    memory_dir: Path | str = DEFAULT_MEMORY_DIR,
+    briefs_dir: Path | str = DEFAULT_BRIEFS_DIR,
+) -> tuple[list[LibraryEntry], list[str]]:
+    """Return valid saved entries and notes for files that could not be read.
+
+    Hand-edited and foreign files are tolerated: a generic Markdown heading is
+    enough to include a file, while unreadable or unrecognizable files are
+    skipped with a note instead of aborting the entire feed generation.
+    """
+    entries: dict[str, LibraryEntry] = {}
+    notes: list[str] = []
+    memory_path = Path(memory_dir).expanduser()
+    briefs_path = Path(briefs_dir).expanduser()
+
+    if memory_path.is_dir():
+        for path in sorted(memory_path.glob("*.md")):
+            try:
+                entry = _parse_markdown(path)
+                _keep_preferred(entries, entry)
+            except (OSError, UnicodeError, ValueError) as exc:
+                notes.append(f"Skipped {path}: {exc}")
+                continue
+
+    if briefs_path.is_dir():
+        for path in sorted(briefs_path.glob("*.json")):
+            try:
+                entry = _parse_briefing(path)
+                _keep_preferred(entries, entry)
+            except (OSError, UnicodeError, ValueError, json.JSONDecodeError) as exc:
+                notes.append(f"Skipped {path}: {exc}")
+                continue
+
+    ordered = sorted(
+        entries.values(),
+        key=lambda entry: (entry.published_date, entry.topic.casefold(), entry.source_path.name),
+        reverse=True,
+    )
+    return ordered, notes
+
+
+def _keep_preferred(entries: dict[str, LibraryEntry], entry: LibraryEntry) -> None:
+    existing = entries.get(entry.entry_id)
+    if existing is None or entry.source_updated_at > existing.source_updated_at:
+        entries[entry.entry_id] = entry
+
+
+def _parse_markdown(path: Path) -> LibraryEntry:
+    content = path.read_text(encoding="utf-8")
+    public_content = _PRIVATE_CORPUS_BLOCK.sub("", content)
+    title_match = _REPORT_TITLE.search(public_content) or _FIRST_TITLE.search(public_content)
+    if not title_match:
+        raise ValueError("no Markdown title found")
+    topic = _clean_inline(title_match.group(1))
+    if not topic:
+        raise ValueError("empty Markdown title")
+    published_date = _markdown_date(public_content, path)
+    headline = _markdown_headline(public_content) or topic
+    summary = _markdown_summary(public_content) or headline
+    return LibraryEntry(
+        slug=slugify(topic),
+        topic=topic,
+        published_date=published_date,
+        headline=headline,
+        summary=summary,
+        source_path=path,
+        content=content,
+        source_updated_at=_source_updated_at(path),
+    )
+
+
+def _markdown_date(content: str, path: Path) -> date:
+    if match := _DATE_RANGE.search(content):
+        return date.fromisoformat(match.group(1))
+    if match := _DATED_FILENAME.search(path.stem):
+        return date.fromisoformat(match.group(1))
+    return datetime.fromtimestamp(path.stat().st_mtime).date()
+
+
+def _markdown_headline(content: str) -> str:
+    if match := _RANKED_HEADLINE.search(content):
+        return _clean_inline(_SCORE_SUFFIX.sub("", match.group(1)))
+    return ""
+
+
+def _markdown_summary(content: str) -> str:
+    learned = re.search(
+        r"^##\s+What I learned\s*$\n+(.+?)(?=\n#{1,3}\s|\n---|\Z)",
+        content,
+        re.MULTILINE | re.DOTALL | re.IGNORECASE,
+    )
+    if learned:
+        for paragraph in re.split(r"\n\s*\n", learned.group(1)):
+            cleaned = _clean_inline(paragraph)
+            if cleaned:
+                return cleaned[:500]
+    evidence = re.search(r"^\s*-\s*Evidence:\s*(.+?)\s*$", content, re.MULTILINE | re.IGNORECASE)
+    if evidence:
+        return _clean_inline(evidence.group(1))[:500]
+    return ""
+
+
+def _parse_briefing(path: Path) -> LibraryEntry:
+    data = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(data, dict):
+        raise ValueError("briefing JSON is not an object")
+    is_weekly = data.get("type") == "weekly" or path.stem.endswith("-weekly")
+    raw_date = path.stem[:10] if is_weekly else data.get("date") or path.stem[:10]
+    try:
+        published_date = date.fromisoformat(str(raw_date))
+    except ValueError as exc:
+        raise ValueError("briefing has no valid date") from exc
+    topic = "Weekly research briefing" if is_weekly else "Daily research briefing"
+    top = data.get("top_finding") if isinstance(data.get("top_finding"), dict) else {}
+    headline = _briefing_public_headline(data, topic)
+    summary = _briefing_summary(data, headline)
+    markdown = _briefing_markdown(data, topic, published_date, summary)
+    local_title = str(top.get("title") or "")
+    if local_title and local_title != headline:
+        local_title = local_title.replace("LAST30DAYS_PRIVATE_CORPUS", "LAST30DAYS_PRIVATE-CORPUS")
+        markdown += (
+            "\n<!-- LAST30DAYS_PRIVATE_CORPUS_START -->\n"
+            "## Local briefing highlight\n\n"
+            f"{local_title}\n"
+            "<!-- LAST30DAYS_PRIVATE_CORPUS_END -->\n"
+        )
+    return LibraryEntry(
+        slug=slugify(topic),
+        topic=topic,
+        published_date=published_date,
+        headline=headline,
+        summary=summary,
+        source_path=path,
+        content=markdown,
+        source_updated_at=_source_updated_at(path),
+        source_format="json",
+    )
+
+
+def _briefing_public_headline(data: dict[str, object], fallback: str) -> str:
+    top = data.get("top_finding")
+    candidates = [top] if isinstance(top, dict) else []
+    # Legacy daily archives lack source on top_finding but retain it on findings.
+    topics = data.get("topics")
+    if isinstance(topics, list):
+        for topic in topics:
+            if not isinstance(topic, dict):
+                continue
+            for field in ("findings", "top_findings"):
+                findings = topic.get(field)
+                if isinstance(findings, list):
+                    candidates.extend(findings)
+
+    public_titles = []
+    for finding in candidates:
+        if not isinstance(finding, dict):
+            continue
+        source = str(finding.get("source") or "").strip().lower()
+        url = str(finding.get("source_url") or "").strip().lower()
+        if not source or source == "corpus" or url.startswith("corpus:"):
+            continue
+        title = finding.get("source_title") or finding.get("title")
+        if title:
+            engagement = finding.get("engagement_score", finding.get("engagement"))
+            score = engagement if isinstance(engagement, (int, float)) else 0
+            public_titles.append((score, str(title)))
+    return max(public_titles, key=lambda item: item[0])[1] if public_titles else fallback
+
+
+def _source_updated_at(path: Path) -> datetime:
+    seconds, nanoseconds = divmod(path.stat().st_mtime_ns, 1_000_000_000)
+    return datetime.fromtimestamp(seconds, tz=timezone.utc).replace(
+        microsecond=nanoseconds // 1_000
+    )
+
+
+def _briefing_summary(data: dict[str, object], fallback: str) -> str:
+    total_new = data.get("total_new")
+    total_topics = data.get("total_topics")
+    if total_new is not None and total_topics is not None:
+        return f"{total_new} new findings across {total_topics} monitored topics. {fallback}"[:500]
+    topics = data.get("topics")
+    if isinstance(topics, list):
+        return f"Updates across {len(topics)} monitored topics. {fallback}"[:500]
+    return fallback[:500]
+
+
+def _briefing_markdown(data: dict[str, object], topic: str, published_date: date, summary: str) -> str:
+    lines = [f"# {topic}", "", f"- Date: {published_date.isoformat()}", "", summary]
+    if data.get("type") == "weekly" and data.get("week_of"):
+        lines[3:3] = [f"- Week of: {data['week_of']}"]
+    topics = data.get("topics")
+    if isinstance(topics, list):
+        lines.extend(["", "## Topics", ""])
+        for item in topics:
+            if not isinstance(item, dict):
+                continue
+            name = str(item.get("name") or "Untitled topic")
+            count = item.get("new_count", item.get("this_week_count", 0))
+            lines.append(f"- **{name}** — {count} new findings")
+    return "\n".join(lines).strip() + "\n"
+
+
+def _clean_inline(value: str) -> str:
+    value = _MARKDOWN_LINK.sub(r"\1", value)
+    value = re.sub(r"^\s*>\s?", "", value)
+    value = re.sub(r"(?<!\w)(\*\*|__)(?=\S)(.+?)(?<=\S)\1(?!\w)", r"\2", value)
+    value = re.sub(r"(?<!\w)([*_])(?=\S)(.+?)(?<=\S)\1(?!\w)", r"\2", value)
+    value = re.sub(r"(?<!\w)`(?=\S)(.+?)(?<=\S)`(?!\w)", r"\1", value)
+    return re.sub(r"\s+", " ", value).strip()