luckiest-co 1.0.47 → 1.0.49

package/skills/trends/scripts/lib/chrome_cookies.py

@@ -21,6 +21,8 @@ import tempfile
 from pathlib import Path
 from typing import Optional
 
+from .cookie_paths import path_exists, path_is_dir
+
 logger = logging.getLogger(__name__)
 
 
@@ -204,6 +206,7 @@ def _extract_chromium_cookies_macos(
     keychain_service: str,
     domain: str,
     cookie_names: list[str],
+    key_cache: Optional[dict[str, Optional[bytes]]] = None,
 ) -> Optional[dict[str, str]]:
     """Extract cookies from any Chromium-based browser on macOS.
 
@@ -220,7 +223,11 @@ def _extract_chromium_cookies_macos(
         Dict mapping cookie name to decrypted value, or None on failure.
         Only includes cookies that were successfully found and decrypted.
     """
-    if not db_path.exists():
+    try:
+        db_path.stat()
+    except PermissionError:
+        raise
+    except OSError:
         logger.info("%s cookies database not found at %s", keychain_service, db_path)
         return None
 
@@ -229,15 +236,24 @@ def _extract_chromium_cookies_macos(
     tmp_path = None
     try:
         tmp_fd, tmp_path = tempfile.mkstemp(suffix=".sqlite")
-        shutil.copy2(str(db_path), tmp_path)
+        # mkstemp creates the file 0600. copy2 would copy the source DB's
+        # permission bits onto the temp file before the chmod below runs,
+        # briefly exposing live cookies when the source DB is looser.
+        shutil.copyfile(str(db_path), tmp_path)
         _lock_temp_cookie_copy(tmp_path)
     except Exception as e:
-        logger.info("Failed to copy %s cookies database: %s", keychain_service, e)
+        browser_read_denied = isinstance(e, PermissionError) and e.filename == str(db_path)
+        if browser_read_denied:
+            logger.info("Permission denied reading %s cookies database", keychain_service)
+        else:
+            logger.info("Failed to copy %s cookies database: %s", keychain_service, e)
         if tmp_path:
             try:
                 Path(tmp_path).unlink(missing_ok=True)
             except Exception:
                 pass
+        if browser_read_denied:
+            raise
         return None
     finally:
         if tmp_fd is not None:
@@ -274,8 +290,13 @@ def _extract_chromium_cookies_macos(
                     # Keychain prompt for browsers that don't hold the requested
                     # cookie, which matters for FROM_BROWSER=auto across several
                     # installed Chromium browsers.
-                    passphrase = _get_chromium_encryption_key(keychain_service)
-                    aes_key = _derive_aes_key(passphrase) if passphrase else None
+                    if key_cache is not None and keychain_service in key_cache:
+                        aes_key = key_cache[keychain_service]
+                    else:
+                        passphrase = _get_chromium_encryption_key(keychain_service)
+                        aes_key = _derive_aes_key(passphrase) if passphrase else None
+                        if key_cache is not None:
+                            key_cache[keychain_service] = aes_key
                     key_fetched = True
                 if aes_key is None:
                     logger.debug("Skipping encrypted cookie %s — no Keychain access", name)
@@ -316,16 +337,14 @@ def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Option
     same modern ``Default/Network/Cookies`` (Chromium >= 96) and legacy
     ``Default/Cookies`` probing as the rest of the Chromium family.
     """
-    db_path = _find_chromium_cookies_db(CHROME_BASE_DIR)
-    if db_path is None:
-        logger.info("Chrome cookies database not found under %s", CHROME_BASE_DIR)
-        return None
-    return _extract_chromium_cookies_macos(
-        db_path, "Chrome Safe Storage", domain, cookie_names
+    return _extract_chromium_cookies_any_profile(
+        CHROME_BASE_DIR, "Chrome Safe Storage", domain, cookie_names
     )
 
 
-def _profile_cookie_db(profile_dir: Path) -> Optional[Path]:
+def _profile_cookie_db(
+    profile_dir: Path, denials: Optional[list[PermissionError]] = None
+) -> Optional[Path]:
     """Return the Cookies DB inside a profile dir, or None.
 
     Prefers the modern ``Network/Cookies`` location (Chromium >= 96 moved the
@@ -333,10 +352,20 @@ def _profile_cookie_db(profile_dir: Path) -> Optional[Path]:
     to the legacy flat ``Cookies`` file. Different browsers and versions use
     different layouts, so both are probed.
     """
+    denied: list[PermissionError] = []
     for rel in ("Network/Cookies", "Cookies"):
         candidate = profile_dir / rel
-        if candidate.exists():
-            return candidate
+        try:
+            if path_exists(candidate):
+                if denials is not None:
+                    denials.extend(denied)
+                return candidate
+        except PermissionError as exc:
+            denied.append(exc)
+    if denials is not None:
+        denials.extend(denied)
+    elif denied:
+        raise denied[0]
     return None
 
 
@@ -351,28 +380,96 @@ def _find_chromium_cookies_db(base_dir: Path) -> Optional[Path]:
     recently used one is the likeliest to hold current cookies. Lexicographic
     sort would visit "Profile 10" before "Profile 2", which can return the
     wrong profile, so we sort by mtime.
+
+    Kept for backward compatibility; new code should use
+    _find_all_chromium_cookies_dbs() to search across all profiles.
     """
-    found = _profile_cookie_db(base_dir / "Default")
-    if found:
-        return found
+    dbs = _find_all_chromium_cookies_dbs(base_dir)
+    return dbs[0] if dbs else None
+
 
-    found = _profile_cookie_db(base_dir)
-    if found:
-        return found
+def _find_all_chromium_cookies_dbs(
+    base_dir: Path, denials: Optional[list[PermissionError]] = None
+) -> list[Path]:
+    """Return ALL candidate Cookies DBs under base_dir, best-guess order first.
 
+    Order: Default, the base dir itself (Opera's flat layout), then numbered
+    "Profile N" dirs by most-recently-modified. Unlike _find_chromium_cookies_db
+    (which returns the first DB that merely EXISTS), this returns every profile
+    so the caller can pick the one that actually holds the target domain's
+    cookies. Needed because a logged-in session often lives in a non-Default
+    profile while Default still has a (guest-only) cookie DB.
+    """
+    paths: list[Path] = []
+    seen: set[Path] = set()
+    denied: list[PermissionError] = []
+
+    def add(p: Optional[Path]) -> None:
+        if p is not None and p not in seen:
+            seen.add(p)
+            paths.append(p)
+
+    add(_profile_cookie_db(base_dir / "Default", denied))
+    add(_profile_cookie_db(base_dir, denied))
     try:
-        candidates = [
-            child for child in base_dir.iterdir()
-            if child.is_dir() and child.name.startswith("Profile ")
-        ]
-        for child in sorted(candidates, key=lambda p: p.stat().st_mtime, reverse=True):
-            found = _profile_cookie_db(child)
-            if found:
-                return found
+        candidates = []
+        for child in base_dir.iterdir():
+            if not child.name.startswith("Profile "):
+                continue
+            try:
+                if path_is_dir(child):
+                    candidates.append((child.stat().st_mtime, child))
+            except PermissionError as exc:
+                denied.append(exc)
+        for _, child in sorted(candidates, reverse=True):
+            add(_profile_cookie_db(child, denied))
+    except PermissionError as exc:
+        denied.append(exc)
     except OSError:
         pass
+    if denials is not None:
+        denials.extend(denied)
+    elif denied and not paths:
+        raise denied[0]
+    return paths
 
-    return None
+
+def _extract_chromium_cookies_any_profile(
+    base_dir: Path, keychain_service: str, domain: str, cookie_names: list[str]
+) -> Optional[dict[str, str]]:
+    """Try every profile under base_dir and return the best cookie match.
+
+    Returns the first profile that yields ALL requested cookie_names. If no
+    profile has the complete set, returns the first partial match found, or
+    None if no profile yielded any. This fixes the single-profile limitation
+    where a guest-only Default profile shadowed a logged-in "Profile N".
+    """
+    denials: list[PermissionError] = []
+    db_paths = _find_all_chromium_cookies_dbs(base_dir, denials)
+    if not db_paths:
+        if denials:
+            raise denials[0]
+        logger.info("%s cookies database not found under %s", keychain_service, base_dir)
+        return None
+    best: Optional[dict[str, str]] = None
+    key_cache: dict[str, Optional[bytes]] = {}
+    for db_path in db_paths:
+        try:
+            got = _extract_chromium_cookies_macos(
+                db_path, keychain_service, domain, cookie_names, key_cache=key_cache
+            )
+        except PermissionError as exc:
+            denials.append(exc)
+            continue
+        if got:
+            if all(name in got for name in cookie_names):
+                logger.debug("Found complete cookie set for %s in %s", domain, db_path)
+                return got
+            if best is None:
+                best = got
+    if denials:
+        raise denials[0]
+    return best
 
 
 def _find_brave_cookies_db() -> Optional[Path]:
@@ -386,11 +483,9 @@ def extract_brave_cookies_macos(domain: str, cookie_names: list[str]) -> Optiona
     Brave uses the same v10 AES-128-CBC encryption as Chrome; only the DB
     path and Keychain service name differ.
     """
-    db_path = _find_brave_cookies_db()
-    if db_path is None:
-        logger.info("Brave cookies database not found under %s", BRAVE_BASE_DIR)
-        return None
-    return _extract_chromium_cookies_macos(db_path, "Brave Safe Storage", domain, cookie_names)
+    return _extract_chromium_cookies_any_profile(
+        BRAVE_BASE_DIR, "Brave Safe Storage", domain, cookie_names
+    )
 
 
 def extract_chromium_browser_cookies_macos(
@@ -407,8 +502,6 @@ def extract_chromium_browser_cookies_macos(
         logger.debug("Unknown Chromium browser: %s", browser)
         return None
     base_dir, keychain_service = spec
-    db_path = _find_chromium_cookies_db(base_dir)
-    if db_path is None:
-        logger.info("%s cookies database not found under %s", keychain_service, base_dir)
-        return None
-    return _extract_chromium_cookies_macos(db_path, keychain_service, domain, cookie_names)
+    return _extract_chromium_cookies_any_profile(
+        base_dir, keychain_service, domain, cookie_names
+    )

package/skills/trends/scripts/lib/cluster.py

@@ -2,59 +2,25 @@
 
 from __future__ import annotations
 
-import re
+from . import dedupe, entity_extract, schema
 
-from . import dedupe, schema
 
-CLUSTERABLE_INTENTS = {"breaking_news", "opinion", "comparison", "prediction"}
+def _cluster_sort_key(candidate: schema.Candidate) -> tuple:
+    """Sort key that partitions stale candidates below fresh ones.
 
-# Words too common to signal shared topic between clusters.
-_ENTITY_STOPWORDS = frozenset({
-    "the", "a", "an", "to", "for", "how", "is", "in", "of", "on", "and",
-    "with", "from", "by", "at", "this", "that", "it", "what", "are", "do",
-    "can", "his", "her", "he", "she", "its", "was", "has", "new", "just",
-    "says", "said", "will", "about", "after", "now", "all", "been", "here",
-    "not", "out", "up", "more", "also", "but", "who", "year", "first",
-    "make", "being", "making", "over", "into", "than", "they", "their",
-    "would", "could", "get", "got", "some", "like", "back", "going",
-    "breaking", "https", "http", "www", "com",
-})
+    Stale items (all dated source_items outside the window) must never lead
+    cluster representatives or render as the cluster title.
+    """
+    return (
+        1 if schema.candidate_out_of_window(candidate) else 0,
+        -candidate.final_score,
+    )
 
+CLUSTERABLE_INTENTS = {"breaking_news", "opinion", "comparison", "prediction"}
 
 def _candidate_text(candidate: schema.Candidate) -> str:
     return " ".join(part for part in [candidate.title, candidate.snippet] if part).strip()
 
-
-def _extract_entities(text: str) -> set[str]:
-    """Extract significant words (proper nouns, numbers, capitalized words) from text.
-
-    Used for cross-source cluster merging where phrasing differs but entities overlap.
-    """
-    # Normalize but preserve word boundaries
-    words = re.sub(r"[^\w\s]", " ", text).split()
-    entities = set()
-    for word in words:
-        lower = word.lower()
-        if lower in _ENTITY_STOPWORDS or len(word) <= 2:
-            continue
-        # Keep words that are: capitalized, ALL CAPS, contain digits, or 4+ chars
-        if word[0].isupper() or word.isupper() or any(c.isdigit() for c in word) or len(word) >= 4:
-            entities.add(lower)
-    return entities
-
-
-def _entity_overlap(entities_a: set[str], entities_b: set[str]) -> float:
-    """Jaccard-style overlap on extracted entities."""
-    if not entities_a or not entities_b:
-        return 0.0
-    intersection = entities_a & entities_b
-    smaller = min(len(entities_a), len(entities_b))
-    # Use overlap coefficient (intersection / min) instead of Jaccard,
-    # because a short tweet about the same event as a long Reddit post
-    # will have fewer total entities but high overlap with the larger set.
-    return len(intersection) / smaller if smaller > 0 else 0.0
-
-
 def _mmr_representatives(
     candidates: list[schema.Candidate],
     text_cache: dict[str, dedupe._PreparedText],
@@ -66,7 +32,7 @@ def _mmr_representatives(
     remaining = list(candidates)
     while remaining and len(selected) < limit:
         if not selected:
-            best = max(remaining, key=lambda candidate: candidate.final_score)
+            best = min(remaining, key=_cluster_sort_key)
             selected.append(best)
             remaining_set.discard(best.candidate_id)
             remaining = [c for c in remaining if c.candidate_id in remaining_set]
@@ -74,12 +40,16 @@ def _mmr_representatives(
 
         selected_preps = [text_cache[c.candidate_id] for c in selected]
 
-        def score(candidate: schema.Candidate) -> float:
+        def score(candidate: schema.Candidate) -> tuple:
             prep = text_cache[candidate.candidate_id]
             diversity_penalty = max(
                 dedupe.prepared_similarity(prep, sp) for sp in selected_preps
             )
-            return (diversity_lambda * candidate.final_score) - ((1 - diversity_lambda) * diversity_penalty * 100)
+            base_score = (diversity_lambda * candidate.final_score) - ((1 - diversity_lambda) * diversity_penalty * 100)
+            return (
+                0 if schema.candidate_out_of_window(candidate) else 1,
+                base_score,
+            )
 
         best = max(remaining, key=score)
         selected.append(best)
@@ -135,7 +105,7 @@ def cluster_candidates(
 
     clusters: list[schema.Cluster] = []
     for index, group in enumerate(groups, start=1):
-        group.sort(key=lambda candidate: candidate.final_score, reverse=True)
+        group.sort(key=_cluster_sort_key)
         cluster_id = f"cluster-{index}"
         representatives = _mmr_representatives(group, text_cache)
         for candidate in group:
@@ -153,7 +123,11 @@ def cluster_candidates(
         )
 
     # Second pass: merge small clusters that share entities across sources.
-    clusters = _merge_entity_clusters(clusters, candidates)
+    clusters = _merge_entity_clusters(
+        clusters,
+        candidates,
+        min_shared_entities=2 if "discover-mode" in plan.notes else 1,
+    )
 
     return sorted(clusters, key=lambda cluster: cluster.score, reverse=True)
 
@@ -161,6 +135,8 @@ def cluster_candidates(
 def _merge_entity_clusters(
     clusters: list[schema.Cluster],
     all_candidates: list[schema.Candidate],
+    *,
+    min_shared_entities: int = 1,
 ) -> list[schema.Cluster]:
     """Merge small clusters that cover the same story across different sources.
 
@@ -182,7 +158,7 @@ def _merge_entity_clusters(
         for cid in cl.candidate_ids:
             cand = candidate_map.get(cid)
             if cand:
-                entities |= _extract_entities(_candidate_text(cand))
+                entities |= entity_extract.extract_text_entities(_candidate_text(cand))
         cluster_entities.append(entities)
 
     # Only merge clusters with <= 3 items (don't merge already-large clusters)
@@ -207,8 +183,9 @@ def _merge_entity_clusters(
             if poly_i != poly_j:
                 continue
 
-            overlap = _entity_overlap(cluster_entities[i], cluster_entities[j])
-            if overlap >= 0.45:
+            shared_entities = cluster_entities[i] & cluster_entities[j]
+            overlap = entity_extract.entity_overlap(cluster_entities[i], cluster_entities[j])
+            if len(shared_entities) >= min_shared_entities and overlap >= 0.45:
                 merged_into[j] = i
 
     if not merged_into:
@@ -236,7 +213,7 @@ def _merge_entity_clusters(
 
         # Pick representatives from combined pool
         combined_candidates = [candidate_map[cid] for cid in combined_cids if cid in candidate_map]
-        combined_candidates.sort(key=lambda c: c.final_score, reverse=True)
+        combined_candidates.sort(key=_cluster_sort_key)
         merge_text_cache = {
             c.candidate_id: dedupe._PreparedText(_candidate_text(c))
             for c in combined_candidates

package/skills/trends/scripts/lib/competitors.py

@@ -19,6 +19,15 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 from . import dates, grounding, log
 from .resolve import _has_backend
 
+# Peer cap vs total vs-entity cap (main + peers).
+COMPETITORS_MIN = 1
+COMPETITORS_MAX = 6
+COMPETITORS_DEFAULT = 2
+COMPARISON_ENTITY_MAX = COMPETITORS_MAX + 1
+# Discovery SERP fan-out is small (3 queries today) but still needs a ceiling
+# so a future query expansion cannot open one worker per query unbounded.
+MAX_DISCOVERY_WORKERS = 3
+
 # A "brand-shaped" token starts with uppercase OR is camelCase with an
 # uppercase letter later. Catches "Anthropic", "OpenAI", "xAI", "iPhone",
 # "eBay", "Hugging", "Face".
@@ -172,7 +181,7 @@ def discover_competitors(
         items, _artifact = grounding.web_search(query, date_range, config)
         return label, items
 
-    with ThreadPoolExecutor(max_workers=len(queries)) as executor:
+    with ThreadPoolExecutor(max_workers=min(len(queries), MAX_DISCOVERY_WORKERS)) as executor:
         futures = {
             executor.submit(_search, label, q): label
             for label, q in queries.items()