luckiest-co 1.0.46 → 1.0.48

package/skills/trends/scripts/store.py

@@ -12,16 +12,19 @@ Database location: ~/.local/share/last30days/research.db
 
 import argparse
 import json
+import os
+import re
 import sqlite3
 import sys
+from contextlib import contextmanager
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Iterator, List, Optional
 
 SCRIPT_DIR = Path(__file__).parent.resolve()
 sys.path.insert(0, str(SCRIPT_DIR))
 
-from lib import schema
+from lib import dedupe, entity_extract, schema
 
 DB_DIR = Path.home() / ".local" / "share" / "last30days"
 DB_PATH = DB_DIR / "research.db"
@@ -34,6 +37,45 @@ def _get_db_path() -> Path:
     return _db_override or DB_PATH
 
 
+@contextmanager
+def scoped_db(db_path: Optional[Path]) -> Iterator[None]:
+    """Route all store access inside the block to ``db_path``.
+
+    ``None`` keeps the shared store. Scoped runs (``--save-dir``) use this so
+    their findings land next to their briefs instead of leaking into the
+    shared research.db that unscoped searches read.
+    """
+    global _db_override
+    if db_path is None:
+        yield
+        return
+    previous = _db_override
+    _db_override = Path(db_path)
+    try:
+        yield
+    finally:
+        _db_override = previous
+
+
+def ensure_private_db_files(db_path: Optional[Path] = None) -> Path:
+    """Create/harden the research database and SQLite sidecars owner-only."""
+    path = db_path or _get_db_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if not path.exists():
+        try:
+            fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
+        except FileExistsError:
+            pass
+        else:
+            os.close(fd)
+    for candidate in (path, Path(f"{path}-wal"), Path(f"{path}-shm")):
+        try:
+            candidate.chmod(0o600)
+        except FileNotFoundError:
+            pass
+    return path
+
+
 SCHEMA_V1 = """
 PRAGMA journal_mode=WAL;
 PRAGMA synchronous=NORMAL;
@@ -123,20 +165,20 @@ CREATE TABLE IF NOT EXISTS settings (
 );
 """
 
-SCHEMA_V1_DEFAULTS = """
-INSERT OR IGNORE INTO schema_version (version) VALUES (1);
-INSERT OR IGNORE INTO settings (key, value) VALUES ('daily_budget', '5.00');
-INSERT OR IGNORE INTO settings (key, value) VALUES ('delivery_channel', '');
-INSERT OR IGNORE INTO settings (key, value) VALUES ('delivery_mode', 'announce');
-INSERT OR IGNORE INTO settings (key, value) VALUES ('briefing_format', 'concise');
-INSERT OR IGNORE INTO settings (key, value) VALUES ('default_schedule', '0 8 * * *');
-"""
+_DEFAULT_SETTINGS = {
+    "daily_budget": "5.00",
+    "delivery_channel": "",
+    "delivery_mode": "announce",
+    "briefing_format": "concise",
+    "default_schedule": "0 8 * * *",
+}
 
 _UPDATABLE_RUN_COLUMNS = frozenset({
     "source_mode",
     "prompt_tokens",
     "completion_tokens",
     "token_cost",
+    "cost_unknown",
     "duration_seconds",
     "status",
     "error_message",
@@ -181,6 +223,28 @@ CREATE INDEX IF NOT EXISTS idx_finding_sightings_topic_seen
     ON finding_sightings(topic_id, seen_at);
 CREATE INDEX IF NOT EXISTS idx_finding_sightings_url
     ON finding_sightings(source_url);
+""",
+    3: """
+CREATE TABLE IF NOT EXISTS discovery_topics (
+    id INTEGER PRIMARY KEY,
+    name TEXT NOT NULL,
+    normalized_name TEXT NOT NULL UNIQUE,
+    entity_key TEXT,
+    domain TEXT,
+    first_surfaced TEXT NOT NULL,
+    last_surfaced TEXT NOT NULL,
+    surface_count INTEGER NOT NULL DEFAULT 1,
+    status TEXT NOT NULL DEFAULT 'surfaced' CHECK(status IN ('surfaced','covered')),
+    covered_at TEXT,
+    last_run_ref TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_discovery_topics_status_surfaced
+    ON discovery_topics(status, last_surfaced);
+""",
+    4: """
+ALTER TABLE research_runs ADD COLUMN cost_unknown INTEGER NOT NULL DEFAULT 1;
+UPDATE research_runs SET cost_unknown = 0 WHERE token_cost > 0;
 """,
 }
 
@@ -208,9 +272,22 @@ def init_db(db_path: Optional[Path] = None) -> Path:
     conn = _connect(path)
     try:
         conn.executescript(SCHEMA_V1)
-        conn.executescript(SCHEMA_V1_DEFAULTS)
+        if not conn.execute("SELECT 1 FROM schema_version WHERE version = 1").fetchone():
+            conn.execute("INSERT OR IGNORE INTO schema_version (version) VALUES (1)")
+        existing_settings = {row["key"] for row in conn.execute("SELECT key FROM settings")}
+        missing_defaults = [
+            (key, value) for key, value in _DEFAULT_SETTINGS.items()
+            if key not in existing_settings
+        ]
+        if missing_defaults:
+            conn.executemany(
+                "INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)",
+                missing_defaults,
+            )
+        conn.commit()
         _run_migrations(conn)
         conn.commit()
+        _backfill_owner_sightings(conn)
     finally:
         conn.close()
 
@@ -219,27 +296,88 @@ def init_db(db_path: Optional[Path] = None) -> Path:
 
 def _run_migrations(conn: sqlite3.Connection):
     """Apply pending schema migrations."""
-    current = conn.execute(
-        "SELECT MAX(version) FROM schema_version"
-    ).fetchone()[0] or 0
+    current = conn.execute("SELECT MAX(version) FROM schema_version").fetchone()[0] or 0
+    if current >= max(MIGRATIONS, default=0):
+        return
+    conn.execute("BEGIN IMMEDIATE")
+    try:
+        current = conn.execute(
+            "SELECT MAX(version) FROM schema_version"
+        ).fetchone()[0] or 0
+        for version in sorted(MIGRATIONS):
+            if version <= current:
+                continue
+            # executescript commits before running; keep DDL and its marker atomic.
+            statement = ""
+            for character in MIGRATIONS[version]:
+                statement += character
+                if character == ";" and sqlite3.complete_statement(statement):
+                    conn.execute(statement)
+                    statement = ""
+            if statement.strip():
+                conn.execute(statement)
+            conn.execute("INSERT INTO schema_version (version) VALUES (?)", (version,))
+        conn.commit()
+    except BaseException:
+        conn.rollback()
+        raise
 
-    for version in sorted(MIGRATIONS.keys()):
-        if version > current:
-            conn.executescript(MIGRATIONS[version])
-            conn.execute(
-                "INSERT INTO schema_version (version) VALUES (?)", (version,)
-            )
+
+def _backfill_owner_sightings(conn: sqlite3.Connection) -> None:
+    """Preserve legacy first observations before aggregate ownership can change."""
+    marker = "_topic_sightings_backfilled_v1"
+    if conn.execute("SELECT 1 FROM settings WHERE key = ?", (marker,)).fetchone():
+        return
+    with conn:
+        conn.execute("BEGIN IMMEDIATE")
+        if conn.execute("SELECT 1 FROM settings WHERE key = ?", (marker,)).fetchone():
+            return
+        conn.execute(
+            """INSERT INTO finding_sightings
+               (finding_id, run_id, topic_id, source, source_url, source_title,
+                engagement_score, relevance_score, seen_at)
+               SELECT f.id, NULL, f.topic_id, f.source, COALESCE(f.source_url, ''),
+                      f.source_title, f.engagement_score, f.relevance_score, f.first_seen
+               FROM findings f
+               WHERE f.topic_id IS NOT NULL AND f.first_seen IS NOT NULL
+                 AND NOT EXISTS (
+                     SELECT 1 FROM finding_sightings s
+                     WHERE s.finding_id = f.id AND s.topic_id = f.topic_id
+                       AND s.seen_at <= f.first_seen
+                 )"""
+        )
+        conn.execute("INSERT INTO settings (key, value) VALUES (?, '1')", (marker,))
 
 
 # --- Topics ---
 
 
+# Ledger dates survive aggregate ownership reassignment; references are a legacy fallback.
+_FINDING_TOPICS_SQL = """
+SELECT finding_id, topic_id,
+       COALESCE(MIN(sighting_seen), MIN(legacy_first_seen)) AS first_seen
+FROM (
+    SELECT id AS finding_id, topic_id, NULL AS sighting_seen, first_seen AS legacy_first_seen
+    FROM findings WHERE topic_id IS NOT NULL
+    UNION ALL
+    SELECT finding_id, topic_id, seen_at, NULL
+    FROM finding_sightings WHERE topic_id IS NOT NULL
+    UNION ALL
+    SELECT f.id, r.topic_id, NULL, r.run_date FROM findings f
+    JOIN research_runs r ON r.id = f.run_id WHERE r.topic_id IS NOT NULL
+)
+GROUP BY finding_id, topic_id
+"""
+
+
 def add_topic(
     name: str,
     search_queries: Optional[List[str]] = None,
     schedule: str = "0 8 * * *",
+    *,
+    update_existing: bool = True,
 ) -> Dict[str, Any]:
-    """Add a topic to the watchlist. Returns the topic dict."""
+    """Add a topic, optionally preserving an existing topic's configuration."""
     init_db()
     conn = _connect()
     try:
@@ -250,8 +388,9 @@ def add_topic(
                ON CONFLICT(name) DO UPDATE SET
                    search_queries = excluded.search_queries,
                    schedule = excluded.schedule,
-                   updated_at = datetime('now')""",
-            (name, queries_json, schedule),
+                   updated_at = datetime('now')
+               WHERE ?""",
+            (name, queries_json, schedule, update_existing),
         )
         conn.commit()
         row = conn.execute(
@@ -267,14 +406,44 @@ def remove_topic(name: str) -> bool:
     init_db()
     conn = _connect()
     try:
+        conn.execute("BEGIN IMMEDIATE")
         row = conn.execute(
             "SELECT id FROM topics WHERE name = ?", (name,)
         ).fetchone()
         if not row:
             return False
         topic_id = row["id"]
-        # Delete findings and runs for this topic
-        conn.execute("DELETE FROM findings WHERE topic_id = ?", (topic_id,))
+        findings = conn.execute(
+            f"""WITH memberships AS ({_FINDING_TOPICS_SQL})
+                SELECT f.id, f.topic_id, f.run_id, r.topic_id AS run_topic_id,
+                       (SELECT MIN(m.topic_id) FROM memberships m
+                        WHERE m.finding_id = f.id AND m.topic_id != ?) AS survivor
+                FROM findings f
+                LEFT JOIN research_runs r ON r.id = f.run_id
+                WHERE f.id IN (SELECT finding_id FROM memberships WHERE topic_id = ?)""",
+            (topic_id, topic_id),
+        ).fetchall()
+        for finding in findings:
+            if finding["survivor"] is None:
+                conn.execute("DELETE FROM findings WHERE id = ?", (finding["id"],))
+                continue
+            owner_id = finding["topic_id"]
+            if owner_id == topic_id:
+                owner_id = finding["survivor"]
+            run_id = finding["run_id"]
+            if finding["run_topic_id"] == topic_id:
+                surviving_run = conn.execute(
+                    """SELECT s.run_id FROM finding_sightings s
+                       JOIN research_runs r ON r.id = s.run_id
+                       WHERE s.finding_id = ? AND s.topic_id != ? AND r.topic_id != ?
+                       ORDER BY s.seen_at DESC, s.id DESC LIMIT 1""",
+                    (finding["id"], topic_id, topic_id),
+                ).fetchone()
+                run_id = surviving_run["run_id"] if surviving_run else None
+            conn.execute(
+                "UPDATE findings SET topic_id = ?, run_id = ? WHERE id = ?",
+                (owner_id, run_id, finding["id"]),
+            )
         conn.execute("DELETE FROM research_runs WHERE topic_id = ?", (topic_id,))
         conn.execute("DELETE FROM topics WHERE id = ?", (topic_id,))
         conn.commit()
@@ -289,8 +458,9 @@ def list_topics() -> List[Dict[str, Any]]:
     conn = _connect()
     try:
         rows = conn.execute(
-            """SELECT t.*,
-                      (SELECT COUNT(*) FROM findings WHERE topic_id = t.id) as finding_count,
+            f"""WITH memberships AS ({_FINDING_TOPICS_SQL})
+               SELECT t.*,
+                      (SELECT COUNT(*) FROM memberships WHERE topic_id = t.id) as finding_count,
                       (SELECT MAX(run_date) FROM research_runs WHERE topic_id = t.id) as last_run,
                       (SELECT status FROM research_runs WHERE topic_id = t.id
                        ORDER BY created_at DESC LIMIT 1) as last_status
@@ -326,7 +496,7 @@ def record_run(
     duration_seconds: float = 0,
     prompt_tokens: int = 0,
     completion_tokens: int = 0,
-    token_cost: float = 0,
+    token_cost: Optional[float] = None,
 ) -> int:
     """Record a research run. Returns the run ID."""
     conn = _connect()
@@ -334,11 +504,12 @@ def record_run(
         cursor = conn.execute(
             """INSERT INTO research_runs
                (topic_id, run_date, source_mode, status, error_message,
-                duration_seconds, prompt_tokens, completion_tokens, token_cost)
-               VALUES (?, datetime('now'), ?, ?, ?, ?, ?, ?, ?)""",
+                duration_seconds, prompt_tokens, completion_tokens, token_cost, cost_unknown)
+               VALUES (?, datetime('now'), ?, ?, ?, ?, ?, ?, ?, ?)""",
             (
                 topic_id, source_mode, status, error_message,
                 duration_seconds, prompt_tokens, completion_tokens, token_cost,
+                int(token_cost is None),
             ),
         )
         conn.commit()
@@ -424,7 +595,7 @@ def store_findings(
 
         for url, f in with_urls:
             existing = existing_by_url.get(url)
-            new_engagement = f.get("engagement_score", 0)
+            new_engagement = f.get("engagement_score") or 0
             if existing:
                 update_rows.append((
                     max(new_engagement, existing["engagement_score"] or 0),
@@ -543,8 +714,8 @@ def _record_sightings(
             finding.get("source", "unknown"),
             url,
             finding.get("source_title") or finding.get("title", ""),
-            finding.get("engagement_score", 0),
-            finding.get("relevance_score", 0),
+            finding.get("engagement_score") if finding.get("engagement_score") is not None else 0,
+            finding.get("relevance_score") if finding.get("relevance_score") is not None else 0,
         ))
 
     if not sighting_rows:
@@ -666,25 +837,31 @@ def _delta_source_counts(
 def get_new_findings(
     topic_id: int,
     since: Optional[str] = None,
+    before: Optional[str] = None,
 ) -> List[Dict[str, Any]]:
-    """Get findings for a topic, optionally since a date."""
+    """Get findings by their first observation for this topic within the date bounds."""
     conn = _connect()
     try:
+        date_filter = ""
+        parameters: List[Any] = [topic_id]
         if since:
-            rows = conn.execute(
-                """SELECT * FROM findings
-                   WHERE topic_id = ? AND first_seen >= ? AND dismissed = 0
-                   ORDER BY first_seen DESC""",
-                (topic_id, since),
-            ).fetchall()
-        else:
-            rows = conn.execute(
-                """SELECT * FROM findings
-                   WHERE topic_id = ? AND dismissed = 0
-                   ORDER BY first_seen DESC""",
-                (topic_id,),
-            ).fetchall()
-        return [dict(r) for r in rows]
+            date_filter += " AND m.first_seen >= ?"
+            parameters.append(since)
+        if before:
+            date_filter += " AND m.first_seen < ?"
+            parameters.append(before)
+        rows = conn.execute(
+            f"""WITH memberships AS ({_FINDING_TOPICS_SQL})
+                SELECT f.*, m.first_seen AS topic_first_seen FROM findings f
+                JOIN memberships m ON m.finding_id = f.id
+                WHERE m.topic_id = ? AND f.dismissed = 0{date_filter}
+                ORDER BY m.first_seen DESC""",
+            parameters,
+        ).fetchall()
+        findings = [dict(r) for r in rows]
+        for finding in findings:
+            finding["first_seen"] = finding.pop("topic_first_seen")
+        return findings
     finally:
         conn.close()
 
@@ -740,6 +917,209 @@ def dismiss_finding(finding_id: int):
     update_finding(finding_id, dismissed=1)
 
 
+# --- Discovery topic queue ---
+
+# Conservative floor for fuzzy queue matching (overlap coefficient of entity
+# tokens via entity_extract.entity_overlap). Tunable: raise toward 1.0 for
+# stricter matching, lower for looser. Matching is annotate-only - a fuzzy
+# match stamps prior-surfacing context onto an incoming topic but NEVER merges
+# queue rows, so a too-loose threshold can mislabel a card yet never lose data.
+DISCOVERY_QUEUE_OVERLAP_THRESHOLD = 0.6
+
+
+def _normalize_discovery_name(name: str) -> str:
+    """Queue identity: lowercased, punctuation-stripped, whitespace-collapsed
+    (thin alias for dedupe.normalize_text)."""
+    return dedupe.normalize_text(name)
+
+
+def _discovery_entity_key(name: str) -> str:
+    """Sorted joined significant tokens, computed once at write time."""
+    return " ".join(sorted(entity_extract.extract_text_entities(name)))
+
+
+def _discovery_anchor_entities(name: str) -> set[str]:
+    """Anchor tokens for fuzzy matching: capitalized, all-caps, or
+    digit-bearing words minus stopwords (product/person/version anchors).
+
+    Generic lowercase words ("chat", "templates") are excluded so two angles
+    on the same subject ("Gemma 4 chat templates" / "Gemma 4 tool calling
+    fixes") cross-match while different subjects sharing filler words don't.
+    """
+    anchors = set()
+    for word in re.sub(r"[^\w\s]", " ", name).split():
+        lower = word.casefold()
+        if lower in entity_extract.ENTITY_STOPWORDS:
+            continue
+        if entity_extract.has_anchor_signal(word):
+            anchors.add(lower)
+    return anchors
+
+
+def record_discovery_surfacing(
+    name: str,
+    domain: str = "",
+    run_ref: str = "",
+    as_of: str = "",
+    inherit_covered_at: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Upsert a queue row by normalized name.
+
+    A fresh topic inserts with surface_count 1; re-surfacing the same
+    normalized name increments the count and refreshes last_surfaced and
+    last_run_ref (first_surfaced never changes). Returns the resulting row.
+
+    A resurfacing with a blank domain (e.g. a global-trending sweep with no
+    domain) never blanks a domain recorded by an earlier, domain-scoped
+    surfacing - the stored domain only changes when the incoming domain is
+    non-empty. ``domain`` is normalized to "" here (never NULL bound) so the
+    column's storage convention stays consistent regardless of whether a
+    caller passes "" or None.
+
+    ``inherit_covered_at`` makes a FRESH row be born covered (status
+    'covered', covered_at set to the given date). Callers pass it when this
+    name fuzzy-matched an already-covered prior row, so a user's covered
+    mark survives judge naming drift instead of forking into a fresh
+    uncovered row. An existing row's status/covered_at are never modified
+    by this function - the ON CONFLICT path deliberately ignores it.
+
+    Idempotency guard: when the existing row's last_run_ref already equals
+    this call's (non-blank) run_ref, the surfacing was ALREADY counted by
+    this run identity - a retry (e.g. a --finalize re-run with a corrected
+    angles file) returns the row unchanged instead of double-counting.
+    Blank run_refs never guard, so callers without a run identity keep the
+    every-call-increments behavior.
+    """
+    init_db()
+    domain = domain or ""
+    normalized = _normalize_discovery_name(name)
+    entity_key = _discovery_entity_key(name)
+    status = "covered" if inherit_covered_at else "surfaced"
+    conn = _connect()
+    try:
+        if run_ref:
+            existing = conn.execute(
+                "SELECT * FROM discovery_topics WHERE normalized_name = ?",
+                (normalized,),
+            ).fetchone()
+            if existing is not None and existing["last_run_ref"] == run_ref:
+                return dict(existing)
+        conn.execute(
+            """INSERT INTO discovery_topics
+               (name, normalized_name, entity_key, domain, first_surfaced,
+                last_surfaced, surface_count, last_run_ref, status, covered_at)
+               VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?)
+               ON CONFLICT(normalized_name) DO UPDATE SET
+                   surface_count = surface_count + 1,
+                   last_surfaced = excluded.last_surfaced,
+                   last_run_ref = excluded.last_run_ref,
+                   domain = CASE WHEN excluded.domain <> '' THEN excluded.domain ELSE domain END""",
+            (name, normalized, entity_key, domain, as_of, as_of, run_ref, status, inherit_covered_at),
+        )
+        conn.commit()
+        row = conn.execute(
+            "SELECT * FROM discovery_topics WHERE normalized_name = ?",
+            (normalized,),
+        ).fetchone()
+        return dict(row)
+    finally:
+        conn.close()
+
+
+def match_discovery_topic(name: str) -> Optional[Dict[str, Any]]:
+    """Find the queue row a topic name refers to, or None.
+
+    Exact normalized-name match wins; otherwise the best entity-overlap match
+    at or above DISCOVERY_QUEUE_OVERLAP_THRESHOLD. Overlap is the better of
+    the full entity_key token overlap and the anchor-token overlap (see
+    _discovery_anchor_entities) - full-token overlap alone dilutes the subject
+    anchor with generic words, so same-subject near-duplicates would never
+    clear a conservative floor. Matching NEVER merges rows: a fuzzy match only
+    annotates the incoming topic with the prior row's context.
+    """
+    init_db()
+    normalized = _normalize_discovery_name(name)
+    conn = _connect()
+    try:
+        row = conn.execute(
+            "SELECT * FROM discovery_topics WHERE normalized_name = ?",
+            (normalized,),
+        ).fetchone()
+        if row:
+            return dict(row)
+
+        entities = entity_extract.extract_text_entities(name)
+        anchors = _discovery_anchor_entities(name)
+        if not entities and not anchors:
+            return None
+        best: Optional[sqlite3.Row] = None
+        best_overlap = 0.0
+        for candidate in conn.execute("SELECT * FROM discovery_topics").fetchall():
+            candidate_entities = set((candidate["entity_key"] or "").split())
+            overlap = max(
+                entity_extract.entity_overlap(entities, candidate_entities),
+                entity_extract.entity_overlap(
+                    anchors, _discovery_anchor_entities(candidate["name"])
+                ),
+            )
+            if overlap > best_overlap:
+                best, best_overlap = candidate, overlap
+        if best is not None and best_overlap >= DISCOVERY_QUEUE_OVERLAP_THRESHOLD:
+            return dict(best)
+        return None
+    finally:
+        conn.close()
+
+
+def list_discovery_queue(status: Optional[str] = None) -> List[Dict[str, Any]]:
+    """List queue rows, newest surfacing first, optionally filtered by status."""
+    init_db()
+    conn = _connect()
+    try:
+        if status:
+            rows = conn.execute(
+                """SELECT * FROM discovery_topics WHERE status = ?
+                   ORDER BY last_surfaced DESC, id DESC""",
+                (status,),
+            ).fetchall()
+        else:
+            rows = conn.execute(
+                "SELECT * FROM discovery_topics ORDER BY last_surfaced DESC, id DESC"
+            ).fetchall()
+        return [dict(r) for r in rows]
+    finally:
+        conn.close()
+
+
+def mark_discovery_covered(name: str, as_of: str) -> Optional[Dict[str, Any]]:
+    """Mark a queued topic covered by EXACT normalized name.
+
+    Returns the updated row, or None when no row matches - callers must error
+    loudly on None, never silently no-op. Fuzzy matching is deliberately not
+    offered here: covering mutates state, so it demands the exact name.
+    """
+    init_db()
+    normalized = _normalize_discovery_name(name)
+    conn = _connect()
+    try:
+        cursor = conn.execute(
+            """UPDATE discovery_topics
+               SET status = 'covered', covered_at = ?
+               WHERE normalized_name = ?""",
+            (as_of, normalized),
+        )
+        conn.commit()
+        if cursor.rowcount == 0:
+            return None
+        row = conn.execute(
+            "SELECT * FROM discovery_topics WHERE normalized_name = ?",
+            (normalized,),
+        ).fetchone()
+        return dict(row)
+    finally:
+        conn.close()
+
+
 # --- Cost Tracking ---
 
 
@@ -763,6 +1143,18 @@ def get_daily_cost(date: Optional[str] = None) -> float:
 # --- Settings ---
 
 
+def get_daily_unknown_cost_runs(date: Optional[str] = None) -> int:
+    conn = _connect()
+    try:
+        date = date or datetime.now(timezone.utc).strftime("%Y-%m-%d")
+        return conn.execute(
+            "SELECT COUNT(*) FROM research_runs WHERE date(run_date) = date(?) AND cost_unknown = 1",
+            (date,),
+        ).fetchone()[0]
+    finally:
+        conn.close()
+
+
 def get_setting(key: str, default: Optional[str] = None) -> Optional[str]:
     """Get a setting value."""
     init_db()
@@ -852,11 +1244,13 @@ def get_trending(days: int = 7) -> List[Dict[str, Any]]:
     try:
         since = (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d")
         rows = conn.execute(
-            """SELECT t.name, t.id,
+            f"""WITH memberships AS ({_FINDING_TOPICS_SQL})
+               SELECT t.name, t.id,
                       COUNT(f.id) as new_findings,
                       COALESCE(SUM(f.engagement_score), 0) as total_engagement
                FROM topics t
-               LEFT JOIN findings f ON f.topic_id = t.id AND f.first_seen >= ?
+               LEFT JOIN memberships m ON m.topic_id = t.id
+               LEFT JOIN findings f ON f.id = m.finding_id AND m.first_seen >= ?
                WHERE t.enabled = 1
                GROUP BY t.id
                ORDER BY new_findings DESC""",