luckiest-co 1.0.46 → 1.0.48

package/skills/trends/scripts/lib/normalize.py

@@ -18,7 +18,12 @@ def filter_by_date_range(
     filtered: list[schema.SourceItem] = []
     for item in items:
         if not item.published_at:
-            if not require_date:
+            basis = item.metadata.get("date_window_basis") if item.source == "grounding" else None
+            if (
+                not require_date
+                or basis == "server_bounds"
+                or basis == "server_start_current"
+            ):
                 filtered.append(item)
             continue
         if item.published_at < from_date or item.published_at > to_date:
@@ -40,12 +45,28 @@ def normalize_source_items(
         "reddit": _normalize_reddit,
         "x": _normalize_x,
         "youtube": _normalize_youtube,
-        "tiktok": lambda s, i, idx, fd, td: _normalize_shortform_video(s, i, idx, fd, td, "TK", "TikTok post"),
-        "instagram": lambda s, i, idx, fd, td: _normalize_shortform_video(s, i, idx, fd, td, "IG", "Instagram reel"),
+        "tiktok": lambda s, i, idx, fd, td: _normalize_shortform_video(
+            s, i, idx, fd, td, "TK", "TikTok post"
+        ),
+        "instagram": lambda s, i, idx, fd, td: _normalize_shortform_video(
+            s, i, idx, fd, td, "IG", "Instagram reel"
+        ),
         "hackernews": _normalize_hackernews,
-        "bluesky": lambda s, i, idx, fd, td: _normalize_microblog(s, i, idx, fd, td, "BS", "Bluesky post"),
-        "truthsocial": lambda s, i, idx, fd, td: _normalize_microblog(s, i, idx, fd, td, "TS", "Truth Social post"),
-        "threads": lambda s, i, idx, fd, td: _normalize_microblog(s, i, idx, fd, td, "TH", "Threads post"),
+        "devto": _normalize_devto,
+        "stocktwits": _normalize_stocktwits,
+        "dripstack": _normalize_dripstack,
+        "bluesky": lambda s, i, idx, fd, td: _normalize_microblog(
+            s, i, idx, fd, td, "BS", "Bluesky post"
+        ),
+        "truthsocial": lambda s, i, idx, fd, td: _normalize_microblog(
+            s, i, idx, fd, td, "TS", "Truth Social post"
+        ),
+        "threads": lambda s, i, idx, fd, td: _normalize_microblog(
+            s, i, idx, fd, td, "TH", "Threads post"
+        ),
+        "telegram": lambda s, i, idx, fd, td: _normalize_microblog(
+            s, i, idx, fd, td, "TG", "Telegram post"
+        ),
         "xquik": _normalize_x,
         "pinterest": _normalize_pinterest,
         "polymarket": _normalize_polymarket,
@@ -53,6 +74,8 @@ def normalize_source_items(
         "arxiv": _normalize_arxiv,
         "techmeme": _normalize_techmeme,
         "trustpilot": _normalize_trustpilot,
+        "amazon": _normalize_amazon,
+        "meta_ads": _normalize_meta_ads,
         "grounding": _normalize_grounding,
         "xiaohongshu": _normalize_grounding,
         "github": _normalize_github,
@@ -63,7 +86,14 @@ def normalize_source_items(
     normalizer = normalizers.get(source)
     if normalizer is None:
         raise ValueError(f"Unsupported source: {source}")
-    normalized = [normalizer(source, item, index, from_date, to_date) for index, item in enumerate(items)]
+    normalized = [
+        normalizer(source, item, index, from_date, to_date)
+        for index, item in enumerate(items)
+    ]
+    if source == "arxiv":
+        # The adapter owns arXiv's 365-day recency contract. Applying the
+        # report window again here drops relevant papers the adapter accepted.
+        return normalized
     if source == "jobs":
         # A careers board is a snapshot of CURRENTLY OPEN roles. An open posting
         # is current evidence regardless of when it was posted, so date-windowing
@@ -72,9 +102,27 @@ def normalize_source_items(
         # Keep the full board; recency is annotated, not used to drop.
         return normalized
     require_date = source == "grounding"
-    filtered = filter_by_date_range(normalized, from_date, to_date, require_date=require_date)
+    filtered = filter_by_date_range(
+        normalized, from_date, to_date, require_date=require_date
+    )
     if filtered:
         return filtered
+    # For evergreen_ok, YouTube search keeps out-of-window videos when fewer
+    # than 3 are recent, then pays for transcripts. A second hard date filter
+    # here dropped those transcribed items to zero (#1043). Keep
+    # transcript-backed retrieved items instead of paying for transcripts that
+    # never appear in the brief. Metadata-only / caption-failed videos are not
+    # that rescue. Recent modes (strict_recent / balanced_recent) get no
+    # rescue: years-old videos on a recent topic are noise, not evidence.
+    if freshness_mode == "evergreen_ok" and source == "youtube" and normalized:
+        transcribed = [
+            item for item in normalized if str(item.snippet or "").strip()
+        ]
+        if transcribed:
+            if require_date:
+                dated = [item for item in transcribed if item.published_at]
+                return dated or transcribed
+            return transcribed
     if freshness_mode == "evergreen_ok" and source == "youtube":
         if require_date:
             return [item for item in normalized if item.published_at]
@@ -86,6 +134,8 @@ def _remap_comments(
     raw: list[Any],
     score_keys: tuple[str, ...],
     excerpt_keys: tuple[str, ...],
+    *,
+    preserve_absent_score: bool = False,
 ) -> list[dict[str, Any]]:
     """Normalize comments from any source into the shared Reddit-compatible shape.
 
@@ -93,19 +143,29 @@ def _remap_comments(
     entity_extract, rerank) all expect `score` and `excerpt`. This helper maps
     per-source field names (YT: likes/text, TikTok: digg_count/text) onto that
     shape while preserving author/date/url passthrough.
+
+    Sources that distinguish an absent vote from a measured zero can opt into
+    preserving the absent value as ``None``.
     """
     out: list[dict[str, Any]] = []
     for raw_c in raw:
         if not isinstance(raw_c, dict):
             continue
-        score = _first_present(raw_c, score_keys, default=0)
+        score = _first_present(
+            raw_c,
+            score_keys,
+            default=None if preserve_absent_score else 0,
+        )
         excerpt = _first_present(raw_c, excerpt_keys, default="")
-        try:
-            score_int = int(score or 0)
-        except (TypeError, ValueError):
-            score_int = 0
+        if score is None and preserve_absent_score:
+            normalized_score = None
+        else:
+            try:
+                normalized_score = int(score or 0)
+            except (TypeError, ValueError):
+                normalized_score = 0
         entry: dict[str, Any] = {
-            "score": score_int,
+            "score": normalized_score,
             "excerpt": str(excerpt or "")[:400],
             "author": str(raw_c.get("author") or ""),
             "date": str(raw_c.get("date") or ""),
@@ -143,7 +203,9 @@ def _domain_from_url(url: str) -> str | None:
     return domain or None
 
 
-def _date_confidence(item: dict[str, Any], from_date: str, to_date: str, default: str = "low") -> str:
+def _date_confidence(
+    item: dict[str, Any], from_date: str, to_date: str, default: str = "low"
+) -> str:
     if item.get("date_confidence"):
         return str(item["date_confidence"])
     date_value = item.get("date")
@@ -187,6 +249,70 @@ def _source_item(
     )
 
 
+def _normalize_stocktwits(
+    source: str,
+    item: dict[str, Any],
+    index: int,
+    from_date: str,
+    to_date: str,
+) -> schema.SourceItem:
+    meta = item.get("metadata") or {}
+    return _source_item(
+        item_id=str(item.get("id") or f"ST{index + 1}"),
+        source=source,
+        title=str(item.get("title") or ""),
+        body=str(item.get("snippet") or ""),
+        url=str(item.get("url") or ""),
+        author=str(item.get("author") or "") or None,
+        container=str(meta.get("symbol") or "") or None,
+        published_at=item.get("date"),
+        date_confidence=_date_confidence(item, from_date, to_date, default="high"),
+        engagement=item.get("engagement") or {},
+        relevance_hint=item.get("relevance", 0.7),
+        why_relevant=str(item.get("why_relevant") or ""),
+        snippet=str(item.get("snippet") or "")[:400],
+        metadata=meta,  # carries sentiment + symbol-level bull/bear aggregate
+    )
+
+
+def _normalize_dripstack(
+    source: str,
+    item: dict[str, Any],
+    index: int,
+    from_date: str,
+    to_date: str,
+) -> schema.SourceItem:
+    """Normalizer for DripStack newsletter search results.
+
+    DripStack returns article metadata from paid financial newsletters.
+    No engagement signal — ranking relies on DripStack's own relevanceScore
+    (0-100, normalized to 0-1) plus recency. The publication name serves as
+    author/attribution (e.g. "SemiAnalysis", "Bloomberg").
+    """
+    meta = item.get("metadata") or {}
+    return _source_item(
+        item_id=str(item.get("id") or f"DS{index + 1}"),
+        source=source,
+        title=str(item.get("title") or ""),
+        body=str(item.get("body") or "")
+        or str(item.get("snippet") or "")
+        or str(item.get("title") or ""),
+        url=str(item.get("url") or ""),
+        author=str(item.get("author") or "") or None,
+        container=str(meta.get("publication_slug") or "") or None,
+        published_at=item.get("date"),
+        date_confidence=_date_confidence(item, from_date, to_date, default="high"),
+        engagement={},
+        relevance_hint=item.get("relevance", 0.5),
+        why_relevant=str(item.get("why_relevant") or ""),
+        snippet=str(item.get("snippet") or "")[:400],
+        metadata={
+            **meta,
+            "publication_slug": meta.get("publication_slug"),
+        },
+    )
+
+
 def _normalize_reddit(
     source: str,
     item: dict[str, Any],
@@ -262,7 +388,9 @@ def _normalize_jobs(
     title = str(item.get("title") or "").strip()
     department = str(item.get("department") or "").strip()
     location = str(item.get("location") or "").strip()
-    body = "\n".join(part for part in [title, department, location, description] if part)
+    body = "\n".join(
+        part for part in [title, department, location, description] if part
+    )
     provider = str(item.get("provider") or "").strip()
     return _source_item(
         item_id=str(item.get("id") or f"J{index + 1}"),
@@ -281,12 +409,15 @@ def _normalize_jobs(
         metadata={
             "provider": provider,
             "department": department,
-            "departments": item.get("departments") or ([department] if department else []),
+            "departments": item.get("departments")
+            or ([department] if department else []),
             "location": location,
             "offices": item.get("offices") or [],
             "board_token": item.get("board_token") or "",
             "source_url": item.get("source_url") or "",
-            "source_domain": item.get("source_domain") or _domain_from_url(str(item.get("url") or "")) or "",
+            "source_domain": item.get("source_domain")
+            or _domain_from_url(str(item.get("url") or ""))
+            or "",
         },
     )
 
@@ -361,9 +492,10 @@ def _normalize_shortform_video(
             "hashtags": item.get("hashtags") or [],
             "top_comments": _remap_comments(
                 item.get("top_comments") or [],
-                # TikTok uses digg_count as the vote field; Instagram has no
-                # comment fetcher today so the key is harmlessly absent.
-                score_keys=("score", "digg_count", "likes"),
+                # Instagram comments use comment_like_count as the vote field
+                # (ScrapeCreators /v2/instagram/post/comments); digg_count/likes
+                # kept for shape compatibility.
+                score_keys=("score", "comment_like_count", "digg_count", "likes"),
                 excerpt_keys=("excerpt", "text"),
             ),
         },
@@ -399,6 +531,41 @@ def _normalize_pinterest(
     )
 
 
+def _normalize_devto(
+    source: str,
+    item: dict[str, Any],
+    index: int,
+    from_date: str,
+    to_date: str,
+) -> schema.SourceItem:
+    top_comments = _remap_comments(
+        item.get("top_comments") or [],
+        score_keys=("replies",),
+        excerpt_keys=("text", "excerpt"),
+        preserve_absent_score=True,
+    )
+    comment_text = _join_comment_excerpts(top_comments, "excerpt")
+    title = str(item.get("title") or "").strip()
+    description = str(item.get("text") or "").strip()
+    body = "\n".join(part for part in [title, description, comment_text] if part)
+    return _source_item(
+        item_id=str(item.get("id") or f"DEV{index + 1}"),
+        source=source,
+        title=title or f"dev.to article {index + 1}",
+        body=body,
+        url=str(item.get("url") or ""),
+        author=str(item.get("author") or ""),
+        container="dev.to",
+        published_at=item.get("date"),
+        date_confidence=_date_confidence(item, from_date, to_date, default="high"),
+        engagement=item.get("engagement") or {},
+        relevance_hint=item.get("relevance", 0.5),
+        why_relevant=str(item.get("why_relevant") or ""),
+        snippet=description or comment_text,
+        metadata={**(item.get("metadata") or {}), "top_comments": top_comments},
+    )
+
+
 def _normalize_hackernews(
     source: str,
     item: dict[str, Any],
@@ -406,10 +573,23 @@ def _normalize_hackernews(
     from_date: str,
     to_date: str,
 ) -> schema.SourceItem:
-    top_comments = item.get("top_comments") or []
-    comment_text = _join_comment_excerpts(top_comments, "text")
+    # HN comments arrive as {author, text, points}; downstream code keys on
+    # score/excerpt, so remap here exactly as the YouTube and TikTok normalisers
+    # do. Without this the per-source floor in render._top_comments_list reads a
+    # `score` that is never present and rejects every HN comment.
+    top_comments = _remap_comments(
+        item.get("top_comments") or [],
+        score_keys=("points", "score"),
+        excerpt_keys=("text", "excerpt"),
+        preserve_absent_score=True,
+    )
+    comment_text = _join_comment_excerpts(top_comments, "excerpt")
     title = str(item.get("title") or "").strip()
-    body = "\n".join(part for part in [title, str(item.get("text") or "").strip(), comment_text] if part)
+    body = "\n".join(
+        part
+        for part in [title, str(item.get("text") or "").strip(), comment_text]
+        if part
+    )
     return _source_item(
         item_id=str(item.get("id") or f"HN{index + 1}"),
         source=source,
@@ -582,6 +762,92 @@ def _normalize_techmeme(
     )
 
 
+def _normalize_meta_ads(
+    source: str,
+    item: dict[str, Any],
+    index: int,
+    from_date: str,
+    to_date: str,
+) -> schema.SourceItem:
+    """Normalizer for Meta Ad Library creatives.
+
+    One item per distinct creative launched inside the window. Identity is the
+    Ad Library permalink, never the landing URL: many creatives for one product
+    share a landing page, and fusion merges by normalized URL, so keying on the
+    landing page would collapse a whole campaign into one candidate and lose
+    every transcript but the first.
+
+    Grounding-exempt on the Amazon precedent. Ad copy is written to sell, not
+    to name the brand -- "Stop making boring drinks" never repeats the topic --
+    and the advertiser page was already resolved by name before any of these
+    were created, so they are on-entity by construction.
+    """
+    body = str(item.get("text") or "").strip()
+    transcript = str(item.get("transcript") or "").strip()
+    advertiser = str(item.get("advertiser") or "").strip()
+    cta = str(item.get("cta") or "").strip()
+    landing = str(item.get("landing_url") or "").strip()
+    placements = [str(p) for p in (item.get("placements") or [])]
+    promo = str(item.get("promo_code") or "").strip()
+
+    # The spoken script is usually the sharper version of the pitch, so it
+    # leads the evidence snippet when present.
+    snippet_parts = [part for part in [body, transcript] if part]
+    # The adapter deliberately keeps creatives that launched inside the window
+    # and have since ended -- a one-week promo push is exactly the signal this
+    # source exists for -- so the wording has to follow the stored state rather
+    # than calling every creative active.
+    running = bool(item.get("is_active"))
+    ended_on = str(item.get("ended_on") or "").strip()
+    if running:
+        state_word = "Running paid creative"
+    elif ended_on:
+        state_word = f"Paid creative that ran until {ended_on}"
+    else:
+        state_word = "Paid creative that has since ended"
+    default_why = f"{state_word} from {advertiser}" if advertiser else state_word
+    context = " │ ".join(
+        part
+        for part in [
+            cta or "",
+            f"code {promo}" if promo else "",
+            landing,
+        ]
+        if part
+    )
+    return _source_item(
+        item_id=str(item.get("id") or f"MA{index + 1}"),
+        source=source,
+        title=str(item.get("title") or "").strip() or body[:140] or f"Meta ad {index + 1}",
+        body=body,
+        url=str(item.get("url") or "").strip(),
+        author=advertiser,
+        container="Meta Ad Library",
+        published_at=item.get("date"),
+        date_confidence=_date_confidence(item, from_date, to_date, default="high"),
+        engagement={"variants": int(item.get("variants") or 1)},
+        relevance_hint=0.8,
+        why_relevant=str(item.get("why_relevant") or "") or default_why,
+        snippet=" ".join(snippet_parts)[:400],
+        metadata={
+            "grounding_exempt": True,
+            "advertiser": advertiser,
+            "page_id": str(item.get("page_id") or ""),
+            "is_active": bool(item.get("is_active")),
+            "ended_on": item.get("ended_on"),
+            "display_format": str(item.get("display_format") or ""),
+            "placements": placements,
+            "cta": cta,
+            "landing_url": landing,
+            "promo_code": promo,
+            "variants": int(item.get("variants") or 1),
+            "has_video": bool(item.get("has_video")),
+            "transcript_snippet": transcript[:400],
+            "ad_context": context,
+        },
+    )
+
+
 def _normalize_trustpilot(
     source: str,
     item: dict[str, Any],
@@ -601,7 +867,8 @@ def _normalize_trustpilot(
     return _source_item(
         item_id=str(item.get("id") or f"TP{index + 1}"),
         source=source,
-        title=title or (f"{name} on Trustpilot" if name else f"Trustpilot reviews {index + 1}"),
+        title=title
+        or (f"{name} on Trustpilot" if name else f"Trustpilot reviews {index + 1}"),
         body=body,
         url=str(item.get("url") or ""),
         author=name or None,
@@ -621,6 +888,87 @@ def _normalize_trustpilot(
     )
 
 
+def _normalize_amazon(
+    source: str,
+    item: dict[str, Any],
+    index: int,
+    from_date: str,
+    to_date: str,
+) -> schema.SourceItem:
+    """Normalizer for Amazon product-and-review signals.
+
+    One item per product. The aggregate rating is current-state evidence, so
+    the item is stamped with today's date on the Trustpilot precedent -- a
+    live 4.4-star average is a fact about now, not about whenever the
+    product launched.
+
+    Reviews arrive already in the shared score/excerpt comment shape (built
+    in the amazon adapter, deliberately not routed through _remap_comments,
+    which would strip the rating/date/verified keys this source needs), so
+    they pass straight through to metadata.
+    """
+    name = str(item.get("name") or "").strip()
+    brand = str(item.get("brand") or "").strip()
+    top_comments = item.get("top_comments") or []
+    comment_text = _join_comment_excerpts(top_comments, "excerpt")
+    rating = item.get("product_rating") if item.get("product_rating") is not None else item.get("rating")
+    ratings_total = item.get("product_rating_count") or item.get("num_ratings") or 0
+
+    headline = " ".join(
+        part for part in [
+            f"{rating}/5" if rating is not None else "",
+            f"({ratings_total:,} ratings)" if ratings_total else "",
+        ] if part
+    )
+    # The brand rides in its own field and is usually absent from the name,
+    # so prepend it -- unless the name already leads with it, which would
+    # otherwise read "Weber Weber Spirit E-325".
+    if brand and not name.lower().startswith(brand.lower()):
+        product_label = f"{brand} {name}".strip()
+    else:
+        product_label = name or brand
+    title = " - ".join(part for part in [product_label, headline] if part)
+    body = "\n".join(part for part in [title, comment_text] if part)
+
+    return _source_item(
+        item_id=str(item.get("asin") or f"AMZ{index + 1}"),
+        source=source,
+        title=title or f"Amazon product {index + 1}",
+        body=body,
+        url=str(item.get("url") or ""),
+        author=brand or None,
+        container="Amazon",
+        published_at=item.get("date"),
+        date_confidence=_date_confidence(item, from_date, to_date, default="low"),
+        engagement=item.get("engagement") or {"ratings": ratings_total},
+        relevance_hint=item.get("relevance", 0.6),
+        why_relevant=str(item.get("why_relevant") or ""),
+        snippet=comment_text[:400],
+        metadata={
+            "asin": str(item.get("asin") or ""),
+            "name": name,
+            "short_name": item.get("short_name") or "",
+            "brand": brand,
+            "rating": item.get("rating"),
+            "num_ratings": item.get("num_ratings") or 0,
+            "price": item.get("price"),
+            "currency": item.get("currency") or "",
+            "badge": item.get("badge") or "",
+            # Recorded, never used as a filter: the flag's distribution
+            # swings with keyword phrasing, so filtering can blank the lane.
+            "sponsored": bool(item.get("sponsored")),
+            "top_comments": top_comments,
+            "product_rating": item.get("product_rating"),
+            "product_rating_count": item.get("product_rating_count") or 0,
+            "star_distribution": item.get("star_distribution") or {},
+            # Relevant by construction: the adapter already gated products
+            # against the model-supplied keyword, and review text rarely
+            # names the product (KTD8).
+            "grounding_exempt": True,
+        },
+    )
+
+
 def _normalize_polymarket(
     source: str,
     item: dict[str, Any],
@@ -635,10 +983,14 @@ def _normalize_polymarket(
         "liquidity": item.get("liquidity") or 0,
     }
     return _source_item(
-        item_id=str(item.get("id") or f"PM{index + 1}"),
+        item_id=str(item.get("event_id") or item.get("id") or f"PM{index + 1}"),
         source=source,
         title=title or question or f"Polymarket event {index + 1}",
-        body="\n".join(part for part in [title, question, str(item.get("price_movement") or "")] if part),
+        body="\n".join(
+            part
+            for part in [title, question, str(item.get("price_movement") or "")]
+            if part
+        ),
         url=str(item.get("url") or ""),
         author=None,
         container="Polymarket",
@@ -649,6 +1001,7 @@ def _normalize_polymarket(
         why_relevant=str(item.get("why_relevant") or ""),
         snippet=str(item.get("price_movement") or ""),
         metadata={
+            "event_id": item.get("event_id"),
             "question": question,
             "end_date": item.get("end_date"),
             "outcome_prices": item.get("outcome_prices") or [],
@@ -657,7 +1010,6 @@ def _normalize_polymarket(
     )
 
 
-
 def _normalize_github(
     source: str,
     item: dict[str, Any],
@@ -690,9 +1042,11 @@ def _normalize_github(
             "labels": metadata.get("labels") or [],
             "state": metadata.get("state", ""),
             "is_pr": metadata.get("is_pr", False),
+            "date_is_snapshot": bool(metadata.get("date_is_snapshot")),
         },
     )
 
+
 def _normalize_grounding(
     source: str,
     item: dict[str, Any],