luckiest-co 1.0.47 → 1.0.49

package/skills/trends/scripts/lib/fanout.py

@@ -16,7 +16,7 @@ from __future__ import annotations
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import Callable
 
-from . import log, schema
+from . import log, schema, youtube_yt
 
 # Sub-runs hit the same upstream APIs as the main topic. Cap parallelism so a
 # 6-way fan-out does not stampede a single backend's rate limit.
@@ -46,11 +46,21 @@ def run_competitor_fanout(
         Ordered list of (entity_name, Report) tuples for runs that succeeded.
         Empty list if every run raised; the caller decides how to surface
         partial-failure modes.
+
+        ``main_topic`` is NOT guaranteed to be present: a main run that raised
+        is dropped like any other. Since the render treats element 0 as the
+        comparison's subject, a caller must verify ``main_topic`` survived
+        before using the list, or it will silently head the report with a
+        competitor.
     """
     if not competitors:
         report = main_runner()
         return [(main_topic, report)]
 
+    # One clear for the whole comparison so entity sub-runs share the YouTube
+    # search cache without inheriting a prior run's results in this process.
+    youtube_yt.reset_search_cache()
+
     workers = min(len(competitors) + 1, MAX_PARALLEL_SUBRUNS)
 
     def _run_one(label: str, fn: Callable[[], schema.Report]) -> tuple[str, schema.Report | None, Exception | None]:

package/skills/trends/scripts/lib/feed.py

@@ -0,0 +1,61 @@
+"""Deterministic Atom rendering for the saved research library."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from datetime import datetime
+from xml.etree import ElementTree as ET
+
+from .library import LibraryEntry
+
+
+ATOM_NS = "http://www.w3.org/2005/Atom"
+ET.register_namespace("", ATOM_NS)
+
+
+def render_atom(
+    entries: Sequence[LibraryEntry],
+    *,
+    library_id: str,
+    entry_urls: Mapping[str, str] | None = None,
+    feed_url: str | None = None,
+    title: str = "last30days research library",
+    author: str = "last30days research library",
+) -> str:
+    """Render an Atom feed whose IDs and timestamps are stable across runs."""
+    urls = entry_urls or {}
+    feed_id = f"urn:last30days:research-library:{library_id}"
+    root = ET.Element(_tag("feed"))
+    ET.SubElement(root, _tag("id")).text = feed_id
+    ET.SubElement(root, _tag("title")).text = title
+    author_node = ET.SubElement(root, _tag("author"))
+    ET.SubElement(author_node, _tag("name")).text = author
+    updated = max((item.source_updated_at for item in entries), default=None)
+    ET.SubElement(root, _tag("updated")).text = (
+        _format_timestamp(updated) if updated else "1970-01-01T00:00:00Z"
+    )
+    if feed_url:
+        ET.SubElement(root, _tag("link"), {"rel": "self", "href": feed_url})
+
+    for item in entries:
+        node = ET.SubElement(root, _tag("entry"))
+        entry_id = item.entry_id.removeprefix("urn:last30days:")
+        ET.SubElement(node, _tag("id")).text = f"{feed_id}:{entry_id}"
+        ET.SubElement(node, _tag("title")).text = item.headline
+        ET.SubElement(node, _tag("updated")).text = _format_timestamp(item.source_updated_at)
+        ET.SubElement(node, _tag("published")).text = f"{item.published_date.isoformat()}T00:00:00Z"
+        ET.SubElement(node, _tag("category"), {"term": item.topic})
+        url = urls.get(item.entry_id, f"briefs/{item.output_name}")
+        ET.SubElement(node, _tag("link"), {"href": url})
+        ET.SubElement(node, _tag("summary"), {"type": "text"}).text = item.summary
+
+    ET.indent(root, space="  ")
+    return '<?xml version="1.0" encoding="utf-8"?>\n' + ET.tostring(root, encoding="unicode") + "\n"
+
+
+def _tag(name: str) -> str:
+    return f"{{{ATOM_NS}}}{name}"
+
+
+def _format_timestamp(value: datetime) -> str:
+    return value.isoformat().replace("+00:00", "Z")

package/skills/trends/scripts/lib/fetch_memo.py

@@ -0,0 +1,135 @@
+"""Run-scoped memo for source fetches.
+
+Subquery streams for TikTok, Instagram, YouTube, and X all search the raw
+topic, and the thin-source retry repeats that search (and, for Polymarket and
+Digg, a planner subquery that equals the retry's core subject), so one run can
+issue the same request several times. :func:`fetch_once` runs a fetch at most once per
+key: the first caller fetches, concurrent callers wait for it (bounded by
+their own timeout), and later callers get a copy of the cached result.
+
+Only clean results are cached. A failure (a raised exception, a result the
+caller's ``cacheable`` predicate rejects, or HTTP failures an adapter
+swallowed into the failure sink) reaches only the callers already waiting on
+that fetch; the next call fetches again, so the thin-source retry still makes
+a real second attempt after a transient failure.
+
+Reset once per command (``last30days.py``) and between tests (``conftest``).
+"""
+
+from __future__ import annotations
+
+import copy
+import threading
+import time
+from typing import Any, Callable, Hashable, TypeVar
+
+from . import health, http
+
+T = TypeVar("T")
+
+
+class MemoWaitTimeout(TimeoutError):
+    """A waiter's budget expired before the in-flight fetch it shares finished."""
+
+    outcome_state = health.TIMEOUT
+
+
+class _Slot:
+    __slots__ = ("event", "outcome")
+
+    def __init__(self) -> None:
+        self.event = threading.Event()
+        # (ok, value_or_exception, recorded_failures) once the owner finishes.
+        self.outcome: tuple[bool, Any, list] | None = None
+
+
+_CACHE: dict[Hashable, Any] = {}
+_INFLIGHT: dict[Hashable, _Slot] = {}
+_LOCK = threading.Lock()
+
+
+def reset() -> None:
+    """Forget cached and in-flight fetches. Called once per command, and by tests."""
+    with _LOCK:
+        _CACHE.clear()
+        _INFLIGHT.clear()
+
+
+def _always(_value: Any) -> bool:
+    return True
+
+
+def _deliver_shared(outcome: tuple[bool, Any, list]) -> Any:
+    ok, value, failures = outcome
+    # The owner's swallowed HTTP failures reach this caller's sink too, or a
+    # shared empty result reads as a clean no-results.
+    for failure in failures:
+        http._record_failure(failure)
+    if not ok:
+        raise value
+    return copy.deepcopy(value)
+
+
+def fetch_once(
+    key: Hashable,
+    fetch: Callable[[], T],
+    *,
+    timeout: float,
+    cacheable: Callable[[T], bool] = _always,
+) -> T:
+    """Return ``fetch()``, running it at most once per ``key`` while it stays clean.
+
+    ``key`` must include every input that changes the request. ``timeout``
+    bounds how long a caller waits on another caller's in-flight fetch; past
+    it, :class:`MemoWaitTimeout` is raised. Every caller gets its own copy.
+    """
+    deadline = time.monotonic() + max(0.0, timeout)
+    while True:
+        with _LOCK:
+            hit = key in _CACHE
+            if hit:
+                cached = _CACHE[key]
+            else:
+                slot = _INFLIGHT.get(key)
+                owner = slot is None
+                if owner:
+                    slot = _Slot()
+                    _INFLIGHT[key] = slot
+        if hit:
+            return copy.deepcopy(cached)
+        if owner:
+            break
+        if not slot.event.wait(max(0.0, deadline - time.monotonic())):
+            raise MemoWaitTimeout(
+                f"waited {timeout:g}s for an identical in-flight fetch"
+            )
+        if slot.outcome is not None:
+            return _deliver_shared(slot.outcome)
+        # The owner was interrupted before publishing (e.g. KeyboardInterrupt):
+        # loop and elect a new owner.
+
+    try:
+        with http.tee_failures() as recorded:
+            try:
+                value = fetch()
+                ok = True
+            except Exception as exc:
+                value = exc
+                ok = False
+        clean = ok and not recorded and cacheable(value)
+        # Waiters and the cache copy from this snapshot, never from the
+        # owner's own value, which the owner's caller may mutate.
+        shared = copy.deepcopy(value) if ok else value
+        slot.outcome = (ok, shared, list(recorded))
+        with _LOCK:
+            # Slot identity guards against a reset() during the fetch.
+            if clean and _INFLIGHT.get(key) is slot:
+                _CACHE[key] = shared
+    finally:
+        with _LOCK:
+            if _INFLIGHT.get(key) is slot:
+                _INFLIGHT.pop(key)
+        slot.event.set()
+    if not ok:
+        raise value
+    return value