luckiest-co 1.0.47 → 1.0.48

package/skills/trends/scripts/lib/hosted.py

@@ -0,0 +1,333 @@
+"""Remote API client for last30days (optional hosted-backend mode).
+
+When both LAST30DAYS_API_KEY and LAST30DAYS_API_BASE are set, the engine
+submits the topic to the configured remote API, polls until the run reaches a
+terminal status, streams narration progress to stderr, and renders the
+server's report. No local provider keys are required in this mode. The
+endpoint comes only from LAST30DAYS_API_BASE - there is no built-in default.
+
+Contract (API v1):
+  POST {base}/search   Authorization: Bearer <key>
+                       {"query": ..., "depth": "quick"|"default"|"deep",
+                        "register"?: "exec"|"dev"|"creator"|"eli5"}
+                       -> 200 {"search_id": "<uuid>", "status": "running"}
+                       -> 200 clarify payload {"needs_clarification": true, ...}
+                       -> 401 {"error"} / 402 {"error","requires_credits",
+                          "balance","needed"} / 429 {"error"}
+  GET  {base}/search?id=<uuid>  same auth header; poll until status is
+                       terminal ("complete" | "error"). Running rows carry
+                       "stderr" (narration + engine lines) and "eta_ms";
+                       terminal complete rows carry "synthesis_text" and
+                       "raw_markdown" (stderr stripped).
+
+This module carries ZERO pricing, rate-card, cost, or billing logic.
+Balance/credit numbers are only ever displayed verbatim from API responses.
+The API key is never printed, logged, or persisted by this module.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+import time
+
+from . import env, http, providers, usage
+from .log import source_log
+
+# Distinct exit code for the clarify gate so the invoking model can tell
+# "re-run with a chosen angle" apart from a plain failure (1).
+EXIT_CLARIFY = 3
+
+POLL_INITIAL_DELAY = 3.0
+POLL_MAX_DELAY = 10.0
+POLL_TIMEOUT_SECONDS = 15 * 60
+# GET is idempotent: retry a few times across network blips before giving up.
+POLL_NETWORK_RETRIES = 3
+# Cadence for the compact elapsed/eta progress line (seconds).
+PROGRESS_LINE_INTERVAL = 15.0
+
+NARRATE_PREFIX = "[narrate] step="
+TERMINAL_STATUSES = {"complete", "error"}
+
+
+def _err(msg: str) -> None:
+    source_log("hosted", msg, tty_only=False)
+
+
+def _api_base() -> str:
+    # Endpoint comes only from the environment - no built-in default. Hosted
+    # mode is gated on this being set (see last30days.py), so by the time this
+    # is called it is populated; an empty value means "not configured".
+    base = (os.environ.get("LAST30DAYS_API_BASE") or "").rstrip("/")
+    if not providers.allowed_base_url_override(base):
+        raise ValueError(
+            "LAST30DAYS_API_BASE must use https:// "
+            "(http:// is allowed only for loopback hosts)."
+        )
+    return base
+
+
+def _billing_url() -> str:
+    """Derive a billing link from the configured base, so no URL is hardcoded.
+    Convention: the base is the API-version root (e.g. ends in /api/v1); drop
+    that segment and point at the account's billing page."""
+    base = _api_base()
+    root = re.sub(r"/api/v\d+$", "", base)
+    return f"{root}/dashboard/billing"
+
+
+def _auth_headers() -> dict[str, str]:
+    # Key is read at call time and placed only in the request header;
+    # it must never be interpolated into any log or output line.
+    key = env.read_secret_env("LAST30DAYS_API_KEY") or ""
+    return {"Authorization": f"Bearer {key}"}
+
+
+def submit(query: str, depth: str, register: str = "default") -> dict:
+    """POST the search. retries=1: a blind POST retry could double-submit."""
+    payload = {"query": query, "depth": depth}
+    if register != "default":
+        payload["register"] = register
+    usage.begin("hosted")
+    return http.post(
+        f"{_api_base()}/search",
+        json_data=payload,
+        headers=_auth_headers(),
+        retries=1,
+    )
+
+
+def poll(search_id: str) -> dict:
+    """GET the search row once. Callers own the retry loop (GET is idempotent)."""
+    return http.get(
+        f"{_api_base()}/search",
+        headers=_auth_headers(),
+        params={"id": search_id},
+        retries=1,
+    )
+
+
+def _parse_error_body(exc: http.HTTPError) -> dict:
+    if not exc.body:
+        return {}
+    try:
+        parsed = json.loads(exc.body)
+    except (json.JSONDecodeError, TypeError):
+        return {}
+    return parsed if isinstance(parsed, dict) else {}
+
+
+def _handle_http_error(exc: http.HTTPError) -> int:
+    body = _parse_error_body(exc)
+    if exc.status_code == 401:
+        _err(
+            "API key rejected: invalid or revoked. Check "
+            "LAST30DAYS_API_KEY (and LAST30DAYS_API_BASE), or unset them "
+            "to fall back to local sources."
+        )
+        return 1
+    if exc.status_code == 402:
+        _err(f"API: {body.get('error') or 'insufficient credits.'}")
+        if body.get("balance") is not None or body.get("needed") is not None:
+            _err(
+                f"Balance: {body.get('balance')} credits. "
+                f"Needed for this search: {body.get('needed')} credits."
+            )
+        _err(f"Add credits at {_billing_url()}")
+        return 1
+    if exc.status_code == 429:
+        _err(
+            f"API rate limit hit: "
+            f"{body.get('error') or 'too many requests.'} "
+            "Wait a minute and re-run."
+        )
+        return 1
+    _err(f"API request failed: {exc}")
+    return 1
+
+
+def _handle_clarify(resp: dict) -> int:
+    question = resp.get("question") or "The API needs a clarification before searching."
+    options = resp.get("options") or []
+    _err(f"Clarification needed before this search runs: {question}")
+    for index, option in enumerate(options, 1):
+        label = option if isinstance(option, str) else json.dumps(option)
+        sys.stderr.write(f"  {index}. {label}\n")
+    sys.stderr.flush()
+    _err(
+        "No search was started. Re-run last30days with the chosen angle "
+        "folded into the topic text."
+    )
+    return EXIT_CLARIFY
+
+
+def _print_new_narration(stderr_blob: str, seen: set[str]) -> bool:
+    """Print each '[narrate] step=' line once, verbatim. Returns True if any new."""
+    printed = False
+    for line in stderr_blob.splitlines():
+        if line.startswith(NARRATE_PREFIX) and line not in seen:
+            seen.add(line)
+            sys.stderr.write(f"{line}\n")
+            printed = True
+    if printed:
+        sys.stderr.flush()
+    return printed
+
+
+def _print_progress_line(elapsed: float, eta_ms) -> None:
+    line = f"elapsed {int(elapsed)}s"
+    if isinstance(eta_ms, (int, float)) and eta_ms > 0:
+        line += f", eta ~{int(eta_ms / 1000)}s"
+    _err(line)
+
+
+def _poll_with_retry(search_id: str) -> dict | None:
+    """Poll once, retrying transient network failures. None means give up
+    (a user-facing message has already been printed)."""
+    last_error: http.HTTPError | None = None
+    for attempt in range(POLL_NETWORK_RETRIES):
+        try:
+            return poll(search_id)
+        except http.HTTPError as exc:
+            if exc.status_code is not None and 400 <= exc.status_code < 500 and exc.status_code != 429:
+                _handle_http_error(exc)
+                return None
+            # Network blip / timeout / 5xx / 429: GET is idempotent, retry.
+            last_error = exc
+            if attempt < POLL_NETWORK_RETRIES - 1:
+                time.sleep(POLL_INITIAL_DELAY)
+    _err(
+        f"API unreachable while polling search {search_id} "
+        f"after {POLL_NETWORK_RETRIES} attempts: {last_error}"
+    )
+    return None
+
+
+def _slugify(value: str) -> str:
+    slug = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")
+    return slug or "last30days"
+
+
+def _save_output(topic: str, content: str, emit: str, save_dir: str, suffix: str):
+    """Mirror local save_output() naming: <slug>-raw[-suffix].<ext>."""
+    from datetime import datetime
+    from pathlib import Path
+
+    path = Path(save_dir).expanduser().resolve()
+    path.mkdir(parents=True, exist_ok=True)
+    slug = _slugify(topic)
+    extension = "json" if emit == "json" else "md"
+    suffix_part = f"-{suffix}" if suffix else ""
+    base = path / f"{slug}-raw{suffix_part}.{extension}"
+    date_str = datetime.now().strftime('%Y-%m-%d')
+    candidates = [base]
+    candidates.append(path / f"{slug}-raw{suffix_part}-{date_str}.{extension}")
+    for i in range(1, 100):
+        candidates.append(path / f"{slug}-raw{suffix_part}-{date_str}-{i}.{extension}")
+    encoded = content.encode("utf-8")
+    for candidate in candidates:
+        try:
+            fd = os.open(candidate, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
+        except FileExistsError:
+            continue
+        with os.fdopen(fd, "wb") as f:
+            f.write(encoded)
+        return candidate
+    # Fallback: all 101 candidates existed (extremely unlikely).
+    raise RuntimeError(
+        f"_save_output: could not find a unique filename after 101 attempts in {path}"
+    )
+
+
+def _render_complete(row: dict, topic: str, emit: str, save_dir, save_suffix: str) -> int:
+    synthesis = row.get("synthesis_text") or ""
+    raw_markdown = row.get("raw_markdown") or ""
+    if emit == "json":
+        payload = {
+            key: row.get(key)
+            for key in ("id", "status", "synthesis_text", "raw_markdown")
+            if key in row
+        }
+        rendered = json.dumps(payload, indent=2, sort_keys=True)
+        save_content = rendered
+    else:
+        # The server report is the content source; it already synthesized.
+        # All markdown-ish emit modes print the synthesis text as-is.
+        rendered = synthesis or raw_markdown
+        save_content = raw_markdown or synthesis
+    if save_dir:
+        out_path = _save_output(topic, save_content, emit, save_dir, save_suffix)
+        sys.stderr.write(f"[last30days] Saved output to {out_path}\n")
+        sys.stderr.flush()
+    print(rendered)
+    return 0
+
+
+def run_hosted(
+    topic: str,
+    depth: str,
+    *,
+    emit: str = "compact",
+    save_dir=None,
+    save_suffix: str = "",
+    register: str = "default",
+) -> int:
+    """Submit topic to the remote API, poll to terminal status, render report."""
+    try:
+        base = _api_base()
+    except ValueError as exc:
+        _err(str(exc))
+        return 1
+    _err(f"Running via last30days API ({base}), depth={depth}")
+    try:
+        resp = submit(topic, depth, register=register)
+    except http.HTTPError as exc:
+        return _handle_http_error(exc)
+
+    if resp.get("needs_clarification"):
+        return _handle_clarify(resp)
+
+    search_id = resp.get("search_id")
+    if not search_id:
+        _err(f"Unexpected API response (no search_id): {json.dumps(resp)[:200]}")
+        return 1
+    _err(f"Search submitted (id: {search_id}). Polling for results...")
+
+    started = time.monotonic()
+    delay = POLL_INITIAL_DELAY
+    seen_narration: set[str] = set()
+    last_progress_line = 0.0
+    while True:
+        elapsed = time.monotonic() - started
+        if elapsed > POLL_TIMEOUT_SECONDS:
+            _err(
+                f"Search did not finish within "
+                f"{POLL_TIMEOUT_SECONDS // 60} minutes (id: {search_id}). "
+                "It may still complete server-side; check the dashboard."
+            )
+            return 1
+        time.sleep(delay)
+        delay = min(delay * 2, POLL_MAX_DELAY)
+
+        row = _poll_with_retry(search_id)
+        if row is None:
+            return 1
+
+        status = row.get("status")
+        narrated = _print_new_narration(row.get("stderr") or "", seen_narration)
+        elapsed = time.monotonic() - started
+        if status not in TERMINAL_STATUSES and (
+            narrated or elapsed - last_progress_line >= PROGRESS_LINE_INTERVAL or last_progress_line == 0.0
+        ):
+            _print_progress_line(elapsed, row.get("eta_ms"))
+            last_progress_line = elapsed
+
+        if status == "error":
+            _err(f"Search failed: {row.get('error') or 'unknown server error'}")
+            return 1
+        if status == "complete":
+            _err(f"Search complete in {int(elapsed)}s.")
+            return _render_complete(row, topic, emit, save_dir, save_suffix)
+        # pending | running -> keep polling

package/skills/trends/scripts/lib/html_publish.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import json
+from collections.abc import Mapping
 from typing import Any, Callable
 from urllib.error import HTTPError, URLError
 from urllib.request import Request, urlopen
@@ -15,6 +16,14 @@ class HtmlPublishError(RuntimeError):
     """Raised when the hosted HTML publish endpoint rejects the artifact."""
 
 
+class HtmlPublishBatchResult(dict[str, dict[str, Any]]):
+    """Successful document publishes plus an optional later failure."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.error: HtmlPublishError | None = None
+
+
 def publish_html(
     html_content: str,
     *,
@@ -62,6 +71,31 @@ def publish_html(
     return result
 
 
+def publish_html_documents(
+    documents: Mapping[str, str],
+    *,
+    password: str | None = None,
+    endpoint: str = DEFAULT_ENDPOINT,
+    opener: Callable[..., Any] | None = None,
+    timeout: int = 30,
+) -> HtmlPublishBatchResult:
+    """Publish a named set of documents, preserving caller order in results."""
+    results = HtmlPublishBatchResult()
+    for name, content in documents.items():
+        try:
+            results[name] = publish_html(
+                content,
+                password=password,
+                endpoint=endpoint,
+                opener=opener,
+                timeout=timeout,
+            )
+        except HtmlPublishError as exc:
+            results.error = exc
+            break
+    return results
+
+
 def _error_message(status: int, detail: str) -> str:
     try:
         payload = json.loads(detail)

package/skills/trends/scripts/lib/html_render.py

@@ -4,9 +4,12 @@ from __future__ import annotations
 
 import html
 import re
+from collections import OrderedDict
+from collections.abc import Mapping, Sequence
 from datetime import date
 
-from . import render, schema
+from . import registers, render, schema
+from .library import LibraryEntry
 
 
 PROSE_LABELS = [
@@ -274,6 +277,47 @@ td:first-child { color: var(--fg); font-weight: 500; }
   font-size: 0.95em;
 }
 
+.library-hero {
+  padding: 1.5rem 0 2.5rem;
+  border-bottom: 1px solid var(--border);
+}
+
+.library-hero h1 { margin-bottom: 0.65rem; }
+.library-hero p { max-width: 42rem; color: var(--fg-muted); }
+.library-hero .subscribe { font-weight: 700; }
+
+.library-topic { margin-top: 3rem; }
+
+.library-topic-heading {
+  display: flex;
+  align-items: baseline;
+  justify-content: space-between;
+  gap: 1rem;
+  border-bottom: 1px solid var(--border);
+}
+
+.library-topic-heading h2 { margin-bottom: 0.65rem; }
+.library-topic-heading a { font-size: 0.85rem; }
+
+.library-entry {
+  display: grid;
+  grid-template-columns: 7rem minmax(0, 1fr);
+  column-gap: 1.25rem;
+  padding: 1.35rem 0;
+  border-bottom: 1px solid var(--border);
+}
+
+.library-entry time {
+  grid-row: 1 / span 2;
+  color: var(--fg-subtle);
+  font-family: 'JetBrains Mono', ui-monospace, 'SF Mono', Menlo, monospace;
+  font-size: 0.8rem;
+}
+
+.library-entry h3 { margin: 0 0 0.35rem; font-size: 1.1rem; }
+.library-entry p { margin: 0; color: var(--fg-muted); }
+.library-empty { padding: 3rem 0; }
+
 @media print {
   :root {
     --bg: #ffffff;
@@ -319,6 +363,8 @@ td:first-child { color: var(--fg); font-weight: 500; }
   h1 { font-size: 25px; }
   .badge { font-size: 12px; }
   th, td { padding: 0.65rem 0.5rem; }
+  .library-entry { grid-template-columns: 1fr; }
+  .library-entry time { grid-row: auto; margin-bottom: 0.5rem; }
 }
 """.strip()
 
@@ -346,9 +392,15 @@ def render_html(
     fun_level: str = "medium",
     save_path: str | None = None,
     synthesis_md: str | None = None,
+    register: str = "default",
 ) -> str:
-    _ = fun_level
-    md = render.render_for_html(report, synthesis_md=synthesis_md, save_path=save_path)
+    md = render.render_for_html(
+        report,
+        synthesis_md=synthesis_md,
+        save_path=save_path,
+        fun_level=fun_level,
+        register=register,
+    )
     md = _strip_evidence_block(md)
     md = _strip_invitation(md)
     md = _strip_canonical_boundary(md)
@@ -383,6 +435,120 @@ def render_html_comparison(
     return _wrap_in_template(body, colophon, topic)
 
 
+LIBRARY_BRIEF_MARKER = "<!-- generated by last30days library feed -->"
+
+
+def render_library_brief(entry: LibraryEntry, *, include_private: bool = True) -> str:
+    """Render a scanned Markdown or JSON briefing as a safe standalone page."""
+    md = _strip_invitation(entry.content)
+    md = _strip_canonical_boundary(md)
+    if not include_private:
+        md = _strip_private_corpus(md)
+    body = _markdown_to_html(md)
+    body = _wrap_engine_footer(body)
+    colophon = (
+        '<footer class="colophon">'
+        f"Saved research · {html.escape(entry.published_date.isoformat())} · "
+        f"{html.escape(entry.topic)}"
+        "</footer>"
+    )
+    rendered = scrub_publishable_digit_runs(
+        _wrap_in_template(body, colophon, entry.headline)
+    )
+    # Ownership marker consumed by the library-feed prune: a generated-looking
+    # filename alone must never be grounds for deletion.
+    return rendered.replace("</body>", f"{LIBRARY_BRIEF_MARKER}\n</body>", 1)
+
+
+_PRIVATE_CORPUS_BLOCK = re.compile(
+    r"<!-- LAST30DAYS_PRIVATE_CORPUS_START -->.*?"
+    r"<!-- LAST30DAYS_PRIVATE_CORPUS_END -->\s*",
+    re.DOTALL,
+)
+
+
+def _strip_private_corpus(markdown: str) -> str:
+    """Remove the renderer-marked local corpus section before publication."""
+    return _PRIVATE_CORPUS_BLOCK.sub("", markdown)
+
+
+def render_library_index(
+    entries: Sequence[LibraryEntry],
+    *,
+    entry_urls: Mapping[str, str] | None = None,
+    feed_url: str | None = "feed.xml",
+) -> str:
+    """Render the reverse-chronological, topic-grouped research index."""
+    urls = entry_urls or {}
+    grouped: OrderedDict[str, list[LibraryEntry]] = OrderedDict()
+    for entry in entries:
+        grouped.setdefault(entry.topic, []).append(entry)
+
+    parts = [
+        '<header class="library-hero">',
+        '<span class="badge">RESEARCH LIBRARY</span>',
+        '<h1>What the community is learning</h1>',
+    ]
+    if feed_url is None:
+        parts.append('<p>Saved last30days briefs, newest first.</p>')
+    else:
+        parts.extend([
+            '<p>Saved last30days briefs, newest first. Follow the Atom feed to keep up.</p>',
+            f'<p><a class="subscribe" href="{html.escape(feed_url, quote=True)}">Subscribe via Atom</a></p>',
+        ])
+    parts.append('</header>')
+    if not entries:
+        parts.append('<section class="library-empty"><h2>No saved briefs yet</h2><p>Run last30days research and this library will fill itself.</p></section>')
+    for topic, topic_entries in grouped.items():
+        latest = topic_entries[0]
+        latest_url = urls.get(latest.entry_id, f"briefs/{latest.output_name}")
+        parts.extend([
+            '<section class="library-topic">',
+            '<div class="library-topic-heading">',
+            f'<h2>{html.escape(topic)}</h2>',
+            f'<a href="{html.escape(latest_url, quote=True)}">Latest</a>',
+            '</div>',
+        ])
+        for entry in topic_entries:
+            url = urls.get(entry.entry_id, f"briefs/{entry.output_name}")
+            parts.extend([
+                '<article class="library-entry">',
+                f'<time datetime="{entry.published_date.isoformat()}">{entry.published_date.isoformat()}</time>',
+                f'<h3><a href="{html.escape(url, quote=True)}">{html.escape(entry.headline)}</a></h3>',
+                f'<p>{html.escape(entry.summary)}</p>',
+                '</article>',
+            ])
+        parts.append('</section>')
+    colophon = '<footer class="colophon">Generated locally by <strong>last30days</strong>.</footer>'
+    rendered = _wrap_in_template("\n".join(parts), colophon, "Research library")
+    return scrub_publishable_digit_runs(rendered)
+
+
+_HREF_PATTERN = re.compile(r'(?P<prefix>\bhref\s*=\s*)(?P<quote>["\'])(?P<url>.*?)(?P=quote)', re.IGNORECASE)
+_LONG_DIGIT_RUN = re.compile(r"\d{13,19}")
+
+
+def scrub_publishable_digit_runs(html_content: str) -> str:
+    """Defuse payment-card-shaped digit runs before hosted publishing.
+
+    ht-ml.app rejects pages containing 13-19 digit runs during its safety scan.
+    Social post IDs commonly have that shape. Link targets are percent-encoded
+    so they still resolve; visible occurrences are shortened for readability.
+    """
+    def scrub_href(match: re.Match[str]) -> str:
+        url = _LONG_DIGIT_RUN.sub(
+            lambda digits: "".join(f"%{ord(char):02X}" for char in digits.group(0)),
+            match.group("url"),
+        )
+        return f'{match.group("prefix")}{match.group("quote")}{url}{match.group("quote")}'
+
+    with_safe_hrefs = _HREF_PATTERN.sub(scrub_href, html_content)
+    return _LONG_DIGIT_RUN.sub(
+        lambda digits: f"{digits.group(0)[:6]}…{digits.group(0)[-4:]}",
+        with_safe_hrefs,
+    )
+
+
 def _strip_evidence_block(md: str) -> str:
     return EVIDENCE_BLOCK_PATTERN.sub("", md)