business-stack 0.1.0

package/backend/app/services/gemini_chat.py

@@ -0,0 +1,113 @@
+"""Gemini generateContent for chat replies over retrieved context."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import httpx
+
+from app.config import Settings
+from app.services.integrations_remote import resolve_gemini_api_key
+
+logger = logging.getLogger(__name__)
+
+_GEMINI_BASE = "https://generativelanguage.googleapis.com/v1beta"
+_MAX_CONTEXT_CHARS = 100_000
+
+
+def _fallback_from_context(context_text: str) -> str:
+    t = context_text.strip()
+    if not t:
+        return (
+            "No matching passages were found in the knowledge base for this question."
+        )
+    head = t[:8000]
+    suffix = "\n\n…(truncated)" if len(t) > 8000 else ""
+    return (
+        "Here are the most relevant passages from your knowledge base "
+        "(chat model unavailable or failed):\n\n"
+        f"{head}{suffix}"
+    )
+
+
+def _extract_text(data: dict[str, Any]) -> str | None:
+    cands = data.get("candidates")
+    if not isinstance(cands, list) or not cands:
+        return None
+    first = cands[0]
+    if not isinstance(first, dict):
+        return None
+    content = first.get("content")
+    if not isinstance(content, dict):
+        return None
+    parts = content.get("parts")
+    if not isinstance(parts, list):
+        return None
+    texts: list[str] = []
+    for p in parts:
+        if isinstance(p, dict) and isinstance(p.get("text"), str):
+            texts.append(p["text"])
+    out = "\n".join(texts).strip()
+    return out or None
+
+
+async def gemini_chat_reply(
+    settings: Settings,
+    *,
+    user_message: str,
+    context_combined: str,
+) -> tuple[str, str]:
+    """
+    Returns (reply, source) where source is ``model`` or ``retrieval_fallback``.
+    """
+    ctx = context_combined[:_MAX_CONTEXT_CHARS]
+    api_key = await resolve_gemini_api_key(settings)
+    if not api_key:
+        return _fallback_from_context(ctx), "retrieval_fallback"
+
+    model = (settings.gemini_chat_model or "gemini-3-flash-preview").strip()
+    url = f"{_GEMINI_BASE}/models/{model}:generateContent"
+    system_text = (
+        "You are a helpful assistant. Answer using ONLY the context below. "
+        "If the context does not contain the answer, say so clearly and suggest "
+        "what might be missing. Be concise.\n\n"
+        f"--- Context ---\n{ctx}\n--- End context ---"
+    )
+    body: dict[str, Any] = {
+        "systemInstruction": {"parts": [{"text": system_text}]},
+        "contents": [
+            {
+                "role": "user",
+                "parts": [{"text": user_message}],
+            },
+        ],
+    }
+    try:
+        async with httpx.AsyncClient(timeout=120.0) as client:
+            r = await client.post(
+                url,
+                params={"key": api_key},
+                json=body,
+            )
+    except httpx.HTTPError:
+        logger.exception("gemini chat HTTP error")
+        return _fallback_from_context(ctx), "retrieval_fallback"
+
+    if r.status_code != 200:
+        logger.warning(
+            "gemini chat HTTP %s: %s",
+            r.status_code,
+            (r.text or "")[:300],
+        )
+        return _fallback_from_context(ctx), "retrieval_fallback"
+
+    try:
+        data = r.json()
+    except ValueError:
+        return _fallback_from_context(ctx), "retrieval_fallback"
+
+    text = _extract_text(data) if isinstance(data, dict) else None
+    if not text:
+        return _fallback_from_context(ctx), "retrieval_fallback"
+    return text, "model"

package/backend/app/services/hooks/__init__.py

@@ -0,0 +1,3 @@
+from app.services.hooks.post_ingest import dispatch_post_ingest_hooks
+
+__all__ = ["dispatch_post_ingest_hooks"]

package/backend/app/services/hooks/post_ingest.py

@@ -0,0 +1,186 @@
+"""
+Post-ingest notifications when a document reaches ``status = 'ok'`` (after embed).
+
+Templates are Python ``str.format`` strings. Available placeholders:
+
+- ``document_id`` — KB document id
+- ``summary`` — full stored summary (may be long)
+- ``summary_short`` — summary truncated to ``post_ingest_summary_max_chars``
+- ``source_link`` — ``canonical_url``, else first link row, else ``(none)``
+- ``canonical_url`` — raw column or empty string
+- ``source_name`` — ``sources.name``
+
+Use ``{{`` and ``}}`` for literal braces in templates.
+
+**Logging**: webhook URLs and payloads are never written to logs. Failures log only
+the destination label (``slack`` / ``discord``) and HTTP status or exception type.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import httpx
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from app.config import Settings
+
+logger = logging.getLogger(__name__)
+
+
+class _FormatMapping(dict[str, Any]):
+    """Missing keys render as ``{key}`` instead of raising ``KeyError``."""
+
+    def __missing__(self, key: str) -> str:
+        return "{" + key + "}"
+
+
+def _apply_template(template: str, ctx: dict[str, Any]) -> str:
+    try:
+        return template.format_map(_FormatMapping(**ctx))
+    except (ValueError, TypeError) as e:
+        logger.warning(
+            "post-ingest template format failed: %s",
+            type(e).__name__,
+        )
+        return template
+
+
+def _truncate(s: str | None, max_chars: int) -> str:
+    if not s:
+        return ""
+    t = s.strip()
+    if len(t) <= max_chars:
+        return t
+    return t[: max(0, max_chars - 1)] + "…"
+
+
+async def _load_hook_context(
+    session: AsyncSession,
+    *,
+    document_id: str,
+    settings: Settings,
+) -> dict[str, Any] | None:
+    r = await session.execute(
+        text(
+            "SELECT d.status, d.summary, d.canonical_url, s.name "
+            "FROM documents d "
+            "JOIN sources s ON s.id = d.source_id "
+            "WHERE d.id = :id LIMIT 1",
+        ),
+        {"id": document_id},
+    )
+    row = r.first()
+    if row is None:
+        return None
+    status, summary, canonical_url, source_name = row[0], row[1], row[2], row[3]
+    if str(status) != "ok":
+        return None
+
+    link_row = await session.execute(
+        text(
+            "SELECT url FROM document_links WHERE document_id = :d "
+            "ORDER BY ordinal LIMIT 1",
+        ),
+        {"d": document_id},
+    )
+    first_link = link_row.scalar_one_or_none()
+    cu = (canonical_url or "").strip() if canonical_url else ""
+    fl = (first_link or "").strip() if first_link else ""
+    source_link = cu or fl or "(none)"
+
+    sm = summary or ""
+    max_c = settings.post_ingest_summary_max_chars
+    return {
+        "document_id": document_id,
+        "summary": sm,
+        "summary_short": _truncate(sm, max_c),
+        "source_link": source_link,
+        "canonical_url": cu,
+        "source_name": str(source_name or ""),
+    }
+
+
+async def _post_slack(*, webhook_url: str, text_body: str) -> None:
+    payload = {"text": text_body[:15000]}
+    async with httpx.AsyncClient(timeout=15.0) as client:
+        r = await client.post(webhook_url, json=payload)
+        r.raise_for_status()
+
+
+async def _post_discord(*, webhook_url: str, content: str) -> None:
+    payload = {"content": content[:2000]}
+    async with httpx.AsyncClient(timeout=15.0) as client:
+        r = await client.post(webhook_url, json=payload)
+        r.raise_for_status()
+
+
+async def dispatch_post_ingest_hooks(
+    session_factory: async_sessionmaker[AsyncSession],
+    *,
+    document_id: str,
+    settings: Settings,
+) -> None:
+    """
+    If the document is ``ok``, render templates and POST to configured webhooks.
+
+    Safe to call when no webhooks are configured (no-op). Never logs secrets.
+    """
+    slack_u = (settings.post_ingest_slack_webhook_url or "").strip()
+    discord_u = (settings.post_ingest_discord_webhook_url or "").strip()
+    if not slack_u and not discord_u:
+        return
+
+    async with session_factory() as session:
+        ctx = await _load_hook_context(
+            session,
+            document_id=document_id,
+            settings=settings,
+        )
+    if ctx is None:
+        logger.debug("post-ingest skipped (missing doc or not ok): %s", document_id)
+        return
+
+    if slack_u:
+        body = _apply_template(settings.post_ingest_slack_template, ctx)
+        try:
+            await _post_slack(webhook_url=slack_u, text_body=body)
+            logger.info(
+                "post-ingest slack dispatch ok for document_id=%s",
+                document_id,
+            )
+        except httpx.HTTPStatusError as e:
+            logger.warning(
+                "post-ingest slack HTTP %s for document_id=%s",
+                e.response.status_code,
+                document_id,
+            )
+        except httpx.HTTPError as e:
+            logger.warning(
+                "post-ingest slack transport %s for document_id=%s",
+                type(e).__name__,
+                document_id,
+            )
+
+    if discord_u:
+        body = _apply_template(settings.post_ingest_discord_template, ctx)
+        try:
+            await _post_discord(webhook_url=discord_u, content=body)
+            logger.info(
+                "post-ingest discord dispatch ok for document_id=%s",
+                document_id,
+            )
+        except httpx.HTTPStatusError as e:
+            logger.warning(
+                "post-ingest discord HTTP %s for document_id=%s",
+                e.response.status_code,
+                document_id,
+            )
+        except httpx.HTTPError as e:
+            logger.warning(
+                "post-ingest discord transport %s for document_id=%s",
+                type(e).__name__,
+                document_id,
+            )

package/backend/app/services/ingestion/persist.py

@@ -0,0 +1,188 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from uuid import uuid4
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.schemas.ingest import RawIngestEnvelope
+
+
+def compute_envelope_dedupe_hash(envelope: RawIngestEnvelope) -> str:
+    """
+    Content-stable SHA-256 for dedupe when ``external_id`` is absent.
+
+    Excludes ``timestamp`` so replays with a new event time still dedupe.
+    If ``metadata.dedupe_content_hash`` is set (hex string), that value wins.
+    """
+    raw = envelope.metadata.get("dedupe_content_hash")
+    if isinstance(raw, str):
+        s = raw.strip().lower()
+        if len(s) >= 16:
+            return s[:128]
+    stable = {
+        "source": envelope.source,
+        "content_type": envelope.content_type,
+        "payload": envelope.payload,
+        "metadata": envelope.metadata,
+    }
+    payload = json.dumps(stable, sort_keys=True, default=str, separators=(",", ":"))
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+def envelope_external_id(envelope: RawIngestEnvelope) -> str | None:
+    raw = envelope.metadata.get("external_id")
+    if raw is None:
+        return None
+    s = str(raw).strip()
+    return s or None
+
+
+async def get_or_create_source_id(
+    session: AsyncSession,
+    *,
+    name: str,
+    connector_type: str,
+) -> int:
+    res = await session.execute(
+        text(
+            "SELECT id FROM sources WHERE name = :name AND connector_type = :ct "
+            "LIMIT 1",
+        ),
+        {"name": name, "ct": connector_type},
+    )
+    row = res.first()
+    if row is not None:
+        return int(row[0])
+    ins = await session.execute(
+        text(
+            "INSERT INTO sources (name, connector_type) "
+            "VALUES (:name, :ct) RETURNING id",
+        ),
+        {"name": name, "ct": connector_type},
+    )
+    return int(ins.scalar_one())
+
+
+async def find_document_by_dedupe(
+    session: AsyncSession,
+    *,
+    source_id: int,
+    external_id: str | None,
+    dedupe_hash: str,
+) -> str | None:
+    if external_id:
+        r = await session.execute(
+            text(
+                "SELECT id FROM documents WHERE source_id = :sid "
+                "AND external_id = :eid LIMIT 1",
+            ),
+            {"sid": source_id, "eid": external_id},
+        )
+        row = r.first()
+        if row is not None:
+            return str(row[0])
+    r2 = await session.execute(
+        text(
+            "SELECT id FROM documents WHERE source_id = :sid "
+            "AND dedupe_content_hash = :h "
+            "AND (external_id IS NULL OR trim(external_id) = '') "
+            "LIMIT 1",
+        ),
+        {"sid": source_id, "h": dedupe_hash},
+    )
+    row2 = r2.first()
+    return str(row2[0]) if row2 is not None else None
+
+
+async def load_raw_envelope_for_document(
+    session: AsyncSession,
+    *,
+    document_id: str,
+) -> RawIngestEnvelope:
+    r = await session.execute(
+        text("SELECT raw_content FROM documents WHERE id = :id LIMIT 1"),
+        {"id": document_id},
+    )
+    row = r.first()
+    if row is None:
+        msg = "document not found"
+        raise ValueError(msg)
+    data = json.loads(str(row[0]))
+    return RawIngestEnvelope.model_validate(data)
+
+
+async def insert_document_partial(
+    session: AsyncSession,
+    *,
+    envelope: RawIngestEnvelope,
+    source_id: int,
+    external_id: str | None,
+    dedupe_content_hash: str,
+) -> str:
+    doc_id = str(uuid4())
+    stored = json.dumps(envelope.model_dump(mode="json"), default=str)
+    ts = envelope.timestamp.isoformat()
+    await session.execute(
+        text(
+            "INSERT INTO documents "
+            "(id, source_id, timestamp, content_type, raw_content, summary, status, "
+            "external_id, dedupe_content_hash, normalization_error) "
+            "VALUES (:id, :sid, :ts, :ctype, :raw, NULL, 'partial', "
+            ":eid, :dhash, NULL)",
+        ),
+        {
+            "id": doc_id,
+            "sid": source_id,
+            "ts": ts,
+            "ctype": envelope.content_type,
+            "raw": stored,
+            "eid": external_id,
+            "dhash": dedupe_content_hash,
+        },
+    )
+    return doc_id
+
+
+async def persist_raw_envelope(
+    session: AsyncSession,
+    envelope: RawIngestEnvelope,
+) -> tuple[str, bool]:
+    """
+    Insert raw document or return an existing id when ``source_id`` +
+    ``external_id`` or ``source_id`` + content hash matches.
+
+    Returns ``(document_id, deduplicated)``.
+    """
+    source_id = await get_or_create_source_id(
+        session,
+        name=envelope.source,
+        connector_type=envelope.source,
+    )
+    ext = envelope_external_id(envelope)
+    dedupe_hash = compute_envelope_dedupe_hash(envelope)
+    existing = await find_document_by_dedupe(
+        session,
+        source_id=source_id,
+        external_id=ext,
+        dedupe_hash=dedupe_hash,
+    )
+    if existing is not None:
+        return existing, True
+    doc_id = await insert_document_partial(
+        session,
+        envelope=envelope,
+        source_id=source_id,
+        external_id=ext,
+        dedupe_content_hash=dedupe_hash,
+    )
+    return doc_id, False
+
+
+async def clear_normalization_error(session: AsyncSession, *, document_id: str) -> None:
+    await session.execute(
+        text("UPDATE documents SET normalization_error = NULL WHERE id = :id"),
+        {"id": document_id},
+    )

package/backend/app/services/integrations_remote.py

@@ -0,0 +1,91 @@
+"""Fetch integration secrets from the Hono gateway (encrypted SQLite)."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from typing import Any
+
+import httpx
+
+from app.config import Settings
+
+logger = logging.getLogger(__name__)
+
+_TTL_S = 60.0
+_lock = asyncio.Lock()
+_cache_payload: dict[str, Any] | None = None
+_cache_at: float = 0.0
+
+
+def clear_remote_integrations_cache() -> None:
+    """Test hook: drop cached GET /internal/integrations response."""
+    global _cache_payload, _cache_at
+    _cache_payload = None
+    _cache_at = 0.0
+
+
+async def _fetch_integrations_json(settings: Settings) -> dict[str, Any] | None:
+    base = (settings.integrations_gateway_url or "").strip().rstrip("/")
+    secret = (settings.integrations_internal_secret or "").strip()
+    if not base or not secret:
+        return None
+    url = f"{base}/internal/integrations"
+    try:
+        async with httpx.AsyncClient(timeout=10.0) as client:
+            r = await client.get(
+                url,
+                headers={"Authorization": f"Bearer {secret}"},
+            )
+    except httpx.HTTPError:
+        logger.warning("integrations gateway request failed: %s", url)
+        return None
+    if r.status_code != 200:
+        logger.warning(
+            "integrations gateway HTTP %s for %s",
+            r.status_code,
+            url,
+        )
+        return None
+    try:
+        data = r.json()
+    except ValueError:
+        return None
+    if not isinstance(data, dict):
+        return None
+    return data
+
+
+async def get_remote_integrations_payload(
+    settings: Settings,
+) -> dict[str, Any] | None:
+    """Cached plaintext integration dict from the gateway (or None)."""
+    global _cache_payload, _cache_at
+    async with _lock:
+        now = time.monotonic()
+        if _cache_payload is not None and now - _cache_at < _TTL_S:
+            return _cache_payload
+        payload = await _fetch_integrations_json(settings)
+        _cache_payload = payload
+        _cache_at = now
+        return payload
+
+
+async def resolve_gemini_api_key(settings: Settings) -> str | None:
+    """
+    Env ``GEMINI_API_KEY`` wins. Otherwise use ``geminiApiKey`` from the gateway
+    integration store when ``INTEGRATIONS_GATEWAY_URL`` and
+    ``INTEGRATIONS_INTERNAL_SECRET`` are set.
+    """
+    env_key = (settings.gemini_api_key or "").strip()
+    if env_key:
+        return env_key
+    data = await get_remote_integrations_payload(settings)
+    if not data:
+        return None
+    raw = data.get("geminiApiKey")
+    if not isinstance(raw, str):
+        return None
+    key = raw.strip()
+    return key or None

package/backend/app/services/link_expansion/__init__.py

@@ -0,0 +1,3 @@
+from app.services.link_expansion.worker import expand_links_from_document
+
+__all__ = ["expand_links_from_document"]

package/backend/app/services/link_expansion/canonical_url.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
+
+
+def host_from_url(url: str) -> str | None:
+    try:
+        p = urlparse(url.strip())
+    except ValueError:
+        return None
+    if not p.netloc:
+        return None
+    return p.netloc.split("@")[-1].split(":")[0].lower()
+
+
+def canonicalize_url(url: str) -> str:
+    """Normalize URL for dedupe (scheme/host lower, sorted query, no fragment)."""
+    raw = url.strip()
+    try:
+        p = urlparse(raw)
+    except ValueError:
+        return raw
+    if not p.scheme or not p.netloc:
+        return raw
+    scheme = p.scheme.lower()
+    netloc = p.netloc.lower()
+    if scheme == "http" and netloc.endswith(":80"):
+        netloc = netloc[:-3]
+    elif scheme == "https" and netloc.endswith(":443"):
+        netloc = netloc[:-4]
+    path = p.path or "/"
+    if len(path) > 1 and path.endswith("/"):
+        path = path.rstrip("/")
+    q = parse_qsl(p.query, keep_blank_values=True)
+    q.sort()
+    query = urlencode(q)
+    return urlunparse((scheme, netloc, path, "", query, ""))
+
+
+def is_http_url(url: str) -> bool:
+    try:
+        p = urlparse(url.strip())
+    except ValueError:
+        return False
+    return p.scheme in ("http", "https") and bool(p.netloc)

package/backend/app/services/link_expansion/domain_policy.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+
+def parse_host_csv(csv: str) -> frozenset[str]:
+    return frozenset(
+        h.strip().lower().split(":")[0] for h in csv.split(",") if h.strip()
+    )
+
+
+def host_allowed(
+    host: str,
+    *,
+    allowlist: frozenset[str],
+    denylist: frozenset[str],
+) -> bool:
+    h = host.lower().split(":")[0]
+    if h in denylist:
+        return False
+    if not allowlist:
+        return True
+    if h in allowlist:
+        return True
+    for allowed in allowlist:
+        if h == allowed or h.endswith(f".{allowed}"):
+            return True
+    return False