business-stack 0.1.0

package/backend/app/services/link_expansion/html_extract.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from html.parser import HTMLParser
+from urllib.parse import urljoin
+
+
+class _LinkCollector(HTMLParser):
+    def __init__(self, base_url: str) -> None:
+        super().__init__(convert_charrefs=True)
+        self.base_url = base_url
+        self.links: list[str] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if tag.lower() != "a":
+            return
+        href = None
+        for k, v in attrs:
+            if k.lower() == "href" and v:
+                href = v
+                break
+        if not href:
+            return
+        h = href.strip()
+        if h.startswith(("#", "mailto:", "javascript:", "tel:")):
+            return
+        self.links.append(urljoin(self.base_url, h))
+
+
+class _TextCollector(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self._skip = 0
+        self.parts: list[str] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        t = tag.lower()
+        if t in ("script", "style", "noscript"):
+            self._skip += 1
+
+    def handle_endtag(self, tag: str) -> None:
+        t = tag.lower()
+        if t in ("script", "style", "noscript") and self._skip > 0:
+            self._skip -= 1
+
+    def handle_data(self, data: str) -> None:
+        if self._skip == 0 and data.strip():
+            self.parts.append(data)
+
+
+def extract_links_from_html(html: str, base_url: str) -> list[str]:
+    p = _LinkCollector(base_url)
+    try:
+        p.feed(html)
+        p.close()
+    except Exception:
+        return list(dict.fromkeys(p.links))
+    return list(dict.fromkeys(p.links))
+
+
+def extract_text_from_html(html: str, max_chars: int = 200_000) -> str:
+    p = _TextCollector()
+    try:
+        p.feed(html)
+        p.close()
+    except Exception:
+        text = " ".join(p.parts)
+        return text[:max_chars]
+    text = " ".join(p.parts)
+    text = " ".join(text.split())
+    if len(text) > max_chars:
+        return text[:max_chars]
+    return text

package/backend/app/services/link_expansion/rate_limit.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import asyncio
+import time
+from collections import defaultdict
+
+
+class PolitenessGate:
+    """Per-host spacing; each wait is max(politeness floor, domain interval gap)."""
+
+    def __init__(
+        self,
+        *,
+        politeness_delay_s: float,
+        per_domain_interval_s: float,
+    ) -> None:
+        self._politeness = politeness_delay_s
+        self._interval = per_domain_interval_s
+        self._locks: dict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
+        self._last_fetch_mono: dict[str, float] = {}
+
+    async def before_request(self, host: str) -> None:
+        key = host.lower()
+        lock = self._locks[key]
+        async with lock:
+            now = time.monotonic()
+            last = self._last_fetch_mono.get(key, 0.0)
+            gap = max(0.0, self._interval - (now - last))
+            wait = max(self._politeness, gap)
+            if wait > 0:
+                await asyncio.sleep(wait)
+            self._last_fetch_mono[key] = time.monotonic()

package/backend/app/services/link_expansion/robots.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import logging
+from urllib.parse import urlparse
+from urllib.robotparser import RobotFileParser
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+async def fetch_robots_parser(
+    client: httpx.AsyncClient,
+    page_url: str,
+    cache: dict[str, RobotFileParser | None],
+    *,
+    timeout_s: float,
+) -> RobotFileParser | None:
+    parsed = urlparse(page_url)
+    if not parsed.scheme or not parsed.netloc:
+        return None
+    host = parsed.netloc.lower()
+    if host in cache:
+        return cache[host]
+    robots_url = f"{parsed.scheme}://{host}/robots.txt"
+    try:
+        r = await client.get(robots_url, timeout=timeout_s)
+        if r.status_code == 200 and (text := r.text):
+            rp = RobotFileParser()
+            rp.parse(text.splitlines())
+            cache[host] = rp
+            return rp
+    except Exception:
+        logger.debug("robots.txt fetch failed for %s", host, exc_info=True)
+    cache[host] = None
+    return None
+
+
+def robots_can_fetch(
+    rp: RobotFileParser | None,
+    user_agent: str,
+    url: str,
+) -> bool:
+    if rp is None:
+        return True
+    return rp.can_fetch(user_agent, url)

package/backend/app/services/link_expansion/schemas.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.config import Settings
+from app.services.link_expansion.domain_policy import parse_host_csv
+
+
+@dataclass(frozen=True)
+class ExpandLinksOptions:
+    max_depth: int
+    allowlist: frozenset[str]
+    denylist: frozenset[str]
+    respect_robots: bool
+
+
+class ExpandLinksRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    document_id: str = Field(..., min_length=1)
+    max_depth: int | None = Field(default=None, ge=1, le=32)
+    domain_allowlist: list[str] | None = None
+    domain_denylist: list[str] | None = None
+    respect_robots: bool | None = None
+
+    def resolve(self, settings: Settings) -> ExpandLinksOptions:
+        allow = (
+            frozenset(h.strip().lower().split(":")[0] for h in self.domain_allowlist)
+            if self.domain_allowlist is not None
+            else parse_host_csv(settings.link_expand_domain_allowlist)
+        )
+        deny = (
+            frozenset(h.strip().lower().split(":")[0] for h in self.domain_denylist)
+            if self.domain_denylist is not None
+            else parse_host_csv(settings.link_expand_domain_denylist)
+        )
+        depth = (
+            self.max_depth
+            if self.max_depth is not None
+            else settings.link_expand_max_depth
+        )
+        robots = (
+            self.respect_robots
+            if self.respect_robots is not None
+            else settings.link_expand_respect_robots
+        )
+        return ExpandLinksOptions(
+            max_depth=depth,
+            allowlist=allow,
+            denylist=deny,
+            respect_robots=robots,
+        )
+
+
+class ExpandLinksJobResult(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    root_document_id: str
+    documents_created: int = 0
+    relationships_created: int = 0
+    urls_skipped_policy: int = 0
+    urls_skipped_robots: int = 0
+    fetch_errors: int = 0
+    errors: list[dict[str, Any]] = Field(default_factory=list)

package/backend/app/services/link_expansion/worker.py

@@ -0,0 +1,458 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+from collections import deque
+from datetime import UTC, datetime
+from typing import Any
+from urllib.robotparser import RobotFileParser
+from uuid import uuid4
+
+import httpx
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from app.config import Settings
+from app.services.ingestion.persist import get_or_create_source_id
+from app.services.link_expansion.canonical_url import (
+    canonicalize_url,
+    host_from_url,
+    is_http_url,
+)
+from app.services.link_expansion.domain_policy import host_allowed
+from app.services.link_expansion.html_extract import (
+    extract_links_from_html,
+    extract_text_from_html,
+)
+from app.services.link_expansion.rate_limit import PolitenessGate
+from app.services.link_expansion.robots import fetch_robots_parser, robots_can_fetch
+from app.services.link_expansion.schemas import ExpandLinksJobResult, ExpandLinksOptions
+
+logger = logging.getLogger(__name__)
+
+_MAX_SUMMARY = 500
+_TEXT_BLOCK_CAP = 200_000
+
+
+async def _document_exists(session: AsyncSession, doc_id: str) -> bool:
+    r = await session.execute(
+        text("SELECT 1 FROM documents WHERE id = :id LIMIT 1"),
+        {"id": doc_id},
+    )
+    return r.first() is not None
+
+
+async def _load_outbound_links(session: AsyncSession, doc_id: str) -> list[str]:
+    r = await session.execute(
+        text(
+            "SELECT url FROM document_links WHERE document_id = :d ORDER BY ordinal",
+        ),
+        {"d": doc_id},
+    )
+    return [row[0] for row in r.fetchall()]
+
+
+async def _find_deduped_document(
+    session: AsyncSession,
+    *,
+    canonical_url: str,
+    content_sha256: str,
+) -> str | None:
+    r = await session.execute(
+        text(
+            "SELECT id FROM documents WHERE canonical_url = :u "
+            "AND content_sha256 = :h LIMIT 1",
+        ),
+        {"u": canonical_url, "h": content_sha256},
+    )
+    row = r.first()
+    return str(row[0]) if row else None
+
+
+async def _insert_relationship(
+    session: AsyncSession,
+    *,
+    parent_id: str,
+    child_id: str,
+    meta: dict[str, Any],
+) -> bool:
+    res = await session.execute(
+        text(
+            "INSERT OR IGNORE INTO relationships "
+            "(parent_document_id, child_document_id, relation_type, weight, meta) "
+            "VALUES (:p, :c, 'link', NULL, :m)",
+        ),
+        {"p": parent_id, "c": child_id, "m": json.dumps(meta)},
+    )
+    return (res.rowcount or 0) > 0
+
+
+async def _insert_partial_error_document(
+    session: AsyncSession,
+    *,
+    source_id: int,
+    canonical_url: str,
+    requested_url: str,
+    error: str,
+    ingest_meta: dict[str, Any],
+) -> str:
+    doc_id = str(uuid4())
+    ts = datetime.now(UTC).isoformat()
+    raw = json.dumps(
+        {
+            "kind": "link_fetch_error",
+            "requested_url": requested_url,
+            "canonical_url": canonical_url,
+            "error": error,
+        },
+    )
+    meta = {**ingest_meta, "fetch_failed": True, "error": error}
+    summary = f"Fetch failed: {error[:200]}"
+    await session.execute(
+        text(
+            "INSERT INTO documents "
+            "(id, source_id, timestamp, content_type, raw_content, summary, "
+            "status, canonical_url, content_sha256, ingest_meta) "
+            "VALUES (:id, :sid, :ts, 'text', :raw, :sum, 'partial', :cu, NULL, :im)",
+        ),
+        {
+            "id": doc_id,
+            "sid": source_id,
+            "ts": ts,
+            "raw": raw,
+            "sum": summary,
+            "cu": canonical_url,
+            "im": json.dumps(meta),
+        },
+    )
+    err_block = json.dumps({"text": summary, "link_expand_error": True})
+    await session.execute(
+        text(
+            "INSERT INTO content_blocks "
+            "(document_id, ordinal, type, storage_uri, inline_ref, mime, sha256, meta) "
+            "VALUES (:did, 0, 'text', NULL, 'link_expand:error', "
+            "'text/plain', NULL, :meta)",
+        ),
+        {"did": doc_id, "meta": err_block},
+    )
+    return doc_id
+
+
+async def _insert_fetched_document(
+    session: AsyncSession,
+    *,
+    source_id: int,
+    canonical_url: str,
+    content_sha256: str,
+    final_url: str,
+    requested_url: str,
+    body: bytes,
+    content_type_header: str | None,
+    outbound_urls: list[str],
+) -> tuple[str, bool]:
+    existing = await _find_deduped_document(
+        session,
+        canonical_url=canonical_url,
+        content_sha256=content_sha256,
+    )
+    if existing:
+        return existing, False
+
+    doc_id = str(uuid4())
+    ts = datetime.now(UTC).isoformat()
+    html = body.decode("utf-8", errors="replace")
+    plain = extract_text_from_html(html, max_chars=_TEXT_BLOCK_CAP)
+    summary = plain[:_MAX_SUMMARY] + ("…" if len(plain) > _MAX_SUMMARY else "")
+    raw = json.dumps(
+        {
+            "kind": "link_fetch",
+            "requested_url": requested_url,
+            "final_url": final_url,
+            "content_type": content_type_header,
+            "bytes": len(body),
+        },
+    )
+    await session.execute(
+        text(
+            "INSERT INTO documents "
+            "(id, source_id, timestamp, content_type, raw_content, summary, status, "
+            "canonical_url, content_sha256, ingest_meta) "
+            "VALUES (:id, :sid, :ts, 'text', :raw, :sum, 'partial', :cu, :h, NULL)",
+        ),
+        {
+            "id": doc_id,
+            "sid": source_id,
+            "ts": ts,
+            "raw": raw,
+            "sum": summary,
+            "cu": canonical_url,
+            "h": content_sha256,
+        },
+    )
+    text_meta = json.dumps({"text": plain[:_TEXT_BLOCK_CAP]})
+    await session.execute(
+        text(
+            "INSERT INTO content_blocks "
+            "(document_id, ordinal, type, storage_uri, inline_ref, mime, sha256, meta) "
+            "VALUES (:did, 0, 'text', NULL, 'link_expand:html', "
+            "'text/plain', NULL, :meta)",
+        ),
+        {"did": doc_id, "meta": text_meta},
+    )
+    for ordinal, u in enumerate(outbound_urls):
+        await session.execute(
+            text(
+                "INSERT INTO document_links (document_id, url, ordinal) "
+                "VALUES (:d, :u, :o)",
+            ),
+            {"d": doc_id, "u": u, "o": ordinal},
+        )
+    return doc_id, True
+
+
+async def expand_links_from_document(
+    session: AsyncSession,
+    *,
+    root_document_id: str,
+    options: ExpandLinksOptions,
+    settings: Settings,
+) -> ExpandLinksJobResult:
+    result = ExpandLinksJobResult(root_document_id=root_document_id)
+    if not await _document_exists(session, root_document_id):
+        result.errors.append(
+            {"code": "not_found", "detail": "document id does not exist"},
+        )
+        return result
+
+    source_id = await get_or_create_source_id(
+        session,
+        name="link_expansion",
+        connector_type="link_expansion",
+    )
+
+    root_links = await _load_outbound_links(session, root_document_id)
+    q: deque[tuple[str, str, int]] = deque()
+    for u in root_links:
+        if is_http_url(u):
+            q.append((root_document_id, u, 1))
+
+    fetch_cache: dict[str, tuple[str, str, str]] = {}
+    expanded_children: set[str] = set()
+
+    gate = PolitenessGate(
+        politeness_delay_s=settings.link_expand_politeness_delay_ms / 1000.0,
+        per_domain_interval_s=settings.link_expand_per_domain_interval_ms / 1000.0,
+    )
+    robots_cache: dict[str, RobotFileParser | None] = {}
+    timeout = settings.link_expand_fetch_timeout_s
+    max_bytes = settings.link_expand_max_response_bytes
+    ua = settings.link_expand_user_agent
+
+    async with httpx.AsyncClient(
+        headers={"User-Agent": ua},
+        follow_redirects=True,
+    ) as client:
+        while q:
+            parent_id, url, link_depth = q.popleft()
+            if link_depth > options.max_depth:
+                continue
+            if not is_http_url(url):
+                result.urls_skipped_policy += 1
+                continue
+
+            req_canon = canonicalize_url(url)
+            host = host_from_url(url)
+            if not host:
+                result.urls_skipped_policy += 1
+                continue
+            if not host_allowed(
+                host,
+                allowlist=options.allowlist,
+                denylist=options.denylist,
+            ):
+                result.urls_skipped_policy += 1
+                continue
+
+            if req_canon in fetch_cache:
+                final_c, sha_hex, child_id = fetch_cache[req_canon]
+                meta = {
+                    "requested_url": url,
+                    "final_url": final_c,
+                    "depth": link_depth,
+                    "cache_hit": True,
+                }
+                if await _insert_relationship(
+                    session,
+                    parent_id=parent_id,
+                    child_id=child_id,
+                    meta=meta,
+                ):
+                    result.relationships_created += 1
+                if link_depth < options.max_depth and child_id not in expanded_children:
+                    expanded_children.add(child_id)
+                    for nxt in await _load_outbound_links(session, child_id):
+                        if is_http_url(nxt):
+                            q.append((child_id, nxt, link_depth + 1))
+                continue
+
+            await gate.before_request(host)
+
+            if options.respect_robots:
+                rp = await fetch_robots_parser(
+                    client,
+                    url,
+                    robots_cache,
+                    timeout_s=min(timeout, 15.0),
+                )
+                if not robots_can_fetch(rp, ua, url):
+                    result.urls_skipped_robots += 1
+                    continue
+
+            try:
+                head = None
+                try:
+                    head = await client.head(url, timeout=timeout)
+                except httpx.HTTPError:
+                    pass
+                if head is not None and head.is_success:
+                    cl = head.headers.get("content-length")
+                    if cl is not None:
+                        try:
+                            if int(cl) > max_bytes:
+                                raise OSError(f"content-length {cl} exceeds cap")
+                        except ValueError:
+                            pass
+                response = await client.get(url, timeout=timeout)
+            except Exception as e:
+                logger.info("link fetch failed %s: %s", url, e)
+                result.fetch_errors += 1
+                result.errors.append({"url": url, "error": str(e)})
+                canon_for_err = canonicalize_url(str(url))
+                child_id = await _insert_partial_error_document(
+                    session,
+                    source_id=source_id,
+                    canonical_url=canon_for_err,
+                    requested_url=url,
+                    error=str(e),
+                    ingest_meta={"requested_url": url},
+                )
+                result.documents_created += 1
+                fetch_cache[req_canon] = (canon_for_err, "", child_id)
+                meta = {
+                    "requested_url": url,
+                    "final_url": canon_for_err,
+                    "depth": link_depth,
+                    "fetch_error": True,
+                }
+                if await _insert_relationship(
+                    session,
+                    parent_id=parent_id,
+                    child_id=child_id,
+                    meta=meta,
+                ):
+                    result.relationships_created += 1
+                continue
+
+            body = response.content
+            if len(body) > max_bytes:
+                err = f"response body {len(body)} exceeds max_bytes={max_bytes}"
+                result.fetch_errors += 1
+                result.errors.append({"url": url, "error": err})
+                final_u = canonicalize_url(str(response.url))
+                child_id = await _insert_partial_error_document(
+                    session,
+                    source_id=source_id,
+                    canonical_url=final_u,
+                    requested_url=url,
+                    error=err,
+                    ingest_meta={"requested_url": url},
+                )
+                result.documents_created += 1
+                fetch_cache[req_canon] = (final_u, "", child_id)
+                meta = {
+                    "requested_url": url,
+                    "final_url": final_u,
+                    "depth": link_depth,
+                    "fetch_error": True,
+                }
+                if await _insert_relationship(
+                    session,
+                    parent_id=parent_id,
+                    child_id=child_id,
+                    meta=meta,
+                ):
+                    result.relationships_created += 1
+                continue
+
+            final_canon = canonicalize_url(str(response.url))
+            sha_hex = hashlib.sha256(body).hexdigest()
+
+            ct = (response.headers.get("content-type") or "").split(";")[0].strip()
+            outbound: list[str] = []
+            if "html" in ct.lower() or body.lstrip()[:1] in (b"<", b"\xef"):
+                outbound = extract_links_from_html(
+                    body.decode("utf-8", errors="replace"),
+                    str(response.url),
+                )
+
+            child_id, inserted_new = await _insert_fetched_document(
+                session,
+                source_id=source_id,
+                canonical_url=final_canon,
+                content_sha256=sha_hex,
+                final_url=str(response.url),
+                requested_url=url,
+                body=body,
+                content_type_header=ct or None,
+                outbound_urls=outbound,
+            )
+            if inserted_new:
+                result.documents_created += 1
+
+            fetch_cache[req_canon] = (final_canon, sha_hex, child_id)
+
+            meta = {
+                "requested_url": url,
+                "final_url": str(response.url),
+                "depth": link_depth,
+                "content_sha256": sha_hex,
+            }
+            if await _insert_relationship(
+                session,
+                parent_id=parent_id,
+                child_id=child_id,
+                meta=meta,
+            ):
+                result.relationships_created += 1
+
+            if link_depth < options.max_depth and child_id not in expanded_children:
+                expanded_children.add(child_id)
+                to_follow = await _load_outbound_links(session, child_id)
+                for nxt in to_follow:
+                    if is_http_url(nxt):
+                        q.append((child_id, nxt, link_depth + 1))
+
+    return result
+
+
+async def run_expand_links_job(
+    *,
+    root_document_id: str,
+    options: ExpandLinksOptions,
+    session_factory: async_sessionmaker[AsyncSession],
+    settings: Settings,
+) -> None:
+    """Background entrypoint: own session + commit."""
+    async with session_factory() as session:
+        try:
+            await expand_links_from_document(
+                session,
+                root_document_id=root_document_id,
+                options=options,
+                settings=settings,
+            )
+            await session.commit()
+        except Exception:
+            logger.exception("link expansion job failed for %s", root_document_id)
+            await session.rollback()

package/backend/app/services/normalization/__init__.py

@@ -0,0 +1,7 @@
+from app.services.normalization.normalizer import normalize_envelope_to_canonical
+from app.services.normalization.persist_normalized import persist_normalized_document
+
+__all__ = [
+    "normalize_envelope_to_canonical",
+    "persist_normalized_document",
+]