ltcai 9.0.0 → 9.1.0

package/latticeai/api/auth.py

@@ -5,7 +5,8 @@ import hashlib
 import logging
 import secrets
 import time
-from typing import Any, Awaitable, Callable, Dict, Optional, Tuple
+from dataclasses import dataclass
+from typing import Any, Awaitable, Callable, Dict, Optional
 from urllib.parse import urlencode
 
 from fastapi import APIRouter, HTTPException, Request
@@ -42,9 +43,20 @@ class UpdateProfileRequest(BaseModel):
     nickname: Optional[str] = None
 
 
-# state → (issued_at, nonce). The nonce binds the eventual ID token to *this*
-# login attempt (replay / token-injection defence); the timestamp expires it.
-_sso_states: Dict[str, Tuple[float, str]] = {}
+@dataclass(frozen=True)
+class _SSOLoginState:
+    """Server-side SSO transaction state consumed exactly once."""
+
+    issued_at: float
+    nonce: str
+    code_verifier: str
+    invite_authorized: bool
+
+
+# The nonce and PKCE verifier bind the callback/token to this login attempt.
+# The signed invite cookie is verified once at login and reduced to a
+# server-side claim, so the callback never trusts a client-supplied invite bit.
+_sso_states: Dict[str, _SSOLoginState] = {}
 
 
 def create_auth_router(
@@ -67,7 +79,10 @@ def create_auth_router(
     open_registration: bool,
     session_ttl: int,
     require_auth: bool = True,
+    secure_cookies: bool = False,
     ensure_identity: Optional[Callable[[str, Dict], None]] = None,
+    invite_gate_enabled: bool = False,
+    invite_authorized: Optional[Callable[[Request], bool]] = None,
     verify_id_token: Callable[..., Dict] = _default_verify_id_token,
     fetch_jwks: Callable[[str], Awaitable[Dict]] = _default_fetch_jwks,
 ) -> APIRouter:
@@ -86,7 +101,19 @@ def create_auth_router(
     @router.post("/register")
     async def register(req: UserRegister, request: Request):
         check_ip_rate_limit(client_ip(request), "register", max_calls=5, window_secs=3600)
-        if not open_registration:
+        invite_claim = bool(
+            invite_gate_enabled
+            and invite_authorized is not None
+            and invite_authorized(request)
+        )
+        if invite_gate_enabled and not invite_claim:
+            raise HTTPException(
+                status_code=403,
+                detail="유효한 서명 초대 권한이 필요합니다.",
+            )
+        # Closed public registration stays closed unless the operator enabled
+        # the invite gate and this exact request carries a valid signed claim.
+        if not open_registration and not invite_claim:
             raise HTTPException(status_code=403, detail="회원가입이 비활성화되어 있습니다. 관리자에게 문의하세요.")
         _enforce_password_policy(req.password)
         email = normalize_email(req.email)
@@ -127,7 +154,14 @@ def create_auth_router(
             "role": role,
             "is_admin": role == "admin",
         })
-        response.set_cookie(key="session_token", value=token, httponly=True, samesite="lax", max_age=session_ttl)
+        response.set_cookie(
+            key="session_token",
+            value=token,
+            httponly=True,
+            secure=secure_cookies,
+            samesite="lax",
+            max_age=session_ttl,
+        )
         return response
 
     @router.get("/auth/sso/config")
@@ -135,7 +169,7 @@ def create_auth_router(
         return public_sso_config()
 
     @router.get("/auth/sso/login")
-    async def sso_login():
+    async def sso_login(request: Request):
         settings = get_sso_settings()
         discovery = await get_sso_discovery()
         if not settings.get("enabled") or not discovery:
@@ -150,7 +184,19 @@ def create_auth_router(
             .rstrip(b"=")
             .decode("ascii")
         )
-        _sso_states[state] = (time.time(), nonce, code_verifier)
+        invite_claim = bool(
+            not invite_gate_enabled
+            or (
+                invite_authorized is not None
+                and invite_authorized(request)
+            )
+        )
+        _sso_states[state] = _SSOLoginState(
+            issued_at=time.time(),
+            nonce=nonce,
+            code_verifier=code_verifier,
+            invite_authorized=invite_claim,
+        )
         params = urlencode({
             "client_id": settings["client_id"],
             "response_type": "code",
@@ -165,12 +211,11 @@ def create_auth_router(
 
     @router.get("/auth/sso/callback")
     async def sso_callback(code: str = "", state: str = "", error: str = ""):
-        if error:
-            return RedirectResponse(f"/?sso_error={error}")
         entry = _sso_states.pop(state, None)
-        if entry is None or time.time() - entry[0] > 300:
+        if entry is None or time.time() - entry.issued_at > 300:
             raise HTTPException(status_code=400, detail="유효하지 않은 SSO 상태입니다.")
-        _, nonce, code_verifier = entry
+        if error:
+            return RedirectResponse(f"/?{urlencode({'sso_error': error})}")
         settings = get_sso_settings()
         discovery = await get_sso_discovery()
         if not settings.get("enabled") or not discovery:
@@ -183,7 +228,7 @@ def create_auth_router(
                 "redirect_uri": settings["redirect_uri"],
                 "client_id": settings["client_id"],
                 "client_secret": settings["client_secret"],
-                "code_verifier": code_verifier,
+                "code_verifier": entry.code_verifier,
             }, headers={"Accept": "application/json"}, timeout=15)
             tokens = r.json()
         id_token = tokens.get("id_token")
@@ -200,7 +245,7 @@ def create_auth_router(
                 jwks=jwks,
                 issuer=issuer,
                 audience=settings["client_id"],
-                nonce=nonce,
+                nonce=entry.nonce,
             )
         except OIDCValidationError as exc:
             logging.warning("SSO ID token rejected: %s", exc)
@@ -213,6 +258,11 @@ def create_auth_router(
             raise HTTPException(status_code=400, detail="이메일을 확인할 수 없습니다.")
         users = load_users()
         if email not in users:
+            if invite_gate_enabled and not entry.invite_authorized:
+                raise HTTPException(
+                    status_code=403,
+                    detail="신규 SSO 계정에는 유효한 서명 초대 권한이 필요합니다.",
+                )
             is_first = len(users) == 0
             users[email] = {
                 "password": "",
@@ -229,7 +279,14 @@ def create_auth_router(
             raise HTTPException(status_code=403, detail="비활성화된 계정입니다.")
         token = create_session(email)
         resp = RedirectResponse("/app", status_code=302)
-        resp.set_cookie("session_token", token, httponly=True, samesite="lax", max_age=session_ttl)
+        resp.set_cookie(
+            "session_token",
+            token,
+            httponly=True,
+            secure=secure_cookies,
+            samesite="lax",
+            max_age=session_ttl,
+        )
         return resp
 
     @router.post("/logout")
@@ -238,7 +295,12 @@ def create_auth_router(
         if token:
             invalidate_session(token)
         response = JSONResponse(content={"status": "ok"})
-        response.delete_cookie("session_token")
+        response.delete_cookie(
+            "session_token",
+            secure=secure_cookies,
+            httponly=True,
+            samesite="lax",
+        )
         return response
 
     @router.post("/account/change-password")

package/latticeai/api/browser.py

@@ -17,18 +17,31 @@ Two layers, both feeding ``IngestionPipeline.ingest``:
 
 from __future__ import annotations
 
+import ipaddress
+import socket
 from html.parser import HTMLParser
 from typing import Any, Callable, Optional, Tuple
-from urllib.parse import urlparse
+from urllib.parse import SplitResult, urljoin, urlsplit, urlunsplit
 
 from fastapi import APIRouter, HTTPException, Request
 from pydantic import BaseModel
 
+from latticeai import __version__
 from lattice_brain.ingestion import IngestionItem
 
 MAX_TAB_BYTES = 4 * 1024 * 1024          # 4 MB per captured tab payload
 MAX_URL_FETCH_BYTES = 4 * 1024 * 1024    # 4 MB cap on a fetched page
 URL_FETCH_TIMEOUT = 12.0                  # seconds
+MAX_URL_LENGTH = 8192
+MAX_URL_REDIRECTS = 5
+
+_REDIRECT_STATUSES = frozenset({301, 302, 303, 307, 308})
+_TEXTUAL_APPLICATION_TYPES = frozenset({
+    "application/json",
+    "application/ld+json",
+    "application/xhtml+xml",
+    "application/xml",
+})
 
 
 class BrowserFetchError(Exception):
@@ -86,7 +99,157 @@ def extract_readable_text(html: str) -> Tuple[str, str]:
     return parser.title.strip(), parser.text()
 
 
-def _default_fetch_url(url: str) -> Tuple[str, str]:
+def _parse_http_url(url: str) -> tuple[str, SplitResult, str, int]:
+    """Parse a URL into safe, unambiguous HTTP connection components."""
+    cleaned = (url or "").strip()
+    if not cleaned:
+        raise ValueError("url is required.")
+    if len(cleaned) > MAX_URL_LENGTH:
+        raise ValueError("URL is too long.")
+    if "\\" in cleaned or any(ord(char) < 32 or ord(char) == 127 for char in cleaned):
+        raise ValueError("Malformed URL.")
+
+    try:
+        parsed = urlsplit(cleaned)
+        hostname = parsed.hostname
+        port = parsed.port
+    except ValueError as exc:
+        raise ValueError("Malformed URL.") from exc
+    if parsed.scheme.lower() not in ("http", "https"):
+        raise ValueError("Only http(s) URLs are supported.")
+    if not parsed.netloc or not hostname:
+        raise ValueError("Malformed URL.")
+    if parsed.username is not None or parsed.password is not None:
+        raise ValueError("URLs containing credentials are not supported.")
+    if "%" in hostname:
+        # Scoped IPv6 literals are interface-local by definition and also have
+        # inconsistent URL parser semantics across HTTP clients.
+        raise ValueError("Scoped IP addresses are not supported.")
+
+    hostname = hostname.rstrip(".")
+    if not hostname:
+        raise ValueError("Malformed URL.")
+    try:
+        ascii_hostname = hostname.encode("idna").decode("ascii").lower()
+    except UnicodeError as exc:
+        raise ValueError("Malformed URL hostname.") from exc
+    if len(ascii_hostname) > 253:
+        raise ValueError("Malformed URL hostname.")
+
+    port = port if port is not None else (443 if parsed.scheme.lower() == "https" else 80)
+    if port < 1 or port > 65535:
+        raise ValueError("Malformed URL port.")
+    return cleaned, parsed, ascii_hostname, port
+
+
+def _is_public_ip(value: str) -> bool:
+    """Return whether *value* is globally routable (IPv4 or IPv6)."""
+    try:
+        address = ipaddress.ip_address(value)
+    except ValueError:
+        return False
+    # Some Python releases report multicast addresses as ``is_global``. Keep
+    # the security classes explicit instead of relying on that single flag.
+    return address.is_global and not any((
+        address.is_loopback,
+        address.is_private,
+        address.is_link_local,
+        address.is_multicast,
+        address.is_unspecified,
+        address.is_reserved,
+        getattr(address, "is_site_local", False),
+    ))
+
+
+def _resolve_public_target(
+    url: str,
+    *,
+    resolver: Optional[Callable[..., Any]] = None,
+) -> tuple[str, SplitResult, str, int, tuple[str, ...]]:
+    """Resolve a URL and reject every non-public address before connecting."""
+    try:
+        cleaned, parsed, hostname, port = _parse_http_url(url)
+    except ValueError as exc:
+        raise BrowserFetchError(str(exc)) from exc
+
+    if hostname == "localhost" or hostname.endswith(".localhost"):
+        raise BrowserFetchError("Local and private network URLs are not allowed.")
+
+    resolve = resolver or socket.getaddrinfo
+    try:
+        records = resolve(
+            hostname,
+            port,
+            family=socket.AF_UNSPEC,
+            type=socket.SOCK_STREAM,
+        )
+    except (OSError, socket.gaierror) as exc:
+        raise BrowserFetchError(f"Could not resolve the page host: {hostname}.") from exc
+
+    addresses: list[str] = []
+    for record in records or ():
+        try:
+            address = str(record[4][0])
+        except (IndexError, TypeError):
+            raise BrowserFetchError("The page host returned an invalid DNS record.")
+        if not _is_public_ip(address):
+            raise BrowserFetchError("Local and private network URLs are not allowed.")
+        if address not in addresses:
+            addresses.append(address)
+    if not addresses:
+        raise BrowserFetchError(f"Could not resolve the page host: {hostname}.")
+    return cleaned, parsed, hostname, port, tuple(addresses)
+
+
+def _origin_host_header(hostname: str, port: int, scheme: str) -> str:
+    try:
+        is_ipv6 = ipaddress.ip_address(hostname).version == 6
+    except ValueError:
+        is_ipv6 = False
+    host = f"[{hostname}]" if is_ipv6 else hostname
+    default_port = 443 if scheme == "https" else 80
+    return host if port == default_port else f"{host}:{port}"
+
+
+def _pinned_request_url(parsed: SplitResult, address: str, port: int) -> str:
+    """Build a URL that connects to a pre-validated DNS result.
+
+    The request retains the original Host header and TLS SNI separately. This
+    closes the DNS-rebinding window between validation and socket connection.
+    """
+    ip = ipaddress.ip_address(address)
+    host = f"[{ip.compressed}]" if ip.version == 6 else ip.compressed
+    default_port = 443 if parsed.scheme.lower() == "https" else 80
+    netloc = host if port == default_port else f"{host}:{port}"
+    return urlunsplit((parsed.scheme.lower(), netloc, parsed.path or "/", parsed.query, ""))
+
+
+def _read_limited_response(response: Any, max_bytes: int) -> bytes:
+    content_length = response.headers.get("content-length")
+    if content_length:
+        try:
+            declared_size = int(content_length)
+        except ValueError:
+            declared_size = -1
+        if declared_size > max_bytes:
+            raise BrowserFetchError("The page is too large to ingest.")
+
+    body = bytearray()
+    for chunk in response.iter_bytes():
+        if len(body) + len(chunk) > max_bytes:
+            raise BrowserFetchError("The page is too large to ingest.")
+        body.extend(chunk)
+    return bytes(body)
+
+
+def _default_fetch_url(
+    url: str,
+    *,
+    resolver: Optional[Callable[..., Any]] = None,
+    transport: Any = None,
+    max_bytes: int = MAX_URL_FETCH_BYTES,
+    max_redirects: int = MAX_URL_REDIRECTS,
+) -> Tuple[str, str]:
     """Fetch a public URL on the local runtime and extract readable text.
 
     Raises :class:`BrowserFetchError` on any non-success (blocked, login wall,
@@ -94,39 +257,113 @@ def _default_fetch_url(url: str) -> Tuple[str, str]:
     """
     import httpx
 
+    if max_bytes < 1 or max_redirects < 0:
+        raise BrowserFetchError("Invalid URL fetch limits.")
+
+    client_options: dict[str, Any] = {
+        "follow_redirects": False,
+        "timeout": URL_FETCH_TIMEOUT,
+        "trust_env": False,
+        "limits": httpx.Limits(max_keepalive_connections=0),
+    }
+    if transport is not None:
+        client_options["transport"] = transport
+
+    current_url = url
     try:
-        with httpx.Client(
-            follow_redirects=True, timeout=URL_FETCH_TIMEOUT,
-            headers={"User-Agent": "LatticeAI-local/3.6 (+local-first knowledge graph)"},
-        ) as client:
-            resp = client.get(url)
+        with httpx.Client(**client_options) as client:
+            for redirect_count in range(max_redirects + 1):
+                current_url, parsed, hostname, port, addresses = _resolve_public_target(
+                    current_url,
+                    resolver=resolver,
+                )
+                next_url: Optional[str] = None
+                connect_errors: list[Exception] = []
+
+                for address in addresses:
+                    request = client.build_request(
+                        "GET",
+                        _pinned_request_url(parsed, address, port),
+                        headers={
+                            "Accept": "text/html, text/plain;q=0.9, application/xhtml+xml;q=0.8",
+                            "Accept-Encoding": "identity",
+                            "Connection": "close",
+                            "Host": _origin_host_header(hostname, port, parsed.scheme.lower()),
+                            "User-Agent": f"LatticeAI-local/{__version__} (+local-first knowledge graph)",
+                        },
+                        extensions={"sni_hostname": hostname},
+                    )
+                    response = None
+                    try:
+                        response = client.send(request, stream=True)
+                    except (httpx.ConnectError, httpx.ConnectTimeout) as exc:
+                        connect_errors.append(exc)
+                        continue
+
+                    try:
+                        if response.status_code in _REDIRECT_STATUSES:
+                            location = (response.headers.get("location") or "").strip()
+                            if not location:
+                                raise BrowserFetchError("The page returned a redirect without a location.")
+                            if redirect_count >= max_redirects:
+                                raise BrowserFetchError("The page redirected too many times.")
+                            # Join against the original public URL, never the pinned
+                            # IP URL exposed only to the transport.
+                            next_url = urljoin(current_url, location)
+                            break
+                        if response.status_code in (401, 403):
+                            raise BrowserFetchError(
+                                "The page is login-required or blocked "
+                                f"(HTTP {response.status_code})."
+                            )
+                        if response.status_code >= 400:
+                            raise BrowserFetchError(
+                                f"The page returned HTTP {response.status_code}."
+                            )
+
+                        content_type = response.headers.get("content-type", "")
+                        media_type = content_type.split(";", 1)[0].strip().lower()
+                        if not (
+                            media_type.startswith("text/")
+                            or media_type in _TEXTUAL_APPLICATION_TYPES
+                        ):
+                            raise BrowserFetchError(
+                                f"Unsupported content type: {content_type or 'unknown'}."
+                            )
+
+                        raw_body = _read_limited_response(response, max_bytes)
+                        try:
+                            encoding = response.encoding or "utf-8"
+                            body = raw_body.decode(encoding, "replace")
+                        except (LookupError, UnicodeError):
+                            body = raw_body.decode("utf-8", "replace")
+                        title, text = extract_readable_text(body)
+                        return (title or current_url, text)
+                    finally:
+                        response.close()
+
+                if next_url is not None:
+                    current_url = next_url
+                    continue
+                if connect_errors:
+                    raise BrowserFetchError(
+                        f"Could not reach the page: {connect_errors[-1]}"
+                    ) from connect_errors[-1]
+                raise BrowserFetchError("Could not reach the page.")
+    except BrowserFetchError:
+        raise
     except httpx.HTTPError as exc:
         raise BrowserFetchError(f"Could not reach the page: {exc}") from exc
 
-    if resp.status_code in (401, 403):
-        raise BrowserFetchError("The page is login-required or blocked (HTTP %s)." % resp.status_code)
-    if resp.status_code >= 400:
-        raise BrowserFetchError(f"The page returned HTTP {resp.status_code}.")
-    content_type = resp.headers.get("content-type", "")
-    if content_type and "html" not in content_type and "text" not in content_type:
-        raise BrowserFetchError(f"Unsupported content type: {content_type or 'unknown'}.")
-    body = resp.text or ""
-    if len(body.encode("utf-8", "ignore")) > MAX_URL_FETCH_BYTES:
-        body = body.encode("utf-8", "ignore")[:MAX_URL_FETCH_BYTES].decode("utf-8", "ignore")
-    title, text = extract_readable_text(body)
-    return (title or url, text)
+    raise BrowserFetchError("The page redirected too many times.")
 
 
 def _validate_http_url(url: str) -> str:
-    url = (url or "").strip()
-    if not url:
-        raise HTTPException(status_code=400, detail="url is required.")
-    parsed = urlparse(url)
-    if parsed.scheme not in ("http", "https"):
-        raise HTTPException(status_code=400, detail="Only http(s) URLs are supported.")
-    if not parsed.netloc:
-        raise HTTPException(status_code=400, detail="Malformed URL.")
-    return url
+    try:
+        cleaned, _parsed, _hostname, _port = _parse_http_url(url)
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    return cleaned
 
 
 # ── request models ───────────────────────────────────────────────────────────
@@ -149,6 +386,7 @@ def create_browser_router(
     *,
     pipeline: Any,
     require_user: Callable[[Request], str],
+    workspace_service: Any = None,
     fetch_url: Optional[Callable[[str], Tuple[str, str]]] = None,
     max_tab_bytes: int = MAX_TAB_BYTES,
 ) -> APIRouter:
@@ -159,11 +397,28 @@ def create_browser_router(
         if pipeline is None or not pipeline.available():
             raise HTTPException(status_code=503, detail="Knowledge Graph ingestion is disabled.")
 
+    def _write_workspace(request: Request, body_workspace: Optional[str], user: str) -> Optional[str]:
+        header_workspace = request.headers.get("X-Workspace-Id")
+        header_workspace = header_workspace.strip() if header_workspace and header_workspace.strip() else None
+        query_workspace = request.query_params.get("workspace_id")
+        query_workspace = query_workspace.strip() if query_workspace and query_workspace.strip() else None
+        supplied = [value for value in (body_workspace, header_workspace, query_workspace) if value]
+        if len(set(supplied)) > 1:
+            raise HTTPException(status_code=403, detail="Workspace selectors must match.")
+        requested = supplied[0] if supplied else None
+        if workspace_service is None:
+            return requested
+        try:
+            return workspace_service.resolve_write_scope(requested, user or None)
+        except PermissionError as exc:
+            raise HTTPException(status_code=403, detail=str(exc)) from exc
+
     @router.post("/api/browser/read-url")
     async def read_url(req: ReadUrlRequest, request: Request):
         """Fetch a public URL locally and ingest it as a web_url source."""
         user = require_user(request)
         _require_pipeline()
+        workspace_id = _write_workspace(request, req.workspace_id, user)
         url = _validate_http_url(req.url)
         try:
             title, text = _fetch(url)
@@ -176,7 +431,7 @@ def create_browser_router(
         res = pipeline.ingest(
             IngestionItem(
                 source_type="web_url", title=title, text=text, source_uri=url,
-                owner=user, workspace_id=req.workspace_id,
+                owner=user, workspace_id=workspace_id,
             ),
             user_email=user,
         )
@@ -187,11 +442,25 @@ def create_browser_router(
         """Ingest a payload captured from the local browser extension."""
         user = require_user(request)
         _require_pipeline()
+        workspace_id = _write_workspace(request, req.workspace_id, user)
         url = _validate_http_url(req.url)
-        # Sanitize: reject an oversized payload before doing any work.
-        for value in (req.text, req.html, req.selected_text):
-            if value and len(value.encode("utf-8", "ignore")) > max_tab_bytes:
-                raise HTTPException(status_code=413, detail="Captured payload is too large.")
+        # Bound the entire capture, not merely each field independently. The
+        # extension commonly supplies full text, selection, and HTML together.
+        captured_bytes = sum(
+            len(value.encode("utf-8", "ignore"))
+            for value in (
+                req.url,
+                req.title,
+                req.text,
+                req.selected_text,
+                req.html,
+                req.captured_at,
+                workspace_id,
+            )
+            if value
+        )
+        if captured_bytes > max_tab_bytes:
+            raise HTTPException(status_code=413, detail="Captured payload is too large.")
         text = (req.text or "").strip()
         if not text and req.html:
             _title, text = extract_readable_text(req.html)
@@ -207,7 +476,7 @@ def create_browser_router(
                 source_uri=url,
                 captured_at=req.captured_at,
                 owner=user,
-                workspace_id=req.workspace_id,
+                workspace_id=workspace_id,
                 metadata={"has_selection": bool(req.selected_text)},
             ),
             user_email=user,