business-stack 0.1.0

package/backend/app/connectors/thread_expansion/twitter.py

@@ -0,0 +1,49 @@
+"""
+Twitter / X thread expansion — **official APIs only**.
+
+Do not use scraping or undocumented HTML/GraphQL access; those may violate the
+platform Terms of Service. Production fetchers must call documented X API
+endpoints under your developer product and agreement.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from app.connectors.thread_expansion.models import ThreadExpansionResult
+
+
+class TwitterThreadExpansionFetcher(ABC):
+    """
+    Fetch a full conversation thread from X/Twitter using **official APIs**.
+
+    Compliance & credentials (operational checklist — not legal advice):
+
+    - **API access**: Requires developer account credentials (e.g. bearer token /
+      OAuth 2.0 user context, depending on endpoint). Keys must be kept server-side
+      and rotated per your security policy.
+    - **Policies**: Usage is subject to the X Developer Agreement, API rules, and
+      rate limits for your access tier. Automated collection must stay within
+      documented allowances.
+    - **Privacy**: Respect user privacy settings and data-retention obligations
+      applicable to your jurisdiction and use case.
+
+    Implementations MUST NOT rely on scraping the public web UI.
+    """
+
+    @abstractmethod
+    async def fetch_full_thread(
+        self,
+        conversation_id: str,
+        *,
+        config: dict[str, Any],
+    ) -> ThreadExpansionResult:
+        """
+        Return every accessible post in the thread identified by ``conversation_id``
+        (typically the root tweet id as returned by the API).
+
+        ``config`` holds implementation-specific settings (e.g. timeout, optional
+        endpoint flags). **Do not** log secrets from ``config``.
+        """
+        raise NotImplementedError

package/backend/app/db.py

@@ -0,0 +1,5 @@
+from sqlalchemy.orm import DeclarativeBase
+
+
+class Base(DeclarativeBase):
+    pass

package/backend/app/dependencies.py

@@ -0,0 +1,34 @@
+from collections.abc import AsyncGenerator
+from typing import Annotated
+
+from fastapi import Depends, Header, HTTPException, Request
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.config import Settings, get_settings
+from app.vectorstore import SqliteVecStore
+
+
+async def get_db(request: Request) -> AsyncGenerator[AsyncSession, None]:
+    factory = request.app.state.session_factory
+    async with factory() as session:
+        yield session
+
+
+def get_vector_store(request: Request) -> SqliteVecStore:
+    return request.app.state.vector_store
+
+
+SettingsDep = Annotated[Settings, Depends(get_settings)]
+DbSession = Annotated[AsyncSession, Depends(get_db)]
+VectorStoreDep = Annotated[SqliteVecStore, Depends(get_vector_store)]
+
+
+async def get_gateway_user_id(
+    x_user_id: Annotated[str | None, Header(alias="x-user-id")] = None,
+) -> str:
+    if not x_user_id or not str(x_user_id).strip():
+        raise HTTPException(status_code=401, detail="Missing x-user-id")
+    return str(x_user_id).strip()
+
+
+GatewayUserDep = Annotated[str, Depends(get_gateway_user_id)]

package/backend/app/logging_config.py

@@ -0,0 +1,35 @@
+import logging
+import sys
+
+from pythonjsonlogger.json import JsonFormatter
+
+from app.middleware.request_id import request_id_ctx
+
+
+class RequestIdFilter(logging.Filter):
+    def filter(self, record: logging.LogRecord) -> bool:
+        if not hasattr(record, "request_id"):
+            rid = request_id_ctx.get()
+            record.request_id = rid if rid is not None else "-"
+        return True
+
+
+def configure_logging(*, log_level: str, log_json: bool) -> None:
+    root = logging.getLogger()
+    root.handlers.clear()
+    root.setLevel(log_level.upper())
+
+    handler = logging.StreamHandler(sys.stdout)
+    handler.addFilter(RequestIdFilter())
+
+    if log_json:
+        formatter = JsonFormatter(
+            "%(asctime)s %(levelname)s %(name)s %(message)s %(request_id)s",
+        )
+    else:
+        fmt = (
+            "%(asctime)s %(levelname)s %(name)s %(message)s [request_id=%(request_id)s]"
+        )
+        formatter = logging.Formatter(fmt)
+    handler.setFormatter(formatter)
+    root.addHandler(handler)

package/backend/app/main.py

@@ -0,0 +1,97 @@
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+from sqlalchemy import event
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+
+from app.config import get_settings
+from app.connectors.registry import init_connectors
+from app.logging_config import configure_logging
+from app.middleware.gateway_identity import GatewayIdentityMiddleware
+from app.middleware.openapi_gateway import OpenApiGatewayMiddleware
+from app.middleware.request_id import RequestIdMiddleware
+from app.openapi_config import APP_DESCRIPTION, OPENAPI_TAGS, attach_custom_openapi
+from app.routers import (
+    admin_pipeline,
+    chat,
+    chunks,
+    entity_extract,
+    example,
+    gemini_embed,
+    health,
+    ingestion,
+    link_expansion,
+    pipeline_status,
+    query,
+    vectors,
+)
+from app.sqlite_ext import load_sqlite_vec_extension, unwrap_sqlite3_connection
+from app.vectorstore import SqliteVecStore
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    settings = get_settings()
+    configure_logging(log_level=settings.log_level, log_json=settings.log_json)
+    settings.data_dir.mkdir(parents=True, exist_ok=True)
+    (settings.data_dir / "blobs").mkdir(parents=True, exist_ok=True)
+    init_connectors()
+    engine = create_async_engine(
+        settings.sqlalchemy_database_url,
+        pool_pre_ping=True,
+    )
+
+    @event.listens_for(engine.sync_engine, "connect")
+    def _sqlite_on_connect(dbapi_connection, _connection_record) -> None:
+        raw = unwrap_sqlite3_connection(dbapi_connection)
+        cursor = raw.cursor()
+        cursor.execute("PRAGMA foreign_keys=ON")
+        cursor.execute("PRAGMA busy_timeout=10000")
+        cursor.close()
+        load_sqlite_vec_extension(dbapi_connection)
+
+    app.state.engine = engine
+    app.state.vector_store = SqliteVecStore(
+        engine,
+        dimension=settings.vector_embedding_dim,
+    )
+    app.state.session_factory = async_sessionmaker(
+        engine,
+        class_=AsyncSession,
+        expire_on_commit=False,
+    )
+    yield
+    await engine.dispose()
+
+
+def create_app() -> FastAPI:
+    app = FastAPI(
+        title="Business Stack API",
+        description=APP_DESCRIPTION,
+        version="0.1.0",
+        openapi_tags=OPENAPI_TAGS,
+        docs_url="/docs",
+        redoc_url="/redoc",
+        openapi_url="/openapi.json",
+        lifespan=lifespan,
+    )
+    attach_custom_openapi(app)
+    app.add_middleware(GatewayIdentityMiddleware)
+    app.add_middleware(RequestIdMiddleware)
+    app.add_middleware(OpenApiGatewayMiddleware)
+    app.include_router(health.router)
+    app.include_router(admin_pipeline.router)
+    app.include_router(ingestion.router)
+    app.include_router(pipeline_status.router)
+    app.include_router(link_expansion.router)
+    app.include_router(chunks.router)
+    app.include_router(gemini_embed.router)
+    app.include_router(entity_extract.router)
+    app.include_router(vectors.router)
+    app.include_router(query.router)
+    app.include_router(chat.router)
+    app.include_router(example.router)
+    return app
+
+
+app = create_app()

package/backend/app/middleware/gateway_identity.py

@@ -0,0 +1,17 @@
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import JSONResponse, Response
+
+_UNAUTH_PATHS = frozenset({"/healthz", "/readyz"})
+
+
+class GatewayIdentityMiddleware(BaseHTTPMiddleware):
+    async def dispatch(self, request: Request, call_next) -> Response:
+        if request.url.path in _UNAUTH_PATHS:
+            return await call_next(request)
+        if not request.headers.get("x-user-id"):
+            return JSONResponse(
+                status_code=401,
+                content={"detail": "Unauthorized"},
+            )
+        return await call_next(request)

package/backend/app/middleware/openapi_gateway.py

@@ -0,0 +1,71 @@
+"""ASGI middleware: OpenAPI UIs only via trusted gateway + correct URL prefix for Swagger/ReDoc."""
+
+from __future__ import annotations
+
+import json
+
+from starlette.types import ASGIApp, Receive, Scope, Send
+
+from app.config import get_settings
+
+
+def _is_openapi_public_path(path: str) -> bool:
+    if path == "/openapi.json" or path == "/redoc" or path == "/docs":
+        return True
+    return path.startswith("/docs/")
+
+
+def _decode_headers(scope: dict) -> dict[str, str]:
+    raw = scope.get("headers") or []
+    out: dict[str, str] = {}
+    for k, v in raw:
+        key = k.decode("latin-1").lower()
+        out[key] = v.decode("latin-1")
+    return out
+
+
+class OpenApiGatewayMiddleware:
+    """Runs outermost: sets ASGI root_path from X-Forwarded-Prefix; gates OpenAPI routes on X-Gateway-Secret."""
+
+    def __init__(self, app: ASGIApp):
+        self.app = app
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] != "http":
+            await self.app(scope, receive, send)
+            return
+
+        settings = get_settings()
+        path = scope["path"]
+        headers = _decode_headers(scope)
+
+        secret = (settings.backend_gateway_secret or "").strip()
+        if secret and _is_openapi_public_path(path):
+            if headers.get("x-gateway-secret") != secret:
+                body = json.dumps(
+                    {
+                        "detail": (
+                            "OpenAPI documentation is only available through the API gateway "
+                            "with a valid X-Gateway-Secret."
+                        ),
+                    },
+                ).encode("utf-8")
+                await send(
+                    {
+                        "type": "http.response.start",
+                        "status": 403,
+                        "headers": [
+                            (b"content-type", b"application/json"),
+                            (b"content-length", str(len(body)).encode("ascii")),
+                        ],
+                    },
+                )
+                await send({"type": "http.response.body", "body": body})
+                return
+
+        prefix = (headers.get("x-forwarded-prefix") or "").strip().rstrip("/")
+        new_scope = scope
+        if prefix:
+            new_scope = {**scope, "root_path": prefix}
+
+        await self.app(new_scope, receive, send)

package/backend/app/middleware/request_id.py

@@ -0,0 +1,23 @@
+from contextvars import ContextVar
+from uuid import uuid4
+
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import Response
+
+REQUEST_ID_HEADER = "X-Request-ID"
+
+request_id_ctx: ContextVar[str | None] = ContextVar("request_id", default=None)
+
+
+class RequestIdMiddleware(BaseHTTPMiddleware):
+    async def dispatch(self, request: Request, call_next) -> Response:
+        incoming = request.headers.get("x-request-id")
+        rid = incoming or str(uuid4())
+        token = request_id_ctx.set(rid)
+        try:
+            response = await call_next(request)
+            response.headers[REQUEST_ID_HEADER] = rid
+            return response
+        finally:
+            request_id_ctx.reset(token)

package/backend/app/openapi_config.py

@@ -0,0 +1,126 @@
+"""OpenAPI (Swagger) metadata and schema customization for the FastAPI app."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from fastapi import FastAPI
+
+from fastapi.openapi.utils import get_openapi
+
+# Must stay in sync with GatewayIdentityMiddleware._UNAUTH_PATHS
+_UNAUTHENTICATED_PATHS = frozenset({"/healthz", "/readyz"})
+
+OPENAPI_TAGS: list[dict[str, str]] = [
+    {
+        "name": "health",
+        "description": "Liveness (`/healthz`) and readiness (`/readyz`) probes. No gateway identity header required.",
+    },
+    {
+        "name": "ingestion",
+        "description": "Ingest raw connector payloads, normalize to canonical documents, and persist blobs.",
+    },
+    {
+        "name": "admin",
+        "description": "Operational endpoints: retry embedding, re-run normalization, clear DLQ state.",
+    },
+    {
+        "name": "query",
+        "description": "RAG retrieval: embed the question (Gemini), search sqlite-vec, return ranked context.",
+    },
+    {
+        "name": "chat",
+        "description": (
+            "Saved chat sessions (SQLite): list/create sessions, load messages, "
+            "complete a turn (RAG + Gemini answer)."
+        ),
+    },
+    {
+        "name": "embeddings",
+        "description": "Trigger or inspect Gemini embedding jobs for ingested documents.",
+    },
+    {
+        "name": "chunks",
+        "description": "Chunk listing and related ingest-time chunk operations.",
+    },
+    {
+        "name": "entities",
+        "description": "Entity extraction over document chunks.",
+    },
+    {
+        "name": "link-expansion",
+        "description": "Configurable URL fetching and link expansion during ingest.",
+    },
+    {
+        "name": "vectors",
+        "description": "Direct vector store utilities (debugging and advanced use).",
+    },
+    {
+        "name": "example",
+        "description": "Sample routes for development.",
+    },
+]
+
+APP_DESCRIPTION = """Internal HTTP API for the Business Stack knowledge base (RAG).
+
+**How to open these docs:** use the Hono gateway URL, e.g. `{gateway}{prefix}/docs` (same prefix as `API_GATEWAY_PREFIX`, default `/api/backend`). The gateway requires a signed-in Better Auth session and injects `X-User-Id` plus `X-Forwarded-Prefix` so Swagger loads `/openapi.json` correctly.
+
+**Direct backend access:** if `BACKEND_GATEWAY_SECRET` is set on the backend and gateway, `/docs`, `/redoc`, and `/openapi.json` return **403** unless the gateway adds matching `X-Gateway-Secret` (clients cannot spoof docs by calling the backend with only `x-user-id`). Leave the secret unset in local dev if you want OpenAPI on `http://127.0.0.1:8000/docs` without the gateway.
+
+**Gateway identity:** every route except `/healthz` and `/readyz` requires `x-user-id`. The gateway adds it after Better Auth session validation, so **Try it out** works with your login cookies and you do not need **Authorize** unless you call the raw backend URL.
+
+**External services:** embeddings and query use **Gemini**. The key may be `GEMINI_API_KEY` or stored in gateway integration settings; the backend uses `INTEGRATIONS_GATEWAY_URL` and `INTEGRATIONS_INTERNAL_SECRET` when env is unset.
+Optional OpenAI is only for LLM-assisted chunking, not `/query`.
+"""
+
+
+def attach_custom_openapi(app: FastAPI) -> None:
+    """Register a custom OpenAPI generator with gateway security and tag descriptions."""
+
+    def custom_openapi() -> dict:
+        if app.openapi_schema:
+            return app.openapi_schema
+        openapi_schema = get_openapi(
+            title=app.title,
+            version=app.version,
+            description=app.description,
+            routes=app.routes,
+            tags=OPENAPI_TAGS,
+        )
+        components = openapi_schema.setdefault("components", {})
+        security_schemes = components.setdefault("securitySchemes", {})
+        security_schemes["GatewayIdentity"] = {
+            "type": "apiKey",
+            "in": "header",
+            "name": "x-user-id",
+            "description": (
+                "End-user id propagated by the API gateway. Required on all routes except "
+                "`GET /healthz` and `GET /readyz`."
+            ),
+        }
+        paths = openapi_schema.get("paths", {})
+        for path, path_item in paths.items():
+            if not isinstance(path_item, dict):
+                continue
+            for method, operation in path_item.items():
+                if method not in (
+                    "get",
+                    "post",
+                    "put",
+                    "patch",
+                    "delete",
+                    "head",
+                    "options",
+                ):
+                    continue
+                if not isinstance(operation, dict):
+                    continue
+                if path in _UNAUTHENTICATED_PATHS:
+                    operation["security"] = []
+                else:
+                    operation["security"] = [{"GatewayIdentity": []}]
+        app.openapi_schema = openapi_schema
+        return openapi_schema
+
+    app.openapi = custom_openapi  # type: ignore[method-assign]

package/backend/app/routers/admin_pipeline.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+
+from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
+from pydantic import BaseModel, ConfigDict, Field
+from sqlalchemy import text
+
+from app.config import get_settings
+from app.dependencies import DbSession, VectorStoreDep
+from app.services.embeddings.dlq import (
+    cleaned_ingest_meta_dict,
+    clear_embedding_dlq,
+    merge_ingest_meta,
+)
+from app.services.embeddings.worker import run_embed_document_job
+from app.services.ingestion.persist import (
+    clear_normalization_error,
+    load_raw_envelope_for_document,
+)
+from app.services.integrations_remote import resolve_gemini_api_key
+from app.services.normalization import (
+    normalize_envelope_to_canonical,
+    persist_normalized_document,
+)
+from app.storage.blobs import BlobStore
+
+router = APIRouter(prefix="/admin", tags=["admin"])
+
+
+class RetryEmbedBody(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    multimodal: bool = Field(
+        default=False,
+        description="Same flag as POST /ingest/documents/{id}/embed",
+    )
+
+
+@router.post("/documents/{document_id}/retry-embedding")
+async def admin_retry_embedding(
+    document_id: str,
+    body: RetryEmbedBody,
+    session: DbSession,
+    request: Request,
+    store: VectorStoreDep,
+    background_tasks: BackgroundTasks,
+) -> dict[str, str | bool]:
+    """
+    Clear DLQ state, reset ingest meta embedding fields, set status to ``partial``,
+    and queue the same background embed job as the public embed endpoint.
+    """
+    settings = get_settings()
+    if not await resolve_gemini_api_key(settings):
+        raise HTTPException(
+            status_code=503,
+            detail=(
+                "Gemini API key is not configured "
+                "(GEMINI_API_KEY or gateway integration store)"
+            ),
+        )
+    async with session.begin():
+        row = await session.execute(
+            text("SELECT 1 FROM documents WHERE id = :id LIMIT 1"),
+            {"id": document_id},
+        )
+        if row.first() is None:
+            raise HTTPException(status_code=404, detail="Document not found")
+        await clear_embedding_dlq(session, document_id=document_id)
+        r2 = await session.execute(
+            text("SELECT ingest_meta FROM documents WHERE id = :id LIMIT 1"),
+            {"id": document_id},
+        )
+        im = r2.scalar_one_or_none()
+        clean = cleaned_ingest_meta_dict(im)
+        new_meta = merge_ingest_meta(
+            json.dumps(clean, default=str),
+            {"embedding_admin_retry_at": datetime.now(UTC).isoformat()},
+        )
+        await session.execute(
+            text(
+                "UPDATE documents SET status = 'partial', ingest_meta = :im "
+                "WHERE id = :id",
+            ),
+            {"im": new_meta, "id": document_id},
+        )
+
+    background_tasks.add_task(
+        run_embed_document_job,
+        document_id=document_id,
+        multimodal=body.multimodal,
+        session_factory=request.app.state.session_factory,
+        settings=settings,
+        store=store,
+    )
+    return {"accepted": True, "document_id": document_id}
+
+
+@router.post("/documents/{document_id}/reprocess-normalization")
+async def admin_reprocess_normalization(
+    document_id: str,
+    session: DbSession,
+) -> dict[str, str | bool]:
+    """Rebuild ``content_blocks`` / ``document_links`` from stored ``raw_content``."""
+    settings = get_settings()
+    blob_store = BlobStore(settings.data_dir / "blobs")
+    async with session.begin():
+        row = await session.execute(
+            text("SELECT 1 FROM documents WHERE id = :id LIMIT 1"),
+            {"id": document_id},
+        )
+        if row.first() is None:
+            raise HTTPException(status_code=404, detail="Document not found")
+        env = await load_raw_envelope_for_document(session, document_id=document_id)
+        canonical = normalize_envelope_to_canonical(
+            document_id=document_id,
+            envelope=env,
+            blob_store=blob_store,
+        )
+        await persist_normalized_document(session, canonical=canonical)
+        await clear_normalization_error(session, document_id=document_id)
+    return {"ok": True, "document_id": document_id}