business-stack 0.1.0

package/backend/app/routers/chat.py

@@ -0,0 +1,206 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Any
+
+from fastapi import APIRouter, HTTPException, Query
+
+from app.dependencies import DbSession, GatewayUserDep, SettingsDep, VectorStoreDep
+from app.schemas.chat import (
+    ChatCompleteRequest,
+    ChatCompleteResponse,
+    ChatMessageOut,
+    ChatSessionCreate,
+    ChatSessionOut,
+)
+from app.services.chat_store import (
+    append_chat_message,
+    create_chat_session,
+    delete_chat_session,
+    get_session_owned,
+    list_chat_messages,
+    list_chat_sessions,
+    maybe_set_title_from_first_message,
+    touch_session_updated,
+)
+from app.services.gemini_chat import gemini_chat_reply
+from app.services.retrieval_service import run_retrieval
+
+router = APIRouter(prefix="/chat", tags=["chat"])
+
+
+@router.post("/sessions", response_model=ChatSessionOut)
+async def create_session(
+    body: ChatSessionCreate,
+    db: DbSession,
+    user_id: GatewayUserDep,
+) -> ChatSessionOut:
+    sid = await create_chat_session(db, user_id=user_id, title=body.title)
+    await db.commit()
+    row = await get_session_owned(db, session_id=sid, user_id=user_id)
+    if not row:
+        raise HTTPException(status_code=500, detail="session create failed") from None
+    return ChatSessionOut(
+        id=str(row["id"]),
+        title=row.get("title"),
+        created_at=str(row["created_at"]),
+        updated_at=str(row["updated_at"]),
+    )
+
+
+@router.get("/sessions", response_model=list[ChatSessionOut])
+async def sessions_list(
+    db: DbSession,
+    user_id: GatewayUserDep,
+    limit: int = Query(default=100, ge=1, le=200),
+) -> list[ChatSessionOut]:
+    rows = await list_chat_sessions(db, user_id=user_id, limit=limit)
+    return [
+        ChatSessionOut(
+            id=str(r["id"]),
+            title=r.get("title"),
+            created_at=str(r["created_at"]),
+            updated_at=str(r["updated_at"]),
+        )
+        for r in rows
+    ]
+
+
+@router.delete("/sessions/{session_id}")
+async def sessions_delete(
+    session_id: str,
+    db: DbSession,
+    user_id: GatewayUserDep,
+) -> dict[str, bool]:
+    ok = await delete_chat_session(db, session_id=session_id, user_id=user_id)
+    await db.commit()
+    if not ok:
+        raise HTTPException(status_code=404, detail="Session not found")
+    return {"deleted": True}
+
+
+@router.get(
+    "/sessions/{session_id}/messages",
+    response_model=list[ChatMessageOut],
+)
+async def messages_list(
+    session_id: str,
+    db: DbSession,
+    user_id: GatewayUserDep,
+    limit: int = Query(default=200, ge=1, le=500),
+) -> list[ChatMessageOut]:
+    row = await get_session_owned(db, session_id=session_id, user_id=user_id)
+    if not row:
+        raise HTTPException(status_code=404, detail="Session not found")
+    rows = await list_chat_messages(db, session_id=session_id, limit=limit)
+    return [
+        ChatMessageOut(
+            id=int(r["id"]),
+            role=str(r["role"]),
+            content=str(r["content"]),
+            meta_json=r.get("meta_json"),
+            created_at=str(r["created_at"]),
+        )
+        for r in rows
+    ]
+
+
+def _sources_meta(candidates: list[dict[str, Any]], *, max_n: int = 12) -> list[dict[str, Any]]:
+    out: list[dict[str, Any]] = []
+    for c in candidates[:max_n]:
+        att = c.get("attribution") or {}
+        if not isinstance(att, dict):
+            continue
+        out.append(
+            {
+                "rank": c.get("rank"),
+                "score": c.get("score"),
+                "document_id": att.get("document_id"),
+                "chunk_id": att.get("chunk_id"),
+                "source_name": att.get("source_name"),
+            },
+        )
+    return out
+
+
+@router.post(
+    "/sessions/{session_id}/complete",
+    response_model=ChatCompleteResponse,
+)
+async def complete_turn(
+    session_id: str,
+    body: ChatCompleteRequest,
+    db: DbSession,
+    user_id: GatewayUserDep,
+    settings: SettingsDep,
+    store: VectorStoreDep,
+) -> ChatCompleteResponse:
+    row = await get_session_owned(db, session_id=session_id, user_id=user_id)
+    if not row:
+        raise HTTPException(status_code=404, detail="Session not found")
+
+    filters = (
+        body.filters.to_vector_filters(now=datetime.now(UTC)) if body.filters else None
+    )
+
+    await append_chat_message(
+        db,
+        session_id=session_id,
+        role="user",
+        content=body.message.strip(),
+        meta=None,
+    )
+    await maybe_set_title_from_first_message(
+        db,
+        session_id=session_id,
+        snippet=body.message.strip(),
+    )
+
+    try:
+        raw = await run_retrieval(
+            session=db,
+            store=store,
+            settings=settings,
+            query=body.message,
+            k=body.k,
+            filters=filters,
+        )
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e)) from e
+    except RuntimeError as e:
+        msg = str(e)
+        if "Gemini API key" in msg or "GEMINI_API_KEY" in msg:
+            raise HTTPException(status_code=503, detail=msg) from e
+        raise HTTPException(status_code=502, detail=msg) from e
+
+    ctx = raw["context"]
+    combined = str(ctx.get("combined_text") or "")
+
+    reply, reply_source = await gemini_chat_reply(
+        settings,
+        user_message=body.message.strip(),
+        context_combined=combined,
+    )
+
+    meta = {
+        "reply_source": reply_source,
+        "sources": _sources_meta(raw["candidates"]),
+        "vector_candidates_considered": raw["vector_candidates_considered"],
+    }
+    mid = await append_chat_message(
+        db,
+        session_id=session_id,
+        role="assistant",
+        content=reply,
+        meta=meta,
+    )
+    await touch_session_updated(db, session_id=session_id)
+    await db.commit()
+
+    return ChatCompleteResponse(
+        assistant_message_id=mid,
+        reply=reply,
+        reply_source=reply_source,
+        embedding_model=raw["embedding_model"],
+        vector_candidates_considered=int(raw["vector_candidates_considered"]),
+    )

package/backend/app/routers/chunks.py

@@ -0,0 +1,36 @@
+from fastapi import APIRouter, HTTPException
+from sqlalchemy import text
+
+from app.config import get_settings
+from app.dependencies import DbSession
+from app.services.chunking.schemas import RebuildChunksRequest, RebuildChunksResponse
+from app.services.chunking.semantic_chunk import rebuild_semantic_chunks
+
+router = APIRouter(prefix="/ingest", tags=["chunks"])
+
+
+@router.post(
+    "/documents/{document_id}/chunks",
+    response_model=RebuildChunksResponse,
+)
+async def rebuild_document_chunks(
+    document_id: str,
+    body: RebuildChunksRequest,
+    session: DbSession,
+) -> RebuildChunksResponse:
+    """Rebuild semantic chunks (structure-first; optional LLM for weak structure)."""
+    row = await session.execute(
+        text("SELECT 1 FROM documents WHERE id = :id LIMIT 1"),
+        {"id": document_id},
+    )
+    if row.first() is None:
+        raise HTTPException(status_code=404, detail="Document not found")
+    settings = get_settings()
+    n = await rebuild_semantic_chunks(
+        session,
+        document_id=document_id,
+        use_llm_weak_structure=body.use_llm_weak_structure,
+        settings=settings,
+    )
+    await session.commit()
+    return RebuildChunksResponse(document_id=document_id, chunks_written=n)

package/backend/app/routers/entity_extract.py

@@ -0,0 +1,31 @@
+from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
+from sqlalchemy import text
+
+from app.config import get_settings
+from app.services.entities.pipeline import run_entity_extraction_job
+
+router = APIRouter(prefix="/ingest", tags=["entities"])
+
+
+@router.post("/documents/{document_id}/entities")
+async def queue_entity_extraction(
+    document_id: str,
+    background_tasks: BackgroundTasks,
+    request: Request,
+) -> dict[str, str | bool]:
+    """Re-run entity extraction for a document (uses current ``document_chunks``)."""
+    settings = get_settings()
+    async with request.app.state.session_factory() as session:
+        row = await session.execute(
+            text("SELECT 1 FROM documents WHERE id = :id LIMIT 1"),
+            {"id": document_id},
+        )
+        if row.first() is None:
+            raise HTTPException(status_code=404, detail="Document not found")
+    background_tasks.add_task(
+        run_entity_extraction_job,
+        document_id=document_id,
+        session_factory=request.app.state.session_factory,
+        settings=settings,
+    )
+    return {"accepted": True, "document_id": document_id}

package/backend/app/routers/example.py

@@ -0,0 +1,8 @@
+from fastapi import APIRouter
+
+router = APIRouter(tags=["example"])
+
+
+@router.get("/hello-world")
+def hello_world() -> dict[str, str]:
+    return {"message": "Hello, World"}

package/backend/app/routers/gemini_embed.py

@@ -0,0 +1,58 @@
+from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
+from pydantic import BaseModel, ConfigDict, Field
+from sqlalchemy import text
+
+from app.config import get_settings
+from app.dependencies import VectorStoreDep
+from app.services.embeddings.worker import run_embed_document_job
+from app.services.integrations_remote import resolve_gemini_api_key
+
+router = APIRouter(prefix="/ingest", tags=["embeddings"])
+
+
+class EmbedDocumentRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    multimodal: bool = Field(
+        default=False,
+        description=(
+            "If true, build interleaved parts from content_blocks in each "
+            "chunk's block span (text + images from blob storage)"
+        ),
+    )
+
+
+@router.post("/documents/{document_id}/embed")
+async def queue_document_embed(
+    document_id: str,
+    body: EmbedDocumentRequest,
+    background_tasks: BackgroundTasks,
+    request: Request,
+    store: VectorStoreDep,
+) -> dict[str, str | bool]:
+    """Schedule Gemini embedding for all ``document_chunks`` (background task)."""
+    settings = get_settings()
+    if not await resolve_gemini_api_key(settings):
+        raise HTTPException(
+            status_code=503,
+            detail=(
+                "Gemini API key is not configured "
+                "(GEMINI_API_KEY or gateway integration store)"
+            ),
+        )
+    async with request.app.state.session_factory() as session:
+        row = await session.execute(
+            text("SELECT 1 FROM documents WHERE id = :id LIMIT 1"),
+            {"id": document_id},
+        )
+        if row.first() is None:
+            raise HTTPException(status_code=404, detail="Document not found")
+    background_tasks.add_task(
+        run_embed_document_job,
+        document_id=document_id,
+        multimodal=body.multimodal,
+        session_factory=request.app.state.session_factory,
+        settings=settings,
+        store=store,
+    )
+    return {"accepted": True, "document_id": document_id}

package/backend/app/routers/health.py

@@ -0,0 +1,28 @@
+import logging
+
+from fastapi import APIRouter, Request
+from fastapi.responses import JSONResponse
+from sqlalchemy import text
+
+router = APIRouter(tags=["health"])
+logger = logging.getLogger(__name__)
+
+
+@router.get("/healthz")
+async def healthz() -> dict[str, str]:
+    return {"status": "ok"}
+
+
+@router.get("/readyz")
+async def readyz(request: Request):
+    engine = request.app.state.engine
+    try:
+        async with engine.connect() as conn:
+            await conn.execute(text("SELECT 1"))
+    except Exception:
+        logger.exception("readiness check failed")
+        return JSONResponse(
+            status_code=503,
+            content={"status": "not_ready"},
+        )
+    return {"status": "ready"}

package/backend/app/routers/ingestion.py

@@ -0,0 +1,146 @@
+import asyncio
+import json
+import traceback
+from typing import Any
+
+from fastapi import APIRouter, HTTPException, Query
+from pydantic import BaseModel, Field
+from sqlalchemy import text
+
+from app.config import get_settings
+from app.connectors.registry import get_connector_class, init_connectors
+from app.dependencies import DbSession
+from app.schemas.canonical import CanonicalDocument
+from app.schemas.ingest import RawIngestEnvelope
+from app.services.chunking.semantic_chunk import rebuild_semantic_chunks
+from app.services.ingestion.persist import (
+    clear_normalization_error,
+    load_raw_envelope_for_document,
+    persist_raw_envelope,
+)
+from app.services.normalization import (
+    normalize_envelope_to_canonical,
+    persist_normalized_document,
+)
+from app.storage.blobs import BlobStore
+
+router = APIRouter(prefix="/ingest", tags=["ingestion"])
+
+
+class RawIngestResponse(BaseModel):
+    document_id: str
+    status: str
+    connector: str
+    normalized: dict[str, Any]
+    canonical: dict[str, Any]
+    deduplicated: bool = Field(
+        default=False,
+        description="True when ingest matched an existing doc (external_id or hash)",
+    )
+    normalization_failed: bool = Field(
+        default=False,
+        description="True when raw was stored but canonical blocks could not be built",
+    )
+
+
+def _canonical_fallback(
+    *,
+    document_id: str,
+    envelope: RawIngestEnvelope,
+) -> CanonicalDocument:
+    """Minimal canonical shape when normalization fails (raw still persisted)."""
+    return CanonicalDocument(
+        id=document_id,
+        source=envelope.source,
+        timestamp=envelope.timestamp,
+        content_blocks=[],
+        raw_content=envelope.payload,
+        entities=[],
+        links=[],
+        tags=[],
+        summary="",
+    )
+
+
+@router.post("/raw", response_model=RawIngestResponse)
+async def ingest_raw(
+    body: RawIngestEnvelope,
+    session: DbSession,
+    connector: str = Query(
+        default="generic",
+        description="Registry key for normalize_raw / validate_config",
+    ),
+) -> RawIngestResponse:
+    init_connectors()
+    try:
+        connector_cls = get_connector_class(connector)
+    except KeyError:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Unknown connector: {connector}",
+        ) from None
+
+    instance = connector_cls()
+    cfg = body.metadata.get("connector_config")
+    if cfg is not None and not isinstance(cfg, dict):
+        raise HTTPException(
+            status_code=400,
+            detail="metadata.connector_config must be an object when set",
+        )
+    config = cfg if isinstance(cfg, dict) else {}
+    try:
+        instance.validate_config(config)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e)) from e
+
+    if connector == "playwright_session":
+        body = await asyncio.to_thread(instance.prepare_envelope, body, config)
+    else:
+        body = instance.prepare_envelope(body, config)
+
+    normalized = instance.normalize_raw(body)
+
+    settings = get_settings()
+    blob_store = BlobStore(settings.data_dir / "blobs")
+
+    norm_failed = False
+    async with session.begin():
+        doc_id, deduped = await persist_raw_envelope(session, body)
+        stored = await load_raw_envelope_for_document(session, document_id=doc_id)
+        try:
+            canonical = normalize_envelope_to_canonical(
+                document_id=doc_id,
+                envelope=stored,
+                blob_store=blob_store,
+            )
+            if not deduped:
+                await persist_normalized_document(session, canonical=canonical)
+                await clear_normalization_error(session, document_id=doc_id)
+                await rebuild_semantic_chunks(
+                    session,
+                    document_id=doc_id,
+                    use_llm_weak_structure=False,
+                    settings=settings,
+                )
+        except Exception as e:
+            norm_failed = True
+            err_obj = {
+                "type": type(e).__name__,
+                "message": str(e),
+                "traceback": traceback.format_exc()[-8000:],
+            }
+            await session.execute(
+                text("UPDATE documents SET normalization_error = :err WHERE id = :id"),
+                {"err": json.dumps(err_obj)[:16000], "id": doc_id},
+            )
+            canonical = _canonical_fallback(document_id=doc_id, envelope=stored)
+
+    return RawIngestResponse(
+        document_id=doc_id,
+        status="partial",
+        connector=connector,
+        normalized=normalized,
+        canonical=canonical.model_dump(mode="json"),
+        deduplicated=deduped,
+        normalization_failed=norm_failed,
+    )

package/backend/app/routers/link_expansion.py

@@ -0,0 +1,34 @@
+from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
+from sqlalchemy import text
+
+from app.config import get_settings
+from app.services.link_expansion.schemas import ExpandLinksRequest
+from app.services.link_expansion.worker import run_expand_links_job
+
+router = APIRouter(prefix="/ingest", tags=["link-expansion"])
+
+
+@router.post("/expand-links")
+async def queue_expand_links(
+    body: ExpandLinksRequest,
+    background_tasks: BackgroundTasks,
+    request: Request,
+) -> dict[str, str | bool]:
+    """Schedule recursive link expansion for a document's outbound links."""
+    settings = get_settings()
+    async with request.app.state.session_factory() as session:
+        row = await session.execute(
+            text("SELECT 1 FROM documents WHERE id = :id LIMIT 1"),
+            {"id": body.document_id},
+        )
+        if row.first() is None:
+            raise HTTPException(status_code=404, detail="Document not found")
+    options = body.resolve(settings)
+    background_tasks.add_task(
+        run_expand_links_job,
+        root_document_id=body.document_id,
+        options=options,
+        session_factory=request.app.state.session_factory,
+        settings=settings,
+    )
+    return {"accepted": True, "document_id": body.document_id}