business-stack 0.1.0

package/backend/app/routers/pipeline_status.py

@@ -0,0 +1,304 @@
+"""Read-only pipeline introspection for a single document (ingest → chunks → embed)."""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from typing import Any, Literal
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel, Field
+from sqlalchemy import text
+
+from app.config import get_settings
+from app.dependencies import DbSession
+
+router = APIRouter(prefix="/ingest", tags=["ingestion"])
+
+PipelineStepState = Literal["ok", "warn", "error", "pending"]
+
+
+class PipelineStepOut(BaseModel):
+    id: str
+    label: str
+    state: PipelineStepState
+    detail: str | None = None
+
+
+class EmbeddingDlqOut(BaseModel):
+    state: str
+    attempt_count: int
+    last_error: str
+    next_retry_at: str | None = None
+    updated_at: str | None = None
+
+
+class DocumentPipelineOut(BaseModel):
+    document_id: str
+    status: str
+    content_type: str | None = None
+    content_block_count: int = 0
+    chunk_count: int = 0
+    vector_row_count: int = 0
+    gemini_embedding_row_count: int = 0
+    normalization_error: dict[str, Any] | str | None = None
+    ingest_meta: dict[str, Any] = Field(default_factory=dict)
+    dlq: EmbeddingDlqOut | None = None
+    steps: list[PipelineStepOut]
+    checked_at: str
+
+
+def _parse_json_obj(raw: str | None) -> dict[str, Any] | str | None:
+    if not raw or not raw.strip():
+        return None
+    try:
+        v = json.loads(raw)
+        return v if isinstance(v, dict) else raw
+    except json.JSONDecodeError:
+        return raw
+
+
+@router.get("/documents/{document_id}/pipeline", response_model=DocumentPipelineOut)
+async def get_document_pipeline(document_id: str, session: DbSession) -> DocumentPipelineOut:
+    settings = get_settings()
+    model = settings.gemini_embedding_model
+
+    row = await session.execute(
+        text(
+            "SELECT status, content_type, normalization_error, ingest_meta "
+            "FROM documents WHERE id = :id LIMIT 1",
+        ),
+        {"id": document_id},
+    )
+    doc = row.first()
+    if doc is None:
+        raise HTTPException(status_code=404, detail="Document not found")
+
+    status, content_type, norm_err_raw, ingest_meta_raw = doc[0], doc[1], doc[2], doc[3]
+    norm_parsed = _parse_json_obj(norm_err_raw)
+
+    im: dict[str, Any] = {}
+    if ingest_meta_raw:
+        try:
+            p = json.loads(ingest_meta_raw)
+            if isinstance(p, dict):
+                im = p
+        except json.JSONDecodeError:
+            im = {"_raw": ingest_meta_raw}
+
+    r_blocks = await session.execute(
+        text("SELECT COUNT(*) FROM content_blocks WHERE document_id = :id"),
+        {"id": document_id},
+    )
+    content_block_count = int(r_blocks.scalar_one())
+
+    r_chunks = await session.execute(
+        text("SELECT COUNT(*) FROM document_chunks WHERE document_id = :id"),
+        {"id": document_id},
+    )
+    chunk_count = int(r_chunks.scalar_one())
+
+    r_vec = await session.execute(
+        text("SELECT COUNT(*) FROM kb_vec_embeddings WHERE document_id = :id"),
+        {"id": document_id},
+    )
+    vector_row_count = int(r_vec.scalar_one())
+
+    r_emb = await session.execute(
+        text(
+            "SELECT COUNT(*) FROM embeddings WHERE document_id = :id AND model = :m",
+        ),
+        {"id": document_id, "m": model},
+    )
+    gemini_embedding_row_count = int(r_emb.scalar_one())
+
+    r_dlq = await session.execute(
+        text(
+            "SELECT state, attempt_count, last_error, next_retry_at, updated_at "
+            "FROM embedding_dlq WHERE document_id = :id LIMIT 1",
+        ),
+        {"id": document_id},
+    )
+    dlq_row = r_dlq.first()
+    dlq: EmbeddingDlqOut | None = None
+    if dlq_row is not None:
+        dlq = EmbeddingDlqOut(
+            state=str(dlq_row[0]),
+            attempt_count=int(dlq_row[1]),
+            last_error=str(dlq_row[2]),
+            next_retry_at=dlq_row[3],
+            updated_at=dlq_row[4],
+        )
+
+    embedding_err = im.get("embedding_error")
+    steps = _build_steps(
+        content_block_count=content_block_count,
+        chunk_count=chunk_count,
+        vector_row_count=vector_row_count,
+        doc_status=str(status),
+        normalization_error=norm_parsed,
+        dlq=dlq,
+        embedding_error=str(embedding_err) if embedding_err else None,
+    )
+
+    return DocumentPipelineOut(
+        document_id=document_id,
+        status=str(status),
+        content_type=str(content_type) if content_type else None,
+        content_block_count=content_block_count,
+        chunk_count=chunk_count,
+        vector_row_count=vector_row_count,
+        gemini_embedding_row_count=gemini_embedding_row_count,
+        normalization_error=norm_parsed,
+        ingest_meta=im,
+        dlq=dlq,
+        steps=steps,
+        checked_at=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
+    )
+
+
+def _build_steps(
+    *,
+    content_block_count: int,
+    chunk_count: int,
+    vector_row_count: int,
+    doc_status: str,
+    normalization_error: dict[str, Any] | str | None,
+    dlq: EmbeddingDlqOut | None,
+    embedding_error: str | None,
+) -> list[PipelineStepOut]:
+    steps: list[PipelineStepOut] = []
+
+    steps.append(
+        PipelineStepOut(
+            id="document",
+            label="Document stored",
+            state="ok",
+            detail="Row exists in `documents`",
+        ),
+    )
+
+    if normalization_error is not None:
+        msg = (
+            normalization_error.get("message", str(normalization_error))
+            if isinstance(normalization_error, dict)
+            else str(normalization_error)
+        )
+        n_state: PipelineStepState = "error"
+        n_detail = msg[:500] + ("…" if len(msg) > 500 else "")
+        steps.append(
+            PipelineStepOut(
+                id="normalize",
+                label="Normalize → content blocks",
+                state=n_state,
+                detail=n_detail,
+            ),
+        )
+    elif content_block_count > 0:
+        steps.append(
+            PipelineStepOut(
+                id="normalize",
+                label="Normalize → content blocks",
+                state="ok",
+                detail=f"{content_block_count} block(s)",
+            ),
+        )
+    else:
+        steps.append(
+            PipelineStepOut(
+                id="normalize",
+                label="Normalize → content blocks",
+                state="warn",
+                detail="No content blocks yet (empty or not normalized)",
+            ),
+        )
+
+    if chunk_count > 0:
+        steps.append(
+            PipelineStepOut(
+                id="chunks",
+                label="Semantic chunks",
+                state="ok",
+                detail=f"{chunk_count} chunk(s)",
+            ),
+        )
+    elif content_block_count > 0:
+        steps.append(
+            PipelineStepOut(
+                id="chunks",
+                label="Semantic chunks",
+                state="error",
+                detail="No rows in `document_chunks` — run POST …/chunks or re-ingest",
+            ),
+        )
+    else:
+        steps.append(
+            PipelineStepOut(
+                id="chunks",
+                label="Semantic chunks",
+                state="pending",
+                detail="Waiting for content blocks",
+            ),
+        )
+
+    err_bits: list[str] = []
+    if embedding_error:
+        err_bits.append(embedding_error[:400])
+    if dlq is not None and dlq.state == "dead":
+        err_bits.append(f"DLQ dead after {dlq.attempt_count} attempt(s)")
+    elif dlq is not None:
+        err_bits.append(f"DLQ {dlq.state} (attempt {dlq.attempt_count})")
+        if dlq.last_error:
+            err_bits.append(dlq.last_error[:400])
+
+    if chunk_count == 0:
+        steps.append(
+            PipelineStepOut(
+                id="embed",
+                label="Embed + vector index",
+                state="pending",
+                detail="Chunks required before embedding",
+            ),
+        )
+    elif vector_row_count > 0:
+        match = vector_row_count == chunk_count
+        steps.append(
+            PipelineStepOut(
+                id="embed",
+                label="Embed + vector index",
+                state="ok" if match else "warn",
+                detail=(
+                    f"{vector_row_count} vector row(s), {chunk_count} chunk(s)"
+                    + ("" if match else " — counts differ; possible partial failure")
+                ),
+            ),
+        )
+    elif doc_status == "failed" or (dlq is not None and dlq.state == "dead"):
+        steps.append(
+            PipelineStepOut(
+                id="embed",
+                label="Embed + vector index",
+                state="error",
+                detail="; ".join(err_bits) if err_bits else "Embedding failed",
+            ),
+        )
+    elif err_bits:
+        steps.append(
+            PipelineStepOut(
+                id="embed",
+                label="Embed + vector index",
+                state="warn",
+                detail="; ".join(err_bits),
+            ),
+        )
+    else:
+        steps.append(
+            PipelineStepOut(
+                id="embed",
+                label="Embed + vector index",
+                state="pending",
+                detail="Queued or running — vectors not in index yet",
+            ),
+        )
+
+    return steps

package/backend/app/routers/query.py

@@ -0,0 +1,63 @@
+from datetime import UTC, datetime
+
+from fastapi import APIRouter, HTTPException
+
+from app.dependencies import DbSession, SettingsDep, VectorStoreDep
+from app.schemas.query import (
+    QueryCandidate,
+    QueryContext,
+    QueryContextMedia,
+    QueryContextSection,
+    QueryRequest,
+    QueryResponse,
+)
+from app.services.retrieval_service import run_retrieval
+
+router = APIRouter(tags=["query"])
+
+
+@router.post("/query", response_model=QueryResponse)
+async def query_retrieve(
+    body: QueryRequest,
+    session: DbSession,
+    store: VectorStoreDep,
+    settings: SettingsDep,
+) -> QueryResponse:
+    """
+    Embed the user query (Gemini), run vector search with optional filters,
+    blend semantic / recency / source weights, hydrate chunks and documents,
+    and return ranked hits plus aggregated multimodal context for generation.
+    """
+    filters = (
+        body.filters.to_vector_filters(now=datetime.now(UTC)) if body.filters else None
+    )
+    try:
+        raw = await run_retrieval(
+            session=session,
+            store=store,
+            settings=settings,
+            query=body.query,
+            k=body.k,
+            filters=filters,
+        )
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e)) from e
+    except RuntimeError as e:
+        msg = str(e)
+        if "Gemini API key" in msg or "GEMINI_API_KEY" in msg:
+            raise HTTPException(status_code=503, detail=msg) from e
+        raise HTTPException(status_code=502, detail=msg) from e
+
+    candidates = [QueryCandidate.model_validate(c) for c in raw["candidates"]]
+    ctx = raw["context"]
+    context = QueryContext(
+        combined_text=ctx["combined_text"],
+        sections=[QueryContextSection.model_validate(s) for s in ctx["sections"]],
+        media=[QueryContextMedia.model_validate(m) for m in ctx["media"]],
+    )
+    return QueryResponse(
+        candidates=candidates,
+        context=context,
+        embedding_model=raw["embedding_model"],
+        vector_candidates_considered=raw["vector_candidates_considered"],
+    )

package/backend/app/routers/vectors.py

@@ -0,0 +1,63 @@
+from fastapi import APIRouter
+
+from app.dependencies import VectorStoreDep
+from app.schemas.vectors import (
+    VectorSearchHit,
+    VectorSearchRequest,
+    VectorSearchResponse,
+    VectorUpsertRequest,
+)
+from app.vectorstore import VectorMeta
+
+router = APIRouter(prefix="/vectors", tags=["vectors"])
+
+
+@router.post("/upsert")
+async def upsert_vectors(
+    body: VectorUpsertRequest,
+    store: VectorStoreDep,
+) -> dict[str, str]:
+    metas = [
+        VectorMeta(
+            document_id=m.document_id,
+            chunk_id=m.chunk_id,
+            source_id=m.source_id,
+            modality=m.modality,
+            ingested_at=m.ingested_at,
+        )
+        for m in body.metas
+    ]
+    await store.upsert(body.embeddings, metas)
+    return {"status": "ok"}
+
+
+@router.delete("/documents/{document_id}")
+async def delete_document_vectors(
+    document_id: str,
+    store: VectorStoreDep,
+) -> dict[str, int]:
+    deleted = await store.delete_document(document_id)
+    return {"deleted": deleted}
+
+
+@router.post("/search", response_model=VectorSearchResponse)
+async def search_vectors(
+    body: VectorSearchRequest,
+    store: VectorStoreDep,
+) -> VectorSearchResponse:
+    filters = body.filters.to_filters() if body.filters else None
+    hits = await store.search(body.query_vector, body.k, filters)
+    return VectorSearchResponse(
+        results=[
+            VectorSearchHit(
+                rowid=h.rowid,
+                distance=h.distance,
+                document_id=h.document_id,
+                chunk_id=h.chunk_id,
+                source_id=h.source_id,
+                modality=h.modality,
+                ingested_at=h.ingested_at,
+            )
+            for h in hits
+        ],
+    )

package/backend/app/schemas/canonical.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+BlockType = Literal["text", "image", "audio", "video", "document"]
+
+
+class CanonicalContentBlock(BaseModel):
+    """One multimodal slice after normalization."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    type: BlockType
+    data: str | None = Field(
+        default=None,
+        description="Inline UTF-8 text or small structured payload summary",
+    )
+    raw_input: str | None = Field(
+        default=None,
+        description="Storage reference e.g. blob://<sha256> or uri:https://...",
+    )
+    mime: str | None = Field(
+        default=None,
+        description="IANA media type when stored externally (e.g. image/png)",
+    )
+
+
+class CanonicalDocument(BaseModel):
+    """Canonical shape for a normalized document (API + persistence source)."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    id: str
+    source: str
+    timestamp: datetime
+    content_blocks: list[CanonicalContentBlock]
+    raw_content: Any
+    entities: list[Any] = Field(default_factory=list)
+    links: list[str] = Field(default_factory=list)
+    tags: list[str] = Field(default_factory=list)
+    summary: str = ""

package/backend/app/schemas/chat.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.schemas.query import QueryFiltersPayload
+
+
+class ChatSessionCreate(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    title: str | None = Field(default=None, max_length=500)
+
+
+class ChatSessionOut(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    id: str
+    title: str | None
+    created_at: str
+    updated_at: str
+
+
+class ChatMessageOut(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    id: int
+    role: str
+    content: str
+    meta_json: str | None
+    created_at: str
+
+
+class ChatCompleteRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    message: str = Field(..., min_length=1, max_length=16_000)
+    k: int = Field(default=8, ge=1, le=50)
+    filters: QueryFiltersPayload | None = None
+
+
+class ChatCompleteResponse(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    assistant_message_id: int
+    reply: str
+    reply_source: str = Field(
+        description="'model' if Gemini answered, else retrieval fallback text",
+    )
+    embedding_model: str
+    vector_candidates_considered: int

package/backend/app/schemas/ingest.py

@@ -0,0 +1,29 @@
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+ContentTypeLiteral = Literal[
+    "text",
+    "image",
+    "audio",
+    "video",
+    "document",
+    "multimodal",
+]
+
+
+class RawIngestEnvelope(BaseModel):
+    """Connector output envelope accepted by POST /ingest/raw."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    source: str = Field(
+        ...,
+        min_length=1,
+        description="Logical source label (e.g. slack, filesystem)",
+    )
+    timestamp: datetime
+    content_type: ContentTypeLiteral
+    payload: Any
+    metadata: dict[str, Any] = Field(default_factory=dict)

package/backend/app/schemas/query.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.vectorstore import VectorSearchFilters
+
+
+def _parse_iso_utc(s: str) -> datetime | None:
+    try:
+        norm = s.replace("Z", "+00:00")
+        dt = datetime.fromisoformat(norm)
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=UTC)
+        return dt.astimezone(UTC)
+    except (TypeError, ValueError):
+        return None
+
+
+def _later_iso(a: str, b: str) -> str:
+    """Return the ISO timestamp that is later in time (stricter lower bound)."""
+    da, db = _parse_iso_utc(a), _parse_iso_utc(b)
+    if da and db:
+        return a if da >= db else b
+    return a if da else b
+
+
+class QueryFiltersPayload(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    document_id: str | None = None
+    source_id: int | None = None
+    modality: str | None = None
+    timestamp_min: str | None = Field(
+        default=None,
+        description="ISO-8601 lower bound on vector ingested_at",
+    )
+    timestamp_max: str | None = Field(
+        default=None,
+        description="ISO-8601 upper bound on vector ingested_at",
+    )
+    newer_than_days: float | None = Field(
+        default=None,
+        ge=0.0,
+        description="Convenience: require ingested_at >= now - this many days",
+    )
+
+    def to_vector_filters(self, *, now: datetime) -> VectorSearchFilters | None:
+        ts_min = self.timestamp_min
+        if self.newer_than_days is not None:
+            cutoff = (now - timedelta(days=float(self.newer_than_days))).astimezone(UTC)
+            cut_iso = cutoff.strftime("%Y-%m-%dT%H:%M:%SZ")
+            ts_min = _later_iso(ts_min, cut_iso) if ts_min else cut_iso
+
+        if (
+            self.document_id is None
+            and self.source_id is None
+            and self.modality is None
+            and ts_min is None
+            and self.timestamp_max is None
+        ):
+            return None
+        return VectorSearchFilters(
+            document_id=self.document_id,
+            source_id=self.source_id,
+            modality=self.modality,
+            timestamp_min=ts_min,
+            timestamp_max=self.timestamp_max,
+        )
+
+
+class QueryRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    query: str = Field(..., min_length=1, max_length=16_000)
+    k: int = Field(default=10, ge=1, le=100)
+    filters: QueryFiltersPayload | None = None
+
+
+class QueryAttribution(BaseModel):
+    vector_rowid: int
+    distance: float
+    document_id: str
+    chunk_id: int
+    source_id: int
+    modality: str
+    ingested_at: str
+    source_name: str | None = None
+    connector_type: str | None = None
+    document_timestamp: str | None = None
+
+
+class QueryChunkPayload(BaseModel):
+    ordinal: int
+    text: str
+    start_block_ordinal: int
+    end_block_ordinal: int
+    meta: str | None = None
+
+
+class QueryDocumentPayload(BaseModel):
+    summary: str | None = None
+    content_type: str
+    timestamp: str
+    status: str
+
+
+class QuerySourcePayload(BaseModel):
+    id: int
+    name: str
+    connector_type: str
+
+
+class QueryCandidate(BaseModel):
+    rank: int
+    score: float
+    semantic_score: float
+    recency_score: float
+    source_weight: float
+    attribution: QueryAttribution
+    document: QueryDocumentPayload | None = None
+    chunk: QueryChunkPayload | None = None
+    source: QuerySourcePayload | None = None
+
+
+class QueryContextSection(BaseModel):
+    document_id: str
+    chunk_id: int
+    chunk_ordinal: int
+    text: str
+
+
+class QueryContextMedia(BaseModel):
+    document_id: str
+    chunk_ordinal: int
+    block_ordinal: int
+    type: str
+    mime: str | None = None
+    storage_uri: str | None = None
+
+
+class QueryContext(BaseModel):
+    combined_text: str
+    sections: list[QueryContextSection]
+    media: list[QueryContextMedia]
+
+
+class QueryResponse(BaseModel):
+    candidates: list[QueryCandidate]
+    context: QueryContext
+    embedding_model: str
+    vector_candidates_considered: int