business-stack 0.1.0

package/backend/app/services/retrieval_service.py

@@ -0,0 +1,351 @@
+from __future__ import annotations
+
+import json
+import logging
+import math
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.config import Settings
+from app.services.embeddings.build_inputs import load_blocks_span
+from app.services.embeddings.gemini_api import batch_embed_contents
+from app.services.embeddings.types import TextPart
+from app.services.integrations_remote import resolve_gemini_api_key
+from app.vectorstore import SqliteVecStore, VectorSearchFilters, VectorSearchResult
+
+logger = logging.getLogger(__name__)
+
+
+def _parse_source_weights(raw: str) -> dict[str, float]:
+    try:
+        data = json.loads(raw)
+    except json.JSONDecodeError:
+        logger.warning("invalid RETRIEVAL_SOURCE_WEIGHTS_JSON; using default")
+        return {"default": 1.0}
+    if not isinstance(data, dict):
+        return {"default": 1.0}
+    out: dict[str, float] = {}
+    for k, v in data.items():
+        try:
+            out[str(k)] = float(v)
+        except (TypeError, ValueError):
+            continue
+    if "default" not in out:
+        out["default"] = 1.0
+    return out
+
+
+def _parse_iso_utc(s: str) -> datetime | None:
+    try:
+        norm = s.replace("Z", "+00:00")
+        dt = datetime.fromisoformat(norm)
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=UTC)
+        return dt.astimezone(UTC)
+    except (TypeError, ValueError):
+        return None
+
+
+def semantic_score_from_distance(distance: float) -> float:
+    """Map sqlite-vec distance to [0, 1]; lower distance → higher score."""
+    if distance < 0 or math.isnan(distance):
+        return 0.0
+    return 1.0 / (1.0 + float(distance))
+
+
+def recency_score_from_ingested_at(
+    ingested_at: str,
+    *,
+    now: datetime,
+    half_life_days: float,
+) -> float:
+    """
+    Exponential half-life decay in [0, 1]: score = 0.5 ** (age_days / half_life).
+    """
+    if half_life_days <= 0:
+        return 1.0
+    parsed = _parse_iso_utc(ingested_at)
+    if parsed is None:
+        return 0.5
+    age = now - parsed
+    age_days = max(0.0, age.total_seconds() / 86_400.0)
+    return float(0.5 ** (age_days / half_life_days))
+
+
+def source_weight_for_connector(
+    connector_type: str,
+    weights: dict[str, float],
+) -> float:
+    w = weights.get(connector_type)
+    if w is None:
+        w = weights["default"]
+    return max(0.0, min(1.0, float(w)))
+
+
+def blended_retrieval_score(
+    *,
+    semantic: float,
+    recency: float,
+    source_w: float,
+    settings: Settings,
+) -> float:
+    return (
+        settings.retrieval_score_semantic_weight * semantic
+        + settings.retrieval_score_recency_weight * recency
+        + settings.retrieval_score_source_weight * source_w
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class _HydratedChunk:
+    chunk_pk: int
+    document_id: str
+    ordinal: int
+    chunk_text: str
+    start_block_ordinal: int
+    end_block_ordinal: int
+    chunk_meta: str | None
+    doc_summary: str | None
+    doc_content_type: str
+    doc_timestamp: str
+    doc_status: str
+    source_id: int
+    source_name: str
+    connector_type: str
+
+
+async def _hydrate_chunks(
+    session: AsyncSession,
+    chunk_pks: list[int],
+) -> dict[int, _HydratedChunk]:
+    if not chunk_pks:
+        return {}
+    placeholders = ", ".join(f":c{i}" for i in range(len(chunk_pks)))
+    params = {f"c{i}": pk for i, pk in enumerate(chunk_pks)}
+    r = await session.execute(
+        text(
+            f"""
+            SELECT
+              dc.id,
+              dc.document_id,
+              dc.ordinal,
+              dc.text,
+              dc.start_block_ordinal,
+              dc.end_block_ordinal,
+              dc.meta,
+              d.summary,
+              d.content_type,
+              d.timestamp,
+              d.status,
+              s.id,
+              s.name,
+              s.connector_type
+            FROM document_chunks dc
+            JOIN documents d ON d.id = dc.document_id
+            JOIN sources s ON s.id = d.source_id
+            WHERE dc.id IN ({placeholders})
+            """,
+        ),
+        params,
+    )
+    out: dict[int, _HydratedChunk] = {}
+    for row in r.fetchall():
+        pk = int(row[0])
+        out[pk] = _HydratedChunk(
+            chunk_pk=pk,
+            document_id=str(row[1]),
+            ordinal=int(row[2]),
+            chunk_text=str(row[3]),
+            start_block_ordinal=int(row[4]),
+            end_block_ordinal=int(row[5]),
+            chunk_meta=row[6] if row[6] else None,
+            doc_summary=row[7] if row[7] else None,
+            doc_content_type=str(row[8]),
+            doc_timestamp=str(row[9]),
+            doc_status=str(row[10]),
+            source_id=int(row[11]),
+            source_name=str(row[12]),
+            connector_type=str(row[13]),
+        )
+    return out
+
+
+async def run_retrieval(
+    *,
+    session: AsyncSession,
+    store: SqliteVecStore,
+    settings: Settings,
+    query: str,
+    k: int,
+    filters: VectorSearchFilters | None,
+    now: datetime | None = None,
+) -> dict[str, Any]:
+    """
+    Embed query (Gemini), vector search with filters, score, hydrate SQLite rows,
+    return ranked candidates + aggregated multimodal context for a generator.
+    """
+    q = query.strip()
+    if not q:
+        msg = "query must be non-empty"
+        raise ValueError(msg)
+    api_key = await resolve_gemini_api_key(settings)
+    if not api_key:
+        msg = "Gemini API key not configured (GEMINI_API_KEY or gateway store)"
+        raise RuntimeError(msg)
+
+    now_utc = (now or datetime.now(UTC)).astimezone(UTC)
+    source_weights = _parse_source_weights(settings.retrieval_source_weights_json)
+
+    vecs = await batch_embed_contents(
+        api_key=api_key,
+        model=settings.gemini_embedding_model,
+        contents=[[TextPart(q)]],
+        settings=settings,
+        task_type=settings.gemini_query_task_type,
+    )
+    if not vecs:
+        msg = "query embedding failed"
+        raise RuntimeError(msg)
+    query_vector = vecs[0]
+
+    mult = max(1, settings.retrieval_vec_candidate_multiplier)
+    vec_k = min(500, max(k, k * mult))
+    raw_hits = await store.search(query_vector, vec_k, filters)
+
+    chunk_ids = [h.chunk_id for h in raw_hits]
+    hydrated = await _hydrate_chunks(session, chunk_ids)
+
+    rescored: list[
+        tuple[float, VectorSearchResult, float, float, float, _HydratedChunk | None]
+    ] = []
+    for h in raw_hits:
+        sem = semantic_score_from_distance(h.distance)
+        rec = recency_score_from_ingested_at(
+            h.ingested_at,
+            now=now_utc,
+            half_life_days=settings.retrieval_recency_half_life_days,
+        )
+        row = hydrated.get(h.chunk_id)
+        src_w = (
+            source_weight_for_connector(row.connector_type, source_weights)
+            if row
+            else source_weights["default"]
+        )
+        total = blended_retrieval_score(
+            semantic=sem,
+            recency=rec,
+            source_w=src_w,
+            settings=settings,
+        )
+        rescored.append((total, h, sem, rec, src_w, row))
+
+    rescored.sort(key=lambda x: x[0], reverse=True)
+    top = rescored[:k]
+
+    candidates: list[dict[str, Any]] = []
+    context_sections: list[dict[str, Any]] = []
+    context_media: list[dict[str, Any]] = []
+
+    for rank, (score, hit, sem, rec, src_w, row) in enumerate(top, start=1):
+        att: dict[str, Any] = {
+            "vector_rowid": hit.rowid,
+            "distance": hit.distance,
+            "document_id": hit.document_id,
+            "chunk_id": hit.chunk_id,
+            "source_id": hit.source_id,
+            "modality": hit.modality,
+            "ingested_at": hit.ingested_at,
+        }
+        chunk_payload: dict[str, Any] | None = None
+        doc_payload: dict[str, Any] | None = None
+        source_payload: dict[str, Any] | None = None
+
+        if row:
+            att.update(
+                {
+                    "source_name": row.source_name,
+                    "connector_type": row.connector_type,
+                    "document_timestamp": row.doc_timestamp,
+                },
+            )
+            chunk_payload = {
+                "ordinal": row.ordinal,
+                "text": row.chunk_text,
+                "start_block_ordinal": row.start_block_ordinal,
+                "end_block_ordinal": row.end_block_ordinal,
+                "meta": row.chunk_meta,
+            }
+            doc_payload = {
+                "summary": row.doc_summary,
+                "content_type": row.doc_content_type,
+                "timestamp": row.doc_timestamp,
+                "status": row.doc_status,
+            }
+            source_payload = {
+                "id": row.source_id,
+                "name": row.source_name,
+                "connector_type": row.connector_type,
+            }
+
+        candidates.append(
+            {
+                "rank": rank,
+                "score": score,
+                "semantic_score": sem,
+                "recency_score": rec,
+                "source_weight": src_w,
+                "attribution": att,
+                "document": doc_payload,
+                "chunk": chunk_payload,
+                "source": source_payload,
+            },
+        )
+
+        if row:
+            context_sections.append(
+                {
+                    "document_id": row.document_id,
+                    "chunk_id": row.chunk_pk,
+                    "chunk_ordinal": row.ordinal,
+                    "text": row.chunk_text,
+                },
+            )
+            blocks = await load_blocks_span(
+                session,
+                row.document_id,
+                row.start_block_ordinal,
+                row.end_block_ordinal,
+            )
+            for b in blocks:
+                if b.type in ("image", "audio", "video", "document"):
+                    context_media.append(
+                        {
+                            "document_id": row.document_id,
+                            "chunk_ordinal": row.ordinal,
+                            "block_ordinal": b.ordinal,
+                            "type": b.type,
+                            "mime": b.mime,
+                            "storage_uri": b.storage_uri,
+                        },
+                    )
+
+    combined = "\n\n".join(
+        f"[{s['document_id']}#{s['chunk_ordinal']}] {s['text']}"
+        for s in context_sections
+        if s.get("text")
+    )
+
+    return {
+        "candidates": candidates,
+        "context": {
+            "combined_text": combined,
+            "sections": context_sections,
+            "media": context_media,
+        },
+        "embedding_model": settings.gemini_embedding_model,
+        "vector_candidates_considered": len(raw_hits),
+    }

package/backend/app/sqlite_ext.py

@@ -0,0 +1,36 @@
+"""Helpers to reach the real sqlite3 connection behind SQLAlchemy + aiosqlite."""
+
+from __future__ import annotations
+
+import sqlite3
+
+import aiosqlite
+import sqlite_vec
+
+
+def unwrap_sqlite3_connection(dbapi_connection: object) -> sqlite3.Connection:
+    """Return the underlying sqlite3.Connection from driver wrappers."""
+    if isinstance(dbapi_connection, aiosqlite.Connection):
+        return dbapi_connection._conn
+
+    inner = getattr(dbapi_connection, "_connection", None)
+    if inner is not None:
+        if isinstance(inner, aiosqlite.Connection):
+            return inner._conn
+        real = getattr(inner, "_conn", None)
+        if isinstance(real, sqlite3.Connection):
+            return real
+    driver = getattr(dbapi_connection, "driver_connection", None)
+    if isinstance(driver, sqlite3.Connection):
+        return driver
+    if isinstance(dbapi_connection, sqlite3.Connection):
+        return dbapi_connection
+    msg = f"Cannot resolve sqlite3.Connection from {type(dbapi_connection)!r}"
+    raise TypeError(msg)
+
+
+def load_sqlite_vec_extension(dbapi_connection: object) -> None:
+    raw = unwrap_sqlite3_connection(dbapi_connection)
+    raw.enable_load_extension(True)
+    sqlite_vec.load(raw)
+    raw.enable_load_extension(False)

package/backend/app/storage/__init__.py

@@ -0,0 +1,3 @@
+from app.storage.blobs import BlobStore
+
+__all__ = ["BlobStore"]

package/backend/app/storage/blobs.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+
+
+class BlobStore:
+    """Content-addressed files under root (e.g. data/blobs/ab/<sha256>)."""
+
+    def __init__(self, root: Path) -> None:
+        self.root = root.resolve()
+
+    def _path_for(self, sha256_hex: str) -> Path:
+        if len(sha256_hex) != 64:
+            msg = "sha256_hex must be 64 hex characters"
+            raise ValueError(msg)
+        prefix = sha256_hex[:2]
+        return self.root / prefix / sha256_hex
+
+    def write(self, data: bytes) -> tuple[str, str]:
+        """Write bytes if missing; return (sha256_hex, storage_uri blob://...)."""
+        digest = hashlib.sha256(data).hexdigest()
+        path = self._path_for(digest)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        if not path.exists():
+            path.write_bytes(data)
+        return digest, f"blob://{digest}"
+
+    def read_bytes(self, sha256_hex: str) -> bytes:
+        return self._path_for(sha256_hex).read_bytes()

package/backend/app/vectorstore/__init__.py

@@ -0,0 +1,13 @@
+from app.vectorstore.sqlite_vec_store import (
+    SqliteVecStore,
+    VectorMeta,
+    VectorSearchFilters,
+    VectorSearchResult,
+)
+
+__all__ = [
+    "SqliteVecStore",
+    "VectorMeta",
+    "VectorSearchFilters",
+    "VectorSearchResult",
+]

package/backend/app/vectorstore/sqlite_vec_store.py

@@ -0,0 +1,242 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+import sqlite_vec
+from sqlalchemy import text
+from sqlalchemy.engine import Connection
+from sqlalchemy.ext.asyncio import AsyncSession
+
+if TYPE_CHECKING:
+    from sqlalchemy.ext.asyncio import AsyncEngine
+
+
+@dataclass(frozen=True, slots=True)
+class VectorMeta:
+    """Row metadata for sqlite-vec (non-null fields at insert time)."""
+
+    document_id: str
+    chunk_id: int = 0
+    source_id: int = -1
+    modality: str = "text"
+    ingested_at: str = ""
+
+
+@dataclass(frozen=True, slots=True)
+class VectorSearchFilters:
+    document_id: str | None = None
+    source_id: int | None = None
+    modality: str | None = None
+    timestamp_min: str | None = None
+    timestamp_max: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class VectorSearchResult:
+    rowid: int
+    distance: float
+    document_id: str
+    chunk_id: int
+    source_id: int
+    modality: str
+    ingested_at: str
+
+
+class SqliteVecStore:
+    """sqlite-vec backed vector storage in the app SQLite file."""
+
+    def __init__(
+        self,
+        engine: AsyncEngine,
+        *,
+        dimension: int,
+        table_name: str = "kb_vec_embeddings",
+    ) -> None:
+        self._engine = engine
+        self._dimension = dimension
+        self._table = table_name
+
+    def _validate_vector(self, v: list[float], *, ctx: str) -> None:
+        if len(v) != self._dimension:
+            msg = f"{ctx}: expected dimension {self._dimension}, got {len(v)}"
+            raise ValueError(msg)
+
+    async def upsert(
+        self,
+        embeddings: list[list[float]],
+        metas: list[VectorMeta],
+    ) -> None:
+        if len(embeddings) != len(metas):
+            msg = "embeddings and metas must have the same length"
+            raise ValueError(msg)
+        for i, (emb, meta) in enumerate(zip(embeddings, metas, strict=True)):
+            self._validate_vector(emb, ctx=f"embeddings[{i}]")
+            if not meta.document_id:
+                raise ValueError("VectorMeta.document_id is required")
+            if not meta.ingested_at:
+                raise ValueError("VectorMeta.ingested_at is required (ISO-8601 text)")
+
+        keys: set[tuple[str, int]] = set()
+        for m in metas:
+            keys.add((m.document_id, m.chunk_id))
+
+        async with self._engine.begin() as conn:
+            await conn.run_sync(
+                self._sync_upsert_keys,
+                keys,
+                embeddings,
+                metas,
+            )
+
+    def _sync_upsert_keys(
+        self,
+        c: Connection,
+        keys: set[tuple[str, int]],
+        embeddings: list[list[float]],
+        metas: list[VectorMeta],
+    ) -> None:
+        for doc_id, chunk_id in keys:
+            c.execute(
+                text(
+                    f"DELETE FROM {self._table} "
+                    "WHERE document_id = :doc AND chunk_id = :chunk",
+                ),
+                {"doc": doc_id, "chunk": chunk_id},
+            )
+        cols = "embedding, document_id, chunk_id, source_id, modality, ingested_at"
+        vals = ":emb, :document_id, :chunk_id, :source_id, :modality, :ingested_at"
+        sql = text(f"INSERT INTO {self._table} ({cols}) VALUES ({vals})")
+        for emb, meta in zip(embeddings, metas, strict=True):
+            blob = sqlite_vec.serialize_float32(emb)
+            c.execute(
+                sql,
+                {
+                    "emb": blob,
+                    "document_id": meta.document_id,
+                    "chunk_id": meta.chunk_id,
+                    "source_id": meta.source_id,
+                    "modality": meta.modality,
+                    "ingested_at": meta.ingested_at,
+                },
+            )
+
+    async def upsert_for_session(
+        self,
+        session: AsyncSession,
+        embeddings: list[list[float]],
+        metas: list[VectorMeta],
+    ) -> None:
+        """Same as ``upsert`` but uses the session's connection (avoids SQLite lock)."""
+        if len(embeddings) != len(metas):
+            msg = "embeddings and metas must have the same length"
+            raise ValueError(msg)
+        for i, (emb, meta) in enumerate(zip(embeddings, metas, strict=True)):
+            self._validate_vector(emb, ctx=f"embeddings[{i}]")
+            if not meta.document_id:
+                raise ValueError("VectorMeta.document_id is required")
+            if not meta.ingested_at:
+                raise ValueError("VectorMeta.ingested_at is required (ISO-8601 text)")
+
+        keys: set[tuple[str, int]] = set()
+        for m in metas:
+            keys.add((m.document_id, m.chunk_id))
+
+        async_conn = await session.connection()
+        await async_conn.run_sync(
+            self._sync_upsert_keys,
+            keys,
+            embeddings,
+            metas,
+        )
+
+    async def delete_document(self, document_id: str) -> int:
+        async with self._engine.begin() as conn:
+            return await conn.run_sync(self._sync_delete_document, document_id)
+
+    def _sync_delete_document(self, c: Connection, document_id: str) -> int:
+        r = c.execute(
+            text(f"DELETE FROM {self._table} WHERE document_id = :doc"),
+            {"doc": document_id},
+        )
+        return r.rowcount or 0
+
+    async def delete_document_for_session(
+        self,
+        session: AsyncSession,
+        document_id: str,
+    ) -> int:
+        """Same as ``delete_document`` but uses the session's SQLite connection."""
+        async_conn = await session.connection()
+        return await async_conn.run_sync(self._sync_delete_document, document_id)
+
+    async def search(
+        self,
+        query_vector: list[float],
+        k: int,
+        filters: VectorSearchFilters | None = None,
+    ) -> list[VectorSearchResult]:
+        if k < 1:
+            raise ValueError("k must be >= 1")
+        self._validate_vector(query_vector, ctx="query_vector")
+        filters = filters or VectorSearchFilters()
+
+        qblob = sqlite_vec.serialize_float32(query_vector)
+        where_parts = [
+            "embedding MATCH :qvec",
+            "k = :k",
+        ]
+        params: dict[str, Any] = {"qvec": qblob, "k": k}
+
+        if filters.document_id is not None:
+            where_parts.append("document_id = :document_id")
+            params["document_id"] = filters.document_id
+        if filters.source_id is not None:
+            where_parts.append("source_id = :source_id")
+            params["source_id"] = filters.source_id
+        if filters.modality is not None:
+            where_parts.append("modality = :modality")
+            params["modality"] = filters.modality
+        if filters.timestamp_min is not None:
+            where_parts.append("ingested_at >= :ts_min")
+            params["ts_min"] = filters.timestamp_min
+        if filters.timestamp_max is not None:
+            where_parts.append("ingested_at <= :ts_max")
+            params["ts_max"] = filters.timestamp_max
+
+        where_sql = " AND ".join(where_parts)
+        stmt = text(
+            f"""
+            SELECT
+              rowid,
+              document_id,
+              chunk_id,
+              source_id,
+              modality,
+              ingested_at,
+              distance
+            FROM {self._table}
+            WHERE {where_sql}
+            """
+        )
+
+        async with self._engine.connect() as conn:
+
+            def _search(c: Connection) -> list[VectorSearchResult]:
+                rows = c.execute(stmt, params).mappings().all()
+                out: list[VectorSearchResult] = []
+                for row in rows:
+                    out.append(
+                        VectorSearchResult(
+                            rowid=int(row["rowid"]),
+                            distance=float(row["distance"]),
+                            document_id=str(row["document_id"]),
+                            chunk_id=int(row["chunk_id"]),
+                            source_id=int(row["source_id"]),
+                            modality=str(row["modality"]),
+                            ingested_at=str(row["ingested_at"]),
+                        )
+                    )
+                return out
+
+            return await conn.run_sync(_search)

package/backend/backend.egg-info/PKG-INFO

@@ -0,0 +1,18 @@
+Metadata-Version: 2.4
+Name: backend
+Version: 0.1.0
+Summary: FastAPI service (internal API behind Hono gateway)
+Requires-Python: >=3.12
+Requires-Dist: fastapi[standard]==0.113.0
+Requires-Dist: pydantic==2.8.0
+Requires-Dist: pydantic-settings>=2.4.0
+Requires-Dist: sqlalchemy[asyncio]>=2.0.36
+Requires-Dist: aiosqlite>=0.20.0
+Requires-Dist: alembic>=1.14.0
+Requires-Dist: python-json-logger>=2.0.7
+Requires-Dist: sqlite-vec>=0.1.9
+Requires-Dist: httpx>=0.27.0
+Provides-Extra: entities
+Requires-Dist: spacy>=3.7.0; extra == "entities"
+Provides-Extra: playwright
+Requires-Dist: playwright>=1.49.0; extra == "playwright"