business-stack 0.1.0

package/backend/app/schemas/vectors.py

@@ -0,0 +1,56 @@
+from pydantic import BaseModel, Field
+
+from app.vectorstore import VectorSearchFilters
+
+
+class VectorMetaPayload(BaseModel):
+    document_id: str
+    chunk_id: int = 0
+    source_id: int = -1
+    modality: str = "text"
+    ingested_at: str = Field(
+        ...,
+        description="ISO-8601 timestamp for filterable range queries on ingested_at",
+    )
+
+
+class VectorUpsertRequest(BaseModel):
+    embeddings: list[list[float]]
+    metas: list[VectorMetaPayload]
+
+
+class VectorSearchFiltersPayload(BaseModel):
+    document_id: str | None = None
+    source_id: int | None = None
+    modality: str | None = None
+    timestamp_min: str | None = None
+    timestamp_max: str | None = None
+
+    def to_filters(self) -> VectorSearchFilters:
+        return VectorSearchFilters(
+            document_id=self.document_id,
+            source_id=self.source_id,
+            modality=self.modality,
+            timestamp_min=self.timestamp_min,
+            timestamp_max=self.timestamp_max,
+        )
+
+
+class VectorSearchRequest(BaseModel):
+    query_vector: list[float]
+    k: int = Field(default=10, ge=1, le=500)
+    filters: VectorSearchFiltersPayload | None = None
+
+
+class VectorSearchHit(BaseModel):
+    rowid: int
+    distance: float
+    document_id: str
+    chunk_id: int
+    source_id: int
+    modality: str
+    ingested_at: str
+
+
+class VectorSearchResponse(BaseModel):
+    results: list[VectorSearchHit]

package/backend/app/services/chat_store.py

@@ -0,0 +1,152 @@
+from __future__ import annotations
+
+import json
+import uuid
+from datetime import UTC, datetime
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+
+def _utc_now_iso() -> str:
+    return datetime.now(UTC).isoformat().replace("+00:00", "Z")
+
+
+async def create_chat_session(
+    session: AsyncSession,
+    *,
+    user_id: str,
+    title: str | None,
+) -> str:
+    sid = str(uuid.uuid4())
+    now = _utc_now_iso()
+    await session.execute(
+        text(
+            "INSERT INTO chat_sessions (id, user_id, title, created_at, updated_at) "
+            "VALUES (:id, :uid, :title, :ca, :ua)",
+        ),
+        {"id": sid, "uid": user_id, "title": title, "ca": now, "ua": now},
+    )
+    return sid
+
+
+async def list_chat_sessions(
+    session: AsyncSession,
+    *,
+    user_id: str,
+    limit: int = 100,
+) -> list[dict[str, str | None]]:
+    r = await session.execute(
+        text(
+            "SELECT id, title, created_at, updated_at FROM chat_sessions "
+            "WHERE user_id = :u ORDER BY updated_at DESC LIMIT :lim",
+        ),
+        {"u": user_id, "lim": limit},
+    )
+    rows = r.mappings().all()
+    return [dict(row) for row in rows]
+
+
+async def get_session_owned(
+    session: AsyncSession,
+    *,
+    session_id: str,
+    user_id: str,
+) -> dict[str, str | None] | None:
+    r = await session.execute(
+        text(
+            "SELECT id, user_id, title, created_at, updated_at FROM chat_sessions "
+            "WHERE id = :id AND user_id = :u LIMIT 1",
+        ),
+        {"id": session_id, "u": user_id},
+    )
+    row = r.mappings().first()
+    return dict(row) if row else None
+
+
+async def touch_session_updated(session: AsyncSession, *, session_id: str) -> None:
+    now = _utc_now_iso()
+    await session.execute(
+        text("UPDATE chat_sessions SET updated_at = :ua WHERE id = :id"),
+        {"ua": now, "id": session_id},
+    )
+
+
+async def maybe_set_title_from_first_message(
+    session: AsyncSession,
+    *,
+    session_id: str,
+    snippet: str,
+) -> None:
+    r = await session.execute(
+        text("SELECT title FROM chat_sessions WHERE id = :id LIMIT 1"),
+        {"id": session_id},
+    )
+    row = r.first()
+    if not row or row[0]:
+        return
+    t = snippet.strip().replace("\n", " ")[:80]
+    if not t:
+        return
+    await session.execute(
+        text("UPDATE chat_sessions SET title = :t WHERE id = :id"),
+        {"t": t, "id": session_id},
+    )
+
+
+async def append_chat_message(
+    session: AsyncSession,
+    *,
+    session_id: str,
+    role: str,
+    content: str,
+    meta: dict | None = None,
+) -> int:
+    now = _utc_now_iso()
+    meta_s = json.dumps(meta, default=str) if meta else None
+    await session.execute(
+        text(
+            "INSERT INTO chat_messages (session_id, role, content, meta_json, created_at) "
+            "VALUES (:sid, :role, :content, :meta, :ca)",
+        ),
+        {
+            "sid": session_id,
+            "role": role,
+            "content": content,
+            "meta": meta_s,
+            "ca": now,
+        },
+    )
+    r = await session.execute(text("SELECT last_insert_rowid()"))
+    rid = r.scalar_one()
+    return int(rid)
+
+
+async def list_chat_messages(
+    session: AsyncSession,
+    *,
+    session_id: str,
+    limit: int = 200,
+) -> list[dict[str, str | int | None]]:
+    r = await session.execute(
+        text(
+            "SELECT id, role, content, meta_json, created_at FROM chat_messages "
+            "WHERE session_id = :sid ORDER BY id ASC LIMIT :lim",
+        ),
+        {"sid": session_id, "lim": limit},
+    )
+    rows = r.mappings().all()
+    return [dict(row) for row in rows]
+
+
+async def delete_chat_session(
+    session: AsyncSession,
+    *,
+    session_id: str,
+    user_id: str,
+) -> bool:
+    r = await session.execute(
+        text("DELETE FROM chat_sessions WHERE id = :id AND user_id = :u"),
+        {"id": session_id, "u": user_id},
+    )
+    return (r.rowcount or 0) > 0

package/backend/app/services/chunking/__init__.py

@@ -0,0 +1,3 @@
+from app.services.chunking.semantic_chunk import rebuild_semantic_chunks
+
+__all__ = ["rebuild_semantic_chunks"]

package/backend/app/services/chunking/llm_boundaries.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+async def suggest_sections_via_llm(
+    text: str,
+    *,
+    api_key: str,
+    model: str,
+    timeout_s: float = 60.0,
+) -> list[str] | None:
+    """Ask the model for semantic section splits; returns section strings or None."""
+    if not text.strip():
+        return []
+    prompt = (
+        "Split the following document into semantic sections for retrieval. "
+        "Prefer coherent topical units. Do not use fixed token sizes. "
+        'Return JSON only: {"sections": ["...", "..."]}\n\n---\n' + text[:48_000]
+    )
+    url = "https://api.openai.com/v1/chat/completions"
+    body: dict[str, Any] = {
+        "model": model,
+        "temperature": 0.2,
+        "response_format": {"type": "json_object"},
+        "messages": [
+            {
+                "role": "system",
+                "content": (
+                    "You output only valid JSON with a sections array of strings."
+                ),
+            },
+            {"role": "user", "content": prompt},
+        ],
+    }
+    try:
+        async with httpx.AsyncClient(timeout=timeout_s) as client:
+            r = await client.post(
+                url,
+                headers={
+                    "Authorization": f"Bearer {api_key}",
+                    "Content-Type": "application/json",
+                },
+                json=body,
+            )
+            r.raise_for_status()
+            data = r.json()
+        content = data["choices"][0]["message"]["content"]
+        parsed = json.loads(content)
+        sections = parsed.get("sections")
+        if not isinstance(sections, list):
+            return None
+        out = [str(s).strip() for s in sections if str(s).strip()]
+        return out or None
+    except Exception:
+        logger.exception("LLM chunk boundary request failed")
+        return None

package/backend/app/services/chunking/schemas.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class SemanticChunkDraft(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    text: str
+    start_block_ordinal: int = Field(ge=0)
+    end_block_ordinal: int = Field(ge=0)
+    meta: dict[str, Any] = Field(default_factory=dict)
+
+
+class RebuildChunksRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    use_llm_weak_structure: bool = Field(
+        default=False,
+        description="If true and structure is weak, call LLM for section boundaries",
+    )
+
+
+class RebuildChunksResponse(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    document_id: str
+    chunks_written: int

package/backend/app/services/chunking/semantic_chunk.py

@@ -0,0 +1,178 @@
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass
+from typing import Any
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.config import Settings
+from app.services.chunking.llm_boundaries import suggest_sections_via_llm
+from app.services.chunking.schemas import SemanticChunkDraft
+from app.services.chunking.splitters import (
+    split_paragraphs,
+    structure_chunks_for_text_block,
+    thread_chunks_from_envelope,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class _BlockRow:
+    ordinal: int
+    type: str
+    meta: dict[str, Any]
+
+
+async def _load_blocks(session: AsyncSession, document_id: str) -> list[_BlockRow]:
+    r = await session.execute(
+        text(
+            "SELECT ordinal, type, meta FROM content_blocks "
+            "WHERE document_id = :d ORDER BY ordinal",
+        ),
+        {"d": document_id},
+    )
+    out: list[_BlockRow] = []
+    for row in r.fetchall():
+        meta: dict[str, Any] = {}
+        if row[2]:
+            try:
+                parsed = json.loads(row[2])
+                if isinstance(parsed, dict):
+                    meta = parsed
+            except json.JSONDecodeError:
+                pass
+        out.append(_BlockRow(ordinal=int(row[0]), type=str(row[1]), meta=meta))
+    return out
+
+
+async def _load_envelope(session: AsyncSession, document_id: str) -> dict[str, Any]:
+    r = await session.execute(
+        text("SELECT raw_content FROM documents WHERE id = :id LIMIT 1"),
+        {"id": document_id},
+    )
+    row = r.first()
+    if row is None or row[0] is None:
+        return {}
+    try:
+        data = json.loads(row[0])
+        return data if isinstance(data, dict) else {}
+    except json.JSONDecodeError:
+        return {}
+
+
+def _align_thread_chunks_to_blocks(
+    drafts: list[SemanticChunkDraft],
+    text_blocks: list[_BlockRow],
+) -> list[SemanticChunkDraft]:
+    if not drafts:
+        return []
+    if not text_blocks:
+        return [
+            d.model_copy(
+                update={"start_block_ordinal": 0, "end_block_ordinal": 0},
+            )
+            for d in drafts
+        ]
+    if len(drafts) == len(text_blocks):
+        return [
+            d.model_copy(
+                update={
+                    "start_block_ordinal": b.ordinal,
+                    "end_block_ordinal": b.ordinal,
+                },
+            )
+            for d, b in zip(drafts, text_blocks, strict=True)
+        ]
+    max_ord = max(b.ordinal for b in text_blocks) if text_blocks else 0
+    return [
+        d.model_copy(
+            update={
+                "start_block_ordinal": 0,
+                "end_block_ordinal": max_ord,
+            },
+        )
+        for d in drafts
+    ]
+
+
+async def rebuild_semantic_chunks(
+    session: AsyncSession,
+    *,
+    document_id: str,
+    use_llm_weak_structure: bool,
+    settings: Settings,
+) -> int:
+    """Replace document_chunks for this document using structure-aware splitting."""
+    envelope = await _load_envelope(session, document_id)
+    blocks = await _load_blocks(session, document_id)
+    text_blocks = [b for b in blocks if b.type == "text" and _block_text_from_row(b)]
+
+    thread_drafts = thread_chunks_from_envelope(envelope)
+    drafts: list[SemanticChunkDraft] = []
+
+    if thread_drafts:
+        drafts = _align_thread_chunks_to_blocks(thread_drafts, text_blocks)
+        logger.debug(
+            "document %s: %s thread chunks",
+            document_id,
+            len(drafts),
+        )
+    else:
+        for b in blocks:
+            if b.type != "text":
+                continue
+            t = _block_text_from_row(b)
+            if not t:
+                continue
+            llm_sections: list[str] | None = None
+            if (
+                use_llm_weak_structure
+                and settings.openai_api_key
+                and len(split_paragraphs(t)) <= 1
+                and len(t) > 800
+            ):
+                llm_sections = await suggest_sections_via_llm(
+                    t,
+                    api_key=settings.openai_api_key,
+                    model=settings.chunk_llm_model,
+                )
+            drafts.extend(
+                structure_chunks_for_text_block(
+                    t,
+                    b.ordinal,
+                    use_llm=use_llm_weak_structure,
+                    llm_sections=llm_sections,
+                ),
+            )
+
+    await session.execute(
+        text("DELETE FROM document_chunks WHERE document_id = :d"),
+        {"d": document_id},
+    )
+    for ordinal, d in enumerate(drafts):
+        await session.execute(
+            text(
+                "INSERT INTO document_chunks "
+                "(document_id, ordinal, text, start_block_ordinal, "
+                "end_block_ordinal, meta) "
+                "VALUES (:did, :ord, :txt, :sb, :eb, :meta)",
+            ),
+            {
+                "did": document_id,
+                "ord": ordinal,
+                "txt": d.text,
+                "sb": d.start_block_ordinal,
+                "eb": d.end_block_ordinal,
+                "meta": json.dumps(d.meta) if d.meta else None,
+            },
+        )
+    return len(drafts)
+
+
+def _block_text_from_row(b: _BlockRow) -> str:
+    t = b.meta.get("text")
+    return t.strip() if isinstance(t, str) else ""

package/backend/app/services/chunking/splitters.py

@@ -0,0 +1,214 @@
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+from app.services.chunking.schemas import SemanticChunkDraft
+
+_HEADING_LINE = re.compile(r"^(#{1,6})\s+(.+)$", re.MULTILINE)
+
+
+def _message_body(item: dict[str, Any]) -> str:
+    for k in ("text", "body", "content", "message", "markdown", "tweet"):
+        v = item.get(k)
+        if isinstance(v, str) and v.strip():
+            return v.strip()
+    return json.dumps(item, ensure_ascii=False)[:16_000]
+
+
+def extract_thread_message_items(
+    envelope: dict[str, Any],
+) -> list[dict[str, Any]] | None:
+    """Detect chat/thread payloads; return list of {thread_id, item, index}."""
+    p = envelope.get("payload")
+    md = envelope.get("metadata") if isinstance(envelope.get("metadata"), dict) else {}
+    tid = md.get("thread_id") or md.get("thread_ts") or md.get("conversation_id")
+
+    if isinstance(p, list) and p and all(isinstance(x, dict) for x in p):
+        return [
+            {"thread_id": tid, "item": x, "index": i, "role": x.get("role")}
+            for i, x in enumerate(p)
+        ]
+
+    if isinstance(p, dict):
+        inner_tid = p.get("thread_id") or tid
+        for key in ("messages", "thread", "replies", "posts"):
+            msgs = p.get(key)
+            if (
+                isinstance(msgs, list)
+                and msgs
+                and all(isinstance(x, dict) for x in msgs)
+            ):
+                return [
+                    {
+                        "thread_id": inner_tid or p.get("thread_id"),
+                        "item": x,
+                        "index": i,
+                        "role": x.get("role"),
+                    }
+                    for i, x in enumerate(msgs)
+                ]
+    return None
+
+
+def thread_chunks_from_envelope(
+    envelope: dict[str, Any],
+    *,
+    default_block_ordinal: int = 0,
+) -> list[SemanticChunkDraft]:
+    items = extract_thread_message_items(envelope)
+    if not items:
+        return []
+    out: list[SemanticChunkDraft] = []
+    for row in items:
+        item = row["item"]
+        if not isinstance(item, dict):
+            continue
+        body = _message_body(item)
+        if not body:
+            continue
+        tid = row.get("thread_id")
+        meta = {
+            "strategy": "thread_message",
+            "message_index": row["index"],
+            "role": row.get("role"),
+        }
+        if tid is not None:
+            meta["thread_id"] = str(tid)
+        if item.get("id") is not None:
+            meta["message_id"] = str(item["id"])
+        out.append(
+            SemanticChunkDraft(
+                text=body,
+                start_block_ordinal=default_block_ordinal,
+                end_block_ordinal=default_block_ordinal,
+                meta=meta,
+            ),
+        )
+    return out
+
+
+def split_markdown_sections(text: str) -> list[tuple[str, dict[str, Any]]]:
+    """Split on ATX headings; each chunk includes its heading line."""
+    if not text.strip():
+        return []
+    if not _HEADING_LINE.search(text):
+        return []
+
+    parts: list[tuple[str, dict[str, Any]]] = []
+    matches = list(_HEADING_LINE.finditer(text))
+    if not matches:
+        return [(text.strip(), {"strategy": "markdown_section", "section_index": 0})]
+
+    preamble = text[: matches[0].start()].strip()
+    if preamble:
+        parts.append(
+            (
+                preamble,
+                {
+                    "strategy": "markdown_section",
+                    "section_index": 0,
+                    "heading": None,
+                },
+            ),
+        )
+
+    for i, m in enumerate(matches):
+        start = m.start()
+        end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
+        chunk = text[start:end].strip()
+        if not chunk:
+            continue
+        level = len(m.group(1))
+        title = m.group(2).strip()
+        parts.append(
+            (
+                chunk,
+                {
+                    "strategy": "markdown_section",
+                    "section_index": len(parts),
+                    "heading_level": level,
+                    "heading": title,
+                },
+            ),
+        )
+    return parts
+
+
+def split_paragraphs(text: str) -> list[str]:
+    raw = text.strip()
+    if not raw:
+        return []
+    blocks = re.split(r"\n\s*\n+", raw)
+    return [b.strip() for b in blocks if b.strip()]
+
+
+def structure_chunks_for_text_block(
+    text: str,
+    block_ordinal: int,
+    *,
+    use_llm: bool,
+    llm_sections: list[str] | None,
+) -> list[SemanticChunkDraft]:
+    """Prefer markdown sections, then paragraphs; optional LLM replaces weak splits."""
+    text = text.strip()
+    if not text:
+        return []
+
+    md_parts = split_markdown_sections(text)
+    if len(md_parts) > 1 or (
+        len(md_parts) == 1 and md_parts[0][1].get("heading") is not None
+    ):
+        return [
+            SemanticChunkDraft(
+                text=seg,
+                start_block_ordinal=block_ordinal,
+                end_block_ordinal=block_ordinal,
+                meta=meta,
+            )
+            for seg, meta in md_parts
+        ]
+
+    paras = split_paragraphs(text)
+    weak = len(paras) <= 1 and len(text) > 400
+
+    if weak and use_llm and llm_sections:
+        return [
+            SemanticChunkDraft(
+                text=s.strip(),
+                start_block_ordinal=block_ordinal,
+                end_block_ordinal=block_ordinal,
+                meta={
+                    "strategy": "llm_section",
+                    "section_index": i,
+                },
+            )
+            for i, s in enumerate(llm_sections)
+            if s.strip()
+        ]
+
+    if len(paras) <= 1:
+        return [
+            SemanticChunkDraft(
+                text=text,
+                start_block_ordinal=block_ordinal,
+                end_block_ordinal=block_ordinal,
+                meta={
+                    "strategy": "fallback_single",
+                    "reason": "weak_structure_no_llm"
+                    if weak and not use_llm
+                    else "short_text",
+                },
+            ),
+        ]
+
+    return [
+        SemanticChunkDraft(
+            text=p,
+            start_block_ordinal=block_ordinal,
+            end_block_ordinal=block_ordinal,
+            meta={"strategy": "paragraph", "paragraph_index": i},
+        )
+        for i, p in enumerate(paras)
+    ]