business-stack 0.1.0

package/backend/backend.egg-info/SOURCES.txt

@@ -0,0 +1,93 @@
+pyproject.toml
+app/__init__.py
+app/config.py
+app/db.py
+app/dependencies.py
+app/logging_config.py
+app/main.py
+app/sqlite_ext.py
+app/connectors/__init__.py
+app/connectors/base.py
+app/connectors/builtins.py
+app/connectors/playwright_session.py
+app/connectors/registry.py
+app/connectors/thread_expansion/__init__.py
+app/connectors/thread_expansion/fakes.py
+app/connectors/thread_expansion/models.py
+app/connectors/thread_expansion/reddit.py
+app/connectors/thread_expansion/twitter.py
+app/middleware/__init__.py
+app/middleware/gateway_identity.py
+app/middleware/request_id.py
+app/routers/__init__.py
+app/routers/chunks.py
+app/routers/entity_extract.py
+app/routers/example.py
+app/routers/gemini_embed.py
+app/routers/health.py
+app/routers/ingestion.py
+app/routers/link_expansion.py
+app/routers/query.py
+app/routers/vectors.py
+app/schemas/__init__.py
+app/schemas/canonical.py
+app/schemas/ingest.py
+app/schemas/query.py
+app/schemas/vectors.py
+app/services/__init__.py
+app/services/retrieval_service.py
+app/services/chunking/__init__.py
+app/services/chunking/llm_boundaries.py
+app/services/chunking/schemas.py
+app/services/chunking/semantic_chunk.py
+app/services/chunking/splitters.py
+app/services/embeddings/__init__.py
+app/services/embeddings/build_inputs.py
+app/services/embeddings/gemini_api.py
+app/services/embeddings/persist.py
+app/services/embeddings/types.py
+app/services/embeddings/worker.py
+app/services/entities/__init__.py
+app/services/entities/gliner_extract.py
+app/services/entities/llm_extract.py
+app/services/entities/pipeline.py
+app/services/entities/spacy_extract.py
+app/services/entities/types.py
+app/services/ingestion/__init__.py
+app/services/ingestion/persist.py
+app/services/link_expansion/__init__.py
+app/services/link_expansion/canonical_url.py
+app/services/link_expansion/domain_policy.py
+app/services/link_expansion/html_extract.py
+app/services/link_expansion/rate_limit.py
+app/services/link_expansion/robots.py
+app/services/link_expansion/schemas.py
+app/services/link_expansion/worker.py
+app/services/normalization/__init__.py
+app/services/normalization/normalizer.py
+app/services/normalization/persist_normalized.py
+app/services/playwright_extract/__init__.py
+app/services/playwright_extract/__main__.py
+app/services/playwright_extract/extract.py
+app/storage/__init__.py
+app/storage/blobs.py
+app/vectorstore/__init__.py
+app/vectorstore/sqlite_vec_store.py
+backend.egg-info/PKG-INFO
+backend.egg-info/SOURCES.txt
+backend.egg-info/dependency_links.txt
+backend.egg-info/entry_points.txt
+backend.egg-info/requires.txt
+backend.egg-info/top_level.txt
+tests/test_chunking.py
+tests/test_entities.py
+tests/test_gemini_embed.py
+tests/test_health.py
+tests/test_ingest_raw.py
+tests/test_link_expansion.py
+tests/test_main.py
+tests/test_normalizer.py
+tests/test_playwright_extract.py
+tests/test_query.py
+tests/test_thread_expansion.py
+tests/test_vectors.py

package/backend/backend.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

package/backend/backend.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[kb.connectors]
+echo = app.connectors.builtins:EchoConnector

package/backend/backend.egg-info/requires.txt

@@ -0,0 +1,15 @@
+fastapi[standard]==0.113.0
+pydantic==2.8.0
+pydantic-settings>=2.4.0
+sqlalchemy[asyncio]>=2.0.36
+aiosqlite>=0.20.0
+alembic>=1.14.0
+python-json-logger>=2.0.7
+sqlite-vec>=0.1.9
+httpx>=0.27.0
+
+[entities]
+spacy>=3.7.0
+
+[playwright]
+playwright>=1.49.0

package/backend/backend.egg-info/top_level.txt

@@ -0,0 +1,4 @@
+app
+data
+dist
+tmp_migrate_kb

package/backend/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "@business-stack/backend",
+  "private": true,
+  "scripts": {
+    "dev": "uv run uvicorn app.main:app --reload --host 127.0.0.1 --port 8000",
+    "start": "uv run uvicorn app.main:app --host 127.0.0.1 --port 8000",
+    "build": "bun -e \"require('fs').mkdirSync('dist',{recursive:true}); require('fs').writeFileSync('dist/.buildstamp','')\"",
+    "lint": "uv run ruff check .",
+    "lint:fix": "uv run ruff check --fix . && uv run ruff format .",
+    "typecheck": "bun -e \"process.exit(0)\"",
+    "test": "uv run pytest",
+    "db:migrate": "uv run alembic upgrade head",
+    "clean": "bun x rimraf@6 .pytest_cache"
+  }
+}

package/backend/pyproject.toml

@@ -0,0 +1,52 @@
+[project]
+name = "backend"
+version = "0.1.0"
+description = "FastAPI service (internal API behind Hono gateway)"
+requires-python = ">=3.12"
+dependencies = [
+  "fastapi[standard]==0.113.0",
+  "pydantic==2.8.0",
+  "pydantic-settings>=2.4.0",
+  "sqlalchemy[asyncio]>=2.0.36",
+  "aiosqlite>=0.20.0",
+  "alembic>=1.14.0",
+  "python-json-logger>=2.0.7",
+  "sqlite-vec>=0.1.9",
+  "httpx>=0.27.0",
+]
+
+[project.optional-dependencies]
+entities = [
+  "spacy>=3.7.0",
+]
+playwright = [
+  "playwright>=1.49.0",
+]
+
+[dependency-groups]
+dev = [
+  "httpx>=0.27.0",
+  "pytest>=8.3.0",
+  "ruff>=0.8.0",
+]
+
+[project.entry-points."kb.connectors"]
+echo = "app.connectors.builtins:EchoConnector"
+
+[build-system]
+requires = ["setuptools>=61"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["."]
+exclude = ["tests*", "alembic*"]
+
+[tool.ruff]
+target-version = "py312"
+line-length = 88
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

package/backend/tests/conftest.py

@@ -0,0 +1,40 @@
+import os
+from pathlib import Path
+
+import pytest
+from alembic.config import Config
+from fastapi.testclient import TestClient
+
+from alembic import command
+from app.config import clear_settings_cache
+from app.main import create_app
+
+BACKEND_ROOT = Path(__file__).resolve().parents[1]
+
+
+def _alembic_upgrade_head() -> None:
+    prev = os.getcwd()
+    os.chdir(BACKEND_ROOT)
+    try:
+        cfg = Config(str(BACKEND_ROOT / "alembic.ini"))
+        command.upgrade(cfg, "head")
+    finally:
+        os.chdir(prev)
+
+
+@pytest.fixture
+def client(tmp_path, monkeypatch) -> TestClient:
+    monkeypatch.setenv("DATA_DIR", str(tmp_path))
+    clear_settings_cache()
+    with TestClient(create_app()) as test_client:
+        yield test_client
+
+
+@pytest.fixture
+def migrated_client(tmp_path, monkeypatch) -> TestClient:
+    monkeypatch.setenv("DATA_DIR", str(tmp_path))
+    clear_settings_cache()
+    _alembic_upgrade_head()
+    clear_settings_cache()
+    with TestClient(create_app()) as test_client:
+        yield test_client

package/backend/tests/test_chat.py

@@ -0,0 +1,92 @@
+from unittest.mock import patch
+
+from fastapi.testclient import TestClient
+
+from app.config import clear_settings_cache
+
+
+def test_chat_requires_gateway_user(migrated_client: TestClient) -> None:
+    response = migrated_client.get("/chat/sessions")
+    assert response.status_code == 401
+
+
+def test_chat_session_crud(migrated_client: TestClient) -> None:
+    headers = {"x-user-id": "user-chat-crud"}
+    create = migrated_client.post("/chat/sessions", json={}, headers=headers)
+    assert create.status_code == 200
+    session_id = create.json()["id"]
+
+    listed = migrated_client.get("/chat/sessions", headers=headers)
+    assert listed.status_code == 200
+    ids = {row["id"] for row in listed.json()}
+    assert session_id in ids
+
+    msgs = migrated_client.get(
+        f"/chat/sessions/{session_id}/messages",
+        headers=headers,
+    )
+    assert msgs.status_code == 200
+    assert msgs.json() == []
+
+    deleted = migrated_client.delete(
+        f"/chat/sessions/{session_id}",
+        headers=headers,
+    )
+    assert deleted.status_code == 200
+
+    gone = migrated_client.get(
+        f"/chat/sessions/{session_id}/messages",
+        headers=headers,
+    )
+    assert gone.status_code == 404
+
+
+def test_chat_complete_persists_messages(
+    migrated_client: TestClient,
+    monkeypatch,
+) -> None:
+    monkeypatch.setenv("GEMINI_API_KEY", "test-key-for-chat-test")
+    clear_settings_cache()
+    headers = {"x-user-id": "user-chat-complete"}
+    sid = migrated_client.post("/chat/sessions", json={}, headers=headers).json()[
+        "id"
+    ]
+
+    async def fake_retrieval(*_args, **_kwargs):
+        return {
+            "candidates": [],
+            "context": {
+                "combined_text": "stub context",
+                "sections": [],
+                "media": [],
+            },
+            "embedding_model": "stub-model",
+            "vector_candidates_considered": 0,
+        }
+
+    async def fake_gemini(*_args, **_kwargs):
+        return ("stub assistant reply", "model")
+
+    with (
+        patch("app.routers.chat.run_retrieval", side_effect=fake_retrieval),
+        patch("app.routers.chat.gemini_chat_reply", side_effect=fake_gemini),
+    ):
+        complete = migrated_client.post(
+            f"/chat/sessions/{sid}/complete",
+            json={"message": "hello from test"},
+            headers=headers,
+        )
+    assert complete.status_code == 200
+    body = complete.json()
+    assert body["reply"] == "stub assistant reply"
+    assert body["reply_source"] == "model"
+
+    rows = migrated_client.get(
+        f"/chat/sessions/{sid}/messages",
+        headers=headers,
+    ).json()
+    assert len(rows) == 2
+    assert rows[0]["role"] == "user"
+    assert rows[0]["content"] == "hello from test"
+    assert rows[1]["role"] == "assistant"
+    assert rows[1]["content"] == "stub assistant reply"

package/backend/tests/test_chunking.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+import json
+import sqlite3
+from datetime import UTC, datetime
+
+from fastapi.testclient import TestClient
+
+from app.config import get_settings
+from app.services.chunking.splitters import (
+    extract_thread_message_items,
+    structure_chunks_for_text_block,
+    thread_chunks_from_envelope,
+)
+
+
+def test_markdown_headings_produce_multiple_chunks() -> None:
+    text = "# Intro\n\nFirst para.\n\n## Details\n\nSecond para."
+    chunks = structure_chunks_for_text_block(
+        text,
+        2,
+        use_llm=False,
+        llm_sections=None,
+    )
+    assert len(chunks) >= 2
+    assert all(c.start_block_ordinal == 2 for c in chunks)
+    assert chunks[0].meta.get("strategy") == "markdown_section"
+
+
+def test_paragraph_split_default() -> None:
+    text = "One block.\n\nTwo block.\n\nThree."
+    chunks = structure_chunks_for_text_block(
+        text,
+        0,
+        use_llm=False,
+        llm_sections=None,
+    )
+    assert len(chunks) == 3
+    assert chunks[1].meta.get("strategy") == "paragraph"
+
+
+def test_weak_structure_single_chunk_without_llm() -> None:
+    text = "x" * 500
+    chunks = structure_chunks_for_text_block(
+        text,
+        1,
+        use_llm=False,
+        llm_sections=None,
+    )
+    assert len(chunks) == 1
+    assert chunks[0].meta.get("strategy") == "fallback_single"
+
+
+def test_llm_sections_used_when_provided() -> None:
+    chunks = structure_chunks_for_text_block(
+        "x" * 900,
+        0,
+        use_llm=True,
+        llm_sections=["part a", "part b"],
+    )
+    assert len(chunks) == 2
+    assert chunks[0].meta.get("strategy") == "llm_section"
+
+
+def test_thread_envelope_metadata() -> None:
+    env = {
+        "payload": [
+            {"text": "hello"},
+            {"text": "world"},
+        ],
+        "metadata": {"thread_id": "thr-9"},
+    }
+    drafts = thread_chunks_from_envelope(env)
+    assert len(drafts) == 2
+    assert drafts[0].meta.get("thread_id") == "thr-9"
+    assert drafts[1].meta.get("message_index") == 1
+
+
+def test_messages_key_in_payload_dict() -> None:
+    env = {
+        "payload": {
+            "messages": [{"body": "a"}, {"body": "b"}],
+            "thread_id": "abc",
+        },
+        "metadata": {},
+    }
+    items = extract_thread_message_items(env)
+    assert items is not None
+    assert len(items) == 2
+
+
+def test_rebuild_chunks_endpoint(migrated_client: TestClient) -> None:
+    h = {"x-user-id": "u1"}
+    r = migrated_client.post(
+        "/ingest/raw?connector=echo",
+        headers=h,
+        json={
+            "source": "s",
+            "timestamp": datetime.now(UTC).isoformat(),
+            "content_type": "text",
+            "payload": "# Title\n\nAlpha.\n\nBeta.",
+            "metadata": {},
+        },
+    )
+    assert r.status_code == 200
+    doc_id = r.json()["document_id"]
+
+    ch = migrated_client.post(
+        f"/ingest/documents/{doc_id}/chunks",
+        headers=h,
+        json={"use_llm_weak_structure": False},
+    )
+    assert ch.status_code == 200
+    data = ch.json()
+    assert data["chunks_written"] >= 1
+
+    settings = get_settings()
+    db_path = (settings.data_dir / settings.sqlite_filename).resolve()
+    conn = sqlite3.connect(str(db_path))
+    rows = conn.execute(
+        "SELECT ordinal, start_block_ordinal, end_block_ordinal, meta "
+        "FROM document_chunks WHERE document_id = ? ORDER BY ordinal",
+        (doc_id,),
+    ).fetchall()
+    conn.close()
+    assert len(rows) >= 1
+    meta0 = json.loads(rows[0][3]) if rows[0][3] else {}
+    assert meta0.get("strategy") in (
+        "markdown_section",
+        "paragraph",
+        "fallback_single",
+    )

package/backend/tests/test_entities.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+import asyncio
+import os
+import sqlite3
+from datetime import UTC, datetime
+from pathlib import Path
+from unittest.mock import AsyncMock, patch
+
+from alembic.config import Config
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+
+from alembic import command
+from app.config import Settings, clear_settings_cache, get_settings
+from app.services.entities.pipeline import extract_and_store_entities_for_document
+from app.services.entities.types import ExtractedMention
+
+
+def test_merge_mentions_keeps_highest_confidence() -> None:
+    from app.services.entities.pipeline import _merge_mentions
+
+    m = _merge_mentions(
+        [
+            ExtractedMention("Acme", "company", 0.5),
+            ExtractedMention("acme", "company", 0.9),
+            ExtractedMention("Acme", "person", 0.8),
+        ],
+    )
+    by_type = {(x.name.lower(), x.type): x for x in m}
+    assert by_type[("acme", "company")].confidence == 0.9
+    assert ("acme", "person") in by_type
+
+
+def test_entity_extract_endpoint_accepts(migrated_client, monkeypatch) -> None:
+    monkeypatch.setenv("GEMINI_API_KEY", "x")
+    clear_settings_cache()
+    h = {"x-user-id": "u1"}
+    ing = migrated_client.post(
+        "/ingest/raw?connector=echo",
+        headers=h,
+        json={
+            "source": "s",
+            "timestamp": datetime.now(UTC).isoformat(),
+            "content_type": "text",
+            "payload": "Alice at Acme in Boston",
+            "metadata": {},
+        },
+    )
+    doc_id = ing.json()["document_id"]
+    migrated_client.post(
+        f"/ingest/documents/{doc_id}/chunks",
+        headers=h,
+        json={},
+    )
+    mock_job = AsyncMock()
+    with patch(
+        "app.routers.entity_extract.run_entity_extraction_job",
+        mock_job,
+    ):
+        r = migrated_client.post(
+            f"/ingest/documents/{doc_id}/entities",
+            headers=h,
+        )
+    assert r.status_code == 200
+    mock_job.assert_called_once()
+
+
+def test_extract_persists_mentions_and_cooccurrence(tmp_path, monkeypatch) -> None:
+    monkeypatch.setenv("DATA_DIR", str(tmp_path))
+    clear_settings_cache()
+
+    backend = Path(__file__).resolve().parents[1]
+    prev = os.getcwd()
+    os.chdir(str(backend))
+    try:
+        command.upgrade(Config(str(backend / "alembic.ini")), "head")
+    finally:
+        os.chdir(prev)
+    clear_settings_cache()
+
+    settings = Settings()
+    engine = create_async_engine(settings.sqlalchemy_database_url)
+    factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+    doc_id = "doc-ent-1"
+
+    async def _seed() -> None:
+        async with factory() as session:
+            await session.execute(
+                text("INSERT INTO sources (name, connector_type) VALUES ('s','s')"),
+            )
+            sid = int(
+                (
+                    await session.execute(text("SELECT id FROM sources LIMIT 1"))
+                ).scalar_one(),
+            )
+            await session.execute(
+                text(
+                    "INSERT INTO documents "
+                    "(id, source_id, timestamp, content_type, raw_content, "
+                    "summary, status) VALUES "
+                    "(:id, :sid, :ts, 'text', '{}', 'x', 'ok')",
+                ),
+                {"id": doc_id, "sid": sid, "ts": datetime.now(UTC).isoformat()},
+            )
+            await session.execute(
+                text(
+                    "INSERT INTO document_chunks "
+                    "(document_id, ordinal, text, start_block_ordinal, "
+                    "end_block_ordinal, meta) VALUES "
+                    "(:d, 0, 'one', 0, 0, NULL)",
+                ),
+                {"d": doc_id},
+            )
+            await session.execute(
+                text(
+                    "INSERT INTO document_chunks "
+                    "(document_id, ordinal, text, start_block_ordinal, "
+                    "end_block_ordinal, meta) VALUES "
+                    "(:d, 1, 'two', 0, 0, NULL)",
+                ),
+                {"d": doc_id},
+            )
+            await session.commit()
+
+    asyncio.run(_seed())
+
+    async def _fake_extract(text: str, _settings: Settings):
+        if "one" in text:
+            return [
+                ExtractedMention("Person A", "person", 0.9),
+                ExtractedMention("Acme Corp", "company", 0.85),
+            ]
+        return [
+            ExtractedMention("Person A", "person", 0.88),
+            ExtractedMention("Boston", "location", 0.8),
+        ]
+
+    async def _run() -> None:
+        with patch(
+            "app.services.entities.pipeline.extract_mentions_for_chunk_text",
+            new=_fake_extract,
+        ):
+            async with factory() as session:
+                nm, nc = await extract_and_store_entities_for_document(
+                    session,
+                    document_id=doc_id,
+                    settings=get_settings(),
+                )
+                await session.commit()
+        assert nm >= 4
+        assert nc == 3
+
+    asyncio.run(_run())
+
+    db_path = (get_settings().data_dir / get_settings().sqlite_filename).resolve()
+    conn = sqlite3.connect(str(db_path))
+    mentions = conn.execute(
+        "SELECT COUNT(*) FROM entity_mentions WHERE document_id = ?",
+        (doc_id,),
+    ).fetchone()[0]
+    co = conn.execute(
+        "SELECT COUNT(*) FROM entity_cooccurrence WHERE document_id = ?",
+        (doc_id,),
+    ).fetchone()[0]
+    conn.close()
+    assert mentions == 4
+    assert co == 3
+
+    asyncio.run(engine.dispose())