business-stack 0.1.0

package/backend/tests/test_gemini_embed.py

@@ -0,0 +1,224 @@
+from __future__ import annotations
+
+import asyncio
+import sqlite3
+import uuid
+from datetime import UTC, datetime
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import httpx
+from fastapi.testclient import TestClient
+
+from app.config import clear_settings_cache, get_settings
+from app.services.embeddings.gemini_api import batch_embed_contents
+from app.services.embeddings.types import InlineDataPart, TextPart
+
+
+def _vec1536() -> list[float]:
+    v = [0.0] * 1536
+    v[0] = 0.25
+    return v
+
+
+def test_batch_embed_contents_parses_response(monkeypatch) -> None:
+    monkeypatch.setenv("VECTOR_EMBEDDING_DIM", "1536")
+    clear_settings_cache()
+    settings = get_settings()
+
+    async def handler(request: httpx.Request) -> httpx.Response:
+        body = {
+            "embeddings": [
+                {"values": _vec1536()},
+                {"values": _vec1536()},
+            ],
+        }
+        return httpx.Response(200, json=body)
+
+    transport = httpx.MockTransport(handler)
+
+    async def _run() -> None:
+        async with httpx.AsyncClient(transport=transport) as client:
+            out = await batch_embed_contents(
+                api_key="k",
+                model="gemini-embedding-001",
+                contents=[[TextPart("a")], [TextPart("b"), TextPart("c")]],
+                settings=settings,
+                client=client,
+            )
+        assert len(out) == 2
+        assert len(out[0]) == 1536
+
+    asyncio.run(_run())
+
+
+def test_embed_endpoint_503_without_key(migrated_client: TestClient) -> None:
+    clear_settings_cache()
+    h = {"x-user-id": "u1"}
+    r = migrated_client.post(
+        "/ingest/documents/x/embed",
+        headers=h,
+        json={"multimodal": False},
+    )
+    assert r.status_code == 503
+
+
+def test_embed_endpoint_accepts_with_key(
+    migrated_client: TestClient,
+    monkeypatch,
+) -> None:
+    monkeypatch.setenv("GEMINI_API_KEY", "test-key")
+    clear_settings_cache()
+
+    h = {"x-user-id": "u1"}
+    ing = migrated_client.post(
+        "/ingest/raw?connector=echo",
+        headers=h,
+        json={
+            "source": "s",
+            "timestamp": datetime.now(UTC).isoformat(),
+            "content_type": "text",
+            "payload": "hello chunk",
+            "metadata": {},
+        },
+    )
+    doc_id = ing.json()["document_id"]
+    ch = migrated_client.post(
+        f"/ingest/documents/{doc_id}/chunks",
+        headers=h,
+        json={"use_llm_weak_structure": False},
+    )
+    assert ch.status_code == 200
+
+    mock_job = AsyncMock()
+    with patch(
+        "app.routers.gemini_embed.run_embed_document_job",
+        mock_job,
+    ):
+        r = migrated_client.post(
+            f"/ingest/documents/{doc_id}/embed",
+            headers=h,
+            json={"multimodal": False},
+        )
+    assert r.status_code == 200
+    assert r.json()["accepted"] is True
+    mock_job.assert_called_once()
+
+
+def test_embed_document_persists_vectors_and_rows(
+    tmp_path,
+    monkeypatch,
+) -> None:
+    db_file = tmp_path / "rag.sqlite"
+    monkeypatch.setenv(
+        "DATABASE_URL",
+        f"sqlite+aiosqlite:///{db_file.as_posix()}",
+    )
+    monkeypatch.setenv("GEMINI_API_KEY", "k")
+    monkeypatch.setenv("VECTOR_EMBEDDING_DIM", "1536")
+    clear_settings_cache()
+
+    import os
+    from pathlib import Path
+
+    from alembic.config import Config
+
+    from alembic import command
+
+    backend = Path(__file__).resolve().parents[1]
+    prev = os.getcwd()
+    os.chdir(str(backend))
+    try:
+        command.upgrade(Config(str(backend / "alembic.ini")), "head")
+    finally:
+        os.chdir(prev)
+    clear_settings_cache()
+
+    from sqlalchemy import text
+    from sqlalchemy.ext.asyncio import (
+        AsyncSession,
+        async_sessionmaker,
+        create_async_engine,
+    )
+
+    from app.config import Settings
+    from app.services.embeddings.worker import embed_document_gemini
+
+    settings = Settings()
+    engine = create_async_engine(settings.sqlalchemy_database_url)
+    factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+    store = MagicMock()
+    store.delete_document_for_session = AsyncMock(return_value=0)
+    store.upsert_for_session = AsyncMock()
+    doc_id = f"d-embed-{uuid.uuid4().hex[:12]}"
+
+    async def _seed() -> None:
+        async with factory() as session:
+            await session.execute(
+                text("INSERT INTO sources (name, connector_type) VALUES ('s','s')"),
+            )
+            sid = int(
+                (
+                    await session.execute(text("SELECT id FROM sources LIMIT 1"))
+                ).scalar_one(),
+            )
+            await session.execute(
+                text(
+                    "INSERT INTO documents "
+                    "(id, source_id, timestamp, content_type, raw_content, "
+                    "summary, status) VALUES "
+                    "(:id, :sid, :ts, 'text', '{}', 'x', 'partial')",
+                ),
+                {"id": doc_id, "sid": sid, "ts": datetime.now(UTC).isoformat()},
+            )
+            await session.execute(
+                text(
+                    "INSERT INTO document_chunks "
+                    "(document_id, ordinal, text, start_block_ordinal, "
+                    "end_block_ordinal, meta) "
+                    "VALUES (:d, 0, 'alpha beta', 0, 0, NULL)",
+                ),
+                {"d": doc_id},
+            )
+            await session.commit()
+
+    asyncio.run(_seed())
+
+    async def _run_embed() -> None:
+        with patch(
+            "app.services.embeddings.worker.batch_embed_contents",
+            new=AsyncMock(return_value=[_vec1536()]),
+        ):
+            async with factory() as session:
+                n = await embed_document_gemini(
+                    session,
+                    document_id=doc_id,
+                    settings=settings,
+                    store=store,
+                    multimodal=False,
+                )
+                await session.commit()
+        assert n == 1
+
+    asyncio.run(_run_embed())
+
+    db_path = db_file.resolve()
+    conn = sqlite3.connect(str(db_path))
+    erows = conn.execute(
+        "SELECT document_id, model, dim FROM embeddings WHERE document_id = ?",
+        (doc_id,),
+    ).fetchall()
+    conn.close()
+    assert len(erows) == 1
+    assert erows[0][1] == "gemini-embedding-001"
+    assert erows[0][2] == 1536
+
+    asyncio.run(engine.dispose())
+
+
+def test_inline_data_part_roundtrip_in_api_dict() -> None:
+    p = InlineDataPart(mime_type="image/png", data=b"\x89PNG\r\n")
+    from app.services.embeddings.gemini_api import _part_to_api_dict
+
+    d = _part_to_api_dict(p)
+    assert d["inlineData"]["mimeType"] == "image/png"
+    assert "data" in d["inlineData"]

package/backend/tests/test_health.py

@@ -0,0 +1,24 @@
+from fastapi.testclient import TestClient
+
+
+def test_healthz_no_gateway_header(client: TestClient) -> None:
+    response = client.get("/healthz")
+    assert response.status_code == 200
+    assert response.json() == {"status": "ok"}
+    assert response.headers.get("x-request-id")
+
+
+def test_readyz_no_gateway_header(client: TestClient) -> None:
+    response = client.get("/readyz")
+    assert response.status_code == 200
+    assert response.json() == {"status": "ready"}
+    assert response.headers.get("x-request-id")
+
+
+def test_request_id_echo(client: TestClient) -> None:
+    response = client.get(
+        "/healthz",
+        headers={"X-Request-ID": "probe-abc"},
+    )
+    assert response.status_code == 200
+    assert response.headers.get("x-request-id") == "probe-abc"

package/backend/tests/test_ingest_raw.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+import json
+import sqlite3
+from datetime import UTC, datetime
+
+from fastapi.testclient import TestClient
+
+from app.config import get_settings
+
+
+def test_ingest_raw_requires_auth(migrated_client: TestClient) -> None:
+    r = migrated_client.post(
+        "/ingest/raw",
+        json={
+            "source": "test",
+            "timestamp": datetime.now(UTC).isoformat(),
+            "content_type": "text",
+            "payload": {"x": 1},
+            "metadata": {},
+        },
+    )
+    assert r.status_code == 401
+
+
+def test_ingest_raw_unknown_connector(migrated_client: TestClient) -> None:
+    h = {"x-user-id": "u1"}
+    r = migrated_client.post(
+        "/ingest/raw?connector=does-not-exist",
+        headers=h,
+        json={
+            "source": "test",
+            "timestamp": datetime.now(UTC).isoformat(),
+            "content_type": "text",
+            "payload": {},
+            "metadata": {},
+        },
+    )
+    assert r.status_code == 400
+
+
+def test_ingest_raw_strict_connector_validation(migrated_client: TestClient) -> None:
+    h = {"x-user-id": "u1"}
+    base = {
+        "source": "src-a",
+        "timestamp": datetime.now(UTC).isoformat(),
+        "content_type": "text",
+        "payload": "hello",
+        "metadata": {"connector_config": {"ok": False}},
+    }
+    bad = migrated_client.post("/ingest/raw?connector=strict", headers=h, json=base)
+    assert bad.status_code == 400
+
+    base["metadata"] = {"connector_config": {"ok": True}}
+    ok = migrated_client.post("/ingest/raw?connector=strict", headers=h, json=base)
+    assert ok.status_code == 200
+    body = ok.json()
+    assert body["status"] == "partial"
+    assert body["normalized"]["strict"] is True
+
+
+def test_ingest_raw_persists_envelope(migrated_client: TestClient) -> None:
+    h = {"x-user-id": "u1"}
+    ts = datetime(2024, 3, 15, 10, 0, 0, tzinfo=UTC)
+    payload = {"body": "hello"}
+    r = migrated_client.post(
+        "/ingest/raw?connector=echo",
+        headers=h,
+        json={
+            "source": "filesystem",
+            "timestamp": ts.isoformat(),
+            "content_type": "multimodal",
+            "payload": payload,
+            "metadata": {"path": "/a/b"},
+        },
+    )
+    assert r.status_code == 200
+    data = r.json()
+    doc_id = data["document_id"]
+    assert data["status"] == "partial"
+    assert data["connector"] == "echo"
+    assert data["normalized"]["metadata_keys"] == ["path"]
+    assert data["canonical"]["id"] == doc_id
+    assert len(data["canonical"]["content_blocks"]) == 1
+    assert data["canonical"]["content_blocks"][0]["type"] == "text"
+    assert data["canonical"]["content_blocks"][0]["data"] == "hello"
+
+    settings = get_settings()
+    db_path = (settings.data_dir / settings.sqlite_filename).resolve()
+    conn = sqlite3.connect(str(db_path))
+    cur = conn.execute(
+        "SELECT id, status, content_type, raw_content, summary "
+        "FROM documents WHERE id = ?",
+        (doc_id,),
+    )
+    row = cur.fetchone()
+    assert row is not None
+    assert row[1] == "partial"
+    assert row[2] == "multimodal"
+    stored = json.loads(row[3])
+    assert stored["source"] == "filesystem"
+    assert stored["payload"] == payload
+    assert row[4] == "hello"
+
+    blocks = conn.execute(
+        "SELECT ordinal, type, meta FROM content_blocks "
+        "WHERE document_id = ? ORDER BY ordinal",
+        (doc_id,),
+    ).fetchall()
+    conn.close()
+    assert len(blocks) == 1
+    assert blocks[0][1] == "text"
+    assert json.loads(blocks[0][2])["text"] == "hello"
+
+
+def test_registry_includes_entrypoint_connector() -> None:
+    from app.connectors.registry import init_connectors, list_connector_keys
+
+    init_connectors()
+    keys = list_connector_keys()
+    assert "generic" in keys
+    assert "strict" in keys
+    assert "echo" in keys

package/backend/tests/test_link_expansion.py

@@ -0,0 +1,241 @@
+from __future__ import annotations
+
+import asyncio
+import os
+import sqlite3
+from datetime import UTC, datetime
+from pathlib import Path
+from unittest.mock import AsyncMock, patch
+
+from alembic.config import Config
+from fastapi.testclient import TestClient
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import (
+    AsyncSession,
+    async_sessionmaker,
+    create_async_engine,
+)
+
+from alembic import command
+from app.config import Settings, clear_settings_cache, get_settings
+from app.services.link_expansion.canonical_url import canonicalize_url, host_from_url
+from app.services.link_expansion.domain_policy import host_allowed, parse_host_csv
+from app.services.link_expansion.schemas import ExpandLinksJobResult, ExpandLinksOptions
+from app.services.link_expansion.worker import expand_links_from_document
+
+
+def test_canonicalize_url_strips_fragment_sorts_query() -> None:
+    u = "HTTPS://Example.COM/path?b=2&a=1#frag"
+    assert canonicalize_url(u) == "https://example.com/path?a=1&b=2"
+
+
+def test_host_allowed_allowlist_denylist() -> None:
+    allow = frozenset({"good.example"})
+    deny = frozenset({"bad.example"})
+    assert host_allowed("good.example", allowlist=allow, denylist=deny) is True
+    assert host_allowed("sub.good.example", allowlist=allow, denylist=deny) is True
+    assert host_allowed("bad.example", allowlist=allow, denylist=deny) is False
+    assert host_allowed("other.com", allowlist=allow, denylist=deny) is False
+    assert host_allowed("open.com", allowlist=frozenset(), denylist=deny) is True
+
+
+def test_parse_host_csv() -> None:
+    assert "a.com" in parse_host_csv(" A.COM , b.com ")
+
+
+class _MockResp:
+    def __init__(
+        self,
+        *,
+        content: bytes = b"",
+        status: int = 200,
+        headers: dict[str, str] | None = None,
+        url: str = "",
+    ) -> None:
+        self.content = content
+        self.status_code = status
+        self.headers = headers or {}
+        self.url = url
+
+    @property
+    def is_success(self) -> bool:
+        return 200 <= self.status_code < 300
+
+    @property
+    def text(self) -> str:
+        return self.content.decode("utf-8", errors="replace")
+
+
+class _MockAsyncClient:
+    def __init__(self, *args: object, **kwargs: object) -> None:
+        pass
+
+    async def __aenter__(self) -> _MockAsyncClient:
+        return self
+
+    async def __aexit__(self, *args: object) -> None:
+        pass
+
+    async def head(self, url: str, timeout: object = None) -> _MockResp:
+        return _MockResp(status=200, headers={}, url=url)
+
+    async def get(self, url: str, timeout: object = None) -> _MockResp:
+        if "robots.txt" in url:
+            body = b"User-agent: *\nDisallow:\n"
+            return _MockResp(content=body, url=url)
+        html = (
+            b"<html><body>Hello "
+            b'<a href="https://child.example/child">x</a></body></html>'
+        )
+        return _MockResp(
+            content=html,
+            headers={"content-type": "text/html"},
+            url="https://example.com/page",
+        )
+
+
+def test_expand_links_creates_child_and_relationship(tmp_path, monkeypatch) -> None:
+    monkeypatch.setenv("DATA_DIR", str(tmp_path))
+    monkeypatch.setenv("LINK_EXPAND_POLITENESS_DELAY_MS", "0")
+    monkeypatch.setenv("LINK_EXPAND_PER_DOMAIN_INTERVAL_MS", "0")
+    clear_settings_cache()
+
+    backend = Path(__file__).resolve().parents[1]
+    prev = os.getcwd()
+    os.chdir(str(backend))
+    try:
+        cfg = Config(str(backend / "alembic.ini"))
+        command.upgrade(cfg, "head")
+    finally:
+        os.chdir(prev)
+    clear_settings_cache()
+
+    settings = Settings()
+    engine = create_async_engine(settings.sqlalchemy_database_url)
+    factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+
+    root_id = "root-doc-1"
+
+    async def _seed() -> None:
+        async with factory() as session:
+            await session.execute(
+                text("INSERT INTO sources (name, connector_type) VALUES ('t', 't')"),
+            )
+            res = await session.execute(text("SELECT id FROM sources LIMIT 1"))
+            sid = int(res.scalar_one())
+            await session.execute(
+                text(
+                    "INSERT INTO documents "
+                    "(id, source_id, timestamp, content_type, raw_content, "
+                    "summary, status) "
+                    "VALUES (:id, :sid, :ts, 'text', '{}', 's', 'partial')",
+                ),
+                {
+                    "id": root_id,
+                    "sid": sid,
+                    "ts": datetime.now(UTC).isoformat(),
+                },
+            )
+            await session.execute(
+                text(
+                    "INSERT INTO document_links (document_id, url, ordinal) "
+                    "VALUES (:id, :u, 0)",
+                ),
+                {"id": root_id, "u": "https://example.com/page"},
+            )
+            await session.commit()
+
+    asyncio.run(_seed())
+
+    opts = ExpandLinksOptions(
+        max_depth=2,
+        allowlist=frozenset(),
+        denylist=frozenset(),
+        respect_robots=True,
+    )
+
+    async def _expand() -> ExpandLinksJobResult:
+        with patch(
+            "app.services.link_expansion.worker.httpx.AsyncClient",
+            _MockAsyncClient,
+        ):
+            async with factory() as session:
+                out = await expand_links_from_document(
+                    session,
+                    root_document_id=root_id,
+                    options=opts,
+                    settings=get_settings(),
+                )
+                await session.commit()
+        return out
+
+    result = asyncio.run(_expand())
+    assert result.relationships_created >= 1
+    assert result.documents_created >= 1
+
+    db_path = (get_settings().data_dir / get_settings().sqlite_filename).resolve()
+    conn = sqlite3.connect(str(db_path))
+    rels = conn.execute(
+        "SELECT parent_document_id, child_document_id, relation_type "
+        "FROM relationships",
+    ).fetchall()
+    conn.close()
+    assert any(r[0] == root_id and r[2] == "link" for r in rels)
+
+    async def _dispose() -> None:
+        await engine.dispose()
+
+    asyncio.run(_dispose())
+
+
+def test_expand_links_endpoint_404(migrated_client: TestClient) -> None:
+    h = {"x-user-id": "u1"}
+    r = migrated_client.post(
+        "/ingest/expand-links",
+        headers=h,
+        json={"document_id": "does-not-exist"},
+    )
+    assert r.status_code == 404
+
+
+def test_expand_links_endpoint_accepts(
+    migrated_client: TestClient,
+    monkeypatch,
+) -> None:
+    monkeypatch.setenv("LINK_EXPAND_POLITENESS_DELAY_MS", "0")
+    monkeypatch.setenv("LINK_EXPAND_PER_DOMAIN_INTERVAL_MS", "0")
+    clear_settings_cache()
+
+    h = {"x-user-id": "u1"}
+    ing = migrated_client.post(
+        "/ingest/raw?connector=echo",
+        headers=h,
+        json={
+            "source": "t",
+            "timestamp": datetime.now(UTC).isoformat(),
+            "content_type": "text",
+            "payload": "see https://example.com/x",
+            "metadata": {},
+        },
+    )
+    assert ing.status_code == 200
+    doc_id = ing.json()["document_id"]
+
+    mock_job = AsyncMock()
+
+    with patch(
+        "app.routers.link_expansion.run_expand_links_job",
+        mock_job,
+    ):
+        r = migrated_client.post(
+            "/ingest/expand-links",
+            headers=h,
+            json={"document_id": doc_id},
+        )
+    assert r.status_code == 200
+    assert r.json() == {"accepted": True, "document_id": doc_id}
+    mock_job.assert_called_once()
+
+
+def test_host_from_url() -> None:
+    assert host_from_url("https://a.B.com/p") == "a.b.com"

package/backend/tests/test_main.py

@@ -0,0 +1,12 @@
+from fastapi.testclient import TestClient
+
+
+def test_hello_world_requires_gateway_identity(client: TestClient) -> None:
+    response = client.get("/hello-world")
+    assert response.status_code == 401
+
+
+def test_hello_world_with_user_header(client: TestClient) -> None:
+    response = client.get("/hello-world", headers={"x-user-id": "user-1"})
+    assert response.status_code == 200
+    assert response.json() == {"message": "Hello, World"}