business-stack 0.1.0

package/backend/tests/test_normalizer.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+import base64
+import hashlib
+from datetime import UTC, datetime
+from pathlib import Path
+
+import pytest
+
+from app.schemas.ingest import RawIngestEnvelope
+from app.services.normalization.normalizer import (
+    extract_urls_from_text,
+    normalize_envelope_to_canonical,
+    parse_blob_sha256,
+)
+from app.storage.blobs import BlobStore
+
+
+def test_text_only_string_payload(tmp_path: Path) -> None:
+    store = BlobStore(tmp_path / "blobs")
+    ts = datetime(2026, 4, 5, 12, 0, tzinfo=UTC)
+    env = RawIngestEnvelope(
+        source="slack",
+        timestamp=ts,
+        content_type="text",
+        payload="Hello from the connector.",
+        metadata={},
+    )
+    doc = normalize_envelope_to_canonical(
+        document_id="doc-1",
+        envelope=env,
+        blob_store=store,
+    )
+    assert doc.id == "doc-1"
+    assert doc.source == "slack"
+    assert doc.timestamp == ts
+    assert len(doc.content_blocks) == 1
+    assert doc.content_blocks[0].type == "text"
+    assert doc.content_blocks[0].data == "Hello from the connector."
+    assert doc.content_blocks[0].raw_input is None
+    assert doc.content_blocks[0].mime == "text/plain"
+    assert doc.links == []
+    assert doc.summary == "Hello from the connector."
+    assert doc.raw_content == "Hello from the connector."
+
+
+def test_text_and_image_multimodal_preserves_blocks_and_blob(tmp_path: Path) -> None:
+    store = BlobStore(tmp_path / "blobs")
+    image_bytes = b"\x89PNG\r\n\x1a\nfake-png-bytes-for-test"
+    b64 = base64.b64encode(image_bytes).decode("ascii")
+    expected_sha = hashlib.sha256(image_bytes).hexdigest()
+    ts = datetime(2026, 4, 5, 12, 0, tzinfo=UTC)
+    env = RawIngestEnvelope(
+        source="twitter",
+        timestamp=ts,
+        content_type="multimodal",
+        payload={
+            "tweet": "Hot take + chart",
+            "image_base64": b64,
+        },
+        metadata={},
+    )
+    doc = normalize_envelope_to_canonical(
+        document_id="doc-2",
+        envelope=env,
+        blob_store=store,
+    )
+    assert len(doc.content_blocks) == 2
+    assert doc.content_blocks[0].type == "text"
+    assert doc.content_blocks[0].data == "Hot take + chart"
+    assert doc.content_blocks[1].type == "image"
+    assert doc.content_blocks[1].raw_input == f"blob://{expected_sha}"
+    sha = parse_blob_sha256(doc.content_blocks[1].raw_input)
+    assert sha == expected_sha
+    blob_path = store._path_for(expected_sha)
+    assert blob_path.is_file()
+    assert blob_path.read_bytes() == image_bytes
+    assert doc.summary.startswith("Hot take")
+
+
+def test_url_metadata_and_inline_urls_deduped(tmp_path: Path) -> None:
+    store = BlobStore(tmp_path / "blobs")
+    ts = datetime(2026, 4, 5, 12, 0, tzinfo=UTC)
+    env = RawIngestEnvelope(
+        source="rss",
+        timestamp=ts,
+        content_type="text",
+        payload="Read https://example.com/a and also https://example.com/b",
+        metadata={
+            "url": "https://example.com/a",
+            "urls": ["https://example.com/c"],
+        },
+    )
+    doc = normalize_envelope_to_canonical(
+        document_id="doc-3",
+        envelope=env,
+        blob_store=store,
+    )
+    assert doc.links == [
+        "https://example.com/a",
+        "https://example.com/b",
+        "https://example.com/c",
+    ]
+
+
+@pytest.mark.parametrize(
+    "text,expected",
+    [
+        ("no urls here", []),
+        ("see https://x.com/foo?q=1", ["https://x.com/foo?q=1"]),
+    ],
+)
+def test_extract_urls_from_text(text: str, expected: list[str]) -> None:
+    assert extract_urls_from_text(text) == expected

package/backend/tests/test_openapi_gateway.py

@@ -0,0 +1,40 @@
+import pytest
+from httpx import ASGITransport, AsyncClient
+
+from app.config import get_settings
+from app.main import create_app
+
+
+@pytest.fixture
+def anyio_backend():
+    return "asyncio"
+
+
+@pytest.mark.anyio
+async def test_openapi_paths_require_gateway_secret_when_configured(monkeypatch):
+    monkeypatch.setenv("BACKEND_GATEWAY_SECRET", "test-gateway-secret")
+    get_settings.cache_clear()
+    app = create_app()
+    transport = ASGITransport(app=app)
+    async with AsyncClient(transport=transport, base_url="http://test") as client:
+        r = await client.get("/docs", headers={"x-user-id": "u1"})
+        assert r.status_code == 403
+        r2 = await client.get(
+            "/docs",
+            headers={
+                "x-user-id": "u1",
+                "x-gateway-secret": "test-gateway-secret",
+            },
+        )
+        assert r2.status_code == 200
+
+
+@pytest.mark.anyio
+async def test_openapi_paths_allow_without_secret(monkeypatch):
+    monkeypatch.delenv("BACKEND_GATEWAY_SECRET", raising=False)
+    get_settings.cache_clear()
+    app = create_app()
+    transport = ASGITransport(app=app)
+    async with AsyncClient(transport=transport, base_url="http://test") as client:
+        r = await client.get("/docs", headers={"x-user-id": "u1"})
+        assert r.status_code == 200

package/backend/tests/test_pipeline_hardening.py

@@ -0,0 +1,285 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import sqlite3
+from datetime import UTC, datetime
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from alembic.config import Config
+from fastapi.testclient import TestClient
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+
+from alembic import command
+from app.config import Settings, clear_settings_cache, get_settings
+from app.services.embeddings.worker import run_embed_document_job
+
+
+def test_ingest_dedupe_external_id(migrated_client: TestClient) -> None:
+    h = {"x-user-id": "u1"}
+    ts1 = datetime(2026, 4, 1, 10, 0, 0, tzinfo=UTC)
+    ts2 = datetime(2026, 4, 2, 11, 0, 0, tzinfo=UTC)
+    base = {
+        "source": "dedupe-src",
+        "content_type": "text",
+        "payload": "same body",
+        "metadata": {"external_id": "ext-42"},
+    }
+    r1 = migrated_client.post(
+        "/ingest/raw?connector=echo",
+        headers=h,
+        json={**base, "timestamp": ts1.isoformat()},
+    )
+    r2 = migrated_client.post(
+        "/ingest/raw?connector=echo",
+        headers=h,
+        json={**base, "timestamp": ts2.isoformat()},
+    )
+    assert r1.status_code == 200
+    assert r2.status_code == 200
+    assert r1.json()["document_id"] == r2.json()["document_id"]
+    assert r2.json()["deduplicated"] is True
+    assert r1.json()["deduplicated"] is False
+
+
+def test_ingest_dedupe_content_hash_no_external_id(
+    migrated_client: TestClient,
+) -> None:
+    h = {"x-user-id": "u1"}
+    ts1 = datetime(2026, 4, 1, 10, 0, 0, tzinfo=UTC)
+    ts2 = datetime(2026, 4, 2, 11, 0, 0, tzinfo=UTC)
+    body = {
+        "source": "hash-src",
+        "content_type": "text",
+        "payload": "identical",
+        "metadata": {},
+    }
+    r1 = migrated_client.post(
+        "/ingest/raw?connector=echo",
+        headers=h,
+        json={**body, "timestamp": ts1.isoformat()},
+    )
+    r2 = migrated_client.post(
+        "/ingest/raw?connector=echo",
+        headers=h,
+        json={**body, "timestamp": ts2.isoformat()},
+    )
+    assert r1.json()["document_id"] == r2.json()["document_id"]
+    assert r2.json()["deduplicated"] is True
+
+
+def test_ingest_normalization_failure_stores_error_and_partial(
+    migrated_client: TestClient,
+    monkeypatch,
+) -> None:
+    h = {"x-user-id": "u1"}
+
+    def boom(*args, **kwargs):
+        raise RuntimeError("normalize boom")
+
+    monkeypatch.setattr(
+        "app.routers.ingestion.normalize_envelope_to_canonical",
+        boom,
+    )
+    r = migrated_client.post(
+        "/ingest/raw?connector=echo",
+        headers=h,
+        json={
+            "source": "norm-fail",
+            "timestamp": datetime.now(UTC).isoformat(),
+            "content_type": "text",
+            "payload": "x",
+            "metadata": {},
+        },
+    )
+    assert r.status_code == 200
+    data = r.json()
+    assert data["normalization_failed"] is True
+    assert data["canonical"]["content_blocks"] == []
+    doc_id = data["document_id"]
+    settings = get_settings()
+    conn = sqlite3.connect(str(settings.data_dir / settings.sqlite_filename))
+    row = conn.execute(
+        "SELECT normalization_error FROM documents WHERE id = ?",
+        (doc_id,),
+    ).fetchone()
+    conn.close()
+    assert row is not None
+    assert row[0] is not None
+    err = json.loads(row[0])
+    assert err["type"] == "RuntimeError"
+
+
+def test_admin_reprocess_normalization(migrated_client: TestClient) -> None:
+    h = {"x-user-id": "u1"}
+
+    def boom(*args, **kwargs):
+        raise RuntimeError("temporary")
+
+    with patch(
+        "app.routers.ingestion.normalize_envelope_to_canonical",
+        side_effect=boom,
+    ):
+        r = migrated_client.post(
+            "/ingest/raw?connector=echo",
+            headers=h,
+            json={
+                "source": "reproc",
+                "timestamp": datetime.now(UTC).isoformat(),
+                "content_type": "text",
+                "payload": "hello reproc",
+                "metadata": {},
+            },
+        )
+    assert r.status_code == 200
+    doc_id = r.json()["document_id"]
+
+    rr = migrated_client.post(
+        f"/admin/documents/{doc_id}/reprocess-normalization",
+        headers=h,
+    )
+    assert rr.status_code == 200
+    settings = get_settings()
+    conn = sqlite3.connect(str(settings.data_dir / settings.sqlite_filename))
+    ne = conn.execute(
+        "SELECT normalization_error FROM documents WHERE id = ?",
+        (doc_id,),
+    ).fetchone()[0]
+    blocks = conn.execute(
+        "SELECT COUNT(*) FROM content_blocks WHERE document_id = ?",
+        (doc_id,),
+    ).fetchone()[0]
+    conn.close()
+    assert ne is None
+    assert blocks >= 1
+
+
+def test_embed_failure_writes_dlq(tmp_path, monkeypatch) -> None:
+    monkeypatch.setenv("DATA_DIR", str(tmp_path))
+    monkeypatch.setenv("GEMINI_API_KEY", "k")
+    monkeypatch.setenv("EMBEDDING_DLQ_MAX_ATTEMPTS", "3")
+    clear_settings_cache()
+
+    backend = Path(__file__).resolve().parents[1]
+    prev = os.getcwd()
+    os.chdir(str(backend))
+    try:
+        command.upgrade(Config(str(backend / "alembic.ini")), "head")
+    finally:
+        os.chdir(prev)
+    clear_settings_cache()
+
+    settings = Settings()
+    engine = create_async_engine(settings.sqlalchemy_database_url)
+    factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+    doc_id = "dlq-doc-1"
+
+    async def _seed() -> None:
+        async with factory() as session:
+            await session.execute(
+                text("INSERT INTO sources (name, connector_type) VALUES ('s','s')"),
+            )
+            sid_row = await session.execute(text("SELECT id FROM sources LIMIT 1"))
+            sid = int(sid_row.scalar_one())
+            await session.execute(
+                text(
+                    "INSERT INTO documents "
+                    "(id, source_id, timestamp, content_type, raw_content, "
+                    "summary, status) VALUES "
+                    "(:id, :sid, :ts, 'text', '{}', 'x', 'partial')",
+                ),
+                {"id": doc_id, "sid": sid, "ts": datetime.now(UTC).isoformat()},
+            )
+            await session.execute(
+                text(
+                    "INSERT INTO document_chunks "
+                    "(document_id, ordinal, text, start_block_ordinal, "
+                    "end_block_ordinal, meta) VALUES (:d, 0, 't', 0, 0, NULL)",
+                ),
+                {"d": doc_id},
+            )
+            await session.commit()
+
+    asyncio.run(_seed())
+
+    async def _fail() -> None:
+        with patch(
+            "app.services.embeddings.worker.batch_embed_contents",
+            new=AsyncMock(side_effect=RuntimeError("api down")),
+        ):
+            await run_embed_document_job(
+                document_id=doc_id,
+                multimodal=False,
+                session_factory=factory,
+                settings=get_settings(),
+                store=MagicMock(),
+            )
+
+    asyncio.run(_fail())
+
+    db_path = (get_settings().data_dir / get_settings().sqlite_filename).resolve()
+    conn = sqlite3.connect(str(db_path))
+    row = conn.execute(
+        "SELECT attempt_count, state FROM embedding_dlq WHERE document_id = ?",
+        (doc_id,),
+    ).fetchone()
+    conn.close()
+    assert row is not None
+    assert int(row[0]) == 1
+    assert row[1] == "pending_retry"
+
+    asyncio.run(engine.dispose())
+
+
+def test_admin_retry_embedding_clears_dlq(
+    migrated_client: TestClient,
+    monkeypatch,
+) -> None:
+    monkeypatch.setenv("GEMINI_API_KEY", "k")
+    clear_settings_cache()
+    h = {"x-user-id": "u1"}
+    ing = migrated_client.post(
+        "/ingest/raw?connector=echo",
+        headers=h,
+        json={
+            "source": "adm",
+            "timestamp": datetime.now(UTC).isoformat(),
+            "content_type": "text",
+            "payload": "p",
+            "metadata": {},
+        },
+    )
+    doc_id = ing.json()["document_id"]
+    settings = get_settings()
+    db_path = str(settings.data_dir / settings.sqlite_filename)
+    conn = sqlite3.connect(db_path)
+    conn.execute(
+        "INSERT INTO embedding_dlq (document_id, last_error, attempt_count, "
+        "next_retry_at, state, multimodal) VALUES (?, ?, 2, NULL, 'dead', 0)",
+        (doc_id, "old err"),
+    )
+    conn.commit()
+    conn.close()
+
+    mock_job = MagicMock()
+    with patch(
+        "app.routers.admin_pipeline.run_embed_document_job",
+        mock_job,
+    ):
+        r = migrated_client.post(
+            f"/admin/documents/{doc_id}/retry-embedding",
+            headers=h,
+            json={"multimodal": False},
+        )
+    assert r.status_code == 200
+    conn = sqlite3.connect(db_path)
+    n = conn.execute(
+        "SELECT COUNT(*) FROM embedding_dlq WHERE document_id = ?",
+        (doc_id,),
+    ).fetchone()[0]
+    conn.close()
+    assert n == 0
+    mock_job.assert_called_once()

package/backend/tests/test_pipeline_status.py

@@ -0,0 +1,71 @@
+from fastapi.testclient import TestClient
+
+
+def test_pipeline_status_404(migrated_client: TestClient) -> None:
+    h = {"x-user-id": "u-pipe"}
+    r = migrated_client.get(
+        "/ingest/documents/missing-id/pipeline",
+        headers=h,
+    )
+    assert r.status_code == 404
+
+
+def test_pipeline_status_after_ingest(migrated_client: TestClient) -> None:
+    h = {"x-user-id": "u-pipe"}
+    env = {
+        "source": "test",
+        "timestamp": "2026-04-05T00:00:00Z",
+        "content_type": "text",
+        "payload": "hello pipeline",
+        "metadata": {},
+    }
+    ing = migrated_client.post(
+        "/ingest/raw?connector=generic",
+        headers=h,
+        json=env,
+    )
+    assert ing.status_code == 200
+    doc_id = ing.json()["document_id"]
+
+    r = migrated_client.get(
+        f"/ingest/documents/{doc_id}/pipeline",
+        headers=h,
+    )
+    assert r.status_code == 200
+    body = r.json()
+    assert body["document_id"] == doc_id
+    assert "steps" in body
+    assert isinstance(body["steps"], list)
+    assert body["chunk_count"] >= 0
+    assert body["content_block_count"] >= 0
+
+
+def test_pipeline_response_shape(migrated_client: TestClient) -> None:
+    h = {"x-user-id": "u-pipe"}
+    env = {
+        "source": "test",
+        "timestamp": "2026-04-05T00:00:01Z",
+        "content_type": "text",
+        "payload": "shape",
+        "metadata": {},
+    }
+    ing = migrated_client.post(
+        "/ingest/raw?connector=generic",
+        headers=h,
+        json=env,
+    )
+    doc_id = ing.json()["document_id"]
+    r = migrated_client.get(
+        f"/ingest/documents/{doc_id}/pipeline",
+        headers=h,
+    )
+    assert r.status_code == 200
+    j = r.json()
+    for key in (
+        "normalization_error",
+        "ingest_meta",
+        "vector_row_count",
+        "gemini_embedding_row_count",
+        "checked_at",
+    ):
+        assert key in j

package/backend/tests/test_playwright_extract.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+import pytest
+from fastapi.testclient import TestClient
+
+from app.services.playwright_extract.extract import (
+    assert_url_allowed,
+    host_matches_allowlist,
+)
+
+
+def test_host_allowlist_exact() -> None:
+    assert host_matches_allowlist("example.com", ["example.com"])
+    assert not host_matches_allowlist("evil.com", ["example.com"])
+
+
+def test_host_allowlist_suffix() -> None:
+    assert host_matches_allowlist("sub.example.com", ["*.example.com"])
+    assert host_matches_allowlist("example.com", ["*.example.com"])
+    assert not host_matches_allowlist("notexample.com", ["*.example.com"])
+
+
+def test_assert_url_allowed_rejects_non_http() -> None:
+    with pytest.raises(ValueError, match="scheme"):
+        assert_url_allowed("file:///etc/passwd", ["x"])
+
+
+def test_assert_url_allowed_empty_hosts() -> None:
+    with pytest.raises(ValueError, match="non-empty"):
+        assert_url_allowed("https://a.com", [])
+
+
+def test_playwright_ingest_enriches_envelope(
+    migrated_client: TestClient,
+    monkeypatch,
+) -> None:
+    from app.services.playwright_extract.extract import PlaywrightExtractResult
+
+    def fake_extract(*args, **kwargs):
+        assert kwargs["allowlisted_hosts"] == ["example.com"]
+        return PlaywrightExtractResult(
+            visible_text="hello page",
+            title="Hi",
+            final_url="https://example.com/after",
+            truncated=False,
+            meta={},
+        )
+
+    monkeypatch.setattr(
+        "app.connectors.playwright_session.extract_visible_text_sync",
+        fake_extract,
+    )
+
+    r = migrated_client.post(
+        "/ingest/raw?connector=playwright_session",
+        headers={"x-user-id": "u1"},
+        json={
+            "source": "pw-test",
+            "timestamp": datetime(2026, 3, 1, tzinfo=UTC).isoformat(),
+            "content_type": "text",
+            "payload": "",
+            "metadata": {
+                "url": "https://example.com/start",
+                "playwright_user_data_dir": r"C:\fake\pw-profile",
+                "connector_config": {
+                    "allowlisted_hosts": ["example.com"],
+                },
+            },
+        },
+    )
+    assert r.status_code == 200, r.text
+    data = r.json()
+    assert data["normalized"]["extracted_text_chars"] == len("hello page")
+    canonical = data["canonical"]
+    assert any(
+        b.get("type") == "text" and "hello page" in (b.get("data") or "")
+        for b in canonical.get("content_blocks", [])
+    )