business-stack 0.1.0

package/backend/app/connectors/base.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from typing import Any, ClassVar
+
+from app.schemas.ingest import RawIngestEnvelope
+
+
+class Connector(ABC):
+    """Source plugin: validate config, optional fetch_iter, normalize raw envelopes."""
+
+    name: ClassVar[str]
+
+    def prepare_envelope(
+        self,
+        envelope: RawIngestEnvelope,
+        config: dict[str, Any],
+    ) -> RawIngestEnvelope:
+        """
+        Mutate or replace the envelope before persistence (default: identity).
+
+        Used for connectors that fetch remote content (e.g. browser session)
+        so ``raw_content`` and canonical normalization see extracted text.
+        """
+        return envelope
+
+    def validate_config(self, config: dict[str, Any]) -> None:
+        """Raise ValueError if *config* is invalid for this connector."""
+
+    def fetch_iter(self, config: dict[str, Any]) -> Iterator[Any]:
+        """Optional batch source; not used by POST /ingest/raw."""
+        raise NotImplementedError(
+            f"Connector {self.name!r} does not implement fetch_iter()",
+        )
+
+    @abstractmethod
+    def normalize_raw(self, envelope: RawIngestEnvelope) -> dict[str, Any]:
+        """Produce a normalized dict for downstream pipeline steps."""

package/backend/app/connectors/builtins.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from typing import Any
+
+from app.connectors.base import Connector
+from app.connectors.registry import register_connector
+from app.schemas.ingest import RawIngestEnvelope
+
+
+class GenericConnector(Connector):
+    name = "generic"
+
+    def normalize_raw(self, envelope: RawIngestEnvelope) -> dict[str, Any]:
+        return {
+            "source": envelope.source,
+            "content_type": envelope.content_type,
+            "payload_type": type(envelope.payload).__name__,
+        }
+
+
+class EchoConnector(Connector):
+    """Passes through key envelope fields for tests and debugging."""
+
+    name = "echo"
+
+    def normalize_raw(self, envelope: RawIngestEnvelope) -> dict[str, Any]:
+        return {
+            "source": envelope.source,
+            "content_type": envelope.content_type,
+            "metadata_keys": sorted(envelope.metadata.keys()),
+        }
+
+
+class StrictConnector(Connector):
+    """Validates metadata.connector_config for integration tests."""
+
+    name = "strict"
+
+    def validate_config(self, config: dict[str, Any]) -> None:
+        if config.get("ok") is not True:
+            raise ValueError("connector_config.ok must be true")
+
+    def normalize_raw(self, envelope: RawIngestEnvelope) -> dict[str, Any]:
+        return {"strict": True, "source": envelope.source}
+
+
+def register_builtin_connectors() -> None:
+    register_connector(GenericConnector)
+    register_connector(StrictConnector)
+    # Side effect: @register_connector on PlaywrightSessionConnector
+    import app.connectors.playwright_session  # noqa: F401

package/backend/app/connectors/playwright_session.py

@@ -0,0 +1,146 @@
+"""
+Ingest connector: fetch **visible text** via Playwright + persistent user-data dir.
+
+Uses the same ``RawIngestEnvelope`` as other connectors; ``prepare_envelope``
+runs the browser **before** the document is persisted so ``raw_content`` and
+canonical normalization include extracted text.
+
+**Envelope shape**
+
+- **URL**: ``metadata.url`` *or* string ``payload`` *or* ``payload["url"]``.
+- **Profile directory**: ``metadata.playwright_user_data_dir`` *or*
+  ``connector_config.user_data_dir`` (config wins if both set — see code).
+
+**connector_config** (safety / tuning)
+
+- ``allowlisted_hosts`` (required): non-empty list of host strings; supports
+  ``*.example.com`` suffix patterns (see ``host_matches_allowlist``).
+- ``navigation_timeout_ms`` (optional, default 30000).
+- ``max_response_chars`` (optional, default 500000).
+- ``headless`` (optional, default true).
+- ``browser_channel`` (optional): e.g. ``"chrome"`` to use installed Google Chrome.
+
+**Credentials / compliance**
+
+- This connector does not ship API keys; authenticated pages rely on **cookies and
+  session state** already stored in the persistent profile directory.
+- You are responsible for site terms, login policies, and data handling.
+
+**Windows / profile locking** (summary)
+
+- Use a **dedicated** profile folder for automation; **close** all browser
+  windows using that profile before ingest; avoid concurrent runs on the same
+  path. Full notes: ``app.services.playwright_extract.extract`` module docstring.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from app.connectors.base import Connector
+from app.connectors.registry import register_connector
+from app.schemas.ingest import RawIngestEnvelope
+from app.services.playwright_extract.extract import extract_visible_text_sync
+
+
+def _resolve_target_url(envelope: RawIngestEnvelope) -> str:
+    md = envelope.metadata.get("url")
+    if isinstance(md, str) and md.strip():
+        return md.strip()
+    if isinstance(envelope.payload, str) and envelope.payload.strip():
+        return envelope.payload.strip()
+    if isinstance(envelope.payload, dict):
+        u = envelope.payload.get("url")
+        if isinstance(u, str) and u.strip():
+            return u.strip()
+    msg = "Set metadata.url or payload (string or {url: ...}) for playwright_session"
+    raise ValueError(msg)
+
+
+def _resolve_user_data_dir(envelope: RawIngestEnvelope, config: dict[str, Any]) -> str:
+    cfg_dir = config.get("user_data_dir")
+    if isinstance(cfg_dir, str) and cfg_dir.strip():
+        return cfg_dir.strip()
+    md_dir = envelope.metadata.get("playwright_user_data_dir")
+    if isinstance(md_dir, str) and md_dir.strip():
+        return md_dir.strip()
+    msg = (
+        "Set connector_config.user_data_dir or metadata.playwright_user_data_dir "
+        "to the persistent Chrome/Chromium user data directory"
+    )
+    raise ValueError(msg)
+
+
+@register_connector
+class PlaywrightSessionConnector(Connector):
+    name = "playwright_session"
+
+    def validate_config(self, config: dict[str, Any]) -> None:
+        hosts = config.get("allowlisted_hosts")
+        if not isinstance(hosts, list) or not hosts:
+            raise ValueError(
+                "connector_config.allowlisted_hosts must be a non-empty list",
+            )
+        if not all(isinstance(h, str) and h.strip() for h in hosts):
+            raise ValueError("allowlisted_hosts entries must be non-empty strings")
+
+    def prepare_envelope(
+        self,
+        envelope: RawIngestEnvelope,
+        config: dict[str, Any],
+    ) -> RawIngestEnvelope:
+        url = _resolve_target_url(envelope)
+        user_data_dir = _resolve_user_data_dir(envelope, config)
+        navigation_timeout_ms = int(config.get("navigation_timeout_ms", 30_000))
+        max_response_chars = int(config.get("max_response_chars", 500_000))
+        headless = bool(config.get("headless", True))
+        ch = config.get("browser_channel")
+        browser_channel = str(ch) if isinstance(ch, str) and ch.strip() else None
+        hosts = [str(h).strip() for h in config["allowlisted_hosts"] if str(h).strip()]
+
+        result = extract_visible_text_sync(
+            url,
+            user_data_dir,
+            allowlisted_hosts=hosts,
+            navigation_timeout_ms=navigation_timeout_ms,
+            max_response_chars=max_response_chars,
+            headless=headless,
+            browser_channel=browser_channel,
+        )
+
+        meta = dict(envelope.metadata)
+        meta["playwright_extraction"] = {
+            "requested_url": url,
+            "final_url": result.final_url,
+            "title": result.title,
+            "truncated": result.truncated,
+            "profile_meta": result.meta,
+        }
+
+        payload: dict[str, Any] = {
+            "text": result.visible_text,
+            "title": result.title,
+            "url": result.final_url,
+            "source_url": url,
+        }
+
+        return envelope.model_copy(
+            update={
+                "content_type": "text",
+                "metadata": meta,
+                "payload": payload,
+            },
+        )
+
+    def normalize_raw(self, envelope: RawIngestEnvelope) -> dict[str, Any]:
+        raw_playwright = envelope.metadata.get("playwright_extraction")
+        text_len = 0
+        if isinstance(envelope.payload, dict):
+            t = envelope.payload.get("text")
+            if isinstance(t, str):
+                text_len = len(t)
+        return {
+            "connector": self.name,
+            "playwright": raw_playwright if isinstance(raw_playwright, dict) else None,
+            "extracted_text_chars": text_len,
+        }

package/backend/app/connectors/registry.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import logging
+from importlib.metadata import entry_points
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from app.connectors.base import Connector
+
+logger = logging.getLogger(__name__)
+
+_CONNECTORS: dict[str, type[Connector]] = {}
+_INITIALIZED = False
+
+
+def register_connector(cls: type[Connector]) -> type[Connector]:
+    key = cls.name
+    if key in _CONNECTORS:
+        logger.warning("Overwriting connector registration for key %r", key)
+    _CONNECTORS[key] = cls
+    return cls
+
+
+def get_connector_class(key: str) -> type[Connector]:
+    return _CONNECTORS[key]
+
+
+def list_connector_keys() -> list[str]:
+    return sorted(_CONNECTORS.keys())
+
+
+def _load_entrypoint_connectors() -> None:
+    from app.connectors.base import Connector as ConnectorBase
+
+    try:
+        eps = entry_points(group="kb.connectors")
+    except TypeError:
+        eps = entry_points().select(group="kb.connectors")
+
+    for ep in eps:
+        try:
+            loaded = ep.load()
+        except Exception:
+            logger.exception("Failed to load connector entry point %r", ep.name)
+            continue
+        if isinstance(loaded, type) and issubclass(loaded, ConnectorBase):
+            register_connector(loaded)
+        else:
+            logger.warning(
+                "Entry point %r did not resolve to a Connector subclass",
+                ep.name,
+            )
+
+
+def _load_builtin_connectors() -> None:
+    from app.connectors.builtins import register_builtin_connectors
+
+    register_builtin_connectors()
+
+
+def init_connectors() -> None:
+    """Idempotent: register built-ins and kb.connectors entry points."""
+    global _INITIALIZED
+    if _INITIALIZED:
+        return
+    _load_builtin_connectors()
+    _load_entrypoint_connectors()
+    _INITIALIZED = True

package/backend/app/connectors/thread_expansion/__init__.py

@@ -0,0 +1,33 @@
+"""
+Thread expansion: social threads via **official APIs** (when implemented).
+
+Models: ``ThreadDocumentNode``, ``ThreadRelationshipEdge``, ``ThreadExpansionResult``
+for a document + ``relationships`` tree (``reply`` / ``thread`` edges).
+
+Abstract fetchers: ``TwitterThreadExpansionFetcher``, ``RedditThreadExpansionFetcher``
+(compliance notes in their docstrings). Fakes: deterministic tests / local use.
+
+Production: implement with documented X and Reddit APIs; do not use scraping.
+"""
+
+from app.connectors.thread_expansion.fakes import (
+    FakeRedditThreadExpansionFetcher,
+    FakeTwitterThreadExpansionFetcher,
+)
+from app.connectors.thread_expansion.models import (
+    ThreadDocumentNode,
+    ThreadExpansionResult,
+    ThreadRelationshipEdge,
+)
+from app.connectors.thread_expansion.reddit import RedditThreadExpansionFetcher
+from app.connectors.thread_expansion.twitter import TwitterThreadExpansionFetcher
+
+__all__ = [
+    "FakeRedditThreadExpansionFetcher",
+    "FakeTwitterThreadExpansionFetcher",
+    "RedditThreadExpansionFetcher",
+    "ThreadDocumentNode",
+    "ThreadExpansionResult",
+    "ThreadRelationshipEdge",
+    "TwitterThreadExpansionFetcher",
+]

package/backend/app/connectors/thread_expansion/fakes.py

@@ -0,0 +1,154 @@
+"""
+Deterministic fake thread fetchers for unit tests and local development.
+
+No network calls; no credentials required.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Any
+
+from app.connectors.thread_expansion.models import (
+    ThreadDocumentNode,
+    ThreadExpansionResult,
+    ThreadRelationshipEdge,
+)
+from app.connectors.thread_expansion.reddit import RedditThreadExpansionFetcher
+from app.connectors.thread_expansion.twitter import TwitterThreadExpansionFetcher
+
+
+class FakeTwitterThreadExpansionFetcher(TwitterThreadExpansionFetcher):
+    """
+    Returns a small linear 3-tweet thread (root → reply → reply).
+
+    ``conversation_id`` is echoed into node ``meta`` only; fetch always returns
+    the same structure for testing.
+    """
+
+    async def fetch_full_thread(
+        self,
+        conversation_id: str,
+        *,
+        config: dict[str, Any],
+    ) -> ThreadExpansionResult:
+        _ = config
+        t0 = datetime(2026, 1, 1, 12, 0, 0, tzinfo=UTC)
+        nodes = [
+            ThreadDocumentNode(
+                document_id="twitter:tw_root",
+                external_id="1999999999999999999",
+                text="Root post about the project launch.",
+                author_handle="example_user",
+                created_at=t0,
+                meta={"conversation_id": conversation_id, "kind": "tweet"},
+            ),
+            ThreadDocumentNode(
+                document_id="twitter:tw_r1",
+                external_id="1999999999999999998",
+                text="Reply: congratulations!",
+                author_handle="fan_one",
+                created_at=t0,
+                meta={"kind": "tweet"},
+            ),
+            ThreadDocumentNode(
+                document_id="twitter:tw_r2",
+                external_id="1999999999999999997",
+                text="Reply: when is v2 shipping?",
+                author_handle="fan_two",
+                created_at=t0,
+                meta={"kind": "tweet"},
+            ),
+        ]
+        edges = [
+            ThreadRelationshipEdge(
+                parent_document_id="twitter:tw_root",
+                child_document_id="twitter:tw_r1",
+                relation_type="reply",
+                meta={"platform": "twitter"},
+            ),
+            ThreadRelationshipEdge(
+                parent_document_id="twitter:tw_r1",
+                child_document_id="twitter:tw_r2",
+                relation_type="reply",
+                meta={"platform": "twitter"},
+            ),
+            ThreadRelationshipEdge(
+                parent_document_id="twitter:tw_root",
+                child_document_id="twitter:tw_r2",
+                relation_type="thread",
+                weight=0.3,
+                meta={"platform": "twitter", "note": "optional weak thread link"},
+            ),
+        ]
+        return ThreadExpansionResult(
+            platform="twitter",
+            thread_key=conversation_id,
+            root_document_id="twitter:tw_root",
+            nodes=nodes,
+            edges=edges,
+        )
+
+
+class FakeRedditThreadExpansionFetcher(RedditThreadExpansionFetcher):
+    """
+    Returns a post plus two nested comments (linear reply chain).
+
+    Ignores ``post_id`` except to set ``thread_key`` for stable tests.
+    """
+
+    async def fetch_post_and_comments(
+        self,
+        post_id: str,
+        *,
+        config: dict[str, Any],
+    ) -> ThreadExpansionResult:
+        _ = config
+        t0 = datetime(2026, 2, 1, 9, 30, 0, tzinfo=UTC)
+        nodes = [
+            ThreadDocumentNode(
+                document_id="reddit:t3_abc123",
+                external_id="t3_abc123",
+                text="Ask HN: Best patterns for thread-safe queues?",
+                author_handle="op_dev",
+                created_at=t0,
+                meta={"subreddit": "test", "score": 42, "permalink": "/r/test/abc"},
+            ),
+            ThreadDocumentNode(
+                document_id="reddit:t1_cmnt1",
+                external_id="t1_cmnt1",
+                text="We use channels and bounded queues.",
+                author_handle="commenter_a",
+                created_at=t0,
+                meta={"depth": 0},
+            ),
+            ThreadDocumentNode(
+                document_id="reddit:t1_cmnt2",
+                external_id="t1_cmnt2",
+                text="Plus backpressure on the producer side.",
+                author_handle="commenter_b",
+                created_at=t0,
+                meta={"depth": 1},
+            ),
+        ]
+        edges = [
+            ThreadRelationshipEdge(
+                parent_document_id="reddit:t3_abc123",
+                child_document_id="reddit:t1_cmnt1",
+                relation_type="reply",
+                meta={"platform": "reddit"},
+            ),
+            ThreadRelationshipEdge(
+                parent_document_id="reddit:t1_cmnt1",
+                child_document_id="reddit:t1_cmnt2",
+                relation_type="reply",
+                meta={"platform": "reddit"},
+            ),
+        ]
+        return ThreadExpansionResult(
+            platform="reddit",
+            thread_key=post_id or "t3_abc123",
+            root_document_id="reddit:t3_abc123",
+            nodes=nodes,
+            edges=edges,
+        )

package/backend/app/connectors/thread_expansion/models.py

@@ -0,0 +1,113 @@
+"""
+Structured thread data for persisting into ``documents`` + ``relationships``.
+
+Use ``relation_type`` values that match the DB check constraint:
+``'link'``, ``'thread'``, ``'reply'``, ``'entity_cooccur'``.
+
+For social threads, model the tree with:
+
+- ``reply`` — direct parent → child reply edge (authoritative tree shape).
+- ``thread`` — optional weak edge (e.g. root → descendant) for “same thread”
+  navigation; omit if you only need the reply tree.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+PlatformLiteral = Literal["twitter", "reddit"]
+ThreadRelationLiteral = Literal["reply", "thread"]
+
+
+class ThreadDocumentNode(BaseModel):
+    """
+    One logical post (tweet, Reddit post/comment) mapped to a KB ``documents`` row.
+
+    ``document_id`` should be stable across fetches (e.g. ``twitter:{tweet_id}``,
+    ``reddit:{fullname}``) so upserts and ``relationships`` stay consistent.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    document_id: str = Field(..., min_length=1)
+    external_id: str = Field(
+        ...,
+        min_length=1,
+        description="Platform-native id (tweet id, Reddit fullname t1_/t3_)",
+    )
+    text: str = Field(default="", description="Primary text for summary / chunking")
+    author_handle: str | None = None
+    created_at: datetime
+    content_type: Literal["text", "multimodal"] = "text"
+    raw_content: dict[str, Any] | None = Field(
+        default=None,
+        description="Optional structured payload stored as JSON in ingest paths",
+    )
+    meta: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Extra attribution: urls, like counts, etc. (JSON-serializable)",
+    )
+
+
+class ThreadRelationshipEdge(BaseModel):
+    """
+    Maps to ``relationships`` (parent_document_id, child_document_id, relation_type).
+
+    Use ``reply`` for the reply tree. Use ``thread`` sparingly for non-reply links
+    (e.g. root → leaf) if your UI or graph queries need them.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    parent_document_id: str = Field(..., min_length=1)
+    child_document_id: str = Field(..., min_length=1)
+    relation_type: ThreadRelationLiteral
+    weight: float | None = None
+    meta: dict[str, Any] = Field(default_factory=dict)
+
+    @model_validator(mode="after")
+    def _no_self_loop(self) -> ThreadRelationshipEdge:
+        if self.parent_document_id == self.child_document_id:
+            msg = "parent_document_id and child_document_id must differ"
+            raise ValueError(msg)
+        return self
+
+
+class ThreadExpansionResult(BaseModel):
+    """
+    Full thread fetch: nodes (documents) + edges (relationships preserving tree shape).
+
+    Persist by inserting/updating one document per node, then inserting edges with
+    ``INSERT OR IGNORE`` (or equivalent) to respect the unique constraint on
+    ``(parent_document_id, child_document_id, relation_type)``.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    platform: PlatformLiteral
+    thread_key: str = Field(
+        ...,
+        min_length=1,
+        description="Stable thread id: X conversation_id, Reddit post fullname, etc.",
+    )
+    root_document_id: str = Field(..., min_length=1)
+    nodes: list[ThreadDocumentNode]
+    edges: list[ThreadRelationshipEdge]
+
+    @model_validator(mode="after")
+    def _graph_integrity(self) -> ThreadExpansionResult:
+        ids = {n.document_id for n in self.nodes}
+        if self.root_document_id not in ids:
+            msg = "root_document_id must match a node.document_id"
+            raise ValueError(msg)
+        for e in self.edges:
+            if e.parent_document_id not in ids:
+                msg = f"edge parent not in nodes: {e.parent_document_id!r}"
+                raise ValueError(msg)
+            if e.child_document_id not in ids:
+                msg = f"edge child not in nodes: {e.child_document_id!r}"
+                raise ValueError(msg)
+        return self

package/backend/app/connectors/thread_expansion/reddit.py

@@ -0,0 +1,53 @@
+"""
+Reddit thread expansion — **official Reddit Data API only**.
+
+Use reddit.com/dev/api and OAuth-based access. Do not bypass or automate the
+site in ways that conflict with Reddit’s Terms or API rules.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from app.connectors.thread_expansion.models import ThreadExpansionResult
+
+
+class RedditThreadExpansionFetcher(ABC):
+    """
+    Fetch a post and its comment tree via the **Reddit Data API**.
+
+    Compliance & credentials (operational checklist — not legal advice):
+
+    - **API access**: Register an app at Reddit to obtain a client id/secret;
+      use OAuth as required for the endpoints you call. Credentials belong in
+      server-side configuration only.
+    - **User-Agent**: Reddit expects a descriptive ``User-Agent`` string
+      (typically including app name and contact). Follow Reddit’s API guidelines.
+    - **Policies**: Comply with Reddit’s API Terms, rate limits, and content
+      policies. Do not use the API for disallowed bulk or commercial uses unless
+      permitted.
+    - **Privacy**: Honor moderator and user visibility (removed/deleted content)
+      as represented by the API.
+
+    Implementations MUST NOT scrape HTML threads as a substitute for the API.
+    """
+
+    @abstractmethod
+    async def fetch_post_and_comments(
+        self,
+        post_id: str,
+        *,
+        config: dict[str, Any],
+    ) -> ThreadExpansionResult:
+        """
+        Return the submission and reachable comments as nodes, with ``reply``
+        edges mirroring Reddit’s parent/child comment relationships.
+
+        ``post_id`` should be the API-facing id (e.g. base36 id or fullname
+        ``t3_xxx`` — implementations should normalize consistently).
+
+        ``config`` holds OAuth tokens, user-agent suffix, sort, depth limits, etc.
+        **Do not** log access tokens or refresh tokens.
+        """
+        raise NotImplementedError