business-stack 0.1.0

package/backend/app/services/normalization/normalizer.py

@@ -0,0 +1,331 @@
+from __future__ import annotations
+
+import base64
+import binascii
+import json
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+from app.schemas.canonical import BlockType, CanonicalContentBlock, CanonicalDocument
+from app.schemas.ingest import RawIngestEnvelope
+from app.storage.blobs import BlobStore
+
+URL_PATTERN = re.compile(
+    r"https?://[^\s<>\"'{}|\\^`\[\]]+",
+    re.IGNORECASE,
+)
+
+MULTIMODAL_TEXT_KEYS: tuple[str, ...] = (
+    "text",
+    "body",
+    "caption",
+    "tweet",
+    "message",
+    "title",
+)
+MULTIMODAL_IMAGE_KEYS: tuple[str, ...] = (
+    "image",
+    "image_base64",
+    "thumbnail",
+    "photo",
+)
+
+
+@dataclass
+class _PendingBlock:
+    block_type: BlockType
+    text: str | None = None
+    media: bytes | None = None
+    media_mime: str | None = None
+
+
+@dataclass
+class _Draft:
+    pending: list[_PendingBlock] = field(default_factory=list)
+    text_segments_for_urls: list[str] = field(default_factory=list)
+
+
+def extract_urls_from_text(text: str) -> list[str]:
+    return URL_PATTERN.findall(text or "")
+
+
+def _extract_metadata_urls(metadata: dict[str, Any]) -> list[str]:
+    out: list[str] = []
+    u = metadata.get("url")
+    if isinstance(u, str) and u.strip():
+        out.append(u.strip())
+    raw_urls = metadata.get("urls")
+    if isinstance(raw_urls, list):
+        out.extend(str(x).strip() for x in raw_urls if isinstance(x, str) and x.strip())
+    return out
+
+
+def dedupe_urls(urls: list[str]) -> list[str]:
+    seen: set[str] = set()
+    out: list[str] = []
+    for u in urls:
+        if u not in seen:
+            seen.add(u)
+            out.append(u)
+    return out
+
+
+def _decode_base64_image(data: str) -> tuple[bytes | None, str | None]:
+    try:
+        raw = base64.b64decode(data, validate=True)
+    except (binascii.Error, ValueError):
+        return None, None
+    return raw, "application/octet-stream"
+
+
+def _tags_from_metadata(metadata: dict[str, Any]) -> list[str]:
+    tags = metadata.get("tags")
+    if not isinstance(tags, list):
+        return []
+    return [str(t) for t in tags if t is not None]
+
+
+def _draft_from_plain_text(text: str) -> _Draft:
+    d = _Draft()
+    stripped = text.strip()
+    if stripped:
+        d.pending.append(_PendingBlock(block_type="text", text=stripped))
+        d.text_segments_for_urls.append(stripped)
+    return d
+
+
+def _append_image_from_value(
+    d: _Draft,
+    value: Any,
+    default_mime: str | None,
+) -> None:
+    if value is None:
+        return
+    if isinstance(value, (bytes, bytearray)):
+        d.pending.append(
+            _PendingBlock(
+                block_type="image",
+                media=bytes(value),
+                media_mime=default_mime or "application/octet-stream",
+            ),
+        )
+        return
+    if isinstance(value, str):
+        decoded, _ = _decode_base64_image(value)
+        if decoded:
+            d.pending.append(
+                _PendingBlock(
+                    block_type="image",
+                    media=decoded,
+                    media_mime=default_mime or "image/png",
+                ),
+            )
+
+
+def _draft_from_dict_payload(payload: dict[str, Any]) -> _Draft:
+    d = _Draft()
+    for key in MULTIMODAL_TEXT_KEYS:
+        if key not in payload:
+            continue
+        val = payload.get(key)
+        if val is None:
+            continue
+        if isinstance(val, str) and val.strip():
+            t = val.strip()
+            d.pending.append(_PendingBlock(block_type="text", text=t))
+            d.text_segments_for_urls.append(t)
+        elif isinstance(val, list):
+            for item in val:
+                if isinstance(item, str) and item.strip():
+                    t = item.strip()
+                    d.pending.append(_PendingBlock(block_type="text", text=t))
+                    d.text_segments_for_urls.append(t)
+
+    for key in MULTIMODAL_IMAGE_KEYS:
+        if key not in payload:
+            continue
+        val = payload[key]
+        if isinstance(val, list):
+            for item in val:
+                _append_image_from_value(d, item, "image/png")
+        else:
+            mime = None
+            if isinstance(val, dict):
+                b64 = val.get("base64") or val.get("data")
+                mime = val.get("mime") or val.get("content_type")
+                if isinstance(b64, str):
+                    _append_image_from_value(d, b64, str(mime) if mime else "image/png")
+            else:
+                _append_image_from_value(d, val, "image/png")
+
+    images = payload.get("images")
+    if isinstance(images, list):
+        for item in images:
+            if isinstance(item, str):
+                _append_image_from_value(d, item, "image/png")
+            elif isinstance(item, dict):
+                b64 = item.get("base64") or item.get("data")
+                mime = item.get("mime") or item.get("content_type")
+                if isinstance(b64, str):
+                    _append_image_from_value(
+                        d,
+                        b64,
+                        str(mime) if mime else "image/png",
+                    )
+
+    return d
+
+
+def _build_draft(envelope: RawIngestEnvelope) -> _Draft:
+    payload = envelope.payload
+    ct = envelope.content_type
+
+    if ct == "text":
+        if isinstance(payload, str):
+            return _draft_from_plain_text(payload)
+        if isinstance(payload, dict):
+            return _draft_from_dict_payload(payload)
+        return _draft_from_plain_text(json.dumps(payload, default=str))
+
+    if ct == "multimodal" and isinstance(payload, dict):
+        return _draft_from_dict_payload(payload)
+
+    if ct == "image":
+        d = _Draft()
+        if isinstance(payload, str):
+            _append_image_from_value(d, payload, "image/png")
+        elif isinstance(payload, dict):
+            b64 = payload.get("base64") or payload.get("data")
+            mime = payload.get("mime") or "image/png"
+            if isinstance(b64, str):
+                _append_image_from_value(d, b64, str(mime))
+        elif isinstance(payload, (bytes, bytearray)):
+            d.pending.append(
+                _PendingBlock(
+                    block_type="image",
+                    media=bytes(payload),
+                    media_mime="application/octet-stream",
+                ),
+            )
+        return d
+
+    if ct in ("audio", "video", "document"):
+        d = _Draft()
+        if isinstance(payload, dict):
+            b64 = payload.get("base64") or payload.get("data")
+            mime = payload.get("mime") or (
+                "audio/mpeg"
+                if ct == "audio"
+                else "video/mp4"
+                if ct == "video"
+                else "application/octet-stream"
+            )
+            if isinstance(b64, str):
+                try:
+                    raw = base64.b64decode(b64, validate=True)
+                except (binascii.Error, ValueError):
+                    raw = None
+                if raw:
+                    d.pending.append(
+                        _PendingBlock(
+                            block_type=ct,
+                            media=raw,
+                            media_mime=str(mime),
+                        ),
+                    )
+            blob = payload.get("bytes")
+            if isinstance(blob, (bytes, bytearray)):
+                d.pending.append(
+                    _PendingBlock(
+                        block_type=ct,
+                        media=bytes(blob),
+                        media_mime=str(mime),
+                    ),
+                )
+        elif isinstance(payload, (bytes, bytearray)):
+            d.pending.append(
+                _PendingBlock(
+                    block_type=ct,
+                    media=bytes(payload),
+                    media_mime="application/octet-stream",
+                ),
+            )
+        return d
+
+    if isinstance(payload, dict):
+        return _draft_from_dict_payload(payload)
+    if isinstance(payload, str):
+        return _draft_from_plain_text(payload)
+    return _draft_from_plain_text(json.dumps(payload, default=str))
+
+
+def _collect_links(envelope: RawIngestEnvelope, draft: _Draft) -> list[str]:
+    found: list[str] = []
+    for seg in draft.text_segments_for_urls:
+        found.extend(extract_urls_from_text(seg))
+    found.extend(_extract_metadata_urls(envelope.metadata))
+    return dedupe_urls(found)
+
+
+def _summary_from_pending(pending: list[_PendingBlock]) -> str:
+    texts = [p.text for p in pending if p.text]
+    joined = " ".join(texts).strip()
+    if len(joined) <= 500:
+        return joined
+    return joined[:499] + "…"
+
+
+def normalize_envelope_to_canonical(
+    *,
+    document_id: str,
+    envelope: RawIngestEnvelope,
+    blob_store: BlobStore,
+) -> CanonicalDocument:
+    """Map raw ingest envelope to canonical document; write media to blob store."""
+    draft = _build_draft(envelope)
+    links = _collect_links(envelope, draft)
+    tags = _tags_from_metadata(envelope.metadata)
+
+    blocks: list[CanonicalContentBlock] = []
+    for p in draft.pending:
+        if p.block_type == "text" and p.text is not None:
+            blocks.append(
+                CanonicalContentBlock(
+                    type="text",
+                    data=p.text,
+                    raw_input=None,
+                    mime="text/plain",
+                ),
+            )
+            continue
+        if p.media is not None and len(p.media) > 0:
+            _sha, uri = blob_store.write(p.media)
+            blocks.append(
+                CanonicalContentBlock(
+                    type=p.block_type,
+                    data=None,
+                    raw_input=uri,
+                    mime=p.media_mime,
+                ),
+            )
+
+    summary = _summary_from_pending(draft.pending)
+
+    return CanonicalDocument(
+        id=document_id,
+        source=envelope.source,
+        timestamp=envelope.timestamp,
+        content_blocks=blocks,
+        raw_content=envelope.payload,
+        entities=[],
+        links=links,
+        tags=tags,
+        summary=summary,
+    )
+
+
+def parse_blob_sha256(raw_input: str | None) -> str | None:
+    if raw_input and raw_input.startswith("blob://"):
+        return raw_input.removeprefix("blob://")
+    return None

package/backend/app/services/normalization/persist_normalized.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import json
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.schemas.canonical import CanonicalDocument
+from app.services.normalization.normalizer import parse_blob_sha256
+
+
+async def persist_normalized_document(
+    session: AsyncSession,
+    *,
+    canonical: CanonicalDocument,
+) -> None:
+    """Replace content_blocks and document_links; set documents.summary."""
+    doc_id = canonical.id
+
+    await session.execute(
+        text("DELETE FROM content_blocks WHERE document_id = :d"),
+        {"d": doc_id},
+    )
+    await session.execute(
+        text("DELETE FROM document_links WHERE document_id = :d"),
+        {"d": doc_id},
+    )
+
+    for ordinal, block in enumerate(canonical.content_blocks):
+        sha = parse_blob_sha256(block.raw_input)
+        meta: dict = {}
+        if block.data is not None:
+            meta["text"] = block.data
+        meta_json = json.dumps(meta) if meta else None
+        storage_uri = block.raw_input if block.raw_input else None
+        await session.execute(
+            text(
+                "INSERT INTO content_blocks "
+                "(document_id, ordinal, type, storage_uri, inline_ref, mime, "
+                "sha256, meta) "
+                "VALUES (:did, :ord, :ty, :su, :ir, :mime, :sha, :meta)",
+            ),
+            {
+                "did": doc_id,
+                "ord": ordinal,
+                "ty": block.type,
+                "su": storage_uri,
+                "ir": "canonical:inline" if block.data and not storage_uri else None,
+                "mime": block.mime,
+                "sha": sha,
+                "meta": meta_json,
+            },
+        )
+
+    for ordinal, url in enumerate(canonical.links):
+        await session.execute(
+            text(
+                "INSERT INTO document_links (document_id, url, ordinal) "
+                "VALUES (:d, :u, :o)",
+            ),
+            {"d": doc_id, "u": url, "o": ordinal},
+        )
+
+    await session.execute(
+        text("UPDATE documents SET summary = :s WHERE id = :id"),
+        {"s": canonical.summary, "id": doc_id},
+    )

package/backend/app/services/playwright_extract/__init__.py

@@ -0,0 +1,13 @@
+from app.services.playwright_extract.extract import (
+    PlaywrightExtractResult,
+    assert_url_allowed,
+    extract_visible_text_sync,
+    host_matches_allowlist,
+)
+
+__all__ = [
+    "PlaywrightExtractResult",
+    "assert_url_allowed",
+    "extract_visible_text_sync",
+    "host_matches_allowlist",
+]

package/backend/app/services/playwright_extract/__main__.py

@@ -0,0 +1,96 @@
+"""
+CLI: extract visible text for a URL using a persistent Chrome/Chromium profile.
+
+Example (Windows, after installing optional deps and browsers)::
+
+    uv run python -m app.services.playwright_extract ^
+      https://example.com/path ^
+      --profile "C:\\\\Users\\\\you\\\\pw-profile" ^
+      --allow-hosts example.com
+
+See ``app.services.playwright_extract.extract`` module docstring for profile
+locking and path guidance.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(
+        description="Playwright visible-text extraction (persistent user data dir)",
+    )
+    p.add_argument("url", help="https? URL to open")
+    p.add_argument(
+        "--profile",
+        required=True,
+        type=Path,
+        help="Persistent browser user data directory (dedicated folder recommended)",
+    )
+    p.add_argument(
+        "--allow-hosts",
+        required=True,
+        help="Comma-separated allowlist (supports *.suffix.com patterns)",
+    )
+    p.add_argument(
+        "--config",
+        type=Path,
+        help="Optional JSON file with keys: navigation_timeout_ms, max_response_chars, "
+        "headless, browser_channel",
+    )
+    p.add_argument(
+        "--headed",
+        action="store_true",
+        help="Run non-headless (overrides config headless)",
+    )
+    args = p.parse_args()
+
+    allowlisted = [h.strip() for h in args.allow_hosts.split(",") if h.strip()]
+    navigation_timeout_ms = 30_000
+    max_response_chars = 500_000
+    headless = not args.headed
+    browser_channel: str | None = None
+
+    if args.config is not None:
+        data = json.loads(args.config.read_text(encoding="utf-8"))
+        if not isinstance(data, dict):
+            print("config file must be a JSON object", file=sys.stderr)
+            return 2
+        navigation_timeout_ms = int(
+            data.get("navigation_timeout_ms", navigation_timeout_ms),
+        )
+        max_response_chars = int(
+            data.get("max_response_chars", max_response_chars),
+        )
+        headless = bool(data.get("headless", headless)) if not args.headed else False
+        ch = data.get("browser_channel")
+        browser_channel = str(ch) if ch else None
+
+    from app.services.playwright_extract.extract import extract_visible_text_sync
+
+    result = extract_visible_text_sync(
+        args.url,
+        str(args.profile.expanduser().resolve()),
+        allowlisted_hosts=allowlisted,
+        navigation_timeout_ms=navigation_timeout_ms,
+        max_response_chars=max_response_chars,
+        headless=headless,
+        browser_channel=browser_channel,
+    )
+    out = {
+        "visible_text": result.visible_text,
+        "title": result.title,
+        "final_url": result.final_url,
+        "truncated": result.truncated,
+        "meta": result.meta,
+    }
+    print(json.dumps(out, indent=2, ensure_ascii=False))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/backend/app/services/playwright_extract/extract.py

@@ -0,0 +1,181 @@
+"""
+Playwright-based extraction of visible page text using a **persistent browser profile**.
+
+Operational notes (especially **Windows**)
+------------------------------------------
+
+**Profile path**
+
+- Use a **dedicated** directory for automation (e.g.
+  ``C:\\Users\\you\\AppData\\Local\\PlaywrightIngestProfile``), not your daily
+  Chrome profile, to reduce lock conflicts and accidental data loss.
+- Paths may be passed as normal Python strings; raw strings (``r"C:\\..."``) help
+  avoid escape issues in scripts.
+- Prefer resolving with ``pathlib.Path`` and ``.expanduser()`` / ``.resolve()``.
+
+**Profile locking ("SingletonLock", "profile in use")**
+
+- The Chromium/Chrome user-data directory must not be open in another browser
+  instance. **Fully quit** regular Chrome (or use a copy of the profile) before
+  ``launch_persistent_context`` runs.
+- Only **one** Playwright persistent context should use a given directory at a
+  time; concurrent ingest requests with the same path will race and may fail.
+- If you need parallel jobs, use **separate** ``user_data_dir`` roots per worker.
+
+**Install**
+
+- Install the optional dependency group (see ``pyproject.toml``), then run
+  ``playwright install chromium`` (or the channel you select). Using
+  ``channel="chrome"`` requires a local Google Chrome install.
+
+This module performs **no** scraping guidance beyond driving a real browser
+session; target sites' terms and robots rules still apply.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import Any
+from urllib.parse import urlparse
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True, slots=True)
+class PlaywrightExtractResult:
+    visible_text: str
+    title: str | None
+    final_url: str
+    truncated: bool
+    meta: dict[str, Any] = field(default_factory=dict)
+
+
+def _scheme_host(url: str) -> tuple[str, str]:
+    parsed = urlparse(url.strip())
+    if parsed.scheme not in ("http", "https"):
+        msg = f"URL scheme must be http or https, got {parsed.scheme!r}"
+        raise ValueError(msg)
+    host = (parsed.hostname or "").lower()
+    if not host:
+        msg = "URL must include a host"
+        raise ValueError(msg)
+    return parsed.scheme, host
+
+
+def host_matches_allowlist(hostname: str, allowlisted_hosts: list[str]) -> bool:
+    """
+    Match hostname against entries.
+
+    - ``example.com`` — exact host match (case-insensitive).
+    - ``*.example.com`` — suffix match (``sub.example.com``, not ``evil-example.com``).
+    """
+    h = hostname.lower().rstrip(".")
+    for raw in allowlisted_hosts:
+        p = raw.strip().lower()
+        if not p:
+            continue
+        if p.startswith("*."):
+            root = p[2:].lstrip(".")
+            if h == root or h.endswith("." + root):
+                return True
+        elif h == p:
+            return True
+    return False
+
+
+def assert_url_allowed(url: str, allowlisted_hosts: list[str]) -> None:
+    if not allowlisted_hosts:
+        msg = "allowlisted_hosts must be a non-empty list"
+        raise ValueError(msg)
+    _, host = _scheme_host(url)
+    if not host_matches_allowlist(host, allowlisted_hosts):
+        msg = f"Host {host!r} is not in allowlisted_hosts"
+        raise ValueError(msg)
+
+
+def extract_visible_text_sync(
+    url: str,
+    user_data_dir: str,
+    *,
+    allowlisted_hosts: list[str],
+    navigation_timeout_ms: int = 30_000,
+    max_response_chars: int = 500_000,
+    headless: bool = True,
+    browser_channel: str | None = None,
+) -> PlaywrightExtractResult:
+    """
+    Launch a persistent context, navigate to ``url``, return visible text + metadata.
+
+    Runs synchronously (intended for ``asyncio.to_thread`` from async routes or
+    direct use from the sync CLI). Requires the ``playwright`` package and
+    installed browser binaries.
+    """
+    assert_url_allowed(url, allowlisted_hosts)
+
+    try:
+        from playwright.sync_api import sync_playwright
+    except ImportError as e:
+        msg = (
+            "playwright is not installed. Install optional deps, e.g. "
+            "`uv sync --extra playwright`, then `playwright install chromium`."
+        )
+        raise RuntimeError(msg) from e
+
+    meta: dict[str, Any] = {
+        "navigation_timeout_ms": navigation_timeout_ms,
+        "max_response_chars": max_response_chars,
+        "user_data_dir": user_data_dir,
+    }
+
+    with sync_playwright() as pw:
+        ctx_kwargs: dict[str, Any] = {
+            "user_data_dir": user_data_dir,
+            "headless": headless,
+            "viewport": {"width": 1280, "height": 720},
+        }
+        if browser_channel:
+            ctx_kwargs["channel"] = browser_channel
+
+        context = pw.chromium.launch_persistent_context(**ctx_kwargs)
+        try:
+            page = context.pages[0] if context.pages else context.new_page()
+            page.goto(
+                url,
+                wait_until="domcontentloaded",
+                timeout=navigation_timeout_ms,
+            )
+            final_url = page.url
+            title = page.title() or None
+            try:
+                raw_text = page.inner_text(
+                    "body",
+                    timeout=min(15_000, navigation_timeout_ms),
+                )
+            except Exception:
+                logger.debug(
+                    "inner_text(body) failed; falling back to evaluate",
+                    exc_info=True,
+                )
+                raw_text = (
+                    page.evaluate(
+                        "() => document.body ? document.body.innerText : ''",
+                    )
+                    or ""
+                )
+
+            truncated = False
+            if len(raw_text) > max_response_chars:
+                raw_text = raw_text[:max_response_chars]
+                truncated = True
+            meta["truncated"] = truncated
+            meta["final_url"] = final_url
+            return PlaywrightExtractResult(
+                visible_text=raw_text.strip(),
+                title=title,
+                final_url=final_url,
+                truncated=truncated,
+                meta=meta,
+            )
+        finally:
+            context.close()