igel-qe-core 1.0.0

package/knowledge_base/config/settings.py

@@ -0,0 +1,280 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+_ENV_PATH = Path(__file__).parent / ".env"
+load_dotenv(_ENV_PATH)
+
+
+def _truthy(raw: str | None, default: bool = False) -> bool:
+    if raw is None or raw == "":
+        return default
+    return raw.strip().lower() in ("1", "true", "yes", "on")
+
+
+class Config:
+    # PostgreSQL
+    DB_HOST: str = os.getenv("DB_HOST", "127.0.0.1")
+    DB_PORT: int = int(os.getenv("DB_PORT", "5432"))
+    DB_NAME: str = os.getenv("DB_NAME", "genai")
+    DB_USER: str = os.getenv("DB_USER", "genai")
+    DB_PASSWORD: str = os.getenv("DB_PASSWORD", "")
+
+    # Azure OpenAI - LLM
+    AZURE_API_KEY: str = os.getenv("AZURE_API_KEY", "")
+    AZURE_ENDPOINT: str = os.getenv("AZURE_ENDPOINT", "")
+    AZURE_DEPLOYMENT: str = os.getenv("AZURE_DEPLOYMENT", "gpt-4.1")
+    AZURE_API_VERSION: str = os.getenv("AZURE_API_VERSION", "2024-12-01-preview")
+
+    # Azure OpenAI - Embeddings
+    AZURE_EMBEDDING_DEPLOYMENT: str = os.getenv(
+        "AZURE_EMBEDDING_DEPLOYMENT", "text-embedding-3-large"
+    )
+    AZURE_EMBEDDINGS_API_VERSION: str = os.getenv(
+        "AZURE_EMBEDDINGS_API_VERSION", "2024-02-01"
+    )
+    EMBEDDING_DIM: int = int(os.getenv("EMBEDDING_DIM", "3072"))
+
+    # Knowledge Base paths (innominds project root)
+    _PROJECT_ROOT: Path = Path(__file__).resolve().parent.parent.parent
+
+    KB_DIR: Path = Path(
+        os.getenv(
+            "KB_DIR",
+            str(_PROJECT_ROOT / "Knowledge Base"),
+        )
+    )
+    MARKDOWN_OUTPUT_DIR: Path = Path(
+        os.getenv(
+            "MARKDOWN_OUTPUT_DIR",
+            str(_PROJECT_ROOT / "Knowledge Base" / "markdown_output"),
+        )
+    )
+
+    # Ingestion settings
+    BATCH_SIZE: int = int(os.getenv("BATCH_SIZE", "32"))
+    MAX_RETRIES: int = int(os.getenv("MAX_RETRIES", "3"))
+    LOG_LEVEL: str = os.getenv("LOG_LEVEL", "INFO")
+
+    # Supported file extensions
+    SUPPORTED_EXTENSIONS: tuple[str, ...] = (
+        ".pdf",
+        ".docx",
+        ".doc",
+        ".csv",
+        ".xlsx",
+        ".xls",
+        ".md",
+        ".mht",
+    )
+    SKIP_EXTENSIONS: tuple[str, ...] = (
+        ".zip",
+        ".py",
+        ".exe",
+        ".png",
+        ".jpg",
+        ".json",
+        ".DS_Store",
+    )
+
+    # ── V3 Retrieval / rerank / GraphRAG ───────────────────────────────────
+    RERANKER_BACKEND: str = os.getenv("RERANKER_BACKEND", "local").strip().lower()
+    # CPU-optimized default: MiniLM (~80MB, fast). For multilingual DE/EN use BAAI/bge-reranker-base
+    RERANKER_MODEL: str = os.getenv(
+        "RERANKER_MODEL",
+        "cross-encoder/ms-marco-MiniLM-L-6-v2",
+    )
+    RERANKER_DEVICE: str = os.getenv("RERANKER_DEVICE", "cpu").strip().lower()
+    RERANKER_CPU_PROFILE: bool = _truthy(os.getenv("RERANKER_CPU_PROFILE"), True)
+    RERANKER_TOP_N: int = int(os.getenv("RERANKER_TOP_N", "16"))
+    RERANKER_BATCH_SIZE: int = int(os.getenv("RERANKER_BATCH_SIZE", "8"))
+    RERANKER_MAX_LENGTH: int = int(os.getenv("RERANKER_MAX_LENGTH", "256"))
+    RERANKER_PASSAGE_CHARS: int = int(os.getenv("RERANKER_PASSAGE_CHARS", "2000"))
+    RERANKER_PARENT_CHARS: int = int(os.getenv("RERANKER_PARENT_CHARS", "1200"))
+    RERANKER_TORCH_THREADS: int = int(os.getenv("RERANKER_TORCH_THREADS", "4"))
+    RERANKER_SCORE_THRESHOLD: float = float(os.getenv("RERANKER_SCORE_THRESHOLD", "0.2"))
+    RETRIEVAL_CANDIDATE_MULTIPLIER: int = int(
+        os.getenv("RETRIEVAL_CANDIDATE_MULTIPLIER", "6"),
+    )
+
+    COHERE_API_KEY: str = os.getenv("COHERE_API_KEY", "")
+    COHERE_RERANK_MODEL: str = os.getenv(
+        "COHERE_RERANK_MODEL",
+        "rerank-multilingual-v3.0",
+    )
+
+    # ── Chunking strategy (env-tunable) ─────────────────────────────────────
+    CHUNK_PARENT_MAX_TOKENS: int = int(os.getenv("CHUNK_PARENT_MAX_TOKENS", "1024"))
+    CHUNK_PARENT_MIN_TOKENS: int = int(os.getenv("CHUNK_PARENT_MIN_TOKENS", "80"))
+    CHUNK_CHILD_MAX_TOKENS: int = int(os.getenv("CHUNK_CHILD_MAX_TOKENS", "220"))
+    CHUNK_CHILD_MIN_TOKENS: int = int(os.getenv("CHUNK_CHILD_MIN_TOKENS", "50"))
+    CHUNK_OVERLAP_TOKENS: int = int(os.getenv("CHUNK_OVERLAP_TOKENS", "64"))
+    CHUNK_CONTEXT_HEADERS: bool = _truthy(os.getenv("CHUNK_CONTEXT_HEADERS"), True)
+
+    # ── Live web retrieval (confidence-gated Stage 2 search) ─────────────────
+    WEB_RETRIEVAL_ENABLED: bool = _truthy(os.getenv("WEB_RETRIEVAL_ENABLED"), True)
+    CONFIDENCE_THRESHOLD: float = float(os.getenv("CONFIDENCE_THRESHOLD", "0.75"))
+    _raw_web_backends = os.getenv("WEB_SEARCH_BACKENDS", "firecrawl,ddgs,searxng")
+    WEB_SEARCH_BACKENDS: tuple[str, ...] = tuple(
+        s.strip().lower() for s in _raw_web_backends.split(",") if s.strip()
+    )
+    _raw_web_sources = os.getenv("WEB_SOURCES", "kb_igel,community")
+    WEB_SOURCES: tuple[str, ...] = tuple(
+        s.strip().lower() for s in _raw_web_sources.split(",") if s.strip()
+    )
+    WEB_MAX_PAGES: int = int(os.getenv("WEB_MAX_PAGES", "5"))
+    WEB_MIN_SCORE: float = float(os.getenv("WEB_MIN_SCORE", "0.15"))
+    WEB_CACHE_TTL_SECONDS: int = int(os.getenv("WEB_CACHE_TTL_SECONDS", "3600"))
+    WEB_BUDGET_RATIO: float = float(os.getenv("WEB_BUDGET_RATIO", "0.12"))
+    WEB_ALLOW_PRIVATE_HOSTS: bool = _truthy(os.getenv("WEB_ALLOW_PRIVATE_HOSTS"), False)
+    DDGS_BACKEND: str = os.getenv("DDGS_BACKEND", "auto")
+    DDGS_REGION: str = os.getenv("DDGS_REGION", "wt-wt")
+    SEARXNG_BASE_URL: str = os.getenv("SEARXNG_BASE_URL", "")
+    FIRECRAWL_BASE_URL: str = os.getenv("FIRECRAWL_BASE_URL", "")
+    FIRECRAWL_API_KEY: str = os.getenv("FIRECRAWL_API_KEY", "")
+    CRAWL4AI_ENABLED: bool = _truthy(os.getenv("CRAWL4AI_ENABLED"), True)
+    CRAWL4AI_TIMEOUT: int = int(os.getenv("CRAWL4AI_TIMEOUT", "30"))
+    _raw_strip = os.getenv("STRIP_PROMOTED_PATTERNS", "ad,sponsored,promo,advertisement")
+    STRIP_PROMOTED_PATTERNS: tuple[str, ...] = tuple(
+        s.strip().lower() for s in _raw_strip.split(",") if s.strip()
+    )
+    _raw_inc = os.getenv("WEB_SEARCH_INCLUDE_DOMAINS", "")
+    WEB_SEARCH_INCLUDE_DOMAINS: tuple[str, ...] = tuple(
+        s.strip().lower() for s in _raw_inc.split(",") if s.strip()
+    )
+    _raw_exc = os.getenv("WEB_SEARCH_EXCLUDE_DOMAINS", "")
+    WEB_SEARCH_EXCLUDE_DOMAINS: tuple[str, ...] = tuple(
+        s.strip().lower() for s in _raw_exc.split(",") if s.strip()
+    )
+    KB_IGEL_BASE_URL: str = os.getenv("KB_IGEL_BASE_URL", "https://kb.igel.com")
+    COMMUNITY_BASE_URL: str = os.getenv("COMMUNITY_BASE_URL", "https://community.igel.com")
+    KB_IGEL_ENABLED: bool = _truthy(os.getenv("KB_IGEL_ENABLED"), True)
+    COMMUNITY_ENABLED: bool = _truthy(os.getenv("COMMUNITY_ENABLED"), True)
+
+    # ── Unified query orchestration (RAG + EKG + Web + MCP) ───────────────────
+    QUERY_EXTERNAL_ENABLED: bool = _truthy(os.getenv("QUERY_EXTERNAL_ENABLED"), True)
+    _raw_mcp = os.getenv("QUERY_MCP_SOURCES", "jira,confluence")
+    QUERY_MCP_SOURCES: tuple[str, ...] = tuple(
+        s.strip().lower() for s in _raw_mcp.split(",") if s.strip()
+    )
+    QUERY_GENERATION_TIMEOUT_SECONDS: int = int(os.getenv("QUERY_GENERATION_TIMEOUT_SECONDS", "90"))
+    ASYNC_THREAD_LIMIT: int = int(os.getenv("ASYNC_THREAD_LIMIT", "16"))
+
+    KG_ENABLED: bool = _truthy(os.getenv("KG_ENABLED"), True)
+    KG_MAX_HOPS: int = min(3, max(0, int(os.getenv("KG_MAX_HOPS", "2"))))
+    CONTEXT_INCLUDE_IMAGES: bool = _truthy(os.getenv("CONTEXT_INCLUDE_IMAGES"), True)
+
+    CONTEXT_TOKEN_BUDGET: int = int(os.getenv("CONTEXT_TOKEN_BUDGET", "8000"))
+    CONTEXT_RESERVE_PROMPT: int = int(os.getenv("CONTEXT_RESERVE_PROMPT", "800"))
+    _raw_ms = os.getenv("CONTEXT_MIN_SCORE")
+    if _raw_ms is None or str(_raw_ms).strip() == "":
+        CONTEXT_MIN_SCORE: float | None = None
+    else:
+        CONTEXT_MIN_SCORE = float(_raw_ms)
+    CONTEXT_MAX_CHUNKS: int = int(os.getenv("CONTEXT_MAX_CHUNKS", "10"))
+    CONTEXT_KG_BUDGET_RATIO: float = float(os.getenv("CONTEXT_KG_BUDGET_RATIO", "0.22"))
+    CONTEXT_IMAGE_BUDGET_RATIO: float = float(
+        os.getenv("CONTEXT_IMAGE_BUDGET_RATIO", "0.08"),
+    )
+
+    GENERATION_RUNS_LOGGING: bool = _truthy(os.getenv("GENERATION_RUNS_LOGGING"), False)
+
+    # ── Evaluation (Phase 0) ─────────────────────────────────────────────────
+    EVAL_ENABLED: bool = _truthy(os.getenv("EVAL_ENABLED"), True)
+    BASELINE_PATH: Path = Path(
+        os.getenv(
+            "BASELINE_PATH",
+            str(Path(__file__).resolve().parent.parent / "eval" / "baseline.json"),
+        )
+    )
+    PRECEDENCE_RULES_PATH: Path = Path(
+        os.getenv(
+            "PRECEDENCE_RULES_PATH",
+            str(Path(__file__).resolve().parent.parent / "core" / "precedence.py"),
+        )
+    )
+
+    # ── Confidence thresholds (Phase 1) ──────────────────────────────────────
+    CONFIDENCE_REJECT_THRESHOLD: float = float(os.getenv("CONFIDENCE_REJECT_THRESHOLD", "0.4"))
+    CONFIDENCE_WARN_THRESHOLD: float = float(os.getenv("CONFIDENCE_WARN_THRESHOLD", "0.6"))
+
+    # ── Source filtering ─────────────────────────────────────────────────────
+    _raw_exclude = os.getenv("RETRIEVAL_EXCLUDE_SOURCES", "")
+    RETRIEVAL_EXCLUDE_SOURCES: tuple[str, ...] = tuple(
+        s.strip().lower() for s in _raw_exclude.split(",") if s.strip()
+    )
+
+    # ── Knowledge graph backend (Phase 3) ──────────────────────────────────
+    KG_BACKEND: str = os.getenv("KG_BACKEND", "postgres").strip().lower()
+    KG_SHADOW_MODE: bool = _truthy(os.getenv("KG_SHADOW_MODE"), False)
+    NEO4J_URI: str = os.getenv("NEO4J_URI", "bolt://localhost:7687")
+    NEO4J_USER: str = os.getenv("NEO4J_USER", "neo4j")
+    NEO4J_PASSWORD: str = os.getenv("NEO4J_PASSWORD", "")
+
+    # ── External connectors (Phase 8 — disabled by default) ──────────────────
+    JIRA_ENABLED: bool = _truthy(os.getenv("JIRA_ENABLED"), False)
+    JIRA_URL: str = os.getenv("JIRA_URL", "")
+    JIRA_USERNAME: str = os.getenv("JIRA_USERNAME", "")
+    JIRA_API_TOKEN: str = os.getenv("JIRA_API_TOKEN", "")
+
+    CONFLUENCE_ENABLED: bool = _truthy(os.getenv("CONFLUENCE_ENABLED"), False)
+    CONFLUENCE_URL: str = os.getenv("CONFLUENCE_URL", "")
+    CONFLUENCE_USERNAME: str = os.getenv("CONFLUENCE_USERNAME", "")
+    CONFLUENCE_API_TOKEN: str = os.getenv("CONFLUENCE_API_TOKEN", "")
+
+    BITBUCKET_ENABLED: bool = _truthy(os.getenv("BITBUCKET_ENABLED"), False)
+    TEAMCITY_ENABLED: bool = _truthy(os.getenv("TEAMCITY_ENABLED"), False)
+    TESTRAIL_ENABLED: bool = _truthy(os.getenv("TESTRAIL_ENABLED"), False)
+    SERVICENOW_ENABLED: bool = _truthy(os.getenv("SERVICENOW_ENABLED"), False)
+
+    # ── Operational memory (Phase 4) ─────────────────────────────────────────
+    COGNEE_ENABLED: bool = _truthy(os.getenv("COGNEE_ENABLED"), True)
+
+    # ── Temporal graph (Phase 6) ─────────────────────────────────────────────
+    GRAPHITI_ENABLED: bool = _truthy(os.getenv("GRAPHITI_ENABLED"), True)
+
+    # ── Agent orchestration (Phase 5) ────────────────────────────────────────
+    AGENTIC_ENABLED: bool = _truthy(os.getenv("AGENTIC_ENABLED"), True)
+
+    # ── Infrastructure (Phase 9) ─────────────────────────────────────────────
+    REDIS_URL: str = os.getenv("REDIS_URL", "redis://localhost:6379/0")
+    MINIO_ENDPOINT: str = os.getenv("MINIO_ENDPOINT", "localhost:9000")
+    MINIO_ACCESS_KEY: str = os.getenv("MINIO_ACCESS_KEY", "")
+    MINIO_SECRET_KEY: str = os.getenv("MINIO_SECRET_KEY", "")
+    API_KEY: str = os.getenv("API_KEY", "")
+    API_KEYS: str = os.getenv("API_KEYS", "")
+    API_KEY_REQUIRED: bool = _truthy(os.getenv("API_KEY_REQUIRED"), True)
+    RATE_LIMIT_ENABLED: bool = _truthy(os.getenv("RATE_LIMIT_ENABLED"), True)
+    JSON_LOGS: bool = _truthy(os.getenv("JSON_LOGS"), True)
+    DB_SSLMODE: str = os.getenv("DB_SSLMODE", "prefer")
+
+    # ── Retrieval optimization (Phase D) ─────────────────────────────────────
+    FTS_LANGUAGE: str = os.getenv("FTS_LANGUAGE", "english")
+    EMBED_CACHE_TTL: int = int(os.getenv("EMBED_CACHE_TTL", "3600"))
+    RETRIEVAL_CACHE_TTL: int = int(os.getenv("RETRIEVAL_CACHE_TTL", "300"))
+    QUERY_TIMEOUT_SECONDS: int = int(os.getenv("QUERY_TIMEOUT_SECONDS", "60"))
+    RETRIEVAL_TRACE_SAMPLE_RATE: float = float(os.getenv("RETRIEVAL_TRACE_SAMPLE_RATE", "0.1"))
+
+    # ── Generation modes (Phase I) ───────────────────────────────────────────
+    DEFAULT_GENERATION_MODE: str = os.getenv("DEFAULT_GENERATION_MODE", "pipeline")
+
+    @property
+    def api_key_map(self) -> dict[str, str]:
+        from knowledge_base.core.request_context import parse_api_keys
+        keys = parse_api_keys(self.API_KEYS)
+        if self.API_KEY and self.API_KEY not in keys:
+            keys[self.API_KEY] = "admin"
+        return keys
+
+    @property
+    def db_dsn(self) -> str:
+        return (
+            f"host={self.DB_HOST} port={self.DB_PORT} "
+            f"dbname={self.DB_NAME} user={self.DB_USER} password={self.DB_PASSWORD}"
+        )
+
+
+cfg = Config()

package/knowledge_base/connectors/__init__.py

@@ -0,0 +1,5 @@
+"""Connector package — Phase 8 (stubbed, config-driven)."""
+
+from knowledge_base.connectors.base import BaseConnector
+
+__all__ = ["BaseConnector"]

package/knowledge_base/connectors/base.py

@@ -0,0 +1,44 @@
+"""Base connector interface — Phase 8."""
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from datetime import datetime
+from typing import Any
+
+
+class BaseConnector(ABC):
+    """All external connectors implement this interface."""
+
+    name: str = "base"
+    enabled: bool = False
+    supports_sync: bool = True
+
+    @abstractmethod
+    def connect(self) -> bool:
+        """Test connection to external system."""
+
+    @abstractmethod
+    def sync(self, since: datetime | None = None) -> list[dict]:
+        """Fetch new/updated items since last sync."""
+
+    @abstractmethod
+    def ingest_to_kg(self, items: list[dict]) -> int:
+        """Store items in knowledge graph. Returns count ingested."""
+
+    @abstractmethod
+    def ingest_to_rag(self, items: list[dict]) -> int:
+        """Store items in PostgreSQL vector store. Returns count ingested."""
+
+    def search(self, query: str, limit: int = 5, timeout: float | None = None) -> list[str]:
+        """Optional query-time URL search for MCP bridge connectors."""
+        return []
+
+    def fetch_page(self, url: str, timeout: float | None = None) -> dict[str, Any] | None:
+        """Optional query-time page fetch for MCP bridge connectors."""
+        return None
+
+    def health_check(self) -> dict[str, Any]:
+        if not self.enabled:
+            return {"status": "disabled", "connector": self.name}
+        ok = self.connect()
+        return {"status": "healthy" if ok else "error", "connector": self.name}

package/knowledge_base/connectors/bitbucket_connector.py

@@ -0,0 +1,25 @@
+"""Bitbucket connector — Phase 8 stub."""
+from __future__ import annotations
+
+from datetime import datetime
+
+from knowledge_base.config import cfg
+from knowledge_base.connectors.base import BaseConnector
+
+
+class BitbucketConnector(BaseConnector):
+    name = "bitbucket"
+    enabled = cfg.BITBUCKET_ENABLED
+    supports_sync = False
+
+    def connect(self) -> bool:
+        return self.enabled
+
+    def sync(self, since: datetime | None = None) -> list[dict]:
+        return []
+
+    def ingest_to_kg(self, items: list[dict]) -> int:
+        return 0
+
+    def ingest_to_rag(self, items: list[dict]) -> int:
+        return 0

package/knowledge_base/connectors/community_connector.py

@@ -0,0 +1,112 @@
+"""IGEL Community / forum connector — fetch public community content."""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+from typing import Any
+from urllib.parse import quote_plus
+
+from knowledge_base.config import cfg
+from knowledge_base.connectors.base import BaseConnector
+from knowledge_base.connectors.web_fetcher import fetch_url, search_confluence_site
+from knowledge_base.parsers.html_parser import parse_html_page
+
+logger = logging.getLogger(__name__)
+
+
+class CommunityConnector(BaseConnector):
+    """Fetch content from IGEL community sites (Discourse / forum pages)."""
+
+    name = "community"
+
+    def __init__(self) -> None:
+        self.base_url = getattr(
+            cfg,
+            "COMMUNITY_BASE_URL",
+            "https://community.igel.com",
+        ).rstrip("/")
+        self.enabled = getattr(cfg, "COMMUNITY_ENABLED", True)
+
+    def connect(self) -> bool:
+        result = fetch_url(self.base_url, cache_ttl=300)
+        return result.status_code in (200, 301, 302)
+
+    def search(self, query: str, *, limit: int | None = None, timeout: float | None = None) -> list[str]:
+        max_pages = limit or getattr(cfg, "WEB_MAX_PAGES", 3)
+        urls: list[str] = []
+        fetch_timeout = min(20.0, timeout) if timeout and timeout > 0 else 20.0
+
+        # Try community search endpoint
+        search_url = f"{self.base_url}/search?q={quote_plus(query)}"
+        result = fetch_url(search_url, cache_ttl=600, timeout=fetch_timeout)
+        if result.status_code == 200 and result.html:
+            import re
+
+            for href in re.findall(r'href="(/t/[^"]+)"', result.html):
+                full = f"{self.base_url}{href.split('?')[0]}"
+                if full not in urls:
+                    urls.append(full)
+                if len(urls) >= max_pages:
+                    return urls
+
+        # Fallback: also search official KB for community-tagged content
+        kb_urls = search_confluence_site(
+            getattr(cfg, "KB_IGEL_BASE_URL", "https://kb.igel.com"),
+            f"{query} community",
+            limit=max(1, max_pages // 2),
+        )
+        urls.extend(kb_urls)
+        return urls[:max_pages]
+
+    def fetch_page(self, url: str, timeout: float | None = None) -> dict[str, Any] | None:
+        fetch_timeout = min(20.0, timeout) if timeout and timeout > 0 else 20.0
+        result = fetch_url(
+            url,
+            cache_ttl=getattr(cfg, "WEB_CACHE_TTL_SECONDS", 3600),
+            timeout=fetch_timeout,
+        )
+        if result.status_code != 200 or not result.html:
+            return None
+        parsed = parse_html_page(result.html, url, source_type="community")
+        return {
+            "url": url,
+            "title": parsed.title,
+            "sections": parsed.sections,
+            "raw_text": parsed.raw_text,
+            "source_type": "community",
+            "product": "IGEL",
+        }
+
+    def sync(self, since: datetime | None = None) -> list[dict]:
+        return []
+
+    def ingest_to_kg(self, items: list[dict]) -> int:
+        if not items:
+            return 0
+        from knowledge_base.db.client import get_conn
+        import json
+        count = 0
+        with get_conn() as conn:
+            with conn.cursor() as cur:
+                for item in items:
+                    title = item.get("title") or item.get("url", "")
+                    if not title:
+                        continue
+                    payload = dict(item)
+                    payload.setdefault("source", "community")
+                    cur.execute(
+                        """
+                        INSERT INTO knowledge_graph_nodes (node_type, value, normalized, metadata)
+                        VALUES ('feature', %s, %s, %s::jsonb)
+                        ON CONFLICT (node_type, normalized) DO UPDATE SET metadata = EXCLUDED.metadata
+                        """,
+                        (title[:4000], title.lower()[:2000], json.dumps(payload)),
+                    )
+                    count += 1
+        return count
+
+    def ingest_to_rag(self, items: list[dict]) -> int:
+        from knowledge_base.ingestion.web_ingest import ingest_web_items
+
+        return ingest_web_items(items, source_label="community.igel.com")