igel-qe-core 1.0.0

package/knowledge_base/alembic/env.py

@@ -0,0 +1,76 @@
+"""
+Alembic Environment Configuration
+==================================
+Connects Alembic to PostgreSQL database for migrations.
+"""
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+
+# Import DB configuration
+import sys
+from pathlib import Path
+
+# Add parent directory to path
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+from knowledge_base.config import cfg
+
+# Alembic Config object
+config = context.config
+
+# Set up logging
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# Override sqlalchemy.url with our config
+config.set_main_option(
+    "sqlalchemy.url",
+    f"postgresql://{cfg.DB_USER}:{cfg.DB_PASSWORD}@{cfg.DB_HOST}:{cfg.DB_PORT}/{cfg.DB_NAME}"
+)
+
+# Target metadata (not used for offline mode, but required)
+target_metadata = None
+
+
+def run_migrations_offline() -> None:
+    """
+    Run migrations in 'offline' mode (generate SQL script).
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """
+    Run migrations in 'online' mode (execute against DB).
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

package/knowledge_base/alembic/script.py.mako

@@ -0,0 +1,5 @@
+"""
+Alembic migration script placeholder.
+=====================================
+Place migration scripts in knowledge_base/alembic/versions/
+"""

package/knowledge_base/alembic/versions/.gitkeep

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

package/knowledge_base/alembic/versions/001_phase0_feedback.py

@@ -0,0 +1,122 @@
+"""Add feedback and evaluation tables for Phase 0
+
+Revision ID: 001_phase0_feedback
+Revises: 
+Create Date: 2024-01-15 10:00:00.000000
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = '001_phase0_feedback'
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Add feedback_events, eval_runs, and eval_results tables."""
+    
+    # feedback_events table
+    op.create_table(
+        'feedback_events',
+        sa.Column('id', postgresql.UUID(as_uuid=True), server_default=sa.text('uuid_generate_v4()'), primary_key=True),
+        sa.Column('query', sa.Text(), nullable=False),
+        sa.Column('generated_output', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
+        sa.Column('generation_metadata', postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False),
+        sa.Column('feedback_type', sa.Text(), nullable=False),
+        sa.Column('feedback_data', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
+        sa.Column('user_id', sa.Text(), nullable=True),
+        sa.Column('session_id', sa.Text(), nullable=True),
+        sa.Column('status', sa.Text(), nullable=False, server_default='pending'),
+        sa.Column('curator_id', sa.Text(), nullable=True),
+        sa.Column('curator_notes', sa.Text(), nullable=True),
+        sa.Column('curated_at', sa.TIMESTAMP(), nullable=True),
+        sa.Column('applied_action', sa.Text(), nullable=True),
+        sa.Column('applied_at', sa.TIMESTAMP(), nullable=True),
+        sa.Column('created_at', sa.TIMESTAMP(), server_default=sa.text('NOW()'), nullable=False),
+        sa.Column('source_context', postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False),
+    )
+    
+    # Indexes for feedback_events
+    op.create_index('idx_feedback_status', 'feedback_events', ['status'])
+    op.create_index('idx_feedback_type', 'feedback_events', ['feedback_type'])
+    op.create_index('idx_feedback_created', 'feedback_events', ['created_at'], postgresql_using='btree', postgresql_ops={'created_at': 'DESC'})
+    op.create_index('idx_feedback_curator', 'feedback_events', ['curator_id'])
+    op.create_index('idx_feedback_query', 'feedback_events', [sa.text("to_tsvector('english', query)")], postgresql_using='gin')
+    
+    # eval_runs table
+    op.create_table(
+        'eval_runs',
+        sa.Column('id', postgresql.UUID(as_uuid=True), server_default=sa.text('uuid_generate_v4()'), primary_key=True),
+        sa.Column('run_id', sa.Text(), nullable=False, unique=True),
+        sa.Column('run_type', sa.Text(), nullable=False),
+        sa.Column('avg_retrieval', sa.Float(), nullable=True),
+        sa.Column('avg_domain', sa.Float(), nullable=True),
+        sa.Column('avg_structural', sa.Float(), nullable=True),
+        sa.Column('avg_executability', sa.Float(), nullable=True),
+        sa.Column('avg_publish', sa.Float(), nullable=True),
+        sa.Column('avg_overall', sa.Float(), nullable=True),
+        sa.Column('pass_rate', sa.Float(), nullable=True),
+        sa.Column('query_count', sa.Integer(), nullable=False),
+        sa.Column('duration_seconds', sa.Float(), nullable=True),
+        sa.Column('system_version', sa.Text(), nullable=True),
+        sa.Column('metadata', postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False),
+        sa.Column('created_at', sa.TIMESTAMP(), server_default=sa.text('NOW()'), nullable=False),
+    )
+    
+    # Indexes for eval_runs
+    op.create_index('idx_eval_runs_id', 'eval_runs', ['run_id'])
+    op.create_index('idx_eval_runs_type', 'eval_runs', ['run_type'])
+    op.create_index('idx_eval_runs_created', 'eval_runs', ['created_at'], postgresql_using='btree', postgresql_ops={'created_at': 'DESC'})
+    
+    # eval_results table
+    op.create_table(
+        'eval_results',
+        sa.Column('id', postgresql.UUID(as_uuid=True), server_default=sa.text('uuid_generate_v4()'), primary_key=True),
+        sa.Column('eval_run_id', postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column('query_id', sa.Text(), nullable=False),
+        sa.Column('query', sa.Text(), nullable=False),
+        sa.Column('generated_output', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
+        sa.Column('retrieval_score', sa.Float(), nullable=True),
+        sa.Column('retrieval_feedback', postgresql.ARRAY(sa.Text()), nullable=True),
+        sa.Column('domain_score', sa.Float(), nullable=True),
+        sa.Column('domain_feedback', postgresql.ARRAY(sa.Text()), nullable=True),
+        sa.Column('structural_score', sa.Float(), nullable=True),
+        sa.Column('structural_feedback', postgresql.ARRAY(sa.Text()), nullable=True),
+        sa.Column('executability_score', sa.Float(), nullable=True),
+        sa.Column('executability_feedback', postgresql.ARRAY(sa.Text()), nullable=True),
+        sa.Column('publish_score', sa.Float(), nullable=True),
+        sa.Column('publish_feedback', postgresql.ARRAY(sa.Text()), nullable=True),
+        sa.Column('overall_score', sa.Float(), nullable=True),
+        sa.Column('passed', sa.Boolean(), nullable=True),
+        sa.Column('duration_seconds', sa.Float(), nullable=True),
+        sa.Column('metadata', postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False),
+        sa.Column('created_at', sa.TIMESTAMP(), server_default=sa.text('NOW()'), nullable=False),
+    )
+    
+    # Foreign key constraint for eval_results
+    op.create_foreign_key(
+        'fk_eval_results_run',
+        'eval_results',
+        'eval_runs',
+        ['eval_run_id'],
+        ['id'],
+        ondelete='CASCADE'
+    )
+    
+    # Indexes for eval_results
+    op.create_index('idx_eval_results_run', 'eval_results', ['eval_run_id'])
+    op.create_index('idx_eval_results_passed', 'eval_results', ['passed'])
+    op.create_index('idx_eval_results_overall', 'eval_results', ['overall_score'], postgresql_using='btree', postgresql_ops={'overall_score': 'DESC'})
+
+
+def downgrade() -> None:
+    """Remove feedback and evaluation tables."""
+    op.drop_table('eval_results')
+    op.drop_table('eval_runs')
+    op.drop_table('feedback_events')

package/knowledge_base/alembic/versions/002_feedback_promotions.py

@@ -0,0 +1,60 @@
+"""Add feedback_promotions and feedback_audit_trail tables (Phase 0 completion)
+
+Revision ID: 002_feedback_promotions
+Revises: 001_phase0_feedback
+Create Date: 2026-05-22
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+revision: str = "002_feedback_promotions"
+down_revision: Union[str, None] = "001_phase0_feedback"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "feedback_promotions",
+        sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("uuid_generate_v4()"), primary_key=True),
+        sa.Column("feedback_event_id", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column("curator_id", sa.Text(), nullable=False),
+        sa.Column("curator_notes", sa.Text(), nullable=True),
+        sa.Column("promoted_content", postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False),
+        sa.Column("status", sa.Text(), server_default="pending", nullable=False),
+        sa.Column("applied_at", sa.TIMESTAMP(), nullable=True),
+        sa.Column("created_at", sa.TIMESTAMP(), server_default=sa.text("NOW()"), nullable=False),
+        sa.ForeignKeyConstraint(["feedback_event_id"], ["feedback_events.id"], ondelete="CASCADE"),
+    )
+    op.create_index("idx_feedback_promotions_status", "feedback_promotions", ["status"])
+    op.create_index("idx_feedback_promotions_event", "feedback_promotions", ["feedback_event_id"])
+
+    op.create_table(
+        "feedback_audit_trail",
+        sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("uuid_generate_v4()"), primary_key=True),
+        sa.Column("promotion_id", postgresql.UUID(as_uuid=True), nullable=True),
+        sa.Column("feedback_event_id", postgresql.UUID(as_uuid=True), nullable=True),
+        sa.Column("action", sa.Text(), nullable=False),
+        sa.Column("actor_id", sa.Text(), nullable=True),
+        sa.Column("details", postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False),
+        sa.Column("created_at", sa.TIMESTAMP(), server_default=sa.text("NOW()"), nullable=False),
+        sa.ForeignKeyConstraint(["promotion_id"], ["feedback_promotions.id"], ondelete="SET NULL"),
+        sa.ForeignKeyConstraint(["feedback_event_id"], ["feedback_events.id"], ondelete="SET NULL"),
+    )
+    op.create_index("idx_feedback_audit_promotion", "feedback_audit_trail", ["promotion_id"])
+    op.create_index("idx_feedback_audit_event", "feedback_audit_trail", ["feedback_event_id"])
+    op.create_index(
+        "idx_feedback_audit_created",
+        "feedback_audit_trail",
+        ["created_at"],
+        postgresql_using="btree",
+        postgresql_ops={"created_at": "DESC"},
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("feedback_audit_trail")
+    op.drop_table("feedback_promotions")

package/knowledge_base/alembic/versions/003_enterprise_hardening.py

@@ -0,0 +1,74 @@
+"""Enterprise hardening migration — HNSW indexes, sync state, retrieval traces."""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+revision: str = "003_enterprise_hardening"
+down_revision: Union[str, None] = "002_feedback_promotions"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        CREATE TABLE IF NOT EXISTS connector_sync_state (
+            connector_name TEXT PRIMARY KEY,
+            last_sync_timestamp TIMESTAMPTZ,
+            cursor TEXT,
+            status TEXT DEFAULT 'idle',
+            metadata JSONB DEFAULT '{}'::jsonb,
+            updated_at TIMESTAMPTZ DEFAULT NOW()
+        )
+        """
+    )
+    op.execute(
+        """
+        CREATE TABLE IF NOT EXISTS retrieval_traces (
+            id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+            request_id TEXT,
+            query TEXT NOT NULL,
+            sources_summary JSONB DEFAULT '{}'::jsonb,
+            debug_json JSONB DEFAULT '{}'::jsonb,
+            timing_ms JSONB DEFAULT '{}'::jsonb,
+            created_at TIMESTAMPTZ DEFAULT NOW()
+        )
+        """
+    )
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS idx_retrieval_traces_created ON retrieval_traces (created_at DESC)"
+    )
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS idx_retrieval_traces_request ON retrieval_traces (request_id)"
+    )
+
+    # HNSW via halfvec for 3072-d embeddings (pgvector 0.7+)
+    op.execute(
+        """
+        DO $$
+        BEGIN
+            IF EXISTS (SELECT 1 FROM pg_type WHERE typname = 'halfvec') THEN
+                EXECUTE '
+                    CREATE INDEX IF NOT EXISTS idx_kb_chunks_embedding_hnsw
+                    ON kb_chunks USING hnsw ((embedding::halfvec(3072)) halfvec_cosine_ops)
+                    WITH (m = 16, ef_construction = 64)
+                ';
+                EXECUTE '
+                    CREATE INDEX IF NOT EXISTS idx_kb_images_embedding_hnsw
+                    ON kb_images USING hnsw ((description_embedding::halfvec(3072)) halfvec_cosine_ops)
+                    WITH (m = 16, ef_construction = 64)
+                ';
+            END IF;
+        EXCEPTION WHEN OTHERS THEN
+            RAISE NOTICE 'HNSW halfvec indexes skipped: %', SQLERRM;
+        END $$;
+        """
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP INDEX IF EXISTS idx_kb_images_embedding_hnsw")
+    op.execute("DROP INDEX IF EXISTS idx_kb_chunks_embedding_hnsw")
+    op.execute("DROP TABLE IF EXISTS retrieval_traces")
+    op.execute("DROP TABLE IF EXISTS connector_sync_state")

package/knowledge_base/api/__init__.py

@@ -0,0 +1 @@
+"""REST API package."""

package/knowledge_base/api/middleware.py

@@ -0,0 +1,78 @@
+"""FastAPI middleware: request tracing, rate limiting, metrics."""
+
+from __future__ import annotations
+
+import time
+from collections import defaultdict, deque
+
+from fastapi import HTTPException, Request
+from starlette.middleware.base import BaseHTTPMiddleware
+
+from knowledge_base.core.metrics import metrics
+from knowledge_base.core.request_context import get_request_id, hash_api_key, new_request_id
+from knowledge_base.config import cfg
+
+
+class RequestTracingMiddleware(BaseHTTPMiddleware):
+    async def dispatch(self, request: Request, call_next):
+        rid = request.headers.get("X-Request-ID") or new_request_id()
+        request.state.request_id = rid
+        start = time.perf_counter()
+        response = await call_next(request)
+        elapsed_ms = (time.perf_counter() - start) * 1000
+        response.headers["X-Request-ID"] = rid
+        metrics.inc("http_requests_total", labels={"path": request.url.path, "method": request.method})
+        metrics.observe("http_request_duration_ms", elapsed_ms, labels={"path": request.url.path})
+        return response
+
+
+class RateLimitMiddleware(BaseHTTPMiddleware):
+    """Simple in-memory rate limiter keyed by API key or client IP."""
+
+    _buckets: dict[str, deque[float]] = defaultdict(deque)
+    _last_gc: float = 0.0
+    _gc_interval_seconds: float = 60.0
+    _max_buckets: int = 20000
+
+    LIMITS = {
+        "/query": 30,
+        "/generate": 10,
+        "/convert": 10,
+        "/feedback": 60,
+        "/generate/v2": 10,
+    }
+
+    @classmethod
+    def _gc_buckets(cls, now: float, window: float) -> None:
+        """Prune stale buckets so cardinality stays bounded over long uptimes."""
+        if now - cls._last_gc < cls._gc_interval_seconds and len(cls._buckets) < cls._max_buckets:
+            return
+        stale_keys: list[str] = []
+        for key, q in list(cls._buckets.items()):
+            while q and now - q[0] > window:
+                q.popleft()
+            if not q:
+                stale_keys.append(key)
+        for key in stale_keys:
+            cls._buckets.pop(key, None)
+        cls._last_gc = now
+
+    async def dispatch(self, request: Request, call_next):
+        if not getattr(cfg, "RATE_LIMIT_ENABLED", True):
+            return await call_next(request)
+        path = request.url.path
+        limit = self.LIMITS.get(path)
+        if not limit:
+            return await call_next(request)
+        key = request.headers.get("X-API-Key") or request.client.host if request.client else "unknown"
+        bucket_key = f"{path}:{hash_api_key(key)}"
+        now = time.time()
+        window = 60.0
+        self._gc_buckets(now, window)
+        q = self._buckets[bucket_key]
+        while q and now - q[0] > window:
+            q.popleft()
+        if len(q) >= limit:
+            raise HTTPException(status_code=429, detail="Rate limit exceeded", headers={"Retry-After": "60"})
+        q.append(now)
+        return await call_next(request)