igel-qe-core 1.0.0

package/knowledge_base/db/schema.py

@@ -0,0 +1,649 @@
+"""
+Creates and manages the PostgreSQL schema for the IGEL Knowledge Base.
+Tables:
+  kb_documents     - one row per source file
+  kb_parent_chunks - large sections (800-1200 tokens), returned as context
+  kb_chunks        - small indexed units (150-300 tokens) with vector embeddings
+  feedback_events  - user feedback for continuous learning (Phase 2)
+  eval_runs        - evaluation run summaries (Phase 0)
+  eval_results     - per-query evaluation results (Phase 0)
+"""
+from __future__ import annotations
+from knowledge_base.db.client import get_conn
+from knowledge_base.config import cfg
+
+_SCHEMA_SQL = f"""
+-- Enable required extensions
+CREATE EXTENSION IF NOT EXISTS vector;
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+
+-- Source documents registry
+CREATE TABLE IF NOT EXISTS kb_documents (
+    id              UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    file_name       TEXT NOT NULL,
+    file_path       TEXT NOT NULL,
+    file_type       TEXT NOT NULL,
+    file_hash       TEXT UNIQUE NOT NULL,
+    product         TEXT,
+    total_parent_chunks INT DEFAULT 0,
+    total_chunks    INT DEFAULT 0,
+    ingested_at     TIMESTAMP DEFAULT NOW(),
+    metadata        JSONB DEFAULT '{{}}'::jsonb
+);
+
+-- Parent chunks: full sections returned during retrieval for full context
+CREATE TABLE IF NOT EXISTS kb_parent_chunks (
+    id              UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    document_id     UUID NOT NULL REFERENCES kb_documents(id) ON DELETE CASCADE,
+    chunk_index     INT NOT NULL,
+    content         TEXT NOT NULL,
+    section_title   TEXT,
+    chunk_type      TEXT,
+    page_number     INT,
+    metadata        JSONB DEFAULT '{{}}'::jsonb,
+    created_at      TIMESTAMP DEFAULT NOW()
+);
+
+-- Child chunks: indexed for vector + full-text search
+CREATE TABLE IF NOT EXISTS kb_chunks (
+    id              UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    parent_chunk_id UUID NOT NULL REFERENCES kb_parent_chunks(id) ON DELETE CASCADE,
+    document_id     UUID NOT NULL REFERENCES kb_documents(id) ON DELETE CASCADE,
+    chunk_index     INT NOT NULL,
+    content         TEXT NOT NULL,
+    embedding       VECTOR({cfg.EMBEDDING_DIM}),
+    token_count     INT,
+    contains_steps  BOOLEAN DEFAULT FALSE,
+    has_table       BOOLEAN DEFAULT FALSE,
+    chunk_type      TEXT,
+    metadata        JSONB DEFAULT '{{}}'::jsonb,
+    created_at      TIMESTAMP DEFAULT NOW()
+);
+
+-- Note: no ANN index on `embedding` — text-embedding-3-large is 3072-dim,
+-- which exceeds pgvector's 2000-dim cap for IVFFlat/HNSW with vector_cosine_ops.
+-- For this corpus size (~100 docs, a few thousand chunks) sequential scan
+-- via vector_cosine_ops operator class is fast enough. If we ever exceed
+-- ~50k chunks, switch to halfvec(3072) + HNSW (4000-dim cap).
+
+-- Full-text search index (BM25-style fallback)
+CREATE INDEX IF NOT EXISTS kb_chunks_fts_idx
+    ON kb_chunks USING gin(to_tsvector('english', content));
+
+-- Lookup indexes
+CREATE INDEX IF NOT EXISTS kb_chunks_doc_idx      ON kb_chunks(document_id);
+CREATE INDEX IF NOT EXISTS kb_chunks_parent_idx   ON kb_chunks(parent_chunk_id);
+CREATE INDEX IF NOT EXISTS kb_chunks_type_idx     ON kb_chunks(chunk_type);
+CREATE INDEX IF NOT EXISTS kb_parent_doc_idx      ON kb_parent_chunks(document_id);
+CREATE INDEX IF NOT EXISTS kb_docs_hash_idx       ON kb_documents(file_hash);
+CREATE INDEX IF NOT EXISTS kb_docs_product_idx    ON kb_documents(product);
+
+-- ─────────────────────────────────────────────────────────────────────────────
+-- Images table: every image extracted from any source document.
+-- Hard FK to kb_documents (cascades on doc delete) and soft FK to nearest section.
+-- Stores binary + multiple text representations (OCR-grade verbatim + semantic).
+-- ─────────────────────────────────────────────────────────────────────────────
+CREATE TABLE IF NOT EXISTS kb_images (
+    id                 UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    document_id        UUID NOT NULL REFERENCES kb_documents(id) ON DELETE CASCADE,
+    parent_chunk_id    UUID REFERENCES kb_parent_chunks(id) ON DELETE SET NULL,
+
+    -- Position / identity
+    page_number        INT,
+    sequence_index     INT NOT NULL,                  -- order across the whole document
+    bbox               JSONB,                         -- {{x0,y0,x1,y1}} on page (PDFs)
+    image_hash         TEXT NOT NULL,                 -- sha256 of bytes (per-document, not globally unique)
+    image_format       TEXT NOT NULL,                 -- png, jpeg, etc.
+    width              INT,
+    height             INT,
+    image_bytes        BYTEA NOT NULL,                -- raw image preserved
+
+    -- Captured text representations (no context loss)
+    image_type         TEXT,                          -- ui_screenshot, diagram, photo, icon, ...
+    verbatim_text      TEXT,                          -- every visible label/button/value
+    ui_elements        JSONB,                         -- structured controls + state
+    semantic_description TEXT,                        -- 2-4 sentence narrative
+    test_relevance     TEXT,                          -- what test cases this illustrates
+    surrounding_text   TEXT,                          -- paragraph above/below in source doc
+    caption            TEXT,                          -- explicit caption if any
+
+    -- Vector embedding of the combined description (for similarity search)
+    description_embedding VECTOR({cfg.EMBEDDING_DIM}),
+
+    metadata           JSONB DEFAULT '{{}}'::jsonb,
+    created_at         TIMESTAMP DEFAULT NOW(),
+
+    UNIQUE (document_id, sequence_index)              -- per-doc dedup; same image in different docs ok
+);
+
+CREATE INDEX IF NOT EXISTS kb_images_doc_idx        ON kb_images(document_id);
+CREATE INDEX IF NOT EXISTS kb_images_parent_idx     ON kb_images(parent_chunk_id);
+CREATE INDEX IF NOT EXISTS kb_images_page_idx       ON kb_images(document_id, page_number);
+CREATE INDEX IF NOT EXISTS kb_images_hash_idx       ON kb_images(image_hash);
+CREATE INDEX IF NOT EXISTS kb_images_fts_idx
+    ON kb_images USING gin(to_tsvector('english',
+        coalesce(verbatim_text,'') || ' ' ||
+        coalesce(semantic_description,'') || ' ' ||
+        coalesce(surrounding_text,'')));
+"""
+
+_KG_TABLES_SQL = """
+CREATE TABLE IF NOT EXISTS knowledge_graph_nodes (
+    id              UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    node_type       TEXT NOT NULL,
+    value           TEXT NOT NULL,
+    normalized      TEXT NOT NULL DEFAULT '',
+    metadata        JSONB DEFAULT '{}'::jsonb,
+    parent_chunk_id UUID REFERENCES kb_parent_chunks(id) ON DELETE SET NULL,
+    created_at      TIMESTAMP DEFAULT NOW(),
+    UNIQUE (node_type, normalized)
+);
+
+CREATE TABLE IF NOT EXISTS knowledge_graph_edges (
+    id              UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    source_node_id  UUID NOT NULL REFERENCES knowledge_graph_nodes(id) ON DELETE CASCADE,
+    target_node_id  UUID NOT NULL REFERENCES knowledge_graph_nodes(id) ON DELETE CASCADE,
+    relation        TEXT NOT NULL DEFAULT 'related_to',
+    weight          DOUBLE PRECISION DEFAULT 1.0,
+    metadata        JSONB DEFAULT '{}'::jsonb,
+    created_at      TIMESTAMP DEFAULT NOW(),
+    UNIQUE (source_node_id, target_node_id, relation)
+);
+"""
+
+_KG_INDEX_SQL = """
+CREATE INDEX IF NOT EXISTS ix_kgn_chunk ON knowledge_graph_nodes(parent_chunk_id);
+CREATE INDEX IF NOT EXISTS ix_kgn_type ON knowledge_graph_nodes(node_type);
+CREATE INDEX IF NOT EXISTS ix_kgn_norm ON knowledge_graph_nodes(normalized);
+CREATE INDEX IF NOT EXISTS ix_kge_src ON knowledge_graph_edges(source_node_id);
+CREATE INDEX IF NOT EXISTS ix_kge_tgt ON knowledge_graph_edges(target_node_id);
+"""
+
+# Backward compat alias
+_KG_GRAPH_SQL = _KG_TABLES_SQL + _KG_INDEX_SQL
+
+
+def ensure_knowledge_graph_tables() -> None:
+    """Create KG tables if absent; migrate legacy tables missing columns."""
+    with get_conn() as conn:
+        with conn.cursor() as cur:
+            cur.execute(_KG_TABLES_SQL)
+            cur.execute(
+                """
+                ALTER TABLE knowledge_graph_nodes
+                ADD COLUMN IF NOT EXISTS parent_chunk_id UUID
+                    REFERENCES kb_parent_chunks(id) ON DELETE SET NULL
+                """
+            )
+            cur.execute(
+                """
+                ALTER TABLE knowledge_graph_nodes
+                ADD COLUMN IF NOT EXISTS normalized TEXT NOT NULL DEFAULT ''
+                """
+            )
+            cur.execute(
+                """
+                ALTER TABLE knowledge_graph_nodes
+                ADD COLUMN IF NOT EXISTS created_at TIMESTAMP DEFAULT NOW()
+                """
+            )
+            cur.execute(_KG_INDEX_SQL)
+    try:
+        from knowledge_base.db.kg_introspect import clear_kg_layout_cache
+        clear_kg_layout_cache()
+    except Exception:
+        pass
+
+
+
+
+def create_schema() -> None:
+    """Ensure schema via Alembic migrations, with legacy DDL fallback."""
+    try:
+        from knowledge_base.db.migrations import run_migrations
+        run_migrations("head")
+    except Exception as exc:
+        import logging
+        logging.getLogger(__name__).warning(
+            "Alembic migration failed (%s) — applying legacy DDL fallback", exc,
+        )
+        with get_conn() as conn:
+            with conn.cursor() as cur:
+                cur.execute(_SCHEMA_SQL)
+        ensure_knowledge_graph_tables()
+        from knowledge_base.db.feedback_schema import ensure_feedback_tables
+        ensure_feedback_tables()
+    print("  Schema ready (Alembic + kb_documents, kb_parent_chunks, kb_chunks, knowledge_graph_*)")
+
+
+def drop_schema() -> None:
+    """Full reset — drops all KB tables. Use only for clean re-ingestion."""
+    sql = """
+    DROP TABLE IF EXISTS knowledge_graph_edges CASCADE;
+    DROP TABLE IF EXISTS knowledge_graph_nodes CASCADE;
+    DROP TABLE IF EXISTS kb_images CASCADE;
+    DROP TABLE IF EXISTS kb_chunks CASCADE;
+    DROP TABLE IF EXISTS kb_parent_chunks CASCADE;
+    DROP TABLE IF EXISTS kb_documents CASCADE;
+    """
+    with get_conn() as conn:
+        with conn.cursor() as cur:
+            cur.execute(sql)
+    print("  Schema dropped.")
+
+
+def document_exists(file_hash: str) -> str | None:
+    """Return document id if already ingested, else None."""
+    with get_conn() as conn:
+        with conn.cursor() as cur:
+            cur.execute(
+                "SELECT id FROM kb_documents WHERE file_hash = %s",
+                (file_hash,)
+            )
+            row = cur.fetchone()
+            return str(row[0]) if row else None
+
+
+def document_has_chunks(document_id: str) -> bool:
+    """True if document has at least one indexed child chunk."""
+    with get_conn() as conn:
+        with conn.cursor() as cur:
+            cur.execute(
+                "SELECT COALESCE(total_chunks, 0) FROM kb_documents WHERE id = %s",
+                (document_id,),
+            )
+            row = cur.fetchone()
+            return bool(row and row[0] and row[0] > 0)
+
+
+def insert_document(
+    file_name: str,
+    file_path: str,
+    file_type: str,
+    file_hash: str,
+    product: str,
+    metadata: dict,
+    *,
+    conn=None,
+) -> str:
+    sql = """
+    INSERT INTO kb_documents (file_name, file_path, file_type, file_hash, product, metadata)
+    VALUES (%s, %s, %s, %s, %s, %s::jsonb)
+    RETURNING id
+    """
+    import json
+
+    params = (file_name, file_path, file_type, file_hash, product, json.dumps(metadata))
+
+    def _exec(cur):
+        cur.execute(sql, params)
+        return str(cur.fetchone()[0])
+
+    if conn is not None:
+        with conn.cursor() as cur:
+            return _exec(cur)
+    with get_conn() as c:
+        with c.cursor() as cur:
+            return _exec(cur)
+
+
+def insert_parent_chunk(
+    document_id: str,
+    chunk_index: int,
+    content: str,
+    section_title: str,
+    chunk_type: str,
+    page_number: int,
+    metadata: dict,
+    *,
+    conn=None,
+) -> str:
+    sql = """
+    INSERT INTO kb_parent_chunks (document_id, chunk_index, content, section_title, chunk_type, page_number, metadata)
+    VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb)
+    RETURNING id
+    """
+    import json
+
+    params = (
+        document_id, chunk_index, content, section_title,
+        chunk_type, page_number, json.dumps(metadata),
+    )
+
+    def _exec(cur):
+        cur.execute(sql, params)
+        return str(cur.fetchone()[0])
+
+    if conn is not None:
+        with conn.cursor() as cur:
+            return _exec(cur)
+    with get_conn() as c:
+        with c.cursor() as cur:
+            return _exec(cur)
+
+
+def insert_chunks_batch(rows: list[dict], *, conn=None) -> None:
+    """Batch insert child chunks with embeddings."""
+    if not rows:
+        return
+    import json
+    from psycopg2.extras import execute_values
+
+    sql = """
+    INSERT INTO kb_chunks
+        (parent_chunk_id, document_id, chunk_index, content, embedding,
+         token_count, contains_steps, has_table, chunk_type, metadata)
+    VALUES %s
+    """
+    values = [
+        (
+            r["parent_chunk_id"],
+            r["document_id"],
+            r["chunk_index"],
+            r["content"],
+            r["embedding"],
+            r["token_count"],
+            r["contains_steps"],
+            r["has_table"],
+            r["chunk_type"],
+            json.dumps(r["metadata"]),
+        )
+        for r in rows
+    ]
+
+    def _exec(cur):
+        execute_values(cur, sql, values, template=None, page_size=50)
+
+    if conn is not None:
+        with conn.cursor() as cur:
+            _exec(cur)
+    else:
+        with get_conn() as c:
+            with c.cursor() as cur:
+                _exec(cur)
+
+
+def update_document_counts(document_id: str, parent_count: int, chunk_count: int, *, conn=None) -> None:
+    sql = "UPDATE kb_documents SET total_parent_chunks=%s, total_chunks=%s WHERE id=%s"
+    params = (parent_count, chunk_count, document_id)
+
+    def _exec(cur):
+        cur.execute(sql, params)
+
+    if conn is not None:
+        with conn.cursor() as cur:
+            _exec(cur)
+    else:
+        with get_conn() as c:
+            with c.cursor() as cur:
+                _exec(cur)
+
+
+def clear_document_chunks(document_id: str, *, conn=None) -> None:
+    """Remove all chunks for a document so it can be re-ingested safely."""
+    def _exec(cur):
+        cur.execute("DELETE FROM kb_chunks WHERE document_id = %s", (document_id,))
+        cur.execute("DELETE FROM kb_parent_chunks WHERE document_id = %s", (document_id,))
+        cur.execute(
+            "UPDATE kb_documents SET total_parent_chunks=0, total_chunks=0 WHERE id=%s",
+            (document_id,),
+        )
+
+    if conn is not None:
+        with conn.cursor() as cur:
+            _exec(cur)
+    else:
+        with get_conn() as c:
+            with c.cursor() as cur:
+                _exec(cur)
+
+
+def ingest_document_atomic(
+    *,
+    file_name: str,
+    file_path: str,
+    file_type: str,
+    file_hash: str,
+    product: str,
+    doc_metadata: dict,
+    parents: list,
+    existing_id: str | None,
+    embed_fn,
+    format_fn,
+    batch_size: int,
+) -> tuple[str, int, int, int]:
+    """
+    Atomically insert document + parent + child chunks in one transaction.
+    Returns (doc_id, parent_count, chunk_count, validation_skipped).
+    """
+    from knowledge_base.db.client import get_transaction
+
+    validation_skipped = 0
+    with get_transaction() as conn:
+        if existing_id:
+            doc_id = existing_id
+            clear_document_chunks(doc_id, conn=conn)
+        else:
+            doc_id = insert_document(
+                file_name=file_name,
+                file_path=file_path,
+                file_type=file_type,
+                file_hash=file_hash,
+                product=product,
+                metadata=doc_metadata,
+                conn=conn,
+            )
+
+        total_children = 0
+        for p_idx, parent in enumerate(parents):
+            parent_id = insert_parent_chunk(
+                document_id=doc_id,
+                chunk_index=p_idx,
+                content=parent.content,
+                section_title=parent.section_title,
+                chunk_type=parent.chunk_type,
+                page_number=parent.page_number,
+                metadata=parent.metadata,
+                conn=conn,
+            )
+            if not parent.children:
+                continue
+
+            valid_children = []
+            for child in parent.children:
+                if getattr(child, "validation", None) and not child.validation.passed:
+                    validation_skipped += 1
+                    continue
+                valid_children.append(child)
+            if not valid_children:
+                continue
+
+            child_texts = [c.content for c in valid_children]
+            embeddings = embed_fn(child_texts, batch_size=batch_size)
+            rows = []
+            for c_idx, (child, embedding) in enumerate(zip(valid_children, embeddings)):
+                rows.append({
+                    "parent_chunk_id": parent_id,
+                    "document_id": doc_id,
+                    "chunk_index": total_children + c_idx,
+                    "content": child.content,
+                    "embedding": format_fn(embedding),
+                    "token_count": child.token_count,
+                    "contains_steps": child.contains_steps,
+                    "has_table": child.has_table,
+                    "chunk_type": child.chunk_type,
+                    "metadata": {**child.metadata, "parent_chunk_index": p_idx},
+                })
+            insert_chunks_batch(rows, conn=conn)
+            total_children += len(rows)
+
+        update_document_counts(doc_id, len(parents), total_children, conn=conn)
+    return doc_id, len(parents), total_children, validation_skipped
+
+
+def get_stats() -> dict:
+    with get_conn() as conn:
+        with conn.cursor() as cur:
+            cur.execute("SELECT COUNT(*) FROM kb_documents")
+            docs = cur.fetchone()[0]
+            cur.execute("SELECT COUNT(*) FROM kb_parent_chunks")
+            parents = cur.fetchone()[0]
+            cur.execute("SELECT COUNT(*) FROM kb_chunks")
+            chunks = cur.fetchone()[0]
+            cur.execute("SELECT COUNT(*) FROM kb_chunks WHERE embedding IS NULL")
+            missing_emb = cur.fetchone()[0]
+            try:
+                cur.execute("SELECT COUNT(*) FROM kb_images")
+                images = cur.fetchone()[0]
+                cur.execute("SELECT COUNT(*) FROM kb_images WHERE description_embedding IS NULL")
+                images_missing_emb = cur.fetchone()[0]
+            except Exception:
+                images, images_missing_emb = 0, 0
+
+            # Knowledge Graph stats
+            kg_nodes = kg_edges = kg_orphan_parents = 0
+            try:
+                cur.execute("SELECT COUNT(*) FROM knowledge_graph_nodes")
+                kg_nodes = cur.fetchone()[0]
+                cur.execute("SELECT COUNT(*) FROM knowledge_graph_edges")
+                kg_edges = cur.fetchone()[0]
+                cur.execute("""
+                    SELECT COUNT(*) FROM kb_parent_chunks p
+                    WHERE NOT EXISTS (
+                        SELECT 1 FROM knowledge_graph_nodes n
+                        WHERE n.parent_chunk_id = p.id
+                    )
+                """)
+                kg_orphan_parents = cur.fetchone()[0]
+            except Exception:
+                pass
+
+    return {
+        "documents": docs,
+        "parent_chunks": parents,
+        "chunks": chunks,
+        "missing_embeddings": missing_emb,
+        "images": images,
+        "images_missing_embeddings": images_missing_emb,
+        "kg_nodes": kg_nodes,
+        "kg_edges": kg_edges,
+        "kg_orphan_parents (no KG entity)": kg_orphan_parents,
+    }
+
+
+# ── Image helpers ─────────────────────────────────────────────────────────────
+
+def insert_image(
+    document_id: str,
+    parent_chunk_id: str | None,
+    sequence_index: int,
+    page_number: int | None,
+    bbox: dict | None,
+    image_hash: str,
+    image_format: str,
+    width: int | None,
+    height: int | None,
+    image_bytes: bytes,
+    image_type: str,
+    verbatim_text: str,
+    ui_elements: list | dict | None,
+    semantic_description: str,
+    test_relevance: str,
+    surrounding_text: str,
+    caption: str | None,
+    description_embedding: str,
+    metadata: dict,
+) -> str:
+    import json
+    sql = """
+    INSERT INTO kb_images (
+        document_id, parent_chunk_id, sequence_index, page_number, bbox,
+        image_hash, image_format, width, height, image_bytes,
+        image_type, verbatim_text, ui_elements, semantic_description,
+        test_relevance, surrounding_text, caption,
+        description_embedding, metadata
+    ) VALUES (
+        %s, %s, %s, %s, %s::jsonb,
+        %s, %s, %s, %s, %s,
+        %s, %s, %s::jsonb, %s,
+        %s, %s, %s,
+        %s, %s::jsonb
+    )
+    ON CONFLICT (document_id, sequence_index) DO UPDATE SET
+        parent_chunk_id      = EXCLUDED.parent_chunk_id,
+        page_number          = EXCLUDED.page_number,
+        bbox                 = EXCLUDED.bbox,
+        image_hash           = EXCLUDED.image_hash,
+        image_format         = EXCLUDED.image_format,
+        width                = EXCLUDED.width,
+        height               = EXCLUDED.height,
+        image_bytes          = EXCLUDED.image_bytes,
+        image_type           = EXCLUDED.image_type,
+        verbatim_text        = EXCLUDED.verbatim_text,
+        ui_elements          = EXCLUDED.ui_elements,
+        semantic_description = EXCLUDED.semantic_description,
+        test_relevance       = EXCLUDED.test_relevance,
+        surrounding_text     = EXCLUDED.surrounding_text,
+        caption              = EXCLUDED.caption,
+        description_embedding = EXCLUDED.description_embedding,
+        metadata             = EXCLUDED.metadata
+    RETURNING id
+    """
+    with get_conn() as conn:
+        with conn.cursor() as cur:
+            cur.execute(sql, (
+                document_id, parent_chunk_id, sequence_index, page_number,
+                json.dumps(bbox) if bbox else None,
+                image_hash, image_format, width, height, image_bytes,
+                image_type, verbatim_text,
+                json.dumps(ui_elements) if ui_elements is not None else None,
+                semantic_description, test_relevance, surrounding_text, caption,
+                description_embedding,
+                json.dumps(metadata),
+            ))
+            return str(cur.fetchone()[0])
+
+
+def find_nearest_parent_chunk(document_id: str, page_number: int | None) -> str | None:
+    """Return the kb_parent_chunks.id whose page_number is closest to the given page,
+    so an image can be linked to the most relevant section."""
+    if page_number is None:
+        # Just pick the first parent chunk for this document
+        sql = "SELECT id FROM kb_parent_chunks WHERE document_id = %s ORDER BY chunk_index ASC LIMIT 1"
+        params = (document_id,)
+    else:
+        sql = """
+        SELECT id FROM kb_parent_chunks
+        WHERE document_id = %s
+        ORDER BY ABS(COALESCE(page_number, 0) - %s) ASC, chunk_index ASC
+        LIMIT 1
+        """
+        params = (document_id, page_number)
+    with get_conn() as conn:
+        with conn.cursor() as cur:
+            cur.execute(sql, params)
+            row = cur.fetchone()
+    return str(row[0]) if row else None
+
+
+def list_documents_for_image_pass() -> list[dict]:
+    """Documents already ingested as text — for the image pass to iterate over."""
+    sql = """
+    SELECT id, file_name, file_path, file_type
+    FROM kb_documents
+    ORDER BY ingested_at ASC
+    """
+    with get_conn() as conn:
+        with conn.cursor() as cur:
+            cur.execute(sql)
+            return [
+                {"id": str(r[0]), "file_name": r[1], "file_path": r[2], "file_type": r[3]}
+                for r in cur.fetchall()
+            ]
+
+
+def document_has_images(document_id: str) -> bool:
+    with get_conn() as conn:
+        with conn.cursor() as cur:
+            cur.execute("SELECT 1 FROM kb_images WHERE document_id = %s LIMIT 1", (document_id,))
+            return cur.fetchone() is not None

package/knowledge_base/deploy/Dockerfile

@@ -0,0 +1,23 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y --no-install-recommends curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY knowledge_base/ knowledge_base/
+COPY alembic.ini alembic.ini
+
+COPY knowledge_base/deploy/requirements.txt requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+
+ENV PYTHONPATH=/app
+ENV TESTGEN_HOST=0.0.0.0
+ENV TESTGEN_PORT=8000
+
+EXPOSE 8000
+
+HEALTHCHECK --interval=30s --timeout=10s --start-period=120s --retries=3 \
+  CMD curl -f http://localhost:8000/health || exit 1
+
+CMD ["uvicorn", "knowledge_base.api.server:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "2"]

package/knowledge_base/deploy/feedback_automation.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Apply pending feedback promotions and ingest to operational memory.
+set -euo pipefail
+cd "$(dirname "$0")/../.."
+export PYTHONPATH="${PWD}:${PYTHONPATH:-}"
+/usr/bin/python3.8 - <<'PY'
+from knowledge_base.db.client import init_pool, close_pool
+from knowledge_base.feedback.applier import apply_pending_promotions
+from knowledge_base.memory.ingest_memory import ingest_curated_feedback
+
+init_pool()
+results = apply_pending_promotions(limit=50)
+print(f"Applied {len(results)} feedback promotions")
+if results:
+    stats = ingest_curated_feedback(limit=len(results))
+    print(f"Memory ingest: {stats}")
+close_pool()
+PY