jumpstart-mode 1.1.13 → 1.1.14

package/.jumpstart/skills/postgresql-pgvector/scripts/check_embedding_dimensions.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+"""
+check_embedding_dimensions.py
+
+Validates that all vector columns in the database match the declared dimensions
+in the embedding_models registry. Reports mismatches and orphaned records.
+
+Usage:
+    DATABASE_URL=postgresql://app:app@localhost:5432/appdb python check_embedding_dimensions.py
+
+Exit codes:
+    0 — No dimension mismatches found
+    1 — One or more mismatches or validation errors detected
+Requires: pip install psycopg2-binary
+"""
+
+import os
+import sys
+
+import psycopg2
+import psycopg2.extras
+
+
+def main() -> None:
+    database_url = os.environ.get("DATABASE_URL")
+    if not database_url:
+        print("ERROR: DATABASE_URL is not set.", file=sys.stderr)
+        sys.exit(1)
+
+    conn_url = database_url.replace("postgresql+asyncpg://", "postgresql://")
+
+    try:
+        conn = psycopg2.connect(conn_url)
+    except psycopg2.OperationalError as exc:
+        print(f"ERROR: Connection failed.\n{exc}", file=sys.stderr)
+        sys.exit(1)
+
+    issues: list[str] = []
+
+    with conn:
+        with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
+
+            # ------------------------------------------------------------------
+            # 1. Check each embedding model's declared dimension
+            # ------------------------------------------------------------------
+            cur.execute("""
+                SELECT id, provider, model_name, dimensions
+                FROM embedding_models
+                WHERE is_active = TRUE;
+            """)
+            models = cur.fetchall()
+
+            if not models:
+                print("WARNING: No active embedding models found in embedding_models table.")
+            else:
+                print(f"Found {len(models)} active embedding model(s):\n")
+                for m in models:
+                    print(f"  {m['provider']} / {m['model_name']} — {m['dimensions']} dims")
+
+            # ------------------------------------------------------------------
+            # 2. Check actual vector column dimension using pg_attribute
+            # ------------------------------------------------------------------
+            cur.execute("""
+                SELECT
+                    a.attname AS column_name,
+                    t.typname AS type_name,
+                    a.atttypmod AS declared_dim
+                FROM pg_attribute a
+                JOIN pg_class c ON c.oid = a.attrelid
+                JOIN pg_type t ON t.oid = a.atttypid
+                WHERE c.relname = 'document_chunks'
+                  AND t.typname = 'vector'
+                  AND a.attnum > 0;
+            """)
+            vector_cols = cur.fetchall()
+
+            if not vector_cols:
+                print("\nWARNING: No vector columns found in document_chunks.")
+            else:
+                for col in vector_cols:
+                    # atttypmod for vector is dimension + 4 (internal offset)
+                    actual_dim = col["declared_dim"] - 4 if col["declared_dim"] > 0 else -1
+                    print(f"\nVector column '{col['column_name']}': declared dimension = {actual_dim}")
+                    for m in models:
+                        if m["dimensions"] != actual_dim:
+                            msg = (
+                                f"MISMATCH: Column '{col['column_name']}' has dimension {actual_dim} "
+                                f"but embedding model '{m['model_name']}' declares {m['dimensions']} dims."
+                            )
+                            issues.append(msg)
+                            print(f"  {msg}")
+                        else:
+                            print(f"  OK: Matches model '{m['model_name']}' ({m['dimensions']} dims)")
+
+            # ------------------------------------------------------------------
+            # 3. Check for chunks with NULL embeddings
+            # ------------------------------------------------------------------
+            cur.execute("""
+                SELECT COUNT(*) AS null_count
+                FROM document_chunks
+                WHERE embedding IS NULL AND is_deleted = FALSE;
+            """)
+            null_count = cur.fetchone()["null_count"]
+            if null_count > 0:
+                msg = f"WARNING: {null_count} active chunk(s) have NULL embeddings."
+                print(f"\n{msg}")
+                issues.append(msg)
+            else:
+                print(f"\nOK: No active chunks with NULL embeddings.")
+
+            # ------------------------------------------------------------------
+            # 4. Check for chunks not linked to a valid embedding model
+            # ------------------------------------------------------------------
+            cur.execute("""
+                SELECT COUNT(*) AS orphan_count
+                FROM document_chunks dc
+                LEFT JOIN embedding_models em ON em.id = dc.embedding_model_id
+                WHERE em.id IS NULL AND dc.is_deleted = FALSE;
+            """)
+            orphan_count = cur.fetchone()["orphan_count"]
+            if orphan_count > 0:
+                msg = f"WARNING: {orphan_count} active chunk(s) reference a missing or deleted embedding model."
+                print(msg)
+                issues.append(msg)
+            else:
+                print("OK: All active chunks are linked to valid embedding models.")
+
+    conn.close()
+
+    print("\n" + "=" * 60)
+    if issues:
+        print(f"RESULT: {len(issues)} issue(s) found. Review above output.")
+        sys.exit(1)
+    else:
+        print("RESULT: All embedding dimension checks passed.")
+        sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

package/.jumpstart/skills/postgresql-pgvector/scripts/explain_query_plan.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""
+explain_query_plan.py
+
+Runs EXPLAIN (ANALYZE, BUFFERS, FORMAT TEXT) on the standard pgvector
+semantic retrieval query and prints the formatted execution plan.
+
+Use this to verify index usage and diagnose slow queries.
+
+Usage:
+    DATABASE_URL=postgresql://app:app@localhost:5432/appdb python explain_query_plan.py
+
+Options (edit QUERY_PARAMS below):
+    TOP_K       — Number of results to retrieve
+    QUERY_TYPE  — 'semantic' or 'hybrid'
+
+Requires: pip install psycopg2-binary numpy
+"""
+
+import os
+import sys
+
+import numpy as np
+import psycopg2
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+TOP_K = 10
+QUERY_TYPE = "semantic"   # 'semantic' or 'hybrid'
+EMBEDDING_DIM = 1536
+SAMPLE_QUERY_TEXT = "AI Innovation Pod delivery model"
+# ---------------------------------------------------------------------------
+
+
+def random_embedding(dim: int) -> list[float]:
+    vec = np.random.randn(dim).astype(np.float32)
+    vec = vec / np.linalg.norm(vec)
+    return vec.tolist()
+
+
+SEMANTIC_SQL = """
+EXPLAIN (ANALYZE, BUFFERS, FORMAT TEXT)
+SELECT
+    dc.id,
+    dc.chunk_text,
+    1 - (dc.embedding <=> %(emb)s::vector) AS similarity
+FROM document_chunks dc
+WHERE dc.is_deleted = FALSE
+ORDER BY dc.embedding <=> %(emb)s::vector
+LIMIT %(k)s
+"""
+
+HYBRID_SQL = """
+EXPLAIN (ANALYZE, BUFFERS, FORMAT TEXT)
+WITH semantic AS (
+    SELECT id, 1 - (embedding <=> %(emb)s::vector) AS sem_score
+    FROM document_chunks
+    WHERE is_deleted = FALSE
+    ORDER BY embedding <=> %(emb)s::vector
+    LIMIT %(candidate_k)s
+),
+keyword AS (
+    SELECT id,
+           ts_rank_cd(
+               to_tsvector('english', chunk_text),
+               plainto_tsquery('english', %(query)s)
+           ) AS kw_score
+    FROM document_chunks
+    WHERE is_deleted = FALSE
+      AND to_tsvector('english', chunk_text) @@ plainto_tsquery('english', %(query)s)
+    LIMIT %(candidate_k)s
+),
+combined AS (
+    SELECT COALESCE(s.id, k.id) AS id,
+           COALESCE(s.sem_score, 0) * 0.7 + COALESCE(k.kw_score, 0) * 0.3 AS score
+    FROM semantic s
+    FULL OUTER JOIN keyword k ON k.id = s.id
+)
+SELECT dc.id, dc.chunk_text, c.score
+FROM combined c
+JOIN document_chunks dc ON dc.id = c.id
+ORDER BY c.score DESC
+LIMIT %(k)s
+"""
+
+
+def main() -> None:
+    database_url = os.environ.get("DATABASE_URL")
+    if not database_url:
+        print("ERROR: DATABASE_URL is not set.", file=sys.stderr)
+        sys.exit(1)
+
+    conn_url = database_url.replace("postgresql+asyncpg://", "postgresql://")
+
+    try:
+        conn = psycopg2.connect(conn_url)
+    except psycopg2.OperationalError as exc:
+        print(f"ERROR: Connection failed.\n{exc}", file=sys.stderr)
+        sys.exit(1)
+
+    embedding = random_embedding(EMBEDDING_DIM)
+
+    with conn:
+        with conn.cursor() as cur:
+            print(f"Query type  : {QUERY_TYPE.upper()}")
+            print(f"Top-k       : {TOP_K}")
+            print(f"Sample query: {SAMPLE_QUERY_TEXT}")
+            print("-" * 60)
+
+            if QUERY_TYPE == "semantic":
+                cur.execute(SEMANTIC_SQL, {"emb": str(embedding), "k": TOP_K})
+            else:
+                cur.execute(HYBRID_SQL, {
+                    "emb": str(embedding),
+                    "query": SAMPLE_QUERY_TEXT,
+                    "k": TOP_K,
+                    "candidate_k": TOP_K * 10,
+                })
+
+            plan_rows = cur.fetchall()
+            plan_text = "\n".join(row[0] for row in plan_rows)
+            print(plan_text)
+
+    conn.close()
+
+    # Heuristic check for index usage
+    if "Index Scan" in plan_text or "hnsw" in plan_text.lower():
+        print("\nOK: Query plan includes an index scan.")
+    else:
+        print(
+            "\nWARNING: No index scan detected. The query may be doing a sequential scan.\n"
+            "Check that the HNSW index exists and ANALYZE has been run on the table."
+        )
+
+
+if __name__ == "__main__":
+    main()

package/.jumpstart/skills/postgresql-pgvector/scripts/seed_sample_chunks.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+"""
+seed_sample_chunks.py
+
+Inserts sample documents, chunks, and synthetic embeddings into a local
+development database for testing and UI development.
+
+Usage:
+    DATABASE_URL=postgresql://app:app@localhost:5432/appdb python seed_sample_chunks.py
+
+Idempotent: skips insertion if a document with the same source_uri already exists.
+Requires: pip install psycopg2-binary numpy
+"""
+
+import os
+import sys
+import uuid
+
+import numpy as np
+import psycopg2
+import psycopg2.extras
+
+EMBEDDING_DIM = 1536
+
+SAMPLE_DOCUMENTS = [
+    {
+        "source_system": "seed",
+        "source_uri": "seed://doc/ai-innovation-pod-overview",
+        "title": "AI Innovation Pod Overview",
+        "file_type": "md",
+        "security_label": "internal",
+    },
+    {
+        "source_system": "seed",
+        "source_uri": "seed://doc/jumpstart-workshop-guide",
+        "title": "Jump Start Workshop Facilitation Guide",
+        "file_type": "md",
+        "security_label": "internal",
+    },
+    {
+        "source_system": "seed",
+        "source_uri": "seed://doc/pgvector-schema-reference",
+        "title": "pgvector Schema Reference",
+        "file_type": "md",
+        "security_label": "internal",
+    },
+]
+
+SAMPLE_CHUNKS = [
+    "The AI Innovation Pod is a mutualized delivery model that combines shared team capacity "
+    "with rapid prototyping to deliver production-grade AI applications.",
+    "Jump Start workshops are designed to accelerate discovery and scoping, leading to a "
+    "qualified opportunity for AI Innovation Pod follow-on work.",
+    "PostgreSQL pgvector enables high-performance vector similarity search using cosine, "
+    "L2, or inner product distance metrics.",
+    "HNSW indexes provide fast approximate nearest neighbor search with configurable "
+    "recall/latency tradeoffs via the ef_search parameter.",
+    "Row-level security in PostgreSQL can enforce tenant isolation at the database level, "
+    "ensuring that one tenant cannot access another tenant's documents.",
+    "Hybrid search combines vector similarity and full-text BM25-style ranking using "
+    "configurable weights to improve retrieval coverage.",
+    "The production readiness checklist includes index creation, RLS validation, embedding "
+    "dimension verification, and retrieval evaluation against a golden dataset.",
+    "Retrieval events should be logged for every query to enable offline evaluation, "
+    "latency monitoring, and user feedback correlation.",
+]
+
+
+def random_embedding(dim: int = EMBEDDING_DIM) -> list[float]:
+    vec = np.random.randn(dim).astype(np.float32)
+    vec = vec / np.linalg.norm(vec)
+    return vec.tolist()
+
+
+def main() -> None:
+    database_url = os.environ.get("DATABASE_URL")
+    if not database_url:
+        print("ERROR: DATABASE_URL is not set.", file=sys.stderr)
+        sys.exit(1)
+
+    conn_url = database_url.replace("postgresql+asyncpg://", "postgresql://")
+    conn = psycopg2.connect(conn_url)
+    psycopg2.extras.register_uuid()
+
+    with conn:
+        with conn.cursor() as cur:
+            # Insert or fetch embedding model
+            embedding_model_id = uuid.uuid4()
+            cur.execute("""
+                INSERT INTO embedding_models (id, provider, model_name, dimensions)
+                VALUES (%s, 'synthetic', 'random-1536', %s)
+                ON CONFLICT (provider, model_name, dimensions) DO NOTHING
+                RETURNING id;
+            """, (str(embedding_model_id), EMBEDDING_DIM))
+            row = cur.fetchone()
+            if row:
+                embedding_model_id = row[0]
+            else:
+                cur.execute("""
+                    SELECT id FROM embedding_models
+                    WHERE provider = 'synthetic' AND model_name = 'random-1536';
+                """)
+                embedding_model_id = cur.fetchone()[0]
+
+            total_chunks = 0
+
+            for doc_data in SAMPLE_DOCUMENTS:
+                # Check if document already exists
+                cur.execute(
+                    "SELECT id FROM documents WHERE source_uri = %s;",
+                    (doc_data["source_uri"],),
+                )
+                existing = cur.fetchone()
+                if existing:
+                    print(f"SKIP: Document '{doc_data['title']}' already exists.")
+                    continue
+
+                doc_id = uuid.uuid4()
+                cur.execute("""
+                    INSERT INTO documents
+                        (id, source_system, source_uri, title, file_type, security_label, metadata)
+                    VALUES (%s, %s, %s, %s, %s, %s, %s);
+                """, (
+                    str(doc_id),
+                    doc_data["source_system"],
+                    doc_data["source_uri"],
+                    doc_data["title"],
+                    doc_data["file_type"],
+                    doc_data["security_label"],
+                    psycopg2.extras.Json({}),
+                ))
+
+                # Insert 2-3 chunks per document
+                chunk_texts = SAMPLE_CHUNKS[total_chunks % len(SAMPLE_CHUNKS):
+                                            (total_chunks % len(SAMPLE_CHUNKS)) + 3]
+                for i, chunk_text in enumerate(chunk_texts):
+                    emb = random_embedding()
+                    cur.execute("""
+                        INSERT INTO document_chunks
+                            (id, document_id, embedding_model_id, chunk_index,
+                             chunk_text, token_count, embedding, metadata)
+                        VALUES (%s, %s, %s, %s, %s, %s, %s::vector, %s);
+                    """, (
+                        str(uuid.uuid4()),
+                        str(doc_id),
+                        str(embedding_model_id),
+                        i,
+                        chunk_text,
+                        len(chunk_text.split()),
+                        str(emb),
+                        psycopg2.extras.Json({"source_system": "seed"}),
+                    ))
+                    total_chunks += 1
+
+                print(f"INSERTED: Document '{doc_data['title']}' with {len(chunk_texts)} chunks.")
+
+    conn.close()
+    print(f"\nSeed complete. Total chunks inserted: {total_chunks}")
+
+
+if __name__ == "__main__":
+    main()

package/.jumpstart/skills/postgresql-pgvector/scripts/validate_pgvector.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""
+validate_pgvector.py
+
+Verifies that the pgvector extension is installed and accessible in the
+target PostgreSQL database.
+
+Usage:
+    DATABASE_URL=postgresql://app:app@localhost:5432/appdb python validate_pgvector.py
+
+Exit codes:
+    0 — pgvector extension found and version reported
+    1 — Extension not installed or connection failed
+"""
+
+import os
+import sys
+
+import psycopg2
+
+
+def main() -> None:
+    database_url = os.environ.get("DATABASE_URL")
+    if not database_url:
+        print("ERROR: DATABASE_URL environment variable is not set.", file=sys.stderr)
+        sys.exit(1)
+
+    # Replace asyncpg driver prefix if DATABASE_URL was copied from FastAPI config
+    conn_url = database_url.replace("postgresql+asyncpg://", "postgresql://")
+
+    try:
+        conn = psycopg2.connect(conn_url)
+    except psycopg2.OperationalError as exc:
+        print(f"ERROR: Could not connect to database.\n{exc}", file=sys.stderr)
+        sys.exit(1)
+
+    with conn:
+        with conn.cursor() as cur:
+            cur.execute("""
+                SELECT extname, extversion
+                FROM pg_extension
+                WHERE extname = 'vector';
+            """)
+            row = cur.fetchone()
+
+            if row is None:
+                print(
+                    "FAIL: pgvector extension is NOT installed.\n"
+                    "Run:  CREATE EXTENSION IF NOT EXISTS vector;",
+                    file=sys.stderr,
+                )
+                conn.close()
+                sys.exit(1)
+
+            ext_name, ext_version = row
+            print(f"OK: Extension '{ext_name}' version {ext_version} is installed.")
+
+            # Check that the vector type is usable
+            cur.execute("SELECT '[1,2,3]'::vector;")
+            result = cur.fetchone()
+            print(f"OK: Vector cast test passed — result: {result[0]}")
+
+    conn.close()
+    print("pgvector validation passed.")
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

package/.jumpstart/skills/postgresql-pgvector/templates/alembic_pgvector_migration.py

@@ -0,0 +1,170 @@
+"""
+Alembic migration: enable pgvector, create RAG tables, create HNSW index.
+
+Revision: 001_pgvector_rag_schema
+Run with:  alembic upgrade head
+Rollback:  alembic downgrade -1
+
+Notes:
+- The HNSW index is created in a separate step (concurrent-safe) after table creation.
+- Adjust vector(1536) to match your embedding model's output dimension.
+- For production migrations, test on a copy of production data first.
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from pgvector.sqlalchemy import Vector
+
+# ---------------------------------------------------------------------------
+# Revision identifiers
+# ---------------------------------------------------------------------------
+revision = "001_pgvector_rag_schema"
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # -----------------------------------------------------------------------
+    # 1. Enable pgvector extension
+    # -----------------------------------------------------------------------
+    op.execute("CREATE EXTENSION IF NOT EXISTS vector;")
+
+    # -----------------------------------------------------------------------
+    # 2. Embedding model registry
+    # -----------------------------------------------------------------------
+    op.create_table(
+        "embedding_models",
+        sa.Column("id", sa.UUID(as_uuid=True), primary_key=True, server_default=sa.text("gen_random_uuid()")),
+        sa.Column("provider", sa.Text, nullable=False),
+        sa.Column("model_name", sa.Text, nullable=False),
+        sa.Column("dimensions", sa.Integer, nullable=False),
+        sa.Column("distance_metric", sa.Text, nullable=False, server_default="cosine"),
+        sa.Column("is_active", sa.Boolean, nullable=False, server_default="TRUE"),
+        sa.Column("created_at", sa.TIMESTAMP(timezone=True), server_default=sa.text("now()")),
+        sa.UniqueConstraint("provider", "model_name", "dimensions", name="uq_embedding_models"),
+    )
+
+    # -----------------------------------------------------------------------
+    # 3. Source documents
+    # -----------------------------------------------------------------------
+    op.create_table(
+        "documents",
+        sa.Column("id", sa.UUID(as_uuid=True), primary_key=True, server_default=sa.text("gen_random_uuid()")),
+        sa.Column("source_system", sa.Text, nullable=False),
+        sa.Column("source_uri", sa.Text, nullable=False, unique=True),
+        sa.Column("title", sa.Text),
+        sa.Column("content_hash", sa.Text),
+        sa.Column("file_type", sa.Text),
+        sa.Column("page_count", sa.Integer),
+        sa.Column("language", sa.Text, server_default="en"),
+        sa.Column("security_label", sa.Text, nullable=False, server_default="internal"),
+        sa.Column("metadata", sa.JSON, nullable=False, server_default="{}"),
+        sa.Column("ingested_at", sa.TIMESTAMP(timezone=True)),
+        sa.Column("is_deleted", sa.Boolean, nullable=False, server_default="FALSE"),
+        sa.Column("created_at", sa.TIMESTAMP(timezone=True), server_default=sa.text("now()")),
+        sa.Column("updated_at", sa.TIMESTAMP(timezone=True), server_default=sa.text("now()")),
+    )
+
+    # -----------------------------------------------------------------------
+    # 4. Document chunks with vector column
+    # -----------------------------------------------------------------------
+    op.create_table(
+        "document_chunks",
+        sa.Column("id", sa.UUID(as_uuid=True), primary_key=True, server_default=sa.text("gen_random_uuid()")),
+        sa.Column("document_id", sa.UUID(as_uuid=True), sa.ForeignKey("documents.id", ondelete="CASCADE"), nullable=False),
+        sa.Column("embedding_model_id", sa.UUID(as_uuid=True), sa.ForeignKey("embedding_models.id"), nullable=False),
+        sa.Column("chunk_index", sa.Integer, nullable=False),
+        sa.Column("chunk_text", sa.Text, nullable=False),
+        sa.Column("token_count", sa.Integer),
+        sa.Column("embedding", Vector(1536)),   # adjust dimension here
+        sa.Column("chunk_strategy", sa.Text, nullable=False, server_default="fixed"),
+        sa.Column("chunk_version", sa.Integer, nullable=False, server_default="1"),
+        sa.Column("metadata", sa.JSON, nullable=False, server_default="{}"),
+        sa.Column("is_deleted", sa.Boolean, nullable=False, server_default="FALSE"),
+        sa.Column("created_at", sa.TIMESTAMP(timezone=True), server_default=sa.text("now()")),
+    )
+    op.create_index("ix_document_chunks_document_id", "document_chunks", ["document_id"])
+
+    # -----------------------------------------------------------------------
+    # 5. Chunk citations
+    # -----------------------------------------------------------------------
+    op.create_table(
+        "chunk_citations",
+        sa.Column("id", sa.UUID(as_uuid=True), primary_key=True, server_default=sa.text("gen_random_uuid()")),
+        sa.Column("chunk_id", sa.UUID(as_uuid=True), sa.ForeignKey("document_chunks.id", ondelete="CASCADE"), nullable=False),
+        sa.Column("source_uri", sa.Text, nullable=False),
+        sa.Column("page_number", sa.Integer),
+        sa.Column("section_heading", sa.Text),
+        sa.Column("start_char", sa.Integer),
+        sa.Column("end_char", sa.Integer),
+        sa.Column("citation_text", sa.Text),
+    )
+
+    # -----------------------------------------------------------------------
+    # 6. Ingestion batches
+    # -----------------------------------------------------------------------
+    op.create_table(
+        "ingestion_batches",
+        sa.Column("id", sa.UUID(as_uuid=True), primary_key=True, server_default=sa.text("gen_random_uuid()")),
+        sa.Column("source_system", sa.Text, nullable=False),
+        sa.Column("batch_label", sa.Text),
+        sa.Column("started_at", sa.TIMESTAMP(timezone=True), server_default=sa.text("now()")),
+        sa.Column("completed_at", sa.TIMESTAMP(timezone=True)),
+        sa.Column("status", sa.Text, nullable=False, server_default="running"),
+        sa.Column("records_processed", sa.Integer, server_default="0"),
+        sa.Column("records_failed", sa.Integer, server_default="0"),
+        sa.Column("error_summary", sa.Text),
+        sa.Column("metadata", sa.JSON, nullable=False, server_default="{}"),
+    )
+
+    # -----------------------------------------------------------------------
+    # 7. Retrieval events
+    # -----------------------------------------------------------------------
+    op.create_table(
+        "retrieval_events",
+        sa.Column("id", sa.UUID(as_uuid=True), primary_key=True, server_default=sa.text("gen_random_uuid()")),
+        sa.Column("trace_id", sa.Text),
+        sa.Column("session_id", sa.Text),
+        sa.Column("user_id", sa.Text),
+        sa.Column("query_text", sa.Text, nullable=False),
+        sa.Column("query_embedding_model", sa.Text),
+        sa.Column("filters", sa.JSON, nullable=False, server_default="{}"),
+        sa.Column("retrieved_chunk_ids", sa.ARRAY(sa.UUID(as_uuid=True))),
+        sa.Column("scores", sa.JSON),
+        sa.Column("top_k", sa.Integer),
+        sa.Column("latency_ms", sa.Integer),
+        sa.Column("result_used", sa.Boolean),
+        sa.Column("created_at", sa.TIMESTAMP(timezone=True), server_default=sa.text("now()")),
+    )
+    op.create_index("ix_retrieval_events_trace_id", "retrieval_events", ["trace_id"])
+
+    # -----------------------------------------------------------------------
+    # 8. HNSW vector index (run after table creation)
+    # -----------------------------------------------------------------------
+    op.execute("""
+        CREATE INDEX IF NOT EXISTS document_chunks_embedding_hnsw_idx
+        ON document_chunks
+        USING hnsw (embedding vector_cosine_ops)
+        WITH (m = 16, ef_construction = 64);
+    """)
+
+    # -----------------------------------------------------------------------
+    # 9. Full-text GIN index
+    # -----------------------------------------------------------------------
+    op.execute("""
+        CREATE INDEX IF NOT EXISTS document_chunks_fts_idx
+        ON document_chunks
+        USING gin (to_tsvector('english', chunk_text));
+    """)
+
+
+def downgrade() -> None:
+    op.drop_index("document_chunks_fts_idx", table_name="document_chunks")
+    op.drop_index("document_chunks_embedding_hnsw_idx", table_name="document_chunks")
+    op.drop_table("retrieval_events")
+    op.drop_table("ingestion_batches")
+    op.drop_table("chunk_citations")
+    op.drop_table("document_chunks")
+    op.drop_table("documents")
+    op.drop_table("embedding_models")