igel-qe-core 1.0.0

package/knowledge_base/ingestion/ingest.py

@@ -0,0 +1,346 @@
+"""
+IGEL Knowledge Base — Document Ingestion Pipeline
+==================================================
+Usage:
+  python -m knowledge_base.ingestion.ingest               # Ingest all new documents
+  python -m knowledge_base.ingestion.ingest --file x.pdf  # Single file
+  python -m knowledge_base.ingestion.ingest --reset       # Drop + rebuild schema, then ingest all
+  python -m knowledge_base.ingestion.ingest --dry-run     # Parse + chunk without writing to DB
+  python -m knowledge_base.ingestion.ingest --stats       # Show DB statistics and exit
+"""
+from __future__ import annotations
+import argparse
+import hashlib
+import logging
+import sys
+import time
+from pathlib import Path
+
+from rich.console import Console
+from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn, TimeElapsedColumn
+from rich.table import Table
+
+from knowledge_base.config import cfg
+from knowledge_base.db.client import init_pool, test_connection, close_pool
+from knowledge_base.db.schema import (
+    create_schema, drop_schema, document_exists,
+    get_stats, ingest_document_atomic,
+)
+from knowledge_base.parsers.pdf_parser import parse_pdf
+from knowledge_base.parsers.markdown_parser import parse_markdown_file
+from knowledge_base.parsers.doc_parser import parse_doc
+from knowledge_base.parsers.csv_parser import parse_csv
+from knowledge_base.chunking.chunker import chunk_sections
+from knowledge_base.chunking.validators import detect_product
+from knowledge_base.embeddings.azure_embedder import embed_texts, format_for_pgvector
+from knowledge_base.ingestion.kg_extractor import extract_for_document
+
+console = Console()
+logging.basicConfig(level=getattr(logging, cfg.LOG_LEVEL), format="%(levelname)s %(message)s")
+logger = logging.getLogger(__name__)
+
+
+# ── File discovery ────────────────────────────────────────────────────────────
+
+def discover_files(kb_dir: Path, single_file: str | None = None) -> list[Path]:
+    if single_file:
+        p = kb_dir / single_file
+        if not p.exists():
+            console.print(f"[red]File not found: {p}[/red]")
+            sys.exit(1)
+        return [p]
+
+    files = []
+    for f in sorted(kb_dir.iterdir()):
+        if not f.is_file():
+            continue
+        if f.name.startswith("~$") or f.name.startswith("."):
+            continue
+        if f.suffix.lower() in cfg.SUPPORTED_EXTENSIONS:
+            files.append(f)
+    return files
+
+
+def file_hash(path: Path) -> str:
+    h = hashlib.md5()
+    with open(path, "rb") as f:
+        for block in iter(lambda: f.read(65536), b""):
+            h.update(block)
+    return h.hexdigest()
+
+
+# ── Parsing dispatcher ────────────────────────────────────────────────────────
+
+def parse_file(path: Path) -> tuple[list, str]:
+    """Returns (sections, parse_method_label)."""
+    suffix = path.suffix.lower()
+
+    if suffix == ".pdf":
+        return parse_pdf(path, cfg.MARKDOWN_OUTPUT_DIR)
+
+    if suffix == ".md":
+        return parse_markdown_file(path), "markdown"
+
+    if suffix in (".doc", ".docx"):
+        return parse_doc(path), "docx"
+
+    if suffix in (".csv", ".xlsx", ".xls"):
+        return parse_csv(path), "csv"
+
+    if suffix == ".mht":
+        return _parse_mht(path), "mht"
+
+    raise ValueError(f"No parser for: {suffix}")
+
+
+def _parse_mht(path: Path) -> list:
+    """MHT = MIME HTML archive — extract readable text."""
+    from knowledge_base.parsers.markdown_parser import Section
+    import re
+
+    raw = path.read_bytes().decode("utf-8", errors="replace")
+    # Strip MIME headers and HTML tags
+    text = re.sub(r"<[^>]+>", " ", raw)
+    text = re.sub(r"&[a-z]+;", " ", text)
+    text = re.sub(r"\s{3,}", "\n\n", text).strip()
+    return [Section(title=path.stem, level=0, content=text)]
+
+
+# ── Core ingestion ────────────────────────────────────────────────────────────
+
+def ingest_file(path: Path, dry_run: bool = False, skip_kg: bool = False) -> dict:
+    result = {
+        "file": path.name,
+        "status": "ok",
+        "parse_method": "?",
+        "parents": 0,
+        "chunks": 0,
+        "kg_nodes": 0,
+        "kg_edges": 0,
+        "warnings": 0,
+        "validation_skipped": 0,
+        "skipped": False,
+        "error": None,
+        "elapsed": 0.0,
+    }
+    t0 = time.time()
+
+    try:
+        if path.name.startswith("~$") or path.name.startswith("."):
+            result["status"] = "skipped"
+            result["skipped"] = True
+            return result
+
+        # Deduplication check — re-ingest if prior run left an empty document
+        fhash = file_hash(path)
+        existing_id: str | None = None
+        if not dry_run:
+            existing_id = document_exists(fhash)
+            if existing_id:
+                from knowledge_base.db.schema import clear_document_chunks, document_has_chunks
+                if document_has_chunks(existing_id):
+                    result["status"] = "skipped"
+                    result["skipped"] = True
+                    return result
+                logger.warning("Re-ingesting %s (previous ingest had 0 chunks)", path.name)
+                clear_document_chunks(existing_id)
+
+        # Parse
+        sections, parse_method = parse_file(path)
+        result["parse_method"] = parse_method
+
+        if not sections:
+            result["status"] = "empty"
+            return result
+
+        # Chunk
+        product = detect_product("", path.name)
+        parents, warnings = chunk_sections(sections, path.name, product)
+        result["parents"] = len(parents)
+        result["warnings"] = len(warnings)
+
+        if warnings:
+            for w in warnings[:5]:   # log first 5 to avoid noise
+                logger.warning(w)
+
+        if dry_run:
+            total_children = sum(len(p.children) for p in parents)
+            result["chunks"] = total_children
+            return result
+
+        doc_id, parent_count, total_children, validation_skipped = ingest_document_atomic(
+            file_name=path.name,
+            file_path=str(path),
+            file_type=path.suffix.lower().lstrip("."),
+            file_hash=fhash,
+            product=product,
+            doc_metadata={"parse_method": parse_method, "section_count": len(sections)},
+            parents=parents,
+            existing_id=existing_id,
+            embed_fn=embed_texts,
+            format_fn=format_for_pgvector,
+            batch_size=cfg.BATCH_SIZE,
+        )
+        result["validation_skipped"] = validation_skipped
+        result["chunks"] = total_children
+
+        if not skip_kg:
+            try:
+                kg_n, kg_e = extract_for_document(doc_id)
+                result["kg_nodes"] = kg_n
+                result["kg_edges"] = kg_e
+            except Exception as kg_err:
+                logger.warning("KG extraction failed for %s: %s", path.name, kg_err)
+
+    except Exception as e:
+        result["status"] = "error"
+        result["error"] = str(e)
+        logger.exception("Failed to ingest %s", path.name)
+
+    result["elapsed"] = round(time.time() - t0, 1)
+    return result
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser(description="IGEL Knowledge Base Ingestion Pipeline")
+    parser.add_argument("--file", type=str, help="Ingest a single file by name")
+    parser.add_argument("--reset", action="store_true", help="Drop and recreate schema before ingesting")
+    parser.add_argument("--dry-run", action="store_true", help="Parse + chunk without writing to DB")
+    parser.add_argument("--skip-kg", action="store_true", help="Skip knowledge graph extraction after ingestion")
+    parser.add_argument("--enrich-orphans", action="store_true", help="Run LLM KG enrichment on orphan parent chunks")
+    parser.add_argument("--stats", action="store_true", help="Print knowledge base statistics and exit")
+    args = parser.parse_args()
+
+    console.rule("[bold blue]IGEL Knowledge Base Ingestion Pipeline[/bold blue]")
+    console.print(f"  KB directory : {cfg.KB_DIR}")
+    console.print(f"  Database     : {cfg.DB_HOST}:{cfg.DB_PORT}/{cfg.DB_NAME}")
+    console.print(f"  Embeddings   : {cfg.AZURE_EMBEDDING_DEPLOYMENT} ({cfg.EMBEDDING_DIM}d)")
+    console.print(f"  Dry run      : {args.dry_run}")
+    console.print(f"  KG sync      : {'disabled' if args.skip_kg else 'automatic'}")
+    console.print()
+
+    if not args.dry_run:
+        console.print("[bold]Connecting to PostgreSQL...[/bold]")
+        init_pool()
+        if not test_connection():
+            console.print("[red]Cannot connect to database. Check .env settings.[/red]")
+            sys.exit(1)
+
+    if args.enrich_orphans and not args.dry_run:
+        from knowledge_base.ingestion.kg_extractor import enrich_orphan_parents
+        console.print("[bold]Enriching orphan parent chunks with LLM KG extraction...[/bold]")
+        init_pool()
+        n, e = enrich_orphan_parents(limit=500)
+        console.print(f"  Enriched: {n} nodes, {e} edges")
+        close_pool()
+        return
+
+    if args.stats:
+        stats = get_stats()
+        t = Table(title="Knowledge Base Statistics")
+        t.add_column("Metric")
+        t.add_column("Value", justify="right")
+        for k, v in stats.items():
+            t.add_row(k.replace("_", " ").title(), str(v))
+        console.print(t)
+        close_pool()
+        return
+
+    if args.reset and not args.dry_run:
+        console.print("[yellow]Resetting schema (drop + recreate)...[/yellow]")
+        drop_schema()
+
+    if not args.dry_run:
+        console.print("[bold]Initializing schema...[/bold]")
+        create_schema()
+
+    files = discover_files(cfg.KB_DIR, args.file)
+    console.print(f"\nFound [bold]{len(files)}[/bold] files to process\n")
+
+    results = []
+
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        TaskProgressColumn(),
+        TimeElapsedColumn(),
+        console=console,
+    ) as progress:
+        task = progress.add_task("Ingesting...", total=len(files))
+
+        for path in files:
+            progress.update(task, description=f"[cyan]{path.name[:50]}[/cyan]")
+            r = ingest_file(path, dry_run=args.dry_run, skip_kg=args.skip_kg)
+            results.append(r)
+            progress.advance(task)
+
+    # ── Summary ───────────────────────────────────────────────────────────────
+    console.print()
+    console.rule("[bold green]Ingestion Complete[/bold green]")
+
+    summary = Table(title="Results Summary")
+    summary.add_column("File", style="cyan", max_width=45)
+    summary.add_column("Status", justify="center")
+    summary.add_column("Method")
+    summary.add_column("Parents", justify="right")
+    summary.add_column("Chunks", justify="right")
+    summary.add_column("KG", justify="right")
+    summary.add_column("Warnings", justify="right")
+    summary.add_column("Time(s)", justify="right")
+
+    ok = skipped = errors = total_parents = total_chunks = total_kg_nodes = total_kg_edges = 0
+    for r in results:
+        if r["skipped"]:
+            skipped += 1
+            status_str = "[yellow]SKIP[/yellow]"
+        elif r["status"] == "error":
+            errors += 1
+            status_str = f"[red]ERROR[/red]"
+        elif r["status"] == "empty":
+            status_str = "[dim]EMPTY[/dim]"
+        else:
+            ok += 1
+            status_str = "[green]OK[/green]"
+
+        total_parents += r["parents"]
+        total_chunks += r["chunks"]
+        total_kg_nodes += r.get("kg_nodes", 0)
+        total_kg_edges += r.get("kg_edges", 0)
+
+        kg_cell = f"{r.get('kg_nodes', 0)}n/{r.get('kg_edges', 0)}e" if r.get("kg_nodes") else ""
+        summary.add_row(
+            r["file"],
+            status_str,
+            r["parse_method"],
+            str(r["parents"]),
+            str(r["chunks"]),
+            kg_cell,
+            str(r["warnings"]) if r["warnings"] else "",
+            str(r["elapsed"]),
+        )
+
+    console.print(summary)
+    console.print()
+    console.print(f"  Processed : [green]{ok}[/green]  |  Skipped: [yellow]{skipped}[/yellow]  |  Errors: [red]{errors}[/red]")
+    console.print(f"  Total parent chunks : {total_parents}")
+    console.print(f"  Total child chunks  : {total_chunks} (these are vector-indexed)")
+    if total_kg_nodes or total_kg_edges:
+        console.print(f"  KG sync             : {total_kg_nodes} node ops, {total_kg_edges} edges created")
+
+    if not args.dry_run:
+        stats = get_stats()
+        console.print(f"\n  DB totals → docs: {stats['documents']}  |  parents: {stats['parent_chunks']}  |  chunks: {stats['chunks']}")
+        if stats["missing_embeddings"] > 0:
+            console.print(f"  [yellow]Warning: {stats['missing_embeddings']} chunks have no embedding[/yellow]")
+        close_pool()
+
+    if errors > 0:
+        console.print(f"\n[red]{errors} file(s) failed — check logs above for details[/red]")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/knowledge_base/ingestion/ingest_images.py

@@ -0,0 +1,330 @@
+"""
+IGEL Knowledge Base — Image Ingestion Pipeline
+==============================================
+Iterates over every document already ingested into kb_documents, extracts
+embedded images, runs them through GPT-4.1 Vision for verbatim + semantic
+description, embeds the combined context, and links each image back to:
+
+  • its source document (kb_documents.id, hard FK, ON DELETE CASCADE)
+  • the nearest text section (kb_parent_chunks.id, soft FK)
+
+Idempotent at the document level: a document already covered (has ≥1 row in
+kb_images) is skipped unless --force is passed.
+
+Usage:
+  python -m knowledge_base.ingestion.ingest_images                   # all docs
+  python -m knowledge_base.ingestion.ingest_images --document NAME   # single doc by file_name
+  python -m knowledge_base.ingestion.ingest_images --force           # re-process even covered docs
+  python -m knowledge_base.ingestion.ingest_images --stats           # show image stats and exit
+  python -m knowledge_base.ingestion.ingest_images --limit N         # process at most N docs
+"""
+from __future__ import annotations
+import argparse
+import logging
+import sys
+import time
+from pathlib import Path
+
+from rich.console import Console
+from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn, TimeElapsedColumn
+from rich.table import Table
+
+from knowledge_base.config import cfg
+from knowledge_base.db.client import init_pool, test_connection, close_pool
+from knowledge_base.db.schema import (
+    create_schema, get_stats,
+    insert_image, find_nearest_parent_chunk,
+    list_documents_for_image_pass, document_has_images,
+)
+from knowledge_base.parsers.image_extractor import extract_images
+from knowledge_base.embeddings.azure_embedder import embed_texts, format_for_pgvector
+from knowledge_base.vision.image_describer import describe_image, build_embedding_text
+
+console = Console()
+logging.basicConfig(level=getattr(logging, cfg.LOG_LEVEL), format="%(levelname)s %(message)s")
+logger = logging.getLogger(__name__)
+
+# Quiet down noisy HTTP loggers from openai/httpx during long runs
+logging.getLogger("httpx").setLevel(logging.WARNING)
+logging.getLogger("openai").setLevel(logging.WARNING)
+
+
+def get_section_title_for_chunk(parent_chunk_id: str | None) -> str:
+    """Look up section_title for the nearest parent chunk (best-effort)."""
+    if not parent_chunk_id:
+        return ""
+    from knowledge_base.db.client import get_conn
+    with get_conn() as conn:
+        with conn.cursor() as cur:
+            cur.execute(
+                "SELECT section_title FROM kb_parent_chunks WHERE id = %s",
+                (parent_chunk_id,)
+            )
+            row = cur.fetchone()
+            return (row[0] if row and row[0] else "") or ""
+
+
+def ingest_one_document(doc: dict, force: bool = False) -> dict:
+    result = {
+        "file": doc["file_name"],
+        "status": "ok",
+        "extracted": 0,
+        "described": 0,
+        "inserted": 0,
+        "errors": 0,
+        "elapsed": 0.0,
+    }
+    t0 = time.time()
+
+    # Skip if already covered
+    if not force and document_has_images(doc["id"]):
+        result["status"] = "skipped"
+        return result
+
+    file_path = Path(doc["file_path"])
+    if not file_path.exists():
+        # Fall back to KB_DIR
+        candidate = cfg.KB_DIR / doc["file_name"]
+        if candidate.exists():
+            file_path = candidate
+        else:
+            result["status"] = "missing_file"
+            return result
+
+    try:
+        images = extract_images(file_path)
+    except Exception as e:
+        logger.exception("extract_images failed for %s", file_path.name)
+        result["status"] = "extract_error"
+        result["errors"] += 1
+        result["elapsed"] = round(time.time() - t0, 1)
+        return result
+
+    result["extracted"] = len(images)
+    if not images:
+        result["status"] = "no_images"
+        result["elapsed"] = round(time.time() - t0, 1)
+        return result
+
+    # Process images: describe + embed + insert
+    embed_texts_buffer: list[str] = []
+    pending: list[dict] = []   # accumulate metadata to insert after batch embedding
+
+    from knowledge_base.vision.image_describer import ImageDescription
+
+    for img in images:
+        parent_chunk_id = find_nearest_parent_chunk(doc["id"], img.page_number)
+        section_title = get_section_title_for_chunk(parent_chunk_id)
+
+        if img.is_recurring:
+            # Recurring header/logo/footer — describe ONCE per unique image (already
+            # deduped by hash), but with a cheap, fixed description so we don't burn
+            # vision tokens on the same logo. Binary is still preserved.
+            desc = ImageDescription(
+                image_type="recurring_header_or_logo",
+                verbatim_text="",
+                ui_elements=[],
+                semantic_description=(
+                    f"Recurring page element appearing on {len(img.occurrence_pages)} of "
+                    f"the document's pages (e.g. header, footer, logo, watermark). "
+                    f"First seen on page {img.page_number}."
+                ),
+                test_relevance="Reference image only",
+                raw_json={"recurring": True, "occurrence_pages": img.occurrence_pages},
+            )
+        else:
+            try:
+                desc = describe_image(
+                    image_bytes=img.image_bytes,
+                    image_format=img.image_format,
+                    document_name=doc["file_name"],
+                    section_title=section_title,
+                    page_number=img.page_number,
+                    caption=img.caption,
+                    surrounding_text=img.surrounding_text,
+                )
+            except Exception as e:
+                logger.exception("describe_image failed for %s img#%d", file_path.name, img.sequence_index)
+                result["errors"] += 1
+                continue
+
+            if desc.error:
+                result["errors"] += 1
+            else:
+                result["described"] += 1
+
+        embed_text = build_embedding_text(
+            desc, doc["file_name"], section_title, img.page_number,
+            img.caption, img.surrounding_text,
+        )
+
+        embed_texts_buffer.append(embed_text)
+        pending.append({
+            "img": img,
+            "desc": desc,
+            "parent_chunk_id": parent_chunk_id,
+            "section_title": section_title,
+            "embed_text": embed_text,
+        })
+
+    # Batch-embed all images for this document
+    if not pending:
+        result["status"] = "no_described"
+        result["elapsed"] = round(time.time() - t0, 1)
+        return result
+
+    try:
+        embeddings = embed_texts(embed_texts_buffer, batch_size=cfg.BATCH_SIZE)
+    except Exception as e:
+        logger.exception("embed_texts failed for %s", file_path.name)
+        result["status"] = "embed_error"
+        result["errors"] += len(pending)
+        result["elapsed"] = round(time.time() - t0, 1)
+        return result
+
+    # Insert
+    for p, embedding in zip(pending, embeddings):
+        img = p["img"]
+        desc = p["desc"]
+        try:
+            insert_image(
+                document_id=doc["id"],
+                parent_chunk_id=p["parent_chunk_id"],
+                sequence_index=img.sequence_index,
+                page_number=img.page_number,
+                bbox=img.bbox,
+                image_hash=img.image_hash,
+                image_format=img.image_format,
+                width=img.width,
+                height=img.height,
+                image_bytes=img.image_bytes,
+                image_type=desc.image_type,
+                verbatim_text=desc.verbatim_text,
+                ui_elements=desc.ui_elements,
+                semantic_description=desc.semantic_description,
+                test_relevance=desc.test_relevance,
+                surrounding_text=img.surrounding_text,
+                caption=img.caption,
+                description_embedding=format_for_pgvector(embedding),
+                metadata={
+                    "section_title": p["section_title"],
+                    "raw_response": desc.raw_json,
+                    "describer_error": desc.error,
+                    "occurrence_pages": img.occurrence_pages,
+                    "is_recurring": img.is_recurring,
+                },
+            )
+            result["inserted"] += 1
+        except Exception as e:
+            logger.exception("insert_image failed for %s img#%d", file_path.name, img.sequence_index)
+            result["errors"] += 1
+
+    result["elapsed"] = round(time.time() - t0, 1)
+    return result
+
+
+def main():
+    parser = argparse.ArgumentParser(description="IGEL Knowledge Base — image ingestion pass")
+    parser.add_argument("--document", type=str, help="Single document by file_name")
+    parser.add_argument("--force", action="store_true", help="Re-process docs that already have images")
+    parser.add_argument("--stats", action="store_true", help="Show image stats and exit")
+    parser.add_argument("--limit", type=int, default=0, help="Process at most N documents")
+    args = parser.parse_args()
+
+    console.rule("[bold blue]IGEL KB — Image Ingestion[/bold blue]")
+    console.print(f"  Database     : {cfg.DB_HOST}:{cfg.DB_PORT}/{cfg.DB_NAME}")
+    console.print(f"  Vision LLM   : {cfg.AZURE_DEPLOYMENT}")
+    console.print(f"  Embeddings   : {cfg.AZURE_EMBEDDING_DEPLOYMENT} ({cfg.EMBEDDING_DIM}d)")
+    console.print()
+
+    init_pool()
+    if not test_connection():
+        console.print("[red]Cannot connect to database[/red]")
+        sys.exit(1)
+    create_schema()  # idempotent — ensures kb_images table exists
+
+    if args.stats:
+        s = get_stats()
+        t = Table(title="KB Stats")
+        for k, v in s.items():
+            t.add_row(k.replace("_", " ").title(), str(v))
+        console.print(t)
+        close_pool()
+        return
+
+    docs = list_documents_for_image_pass()
+    if args.document:
+        docs = [d for d in docs if d["file_name"] == args.document]
+        if not docs:
+            console.print(f"[red]No document found with file_name={args.document}[/red]")
+            sys.exit(1)
+    if args.limit:
+        docs = docs[: args.limit]
+
+    console.print(f"Found [bold]{len(docs)}[/bold] documents to process\n")
+
+    results = []
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        TaskProgressColumn(),
+        TimeElapsedColumn(),
+        console=console,
+    ) as progress:
+        task = progress.add_task("Image ingest...", total=len(docs))
+
+        for d in docs:
+            progress.update(task, description=f"[cyan]{d['file_name'][:60]}[/cyan]")
+            r = ingest_one_document(d, force=args.force)
+            results.append(r)
+            progress.advance(task)
+
+    # Summary
+    console.print()
+    console.rule("[bold green]Image Ingest Complete[/bold green]")
+    summary = Table(title="Per-Document Image Stats")
+    summary.add_column("File", style="cyan", max_width=45)
+    summary.add_column("Status", justify="center")
+    summary.add_column("Extracted", justify="right")
+    summary.add_column("Described", justify="right")
+    summary.add_column("Inserted", justify="right")
+    summary.add_column("Errors", justify="right")
+    summary.add_column("Time(s)", justify="right")
+
+    total_extracted = total_inserted = total_errors = 0
+    skipped = no_images = ok = 0
+    for r in results:
+        status = r["status"]
+        if status == "skipped":
+            status_str = "[yellow]SKIP[/yellow]"; skipped += 1
+        elif status == "no_images":
+            status_str = "[dim]NONE[/dim]"; no_images += 1
+        elif status == "ok":
+            status_str = "[green]OK[/green]"; ok += 1
+        else:
+            status_str = f"[red]{status.upper()}[/red]"
+
+        total_extracted += r["extracted"]
+        total_inserted += r["inserted"]
+        total_errors += r["errors"]
+
+        summary.add_row(
+            r["file"], status_str,
+            str(r["extracted"]), str(r["described"]), str(r["inserted"]),
+            str(r["errors"]) if r["errors"] else "",
+            str(r["elapsed"]),
+        )
+
+    console.print(summary)
+    console.print()
+    console.print(f"  Processed: [green]{ok}[/green]  |  Skipped: [yellow]{skipped}[/yellow]  |  No images: [dim]{no_images}[/dim]")
+    console.print(f"  Total extracted: {total_extracted}  |  Inserted: {total_inserted}  |  Errors: {total_errors}")
+
+    s = get_stats()
+    console.print(f"\n  DB totals -> images: {s['images']}  |  missing embeddings: {s['images_missing_embeddings']}")
+    close_pool()
+
+
+if __name__ == "__main__":
+    main()