own-rag-cli 0.0.11-snapshot → 0.0.13-snapshot

package/README.md

@@ -1,4 +1,4 @@
-# MCP binary checksum (SHA-256, payload without shebang): `286e52abcbc295893b501fc8aae092420c452a4d3a4242e25c88e646398190b6`
+# MCP binary checksum (SHA-256, payload without shebang): `c74a07c0d8349fad54f3c5aa876296f9ed20239ffa0f2e6483e57796c7831e2a`
 
 # own-rag
 
@@ -60,10 +60,15 @@ rag run /path/to/project ".js,.py,.md,.java"  # indexa só essas extensões
 rag run /path/to/project --skip-index --extensions ".md,.txt"
 rag monitor                     # interactive Chroma monitor
 rag monitor full                # full monitor dashboard
-rag remove                      # full local uninstall (double confirmation)
-rag remove --force              # uninstall without confirmation prompts
+rag remove                      # uninstall local setup (preserves downloaded models)
+rag remove --force              # uninstall setup without confirmation prompts
+rag clean models                # list cached models and remove one or 'all' (double confirmation)
 ```
 
+Notes:
+- `rag remove` keeps downloaded model cache.
+- Use `rag clean models` to remove selected model cache entries.
+
 About extension filter in `rag run`:
 - You can pass extensions as CSV in the 3rd/4th argument, for example:
   - `rag run ./path .js,.py,.md,.java`

package/README.pt-br.md

@@ -55,10 +55,15 @@ chmod +x rag-setup-macos.run
 rag run /caminho/do/projeto    # setup completo + indexação
 rag monitor                    # monitor interativo do Chroma
 rag monitor full               # painel completo do monitor
-rag remove                     # desinstalação completa local (dupla confirmação)
-rag remove --force             # desinstala sem prompts de confirmação
+rag remove                     # desinstala setup local (preserva modelos baixados)
+rag remove --force             # desinstala setup sem prompts de confirmação
+rag clean models               # lista modelos em cache e remove um ou 'all' (dupla confirmação)
 ```
 
+Notas:
+- `rag remove` mantém o cache de modelos baixados.
+- Use `rag clean models` para limpar modelos específicos ou todos (`all`).
+
 ## Indexacao a partir de URL (HTTP/HTTPS)
 
 Agora o `rag run` aceita URL remota alem de pasta local.

package/bin/mcp_server.py

@@ -16,6 +16,8 @@ import json
 import logging
 import getpass
 import shutil
+import threading
+import time
 from collections.abc import Iterator
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from dataclasses import dataclass
@@ -392,6 +394,21 @@ MAX_FILE_SIZE_BYTES = 500 * 1024  # 500 KB
 TOP_K_RESULTS = 7
 MAX_QUERY_RESULTS = 30
 
+WARMUP_ENABLED = os.environ.get("MCP_WARMUP_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
+WARMUP_MODE = os.environ.get("MCP_WARMUP_MODE", "single").strip().lower()
+if WARMUP_MODE not in {"single", "ensemble"}:
+    WARMUP_MODE = "single"
+WARMUP_QUERY = os.environ.get("MCP_WARMUP_QUERY", "warmup semantic search")
+WARMUP_TOP_K = max(1, min(_parse_int(os.environ.get("MCP_WARMUP_TOP_K", "1"), 1, min_value=1), 20))
+
+KEEPALIVE_ENABLED = os.environ.get("MCP_KEEPALIVE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
+KEEPALIVE_MODE = os.environ.get("MCP_KEEPALIVE_MODE", "single").strip().lower()
+if KEEPALIVE_MODE not in {"single", "ensemble"}:
+    KEEPALIVE_MODE = "single"
+KEEPALIVE_QUERY = os.environ.get("MCP_KEEPALIVE_QUERY", WARMUP_QUERY)
+KEEPALIVE_TOP_K = max(1, min(_parse_int(os.environ.get("MCP_KEEPALIVE_TOP_K", "1"), 1, min_value=1), 20))
+KEEPALIVE_INTERVAL_SEC = _parse_int(os.environ.get("MCP_KEEPALIVE_INTERVAL_SEC", "600"), 600, min_value=60)
+
 # Filtros de varredura
 IGNORED_DIRS = {
     ".git", "node_modules", "__pycache__", ".venv", "venv", "env",
@@ -1339,6 +1356,88 @@ def _run_ensemble_mode(query: str, top_k: int) -> tuple[list[FusedHit], list[str
     return reranked_hits, branch_errors, rerank_applied, rerank_error
 
 
+def _execute_background_search(mode: str, query: str, top_k: int, source: str) -> None:
+    started = time.perf_counter()
+    try:
+        if mode == "ensemble":
+            hits, branch_errors, rerank_applied, rerank_error = _run_ensemble_mode(query, top_k)
+        else:
+            hits, branch_errors, rerank_applied, rerank_error = _run_single_mode(query, top_k)
+
+        elapsed_ms = int((time.perf_counter() - started) * 1000)
+        log.info(
+            "%s concluido: mode=%s top_k=%s elapsed_ms=%s hits=%s branch_errors=%s rerank=%s",
+            source,
+            mode,
+            top_k,
+            elapsed_ms,
+            len(hits),
+            len(branch_errors),
+            rerank_applied,
+        )
+        _log_tool_usage(
+            event="maintenance_call",
+            tool_name=source,
+            details={
+                "status": "ok",
+                "mode": mode,
+                "query_len": len(query),
+                "top_k": top_k,
+                "elapsed_ms": elapsed_ms,
+                "result_count": len(hits),
+                "branch_errors": len(branch_errors),
+                "rerank_applied": rerank_applied,
+                "rerank_error": rerank_error,
+            },
+        )
+    except Exception as e:
+        elapsed_ms = int((time.perf_counter() - started) * 1000)
+        log.warning("%s falhou após %sms: %s", source, elapsed_ms, e)
+        _log_tool_usage(
+            event="maintenance_call",
+            tool_name=source,
+            details={
+                "status": "error",
+                "mode": mode,
+                "query_len": len(query),
+                "top_k": top_k,
+                "elapsed_ms": elapsed_ms,
+                "error": str(e),
+            },
+        )
+
+
+_keepalive_thread: threading.Thread | None = None
+_keepalive_stop_event = threading.Event()
+
+
+def _start_keepalive_thread() -> None:
+    global _keepalive_thread
+    if not KEEPALIVE_ENABLED:
+        log.info("Keepalive desabilitado (MCP_KEEPALIVE_ENABLED=false).")
+        return
+    if _keepalive_thread is not None and _keepalive_thread.is_alive():
+        return
+
+    def _runner() -> None:
+        log.info(
+            "Keepalive iniciado: intervalo=%ss mode=%s top_k=%s",
+            KEEPALIVE_INTERVAL_SEC,
+            KEEPALIVE_MODE,
+            KEEPALIVE_TOP_K,
+        )
+        while not _keepalive_stop_event.wait(KEEPALIVE_INTERVAL_SEC):
+            _execute_background_search(
+                mode=KEEPALIVE_MODE,
+                query=KEEPALIVE_QUERY,
+                top_k=KEEPALIVE_TOP_K,
+                source="semantic_search_keepalive",
+            )
+
+    _keepalive_thread = threading.Thread(target=_runner, name="rag-keepalive", daemon=True)
+    _keepalive_thread.start()
+
+
 # ---------------------------------------------------------------------------
 # Servidor MCP via FastMCP
 # ---------------------------------------------------------------------------
@@ -1799,6 +1898,16 @@ if __name__ == "__main__":
     log.info("Reranker: %s (enabled=%s, quant=%s)", RERANK_MODEL_ID, RERANK_ENABLED, RERANKER_QUANTIZATION)
     log.info("Pasta de modelos locais: %s", MODEL_DIR)
     log.info("Uso MCP será registrado em: %s", MCP_USAGE_LOG_PATH)
+    log.info(
+        "Warmup: enabled=%s mode=%s top_k=%s | Keepalive: enabled=%s interval=%ss mode=%s top_k=%s",
+        WARMUP_ENABLED,
+        WARMUP_MODE,
+        WARMUP_TOP_K,
+        KEEPALIVE_ENABLED,
+        KEEPALIVE_INTERVAL_SEC,
+        KEEPALIVE_MODE,
+        KEEPALIVE_TOP_K,
+    )
 
     # Pré-aquece somente conexão Chroma; modelos ficam lazy para poupar RAM.
     try:
@@ -1810,4 +1919,16 @@ if __name__ == "__main__":
         log.error("Falha ao inicializar ChromaDB: %s", e)
         log.error("O servidor continuará, mas as ferramentas retornarão erro até o ChromaDB estar disponível.")
 
+    if WARMUP_ENABLED:
+        _execute_background_search(
+            mode=WARMUP_MODE,
+            query=WARMUP_QUERY,
+            top_k=WARMUP_TOP_K,
+            source="semantic_search_warmup",
+        )
+    else:
+        log.info("Warmup desabilitado (MCP_WARMUP_ENABLED=false).")
+
+    _start_keepalive_thread()
+
     mcp.run(transport="stdio")

package/bin/postinstall.sh

@@ -16,6 +16,8 @@ MONITOR_SRC="${PACKAGE_ROOT}/chroma_monitor.sh"
 MONITOR_DEST="${LOCAL_BIN_DIR}/chroma_monitor.sh"
 REMOVE_SRC="${PACKAGE_ROOT}/bin/rag-remove.sh"
 REMOVE_DEST="${LOCAL_BIN_DIR}/rag-remove.sh"
+CLEAN_MODELS_SRC="${PACKAGE_ROOT}/bin/rag-clean-models.sh"
+CLEAN_MODELS_DEST="${LOCAL_BIN_DIR}/rag-clean-models.sh"
 OWN_RAG_CONFIG_FILE="${HOME}/.own-rag-cli.json"
 MCP_SERVER_COMMAND="${HOME}/.local/bin/mcp-rag-server"
 CLAUDE_CONFIG_FILE="${HOME}/.claude.json"
@@ -359,6 +361,12 @@ if [[ -f "${REMOVE_SRC}" ]]; then
     log_info "Remoção copiada: ${REMOVE_DEST}"
 fi
 
+if [[ -f "${CLEAN_MODELS_SRC}" ]]; then
+    cp "${CLEAN_MODELS_SRC}" "${CLEAN_MODELS_DEST}"
+    chmod +x "${CLEAN_MODELS_DEST}"
+    log_info "Limpeza de modelos copiada: ${CLEAN_MODELS_DEST}"
+fi
+
 DOCKER_COMPOSE_CMD=""
 if command -v docker >/dev/null 2>&1; then
     if docker compose version >/dev/null 2>&1; then
@@ -399,4 +407,4 @@ if [[ -f "${HOME}/.profile" ]]; then
     source "${HOME}/.profile" || true
 fi
 
-log_info "Postinstall concluído. Use: rag run [path] | rag monitor | rag remove"
+log_info "Postinstall concluído. Use: rag run [path] | rag monitor | rag remove | rag clean models"

package/bin/rag-clean-models.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+    cat <<'EOF'
+Uso:
+  rag clean models
+
+Descrição:
+  Lista os modelos baixados em cache e permite apagar um modelo específico
+  ou todos ("all"), com confirmação dupla.
+EOF
+}
+
+log_info() { printf "[+] %s\n" "$*"; }
+log_warn() { printf "[!] %s\n" "$*" >&2; }
+
+if [[ $# -gt 0 ]]; then
+    case "${1:-}" in
+        --help|-h)
+            usage
+            exit 0
+            ;;
+        *)
+            echo "Opção desconhecida: $1" >&2
+            usage
+            exit 1
+            ;;
+    esac
+fi
+
+if [[ ! -t 0 ]]; then
+    echo "[ERRO] Terminal não interativo. Este comando exige confirmação dupla." >&2
+    exit 1
+fi
+
+declare -a MODEL_DIRS=()
+tmp_models_file="$(mktemp "${TMPDIR:-/tmp}/rag-models.XXXXXX")"
+cleanup_tmp_files() {
+    rm -f "${tmp_models_file}" "${tmp_models_sorted_file:-}" 2>/dev/null || true
+}
+trap cleanup_tmp_files EXIT
+
+while IFS= read -r -d '' marker_file; do
+    model_dir="$(dirname "${marker_file}")"
+    printf '%s\n' "${model_dir}" >> "${tmp_models_file}"
+done < <(find "${HOME}/.cache" -type f -name ".download_complete" -print0 2>/dev/null)
+
+if [[ ! -s "${tmp_models_file}" ]]; then
+    log_info "Nenhum modelo encontrado para limpeza em ${HOME}/.cache."
+    exit 0
+fi
+
+tmp_models_sorted_file="$(mktemp "${TMPDIR:-/tmp}/rag-models-sorted.XXXXXX")"
+LC_ALL=C sort -u "${tmp_models_file}" > "${tmp_models_sorted_file}"
+
+while IFS= read -r model_path; do
+    [[ -z "${model_path}" ]] && continue
+    MODEL_DIRS+=("${model_path}")
+done < "${tmp_models_sorted_file}"
+
+echo "Modelos encontrados:"
+for idx in "${!MODEL_DIRS[@]}"; do
+    printf "  [%d] %s\n" "$((idx + 1))" "${MODEL_DIRS[$idx]}"
+done
+echo "  [all] Remover todos os modelos listados"
+echo ""
+
+read -r -p "Escolha o número do modelo para remover ou 'all': " selection
+selection="$(printf '%s' "${selection}" | tr '[:upper:]' '[:lower:]')"
+selection="${selection#"${selection%%[![:space:]]*}"}"
+selection="${selection%"${selection##*[![:space:]]}"}"
+
+declare -a TARGETS=()
+if [[ "${selection}" == "all" ]]; then
+    TARGETS=("${MODEL_DIRS[@]}")
+elif [[ "${selection}" =~ ^[0-9]+$ ]]; then
+    selected_index=$((selection - 1))
+    if (( selected_index < 0 || selected_index >= ${#MODEL_DIRS[@]} )); then
+        log_warn "Seleção inválida."
+        exit 1
+    fi
+    TARGETS=("${MODEL_DIRS[$selected_index]}")
+else
+    log_warn "Seleção inválida. Use um número da lista ou 'all'."
+    exit 1
+fi
+
+echo ""
+echo "Alvos para remoção:"
+for target in "${TARGETS[@]}"; do
+    echo "  - ${target}"
+done
+echo ""
+
+read -r -p "Confirma a remoção dos modelos selecionados? [s/N] " answer
+answer="$(printf '%s' "${answer}" | tr '[:upper:]' '[:lower:]')"
+if [[ "${answer}" != "s" && "${answer}" != "sim" && "${answer}" != "y" && "${answer}" != "yes" ]]; then
+    echo "Limpeza cancelada."
+    exit 0
+fi
+
+read -r -p "Digite LIMPAR para confirmar definitivamente: " confirm_word
+if [[ "${confirm_word}" != "LIMPAR" ]]; then
+    echo "Confirmação inválida. Limpeza cancelada."
+    exit 1
+fi
+
+declare -a FAILED_PATHS=()
+for target in "${TARGETS[@]}"; do
+    if [[ -e "${target}" || -L "${target}" ]]; then
+        if rm -rf "${target}" 2>/dev/null; then
+            log_info "Removido: ${target}"
+        else
+            FAILED_PATHS+=("${target}")
+            log_warn "Falha ao remover (permissão): ${target}"
+        fi
+    fi
+done
+
+if [[ ${#FAILED_PATHS[@]} -gt 0 ]]; then
+    echo ""
+    log_warn "Limpeza concluída com pendências:"
+    for failed_path in "${FAILED_PATHS[@]}"; do
+        echo "    - ${failed_path}"
+    done
+    log_warn "Execute com sudo para limpar pendências, por exemplo:"
+    echo "    sudo rm -rf ${FAILED_PATHS[0]}"
+    exit 1
+fi
+
+log_info "Limpeza de modelos concluída."

package/bin/rag-remove.sh

@@ -8,7 +8,8 @@ Uso:
   rag remove --force
 
 Descrição:
-  Remove toda a instalação local do own-rag (venv, banco, docker, binários e configuração MCP).
+  Remove a instalação local do own-rag (venv, banco, docker, binários e configuração MCP).
+  Modelos em cache NÃO são removidos por este comando.
 
 Opções:
   --force, -f    pula a confirmação dupla.
@@ -96,15 +97,15 @@ declare -a REMOVE_PATHS=(
     "${HOME}/.rag_venv"
     "${HOME}/.rag_db"
     "${HOME}/docker-chromadb"
-    "${HOME}/.cache/own-rag-cli"
-    "${HOME}/.cache/my-custom-rag-python"
-    "${HOME}/.cache/ny-custom-rag-python"
+    "${HOME}/.cache/own-rag-cli/logs"
+    "${HOME}/.cache/own-rag-cli/indexer_tuning.json"
     "${HOME}/.own-rag-cli.json"
     "${HOME}/.local/bin/mcp-rag-server"
     "${HOME}/.local/bin/download_model_from_hugginface.py"
     "${HOME}/.local/bin/download_model_from_modelscope.py"
     "${HOME}/.local/bin/rag-wrapper.sh"
     "${HOME}/.local/bin/rag-remove.sh"
+    "${HOME}/.local/bin/rag-clean-models.sh"
     "${HOME}/.local/bin/rag-setup.run"
     "${HOME}/.local/bin/rag-setup-macos.run"
     "${HOME}/.local/bin/chroma_monitor.sh"
@@ -231,3 +232,8 @@ if [[ ${#FAILED_PATHS[@]} -gt 0 ]]; then
 else
     log_info "Remoção completa concluída."
 fi
+
+echo ""
+log_info "Modelos baixados foram preservados no cache."
+echo "    Para apagar modelos não desejados, execute: rag clean models"
+echo "    O comando lista os modelos e permite escolher um específico ou 'all'."

package/bin/rag-wrapper.sh

@@ -11,6 +11,8 @@ LOCAL_MONITOR_SCRIPT="${LOCAL_BIN_DIR}/chroma_monitor.sh"
 PACKAGE_MONITOR_SCRIPT="${SCRIPT_DIR}/../chroma_monitor.sh"
 LOCAL_REMOVE_SCRIPT="${LOCAL_BIN_DIR}/rag-remove.sh"
 PACKAGE_REMOVE_SCRIPT="${SCRIPT_DIR}/../bin/rag-remove.sh"
+LOCAL_CLEAN_MODELS_SCRIPT="${LOCAL_BIN_DIR}/rag-clean-models.sh"
+PACKAGE_CLEAN_MODELS_SCRIPT="${SCRIPT_DIR}/../bin/rag-clean-models.sh"
 
 URL_TEMP_DIR=""
 
@@ -101,6 +103,28 @@ find_remove_runner() {
     return 1
 }
 
+find_clean_models_runner() {
+    local os_name="$1"
+    if [[ "${os_name}" == "unknown" ]]; then
+        return 1
+    fi
+
+    if [[ -x "${LOCAL_CLEAN_MODELS_SCRIPT}" ]]; then
+        echo "${LOCAL_CLEAN_MODELS_SCRIPT}"
+        return 0
+    fi
+    if [[ -x "${PACKAGE_CLEAN_MODELS_SCRIPT}" ]]; then
+        echo "${PACKAGE_CLEAN_MODELS_SCRIPT}"
+        return 0
+    fi
+    if [[ -f "${PACKAGE_CLEAN_MODELS_SCRIPT}" ]]; then
+        chmod +x "${PACKAGE_CLEAN_MODELS_SCRIPT}" || true
+        echo "${PACKAGE_CLEAN_MODELS_SCRIPT}"
+        return 0
+    fi
+    return 1
+}
+
 cleanup_url_temp_dir() {
     if [[ -n "${URL_TEMP_DIR}" && -d "${URL_TEMP_DIR}" ]]; then
         rm -rf "${URL_TEMP_DIR}" || true
@@ -235,6 +259,7 @@ Uso:
   rag run [path|url] [opcoes] --extensions ".js,.py,.md"
   rag monitor [command]
   rag remove
+  rag clean models
 
 Exemplos:
   rag run .
@@ -246,6 +271,7 @@ Exemplos:
   rag monitor
   rag monitor full
   rag remove
+  rag clean models
 EOF
 }
 
@@ -377,6 +403,25 @@ case "${command_name}" in
         fi
         exec "${remove_runner}" "$@"
         ;;
+    clean)
+        clean_target="${1:-}"
+        shift || true
+
+        case "${clean_target}" in
+            models)
+                if ! clean_models_runner="$(find_clean_models_runner "${os_name}")"; then
+                    echo "Erro: rag-clean-models.sh não encontrado." >&2
+                    echo "Esperado em: ${LOCAL_CLEAN_MODELS_SCRIPT} ou ${PACKAGE_CLEAN_MODELS_SCRIPT}" >&2
+                    exit 1
+                fi
+                exec "${clean_models_runner}" "$@"
+                ;;
+            *)
+                echo "Erro: uso inválido. Use: rag clean models" >&2
+                exit 1
+                ;;
+        esac
+        ;;
     *)
         usage
         exit 1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "own-rag-cli",
-  "version": "0.0.11-snapshot",
+  "version": "0.0.13-snapshot",
   "description": "Local RAG setup with ChromaDB + MCP server (Jina/BGE hybrid support).",
   "license": "MIT",
   "private": false,
@@ -37,6 +37,7 @@
     "bin/requirements.txt",
     "bin/rag-wrapper.sh",
     "bin/rag-remove.sh",
+    "bin/rag-clean-models.sh",
     "bin/postinstall.sh",
     "bin/indexer_full.py",
     "bin/mcp_server.py",