own-rag-cli 0.0.1-snapshot → 0.0.2-snapshot
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -3
- package/bin/indexer_full.py +189 -43
- package/bin/mcp_server.py +163 -18
- package/bin/postinstall.sh +28 -0
- package/bin/rag-remove.sh +3 -0
- package/chroma_monitor.sh +92 -18
- package/package.json +1 -1
- package/rag-setup-macos.run +295 -71
- package/rag-setup.run +295 -73
package/README.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# MCP binary checksum (SHA-256, payload without shebang): `
|
|
1
|
+
# MCP binary checksum (SHA-256, payload without shebang): `74bb1de161e5ebeef6691397fc1403bc58c221b9302f67fd770b1097f8ff76d8`
|
|
2
2
|
|
|
3
3
|
# own-rag
|
|
4
4
|
|
|
@@ -63,7 +63,43 @@ rag remove
|
|
|
63
63
|
4. Optionally updates MCP config files (`.claude.json`, Cursor config).
|
|
64
64
|
5. Indexes the project.
|
|
65
65
|
|
|
66
|
-
##
|
|
66
|
+
## Permissions and Tuning Config
|
|
67
|
+
|
|
68
|
+
- Autotune/indexer tuning is persisted by default in:
|
|
69
|
+
- `~/.cache/own-rag-cli/indexer_tuning.json`
|
|
70
|
+
- This avoids permission issues when `~/.rag_db` is owned by `root`.
|
|
71
|
+
- If you want to store tuning inside `~/.rag_db`, grant permissions and set:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
sudo chown -R "$USER":"$USER" ~/.rag_db
|
|
75
|
+
export MCP_INDEXER_CONFIG_FILE="$HOME/.rag_db/indexer_tuning.json"
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## Chroma Endpoint Config
|
|
79
|
+
|
|
80
|
+
- Runtime config file: `~/.own-rag-cli.json`
|
|
81
|
+
- Default generated content:
|
|
82
|
+
|
|
83
|
+
```json
|
|
84
|
+
{
|
|
85
|
+
"chroma": {
|
|
86
|
+
"scheme": "http",
|
|
87
|
+
"host": "localhost",
|
|
88
|
+
"port": 8000
|
|
89
|
+
},
|
|
90
|
+
"indexing": {
|
|
91
|
+
"embedding_batch_size": 4,
|
|
92
|
+
"batch_count": 4
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
- You can point to remote ChromaDB by changing `scheme`, `host`, and `port`.
|
|
98
|
+
- `indexing.embedding_batch_size` / `indexing.batch_count` are updated after indexing/autotune.
|
|
99
|
+
- `rag-setup.run` / `rag-setup-macos.run` persist chosen values into this file.
|
|
100
|
+
- If `host` is not local (`localhost`, `127.0.0.1`, `::1`), setup skips local Docker startup.
|
|
101
|
+
|
|
102
|
+
## ChromaDB Port Behavior (Local)
|
|
67
103
|
|
|
68
104
|
- Default host port is `8000`.
|
|
69
105
|
- During ChromaDB install/reinstall, setup asks for the port.
|
|
@@ -75,7 +111,7 @@ rag remove
|
|
|
75
111
|
- Selected port is propagated to:
|
|
76
112
|
- Docker Compose mapping
|
|
77
113
|
- health checks
|
|
78
|
-
-
|
|
114
|
+
- `~/.own-rag-cli.json`
|
|
79
115
|
- indexer runtime (`MCP_CHROMA_PORT`)
|
|
80
116
|
|
|
81
117
|
## Performance Profiles
|
|
@@ -95,6 +131,9 @@ Este modo pode elevar consideravelmente o consumo de memória e causar encerrame
|
|
|
95
131
|
- `MCP_EMBEDDING_MODEL=jina|bge|hybrid`
|
|
96
132
|
- `MCP_JINA_QUANTIZATION=default|dynamic-int8`
|
|
97
133
|
- `MCP_PERF_PROFILE=autotune|max-performance`
|
|
134
|
+
- `OWN_RAG_CLI_CONFIG_FILE=~/.own-rag-cli.json`
|
|
135
|
+
- `MCP_CHROMA_SCHEME=http|https`
|
|
136
|
+
- `MCP_CHROMA_HOST=localhost|<host>`
|
|
98
137
|
- `MCP_CHROMA_PORT=8000`
|
|
99
138
|
- `MCP_CHUNK_SIZE`
|
|
100
139
|
- `MCP_CHUNK_OVERLAP`
|
package/bin/indexer_full.py
CHANGED
|
@@ -22,6 +22,7 @@ from collections.abc import Iterator
|
|
|
22
22
|
from pathlib import Path
|
|
23
23
|
from dataclasses import dataclass
|
|
24
24
|
from datetime import datetime
|
|
25
|
+
from urllib.parse import urlparse
|
|
25
26
|
|
|
26
27
|
# Evita avisos "advisory" ruidosos do transformers no fluxo interativo.
|
|
27
28
|
os.environ.setdefault("TRANSFORMERS_NO_ADVISORY_WARNINGS", "1")
|
|
@@ -55,8 +56,126 @@ def _env_int(name: str, default: int, *, min_value: int = 1) -> int:
|
|
|
55
56
|
except ValueError:
|
|
56
57
|
return max(min_value, default)
|
|
57
58
|
|
|
58
|
-
|
|
59
|
-
|
|
59
|
+
|
|
60
|
+
def _env_str(name: str, default: str) -> str:
|
|
61
|
+
raw = os.environ.get(name)
|
|
62
|
+
if raw is None:
|
|
63
|
+
return default
|
|
64
|
+
stripped = raw.strip()
|
|
65
|
+
return stripped if stripped else default
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _parse_positive_int(raw: object, *, min_value: int = 1) -> int | None:
|
|
69
|
+
try:
|
|
70
|
+
parsed = int(str(raw).strip())
|
|
71
|
+
except Exception:
|
|
72
|
+
return None
|
|
73
|
+
if parsed < min_value:
|
|
74
|
+
return None
|
|
75
|
+
return parsed
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
OWN_RAG_CLI_CONFIG_PATH = Path(
|
|
79
|
+
os.environ.get("OWN_RAG_CLI_CONFIG_FILE", str(Path.home() / ".own-rag-cli.json"))
|
|
80
|
+
).expanduser()
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _load_own_rag_cli_config_payload() -> dict[str, object]:
|
|
84
|
+
try:
|
|
85
|
+
if not OWN_RAG_CLI_CONFIG_PATH.exists():
|
|
86
|
+
return {}
|
|
87
|
+
payload = json.loads(OWN_RAG_CLI_CONFIG_PATH.read_text(encoding="utf-8"))
|
|
88
|
+
return payload if isinstance(payload, dict) else {}
|
|
89
|
+
except Exception:
|
|
90
|
+
return {}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _extract_chroma_endpoint_from_config(config_payload: object) -> tuple[str, str, int]:
|
|
94
|
+
scheme = "http"
|
|
95
|
+
host = "localhost"
|
|
96
|
+
port = 8000
|
|
97
|
+
|
|
98
|
+
def _read_scheme(raw: object) -> None:
|
|
99
|
+
nonlocal scheme
|
|
100
|
+
if not isinstance(raw, str):
|
|
101
|
+
return
|
|
102
|
+
lowered = raw.strip().lower()
|
|
103
|
+
if lowered in {"http", "https"}:
|
|
104
|
+
scheme = lowered
|
|
105
|
+
|
|
106
|
+
def _read_host(raw: object) -> None:
|
|
107
|
+
nonlocal scheme, host, port
|
|
108
|
+
if not isinstance(raw, str):
|
|
109
|
+
return
|
|
110
|
+
candidate = raw.strip()
|
|
111
|
+
if not candidate:
|
|
112
|
+
return
|
|
113
|
+
parsed = urlparse(candidate if "://" in candidate else f"//{candidate}")
|
|
114
|
+
if parsed.scheme in {"http", "https"}:
|
|
115
|
+
scheme = parsed.scheme
|
|
116
|
+
if parsed.hostname:
|
|
117
|
+
host = parsed.hostname
|
|
118
|
+
elif "://" not in candidate:
|
|
119
|
+
host = candidate.strip().strip("/")
|
|
120
|
+
if parsed.port and 1 <= parsed.port <= 65535:
|
|
121
|
+
port = parsed.port
|
|
122
|
+
|
|
123
|
+
def _read_port(raw: object) -> None:
|
|
124
|
+
nonlocal port
|
|
125
|
+
try:
|
|
126
|
+
parsed = int(str(raw).strip())
|
|
127
|
+
except Exception:
|
|
128
|
+
return
|
|
129
|
+
if 1 <= parsed <= 65535:
|
|
130
|
+
port = parsed
|
|
131
|
+
|
|
132
|
+
if isinstance(config_payload, dict):
|
|
133
|
+
chroma_obj = config_payload.get("chroma")
|
|
134
|
+
if isinstance(chroma_obj, dict):
|
|
135
|
+
_read_scheme(chroma_obj.get("scheme"))
|
|
136
|
+
_read_host(chroma_obj.get("host"))
|
|
137
|
+
_read_port(chroma_obj.get("port"))
|
|
138
|
+
|
|
139
|
+
# Compatibilidade com formatos flat antigos.
|
|
140
|
+
_read_scheme(config_payload.get("CHROMA_SCHEME"))
|
|
141
|
+
_read_host(config_payload.get("CHROMA_HOST"))
|
|
142
|
+
_read_port(config_payload.get("CHROMA_PORT"))
|
|
143
|
+
|
|
144
|
+
return scheme, host, port
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _extract_embedding_batch_from_config(config_payload: object) -> int | None:
|
|
148
|
+
if not isinstance(config_payload, dict):
|
|
149
|
+
return None
|
|
150
|
+
|
|
151
|
+
indexing = config_payload.get("indexing")
|
|
152
|
+
if isinstance(indexing, dict):
|
|
153
|
+
for key in ("embedding_batch_size", "batch_count"):
|
|
154
|
+
parsed = _parse_positive_int(indexing.get(key), min_value=1)
|
|
155
|
+
if parsed is not None:
|
|
156
|
+
return parsed
|
|
157
|
+
|
|
158
|
+
for key in ("MCP_EMBEDDING_BATCH_SIZE", "embedding_batch_size", "batch_count"):
|
|
159
|
+
parsed = _parse_positive_int(config_payload.get(key), min_value=1)
|
|
160
|
+
if parsed is not None:
|
|
161
|
+
return parsed
|
|
162
|
+
return None
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _load_cli_chroma_defaults() -> tuple[str, str, int]:
|
|
166
|
+
payload = _load_own_rag_cli_config_payload()
|
|
167
|
+
if not payload:
|
|
168
|
+
return ("http", "localhost", 8000)
|
|
169
|
+
return _extract_chroma_endpoint_from_config(payload)
|
|
170
|
+
|
|
171
|
+
_DEFAULT_CHROMA_SCHEME, _DEFAULT_CHROMA_HOST, _DEFAULT_CHROMA_PORT = _load_cli_chroma_defaults()
|
|
172
|
+
_DEFAULT_EMBEDDING_BATCH_SIZE = _extract_embedding_batch_from_config(_load_own_rag_cli_config_payload()) or 4
|
|
173
|
+
CHROMA_SCHEME = _env_str("MCP_CHROMA_SCHEME", _DEFAULT_CHROMA_SCHEME).lower()
|
|
174
|
+
if CHROMA_SCHEME not in {"http", "https"}:
|
|
175
|
+
CHROMA_SCHEME = _DEFAULT_CHROMA_SCHEME
|
|
176
|
+
CHROMA_HOST = _env_str("MCP_CHROMA_HOST", _DEFAULT_CHROMA_HOST)
|
|
177
|
+
CHROMA_PORT = _env_int("MCP_CHROMA_PORT", _DEFAULT_CHROMA_PORT, min_value=1)
|
|
178
|
+
CHROMA_SSL = CHROMA_SCHEME == "https"
|
|
60
179
|
COLLECTION_CODE_JINA = "code_vectors_jina"
|
|
61
180
|
COLLECTION_DOC_BGE = "doc_vectors_bge"
|
|
62
181
|
|
|
@@ -104,12 +223,18 @@ MAX_FILE_SIZE_BYTES = 500 * 1024 # 500 KB
|
|
|
104
223
|
# Parâmetros do splitter e batch (perfil low-memory por padrão).
|
|
105
224
|
CHUNK_SIZE = _env_int("MCP_CHUNK_SIZE", 3000, min_value=256)
|
|
106
225
|
CHUNK_OVERLAP = min(CHUNK_SIZE - 1, _env_int("MCP_CHUNK_OVERLAP", 400, min_value=0))
|
|
107
|
-
EMBEDDING_BATCH_SIZE = _env_int(
|
|
226
|
+
EMBEDDING_BATCH_SIZE = _env_int(
|
|
227
|
+
"MCP_EMBEDDING_BATCH_SIZE",
|
|
228
|
+
_DEFAULT_EMBEDDING_BATCH_SIZE,
|
|
229
|
+
min_value=1,
|
|
230
|
+
)
|
|
108
231
|
DEFAULT_PERF_PROFILE = "autotune"
|
|
109
232
|
INDEXER_CONFIG_PATH = Path(
|
|
110
|
-
os.environ.get(
|
|
233
|
+
os.environ.get(
|
|
234
|
+
"MCP_INDEXER_CONFIG_FILE",
|
|
235
|
+
str(Path.home() / ".cache" / "own-rag-cli" / "indexer_tuning.json"),
|
|
236
|
+
)
|
|
111
237
|
).expanduser()
|
|
112
|
-
INDEXER_CONFIG_FALLBACK_PATH = Path.home() / ".cache" / "my-custom-rag-python" / "indexer_tuning.json"
|
|
113
238
|
|
|
114
239
|
# Modelo de embeddings (roda na CPU)
|
|
115
240
|
JINA_V3_EMBEDDING_MODEL = "jinaai/jina-embeddings-v3"
|
|
@@ -118,7 +243,7 @@ BGE_EMBEDDING_MODEL = "BAAI/bge-m3"
|
|
|
118
243
|
DEFAULT_EMBEDDING_MODEL_CHOICE = "jina"
|
|
119
244
|
DEFAULT_JINA_QUANTIZATION = "dynamic-int8"
|
|
120
245
|
MODEL_CACHE_BASE_DIR = Path(
|
|
121
|
-
os.environ.get("MCP_MODEL_DIR", str(Path.home() / ".cache" / "
|
|
246
|
+
os.environ.get("MCP_MODEL_DIR", str(Path.home() / ".cache" / "own-rag-cli" / "models"))
|
|
122
247
|
).expanduser()
|
|
123
248
|
JINA_RECOMMENDED_RAM_GB_DEFAULT = 64
|
|
124
249
|
JINA_RECOMMENDED_RAM_GB_DYNAMIC_INT8 = 48
|
|
@@ -349,52 +474,72 @@ def resolve_embedding_config(
|
|
|
349
474
|
return model_choice, jina_quantization
|
|
350
475
|
|
|
351
476
|
|
|
352
|
-
def _indexer_config_candidates() -> list[Path]:
|
|
353
|
-
candidates = [INDEXER_CONFIG_PATH]
|
|
354
|
-
if INDEXER_CONFIG_FALLBACK_PATH not in candidates:
|
|
355
|
-
candidates.append(INDEXER_CONFIG_FALLBACK_PATH)
|
|
356
|
-
return candidates
|
|
357
|
-
|
|
358
|
-
|
|
359
477
|
def load_indexer_tuning_config(force_reconfigure: bool) -> dict[str, object]:
|
|
360
478
|
if force_reconfigure:
|
|
361
479
|
return {}
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
continue
|
|
480
|
+
try:
|
|
481
|
+
if not INDEXER_CONFIG_PATH.exists():
|
|
482
|
+
return {}
|
|
483
|
+
data = json.loads(INDEXER_CONFIG_PATH.read_text(encoding="utf-8"))
|
|
484
|
+
if isinstance(data, dict):
|
|
485
|
+
return data
|
|
486
|
+
except Exception:
|
|
487
|
+
return {}
|
|
371
488
|
return {}
|
|
372
489
|
|
|
373
490
|
|
|
491
|
+
def _persist_batch_to_cli_config(batch_size: int) -> None:
|
|
492
|
+
payload = _load_own_rag_cli_config_payload()
|
|
493
|
+
chroma = payload.get("chroma")
|
|
494
|
+
if not isinstance(chroma, dict):
|
|
495
|
+
chroma = {
|
|
496
|
+
"scheme": CHROMA_SCHEME,
|
|
497
|
+
"host": CHROMA_HOST,
|
|
498
|
+
"port": CHROMA_PORT,
|
|
499
|
+
}
|
|
500
|
+
payload["chroma"] = chroma
|
|
501
|
+
|
|
502
|
+
indexing = payload.get("indexing")
|
|
503
|
+
if not isinstance(indexing, dict):
|
|
504
|
+
indexing = {}
|
|
505
|
+
indexing["embedding_batch_size"] = batch_size
|
|
506
|
+
indexing["batch_count"] = batch_size
|
|
507
|
+
payload["indexing"] = indexing
|
|
508
|
+
|
|
509
|
+
try:
|
|
510
|
+
OWN_RAG_CLI_CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
511
|
+
OWN_RAG_CLI_CONFIG_PATH.write_text(
|
|
512
|
+
json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
|
|
513
|
+
encoding="utf-8",
|
|
514
|
+
)
|
|
515
|
+
except Exception as e:
|
|
516
|
+
print(
|
|
517
|
+
f"[AVISO] Não foi possível atualizar batch em "
|
|
518
|
+
f"{OWN_RAG_CLI_CONFIG_PATH}: {_format_exception(e)}"
|
|
519
|
+
)
|
|
520
|
+
|
|
521
|
+
|
|
374
522
|
def save_indexer_tuning_config(config: dict[str, object]) -> None:
|
|
523
|
+
batch_size = _parse_positive_int(config.get("embedding_batch_size"), min_value=1)
|
|
375
524
|
payload = {
|
|
376
525
|
**config,
|
|
377
526
|
"updated_at": int(time()),
|
|
378
527
|
}
|
|
379
|
-
|
|
528
|
+
if batch_size is not None:
|
|
529
|
+
payload["batch_count"] = batch_size
|
|
380
530
|
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
f"(destino primário sem permissão: {INDEXER_CONFIG_PATH})"
|
|
391
|
-
)
|
|
392
|
-
return
|
|
393
|
-
except Exception as e:
|
|
394
|
-
write_errors.append((candidate, e))
|
|
531
|
+
try:
|
|
532
|
+
INDEXER_CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
533
|
+
INDEXER_CONFIG_PATH.write_text(
|
|
534
|
+
json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
|
|
535
|
+
encoding="utf-8",
|
|
536
|
+
)
|
|
537
|
+
print(f"[CONFIG] Configuração persistida em: {INDEXER_CONFIG_PATH}")
|
|
538
|
+
except Exception as e:
|
|
539
|
+
print(f"[AVISO] Não foi possível persistir configuração em {INDEXER_CONFIG_PATH}: {_format_exception(e)}")
|
|
395
540
|
|
|
396
|
-
|
|
397
|
-
|
|
541
|
+
if batch_size is not None:
|
|
542
|
+
_persist_batch_to_cli_config(batch_size)
|
|
398
543
|
|
|
399
544
|
|
|
400
545
|
def resolve_perf_profile(perf_profile_arg: str | None, persisted_config: dict[str, object]) -> str:
|
|
@@ -921,15 +1066,14 @@ def load_embedding_model(model_choice: str, jina_quantization: str) -> SentenceT
|
|
|
921
1066
|
def connect_to_chroma() -> chromadb.HttpClient:
|
|
922
1067
|
"""Conecta ao ChromaDB via HTTP e valida a conexão."""
|
|
923
1068
|
try:
|
|
924
|
-
client = chromadb.HttpClient(host=CHROMA_HOST, port=CHROMA_PORT)
|
|
1069
|
+
client = chromadb.HttpClient(host=CHROMA_HOST, port=CHROMA_PORT, ssl=CHROMA_SSL)
|
|
925
1070
|
# Faz um heartbeat para confirmar que o servidor está no ar
|
|
926
1071
|
client.heartbeat()
|
|
927
|
-
print(f"[+] Conectado ao ChromaDB em {CHROMA_HOST}:{CHROMA_PORT}")
|
|
1072
|
+
print(f"[+] Conectado ao ChromaDB em {CHROMA_SCHEME}://{CHROMA_HOST}:{CHROMA_PORT}")
|
|
928
1073
|
return client
|
|
929
1074
|
except Exception as e:
|
|
930
1075
|
print(f"[ERRO] Não foi possível conectar ao ChromaDB: {e}")
|
|
931
|
-
print("
|
|
932
|
-
print(" docker compose up -d")
|
|
1076
|
+
print(f" Endpoint configurado: {CHROMA_SCHEME}://{CHROMA_HOST}:{CHROMA_PORT}")
|
|
933
1077
|
sys.exit(1)
|
|
934
1078
|
|
|
935
1079
|
|
|
@@ -1141,6 +1285,8 @@ def main():
|
|
|
1141
1285
|
persisted_chunk_size = _parse_config_int(persisted_config, "chunk_size")
|
|
1142
1286
|
persisted_chunk_overlap = _parse_config_int(persisted_config, "chunk_overlap")
|
|
1143
1287
|
persisted_batch_size = _parse_config_int(persisted_config, "embedding_batch_size")
|
|
1288
|
+
if persisted_batch_size is None:
|
|
1289
|
+
persisted_batch_size = _parse_config_int(persisted_config, "batch_count")
|
|
1144
1290
|
|
|
1145
1291
|
effective_chunk_size = CHUNK_SIZE
|
|
1146
1292
|
if not chunk_size_locked and persisted_chunk_size is not None:
|
package/bin/mcp_server.py
CHANGED
|
@@ -21,6 +21,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
21
21
|
from dataclasses import dataclass
|
|
22
22
|
from datetime import datetime, timezone
|
|
23
23
|
from pathlib import Path
|
|
24
|
+
from urllib.parse import urlparse
|
|
24
25
|
|
|
25
26
|
# Evita mensagens advisory do transformers em stderr durante a carga do modelo.
|
|
26
27
|
os.environ.setdefault("TRANSFORMERS_NO_ADVISORY_WARNINGS", "1")
|
|
@@ -118,7 +119,13 @@ def _log_tool_usage(event: str, tool_name: str, details: dict[str, object] | Non
|
|
|
118
119
|
|
|
119
120
|
|
|
120
121
|
INDEXER_CONFIG_PATH = Path(
|
|
121
|
-
os.environ.get(
|
|
122
|
+
os.environ.get(
|
|
123
|
+
"MCP_INDEXER_CONFIG_FILE",
|
|
124
|
+
str(Path.home() / ".cache" / "own-rag-cli" / "indexer_tuning.json"),
|
|
125
|
+
)
|
|
126
|
+
).expanduser()
|
|
127
|
+
OWN_RAG_CLI_CONFIG_PATH = Path(
|
|
128
|
+
os.environ.get("OWN_RAG_CLI_CONFIG_FILE", str(Path.home() / ".own-rag-cli.json"))
|
|
122
129
|
).expanduser()
|
|
123
130
|
|
|
124
131
|
|
|
@@ -135,6 +142,16 @@ def _load_indexer_tuning_config() -> dict[str, object]:
|
|
|
135
142
|
INDEXER_TUNING_CONFIG = _load_indexer_tuning_config()
|
|
136
143
|
|
|
137
144
|
|
|
145
|
+
def _load_own_rag_cli_config_payload() -> dict[str, object]:
|
|
146
|
+
try:
|
|
147
|
+
if not OWN_RAG_CLI_CONFIG_PATH.exists():
|
|
148
|
+
return {}
|
|
149
|
+
payload = json.loads(OWN_RAG_CLI_CONFIG_PATH.read_text(encoding="utf-8"))
|
|
150
|
+
return payload if isinstance(payload, dict) else {}
|
|
151
|
+
except Exception:
|
|
152
|
+
return {}
|
|
153
|
+
|
|
154
|
+
|
|
138
155
|
def _config_str(env_name: str, config_key: str, default: str) -> str:
|
|
139
156
|
env_raw = os.environ.get(env_name)
|
|
140
157
|
if env_raw is not None and env_raw.strip():
|
|
@@ -145,7 +162,108 @@ def _config_str(env_name: str, config_key: str, default: str) -> str:
|
|
|
145
162
|
return default
|
|
146
163
|
|
|
147
164
|
|
|
148
|
-
def
|
|
165
|
+
def _parse_int(raw: object, default: int, *, min_value: int = 1) -> int:
|
|
166
|
+
try:
|
|
167
|
+
parsed = int(str(raw).strip())
|
|
168
|
+
except Exception:
|
|
169
|
+
return max(min_value, default)
|
|
170
|
+
return max(min_value, parsed)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _extract_chroma_endpoint_from_config(config_payload: object) -> tuple[str, str, int]:
|
|
174
|
+
scheme = "http"
|
|
175
|
+
host = "localhost"
|
|
176
|
+
port = 8000
|
|
177
|
+
|
|
178
|
+
def _read_scheme(raw: object) -> None:
|
|
179
|
+
nonlocal scheme
|
|
180
|
+
if not isinstance(raw, str):
|
|
181
|
+
return
|
|
182
|
+
lowered = raw.strip().lower()
|
|
183
|
+
if lowered in {"http", "https"}:
|
|
184
|
+
scheme = lowered
|
|
185
|
+
|
|
186
|
+
def _read_host(raw: object) -> None:
|
|
187
|
+
nonlocal scheme, host, port
|
|
188
|
+
if not isinstance(raw, str):
|
|
189
|
+
return
|
|
190
|
+
candidate = raw.strip()
|
|
191
|
+
if not candidate:
|
|
192
|
+
return
|
|
193
|
+
parsed = urlparse(candidate if "://" in candidate else f"//{candidate}")
|
|
194
|
+
if parsed.scheme in {"http", "https"}:
|
|
195
|
+
scheme = parsed.scheme
|
|
196
|
+
if parsed.hostname:
|
|
197
|
+
host = parsed.hostname
|
|
198
|
+
elif "://" not in candidate:
|
|
199
|
+
host = candidate.strip().strip("/")
|
|
200
|
+
if parsed.port and 1 <= parsed.port <= 65535:
|
|
201
|
+
port = parsed.port
|
|
202
|
+
|
|
203
|
+
def _read_port(raw: object) -> None:
|
|
204
|
+
nonlocal port
|
|
205
|
+
try:
|
|
206
|
+
parsed = int(str(raw).strip())
|
|
207
|
+
except Exception:
|
|
208
|
+
return
|
|
209
|
+
if 1 <= parsed <= 65535:
|
|
210
|
+
port = parsed
|
|
211
|
+
|
|
212
|
+
if isinstance(config_payload, dict):
|
|
213
|
+
chroma_obj = config_payload.get("chroma")
|
|
214
|
+
if isinstance(chroma_obj, dict):
|
|
215
|
+
_read_scheme(chroma_obj.get("scheme"))
|
|
216
|
+
_read_host(chroma_obj.get("host"))
|
|
217
|
+
_read_port(chroma_obj.get("port"))
|
|
218
|
+
|
|
219
|
+
# Compatibilidade com formatos flat antigos.
|
|
220
|
+
_read_scheme(config_payload.get("CHROMA_SCHEME"))
|
|
221
|
+
_read_host(config_payload.get("CHROMA_HOST"))
|
|
222
|
+
_read_port(config_payload.get("CHROMA_PORT"))
|
|
223
|
+
|
|
224
|
+
return scheme, host, port
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _extract_embedding_batch_from_config(config_payload: object) -> int | None:
|
|
228
|
+
if not isinstance(config_payload, dict):
|
|
229
|
+
return None
|
|
230
|
+
|
|
231
|
+
def _read(raw: object) -> int | None:
|
|
232
|
+
try:
|
|
233
|
+
parsed = int(str(raw).strip())
|
|
234
|
+
except Exception:
|
|
235
|
+
return None
|
|
236
|
+
return parsed if parsed >= 1 else None
|
|
237
|
+
|
|
238
|
+
indexing = config_payload.get("indexing")
|
|
239
|
+
if isinstance(indexing, dict):
|
|
240
|
+
for key in ("embedding_batch_size", "batch_count"):
|
|
241
|
+
parsed = _read(indexing.get(key))
|
|
242
|
+
if parsed is not None:
|
|
243
|
+
return parsed
|
|
244
|
+
|
|
245
|
+
for key in ("MCP_EMBEDDING_BATCH_SIZE", "embedding_batch_size", "batch_count"):
|
|
246
|
+
parsed = _read(config_payload.get(key))
|
|
247
|
+
if parsed is not None:
|
|
248
|
+
return parsed
|
|
249
|
+
return None
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _load_cli_chroma_defaults() -> tuple[str, str, int]:
|
|
253
|
+
payload = _load_own_rag_cli_config_payload()
|
|
254
|
+
if not payload:
|
|
255
|
+
return ("http", "localhost", 8000)
|
|
256
|
+
return _extract_chroma_endpoint_from_config(payload)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _config_int(
|
|
260
|
+
env_name: str,
|
|
261
|
+
config_key: str,
|
|
262
|
+
default: int,
|
|
263
|
+
*,
|
|
264
|
+
min_value: int = 1,
|
|
265
|
+
fallback_config_keys: tuple[str, ...] = (),
|
|
266
|
+
) -> int:
|
|
149
267
|
env_raw = os.environ.get(env_name)
|
|
150
268
|
if env_raw is not None and env_raw.strip():
|
|
151
269
|
try:
|
|
@@ -153,20 +271,41 @@ def _config_int(env_name: str, config_key: str, default: int, *, min_value: int
|
|
|
153
271
|
except ValueError:
|
|
154
272
|
pass
|
|
155
273
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
274
|
+
for key in (config_key, *fallback_config_keys):
|
|
275
|
+
cfg_raw = INDEXER_TUNING_CONFIG.get(key)
|
|
276
|
+
if isinstance(cfg_raw, int):
|
|
277
|
+
return max(min_value, cfg_raw)
|
|
278
|
+
if isinstance(cfg_raw, str):
|
|
279
|
+
try:
|
|
280
|
+
return max(min_value, int(cfg_raw))
|
|
281
|
+
except ValueError:
|
|
282
|
+
pass
|
|
164
283
|
|
|
165
284
|
return max(min_value, default)
|
|
166
285
|
|
|
167
286
|
|
|
168
|
-
|
|
169
|
-
|
|
287
|
+
_DEFAULT_CHROMA_SCHEME, _DEFAULT_CHROMA_HOST, _DEFAULT_CHROMA_PORT = _load_cli_chroma_defaults()
|
|
288
|
+
_DEFAULT_EMBEDDING_BATCH_SIZE = _extract_embedding_batch_from_config(_load_own_rag_cli_config_payload()) or 4
|
|
289
|
+
CHROMA_SCHEME = (
|
|
290
|
+
os.environ.get("CHROMA_SCHEME")
|
|
291
|
+
or os.environ.get("MCP_CHROMA_SCHEME")
|
|
292
|
+
or _DEFAULT_CHROMA_SCHEME
|
|
293
|
+
).strip().lower()
|
|
294
|
+
if CHROMA_SCHEME not in {"http", "https"}:
|
|
295
|
+
CHROMA_SCHEME = _DEFAULT_CHROMA_SCHEME
|
|
296
|
+
CHROMA_HOST = (
|
|
297
|
+
os.environ.get("CHROMA_HOST")
|
|
298
|
+
or os.environ.get("MCP_CHROMA_HOST")
|
|
299
|
+
or _DEFAULT_CHROMA_HOST
|
|
300
|
+
).strip() or _DEFAULT_CHROMA_HOST
|
|
301
|
+
CHROMA_PORT = _parse_int(
|
|
302
|
+
os.environ.get("CHROMA_PORT")
|
|
303
|
+
or os.environ.get("MCP_CHROMA_PORT")
|
|
304
|
+
or _DEFAULT_CHROMA_PORT,
|
|
305
|
+
_DEFAULT_CHROMA_PORT,
|
|
306
|
+
min_value=1,
|
|
307
|
+
)
|
|
308
|
+
CHROMA_SSL = CHROMA_SCHEME == "https"
|
|
170
309
|
|
|
171
310
|
# Coleções separadas por especialização de embedding
|
|
172
311
|
COLLECTION_CODE_JINA = "code_vectors_jina"
|
|
@@ -225,13 +364,19 @@ if RERANKER_QUANTIZATION not in {"default", "dynamic-int8"}:
|
|
|
225
364
|
RERANKER_QUANTIZATION = "dynamic-int8"
|
|
226
365
|
|
|
227
366
|
RRF_K = int(os.environ.get("MCP_RRF_K", "60"))
|
|
228
|
-
EMBEDDING_BATCH_SIZE = _config_int(
|
|
367
|
+
EMBEDDING_BATCH_SIZE = _config_int(
|
|
368
|
+
"MCP_EMBEDDING_BATCH_SIZE",
|
|
369
|
+
"embedding_batch_size",
|
|
370
|
+
_DEFAULT_EMBEDDING_BATCH_SIZE,
|
|
371
|
+
min_value=1,
|
|
372
|
+
fallback_config_keys=("batch_count",),
|
|
373
|
+
)
|
|
229
374
|
|
|
230
375
|
_env_model_dir = os.environ.get("MCP_MODEL_DIR")
|
|
231
376
|
MODEL_DIR = (
|
|
232
377
|
Path(_env_model_dir).expanduser()
|
|
233
378
|
if _env_model_dir
|
|
234
|
-
else Path.home() / ".cache" / "
|
|
379
|
+
else Path.home() / ".cache" / "own-rag-cli" / "models"
|
|
235
380
|
)
|
|
236
381
|
|
|
237
382
|
# Parâmetros do splitter (alinhados com indexer_full.py, perfil low-memory)
|
|
@@ -355,9 +500,9 @@ def _model_cache_dir(base_dir: Path, model_id: str) -> Path:
|
|
|
355
500
|
def _get_chroma_client() -> chromadb.HttpClient:
|
|
356
501
|
global _chroma_client
|
|
357
502
|
if _chroma_client is None:
|
|
358
|
-
_chroma_client = chromadb.HttpClient(host=CHROMA_HOST, port=CHROMA_PORT)
|
|
503
|
+
_chroma_client = chromadb.HttpClient(host=CHROMA_HOST, port=CHROMA_PORT, ssl=CHROMA_SSL)
|
|
359
504
|
_chroma_client.heartbeat()
|
|
360
|
-
log.info("Conectado ao ChromaDB em %s:%s", CHROMA_HOST, CHROMA_PORT)
|
|
505
|
+
log.info("Conectado ao ChromaDB em %s://%s:%s", CHROMA_SCHEME, CHROMA_HOST, CHROMA_PORT)
|
|
361
506
|
return _chroma_client
|
|
362
507
|
|
|
363
508
|
|
|
@@ -376,7 +521,7 @@ def get_chroma_collection(collection_name: str) -> chromadb.Collection:
|
|
|
376
521
|
except Exception as e:
|
|
377
522
|
raise RuntimeError(
|
|
378
523
|
f"Não foi possível acessar a coleção '{collection_name}' no ChromaDB "
|
|
379
|
-
f"({CHROMA_HOST}:{CHROMA_PORT}). Erro: {e}"
|
|
524
|
+
f"({CHROMA_SCHEME}://{CHROMA_HOST}:{CHROMA_PORT}). Erro: {e}"
|
|
380
525
|
)
|
|
381
526
|
|
|
382
527
|
|
|
@@ -1402,7 +1547,7 @@ def index_specific_folder(folder_path: str) -> str:
|
|
|
1402
1547
|
|
|
1403
1548
|
if __name__ == "__main__":
|
|
1404
1549
|
log.info("Iniciando servidor MCP RAG (stdio)...")
|
|
1405
|
-
log.info("ChromaDB: %s:%s", CHROMA_HOST, CHROMA_PORT)
|
|
1550
|
+
log.info("ChromaDB: %s://%s:%s", CHROMA_SCHEME, CHROMA_HOST, CHROMA_PORT)
|
|
1406
1551
|
log.info(
|
|
1407
1552
|
"Coleções: %s (%s), %s (%s)",
|
|
1408
1553
|
COLLECTION_CODE_JINA,
|
package/bin/postinstall.sh
CHANGED
|
@@ -16,6 +16,7 @@ MONITOR_SRC="${PACKAGE_ROOT}/chroma_monitor.sh"
|
|
|
16
16
|
MONITOR_DEST="${LOCAL_BIN_DIR}/chroma_monitor.sh"
|
|
17
17
|
REMOVE_SRC="${PACKAGE_ROOT}/bin/rag-remove.sh"
|
|
18
18
|
REMOVE_DEST="${LOCAL_BIN_DIR}/rag-remove.sh"
|
|
19
|
+
OWN_RAG_CONFIG_FILE="${HOME}/.own-rag-cli.json"
|
|
19
20
|
|
|
20
21
|
COMPOSE_SOURCE="${PACKAGE_ROOT}/bin/docker-compose.yml"
|
|
21
22
|
COMPOSE_DIR="${HOME}/docker-chromadb"
|
|
@@ -26,7 +27,34 @@ ALIAS_LINE="alias rag='~/.local/bin/rag-wrapper.sh'"
|
|
|
26
27
|
log_info() { printf "[+] %s\n" "$*"; }
|
|
27
28
|
log_warn() { printf "[!] %s\n" "$*" >&2; }
|
|
28
29
|
|
|
30
|
+
ensure_own_rag_cli_config() {
|
|
31
|
+
python3 - "${OWN_RAG_CONFIG_FILE}" <<'PYEOF'
|
|
32
|
+
import json
|
|
33
|
+
import sys
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
|
|
36
|
+
cfg = Path(sys.argv[1]).expanduser()
|
|
37
|
+
if cfg.exists():
|
|
38
|
+
raise SystemExit(0)
|
|
39
|
+
cfg.parent.mkdir(parents=True, exist_ok=True)
|
|
40
|
+
payload = {
|
|
41
|
+
"chroma": {
|
|
42
|
+
"scheme": "http",
|
|
43
|
+
"host": "localhost",
|
|
44
|
+
"port": 8000,
|
|
45
|
+
},
|
|
46
|
+
"indexing": {
|
|
47
|
+
"embedding_batch_size": 4,
|
|
48
|
+
"batch_count": 4,
|
|
49
|
+
},
|
|
50
|
+
}
|
|
51
|
+
cfg.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
52
|
+
PYEOF
|
|
53
|
+
}
|
|
54
|
+
|
|
29
55
|
mkdir -p "${LOCAL_BIN_DIR}"
|
|
56
|
+
ensure_own_rag_cli_config
|
|
57
|
+
log_info "Config criada/ok: ${OWN_RAG_CONFIG_FILE}"
|
|
30
58
|
|
|
31
59
|
if [[ -f "${WRAPPER_SRC}" ]]; then
|
|
32
60
|
cp "${WRAPPER_SRC}" "${WRAPPER_DEST}"
|
package/bin/rag-remove.sh
CHANGED
|
@@ -96,8 +96,10 @@ declare -a REMOVE_PATHS=(
|
|
|
96
96
|
"${HOME}/.rag_venv"
|
|
97
97
|
"${HOME}/.rag_db"
|
|
98
98
|
"${HOME}/docker-chromadb"
|
|
99
|
+
"${HOME}/.cache/own-rag-cli"
|
|
99
100
|
"${HOME}/.cache/my-custom-rag-python"
|
|
100
101
|
"${HOME}/.cache/ny-custom-rag-python"
|
|
102
|
+
"${HOME}/.own-rag-cli.json"
|
|
101
103
|
"${HOME}/.local/bin/mcp-rag-server"
|
|
102
104
|
"${HOME}/.local/bin/download_model_from_hugginface.py"
|
|
103
105
|
"${HOME}/.local/bin/download_model_from_modelscope.py"
|
|
@@ -183,6 +185,7 @@ PY
|
|
|
183
185
|
|
|
184
186
|
if command -v npm >/dev/null 2>&1; then
|
|
185
187
|
npm uninstall -g own-rag >/dev/null 2>&1 || true
|
|
188
|
+
npm uninstall -g own-rag-cli >/dev/null 2>&1 || true
|
|
186
189
|
fi
|
|
187
190
|
|
|
188
191
|
if [[ ${#FAILED_PATHS[@]} -gt 0 ]]; then
|