@allansantos-dev/smart-tool 0.9.78 → 0.9.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -55,6 +55,7 @@ import index_inventory
55
55
  import usage_meter
56
56
  import document_read
57
57
  import document_text
58
+ import dependency_notice
58
59
  import dossier
59
60
  import scout
60
61
  import embedding_cache
@@ -76,6 +77,7 @@ import site_memory
76
77
  import web_search_adapters
77
78
  import browser_service_client
78
79
  import local_embedder
80
+ import low_priority
79
81
  import web_search_health
80
82
  import web_fetch
81
83
  import version
@@ -172,6 +174,12 @@ SITE_MEMORY_MAX_TOPICS = 50
172
174
  # em até 0,064 e pares úteis a partir de 0,148. 0,12 separa as duas amostras
173
175
  # sem endurecer o limiar de embedding (0,35), que é só filtro de candidatos.
174
176
  SITE_MEMORY_RERANK_MIN_SCORE = 0.12
177
+ # The cosine scale depends on the embedder: granite (the embeddings home) puts unrelated web queries at 0.48 to 0.73,
178
+ # where text-embedding-3-small puts them at 0.09 to 0.54. On the same 20 near and 20 far pairs
179
+ # (docs/spikes/web_memory_threshold_spike.py), 0.62 and 0.68 keep every near pair and as many far ones as 0.35 and 0.45
180
+ # keep on the gateway model. The configured site memory threshold is the gateway's.
181
+ HOME_SITE_MEMORY_SIMILARITY = 0.62
182
+ SIMILAR_SEARCH_MIN_COSINE = {"gateway": 0.45, "home": 0.68}
175
183
 
176
184
 
177
185
  def _parse_bounded_int(value, default, min_v, max_v, field_name):
@@ -204,7 +212,10 @@ SMART_SEARCH_RESULT_WAIT_S = 15
204
212
  SMART_SEARCH_MAX_PENDING = 8
205
213
  BACKGROUND_MAX_PENDING = 8
206
214
  INTERACTIVE_PRIORITY = 1
207
- SMART_SEARCH_EXECUTOR = project_scheduler.ProjectScheduler(workers=3)
215
+ # Slowest first-index speed measured on a CPU-only embedder (docs/research/2026-11-16-indexacao-cpu-busca-durante-o-indice.md).
216
+ CPU_INDEX_FILES_PER_MINUTE = 180
217
+ _INDEX_RUNNING = {}
218
+ SMART_SEARCH_EXECUTOR = project_scheduler.ProjectScheduler(workers=3, reserved_lane="search")
208
219
  _PROJECT_LOCKS = {}
209
220
  _PROJECT_LOCKS_LOCK = threading.Lock()
210
221
  _PROJECT_MONITOR = None
@@ -373,10 +384,10 @@ SERVER_INSTRUCTIONS = (
373
384
  TOOLS = [
374
385
  {
375
386
  "name": "project_manage",
376
- "description": "Manages folders and indexing. Every action takes project_root (the folder) or project_id. list gives a short line per project (a project whose folder is gone shows as unavailable with the reason); status shows storage, scope and jobs, and with a path inside a registered project answers for the deepest one that holds it (resolved_from). graph with symbol (function, Class.method, class name, or path::name when the name repeats) answers what an edit touches: definition, callers with line, calls, tests that reach it through static calls (depth hops, default 3) and files importing it; use it before changing a function to know what to update. affected_tests answers which tests to run after editing: every test file importing a changed file (git diff against base, default HEAD, untracked included) plus changed tests, likeliest failures first, with the command to run them and run_all when config or unanalyzed code changed. pr_impact (base, default HEAD; main or origin/main for a branch) returns a Markdown report of the change for a pull request: the functions the diff touches with their callers, the documentation entries it leaves out of date and the tests to run with their command; it publishes nothing. graph without symbol reads the indexed view: coverage, references and static calls in Java, Angular, Python, Go and JS/TS/JSX/TSX; includes HTML/CSS/Markdown references and TXT/DOCX coverage; with file_path, narrows to that file's symbols, imports and calls (lists capped by limit). inspect lists the files of a view_id; with file_path, returns only that file's chunks. usage shows consumption and reuse; docs lists public functions without a docstring (Python docstring, JSDoc, Javadoc, Go doc comment starting with the name; private, nested and override functions excluded) with coverage percent, excluding tests unless include_tests=true, for documenting a project that started without it; docs generate=true writes the documentation scope to docs/scope in the project in three levels tied to the code: technical pages by folder (an entry per public function anchored on path::name with signature, first doc line, layer, callers, calls and tests, plus a Mermaid flow by layer), functional.md (each entry point and the functions its flow reaches, by layer) and simple.md; the prose between the prose markers is kept by regeneration; docs write_prose=true writes the empty prose blocks with the Docs model of the setup (each paragraph checked against the code first) within about 40 s and says how many are left, call it again until none is; docs check=true reports anchors whose function changed signature or is gone and broken links; docs validate=true checks every claim of docs/scope against the code (anchors, links, line numbers, callers and tests, coverage, entry points, command-line options, prose names, counts, glossary, navigation, versions) and writes docs/scope/validation.md with pass/fail per check, the evidence, a score and the verdict (generate runs it too); docs with symbol lists where that function is documented; layers declares the project's layers (ordered highest first, each with path globs and annotations): without layers it returns the saved rule or a suggestion read from folder names and annotations, with files, symbols and calls between layers (direction down or up); pass layers to preview a rule and the same layers with confirm=true to save it in the project (.smart-tool/layers.json); duplicates lists duplicated functions (identical, near-identical and semantic bodies, min_similarity default 0.90; below that most pairs measured were false positives), excluding tests unless include_tests=true; fix real duplicates; only dismiss with duplicates_dismiss (finding_id, reason=false_positive|intentional, note) what is not a duplicate or is a copy kept on purpose: the finding stays hidden until the code changes, and duplicates_restore reopens it; integration shows the MCP client, hook and tool usage metrics; install_hook_preview shows the change that would install the hook redirecting Grep/Read to smart_search and WebSearch/WebFetch to web_search/web_fetch in this session's client and returns a plan_id; install_hook writes it with that plan_id and confirm=true, with a backup, only after the user approves the change; search_limits saves the project's default top_k per block ({code, test, doc}, each 1-20, total at most 20); storage shows disk usage and views; pin_view protects a view; each view carries stale (not updated for 30 days, the test of the daily cleanup that removes them) and stale_bytes sums them; cleanup_preview and cleanup_commit remove selected views after confirmation; compare compares two stored views, without checkout or embeddings. register adds a folder; preview checks the scope; index/rebuild update; pause/cancel keep checkpoints; resume continues; policy picks on_search/eager; scope adjusts the scope; remove deletes only the local index. setup_audit (project_root) reports what Claude Code and Codex actually load for that folder (settings layers, plugins, hooks, MCP servers, skills, CLAUDE.md/AGENTS.md) and where they collide: hooks deciding on the same tool in parallel, missing hook scripts, silenced hooks, MCP servers down or offering duplicate tools, skill name and budget problems, AGENTS.md that only Codex reads, broken @imports; what could not be checked is listed as not verified; each finding carries its fix (the file to edit or the command to run), applied by the agent when the user agrees, never by Smart Tool; mode=last returns the stored report instead of auditing again. redirect_proposals (no project_root) lists the redirect patterns proposed from allowed searches, most matched first, with a guide; redirect_decide decides them by ids: accept, instruct (tip without blocking), reject, or remove an accepted one. junk (no project_root) lists the registered projects that were never projects: a personal, system, application-data or build folder, a subfolder of a repository or of another project, a folder of projects, or a folder that no longer exists (on a drive or share that is mounted); with confirm=true it unregisters them and deletes only their index files, and the manifest says what went. A project is the folder where the session started, so smart_search and register refuse a folder that is not a project and index a new one only from the session that started in it. Jobs via smart_search_result.",
387
+ "description": "Manages folders and indexing. Every action takes project_root (the folder) or project_id. list gives a short line per project (a project whose folder is gone shows as unavailable with the reason); status shows storage, scope and jobs, and with a path inside a registered project answers for the deepest one that holds it (resolved_from). deps lists the known OSV advisories of every dependency the manifests of the project declare (package.json, requirements*.txt, pyproject.toml, go.mod), not only the ones an edit changed, at the version a package-lock.json beside a package.json resolved when it satisfies the range; only package names and versions leave the machine. graph with symbol (function, Class.method, class name, or path::name when the name repeats) answers what an edit touches: definition, callers with line, calls, tests that reach it through static calls (depth hops, default 3) and files importing it; use it before changing a function to know what to update. affected_tests answers which tests to run after editing: every test file importing a changed file (git diff against base, default HEAD, untracked included) plus changed tests, likeliest failures first, with the command to run them and run_all when config or unanalyzed code changed. pr_impact (base, default HEAD; main or origin/main for a branch) returns a Markdown report of the change for a pull request: the functions the diff touches with their callers and the tests that reach them (those no test reaches are listed apart), the documentation entries it leaves out of date, the notes the edit hook would have added to the whole diff (comments, copies, dependency advisories) and the tests to run with their command; it publishes nothing. hotspots (limit) ranks the code files by commits times lines, the riskiest to change; dead_code lists the functions no code calls and those only tests call (static call graph, candidates); import_cycles lists the groups of files that import each other; comments lists comments in the wrong place (commented-out code, history tags, loose docs, long inline blocks) across the code files, not only in an edit. graph without symbol reads the indexed view: coverage, references and static calls in Java, Angular, Python, Go and JS/TS/JSX/TSX; includes HTML/CSS/Markdown references and TXT/DOCX coverage; with file_path, narrows to that file's symbols, imports and calls (lists capped by limit). inspect lists the files of a view_id; with file_path, returns only that file's chunks. usage shows consumption and reuse; docs lists public functions without a docstring (Python docstring, JSDoc, Javadoc, Go doc comment starting with the name; private, nested and override functions excluded) with coverage percent, excluding tests unless include_tests=true, for documenting a project that started without it; docs generate=true writes the documentation scope to docs/scope in the project in three levels tied to the code: technical pages by folder (an entry per public function anchored on path::name with signature, first doc line, layer, callers, calls and tests, plus a Mermaid flow by layer), functional.md (each entry point and the functions its flow reaches, by layer) and simple.md; the prose between the prose markers is kept by regeneration; docs write_prose=true writes the empty prose blocks with the Docs model of the setup (each paragraph checked against the code first) within about 40 s and says how many are left, call it again until none is; docs check=true reports anchors whose function changed signature or is gone and broken links; docs validate=true checks every claim of docs/scope against the code (anchors, links, line numbers, callers and tests, coverage, entry points, command-line options, prose names, counts, glossary, navigation, versions) and writes docs/scope/validation.md with pass/fail per check, the evidence, a score and the verdict (generate runs it too); docs with symbol lists where that function is documented; layers declares the project's layers (ordered highest first, each with path globs and annotations): without layers it returns the saved rule or a suggestion read from folder names and annotations, with files, symbols and calls between layers (direction down or up); pass layers to preview a rule and the same layers with confirm=true to save it in the project (.smart-tool/layers.json); duplicates lists duplicated functions (identical, near-identical and semantic bodies, min_similarity default 0.90; below that most pairs measured were false positives), excluding tests unless include_tests=true; fix real duplicates; only dismiss with duplicates_dismiss (finding_id, reason=false_positive|intentional, note) what is not a duplicate or is a copy kept on purpose: the finding stays hidden until the code changes, and duplicates_restore reopens it; integration shows the MCP client, hook and tool usage metrics; install_hook_preview shows the change that would install the hook redirecting Grep/Read to smart_search and WebSearch/WebFetch to web_search/web_fetch in this session's client and returns a plan_id; install_hook writes it with that plan_id and confirm=true, with a backup, only after the user approves the change; search_limits saves the project's default top_k per block ({code, test, doc}, each 1-20, total at most 20); storage shows disk usage and views; pin_view protects a view; each view carries stale (not updated for 30 days, the test of the daily cleanup that removes them) and stale_bytes sums them; cleanup_preview and cleanup_commit remove selected views after confirmation; compare compares two stored views, without checkout or embeddings. register adds a folder; preview checks the scope; index/rebuild update; pause/cancel keep checkpoints; resume continues; policy picks on_search/eager; scope adjusts the scope; remove deletes only the local index. setup_audit (project_root) reports what Claude Code and Codex actually load for that folder (settings layers, plugins, hooks, MCP servers, skills, CLAUDE.md/AGENTS.md) and where they collide: hooks deciding on the same tool in parallel, missing hook scripts, silenced hooks, MCP servers down or offering duplicate tools, skill name and budget problems, AGENTS.md that only Codex reads, broken @imports; what could not be checked is listed as not verified; each finding carries its fix (the file to edit or the command to run), applied by the agent when the user agrees, never by Smart Tool; mode=last returns the stored report instead of auditing again. redirect_proposals (no project_root) lists the redirect patterns proposed from allowed searches, most matched first, with a guide; redirect_decide decides them by ids: accept, instruct (tip without blocking), reject, or remove an accepted one. junk (no project_root) lists the registered projects that were never projects: a personal, system, application-data or build folder, a subfolder of a repository or of another project, a folder of projects, or a folder that no longer exists (on a drive or share that is mounted); with confirm=true it unregisters them and deletes only their index files, and the manifest says what went. A project is the folder where the session started, so smart_search and register refuse a folder that is not a project and index a new one only from the session that started in it. Jobs via smart_search_result.",
377
388
  "inputSchema": {
378
389
  "type": "object", "properties": {
379
- "action": {"type": "string", "enum": ["list", "status", "register", "preview", "index", "rebuild", "pause", "cancel", "resume", "watch", "scope", "remove", "relocate", "probe", "policy", "inspect", "graph", "usage", "storage", "pin_view", "cleanup_preview", "cleanup_commit", "compare", "search_limits", "integration", "install_hook_preview", "install_hook", "duplicates", "duplicates_dismiss", "duplicates_restore", "docs", "layers", "affected_tests", "pr_impact", "setup_audit", "redirect_proposals", "redirect_decide", "junk"]},
390
+ "action": {"type": "string", "enum": ["list", "status", "register", "preview", "index", "rebuild", "pause", "cancel", "resume", "watch", "scope", "remove", "relocate", "probe", "policy", "inspect", "graph", "usage", "storage", "pin_view", "cleanup_preview", "cleanup_commit", "compare", "search_limits", "integration", "install_hook_preview", "install_hook", "duplicates", "duplicates_dismiss", "duplicates_restore", "docs", "layers", "affected_tests", "pr_impact", "hotspots", "dead_code", "import_cycles", "comments", "setup_audit", "redirect_proposals", "redirect_decide", "junk", "deps"]},
380
391
  "ids": {"type": "array", "items": {"type": "string"}, "description": "redirect_decide: proposal ids from redirect_proposals (accepted ids for remove)"},
381
392
  "decision": {"type": "string", "enum": ["accept", "instruct", "reject", "remove"], "description": "redirect_decide: accept redirects matching calls to smart_search; instruct lets them run with a tip naming smart_search; reject discards; remove deletes an accepted pattern"},
382
393
  "finding_id": {"type": "string"},
@@ -424,7 +435,7 @@ TOOLS = [
424
435
  "name": "smart_search",
425
436
  "_meta": ALWAYS_LOAD,
426
437
  "description": (
427
- "Semantic search (embed+rerank) over the project's index: the first step to understand code in a "
438
+ "Search over the project's index (lexical, plus semantic with an embedding model and reranked with a rerank model, when configured): the first step to understand code in a "
428
439
  "project, finding functions, flows and reusable code by meaning, ranked, with name:line. Use it before "
429
440
  "a series of Grep/Read; use Grep only for an exact literal or inside a known file. "
430
441
  "Results come in three blocks, code, test and doc, plus the project profile with the natural "
@@ -454,6 +465,10 @@ TOOLS = [
454
465
  "test": {"type": "integer", "minimum": 1, "maximum": 20},
455
466
  "doc": {"type": "integer", "minimum": 1, "maximum": 20}}}],
456
467
  "description": "Results per block: one number for all blocks or {code, test, doc}; each 1-20, total at most 20. Defaults to the project's search_limits, else 5"},
468
+ "folder": {"anyOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}, "maxItems": 10}],
469
+ "description": "Only files under this folder of the project (a project-relative path, or a list of them); applied before ranking, so the results are the best inside the folder"},
470
+ "layer": {"type": "string",
471
+ "description": "Only files of this layer, by the path globs of the project's confirmed layers (project_manage action=layers); applied before ranking"},
457
472
  "detail": {"type": "string", "enum": ["compact", "full"], "default": "compact",
458
473
  "description": "compact: 240-character excerpts, and ranking (when rerank) and location_kind (when source_line) left out of each result; full: 500-character excerpts with every field"},
459
474
  "mode": {"type": "string", "enum": ["search", "dossier"], "default": "search",
@@ -462,6 +477,10 @@ TOOLS = [
462
477
  "description": "mode=dossier only: scan budget; past it the answer is marked degraded"},
463
478
  "max_hits": {"type": "integer", "minimum": 1, "maximum": 10, "default": 5,
464
479
  "description": "mode=dossier only: hits returned"},
480
+ "history": {"type": "boolean", "default": False,
481
+ "description": "mode=dossier only: also search the messages of the project's commits that have a body (hits with source git and id git:<hash>); the first read of a repository can take longer than timeout_ms"},
482
+ "decision_notes": {"type": "boolean", "default": False,
483
+ "description": "mode=search only: append the project's decision notes (docs/research, the backlog, CLAUDE.md, AGENTS.md and commit messages) under decision_notes"},
465
484
  },
466
485
  "required": ["project_root"],
467
486
  },
@@ -573,21 +592,22 @@ TOOLS = [
573
592
  {
574
593
  "name": "read_document",
575
594
  "description": (
576
- "Reads a PDF, Word (.docx), PowerPoint (.pptx) or Excel (.xlsx) file as compact text, a range at a time, "
577
- "inside Smart Tool so the file never enters your context whole. The answer opens with the facts: type, "
595
+ "Reads a PDF, Word (.docx), PowerPoint (.pptx), Excel (.xlsx), HTML (.html, .htm), CSV (.csv) or EPUB "
596
+ "(.epub) file as compact text, a range at a time, inside Smart Tool so the file never enters your context whole. The answer opens with the facts: type, "
578
597
  "size, page, slide, sheet or part count and the estimated tokens of all the text; then comes the text of "
579
598
  "the range you asked for (default: from the start, up to max_chars) with a `next:` line naming the range "
580
599
  "to continue. PDF pages come with the layout noise removed (ligatures, hyphenation, wrapped lines); docx "
581
600
  "as markdown (headings, lists, tables) in parts of about 6,000 characters because a docx has no fixed "
582
601
  "pages; pptx per slide with the speaker notes; xlsx per sheet, one tab-separated line per row with the "
583
- "cached formula values. Use info_only first to size a large file and pick the pages you need. A scanned "
602
+ "cached formula values; html as markdown in parts like a docx; csv as one sheet, one tab-separated line per row; "
603
+ "epub per chapter in reading order. Use info_only first to size a large file and pick the pages you need. A scanned "
584
604
  "PDF (no text layer) fails with that reason; there is no OCR. Not for plain text or code: use Read."
585
605
  ),
586
606
  "inputSchema": {
587
607
  "type": "object",
588
608
  "properties": {
589
- "path": {"type": "string", "description": "Absolute path of the .pdf, .docx, .pptx or .xlsx file"},
590
- "range": {"type": "string", "description": "Pages, slides, docx parts or sheets to read: `3`, `3-5`, `7-` (to the end), `-4` (the first four) or a comma list like `1,4-6`; a sheet name also works for a workbook. Default: from the first one"},
609
+ "path": {"type": "string", "description": "Absolute path of the .pdf, .docx, .pptx, .xlsx, .html, .htm, .csv or .epub file"},
610
+ "range": {"type": "string", "description": "Pages, slides, parts, chapters or sheets to read: `3`, `3-5`, `7-` (to the end), `-4` (the first four) or a comma list like `1,4-6`; a sheet name also works for a workbook. Default: from the first one"},
591
611
  "max_chars": {"type": "integer", "minimum": 1000, "maximum": document_read.MAX_CHARS_LIMIT, "default": document_read.DEFAULT_MAX_CHARS, "description": "Cap of the text returned (about four characters per token)"},
592
612
  "info_only": {"type": "boolean", "default": False, "description": "Return only the facts (type, size, counts, estimated tokens), no text"},
593
613
  },
@@ -804,8 +824,9 @@ def _embed(model, texts, token, deadline=None, on_retry=None, cancel_check=None)
804
824
 
805
825
 
806
826
  def _rerank(model, query, documents, top_n, token, deadline=None):
807
- """Rows `{index, relevance_score}` of the best `top_n` documents. Only an HTTP answer that is worth repeating
808
- (429, 5xx) is asked again, after the wait of `_wait_to_retry`; a timeout fails at once so a search does not stall."""
827
+ """Rows `{index, relevance_score}` of the best `top_n` documents from the gateway. Only an HTTP answer that is worth
828
+ repeating (429, 5xx) is asked again, after the wait of `_wait_to_retry`; a timeout fails at once so a search does
829
+ not stall."""
809
830
  if not documents:
810
831
  return []
811
832
  top_n = min(top_n, len(documents))
@@ -822,6 +843,20 @@ def _rerank(model, query, documents, top_n, token, deadline=None):
822
843
  if attempt >= EMBED_MAX_ATTEMPTS or not _transient_embed_error(exc):
823
844
  raise
824
845
  _wait_to_retry(exc, attempt, deadline)
846
+ return _rerank_rows(response, documents, top_n)
847
+
848
+
849
+ def _home_rerank(query, documents, top_n, deadline=None):
850
+ """`_rerank` through the embeddings home, which forwards to the rerank host configured in it."""
851
+ if not documents:
852
+ return []
853
+ top_n = min(top_n, len(documents))
854
+ return _rerank_rows(local_embedder.rerank(query, documents, top_n, deadline=deadline), documents, top_n)
855
+
856
+
857
+ def _rerank_rows(response, documents, top_n):
858
+ """The best `top_n` rows of a rerank answer in the Cohere form, highest score first. Scores are opaque: only their
859
+ order within one answer counts. A repeated or out-of-range index, or a missing score, raises."""
825
860
  rows = response.get("results", []) if isinstance(response, dict) else []
826
861
  if not isinstance(rows, list) or not rows:
827
862
  raise ValueError("The reranker returned no valid results.")
@@ -833,7 +868,46 @@ def _rerank(model, query, documents, top_n, token, deadline=None):
833
868
  not isinstance(score, (int, float)) or not math.isfinite(score)):
834
869
  raise ValueError("The reranker returned an invalid index or score.")
835
870
  seen.add(idx)
836
- return rows[:top_n]
871
+ return sorted(rows, key=lambda row: row["relevance_score"], reverse=True)[:top_n]
872
+
873
+
874
+ def _reranker(cfg, token):
875
+ """The reranker as `rerank(query, documents, top_n, deadline)`, or None when there is none: the gateway's
876
+ rerank_model when the user chose the gateway, else the embeddings home when a rerank host is configured in it."""
877
+ if config.embedder(cfg) == "gateway":
878
+ model = cfg.get("rerank_model")
879
+ return (lambda query, documents, top_n, deadline=None:
880
+ _rerank(model, query, documents, top_n, token, deadline=deadline)) if model else None
881
+ return _home_rerank if local_embedder.rerank_configured() else None
882
+
883
+
884
+ def _web_embedder(cfg):
885
+ """Embedding model of the web memory, the same choice as the project indexes (config `embedder`); None when there
886
+ is none: no gateway embedding_model, or the embeddings home not running or unusable."""
887
+ try:
888
+ model = local_embedder.resolve(cfg)
889
+ except local_embedder.IdentityError as exc:
890
+ _log(f"web memory without vectors: {exc}")
891
+ return None
892
+ return None if model in (None, indexer.LEXICAL_MODEL) else model
893
+
894
+
895
+ def _web_memory_models(cfg, token):
896
+ """`(embedding_model, rerank, problem)` of the web memory (sources that helped before, reuse of similar searches).
897
+ It follows config `embedder` like the project indexes: a stopped home is never replaced by the gateway. `problem`
898
+ says what is missing, empty when both are there."""
899
+ try:
900
+ model = local_embedder.resolve(cfg)
901
+ rerank = _reranker(cfg, token)
902
+ except local_embedder.IdentityError as exc:
903
+ return None, None, str(exc)
904
+ model = None if model in (None, indexer.LEXICAL_MODEL) else model
905
+ if config.embedder(cfg) == "gateway":
906
+ missing = [name for name, value in (("embedding_model", model), ("rerank_model", rerank)) if not value]
907
+ return model, rerank, " and ".join(missing) + " missing" if missing else ""
908
+ if not model:
909
+ return None, rerank, "the embeddings home is not running"
910
+ return model, rerank, "" if rerank else "no rerank host configured in the embeddings home"
837
911
 
838
912
 
839
913
  LOG_PATH = os.path.join(RUN_DIR, "daemon.log")
@@ -1018,6 +1092,52 @@ def _cold_overlay(root, queries):
1018
1092
  return {"drop": [], "chunks": chunks}, warning
1019
1093
 
1020
1094
 
1095
+ def _duration(seconds):
1096
+ return f"{round(seconds)} s" if seconds < 90 else f"{-(-round(seconds) // 60)} min"
1097
+
1098
+
1099
+ def _indexing_notice(root, model, indexed, total):
1100
+ """Where the running index build of `root` stands and how long it should take: the speed so far once files were
1101
+ indexed, else, on a CPU-only embedder, the slowest measured speed as an upper bound."""
1102
+ with _JOBS_LOCK:
1103
+ job = dict(_jobs.get(_INDEX_RUNNING.get(project_identity.canonical_root(root))) or {})
1104
+ done = max(indexed, job.get("processed_files") or 0)
1105
+ total = max(job.get("total_files") or 0, total, done)
1106
+ since = next((at for phase, at in reversed(job.get("phases", [])) if phase == "indexing"), None)
1107
+ left = total - done
1108
+ if done and since and time.time() > since:
1109
+ eta = f"about {_duration(left * (time.time() - since) / done)} left at the speed so far"
1110
+ elif local_embedder.runs_on_cpu(model):
1111
+ eta = f"expected time on this CPU-only embedder: up to about {_duration(left * 60 / CPU_INDEX_FILES_PER_MINUTE)}"
1112
+ else:
1113
+ eta = "time left not known yet"
1114
+ return (f"Indexing in progress: {done} of {total} files indexed ({100 * done // max(total, 1)}%), {eta}. Results "
1115
+ "rank the indexed files by meaning and keywords and read the others from disk by keyword; repeat the "
1116
+ "search later for the full ranking.")
1117
+
1118
+
1119
+ def _partial_search(root, model, texts, token, job_id):
1120
+ """`(vectors, path, overlay, notice)` for a search made while the index of `root` is being built: the query is
1121
+ embedded and searched in the part of the new index already committed, and the files not committed yet are read
1122
+ from disk by keyword."""
1123
+ path, indexed = indexer.partial_index(root, model)
1124
+ scope = index_scope.load_scope(root)
1125
+ files = index_scope.resolve_included_files(root, scope if index_scope.valid_scope(scope) else {"include": []})
1126
+ notice = _indexing_notice(root, model, len(indexed), len(files))
1127
+ vectors = {}
1128
+ if path:
1129
+ try:
1130
+ vectors = dict(zip(texts, _embed(model, texts, token, deadline=_job_deadline(job_id),
1131
+ cancel_check=lambda: _job_checkpoint(job_id))))
1132
+ except ProjectJobCancelled:
1133
+ raise
1134
+ except Exception as exc:
1135
+ notice += " Embedding unavailable, so the indexed files were searched by keyword only. " + _project_error(exc)
1136
+ chunks, _scanned = indexer.keyword_chunks(root, [rel for rel in files if rel not in indexed], texts,
1137
+ COLD_SCAN_BUDGET_S)
1138
+ return vectors, path, {"drop": [], "chunks": chunks}, notice
1139
+
1140
+
1021
1141
  def _lag_overlay(root, lag):
1022
1142
  """`(overlay, warning)` for `indexer.search`: the stale files read from disk, searched by keyword in place of their
1023
1143
  outdated chunks, and the notice telling the agent that indexing is in progress. `(None, "")` when the index is
@@ -1137,31 +1257,43 @@ def _prepare_project_index(arguments, job_id, cfg, token, query_vector=None, pre
1137
1257
  "sample_paths": included[:30]}
1138
1258
  project_store.update(project_identity.project_id(root), preview=result)
1139
1259
  return result
1140
- model = model or local_embedder.resolve(cfg.get("embedding_model"), indexer.info(root).get("model_id"))
1260
+ model = model or local_embedder.resolve(cfg, indexer.info(root).get("model_id"))
1261
+ if model is None:
1262
+ raise RuntimeError(f"The embeddings home is not running: this folder's index holds "
1263
+ f"{indexer.info(root).get('model_id')} and keeps answering; re-embedding starts when the "
1264
+ "home answers.")
1141
1265
  lexical = model == indexer.LEXICAL_MODEL
1142
1266
  current = indexer.info(root)
1143
1267
  known_dimensions = current.get("dimensions") if current.get("model_id") == model else None
1268
+ gentle = not lexical and local_embedder.runs_on_cpu(model)
1144
1269
  if not lexical and query_vector is None and not known_dimensions:
1145
1270
  _update_job(job_id, phase="model_probe")
1146
- query_vector = _embed(model, ["Smart Tool: indexing dimension check."], token,
1147
- deadline=_job_deadline(job_id), cancel_check=checkpoint,
1148
- on_retry=lambda attempt, items: _update_job(job_id, phase="indexing_retry",
1149
- retry_attempt=attempt, batch_items=items))[0]
1271
+ with low_priority.background() if gentle else contextlib.nullcontext():
1272
+ query_vector = _embed(model, ["Smart Tool: indexing dimension check."], token,
1273
+ deadline=_job_deadline(job_id), cancel_check=checkpoint,
1274
+ on_retry=lambda attempt, items: _update_job(job_id, phase="indexing_retry",
1275
+ retry_attempt=attempt, batch_items=items))[0]
1150
1276
  checkpoint()
1151
1277
  _update_job(job_id, phase="indexing")
1152
1278
  full_check = bool(arguments.get("full_check") or project.get("needs_full_check") or
1153
1279
  time.time() - current.get("full_scan_at", 0) >= project_monitor.FULL_CHECK_S)
1154
- stats = indexer.reindex(
1155
- root, included,
1156
- None if lexical else lambda texts: _embed(model, texts, token, deadline=_job_deadline(job_id),
1157
- cancel_check=checkpoint, on_retry=lambda attempt, items: _update_job(
1158
- job_id, phase="indexing_retry", retry_attempt=attempt, batch_items=items)),
1159
- on_progress=lambda processed, total: _update_job(
1160
- job_id, phase="indexing", processed_files=processed, total_files=total),
1161
- model_id=model, expected_dimensions=None if lexical else known_dimensions or len(query_vector),
1162
- force_rebuild=bool(arguments.get("force_rebuild")), full_check=full_check,
1163
- changed_paths=set(arguments.get("changed_paths") or []) | set(project.get('dirty_paths', [])), cancel_check=checkpoint,
1164
- )
1280
+
1281
+ def embed_batch(texts):
1282
+ with low_priority.background() if gentle else contextlib.nullcontext():
1283
+ return _embed(model, texts, token, deadline=_job_deadline(job_id), cancel_check=checkpoint,
1284
+ on_retry=lambda attempt, items: _update_job(
1285
+ job_id, phase="indexing_retry", retry_attempt=attempt, batch_items=items))
1286
+
1287
+ with low_priority.background() if gentle else contextlib.nullcontext():
1288
+ stats = indexer.reindex(
1289
+ root, included, None if lexical else embed_batch,
1290
+ on_progress=lambda processed, total: _update_job(
1291
+ job_id, phase="indexing", processed_files=processed, total_files=total),
1292
+ model_id=model, expected_dimensions=None if lexical else known_dimensions or len(query_vector),
1293
+ force_rebuild=bool(arguments.get("force_rebuild")), full_check=full_check,
1294
+ changed_paths=set(arguments.get("changed_paths") or []) | set(project.get('dirty_paths', [])),
1295
+ cancel_check=checkpoint,
1296
+ )
1165
1297
  _update_job(job_id, stats=stats, index_updated=stats.get("ready", True))
1166
1298
  project_store.update(project_identity.project_id(root), index=indexer.info(root),
1167
1299
  last_stats=stats, last_checked=time.time(), scope_summary={
@@ -1188,15 +1320,20 @@ def _grouped_yaml(profile, blocks, warning):
1188
1320
  return "\n".join(lines) + "\n"
1189
1321
 
1190
1322
 
1191
- LOCAL_EMBEDDER_DOWN_WARNING = ("The mcp-memory local embedder is configured but not responding: index and search are lexical only "
1192
- "until it is back (check the mcp-memory embedding service).")
1323
+ HOME_DOWN_WARNING = ("The embeddings home is not running: index and search are lexical only until it answers (start or "
1324
+ "reinstall casa-de-embeddings; it announces itself in ~/.casa-de-embeddings/run/daemon.json).")
1325
+ RERANK_MISSING_WARNING = {
1326
+ "gateway": "Rerank not configured: hybrid lexical/vector order, less precise at the top. Configure a rerank_model.",
1327
+ "home": ("Rerank not configured: hybrid lexical/vector order, less precise at the top. Configure a rerank host in the "
1328
+ "embeddings home."),
1329
+ }
1193
1330
  EMPTY_RESULT_NOTICE = ("No matches. Do not fall back to Grep, Read of folders or shell search: retry smart_search with other "
1194
1331
  "identifier terms, another group, or a wider top_k, and if it stays empty report that the code was not found.")
1195
1332
  LEXICAL_WARNING = ("No embedding model: index and search are lexical only (exact words, no synonyms or cross-language matches). "
1196
1333
  "Configure an embedding_model for semantic search.")
1197
1334
 
1198
1335
 
1199
- def _rerank_blocks(model, groups, queries, found, vectors, token, deadline=None):
1336
+ def _rerank_blocks(rerank, groups, queries, found, vectors, deadline=None):
1200
1337
  """`({group: rows}, {group: error})` of the blocks that have candidates and a query vector. One rerank request goes
1201
1338
  out per distinct query, not per block: the candidates of the blocks that share a query travel together and come back
1202
1339
  split by block (`index` is the position inside the block's candidates). A reranker scores each document against the
@@ -1210,7 +1347,7 @@ def _rerank_blocks(model, groups, queries, found, vectors, token, deadline=None)
1210
1347
  owners = [(group, position) for group in members for position in range(len(found[group]))]
1211
1348
  documents = [found[group][position][4] for group, position in owners]
1212
1349
  try:
1213
- rows = _rerank(model, query, documents, len(documents), token, deadline=deadline)
1350
+ rows = rerank(query, documents, len(documents), deadline=deadline)
1214
1351
  except Exception as exc:
1215
1352
  failed.update({group: _project_error(exc) for group in members})
1216
1353
  continue
@@ -1245,6 +1382,42 @@ def _nested_projects_note(root):
1245
1382
  f"results; search them with smart_search on their folder: {'; '.join(parts)}{more}.")
1246
1383
 
1247
1384
 
1385
+ def _switch_search(root, old, target, texts, token, problem, cfg, job_id):
1386
+ """`(vectors, search_model, overlay, notice)` for a search of a folder whose index holds `old` while the embedder
1387
+ is now `target` (None while the embeddings home is not running). The old index answers, with the query embedded by
1388
+ its own model when that model can still be asked (a gateway model and the gateway answers), else by keyword; the
1389
+ files changed since are read from disk by keyword. The re-embedding into `target` runs in the background, queued
1390
+ here when it is not; the folder switches to the new index only when it is complete."""
1391
+ vectors, search_model, failure = {}, None, ""
1392
+ if old != indexer.LEXICAL_MODEL and not local_embedder.is_local(old) and not problem:
1393
+ try:
1394
+ vectors = dict(zip(texts, _embed(old, texts, token, deadline=_job_deadline(job_id),
1395
+ cancel_check=lambda: _job_checkpoint(job_id))))
1396
+ search_model = old
1397
+ except ProjectJobCancelled:
1398
+ raise
1399
+ except Exception as exc:
1400
+ failure = " Embedding the query with that model failed. " + _project_error(exc)
1401
+ lag = _index_lag(root, old, cfg, problem, job_id)
1402
+ overlay, stale = _lag_overlay(root, lag) if lag else (None, "")
1403
+ how = "ranked by meaning and keywords" if search_model else "by keyword only"
1404
+ if target is None:
1405
+ notice = (f"The embeddings home is not running: results come from this folder's index built with {old}, {how}. "
1406
+ "Re-embedding with the home starts when it answers.")
1407
+ else:
1408
+ key = project_identity.canonical_root(root)
1409
+ with _JOBS_LOCK:
1410
+ job = dict(_jobs.get(_INDEX_RUNNING.get(key)) or {})
1411
+ queued = "" if key in _INDEX_RUNNING or _index_update_running(root) else \
1412
+ _refresh_in_background(root, {"refresh": True})
1413
+ progress = (f" ({job['processed_files']} of {job['total_files']} files so far)"
1414
+ if job.get("total_files") else "")
1415
+ notice = (f"Re-embedding in progress: this folder's index was built with {old} and is being rebuilt with "
1416
+ f"{target} in the background{progress}. Results come from the old index, {how}; the folder switches "
1417
+ "to the new index when it is complete." + (" " + queued if queued else ""))
1418
+ return vectors, search_model, overlay, " ".join(text for text in (notice + failure, stale) if text)
1419
+
1420
+
1248
1421
  def _perform_smart_search_isolated(arguments, job_id):
1249
1422
  _update_job(job_id, phase="auth")
1250
1423
  problem = _gateway_problem()
@@ -1255,18 +1428,28 @@ def _perform_smart_search_isolated(arguments, job_id):
1255
1428
  queries = {"code": arguments["query"], "test": arguments["query"], "doc": arguments["doc_query"]}
1256
1429
  texts = list(dict.fromkeys(queries[group] for group in groups))
1257
1430
  cfg = config.load_config()
1258
- model = local_embedder.resolve(cfg.get("embedding_model"), indexer.info(root).get("model_id"))
1431
+ current = indexer.info(root)
1432
+ model = local_embedder.resolve(cfg, current.get("model_id"))
1433
+ unreachable = problem and model not in (None, indexer.LEXICAL_MODEL) and not local_embedder.is_local(model)
1434
+ old = (current.get("model_id") if current.get("chunks") and current.get("model_id") != model and not unreachable
1435
+ else None)
1436
+ model = indexer.LEXICAL_MODEL if model is None and old is None else model
1259
1437
  warnings = []
1260
1438
  vectors = {}
1261
1439
  overlay = None
1262
1440
  search_model = model
1441
+ search_path = None
1263
1442
  source = "index"
1264
1443
  if problem:
1265
1444
  model_client.OFFLINE.set(problem)
1266
1445
  token = "" if problem else model_client.get_token()
1267
- if model == indexer.LEXICAL_MODEL:
1268
- warnings.append(LOCAL_EMBEDDER_DOWN_WARNING if local_embedder.configured() and config.local_models_mode(cfg) != "off"
1269
- else LEXICAL_WARNING)
1446
+ rerank = _reranker(cfg, token)
1447
+ if old is not None:
1448
+ vectors, search_model, overlay, notice = _switch_search(root, old, model, texts, token, problem, cfg, job_id)
1449
+ source = "index+disk" if overlay else source
1450
+ warnings.append(notice)
1451
+ elif model == indexer.LEXICAL_MODEL:
1452
+ warnings.append(HOME_DOWN_WARNING if config.embedder(cfg) == "home" else LEXICAL_WARNING)
1270
1453
  try:
1271
1454
  if not arguments.get("_existing_index_only"):
1272
1455
  _prepare_project_index(arguments, job_id, cfg, token, model=model)
@@ -1277,6 +1460,11 @@ def _perform_smart_search_isolated(arguments, job_id):
1277
1460
  raise
1278
1461
  warnings.append("Lexical index not updated; results from the last ready index. " + _project_error(exc))
1279
1462
  search_model = None
1463
+ elif arguments.get("_during_index") and _index_lag(root, model, cfg, problem, job_id) is None:
1464
+ vectors, search_path, overlay, notice = _partial_search(root, model, texts, token, job_id)
1465
+ search_model = model if vectors else None
1466
+ source = "partial"
1467
+ warnings.append(notice)
1280
1468
  elif _cold_index(root, cfg, problem):
1281
1469
  overlay, notice = _cold_overlay(root, texts)
1282
1470
  search_model = None
@@ -1306,7 +1494,8 @@ def _perform_smart_search_isolated(arguments, job_id):
1306
1494
  if lag is not None:
1307
1495
  overlay, notice = _lag_overlay(root, lag)
1308
1496
  source = "index+disk" if overlay else source
1309
- warnings += [text for text in (notice, _refresh_in_background(root, lag)) if text]
1497
+ warnings += [text for text in (notice, None if arguments.get("_during_index")
1498
+ else _refresh_in_background(root, lag)) if text]
1310
1499
  elif not arguments.get("_existing_index_only"):
1311
1500
  try:
1312
1501
  _prepare_project_index(arguments, job_id, cfg, token, query_vector=next(iter(vectors.values())),
@@ -1320,8 +1509,8 @@ def _perform_smart_search_isolated(arguments, job_id):
1320
1509
  search_model = None
1321
1510
  if indexer.info(root).get("model_id") != model:
1322
1511
  vectors = {}
1323
- if vectors and not cfg.get("rerank_model"):
1324
- warnings.append("Rerank not configured: hybrid lexical/vector order, less precise at the top. Configure a rerank_model.")
1512
+ if vectors and not rerank:
1513
+ warnings.append(RERANK_MISSING_WARNING[config.embedder(cfg)])
1325
1514
  _job_checkpoint(job_id)
1326
1515
  saved_scope = index_scope.load_scope(root) or {}
1327
1516
  profile = index_profile.current(saved_scope.get("profile"))
@@ -1348,15 +1537,18 @@ def _perform_smart_search_isolated(arguments, job_id):
1348
1537
  query_terms = _symbol_terms(" ".join(queries[group] for group in groups if group != "doc"))
1349
1538
  detail = arguments.get("detail", "compact")
1350
1539
  _update_job(job_id, phase="searching")
1540
+ in_scope = _scope_filter(root, arguments)
1351
1541
  found = {group: indexer.search(root, vectors.get(queries[group]), top_k=max(limits[group] * 4, candidate_pool),
1352
1542
  query=queries[group], model_id=search_model, overlay=overlay,
1353
- include=lambda path, group=group: index_profile.kind(path, profile) == group)
1543
+ path=search_path,
1544
+ include=lambda path, group=group: index_profile.kind(path, profile) == group
1545
+ and (in_scope is None or in_scope(path)))
1354
1546
  for group in groups}
1355
1547
 
1356
1548
  reranked, rerank_failed = {}, {}
1357
- if cfg.get("rerank_model"):
1549
+ if rerank:
1358
1550
  _update_job(job_id, phase="reranking")
1359
- reranked, rerank_failed = _rerank_blocks(cfg["rerank_model"], groups, queries, found, vectors, token,
1551
+ reranked, rerank_failed = _rerank_blocks(rerank, groups, queries, found, vectors,
1360
1552
  deadline=_job_deadline(job_id))
1361
1553
 
1362
1554
  def search_block(group):
@@ -1389,7 +1581,8 @@ def _perform_smart_search_isolated(arguments, job_id):
1389
1581
  warnings.append(problem)
1390
1582
  _job_checkpoint(job_id)
1391
1583
  if not any(blocks.values()):
1392
- notes.append(EMPTY_RESULT_NOTICE)
1584
+ notes.append(EMPTY_RESULT_NOTICE + (" The folder and layer filters of this search may be too narrow."
1585
+ if in_scope else ""))
1393
1586
  nested = _nested_projects_note(root)
1394
1587
  if nested:
1395
1588
  notes.append(nested)
@@ -1411,8 +1604,8 @@ def _perform_project_index_isolated(arguments, job_id, preview=False):
1411
1604
  problem = _gateway_problem()
1412
1605
  cfg = config.load_config()
1413
1606
  if problem:
1414
- model = local_embedder.resolve(cfg.get("embedding_model"), indexer.info(arguments["project_root"]).get("model_id"))
1415
- if model != indexer.LEXICAL_MODEL and not local_embedder.is_local(model):
1607
+ model = local_embedder.resolve(cfg, indexer.info(arguments["project_root"]).get("model_id"))
1608
+ if model not in (None, indexer.LEXICAL_MODEL) and not local_embedder.is_local(model):
1416
1609
  raise RuntimeError(problem)
1417
1610
  model_client.OFFLINE.set(problem)
1418
1611
  return _prepare_project_index(arguments, job_id, cfg, "", preview=preview, model=model)
@@ -1446,9 +1639,19 @@ def _run_project_job_in_view(job_id, arguments, kind):
1446
1639
  if arguments.get('_requested_view') and arguments['_requested_view'] != index_views.public(view):
1447
1640
  raise index_views.ViewChanged('The branch or commit changed while the job was waiting. Start again on the current view.')
1448
1641
  _update_job(job_id, view=index_views.public(view))
1449
- with _project_lock(arguments["project_root"]):
1450
- result = (_perform_smart_search(arguments, job_id) if kind == "smart_search"
1451
- else _perform_project_index(arguments, job_id, preview=kind == "project_preview"))
1642
+ key = project_identity.canonical_root(arguments["project_root"])
1643
+ if kind == "smart_search" and key in _INDEX_RUNNING:
1644
+ result = _perform_smart_search({**arguments, "_existing_index_only": True, "_during_index": True}, job_id)
1645
+ else:
1646
+ with _project_lock(arguments["project_root"]):
1647
+ if kind == "project_index":
1648
+ _INDEX_RUNNING[key] = job_id
1649
+ try:
1650
+ result = (_perform_smart_search(arguments, job_id) if kind == "smart_search"
1651
+ else _perform_project_index(arguments, job_id, preview=kind == "project_preview"))
1652
+ finally:
1653
+ if kind == "project_index":
1654
+ _INDEX_RUNNING.pop(key, None)
1452
1655
  _set_job(job_id, status="done", phase="done", result=result, error=None)
1453
1656
  current = project_store.get(project_id) or {}
1454
1657
  with _JOBS_LOCK:
@@ -1531,7 +1734,8 @@ def _submit_project_job_locked(kind, arguments):
1531
1734
  enabled=kind != "project_preview" or project.get("enabled", False))
1532
1735
  try:
1533
1736
  future = SMART_SEARCH_EXECUTOR.submit(root, _run_project_job, job_id, arguments, kind,
1534
- priority=INTERACTIVE_PRIORITY if interactive else 0)
1737
+ priority=INTERACTIVE_PRIORITY if interactive else 0,
1738
+ lane="search" if interactive else "index")
1535
1739
  with _JOBS_LOCK:
1536
1740
  if not future.done():
1537
1741
  _JOB_FUTURES[job_id] = future
@@ -1646,8 +1850,9 @@ def _optional_text(arguments, name):
1646
1850
 
1647
1851
 
1648
1852
  _SMART_SEARCH_FIELDS = {"project_root", "query_identifiers", "query_comments", "doc_query",
1649
- "single_language_reason", "group", "top_k", "detail"}
1650
- _DOSSIER_FIELDS = {"project_root", "query_identifiers", "query_comments", "doc_query", "mode", "timeout_ms", "max_hits"}
1853
+ "single_language_reason", "group", "top_k", "detail", "folder", "layer"}
1854
+ _DOSSIER_FIELDS = {"project_root", "query_identifiers", "query_comments", "doc_query", "mode", "timeout_ms", "max_hits",
1855
+ "history"}
1651
1856
 
1652
1857
 
1653
1858
  def _reject_kind(message):
@@ -1671,6 +1876,11 @@ def _handle_smart_search(arguments):
1671
1876
  if mode != "search":
1672
1877
  raise ValueError("mode must be search or dossier.")
1673
1878
  arguments = {key: value for key, value in arguments.items() if key != "mode"}
1879
+ want_notes = arguments.pop("decision_notes", False)
1880
+ if not isinstance(want_notes, bool):
1881
+ raise ValueError("decision_notes must be true or false.")
1882
+ notes_text = " ".join(arguments[key] for key in ("query_identifiers", "query_comments", "doc_query")
1883
+ if isinstance(arguments.get(key), str))
1674
1884
  try:
1675
1885
  arguments = _smart_search_arguments(arguments)
1676
1886
  except ValueError as exc:
@@ -1688,7 +1898,17 @@ def _handle_smart_search(arguments):
1688
1898
  return _agent_search_answer(_smart_search_job_payload(job_id))
1689
1899
  result = future.result()
1690
1900
  _mark_notice_delivered(job_id)
1691
- return result
1901
+ return _with_decision_notes(result, arguments["project_root"], notes_text) if want_notes else result
1902
+
1903
+
1904
+ def _with_decision_notes(result, root, text):
1905
+ """The finished search answer with a `decision_notes:` line (JSON, which is valid YAML) after it: the hits of the
1906
+ project's decision notes for `text`, commit messages included, and why any of them is missing."""
1907
+ if not isinstance(result, str) or not dossier.focus_terms(text):
1908
+ return result
1909
+ found = dossier.query(root, text, max_hits=scout.DECISION_NOTES_MAX, history=True)
1910
+ notes = {"decision_notes": found["hits"], **({"decision_notes_warnings": found["reasons"]} if found["reasons"] else {})}
1911
+ return result.rstrip("\n") + "\n" + json.dumps(notes, ensure_ascii=False, separators=(",", ":")) + "\n"
1692
1912
 
1693
1913
 
1694
1914
  def _mark_notice_delivered(job_id):
@@ -1814,6 +2034,52 @@ def _update_notices():
1814
2034
  return []
1815
2035
 
1816
2036
 
2037
+ def _search_scope(project_root, folder, layer):
2038
+ """The folder and layer filters of a smart_search, validated.
2039
+
2040
+ Parameters: project_root (str); folder (str, list of str or None) project-relative folders; layer (str or None) a
2041
+ layer of the project's confirmed layers. Returns: tuple (list of folders with forward slashes and no outer
2042
+ slashes, layer name or None). Raises: ValueError for a folder that is absolute or climbs out of the project, a layer
2043
+ the project does not declare and a project without confirmed layers.
2044
+ """
2045
+ folders = [folder] if isinstance(folder, str) else folder
2046
+ if folders is not None and (not isinstance(folders, list) or not folders
2047
+ or not all(isinstance(item, str) and item.strip() for item in folders)):
2048
+ raise ValueError("folder must be a project-relative path or a list of them.")
2049
+ clean = []
2050
+ for item in folders or []:
2051
+ parts = [part for part in item.strip().replace("\\", "/").split("/") if part not in ("", ".")]
2052
+ if item.strip().startswith(("/", "\\")) or re.match(r"[A-Za-z]:", item.strip()) or ".." in parts:
2053
+ raise ValueError(f"folder {item!r} must be relative to the project root and stay inside it.")
2054
+ clean.append("/".join(parts))
2055
+ if layer is None:
2056
+ return clean, None
2057
+ if not isinstance(layer, str) or not layer.strip():
2058
+ raise ValueError("layer must be the name of a layer of the project.")
2059
+ import layers
2060
+ rule = layers.load(project_root)
2061
+ if rule is None:
2062
+ raise ValueError("layer needs confirmed layers: this project has none (project_manage action=layers).")
2063
+ names = [item["name"] for item in rule.layers]
2064
+ if layer not in names:
2065
+ raise ValueError(f"layer {layer!r} is not declared by this project; its layers are {', '.join(names)}.")
2066
+ return clean, layer
2067
+
2068
+
2069
+ def _scope_filter(root, arguments):
2070
+ """Predicate over project-relative paths for the folder and layer filters of a search, or None without filters."""
2071
+ folders, layer = arguments.get("folders") or [], arguments.get("layer")
2072
+ if not folders and not layer:
2073
+ return None
2074
+ import layers
2075
+ rule = layers.load(root) if layer else None
2076
+
2077
+ def inside(path):
2078
+ return (not folders or any(not folder or path == folder or path.startswith(folder + "/") for folder in folders)) \
2079
+ and (rule is None or rule.assign(path) == layer)
2080
+ return inside
2081
+
2082
+
1817
2083
  def _smart_search_arguments(arguments):
1818
2084
  if not isinstance(arguments, dict):
1819
2085
  raise ValueError("smart_search requires its arguments as a JSON object")
@@ -1853,9 +2119,11 @@ def _smart_search_arguments(arguments):
1853
2119
  detail = arguments.get("detail", "compact")
1854
2120
  if detail not in SEARCH_DETAILS:
1855
2121
  raise ValueError("detail must be compact or full.")
2122
+ folders, layer = _search_scope(project_root, arguments.get("folder"), arguments.get("layer"))
1856
2123
  query = " ".join(part for part in (identifiers, comments) if part) or doc_query
1857
2124
  arguments = {"project_root": project_root, "query": query, "top_k": limits, "group": group, "detail": detail,
1858
2125
  "doc_query": doc_query or comments or identifiers,
2126
+ **({"folders": folders} if folders else {}), **({"layer": layer} if layer else {}),
1859
2127
  **({"single_language_reason": reason} if reason else {})}
1860
2128
  session, client = _session_client()
1861
2129
  if session is not None and "hook_notice" not in session:
@@ -2133,8 +2401,7 @@ def _find_duplicates(root, arguments, dismissed):
2133
2401
  if type(include_tests) is not bool:
2134
2402
  raise ValueError("include_tests must be true or false.")
2135
2403
  limit = _parse_bounded_int(arguments.get("limit"), default=30, min_v=1, max_v=200, field_name="limit")
2136
- configured_model = local_embedder.resolve(config.load_config().get("embedding_model"),
2137
- indexer.info(root).get("model_id"))
2404
+ configured_model = local_embedder.resolve(config.load_config(), indexer.info(root).get("model_id"))
2138
2405
  if configured_model == indexer.LEXICAL_MODEL:
2139
2406
  configured_model = None
2140
2407
  view = index_views.describe(root)
@@ -2211,6 +2478,14 @@ def _project_action(arguments):
2211
2478
  return _redirect_action(arguments)
2212
2479
  if arguments.get("action") == "junk":
2213
2480
  return _junk_action(arguments)
2481
+ if arguments.get("action") == "deps":
2482
+ root = arguments.get("project_root")
2483
+ if not isinstance(root, str) or not os.path.isdir(root):
2484
+ raise ValueError("deps needs project_root: the folder whose dependency manifests to check against OSV.")
2485
+ try:
2486
+ return dependency_notice.audit(root)
2487
+ except dependency_notice.DependencyCheckError as exc:
2488
+ raise ValueError(f"dependency check could not run: {exc}") from None
2214
2489
  if arguments.get("action") == "setup_audit":
2215
2490
  root = arguments.get("project_root")
2216
2491
  if not isinstance(root, str) or not os.path.isdir(root):
@@ -2232,7 +2507,7 @@ def _project_action(arguments):
2232
2507
  action = arguments.get("action", "list")
2233
2508
  if action == "probe":
2234
2509
  return _probe_index_models()
2235
- if action in ('graph','compare','duplicates','duplicates_dismiss','duplicates_restore','docs','layers','affected_tests','pr_impact'):
2510
+ if action in ('graph','compare','duplicates','duplicates_dismiss','duplicates_restore','docs','layers','affected_tests','pr_impact','hotspots','dead_code','import_cycles','comments'):
2236
2511
  # Análise somente leitura pode demorar; não prende o lock global de ações.
2237
2512
  key=arguments.get('project_id')
2238
2513
  if not isinstance(key,str) or not re.fullmatch(r'[a-f0-9]{16}',key):
@@ -2273,6 +2548,10 @@ def _project_action(arguments):
2273
2548
  if action=='layers':
2274
2549
  import layers
2275
2550
  return layers.report(project['root'],arguments.get('layers'),arguments.get('confirm',False),arguments.get('view_id'))
2551
+ if action in ('hotspots','dead_code','import_cycles','comments'):
2552
+ import code_health
2553
+ limit=_parse_bounded_int(arguments.get('limit'),default=25,min_v=1,max_v=200,field_name='limit')
2554
+ return getattr(code_health,action)(project['root'],arguments.get('view_id'),limit)
2276
2555
  if action=='pr_impact':
2277
2556
  import pr_impact
2278
2557
  limit=_parse_bounded_int(arguments.get('limit'),default=30,min_v=1,max_v=200,field_name='limit')
@@ -2312,22 +2591,27 @@ def _project_action(arguments):
2312
2591
  return result
2313
2592
 
2314
2593
 
2594
+ HOME_RERANK = "embeddings home rerank host"
2595
+
2596
+
2315
2597
  def _probe_index_models():
2316
2598
  global _GATEWAY_STATUS
2317
2599
  with _GATEWAY_LOCK:
2318
2600
  cfg = config.load_config()
2319
- embedding = local_embedder.resolve(cfg.get("embedding_model"))
2601
+ gateway = config.embedder(cfg) == "gateway"
2602
+ embedding = local_embedder.resolve(cfg)
2320
2603
  embedding = None if embedding == indexer.LEXICAL_MODEL else embedding
2321
- local_down = (config.local_models_mode(cfg) == "prefer" and local_embedder.configured()
2322
- and not local_embedder.info())
2323
- models = [embedding, cfg.get("rerank_model")]
2604
+ home_down = not gateway and not embedding
2605
+ rerank_model = cfg.get("rerank_model") if gateway else HOME_RERANK if local_embedder.rerank_configured() else None
2606
+ models = [embedding, rerank_model]
2324
2607
  if _GATEWAY_STATUS.get("models") == models and time.time() - _GATEWAY_STATUS.get("checked_at", 0) < 30:
2325
2608
  return {"gateway": dict(_GATEWAY_STATUS), "cached": True}
2326
2609
  result = {"checked_at": time.time(), "models": models}
2327
2610
  try:
2328
- if not local_embedder.is_local(embedding) or cfg.get("rerank_model"):
2611
+ token = ""
2612
+ if gateway:
2329
2613
  _require_valid_credential()
2330
- token = "" if local_embedder.is_local(embedding) and not cfg.get("rerank_model") else model_client.get_token()
2614
+ token = model_client.get_token()
2331
2615
  except Exception as exc:
2332
2616
  result.update(status="needs_config", error=_project_error(exc))
2333
2617
  else:
@@ -2335,10 +2619,8 @@ def _probe_index_models():
2335
2619
  if not model:
2336
2620
  result[kind] = {"status": "not_configured"}
2337
2621
  continue
2338
- if kind == "embedding" and local_down:
2339
- result[kind] = {"status": "error", "error": "Prefer-local mode: the mcp-memory local embedder is not "
2340
- "responding; local indexes have no vector part until it is back.",
2341
- "elapsed_s": 0.0}
2622
+ if kind == "embedding" and home_down:
2623
+ result[kind] = {"status": "error", "error": HOME_DOWN_WARNING, "elapsed_s": 0.0}
2342
2624
  continue
2343
2625
  start = time.monotonic()
2344
2626
  try:
@@ -2346,7 +2628,10 @@ def _probe_index_models():
2346
2628
  vector = _embed(model, ["Smart Tool: indexing diagnostics."], token, deadline=start + 60)[0]
2347
2629
  result[kind] = {"status": "ok", "dimensions": len(vector)}
2348
2630
  else:
2349
- _rerank(model, "indexing", ["Indexing files for search."], 1, token, deadline=start + 30)
2631
+ if model == HOME_RERANK:
2632
+ _home_rerank("indexing", ["Indexing files for search."], 1, deadline=start + 30)
2633
+ else:
2634
+ _rerank(model, "indexing", ["Indexing files for search."], 1, token, deadline=start + 30)
2350
2635
  result[kind] = {"status": "ok"}
2351
2636
  except Exception as exc:
2352
2637
  result[kind] = {"status": "error", "error": _project_error(exc)}
@@ -2675,14 +2960,14 @@ def _site_memory_correlate(query, records=None, deadline=None, include_content=T
2675
2960
  if not records:
2676
2961
  return set(), []
2677
2962
  cfg = config.load_config()
2678
- embedding_model = cfg.get("embedding_model")
2679
- rerank_model = cfg.get("rerank_model")
2680
- if not embedding_model or not rerank_model:
2681
- _web_model_warn("Site memory off (sources that helped before are not prioritized): embedding_model or rerank_model missing.")
2963
+ token = model_client.get_token() if config.embedder(cfg) == "gateway" else ""
2964
+ embedding_model, rerank, problem = _web_memory_models(cfg, token)
2965
+ if problem:
2966
+ _web_model_warn(f"Site memory off (sources that helped before are not prioritized): {problem}.")
2682
2967
  return set(), []
2683
- threshold = config.clamp_site_memory_similarity_threshold(cfg.get("site_memory_similarity_threshold"))
2968
+ threshold = (config.clamp_site_memory_similarity_threshold(cfg.get("site_memory_similarity_threshold"))
2969
+ if config.embedder(cfg) == "gateway" else HOME_SITE_MEMORY_SIMILARITY)
2684
2970
  topics = list({r["topic"] for r in records})[:SITE_MEMORY_MAX_TOPICS]
2685
- token = model_client.get_token()
2686
2971
  timed = {"deadline": deadline} if deadline is not None else {}
2687
2972
  vectors = _embed(embedding_model, [query] + topics, token, **timed)
2688
2973
  query_vec, topic_vecs = vectors[0], vectors[1:]
@@ -2693,7 +2978,7 @@ def _site_memory_correlate(query, records=None, deadline=None, include_content=T
2693
2978
  if not candidates:
2694
2979
  research_metrics.log_site_memory(query, threshold, scored_topics, recalled=0)
2695
2980
  return set(), []
2696
- reranked = _rerank(rerank_model, query, candidates, len(candidates), token, **timed)
2981
+ reranked = rerank(query, candidates, len(candidates), **timed)
2697
2982
  rerank_scores = {}
2698
2983
  for item in reranked:
2699
2984
  idx = item.get("index")
@@ -3198,7 +3483,8 @@ def _cache_match(depth, query, scope, cfg, token, deadline):
3198
3483
  for entry in entries:
3199
3484
  if _normalized(entry.get("query")) == normalized:
3200
3485
  stored = entry.get("topic_vector") or {}
3201
- return entry, (stored.get("values") if stored.get("model") == cfg.get("embedding_model") else None)
3486
+ model = _web_embedder(cfg)
3487
+ return entry, (stored.get("values") if model and stored.get("model") == model else None)
3202
3488
  if freshness.npm_version_only(query) or freshness.npm_version_security_only(query):
3203
3489
  package = freshness.npm_package(query)
3204
3490
  wants_security = freshness.npm_security_intent(query)
@@ -3209,12 +3495,11 @@ def _cache_match(depth, query, scope, cfg, token, deadline):
3209
3495
  return entry, None
3210
3496
  if model_client.OFFLINE.get():
3211
3497
  return None, None
3212
- embed_model, rerank_model = cfg.get("embedding_model"), cfg.get("rerank_model")
3498
+ embed_model, rerank, problem = _web_memory_models(cfg, token)
3213
3499
  match_model = cfg.get("router_model") or cfg.get("research_model")
3214
- if not embed_model or not rerank_model or not match_model:
3215
- missing = [name for name, value in (("embedding_model", embed_model), ("rerank_model", rerank_model),
3216
- ("router_model", match_model)) if not value]
3217
- _web_model_warn("Reuse of similar searches off (only the same question is reused): missing " + ", ".join(missing) + ".")
3500
+ if problem or not match_model:
3501
+ reasons = [text for text in (problem, "" if match_model else "router_model missing") if text]
3502
+ _web_model_warn("Reuse of similar searches off (only the same question is reused): " + "; ".join(reasons) + ".")
3218
3503
  return None, None
3219
3504
  entries = [entry for entry in entries
3220
3505
  if (entry.get("topic_vector") or {}).get("model") == embed_model]
@@ -3224,11 +3509,11 @@ def _cache_match(depth, query, scope, cfg, token, deadline):
3224
3509
  vector = _embed(embed_model, [query], token, deadline=deadline)[0]
3225
3510
  scored = sorted(((indexer.cosine_similarity(vector, entry["topic_vector"]["values"]), i)
3226
3511
  for i, entry in enumerate(entries)), reverse=True)
3227
- candidates = [i for score, i in scored[:10] if score >= 0.45]
3512
+ candidates = [i for score, i in scored[:10] if score >= SIMILAR_SEARCH_MIN_COSINE[config.embedder(cfg)]]
3228
3513
  if not candidates:
3229
3514
  return None, vector
3230
3515
  topics = [str(entries[i].get("query") or "") for i in candidates]
3231
- ranked = _rerank(rerank_model, query, topics, min(3, len(candidates)), token, deadline=deadline)
3516
+ ranked = rerank(query, topics, min(3, len(candidates)), deadline=deadline)
3232
3517
  for row in ranked[:3]:
3233
3518
  index = row.get("index")
3234
3519
  if not isinstance(index, int) or not 0 <= index < len(candidates):
@@ -3300,7 +3585,7 @@ def _validated_cache_result(depth, query, scope, cfg, token, deadline, requested
3300
3585
  hit = _revalidated(depth, entry, cfg, deadline)
3301
3586
  if hit and topic_vector is not None and not pending and _normalized(entry["query"]) != _normalized(query):
3302
3587
  research_cache.put(_cache_namespace(depth, scope), query, entry["result"], entry["classification"],
3303
- cfg.get("embedding_model"), topic_vector, origin=entry)
3588
+ _web_embedder(cfg), topic_vector, origin=entry)
3304
3589
  return hit, topic_vector
3305
3590
 
3306
3591
 
@@ -3392,7 +3677,7 @@ def _store_fresh_research(depth, query, scope, result, sources, cfg, topic_vecto
3392
3677
  classification["npm_advisories"] = (osv_source["osv_advisory_ids"]
3393
3678
  if osv_source else None)
3394
3679
  namespace = _cache_namespace(depth, scope)
3395
- embed_model = cfg.get("embedding_model")
3680
+ embed_model = _web_embedder(cfg)
3396
3681
  saved_at = research_cache.put(namespace, query, result, classification,
3397
3682
  embed_model if topic_vector else None, topic_vector)
3398
3683
  if saved_at is not None and not model_client.OFFLINE.get():
@@ -3421,10 +3706,11 @@ def _complete_research(namespace, query, saved_at, sources, pending, cfg, topic_
3421
3706
  _log(f"volatility not judged; reuse requires a check: {classification['error']}")
3422
3707
  classification.update({key: value for key, value in pending.items()
3423
3708
  if key.startswith("npm_") or key == "security_intent"})
3424
- embed_model = cfg.get("embedding_model")
3709
+ embed_model = _web_embedder(cfg)
3425
3710
  if topic_vector is None and embed_model:
3426
3711
  try:
3427
- topic_vector = _embed(embed_model, [query], model_client.get_token())[0]
3712
+ topic_vector = _embed(embed_model, [query], "" if local_embedder.is_local(embed_model)
3713
+ else model_client.get_token())[0]
3428
3714
  except Exception as exc:
3429
3715
  _log(f"research vector not generated; reuse by exact text only: {type(exc).__name__}: {exc}")
3430
3716
  if not research_cache.complete(namespace, query, saved_at, classification,
@@ -3998,10 +4284,11 @@ def _render_page(url, deadline):
3998
4284
  def _web_fetch_embed(texts, deadline):
3999
4285
  """Vectors for the pieces of a long page and the prompt, with the pieces cached per embedding model. Fails loudly
4000
4286
  without an embedding model: a long page is never cut back to its start in silence."""
4001
- model = local_embedder.resolve(config.load_config().get("embedding_model"), None)
4287
+ model = local_embedder.resolve(config.load_config(), None)
4002
4288
  if not model or model == indexer.LEXICAL_MODEL:
4003
4289
  raise RuntimeError("This page is longer than 60,000 characters and choosing its relevant parts needs an "
4004
- "embedding_model, which is not configured")
4290
+ "embedder (the embeddings home running, or the gateway's embedding_model), which is not "
4291
+ "available")
4005
4292
  try:
4006
4293
  token = model_client.get_token()
4007
4294
  probe = _embed(model, texts[-1:], token, deadline=deadline)
@@ -4257,7 +4544,7 @@ def _session_audit_hook(handler):
4257
4544
 
4258
4545
 
4259
4546
  def _handle_read_document(arguments):
4260
- """read_document: facts and a page, slide, sheet or part range of a PDF, docx, pptx or xlsx as compact text (see
4547
+ """read_document: facts and a page, slide, sheet, part or chapter range of a PDF, docx, pptx, xlsx, html, csv or epub as compact text (see
4261
4548
  document_read). Fails loudly, with the reason, for a path that is not absolute or readable, an unsupported or
4262
4549
  oversized file, a password, a scanned PDF, a corrupt or hostile package and a range outside the document; one metric
4263
4550
  row records the kind, the estimated tokens of the whole text and of the answer."""
@@ -4420,13 +4707,15 @@ _SETUP_SAVES = {
4420
4707
  "/setup/models": setup_ui.handle_models_save,
4421
4708
  "/setup/models/upgrade": setup_ui.handle_upgrade_action,
4422
4709
  "/setup/providers": setup_ui.handle_providers_save,
4423
- "/setup/local-models": setup_ui.handle_local_models_save,
4710
+ "/setup/embedder": setup_ui.handle_embedder_save,
4424
4711
  "/setup/integration": setup_ui.handle_integration_save,
4425
4712
  "/setup/gateway": setup_ui.handle_gateway_save,
4426
4713
  "/setup/autostart": setup_ui.handle_autostart_action,
4714
+ "/setup/updates": setup_ui.handle_updates_save,
4427
4715
  "/setup/contact": setup_ui.handle_contact_save,
4428
4716
  "/setup/language": setup_ui.handle_language_save,
4429
4717
  "/setup/audit": setup_ui.handle_audit,
4718
+ "/setup/usage/mute": setup_ui.handle_usage_mute,
4430
4719
  }
4431
4720
 
4432
4721
 
@@ -4563,6 +4852,18 @@ class MCPHandler(BaseHTTPRequestHandler):
4563
4852
  except Exception as exc:
4564
4853
  self._write_json(500, {"error": _sanitize_text(str(exc))})
4565
4854
  return
4855
+ if self.path == "/setup/usage":
4856
+ try:
4857
+ self._write_json(200, setup_ui.handle_usage_get())
4858
+ except Exception as exc:
4859
+ self._write_json(500, {"error": _sanitize_text(str(exc))})
4860
+ return
4861
+ if self.path == "/setup/web-search/requesters":
4862
+ try:
4863
+ self._write_json(200, setup_ui.handle_web_requesters_get())
4864
+ except Exception as exc:
4865
+ self._write_json(500, {"error": _sanitize_text(str(exc))})
4866
+ return
4566
4867
  if self.path in ("/setup/models/upgrade", "/setup/models/health"):
4567
4868
  try:
4568
4869
  self._write_json(200, setup_ui.handle_upgrade_get() if self.path.endswith("upgrade")
@@ -4593,9 +4894,9 @@ class MCPHandler(BaseHTTPRequestHandler):
4593
4894
  except Exception as exc:
4594
4895
  self._write_json(500, {"error": _sanitize_text(str(exc))})
4595
4896
  return
4596
- if self.path == "/setup/local-models":
4897
+ if self.path == "/setup/embedder":
4597
4898
  try:
4598
- self._write_json(200, setup_ui.handle_local_models_get())
4899
+ self._write_json(200, setup_ui.handle_embedder_get())
4599
4900
  except Exception as exc:
4600
4901
  self._write_json(500, {"error": _sanitize_text(str(exc))})
4601
4902
  return
@@ -4611,6 +4912,12 @@ class MCPHandler(BaseHTTPRequestHandler):
4611
4912
  except Exception as exc:
4612
4913
  self._write_json(500, {"error": _sanitize_text(str(exc))})
4613
4914
  return
4915
+ if self.path == "/setup/updates":
4916
+ try:
4917
+ self._write_json(200, setup_ui.handle_updates_get())
4918
+ except Exception as exc:
4919
+ self._write_json(500, {"error": _sanitize_text(str(exc))})
4920
+ return
4614
4921
  self._write_json(404, {"error": "not found"})
4615
4922
 
4616
4923
  def _post_setup(self, save):