@allansantos-dev/smart-tool 0.9.78 → 0.9.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +79 -0
- package/README.md +65 -18
- package/affected_tests.py +4 -0
- package/browser_control.py +10 -3
- package/browser_service_client.py +3 -32
- package/chromium_flags.py +30 -0
- package/code_graph.py +39 -17
- package/code_graph_go.py +6 -6
- package/code_graph_java.mjs +10 -9
- package/code_graph_js.cjs +8 -8
- package/code_health.py +180 -0
- package/code_impact.py +16 -2
- package/config.py +41 -8
- package/credential_check.py +38 -0
- package/dependency_notice.py +262 -14
- package/doc_scope.py +1 -1
- package/document_converter.py +221 -0
- package/document_read.py +100 -25
- package/document_text.py +194 -6
- package/dossier.py +35 -8
- package/duplicates.py +1 -1
- package/embedding_cache.py +32 -8
- package/freshness.py +3 -2
- package/hook_decision.py +65 -14
- package/indexer.py +92 -3
- package/layer_check.py +57 -0
- package/layers.py +3 -3
- package/local_embedder.py +114 -57
- package/locales/pt-BR.json +52 -17
- package/low_priority.py +41 -0
- package/package.json +1 -1
- package/pr_impact.py +113 -8
- package/process_limits.py +108 -0
- package/project_monitor.py +33 -7
- package/project_scheduler.py +12 -4
- package/scout.py +6 -2
- package/setup_audit.py +34 -1
- package/setup_ui.py +210 -39
- package/smart_tool_daemon.py +398 -91
- package/update_check.py +18 -0
- package/usage_meter.py +78 -0
- package/version.py +1 -1
- package/web_adapters/browser/crawl4ai_search.py +7 -3
- package/web_fetch.py +24 -0
- package/web_search_health.py +31 -0
package/smart_tool_daemon.py
CHANGED
|
@@ -55,6 +55,7 @@ import index_inventory
|
|
|
55
55
|
import usage_meter
|
|
56
56
|
import document_read
|
|
57
57
|
import document_text
|
|
58
|
+
import dependency_notice
|
|
58
59
|
import dossier
|
|
59
60
|
import scout
|
|
60
61
|
import embedding_cache
|
|
@@ -76,6 +77,7 @@ import site_memory
|
|
|
76
77
|
import web_search_adapters
|
|
77
78
|
import browser_service_client
|
|
78
79
|
import local_embedder
|
|
80
|
+
import low_priority
|
|
79
81
|
import web_search_health
|
|
80
82
|
import web_fetch
|
|
81
83
|
import version
|
|
@@ -172,6 +174,12 @@ SITE_MEMORY_MAX_TOPICS = 50
|
|
|
172
174
|
# em até 0,064 e pares úteis a partir de 0,148. 0,12 separa as duas amostras
|
|
173
175
|
# sem endurecer o limiar de embedding (0,35), que é só filtro de candidatos.
|
|
174
176
|
SITE_MEMORY_RERANK_MIN_SCORE = 0.12
|
|
177
|
+
# The cosine scale depends on the embedder: granite (the embeddings home) puts unrelated web queries at 0.48 to 0.73,
|
|
178
|
+
# where text-embedding-3-small puts them at 0.09 to 0.54. On the same 20 near and 20 far pairs
|
|
179
|
+
# (docs/spikes/web_memory_threshold_spike.py), 0.62 and 0.68 keep every near pair and as many far ones as 0.35 and 0.45
|
|
180
|
+
# keep on the gateway model. The configured site memory threshold is the gateway's.
|
|
181
|
+
HOME_SITE_MEMORY_SIMILARITY = 0.62
|
|
182
|
+
SIMILAR_SEARCH_MIN_COSINE = {"gateway": 0.45, "home": 0.68}
|
|
175
183
|
|
|
176
184
|
|
|
177
185
|
def _parse_bounded_int(value, default, min_v, max_v, field_name):
|
|
@@ -204,7 +212,10 @@ SMART_SEARCH_RESULT_WAIT_S = 15
|
|
|
204
212
|
SMART_SEARCH_MAX_PENDING = 8
|
|
205
213
|
BACKGROUND_MAX_PENDING = 8
|
|
206
214
|
INTERACTIVE_PRIORITY = 1
|
|
207
|
-
|
|
215
|
+
# Slowest first-index speed measured on a CPU-only embedder (docs/research/2026-11-16-indexacao-cpu-busca-durante-o-indice.md).
|
|
216
|
+
CPU_INDEX_FILES_PER_MINUTE = 180
|
|
217
|
+
_INDEX_RUNNING = {}
|
|
218
|
+
SMART_SEARCH_EXECUTOR = project_scheduler.ProjectScheduler(workers=3, reserved_lane="search")
|
|
208
219
|
_PROJECT_LOCKS = {}
|
|
209
220
|
_PROJECT_LOCKS_LOCK = threading.Lock()
|
|
210
221
|
_PROJECT_MONITOR = None
|
|
@@ -373,10 +384,10 @@ SERVER_INSTRUCTIONS = (
|
|
|
373
384
|
TOOLS = [
|
|
374
385
|
{
|
|
375
386
|
"name": "project_manage",
|
|
376
|
-
"description": "Manages folders and indexing. Every action takes project_root (the folder) or project_id. list gives a short line per project (a project whose folder is gone shows as unavailable with the reason); status shows storage, scope and jobs, and with a path inside a registered project answers for the deepest one that holds it (resolved_from). graph with symbol (function, Class.method, class name, or path::name when the name repeats) answers what an edit touches: definition, callers with line, calls, tests that reach it through static calls (depth hops, default 3) and files importing it; use it before changing a function to know what to update. affected_tests answers which tests to run after editing: every test file importing a changed file (git diff against base, default HEAD, untracked included) plus changed tests, likeliest failures first, with the command to run them and run_all when config or unanalyzed code changed. pr_impact (base, default HEAD; main or origin/main for a branch) returns a Markdown report of the change for a pull request: the functions the diff touches with their callers, the documentation entries it leaves out of date and the tests to run with their command; it publishes nothing. graph without symbol reads the indexed view: coverage, references and static calls in Java, Angular, Python, Go and JS/TS/JSX/TSX; includes HTML/CSS/Markdown references and TXT/DOCX coverage; with file_path, narrows to that file's symbols, imports and calls (lists capped by limit). inspect lists the files of a view_id; with file_path, returns only that file's chunks. usage shows consumption and reuse; docs lists public functions without a docstring (Python docstring, JSDoc, Javadoc, Go doc comment starting with the name; private, nested and override functions excluded) with coverage percent, excluding tests unless include_tests=true, for documenting a project that started without it; docs generate=true writes the documentation scope to docs/scope in the project in three levels tied to the code: technical pages by folder (an entry per public function anchored on path::name with signature, first doc line, layer, callers, calls and tests, plus a Mermaid flow by layer), functional.md (each entry point and the functions its flow reaches, by layer) and simple.md; the prose between the prose markers is kept by regeneration; docs write_prose=true writes the empty prose blocks with the Docs model of the setup (each paragraph checked against the code first) within about 40 s and says how many are left, call it again until none is; docs check=true reports anchors whose function changed signature or is gone and broken links; docs validate=true checks every claim of docs/scope against the code (anchors, links, line numbers, callers and tests, coverage, entry points, command-line options, prose names, counts, glossary, navigation, versions) and writes docs/scope/validation.md with pass/fail per check, the evidence, a score and the verdict (generate runs it too); docs with symbol lists where that function is documented; layers declares the project's layers (ordered highest first, each with path globs and annotations): without layers it returns the saved rule or a suggestion read from folder names and annotations, with files, symbols and calls between layers (direction down or up); pass layers to preview a rule and the same layers with confirm=true to save it in the project (.smart-tool/layers.json); duplicates lists duplicated functions (identical, near-identical and semantic bodies, min_similarity default 0.90; below that most pairs measured were false positives), excluding tests unless include_tests=true; fix real duplicates; only dismiss with duplicates_dismiss (finding_id, reason=false_positive|intentional, note) what is not a duplicate or is a copy kept on purpose: the finding stays hidden until the code changes, and duplicates_restore reopens it; integration shows the MCP client, hook and tool usage metrics; install_hook_preview shows the change that would install the hook redirecting Grep/Read to smart_search and WebSearch/WebFetch to web_search/web_fetch in this session's client and returns a plan_id; install_hook writes it with that plan_id and confirm=true, with a backup, only after the user approves the change; search_limits saves the project's default top_k per block ({code, test, doc}, each 1-20, total at most 20); storage shows disk usage and views; pin_view protects a view; each view carries stale (not updated for 30 days, the test of the daily cleanup that removes them) and stale_bytes sums them; cleanup_preview and cleanup_commit remove selected views after confirmation; compare compares two stored views, without checkout or embeddings. register adds a folder; preview checks the scope; index/rebuild update; pause/cancel keep checkpoints; resume continues; policy picks on_search/eager; scope adjusts the scope; remove deletes only the local index. setup_audit (project_root) reports what Claude Code and Codex actually load for that folder (settings layers, plugins, hooks, MCP servers, skills, CLAUDE.md/AGENTS.md) and where they collide: hooks deciding on the same tool in parallel, missing hook scripts, silenced hooks, MCP servers down or offering duplicate tools, skill name and budget problems, AGENTS.md that only Codex reads, broken @imports; what could not be checked is listed as not verified; each finding carries its fix (the file to edit or the command to run), applied by the agent when the user agrees, never by Smart Tool; mode=last returns the stored report instead of auditing again. redirect_proposals (no project_root) lists the redirect patterns proposed from allowed searches, most matched first, with a guide; redirect_decide decides them by ids: accept, instruct (tip without blocking), reject, or remove an accepted one. junk (no project_root) lists the registered projects that were never projects: a personal, system, application-data or build folder, a subfolder of a repository or of another project, a folder of projects, or a folder that no longer exists (on a drive or share that is mounted); with confirm=true it unregisters them and deletes only their index files, and the manifest says what went. A project is the folder where the session started, so smart_search and register refuse a folder that is not a project and index a new one only from the session that started in it. Jobs via smart_search_result.",
|
|
387
|
+
"description": "Manages folders and indexing. Every action takes project_root (the folder) or project_id. list gives a short line per project (a project whose folder is gone shows as unavailable with the reason); status shows storage, scope and jobs, and with a path inside a registered project answers for the deepest one that holds it (resolved_from). deps lists the known OSV advisories of every dependency the manifests of the project declare (package.json, requirements*.txt, pyproject.toml, go.mod), not only the ones an edit changed, at the version a package-lock.json beside a package.json resolved when it satisfies the range; only package names and versions leave the machine. graph with symbol (function, Class.method, class name, or path::name when the name repeats) answers what an edit touches: definition, callers with line, calls, tests that reach it through static calls (depth hops, default 3) and files importing it; use it before changing a function to know what to update. affected_tests answers which tests to run after editing: every test file importing a changed file (git diff against base, default HEAD, untracked included) plus changed tests, likeliest failures first, with the command to run them and run_all when config or unanalyzed code changed. pr_impact (base, default HEAD; main or origin/main for a branch) returns a Markdown report of the change for a pull request: the functions the diff touches with their callers and the tests that reach them (those no test reaches are listed apart), the documentation entries it leaves out of date, the notes the edit hook would have added to the whole diff (comments, copies, dependency advisories) and the tests to run with their command; it publishes nothing. hotspots (limit) ranks the code files by commits times lines, the riskiest to change; dead_code lists the functions no code calls and those only tests call (static call graph, candidates); import_cycles lists the groups of files that import each other; comments lists comments in the wrong place (commented-out code, history tags, loose docs, long inline blocks) across the code files, not only in an edit. graph without symbol reads the indexed view: coverage, references and static calls in Java, Angular, Python, Go and JS/TS/JSX/TSX; includes HTML/CSS/Markdown references and TXT/DOCX coverage; with file_path, narrows to that file's symbols, imports and calls (lists capped by limit). inspect lists the files of a view_id; with file_path, returns only that file's chunks. usage shows consumption and reuse; docs lists public functions without a docstring (Python docstring, JSDoc, Javadoc, Go doc comment starting with the name; private, nested and override functions excluded) with coverage percent, excluding tests unless include_tests=true, for documenting a project that started without it; docs generate=true writes the documentation scope to docs/scope in the project in three levels tied to the code: technical pages by folder (an entry per public function anchored on path::name with signature, first doc line, layer, callers, calls and tests, plus a Mermaid flow by layer), functional.md (each entry point and the functions its flow reaches, by layer) and simple.md; the prose between the prose markers is kept by regeneration; docs write_prose=true writes the empty prose blocks with the Docs model of the setup (each paragraph checked against the code first) within about 40 s and says how many are left, call it again until none is; docs check=true reports anchors whose function changed signature or is gone and broken links; docs validate=true checks every claim of docs/scope against the code (anchors, links, line numbers, callers and tests, coverage, entry points, command-line options, prose names, counts, glossary, navigation, versions) and writes docs/scope/validation.md with pass/fail per check, the evidence, a score and the verdict (generate runs it too); docs with symbol lists where that function is documented; layers declares the project's layers (ordered highest first, each with path globs and annotations): without layers it returns the saved rule or a suggestion read from folder names and annotations, with files, symbols and calls between layers (direction down or up); pass layers to preview a rule and the same layers with confirm=true to save it in the project (.smart-tool/layers.json); duplicates lists duplicated functions (identical, near-identical and semantic bodies, min_similarity default 0.90; below that most pairs measured were false positives), excluding tests unless include_tests=true; fix real duplicates; only dismiss with duplicates_dismiss (finding_id, reason=false_positive|intentional, note) what is not a duplicate or is a copy kept on purpose: the finding stays hidden until the code changes, and duplicates_restore reopens it; integration shows the MCP client, hook and tool usage metrics; install_hook_preview shows the change that would install the hook redirecting Grep/Read to smart_search and WebSearch/WebFetch to web_search/web_fetch in this session's client and returns a plan_id; install_hook writes it with that plan_id and confirm=true, with a backup, only after the user approves the change; search_limits saves the project's default top_k per block ({code, test, doc}, each 1-20, total at most 20); storage shows disk usage and views; pin_view protects a view; each view carries stale (not updated for 30 days, the test of the daily cleanup that removes them) and stale_bytes sums them; cleanup_preview and cleanup_commit remove selected views after confirmation; compare compares two stored views, without checkout or embeddings. register adds a folder; preview checks the scope; index/rebuild update; pause/cancel keep checkpoints; resume continues; policy picks on_search/eager; scope adjusts the scope; remove deletes only the local index. setup_audit (project_root) reports what Claude Code and Codex actually load for that folder (settings layers, plugins, hooks, MCP servers, skills, CLAUDE.md/AGENTS.md) and where they collide: hooks deciding on the same tool in parallel, missing hook scripts, silenced hooks, MCP servers down or offering duplicate tools, skill name and budget problems, AGENTS.md that only Codex reads, broken @imports; what could not be checked is listed as not verified; each finding carries its fix (the file to edit or the command to run), applied by the agent when the user agrees, never by Smart Tool; mode=last returns the stored report instead of auditing again. redirect_proposals (no project_root) lists the redirect patterns proposed from allowed searches, most matched first, with a guide; redirect_decide decides them by ids: accept, instruct (tip without blocking), reject, or remove an accepted one. junk (no project_root) lists the registered projects that were never projects: a personal, system, application-data or build folder, a subfolder of a repository or of another project, a folder of projects, or a folder that no longer exists (on a drive or share that is mounted); with confirm=true it unregisters them and deletes only their index files, and the manifest says what went. A project is the folder where the session started, so smart_search and register refuse a folder that is not a project and index a new one only from the session that started in it. Jobs via smart_search_result.",
|
|
377
388
|
"inputSchema": {
|
|
378
389
|
"type": "object", "properties": {
|
|
379
|
-
"action": {"type": "string", "enum": ["list", "status", "register", "preview", "index", "rebuild", "pause", "cancel", "resume", "watch", "scope", "remove", "relocate", "probe", "policy", "inspect", "graph", "usage", "storage", "pin_view", "cleanup_preview", "cleanup_commit", "compare", "search_limits", "integration", "install_hook_preview", "install_hook", "duplicates", "duplicates_dismiss", "duplicates_restore", "docs", "layers", "affected_tests", "pr_impact", "setup_audit", "redirect_proposals", "redirect_decide", "junk"]},
|
|
390
|
+
"action": {"type": "string", "enum": ["list", "status", "register", "preview", "index", "rebuild", "pause", "cancel", "resume", "watch", "scope", "remove", "relocate", "probe", "policy", "inspect", "graph", "usage", "storage", "pin_view", "cleanup_preview", "cleanup_commit", "compare", "search_limits", "integration", "install_hook_preview", "install_hook", "duplicates", "duplicates_dismiss", "duplicates_restore", "docs", "layers", "affected_tests", "pr_impact", "hotspots", "dead_code", "import_cycles", "comments", "setup_audit", "redirect_proposals", "redirect_decide", "junk", "deps"]},
|
|
380
391
|
"ids": {"type": "array", "items": {"type": "string"}, "description": "redirect_decide: proposal ids from redirect_proposals (accepted ids for remove)"},
|
|
381
392
|
"decision": {"type": "string", "enum": ["accept", "instruct", "reject", "remove"], "description": "redirect_decide: accept redirects matching calls to smart_search; instruct lets them run with a tip naming smart_search; reject discards; remove deletes an accepted pattern"},
|
|
382
393
|
"finding_id": {"type": "string"},
|
|
@@ -424,7 +435,7 @@ TOOLS = [
|
|
|
424
435
|
"name": "smart_search",
|
|
425
436
|
"_meta": ALWAYS_LOAD,
|
|
426
437
|
"description": (
|
|
427
|
-
"
|
|
438
|
+
"Search over the project's index (lexical, plus semantic with an embedding model and reranked with a rerank model, when configured): the first step to understand code in a "
|
|
428
439
|
"project, finding functions, flows and reusable code by meaning, ranked, with name:line. Use it before "
|
|
429
440
|
"a series of Grep/Read; use Grep only for an exact literal or inside a known file. "
|
|
430
441
|
"Results come in three blocks, code, test and doc, plus the project profile with the natural "
|
|
@@ -454,6 +465,10 @@ TOOLS = [
|
|
|
454
465
|
"test": {"type": "integer", "minimum": 1, "maximum": 20},
|
|
455
466
|
"doc": {"type": "integer", "minimum": 1, "maximum": 20}}}],
|
|
456
467
|
"description": "Results per block: one number for all blocks or {code, test, doc}; each 1-20, total at most 20. Defaults to the project's search_limits, else 5"},
|
|
468
|
+
"folder": {"anyOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}, "maxItems": 10}],
|
|
469
|
+
"description": "Only files under this folder of the project (a project-relative path, or a list of them); applied before ranking, so the results are the best inside the folder"},
|
|
470
|
+
"layer": {"type": "string",
|
|
471
|
+
"description": "Only files of this layer, by the path globs of the project's confirmed layers (project_manage action=layers); applied before ranking"},
|
|
457
472
|
"detail": {"type": "string", "enum": ["compact", "full"], "default": "compact",
|
|
458
473
|
"description": "compact: 240-character excerpts, and ranking (when rerank) and location_kind (when source_line) left out of each result; full: 500-character excerpts with every field"},
|
|
459
474
|
"mode": {"type": "string", "enum": ["search", "dossier"], "default": "search",
|
|
@@ -462,6 +477,10 @@ TOOLS = [
|
|
|
462
477
|
"description": "mode=dossier only: scan budget; past it the answer is marked degraded"},
|
|
463
478
|
"max_hits": {"type": "integer", "minimum": 1, "maximum": 10, "default": 5,
|
|
464
479
|
"description": "mode=dossier only: hits returned"},
|
|
480
|
+
"history": {"type": "boolean", "default": False,
|
|
481
|
+
"description": "mode=dossier only: also search the messages of the project's commits that have a body (hits with source git and id git:<hash>); the first read of a repository can take longer than timeout_ms"},
|
|
482
|
+
"decision_notes": {"type": "boolean", "default": False,
|
|
483
|
+
"description": "mode=search only: append the project's decision notes (docs/research, the backlog, CLAUDE.md, AGENTS.md and commit messages) under decision_notes"},
|
|
465
484
|
},
|
|
466
485
|
"required": ["project_root"],
|
|
467
486
|
},
|
|
@@ -573,21 +592,22 @@ TOOLS = [
|
|
|
573
592
|
{
|
|
574
593
|
"name": "read_document",
|
|
575
594
|
"description": (
|
|
576
|
-
"Reads a PDF, Word (.docx), PowerPoint (.pptx)
|
|
577
|
-
"inside Smart Tool so the file never enters your context whole. The answer opens with the facts: type, "
|
|
595
|
+
"Reads a PDF, Word (.docx), PowerPoint (.pptx), Excel (.xlsx), HTML (.html, .htm), CSV (.csv) or EPUB "
|
|
596
|
+
"(.epub) file as compact text, a range at a time, inside Smart Tool so the file never enters your context whole. The answer opens with the facts: type, "
|
|
578
597
|
"size, page, slide, sheet or part count and the estimated tokens of all the text; then comes the text of "
|
|
579
598
|
"the range you asked for (default: from the start, up to max_chars) with a `next:` line naming the range "
|
|
580
599
|
"to continue. PDF pages come with the layout noise removed (ligatures, hyphenation, wrapped lines); docx "
|
|
581
600
|
"as markdown (headings, lists, tables) in parts of about 6,000 characters because a docx has no fixed "
|
|
582
601
|
"pages; pptx per slide with the speaker notes; xlsx per sheet, one tab-separated line per row with the "
|
|
583
|
-
"cached formula values
|
|
602
|
+
"cached formula values; html as markdown in parts like a docx; csv as one sheet, one tab-separated line per row; "
|
|
603
|
+
"epub per chapter in reading order. Use info_only first to size a large file and pick the pages you need. A scanned "
|
|
584
604
|
"PDF (no text layer) fails with that reason; there is no OCR. Not for plain text or code: use Read."
|
|
585
605
|
),
|
|
586
606
|
"inputSchema": {
|
|
587
607
|
"type": "object",
|
|
588
608
|
"properties": {
|
|
589
|
-
"path": {"type": "string", "description": "Absolute path of the .pdf, .docx, .pptx or .
|
|
590
|
-
"range": {"type": "string", "description": "Pages, slides,
|
|
609
|
+
"path": {"type": "string", "description": "Absolute path of the .pdf, .docx, .pptx, .xlsx, .html, .htm, .csv or .epub file"},
|
|
610
|
+
"range": {"type": "string", "description": "Pages, slides, parts, chapters or sheets to read: `3`, `3-5`, `7-` (to the end), `-4` (the first four) or a comma list like `1,4-6`; a sheet name also works for a workbook. Default: from the first one"},
|
|
591
611
|
"max_chars": {"type": "integer", "minimum": 1000, "maximum": document_read.MAX_CHARS_LIMIT, "default": document_read.DEFAULT_MAX_CHARS, "description": "Cap of the text returned (about four characters per token)"},
|
|
592
612
|
"info_only": {"type": "boolean", "default": False, "description": "Return only the facts (type, size, counts, estimated tokens), no text"},
|
|
593
613
|
},
|
|
@@ -804,8 +824,9 @@ def _embed(model, texts, token, deadline=None, on_retry=None, cancel_check=None)
|
|
|
804
824
|
|
|
805
825
|
|
|
806
826
|
def _rerank(model, query, documents, top_n, token, deadline=None):
|
|
807
|
-
"""Rows `{index, relevance_score}` of the best `top_n` documents. Only an HTTP answer that is worth
|
|
808
|
-
(429, 5xx) is asked again, after the wait of `_wait_to_retry`; a timeout fails at once so a search does
|
|
827
|
+
"""Rows `{index, relevance_score}` of the best `top_n` documents from the gateway. Only an HTTP answer that is worth
|
|
828
|
+
repeating (429, 5xx) is asked again, after the wait of `_wait_to_retry`; a timeout fails at once so a search does
|
|
829
|
+
not stall."""
|
|
809
830
|
if not documents:
|
|
810
831
|
return []
|
|
811
832
|
top_n = min(top_n, len(documents))
|
|
@@ -822,6 +843,20 @@ def _rerank(model, query, documents, top_n, token, deadline=None):
|
|
|
822
843
|
if attempt >= EMBED_MAX_ATTEMPTS or not _transient_embed_error(exc):
|
|
823
844
|
raise
|
|
824
845
|
_wait_to_retry(exc, attempt, deadline)
|
|
846
|
+
return _rerank_rows(response, documents, top_n)
|
|
847
|
+
|
|
848
|
+
|
|
849
|
+
def _home_rerank(query, documents, top_n, deadline=None):
|
|
850
|
+
"""`_rerank` through the embeddings home, which forwards to the rerank host configured in it."""
|
|
851
|
+
if not documents:
|
|
852
|
+
return []
|
|
853
|
+
top_n = min(top_n, len(documents))
|
|
854
|
+
return _rerank_rows(local_embedder.rerank(query, documents, top_n, deadline=deadline), documents, top_n)
|
|
855
|
+
|
|
856
|
+
|
|
857
|
+
def _rerank_rows(response, documents, top_n):
|
|
858
|
+
"""The best `top_n` rows of a rerank answer in the Cohere form, highest score first. Scores are opaque: only their
|
|
859
|
+
order within one answer counts. A repeated or out-of-range index, or a missing score, raises."""
|
|
825
860
|
rows = response.get("results", []) if isinstance(response, dict) else []
|
|
826
861
|
if not isinstance(rows, list) or not rows:
|
|
827
862
|
raise ValueError("The reranker returned no valid results.")
|
|
@@ -833,7 +868,46 @@ def _rerank(model, query, documents, top_n, token, deadline=None):
|
|
|
833
868
|
not isinstance(score, (int, float)) or not math.isfinite(score)):
|
|
834
869
|
raise ValueError("The reranker returned an invalid index or score.")
|
|
835
870
|
seen.add(idx)
|
|
836
|
-
return rows[:top_n]
|
|
871
|
+
return sorted(rows, key=lambda row: row["relevance_score"], reverse=True)[:top_n]
|
|
872
|
+
|
|
873
|
+
|
|
874
|
+
def _reranker(cfg, token):
|
|
875
|
+
"""The reranker as `rerank(query, documents, top_n, deadline)`, or None when there is none: the gateway's
|
|
876
|
+
rerank_model when the user chose the gateway, else the embeddings home when a rerank host is configured in it."""
|
|
877
|
+
if config.embedder(cfg) == "gateway":
|
|
878
|
+
model = cfg.get("rerank_model")
|
|
879
|
+
return (lambda query, documents, top_n, deadline=None:
|
|
880
|
+
_rerank(model, query, documents, top_n, token, deadline=deadline)) if model else None
|
|
881
|
+
return _home_rerank if local_embedder.rerank_configured() else None
|
|
882
|
+
|
|
883
|
+
|
|
884
|
+
def _web_embedder(cfg):
|
|
885
|
+
"""Embedding model of the web memory, the same choice as the project indexes (config `embedder`); None when there
|
|
886
|
+
is none: no gateway embedding_model, or the embeddings home not running or unusable."""
|
|
887
|
+
try:
|
|
888
|
+
model = local_embedder.resolve(cfg)
|
|
889
|
+
except local_embedder.IdentityError as exc:
|
|
890
|
+
_log(f"web memory without vectors: {exc}")
|
|
891
|
+
return None
|
|
892
|
+
return None if model in (None, indexer.LEXICAL_MODEL) else model
|
|
893
|
+
|
|
894
|
+
|
|
895
|
+
def _web_memory_models(cfg, token):
|
|
896
|
+
"""`(embedding_model, rerank, problem)` of the web memory (sources that helped before, reuse of similar searches).
|
|
897
|
+
It follows config `embedder` like the project indexes: a stopped home is never replaced by the gateway. `problem`
|
|
898
|
+
says what is missing, empty when both are there."""
|
|
899
|
+
try:
|
|
900
|
+
model = local_embedder.resolve(cfg)
|
|
901
|
+
rerank = _reranker(cfg, token)
|
|
902
|
+
except local_embedder.IdentityError as exc:
|
|
903
|
+
return None, None, str(exc)
|
|
904
|
+
model = None if model in (None, indexer.LEXICAL_MODEL) else model
|
|
905
|
+
if config.embedder(cfg) == "gateway":
|
|
906
|
+
missing = [name for name, value in (("embedding_model", model), ("rerank_model", rerank)) if not value]
|
|
907
|
+
return model, rerank, " and ".join(missing) + " missing" if missing else ""
|
|
908
|
+
if not model:
|
|
909
|
+
return None, rerank, "the embeddings home is not running"
|
|
910
|
+
return model, rerank, "" if rerank else "no rerank host configured in the embeddings home"
|
|
837
911
|
|
|
838
912
|
|
|
839
913
|
LOG_PATH = os.path.join(RUN_DIR, "daemon.log")
|
|
@@ -1018,6 +1092,52 @@ def _cold_overlay(root, queries):
|
|
|
1018
1092
|
return {"drop": [], "chunks": chunks}, warning
|
|
1019
1093
|
|
|
1020
1094
|
|
|
1095
|
+
def _duration(seconds):
|
|
1096
|
+
return f"{round(seconds)} s" if seconds < 90 else f"{-(-round(seconds) // 60)} min"
|
|
1097
|
+
|
|
1098
|
+
|
|
1099
|
+
def _indexing_notice(root, model, indexed, total):
|
|
1100
|
+
"""Where the running index build of `root` stands and how long it should take: the speed so far once files were
|
|
1101
|
+
indexed, else, on a CPU-only embedder, the slowest measured speed as an upper bound."""
|
|
1102
|
+
with _JOBS_LOCK:
|
|
1103
|
+
job = dict(_jobs.get(_INDEX_RUNNING.get(project_identity.canonical_root(root))) or {})
|
|
1104
|
+
done = max(indexed, job.get("processed_files") or 0)
|
|
1105
|
+
total = max(job.get("total_files") or 0, total, done)
|
|
1106
|
+
since = next((at for phase, at in reversed(job.get("phases", [])) if phase == "indexing"), None)
|
|
1107
|
+
left = total - done
|
|
1108
|
+
if done and since and time.time() > since:
|
|
1109
|
+
eta = f"about {_duration(left * (time.time() - since) / done)} left at the speed so far"
|
|
1110
|
+
elif local_embedder.runs_on_cpu(model):
|
|
1111
|
+
eta = f"expected time on this CPU-only embedder: up to about {_duration(left * 60 / CPU_INDEX_FILES_PER_MINUTE)}"
|
|
1112
|
+
else:
|
|
1113
|
+
eta = "time left not known yet"
|
|
1114
|
+
return (f"Indexing in progress: {done} of {total} files indexed ({100 * done // max(total, 1)}%), {eta}. Results "
|
|
1115
|
+
"rank the indexed files by meaning and keywords and read the others from disk by keyword; repeat the "
|
|
1116
|
+
"search later for the full ranking.")
|
|
1117
|
+
|
|
1118
|
+
|
|
1119
|
+
def _partial_search(root, model, texts, token, job_id):
|
|
1120
|
+
"""`(vectors, path, overlay, notice)` for a search made while the index of `root` is being built: the query is
|
|
1121
|
+
embedded and searched in the part of the new index already committed, and the files not committed yet are read
|
|
1122
|
+
from disk by keyword."""
|
|
1123
|
+
path, indexed = indexer.partial_index(root, model)
|
|
1124
|
+
scope = index_scope.load_scope(root)
|
|
1125
|
+
files = index_scope.resolve_included_files(root, scope if index_scope.valid_scope(scope) else {"include": []})
|
|
1126
|
+
notice = _indexing_notice(root, model, len(indexed), len(files))
|
|
1127
|
+
vectors = {}
|
|
1128
|
+
if path:
|
|
1129
|
+
try:
|
|
1130
|
+
vectors = dict(zip(texts, _embed(model, texts, token, deadline=_job_deadline(job_id),
|
|
1131
|
+
cancel_check=lambda: _job_checkpoint(job_id))))
|
|
1132
|
+
except ProjectJobCancelled:
|
|
1133
|
+
raise
|
|
1134
|
+
except Exception as exc:
|
|
1135
|
+
notice += " Embedding unavailable, so the indexed files were searched by keyword only. " + _project_error(exc)
|
|
1136
|
+
chunks, _scanned = indexer.keyword_chunks(root, [rel for rel in files if rel not in indexed], texts,
|
|
1137
|
+
COLD_SCAN_BUDGET_S)
|
|
1138
|
+
return vectors, path, {"drop": [], "chunks": chunks}, notice
|
|
1139
|
+
|
|
1140
|
+
|
|
1021
1141
|
def _lag_overlay(root, lag):
|
|
1022
1142
|
"""`(overlay, warning)` for `indexer.search`: the stale files read from disk, searched by keyword in place of their
|
|
1023
1143
|
outdated chunks, and the notice telling the agent that indexing is in progress. `(None, "")` when the index is
|
|
@@ -1137,31 +1257,43 @@ def _prepare_project_index(arguments, job_id, cfg, token, query_vector=None, pre
|
|
|
1137
1257
|
"sample_paths": included[:30]}
|
|
1138
1258
|
project_store.update(project_identity.project_id(root), preview=result)
|
|
1139
1259
|
return result
|
|
1140
|
-
model = model or local_embedder.resolve(cfg
|
|
1260
|
+
model = model or local_embedder.resolve(cfg, indexer.info(root).get("model_id"))
|
|
1261
|
+
if model is None:
|
|
1262
|
+
raise RuntimeError(f"The embeddings home is not running: this folder's index holds "
|
|
1263
|
+
f"{indexer.info(root).get('model_id')} and keeps answering; re-embedding starts when the "
|
|
1264
|
+
"home answers.")
|
|
1141
1265
|
lexical = model == indexer.LEXICAL_MODEL
|
|
1142
1266
|
current = indexer.info(root)
|
|
1143
1267
|
known_dimensions = current.get("dimensions") if current.get("model_id") == model else None
|
|
1268
|
+
gentle = not lexical and local_embedder.runs_on_cpu(model)
|
|
1144
1269
|
if not lexical and query_vector is None and not known_dimensions:
|
|
1145
1270
|
_update_job(job_id, phase="model_probe")
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1271
|
+
with low_priority.background() if gentle else contextlib.nullcontext():
|
|
1272
|
+
query_vector = _embed(model, ["Smart Tool: indexing dimension check."], token,
|
|
1273
|
+
deadline=_job_deadline(job_id), cancel_check=checkpoint,
|
|
1274
|
+
on_retry=lambda attempt, items: _update_job(job_id, phase="indexing_retry",
|
|
1275
|
+
retry_attempt=attempt, batch_items=items))[0]
|
|
1150
1276
|
checkpoint()
|
|
1151
1277
|
_update_job(job_id, phase="indexing")
|
|
1152
1278
|
full_check = bool(arguments.get("full_check") or project.get("needs_full_check") or
|
|
1153
1279
|
time.time() - current.get("full_scan_at", 0) >= project_monitor.FULL_CHECK_S)
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1280
|
+
|
|
1281
|
+
def embed_batch(texts):
|
|
1282
|
+
with low_priority.background() if gentle else contextlib.nullcontext():
|
|
1283
|
+
return _embed(model, texts, token, deadline=_job_deadline(job_id), cancel_check=checkpoint,
|
|
1284
|
+
on_retry=lambda attempt, items: _update_job(
|
|
1285
|
+
job_id, phase="indexing_retry", retry_attempt=attempt, batch_items=items))
|
|
1286
|
+
|
|
1287
|
+
with low_priority.background() if gentle else contextlib.nullcontext():
|
|
1288
|
+
stats = indexer.reindex(
|
|
1289
|
+
root, included, None if lexical else embed_batch,
|
|
1290
|
+
on_progress=lambda processed, total: _update_job(
|
|
1291
|
+
job_id, phase="indexing", processed_files=processed, total_files=total),
|
|
1292
|
+
model_id=model, expected_dimensions=None if lexical else known_dimensions or len(query_vector),
|
|
1293
|
+
force_rebuild=bool(arguments.get("force_rebuild")), full_check=full_check,
|
|
1294
|
+
changed_paths=set(arguments.get("changed_paths") or []) | set(project.get('dirty_paths', [])),
|
|
1295
|
+
cancel_check=checkpoint,
|
|
1296
|
+
)
|
|
1165
1297
|
_update_job(job_id, stats=stats, index_updated=stats.get("ready", True))
|
|
1166
1298
|
project_store.update(project_identity.project_id(root), index=indexer.info(root),
|
|
1167
1299
|
last_stats=stats, last_checked=time.time(), scope_summary={
|
|
@@ -1188,15 +1320,20 @@ def _grouped_yaml(profile, blocks, warning):
|
|
|
1188
1320
|
return "\n".join(lines) + "\n"
|
|
1189
1321
|
|
|
1190
1322
|
|
|
1191
|
-
|
|
1192
|
-
|
|
1323
|
+
HOME_DOWN_WARNING = ("The embeddings home is not running: index and search are lexical only until it answers (start or "
|
|
1324
|
+
"reinstall casa-de-embeddings; it announces itself in ~/.casa-de-embeddings/run/daemon.json).")
|
|
1325
|
+
RERANK_MISSING_WARNING = {
|
|
1326
|
+
"gateway": "Rerank not configured: hybrid lexical/vector order, less precise at the top. Configure a rerank_model.",
|
|
1327
|
+
"home": ("Rerank not configured: hybrid lexical/vector order, less precise at the top. Configure a rerank host in the "
|
|
1328
|
+
"embeddings home."),
|
|
1329
|
+
}
|
|
1193
1330
|
EMPTY_RESULT_NOTICE = ("No matches. Do not fall back to Grep, Read of folders or shell search: retry smart_search with other "
|
|
1194
1331
|
"identifier terms, another group, or a wider top_k, and if it stays empty report that the code was not found.")
|
|
1195
1332
|
LEXICAL_WARNING = ("No embedding model: index and search are lexical only (exact words, no synonyms or cross-language matches). "
|
|
1196
1333
|
"Configure an embedding_model for semantic search.")
|
|
1197
1334
|
|
|
1198
1335
|
|
|
1199
|
-
def _rerank_blocks(
|
|
1336
|
+
def _rerank_blocks(rerank, groups, queries, found, vectors, deadline=None):
|
|
1200
1337
|
"""`({group: rows}, {group: error})` of the blocks that have candidates and a query vector. One rerank request goes
|
|
1201
1338
|
out per distinct query, not per block: the candidates of the blocks that share a query travel together and come back
|
|
1202
1339
|
split by block (`index` is the position inside the block's candidates). A reranker scores each document against the
|
|
@@ -1210,7 +1347,7 @@ def _rerank_blocks(model, groups, queries, found, vectors, token, deadline=None)
|
|
|
1210
1347
|
owners = [(group, position) for group in members for position in range(len(found[group]))]
|
|
1211
1348
|
documents = [found[group][position][4] for group, position in owners]
|
|
1212
1349
|
try:
|
|
1213
|
-
rows =
|
|
1350
|
+
rows = rerank(query, documents, len(documents), deadline=deadline)
|
|
1214
1351
|
except Exception as exc:
|
|
1215
1352
|
failed.update({group: _project_error(exc) for group in members})
|
|
1216
1353
|
continue
|
|
@@ -1245,6 +1382,42 @@ def _nested_projects_note(root):
|
|
|
1245
1382
|
f"results; search them with smart_search on their folder: {'; '.join(parts)}{more}.")
|
|
1246
1383
|
|
|
1247
1384
|
|
|
1385
|
+
def _switch_search(root, old, target, texts, token, problem, cfg, job_id):
|
|
1386
|
+
"""`(vectors, search_model, overlay, notice)` for a search of a folder whose index holds `old` while the embedder
|
|
1387
|
+
is now `target` (None while the embeddings home is not running). The old index answers, with the query embedded by
|
|
1388
|
+
its own model when that model can still be asked (a gateway model and the gateway answers), else by keyword; the
|
|
1389
|
+
files changed since are read from disk by keyword. The re-embedding into `target` runs in the background, queued
|
|
1390
|
+
here when it is not; the folder switches to the new index only when it is complete."""
|
|
1391
|
+
vectors, search_model, failure = {}, None, ""
|
|
1392
|
+
if old != indexer.LEXICAL_MODEL and not local_embedder.is_local(old) and not problem:
|
|
1393
|
+
try:
|
|
1394
|
+
vectors = dict(zip(texts, _embed(old, texts, token, deadline=_job_deadline(job_id),
|
|
1395
|
+
cancel_check=lambda: _job_checkpoint(job_id))))
|
|
1396
|
+
search_model = old
|
|
1397
|
+
except ProjectJobCancelled:
|
|
1398
|
+
raise
|
|
1399
|
+
except Exception as exc:
|
|
1400
|
+
failure = " Embedding the query with that model failed. " + _project_error(exc)
|
|
1401
|
+
lag = _index_lag(root, old, cfg, problem, job_id)
|
|
1402
|
+
overlay, stale = _lag_overlay(root, lag) if lag else (None, "")
|
|
1403
|
+
how = "ranked by meaning and keywords" if search_model else "by keyword only"
|
|
1404
|
+
if target is None:
|
|
1405
|
+
notice = (f"The embeddings home is not running: results come from this folder's index built with {old}, {how}. "
|
|
1406
|
+
"Re-embedding with the home starts when it answers.")
|
|
1407
|
+
else:
|
|
1408
|
+
key = project_identity.canonical_root(root)
|
|
1409
|
+
with _JOBS_LOCK:
|
|
1410
|
+
job = dict(_jobs.get(_INDEX_RUNNING.get(key)) or {})
|
|
1411
|
+
queued = "" if key in _INDEX_RUNNING or _index_update_running(root) else \
|
|
1412
|
+
_refresh_in_background(root, {"refresh": True})
|
|
1413
|
+
progress = (f" ({job['processed_files']} of {job['total_files']} files so far)"
|
|
1414
|
+
if job.get("total_files") else "")
|
|
1415
|
+
notice = (f"Re-embedding in progress: this folder's index was built with {old} and is being rebuilt with "
|
|
1416
|
+
f"{target} in the background{progress}. Results come from the old index, {how}; the folder switches "
|
|
1417
|
+
"to the new index when it is complete." + (" " + queued if queued else ""))
|
|
1418
|
+
return vectors, search_model, overlay, " ".join(text for text in (notice + failure, stale) if text)
|
|
1419
|
+
|
|
1420
|
+
|
|
1248
1421
|
def _perform_smart_search_isolated(arguments, job_id):
|
|
1249
1422
|
_update_job(job_id, phase="auth")
|
|
1250
1423
|
problem = _gateway_problem()
|
|
@@ -1255,18 +1428,28 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
1255
1428
|
queries = {"code": arguments["query"], "test": arguments["query"], "doc": arguments["doc_query"]}
|
|
1256
1429
|
texts = list(dict.fromkeys(queries[group] for group in groups))
|
|
1257
1430
|
cfg = config.load_config()
|
|
1258
|
-
|
|
1431
|
+
current = indexer.info(root)
|
|
1432
|
+
model = local_embedder.resolve(cfg, current.get("model_id"))
|
|
1433
|
+
unreachable = problem and model not in (None, indexer.LEXICAL_MODEL) and not local_embedder.is_local(model)
|
|
1434
|
+
old = (current.get("model_id") if current.get("chunks") and current.get("model_id") != model and not unreachable
|
|
1435
|
+
else None)
|
|
1436
|
+
model = indexer.LEXICAL_MODEL if model is None and old is None else model
|
|
1259
1437
|
warnings = []
|
|
1260
1438
|
vectors = {}
|
|
1261
1439
|
overlay = None
|
|
1262
1440
|
search_model = model
|
|
1441
|
+
search_path = None
|
|
1263
1442
|
source = "index"
|
|
1264
1443
|
if problem:
|
|
1265
1444
|
model_client.OFFLINE.set(problem)
|
|
1266
1445
|
token = "" if problem else model_client.get_token()
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1446
|
+
rerank = _reranker(cfg, token)
|
|
1447
|
+
if old is not None:
|
|
1448
|
+
vectors, search_model, overlay, notice = _switch_search(root, old, model, texts, token, problem, cfg, job_id)
|
|
1449
|
+
source = "index+disk" if overlay else source
|
|
1450
|
+
warnings.append(notice)
|
|
1451
|
+
elif model == indexer.LEXICAL_MODEL:
|
|
1452
|
+
warnings.append(HOME_DOWN_WARNING if config.embedder(cfg) == "home" else LEXICAL_WARNING)
|
|
1270
1453
|
try:
|
|
1271
1454
|
if not arguments.get("_existing_index_only"):
|
|
1272
1455
|
_prepare_project_index(arguments, job_id, cfg, token, model=model)
|
|
@@ -1277,6 +1460,11 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
1277
1460
|
raise
|
|
1278
1461
|
warnings.append("Lexical index not updated; results from the last ready index. " + _project_error(exc))
|
|
1279
1462
|
search_model = None
|
|
1463
|
+
elif arguments.get("_during_index") and _index_lag(root, model, cfg, problem, job_id) is None:
|
|
1464
|
+
vectors, search_path, overlay, notice = _partial_search(root, model, texts, token, job_id)
|
|
1465
|
+
search_model = model if vectors else None
|
|
1466
|
+
source = "partial"
|
|
1467
|
+
warnings.append(notice)
|
|
1280
1468
|
elif _cold_index(root, cfg, problem):
|
|
1281
1469
|
overlay, notice = _cold_overlay(root, texts)
|
|
1282
1470
|
search_model = None
|
|
@@ -1306,7 +1494,8 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
1306
1494
|
if lag is not None:
|
|
1307
1495
|
overlay, notice = _lag_overlay(root, lag)
|
|
1308
1496
|
source = "index+disk" if overlay else source
|
|
1309
|
-
warnings += [text for text in (notice,
|
|
1497
|
+
warnings += [text for text in (notice, None if arguments.get("_during_index")
|
|
1498
|
+
else _refresh_in_background(root, lag)) if text]
|
|
1310
1499
|
elif not arguments.get("_existing_index_only"):
|
|
1311
1500
|
try:
|
|
1312
1501
|
_prepare_project_index(arguments, job_id, cfg, token, query_vector=next(iter(vectors.values())),
|
|
@@ -1320,8 +1509,8 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
1320
1509
|
search_model = None
|
|
1321
1510
|
if indexer.info(root).get("model_id") != model:
|
|
1322
1511
|
vectors = {}
|
|
1323
|
-
if vectors and not
|
|
1324
|
-
warnings.append(
|
|
1512
|
+
if vectors and not rerank:
|
|
1513
|
+
warnings.append(RERANK_MISSING_WARNING[config.embedder(cfg)])
|
|
1325
1514
|
_job_checkpoint(job_id)
|
|
1326
1515
|
saved_scope = index_scope.load_scope(root) or {}
|
|
1327
1516
|
profile = index_profile.current(saved_scope.get("profile"))
|
|
@@ -1348,15 +1537,18 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
1348
1537
|
query_terms = _symbol_terms(" ".join(queries[group] for group in groups if group != "doc"))
|
|
1349
1538
|
detail = arguments.get("detail", "compact")
|
|
1350
1539
|
_update_job(job_id, phase="searching")
|
|
1540
|
+
in_scope = _scope_filter(root, arguments)
|
|
1351
1541
|
found = {group: indexer.search(root, vectors.get(queries[group]), top_k=max(limits[group] * 4, candidate_pool),
|
|
1352
1542
|
query=queries[group], model_id=search_model, overlay=overlay,
|
|
1353
|
-
|
|
1543
|
+
path=search_path,
|
|
1544
|
+
include=lambda path, group=group: index_profile.kind(path, profile) == group
|
|
1545
|
+
and (in_scope is None or in_scope(path)))
|
|
1354
1546
|
for group in groups}
|
|
1355
1547
|
|
|
1356
1548
|
reranked, rerank_failed = {}, {}
|
|
1357
|
-
if
|
|
1549
|
+
if rerank:
|
|
1358
1550
|
_update_job(job_id, phase="reranking")
|
|
1359
|
-
reranked, rerank_failed = _rerank_blocks(
|
|
1551
|
+
reranked, rerank_failed = _rerank_blocks(rerank, groups, queries, found, vectors,
|
|
1360
1552
|
deadline=_job_deadline(job_id))
|
|
1361
1553
|
|
|
1362
1554
|
def search_block(group):
|
|
@@ -1389,7 +1581,8 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
1389
1581
|
warnings.append(problem)
|
|
1390
1582
|
_job_checkpoint(job_id)
|
|
1391
1583
|
if not any(blocks.values()):
|
|
1392
|
-
notes.append(EMPTY_RESULT_NOTICE
|
|
1584
|
+
notes.append(EMPTY_RESULT_NOTICE + (" The folder and layer filters of this search may be too narrow."
|
|
1585
|
+
if in_scope else ""))
|
|
1393
1586
|
nested = _nested_projects_note(root)
|
|
1394
1587
|
if nested:
|
|
1395
1588
|
notes.append(nested)
|
|
@@ -1411,8 +1604,8 @@ def _perform_project_index_isolated(arguments, job_id, preview=False):
|
|
|
1411
1604
|
problem = _gateway_problem()
|
|
1412
1605
|
cfg = config.load_config()
|
|
1413
1606
|
if problem:
|
|
1414
|
-
model = local_embedder.resolve(cfg
|
|
1415
|
-
if model
|
|
1607
|
+
model = local_embedder.resolve(cfg, indexer.info(arguments["project_root"]).get("model_id"))
|
|
1608
|
+
if model not in (None, indexer.LEXICAL_MODEL) and not local_embedder.is_local(model):
|
|
1416
1609
|
raise RuntimeError(problem)
|
|
1417
1610
|
model_client.OFFLINE.set(problem)
|
|
1418
1611
|
return _prepare_project_index(arguments, job_id, cfg, "", preview=preview, model=model)
|
|
@@ -1446,9 +1639,19 @@ def _run_project_job_in_view(job_id, arguments, kind):
|
|
|
1446
1639
|
if arguments.get('_requested_view') and arguments['_requested_view'] != index_views.public(view):
|
|
1447
1640
|
raise index_views.ViewChanged('The branch or commit changed while the job was waiting. Start again on the current view.')
|
|
1448
1641
|
_update_job(job_id, view=index_views.public(view))
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1642
|
+
key = project_identity.canonical_root(arguments["project_root"])
|
|
1643
|
+
if kind == "smart_search" and key in _INDEX_RUNNING:
|
|
1644
|
+
result = _perform_smart_search({**arguments, "_existing_index_only": True, "_during_index": True}, job_id)
|
|
1645
|
+
else:
|
|
1646
|
+
with _project_lock(arguments["project_root"]):
|
|
1647
|
+
if kind == "project_index":
|
|
1648
|
+
_INDEX_RUNNING[key] = job_id
|
|
1649
|
+
try:
|
|
1650
|
+
result = (_perform_smart_search(arguments, job_id) if kind == "smart_search"
|
|
1651
|
+
else _perform_project_index(arguments, job_id, preview=kind == "project_preview"))
|
|
1652
|
+
finally:
|
|
1653
|
+
if kind == "project_index":
|
|
1654
|
+
_INDEX_RUNNING.pop(key, None)
|
|
1452
1655
|
_set_job(job_id, status="done", phase="done", result=result, error=None)
|
|
1453
1656
|
current = project_store.get(project_id) or {}
|
|
1454
1657
|
with _JOBS_LOCK:
|
|
@@ -1531,7 +1734,8 @@ def _submit_project_job_locked(kind, arguments):
|
|
|
1531
1734
|
enabled=kind != "project_preview" or project.get("enabled", False))
|
|
1532
1735
|
try:
|
|
1533
1736
|
future = SMART_SEARCH_EXECUTOR.submit(root, _run_project_job, job_id, arguments, kind,
|
|
1534
|
-
priority=INTERACTIVE_PRIORITY if interactive else 0
|
|
1737
|
+
priority=INTERACTIVE_PRIORITY if interactive else 0,
|
|
1738
|
+
lane="search" if interactive else "index")
|
|
1535
1739
|
with _JOBS_LOCK:
|
|
1536
1740
|
if not future.done():
|
|
1537
1741
|
_JOB_FUTURES[job_id] = future
|
|
@@ -1646,8 +1850,9 @@ def _optional_text(arguments, name):
|
|
|
1646
1850
|
|
|
1647
1851
|
|
|
1648
1852
|
_SMART_SEARCH_FIELDS = {"project_root", "query_identifiers", "query_comments", "doc_query",
|
|
1649
|
-
"single_language_reason", "group", "top_k", "detail"}
|
|
1650
|
-
_DOSSIER_FIELDS = {"project_root", "query_identifiers", "query_comments", "doc_query", "mode", "timeout_ms", "max_hits"
|
|
1853
|
+
"single_language_reason", "group", "top_k", "detail", "folder", "layer"}
|
|
1854
|
+
_DOSSIER_FIELDS = {"project_root", "query_identifiers", "query_comments", "doc_query", "mode", "timeout_ms", "max_hits",
|
|
1855
|
+
"history"}
|
|
1651
1856
|
|
|
1652
1857
|
|
|
1653
1858
|
def _reject_kind(message):
|
|
@@ -1671,6 +1876,11 @@ def _handle_smart_search(arguments):
|
|
|
1671
1876
|
if mode != "search":
|
|
1672
1877
|
raise ValueError("mode must be search or dossier.")
|
|
1673
1878
|
arguments = {key: value for key, value in arguments.items() if key != "mode"}
|
|
1879
|
+
want_notes = arguments.pop("decision_notes", False)
|
|
1880
|
+
if not isinstance(want_notes, bool):
|
|
1881
|
+
raise ValueError("decision_notes must be true or false.")
|
|
1882
|
+
notes_text = " ".join(arguments[key] for key in ("query_identifiers", "query_comments", "doc_query")
|
|
1883
|
+
if isinstance(arguments.get(key), str))
|
|
1674
1884
|
try:
|
|
1675
1885
|
arguments = _smart_search_arguments(arguments)
|
|
1676
1886
|
except ValueError as exc:
|
|
@@ -1688,7 +1898,17 @@ def _handle_smart_search(arguments):
|
|
|
1688
1898
|
return _agent_search_answer(_smart_search_job_payload(job_id))
|
|
1689
1899
|
result = future.result()
|
|
1690
1900
|
_mark_notice_delivered(job_id)
|
|
1691
|
-
return result
|
|
1901
|
+
return _with_decision_notes(result, arguments["project_root"], notes_text) if want_notes else result
|
|
1902
|
+
|
|
1903
|
+
|
|
1904
|
+
def _with_decision_notes(result, root, text):
|
|
1905
|
+
"""The finished search answer with a `decision_notes:` line (JSON, which is valid YAML) after it: the hits of the
|
|
1906
|
+
project's decision notes for `text`, commit messages included, and why any of them is missing."""
|
|
1907
|
+
if not isinstance(result, str) or not dossier.focus_terms(text):
|
|
1908
|
+
return result
|
|
1909
|
+
found = dossier.query(root, text, max_hits=scout.DECISION_NOTES_MAX, history=True)
|
|
1910
|
+
notes = {"decision_notes": found["hits"], **({"decision_notes_warnings": found["reasons"]} if found["reasons"] else {})}
|
|
1911
|
+
return result.rstrip("\n") + "\n" + json.dumps(notes, ensure_ascii=False, separators=(",", ":")) + "\n"
|
|
1692
1912
|
|
|
1693
1913
|
|
|
1694
1914
|
def _mark_notice_delivered(job_id):
|
|
@@ -1814,6 +2034,52 @@ def _update_notices():
|
|
|
1814
2034
|
return []
|
|
1815
2035
|
|
|
1816
2036
|
|
|
2037
|
+
def _search_scope(project_root, folder, layer):
|
|
2038
|
+
"""The folder and layer filters of a smart_search, validated.
|
|
2039
|
+
|
|
2040
|
+
Parameters: project_root (str); folder (str, list of str or None) project-relative folders; layer (str or None) a
|
|
2041
|
+
layer of the project's confirmed layers. Returns: tuple (list of folders with forward slashes and no outer
|
|
2042
|
+
slashes, layer name or None). Raises: ValueError for a folder that is absolute or climbs out of the project, a layer
|
|
2043
|
+
the project does not declare and a project without confirmed layers.
|
|
2044
|
+
"""
|
|
2045
|
+
folders = [folder] if isinstance(folder, str) else folder
|
|
2046
|
+
if folders is not None and (not isinstance(folders, list) or not folders
|
|
2047
|
+
or not all(isinstance(item, str) and item.strip() for item in folders)):
|
|
2048
|
+
raise ValueError("folder must be a project-relative path or a list of them.")
|
|
2049
|
+
clean = []
|
|
2050
|
+
for item in folders or []:
|
|
2051
|
+
parts = [part for part in item.strip().replace("\\", "/").split("/") if part not in ("", ".")]
|
|
2052
|
+
if item.strip().startswith(("/", "\\")) or re.match(r"[A-Za-z]:", item.strip()) or ".." in parts:
|
|
2053
|
+
raise ValueError(f"folder {item!r} must be relative to the project root and stay inside it.")
|
|
2054
|
+
clean.append("/".join(parts))
|
|
2055
|
+
if layer is None:
|
|
2056
|
+
return clean, None
|
|
2057
|
+
if not isinstance(layer, str) or not layer.strip():
|
|
2058
|
+
raise ValueError("layer must be the name of a layer of the project.")
|
|
2059
|
+
import layers
|
|
2060
|
+
rule = layers.load(project_root)
|
|
2061
|
+
if rule is None:
|
|
2062
|
+
raise ValueError("layer needs confirmed layers: this project has none (project_manage action=layers).")
|
|
2063
|
+
names = [item["name"] for item in rule.layers]
|
|
2064
|
+
if layer not in names:
|
|
2065
|
+
raise ValueError(f"layer {layer!r} is not declared by this project; its layers are {', '.join(names)}.")
|
|
2066
|
+
return clean, layer
|
|
2067
|
+
|
|
2068
|
+
|
|
2069
|
+
def _scope_filter(root, arguments):
|
|
2070
|
+
"""Predicate over project-relative paths for the folder and layer filters of a search, or None without filters."""
|
|
2071
|
+
folders, layer = arguments.get("folders") or [], arguments.get("layer")
|
|
2072
|
+
if not folders and not layer:
|
|
2073
|
+
return None
|
|
2074
|
+
import layers
|
|
2075
|
+
rule = layers.load(root) if layer else None
|
|
2076
|
+
|
|
2077
|
+
def inside(path):
|
|
2078
|
+
return (not folders or any(not folder or path == folder or path.startswith(folder + "/") for folder in folders)) \
|
|
2079
|
+
and (rule is None or rule.assign(path) == layer)
|
|
2080
|
+
return inside
|
|
2081
|
+
|
|
2082
|
+
|
|
1817
2083
|
def _smart_search_arguments(arguments):
|
|
1818
2084
|
if not isinstance(arguments, dict):
|
|
1819
2085
|
raise ValueError("smart_search requires its arguments as a JSON object")
|
|
@@ -1853,9 +2119,11 @@ def _smart_search_arguments(arguments):
|
|
|
1853
2119
|
detail = arguments.get("detail", "compact")
|
|
1854
2120
|
if detail not in SEARCH_DETAILS:
|
|
1855
2121
|
raise ValueError("detail must be compact or full.")
|
|
2122
|
+
folders, layer = _search_scope(project_root, arguments.get("folder"), arguments.get("layer"))
|
|
1856
2123
|
query = " ".join(part for part in (identifiers, comments) if part) or doc_query
|
|
1857
2124
|
arguments = {"project_root": project_root, "query": query, "top_k": limits, "group": group, "detail": detail,
|
|
1858
2125
|
"doc_query": doc_query or comments or identifiers,
|
|
2126
|
+
**({"folders": folders} if folders else {}), **({"layer": layer} if layer else {}),
|
|
1859
2127
|
**({"single_language_reason": reason} if reason else {})}
|
|
1860
2128
|
session, client = _session_client()
|
|
1861
2129
|
if session is not None and "hook_notice" not in session:
|
|
@@ -2133,8 +2401,7 @@ def _find_duplicates(root, arguments, dismissed):
|
|
|
2133
2401
|
if type(include_tests) is not bool:
|
|
2134
2402
|
raise ValueError("include_tests must be true or false.")
|
|
2135
2403
|
limit = _parse_bounded_int(arguments.get("limit"), default=30, min_v=1, max_v=200, field_name="limit")
|
|
2136
|
-
configured_model = local_embedder.resolve(config.load_config().get("
|
|
2137
|
-
indexer.info(root).get("model_id"))
|
|
2404
|
+
configured_model = local_embedder.resolve(config.load_config(), indexer.info(root).get("model_id"))
|
|
2138
2405
|
if configured_model == indexer.LEXICAL_MODEL:
|
|
2139
2406
|
configured_model = None
|
|
2140
2407
|
view = index_views.describe(root)
|
|
@@ -2211,6 +2478,14 @@ def _project_action(arguments):
|
|
|
2211
2478
|
return _redirect_action(arguments)
|
|
2212
2479
|
if arguments.get("action") == "junk":
|
|
2213
2480
|
return _junk_action(arguments)
|
|
2481
|
+
if arguments.get("action") == "deps":
|
|
2482
|
+
root = arguments.get("project_root")
|
|
2483
|
+
if not isinstance(root, str) or not os.path.isdir(root):
|
|
2484
|
+
raise ValueError("deps needs project_root: the folder whose dependency manifests to check against OSV.")
|
|
2485
|
+
try:
|
|
2486
|
+
return dependency_notice.audit(root)
|
|
2487
|
+
except dependency_notice.DependencyCheckError as exc:
|
|
2488
|
+
raise ValueError(f"dependency check could not run: {exc}") from None
|
|
2214
2489
|
if arguments.get("action") == "setup_audit":
|
|
2215
2490
|
root = arguments.get("project_root")
|
|
2216
2491
|
if not isinstance(root, str) or not os.path.isdir(root):
|
|
@@ -2232,7 +2507,7 @@ def _project_action(arguments):
|
|
|
2232
2507
|
action = arguments.get("action", "list")
|
|
2233
2508
|
if action == "probe":
|
|
2234
2509
|
return _probe_index_models()
|
|
2235
|
-
if action in ('graph','compare','duplicates','duplicates_dismiss','duplicates_restore','docs','layers','affected_tests','pr_impact'):
|
|
2510
|
+
if action in ('graph','compare','duplicates','duplicates_dismiss','duplicates_restore','docs','layers','affected_tests','pr_impact','hotspots','dead_code','import_cycles','comments'):
|
|
2236
2511
|
# Análise somente leitura pode demorar; não prende o lock global de ações.
|
|
2237
2512
|
key=arguments.get('project_id')
|
|
2238
2513
|
if not isinstance(key,str) or not re.fullmatch(r'[a-f0-9]{16}',key):
|
|
@@ -2273,6 +2548,10 @@ def _project_action(arguments):
|
|
|
2273
2548
|
if action=='layers':
|
|
2274
2549
|
import layers
|
|
2275
2550
|
return layers.report(project['root'],arguments.get('layers'),arguments.get('confirm',False),arguments.get('view_id'))
|
|
2551
|
+
if action in ('hotspots','dead_code','import_cycles','comments'):
|
|
2552
|
+
import code_health
|
|
2553
|
+
limit=_parse_bounded_int(arguments.get('limit'),default=25,min_v=1,max_v=200,field_name='limit')
|
|
2554
|
+
return getattr(code_health,action)(project['root'],arguments.get('view_id'),limit)
|
|
2276
2555
|
if action=='pr_impact':
|
|
2277
2556
|
import pr_impact
|
|
2278
2557
|
limit=_parse_bounded_int(arguments.get('limit'),default=30,min_v=1,max_v=200,field_name='limit')
|
|
@@ -2312,22 +2591,27 @@ def _project_action(arguments):
|
|
|
2312
2591
|
return result
|
|
2313
2592
|
|
|
2314
2593
|
|
|
2594
|
+
HOME_RERANK = "embeddings home rerank host"
|
|
2595
|
+
|
|
2596
|
+
|
|
2315
2597
|
def _probe_index_models():
|
|
2316
2598
|
global _GATEWAY_STATUS
|
|
2317
2599
|
with _GATEWAY_LOCK:
|
|
2318
2600
|
cfg = config.load_config()
|
|
2319
|
-
|
|
2601
|
+
gateway = config.embedder(cfg) == "gateway"
|
|
2602
|
+
embedding = local_embedder.resolve(cfg)
|
|
2320
2603
|
embedding = None if embedding == indexer.LEXICAL_MODEL else embedding
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
models = [embedding,
|
|
2604
|
+
home_down = not gateway and not embedding
|
|
2605
|
+
rerank_model = cfg.get("rerank_model") if gateway else HOME_RERANK if local_embedder.rerank_configured() else None
|
|
2606
|
+
models = [embedding, rerank_model]
|
|
2324
2607
|
if _GATEWAY_STATUS.get("models") == models and time.time() - _GATEWAY_STATUS.get("checked_at", 0) < 30:
|
|
2325
2608
|
return {"gateway": dict(_GATEWAY_STATUS), "cached": True}
|
|
2326
2609
|
result = {"checked_at": time.time(), "models": models}
|
|
2327
2610
|
try:
|
|
2328
|
-
|
|
2611
|
+
token = ""
|
|
2612
|
+
if gateway:
|
|
2329
2613
|
_require_valid_credential()
|
|
2330
|
-
|
|
2614
|
+
token = model_client.get_token()
|
|
2331
2615
|
except Exception as exc:
|
|
2332
2616
|
result.update(status="needs_config", error=_project_error(exc))
|
|
2333
2617
|
else:
|
|
@@ -2335,10 +2619,8 @@ def _probe_index_models():
|
|
|
2335
2619
|
if not model:
|
|
2336
2620
|
result[kind] = {"status": "not_configured"}
|
|
2337
2621
|
continue
|
|
2338
|
-
if kind == "embedding" and
|
|
2339
|
-
result[kind] = {"status": "error", "error": "
|
|
2340
|
-
"responding; local indexes have no vector part until it is back.",
|
|
2341
|
-
"elapsed_s": 0.0}
|
|
2622
|
+
if kind == "embedding" and home_down:
|
|
2623
|
+
result[kind] = {"status": "error", "error": HOME_DOWN_WARNING, "elapsed_s": 0.0}
|
|
2342
2624
|
continue
|
|
2343
2625
|
start = time.monotonic()
|
|
2344
2626
|
try:
|
|
@@ -2346,7 +2628,10 @@ def _probe_index_models():
|
|
|
2346
2628
|
vector = _embed(model, ["Smart Tool: indexing diagnostics."], token, deadline=start + 60)[0]
|
|
2347
2629
|
result[kind] = {"status": "ok", "dimensions": len(vector)}
|
|
2348
2630
|
else:
|
|
2349
|
-
|
|
2631
|
+
if model == HOME_RERANK:
|
|
2632
|
+
_home_rerank("indexing", ["Indexing files for search."], 1, deadline=start + 30)
|
|
2633
|
+
else:
|
|
2634
|
+
_rerank(model, "indexing", ["Indexing files for search."], 1, token, deadline=start + 30)
|
|
2350
2635
|
result[kind] = {"status": "ok"}
|
|
2351
2636
|
except Exception as exc:
|
|
2352
2637
|
result[kind] = {"status": "error", "error": _project_error(exc)}
|
|
@@ -2675,14 +2960,14 @@ def _site_memory_correlate(query, records=None, deadline=None, include_content=T
|
|
|
2675
2960
|
if not records:
|
|
2676
2961
|
return set(), []
|
|
2677
2962
|
cfg = config.load_config()
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
if
|
|
2681
|
-
_web_model_warn("Site memory off (sources that helped before are not prioritized):
|
|
2963
|
+
token = model_client.get_token() if config.embedder(cfg) == "gateway" else ""
|
|
2964
|
+
embedding_model, rerank, problem = _web_memory_models(cfg, token)
|
|
2965
|
+
if problem:
|
|
2966
|
+
_web_model_warn(f"Site memory off (sources that helped before are not prioritized): {problem}.")
|
|
2682
2967
|
return set(), []
|
|
2683
|
-
threshold = config.clamp_site_memory_similarity_threshold(cfg.get("site_memory_similarity_threshold"))
|
|
2968
|
+
threshold = (config.clamp_site_memory_similarity_threshold(cfg.get("site_memory_similarity_threshold"))
|
|
2969
|
+
if config.embedder(cfg) == "gateway" else HOME_SITE_MEMORY_SIMILARITY)
|
|
2684
2970
|
topics = list({r["topic"] for r in records})[:SITE_MEMORY_MAX_TOPICS]
|
|
2685
|
-
token = model_client.get_token()
|
|
2686
2971
|
timed = {"deadline": deadline} if deadline is not None else {}
|
|
2687
2972
|
vectors = _embed(embedding_model, [query] + topics, token, **timed)
|
|
2688
2973
|
query_vec, topic_vecs = vectors[0], vectors[1:]
|
|
@@ -2693,7 +2978,7 @@ def _site_memory_correlate(query, records=None, deadline=None, include_content=T
|
|
|
2693
2978
|
if not candidates:
|
|
2694
2979
|
research_metrics.log_site_memory(query, threshold, scored_topics, recalled=0)
|
|
2695
2980
|
return set(), []
|
|
2696
|
-
reranked =
|
|
2981
|
+
reranked = rerank(query, candidates, len(candidates), **timed)
|
|
2697
2982
|
rerank_scores = {}
|
|
2698
2983
|
for item in reranked:
|
|
2699
2984
|
idx = item.get("index")
|
|
@@ -3198,7 +3483,8 @@ def _cache_match(depth, query, scope, cfg, token, deadline):
|
|
|
3198
3483
|
for entry in entries:
|
|
3199
3484
|
if _normalized(entry.get("query")) == normalized:
|
|
3200
3485
|
stored = entry.get("topic_vector") or {}
|
|
3201
|
-
|
|
3486
|
+
model = _web_embedder(cfg)
|
|
3487
|
+
return entry, (stored.get("values") if model and stored.get("model") == model else None)
|
|
3202
3488
|
if freshness.npm_version_only(query) or freshness.npm_version_security_only(query):
|
|
3203
3489
|
package = freshness.npm_package(query)
|
|
3204
3490
|
wants_security = freshness.npm_security_intent(query)
|
|
@@ -3209,12 +3495,11 @@ def _cache_match(depth, query, scope, cfg, token, deadline):
|
|
|
3209
3495
|
return entry, None
|
|
3210
3496
|
if model_client.OFFLINE.get():
|
|
3211
3497
|
return None, None
|
|
3212
|
-
embed_model,
|
|
3498
|
+
embed_model, rerank, problem = _web_memory_models(cfg, token)
|
|
3213
3499
|
match_model = cfg.get("router_model") or cfg.get("research_model")
|
|
3214
|
-
if
|
|
3215
|
-
|
|
3216
|
-
|
|
3217
|
-
_web_model_warn("Reuse of similar searches off (only the same question is reused): missing " + ", ".join(missing) + ".")
|
|
3500
|
+
if problem or not match_model:
|
|
3501
|
+
reasons = [text for text in (problem, "" if match_model else "router_model missing") if text]
|
|
3502
|
+
_web_model_warn("Reuse of similar searches off (only the same question is reused): " + "; ".join(reasons) + ".")
|
|
3218
3503
|
return None, None
|
|
3219
3504
|
entries = [entry for entry in entries
|
|
3220
3505
|
if (entry.get("topic_vector") or {}).get("model") == embed_model]
|
|
@@ -3224,11 +3509,11 @@ def _cache_match(depth, query, scope, cfg, token, deadline):
|
|
|
3224
3509
|
vector = _embed(embed_model, [query], token, deadline=deadline)[0]
|
|
3225
3510
|
scored = sorted(((indexer.cosine_similarity(vector, entry["topic_vector"]["values"]), i)
|
|
3226
3511
|
for i, entry in enumerate(entries)), reverse=True)
|
|
3227
|
-
candidates = [i for score, i in scored[:10] if score >=
|
|
3512
|
+
candidates = [i for score, i in scored[:10] if score >= SIMILAR_SEARCH_MIN_COSINE[config.embedder(cfg)]]
|
|
3228
3513
|
if not candidates:
|
|
3229
3514
|
return None, vector
|
|
3230
3515
|
topics = [str(entries[i].get("query") or "") for i in candidates]
|
|
3231
|
-
ranked =
|
|
3516
|
+
ranked = rerank(query, topics, min(3, len(candidates)), deadline=deadline)
|
|
3232
3517
|
for row in ranked[:3]:
|
|
3233
3518
|
index = row.get("index")
|
|
3234
3519
|
if not isinstance(index, int) or not 0 <= index < len(candidates):
|
|
@@ -3300,7 +3585,7 @@ def _validated_cache_result(depth, query, scope, cfg, token, deadline, requested
|
|
|
3300
3585
|
hit = _revalidated(depth, entry, cfg, deadline)
|
|
3301
3586
|
if hit and topic_vector is not None and not pending and _normalized(entry["query"]) != _normalized(query):
|
|
3302
3587
|
research_cache.put(_cache_namespace(depth, scope), query, entry["result"], entry["classification"],
|
|
3303
|
-
cfg
|
|
3588
|
+
_web_embedder(cfg), topic_vector, origin=entry)
|
|
3304
3589
|
return hit, topic_vector
|
|
3305
3590
|
|
|
3306
3591
|
|
|
@@ -3392,7 +3677,7 @@ def _store_fresh_research(depth, query, scope, result, sources, cfg, topic_vecto
|
|
|
3392
3677
|
classification["npm_advisories"] = (osv_source["osv_advisory_ids"]
|
|
3393
3678
|
if osv_source else None)
|
|
3394
3679
|
namespace = _cache_namespace(depth, scope)
|
|
3395
|
-
embed_model = cfg
|
|
3680
|
+
embed_model = _web_embedder(cfg)
|
|
3396
3681
|
saved_at = research_cache.put(namespace, query, result, classification,
|
|
3397
3682
|
embed_model if topic_vector else None, topic_vector)
|
|
3398
3683
|
if saved_at is not None and not model_client.OFFLINE.get():
|
|
@@ -3421,10 +3706,11 @@ def _complete_research(namespace, query, saved_at, sources, pending, cfg, topic_
|
|
|
3421
3706
|
_log(f"volatility not judged; reuse requires a check: {classification['error']}")
|
|
3422
3707
|
classification.update({key: value for key, value in pending.items()
|
|
3423
3708
|
if key.startswith("npm_") or key == "security_intent"})
|
|
3424
|
-
embed_model = cfg
|
|
3709
|
+
embed_model = _web_embedder(cfg)
|
|
3425
3710
|
if topic_vector is None and embed_model:
|
|
3426
3711
|
try:
|
|
3427
|
-
topic_vector = _embed(embed_model, [query],
|
|
3712
|
+
topic_vector = _embed(embed_model, [query], "" if local_embedder.is_local(embed_model)
|
|
3713
|
+
else model_client.get_token())[0]
|
|
3428
3714
|
except Exception as exc:
|
|
3429
3715
|
_log(f"research vector not generated; reuse by exact text only: {type(exc).__name__}: {exc}")
|
|
3430
3716
|
if not research_cache.complete(namespace, query, saved_at, classification,
|
|
@@ -3998,10 +4284,11 @@ def _render_page(url, deadline):
|
|
|
3998
4284
|
def _web_fetch_embed(texts, deadline):
|
|
3999
4285
|
"""Vectors for the pieces of a long page and the prompt, with the pieces cached per embedding model. Fails loudly
|
|
4000
4286
|
without an embedding model: a long page is never cut back to its start in silence."""
|
|
4001
|
-
model = local_embedder.resolve(config.load_config()
|
|
4287
|
+
model = local_embedder.resolve(config.load_config(), None)
|
|
4002
4288
|
if not model or model == indexer.LEXICAL_MODEL:
|
|
4003
4289
|
raise RuntimeError("This page is longer than 60,000 characters and choosing its relevant parts needs an "
|
|
4004
|
-
"embedding_model, which is not
|
|
4290
|
+
"embedder (the embeddings home running, or the gateway's embedding_model), which is not "
|
|
4291
|
+
"available")
|
|
4005
4292
|
try:
|
|
4006
4293
|
token = model_client.get_token()
|
|
4007
4294
|
probe = _embed(model, texts[-1:], token, deadline=deadline)
|
|
@@ -4257,7 +4544,7 @@ def _session_audit_hook(handler):
|
|
|
4257
4544
|
|
|
4258
4545
|
|
|
4259
4546
|
def _handle_read_document(arguments):
|
|
4260
|
-
"""read_document: facts and a page, slide, sheet or
|
|
4547
|
+
"""read_document: facts and a page, slide, sheet, part or chapter range of a PDF, docx, pptx, xlsx, html, csv or epub as compact text (see
|
|
4261
4548
|
document_read). Fails loudly, with the reason, for a path that is not absolute or readable, an unsupported or
|
|
4262
4549
|
oversized file, a password, a scanned PDF, a corrupt or hostile package and a range outside the document; one metric
|
|
4263
4550
|
row records the kind, the estimated tokens of the whole text and of the answer."""
|
|
@@ -4420,13 +4707,15 @@ _SETUP_SAVES = {
|
|
|
4420
4707
|
"/setup/models": setup_ui.handle_models_save,
|
|
4421
4708
|
"/setup/models/upgrade": setup_ui.handle_upgrade_action,
|
|
4422
4709
|
"/setup/providers": setup_ui.handle_providers_save,
|
|
4423
|
-
"/setup/
|
|
4710
|
+
"/setup/embedder": setup_ui.handle_embedder_save,
|
|
4424
4711
|
"/setup/integration": setup_ui.handle_integration_save,
|
|
4425
4712
|
"/setup/gateway": setup_ui.handle_gateway_save,
|
|
4426
4713
|
"/setup/autostart": setup_ui.handle_autostart_action,
|
|
4714
|
+
"/setup/updates": setup_ui.handle_updates_save,
|
|
4427
4715
|
"/setup/contact": setup_ui.handle_contact_save,
|
|
4428
4716
|
"/setup/language": setup_ui.handle_language_save,
|
|
4429
4717
|
"/setup/audit": setup_ui.handle_audit,
|
|
4718
|
+
"/setup/usage/mute": setup_ui.handle_usage_mute,
|
|
4430
4719
|
}
|
|
4431
4720
|
|
|
4432
4721
|
|
|
@@ -4563,6 +4852,18 @@ class MCPHandler(BaseHTTPRequestHandler):
|
|
|
4563
4852
|
except Exception as exc:
|
|
4564
4853
|
self._write_json(500, {"error": _sanitize_text(str(exc))})
|
|
4565
4854
|
return
|
|
4855
|
+
if self.path == "/setup/usage":
|
|
4856
|
+
try:
|
|
4857
|
+
self._write_json(200, setup_ui.handle_usage_get())
|
|
4858
|
+
except Exception as exc:
|
|
4859
|
+
self._write_json(500, {"error": _sanitize_text(str(exc))})
|
|
4860
|
+
return
|
|
4861
|
+
if self.path == "/setup/web-search/requesters":
|
|
4862
|
+
try:
|
|
4863
|
+
self._write_json(200, setup_ui.handle_web_requesters_get())
|
|
4864
|
+
except Exception as exc:
|
|
4865
|
+
self._write_json(500, {"error": _sanitize_text(str(exc))})
|
|
4866
|
+
return
|
|
4566
4867
|
if self.path in ("/setup/models/upgrade", "/setup/models/health"):
|
|
4567
4868
|
try:
|
|
4568
4869
|
self._write_json(200, setup_ui.handle_upgrade_get() if self.path.endswith("upgrade")
|
|
@@ -4593,9 +4894,9 @@ class MCPHandler(BaseHTTPRequestHandler):
|
|
|
4593
4894
|
except Exception as exc:
|
|
4594
4895
|
self._write_json(500, {"error": _sanitize_text(str(exc))})
|
|
4595
4896
|
return
|
|
4596
|
-
if self.path == "/setup/
|
|
4897
|
+
if self.path == "/setup/embedder":
|
|
4597
4898
|
try:
|
|
4598
|
-
self._write_json(200, setup_ui.
|
|
4899
|
+
self._write_json(200, setup_ui.handle_embedder_get())
|
|
4599
4900
|
except Exception as exc:
|
|
4600
4901
|
self._write_json(500, {"error": _sanitize_text(str(exc))})
|
|
4601
4902
|
return
|
|
@@ -4611,6 +4912,12 @@ class MCPHandler(BaseHTTPRequestHandler):
|
|
|
4611
4912
|
except Exception as exc:
|
|
4612
4913
|
self._write_json(500, {"error": _sanitize_text(str(exc))})
|
|
4613
4914
|
return
|
|
4915
|
+
if self.path == "/setup/updates":
|
|
4916
|
+
try:
|
|
4917
|
+
self._write_json(200, setup_ui.handle_updates_get())
|
|
4918
|
+
except Exception as exc:
|
|
4919
|
+
self._write_json(500, {"error": _sanitize_text(str(exc))})
|
|
4920
|
+
return
|
|
4614
4921
|
self._write_json(404, {"error": "not found"})
|
|
4615
4922
|
|
|
4616
4923
|
def _post_setup(self, save):
|