@allansantos-dev/smart-tool 0.9.79 → 0.9.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +132 -0
- package/README.md +82 -18
- package/browser_control.py +10 -3
- package/browser_service_client.py +3 -32
- package/chromium_flags.py +30 -0
- package/client_hooks.py +10 -8
- package/code_health.py +182 -0
- package/code_impact.py +14 -2
- package/config.py +76 -8
- package/credential_check.py +38 -0
- package/dependency_notice.py +262 -14
- package/document_converter.py +221 -0
- package/document_read.py +100 -25
- package/document_text.py +194 -6
- package/dossier.py +71 -9
- package/embedding_cache.py +32 -8
- package/freshness.py +3 -2
- package/hook_decision.py +289 -18
- package/hooks/pretool_router.py +5 -4
- package/indexer.py +92 -3
- package/install.py +66 -27
- package/install_runtime.py +24 -10
- package/layer_check.py +57 -0
- package/link_check.py +69 -0
- package/local_embedder.py +115 -58
- package/locales/pt-BR.json +122 -18
- package/low_priority.py +41 -0
- package/model_client.py +1 -1
- package/model_health.py +24 -6
- package/notice_action.py +9 -0
- package/package.json +1 -1
- package/pr_comment.py +94 -0
- package/pr_impact.py +292 -8
- package/process_limits.py +108 -0
- package/project_admission.py +17 -15
- package/project_monitor.py +33 -7
- package/project_scheduler.py +12 -4
- package/removed_callers.py +146 -0
- package/scout.py +6 -2
- package/setup_audit.py +34 -1
- package/setup_ui.py +241 -41
- package/smart_tool_daemon.py +403 -91
- package/tray.py +29 -9
- package/update_check.py +18 -0
- package/update_splash.py +2 -2
- package/usage_meter.py +78 -0
- package/version.py +1 -1
- package/web_adapters/browser/crawl4ai_search.py +7 -3
- package/web_document_graph.py +12 -1
- package/web_fetch.py +24 -0
- package/web_search_health.py +31 -0
package/smart_tool_daemon.py
CHANGED
|
@@ -55,6 +55,7 @@ import index_inventory
|
|
|
55
55
|
import usage_meter
|
|
56
56
|
import document_read
|
|
57
57
|
import document_text
|
|
58
|
+
import dependency_notice
|
|
58
59
|
import dossier
|
|
59
60
|
import scout
|
|
60
61
|
import embedding_cache
|
|
@@ -76,6 +77,7 @@ import site_memory
|
|
|
76
77
|
import web_search_adapters
|
|
77
78
|
import browser_service_client
|
|
78
79
|
import local_embedder
|
|
80
|
+
import low_priority
|
|
79
81
|
import web_search_health
|
|
80
82
|
import web_fetch
|
|
81
83
|
import version
|
|
@@ -172,6 +174,12 @@ SITE_MEMORY_MAX_TOPICS = 50
|
|
|
172
174
|
# em até 0,064 e pares úteis a partir de 0,148. 0,12 separa as duas amostras
|
|
173
175
|
# sem endurecer o limiar de embedding (0,35), que é só filtro de candidatos.
|
|
174
176
|
SITE_MEMORY_RERANK_MIN_SCORE = 0.12
|
|
177
|
+
# The cosine scale depends on the embedder: granite (the embeddings home) puts unrelated web queries at 0.48 to 0.73,
|
|
178
|
+
# where text-embedding-3-small puts them at 0.09 to 0.54. On the same 20 near and 20 far pairs
|
|
179
|
+
# (docs/spikes/web_memory_threshold_spike.py), 0.62 and 0.68 keep every near pair and as many far ones as 0.35 and 0.45
|
|
180
|
+
# keep on the gateway model. The configured site memory threshold is the gateway's.
|
|
181
|
+
HOME_SITE_MEMORY_SIMILARITY = 0.62
|
|
182
|
+
SIMILAR_SEARCH_MIN_COSINE = {"gateway": 0.45, "home": 0.68}
|
|
175
183
|
|
|
176
184
|
|
|
177
185
|
def _parse_bounded_int(value, default, min_v, max_v, field_name):
|
|
@@ -204,7 +212,10 @@ SMART_SEARCH_RESULT_WAIT_S = 15
|
|
|
204
212
|
SMART_SEARCH_MAX_PENDING = 8
|
|
205
213
|
BACKGROUND_MAX_PENDING = 8
|
|
206
214
|
INTERACTIVE_PRIORITY = 1
|
|
207
|
-
|
|
215
|
+
# Slowest first-index speed measured on a CPU-only embedder (docs/research/2026-11-16-indexacao-cpu-busca-durante-o-indice.md).
|
|
216
|
+
CPU_INDEX_FILES_PER_MINUTE = 180
|
|
217
|
+
_INDEX_RUNNING = {}
|
|
218
|
+
SMART_SEARCH_EXECUTOR = project_scheduler.ProjectScheduler(workers=3, reserved_lane="search")
|
|
208
219
|
_PROJECT_LOCKS = {}
|
|
209
220
|
_PROJECT_LOCKS_LOCK = threading.Lock()
|
|
210
221
|
_PROJECT_MONITOR = None
|
|
@@ -373,10 +384,10 @@ SERVER_INSTRUCTIONS = (
|
|
|
373
384
|
TOOLS = [
|
|
374
385
|
{
|
|
375
386
|
"name": "project_manage",
|
|
376
|
-
"description": "Manages folders and indexing. Every action takes project_root (the folder) or project_id. list gives a short line per project (a project whose folder is gone shows as unavailable with the reason); status shows storage, scope and jobs, and with a path inside a registered project answers for the deepest one that holds it (resolved_from). graph with symbol (function, Class.method, class name, or path::name when the name repeats) answers what an edit touches: definition, callers with line, calls, tests that reach it through static calls (depth hops, default 3) and files importing it; use it before changing a function to know what to update. affected_tests answers which tests to run after editing: every test file importing a changed file (git diff against base, default HEAD, untracked included) plus changed tests, likeliest failures first, with the command to run them and run_all when config or unanalyzed code changed. pr_impact (base, default HEAD; main or origin/main for a branch) returns a Markdown report of the change for a pull request: the functions the diff touches with their callers, the documentation entries it leaves out of date and the tests to run with their command; it publishes nothing. graph without symbol reads the indexed view: coverage, references and static calls in Java, Angular, Python, Go and JS/TS/JSX/TSX; includes HTML/CSS/Markdown references and TXT/DOCX coverage; with file_path, narrows to that file's symbols, imports and calls (lists capped by limit). inspect lists the files of a view_id; with file_path, returns only that file's chunks. usage shows consumption and reuse; docs lists public functions without a docstring (Python docstring, JSDoc, Javadoc, Go doc comment starting with the name; private, nested and override functions excluded) with coverage percent, excluding tests unless include_tests=true, for documenting a project that started without it; docs generate=true writes the documentation scope to docs/scope in the project in three levels tied to the code: technical pages by folder (an entry per public function anchored on path::name with signature, first doc line, layer, callers, calls and tests, plus a Mermaid flow by layer), functional.md (each entry point and the functions its flow reaches, by layer) and simple.md; the prose between the prose markers is kept by regeneration; docs write_prose=true writes the empty prose blocks with the Docs model of the setup (each paragraph checked against the code first) within about 40 s and says how many are left, call it again until none is; docs check=true reports anchors whose function changed signature or is gone and broken links; docs validate=true checks every claim of docs/scope against the code (anchors, links, line numbers, callers and tests, coverage, entry points, command-line options, prose names, counts, glossary, navigation, versions) and writes docs/scope/validation.md with pass/fail per check, the evidence, a score and the verdict (generate runs it too); docs with symbol lists where that function is documented; layers declares the project's layers (ordered highest first, each with path globs and annotations): without layers it returns the saved rule or a suggestion read from folder names and annotations, with files, symbols and calls between layers (direction down or up); pass layers to preview a rule and the same layers with confirm=true to save it in the project (.smart-tool/layers.json); duplicates lists duplicated functions (identical, near-identical and semantic bodies, min_similarity default 0.90; below that most pairs measured were false positives), excluding tests unless include_tests=true; fix real duplicates; only dismiss with duplicates_dismiss (finding_id, reason=false_positive|intentional, note) what is not a duplicate or is a copy kept on purpose: the finding stays hidden until the code changes, and duplicates_restore reopens it; integration shows the MCP client, hook and tool usage metrics; install_hook_preview shows the change that would install the hook redirecting Grep/Read to smart_search and WebSearch/WebFetch to web_search/web_fetch in this session's client and returns a plan_id; install_hook writes it with that plan_id and confirm=true, with a backup, only after the user approves the change; search_limits saves the project's default top_k per block ({code, test, doc}, each 1-20, total at most 20); storage shows disk usage and views; pin_view protects a view; each view carries stale (not updated for 30 days, the test of the daily cleanup that removes them) and stale_bytes sums them; cleanup_preview and cleanup_commit remove selected views after confirmation; compare compares two stored views, without checkout or embeddings. register adds a folder; preview checks the scope; index/rebuild update; pause/cancel keep checkpoints; resume continues; policy picks on_search/eager; scope adjusts the scope; remove deletes only the local index. setup_audit (project_root) reports what Claude Code and Codex actually load for that folder (settings layers, plugins, hooks, MCP servers, skills, CLAUDE.md/AGENTS.md) and where they collide: hooks deciding on the same tool in parallel, missing hook scripts, silenced hooks, MCP servers down or offering duplicate tools, skill name and budget problems, AGENTS.md that only Codex reads, broken @imports; what could not be checked is listed as not verified; each finding carries its fix (the file to edit or the command to run), applied by the agent when the user agrees, never by Smart Tool; mode=last returns the stored report instead of auditing again. redirect_proposals (no project_root) lists the redirect patterns proposed from allowed searches, most matched first, with a guide; redirect_decide decides them by ids: accept, instruct (tip without blocking), reject, or remove an accepted one. junk (no project_root) lists the registered projects that were never projects: a personal, system, application-data or build folder, a subfolder of a repository or of another project, a folder of projects, or a folder that no longer exists (on a drive or share that is mounted); with confirm=true it unregisters them and deletes only their index files, and the manifest says what went. A project is the folder where the session started, so smart_search and register refuse a folder that is not a project and index a new one only from the session that started in it. Jobs via smart_search_result.",
|
|
387
|
+
"description": "Manages folders and indexing. Every action takes project_root (the folder) or project_id. list gives a short line per project (a project whose folder is gone shows as unavailable with the reason); status shows storage, scope and jobs, and with a path inside a registered project answers for the deepest one that holds it (resolved_from). deps lists the known OSV advisories of every dependency the manifests of the project declare (package.json, requirements*.txt, pyproject.toml, go.mod), not only the ones an edit changed, at the version a package-lock.json beside a package.json resolved when it satisfies the range; only package names and versions leave the machine. graph with symbol (function, Class.method, class name, or path::name when the name repeats) answers what an edit touches: definition, callers with line, calls, tests that reach it through static calls (depth hops, default 3) and files importing it; use it before changing a function to know what to update. affected_tests answers which tests to run after editing: every test file importing a changed file (git diff against base, default HEAD, untracked included) plus changed tests, likeliest failures first, with the command to run them and run_all when config or unanalyzed code changed. pr_impact (base, default HEAD; main or origin/main for a branch) returns a Markdown report of the change for a pull request: the functions the diff touches with their callers and the tests that reach them (those no test reaches are listed apart), the documentation entries it leaves out of date, the notes the edit hook would have added to the whole diff (comments, copies, dependency advisories) and the tests to run with their command; it publishes nothing. pr_impact_comment (pr, base required) posts that report as a comment on pull request pr through the gh CLI, from the project folder, and returns the comment URL; a rerun replaces the previous Smart Tool comment (marked by a hidden HTML marker) instead of stacking another, and the body carries no local path; it is a write to GitHub, so it refuses unless the user turned pr_comment on in the setup screen (Agent rules, off by default), and when gh is missing or not logged in; never call it without the user asking for the comment. hotspots (limit) ranks the code files by commits times lines, the riskiest to change; dead_code lists the functions no code calls and those only tests call (static call graph, candidates); import_cycles lists the groups of files that import each other; comments lists comments in the wrong place (commented-out code, history tags, loose docs, long inline blocks) across the code files, not only in an edit. graph without symbol reads the indexed view: coverage, references and static calls in Java, Angular, Python, Go and JS/TS/JSX/TSX; includes HTML/CSS/Markdown references and TXT/DOCX coverage; with file_path, narrows to that file's symbols, imports and calls (lists capped by limit). inspect lists the files of a view_id; with file_path, returns only that file's chunks. usage shows consumption and reuse; docs lists public functions without a docstring (Python docstring, JSDoc, Javadoc, Go doc comment starting with the name; private, nested and override functions excluded) with coverage percent, excluding tests unless include_tests=true, for documenting a project that started without it; docs generate=true writes the documentation scope to docs/scope in the project in three levels tied to the code: technical pages by folder (an entry per public function anchored on path::name with signature, first doc line, layer, callers, calls and tests, plus a Mermaid flow by layer), functional.md (each entry point and the functions its flow reaches, by layer) and simple.md; the prose between the prose markers is kept by regeneration; docs write_prose=true writes the empty prose blocks with the Docs model of the setup (each paragraph checked against the code first) within about 40 s and says how many are left, call it again until none is; docs check=true reports anchors whose function changed signature or is gone and broken links; docs validate=true checks every claim of docs/scope against the code (anchors, links, line numbers, callers and tests, coverage, entry points, command-line options, prose names, counts, glossary, navigation, versions) and writes docs/scope/validation.md with pass/fail per check, the evidence, a score and the verdict (generate runs it too); docs with symbol lists where that function is documented; layers declares the project's layers (ordered highest first, each with path globs and annotations): without layers it returns the saved rule or a suggestion read from folder names and annotations, with files, symbols and calls between layers (direction down or up); pass layers to preview a rule and the same layers with confirm=true to save it in the project (.smart-tool/layers.json); duplicates lists duplicated functions (identical, near-identical and semantic bodies, min_similarity default 0.90; below that most pairs measured were false positives), excluding tests unless include_tests=true; fix real duplicates; only dismiss with duplicates_dismiss (finding_id, reason=false_positive|intentional, note) what is not a duplicate or is a copy kept on purpose: the finding stays hidden until the code changes, and duplicates_restore reopens it; integration shows the MCP client, hook and tool usage metrics; install_hook_preview shows the change that would install the hook redirecting Grep/Read to smart_search and WebSearch/WebFetch to web_search/web_fetch in this session's client and returns a plan_id; install_hook writes it with that plan_id and confirm=true, with a backup, only after the user approves the change; search_limits saves the project's default top_k per block ({code, test, doc}, each 1-20, total at most 20); storage shows disk usage and views; pin_view protects a view; each view carries stale (not updated for 30 days, the test of the daily cleanup that removes them) and stale_bytes sums them; cleanup_preview and cleanup_commit remove selected views after confirmation; compare compares two stored views, without checkout or embeddings. register adds a folder; preview checks the scope; index/rebuild update; pause/cancel keep checkpoints; resume continues; policy picks on_search/eager; scope adjusts the scope; remove deletes only the local index. setup_audit (project_root) reports what Claude Code and Codex actually load for that folder (settings layers, plugins, hooks, MCP servers, skills, CLAUDE.md/AGENTS.md) and where they collide: hooks deciding on the same tool in parallel, missing hook scripts, silenced hooks, MCP servers down or offering duplicate tools, skill name and budget problems, AGENTS.md that only Codex reads, broken @imports; what could not be checked is listed as not verified; each finding carries its fix (the file to edit or the command to run), applied by the agent when the user agrees, never by Smart Tool; mode=last returns the stored report instead of auditing again. redirect_proposals (no project_root) lists the redirect patterns proposed from allowed searches, most matched first, with a guide; redirect_decide decides them by ids: accept, instruct (tip without blocking), reject, or remove an accepted one. junk (no project_root) lists the registered projects that were never projects: a personal, system, application-data or build folder, a subfolder of a repository or of another project, a folder of projects, or a folder that no longer exists (on a drive or share that is mounted); with confirm=true it unregisters them and deletes only their index files, and the manifest says what went. A project is the folder where the session started, so smart_search and register refuse a folder that is not a project and index a new one only from the session that started in it. Jobs via smart_search_result.",
|
|
377
388
|
"inputSchema": {
|
|
378
389
|
"type": "object", "properties": {
|
|
379
|
-
"action": {"type": "string", "enum": ["list", "status", "register", "preview", "index", "rebuild", "pause", "cancel", "resume", "watch", "scope", "remove", "relocate", "probe", "policy", "inspect", "graph", "usage", "storage", "pin_view", "cleanup_preview", "cleanup_commit", "compare", "search_limits", "integration", "install_hook_preview", "install_hook", "duplicates", "duplicates_dismiss", "duplicates_restore", "docs", "layers", "affected_tests", "pr_impact", "setup_audit", "redirect_proposals", "redirect_decide", "junk"]},
|
|
390
|
+
"action": {"type": "string", "enum": ["list", "status", "register", "preview", "index", "rebuild", "pause", "cancel", "resume", "watch", "scope", "remove", "relocate", "probe", "policy", "inspect", "graph", "usage", "storage", "pin_view", "cleanup_preview", "cleanup_commit", "compare", "search_limits", "integration", "install_hook_preview", "install_hook", "duplicates", "duplicates_dismiss", "duplicates_restore", "docs", "layers", "affected_tests", "pr_impact", "pr_impact_comment", "hotspots", "dead_code", "import_cycles", "comments", "setup_audit", "redirect_proposals", "redirect_decide", "junk", "deps"]},
|
|
380
391
|
"ids": {"type": "array", "items": {"type": "string"}, "description": "redirect_decide: proposal ids from redirect_proposals (accepted ids for remove)"},
|
|
381
392
|
"decision": {"type": "string", "enum": ["accept", "instruct", "reject", "remove"], "description": "redirect_decide: accept redirects matching calls to smart_search; instruct lets them run with a tip naming smart_search; reject discards; remove deletes an accepted pattern"},
|
|
382
393
|
"finding_id": {"type": "string"},
|
|
@@ -409,6 +420,7 @@ TOOLS = [
|
|
|
409
420
|
"mode": {"type": "string", "enum": ["run", "last"], "default": "run", "description": "setup_audit: run audits now; last reads the stored report (the service audits registered projects every 30 minutes) without running anything, and fails when none is stored"},
|
|
410
421
|
"probe": {"type": "boolean", "default": True, "description": "setup_audit: start/call each MCP server (initialize + tools/list); false reads configs only"},
|
|
411
422
|
"base": {"type": "string", "default": "HEAD", "description": "affected_tests: git revision the changes are compared with (HEAD = uncommitted work; main or origin/main for a branch)"},
|
|
423
|
+
"pr": {"type": "integer", "minimum": 1, "description": "pr_impact_comment: number of the pull request to comment on"},
|
|
412
424
|
"symbol": {"type": "string", "description": "graph: function, Class.method, class name or path::name to get its callers, calls, tests and importers; docs: where that function is documented"},
|
|
413
425
|
"depth": {"type": "integer", "minimum": 1, "maximum": 4, "default": 3, "description": "graph with symbol: call hops searched for tests"},
|
|
414
426
|
"project_root": {"type": "string"}, "project_id": {"type": "string"},
|
|
@@ -424,7 +436,7 @@ TOOLS = [
|
|
|
424
436
|
"name": "smart_search",
|
|
425
437
|
"_meta": ALWAYS_LOAD,
|
|
426
438
|
"description": (
|
|
427
|
-
"
|
|
439
|
+
"Search over the project's index (lexical, plus semantic with an embedding model and reranked with a rerank model, when configured): the first step to understand code in a "
|
|
428
440
|
"project, finding functions, flows and reusable code by meaning, ranked, with name:line. Use it before "
|
|
429
441
|
"a series of Grep/Read; use Grep only for an exact literal or inside a known file. "
|
|
430
442
|
"Results come in three blocks, code, test and doc, plus the project profile with the natural "
|
|
@@ -454,6 +466,10 @@ TOOLS = [
|
|
|
454
466
|
"test": {"type": "integer", "minimum": 1, "maximum": 20},
|
|
455
467
|
"doc": {"type": "integer", "minimum": 1, "maximum": 20}}}],
|
|
456
468
|
"description": "Results per block: one number for all blocks or {code, test, doc}; each 1-20, total at most 20. Defaults to the project's search_limits, else 5"},
|
|
469
|
+
"folder": {"anyOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}, "maxItems": 10}],
|
|
470
|
+
"description": "Only files under this folder of the project (a project-relative path, or a list of them); applied before ranking, so the results are the best inside the folder"},
|
|
471
|
+
"layer": {"type": "string",
|
|
472
|
+
"description": "Only files of this layer, by the path globs of the project's confirmed layers (project_manage action=layers); applied before ranking"},
|
|
457
473
|
"detail": {"type": "string", "enum": ["compact", "full"], "default": "compact",
|
|
458
474
|
"description": "compact: 240-character excerpts, and ranking (when rerank) and location_kind (when source_line) left out of each result; full: 500-character excerpts with every field"},
|
|
459
475
|
"mode": {"type": "string", "enum": ["search", "dossier"], "default": "search",
|
|
@@ -462,6 +478,10 @@ TOOLS = [
|
|
|
462
478
|
"description": "mode=dossier only: scan budget; past it the answer is marked degraded"},
|
|
463
479
|
"max_hits": {"type": "integer", "minimum": 1, "maximum": 10, "default": 5,
|
|
464
480
|
"description": "mode=dossier only: hits returned"},
|
|
481
|
+
"history": {"type": "boolean", "default": False,
|
|
482
|
+
"description": "mode=dossier only: also search the messages of the project's commits that have a body (hits with source git and id git:<hash>); the first read of a repository can take longer than timeout_ms"},
|
|
483
|
+
"decision_notes": {"type": "boolean", "default": False,
|
|
484
|
+
"description": "mode=search only: append the project's decision notes (docs/research, the backlog, CLAUDE.md, AGENTS.md and commit messages) under decision_notes"},
|
|
465
485
|
},
|
|
466
486
|
"required": ["project_root"],
|
|
467
487
|
},
|
|
@@ -573,21 +593,22 @@ TOOLS = [
|
|
|
573
593
|
{
|
|
574
594
|
"name": "read_document",
|
|
575
595
|
"description": (
|
|
576
|
-
"Reads a PDF, Word (.docx), PowerPoint (.pptx)
|
|
577
|
-
"inside Smart Tool so the file never enters your context whole. The answer opens with the facts: type, "
|
|
596
|
+
"Reads a PDF, Word (.docx), PowerPoint (.pptx), Excel (.xlsx), HTML (.html, .htm), CSV (.csv) or EPUB "
|
|
597
|
+
"(.epub) file as compact text, a range at a time, inside Smart Tool so the file never enters your context whole. The answer opens with the facts: type, "
|
|
578
598
|
"size, page, slide, sheet or part count and the estimated tokens of all the text; then comes the text of "
|
|
579
599
|
"the range you asked for (default: from the start, up to max_chars) with a `next:` line naming the range "
|
|
580
600
|
"to continue. PDF pages come with the layout noise removed (ligatures, hyphenation, wrapped lines); docx "
|
|
581
601
|
"as markdown (headings, lists, tables) in parts of about 6,000 characters because a docx has no fixed "
|
|
582
602
|
"pages; pptx per slide with the speaker notes; xlsx per sheet, one tab-separated line per row with the "
|
|
583
|
-
"cached formula values
|
|
603
|
+
"cached formula values; html as markdown in parts like a docx; csv as one sheet, one tab-separated line per row; "
|
|
604
|
+
"epub per chapter in reading order. Use info_only first to size a large file and pick the pages you need. A scanned "
|
|
584
605
|
"PDF (no text layer) fails with that reason; there is no OCR. Not for plain text or code: use Read."
|
|
585
606
|
),
|
|
586
607
|
"inputSchema": {
|
|
587
608
|
"type": "object",
|
|
588
609
|
"properties": {
|
|
589
|
-
"path": {"type": "string", "description": "Absolute path of the .pdf, .docx, .pptx or .
|
|
590
|
-
"range": {"type": "string", "description": "Pages, slides,
|
|
610
|
+
"path": {"type": "string", "description": "Absolute path of the .pdf, .docx, .pptx, .xlsx, .html, .htm, .csv or .epub file"},
|
|
611
|
+
"range": {"type": "string", "description": "Pages, slides, parts, chapters or sheets to read: `3`, `3-5`, `7-` (to the end), `-4` (the first four) or a comma list like `1,4-6`; a sheet name also works for a workbook. Default: from the first one"},
|
|
591
612
|
"max_chars": {"type": "integer", "minimum": 1000, "maximum": document_read.MAX_CHARS_LIMIT, "default": document_read.DEFAULT_MAX_CHARS, "description": "Cap of the text returned (about four characters per token)"},
|
|
592
613
|
"info_only": {"type": "boolean", "default": False, "description": "Return only the facts (type, size, counts, estimated tokens), no text"},
|
|
593
614
|
},
|
|
@@ -804,8 +825,9 @@ def _embed(model, texts, token, deadline=None, on_retry=None, cancel_check=None)
|
|
|
804
825
|
|
|
805
826
|
|
|
806
827
|
def _rerank(model, query, documents, top_n, token, deadline=None):
|
|
807
|
-
"""Rows `{index, relevance_score}` of the best `top_n` documents. Only an HTTP answer that is worth
|
|
808
|
-
(429, 5xx) is asked again, after the wait of `_wait_to_retry`; a timeout fails at once so a search does
|
|
828
|
+
"""Rows `{index, relevance_score}` of the best `top_n` documents from the gateway. Only an HTTP answer that is worth
|
|
829
|
+
repeating (429, 5xx) is asked again, after the wait of `_wait_to_retry`; a timeout fails at once so a search does
|
|
830
|
+
not stall."""
|
|
809
831
|
if not documents:
|
|
810
832
|
return []
|
|
811
833
|
top_n = min(top_n, len(documents))
|
|
@@ -822,6 +844,20 @@ def _rerank(model, query, documents, top_n, token, deadline=None):
|
|
|
822
844
|
if attempt >= EMBED_MAX_ATTEMPTS or not _transient_embed_error(exc):
|
|
823
845
|
raise
|
|
824
846
|
_wait_to_retry(exc, attempt, deadline)
|
|
847
|
+
return _rerank_rows(response, documents, top_n)
|
|
848
|
+
|
|
849
|
+
|
|
850
|
+
def _home_rerank(query, documents, top_n, deadline=None):
|
|
851
|
+
"""`_rerank` through the embeddings home, which forwards to the rerank host configured in it."""
|
|
852
|
+
if not documents:
|
|
853
|
+
return []
|
|
854
|
+
top_n = min(top_n, len(documents))
|
|
855
|
+
return _rerank_rows(local_embedder.rerank(query, documents, top_n, deadline=deadline), documents, top_n)
|
|
856
|
+
|
|
857
|
+
|
|
858
|
+
def _rerank_rows(response, documents, top_n):
|
|
859
|
+
"""The best `top_n` rows of a rerank answer in the Cohere form, highest score first. Scores are opaque: only their
|
|
860
|
+
order within one answer counts. A repeated or out-of-range index, or a missing score, raises."""
|
|
825
861
|
rows = response.get("results", []) if isinstance(response, dict) else []
|
|
826
862
|
if not isinstance(rows, list) or not rows:
|
|
827
863
|
raise ValueError("The reranker returned no valid results.")
|
|
@@ -833,7 +869,46 @@ def _rerank(model, query, documents, top_n, token, deadline=None):
|
|
|
833
869
|
not isinstance(score, (int, float)) or not math.isfinite(score)):
|
|
834
870
|
raise ValueError("The reranker returned an invalid index or score.")
|
|
835
871
|
seen.add(idx)
|
|
836
|
-
return rows[:top_n]
|
|
872
|
+
return sorted(rows, key=lambda row: row["relevance_score"], reverse=True)[:top_n]
|
|
873
|
+
|
|
874
|
+
|
|
875
|
+
def _reranker(cfg, token):
|
|
876
|
+
"""The reranker as `rerank(query, documents, top_n, deadline)`, or None when there is none: the gateway's
|
|
877
|
+
rerank_model when the user chose the gateway, else the embeddings home when a rerank host is configured in it."""
|
|
878
|
+
if config.embedder(cfg) == "gateway":
|
|
879
|
+
model = cfg.get("rerank_model")
|
|
880
|
+
return (lambda query, documents, top_n, deadline=None:
|
|
881
|
+
_rerank(model, query, documents, top_n, token, deadline=deadline)) if model else None
|
|
882
|
+
return _home_rerank if local_embedder.rerank_configured() else None
|
|
883
|
+
|
|
884
|
+
|
|
885
|
+
def _web_embedder(cfg):
|
|
886
|
+
"""Embedding model of the web memory, the same choice as the project indexes (config `embedder`); None when there
|
|
887
|
+
is none: no gateway embedding_model, or the embeddings home not running or unusable."""
|
|
888
|
+
try:
|
|
889
|
+
model = local_embedder.resolve(cfg)
|
|
890
|
+
except local_embedder.IdentityError as exc:
|
|
891
|
+
_log(f"web memory without vectors: {exc}")
|
|
892
|
+
return None
|
|
893
|
+
return None if model in (None, indexer.LEXICAL_MODEL) else model
|
|
894
|
+
|
|
895
|
+
|
|
896
|
+
def _web_memory_models(cfg, token):
|
|
897
|
+
"""`(embedding_model, rerank, problem)` of the web memory (sources that helped before, reuse of similar searches).
|
|
898
|
+
It follows config `embedder` like the project indexes: a stopped home is never replaced by the gateway. `problem`
|
|
899
|
+
says what is missing, empty when both are there."""
|
|
900
|
+
try:
|
|
901
|
+
model = local_embedder.resolve(cfg)
|
|
902
|
+
rerank = _reranker(cfg, token)
|
|
903
|
+
except local_embedder.IdentityError as exc:
|
|
904
|
+
return None, None, str(exc)
|
|
905
|
+
model = None if model in (None, indexer.LEXICAL_MODEL) else model
|
|
906
|
+
if config.embedder(cfg) == "gateway":
|
|
907
|
+
missing = [name for name, value in (("embedding_model", model), ("rerank_model", rerank)) if not value]
|
|
908
|
+
return model, rerank, " and ".join(missing) + " missing" if missing else ""
|
|
909
|
+
if not model:
|
|
910
|
+
return None, rerank, "the embeddings home is not running"
|
|
911
|
+
return model, rerank, "" if rerank else "no rerank host configured in the embeddings home"
|
|
837
912
|
|
|
838
913
|
|
|
839
914
|
LOG_PATH = os.path.join(RUN_DIR, "daemon.log")
|
|
@@ -1018,6 +1093,52 @@ def _cold_overlay(root, queries):
|
|
|
1018
1093
|
return {"drop": [], "chunks": chunks}, warning
|
|
1019
1094
|
|
|
1020
1095
|
|
|
1096
|
+
def _duration(seconds):
|
|
1097
|
+
return f"{round(seconds)} s" if seconds < 90 else f"{-(-round(seconds) // 60)} min"
|
|
1098
|
+
|
|
1099
|
+
|
|
1100
|
+
def _indexing_notice(root, model, indexed, total):
|
|
1101
|
+
"""Where the running index build of `root` stands and how long it should take: the speed so far once files were
|
|
1102
|
+
indexed, else, on a CPU-only embedder, the slowest measured speed as an upper bound."""
|
|
1103
|
+
with _JOBS_LOCK:
|
|
1104
|
+
job = dict(_jobs.get(_INDEX_RUNNING.get(project_identity.canonical_root(root))) or {})
|
|
1105
|
+
done = max(indexed, job.get("processed_files") or 0)
|
|
1106
|
+
total = max(job.get("total_files") or 0, total, done)
|
|
1107
|
+
since = next((at for phase, at in reversed(job.get("phases", [])) if phase == "indexing"), None)
|
|
1108
|
+
left = total - done
|
|
1109
|
+
if done and since and time.time() > since:
|
|
1110
|
+
eta = f"about {_duration(left * (time.time() - since) / done)} left at the speed so far"
|
|
1111
|
+
elif local_embedder.runs_on_cpu(model):
|
|
1112
|
+
eta = f"expected time on this CPU-only embedder: up to about {_duration(left * 60 / CPU_INDEX_FILES_PER_MINUTE)}"
|
|
1113
|
+
else:
|
|
1114
|
+
eta = "time left not known yet"
|
|
1115
|
+
return (f"Indexing in progress: {done} of {total} files indexed ({100 * done // max(total, 1)}%), {eta}. Results "
|
|
1116
|
+
"rank the indexed files by meaning and keywords and read the others from disk by keyword; repeat the "
|
|
1117
|
+
"search later for the full ranking.")
|
|
1118
|
+
|
|
1119
|
+
|
|
1120
|
+
def _partial_search(root, model, texts, token, job_id):
|
|
1121
|
+
"""`(vectors, path, overlay, notice)` for a search made while the index of `root` is being built: the query is
|
|
1122
|
+
embedded and searched in the part of the new index already committed, and the files not committed yet are read
|
|
1123
|
+
from disk by keyword."""
|
|
1124
|
+
path, indexed = indexer.partial_index(root, model)
|
|
1125
|
+
scope = index_scope.load_scope(root)
|
|
1126
|
+
files = index_scope.resolve_included_files(root, scope if index_scope.valid_scope(scope) else {"include": []})
|
|
1127
|
+
notice = _indexing_notice(root, model, len(indexed), len(files))
|
|
1128
|
+
vectors = {}
|
|
1129
|
+
if path:
|
|
1130
|
+
try:
|
|
1131
|
+
vectors = dict(zip(texts, _embed(model, texts, token, deadline=_job_deadline(job_id),
|
|
1132
|
+
cancel_check=lambda: _job_checkpoint(job_id))))
|
|
1133
|
+
except ProjectJobCancelled:
|
|
1134
|
+
raise
|
|
1135
|
+
except Exception as exc:
|
|
1136
|
+
notice += " Embedding unavailable, so the indexed files were searched by keyword only. " + _project_error(exc)
|
|
1137
|
+
chunks, _scanned = indexer.keyword_chunks(root, [rel for rel in files if rel not in indexed], texts,
|
|
1138
|
+
COLD_SCAN_BUDGET_S)
|
|
1139
|
+
return vectors, path, {"drop": [], "chunks": chunks}, notice
|
|
1140
|
+
|
|
1141
|
+
|
|
1021
1142
|
def _lag_overlay(root, lag):
|
|
1022
1143
|
"""`(overlay, warning)` for `indexer.search`: the stale files read from disk, searched by keyword in place of their
|
|
1023
1144
|
outdated chunks, and the notice telling the agent that indexing is in progress. `(None, "")` when the index is
|
|
@@ -1137,31 +1258,43 @@ def _prepare_project_index(arguments, job_id, cfg, token, query_vector=None, pre
|
|
|
1137
1258
|
"sample_paths": included[:30]}
|
|
1138
1259
|
project_store.update(project_identity.project_id(root), preview=result)
|
|
1139
1260
|
return result
|
|
1140
|
-
model = model or local_embedder.resolve(cfg
|
|
1261
|
+
model = model or local_embedder.resolve(cfg, indexer.info(root).get("model_id"))
|
|
1262
|
+
if model is None:
|
|
1263
|
+
raise RuntimeError(f"The embeddings home is not running: this folder's index holds "
|
|
1264
|
+
f"{indexer.info(root).get('model_id')} and keeps answering; re-embedding starts when the "
|
|
1265
|
+
"home answers.")
|
|
1141
1266
|
lexical = model == indexer.LEXICAL_MODEL
|
|
1142
1267
|
current = indexer.info(root)
|
|
1143
1268
|
known_dimensions = current.get("dimensions") if current.get("model_id") == model else None
|
|
1269
|
+
gentle = not lexical and local_embedder.runs_on_cpu(model)
|
|
1144
1270
|
if not lexical and query_vector is None and not known_dimensions:
|
|
1145
1271
|
_update_job(job_id, phase="model_probe")
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1272
|
+
with low_priority.background() if gentle else contextlib.nullcontext():
|
|
1273
|
+
query_vector = _embed(model, ["Smart Tool: indexing dimension check."], token,
|
|
1274
|
+
deadline=_job_deadline(job_id), cancel_check=checkpoint,
|
|
1275
|
+
on_retry=lambda attempt, items: _update_job(job_id, phase="indexing_retry",
|
|
1276
|
+
retry_attempt=attempt, batch_items=items))[0]
|
|
1150
1277
|
checkpoint()
|
|
1151
1278
|
_update_job(job_id, phase="indexing")
|
|
1152
1279
|
full_check = bool(arguments.get("full_check") or project.get("needs_full_check") or
|
|
1153
1280
|
time.time() - current.get("full_scan_at", 0) >= project_monitor.FULL_CHECK_S)
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1281
|
+
|
|
1282
|
+
def embed_batch(texts):
|
|
1283
|
+
with low_priority.background() if gentle else contextlib.nullcontext():
|
|
1284
|
+
return _embed(model, texts, token, deadline=_job_deadline(job_id), cancel_check=checkpoint,
|
|
1285
|
+
on_retry=lambda attempt, items: _update_job(
|
|
1286
|
+
job_id, phase="indexing_retry", retry_attempt=attempt, batch_items=items))
|
|
1287
|
+
|
|
1288
|
+
with low_priority.background() if gentle else contextlib.nullcontext():
|
|
1289
|
+
stats = indexer.reindex(
|
|
1290
|
+
root, included, None if lexical else embed_batch,
|
|
1291
|
+
on_progress=lambda processed, total: _update_job(
|
|
1292
|
+
job_id, phase="indexing", processed_files=processed, total_files=total),
|
|
1293
|
+
model_id=model, expected_dimensions=None if lexical else known_dimensions or len(query_vector),
|
|
1294
|
+
force_rebuild=bool(arguments.get("force_rebuild")), full_check=full_check,
|
|
1295
|
+
changed_paths=set(arguments.get("changed_paths") or []) | set(project.get('dirty_paths', [])),
|
|
1296
|
+
cancel_check=checkpoint,
|
|
1297
|
+
)
|
|
1165
1298
|
_update_job(job_id, stats=stats, index_updated=stats.get("ready", True))
|
|
1166
1299
|
project_store.update(project_identity.project_id(root), index=indexer.info(root),
|
|
1167
1300
|
last_stats=stats, last_checked=time.time(), scope_summary={
|
|
@@ -1188,15 +1321,20 @@ def _grouped_yaml(profile, blocks, warning):
|
|
|
1188
1321
|
return "\n".join(lines) + "\n"
|
|
1189
1322
|
|
|
1190
1323
|
|
|
1191
|
-
|
|
1192
|
-
|
|
1324
|
+
HOME_DOWN_WARNING = ("The embeddings home is not running: index and search are lexical only until it answers (start or "
|
|
1325
|
+
"reinstall casa-de-embeddings; it announces itself in ~/.casa-de-embeddings/run/daemon.json).")
|
|
1326
|
+
RERANK_MISSING_WARNING = {
|
|
1327
|
+
"gateway": "Rerank not configured: hybrid lexical/vector order, less precise at the top. Configure a rerank_model.",
|
|
1328
|
+
"home": ("Rerank not configured: hybrid lexical/vector order, less precise at the top. Configure a rerank host in the "
|
|
1329
|
+
"embeddings home."),
|
|
1330
|
+
}
|
|
1193
1331
|
EMPTY_RESULT_NOTICE = ("No matches. Do not fall back to Grep, Read of folders or shell search: retry smart_search with other "
|
|
1194
1332
|
"identifier terms, another group, or a wider top_k, and if it stays empty report that the code was not found.")
|
|
1195
1333
|
LEXICAL_WARNING = ("No embedding model: index and search are lexical only (exact words, no synonyms or cross-language matches). "
|
|
1196
1334
|
"Configure an embedding_model for semantic search.")
|
|
1197
1335
|
|
|
1198
1336
|
|
|
1199
|
-
def _rerank_blocks(
|
|
1337
|
+
def _rerank_blocks(rerank, groups, queries, found, vectors, deadline=None):
|
|
1200
1338
|
"""`({group: rows}, {group: error})` of the blocks that have candidates and a query vector. One rerank request goes
|
|
1201
1339
|
out per distinct query, not per block: the candidates of the blocks that share a query travel together and come back
|
|
1202
1340
|
split by block (`index` is the position inside the block's candidates). A reranker scores each document against the
|
|
@@ -1210,7 +1348,7 @@ def _rerank_blocks(model, groups, queries, found, vectors, token, deadline=None)
|
|
|
1210
1348
|
owners = [(group, position) for group in members for position in range(len(found[group]))]
|
|
1211
1349
|
documents = [found[group][position][4] for group, position in owners]
|
|
1212
1350
|
try:
|
|
1213
|
-
rows =
|
|
1351
|
+
rows = rerank(query, documents, len(documents), deadline=deadline)
|
|
1214
1352
|
except Exception as exc:
|
|
1215
1353
|
failed.update({group: _project_error(exc) for group in members})
|
|
1216
1354
|
continue
|
|
@@ -1245,6 +1383,42 @@ def _nested_projects_note(root):
|
|
|
1245
1383
|
f"results; search them with smart_search on their folder: {'; '.join(parts)}{more}.")
|
|
1246
1384
|
|
|
1247
1385
|
|
|
1386
|
+
def _switch_search(root, old, target, texts, token, problem, cfg, job_id):
|
|
1387
|
+
"""`(vectors, search_model, overlay, notice)` for a search of a folder whose index holds `old` while the embedder
|
|
1388
|
+
is now `target` (None while the embeddings home is not running). The old index answers, with the query embedded by
|
|
1389
|
+
its own model when that model can still be asked (a gateway model and the gateway answers), else by keyword; the
|
|
1390
|
+
files changed since are read from disk by keyword. The re-embedding into `target` runs in the background, queued
|
|
1391
|
+
here when it is not; the folder switches to the new index only when it is complete."""
|
|
1392
|
+
vectors, search_model, failure = {}, None, ""
|
|
1393
|
+
if old != indexer.LEXICAL_MODEL and not local_embedder.is_local(old) and not problem:
|
|
1394
|
+
try:
|
|
1395
|
+
vectors = dict(zip(texts, _embed(old, texts, token, deadline=_job_deadline(job_id),
|
|
1396
|
+
cancel_check=lambda: _job_checkpoint(job_id))))
|
|
1397
|
+
search_model = old
|
|
1398
|
+
except ProjectJobCancelled:
|
|
1399
|
+
raise
|
|
1400
|
+
except Exception as exc:
|
|
1401
|
+
failure = " Embedding the query with that model failed. " + _project_error(exc)
|
|
1402
|
+
lag = _index_lag(root, old, cfg, problem, job_id)
|
|
1403
|
+
overlay, stale = _lag_overlay(root, lag) if lag else (None, "")
|
|
1404
|
+
how = "ranked by meaning and keywords" if search_model else "by keyword only"
|
|
1405
|
+
if target is None:
|
|
1406
|
+
notice = (f"The embeddings home is not running: results come from this folder's index built with {old}, {how}. "
|
|
1407
|
+
"Re-embedding with the home starts when it answers.")
|
|
1408
|
+
else:
|
|
1409
|
+
key = project_identity.canonical_root(root)
|
|
1410
|
+
with _JOBS_LOCK:
|
|
1411
|
+
job = dict(_jobs.get(_INDEX_RUNNING.get(key)) or {})
|
|
1412
|
+
queued = "" if key in _INDEX_RUNNING or _index_update_running(root) else \
|
|
1413
|
+
_refresh_in_background(root, {"refresh": True})
|
|
1414
|
+
progress = (f" ({job['processed_files']} of {job['total_files']} files so far)"
|
|
1415
|
+
if job.get("total_files") else "")
|
|
1416
|
+
notice = (f"Re-embedding in progress: this folder's index was built with {old} and is being rebuilt with "
|
|
1417
|
+
f"{target} in the background{progress}. Results come from the old index, {how}; the folder switches "
|
|
1418
|
+
"to the new index when it is complete." + (" " + queued if queued else ""))
|
|
1419
|
+
return vectors, search_model, overlay, " ".join(text for text in (notice + failure, stale) if text)
|
|
1420
|
+
|
|
1421
|
+
|
|
1248
1422
|
def _perform_smart_search_isolated(arguments, job_id):
|
|
1249
1423
|
_update_job(job_id, phase="auth")
|
|
1250
1424
|
problem = _gateway_problem()
|
|
@@ -1255,18 +1429,28 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
1255
1429
|
queries = {"code": arguments["query"], "test": arguments["query"], "doc": arguments["doc_query"]}
|
|
1256
1430
|
texts = list(dict.fromkeys(queries[group] for group in groups))
|
|
1257
1431
|
cfg = config.load_config()
|
|
1258
|
-
|
|
1432
|
+
current = indexer.info(root)
|
|
1433
|
+
model = local_embedder.resolve(cfg, current.get("model_id"))
|
|
1434
|
+
unreachable = problem and model not in (None, indexer.LEXICAL_MODEL) and not local_embedder.is_local(model)
|
|
1435
|
+
old = (current.get("model_id") if current.get("chunks") and current.get("model_id") != model and not unreachable
|
|
1436
|
+
else None)
|
|
1437
|
+
model = indexer.LEXICAL_MODEL if model is None and old is None else model
|
|
1259
1438
|
warnings = []
|
|
1260
1439
|
vectors = {}
|
|
1261
1440
|
overlay = None
|
|
1262
1441
|
search_model = model
|
|
1442
|
+
search_path = None
|
|
1263
1443
|
source = "index"
|
|
1264
1444
|
if problem:
|
|
1265
1445
|
model_client.OFFLINE.set(problem)
|
|
1266
1446
|
token = "" if problem else model_client.get_token()
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1447
|
+
rerank = _reranker(cfg, token)
|
|
1448
|
+
if old is not None:
|
|
1449
|
+
vectors, search_model, overlay, notice = _switch_search(root, old, model, texts, token, problem, cfg, job_id)
|
|
1450
|
+
source = "index+disk" if overlay else source
|
|
1451
|
+
warnings.append(notice)
|
|
1452
|
+
elif model == indexer.LEXICAL_MODEL:
|
|
1453
|
+
warnings.append(HOME_DOWN_WARNING if config.embedder(cfg) == "home" else LEXICAL_WARNING)
|
|
1270
1454
|
try:
|
|
1271
1455
|
if not arguments.get("_existing_index_only"):
|
|
1272
1456
|
_prepare_project_index(arguments, job_id, cfg, token, model=model)
|
|
@@ -1277,6 +1461,11 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
1277
1461
|
raise
|
|
1278
1462
|
warnings.append("Lexical index not updated; results from the last ready index. " + _project_error(exc))
|
|
1279
1463
|
search_model = None
|
|
1464
|
+
elif arguments.get("_during_index") and _index_lag(root, model, cfg, problem, job_id) is None:
|
|
1465
|
+
vectors, search_path, overlay, notice = _partial_search(root, model, texts, token, job_id)
|
|
1466
|
+
search_model = model if vectors else None
|
|
1467
|
+
source = "partial"
|
|
1468
|
+
warnings.append(notice)
|
|
1280
1469
|
elif _cold_index(root, cfg, problem):
|
|
1281
1470
|
overlay, notice = _cold_overlay(root, texts)
|
|
1282
1471
|
search_model = None
|
|
@@ -1306,7 +1495,8 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
1306
1495
|
if lag is not None:
|
|
1307
1496
|
overlay, notice = _lag_overlay(root, lag)
|
|
1308
1497
|
source = "index+disk" if overlay else source
|
|
1309
|
-
warnings += [text for text in (notice,
|
|
1498
|
+
warnings += [text for text in (notice, None if arguments.get("_during_index")
|
|
1499
|
+
else _refresh_in_background(root, lag)) if text]
|
|
1310
1500
|
elif not arguments.get("_existing_index_only"):
|
|
1311
1501
|
try:
|
|
1312
1502
|
_prepare_project_index(arguments, job_id, cfg, token, query_vector=next(iter(vectors.values())),
|
|
@@ -1320,8 +1510,8 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
1320
1510
|
search_model = None
|
|
1321
1511
|
if indexer.info(root).get("model_id") != model:
|
|
1322
1512
|
vectors = {}
|
|
1323
|
-
if vectors and not
|
|
1324
|
-
warnings.append(
|
|
1513
|
+
if vectors and not rerank:
|
|
1514
|
+
warnings.append(RERANK_MISSING_WARNING[config.embedder(cfg)])
|
|
1325
1515
|
_job_checkpoint(job_id)
|
|
1326
1516
|
saved_scope = index_scope.load_scope(root) or {}
|
|
1327
1517
|
profile = index_profile.current(saved_scope.get("profile"))
|
|
@@ -1348,15 +1538,18 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
1348
1538
|
query_terms = _symbol_terms(" ".join(queries[group] for group in groups if group != "doc"))
|
|
1349
1539
|
detail = arguments.get("detail", "compact")
|
|
1350
1540
|
_update_job(job_id, phase="searching")
|
|
1541
|
+
in_scope = _scope_filter(root, arguments)
|
|
1351
1542
|
found = {group: indexer.search(root, vectors.get(queries[group]), top_k=max(limits[group] * 4, candidate_pool),
|
|
1352
1543
|
query=queries[group], model_id=search_model, overlay=overlay,
|
|
1353
|
-
|
|
1544
|
+
path=search_path,
|
|
1545
|
+
include=lambda path, group=group: index_profile.kind(path, profile) == group
|
|
1546
|
+
and (in_scope is None or in_scope(path)))
|
|
1354
1547
|
for group in groups}
|
|
1355
1548
|
|
|
1356
1549
|
reranked, rerank_failed = {}, {}
|
|
1357
|
-
if
|
|
1550
|
+
if rerank:
|
|
1358
1551
|
_update_job(job_id, phase="reranking")
|
|
1359
|
-
reranked, rerank_failed = _rerank_blocks(
|
|
1552
|
+
reranked, rerank_failed = _rerank_blocks(rerank, groups, queries, found, vectors,
|
|
1360
1553
|
deadline=_job_deadline(job_id))
|
|
1361
1554
|
|
|
1362
1555
|
def search_block(group):
|
|
@@ -1389,7 +1582,8 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
1389
1582
|
warnings.append(problem)
|
|
1390
1583
|
_job_checkpoint(job_id)
|
|
1391
1584
|
if not any(blocks.values()):
|
|
1392
|
-
notes.append(EMPTY_RESULT_NOTICE
|
|
1585
|
+
notes.append(EMPTY_RESULT_NOTICE + (" The folder and layer filters of this search may be too narrow."
|
|
1586
|
+
if in_scope else ""))
|
|
1393
1587
|
nested = _nested_projects_note(root)
|
|
1394
1588
|
if nested:
|
|
1395
1589
|
notes.append(nested)
|
|
@@ -1411,8 +1605,8 @@ def _perform_project_index_isolated(arguments, job_id, preview=False):
|
|
|
1411
1605
|
problem = _gateway_problem()
|
|
1412
1606
|
cfg = config.load_config()
|
|
1413
1607
|
if problem:
|
|
1414
|
-
model = local_embedder.resolve(cfg
|
|
1415
|
-
if model
|
|
1608
|
+
model = local_embedder.resolve(cfg, indexer.info(arguments["project_root"]).get("model_id"))
|
|
1609
|
+
if model not in (None, indexer.LEXICAL_MODEL) and not local_embedder.is_local(model):
|
|
1416
1610
|
raise RuntimeError(problem)
|
|
1417
1611
|
model_client.OFFLINE.set(problem)
|
|
1418
1612
|
return _prepare_project_index(arguments, job_id, cfg, "", preview=preview, model=model)
|
|
@@ -1446,9 +1640,19 @@ def _run_project_job_in_view(job_id, arguments, kind):
|
|
|
1446
1640
|
if arguments.get('_requested_view') and arguments['_requested_view'] != index_views.public(view):
|
|
1447
1641
|
raise index_views.ViewChanged('The branch or commit changed while the job was waiting. Start again on the current view.')
|
|
1448
1642
|
_update_job(job_id, view=index_views.public(view))
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1643
|
+
key = project_identity.canonical_root(arguments["project_root"])
|
|
1644
|
+
if kind == "smart_search" and key in _INDEX_RUNNING:
|
|
1645
|
+
result = _perform_smart_search({**arguments, "_existing_index_only": True, "_during_index": True}, job_id)
|
|
1646
|
+
else:
|
|
1647
|
+
with _project_lock(arguments["project_root"]):
|
|
1648
|
+
if kind == "project_index":
|
|
1649
|
+
_INDEX_RUNNING[key] = job_id
|
|
1650
|
+
try:
|
|
1651
|
+
result = (_perform_smart_search(arguments, job_id) if kind == "smart_search"
|
|
1652
|
+
else _perform_project_index(arguments, job_id, preview=kind == "project_preview"))
|
|
1653
|
+
finally:
|
|
1654
|
+
if kind == "project_index":
|
|
1655
|
+
_INDEX_RUNNING.pop(key, None)
|
|
1452
1656
|
_set_job(job_id, status="done", phase="done", result=result, error=None)
|
|
1453
1657
|
current = project_store.get(project_id) or {}
|
|
1454
1658
|
with _JOBS_LOCK:
|
|
@@ -1531,7 +1735,8 @@ def _submit_project_job_locked(kind, arguments):
|
|
|
1531
1735
|
enabled=kind != "project_preview" or project.get("enabled", False))
|
|
1532
1736
|
try:
|
|
1533
1737
|
future = SMART_SEARCH_EXECUTOR.submit(root, _run_project_job, job_id, arguments, kind,
|
|
1534
|
-
priority=INTERACTIVE_PRIORITY if interactive else 0
|
|
1738
|
+
priority=INTERACTIVE_PRIORITY if interactive else 0,
|
|
1739
|
+
lane="search" if interactive else "index")
|
|
1535
1740
|
with _JOBS_LOCK:
|
|
1536
1741
|
if not future.done():
|
|
1537
1742
|
_JOB_FUTURES[job_id] = future
|
|
@@ -1646,8 +1851,9 @@ def _optional_text(arguments, name):
|
|
|
1646
1851
|
|
|
1647
1852
|
|
|
1648
1853
|
_SMART_SEARCH_FIELDS = {"project_root", "query_identifiers", "query_comments", "doc_query",
|
|
1649
|
-
"single_language_reason", "group", "top_k", "detail"}
|
|
1650
|
-
_DOSSIER_FIELDS = {"project_root", "query_identifiers", "query_comments", "doc_query", "mode", "timeout_ms", "max_hits"
|
|
1854
|
+
"single_language_reason", "group", "top_k", "detail", "folder", "layer"}
|
|
1855
|
+
_DOSSIER_FIELDS = {"project_root", "query_identifiers", "query_comments", "doc_query", "mode", "timeout_ms", "max_hits",
|
|
1856
|
+
"history"}
|
|
1651
1857
|
|
|
1652
1858
|
|
|
1653
1859
|
def _reject_kind(message):
|
|
@@ -1671,6 +1877,11 @@ def _handle_smart_search(arguments):
|
|
|
1671
1877
|
if mode != "search":
|
|
1672
1878
|
raise ValueError("mode must be search or dossier.")
|
|
1673
1879
|
arguments = {key: value for key, value in arguments.items() if key != "mode"}
|
|
1880
|
+
want_notes = arguments.pop("decision_notes", False)
|
|
1881
|
+
if not isinstance(want_notes, bool):
|
|
1882
|
+
raise ValueError("decision_notes must be true or false.")
|
|
1883
|
+
notes_text = " ".join(arguments[key] for key in ("query_identifiers", "query_comments", "doc_query")
|
|
1884
|
+
if isinstance(arguments.get(key), str))
|
|
1674
1885
|
try:
|
|
1675
1886
|
arguments = _smart_search_arguments(arguments)
|
|
1676
1887
|
except ValueError as exc:
|
|
@@ -1688,7 +1899,17 @@ def _handle_smart_search(arguments):
|
|
|
1688
1899
|
return _agent_search_answer(_smart_search_job_payload(job_id))
|
|
1689
1900
|
result = future.result()
|
|
1690
1901
|
_mark_notice_delivered(job_id)
|
|
1691
|
-
return result
|
|
1902
|
+
return _with_decision_notes(result, arguments["project_root"], notes_text) if want_notes else result
|
|
1903
|
+
|
|
1904
|
+
|
|
1905
|
+
def _with_decision_notes(result, root, text):
|
|
1906
|
+
"""The finished search answer with a `decision_notes:` line (JSON, which is valid YAML) after it: the hits of the
|
|
1907
|
+
project's decision notes for `text`, commit messages included, and why any of them is missing."""
|
|
1908
|
+
if not isinstance(result, str) or not dossier.focus_terms(text):
|
|
1909
|
+
return result
|
|
1910
|
+
found = dossier.query(root, text, max_hits=scout.DECISION_NOTES_MAX, history=True)
|
|
1911
|
+
notes = {"decision_notes": found["hits"], **({"decision_notes_warnings": found["reasons"]} if found["reasons"] else {})}
|
|
1912
|
+
return result.rstrip("\n") + "\n" + json.dumps(notes, ensure_ascii=False, separators=(",", ":")) + "\n"
|
|
1692
1913
|
|
|
1693
1914
|
|
|
1694
1915
|
def _mark_notice_delivered(job_id):
|
|
@@ -1814,6 +2035,52 @@ def _update_notices():
|
|
|
1814
2035
|
return []
|
|
1815
2036
|
|
|
1816
2037
|
|
|
2038
|
+
def _search_scope(project_root, folder, layer):
|
|
2039
|
+
"""The folder and layer filters of a smart_search, validated.
|
|
2040
|
+
|
|
2041
|
+
Parameters: project_root (str); folder (str, list of str or None) project-relative folders; layer (str or None) a
|
|
2042
|
+
layer of the project's confirmed layers. Returns: tuple (list of folders with forward slashes and no outer
|
|
2043
|
+
slashes, layer name or None). Raises: ValueError for a folder that is absolute or climbs out of the project, a layer
|
|
2044
|
+
the project does not declare and a project without confirmed layers.
|
|
2045
|
+
"""
|
|
2046
|
+
folders = [folder] if isinstance(folder, str) else folder
|
|
2047
|
+
if folders is not None and (not isinstance(folders, list) or not folders
|
|
2048
|
+
or not all(isinstance(item, str) and item.strip() for item in folders)):
|
|
2049
|
+
raise ValueError("folder must be a project-relative path or a list of them.")
|
|
2050
|
+
clean = []
|
|
2051
|
+
for item in folders or []:
|
|
2052
|
+
parts = [part for part in item.strip().replace("\\", "/").split("/") if part not in ("", ".")]
|
|
2053
|
+
if item.strip().startswith(("/", "\\")) or re.match(r"[A-Za-z]:", item.strip()) or ".." in parts:
|
|
2054
|
+
raise ValueError(f"folder {item!r} must be relative to the project root and stay inside it.")
|
|
2055
|
+
clean.append("/".join(parts))
|
|
2056
|
+
if layer is None:
|
|
2057
|
+
return clean, None
|
|
2058
|
+
if not isinstance(layer, str) or not layer.strip():
|
|
2059
|
+
raise ValueError("layer must be the name of a layer of the project.")
|
|
2060
|
+
import layers
|
|
2061
|
+
rule = layers.load(project_root)
|
|
2062
|
+
if rule is None:
|
|
2063
|
+
raise ValueError("layer needs confirmed layers: this project has none (project_manage action=layers).")
|
|
2064
|
+
names = [item["name"] for item in rule.layers]
|
|
2065
|
+
if layer not in names:
|
|
2066
|
+
raise ValueError(f"layer {layer!r} is not declared by this project; its layers are {', '.join(names)}.")
|
|
2067
|
+
return clean, layer
|
|
2068
|
+
|
|
2069
|
+
|
|
2070
|
+
def _scope_filter(root, arguments):
|
|
2071
|
+
"""Predicate over project-relative paths for the folder and layer filters of a search, or None without filters."""
|
|
2072
|
+
folders, layer = arguments.get("folders") or [], arguments.get("layer")
|
|
2073
|
+
if not folders and not layer:
|
|
2074
|
+
return None
|
|
2075
|
+
import layers
|
|
2076
|
+
rule = layers.load(root) if layer else None
|
|
2077
|
+
|
|
2078
|
+
def inside(path):
|
|
2079
|
+
return (not folders or any(not folder or path == folder or path.startswith(folder + "/") for folder in folders)) \
|
|
2080
|
+
and (rule is None or rule.assign(path) == layer)
|
|
2081
|
+
return inside
|
|
2082
|
+
|
|
2083
|
+
|
|
1817
2084
|
def _smart_search_arguments(arguments):
|
|
1818
2085
|
if not isinstance(arguments, dict):
|
|
1819
2086
|
raise ValueError("smart_search requires its arguments as a JSON object")
|
|
@@ -1853,9 +2120,11 @@ def _smart_search_arguments(arguments):
|
|
|
1853
2120
|
detail = arguments.get("detail", "compact")
|
|
1854
2121
|
if detail not in SEARCH_DETAILS:
|
|
1855
2122
|
raise ValueError("detail must be compact or full.")
|
|
2123
|
+
folders, layer = _search_scope(project_root, arguments.get("folder"), arguments.get("layer"))
|
|
1856
2124
|
query = " ".join(part for part in (identifiers, comments) if part) or doc_query
|
|
1857
2125
|
arguments = {"project_root": project_root, "query": query, "top_k": limits, "group": group, "detail": detail,
|
|
1858
2126
|
"doc_query": doc_query or comments or identifiers,
|
|
2127
|
+
**({"folders": folders} if folders else {}), **({"layer": layer} if layer else {}),
|
|
1859
2128
|
**({"single_language_reason": reason} if reason else {})}
|
|
1860
2129
|
session, client = _session_client()
|
|
1861
2130
|
if session is not None and "hook_notice" not in session:
|
|
@@ -2133,8 +2402,7 @@ def _find_duplicates(root, arguments, dismissed):
|
|
|
2133
2402
|
if type(include_tests) is not bool:
|
|
2134
2403
|
raise ValueError("include_tests must be true or false.")
|
|
2135
2404
|
limit = _parse_bounded_int(arguments.get("limit"), default=30, min_v=1, max_v=200, field_name="limit")
|
|
2136
|
-
configured_model = local_embedder.resolve(config.load_config().get("
|
|
2137
|
-
indexer.info(root).get("model_id"))
|
|
2405
|
+
configured_model = local_embedder.resolve(config.load_config(), indexer.info(root).get("model_id"))
|
|
2138
2406
|
if configured_model == indexer.LEXICAL_MODEL:
|
|
2139
2407
|
configured_model = None
|
|
2140
2408
|
view = index_views.describe(root)
|
|
@@ -2211,6 +2479,14 @@ def _project_action(arguments):
|
|
|
2211
2479
|
return _redirect_action(arguments)
|
|
2212
2480
|
if arguments.get("action") == "junk":
|
|
2213
2481
|
return _junk_action(arguments)
|
|
2482
|
+
if arguments.get("action") == "deps":
|
|
2483
|
+
root = arguments.get("project_root")
|
|
2484
|
+
if not isinstance(root, str) or not os.path.isdir(root):
|
|
2485
|
+
raise ValueError("deps needs project_root: the folder whose dependency manifests to check against OSV.")
|
|
2486
|
+
try:
|
|
2487
|
+
return dependency_notice.audit(root)
|
|
2488
|
+
except dependency_notice.DependencyCheckError as exc:
|
|
2489
|
+
raise ValueError(f"dependency check could not run: {exc}") from None
|
|
2214
2490
|
if arguments.get("action") == "setup_audit":
|
|
2215
2491
|
root = arguments.get("project_root")
|
|
2216
2492
|
if not isinstance(root, str) or not os.path.isdir(root):
|
|
@@ -2232,7 +2508,7 @@ def _project_action(arguments):
|
|
|
2232
2508
|
action = arguments.get("action", "list")
|
|
2233
2509
|
if action == "probe":
|
|
2234
2510
|
return _probe_index_models()
|
|
2235
|
-
if action in ('graph','compare','duplicates','duplicates_dismiss','duplicates_restore','docs','layers','affected_tests','pr_impact'):
|
|
2511
|
+
if action in ('graph','compare','duplicates','duplicates_dismiss','duplicates_restore','docs','layers','affected_tests','pr_impact','pr_impact_comment','hotspots','dead_code','import_cycles','comments'):
|
|
2236
2512
|
# Análise somente leitura pode demorar; não prende o lock global de ações.
|
|
2237
2513
|
key=arguments.get('project_id')
|
|
2238
2514
|
if not isinstance(key,str) or not re.fullmatch(r'[a-f0-9]{16}',key):
|
|
@@ -2273,10 +2549,18 @@ def _project_action(arguments):
|
|
|
2273
2549
|
if action=='layers':
|
|
2274
2550
|
import layers
|
|
2275
2551
|
return layers.report(project['root'],arguments.get('layers'),arguments.get('confirm',False),arguments.get('view_id'))
|
|
2552
|
+
if action in ('hotspots','dead_code','import_cycles','comments'):
|
|
2553
|
+
import code_health
|
|
2554
|
+
limit=_parse_bounded_int(arguments.get('limit'),default=25,min_v=1,max_v=200,field_name='limit')
|
|
2555
|
+
return getattr(code_health,action)(project['root'],arguments.get('view_id'),limit)
|
|
2276
2556
|
if action=='pr_impact':
|
|
2277
2557
|
import pr_impact
|
|
2278
2558
|
limit=_parse_bounded_int(arguments.get('limit'),default=30,min_v=1,max_v=200,field_name='limit')
|
|
2279
2559
|
return pr_impact.report(project['root'],arguments.get('base','HEAD'),limit,arguments.get('view_id'))
|
|
2560
|
+
if action=='pr_impact_comment':
|
|
2561
|
+
import pr_comment
|
|
2562
|
+
limit=_parse_bounded_int(arguments.get('limit'),default=30,min_v=1,max_v=200,field_name='limit')
|
|
2563
|
+
return pr_comment.publish(project['root'],arguments.get('pr'),arguments.get('base'),limit,arguments.get('view_id'))
|
|
2280
2564
|
if action=='affected_tests':
|
|
2281
2565
|
import affected_tests
|
|
2282
2566
|
limit=_parse_bounded_int(arguments.get('limit'),default=30,min_v=1,max_v=200,field_name='limit')
|
|
@@ -2312,22 +2596,27 @@ def _project_action(arguments):
|
|
|
2312
2596
|
return result
|
|
2313
2597
|
|
|
2314
2598
|
|
|
2599
|
+
HOME_RERANK = "embeddings home rerank host"
|
|
2600
|
+
|
|
2601
|
+
|
|
2315
2602
|
def _probe_index_models():
|
|
2316
2603
|
global _GATEWAY_STATUS
|
|
2317
2604
|
with _GATEWAY_LOCK:
|
|
2318
2605
|
cfg = config.load_config()
|
|
2319
|
-
|
|
2606
|
+
gateway = config.embedder(cfg) == "gateway"
|
|
2607
|
+
embedding = local_embedder.resolve(cfg)
|
|
2320
2608
|
embedding = None if embedding == indexer.LEXICAL_MODEL else embedding
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
models = [embedding,
|
|
2609
|
+
home_down = not gateway and not embedding
|
|
2610
|
+
rerank_model = cfg.get("rerank_model") if gateway else HOME_RERANK if local_embedder.rerank_configured() else None
|
|
2611
|
+
models = [embedding, rerank_model]
|
|
2324
2612
|
if _GATEWAY_STATUS.get("models") == models and time.time() - _GATEWAY_STATUS.get("checked_at", 0) < 30:
|
|
2325
2613
|
return {"gateway": dict(_GATEWAY_STATUS), "cached": True}
|
|
2326
2614
|
result = {"checked_at": time.time(), "models": models}
|
|
2327
2615
|
try:
|
|
2328
|
-
|
|
2616
|
+
token = ""
|
|
2617
|
+
if gateway:
|
|
2329
2618
|
_require_valid_credential()
|
|
2330
|
-
|
|
2619
|
+
token = model_client.get_token()
|
|
2331
2620
|
except Exception as exc:
|
|
2332
2621
|
result.update(status="needs_config", error=_project_error(exc))
|
|
2333
2622
|
else:
|
|
@@ -2335,10 +2624,8 @@ def _probe_index_models():
|
|
|
2335
2624
|
if not model:
|
|
2336
2625
|
result[kind] = {"status": "not_configured"}
|
|
2337
2626
|
continue
|
|
2338
|
-
if kind == "embedding" and
|
|
2339
|
-
result[kind] = {"status": "error", "error": "
|
|
2340
|
-
"responding; local indexes have no vector part until it is back.",
|
|
2341
|
-
"elapsed_s": 0.0}
|
|
2627
|
+
if kind == "embedding" and home_down:
|
|
2628
|
+
result[kind] = {"status": "error", "error": HOME_DOWN_WARNING, "elapsed_s": 0.0}
|
|
2342
2629
|
continue
|
|
2343
2630
|
start = time.monotonic()
|
|
2344
2631
|
try:
|
|
@@ -2346,7 +2633,10 @@ def _probe_index_models():
|
|
|
2346
2633
|
vector = _embed(model, ["Smart Tool: indexing diagnostics."], token, deadline=start + 60)[0]
|
|
2347
2634
|
result[kind] = {"status": "ok", "dimensions": len(vector)}
|
|
2348
2635
|
else:
|
|
2349
|
-
|
|
2636
|
+
if model == HOME_RERANK:
|
|
2637
|
+
_home_rerank("indexing", ["Indexing files for search."], 1, deadline=start + 30)
|
|
2638
|
+
else:
|
|
2639
|
+
_rerank(model, "indexing", ["Indexing files for search."], 1, token, deadline=start + 30)
|
|
2350
2640
|
result[kind] = {"status": "ok"}
|
|
2351
2641
|
except Exception as exc:
|
|
2352
2642
|
result[kind] = {"status": "error", "error": _project_error(exc)}
|
|
@@ -2675,14 +2965,14 @@ def _site_memory_correlate(query, records=None, deadline=None, include_content=T
|
|
|
2675
2965
|
if not records:
|
|
2676
2966
|
return set(), []
|
|
2677
2967
|
cfg = config.load_config()
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
if
|
|
2681
|
-
_web_model_warn("Site memory off (sources that helped before are not prioritized):
|
|
2968
|
+
token = model_client.get_token() if config.embedder(cfg) == "gateway" else ""
|
|
2969
|
+
embedding_model, rerank, problem = _web_memory_models(cfg, token)
|
|
2970
|
+
if problem:
|
|
2971
|
+
_web_model_warn(f"Site memory off (sources that helped before are not prioritized): {problem}.")
|
|
2682
2972
|
return set(), []
|
|
2683
|
-
threshold = config.clamp_site_memory_similarity_threshold(cfg.get("site_memory_similarity_threshold"))
|
|
2973
|
+
threshold = (config.clamp_site_memory_similarity_threshold(cfg.get("site_memory_similarity_threshold"))
|
|
2974
|
+
if config.embedder(cfg) == "gateway" else HOME_SITE_MEMORY_SIMILARITY)
|
|
2684
2975
|
topics = list({r["topic"] for r in records})[:SITE_MEMORY_MAX_TOPICS]
|
|
2685
|
-
token = model_client.get_token()
|
|
2686
2976
|
timed = {"deadline": deadline} if deadline is not None else {}
|
|
2687
2977
|
vectors = _embed(embedding_model, [query] + topics, token, **timed)
|
|
2688
2978
|
query_vec, topic_vecs = vectors[0], vectors[1:]
|
|
@@ -2693,7 +2983,7 @@ def _site_memory_correlate(query, records=None, deadline=None, include_content=T
|
|
|
2693
2983
|
if not candidates:
|
|
2694
2984
|
research_metrics.log_site_memory(query, threshold, scored_topics, recalled=0)
|
|
2695
2985
|
return set(), []
|
|
2696
|
-
reranked =
|
|
2986
|
+
reranked = rerank(query, candidates, len(candidates), **timed)
|
|
2697
2987
|
rerank_scores = {}
|
|
2698
2988
|
for item in reranked:
|
|
2699
2989
|
idx = item.get("index")
|
|
@@ -3198,7 +3488,8 @@ def _cache_match(depth, query, scope, cfg, token, deadline):
|
|
|
3198
3488
|
for entry in entries:
|
|
3199
3489
|
if _normalized(entry.get("query")) == normalized:
|
|
3200
3490
|
stored = entry.get("topic_vector") or {}
|
|
3201
|
-
|
|
3491
|
+
model = _web_embedder(cfg)
|
|
3492
|
+
return entry, (stored.get("values") if model and stored.get("model") == model else None)
|
|
3202
3493
|
if freshness.npm_version_only(query) or freshness.npm_version_security_only(query):
|
|
3203
3494
|
package = freshness.npm_package(query)
|
|
3204
3495
|
wants_security = freshness.npm_security_intent(query)
|
|
@@ -3209,12 +3500,11 @@ def _cache_match(depth, query, scope, cfg, token, deadline):
|
|
|
3209
3500
|
return entry, None
|
|
3210
3501
|
if model_client.OFFLINE.get():
|
|
3211
3502
|
return None, None
|
|
3212
|
-
embed_model,
|
|
3503
|
+
embed_model, rerank, problem = _web_memory_models(cfg, token)
|
|
3213
3504
|
match_model = cfg.get("router_model") or cfg.get("research_model")
|
|
3214
|
-
if
|
|
3215
|
-
|
|
3216
|
-
|
|
3217
|
-
_web_model_warn("Reuse of similar searches off (only the same question is reused): missing " + ", ".join(missing) + ".")
|
|
3505
|
+
if problem or not match_model:
|
|
3506
|
+
reasons = [text for text in (problem, "" if match_model else "router_model missing") if text]
|
|
3507
|
+
_web_model_warn("Reuse of similar searches off (only the same question is reused): " + "; ".join(reasons) + ".")
|
|
3218
3508
|
return None, None
|
|
3219
3509
|
entries = [entry for entry in entries
|
|
3220
3510
|
if (entry.get("topic_vector") or {}).get("model") == embed_model]
|
|
@@ -3224,11 +3514,11 @@ def _cache_match(depth, query, scope, cfg, token, deadline):
|
|
|
3224
3514
|
vector = _embed(embed_model, [query], token, deadline=deadline)[0]
|
|
3225
3515
|
scored = sorted(((indexer.cosine_similarity(vector, entry["topic_vector"]["values"]), i)
|
|
3226
3516
|
for i, entry in enumerate(entries)), reverse=True)
|
|
3227
|
-
candidates = [i for score, i in scored[:10] if score >=
|
|
3517
|
+
candidates = [i for score, i in scored[:10] if score >= SIMILAR_SEARCH_MIN_COSINE[config.embedder(cfg)]]
|
|
3228
3518
|
if not candidates:
|
|
3229
3519
|
return None, vector
|
|
3230
3520
|
topics = [str(entries[i].get("query") or "") for i in candidates]
|
|
3231
|
-
ranked =
|
|
3521
|
+
ranked = rerank(query, topics, min(3, len(candidates)), deadline=deadline)
|
|
3232
3522
|
for row in ranked[:3]:
|
|
3233
3523
|
index = row.get("index")
|
|
3234
3524
|
if not isinstance(index, int) or not 0 <= index < len(candidates):
|
|
@@ -3300,7 +3590,7 @@ def _validated_cache_result(depth, query, scope, cfg, token, deadline, requested
|
|
|
3300
3590
|
hit = _revalidated(depth, entry, cfg, deadline)
|
|
3301
3591
|
if hit and topic_vector is not None and not pending and _normalized(entry["query"]) != _normalized(query):
|
|
3302
3592
|
research_cache.put(_cache_namespace(depth, scope), query, entry["result"], entry["classification"],
|
|
3303
|
-
cfg
|
|
3593
|
+
_web_embedder(cfg), topic_vector, origin=entry)
|
|
3304
3594
|
return hit, topic_vector
|
|
3305
3595
|
|
|
3306
3596
|
|
|
@@ -3392,7 +3682,7 @@ def _store_fresh_research(depth, query, scope, result, sources, cfg, topic_vecto
|
|
|
3392
3682
|
classification["npm_advisories"] = (osv_source["osv_advisory_ids"]
|
|
3393
3683
|
if osv_source else None)
|
|
3394
3684
|
namespace = _cache_namespace(depth, scope)
|
|
3395
|
-
embed_model = cfg
|
|
3685
|
+
embed_model = _web_embedder(cfg)
|
|
3396
3686
|
saved_at = research_cache.put(namespace, query, result, classification,
|
|
3397
3687
|
embed_model if topic_vector else None, topic_vector)
|
|
3398
3688
|
if saved_at is not None and not model_client.OFFLINE.get():
|
|
@@ -3421,10 +3711,11 @@ def _complete_research(namespace, query, saved_at, sources, pending, cfg, topic_
|
|
|
3421
3711
|
_log(f"volatility not judged; reuse requires a check: {classification['error']}")
|
|
3422
3712
|
classification.update({key: value for key, value in pending.items()
|
|
3423
3713
|
if key.startswith("npm_") or key == "security_intent"})
|
|
3424
|
-
embed_model = cfg
|
|
3714
|
+
embed_model = _web_embedder(cfg)
|
|
3425
3715
|
if topic_vector is None and embed_model:
|
|
3426
3716
|
try:
|
|
3427
|
-
topic_vector = _embed(embed_model, [query],
|
|
3717
|
+
topic_vector = _embed(embed_model, [query], "" if local_embedder.is_local(embed_model)
|
|
3718
|
+
else model_client.get_token())[0]
|
|
3428
3719
|
except Exception as exc:
|
|
3429
3720
|
_log(f"research vector not generated; reuse by exact text only: {type(exc).__name__}: {exc}")
|
|
3430
3721
|
if not research_cache.complete(namespace, query, saved_at, classification,
|
|
@@ -3998,10 +4289,11 @@ def _render_page(url, deadline):
|
|
|
3998
4289
|
def _web_fetch_embed(texts, deadline):
|
|
3999
4290
|
"""Vectors for the pieces of a long page and the prompt, with the pieces cached per embedding model. Fails loudly
|
|
4000
4291
|
without an embedding model: a long page is never cut back to its start in silence."""
|
|
4001
|
-
model = local_embedder.resolve(config.load_config()
|
|
4292
|
+
model = local_embedder.resolve(config.load_config(), None)
|
|
4002
4293
|
if not model or model == indexer.LEXICAL_MODEL:
|
|
4003
4294
|
raise RuntimeError("This page is longer than 60,000 characters and choosing its relevant parts needs an "
|
|
4004
|
-
"embedding_model, which is not
|
|
4295
|
+
"embedder (the embeddings home running, or the gateway's embedding_model), which is not "
|
|
4296
|
+
"available")
|
|
4005
4297
|
try:
|
|
4006
4298
|
token = model_client.get_token()
|
|
4007
4299
|
probe = _embed(model, texts[-1:], token, deadline=deadline)
|
|
@@ -4257,7 +4549,7 @@ def _session_audit_hook(handler):
|
|
|
4257
4549
|
|
|
4258
4550
|
|
|
4259
4551
|
def _handle_read_document(arguments):
|
|
4260
|
-
"""read_document: facts and a page, slide, sheet or
|
|
4552
|
+
"""read_document: facts and a page, slide, sheet, part or chapter range of a PDF, docx, pptx, xlsx, html, csv or epub as compact text (see
|
|
4261
4553
|
document_read). Fails loudly, with the reason, for a path that is not absolute or readable, an unsupported or
|
|
4262
4554
|
oversized file, a password, a scanned PDF, a corrupt or hostile package and a range outside the document; one metric
|
|
4263
4555
|
row records the kind, the estimated tokens of the whole text and of the answer."""
|
|
@@ -4420,13 +4712,15 @@ _SETUP_SAVES = {
|
|
|
4420
4712
|
"/setup/models": setup_ui.handle_models_save,
|
|
4421
4713
|
"/setup/models/upgrade": setup_ui.handle_upgrade_action,
|
|
4422
4714
|
"/setup/providers": setup_ui.handle_providers_save,
|
|
4423
|
-
"/setup/
|
|
4715
|
+
"/setup/embedder": setup_ui.handle_embedder_save,
|
|
4424
4716
|
"/setup/integration": setup_ui.handle_integration_save,
|
|
4425
4717
|
"/setup/gateway": setup_ui.handle_gateway_save,
|
|
4426
4718
|
"/setup/autostart": setup_ui.handle_autostart_action,
|
|
4719
|
+
"/setup/updates": setup_ui.handle_updates_save,
|
|
4427
4720
|
"/setup/contact": setup_ui.handle_contact_save,
|
|
4428
4721
|
"/setup/language": setup_ui.handle_language_save,
|
|
4429
4722
|
"/setup/audit": setup_ui.handle_audit,
|
|
4723
|
+
"/setup/usage/mute": setup_ui.handle_usage_mute,
|
|
4430
4724
|
}
|
|
4431
4725
|
|
|
4432
4726
|
|
|
@@ -4563,6 +4857,18 @@ class MCPHandler(BaseHTTPRequestHandler):
|
|
|
4563
4857
|
except Exception as exc:
|
|
4564
4858
|
self._write_json(500, {"error": _sanitize_text(str(exc))})
|
|
4565
4859
|
return
|
|
4860
|
+
if self.path == "/setup/usage":
|
|
4861
|
+
try:
|
|
4862
|
+
self._write_json(200, setup_ui.handle_usage_get())
|
|
4863
|
+
except Exception as exc:
|
|
4864
|
+
self._write_json(500, {"error": _sanitize_text(str(exc))})
|
|
4865
|
+
return
|
|
4866
|
+
if self.path == "/setup/web-search/requesters":
|
|
4867
|
+
try:
|
|
4868
|
+
self._write_json(200, setup_ui.handle_web_requesters_get())
|
|
4869
|
+
except Exception as exc:
|
|
4870
|
+
self._write_json(500, {"error": _sanitize_text(str(exc))})
|
|
4871
|
+
return
|
|
4566
4872
|
if self.path in ("/setup/models/upgrade", "/setup/models/health"):
|
|
4567
4873
|
try:
|
|
4568
4874
|
self._write_json(200, setup_ui.handle_upgrade_get() if self.path.endswith("upgrade")
|
|
@@ -4593,9 +4899,9 @@ class MCPHandler(BaseHTTPRequestHandler):
|
|
|
4593
4899
|
except Exception as exc:
|
|
4594
4900
|
self._write_json(500, {"error": _sanitize_text(str(exc))})
|
|
4595
4901
|
return
|
|
4596
|
-
if self.path == "/setup/
|
|
4902
|
+
if self.path == "/setup/embedder":
|
|
4597
4903
|
try:
|
|
4598
|
-
self._write_json(200, setup_ui.
|
|
4904
|
+
self._write_json(200, setup_ui.handle_embedder_get())
|
|
4599
4905
|
except Exception as exc:
|
|
4600
4906
|
self._write_json(500, {"error": _sanitize_text(str(exc))})
|
|
4601
4907
|
return
|
|
@@ -4611,6 +4917,12 @@ class MCPHandler(BaseHTTPRequestHandler):
|
|
|
4611
4917
|
except Exception as exc:
|
|
4612
4918
|
self._write_json(500, {"error": _sanitize_text(str(exc))})
|
|
4613
4919
|
return
|
|
4920
|
+
if self.path == "/setup/updates":
|
|
4921
|
+
try:
|
|
4922
|
+
self._write_json(200, setup_ui.handle_updates_get())
|
|
4923
|
+
except Exception as exc:
|
|
4924
|
+
self._write_json(500, {"error": _sanitize_text(str(exc))})
|
|
4925
|
+
return
|
|
4614
4926
|
self._write_json(404, {"error": "not found"})
|
|
4615
4927
|
|
|
4616
4928
|
def _post_setup(self, save):
|