@allansantos-dev/smart-tool 0.9.78 → 0.9.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +79 -0
- package/README.md +65 -18
- package/affected_tests.py +4 -0
- package/browser_control.py +10 -3
- package/browser_service_client.py +3 -32
- package/chromium_flags.py +30 -0
- package/code_graph.py +39 -17
- package/code_graph_go.py +6 -6
- package/code_graph_java.mjs +10 -9
- package/code_graph_js.cjs +8 -8
- package/code_health.py +180 -0
- package/code_impact.py +16 -2
- package/config.py +41 -8
- package/credential_check.py +38 -0
- package/dependency_notice.py +262 -14
- package/doc_scope.py +1 -1
- package/document_converter.py +221 -0
- package/document_read.py +100 -25
- package/document_text.py +194 -6
- package/dossier.py +35 -8
- package/duplicates.py +1 -1
- package/embedding_cache.py +32 -8
- package/freshness.py +3 -2
- package/hook_decision.py +65 -14
- package/indexer.py +92 -3
- package/layer_check.py +57 -0
- package/layers.py +3 -3
- package/local_embedder.py +114 -57
- package/locales/pt-BR.json +52 -17
- package/low_priority.py +41 -0
- package/package.json +1 -1
- package/pr_impact.py +113 -8
- package/process_limits.py +108 -0
- package/project_monitor.py +33 -7
- package/project_scheduler.py +12 -4
- package/scout.py +6 -2
- package/setup_audit.py +34 -1
- package/setup_ui.py +210 -39
- package/smart_tool_daemon.py +398 -91
- package/update_check.py +18 -0
- package/usage_meter.py +78 -0
- package/version.py +1 -1
- package/web_adapters/browser/crawl4ai_search.py +7 -3
- package/web_fetch.py +24 -0
- package/web_search_health.py +31 -0
package/locales/pt-BR.json
CHANGED
|
@@ -61,11 +61,7 @@
|
|
|
61
61
|
"Quick search time limit (s)": "Tempo limite da busca rápida (s)",
|
|
62
62
|
"How long a quick web search waits for its sources.": "Quanto tempo uma busca rápida na web espera pelas suas fontes.",
|
|
63
63
|
"Per-site memory similarity": "Similaridade da memória por site",
|
|
64
|
-
"How close a new question must be to a remembered page of the same site to reuse it.": "Quão parecida uma nova pergunta precisa ser de uma página lembrada do mesmo site para reaproveitá-la.",
|
|
65
64
|
"Save models": "Salvar modelos",
|
|
66
|
-
"3. Local models": "3. Modelos locais",
|
|
67
|
-
"Embedding from the mcp-memory sidecar on this machine. Switching between gateway and local rebuilds the project indexes; going back to a model used before reuses its vector cache.": "Embedding do sidecar mcp-memory neste computador. Alternar entre gateway e local reconstrói os índices dos projetos; voltar a um modelo usado antes reaproveita o cache de vetores.",
|
|
68
|
-
"When to use the local model": "Quando usar o modelo local",
|
|
69
65
|
"web_search asks several providers at once. All of them work without a key on their public free tier; add your own key only if you need more volume.": "web_search consulta vários provedores ao mesmo tempo. Todos funcionam sem chave no plano gratuito público; adicione a sua própria chave só se precisar de mais volume.",
|
|
70
66
|
"Providers": "Provedores",
|
|
71
67
|
"A key is tested with a real search before it is saved, and is protected by DPAPI in this Windows profile.": "Uma chave é testada com uma busca real antes de ser salva e fica protegida por DPAPI neste perfil do Windows.",
|
|
@@ -215,16 +211,6 @@
|
|
|
215
211
|
"Saving…": "Salvando…",
|
|
216
212
|
"Models saved.": "Modelos salvos.",
|
|
217
213
|
"Recommended": "Recomendado",
|
|
218
|
-
"Only as a fallback": "Só como alternativa",
|
|
219
|
-
"Use the local model only when the gateway does not have the embedding model.": "Use o modelo local apenas quando o gateway não tiver o modelo de embedding.",
|
|
220
|
-
"Prefer local": "Preferir local",
|
|
221
|
-
"Use the local model even when the gateway has it; your code vectors stay on this machine.": "Use o modelo local mesmo quando o gateway o tiver; os vetores do seu código ficam neste computador.",
|
|
222
|
-
"Never": "Nunca",
|
|
223
|
-
"Always use the gateway.": "Usar sempre o gateway.",
|
|
224
|
-
"Saved. The indexes adjust on the next search or check.": "Salvo. Os índices se ajustam na próxima busca ou verificação.",
|
|
225
|
-
"No sidecar on this machine": "Nenhum sidecar neste computador",
|
|
226
|
-
"Sidecar answering:": "Sidecar respondendo:",
|
|
227
|
-
"Sidecar not answering (": "Sidecar não responde (",
|
|
228
214
|
"Status unavailable": "Status indisponível",
|
|
229
215
|
"Redirect behavior": "Comportamento do redirecionamento",
|
|
230
216
|
"Applies on the next tool call.": "Vale na próxima chamada de ferramenta.",
|
|
@@ -674,7 +660,6 @@
|
|
|
674
660
|
"per-site memory similarity threshold": "limiar de similaridade da memória por site",
|
|
675
661
|
"Send the adapter and the field values in a JSON object.": "Envie o adaptador e os valores dos campos em um objeto JSON.",
|
|
676
662
|
"Setup page session token missing or invalid; open {0}/setup again.": "O token de sessão da página de configuração está ausente ou é inválido; abra {0}/setup de novo.",
|
|
677
|
-
"Invalid mode: use fallback, off or prefer.": "Modo inválido: use fallback, off ou prefer.",
|
|
678
663
|
"Invalid client: use claude or codex.": "Cliente inválido: use claude ou codex.",
|
|
679
664
|
"Invalid request.": "Requisição inválida.",
|
|
680
665
|
"Send the contact as text.": "Envie o contato como texto.",
|
|
@@ -812,7 +797,6 @@
|
|
|
812
797
|
", {0} with your key.": ", {0} com a sua chave.",
|
|
813
798
|
"{0}{1} your attention.": "{0}{1} da sua atenção.",
|
|
814
799
|
"{0}{1} Protected by DPAPI.": "{0}{1} Protegido pelo DPAPI.",
|
|
815
|
-
"Sidecar not answering ({0})": "Sidecar sem resposta ({0})",
|
|
816
800
|
"Not available on this machine ({0}). Pages are still read by web_fetch.": "Indisponível neste computador ({0}). As páginas continuam sendo lidas pelo web_fetch.",
|
|
817
801
|
"Registered at {0}. Gives the agent smart_search, web_search and web_fetch.": "Registrado em {0}. Dá ao agente smart_search, web_search e web_fetch.",
|
|
818
802
|
"Registering the MCP server in {0}…": "Registrando o servidor MCP no {0}…",
|
|
@@ -910,7 +894,58 @@
|
|
|
910
894
|
"Tip only": "Só dica",
|
|
911
895
|
"Patterns you accepted (": "Padrões que você aceitou (",
|
|
912
896
|
"Patterns you accepted ({0})": "Padrões que você aceitou ({0})",
|
|
913
|
-
"{0} holds a setup audit of {1} from an older Smart Tool, whose findings have no fix: run project_manage action=setup_audit mode=run to replace it.": "{0} guarda uma auditoria da configuração de {1} feita por um Smart Tool mais antigo, cujos achados não trazem a correção: rode project_manage action=setup_audit mode=run para substituí-la."
|
|
897
|
+
"{0} holds a setup audit of {1} from an older Smart Tool, whose findings have no fix: run project_manage action=setup_audit mode=run to replace it.": "{0} guarda uma auditoria da configuração de {1} feita por um Smart Tool mais antigo, cujos achados não trazem a correção: rode project_manage action=setup_audit mode=run para substituí-la.",
|
|
898
|
+
"The embeddings home (casa-de-embeddings) is the embedding service of this machine, shared with other tools; its rerank connector forwards to the rerank host configured in it. Switching the embedder re-embeds each project in the background: the old index keeps answering until the new one is complete, then the old vectors are deleted.": "A casa de embeddings (casa-de-embeddings) é o serviço de embeddings desta máquina, compartilhado com outras ferramentas; o conector de rerank dela repassa ao host de rerank configurado nela. Trocar o embedder reembute cada projeto em segundo plano: o índice antigo continua respondendo até o novo ficar completo, e depois os vetores antigos são apagados.",
|
|
899
|
+
"Embedder for the project indexes": "Embedder dos índices dos projetos",
|
|
900
|
+
"Embeddings home": "Casa de embeddings",
|
|
901
|
+
"The embedding service of this machine; your code goes to the address it announces, not to the gateway.": "O serviço de embeddings desta máquina; seu código vai para o endereço que ela anuncia, não para o gateway.",
|
|
902
|
+
"The gateway's embedding_model and rerank_model; your code goes to the gateway.": "O embedding_model e o rerank_model do gateway; seu código vai para o gateway.",
|
|
903
|
+
"Saved. Each project is re-embedded in the background; its old index answers meanwhile.": "Salvo. Cada projeto é reembutido em segundo plano; o índice antigo responde enquanto isso.",
|
|
904
|
+
"Embeddings home not running": "Casa de embeddings não está rodando",
|
|
905
|
+
"Embeddings home not answering (": "Casa de embeddings não responde (",
|
|
906
|
+
"Gateway": "Gateway",
|
|
907
|
+
"Embeddings home answering:": "Casa de embeddings respondendo:",
|
|
908
|
+
"· rerank host configured": "· host de rerank configurado",
|
|
909
|
+
"Embeddings home not answering ({0})": "Casa de embeddings sem resposta ({0})",
|
|
910
|
+
"Invalid embedder: use home or gateway.": "Embedder inválido: use home ou gateway.",
|
|
911
|
+
"How close a new question must be to a remembered page of the same site to reuse it, with the gateway embedder; the embeddings home uses 0.62, measured on its model.": "Quão próxima uma pergunta nova precisa estar de uma página lembrada do mesmo site para reaproveitá-la, com o embedder do gateway; a casa de embeddings usa 0,62, medido no modelo dela.",
|
|
912
|
+
"Model use (last 30 days)": "Uso de modelos (últimos 30 dias)",
|
|
913
|
+
"Calls, failures and, when the gateway reports them, tokens and cost, per purpose and over every project. A purpose whose calls in one hour pass twice its own busiest hour of the previous week (and at least 100) is flagged here; it never raises a notification.": "Chamadas, falhas e, quando o gateway informa, tokens e custo, por finalidade e em todos os projetos. Uma finalidade cujas chamadas em uma hora passam do dobro da sua própria hora mais cheia da semana anterior (e de pelo menos 100) é sinalizada aqui; nunca gera notificação.",
|
|
914
|
+
"Volume alert": "Alerta de volume",
|
|
915
|
+
"Prompt tokens": "Tokens de entrada",
|
|
916
|
+
"Output tokens": "Tokens de saída",
|
|
917
|
+
"Reported cost": "Custo informado",
|
|
918
|
+
"No model call in the last 30 days.": "Nenhuma chamada de modelo nos últimos 30 dias.",
|
|
919
|
+
"volume alert": "alerta de volume",
|
|
920
|
+
"Could not read the usage meter:": "Não foi possível ler o medidor de uso:",
|
|
921
|
+
"Volume alert muted for 7 days.": "Alerta de volume silenciado por 7 dias.",
|
|
922
|
+
"Automatic updates": "Atualizações automáticas",
|
|
923
|
+
"Smart Tool checks npm at start and every 24 hours and, when this is on, installs a newer version by itself. The switch is the same as in the tray menu.": "O Smart Tool consulta o npm ao iniciar e a cada 24 horas e, com isto ligado, instala sozinho uma versão mais nova. A chave é a mesma do menu da bandeja.",
|
|
924
|
+
"Installed: {0}.": "Instalada: {0}.",
|
|
925
|
+
"Last check: {0}.": "Última verificação: {0}.",
|
|
926
|
+
"Not checked yet.": "Ainda não verificado.",
|
|
927
|
+
"Smart Tool is up to date.": "O Smart Tool está atualizado.",
|
|
928
|
+
"Updated from {0} to {1} on {2}.": "Atualizado da {0} para a {1} em {2}.",
|
|
929
|
+
"Automatic updates turned on.": "Atualizações automáticas ligadas.",
|
|
930
|
+
"Automatic updates turned off.": "Atualizações automáticas desligadas.",
|
|
931
|
+
"Send {\"enabled\": true or false}.": "Envie {\"enabled\": true ou false}.",
|
|
932
|
+
"{0}=off is set: automatic updates stay off until it is removed.": "{0}=off está definida: as atualizações automáticas ficam desligadas até removê-la.",
|
|
933
|
+
"Who asked the searches": "Quem pediu as buscas",
|
|
934
|
+
"The recent web search calls grouped by the client and project folder that made them. The text of a search is never recorded.": "As chamadas recentes de busca na web agrupadas pelo cliente e pela pasta do projeto que as fez. O texto de uma busca nunca é gravado.",
|
|
935
|
+
"Sessions": "Sessões",
|
|
936
|
+
"Attributed by": "Atribuído por",
|
|
937
|
+
"No web search recorded yet.": "Nenhuma busca na web registrada ainda.",
|
|
938
|
+
"Could not read the search log:": "Não foi possível ler o registro de buscas:",
|
|
939
|
+
"Calls that go up the layers on edit": "Chamadas que sobem as camadas na edição",
|
|
940
|
+
"When the project has confirmed layers (.smart-tool/layers.json), warns about a call an edit adds from a lower layer to a function of a higher one. A project without layers is never checked. It never blocks.": "Quando o projeto tem camadas confirmadas (.smart-tool/layers.json), avisa sobre uma chamada que a edição adiciona de uma camada inferior para uma função de uma superior. Projeto sem camadas nunca é verificado. Nunca bloqueia.",
|
|
941
|
+
"The edit runs; the agent is told which new calls go up the layers.": "A edição roda; o agente é avisado de quais chamadas novas sobem as camadas.",
|
|
942
|
+
"No layer checks.": "Sem verificação de camadas.",
|
|
943
|
+
"{0} volume alert": "{0} alerta de volume",
|
|
944
|
+
"{0} volume alerts": "{0} alertas de volume",
|
|
945
|
+
"In {0}, move each listed package to a version past its advisories; project_manage action=deps lists every advisory ID.": "Em {0}, mova cada pacote listado para uma versão além dos seus avisos; project_manage action=deps lista cada ID de aviso.",
|
|
946
|
+
"dependency advisories": "avisos de dependências",
|
|
947
|
+
"Fix the cause named in the detail (OSV unreachable, or the advisory cache file), then run the audit again.": "Corrija a causa indicada no detalhe (OSV inacessível ou o arquivo de cache de avisos) e rode a auditoria de novo.",
|
|
948
|
+
"; {0} more": "; mais {0}"
|
|
914
949
|
},
|
|
915
950
|
"ignore": [
|
|
916
951
|
"and {0} more",
|
package/low_priority.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Lowers the scheduling priority of the calling thread for a block, so a long index build on a machine without a GPU
|
|
2
|
+
leaves the processor to the user's own programs. Only Windows has a per-thread priority that can be put back; on any
|
|
3
|
+
other system the block runs unchanged (`current()` returns None)."""
|
|
4
|
+
import contextlib
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
BELOW_NORMAL = -1
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _kernel():
|
|
11
|
+
import ctypes
|
|
12
|
+
kernel = ctypes.windll.kernel32
|
|
13
|
+
kernel.GetCurrentThread.restype = ctypes.c_void_p
|
|
14
|
+
kernel.GetThreadPriority.argtypes = [ctypes.c_void_p]
|
|
15
|
+
kernel.SetThreadPriority.argtypes = [ctypes.c_void_p, ctypes.c_int]
|
|
16
|
+
return kernel
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def current():
|
|
20
|
+
"""Priority of the calling thread (0 is normal, -1 below normal); None where it cannot be read."""
|
|
21
|
+
if sys.platform != "win32":
|
|
22
|
+
return None
|
|
23
|
+
kernel = _kernel()
|
|
24
|
+
return kernel.GetThreadPriority(kernel.GetCurrentThread())
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@contextlib.contextmanager
|
|
28
|
+
def background():
|
|
29
|
+
"""Runs the block with the calling thread below normal priority and restores the previous one afterwards."""
|
|
30
|
+
if sys.platform != "win32":
|
|
31
|
+
yield
|
|
32
|
+
return
|
|
33
|
+
kernel = _kernel()
|
|
34
|
+
thread = kernel.GetCurrentThread()
|
|
35
|
+
previous = kernel.GetThreadPriority(thread)
|
|
36
|
+
if not kernel.SetThreadPriority(thread, BELOW_NORMAL):
|
|
37
|
+
raise OSError("SetThreadPriority failed; the index build cannot run at low priority.")
|
|
38
|
+
try:
|
|
39
|
+
yield
|
|
40
|
+
finally:
|
|
41
|
+
kernel.SetThreadPriority(thread, previous)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@allansantos-dev/smart-tool",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.80",
|
|
4
4
|
"description": "Local MCP server that gives coding agents (Claude Code, Codex) cheaper, sharper tools than their built-in search. Windows.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Smart Tool contributors",
|
package/pr_impact.py
CHANGED
|
@@ -1,26 +1,105 @@
|
|
|
1
1
|
"""Impact report of a change set, in Markdown, for the description or a comment of a pull request.
|
|
2
2
|
|
|
3
3
|
It joins what Smart Tool already computes: the functions the diff touches and the tests to run (affected_tests), the
|
|
4
|
-
callers of each touched function (code_impact)
|
|
4
|
+
callers of each touched function and the tests that reach it (code_impact), the documentation anchors the change leaves
|
|
5
|
+
out of date (doc_scope), the files that usually change with the diff's files and are missing from it (code_health), the decision
|
|
6
|
+
notes that name a touched function (dossier) and the notes the edit hook would have added to the whole diff (hook_decision).
|
|
5
7
|
Nothing is published: the agent pastes the report where it belongs.
|
|
6
8
|
"""
|
|
7
9
|
import os
|
|
10
|
+
import re
|
|
8
11
|
|
|
9
12
|
import affected_tests
|
|
13
|
+
import code_health
|
|
10
14
|
import code_impact
|
|
15
|
+
import config
|
|
11
16
|
import doc_scope
|
|
17
|
+
import dossier
|
|
18
|
+
import hook_decision
|
|
12
19
|
|
|
13
20
|
MAX_FUNCTIONS = 15
|
|
14
21
|
MAX_CALLERS = 8
|
|
22
|
+
TEST_DEPTH = 3
|
|
23
|
+
MAX_CHECKED_FILES = 50
|
|
24
|
+
MAX_COUPLED = 10
|
|
25
|
+
MAX_NOTES_PER_FUNCTION = 2
|
|
15
26
|
|
|
16
27
|
|
|
17
28
|
def _callers(root, label, view_id):
|
|
18
|
-
"""Callers of one touched function, or None when the function is new and nothing calls it yet."""
|
|
29
|
+
"""Callers and reaching tests of one touched function, or None when the function is new and nothing calls it yet."""
|
|
19
30
|
try:
|
|
20
|
-
found = code_impact.impact(root, label, view_id, depth=
|
|
31
|
+
found = code_impact.impact(root, label, view_id, depth=TEST_DEPTH, limit=MAX_CALLERS)
|
|
21
32
|
except ValueError:
|
|
22
33
|
return None
|
|
23
|
-
return found["callers"], found["counts"]["callers"]
|
|
34
|
+
return found["callers"], found["counts"]["callers"], found["counts"]["tests"]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _hook_notes(root, base, changes):
|
|
38
|
+
"""The edit hook's notes over the changed files of the diff, which commits made without the hook never got.
|
|
39
|
+
|
|
40
|
+
Parameters: root (str) project folder; base (str) git revision; changes (dict) the `affected_tests.affected` answer.
|
|
41
|
+
Returns: tuple (str notes, "" when nothing is worth saying; str problem, "" when the checks ran).
|
|
42
|
+
"""
|
|
43
|
+
paths = sorted({*changes["changed"]["code"], *changes["changed"]["tests"], *changes["changed"]["other"]})
|
|
44
|
+
triples = []
|
|
45
|
+
for rel in paths[:MAX_CHECKED_FILES]:
|
|
46
|
+
full = os.path.join(root, *rel.split("/"))
|
|
47
|
+
if not os.path.isfile(full):
|
|
48
|
+
continue
|
|
49
|
+
try:
|
|
50
|
+
with open(full, encoding="utf-8") as stream:
|
|
51
|
+
after = stream.read()
|
|
52
|
+
except (OSError, UnicodeDecodeError):
|
|
53
|
+
continue
|
|
54
|
+
try:
|
|
55
|
+
before = affected_tests._git(root, "show", f"{base}:./{rel}")
|
|
56
|
+
except ValueError:
|
|
57
|
+
before = None
|
|
58
|
+
triples.append((full, before, after))
|
|
59
|
+
try:
|
|
60
|
+
return hook_decision.review_diff(triples, config.load_config()), ""
|
|
61
|
+
except (OSError, ValueError, RuntimeError) as exc:
|
|
62
|
+
return "", f"the edit checks could not run: {exc}"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _coupled(root, view_id, changes):
|
|
66
|
+
"""Files that usually change with the diff's files but are not in it, or the reason none could be computed."""
|
|
67
|
+
paths = sorted({*changes["changed"]["code"], *changes["changed"]["tests"], *changes["changed"]["other"]})
|
|
68
|
+
try:
|
|
69
|
+
found = code_health.coupled_with(root, paths, view_id, MAX_COUPLED)
|
|
70
|
+
except ValueError as exc:
|
|
71
|
+
return [], f"The change coupling was not read: {exc}"
|
|
72
|
+
return found["files"], f"The change coupling was not read: {found['note']}" if found["note"] else None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _name_pattern(name):
|
|
76
|
+
"""How a note must mention a function to count. A compound name (snake_case or camelCase) is distinctive, so the bare
|
|
77
|
+
word counts; a single word such as `load` or `report` is also ordinary prose, so only a code reference counts:
|
|
78
|
+
backticks, or a call with parentheses."""
|
|
79
|
+
escaped = re.escape(name)
|
|
80
|
+
if "_" in name or re.search(r"[a-z][A-Z]", name):
|
|
81
|
+
return re.compile(rf"\b(?:{escaped}|{re.escape(name.replace('_', ' '))})\b", re.I)
|
|
82
|
+
return re.compile(rf"`[\w.]*\b{escaped}\b[\w.()]*`|\b{escaped}\(")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _decisions(root, labels):
|
|
86
|
+
"""Decision notes and commit messages that name each touched function: the dossier search over the project's notes and
|
|
87
|
+
commit history, keeping only hits that contain the function's own name, as a term match alone is too loose.
|
|
88
|
+
|
|
89
|
+
Returns: tuple (dict label -> list of hits, str problem, "" when the search ran whole).
|
|
90
|
+
"""
|
|
91
|
+
found, reasons = {}, []
|
|
92
|
+
for label in labels:
|
|
93
|
+
name = label.partition("::")[2].rpartition(".")[2]
|
|
94
|
+
if not name:
|
|
95
|
+
continue
|
|
96
|
+
answer = dossier.query(root, name.replace("_", " "), max_hits=MAX_NOTES_PER_FUNCTION * 3, history=True)
|
|
97
|
+
reasons += [reason for reason in answer["reasons"] if reason not in reasons]
|
|
98
|
+
named = _name_pattern(name)
|
|
99
|
+
hits = [hit for hit in answer["hits"] if named.search(f"{hit['title']} {hit['snippet']}")]
|
|
100
|
+
if hits:
|
|
101
|
+
found[label] = hits[:MAX_NOTES_PER_FUNCTION]
|
|
102
|
+
return found, f"The notes search was cut: {'; '.join(reasons)}" if reasons else ""
|
|
24
103
|
|
|
25
104
|
|
|
26
105
|
def _stale(root, view_id, paths):
|
|
@@ -42,23 +121,31 @@ def report(root, base="HEAD", limit=30, view_id=None):
|
|
|
42
121
|
touched = changes["touched_functions"][:MAX_FUNCTIONS]
|
|
43
122
|
callers = {label: _callers(root, label, view_id) for label in touched}
|
|
44
123
|
stale, stale_note = _stale(root, view_id, changed_code)
|
|
124
|
+
untested = [label for label in touched if callers[label] is None or callers[label][2] == 0]
|
|
125
|
+
notes, notes_problem = _hook_notes(root, base, changes)
|
|
126
|
+
coupled, coupled_note = _coupled(root, view_id, changes)
|
|
127
|
+
decisions, decisions_note = _decisions(root, touched)
|
|
45
128
|
lines = [f"## Impact of the change against `{base}`", "",
|
|
46
129
|
f"{changes['counts']['changed_files']} changed file(s), {len(changes['touched_functions'])} touched "
|
|
47
130
|
f"function(s), {changes['counts']['tests']} test file(s) to run.", ""]
|
|
48
131
|
if changes["touched_functions"]:
|
|
49
|
-
lines += ["### Touched functions and who calls them", "",
|
|
132
|
+
lines += ["### Touched functions and who calls them", "",
|
|
133
|
+
"| Function | Callers | Tests | Called from |", "|---|---|---|---|"]
|
|
50
134
|
for label in touched:
|
|
51
135
|
reached = callers[label]
|
|
52
136
|
if reached is None:
|
|
53
|
-
lines.append(f"| `{label}` | 0 | new: nothing calls it yet |")
|
|
137
|
+
lines.append(f"| `{label}` | 0 | none | new: nothing calls it yet |")
|
|
54
138
|
continue
|
|
55
|
-
sites, total = reached
|
|
139
|
+
sites, total, tests = reached
|
|
56
140
|
shown = ", ".join(f"`{s['function']}` ({s['path']}:{s['line']})" for s in sites)
|
|
57
141
|
more = f" (+{total - len(sites)} more)" if total > len(sites) else ""
|
|
58
|
-
lines.append(f"| `{label}` | {total} | {shown or '-'}{more} |")
|
|
142
|
+
lines.append(f"| `{label}` | {total} | {tests or 'none'} | {shown or '-'}{more} |")
|
|
59
143
|
if len(changes["touched_functions"]) > len(touched):
|
|
60
144
|
lines.append(f"\n{len(changes['touched_functions']) - len(touched)} more touched function(s) not listed.")
|
|
61
145
|
lines.append("")
|
|
146
|
+
lines += ["### Touched functions no test reaches", ""]
|
|
147
|
+
lines += [f"- `{label}`" for label in untested] if untested else ["Every touched function is reached by a test."]
|
|
148
|
+
lines.append("")
|
|
62
149
|
lines += ["### Documentation left out of date", ""]
|
|
63
150
|
if stale_note:
|
|
64
151
|
lines += [stale_note, ""]
|
|
@@ -67,6 +154,21 @@ def report(root, base="HEAD", limit=30, view_id=None):
|
|
|
67
154
|
lines += ["", "Regenerate with `project_manage action=docs generate=true` and review the prose of these entries.", ""]
|
|
68
155
|
else:
|
|
69
156
|
lines += ["No documented entry in the changed files is out of date.", ""]
|
|
157
|
+
lines += ["### Edit checks over the whole diff", "", notes or notes_problem or "The edit checks find nothing to add.", ""]
|
|
158
|
+
if touched:
|
|
159
|
+
lines += ["### Decision notes of the touched functions", ""]
|
|
160
|
+
lines += [f"- `{label}`: `{hit['id']}` {hit['title']}" for label, hits in decisions.items() for hit in hits]
|
|
161
|
+
if not decisions:
|
|
162
|
+
lines.append("No decision note or commit message names a touched function.")
|
|
163
|
+
lines += ([decisions_note] if decisions_note else []) + [""]
|
|
164
|
+
lines += ["### Files that usually change with these", ""]
|
|
165
|
+
if coupled_note:
|
|
166
|
+
lines += [coupled_note, ""]
|
|
167
|
+
elif coupled:
|
|
168
|
+
lines += [f"- `{f['path']}` changed in {f['shared']} commits with `{f['with']}` ({f['degree']}%) and is not in this diff"
|
|
169
|
+
for f in coupled] + [""]
|
|
170
|
+
else:
|
|
171
|
+
lines += ["No file that usually changes with these is missing from the diff.", ""]
|
|
70
172
|
lines += ["### Tests to run", ""]
|
|
71
173
|
if changes["tests"]:
|
|
72
174
|
lines += [f"- `{t['path']}`: {t['why']}" for t in changes["tests"]]
|
|
@@ -78,7 +180,10 @@ def report(root, base="HEAD", limit=30, view_id=None):
|
|
|
78
180
|
lines.append("No test reaches the changed code through imports or static calls.")
|
|
79
181
|
if changes["run_all"]:
|
|
80
182
|
lines += ["", "Run the whole suite: " + "; ".join(changes["run_all_reasons"]) + "."]
|
|
183
|
+
if changes.get("truncated"):
|
|
184
|
+
lines += ["", changes["note"]]
|
|
81
185
|
return {"base": base, "markdown": "\n".join(lines).rstrip("\n") + "\n", "touched": touched,
|
|
82
186
|
"callers": {label: (reached[1] if reached else 0) for label, reached in callers.items()},
|
|
187
|
+
"untested": untested, "coupled": coupled, "decisions": decisions, "edit_checks": notes or notes_problem,
|
|
83
188
|
"stale_anchors": stale, "stale_note": stale_note, "tests": changes["tests"],
|
|
84
189
|
"commands": changes["commands"], "run_all": changes["run_all"], "counts": changes["counts"]}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""Child processes under a time cap and a memory cap whose whole tree dies with them."""
|
|
2
|
+
import os
|
|
3
|
+
import subprocess
|
|
4
|
+
|
|
5
|
+
_IS_WINDOWS = os.name == "nt"
|
|
6
|
+
JOB_OBJECT_LIMIT_JOB_MEMORY = 0x200
|
|
7
|
+
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x2000
|
|
8
|
+
STDERR_TAIL_BYTES = 2000
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def kill_on_close_job(job_memory_bytes=None):
|
|
12
|
+
"""A Windows job object that kills every process in it when its handle closes, and, with `job_memory_bytes`, caps
|
|
13
|
+
the memory the whole tree may commit.
|
|
14
|
+
|
|
15
|
+
Parameters: job_memory_bytes (cap for all processes of the job together, None for no cap). Returns: `(kernel32,
|
|
16
|
+
job)`, the library and the job handle. Raises: OSError when the system refuses the job."""
|
|
17
|
+
import ctypes
|
|
18
|
+
from ctypes import wintypes
|
|
19
|
+
|
|
20
|
+
class IoCounters(ctypes.Structure):
|
|
21
|
+
_fields_ = [(name, ctypes.c_ulonglong) for name in
|
|
22
|
+
("read_ops", "write_ops", "other_ops", "read_bytes", "write_bytes", "other_bytes")]
|
|
23
|
+
|
|
24
|
+
class BasicLimits(ctypes.Structure):
|
|
25
|
+
_fields_ = [("per_process_time", ctypes.c_longlong), ("per_job_time", ctypes.c_longlong),
|
|
26
|
+
("flags", wintypes.DWORD), ("min_ws", ctypes.c_size_t), ("max_ws", ctypes.c_size_t),
|
|
27
|
+
("active_processes", wintypes.DWORD), ("affinity", ctypes.c_size_t),
|
|
28
|
+
("priority", wintypes.DWORD), ("scheduling", wintypes.DWORD)]
|
|
29
|
+
|
|
30
|
+
class ExtendedLimits(ctypes.Structure):
|
|
31
|
+
_fields_ = [("basic", BasicLimits), ("io", IoCounters), ("process_memory", ctypes.c_size_t),
|
|
32
|
+
("job_memory", ctypes.c_size_t), ("peak_process_memory", ctypes.c_size_t),
|
|
33
|
+
("peak_job_memory", ctypes.c_size_t)]
|
|
34
|
+
|
|
35
|
+
kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
|
36
|
+
kernel32.CreateJobObjectW.restype = wintypes.HANDLE
|
|
37
|
+
job = kernel32.CreateJobObjectW(None, None)
|
|
38
|
+
if not job:
|
|
39
|
+
raise ctypes.WinError(ctypes.get_last_error())
|
|
40
|
+
limits = ExtendedLimits()
|
|
41
|
+
limits.basic.flags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
|
|
42
|
+
if job_memory_bytes:
|
|
43
|
+
limits.basic.flags |= JOB_OBJECT_LIMIT_JOB_MEMORY
|
|
44
|
+
limits.job_memory = job_memory_bytes
|
|
45
|
+
if not kernel32.SetInformationJobObject(wintypes.HANDLE(job), 9, ctypes.byref(limits), ctypes.sizeof(limits)):
|
|
46
|
+
raise ctypes.WinError(ctypes.get_last_error())
|
|
47
|
+
return kernel32, job
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _stderr_tail(path):
|
|
51
|
+
with open(path, "rb") as stream:
|
|
52
|
+
stream.seek(0, os.SEEK_END)
|
|
53
|
+
stream.seek(max(0, stream.tell() - STDERR_TAIL_BYTES))
|
|
54
|
+
return stream.read().decode("utf-8", "replace").strip()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def run_capped(argv, workdir, timeout, memory_bytes):
|
|
58
|
+
"""Runs `argv` in `workdir` with no window, no stdin, its output discarded and its error stream kept in a file there.
|
|
59
|
+
At the deadline the whole process tree is killed; a process that passes `memory_bytes` is killed by the system
|
|
60
|
+
(Windows job memory limit; elsewhere an address-space limit).
|
|
61
|
+
|
|
62
|
+
Parameters: argv, workdir (an existing folder), timeout (seconds), memory_bytes (cap for the tree). Returns:
|
|
63
|
+
`(returncode, stderr_tail)`, the tail being the last 2,000 bytes of the error stream as text. Raises:
|
|
64
|
+
subprocess.TimeoutExpired after killing the tree; OSError when the program cannot start."""
|
|
65
|
+
error_path = os.path.join(workdir, "stderr.txt")
|
|
66
|
+
with open(error_path, "wb") as errors:
|
|
67
|
+
if _IS_WINDOWS:
|
|
68
|
+
returncode = _run_in_job(argv, workdir, timeout, memory_bytes, errors)
|
|
69
|
+
else:
|
|
70
|
+
returncode = _run_in_group(argv, workdir, timeout, memory_bytes, errors)
|
|
71
|
+
return returncode, _stderr_tail(error_path)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _run_in_job(argv, workdir, timeout, memory_bytes, errors):
|
|
75
|
+
import ctypes
|
|
76
|
+
from ctypes import wintypes
|
|
77
|
+
kernel32, job = kill_on_close_job(memory_bytes)
|
|
78
|
+
try:
|
|
79
|
+
proc = subprocess.Popen(argv, cwd=workdir, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=errors,
|
|
80
|
+
creationflags=subprocess.CREATE_NO_WINDOW)
|
|
81
|
+
if not kernel32.AssignProcessToJobObject(wintypes.HANDLE(job), wintypes.HANDLE(proc._handle)):
|
|
82
|
+
proc.kill()
|
|
83
|
+
raise ctypes.WinError(ctypes.get_last_error())
|
|
84
|
+
try:
|
|
85
|
+
return proc.wait(timeout)
|
|
86
|
+
except subprocess.TimeoutExpired:
|
|
87
|
+
kernel32.TerminateJobObject(wintypes.HANDLE(job), 1)
|
|
88
|
+
proc.wait()
|
|
89
|
+
raise
|
|
90
|
+
finally:
|
|
91
|
+
kernel32.CloseHandle(wintypes.HANDLE(job))
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _run_in_group(argv, workdir, timeout, memory_bytes, errors):
|
|
95
|
+
import resource
|
|
96
|
+
import signal
|
|
97
|
+
|
|
98
|
+
def limit_memory():
|
|
99
|
+
resource.setrlimit(resource.RLIMIT_AS, (memory_bytes, memory_bytes))
|
|
100
|
+
|
|
101
|
+
proc = subprocess.Popen(argv, cwd=workdir, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=errors,
|
|
102
|
+
start_new_session=True, preexec_fn=limit_memory)
|
|
103
|
+
try:
|
|
104
|
+
return proc.wait(timeout)
|
|
105
|
+
except subprocess.TimeoutExpired:
|
|
106
|
+
os.killpg(proc.pid, signal.SIGKILL)
|
|
107
|
+
proc.wait()
|
|
108
|
+
raise
|
package/project_monitor.py
CHANGED
|
@@ -70,11 +70,18 @@ def snapshot(root, scope):
|
|
|
70
70
|
|
|
71
71
|
|
|
72
72
|
def index_model(current=None):
|
|
73
|
-
return local_embedder.resolve(config.load_config()
|
|
73
|
+
return local_embedder.resolve(config.load_config(), current)
|
|
74
74
|
|
|
75
75
|
|
|
76
76
|
def needs_model_rebuild(meta, configured_model):
|
|
77
|
-
|
|
77
|
+
"""Whether the index must be built again for `configured_model`; never while the target model is unknown (None:
|
|
78
|
+
the embeddings home is not running and the index keeps answering as it is)."""
|
|
79
|
+
return configured_model is not None and (meta.get("model_id") != configured_model or not meta.get("exists"))
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def needs_model_switch(meta, configured_model):
|
|
83
|
+
"""Whether an existing index holds another model than `configured_model` and must be re-embedded."""
|
|
84
|
+
return bool(meta.get("chunks")) and needs_model_rebuild(meta, configured_model)
|
|
78
85
|
|
|
79
86
|
class ProjectMonitor:
|
|
80
87
|
def __init__(self, enqueue, watcher_factory=directory_watch.DirectoryWatcher):
|
|
@@ -92,6 +99,7 @@ class ProjectMonitor:
|
|
|
92
99
|
self._gap_level = {}
|
|
93
100
|
self._filters = {}
|
|
94
101
|
self._views = {}
|
|
102
|
+
self._switch = set()
|
|
95
103
|
self._lock = threading.RLock()
|
|
96
104
|
self._stop = threading.Event()
|
|
97
105
|
self._thread = None
|
|
@@ -238,6 +246,7 @@ class ProjectMonitor:
|
|
|
238
246
|
projects = project_store.all_projects()
|
|
239
247
|
self._sync_watchers(projects, now)
|
|
240
248
|
busy = project_store.projects_with_pending_job()
|
|
249
|
+
switching = bool(busy)
|
|
241
250
|
for project in projects:
|
|
242
251
|
key = project["id"]
|
|
243
252
|
if not monitored(project):
|
|
@@ -269,7 +278,13 @@ class ProjectMonitor:
|
|
|
269
278
|
if scope is not None:
|
|
270
279
|
changed |= snapshot(project["root"], scope) != indexer.manifest_snapshot(project["root"])
|
|
271
280
|
full = time.time() - meta.get("full_scan_at", 0) >= FULL_CHECK_S
|
|
272
|
-
|
|
281
|
+
target = index_model(meta.get("model_id"))
|
|
282
|
+
free = local_embedder.is_local(target) or target == indexer.LEXICAL_MODEL
|
|
283
|
+
if free and needs_model_switch(meta, target):
|
|
284
|
+
self._switch.add(key)
|
|
285
|
+
else:
|
|
286
|
+
self._switch.discard(key)
|
|
287
|
+
changed |= needs_model_rebuild(meta, target)
|
|
273
288
|
project_store.update(key, last_checked=time.time())
|
|
274
289
|
if changed or full or project.get("dirty_seq", 0) > project.get("indexed_seq", 0):
|
|
275
290
|
with self._lock:
|
|
@@ -281,8 +296,17 @@ class ProjectMonitor:
|
|
|
281
296
|
if first:
|
|
282
297
|
project_store.mark_dirty(key, "Reconciliation found a pending update.")
|
|
283
298
|
project_store.record_changes(key, full=full, scope=rescope)
|
|
284
|
-
#
|
|
285
|
-
#
|
|
299
|
+
# A switch to a model that costs nothing (the embeddings home, or lexical) runs in every update mode, one
|
|
300
|
+
# project at a time, so the old index keeps answering and searches keep a free worker. Outside eager mode
|
|
301
|
+
# any other change only stays pending for the next search, which pays the embeddings.
|
|
302
|
+
if key in self._switch:
|
|
303
|
+
if not switching:
|
|
304
|
+
switching = True
|
|
305
|
+
with self._lock:
|
|
306
|
+
self._pending.setdefault(key, {"paths": set(), "last": now - DEBOUNCE_S, "full": False,
|
|
307
|
+
"force_scope": False, "revision": 0})
|
|
308
|
+
self._dispatch(key, now)
|
|
309
|
+
continue
|
|
286
310
|
if project.get('update_mode', 'on_search') != 'eager':
|
|
287
311
|
with self._lock:
|
|
288
312
|
self._pending.pop(key, None)
|
|
@@ -314,7 +338,8 @@ class ProjectMonitor:
|
|
|
314
338
|
|
|
315
339
|
def _dispatch_quiet(self, now):
|
|
316
340
|
"""Between full passes: sends the projects whose debounce ended, without listing every project of the registry.
|
|
317
|
-
A project that is not eager, is busy or is no longer monitored is left to the next full
|
|
341
|
+
A project that is not eager, is busy, is switching model or is no longer monitored is left to the next full
|
|
342
|
+
pass."""
|
|
318
343
|
self._flush_notes(now)
|
|
319
344
|
with self._lock:
|
|
320
345
|
ready = [key for key, pending in self._pending.items()
|
|
@@ -324,7 +349,8 @@ class ProjectMonitor:
|
|
|
324
349
|
busy = project_store.projects_with_pending_job()
|
|
325
350
|
for key in ready:
|
|
326
351
|
project = project_store.get(key)
|
|
327
|
-
if (project and key not in busy and
|
|
352
|
+
if (project and key not in busy and key not in self._switch and monitored(project)
|
|
353
|
+
and project.get("update_mode", "on_search") == "eager"):
|
|
328
354
|
self._dispatch(key, now)
|
|
329
355
|
|
|
330
356
|
def _run(self):
|
package/project_scheduler.py
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
"""Fila justa entre raízes; jobs aguardando a mesma pasta não ocupam workers. Um job de prioridade maior passa à frente
|
|
2
|
-
dos de prioridade menor ainda na fila (o que já está rodando não é interrompido).
|
|
2
|
+
dos de prioridade menor ainda na fila (o que já está rodando não é interrompido). Pistas (`lane`) distintas da mesma pasta
|
|
3
|
+
rodam em paralelo. Com `reserved_lane`, as outras pistas nunca ocupam o último worker livre: uma prioridade só ordena
|
|
4
|
+
a fila e não libera worker, então sem a reserva uma busca espera o fim de um job longo."""
|
|
3
5
|
import collections
|
|
4
6
|
import concurrent.futures
|
|
5
7
|
import threading
|
|
@@ -8,16 +10,19 @@ import project_identity
|
|
|
8
10
|
|
|
9
11
|
|
|
10
12
|
class ProjectScheduler:
|
|
11
|
-
def __init__(self, workers=3):
|
|
13
|
+
def __init__(self, workers=3, reserved_lane=None):
|
|
12
14
|
self.workers = workers
|
|
15
|
+
self.reserved_lane = reserved_lane
|
|
13
16
|
self._pool = concurrent.futures.ThreadPoolExecutor(max_workers=workers, thread_name_prefix="project")
|
|
14
17
|
self._lock = threading.RLock()
|
|
15
18
|
self._queues = collections.OrderedDict()
|
|
16
19
|
self._running = set()
|
|
17
20
|
self._closed = False
|
|
18
21
|
|
|
19
|
-
def submit(self, root, function, *args, priority=0):
|
|
20
|
-
|
|
22
|
+
def submit(self, root, function, *args, priority=0, lane=None):
|
|
23
|
+
"""Queues `function`. Jobs of one folder run one at a time within a lane; jobs of one folder in different
|
|
24
|
+
lanes run side by side (the caller keeps them from writing the same data)."""
|
|
25
|
+
key = (project_identity.canonical_root(root), lane)
|
|
21
26
|
future = concurrent.futures.Future()
|
|
22
27
|
with self._lock:
|
|
23
28
|
if self._closed:
|
|
@@ -31,6 +36,9 @@ class ProjectScheduler:
|
|
|
31
36
|
def _dispatch(self):
|
|
32
37
|
while len(self._running) < self.workers:
|
|
33
38
|
waiting = [root for root in self._queues if root not in self._running]
|
|
39
|
+
if self.reserved_lane is not None and sum(
|
|
40
|
+
lane != self.reserved_lane for _root, lane in self._running) >= self.workers - 1:
|
|
41
|
+
waiting = [key for key in waiting if key[1] == self.reserved_lane]
|
|
34
42
|
if not waiting:
|
|
35
43
|
break
|
|
36
44
|
key = max(waiting, key=lambda root: self._queues[root][0][3])
|
package/scout.py
CHANGED
|
@@ -102,7 +102,8 @@ def is_intent_question(question):
|
|
|
102
102
|
|
|
103
103
|
def disk_lines(root, path, start, end):
|
|
104
104
|
"""`[(line_number, text)]` of lines `start..end` of the file as it is on disk now, or None when the file cannot be
|
|
105
|
-
read by the index rules (secret, dotfile, too large, binary, gone) or holds no line text (docx, pdf, pptx, xlsx
|
|
105
|
+
read by the index rules (secret, dotfile, too large, binary, gone) or holds no line text (docx, pdf, pptx, xlsx;
|
|
106
|
+
`indexer.indexed_lines` serves those)."""
|
|
106
107
|
if document_text.location_kind(path) != "source_line":
|
|
107
108
|
return None
|
|
108
109
|
data, encoding = indexer.read_indexable(root, path)
|
|
@@ -152,7 +153,8 @@ def evidence(root, rows, terms, max_files, window_lines):
|
|
|
152
153
|
"""Evidence items `{id, path, start, end, lines, symbols}` for the first `max_files` files of `rows` (search rows
|
|
153
154
|
`{path, lines: "a-b", snippet, symbols}`), at most WINDOWS_PER_FILE non-overlapping windows per file, each the
|
|
154
155
|
`window_lines` lines of the chunk with most `terms`, read from disk now. A file that cannot be read from disk keeps
|
|
155
|
-
the indexed snippet with its chunk range
|
|
156
|
+
the indexed snippet with its chunk range, except a PDF or Office file, whose window is chosen from the lines the
|
|
157
|
+
index stored (it never extracts the document again)."""
|
|
156
158
|
items, files = [], []
|
|
157
159
|
for row in rows:
|
|
158
160
|
path = row["path"]
|
|
@@ -168,6 +170,8 @@ def evidence(root, rows, terms, max_files, window_lines):
|
|
|
168
170
|
if len(mine) >= WINDOWS_PER_FILE or any(item["start"] <= end and start <= item["end"] for item in mine):
|
|
169
171
|
continue
|
|
170
172
|
lines = disk_lines(root, path, start, end)
|
|
173
|
+
if lines is None and document_text.is_extracted(path):
|
|
174
|
+
lines = indexer.indexed_lines(root, path, start, end)
|
|
171
175
|
if lines:
|
|
172
176
|
window = best_window(lines, terms, window_lines)
|
|
173
177
|
if not window:
|