@allansantos-dev/smart-tool 0.9.81 → 0.9.83

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/CHANGELOG.md +115 -0
  2. package/NOTICE +1 -1
  3. package/README.md +73 -14
  4. package/affected_tests.py +9 -8
  5. package/agent_redirect.py +32 -0
  6. package/autostart.py +32 -25
  7. package/browser_bridge.py +320 -0
  8. package/browser_control.py +76 -39
  9. package/browser_service_client.py +1 -7
  10. package/client_hooks.py +41 -42
  11. package/code_graph.py +66 -21
  12. package/code_graph_java.mjs +21 -12
  13. package/code_health.py +14 -9
  14. package/codex_config.py +3 -0
  15. package/commit_check.py +184 -0
  16. package/daemon_launcher.py +37 -5
  17. package/doc_candidates.py +63 -0
  18. package/doc_check.py +9 -3
  19. package/doc_extras.py +2 -3
  20. package/doc_insights.py +26 -2
  21. package/doc_prose.py +59 -11
  22. package/doc_registry.py +241 -0
  23. package/doc_scope.py +205 -39
  24. package/doc_turn.py +205 -0
  25. package/doc_validate.py +52 -9
  26. package/document_converter.py +4 -4
  27. package/document_read.py +19 -11
  28. package/document_url.py +107 -0
  29. package/edit_shape.py +119 -0
  30. package/endpoint_sync.py +3 -21
  31. package/engine_home.py +95 -0
  32. package/hook_call.py +133 -0
  33. package/hook_decision.py +142 -7
  34. package/hooks/pre_commit.py +45 -0
  35. package/hooks/pretool_router.py +36 -30
  36. package/hooks/turn_end.py +40 -0
  37. package/index_views.py +5 -0
  38. package/install.py +24 -9
  39. package/lazy_mcp_proxy.py +10 -9
  40. package/local_embedder.py +8 -1
  41. package/locales/pt-BR.json +1 -2
  42. package/model_breaker.py +6 -5
  43. package/package.json +2 -2
  44. package/process_limits.py +17 -3
  45. package/project_identity.py +19 -1
  46. package/project_scan.py +98 -0
  47. package/release_notes.py +125 -0
  48. package/rename_preview.py +95 -0
  49. package/requester.py +1 -1
  50. package/scout_access.py +48 -0
  51. package/shell_reads.py +67 -0
  52. package/smart_tool_daemon.py +245 -50
  53. package/tray.py +31 -0
  54. package/venv_launch.py +67 -0
  55. package/version.py +1 -1
  56. package/web_adapters/node/engine-home.mjs +42 -0
  57. package/web_adapters/node/package-lock.json +13 -0
  58. package/web_adapters/node/package.json +1 -0
  59. package/web_fetch_many.py +61 -0
package/install.py CHANGED
@@ -16,8 +16,10 @@ from pathlib import Path
16
16
  import atomic_io
17
17
  import client_hooks
18
18
  import config
19
+ import daemon_launcher
19
20
  import i18n
20
21
  import paths
22
+ import venv_launch
21
23
  import version
22
24
 
23
25
  SOURCE = Path(__file__).resolve().parent
@@ -27,6 +29,7 @@ ENTRY_SCRIPTS = ("tray.py", "smart_tool_daemon.py", "pretool_router.py", "sessio
27
29
  HOOK_SCRIPTS = ("pretool_router.py", "session_audit.py")
28
30
  STARTUP_SCRIPT = Path(os.environ.get("APPDATA", str(Path.home() / "AppData" / "Roaming"))) / "Microsoft" / "Windows" / \
29
31
  "Start Menu" / "Programs" / "Startup" / "SmartTool-Tray.vbs"
32
+ TASK_SCRIPT = Path(paths.DATA_DIR) / "tray-start.vbs"
30
33
  MANIFEST = ".install-manifest.json"
31
34
  DEPENDENCY_INPUTS = ("requirements.txt", "install_runtime.py", "node_locator.py", "web_adapters/node/package.json",
32
35
  "web_adapters/node/package-lock.json", "web_adapters/browser/requirements.txt",
@@ -113,11 +116,15 @@ def _same_folder(path, folder):
113
116
 
114
117
 
115
118
  def stop_processes(target):
119
+ """Closes every process running a script of the installation in `target` (daemon, tray, browser proxies); a closed
120
+ daemon gets its reason in the daemon log, since a forced stop leaves it no chance to write one."""
116
121
  def running():
117
- return [pid for pid, line in _processes() for script in _script_paths(line)
122
+ return [(pid, Path(script).name) for pid, line in _processes() for script in _script_paths(line)
118
123
  if Path(script).name not in HOOK_SCRIPTS and _same_folder(_script_folder(script), target)]
119
124
 
120
- for pid in running():
125
+ for pid, script in running():
126
+ if script == Path(daemon_launcher.DAEMON_SCRIPT).name:
127
+ daemon_launcher.note_stop(f"daemon pid {pid} closed by the installer to update {target}")
121
128
  powershell(f"Stop-Process -Id {pid} -Force -ErrorAction SilentlyContinue")
122
129
  deadline = time.monotonic() + 10
123
130
  while True:
@@ -192,12 +199,19 @@ def _task_definition():
192
199
  return result.stdout if result.returncode == 0 else ""
193
200
 
194
201
 
202
+ def _script_text(path):
203
+ """Text of a launch script or hook file, "" when absent: VBScript files are written in UTF-16, the rest in UTF-8."""
204
+ if not path.is_file():
205
+ return ""
206
+ data = path.read_bytes()
207
+ return data.decode("utf-16") if data[:2] in (b"\xff\xfe", b"\xfe\xff") else data.decode("utf-8", errors="replace")
208
+
209
+
195
210
  def other_installs(target):
196
211
  """Installer-made Smart Tool folders other than target, found through what still runs or points at them: tray and
197
- daemon processes, the logon task or Startup script, and agent hook commands."""
212
+ daemon processes, the logon task and the script it runs, the Startup script, and agent hook commands."""
198
213
  texts = [line for _pid, line in _processes()] + [_task_definition()]
199
- texts += [path.read_text(encoding="utf-8", errors="replace") for path in (STARTUP_SCRIPT, *hook_files())
200
- if path.is_file()]
214
+ texts += [_script_text(path) for path in (TASK_SCRIPT, STARTUP_SCRIPT, *hook_files())]
201
215
  folders = {_install_folder(script) for text in texts for script in _script_paths(text)}
202
216
  return sorted(folder for folder in folders
203
217
  if folder and not _same_folder(folder, target) and not _same_folder(folder, SOURCE))
@@ -231,9 +245,9 @@ def remove_other_installs(target):
231
245
  path.write_text(text, encoding="utf-8")
232
246
  client_hooks.record_codex_trust(trust, str(path.with_name("config.toml")))
233
247
  print(t("Hooks in {path} now point to {target}.", path=path, target=target))
234
- startup = STARTUP_SCRIPT.read_text(encoding="utf-8", errors="replace") if STARTUP_SCRIPT.is_file() else ""
235
- if any(_same_folder(_script_folder(script), folder) for script in _script_paths(startup)):
236
- STARTUP_SCRIPT.unlink()
248
+ for script_path in (TASK_SCRIPT, STARTUP_SCRIPT):
249
+ if any(_same_folder(_script_folder(script), folder) for script in _script_paths(_script_text(script_path))):
250
+ script_path.unlink()
237
251
  shutil.rmtree(folder)
238
252
  removed.append(folder)
239
253
  return removed
@@ -368,7 +382,8 @@ def install_dependencies(target, python):
368
382
  def start(target, python, pythonw):
369
383
  subprocess.run([str(python), "-c", "import autostart, json; print(json.dumps(autostart.install()))"],
370
384
  cwd=target, check=True, capture_output=True, text=True, timeout=60)
371
- subprocess.Popen([str(pythonw if pythonw.exists() else python), str(target / "tray.py")], cwd=target,
385
+ interpreter, additions = venv_launch.direct(pythonw if pythonw.exists() else python)
386
+ subprocess.Popen([interpreter, str(target / "tray.py")], cwd=target, env=venv_launch.environment(additions),
372
387
  stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
373
388
  creationflags=getattr(subprocess, "DETACHED_PROCESS", 0) | getattr(subprocess, "CREATE_NO_WINDOW", 0))
374
389
 
package/lazy_mcp_proxy.py CHANGED
@@ -37,6 +37,15 @@ def load_handshake(path):
37
37
  return data
38
38
 
39
39
 
40
+ def local_initialize(handshake, params):
41
+ """The cached initialize result, speaking the protocol version the client asked for when it is a known one."""
42
+ result = dict(handshake["initialize"])
43
+ asked = (params or {}).get("protocolVersion")
44
+ if asked in KNOWN_PROTOCOLS:
45
+ result["protocolVersion"] = asked
46
+ return result
47
+
48
+
40
49
  class Proxy:
41
50
  """One client conversation: local answers until a request needs the real server, then a pass-through."""
42
51
 
@@ -58,14 +67,6 @@ class Proxy:
58
67
  def answer(self, request_id, result):
59
68
  self.send({"jsonrpc": "2.0", "id": request_id, "result": result})
60
69
 
61
- def local_initialize(self, params):
62
- """The cached initialize result, speaking the protocol version the client asked for when it is a known one."""
63
- result = dict(self.handshake["initialize"])
64
- asked = (params or {}).get("protocolVersion")
65
- if asked in KNOWN_PROTOCOLS:
66
- result["protocolVersion"] = asked
67
- return result
68
-
69
70
  def start_child(self):
70
71
  """Starts the real server and replays the client's initialize to it; its answer is not forwarded (the client
71
72
  already got the cached one). RuntimeError when it exits or answers with an error."""
@@ -141,7 +142,7 @@ class Proxy:
141
142
  if method == "initialize":
142
143
  self.client_init = message.get("params")
143
144
  if self.child is None and request_id is not None and method == "initialize":
144
- return self.answer(request_id, self.local_initialize(message.get("params")))
145
+ return self.answer(request_id, local_initialize(self.handshake, message.get("params")))
145
146
  if self.child is None and request_id is not None and method == "ping":
146
147
  return self.answer(request_id, {})
147
148
  if self.child is None and request_id is not None and method == "tools/list":
package/local_embedder.py CHANGED
@@ -25,6 +25,7 @@ MIN_API_VERSION = 1
25
25
  BATCH = 16
26
26
  INFO_TTL_S = 60
27
27
  MAX_ATTEMPTS = 3
28
+ HOME_FAILURE_CODES = (*model_breaker.BUSY_CODES, 500)
28
29
  REQUEST_TIMEOUT_S = 60
29
30
  RERANK_TIMEOUT_S = 15
30
31
  _lock = threading.Lock()
@@ -189,6 +190,11 @@ def _retry_delay(failure, attempt):
189
190
 
190
191
 
191
192
  def embed(model, texts, deadline=None, cancel_check=None, on_retry=None):
193
+ """Vectors of `texts` from the embeddings home, in batches of BATCH, each tried up to MAX_ATTEMPTS times.
194
+
195
+ Returns the vectors in input order. Raises IndexCompatibilityError when the home serves another model,
196
+ model_breaker.CircuitOpen once repeated busy or failing answers opened the circuit (no request is sent while it is
197
+ open), RuntimeError when the home is not running or a batch fails every attempt."""
192
198
  base = url()
193
199
  if not base:
194
200
  raise RuntimeError(f"The embeddings home is not running (no announce file at {ANNOUNCE}).")
@@ -217,8 +223,9 @@ def embed(model, texts, deadline=None, cancel_check=None, on_retry=None):
217
223
  with model_breaker.slot(model):
218
224
  got = _request(conn, "POST", "/embed", {"inputs": part})
219
225
  except (OSError, http.client.HTTPException) as exc:
220
- model_breaker.note(model, exc)
226
+ model_breaker.note(model, exc, HOME_FAILURE_CODES)
221
227
  model_health.record("/v1/embeddings", {"model": model}, exc)
228
+ model_breaker.guard(model)
222
229
  conn.close()
223
230
  conn = _connection(base, _left(deadline))
224
231
  if attempt == MAX_ATTEMPTS:
@@ -845,7 +845,6 @@
845
845
  "Its {0} entry does not cover {1}.": "A entrada {0} dele não cobre {1}.",
846
846
  "It calls {0} instead of {1}.": "Ele chama {0} em vez de {1}.",
847
847
  "Its tool list is ordered or written differently from the current one.": "A lista de ferramentas dele está em outra ordem ou escrita de outro jeito que a atual.",
848
- "The daemon announcement ({0}) is outside the home folder; the hook cannot name it as its port file.": "O anúncio do daemon ({0}) está fora da pasta do usuário; o hook não consegue nomeá-lo como seu arquivo de porta.",
849
848
  "Path contains a space and has no 8.3 short name ({0}); the hook was not installed.": "O caminho tem um espaço e não tem nome curto 8.3 ({0}); o hook não foi instalado.",
850
849
  "{0} does not have the expected format (object with 'hooks'); nothing was changed.": "{0} não tem o formato esperado (objeto com 'hooks'); nada foi alterado.",
851
850
  "{0} does not appear in {1} after writing; the file was restored.": "{0} não aparece em {1} depois de gravar; o arquivo foi restaurado.",
@@ -855,7 +854,6 @@
855
854
  "Codex does not trust the hook in {0} after recording its approval.": "O Codex não confia no hook em {0} depois de gravar a aprovação.",
856
855
  "Codex does not trust the audit hook in {0} after recording its approval.": "O Codex não confia no hook de auditoria em {0} depois de gravar a aprovação.",
857
856
  "The hook approval does not read back from {0} after writing; the file was restored.": "A aprovação do hook não é lida de volta em {0} depois de gravar; o arquivo foi restaurado.",
858
- "Smart Tool daemon has no registered URL; start the daemon before installing the hook.": "O serviço do Smart Tool não tem URL registrada; inicie o serviço antes de instalar o hook.",
859
857
  "{0}: hooks.{1} is not a list; nothing was changed.": "{0}: hooks.{1} não é uma lista; nada foi alterado.",
860
858
  "Smart Tool updated from {old} to {new}.": "Smart Tool atualizado da versão {old} para a {new}.",
861
859
  "The update to {target} failed; Smart Tool {version} is still installed. It is tried again in 24 h, or now from this menu.": "A atualização para a versão {target} falhou; o Smart Tool {version} continua instalado. Ela será tentada de novo em 24 h, ou agora por este menu.",
@@ -1018,6 +1016,7 @@
1018
1016
  },
1019
1017
  "ignore": [
1020
1018
  "and {0} more",
1019
+ "Stop",
1021
1020
  "saved msg",
1022
1021
  "more row-more",
1023
1022
  "callout info",
package/model_breaker.py CHANGED
@@ -13,7 +13,7 @@ import urllib.error
13
13
  FAILURES_TO_OPEN = 3
14
14
  PAUSE_S = 30
15
15
  MAX_IN_FLIGHT = 6
16
- _BUSY_CODES = (429, 503)
16
+ BUSY_CODES = (429, 503)
17
17
  _lock = threading.Lock()
18
18
  _failures = {}
19
19
  _open_until = {}
@@ -35,12 +35,13 @@ def guard(model):
35
35
  "HTTP 429/503 answers from the gateway; no request was sent.")
36
36
 
37
37
 
38
- def note(model, failure=None):
39
- """Records the outcome of a request: a success closes the circuit, a 429/503 counts toward opening it, any other
40
- failure changes nothing. After the pause the first failure opens it again."""
38
+ def note(model, failure=None, codes=BUSY_CODES):
39
+ """Records the outcome of a request: a success closes the circuit, an HTTP status in `codes` counts toward opening
40
+ it, any other failure changes nothing. After the pause the first failure opens it again. `codes` is the gateway's
41
+ busy answers by default; the embeddings home adds its 500, an inference that fails until the home recovers."""
41
42
  if not model:
42
43
  return
43
- busy = isinstance(failure, urllib.error.HTTPError) and failure.code in _BUSY_CODES
44
+ busy = isinstance(failure, urllib.error.HTTPError) and failure.code in codes
44
45
  with _lock:
45
46
  if failure is None:
46
47
  _failures[model] = 0
package/package.json CHANGED
@@ -1,9 +1,9 @@
1
1
  {
2
2
  "name": "@allansantos-dev/smart-tool",
3
- "version": "0.9.81",
3
+ "version": "0.9.83",
4
4
  "description": "Local MCP server that gives coding agents (Claude Code, Codex) cheaper, sharper tools than their built-in search. Windows.",
5
5
  "license": "Apache-2.0",
6
- "author": "Smart Tool contributors",
6
+ "author": "Allan Santos",
7
7
  "homepage": "https://allansantos-dv.github.io/smart-tool/",
8
8
  "repository": {
9
9
  "type": "git",
package/process_limits.py CHANGED
@@ -47,13 +47,27 @@ def kill_on_close_job(job_memory_bytes=None):
47
47
  return kernel32, job
48
48
 
49
49
 
50
- def _stderr_tail(path):
50
+ def file_tail(path, limit=STDERR_TAIL_BYTES):
51
+ """The last `limit` bytes of the file at `path`, decoded and stripped. Raises OSError when it cannot be read."""
51
52
  with open(path, "rb") as stream:
52
53
  stream.seek(0, os.SEEK_END)
53
- stream.seek(max(0, stream.tell() - STDERR_TAIL_BYTES))
54
+ stream.seek(max(0, stream.tell() - limit))
54
55
  return stream.read().decode("utf-8", "replace").strip()
55
56
 
56
57
 
58
+ def bind_to_kill_on_close_job(job, proc):
59
+ """Puts `proc` in the kill-on-close job `job` ((kernel32, handle) from `kill_on_close_job`; created when None) and
60
+ returns the job. Raises OSError when the system refuses: an unbound child would outlive its owner."""
61
+ import ctypes
62
+ from ctypes import wintypes
63
+ if job is None:
64
+ job = kill_on_close_job()
65
+ kernel32, handle = job
66
+ if not kernel32.AssignProcessToJobObject(wintypes.HANDLE(handle), wintypes.HANDLE(proc._handle)):
67
+ raise ctypes.WinError(ctypes.get_last_error())
68
+ return job
69
+
70
+
57
71
  def run_capped(argv, workdir, timeout, memory_bytes):
58
72
  """Runs `argv` in `workdir` with no window, no stdin, its output discarded and its error stream kept in a file there.
59
73
  At the deadline the whole process tree is killed; a process that passes `memory_bytes` is killed by the system
@@ -68,7 +82,7 @@ def run_capped(argv, workdir, timeout, memory_bytes):
68
82
  returncode = _run_in_job(argv, workdir, timeout, memory_bytes, errors)
69
83
  else:
70
84
  returncode = _run_in_group(argv, workdir, timeout, memory_bytes, errors)
71
- return returncode, _stderr_tail(error_path)
85
+ return returncode, file_tail(error_path)
72
86
 
73
87
 
74
88
  def _run_in_job(argv, workdir, timeout, memory_bytes, errors):
@@ -1,8 +1,10 @@
1
1
  """Identidade única de raízes e contenção de caminhos, inclusive junctions Windows."""
2
2
  import hashlib
3
3
  import os
4
+ import stat
4
5
  import time
5
6
 
7
+ _REPARSE_POINT = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
6
8
  RESOLVED_TTL_S = 3
7
9
  RESOLVED_MAX = 4096
8
10
  _resolved = {}
@@ -43,10 +45,26 @@ def legacy_ids(root):
43
45
  for path in variants))
44
46
 
45
47
 
48
+ def _canonical_file(candidate):
49
+ """canonical_root of a file inside a folder: the folder's resolution (cached like a root) plus the file name, when
50
+ one lstat shows the file itself is no symlink or junction. Every file of an index pass was resolved on its own,
51
+ the costliest call of a pass (one handle per file); this keeps one resolution per folder. A path that cannot be
52
+ read is resolved as before."""
53
+ absolute = os.path.abspath(os.path.expanduser(os.fspath(candidate)))
54
+ folder, name = os.path.split(absolute)
55
+ try:
56
+ leaf = os.lstat(absolute)
57
+ except OSError:
58
+ return canonical_root(absolute)
59
+ if not name or stat.S_ISLNK(leaf.st_mode) or getattr(leaf, "st_file_attributes", 0) & _REPARSE_POINT:
60
+ return canonical_root(absolute)
61
+ return os.path.normcase(os.path.join(display_root(folder), name))
62
+
63
+
46
64
  def within_root(root, candidate):
47
65
  try:
48
66
  base = canonical_root(root)
49
- resolved = canonical_root(candidate)
67
+ resolved = _canonical_file(candidate)
50
68
  return os.path.commonpath([base, resolved]) == base
51
69
  except (OSError, ValueError, TypeError):
52
70
  return False
@@ -0,0 +1,98 @@
1
+ """Whole-project scans of the files git tracks, read from disk: credentials committed earlier or sitting in a `.env`
2
+ file (the edit hook only sees the lines an edit writes, and the indexer skips secret files) and relative links of HTML
3
+ and Markdown files that point to nothing. Nothing is written and no value of a credential is returned.
4
+ """
5
+ import os
6
+ import subprocess
7
+
8
+ import credential_check
9
+ import link_check
10
+
11
+ DEFAULT_LIMIT = 25
12
+ MAX_BYTES = 2_000_000
13
+
14
+
15
+ def tracked_files(root):
16
+ """Project-relative paths of the files git tracks under `root` that exist on disk.
17
+
18
+ Parameters: root (str) project folder.
19
+ Returns: list of str with forward slashes, in git's order.
20
+ Raises: ValueError when git cannot list the folder (not a repository, git missing, timeout).
21
+ """
22
+ try:
23
+ listed = subprocess.run(["git", "ls-files", "-z"], cwd=root, capture_output=True, timeout=30,
24
+ creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
25
+ except (OSError, subprocess.TimeoutExpired) as exc:
26
+ raise ValueError(f"git ls-files could not run in {root}: {exc}") from exc
27
+ if listed.returncode:
28
+ raise ValueError(f"git ls-files failed in {root}: {listed.stderr.decode('utf-8', 'replace').strip()[:300]}")
29
+ return [rel for rel in listed.stdout.decode("utf-8", "replace").split("\0")
30
+ if rel and os.path.isfile(os.path.join(root, rel))]
31
+
32
+
33
+ def _text(path):
34
+ """Contents of a file as UTF-8 text; None for a binary, oversized, non-UTF-8 or unreadable file."""
35
+ try:
36
+ if os.path.getsize(path) > MAX_BYTES:
37
+ return None
38
+ with open(path, "rb") as stream:
39
+ data = stream.read()
40
+ return None if b"\0" in data else data.decode("utf-8")
41
+ except (OSError, UnicodeDecodeError):
42
+ return None
43
+
44
+
45
+ def tracked_texts(root, extensions=None):
46
+ """The tracked files of the project with their text, one at a time.
47
+
48
+ Parameters: root (str) project folder; extensions (tuple of str or None) lowercase suffixes to keep, all files when None.
49
+ Returns: iterator of (path, text); text is None for a file that is not readable text.
50
+ Raises: ValueError when git cannot list the folder.
51
+ """
52
+ for rel in tracked_files(root):
53
+ if extensions is None or rel.lower().endswith(extensions):
54
+ yield rel, _text(os.path.join(root, rel))
55
+
56
+
57
+ def credentials(root, limit=DEFAULT_LIMIT):
58
+ """Credential shapes in every tracked text file, with the file and line and never the value.
59
+
60
+ Parameters: root (str) project folder; limit (int) findings returned.
61
+ Returns: dict {"findings": [{"path", "line", "kind"}], "total": int, "by_kind": {kind: count}, "files_checked": int,
62
+ "files_skipped": int}; skipped files are binary, over `MAX_BYTES` or not UTF-8.
63
+ Raises: ValueError when git cannot list the folder.
64
+ """
65
+ findings, checked, skipped = [], 0, 0
66
+ for rel, text in tracked_texts(root):
67
+ if text is None:
68
+ skipped += 1
69
+ continue
70
+ checked += 1
71
+ findings += [{"path": rel, **item} for item in credential_check.review(None, text)]
72
+ by_kind = {}
73
+ for item in findings:
74
+ by_kind[item["kind"]] = by_kind.get(item["kind"], 0) + 1
75
+ return {"findings": findings[:limit], "total": len(findings), "by_kind": by_kind, "files_checked": checked,
76
+ "files_skipped": skipped}
77
+
78
+
79
+ def links(root, limit=DEFAULT_LIMIT):
80
+ """Relative links of the tracked HTML and Markdown files that point to a missing file, and the links that could not
81
+ be decided (root-relative, template expressions, pages with a base element, unparsable files).
82
+
83
+ Parameters: root (str) project folder; limit (int) rows returned in each list.
84
+ Returns: dict {"broken": [{"path", "line", "link"}], "unchecked": [{"path", "line", "link", "why"}],
85
+ "total_broken": int, "total_unchecked": int, "files_checked": int, "files_skipped": int}.
86
+ Raises: ValueError when git cannot list the folder.
87
+ """
88
+ broken, unchecked, checked, skipped = [], [], 0, 0
89
+ for rel, text in tracked_texts(root, link_check.EXTENSIONS):
90
+ if text is None:
91
+ skipped += 1
92
+ continue
93
+ checked += 1
94
+ found = link_check.review(os.path.join(root, rel), None, text)
95
+ broken += [{"path": rel, **item} for item in found["broken"]]
96
+ unchecked += [{"path": rel, **item} for item in found["unchecked"]]
97
+ return {"broken": broken[:limit], "unchecked": unchecked[:limit], "total_broken": len(broken),
98
+ "total_unchecked": len(unchecked), "files_checked": checked, "files_skipped": skipped}
@@ -0,0 +1,125 @@
1
+ """A Markdown draft for the next release, built without a model from facts the project already holds: the commit
2
+ subjects since a base, the public functions the diff touches with their caller and test counts, the decision notes that
3
+ name them and the documentation entries the diff leaves out of date. The sections come from the builders of
4
+ `pr_impact`; every line is a commit subject, a name or a count read from the project, never written text. Nothing is
5
+ published or written.
6
+ """
7
+ import affected_tests
8
+ import doc_check
9
+ import pr_impact
10
+
11
+ DEFAULT_LIMIT = 30
12
+ CALLBACK_SUFFIX = " (callback"
13
+
14
+
15
+ def _base(root, base):
16
+ """The revision the draft starts from.
17
+
18
+ Parameters: root (str) project folder; base (str or None) a tag, branch or commit, the latest tag of the history of
19
+ HEAD when None.
20
+ Returns: str the revision.
21
+ Raises: ValueError when the folder is not a git repository, no tag exists for the default, or the base is not a commit.
22
+ """
23
+ affected_tests._git(root, "rev-parse", "--git-dir")
24
+ if base is None:
25
+ try:
26
+ return affected_tests._git(root, "describe", "--tags", "--abbrev=0").strip()
27
+ except ValueError as exc:
28
+ raise ValueError(f"The repository has no tag to start from ({exc}). Pass base: a tag, branch or commit.") from None
29
+ if not isinstance(base, str) or not base.strip() or base.startswith("-"):
30
+ raise ValueError("base must be a git tag, branch or commit.")
31
+ try:
32
+ affected_tests._git(root, "rev-parse", "--verify", "--quiet", f"{base}^{{commit}}")
33
+ except ValueError:
34
+ raise ValueError(f"base {base!r} is not a commit of this repository.") from None
35
+ return base
36
+
37
+
38
+ def _commits(root, base, limit):
39
+ """Hash and subject of the commits reachable from HEAD and not from the base, newest first.
40
+
41
+ Returns: tuple (list of {"hash", "subject"} up to `limit`, int total).
42
+ """
43
+ rows = [line.partition("\0")[::2] for line in
44
+ affected_tests._git(root, "log", "--format=%h%x00%s", f"{base}..HEAD").splitlines() if line]
45
+ return [{"hash": short, "subject": subject} for short, subject in rows[:limit]], len(rows)
46
+
47
+
48
+ def _public(label):
49
+ """Whether a `path::name` label names a public function: no segment of the name is private (a leading `_` that is not a
50
+ dunder, or `#`)."""
51
+ qualified = label.rpartition("::")[2].split("(")[0]
52
+ return all(doc_check._public_name(part) and not part.startswith("#") for part in qualified.split("."))
53
+
54
+
55
+ def _commit_lines(commits, total, base):
56
+ """The Markdown section of the commit subjects."""
57
+ if not total:
58
+ return ["### Commits", "", f"No commit since {base}.", ""]
59
+ lines = [f"### Commits ({total})", ""] + [f"- {item['hash']} {item['subject']}" for item in commits]
60
+ if total > len(commits):
61
+ lines.append(f"- ... and {total - len(commits)} more commit(s)")
62
+ return lines + [""]
63
+
64
+
65
+ def _function_lines(base, shown, hidden, callers):
66
+ """The Markdown section of the touched public functions with their caller and test counts."""
67
+ lines = ["### Public functions touched and who calls them", "", f"Diff of {base} against the working tree.", ""]
68
+ if not shown:
69
+ return lines + ["No public function is touched by the diff.", ""]
70
+ lines += ["| Function | Callers | Tests |", "|---|---|---|"]
71
+ for label in shown:
72
+ reached = callers[label]
73
+ lines.append(f"| `{label}` | not in the graph | - |" if reached is None
74
+ else f"| `{label}` | {reached[1]} | {reached[2] or 'none'} |")
75
+ return lines + (["", f"{hidden} more public function(s) not listed."] if hidden else []) + [""]
76
+
77
+
78
+ def _decision_lines(shown, decisions, note):
79
+ """The Markdown section of the decision notes that name a touched function, from `pr_impact._decisions`."""
80
+ lines = ["### Decision notes of the touched functions", ""]
81
+ lines += [f"- `{label}`: `{hit['id']}` {hit['title']}" for label, hits in decisions.items() for hit in hits]
82
+ if not decisions:
83
+ lines.append("No decision note or commit message names a touched public function." if shown
84
+ else "No public function is touched, so no decision note was searched.")
85
+ return lines + ([note] if note else []) + [""]
86
+
87
+
88
+ def _stale_lines(stale, note, limit):
89
+ """The Markdown section of the documentation entries the diff leaves out of date, from `pr_impact._stale`."""
90
+ lines = ["### Documentation left out of date", ""]
91
+ if note:
92
+ return lines + [note, ""]
93
+ if not stale:
94
+ return lines + ["No documented entry in the changed files is out of date.", ""]
95
+ lines += [f"- `{f['id']}`: {f['issue'].replace('_', ' ')} ({f['file']}:{f['line']})" for f in stale[:limit]]
96
+ if len(stale) > limit:
97
+ lines.append(f"- ... and {len(stale) - limit} more")
98
+ return lines + [""]
99
+
100
+
101
+ def draft(root, base=None, limit=DEFAULT_LIMIT, view_id=None):
102
+ """The release notes draft since a base.
103
+
104
+ Parameters: root (str) project folder; base (str or None) tag, branch or commit the draft starts from, the latest tag
105
+ when None; limit (int) rows in each list; view_id (str or None) index view.
106
+ Returns: dict {"base", "markdown" (headed `## Draft since <base>`), "commits": [{"hash", "subject"}], "commit_total",
107
+ "functions": [path::name shown], "callers": {path::name: caller count}, "decisions", "stale_anchors", "stale_note"}.
108
+ Raises: ValueError when the folder is not a git repository, there is no tag and no base was given, or the base is not a
109
+ commit; RuntimeError when the code graph cannot be built.
110
+ """
111
+ base = _base(root, base)
112
+ changes = affected_tests.affected(root, base, pr_impact.ALL_ROWS, view_id)
113
+ touched = list(dict.fromkeys(label.partition(CALLBACK_SUFFIX)[0] for label in changes["touched_functions"]))
114
+ public = [label for label in touched if _public(label)]
115
+ shown = public[:limit]
116
+ callers = {label: pr_impact._callers(root, label, view_id) for label in shown}
117
+ decisions, decisions_note = pr_impact._decisions(root, shown)
118
+ stale, stale_note = pr_impact._stale(root, view_id, set(changes["changed"]["code"]))
119
+ commits, total = _commits(root, base, limit)
120
+ lines = [f"## Draft since {base}", ""] + _commit_lines(commits, total, base)
121
+ lines += _function_lines(base, shown, len(public) - len(shown), callers)
122
+ lines += _decision_lines(shown, decisions, decisions_note) + _stale_lines(stale, stale_note, limit)
123
+ return {"base": base, "markdown": "\n".join(lines).rstrip("\n") + "\n", "commits": commits, "commit_total": total,
124
+ "functions": shown, "callers": {label: reached[1] if reached else 0 for label, reached in callers.items()},
125
+ "decisions": decisions, "stale_anchors": stale[:limit], "stale_note": stale_note}
@@ -0,0 +1,95 @@
1
+ """What renaming a function leaves behind, asked before the rename: where it is defined, who calls it (the calls the
2
+ graph resolved and the calls it left unresolved under that name), the tests that reach it, the files that import its
3
+ module, the anchors of the generated documents and the lines of the README and `docs` text that name it. Nothing is
4
+ written; the lists are the places an agent edits or checks after changing the name.
5
+ """
6
+ import posixpath
7
+
8
+ import code_graph
9
+ import code_impact
10
+ import doc_scope
11
+ import dossier
12
+ import project_scan
13
+ import removed_callers
14
+
15
+ DEFAULT_LIMIT = 30
16
+ TEST_DEPTH = 3
17
+ DOC_EXTENSIONS = (".md", ".rst", ".txt", ".html")
18
+ MAX_LINE_CHARS = 160
19
+ NOTE = ("A rename also needs a text search of the old name for what static analysis cannot see: strings, dynamic access "
20
+ "and names built at run time.")
21
+
22
+
23
+ def _is_doc(rel):
24
+ """Whether a tracked file is a README or sits in a `docs` folder, outside the generated documents (their anchors are
25
+ listed apart)."""
26
+ if rel.startswith(doc_scope.DOCS_DIR + "/") or not rel.lower().endswith(DOC_EXTENSIONS):
27
+ return False
28
+ return posixpath.basename(rel).upper().startswith("README") or "docs" in rel.split("/")[:-1]
29
+
30
+
31
+ def _unresolved_calls(data, definitions, limit):
32
+ """Calls the graph could not resolve whose expression can mean one of the definitions, as `removed_callers` reads them.
33
+
34
+ Returns: tuple (list of {"path", "line", "expression"} up to `limit`, int total).
35
+ """
36
+ found = {}
37
+ for definition in definitions:
38
+ short = definition["name"].rpartition(".")[2]
39
+ stem = posixpath.splitext(posixpath.basename(definition["path"]))[0]
40
+ for call in data.get("unresolved") or []:
41
+ if removed_callers._names(call["expression"], short, stem, "." in definition["name"]):
42
+ found[(call["path"], call["line"], call["expression"])] = None
43
+ rows = [{"path": path, "line": line, "expression": expression} for path, line, expression in sorted(found)]
44
+ return rows[:limit], len(rows)
45
+
46
+
47
+ def _mentions(root, definitions, limit):
48
+ """Lines of the tracked README and `docs` text that name a definition: `path::name`, `module.name` or `Owner.name`,
49
+ the name in backticks, or a compound name on its own (`dossier.mention_test`, which keeps a single plain word out).
50
+
51
+ Returns: tuple (list of {"path", "line", "text"} up to `limit`, int total).
52
+ Raises: ValueError when git cannot list the folder.
53
+ """
54
+ tests = [dossier.mention_test(item["path"], item["name"]) for item in definitions]
55
+ literals = [f"{item['path']}::{item['name']}" for item in definitions]
56
+ quoted = [f"`{item['name'].rpartition('.')[2]}`" for item in definitions]
57
+ quoted += [text[:-1] + "(" for text in quoted]
58
+ found, total = [], 0
59
+ for rel, text in project_scan.tracked_texts(root, DOC_EXTENSIONS):
60
+ if text is None or not _is_doc(rel):
61
+ continue
62
+ for number, line in enumerate(text.splitlines(), 1):
63
+ if any(test(line) for test in tests) or any(literal in line for literal in literals) \
64
+ or any(mark in line for mark in quoted):
65
+ total += 1
66
+ if len(found) < limit:
67
+ found.append({"path": rel, "line": number, "text": line.strip()[:MAX_LINE_CHARS]})
68
+ return found, total
69
+
70
+
71
+ def preview(root, symbol, view_id=None, limit=DEFAULT_LIMIT):
72
+ """Everything a rename of one function touches.
73
+
74
+ Parameters: root (str) project folder; symbol (str) function, `Class.method`, class or `path::name`; view_id (str or
75
+ None) index view; limit (int) rows in each list.
76
+ Returns: dict {"symbol", "definitions": [{"name", "kind", "path", "lines"}], "callers": [{"function", "path", "line"}],
77
+ "unresolved_calls": [{"path", "line", "expression"}], "tests", "imported_by": [path], "doc_anchors": [{"file", "line",
78
+ "id", "level", "current"}], "doc_mentions": [{"path", "line", "text"}], "counts": {list: total}, "doc_scope": str
79
+ ("" when docs/scope exists, otherwise why no anchor could be listed), "note": str}.
80
+ Raises: ValueError when the symbol is not found (with the closest names), the folder is not a git repository or a page of
81
+ docs/scope is in an older format; RuntimeError when the code graph cannot be built.
82
+ """
83
+ found = code_impact.impact(root, symbol, view_id, TEST_DEPTH, limit)
84
+ data = code_graph.build_current(root, view_id, shared=True)
85
+ unresolved, unresolved_total = _unresolved_calls(data, found["definitions"], limit)
86
+ mentions, mentions_total = _mentions(root, found["definitions"], limit)
87
+ anchored = doc_scope.anchors(root, symbol, view_id)
88
+ anchors = anchored["anchors"]
89
+ counts = {**found["counts"], "unresolved_calls": unresolved_total, "doc_anchors": len(anchors),
90
+ "doc_mentions": mentions_total}
91
+ counts.pop("calls")
92
+ return {"symbol": found["symbol"], "definitions": found["definitions"], "callers": found["callers"],
93
+ "unresolved_calls": unresolved, "tests": found["tests"], "imported_by": found["imported_by"],
94
+ "doc_anchors": anchors[:limit], "doc_mentions": mentions, "counts": counts,
95
+ "doc_scope": anchored.get("guidance", ""), "note": f"{found['note']} {NOTE}"}
package/requester.py CHANGED
@@ -13,7 +13,7 @@ import json
13
13
  import threading
14
14
  import time
15
15
 
16
- WEB_TOOLS = ("web_search", "web_fetch")
16
+ WEB_TOOLS = ("web_search", "web_fetch", "web_fetch_many")
17
17
  PROJECT_TOOLS = ("smart_search", "smart_scout", "project_manage")
18
18
  NOTE_TTL_S = 60
19
19
  MAX_NOTES = 256