@allansantos-dev/smart-tool 0.9.80 → 0.9.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +129 -0
- package/NOTICE +1 -1
- package/README.md +59 -14
- package/agent_redirect.py +32 -0
- package/autostart.py +32 -25
- package/browser_bridge.py +320 -0
- package/browser_control.py +76 -39
- package/browser_service_client.py +1 -7
- package/client_hooks.py +42 -41
- package/code_graph.py +56 -15
- package/code_graph_java.mjs +21 -12
- package/code_health.py +19 -12
- package/code_impact.py +1 -1
- package/codex_config.py +3 -0
- package/config.py +35 -0
- package/daemon_launcher.py +37 -5
- package/doc_candidates.py +63 -0
- package/doc_check.py +9 -3
- package/doc_extras.py +2 -3
- package/doc_insights.py +26 -2
- package/doc_prose.py +59 -11
- package/doc_registry.py +241 -0
- package/doc_scope.py +205 -39
- package/doc_turn.py +205 -0
- package/doc_validate.py +52 -9
- package/document_read.py +4 -2
- package/dossier.py +40 -5
- package/edit_shape.py +119 -0
- package/endpoint_sync.py +3 -21
- package/engine_home.py +95 -0
- package/hook_call.py +132 -0
- package/hook_decision.py +337 -23
- package/hooks/pretool_router.py +36 -29
- package/hooks/turn_end.py +40 -0
- package/index_views.py +5 -0
- package/install.py +90 -36
- package/install_runtime.py +24 -10
- package/lazy_mcp_proxy.py +10 -9
- package/link_check.py +69 -0
- package/local_embedder.py +9 -2
- package/locales/pt-BR.json +72 -4
- package/model_breaker.py +6 -5
- package/model_client.py +1 -1
- package/model_health.py +24 -6
- package/notice_action.py +9 -0
- package/package.json +2 -2
- package/pr_comment.py +94 -0
- package/pr_impact.py +216 -35
- package/process_limits.py +17 -3
- package/project_admission.py +17 -15
- package/project_identity.py +19 -1
- package/project_scan.py +98 -0
- package/removed_callers.py +146 -0
- package/scout_access.py +48 -0
- package/setup_ui.py +33 -4
- package/shell_reads.py +67 -0
- package/smart_tool_daemon.py +149 -39
- package/tray.py +60 -9
- package/update_splash.py +2 -2
- package/venv_launch.py +67 -0
- package/version.py +1 -1
- package/web_adapters/node/engine-home.mjs +42 -0
- package/web_adapters/node/package-lock.json +13 -0
- package/web_adapters/node/package.json +1 -0
- package/web_document_graph.py +12 -1
package/install.py
CHANGED
|
@@ -15,7 +15,11 @@ from pathlib import Path
|
|
|
15
15
|
|
|
16
16
|
import atomic_io
|
|
17
17
|
import client_hooks
|
|
18
|
+
import config
|
|
19
|
+
import daemon_launcher
|
|
20
|
+
import i18n
|
|
18
21
|
import paths
|
|
22
|
+
import venv_launch
|
|
19
23
|
import version
|
|
20
24
|
|
|
21
25
|
SOURCE = Path(__file__).resolve().parent
|
|
@@ -25,6 +29,7 @@ ENTRY_SCRIPTS = ("tray.py", "smart_tool_daemon.py", "pretool_router.py", "sessio
|
|
|
25
29
|
HOOK_SCRIPTS = ("pretool_router.py", "session_audit.py")
|
|
26
30
|
STARTUP_SCRIPT = Path(os.environ.get("APPDATA", str(Path.home() / "AppData" / "Roaming"))) / "Microsoft" / "Windows" / \
|
|
27
31
|
"Start Menu" / "Programs" / "Startup" / "SmartTool-Tray.vbs"
|
|
32
|
+
TASK_SCRIPT = Path(paths.DATA_DIR) / "tray-start.vbs"
|
|
28
33
|
MANIFEST = ".install-manifest.json"
|
|
29
34
|
DEPENDENCY_INPUTS = ("requirements.txt", "install_runtime.py", "node_locator.py", "web_adapters/node/package.json",
|
|
30
35
|
"web_adapters/node/package-lock.json", "web_adapters/browser/requirements.txt",
|
|
@@ -36,7 +41,32 @@ BLOCKED = {"__pycache__", ".token-guard", ".git", ".pytest_cache", ".data", ".ar
|
|
|
36
41
|
".scratch", ".memory", "docs", "reports", "site", "design", "release", "npm", ".github", "AGENTS.md", "CLAUDE.md", "DESIGN.md", "PRODUCT.md", "install.py", "uninstall.py", ".gitignore", ".gitattributes", MANIFEST}
|
|
37
42
|
PACKAGE_ONLY = {"bin", "package.json", ".npmignore"}
|
|
38
43
|
REQUIRED_TOOLS = {"smart_search", "smart_search_result", "web_search", "project_manage"}
|
|
44
|
+
LANGUAGE = "en"
|
|
39
45
|
_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
46
|
+
def installer_language(requested=None):
|
|
47
|
+
"""The language of the installer's messages: requested (--language), else the one picked in the settings, else the
|
|
48
|
+
Windows display language. Settings that cannot be read, or that hold an unknown language, are reported on stderr
|
|
49
|
+
and the Windows language is used."""
|
|
50
|
+
if requested:
|
|
51
|
+
return requested
|
|
52
|
+
problem, chosen = config.corrupted_reason(), ""
|
|
53
|
+
if not problem:
|
|
54
|
+
try:
|
|
55
|
+
chosen = config.language()
|
|
56
|
+
except (ValueError, config.ConfigCorruptedError) as exc:
|
|
57
|
+
problem = str(exc)
|
|
58
|
+
fallback = i18n.system_language()
|
|
59
|
+
if problem:
|
|
60
|
+
print(i18n.t("Could not read the Smart Tool settings, so the Windows language is used instead: {problem}",
|
|
61
|
+
lang=fallback, problem=problem), file=sys.stderr)
|
|
62
|
+
return chosen or fallback
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def t(text, **values):
|
|
66
|
+
"""text (an English catalog key) in LANGUAGE, with its {placeholders} filled from values."""
|
|
67
|
+
return i18n.t(text, lang=LANGUAGE, **values)
|
|
68
|
+
|
|
69
|
+
|
|
40
70
|
def ships(rel):
|
|
41
71
|
parts = Path(rel).parts
|
|
42
72
|
*folders, name = parts
|
|
@@ -86,11 +116,15 @@ def _same_folder(path, folder):
|
|
|
86
116
|
|
|
87
117
|
|
|
88
118
|
def stop_processes(target):
|
|
119
|
+
"""Closes every process running a script of the installation in `target` (daemon, tray, browser proxies); a closed
|
|
120
|
+
daemon gets its reason in the daemon log, since a forced stop leaves it no chance to write one."""
|
|
89
121
|
def running():
|
|
90
|
-
return [pid for pid, line in _processes() for script in _script_paths(line)
|
|
122
|
+
return [(pid, Path(script).name) for pid, line in _processes() for script in _script_paths(line)
|
|
91
123
|
if Path(script).name not in HOOK_SCRIPTS and _same_folder(_script_folder(script), target)]
|
|
92
124
|
|
|
93
|
-
for pid in running():
|
|
125
|
+
for pid, script in running():
|
|
126
|
+
if script == Path(daemon_launcher.DAEMON_SCRIPT).name:
|
|
127
|
+
daemon_launcher.note_stop(f"daemon pid {pid} closed by the installer to update {target}")
|
|
94
128
|
powershell(f"Stop-Process -Id {pid} -Force -ErrorAction SilentlyContinue")
|
|
95
129
|
deadline = time.monotonic() + 10
|
|
96
130
|
while True:
|
|
@@ -98,7 +132,7 @@ def stop_processes(target):
|
|
|
98
132
|
if not left:
|
|
99
133
|
return
|
|
100
134
|
if time.monotonic() > deadline:
|
|
101
|
-
raise RuntimeError(
|
|
135
|
+
raise RuntimeError(t("Could not close the running Smart Tool ({count} process(es) still active).", count=len(left)))
|
|
102
136
|
time.sleep(0.5)
|
|
103
137
|
|
|
104
138
|
|
|
@@ -123,7 +157,7 @@ def wait_port_free(target):
|
|
|
123
157
|
stop_processes(target)
|
|
124
158
|
time.sleep(0.5)
|
|
125
159
|
if _daemon_up():
|
|
126
|
-
raise RuntimeError("Another Smart Tool daemon is still active on the registered port; close it before updating.")
|
|
160
|
+
raise RuntimeError(t("Another Smart Tool daemon is still active on the registered port; close it before updating."))
|
|
127
161
|
|
|
128
162
|
|
|
129
163
|
def _daemon_up():
|
|
@@ -165,12 +199,19 @@ def _task_definition():
|
|
|
165
199
|
return result.stdout if result.returncode == 0 else ""
|
|
166
200
|
|
|
167
201
|
|
|
202
|
+
def _script_text(path):
|
|
203
|
+
"""Text of a launch script or hook file, "" when absent: VBScript files are written in UTF-16, the rest in UTF-8."""
|
|
204
|
+
if not path.is_file():
|
|
205
|
+
return ""
|
|
206
|
+
data = path.read_bytes()
|
|
207
|
+
return data.decode("utf-16") if data[:2] in (b"\xff\xfe", b"\xfe\xff") else data.decode("utf-8", errors="replace")
|
|
208
|
+
|
|
209
|
+
|
|
168
210
|
def other_installs(target):
|
|
169
211
|
"""Installer-made Smart Tool folders other than target, found through what still runs or points at them: tray and
|
|
170
|
-
daemon processes, the logon task
|
|
212
|
+
daemon processes, the logon task and the script it runs, the Startup script, and agent hook commands."""
|
|
171
213
|
texts = [line for _pid, line in _processes()] + [_task_definition()]
|
|
172
|
-
texts += [path
|
|
173
|
-
if path.is_file()]
|
|
214
|
+
texts += [_script_text(path) for path in (TASK_SCRIPT, STARTUP_SCRIPT, *hook_files())]
|
|
174
215
|
folders = {_install_folder(script) for text in texts for script in _script_paths(text)}
|
|
175
216
|
return sorted(folder for folder in folders
|
|
176
217
|
if folder and not _same_folder(folder, target) and not _same_folder(folder, SOURCE))
|
|
@@ -193,7 +234,7 @@ def remove_other_installs(target):
|
|
|
193
234
|
removed = []
|
|
194
235
|
codex = hook_files()[1]
|
|
195
236
|
for folder in other_installs(target):
|
|
196
|
-
print(
|
|
237
|
+
print(t("Older Smart Tool installation found at {folder}; removing it.", folder=folder))
|
|
197
238
|
stop_processes(folder)
|
|
198
239
|
for path in hook_files():
|
|
199
240
|
original = path.read_text(encoding="utf-8") if path.is_file() else ""
|
|
@@ -203,10 +244,10 @@ def remove_other_installs(target):
|
|
|
203
244
|
atomic_io.backup_file(str(path))
|
|
204
245
|
path.write_text(text, encoding="utf-8")
|
|
205
246
|
client_hooks.record_codex_trust(trust, str(path.with_name("config.toml")))
|
|
206
|
-
print(
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
247
|
+
print(t("Hooks in {path} now point to {target}.", path=path, target=target))
|
|
248
|
+
for script_path in (TASK_SCRIPT, STARTUP_SCRIPT):
|
|
249
|
+
if any(_same_folder(_script_folder(script), folder) for script in _script_paths(_script_text(script_path))):
|
|
250
|
+
script_path.unlink()
|
|
210
251
|
shutil.rmtree(folder)
|
|
211
252
|
removed.append(folder)
|
|
212
253
|
return removed
|
|
@@ -327,7 +368,7 @@ def prune_camoufox(target):
|
|
|
327
368
|
browser = target / "web_adapters" / "browser"
|
|
328
369
|
result = subprocess.run([str(browser / ".venv" / "Scripts" / "python.exe"), str(browser / "prune_camoufox.py")])
|
|
329
370
|
if result.returncode:
|
|
330
|
-
print(
|
|
371
|
+
print(t("Old Camoufox builds not pruned: prune_camoufox.py exited {code}.", code=result.returncode), file=sys.stderr)
|
|
331
372
|
|
|
332
373
|
|
|
333
374
|
def install_dependencies(target, python):
|
|
@@ -335,13 +376,14 @@ def install_dependencies(target, python):
|
|
|
335
376
|
venv.create(python.parent.parent, with_pip=True)
|
|
336
377
|
subprocess.run([str(python), "-m", "pip", "install", "--disable-pip-version-check", "-q", "-r",
|
|
337
378
|
str(target / "requirements.txt")], check=True)
|
|
338
|
-
subprocess.run([str(python), str(target / "install_runtime.py")], check=True)
|
|
379
|
+
subprocess.run([str(python), str(target / "install_runtime.py"), "--language", LANGUAGE], check=True)
|
|
339
380
|
|
|
340
381
|
|
|
341
382
|
def start(target, python, pythonw):
|
|
342
383
|
subprocess.run([str(python), "-c", "import autostart, json; print(json.dumps(autostart.install()))"],
|
|
343
384
|
cwd=target, check=True, capture_output=True, text=True, timeout=60)
|
|
344
|
-
|
|
385
|
+
interpreter, additions = venv_launch.direct(pythonw if pythonw.exists() else python)
|
|
386
|
+
subprocess.Popen([interpreter, str(target / "tray.py")], cwd=target, env=venv_launch.environment(additions),
|
|
345
387
|
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
346
388
|
creationflags=getattr(subprocess, "DETACHED_PROCESS", 0) | getattr(subprocess, "CREATE_NO_WINDOW", 0))
|
|
347
389
|
|
|
@@ -357,17 +399,20 @@ def verify(target, python):
|
|
|
357
399
|
"print(json.dumps({'version':running,'tools':sorted(r['name'] for r in json.load(opener.open(req,timeout=10))['result']['tools'])}))")
|
|
358
400
|
result = subprocess.run([str(python), "-c", script], cwd=target, capture_output=True, text=True, timeout=60)
|
|
359
401
|
if result.returncode:
|
|
360
|
-
|
|
402
|
+
detail = ((result.stderr or result.stdout).strip().splitlines() or [t("no output")])[-1]
|
|
403
|
+
raise RuntimeError(t("Daemon did not respond: {detail}", detail=detail))
|
|
361
404
|
served = json.loads(result.stdout.strip().splitlines()[-1])
|
|
362
405
|
missing = REQUIRED_TOOLS - set(served["tools"])
|
|
363
406
|
if missing:
|
|
364
|
-
raise RuntimeError("Missing MCP tools: "
|
|
407
|
+
raise RuntimeError(t("Missing MCP tools: {tools}", tools=", ".join(sorted(missing))))
|
|
365
408
|
if served["version"] != version.VERSION:
|
|
366
|
-
raise RuntimeError(
|
|
409
|
+
raise RuntimeError(t("The daemon runs version {running}, not the {expected} being installed",
|
|
410
|
+
running=served["version"], expected=version.VERSION))
|
|
367
411
|
pid = json.loads(REGISTRY.read_text(encoding="utf-8"))["pid"]
|
|
368
412
|
command = powershell(f"(Get-CimInstance Win32_Process -Filter 'ProcessId = {int(pid)}').CommandLine").stdout
|
|
369
413
|
if str(target / "smart_tool_daemon.py").lower() not in command.lower():
|
|
370
|
-
raise RuntimeError(
|
|
414
|
+
raise RuntimeError(t("The active daemon (pid {pid}) does not belong to this installation: {command}",
|
|
415
|
+
pid=pid, command=command.strip()[:200]))
|
|
371
416
|
|
|
372
417
|
|
|
373
418
|
def enable_browser_control(target, python):
|
|
@@ -382,11 +427,12 @@ def enable_browser_control(target, python):
|
|
|
382
427
|
except (OSError, subprocess.TimeoutExpired, ValueError, IndexError) as exc:
|
|
383
428
|
result, outcome = None, f"{type(exc).__name__}: {exc}"
|
|
384
429
|
if not isinstance(outcome, dict):
|
|
385
|
-
detail = outcome or ((result.stderr or result.stdout).strip().splitlines() or ["no output"])[-1]
|
|
386
|
-
print(
|
|
430
|
+
detail = outcome or ((result.stderr or result.stdout).strip().splitlines() or [t("no output")])[-1]
|
|
431
|
+
print(t("Browser control not set up: {detail}", detail=detail))
|
|
387
432
|
return
|
|
388
433
|
for client, state in outcome.items():
|
|
389
|
-
print(
|
|
434
|
+
print(t("Browser control in {client}: {state}", client=client, state=state["state"])
|
|
435
|
+
+ (f" ({state['reason']})" if state.get("reason") else ""))
|
|
390
436
|
|
|
391
437
|
|
|
392
438
|
MIN_PYTHON = (3, 11)
|
|
@@ -404,18 +450,23 @@ def main():
|
|
|
404
450
|
help="install without running tests/run.py (npm package, whose tests ran in CI; emergency only otherwise)")
|
|
405
451
|
parser.add_argument("--expect-version",
|
|
406
452
|
help="refuse to install unless this package is that version (the update screen passes the one it announced)")
|
|
453
|
+
parser.add_argument("--language", choices=config.LANGUAGES,
|
|
454
|
+
help="language of the messages (default: the setting, else the Windows display language)")
|
|
407
455
|
args = parser.parse_args()
|
|
456
|
+
global LANGUAGE
|
|
457
|
+
LANGUAGE = installer_language(args.language)
|
|
408
458
|
if args.expect_version and args.expect_version != version.VERSION:
|
|
409
|
-
raise SystemExit(
|
|
410
|
-
|
|
459
|
+
raise SystemExit(t("This package is Smart Tool {version}, not the {requested} requested; nothing was changed.",
|
|
460
|
+
version=version.VERSION, requested=args.expect_version))
|
|
411
461
|
paths.refuse_overridden_folders("install")
|
|
412
462
|
target = args.target.resolve()
|
|
413
463
|
if target == SOURCE:
|
|
414
|
-
raise SystemExit("The target cannot be the source folder itself.")
|
|
464
|
+
raise SystemExit(t("The target cannot be the source folder itself."))
|
|
415
465
|
if args.skip_tests:
|
|
416
|
-
print("Tests skipped by --skip-tests.")
|
|
466
|
+
print(t("Tests skipped by --skip-tests."))
|
|
417
467
|
elif not (SOURCE / "tests" / "run.py").is_file():
|
|
418
|
-
raise SystemExit(
|
|
468
|
+
raise SystemExit(t("No test suite in {source}; a packaged install runs with --skip-tests "
|
|
469
|
+
"(npx @allansantos-dev/smart-tool@{version} install).", source=SOURCE, version=version.VERSION))
|
|
419
470
|
else:
|
|
420
471
|
interpreter = python_paths(target)[0] if python_paths(target)[0].is_file() else Path(sys.executable)
|
|
421
472
|
tests = subprocess.run([str(interpreter), str(SOURCE / "tests" / "run.py")], capture_output=True, text=True,
|
|
@@ -424,11 +475,13 @@ def main():
|
|
|
424
475
|
if tests.stderr.strip():
|
|
425
476
|
print(tests.stderr.rstrip()[-3000:], file=sys.stderr)
|
|
426
477
|
if tests.returncode:
|
|
427
|
-
raise SystemExit("Tests failed; installation aborted without touching the installed version.")
|
|
478
|
+
raise SystemExit(t("Tests failed; installation aborted without touching the installed version."))
|
|
428
479
|
python, pythonw = python_paths(target)
|
|
429
480
|
files = source_files()
|
|
430
481
|
existed = (target / "smart_tool_daemon.py").is_file()
|
|
431
|
-
|
|
482
|
+
where = {"target": target, "count": len(files), "source": SOURCE}
|
|
483
|
+
print(t("Updating Smart Tool at {target} ({count} files from {source})", **where) if existed
|
|
484
|
+
else t("Installing Smart Tool at {target} ({count} files from {source})", **where))
|
|
432
485
|
target.mkdir(parents=True, exist_ok=True)
|
|
433
486
|
remove_other_installs(target)
|
|
434
487
|
mark_updating(True)
|
|
@@ -439,8 +492,8 @@ def main():
|
|
|
439
492
|
if not args.no_start:
|
|
440
493
|
enable_browser_control(target, python)
|
|
441
494
|
for folder in prune_backups(target):
|
|
442
|
-
print(
|
|
443
|
-
print("Smart Tool ready." if not args.no_start else "Files installed; daemon not restarted (--no-start).")
|
|
495
|
+
print(t("Removed the old copy {folder}.", folder=folder))
|
|
496
|
+
print(t("Smart Tool ready.") if not args.no_start else t("Files installed; daemon not restarted (--no-start)."))
|
|
444
497
|
|
|
445
498
|
|
|
446
499
|
def prune_backups(target):
|
|
@@ -456,7 +509,7 @@ def prune_backups(target):
|
|
|
456
509
|
shutil.rmtree(folder)
|
|
457
510
|
removed.append(str(folder))
|
|
458
511
|
except OSError as exc:
|
|
459
|
-
print(
|
|
512
|
+
print(t("Could not remove the old copy {folder}: {error}", folder=folder, error=exc), file=sys.stderr)
|
|
460
513
|
return removed
|
|
461
514
|
|
|
462
515
|
|
|
@@ -474,19 +527,19 @@ def _update(args, target, files, existed, python, pythonw):
|
|
|
474
527
|
try:
|
|
475
528
|
copy_files(target, files, backup, changed, removed)
|
|
476
529
|
clear_bytecode(target)
|
|
477
|
-
print(
|
|
530
|
+
print(t("Files changed: {changed}; removed as obsolete: {removed}", changed=len(changed), removed=len(removed)))
|
|
478
531
|
if run_deps:
|
|
479
532
|
frozen = freeze(python)
|
|
480
533
|
deps_touched = True
|
|
481
534
|
install_dependencies(target, python)
|
|
482
535
|
elif not args.skip_deps:
|
|
483
|
-
print("Dependencies unchanged since the last update; pip and install_runtime.py skipped.")
|
|
536
|
+
print(t("Dependencies unchanged since the last update; pip and install_runtime.py skipped."))
|
|
484
537
|
if not args.no_start:
|
|
485
538
|
mark_updating(False)
|
|
486
539
|
start(target, python, pythonw)
|
|
487
540
|
verify(target, python)
|
|
488
541
|
except BaseException as exc:
|
|
489
|
-
print(
|
|
542
|
+
print(t("Failed: {error}. Restoring the previous version.", error=exc), file=sys.stderr)
|
|
490
543
|
problems = []
|
|
491
544
|
steps = [lambda: mark_updating(True), lambda: stop_processes(target),
|
|
492
545
|
lambda: restore(target, backup, changed, removed)]
|
|
@@ -500,7 +553,8 @@ def _update(args, target, files, existed, python, pythonw):
|
|
|
500
553
|
except Exception as error:
|
|
501
554
|
problems.append(str(error))
|
|
502
555
|
if problems:
|
|
503
|
-
print(
|
|
556
|
+
print(t("Incomplete rollback; backup kept at {backup}: {problems}", backup=backup, problems=" | ".join(problems)),
|
|
557
|
+
file=sys.stderr)
|
|
504
558
|
raise SystemExit(1)
|
|
505
559
|
shutil.rmtree(backup, ignore_errors=True)
|
|
506
560
|
raise SystemExit(1)
|
package/install_runtime.py
CHANGED
|
@@ -21,6 +21,7 @@ import venv
|
|
|
21
21
|
import zipfile
|
|
22
22
|
from pathlib import Path
|
|
23
23
|
|
|
24
|
+
import i18n
|
|
24
25
|
import node_locator
|
|
25
26
|
|
|
26
27
|
|
|
@@ -43,6 +44,12 @@ MOLI_URL = f"https://github.com/lexmount/moli/releases/download/v{MOLI_VERSION}/
|
|
|
43
44
|
MOLI_SHA256 = "64c00ce02d6db8e1c22c55e88fbcaaf3e7517e930f24c25ff46a457b421040c8"
|
|
44
45
|
MOLI_DIR = BROWSER_DIR / "moli"
|
|
45
46
|
MOLI_EXE = MOLI_DIR / "moli.exe"
|
|
47
|
+
LANGUAGE = "en"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def t(text, **values):
|
|
51
|
+
"""text (an English catalog key) in LANGUAGE, with its {placeholders} filled from values."""
|
|
52
|
+
return i18n.t(text, lang=LANGUAGE, **values)
|
|
46
53
|
|
|
47
54
|
|
|
48
55
|
def _node_supported(node):
|
|
@@ -119,7 +126,7 @@ def _replace_folder(unpacked, final, temp_dir):
|
|
|
119
126
|
|
|
120
127
|
def _install_node():
|
|
121
128
|
"""Baixa o Node LTS oficial e confere SHA-256 antes de extrair."""
|
|
122
|
-
print("Downloading Node LTS for the Smart Tool web runtime...", flush=True)
|
|
129
|
+
print(t("Downloading Node LTS for the Smart Tool web runtime..."), flush=True)
|
|
123
130
|
with urllib.request.urlopen("https://nodejs.org/dist/index.json", timeout=30) as response:
|
|
124
131
|
releases = json.load(response)
|
|
125
132
|
release = next((row for row in releases if row.get("lts") and
|
|
@@ -144,7 +151,7 @@ def _install_node():
|
|
|
144
151
|
if not (unpacked / "node.exe").is_file() or not (unpacked / "npm.cmd").is_file():
|
|
145
152
|
raise RuntimeError("Node package does not contain the expected executables.")
|
|
146
153
|
_replace_folder(unpacked, NODE_RUNTIME, temp_dir)
|
|
147
|
-
print(
|
|
154
|
+
print(t("Node {version} installed inside Smart Tool.", version=version))
|
|
148
155
|
|
|
149
156
|
|
|
150
157
|
def _moli_ready():
|
|
@@ -156,7 +163,7 @@ def _moli_ready():
|
|
|
156
163
|
|
|
157
164
|
def _install_moli():
|
|
158
165
|
"""Downloads the pinned Moli release from GitHub, checks its SHA-256 and unpacks it into web_adapters/browser."""
|
|
159
|
-
print(
|
|
166
|
+
print(t("Downloading Moli {version} (structured-first headless browser, 43 MB)...", version=MOLI_VERSION), flush=True)
|
|
160
167
|
BROWSER_DIR.mkdir(parents=True, exist_ok=True)
|
|
161
168
|
with tempfile.TemporaryDirectory(prefix="moli-setup-", dir=BROWSER_DIR) as temp:
|
|
162
169
|
temp_dir = Path(temp)
|
|
@@ -167,7 +174,7 @@ def _install_moli():
|
|
|
167
174
|
if not (unpacked / "moli.exe").is_file():
|
|
168
175
|
raise RuntimeError("The Moli package does not contain moli.exe.")
|
|
169
176
|
_replace_folder(unpacked, MOLI_DIR, temp_dir)
|
|
170
|
-
print(
|
|
177
|
+
print(t("Moli {version} installed inside Smart Tool.", version=MOLI_VERSION))
|
|
171
178
|
|
|
172
179
|
|
|
173
180
|
def _run(args, *, cwd=None, label="Web dependency", idle_s=600, stream=False, attempts=1):
|
|
@@ -179,15 +186,19 @@ def _run(args, *, cwd=None, label="Web dependency", idle_s=600, stream=False, at
|
|
|
179
186
|
env.setdefault("NODE_USE_SYSTEM_CA", "1")
|
|
180
187
|
flags = (subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW) if os.name == "nt" else 0
|
|
181
188
|
for attempt in range(1, attempts + 1):
|
|
182
|
-
|
|
189
|
+
if attempt > 1:
|
|
190
|
+
print(t("{label}: starting (attempt {attempt} of {attempts})...", label=label, attempt=attempt,
|
|
191
|
+
attempts=attempts), flush=True)
|
|
192
|
+
else:
|
|
193
|
+
print(t("{label}: starting...", label=label), flush=True)
|
|
183
194
|
try:
|
|
184
195
|
_run_once(args, cwd, env, flags, label, idle_s, stream)
|
|
185
|
-
print(
|
|
196
|
+
print(t("{label}: done.", label=label), flush=True)
|
|
186
197
|
return
|
|
187
198
|
except RuntimeError as exc:
|
|
188
199
|
if attempt == attempts:
|
|
189
200
|
raise
|
|
190
|
-
print(
|
|
201
|
+
print(t("{error}; trying again.", error=exc), flush=True)
|
|
191
202
|
|
|
192
203
|
|
|
193
204
|
def _run_once(args, cwd, env, flags, label, idle_s, stream):
|
|
@@ -229,7 +240,7 @@ def _run_once(args, cwd, env, flags, label, idle_s, stream):
|
|
|
229
240
|
raise RuntimeError(f"{label} stopped: no output for {idle_s}s; last output: {tail[-1] if tail else 'none'}")
|
|
230
241
|
if not stream and now - noted >= 30:
|
|
231
242
|
noted = now
|
|
232
|
-
print(
|
|
243
|
+
print(t("{label}: waiting for {seconds}s...", label=label, seconds=int(now - started)), flush=True)
|
|
233
244
|
time.sleep(0.5)
|
|
234
245
|
reader.join(timeout=10)
|
|
235
246
|
if process.returncode:
|
|
@@ -374,7 +385,7 @@ def install(reuse_browser_runtime=None):
|
|
|
374
385
|
status = _check()
|
|
375
386
|
if not all(status.values()):
|
|
376
387
|
raise RuntimeError(f"Incomplete web runtime: {status}")
|
|
377
|
-
print("Smart Tool runtime installed: Node, open-websearch, TypeScript, Java, Angular, Playwright MCP, Camoufox, Crawl4AI and Moli.")
|
|
388
|
+
print(t("Smart Tool runtime installed: Node, open-websearch, TypeScript, Java, Angular, Playwright MCP, Camoufox, Crawl4AI and Moli."))
|
|
378
389
|
|
|
379
390
|
|
|
380
391
|
def main():
|
|
@@ -382,7 +393,10 @@ def main():
|
|
|
382
393
|
parser.add_argument("--check", action="store_true", help="Show the status without installing")
|
|
383
394
|
parser.add_argument("--reuse-browser-runtime", metavar="FOLDER",
|
|
384
395
|
help="Reuse an existing venv without duplicating files (useful when disk is low)")
|
|
396
|
+
parser.add_argument("--language", choices=i18n.config.LANGUAGES, default="en", help="language of the messages")
|
|
385
397
|
args = parser.parse_args()
|
|
398
|
+
global LANGUAGE
|
|
399
|
+
LANGUAGE = args.language
|
|
386
400
|
if args.check:
|
|
387
401
|
status = _check()
|
|
388
402
|
for name, ready in status.items():
|
|
@@ -391,7 +405,7 @@ def main():
|
|
|
391
405
|
try:
|
|
392
406
|
install(args.reuse_browser_runtime)
|
|
393
407
|
except (OSError, RuntimeError, subprocess.SubprocessError) as exc:
|
|
394
|
-
print(
|
|
408
|
+
print(t("Web runtime installation failed: {error}", error=exc), file=sys.stderr)
|
|
395
409
|
return 1
|
|
396
410
|
return 0
|
|
397
411
|
|
package/lazy_mcp_proxy.py
CHANGED
|
@@ -37,6 +37,15 @@ def load_handshake(path):
|
|
|
37
37
|
return data
|
|
38
38
|
|
|
39
39
|
|
|
40
|
+
def local_initialize(handshake, params):
|
|
41
|
+
"""The cached initialize result, speaking the protocol version the client asked for when it is a known one."""
|
|
42
|
+
result = dict(handshake["initialize"])
|
|
43
|
+
asked = (params or {}).get("protocolVersion")
|
|
44
|
+
if asked in KNOWN_PROTOCOLS:
|
|
45
|
+
result["protocolVersion"] = asked
|
|
46
|
+
return result
|
|
47
|
+
|
|
48
|
+
|
|
40
49
|
class Proxy:
|
|
41
50
|
"""One client conversation: local answers until a request needs the real server, then a pass-through."""
|
|
42
51
|
|
|
@@ -58,14 +67,6 @@ class Proxy:
|
|
|
58
67
|
def answer(self, request_id, result):
|
|
59
68
|
self.send({"jsonrpc": "2.0", "id": request_id, "result": result})
|
|
60
69
|
|
|
61
|
-
def local_initialize(self, params):
|
|
62
|
-
"""The cached initialize result, speaking the protocol version the client asked for when it is a known one."""
|
|
63
|
-
result = dict(self.handshake["initialize"])
|
|
64
|
-
asked = (params or {}).get("protocolVersion")
|
|
65
|
-
if asked in KNOWN_PROTOCOLS:
|
|
66
|
-
result["protocolVersion"] = asked
|
|
67
|
-
return result
|
|
68
|
-
|
|
69
70
|
def start_child(self):
|
|
70
71
|
"""Starts the real server and replays the client's initialize to it; its answer is not forwarded (the client
|
|
71
72
|
already got the cached one). RuntimeError when it exits or answers with an error."""
|
|
@@ -141,7 +142,7 @@ class Proxy:
|
|
|
141
142
|
if method == "initialize":
|
|
142
143
|
self.client_init = message.get("params")
|
|
143
144
|
if self.child is None and request_id is not None and method == "initialize":
|
|
144
|
-
return self.answer(request_id, self.
|
|
145
|
+
return self.answer(request_id, local_initialize(self.handshake, message.get("params")))
|
|
145
146
|
if self.child is None and request_id is not None and method == "ping":
|
|
146
147
|
return self.answer(request_id, {})
|
|
147
148
|
if self.child is None and request_id is not None and method == "tools/list":
|
package/link_check.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Local links an edit of an HTML or Markdown file adds or changes, checked against the files on disk, for the edit hook.
|
|
2
|
+
Links come from `web_document_graph.analyze` (the same reader the code graph uses). A link counts only when its text is
|
|
3
|
+
new in the file (a link that was already broken is not the edit's), and only relative paths are looked up: http(s),
|
|
4
|
+
mailto, tel, data and #anchors are skipped. What cannot be decided is returned as unchecked with the reason instead of
|
|
5
|
+
being dropped: a root-relative path (the web root is not known), a template expression, a page with a <base> element,
|
|
6
|
+
a file that cannot be read, HTML that did not parse."""
|
|
7
|
+
import collections
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
|
|
11
|
+
import web_document_graph
|
|
12
|
+
|
|
13
|
+
EXTENSIONS = (".html", ".htm", ".md", ".markdown")
|
|
14
|
+
_TEMPLATE = re.compile(r"\{\{|\{%|<%|<\?|\$\{")
|
|
15
|
+
_BASE = re.compile(r"<base[\s>]", re.I)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _links(origin, text):
|
|
19
|
+
"""The local links of text (their specifiers and lines) and the parser diagnostics, in document order."""
|
|
20
|
+
found = web_document_graph.analyze({origin: text}, ())
|
|
21
|
+
return [item for item in found["dependencies"] if item["resolution"] == "unresolved"], found["diagnostics"]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _missing(target):
|
|
25
|
+
"""Whether target does not exist; any other failure to look it up raises OSError."""
|
|
26
|
+
try:
|
|
27
|
+
os.stat(target)
|
|
28
|
+
except (FileNotFoundError, NotADirectoryError):
|
|
29
|
+
return True
|
|
30
|
+
return False
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def review(path, before, after):
|
|
34
|
+
"""The local links the edit introduced that point to nothing, and the ones that could not be checked.
|
|
35
|
+
|
|
36
|
+
Parameters: path (str) absolute path of the edited file; before (str or None) text before the edit; after (str) text
|
|
37
|
+
after it.
|
|
38
|
+
Returns: dict with `broken` (list of {line, link}) and `unchecked` (list of {line, link, why}; line and link are None
|
|
39
|
+
for a parse failure of the whole file); both empty for a file that is not HTML or Markdown.
|
|
40
|
+
"""
|
|
41
|
+
result = {"broken": [], "unchecked": []}
|
|
42
|
+
if os.path.splitext(path)[1].lower() not in EXTENSIONS:
|
|
43
|
+
return result
|
|
44
|
+
origin = os.path.abspath(path).replace(os.sep, "/")
|
|
45
|
+
known = collections.Counter(item["specifier"] for item in _links(origin, before)[0]) if before else collections.Counter()
|
|
46
|
+
links, diagnostics = _links(origin, after)
|
|
47
|
+
result["unchecked"] += [{"line": None, "link": None, "why": item["reason"]} for item in diagnostics]
|
|
48
|
+
has_base = bool(_BASE.search(after)) and origin.lower().endswith((".html", ".htm"))
|
|
49
|
+
for item in links:
|
|
50
|
+
link = item["specifier"]
|
|
51
|
+
if known[link] > 0:
|
|
52
|
+
known[link] -= 1
|
|
53
|
+
continue
|
|
54
|
+
why = None
|
|
55
|
+
if has_base:
|
|
56
|
+
why = "the page has a <base> element, which changes where relative links lead"
|
|
57
|
+
elif _TEMPLATE.search(link):
|
|
58
|
+
why = "it is a template expression, not a path"
|
|
59
|
+
elif link.startswith("/"):
|
|
60
|
+
why = "it is relative to the root of a site or repository, which is not known here"
|
|
61
|
+
else:
|
|
62
|
+
try:
|
|
63
|
+
if _missing(web_document_graph.local_target(origin, link)):
|
|
64
|
+
result["broken"].append({"line": item["line"], "link": link})
|
|
65
|
+
except (OSError, ValueError) as exc:
|
|
66
|
+
why = f"it could not be looked up: {exc}"
|
|
67
|
+
if why:
|
|
68
|
+
result["unchecked"].append({"line": item["line"], "link": link, "why": why})
|
|
69
|
+
return result
|
package/local_embedder.py
CHANGED
|
@@ -25,6 +25,7 @@ MIN_API_VERSION = 1
|
|
|
25
25
|
BATCH = 16
|
|
26
26
|
INFO_TTL_S = 60
|
|
27
27
|
MAX_ATTEMPTS = 3
|
|
28
|
+
HOME_FAILURE_CODES = (*model_breaker.BUSY_CODES, 500)
|
|
28
29
|
REQUEST_TIMEOUT_S = 60
|
|
29
30
|
RERANK_TIMEOUT_S = 15
|
|
30
31
|
_lock = threading.Lock()
|
|
@@ -189,6 +190,11 @@ def _retry_delay(failure, attempt):
|
|
|
189
190
|
|
|
190
191
|
|
|
191
192
|
def embed(model, texts, deadline=None, cancel_check=None, on_retry=None):
|
|
193
|
+
"""Vectors of `texts` from the embeddings home, in batches of BATCH, each tried up to MAX_ATTEMPTS times.
|
|
194
|
+
|
|
195
|
+
Returns the vectors in input order. Raises IndexCompatibilityError when the home serves another model,
|
|
196
|
+
model_breaker.CircuitOpen once repeated busy or failing answers opened the circuit (no request is sent while it is
|
|
197
|
+
open), RuntimeError when the home is not running or a batch fails every attempt."""
|
|
192
198
|
base = url()
|
|
193
199
|
if not base:
|
|
194
200
|
raise RuntimeError(f"The embeddings home is not running (no announce file at {ANNOUNCE}).")
|
|
@@ -211,14 +217,15 @@ def embed(model, texts, deadline=None, cancel_check=None, on_retry=None):
|
|
|
211
217
|
for attempt in range(1, MAX_ATTEMPTS + 1):
|
|
212
218
|
if cancel_check:
|
|
213
219
|
cancel_check()
|
|
214
|
-
|
|
220
|
+
model_health.guard("/v1/embeddings", model)
|
|
215
221
|
conn.timeout = _left(deadline)
|
|
216
222
|
try:
|
|
217
223
|
with model_breaker.slot(model):
|
|
218
224
|
got = _request(conn, "POST", "/embed", {"inputs": part})
|
|
219
225
|
except (OSError, http.client.HTTPException) as exc:
|
|
220
|
-
model_breaker.note(model, exc)
|
|
226
|
+
model_breaker.note(model, exc, HOME_FAILURE_CODES)
|
|
221
227
|
model_health.record("/v1/embeddings", {"model": model}, exc)
|
|
228
|
+
model_breaker.guard(model)
|
|
222
229
|
conn.close()
|
|
223
230
|
conn = _connection(base, _left(deadline))
|
|
224
231
|
if attempt == MAX_ATTEMPTS:
|