@wendellresearch-ai/benchwarmers 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +147 -147
- package/BENCHMARKS.md +262 -262
- package/GATEWAY.md +272 -272
- package/NOTICE +28 -28
- package/PRIVACY.md +46 -46
- package/README.md +368 -314
- package/SECURITY.md +83 -83
- package/TERMS.md +22 -22
- package/USAGE.md +170 -163
- package/VERSION +1 -1
- package/bin/benchwarmers.js +16 -74
- package/cli.py +666 -0
- package/examples/config/gateway.json +5 -5
- package/examples/config/model-roles.json +11 -11
- package/gateway.py +755 -755
- package/launcher.py +120 -0
- package/mcp_server.py +266 -266
- package/metrics/alerts.py +116 -116
- package/metrics/aliases.py +41 -41
- package/metrics/bench_collections.py +141 -141
- package/metrics/catalog.py +295 -295
- package/metrics/client.py +394 -394
- package/metrics/cliproxy.py +191 -191
- package/metrics/connect.py +691 -691
- package/metrics/cost.py +152 -152
- package/metrics/decide.py +227 -227
- package/metrics/discover.py +318 -318
- package/metrics/endpoints.py +436 -436
- package/metrics/guard.py +617 -592
- package/metrics/learn.py +221 -221
- package/metrics/litellm_hub.py +206 -206
- package/metrics/model_roles.py +84 -84
- package/metrics/offload.py +166 -166
- package/metrics/owui.py +205 -205
- package/metrics/paths.py +60 -60
- package/metrics/route.py +579 -579
- package/metrics/scheduler.py +275 -275
- package/metrics/sources.py +167 -167
- package/metrics/wrong.py +144 -144
- package/package.json +68 -65
- package/perf/README.md +89 -89
- package/perf/run.py +261 -261
- package/scripts/auto_retire.py +367 -367
- package/scripts/check.py +108 -106
- package/scripts/check_deprecations.py +378 -378
- package/scripts/check_themes.py +103 -103
- package/scripts/init.py +124 -124
- package/scripts/list_served.py +78 -78
- package/scripts/notice.py +78 -78
- package/scripts/onboard.py +401 -331
- package/scripts/provider_tags.py +129 -129
- package/scripts/reprefix.py +136 -136
- package/scripts/sync_config.py +105 -105
- package/scripts/test_auth.py +255 -255
- package/scripts/test_cli.py +189 -0
- package/scripts/test_client.py +233 -233
- package/scripts/test_connect.py +222 -222
- package/scripts/test_first_run.py +180 -180
- package/scripts/test_hubs.py +328 -328
- package/scripts/test_launcher.py +67 -0
- package/scripts/test_llm_log.py +85 -81
- package/scripts/test_mcp.py +178 -178
- package/scripts/test_merged.py +214 -214
- package/scripts/test_no_owui.py +291 -291
- package/scripts/test_owui.py +90 -90
- package/scripts/test_serve_edit.py +448 -438
- package/scripts/test_verdict.py +98 -98
- package/scripts/test_vision.py +228 -228
- package/serve.py +3904 -3764
- package/suite/mine.py +154 -154
- package/suite/pace.py +145 -145
- package/suite/run.py +1046 -1042
- package/suite/seed.py +421 -421
- package/suite/tasks/claim-alias-unbound/fixtures/onboard.py +286 -286
- package/suite/tasks/claim-error-text-string/fixtures/decide.py +187 -187
- package/suite/tasks/claim-list-body/fixtures/gateway.py +235 -235
- package/suite/tasks/claim-member-nominated-twice/fixtures/gateway.py +284 -284
- package/suite/tasks/claim-nominee-uses-slot/fixtures/gateway.py +284 -284
- package/suite/tasks/claim-silent-exit/fixtures/gateway.py +235 -235
- package/suite/tasks/claim-sse-hang/fixtures/gateway.py +235 -235
- package/suite/tasks/claim-unknown-nominee-crash/fixtures/gateway.py +284 -284
- package/suite/tasks/rebench-matchms-809-bug/task.json +23 -23
- package/suite/tasks/rebench-matchms-809-control/task.json +27 -27
- package/suite/tasks/rebench-more-itertools-1028-bug/task.json +23 -23
- package/suite/tasks/rebench-more-itertools-1028-control/task.json +27 -27
- package/suite/tasks/rebench-narwhals-2864-bug/task.json +23 -23
- package/suite/tasks/rebench-narwhals-2864-control/task.json +27 -27
- package/suite/tasks/rebench-quantecon-py-769-bug/task.json +23 -23
- package/suite/tasks/rebench-quantecon-py-769-control/task.json +27 -27
- package/suite/tasks/rebench-scrapy-6867-bug/task.json +23 -23
- package/suite/tasks/rebench-scrapy-6867-control/task.json +27 -27
- package/ui/bench.css +430 -425
- package/ui/bench.js +1756 -1576
- package/ui/themes/dark.css +35 -35
- package/ui/themes/hardwood.css +52 -52
- package/ui/themes/rainforest.css +32 -32
package/metrics/scheduler.py
CHANGED
|
@@ -1,275 +1,275 @@
|
|
|
1
|
-
"""The bench's own scheduler: the recurring checks, without Windows Task Scheduler.
|
|
2
|
-
|
|
3
|
-
python metrics/scheduler.py # mode, every job, when each is next due
|
|
4
|
-
python metrics/scheduler.py --selftest
|
|
5
|
-
|
|
6
|
-
RELEASE-PLAN.md Phase 3. The checks used to exist only as Windows scheduled
|
|
7
|
-
tasks (scripts/schedule.ps1), which macOS, Linux and Docker do not have. serve.py
|
|
8
|
-
starts this in a background thread; the jobs below are the same seven, at the
|
|
9
|
-
same times, with the same arguments (schedule.ps1 explains each choice).
|
|
10
|
-
|
|
11
|
-
MODE, from config/scheduler.json:
|
|
12
|
-
absent / "off" nothing runs. A fresh install spends no model calls until
|
|
13
|
-
someone opts in.
|
|
14
|
-
"log-only" at each due time, write "would run X" to runs/scheduler.log
|
|
15
|
-
and run nothing. The switch-over week: it runs beside the
|
|
16
|
-
Windows tasks, which still do the work, so its timing can be
|
|
17
|
-
checked against theirs without doubling a single call.
|
|
18
|
-
"run" run the job (the repo's venv Python), one at a time per job,
|
|
19
|
-
with the Windows tasks switched off in the same step.
|
|
20
|
-
`"jobs": {"perf": false}` leaves a job out.
|
|
21
|
-
|
|
22
|
-
A run missed while the dashboard was down is caught up once when it next
|
|
23
|
-
starts (the Windows tasks' "start when available"). The last due time of each
|
|
24
|
-
job is kept in runs/scheduler-state.json; a lock beside it means only one
|
|
25
|
-
scheduler runs however many dashboards are started.
|
|
26
|
-
"""
|
|
27
|
-
|
|
28
|
-
from __future__ import annotations
|
|
29
|
-
|
|
30
|
-
import datetime as dt
|
|
31
|
-
import json
|
|
32
|
-
import pathlib
|
|
33
|
-
import subprocess
|
|
34
|
-
import sys
|
|
35
|
-
import threading
|
|
36
|
-
import time
|
|
37
|
-
|
|
38
|
-
ROOT = pathlib.Path(__file__).resolve().parent.parent
|
|
39
|
-
from paths import DATA # noqa: E402 -- the install's data folder (metrics/paths.py)
|
|
40
|
-
CONFIG = DATA / "config" / "scheduler.json"
|
|
41
|
-
LOG = DATA / "runs" / "scheduler.log"
|
|
42
|
-
STATE = DATA / "runs" / "scheduler-state.json"
|
|
43
|
-
TICK_S = 30
|
|
44
|
-
RUN_LIMIT_S = 2 * 3600 # schedule.ps1's ExecutionTimeLimit
|
|
45
|
-
CATCH_UP_DAYS = 7 # older misses are not worth a burst of calls
|
|
46
|
-
|
|
47
|
-
# name -> (script and arguments, slots). A slot is (weekday or None, "HH:MM");
|
|
48
|
-
# weekday 0 is Monday. Mirrors scripts/schedule.ps1, which says why each runs when.
|
|
49
|
-
JOBS = {
|
|
50
|
-
"availability": (["perf/run.py", "-n", "1", "--tiny-only", "--timeout", "90", "--client", "builtin"],
|
|
51
|
-
[(None, "05:00")]),
|
|
52
|
-
"catalog": (["metrics/catalog.py"], [(None, "06:00")]),
|
|
53
|
-
"suite-free": (["suite/run.py", "--next-free"], [(None, "06:00")]),
|
|
54
|
-
"deprecations": (["scripts/check_deprecations.py", "--quiet"], [(None, "08:00")]),
|
|
55
|
-
"perf": (["perf/run.py", "-n", "3", "--timeout", "90", "--client", "builtin"], [(0, "07:00")]),
|
|
56
|
-
"capabilities": (["metrics/probe_caps.py", "--gap", "30"], [(6, "07:00")]),
|
|
57
|
-
"suite": (["suite/run.py", "--next"], [(None, t) for t in ("03:00", "09:00", "15:00", "21:00")]),
|
|
58
|
-
}
|
|
59
|
-
# Each mode as the Settings page names and explains it (the docstring above).
|
|
60
|
-
MODES = {"off": ("Off", "Nothing runs. A fresh install spends no model calls until someone opts in."),
|
|
61
|
-
"log-only": ("Log only", "At each due time it writes \"would run\" to runs/scheduler.log and runs"
|
|
62
|
-
" nothing: for checking its times against another scheduler first."),
|
|
63
|
-
"run": ("Run", "Runs each job at its time, one at a time per job, making the calls the job makes.")}
|
|
64
|
-
# What each job is for, in a sentence, for the Settings page.
|
|
65
|
-
ABOUT = {
|
|
66
|
-
"availability": "Is each model still answering? One one-word prompt per model every morning;"
|
|
67
|
-
" feeds the status on At a glance and auto-retire.",
|
|
68
|
-
"catalog": "Refreshes what is known about models (context, output limit, price, free tier) and the"
|
|
69
|
-
" Discover list from public feeds. No model calls.",
|
|
70
|
-
"suite-free": "Scores one free model on the review suite: up to six tasks it has not answered yet,"
|
|
71
|
-
" within the free tier's shared daily budget.",
|
|
72
|
-
"deprecations": "Checks which models each endpoint still serves, and retires models that are gone or"
|
|
73
|
-
" have failed for days (auto-retire), after one confirming call.",
|
|
74
|
-
"perf": "Weekly speed test: a short and a long prompt, three samples per model, for typical and"
|
|
75
|
-
" slow response times and output speed. The trend is on Benchmarks.",
|
|
76
|
-
"capabilities": "Weekly check of what each model can do: tool calling and reasoning, two short calls"
|
|
77
|
-
" per model, spaced out.",
|
|
78
|
-
"suite": "Scores one paid model on the review suite every six hours: up to six tasks it has not"
|
|
79
|
-
" answered yet. These scores are what the router ranks by.",
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
def settings() -> dict:
|
|
84
|
-
try:
|
|
85
|
-
data = json.loads(CONFIG.read_text(encoding="utf-8"))
|
|
86
|
-
except (OSError, ValueError):
|
|
87
|
-
return {"mode": "off", "jobs": {}}
|
|
88
|
-
return {"mode": data.get("mode", "off"), "jobs": data.get("jobs") or {}}
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
def enabled(conf: dict) -> list[str]:
|
|
92
|
-
return [n for n in JOBS if conf["jobs"].get(n, True)]
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
def due_between(slots, start: dt.datetime, end: dt.datetime) -> list[dt.datetime]:
|
|
96
|
-
"""Every slot time in (start, end], oldest first."""
|
|
97
|
-
out = []
|
|
98
|
-
day = start.date()
|
|
99
|
-
while day <= end.date():
|
|
100
|
-
for weekday, hhmm in slots:
|
|
101
|
-
if weekday is not None and day.weekday() != weekday:
|
|
102
|
-
continue
|
|
103
|
-
h, m = map(int, hhmm.split(":"))
|
|
104
|
-
when = dt.datetime.combine(day, dt.time(h, m))
|
|
105
|
-
if start < when <= end:
|
|
106
|
-
out.append(when)
|
|
107
|
-
day += dt.timedelta(days=1)
|
|
108
|
-
return sorted(out)
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
def next_due(slots, after: dt.datetime) -> dt.datetime:
|
|
112
|
-
return due_between(slots, after, after + dt.timedelta(days=8))[0]
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
def _log(line: str) -> None:
|
|
116
|
-
LOG.parent.mkdir(parents=True, exist_ok=True)
|
|
117
|
-
with LOG.open("a", encoding="utf-8") as fh:
|
|
118
|
-
fh.write(f"{dt.datetime.now():%Y-%m-%d %H:%M:%S} {line}\n")
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
def _run(name: str, args: list[str], due: dt.datetime, running: dict) -> None:
|
|
122
|
-
"""One job, in its own thread, recorded in the log with its exit code."""
|
|
123
|
-
sys.path.insert(0, str(ROOT / "suite"))
|
|
124
|
-
from run import bench_python
|
|
125
|
-
started = time.monotonic()
|
|
126
|
-
try:
|
|
127
|
-
done = subprocess.run([bench_python(), *args], cwd=ROOT, capture_output=True, text=True,
|
|
128
|
-
encoding="utf-8", errors="replace", timeout=RUN_LIMIT_S)
|
|
129
|
-
result = f"exit {done.returncode}"
|
|
130
|
-
except subprocess.TimeoutExpired:
|
|
131
|
-
result = f"stopped at the {RUN_LIMIT_S // 3600} h limit"
|
|
132
|
-
except OSError as exc:
|
|
133
|
-
result = f"could not start ({exc})"
|
|
134
|
-
finally:
|
|
135
|
-
running.pop(name, None)
|
|
136
|
-
_log(f"ran {name} (due {due:%H:%M}): {result} in {time.monotonic() - started:.0f}s")
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
def tick(now: dt.datetime, state: dict, conf: dict, running: dict, start_job=None) -> list[str]:
|
|
140
|
-
"""Act on everything due since each job's last due time. Changes `state`
|
|
141
|
-
in place; returns what it logged (for the self-check)."""
|
|
142
|
-
logged = []
|
|
143
|
-
for name in enabled(conf):
|
|
144
|
-
args, slots = JOBS[name]
|
|
145
|
-
last = dt.datetime.fromisoformat(state[name]) if name in state else now
|
|
146
|
-
due = due_between(slots, max(last, now - dt.timedelta(days=CATCH_UP_DAYS)), now)
|
|
147
|
-
if not due:
|
|
148
|
-
state.setdefault(name, now.isoformat(timespec="seconds"))
|
|
149
|
-
continue
|
|
150
|
-
latest = due[-1]
|
|
151
|
-
missed = " (catching up a run missed while stopped)" if now - latest > dt.timedelta(minutes=5) else ""
|
|
152
|
-
state[name] = latest.isoformat(timespec="seconds")
|
|
153
|
-
if conf["mode"] == "log-only":
|
|
154
|
-
line = f"would run {name} (due {latest:%a %H:%M}){missed}: {' '.join(args)}"
|
|
155
|
-
elif name in running:
|
|
156
|
-
line = f"skipped {name} (due {latest:%H:%M}): the previous run is still going"
|
|
157
|
-
else:
|
|
158
|
-
line = f"starting {name} (due {latest:%H:%M}){missed}"
|
|
159
|
-
running[name] = True
|
|
160
|
-
(start_job or _start)(name, args, latest, running)
|
|
161
|
-
_log(line)
|
|
162
|
-
logged.append(line)
|
|
163
|
-
return logged
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
def _start(name, args, due, running) -> None:
|
|
167
|
-
threading.Thread(target=_run, args=(name, args, due, running), daemon=True).start()
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
_stop = threading.Event()
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
def _loop() -> None:
|
|
174
|
-
import locked
|
|
175
|
-
# Its own lock file, not the state file's: update() below locks that one
|
|
176
|
-
# for each write, and holding it here as well would wait on itself.
|
|
177
|
-
try:
|
|
178
|
-
instance = locked.lock(DATA / "runs" / "scheduler", timeout=0)
|
|
179
|
-
instance.__enter__() # one scheduler per machine
|
|
180
|
-
except TimeoutError:
|
|
181
|
-
return # another dashboard already runs it
|
|
182
|
-
try:
|
|
183
|
-
running: dict = {}
|
|
184
|
-
conf = settings()
|
|
185
|
-
_log(f"scheduler started, mode {conf['mode']}, jobs {', '.join(enabled(conf))}")
|
|
186
|
-
while True:
|
|
187
|
-
try:
|
|
188
|
-
conf = settings() # a mode change applies without a restart
|
|
189
|
-
if conf["mode"] in ("log-only", "run"):
|
|
190
|
-
locked.update(STATE, lambda s: tick(dt.datetime.now().replace(microsecond=0),
|
|
191
|
-
s, conf, running))
|
|
192
|
-
except Exception as exc: # noqa: BLE001 -- one bad tick must not end it
|
|
193
|
-
_log(f"tick failed: {type(exc).__name__}: {exc}")
|
|
194
|
-
if _stop.wait(TICK_S):
|
|
195
|
-
return
|
|
196
|
-
finally:
|
|
197
|
-
instance.__exit__(None, None, None)
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
def start() -> threading.Thread:
|
|
201
|
-
"""Called once by serve.py. Does nothing visible when the mode is off."""
|
|
202
|
-
_stop.clear()
|
|
203
|
-
thread = threading.Thread(target=_loop, name="scheduler", daemon=True)
|
|
204
|
-
thread.start()
|
|
205
|
-
return thread
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
def _selftest() -> None:
|
|
209
|
-
mon = dt.datetime(2026, 10, 19, 4, 0) # a Monday
|
|
210
|
-
slots = JOBS["suite"][1]
|
|
211
|
-
assert [d.hour for d in due_between(slots, mon, mon + dt.timedelta(days=1))] == [9, 15, 21, 3]
|
|
212
|
-
assert next_due(JOBS["perf"][1], mon) == dt.datetime(2026, 10, 19, 7, 0)
|
|
213
|
-
assert next_due(JOBS["perf"][1], mon.replace(hour=8)) == dt.datetime(2026, 10, 26, 7, 0)
|
|
214
|
-
assert next_due(JOBS["capabilities"][1], mon) == dt.datetime(2026, 10, 25, 7, 0)
|
|
215
|
-
conf = {"mode": "log-only", "jobs": {n: n == "availability" for n in JOBS}}
|
|
216
|
-
state: dict = {}
|
|
217
|
-
global _log
|
|
218
|
-
real_log, _log = _log, (lambda line: None)
|
|
219
|
-
try:
|
|
220
|
-
assert tick(mon, state, conf, {}) == [] and state == {"availability": mon.isoformat()}
|
|
221
|
-
lines = tick(mon.replace(hour=5, minute=1), state, conf, {})
|
|
222
|
-
assert len(lines) == 1 and lines[0].startswith("would run availability"), lines
|
|
223
|
-
assert tick(mon.replace(hour=5, minute=2), state, conf, {}) == [] # once per due time
|
|
224
|
-
# Stopped over two mornings: one catch-up, not two runs.
|
|
225
|
-
lines = tick(mon.replace(hour=5) + dt.timedelta(days=2, hours=3), state, conf, {})
|
|
226
|
-
assert len(lines) == 1 and "catching up" in lines[0], lines
|
|
227
|
-
started = []
|
|
228
|
-
run_conf = {"mode": "run", "jobs": conf["jobs"]}
|
|
229
|
-
lines = tick(mon.replace(hour=5) + dt.timedelta(days=3, minutes=1), state, run_conf,
|
|
230
|
-
{}, start_job=lambda *a: started.append(a[0]))
|
|
231
|
-
assert started == ["availability"] and lines[0].startswith("starting availability"), lines
|
|
232
|
-
lines = tick(mon.replace(hour=5) + dt.timedelta(days=4, minutes=1), state, run_conf,
|
|
233
|
-
{"availability": True}, start_job=lambda *a: started.append(a[0]))
|
|
234
|
-
assert started == ["availability"] and lines[0].startswith("skipped"), lines # no overlap
|
|
235
|
-
finally:
|
|
236
|
-
_log = real_log
|
|
237
|
-
# The loop itself, briefly, on temp files: it must write its state while
|
|
238
|
-
# holding the one-per-machine lock (the first version waited on itself).
|
|
239
|
-
import tempfile
|
|
240
|
-
global DATA, CONFIG, LOG, STATE, TICK_S
|
|
241
|
-
real = DATA, CONFIG, LOG, STATE, TICK_S
|
|
242
|
-
with tempfile.TemporaryDirectory() as tmp:
|
|
243
|
-
DATA = pathlib.Path(tmp) # the lock and state go here, not to the real data
|
|
244
|
-
CONFIG, LOG, STATE, TICK_S = DATA / "s.json", DATA / "runs" / "s.log", DATA / "runs" / "s.json", 0.05
|
|
245
|
-
CONFIG.write_text('{"mode": "log-only"}', encoding="utf-8")
|
|
246
|
-
thread = start()
|
|
247
|
-
try:
|
|
248
|
-
for _ in range(100):
|
|
249
|
-
if STATE.exists():
|
|
250
|
-
break
|
|
251
|
-
time.sleep(0.05)
|
|
252
|
-
assert STATE.exists() and set(json.loads(STATE.read_text())) == set(JOBS), "loop wrote no state"
|
|
253
|
-
finally:
|
|
254
|
-
_stop.set() # stopped, and its lock let go, before
|
|
255
|
-
thread.join(5) # the real paths come back
|
|
256
|
-
assert not thread.is_alive(), "the loop did not stop"
|
|
257
|
-
DATA, CONFIG, LOG, STATE, TICK_S = real
|
|
258
|
-
print("ok")
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
def main() -> None:
|
|
262
|
-
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
|
263
|
-
if "--selftest" in sys.argv:
|
|
264
|
-
_selftest()
|
|
265
|
-
return
|
|
266
|
-
conf, now = settings(), dt.datetime.now()
|
|
267
|
-
print(f"mode: {conf['mode']} (config/scheduler.json)")
|
|
268
|
-
for name in JOBS:
|
|
269
|
-
args, slots = JOBS[name]
|
|
270
|
-
on = "on " if name in enabled(conf) else "off"
|
|
271
|
-
print(f" {on} {name:<13} next {next_due(slots, now):%a %d %b %H:%M} {' '.join(args)}")
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
if __name__ == "__main__":
|
|
275
|
-
main()
|
|
1
|
+
"""The bench's own scheduler: the recurring checks, without Windows Task Scheduler.
|
|
2
|
+
|
|
3
|
+
python metrics/scheduler.py # mode, every job, when each is next due
|
|
4
|
+
python metrics/scheduler.py --selftest
|
|
5
|
+
|
|
6
|
+
RELEASE-PLAN.md Phase 3. The checks used to exist only as Windows scheduled
|
|
7
|
+
tasks (scripts/schedule.ps1), which macOS, Linux and Docker do not have. serve.py
|
|
8
|
+
starts this in a background thread; the jobs below are the same seven, at the
|
|
9
|
+
same times, with the same arguments (schedule.ps1 explains each choice).
|
|
10
|
+
|
|
11
|
+
MODE, from config/scheduler.json:
|
|
12
|
+
absent / "off" nothing runs. A fresh install spends no model calls until
|
|
13
|
+
someone opts in.
|
|
14
|
+
"log-only" at each due time, write "would run X" to runs/scheduler.log
|
|
15
|
+
and run nothing. The switch-over week: it runs beside the
|
|
16
|
+
Windows tasks, which still do the work, so its timing can be
|
|
17
|
+
checked against theirs without doubling a single call.
|
|
18
|
+
"run" run the job (the repo's venv Python), one at a time per job,
|
|
19
|
+
with the Windows tasks switched off in the same step.
|
|
20
|
+
`"jobs": {"perf": false}` leaves a job out.
|
|
21
|
+
|
|
22
|
+
A run missed while the dashboard was down is caught up once when it next
|
|
23
|
+
starts (the Windows tasks' "start when available"). The last due time of each
|
|
24
|
+
job is kept in runs/scheduler-state.json; a lock beside it means only one
|
|
25
|
+
scheduler runs however many dashboards are started.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import datetime as dt
|
|
31
|
+
import json
|
|
32
|
+
import pathlib
|
|
33
|
+
import subprocess
|
|
34
|
+
import sys
|
|
35
|
+
import threading
|
|
36
|
+
import time
|
|
37
|
+
|
|
38
|
+
ROOT = pathlib.Path(__file__).resolve().parent.parent
|
|
39
|
+
from paths import DATA # noqa: E402 -- the install's data folder (metrics/paths.py)
|
|
40
|
+
CONFIG = DATA / "config" / "scheduler.json"
|
|
41
|
+
LOG = DATA / "runs" / "scheduler.log"
|
|
42
|
+
STATE = DATA / "runs" / "scheduler-state.json"
|
|
43
|
+
TICK_S = 30
|
|
44
|
+
RUN_LIMIT_S = 2 * 3600 # schedule.ps1's ExecutionTimeLimit
|
|
45
|
+
CATCH_UP_DAYS = 7 # older misses are not worth a burst of calls
|
|
46
|
+
|
|
47
|
+
# name -> (script and arguments, slots). A slot is (weekday or None, "HH:MM");
|
|
48
|
+
# weekday 0 is Monday. Mirrors scripts/schedule.ps1, which says why each runs when.
|
|
49
|
+
JOBS = {
|
|
50
|
+
"availability": (["perf/run.py", "-n", "1", "--tiny-only", "--timeout", "90", "--client", "builtin"],
|
|
51
|
+
[(None, "05:00")]),
|
|
52
|
+
"catalog": (["metrics/catalog.py"], [(None, "06:00")]),
|
|
53
|
+
"suite-free": (["suite/run.py", "--next-free"], [(None, "06:00")]),
|
|
54
|
+
"deprecations": (["scripts/check_deprecations.py", "--quiet"], [(None, "08:00")]),
|
|
55
|
+
"perf": (["perf/run.py", "-n", "3", "--timeout", "90", "--client", "builtin"], [(0, "07:00")]),
|
|
56
|
+
"capabilities": (["metrics/probe_caps.py", "--gap", "30"], [(6, "07:00")]),
|
|
57
|
+
"suite": (["suite/run.py", "--next"], [(None, t) for t in ("03:00", "09:00", "15:00", "21:00")]),
|
|
58
|
+
}
|
|
59
|
+
# Each mode as the Settings page names and explains it (the docstring above).
|
|
60
|
+
MODES = {"off": ("Off", "Nothing runs. A fresh install spends no model calls until someone opts in."),
|
|
61
|
+
"log-only": ("Log only", "At each due time it writes \"would run\" to runs/scheduler.log and runs"
|
|
62
|
+
" nothing: for checking its times against another scheduler first."),
|
|
63
|
+
"run": ("Run", "Runs each job at its time, one at a time per job, making the calls the job makes.")}
|
|
64
|
+
# What each job is for, in a sentence, for the Settings page.
|
|
65
|
+
ABOUT = {
|
|
66
|
+
"availability": "Is each model still answering? One one-word prompt per model every morning;"
|
|
67
|
+
" feeds the status on At a glance and auto-retire.",
|
|
68
|
+
"catalog": "Refreshes what is known about models (context, output limit, price, free tier) and the"
|
|
69
|
+
" Discover list from public feeds. No model calls.",
|
|
70
|
+
"suite-free": "Scores one free model on the review suite: up to six tasks it has not answered yet,"
|
|
71
|
+
" within the free tier's shared daily budget.",
|
|
72
|
+
"deprecations": "Checks which models each endpoint still serves, and retires models that are gone or"
|
|
73
|
+
" have failed for days (auto-retire), after one confirming call.",
|
|
74
|
+
"perf": "Weekly speed test: a short and a long prompt, three samples per model, for typical and"
|
|
75
|
+
" slow response times and output speed. The trend is on Benchmarks.",
|
|
76
|
+
"capabilities": "Weekly check of what each model can do: tool calling and reasoning, two short calls"
|
|
77
|
+
" per model, spaced out.",
|
|
78
|
+
"suite": "Scores one paid model on the review suite every six hours: up to six tasks it has not"
|
|
79
|
+
" answered yet. These scores are what the router ranks by.",
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def settings() -> dict:
|
|
84
|
+
try:
|
|
85
|
+
data = json.loads(CONFIG.read_text(encoding="utf-8"))
|
|
86
|
+
except (OSError, ValueError):
|
|
87
|
+
return {"mode": "off", "jobs": {}}
|
|
88
|
+
return {"mode": data.get("mode", "off"), "jobs": data.get("jobs") or {}}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def enabled(conf: dict) -> list[str]:
|
|
92
|
+
return [n for n in JOBS if conf["jobs"].get(n, True)]
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def due_between(slots, start: dt.datetime, end: dt.datetime) -> list[dt.datetime]:
|
|
96
|
+
"""Every slot time in (start, end], oldest first."""
|
|
97
|
+
out = []
|
|
98
|
+
day = start.date()
|
|
99
|
+
while day <= end.date():
|
|
100
|
+
for weekday, hhmm in slots:
|
|
101
|
+
if weekday is not None and day.weekday() != weekday:
|
|
102
|
+
continue
|
|
103
|
+
h, m = map(int, hhmm.split(":"))
|
|
104
|
+
when = dt.datetime.combine(day, dt.time(h, m))
|
|
105
|
+
if start < when <= end:
|
|
106
|
+
out.append(when)
|
|
107
|
+
day += dt.timedelta(days=1)
|
|
108
|
+
return sorted(out)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def next_due(slots, after: dt.datetime) -> dt.datetime:
|
|
112
|
+
return due_between(slots, after, after + dt.timedelta(days=8))[0]
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _log(line: str) -> None:
|
|
116
|
+
LOG.parent.mkdir(parents=True, exist_ok=True)
|
|
117
|
+
with LOG.open("a", encoding="utf-8") as fh:
|
|
118
|
+
fh.write(f"{dt.datetime.now():%Y-%m-%d %H:%M:%S} {line}\n")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _run(name: str, args: list[str], due: dt.datetime, running: dict) -> None:
|
|
122
|
+
"""One job, in its own thread, recorded in the log with its exit code."""
|
|
123
|
+
sys.path.insert(0, str(ROOT / "suite"))
|
|
124
|
+
from run import bench_python
|
|
125
|
+
started = time.monotonic()
|
|
126
|
+
try:
|
|
127
|
+
done = subprocess.run([bench_python(), *args], cwd=ROOT, capture_output=True, text=True,
|
|
128
|
+
encoding="utf-8", errors="replace", timeout=RUN_LIMIT_S)
|
|
129
|
+
result = f"exit {done.returncode}"
|
|
130
|
+
except subprocess.TimeoutExpired:
|
|
131
|
+
result = f"stopped at the {RUN_LIMIT_S // 3600} h limit"
|
|
132
|
+
except OSError as exc:
|
|
133
|
+
result = f"could not start ({exc})"
|
|
134
|
+
finally:
|
|
135
|
+
running.pop(name, None)
|
|
136
|
+
_log(f"ran {name} (due {due:%H:%M}): {result} in {time.monotonic() - started:.0f}s")
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def tick(now: dt.datetime, state: dict, conf: dict, running: dict, start_job=None) -> list[str]:
|
|
140
|
+
"""Act on everything due since each job's last due time. Changes `state`
|
|
141
|
+
in place; returns what it logged (for the self-check)."""
|
|
142
|
+
logged = []
|
|
143
|
+
for name in enabled(conf):
|
|
144
|
+
args, slots = JOBS[name]
|
|
145
|
+
last = dt.datetime.fromisoformat(state[name]) if name in state else now
|
|
146
|
+
due = due_between(slots, max(last, now - dt.timedelta(days=CATCH_UP_DAYS)), now)
|
|
147
|
+
if not due:
|
|
148
|
+
state.setdefault(name, now.isoformat(timespec="seconds"))
|
|
149
|
+
continue
|
|
150
|
+
latest = due[-1]
|
|
151
|
+
missed = " (catching up a run missed while stopped)" if now - latest > dt.timedelta(minutes=5) else ""
|
|
152
|
+
state[name] = latest.isoformat(timespec="seconds")
|
|
153
|
+
if conf["mode"] == "log-only":
|
|
154
|
+
line = f"would run {name} (due {latest:%a %H:%M}){missed}: {' '.join(args)}"
|
|
155
|
+
elif name in running:
|
|
156
|
+
line = f"skipped {name} (due {latest:%H:%M}): the previous run is still going"
|
|
157
|
+
else:
|
|
158
|
+
line = f"starting {name} (due {latest:%H:%M}){missed}"
|
|
159
|
+
running[name] = True
|
|
160
|
+
(start_job or _start)(name, args, latest, running)
|
|
161
|
+
_log(line)
|
|
162
|
+
logged.append(line)
|
|
163
|
+
return logged
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _start(name, args, due, running) -> None:
|
|
167
|
+
threading.Thread(target=_run, args=(name, args, due, running), daemon=True).start()
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
_stop = threading.Event()
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _loop() -> None:
|
|
174
|
+
import locked
|
|
175
|
+
# Its own lock file, not the state file's: update() below locks that one
|
|
176
|
+
# for each write, and holding it here as well would wait on itself.
|
|
177
|
+
try:
|
|
178
|
+
instance = locked.lock(DATA / "runs" / "scheduler", timeout=0)
|
|
179
|
+
instance.__enter__() # one scheduler per machine
|
|
180
|
+
except TimeoutError:
|
|
181
|
+
return # another dashboard already runs it
|
|
182
|
+
try:
|
|
183
|
+
running: dict = {}
|
|
184
|
+
conf = settings()
|
|
185
|
+
_log(f"scheduler started, mode {conf['mode']}, jobs {', '.join(enabled(conf))}")
|
|
186
|
+
while True:
|
|
187
|
+
try:
|
|
188
|
+
conf = settings() # a mode change applies without a restart
|
|
189
|
+
if conf["mode"] in ("log-only", "run"):
|
|
190
|
+
locked.update(STATE, lambda s: tick(dt.datetime.now().replace(microsecond=0),
|
|
191
|
+
s, conf, running))
|
|
192
|
+
except Exception as exc: # noqa: BLE001 -- one bad tick must not end it
|
|
193
|
+
_log(f"tick failed: {type(exc).__name__}: {exc}")
|
|
194
|
+
if _stop.wait(TICK_S):
|
|
195
|
+
return
|
|
196
|
+
finally:
|
|
197
|
+
instance.__exit__(None, None, None)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def start() -> threading.Thread:
|
|
201
|
+
"""Called once by serve.py. Does nothing visible when the mode is off."""
|
|
202
|
+
_stop.clear()
|
|
203
|
+
thread = threading.Thread(target=_loop, name="scheduler", daemon=True)
|
|
204
|
+
thread.start()
|
|
205
|
+
return thread
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _selftest() -> None:
|
|
209
|
+
mon = dt.datetime(2026, 10, 19, 4, 0) # a Monday
|
|
210
|
+
slots = JOBS["suite"][1]
|
|
211
|
+
assert [d.hour for d in due_between(slots, mon, mon + dt.timedelta(days=1))] == [9, 15, 21, 3]
|
|
212
|
+
assert next_due(JOBS["perf"][1], mon) == dt.datetime(2026, 10, 19, 7, 0)
|
|
213
|
+
assert next_due(JOBS["perf"][1], mon.replace(hour=8)) == dt.datetime(2026, 10, 26, 7, 0)
|
|
214
|
+
assert next_due(JOBS["capabilities"][1], mon) == dt.datetime(2026, 10, 25, 7, 0)
|
|
215
|
+
conf = {"mode": "log-only", "jobs": {n: n == "availability" for n in JOBS}}
|
|
216
|
+
state: dict = {}
|
|
217
|
+
global _log
|
|
218
|
+
real_log, _log = _log, (lambda line: None)
|
|
219
|
+
try:
|
|
220
|
+
assert tick(mon, state, conf, {}) == [] and state == {"availability": mon.isoformat()}
|
|
221
|
+
lines = tick(mon.replace(hour=5, minute=1), state, conf, {})
|
|
222
|
+
assert len(lines) == 1 and lines[0].startswith("would run availability"), lines
|
|
223
|
+
assert tick(mon.replace(hour=5, minute=2), state, conf, {}) == [] # once per due time
|
|
224
|
+
# Stopped over two mornings: one catch-up, not two runs.
|
|
225
|
+
lines = tick(mon.replace(hour=5) + dt.timedelta(days=2, hours=3), state, conf, {})
|
|
226
|
+
assert len(lines) == 1 and "catching up" in lines[0], lines
|
|
227
|
+
started = []
|
|
228
|
+
run_conf = {"mode": "run", "jobs": conf["jobs"]}
|
|
229
|
+
lines = tick(mon.replace(hour=5) + dt.timedelta(days=3, minutes=1), state, run_conf,
|
|
230
|
+
{}, start_job=lambda *a: started.append(a[0]))
|
|
231
|
+
assert started == ["availability"] and lines[0].startswith("starting availability"), lines
|
|
232
|
+
lines = tick(mon.replace(hour=5) + dt.timedelta(days=4, minutes=1), state, run_conf,
|
|
233
|
+
{"availability": True}, start_job=lambda *a: started.append(a[0]))
|
|
234
|
+
assert started == ["availability"] and lines[0].startswith("skipped"), lines # no overlap
|
|
235
|
+
finally:
|
|
236
|
+
_log = real_log
|
|
237
|
+
# The loop itself, briefly, on temp files: it must write its state while
|
|
238
|
+
# holding the one-per-machine lock (the first version waited on itself).
|
|
239
|
+
import tempfile
|
|
240
|
+
global DATA, CONFIG, LOG, STATE, TICK_S
|
|
241
|
+
real = DATA, CONFIG, LOG, STATE, TICK_S
|
|
242
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
243
|
+
DATA = pathlib.Path(tmp) # the lock and state go here, not to the real data
|
|
244
|
+
CONFIG, LOG, STATE, TICK_S = DATA / "s.json", DATA / "runs" / "s.log", DATA / "runs" / "s.json", 0.05
|
|
245
|
+
CONFIG.write_text('{"mode": "log-only"}', encoding="utf-8")
|
|
246
|
+
thread = start()
|
|
247
|
+
try:
|
|
248
|
+
for _ in range(100):
|
|
249
|
+
if STATE.exists():
|
|
250
|
+
break
|
|
251
|
+
time.sleep(0.05)
|
|
252
|
+
assert STATE.exists() and set(json.loads(STATE.read_text())) == set(JOBS), "loop wrote no state"
|
|
253
|
+
finally:
|
|
254
|
+
_stop.set() # stopped, and its lock let go, before
|
|
255
|
+
thread.join(5) # the real paths come back
|
|
256
|
+
assert not thread.is_alive(), "the loop did not stop"
|
|
257
|
+
DATA, CONFIG, LOG, STATE, TICK_S = real
|
|
258
|
+
print("ok")
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def main() -> None:
|
|
262
|
+
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
|
263
|
+
if "--selftest" in sys.argv:
|
|
264
|
+
_selftest()
|
|
265
|
+
return
|
|
266
|
+
conf, now = settings(), dt.datetime.now()
|
|
267
|
+
print(f"mode: {conf['mode']} (config/scheduler.json)")
|
|
268
|
+
for name in JOBS:
|
|
269
|
+
args, slots = JOBS[name]
|
|
270
|
+
on = "on " if name in enabled(conf) else "off"
|
|
271
|
+
print(f" {on} {name:<13} next {next_due(slots, now):%a %d %b %H:%M} {' '.join(args)}")
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
if __name__ == "__main__":
|
|
275
|
+
main()
|