@wendellresearch-ai/benchwarmers 0.3.3 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/GATEWAY.md +9 -5
  2. package/PRIVACY.md +9 -3
  3. package/README.md +31 -8
  4. package/USAGE.md +81 -15
  5. package/VERSION +1 -1
  6. package/cli.py +17 -5
  7. package/gateway.py +34 -2
  8. package/launcher.py +2 -1
  9. package/metrics/aliases.py +24 -3
  10. package/metrics/discover.py +68 -18
  11. package/metrics/endpoints.py +36 -4
  12. package/metrics/model_roles.py +3 -1
  13. package/metrics/pacing.py +23 -3
  14. package/metrics/quotaview.py +460 -0
  15. package/metrics/route.py +3 -2
  16. package/metrics/scheduler.py +2 -1
  17. package/metrics/sources.py +29 -6
  18. package/metrics/wrong.py +3 -1
  19. package/package.json +1 -1
  20. package/scripts/check.py +3 -0
  21. package/scripts/onboard.py +12 -1
  22. package/scripts/test_hubs.py +10 -5
  23. package/scripts/test_no_owui.py +14 -2
  24. package/scripts/test_quota.py +12 -2
  25. package/scripts/test_serve_edit.py +58 -5
  26. package/scripts/test_server_json.py +22 -2
  27. package/serve.py +570 -128
  28. package/suite/README.md +33 -0
  29. package/suite/pace.py +8 -4
  30. package/suite/run.py +144 -16
  31. package/suite/tasks/vision-bars/fixtures/bars.png +0 -0
  32. package/suite/tasks/vision-bars/task.json +40 -0
  33. package/suite/tasks/vision-invoice/fixtures/invoice.png +0 -0
  34. package/suite/tasks/vision-invoice/task.json +40 -0
  35. package/suite/tasks/vision-label/fixtures/label.png +0 -0
  36. package/suite/tasks/vision-label/task.json +52 -0
  37. package/suite/tasks/vision-receipt/fixtures/receipt.png +0 -0
  38. package/suite/tasks/vision-receipt/task.json +44 -0
  39. package/suite/tasks/vision-screen/fixtures/screen.png +0 -0
  40. package/suite/tasks/vision-screen/task.json +48 -0
  41. package/suite/tasks/vision-shapes/fixtures/shapes.png +0 -0
  42. package/suite/tasks/vision-shapes/task.json +40 -0
  43. package/suite/tasks/vision-table/fixtures/table.png +0 -0
  44. package/suite/tasks/vision-table/task.json +40 -0
  45. package/suite/test_vision_tasks.py +201 -0
  46. package/suite/vision_keys.py +74 -0
  47. package/suite/vision_sample.py +232 -0
  48. package/suite/vision_tasks.py +338 -0
  49. package/ui/bench.css +71 -6
  50. package/ui/bench.js +543 -49
@@ -0,0 +1,338 @@
1
+ """Make the vision tasks: pictures drawn here, each with its answer key.
2
+
3
+ python suite/vision_tasks.py # (re)writes suite/tasks/vision-*/
4
+
5
+ Needs Pillow, to draw (`pip install pillow`); nothing else does. The suite
6
+ reads the PNGs and task.json files this writes, which are committed, and
7
+ suite/run.py copies them into the data folder (run.seed_shipped).
8
+
9
+ THE KEY EXISTS BY CONSTRUCTION, as with suite/seed.py: every number and name
10
+ in a picture comes from the same data its task's answer key is built from, so
11
+ the two cannot disagree. The pictures are made, not photographed: nobody's
12
+ receipt, card or screen, so they ship with the public repository. What they
13
+ test is reading -- printed text, digits that must be exact, a chart against
14
+ its gridlines, a table cell by row and column, a screen's state, counting,
15
+ a page turned on its side -- the jobs vision seats are given.
16
+
17
+ ONE FIELD IN EACH ASKS FOR SOMETHING THE PICTURE DOES NOT SHOW. The prompt
18
+ says to write `not shown` then; a value is an invention, scored like a trap
19
+ in the text suite. Reading an image confidently wrong is the failure that
20
+ costs, the same as asserting a bug that is not there.
21
+
22
+ Answers are lines, `key: value`. Scoring is the suite's own (run.score):
23
+ regexes over the lowered answer, tolerant of markdown around the key.
24
+ """
25
+
26
+ from __future__ import annotations
27
+
28
+ import json
29
+ import math
30
+ import pathlib
31
+
32
+ from PIL import Image, ImageDraw, ImageFont
33
+
34
+ from vision_keys import NOT_SHOWN, PROMPT, clause
35
+
36
+ HERE = pathlib.Path(__file__).resolve().parent
37
+ OUT = HERE / "tasks"
38
+
39
+
40
+ def font(size: int, kind: str = "sans"):
41
+ names = {"sans": ("arial.ttf", "DejaVuSans.ttf", "LiberationSans-Regular.ttf"),
42
+ "bold": ("arialbd.ttf", "DejaVuSans-Bold.ttf", "LiberationSans-Bold.ttf"),
43
+ "mono": ("consola.ttf", "DejaVuSansMono.ttf", "LiberationMono-Regular.ttf")}[kind]
44
+ for name in names:
45
+ try:
46
+ return ImageFont.truetype(name, size)
47
+ except OSError:
48
+ continue
49
+ return ImageFont.load_default(size)
50
+
51
+
52
+ def write(task_id: str, title: str, image: Image.Image, lines: list[str], find: list[dict],
53
+ not_claim: list[dict], why: str) -> None:
54
+ folder = OUT / task_id
55
+ (folder / "fixtures").mkdir(parents=True, exist_ok=True)
56
+ name = task_id.removeprefix("vision-") + ".png"
57
+ image.save(folder / "fixtures" / name, optimize=True)
58
+ spec = {"id": task_id, "title": title, "shape": "vision", "images": [name],
59
+ "prompt": PROMPT.format(lines="\n".join(lines)), "min_answer_chars": 20,
60
+ "must_find": find, "must_not_claim": not_claim, "why": why,
61
+ "made_by": "suite/vision_tasks.py"}
62
+ (folder / "task.json").write_text(json.dumps(spec, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
63
+ print(f"wrote {folder.relative_to(HERE.parent)} ({len(find)} fields)")
64
+
65
+
66
+ # ---- a receipt: printed text, a date to rewrite, a sum to find ----------
67
+
68
+ def receipt() -> None:
69
+ items = [("Oat milk 1L", 3.49), ("Sourdough loaf", 5.25), ("Blueberries 250g", 4.99), ("Cheddar 200g", 6.80)]
70
+ subtotal = round(sum(p for _, p in items), 2)
71
+ tax = round(subtotal * 0.08, 2)
72
+ total = round(subtotal + tax, 2)
73
+ img = Image.new("RGB", (520, 760), "#f4f1ea")
74
+ d = ImageDraw.Draw(img)
75
+ mono, big = font(22, "mono"), font(30, "bold")
76
+ d.text((260, 50), "HARBOR LANE GROCERY", font=big, fill="#222", anchor="mm")
77
+ d.text((260, 92), "212 Quay Street", font=mono, fill="#333", anchor="mm")
78
+ d.text((260, 122), "Mar 14, 2026 17:42", font=mono, fill="#333", anchor="mm")
79
+ d.line((40, 160, 480, 160), fill="#888", width=2)
80
+ y = 190
81
+ for name, price in items:
82
+ d.text((50, y), name, font=mono, fill="#222")
83
+ d.text((470, y), f"{price:.2f}", font=mono, fill="#222", anchor="ra")
84
+ y += 44
85
+ d.line((40, y + 6, 480, y + 6), fill="#888", width=2)
86
+ y += 30
87
+ for label, value in (("SUBTOTAL", subtotal), ("TAX 8%", tax)):
88
+ d.text((50, y), label, font=mono, fill="#222")
89
+ d.text((470, y), f"{value:.2f}", font=mono, fill="#222", anchor="ra")
90
+ y += 40
91
+ d.text((50, y + 10), "TOTAL", font=big, fill="#111")
92
+ d.text((470, y + 10), f"{total:.2f}", font=big, fill="#111", anchor="ra")
93
+ d.text((260, y + 110), "PAID BY CARD", font=mono, fill="#333", anchor="mm")
94
+ d.text((260, y + 150), "THANK YOU", font=mono, fill="#333", anchor="mm")
95
+ priciest = max(items, key=lambda i: i[1])[0].split()[0].lower()
96
+ write("vision-receipt", "Read a receipt", img,
97
+ ["store: <name>", "date: <YYYY-MM-DD>", "items: <how many line items were bought>",
98
+ "priciest: <the most expensive item>", "total: <amount paid>", "tip: <tip amount>"],
99
+ [clause("store", r"harbor lane", "store: Harbor Lane Grocery"),
100
+ clause("date", r"2026-03-14\b", "date: 2026-03-14"),
101
+ clause("items", r"4\b", f"items: {len(items)}"),
102
+ clause("priciest", rf"{priciest}", "priciest: Cheddar 200g"),
103
+ clause("total", rf"\$?\s*{total:.2f}\b".replace(".", r"\."), f"total: {total:.2f}"),
104
+ clause("tip", NOT_SHOWN, "tip: not shown")],
105
+ [clause("tip", r"\$?\s*\d", "tip: a value (the receipt shows none)")],
106
+ "Printed text, a date written differently from how it is asked for, a count and a maximum.")
107
+
108
+
109
+ # ---- a grading label: digits that must be exact -------------------------
110
+
111
+ def label() -> None:
112
+ img = Image.new("RGB", (900, 300), "#dfe3e8")
113
+ d = ImageDraw.Draw(img)
114
+ d.rounded_rectangle((20, 20, 880, 280), radius=14, fill="#ffffff", outline="#1d3557", width=4)
115
+ d.rectangle((20, 20, 880, 70), fill="#1d3557")
116
+ d.text((450, 45), "NORTHLINE GRADING", font=font(28, "bold"), fill="#ffffff", anchor="mm")
117
+ sans, small = font(26, "bold"), font(20)
118
+ d.text((50, 95), "2019 HARBOR SERIES", font=sans, fill="#111")
119
+ d.text((50, 135), "#148 MAREN OKAFOR", font=sans, fill="#111")
120
+ d.text((50, 175), "GOLD REFRACTOR", font=sans, fill="#111")
121
+ d.text((50, 225), "CEN 9.5 COR 9 EDG 9.5 SUR 8.5", font=small, fill="#333")
122
+ d.text((760, 110), "MINT", font=font(30, "bold"), fill="#111", anchor="mm")
123
+ d.text((760, 175), "9", font=font(80, "bold"), fill="#b22222", anchor="mm")
124
+ d.text((620, 240), "CERT 40731958", font=font(22, "mono"), fill="#111", anchor="mm")
125
+ x = 520 # a barcode's look; it encodes nothing
126
+ for i, w in enumerate((3, 1, 2, 1, 1, 3, 2, 1, 3, 1, 2, 2, 1, 3, 1, 1, 2, 3, 1, 2)):
127
+ if i % 2 == 0:
128
+ d.rectangle((x, 90, x + w * 2, 150), fill="#111")
129
+ x += w * 2 + 2
130
+ write("vision-label", "Read a graded card's label", img,
131
+ ["grader: <company>", "grade: <the number>", "year: <year>", "number: <card number>",
132
+ "player: <name>", "cert: <certification number>", "surface: <surface subgrade>",
133
+ "autograph: <is the card signed: yes or no>"],
134
+ [clause("grader", r"northline", "grader: Northline"),
135
+ clause("grade", r"(mint\s*)?9\b(?!\.)", "grade: 9"),
136
+ clause("year", r"2019\b", "year: 2019"),
137
+ clause("number", r"#?\s*148\b", "number: 148"),
138
+ clause("player", r"maren okafor", "player: Maren Okafor"),
139
+ clause("cert", r"#?\s*40731958\b", "cert: 40731958"),
140
+ clause("surface", r"8\.5\b", "surface: 8.5"),
141
+ # A signed card's label says AUTO; this one does not, so "no" reads it right.
142
+ clause("autograph", r"(no\b|" + NOT_SHOWN[1:], "autograph: no, or not shown")],
143
+ [clause("autograph", r"(yes|signed|auto)", "autograph: yes (the label says nothing of one)")],
144
+ "A label read field by field; the cert number must be exact to the digit.")
145
+
146
+
147
+ # ---- a bar chart: values against gridlines ------------------------------
148
+
149
+ def bars() -> None:
150
+ teams = [("Billing", 45), ("Mobile", 25), ("Web", 35), ("Data", 15), ("Infra", 30)]
151
+ img = Image.new("RGB", (820, 560), "#ffffff")
152
+ d = ImageDraw.Draw(img)
153
+ d.text((410, 35), "Support tickets by team, Q2", font=font(26, "bold"), fill="#111", anchor="mm")
154
+ left, right, top, bottom, most = 90, 780, 80, 470, 50
155
+ for v in range(0, most + 1, 5):
156
+ y = bottom - (bottom - top) * v / most
157
+ d.line((left, y, right, y), fill="#d0d0d0" if v % 10 else "#a8a8a8", width=1)
158
+ if v % 10 == 0:
159
+ d.text((left - 12, y), str(v), font=font(18), fill="#333", anchor="rm")
160
+ d.line((left, top, left, bottom), fill="#333", width=2)
161
+ d.line((left, bottom, right, bottom), fill="#333", width=2)
162
+ slot = (right - left) / len(teams)
163
+ for i, (name, v) in enumerate(teams):
164
+ x0 = left + slot * i + slot * 0.2
165
+ x1 = left + slot * (i + 1) - slot * 0.2
166
+ d.rectangle((x0, bottom - (bottom - top) * v / most, x1, bottom - 1), fill="#3a6ea5")
167
+ d.text(((x0 + x1) / 2, bottom + 22), name, font=font(20), fill="#111", anchor="mm")
168
+ d.text((30, (top + bottom) / 2), "tickets", font=font(18), fill="#333", anchor="mm")
169
+ tall, short = max(teams, key=lambda t: t[1])[0], min(teams, key=lambda t: t[1])[0]
170
+ over = sum(v > 30 for _, v in teams)
171
+ write("vision-bars", "Read a bar chart", img,
172
+ ["tallest: <team>", "shortest: <team>", "web: <Web's value>",
173
+ "over_30: <how many teams are above 30, not counting 30 itself>", "target: <the target line's value>"],
174
+ [clause("tallest", tall.lower(), f"tallest: {tall}"),
175
+ clause("shortest", short.lower(), f"shortest: {short}"),
176
+ clause("web", r"35\b", "web: 35"),
177
+ clause("over_30", rf"{over}\b", f"over_30: {over}"),
178
+ clause("target", NOT_SHOWN, "target: not shown")],
179
+ [clause("target", r"\d", "target: a value (the chart has no target line)")],
180
+ "Values read against gridlines every 5, labelled every 10; one count asks for care at the boundary.")
181
+
182
+
183
+ # ---- a table: a cell by row and column ----------------------------------
184
+
185
+ def table() -> None:
186
+ cols = ["Q1", "Q2", "Q3", "Q4"]
187
+ rows = [("North", [1240, 1315, 1402, 1388]), ("South", [980, 1045, 1120, 1206]),
188
+ ("East", [1510, 1470, 1535, 1602]), ("West", [870, 905, 948, 1011]),
189
+ ("Central", [1105, 1160, 1098, 1175])]
190
+ x0, y0, cw, rh, first = 40, 70, 150, 54, 180
191
+ img = Image.new("RGB", (x0 * 2 + first + cw * len(cols), 420), "#ffffff")
192
+ d = ImageDraw.Draw(img)
193
+ d.text((img.width / 2, 32), "Units shipped by region", font=font(26, "bold"), fill="#111", anchor="mm")
194
+ sans, bold = font(22), font(22, "bold")
195
+ d.rectangle((x0, y0, x0 + first + cw * len(cols), y0 + rh), fill="#e8eef5")
196
+ d.text((x0 + 14, y0 + rh / 2), "Region", font=bold, fill="#111", anchor="lm")
197
+ for j, c in enumerate(cols):
198
+ d.text((x0 + first + cw * (j + 1) - 16, y0 + rh / 2), c, font=bold, fill="#111", anchor="rm")
199
+ for i, (name, vals) in enumerate(rows):
200
+ y = y0 + rh * (i + 1)
201
+ if i % 2:
202
+ d.rectangle((x0, y, x0 + first + cw * len(cols), y + rh), fill="#f6f6f6")
203
+ d.text((x0 + 14, y + rh / 2), name, font=sans, fill="#111", anchor="lm")
204
+ for j, v in enumerate(vals):
205
+ d.text((x0 + first + cw * (j + 1) - 16, y + rh / 2), f"{v:,}", font=sans, fill="#111", anchor="rm")
206
+ d.rectangle((x0, y0, x0 + first + cw * len(cols), y0 + rh * (len(rows) + 1)), outline="#999", width=2)
207
+ val = {name: dict(zip(cols, vals)) for name, vals in rows}
208
+ q4 = max(rows, key=lambda r: r[1][3])[0]
209
+
210
+ def num(n: int) -> str:
211
+ s = f"{n:,}"
212
+ return s.replace(",", ",?") + r"\b" if "," in s else s + r"\b"
213
+ write("vision-table", "Read a table", img,
214
+ ["south_q3: <number>", "west_q1: <number>", "central_q3: <number>",
215
+ "q4_highest: <the region with the most units in Q4>", "northeast_q2: <number>"],
216
+ [clause("south_q3", num(val["South"]["Q3"]), "south_q3: 1,120"),
217
+ clause("west_q1", num(val["West"]["Q1"]), "west_q1: 870"),
218
+ clause("central_q3", num(val["Central"]["Q3"]), "central_q3: 1,098"),
219
+ clause("q4_highest", q4.lower(), f"q4_highest: {q4}"),
220
+ clause("northeast_q2", NOT_SHOWN, "northeast_q2: not shown")],
221
+ [clause("northeast_q2", r"\d", "northeast_q2: a value (there is no Northeast row)")],
222
+ "Cells found by row and column; Central's Q3 dips, so a value from the wrong column shows.")
223
+
224
+
225
+ # ---- a settings screen: state, not just text -----------------------------
226
+
227
+ def screen() -> None:
228
+ img = Image.new("RGB", (760, 640), "#e9ecef")
229
+ d = ImageDraw.Draw(img)
230
+ d.rounded_rectangle((30, 30, 730, 610), radius=12, fill="#ffffff", outline="#c8ccd0", width=2)
231
+ d.text((60, 62), "Sync settings", font=font(28, "bold"), fill="#111")
232
+ d.rounded_rectangle((60, 110, 700, 160), radius=8, fill="#fdecea", outline="#e0a39d")
233
+ d.text((80, 135), "Last sync failed: storage quota exceeded (E-213)", font=font(20), fill="#8a1c12", anchor="lm")
234
+ toggles = [("Sync on mobile data", False), ("Back up photos", True),
235
+ ("Notify on conflicts", True), ("Pause when battery is low", False)]
236
+ y = 195
237
+ for name, on in toggles:
238
+ d.text((60, y + 15), name, font=font(22), fill="#111", anchor="lm")
239
+ d.rounded_rectangle((620, y, 690, y + 30), radius=15, fill="#2b7a3d" if on else "#c4c4c4")
240
+ cx = 675 if on else 635
241
+ d.ellipse((cx - 12, y + 3, cx + 12, y + 27), fill="#ffffff")
242
+ y += 52
243
+ d.text((60, y + 10), "When files conflict", font=font(22, "bold"), fill="#111")
244
+ options, chosen = ["Prefer the newer file", "Keep both", "Prefer this device"], "Keep both"
245
+ y += 50
246
+ for o in options:
247
+ d.ellipse((62, y, 84, y + 22), outline="#555", width=2)
248
+ if o == chosen:
249
+ d.ellipse((68, y + 6, 78, y + 16), fill="#1f5fbf")
250
+ d.text((96, y + 11), o, font=font(21), fill="#111", anchor="lm")
251
+ y += 36
252
+ d.rounded_rectangle((420, 548, 550, 590), radius=8, fill="#eeeeee", outline="#d5d5d5")
253
+ d.text((485, 569), "Sync now", font=font(20), fill="#a8a8a8", anchor="mm")
254
+ d.rounded_rectangle((570, 548, 700, 590), radius=8, fill="#1f5fbf")
255
+ d.text((635, 569), "Save", font=font(20, "bold"), fill="#ffffff", anchor="mm")
256
+ write("vision-screen", "Read a settings screen", img,
257
+ ["toggles_on: <every switch that is on>", "conflict: <the option chosen>",
258
+ "disabled: <the button that cannot be pressed>", "error_code: <code>", "account: <the signed-in email>"],
259
+ [clause("toggles_on", r".*back up photos", "toggles_on: Back up photos"),
260
+ clause("toggles_on", r".*notify on conflicts", "toggles_on: Notify on conflicts"),
261
+ clause("conflict", r"keep both", "conflict: Keep both"),
262
+ clause("disabled", r"sync now", "disabled: Sync now"),
263
+ clause("error_code", r"e-?213\b", "error_code: E-213"),
264
+ clause("account", NOT_SHOWN, "account: not shown")],
265
+ [clause("toggles_on", r".*(mobile data|battery)", "toggles_on: a switch that is off"),
266
+ clause("account", r"\S+@\S+", "account: an email (the screen shows none)")],
267
+ "A screen's state: which switches are on, which option is picked, which button is greyed out.")
268
+
269
+
270
+ # ---- shapes: counting and where things are -------------------------------
271
+
272
+ def shapes() -> None:
273
+ img = Image.new("RGB", (800, 600), "#ffffff")
274
+ d = ImageDraw.Draw(img)
275
+ blue = [(110, 120), (250, 90), (180, 300), (390, 210), (120, 480), (330, 420)]
276
+ red = [(520, 330), (640, 470), (260, 540)]
277
+ green = [(470, 120), (560, 520)]
278
+ for x, y in blue:
279
+ d.ellipse((x - 34, y - 34, x + 34, y + 34), fill="#2f6fd0")
280
+ for x, y in red:
281
+ d.rectangle((x - 32, y - 32, x + 32, y + 32), fill="#d33a2f")
282
+ for x, y in green:
283
+ d.polygon([(x, y - 38), (x - 38, y + 30), (x + 38, y + 30)], fill="#2e9e4f")
284
+ cx, cy, r = 715, 85, 50 # the star, in the top-right corner
285
+ d.polygon([(cx + (r if i % 2 == 0 else r * 0.45) * math.cos(math.radians(-90 + 36 * i)),
286
+ cy + (r if i % 2 == 0 else r * 0.45) * math.sin(math.radians(-90 + 36 * i))) for i in range(10)],
287
+ fill="#f2c200", outline="#b38f00")
288
+ write("vision-shapes", "Count shapes and say where one is", img,
289
+ ["blue_circles: <how many>", "red_squares: <how many>", "green_triangles: <how many>",
290
+ "top_right: <the shape in the top-right corner, with its colour>", "purple_shapes: <how many>"],
291
+ [clause("blue_circles", rf"{len(blue)}\b", f"blue_circles: {len(blue)}"),
292
+ clause("red_squares", rf"{len(red)}\b", f"red_squares: {len(red)}"),
293
+ clause("green_triangles", rf"{len(green)}\b", f"green_triangles: {len(green)}"),
294
+ clause("top_right", r".*star", "top_right: a yellow star"),
295
+ clause("purple_shapes", r"(0\b|none\b|zero\b|no\b|" + NOT_SHOWN[1:], "purple_shapes: 0")],
296
+ [clause("purple_shapes", r"[1-9]", "purple_shapes: a count above 0 (there are none)")],
297
+ "Counting to six without losing one, and a corner told apart from its neighbours.")
298
+
299
+
300
+ # ---- an invoice turned on its side ---------------------------------------
301
+
302
+ def invoice() -> None:
303
+ page = Image.new("RGB", (640, 820), "#ffffff")
304
+ d = ImageDraw.Draw(page)
305
+ d.text((40, 40), "INVOICE", font=font(40, "bold"), fill="#111")
306
+ d.text((40, 100), "INV-2026-0417", font=font(26, "mono"), fill="#111")
307
+ d.text((40, 160), "Issued: 2026-04-02", font=font(22), fill="#333")
308
+ d.text((40, 195), "Due: 2026-05-02", font=font(22), fill="#333")
309
+ d.text((40, 260), "Bill to", font=font(20, "bold"), fill="#555")
310
+ d.text((40, 290), "Kestrel Labs BV", font=font(24), fill="#111")
311
+ d.text((40, 322), "Havenstraat 18, Rotterdam", font=font(20), fill="#333")
312
+ lines = [("Model evaluation, March", 960.00), ("Hosting, March", 324.50)]
313
+ y = 400
314
+ d.line((40, y - 10, 600, y - 10), fill="#bbb", width=2)
315
+ for name, amount in lines:
316
+ d.text((40, y), name, font=font(22), fill="#111")
317
+ d.text((600, y), f"EUR {amount:,.2f}", font=font(22), fill="#111", anchor="ra")
318
+ y += 44
319
+ d.line((40, y + 4, 600, y + 4), fill="#bbb", width=2)
320
+ total = sum(a for _, a in lines)
321
+ d.text((40, y + 30), "Amount due", font=font(26, "bold"), fill="#111")
322
+ d.text((600, y + 30), f"EUR {total:,.2f}", font=font(26, "bold"), fill="#111", anchor="ra")
323
+ img = page.rotate(90, expand=True, fillcolor="#ffffff") # turned anticlockwise: reads bottom to top
324
+ write("vision-invoice", "Read an invoice turned on its side", img,
325
+ ["invoice: <invoice number>", "due: <YYYY-MM-DD>", "amount_due: <amount>", "bill_to: <company>",
326
+ "po_number: <purchase order number>"],
327
+ [clause("invoice", r"inv-2026-0417\b", "invoice: INV-2026-0417"),
328
+ clause("due", r"2026-05-02\b", "due: 2026-05-02"),
329
+ clause("amount_due", r"(eur|€)?\s*1,?284\.50\b", "amount_due: EUR 1,284.50"),
330
+ clause("bill_to", r"kestrel labs", "bill_to: Kestrel Labs BV"),
331
+ clause("po_number", NOT_SHOWN, "po_number: not shown")],
332
+ [clause("po_number", r"\d", "po_number: a value (the invoice has none)")],
333
+ "A page a scanner or a phone delivered sideways; two dates close together, only one asked for.")
334
+
335
+
336
+ if __name__ == "__main__":
337
+ for make in (receipt, label, bars, table, screen, shapes, invoice):
338
+ make()
package/ui/bench.css CHANGED
@@ -283,9 +283,21 @@ dialog h2 { margin: 0 0 .2rem; font-size: 1.375rem; font-weight: 700; }
283
283
  :is(dialog, .form-panel) .rules { font-size: .875rem; color: var(--muted); margin: .35rem 0 0; }
284
284
  :is(dialog, .form-panel) label.check { display: flex; align-items: center; gap: .5rem; margin-top: 1rem; cursor: pointer; }
285
285
  :is(dialog, .form-panel) fieldset { border: 0; padding: 0; margin: 1rem 0 0; }
286
- :is(dialog, .form-panel) .members { display: grid; grid-template-columns: repeat(auto-fill, minmax(14rem, 1fr)); gap: 0 1rem;
287
- max-height: 14rem; overflow: auto; }
288
- :is(dialog, .form-panel) .members label.check { margin-top: .35rem; font-size: .875rem; }
286
+ :is(dialog, .form-panel) .members { max-height: 18rem; overflow: auto; border-top: 1px solid var(--line); }
287
+ /* One row per model: its name and what it holds, then a chip per provider it
288
+ is on, each ticking that alias (bench.js fillForm). */
289
+ :is(dialog, .form-panel) .members .m-row { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: .2rem 1rem;
290
+ align-items: center; padding: .4rem 0; border-bottom: 1px solid var(--line); }
291
+ :is(dialog, .form-panel) .members .m-name { display: flex; flex-wrap: wrap; align-items: baseline; gap: 0 .5rem;
292
+ font-size: .875rem; overflow-wrap: anywhere; }
293
+ :is(dialog, .form-panel) .members .m-facts { color: var(--muted); font-size: .8125rem; }
294
+ :is(dialog, .form-panel) .members .m-routes { display: flex; flex-wrap: wrap; justify-content: flex-end; gap: .3rem; }
295
+ :is(dialog, .form-panel) .members label.check { margin-top: 0; font-size: .8125rem; gap: .3rem; padding: .15rem .5rem;
296
+ border: 1px solid var(--line); border-radius: var(--radius); background: var(--paper-2); }
297
+ :is(dialog, .form-panel) .members label.check:has(input:checked) { border-color: var(--bench-seat, var(--brand));
298
+ background: var(--band); }
299
+ #c-form .seats { min-height: 2rem; }
300
+ #c-form .seat .k.after { margin: 0 0 0 .4rem; }
289
301
  :is(dialog, .form-panel) input.member-filter { font-size: .9375rem; padding: .4rem .6rem; margin-bottom: .2rem; }
290
302
  :is(dialog, .form-panel) .members input:disabled + span { color: var(--muted); }
291
303
  /* Ruled out by the collection's tags (bench.js applyTags): greyed; a ticked
@@ -300,7 +312,8 @@ dialog h2 { margin: 0 0 .2rem; font-size: 1.375rem; font-weight: 700; }
300
312
  :is(dialog, .form-panel) .profile .chip { margin-right: .25rem; }
301
313
  :is(dialog, .form-panel) .profile .src { font-size: .75rem; color: var(--muted); }
302
314
  :is(dialog, .form-panel) .settings { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 0 1rem; }
303
- @media (max-width: 40rem) { :is(dialog, .form-panel) .settings { grid-template-columns: minmax(0, 1fr); } }
315
+ @media (max-width: 40rem) { :is(dialog, .form-panel) :is(.settings, .members .m-row) { grid-template-columns: minmax(0, 1fr); }
316
+ :is(dialog, .form-panel) .members .m-routes { justify-content: flex-start; } }
304
317
  :is(dialog, .form-panel) fieldset label.check { margin-top: .4rem; flex-wrap: wrap; }
305
318
  :is(dialog, .form-panel) fieldset label.check .muted { font-size: .875rem; }
306
319
  :is(dialog, .form-panel) .errors { color: var(--err); font-size: .9375rem; margin: .8rem 0 0; }
@@ -384,6 +397,40 @@ pre.log { background: var(--log-ground, var(--ink-deep)); color: var(--log-ink,
384
397
  white-space: pre-wrap; margin: .5rem 0 0; }
385
398
  pre.log.answer { max-height: 40vh; }
386
399
 
400
+ /* Quota: a tile per provider, as many columns as fit; a group of windows (a
401
+ 5-hour and a weekly limit of one model group) boxed together; a meter of
402
+ ten segments filled like a bar. */
403
+ .q-tiles { display: grid; grid-template-columns: repeat(auto-fill, minmax(16rem, 1fr)); gap: .75rem; align-items: start; margin-top: .75rem; }
404
+ .q-tile { display: flex; flex-direction: column; gap: .45rem; padding: .75rem .85rem; background: var(--panel);
405
+ border: 1px solid var(--line); border-radius: var(--radius); cursor: grab; }
406
+ .q-tile:focus-visible { outline: 2px solid var(--ink); outline-offset: 2px; }
407
+ .q-tile.out { border-color: var(--err); }
408
+ .q-tile.dragging { opacity: .45; }
409
+ .q-tile .q-head { display: flex; justify-content: space-between; align-items: baseline; gap: .5rem; }
410
+ .q-tile .nm { font-size: 1.0625rem; }
411
+ .q-tag, .q-small { font-size: .8125rem; color: var(--muted); }
412
+ .q-group { border: 1px solid var(--line); border-radius: var(--radius); padding: .45rem .55rem .55rem; display: flex; flex-direction: column; gap: .3rem; }
413
+ .q-group.bare { border: 0; padding: 0; }
414
+ .q-group h4 { margin: 0; font-size: .8125rem; font-weight: 500; color: var(--muted); }
415
+ .q-line { display: flex; justify-content: space-between; gap: .5rem; font-size: .875rem; }
416
+ .q-line b { font-weight: 600; }
417
+ .q-line .mono { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; min-width: 0; }
418
+ .segs { display: flex; gap: 2px; }
419
+ .segs span { flex: 1; height: .5rem; border-radius: 2px; background: var(--step-1); overflow: hidden; }
420
+ .segs i { display: block; height: 100%; background: var(--ok); }
421
+ .segs.warn i { background: var(--warn); }
422
+ .segs.err i { background: var(--err); }
423
+ .q-rest { font-size: .8125rem; background: var(--warn-ground); border-radius: var(--radius); padding: .2rem .45rem; }
424
+ .warn-t { color: var(--warn); }
425
+ .q-msg { font-size: .8125rem; }
426
+ .q-msg:empty { display: none; }
427
+ .q-tile .q-foot { display: flex; flex-wrap: wrap; gap: .4rem; justify-content: flex-end; margin-top: auto;
428
+ padding-top: .5rem; border-top: 1px solid var(--line); }
429
+ .q-tile .q-foot .chip { margin-left: .2rem; font-size: .75rem; }
430
+ .q-legend { display: flex; flex-wrap: wrap; gap: 1rem; font-size: .875rem; margin: 0 0 .4rem; }
431
+ .q-legend i { display: inline-block; width: .8rem; height: .7rem; border-radius: 2px; margin-right: .35rem; vertical-align: -1px; }
432
+ #q-models { max-width: min(48rem, 94vw); }
433
+
387
434
  /* Benchmarks charts (Observable Plot) */
388
435
  .chart { margin: .5rem 0 .75rem; overflow-x: auto; }
389
436
  .chart.muted { font-size: .875rem; }
@@ -394,10 +441,26 @@ h3.chart-head { font-size: 1rem; margin: 1rem 0 .2rem; }
394
441
  .list-actions { display: flex; align-items: center; gap: 1rem; margin: .75rem 0; flex-wrap: wrap; }
395
442
  .list-actions .done { margin: 0; color: var(--ok); font-size: .9375rem; }
396
443
  .list-actions .errors { color: var(--err); font-size: .9375rem; }
397
- button.c-row { width: 100%; text-align: left; font: inherit; color: inherit; background: none; border: 0;
398
- border-radius: var(--radius); cursor: pointer; }
444
+ button.c-row { width: calc(100% + 1.5rem); padding: .9rem .75rem; text-align: left; font: inherit; color: inherit;
445
+ background: none; border: 0; border-top: 1px solid var(--line); border-radius: 0; cursor: pointer; }
446
+ button.c-row:first-child { border-top: 0; }
399
447
  button.c-row:hover, button.c-row:focus-visible { background: var(--step-1); }
400
448
  .c-row .open-hint { color: var(--muted); font-size: .875rem; white-space: nowrap; }
449
+ .c-row .top { display: flex; flex-wrap: wrap; align-items: baseline; gap: .3rem; }
450
+ .c-row .top .alias { font-size: 1.125rem; font-weight: 500; margin-right: .5rem; }
451
+ .c-row .desc { margin: .1rem 0 0; max-width: 75ch; }
452
+ .c-row .together { margin: 0; font-size: .9375rem; max-width: 80ch; }
453
+ /* The panel as it sits: each member a seat topped in the bench's colour, as
454
+ the header's bench is; an explorer seat empty and dashed; the merger outlined
455
+ after an arrow. The bordered seats trade padding for border, so all are one size. */
456
+ :is(.c-row, #c-form) .seats { display: flex; flex-wrap: wrap; align-items: center; gap: .4rem; margin: .6rem 0 .5rem; }
457
+ :is(.c-row, #c-form) .seat { font-family: var(--font-mono); font-size: .8125rem; padding: .3rem .6rem; background: var(--band);
458
+ border-top: 3px solid var(--bench-seat, var(--brand)); border-radius: var(--radius); }
459
+ :is(.c-row, #c-form) .seat:is(.explorer, .merger) { background: transparent; border: 1px dashed var(--muted); color: var(--muted);
460
+ padding: calc(.3rem + 2px) calc(.6rem - 1px) calc(.3rem - 1px); }
461
+ .c-row .seat.merger { border-style: solid; border-color: var(--ink); color: var(--ink); }
462
+ :is(.c-row, #c-form) .seat .k { font-family: var(--font-display); color: var(--muted); margin-right: .35rem; }
463
+ .c-row .merge { display: inline-flex; align-items: center; gap: .4rem; color: var(--muted); }
401
464
  /* suggested collection tags under the field */
402
465
  .tag-picks { display: flex; flex-wrap: wrap; gap: .35rem; margin-top: .45rem; }
403
466
  .tag-picks .chip { cursor: pointer; border: 0; font-family: inherit; }
@@ -424,6 +487,8 @@ dialog .pick { display: flex; gap: 1.5rem; }
424
487
  .ep .acts { justify-content: flex-start; }
425
488
  details.key dl { grid-template-columns: minmax(0, 1fr); }
426
489
  .toolbar input[type=search] { min-width: 0; width: 100%; }
490
+ button.c-row { grid-template-columns: minmax(0, 1fr); } /* the seats get the width; the whole row opens */
491
+ .c-row .open-hint { display: none; }
427
492
  }
428
493
  @media (prefers-reduced-motion: reduce) {
429
494
  * { transition: none !important; }