mlx-bun 0.0.4 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +175 -93
  2. package/docs/dag/training-inference-map.html +575 -0
  3. package/docs/design/adapters-end-to-end.md +50 -46
  4. package/docs/design/compat-cli-surface-design.md +10 -8
  5. package/docs/design/docs-reorg-plan.md +14 -1
  6. package/docs/design/hlg-sampling.md +12 -2
  7. package/docs/design/memory-system.md +193 -295
  8. package/docs/design/optimization_plan.md +4 -2
  9. package/docs/design/orpo-dynamic-lambda.md +179 -0
  10. package/docs/design/orpo-future-enhancements.md +340 -0
  11. package/docs/design/orpo-training.md +1212 -0
  12. package/docs/design/parallel-slots.md +34 -26
  13. package/docs/design/parity-tier-dag.md +143 -0
  14. package/docs/design/runtime-isolation.md +1 -1
  15. package/docs/design/segmented-backward-training.md +58 -39
  16. package/docs/design/tauri-desktop-app.md +127 -0
  17. package/docs/design/w4a16-compute-precision-spike.md +232 -0
  18. package/docs/investigations/orpo-base-uf-experiment-and-directions.md +120 -0
  19. package/docs/investigations/orpo-flash-cce-pin-leak.md +167 -0
  20. package/docs/investigations/orpo-uf-testing-handoff.md +126 -0
  21. package/docs/{segmented-backward-handoff.md → investigations/segmented-backward-handoff.md} +1 -1
  22. package/docs/investigations/starter-model-port-handoff.md +32 -13
  23. package/docs/investigations/steel-flash-cce-handoff.md +489 -0
  24. package/docs/investigations/trainer-validation-experiment.md +173 -0
  25. package/docs/planning/IDEAS.md +29 -0
  26. package/docs/planning/PRODUCT_ROADMAP.md +147 -27
  27. package/docs/planning/chunk-task-roadmap.md +1 -1
  28. package/docs/planning/journal.md +161 -0
  29. package/docs/reference/distribution.md +61 -5
  30. package/docs/reference/library-api.md +29 -14
  31. package/docs/reference/memory.md +77 -0
  32. package/docs/reference/orpo-quickstart.md +130 -0
  33. package/docs/reference/server-api.md +105 -7
  34. package/docs/reference/server-config.md +12 -5
  35. package/docs/reference/training.md +296 -0
  36. package/package.json +1 -1
  37. package/src/cli.ts +651 -18
  38. package/src/eval/ifeval.ts +138 -0
  39. package/src/eval/runner.ts +76 -6
  40. package/src/eval/tasks/bfcl.ts +49 -15
  41. package/src/eval/tasks/gsm8k.ts +21 -9
  42. package/src/eval/tasks/hashhop.ts +33 -4
  43. package/src/eval/tasks/humaneval.ts +30 -8
  44. package/src/eval/tasks/ifeval.ts +52 -18
  45. package/src/eval/tasks/mmlu.ts +85 -74
  46. package/src/eval/ultrafeedback.ts +60 -0
  47. package/src/jobs/runner.ts +6 -0
  48. package/src/lora.ts +12 -7
  49. package/src/memory/chunk.ts +25 -0
  50. package/src/memory/cluster.ts +24 -0
  51. package/src/memory/db.ts +114 -0
  52. package/src/memory/ingest.ts +22 -0
  53. package/src/memory/pipeline.ts +63 -0
  54. package/src/memory/prompts.ts +23 -0
  55. package/src/memory/schedule.ts +163 -0
  56. package/src/memory/synthesize.ts +29 -0
  57. package/src/memory/tools.ts +399 -0
  58. package/src/memory/vault.ts +694 -0
  59. package/src/memory/wikify.ts +23 -0
  60. package/src/mlx/metal-kernel.ts +31 -4
  61. package/src/model/flash-attention.ts +37 -19
  62. package/src/model/fused-geglu-kernel.ts +111 -0
  63. package/src/model/gemma4-base.ts +39 -1
  64. package/src/model/gemma4.ts +196 -21
  65. package/src/model/minicpm5.ts +44 -3
  66. package/src/pi-provider.ts +29 -13
  67. package/src/pi-session.ts +81 -0
  68. package/src/pi-terminal.ts +13 -24
  69. package/src/pi-web.ts +246 -166
  70. package/src/quantize/job.ts +17 -2
  71. package/src/quantize/quantizer.ts +26 -2
  72. package/src/server.ts +216 -47
  73. package/src/tokenizer.ts +6 -2
  74. package/src/train/dataset.ts +29 -6
  75. package/src/train/flash-cce.ts +1175 -0
  76. package/src/train/job.ts +64 -3
  77. package/src/train/lora-params.ts +53 -4
  78. package/src/train/loss.ts +947 -0
  79. package/src/train/optimizer.ts +10 -3
  80. package/src/train/prefix-shared.ts +435 -0
  81. package/src/train/rank.ts +36 -0
  82. package/src/train/segmented.ts +1148 -3
  83. package/src/train/steel-qmm-header.ts +1461 -0
  84. package/src/train/trainer.ts +570 -54
  85. package/src/train/watch.ts +542 -0
  86. package/src/web/app.html +285 -19
  87. package/src/web/skills/memory/SKILL.md +97 -0
  88. package/src/web/skills/web-research/SKILL.md +3 -1
  89. package/src/web/skills.ts +46 -8
  90. /package/{docs → src/assets}/curve-designer.html +0 -0
@@ -0,0 +1,542 @@
1
+ // `mlx-bun train-watch <adapter-dir>` — a mactop-style live dashboard for a
2
+ // training run. It tails the trainer's append-only `<adapter>/metrics.jsonl`
3
+ // (written by trainLora) and renders auto-scaled line charts for loss, margin,
4
+ // accuracy (train line + val markers) and memory, plus a progress/ETA header.
5
+ //
6
+ // Decoupled by design: the trainer only writes the stream, this viewer only
7
+ // reads it. So it attaches/detaches freely to a LIVE run, replays a FINISHED
8
+ // one, and never touches training. The renderer is a PURE function
9
+ // (renderFrame) over parsed state, so it can be exercised against a captured
10
+ // fixture with no TTY.
11
+
12
+ import { readFileSync } from "node:fs";
13
+
14
+ // ---------------------------------------------------------------------------
15
+ // Parsed run state
16
+ // ---------------------------------------------------------------------------
17
+
18
+ export interface Pt { step: number; t: number; y: number }
19
+ /** A validation point — sparse (one per eval step). Accuracy carries its raw
20
+ * correct/total so it can be shown as a count, not a bare ratio. */
21
+ export interface ValPt extends Pt { nc?: number; nt?: number }
22
+ export interface RunState {
23
+ meta: {
24
+ method: string; model: string; iters: number; lambda?: number;
25
+ lr?: number; seq?: number; startedAt?: number; adapterPath?: string;
26
+ };
27
+ // Dense per-iter series (a line per metric).
28
+ loss: Pt[];
29
+ margin: Pt[];
30
+ trainAcc: Pt[]; // y ∈ {0,1} at B=1 — render as a rolling fraction, not raw
31
+ mem: Pt[]; // y = active GB (resident — flat = no leak)
32
+ peak: Pt[]; // y = peak GB (cumulative high-water — the climb-to-plateau)
33
+ // Sparse per-eval series (markers, not lines — they only exist at eval steps).
34
+ valAcc: ValPt[];
35
+ valLoss: Pt[];
36
+ peakGb: number; // running high-water
37
+ lastStep: number;
38
+ lastT: number;
39
+ done: boolean;
40
+ }
41
+
42
+ function emptyState(): RunState {
43
+ return {
44
+ meta: { method: "?", model: "", iters: 0 },
45
+ loss: [], margin: [], trainAcc: [], mem: [], peak: [],
46
+ valAcc: [], valLoss: [],
47
+ peakGb: 0, lastStep: 0, lastT: 0, done: false,
48
+ };
49
+ }
50
+
51
+ /** Fold one parsed JSONL record into the state. */
52
+ function ingest(s: RunState, o: Record<string, unknown>): void {
53
+ const type = o.type;
54
+ if (type === "meta") {
55
+ s.meta = {
56
+ method: String(o.method ?? "?"),
57
+ model: String(o.model ?? ""),
58
+ iters: Number(o.iters ?? 0),
59
+ lambda: o.orpo_lambda != null ? Number(o.orpo_lambda) : undefined,
60
+ lr: o.learning_rate != null ? Number(o.learning_rate) : undefined,
61
+ seq: o.max_seq_length != null ? Number(o.max_seq_length) : undefined,
62
+ startedAt: o.t != null ? Number(o.t) : undefined,
63
+ adapterPath: o.adapter_path != null ? String(o.adapter_path) : undefined,
64
+ };
65
+ return;
66
+ }
67
+ if (type !== "metric") return;
68
+ const step = Number(o.step), t = Number(o.t);
69
+ if (!Number.isFinite(step)) return;
70
+ if (o.kind === "train") {
71
+ if (Number.isFinite(Number(o.loss))) s.loss.push({ step, t, y: Number(o.loss) });
72
+ if (Number.isFinite(Number(o.margin))) s.margin.push({ step, t, y: Number(o.margin) });
73
+ if (Number.isFinite(Number(o.accuracy))) s.trainAcc.push({ step, t, y: Number(o.accuracy) });
74
+ if (Number.isFinite(Number(o.active_gb))) s.mem.push({ step, t, y: Number(o.active_gb) });
75
+ if (Number.isFinite(Number(o.peak_gb))) {
76
+ s.peak.push({ step, t, y: Number(o.peak_gb) });
77
+ s.peakGb = Math.max(s.peakGb, Number(o.peak_gb));
78
+ }
79
+ s.lastStep = Math.max(s.lastStep, step);
80
+ if (Number.isFinite(t)) s.lastT = Math.max(s.lastT, t);
81
+ } else if (o.kind === "val") {
82
+ if (Number.isFinite(Number(o.accuracy)))
83
+ s.valAcc.push({ step, t, y: Number(o.accuracy),
84
+ nc: o.n_correct != null ? Number(o.n_correct) : undefined,
85
+ nt: o.n_total != null ? Number(o.n_total) : undefined });
86
+ if (Number.isFinite(Number(o.loss))) s.valLoss.push({ step, t, y: Number(o.loss) });
87
+ }
88
+ }
89
+
90
+ /** Build full state from a metrics.jsonl text blob (replay-from-start). */
91
+ export function parseStream(text: string): RunState {
92
+ const s = emptyState();
93
+ for (const line of text.split("\n")) {
94
+ const t = line.trim();
95
+ if (!t) continue;
96
+ try { ingest(s, JSON.parse(t) as Record<string, unknown>); } catch { /* skip partial/garbled */ }
97
+ }
98
+ if (s.meta.iters > 0 && s.lastStep >= s.meta.iters) s.done = true;
99
+ return s;
100
+ }
101
+
102
+ /** Median step time (seconds) from the last K train timestamps. */
103
+ export function sPerStep(s: RunState, k = 30): number {
104
+ const pts = s.loss.slice(-k - 1);
105
+ const d: number[] = [];
106
+ for (let i = 1; i < pts.length; i++) {
107
+ const dt = (pts[i]!.t - pts[i - 1]!.t) / 1000;
108
+ if (dt > 0 && dt < 600) d.push(dt);
109
+ }
110
+ if (d.length === 0) return 0;
111
+ d.sort((a, b) => a - b);
112
+ return d[Math.floor(d.length / 2)]!;
113
+ }
114
+
115
+ // ---------------------------------------------------------------------------
116
+ // Cell grid compositor — the whole frame is built into a typed grid, then
117
+ // run-length colored on render. Unifies header / charts / axis labels / markers
118
+ // and sidesteps any mid-ANSI string splicing.
119
+ // ---------------------------------------------------------------------------
120
+
121
+ const DEFAULT_FG = -1;
122
+ interface Cell { ch: string; fg: number }
123
+
124
+ // 256-color palette (xterm).
125
+ const C = {
126
+ dim: 240, axis: 244, label: 250, title: 255,
127
+ loss: 203, margin: 78, accT: 45, accV: 207, mem: 221, ok: 78, warn: 203, accent: 81,
128
+ };
129
+
130
+ class Grid {
131
+ rows: number; cols: number; cells: Cell[];
132
+ constructor(rows: number, cols: number) {
133
+ this.rows = rows; this.cols = cols;
134
+ this.cells = new Array(rows * cols);
135
+ for (let i = 0; i < this.cells.length; i++) this.cells[i] = { ch: " ", fg: DEFAULT_FG };
136
+ }
137
+ put(r: number, c: number, ch: string, fg = DEFAULT_FG): void {
138
+ if (r < 0 || r >= this.rows || c < 0 || c >= this.cols) return;
139
+ const cell = this.cells[r * this.cols + c]!;
140
+ cell.ch = ch; cell.fg = fg;
141
+ }
142
+ text(r: number, c: number, str: string, fg = DEFAULT_FG): void {
143
+ for (let i = 0; i < str.length; i++) this.put(r, c + i, str[i]!, fg);
144
+ }
145
+ /** Right-align str ending at column `cEnd`. */
146
+ textRight(r: number, cEnd: number, str: string, fg = DEFAULT_FG): void {
147
+ this.text(r, cEnd - str.length + 1, str, fg);
148
+ }
149
+ render(): string[] {
150
+ const out: string[] = [];
151
+ for (let r = 0; r < this.rows; r++) {
152
+ let line = "", curFg = DEFAULT_FG;
153
+ for (let c = 0; c < this.cols; c++) {
154
+ const cell = this.cells[r * this.cols + c]!;
155
+ if (cell.fg !== curFg) {
156
+ line += cell.fg === DEFAULT_FG ? "\x1b[0m" : `\x1b[38;5;${cell.fg}m`;
157
+ curFg = cell.fg;
158
+ }
159
+ line += cell.ch;
160
+ }
161
+ if (curFg !== DEFAULT_FG) line += "\x1b[0m";
162
+ out.push(line);
163
+ }
164
+ return out;
165
+ }
166
+ }
167
+
168
+ // ---------------------------------------------------------------------------
169
+ // Braille (2×4 dot) canvas — sub-cell line resolution for the charts.
170
+ // ---------------------------------------------------------------------------
171
+
172
+ // dot bit for (col∈{0,1}, row∈{0..3}) within a cell.
173
+ const DOT = [
174
+ [0x01, 0x08],
175
+ [0x02, 0x10],
176
+ [0x04, 0x20],
177
+ [0x40, 0x80],
178
+ ];
179
+
180
+ class Braille {
181
+ cols: number; rows: number; data: Uint8Array;
182
+ constructor(cols: number, rows: number) {
183
+ this.cols = cols; this.rows = rows;
184
+ this.data = new Uint8Array(cols * rows);
185
+ }
186
+ setDot(dx: number, dy: number): void {
187
+ const cx = dx >> 1, cy = dy >> 2;
188
+ if (cx < 0 || cx >= this.cols || cy < 0 || cy >= this.rows) return;
189
+ this.data[cy * this.cols + cx]! |= DOT[dy & 3]![dx & 1]!;
190
+ }
191
+ line(x0: number, y0: number, x1: number, y1: number): void {
192
+ let dx = Math.abs(x1 - x0), dy = -Math.abs(y1 - y0);
193
+ const sx = x0 < x1 ? 1 : -1, sy = y0 < y1 ? 1 : -1;
194
+ let err = dx + dy;
195
+ for (;;) {
196
+ this.setDot(x0, y0);
197
+ if (x0 === x1 && y0 === y1) break;
198
+ const e2 = 2 * err;
199
+ if (e2 >= dy) { err += dy; x0 += sx; }
200
+ if (e2 <= dx) { err += dx; y0 += sy; }
201
+ }
202
+ }
203
+ char(cx: number, cy: number): string | null {
204
+ const b = this.data[cy * this.cols + cx]!;
205
+ return b === 0 ? null : String.fromCodePoint(0x2800 | b);
206
+ }
207
+ }
208
+
209
+ // ---------------------------------------------------------------------------
210
+ // One chart panel: title row + y-axis labels + braille plot + x-axis labels.
211
+ // ---------------------------------------------------------------------------
212
+
213
+ interface ChartOpts {
214
+ title: string;
215
+ series: Pt[];
216
+ color: number;
217
+ fixed?: [number, number]; // fixed y-range (e.g. accuracy 0..1)
218
+ fmtY: (v: number) => string;
219
+ headline?: string; // colored value shown after the title
220
+ alert?: string; // warn-colored note after the headline (e.g. "▲ rising")
221
+ zeroLine?: boolean; // dashed y=0 reference
222
+ refLine?: number; // dashed reference rule at an arbitrary y (e.g. 0.5 chance)
223
+ markers?: { pts: Pt[]; color: number }; // discrete sparse overlay (val points)
224
+ }
225
+
226
+ function fmtNum(v: number): string {
227
+ if (!Number.isFinite(v)) return "—";
228
+ const a = Math.abs(v);
229
+ if (a >= 100) return v.toFixed(0);
230
+ if (a >= 10) return v.toFixed(1);
231
+ if (a >= 1) return v.toFixed(2);
232
+ return v.toFixed(3);
233
+ }
234
+
235
+ function compactInt(n: number): string {
236
+ if (n >= 1_000_000) return (n / 1_000_000).toFixed(n % 1_000_000 ? 1 : 0) + "M";
237
+ if (n >= 10_000) return (n / 1000).toFixed(0) + "k";
238
+ return String(n);
239
+ }
240
+
241
+ /** Draw a chart into `grid` at top-left (r0,c0) spanning (h rows, w cols). */
242
+ function drawChart(grid: Grid, r0: number, c0: number, h: number, w: number, o: ChartOpts): void {
243
+ // Title line: title · headline (series color) · alert (warn).
244
+ grid.text(r0, c0, o.title, C.title);
245
+ let hx = c0 + o.title.length + 1;
246
+ if (o.headline) { grid.text(r0, hx, o.headline, o.color); hx += o.headline.length + 2; }
247
+ if (o.alert) grid.text(r0, hx, o.alert, C.warn);
248
+
249
+ const plotTop = r0 + 1;
250
+ const plotRows = h - 2; // minus title + x-axis row
251
+ if (plotRows < 1 || w < 12) return;
252
+
253
+ const pts = o.series;
254
+ // y-range.
255
+ let ymin: number, ymax: number;
256
+ if (o.fixed) { [ymin, ymax] = o.fixed; }
257
+ else if (pts.length === 0) { ymin = 0; ymax = 1; }
258
+ else {
259
+ ymin = Infinity; ymax = -Infinity;
260
+ for (const p of pts) { if (p.y < ymin) ymin = p.y; if (p.y > ymax) ymax = p.y; }
261
+ if (o.markers) for (const p of o.markers.pts) { if (p.y < ymin) ymin = p.y; if (p.y > ymax) ymax = p.y; }
262
+ if (o.zeroLine) { ymin = Math.min(ymin, 0); ymax = Math.max(ymax, 0); }
263
+ if (ymin === ymax) { ymin -= 1; ymax += 1; }
264
+ else { const pad = (ymax - ymin) * 0.08; ymin -= pad; ymax += pad; }
265
+ }
266
+
267
+ // gutter for y labels.
268
+ const gutter = Math.max(o.fmtY(ymin).length, o.fmtY(ymax).length) + 1;
269
+ const plotCols = w - gutter;
270
+ if (plotCols < 4) return;
271
+
272
+ // y-axis labels: top=max, bottom=min, mid where room.
273
+ grid.textRight(plotTop, c0 + gutter - 2, o.fmtY(ymax), C.axis);
274
+ grid.textRight(plotTop + plotRows - 1, c0 + gutter - 2, o.fmtY(ymin), C.axis);
275
+ if (plotRows >= 5) grid.textRight(plotTop + (plotRows >> 1), c0 + gutter - 2, o.fmtY((ymin + ymax) / 2), C.axis);
276
+ // vertical axis rule.
277
+ for (let r = 0; r < plotRows; r++) grid.put(plotTop + r, c0 + gutter - 1, "│", C.dim);
278
+
279
+ // x-range (step).
280
+ const xmin = pts.length ? pts[0]!.step : 0;
281
+ const xmax = pts.length ? pts[pts.length - 1]!.step : 1;
282
+ const xspan = Math.max(1, xmax - xmin);
283
+
284
+ const nDotX = plotCols * 2, nDotY = plotRows * 4;
285
+ const canvas = new Braille(plotCols, plotRows);
286
+ const mapX = (step: number) => Math.round(((step - xmin) / xspan) * (nDotX - 1));
287
+ const mapY = (y: number) => Math.round((1 - (y - ymin) / (ymax - ymin)) * (nDotY - 1));
288
+
289
+ // zero reference: a dim dotted grid rule the data line crosses (drawn first,
290
+ // so the braille blit below overwrites it only where there's data).
291
+ const drawRule = (yval: number) => {
292
+ if (!(yval > ymin && yval < ymax)) return;
293
+ const rRow = plotTop + Math.round((1 - (yval - ymin) / (ymax - ymin)) * (plotRows - 1));
294
+ for (let c = 0; c < plotCols; c++) grid.put(rRow, c0 + gutter + c, "┄", C.dim);
295
+ };
296
+ if (o.zeroLine) drawRule(0);
297
+ if (o.refLine != null) drawRule(o.refLine);
298
+
299
+ // Downsample to keep line cost bounded on long runs.
300
+ if (pts.length > 0) {
301
+ const stride = Math.max(1, Math.floor(pts.length / (nDotX * 2)));
302
+ let prevX = mapX(pts[0]!.step), prevY = mapY(pts[0]!.y);
303
+ canvas.setDot(prevX, prevY);
304
+ for (let i = stride; i < pts.length; i += stride) {
305
+ const cx = mapX(pts[i]!.step), cy = mapY(pts[i]!.y);
306
+ canvas.line(prevX, prevY, cx, cy);
307
+ prevX = cx; prevY = cy;
308
+ }
309
+ // always connect the final point.
310
+ const last = pts[pts.length - 1]!;
311
+ canvas.line(prevX, prevY, mapX(last.step), mapY(last.y));
312
+ }
313
+
314
+ // Blit braille into grid.
315
+ for (let r = 0; r < plotRows; r++)
316
+ for (let c = 0; c < plotCols; c++) {
317
+ const ch = canvas.char(c, r);
318
+ if (ch) grid.put(plotTop + r, c0 + gutter + c, ch, o.color);
319
+ }
320
+
321
+ // Discrete markers (val accuracy) overlaid as ●.
322
+ if (o.markers) {
323
+ for (const p of o.markers.pts) {
324
+ const cx = c0 + gutter + Math.round(((p.step - xmin) / xspan) * (plotCols - 1));
325
+ const cy = plotTop + Math.round((1 - (p.y - ymin) / (ymax - ymin)) * (plotRows - 1));
326
+ grid.put(cy, cx, "●", o.markers.color);
327
+ }
328
+ }
329
+
330
+ // x-axis labels row.
331
+ const axisRow = plotTop + plotRows;
332
+ grid.text(axisRow, c0 + gutter, compactInt(xmin), C.axis);
333
+ grid.textRight(axisRow, c0 + gutter + plotCols - 1, compactInt(xmax), C.axis);
334
+ }
335
+
336
+ // ---------------------------------------------------------------------------
337
+ // Full-frame renderer (pure).
338
+ // ---------------------------------------------------------------------------
339
+
340
+ /** Rolling MEDIAN over a trailing window — robust trend for a metric whose
341
+ * per-step value is per-example (B=1 loss spikes on a long/hard pair are
342
+ * rejected, unlike a mean). This is what makes the loss trend legible. */
343
+ function rollingMedian(pts: Pt[], w: number): Pt[] {
344
+ const out: Pt[] = [];
345
+ for (let i = 0; i < pts.length; i++) {
346
+ const lo = Math.max(0, i - w + 1);
347
+ const win: number[] = [];
348
+ for (let j = lo; j <= i; j++) win.push(pts[j]!.y);
349
+ win.sort((a, b) => a - b);
350
+ out.push({ step: pts[i]!.step, t: pts[i]!.t, y: win[win.length >> 1]! });
351
+ }
352
+ return out;
353
+ }
354
+
355
+ /** Light trailing mean — smooths the piecewise-constant plateaus a rolling
356
+ * median leaves behind (median despikes; this mean removes the "banding" those
357
+ * flat runs make in the braille line). */
358
+ function rollingMean(pts: Pt[], w: number): Pt[] {
359
+ const out: Pt[] = [];
360
+ let sum = 0;
361
+ for (let i = 0; i < pts.length; i++) {
362
+ sum += pts[i]!.y;
363
+ if (i >= w) sum -= pts[i - w]!.y;
364
+ out.push({ step: pts[i]!.step, t: pts[i]!.t, y: sum / Math.min(i + 1, w) });
365
+ }
366
+ return out;
367
+ }
368
+
369
+ function minPt(pts: Pt[]): { y: number; step: number } | null {
370
+ if (!pts.length) return null;
371
+ let y = Infinity, step = pts[0]!.step;
372
+ for (const p of pts) if (p.y < y) { y = p.y; step = p.step; }
373
+ return { y, step };
374
+ }
375
+
376
+ /** Pull a readable model name from an HF cache path
377
+ * (".../models--mlx-community--MiniCPM5-1B-OptiQ-4bit/snapshots/<hash>") or a
378
+ * plain repo id, dropping the quant suffix. */
379
+ function shortModel(p: string): string {
380
+ const m = p.match(/models--[^/]*--([^/]+)/);
381
+ const name = m ? m[1]! : (p.split("/").filter((s) => s && s !== "snapshots" && !/^[0-9a-f]{16,}$/.test(s)).pop() ?? p);
382
+ return name.replace(/-OptiQ-4bit$/i, "");
383
+ }
384
+
385
+ function fmtDuration(sec: number): string {
386
+ if (!Number.isFinite(sec) || sec < 0) return "—";
387
+ const h = Math.floor(sec / 3600), m = Math.floor((sec % 3600) / 60), s = Math.floor(sec % 60);
388
+ if (h > 0) return `${h}h${String(m).padStart(2, "0")}m`;
389
+ if (m > 0) return `${m}m${String(s).padStart(2, "0")}s`;
390
+ return `${s}s`;
391
+ }
392
+
393
+ export function renderFrame(st: RunState, cols: number, rows: number): string[] {
394
+ const grid = new Grid(rows, cols);
395
+ const sps = sPerStep(st);
396
+ const iters = st.meta.iters || st.lastStep || 1;
397
+ const frac = Math.min(1, st.lastStep / iters);
398
+ const eta = sps > 0 ? (iters - st.lastStep) * sps : NaN;
399
+ const elapsed = st.meta.startedAt && st.lastT ? (st.lastT - st.meta.startedAt) / 1000 : NaN;
400
+
401
+ // Header line 1: identity + progress.
402
+ const modelShort = shortModel(st.meta.model);
403
+ const statusDot = st.done ? "✓" : "●";
404
+ const statusFg = st.done ? C.ok : C.accent;
405
+ grid.put(0, 0, statusDot, statusFg);
406
+ grid.text(0, 2, `${st.meta.method} · ${modelShort}`, C.title);
407
+ const hdr = st.done
408
+ ? `step ${st.lastStep}/${iters} · complete · ${fmtDuration(elapsed)}`
409
+ : `step ${st.lastStep}/${iters} · ${(frac * 100).toFixed(1)}% · ETA ${fmtDuration(eta)} · ${sps ? sps.toFixed(2) + "s/step" : "—"} · ${fmtDuration(elapsed)}`;
410
+ grid.textRight(0, cols - 1, hdr, C.label);
411
+
412
+ // Progress bar (row 1).
413
+ const barW = cols - 2;
414
+ const filled = Math.round(frac * barW);
415
+ for (let i = 0; i < barW; i++) grid.put(1, 1 + i, i < filled ? "█" : "░", i < filled ? statusFg : C.dim);
416
+
417
+ // Legend (row 2): trend lines are smoothed because B=1 per-step is per-example;
418
+ // ● markers are val (sparse, one per eval step).
419
+ grid.text(2, 1, "loss = orpo/step · margin = confidence · accuracy = correct on val · ● = val @ eval", C.dim);
420
+
421
+ // Chart stack (rows 2..).
422
+ const top = 3;
423
+ const avail = rows - top;
424
+ if (avail < 6) return grid.render();
425
+
426
+ const cur = (a: Pt[]) => (a.length ? a[a.length - 1]!.y : NaN);
427
+ const pct = (v: number) => (Number.isFinite(v) ? `${(v * 100).toFixed(1)}%` : "—");
428
+ const SMOOTH = 15;
429
+
430
+ // Loss: at B=1 the per-step loss is per-EXAMPLE (a long/hard pair spikes it),
431
+ // so the displayed trend is a rolling MEDIAN (robust to those spikes) plus the
432
+ // sparse val-loss markers. The min/rising detector runs on the median — the
433
+ // "loss bottomed out and turned back up" early-stop signal.
434
+ const lossMed = rollingMean(rollingMedian(st.loss, SMOOTH), 5); // median despikes, mean de-bands
435
+ const lm = minPt(lossMed);
436
+ const recentLoss = cur(lossMed);
437
+ const rising = !!(lm && lossMed.length > SMOOTH + 5 && lm.step < st.lastStep - SMOOTH
438
+ && recentLoss > lm.y * 1.05);
439
+
440
+ const marginMed = rollingMean(rollingMedian(st.margin, SMOOTH), 5); // median despikes, mean de-bands
441
+
442
+ // Val accuracy — the metric that matters: of the held-out val set, how many
443
+ // pairs it correctly prefers (chosen > rejected). Sparse (one per eval step),
444
+ // drawn as a line + markers; 50% is chance.
445
+ const lastVal = st.valAcc[st.valAcc.length - 1];
446
+ const bestVal = st.valAcc.reduce<ValPt | null>((b, p) => (!b || p.y > b.y ? p : b), null);
447
+
448
+ const panels: ChartOpts[] = [
449
+ {
450
+ title: "loss · orpo per step", series: lossMed, color: C.loss, fmtY: fmtNum,
451
+ markers: st.valLoss.length ? { pts: st.valLoss, color: C.accV } : undefined,
452
+ headline: `${fmtNum(recentLoss)}${lm ? ` ↓min ${fmtNum(lm.y)} @ ${lm.step}` : ""}`,
453
+ alert: rising ? `▲ rising +${fmtNum(recentLoss - lm!.y)} since ${lm!.step}` : undefined,
454
+ },
455
+ {
456
+ title: "margin · confidence in the choice", series: marginMed, color: C.margin,
457
+ fmtY: (v) => (v >= 0 ? "+" : "") + fmtNum(v),
458
+ zeroLine: true,
459
+ headline: `${cur(marginMed) >= 0 ? "+" : ""}${fmtNum(cur(marginMed))} ${cur(marginMed) > 0 ? "leans chosen" : "leans rejected"}`,
460
+ },
461
+ {
462
+ title: "accuracy · correct choices on val", series: st.valAcc, color: C.accT, fixed: [0, 1],
463
+ fmtY: (v) => `${Math.round(v * 100)}%`,
464
+ refLine: 0.5,
465
+ markers: { pts: st.valAcc, color: C.accT },
466
+ headline: lastVal
467
+ ? `${lastVal.nc ?? "?"}/${lastVal.nt ?? "?"} correct = ${pct(lastVal.y)}`
468
+ + (bestVal ? ` · best ${pct(bestVal.y)} @ ${bestVal.step}` : "")
469
+ : "awaiting first eval (50% = chance)",
470
+ },
471
+ {
472
+ title: "memory", series: st.peak, color: C.mem, fmtY: (v) => v.toFixed(1) + "G",
473
+ headline: `peak ${st.peakGb.toFixed(2)}GB · active ${fmtNum(cur(st.mem))}GB`,
474
+ },
475
+ ];
476
+
477
+ const ph = Math.floor(avail / panels.length);
478
+ if (ph < 4) {
479
+ // Too short: only loss.
480
+ drawChart(grid, top, 1, avail, cols - 2, panels[0]!);
481
+ return grid.render();
482
+ }
483
+ panels.forEach((p, i) => drawChart(grid, top + i * ph, 1, ph, cols - 2, p));
484
+ return grid.render();
485
+ }
486
+
487
+ // ---------------------------------------------------------------------------
488
+ // Live TUI loop.
489
+ // ---------------------------------------------------------------------------
490
+
491
+ const ALT_ON = "\x1b[?1049h\x1b[?25l", ALT_OFF = "\x1b[?25h\x1b[?1049l\x1b[0m";
492
+ const HOME = "\x1b[H", CLEAR = "\x1b[2J";
493
+
494
+ export async function runWatch(adapterDir: string): Promise<void> {
495
+ const path = `${adapterDir}/metrics.jsonl`;
496
+ if (!(await Bun.file(path).exists())) {
497
+ console.error(`no metrics.jsonl in ${adapterDir} — is this an mlx-bun training run dir?`);
498
+ console.error(`(the trainer writes ${path} as it runs; point train-watch at the --adapter dir)`);
499
+ process.exit(1);
500
+ }
501
+
502
+ let lastRender = "";
503
+ const draw = (st: RunState) => {
504
+ const cols = process.stdout.columns ?? 100;
505
+ const rows = process.stdout.rows ?? 30;
506
+ const frame = HOME + renderFrame(st, cols, rows).join("\n");
507
+ if (frame !== lastRender) { process.stdout.write(frame); lastRender = frame; }
508
+ };
509
+
510
+ const restore = () => { process.stdout.write(ALT_OFF); };
511
+ const quit = (code = 0) => {
512
+ if (process.stdin.isTTY) process.stdin.setRawMode(false);
513
+ restore();
514
+ process.exit(code);
515
+ };
516
+
517
+ process.stdout.write(ALT_ON + CLEAR);
518
+ if (process.stdin.isTTY) {
519
+ process.stdin.setRawMode(true);
520
+ process.stdin.resume();
521
+ process.stdin.on("data", (b) => {
522
+ const s = b.toString();
523
+ if (s === "q" || s === "\x03" /* ^C */) quit(0);
524
+ });
525
+ }
526
+ process.on("SIGINT", () => quit(0));
527
+ process.on("SIGTERM", () => quit(0));
528
+ process.stdout.on("resize", () => { lastRender = ""; });
529
+
530
+ // Poll the file; reparse fully each tick (it's small, single-digit MB) — keeps
531
+ // the loop dead simple and robust to truncation/rotation. Stop redrawing once
532
+ // the run is done AND the file has settled.
533
+ let stableTicks = 0;
534
+ for (;;) {
535
+ let st: RunState;
536
+ try { st = parseStream(readFileSync(path, "utf8")); }
537
+ catch { st = emptyState(); }
538
+ draw(st);
539
+ if (st.done) { if (++stableTicks > 8) { /* keep showing final frame, slow poll */ } }
540
+ await Bun.sleep(st.done ? 1000 : 250);
541
+ }
542
+ }